From 5b9304067af6e9ab2d1bc39154bb8625f35fd954 Mon Sep 17 00:00:00 2001 From: harshasrinivas Date: Sat, 25 Mar 2017 05:38:21 +0530 Subject: [PATCH 1/2] Warn in case of multiple spiders - scrapy runspider --- scrapy/commands/runspider.py | 10 ++++++++++ tests/test_commands.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index a98033dd1..59caa08b3 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -1,5 +1,6 @@ import sys import os +import warnings from importlib import import_module from scrapy.utils.spider import iter_spider_classes @@ -83,7 +84,16 @@ class Command(ScrapyCommand): spclasses = list(iter_spider_classes(module)) if not spclasses: raise UsageError("No spider found in file: %s\n" % filename) + + self._check_multiple_spiders(spclasses) + spidercls = spclasses.pop() self.crawler_process.crawl(spidercls, **opts.spargs) self.crawler_process.start() + + def _check_multiple_spiders(self, spclasses): + if len(spclasses) > 1: + msg = ("\n\nThere are multiple spiders in this file: " + "Only the last-defined spider '{}' will be run.\n".format(spclasses[-1].name)) + warnings.warn(msg, UserWarning) diff --git a/tests/test_commands.py b/tests/test_commands.py index 922098668..f6dcdc03f 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -2,6 +2,7 @@ import os import sys import subprocess import tempfile +import warnings from time import sleep from os.path import exists, join, abspath from shutil import rmtree, copytree @@ -193,6 +194,24 @@ class MySpider(scrapy.Spider): return [] """ + debug_log_multiple_spiders = """ +import scrapy + +class FirstSpider(scrapy.Spider): + name = 'spider1' + + def start_requests(self): + self.logger.debug("spider1 Works!") + return [] + +class SecondSpider(scrapy.Spider): + name = 'spider2' + + def start_requests(self): + self.logger.debug("spider2 Works!") + return [] +""" + @contextmanager def _create_file(self, content, name): tmpdir = self.mktemp() @@ -267,6 +286,15 @@ class BadSpider(scrapy.Spider): self.assertIn("start_requests", log) self.assertIn("badspider.py", log) + def test_runspider_multiple_spiders(self): + log = self.get_log(self.debug_log_multiple_spiders) + self.assertIn("multiple spiders in this file", log) + self.assertIn("DEBUG: spider2 Works!", log) + self.assertIn("INFO: Spider opened", log) + self.assertIn("INFO: Closing spider (finished)", log) + self.assertIn("INFO: Spider closed (finished)", log) + + class BenchCommandTest(CommandTest): From 09857515e76e42a42eb0d7fe6f8696f1911456da Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Thu, 6 Aug 2026 16:09:36 +0200 Subject: [PATCH 2/2] runspider: warn about extra spiders in the file and add --spider --- docs/topics/commands.rst | 10 +++++- scrapy/commands/runspider.py | 38 ++++++++++++++------- tests/test_command_runspider.py | 58 +++++++++++++++++++++------------ 3 files changed, 73 insertions(+), 33 deletions(-) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 50da4593a..ef9283340 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -575,7 +575,15 @@ runspider Run the spider defined in the given Python file, without requiring a project. -Supported options: the same as :command:`crawl`. +If the file defines more than one spider, the last one is run, and a warning +is logged. Use ``--spider`` to run a different one. + +Supported options: the same as :command:`crawl`, plus: + +* ``--spider=NAME``: run the spider with the given + :attr:`~scrapy.Spider.name` + + .. versionadded:: VERSION Example usage:: diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 28fdf65d8..797bbf29e 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -1,7 +1,7 @@ from __future__ import annotations +import logging import sys -import warnings from importlib import import_module from pathlib import Path from typing import TYPE_CHECKING, Any, ClassVar @@ -17,6 +17,9 @@ if TYPE_CHECKING: from types import ModuleType +logger = logging.getLogger(__name__) + + def _import_file(filepath: str | PathLike[str]) -> ModuleType: abspath = Path(filepath).resolve() if abspath.suffix not in {".py", ".pyw"}: @@ -44,6 +47,14 @@ class Command(BaseRunSpiderCommand): def long_desc(self) -> str: return "Run the spider defined in the given file" + def add_options(self, parser: argparse.ArgumentParser) -> None: + super().add_options(parser) + parser.add_argument( + "--spider", + metavar="NAME", + help="run the spider with this name, if the file defines more than one", + ) + def run(self, args: list[str], opts: argparse.Namespace) -> None: if len(args) != 1: raise UsageError @@ -57,10 +68,21 @@ class Command(BaseRunSpiderCommand): spclasses = list(iter_spider_classes(module)) if not spclasses: raise UsageError(f"No spider found in file: {filename}\n") - - self._check_multiple_spiders(spclasses) - - spidercls = spclasses.pop() + if opts.spider: + try: + spidercls = next(c for c in spclasses if c.name == opts.spider) + except StopIteration: + raise UsageError( + f"No spider named {opts.spider!r} found in file: {filename}\n" + ) from None + else: + spidercls = spclasses[-1] + if len(spclasses) > 1: + names = ", ".join(repr(c.name) for c in spclasses) + logger.warning( + f"{filename} defines more than one spider ({names}), running " + f"{spidercls.name!r}. Use --spider to run a different one." + ) assert self.crawler_process self.crawler_process.crawl(spidercls, **opts.spargs) @@ -68,9 +90,3 @@ class Command(BaseRunSpiderCommand): if self.crawler_process.bootstrap_failed: self.exitcode = 1 - - def _check_multiple_spiders(self, spclasses): - if len(spclasses) > 1: - msg = ("\n\nThere are multiple spiders in this file: " - "Only the last-defined spider '{}' will be run.\n".format(spclasses[-1].name)) - warnings.warn(msg, UserWarning) diff --git a/tests/test_command_runspider.py b/tests/test_command_runspider.py index e29e41fad..3db35dd33 100644 --- a/tests/test_command_runspider.py +++ b/tests/test_command_runspider.py @@ -31,6 +31,26 @@ class MySpider(scrapy.Spider): yield """ + multiple_spiders = """ +import scrapy + +class FirstSpider(scrapy.Spider): + name = 'spider1' + + async def start(self): + self.logger.debug("spider1 Works!") + return + yield + +class SecondSpider(scrapy.Spider): + name = 'spider2' + + async def start(self): + self.logger.debug("spider2 Works!") + return + yield +""" + badspider = """ import scrapy @@ -133,30 +153,26 @@ class MySpider(scrapy.Spider): assert ("[scrapy.core.engine]" in log1) is not value def test_runspider_multiple_spiders(self, tmp_path: Path) -> None: - code = """ -import scrapy - -class FirstSpider(scrapy.Spider): - name = 'spider1' - - async def start(self): - self.logger.debug("spider1 Works!") - return - yield - -class SecondSpider(scrapy.Spider): - name = 'spider2' - - async def start(self): - self.logger.debug("spider2 Works!") - return - yield -""" - log = self.get_log(tmp_path, code) - assert "multiple spiders in this file" in log + log = self.get_log(tmp_path, self.multiple_spiders) + assert "defines more than one spider ('spider1', 'spider2')" in log + assert "running 'spider2'" in log assert "DEBUG: spider2 Works!" in log assert "INFO: Spider closed (finished)" in log + def test_runspider_spider_option(self, tmp_path: Path) -> None: + log = self.get_log( + tmp_path, self.multiple_spiders, args=("--spider", "spider1") + ) + assert "defines more than one spider" not in log + assert "DEBUG: spider1 Works!" in log + assert "INFO: Spider closed (finished)" in log + + def test_runspider_spider_option_not_found(self, tmp_path: Path) -> None: + log = self.get_log( + tmp_path, self.multiple_spiders, args=("--spider", "spider3") + ) + assert "No spider named 'spider3' found in file" in log + def test_runspider_no_spider_found(self, tmp_path: Path) -> None: log = self.get_log(tmp_path, "from scrapy.spiders import Spider\n") assert "No spider found in file" in log