From 37ec53e74879c4165d22b6fe53be09b477d2e06e Mon Sep 17 00:00:00 2001 From: alchemypls Date: Sun, 22 Jun 2025 01:45:47 -0700 Subject: [PATCH 1/2] docs: improve ScrapyCommand documentation for issue #6844 --- docs/topics/commands.rst | 87 ++++++++++++++++++++++++++++++++----- scrapy/commands/__init__.py | 2 + 2 files changed, 78 insertions(+), 11 deletions(-) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 4994fe1d6..82712eba5 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -629,25 +629,74 @@ Custom project commands ======================= You can also add your custom project commands by using the -:setting:`COMMANDS_MODULE` setting. See the Scrapy commands in -`scrapy/commands`_ for examples on how to implement your commands. +:setting:`COMMANDS_MODULE` setting. This allows you to create project-specific +commands that are automatically discovered and made available through the +``scrapy`` command-line tool. -.. _scrapy/commands: https://github.com/scrapy/scrapy/tree/master/scrapy/commands -.. setting:: COMMANDS_MODULE +Creating custom commands +------------------------ -COMMANDS_MODULE ---------------- +To create a custom command, inherit from the :class:`~scrapy.commands.ScrapyCommand` class +and implement the required methods. This allows you to extend Scrapy's command-line +interface with your own functionality, such as project-specific utilities, data +processing tools, or deployment helpers. -Default: ``''`` (empty string) +When you create a custom command, you define its behavior by setting class attributes +and overriding specific methods. Here's what you need to know: -A module to use for looking up custom Scrapy commands. This is used to add custom -commands for your Scrapy project. +**Attributes you can set:** -Example: +* ``requires_project`` (bool): If True, the command requires a Scrapy project to be present (default: False) +* ``requires_crawler_process`` (bool): If True, the command requires a crawler process to be available (default: True) +* ``default_settings`` (dict): Default settings to use for this command instead of global defaults (default: {}) +* ``exitcode`` (int): Exit code to return when command completes (default: 0) + +**Methods you must override:** + +* :meth:`~scrapy.commands.ScrapyCommand.syntax`: Return command syntax (preferably one-line, without command name) +* :meth:`~scrapy.commands.ScrapyCommand.short_desc`: Return a short description of the command +* :meth:`~scrapy.commands.ScrapyCommand.run`: Main entry point for command execution (must implement) + +**Methods you can override:** + +* :meth:`~scrapy.commands.ScrapyCommand.long_desc`: Return a detailed description (can contain newlines) +* :meth:`~scrapy.commands.ScrapyCommand.help`: Return extensive help text (can contain newlines) +* :meth:`~scrapy.commands.ScrapyCommand.add_options`: Add command-specific options to argument parser +* :meth:`~scrapy.commands.ScrapyCommand.process_options`: Process parsed command-line options + +**Example custom command:** .. code-block:: python - COMMANDS_MODULE = "mybot.commands" + from scrapy.commands import ScrapyCommand + import argparse + + + class MyCustomCommand(ScrapyCommand): + requires_project = True + + def syntax(self): + return "[options] " + + def short_desc(self): + return "Run my custom command" + + def add_options(self, parser): + super().add_options(parser) + parser.add_argument("--my-option", help="My custom option") + + def run(self, args, opts): + # Command implementation here + spider_name = args[0] if args else None + print(f"Running custom command for spider: {spider_name}") + +For real examples, see the built-in Scrapy commands in the `scrapy/commands`_ directory. + +.. _scrapy/commands: https://github.com/scrapy/scrapy/tree/master/scrapy/commands + +.. autoclass:: scrapy.commands.ScrapyCommand + :members: + :undoc-members: .. _Deploying your project: https://scrapyd.readthedocs.io/en/latest/deploy.html @@ -674,3 +723,19 @@ The following example adds ``my_command`` command: ], }, ) + +.. setting:: COMMANDS_MODULE + +COMMANDS_MODULE +--------------- + +Default: ``''`` (empty string) + +A module to use for looking up custom Scrapy commands. This is used to add custom +commands for your Scrapy project. + +Example: + +.. code-block:: python + + COMMANDS_MODULE = "mybot.commands" diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 4ce070e6e..ddf5f950f 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -23,6 +23,8 @@ if TYPE_CHECKING: class ScrapyCommand: + """Base class for all Scrapy commands.""" + requires_project: bool = False requires_crawler_process: bool = True crawler_process: CrawlerProcessBase | None = None # set in scrapy.cmdline From 028777d617d489460b2289f079a6614a90f6af37 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 17 Jul 2026 17:58:03 +0500 Subject: [PATCH 2/2] Improvements. --- docs/topics/commands.rst | 36 +++++++++++++++++++++++++----------- scrapy/commands/__init__.py | 17 +++++------------ 2 files changed, 30 insertions(+), 23 deletions(-) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 603789e2f..cc94bdab2 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -662,23 +662,37 @@ and overriding specific methods. Here's what you need to know: **Attributes you can set:** -* ``requires_project`` (bool): If True, the command requires a Scrapy project to be present (default: False) -* ``requires_crawler_process`` (bool): If True, the command requires a crawler process to be available (default: True) -* ``default_settings`` (dict): Default settings to use for this command instead of global defaults (default: {}) -* ``exitcode`` (int): Exit code to return when command completes (default: 0) +* :attr:`~scrapy.commands.ScrapyCommand.requires_project` (bool): If ``True``, + the command only runs inside a Scrapy project (default: ``False``). +* :attr:`~scrapy.commands.ScrapyCommand.requires_crawler_process` (bool): If + ``True``, a :class:`~scrapy.crawler.AsyncCrawlerProcess` or + :class:`~scrapy.crawler.CrawlerProcess` instance will be created by Scrapy + when the command runs and made available in the + :attr:`~scrapy.commands.ScrapyCommand.crawler_process` attribute (default: + ``True``). +* :attr:`~scrapy.commands.ScrapyCommand.default_settings` (dict): Settings that + will override the default ones when running this command (default: ``{}``). +* :attr:`~scrapy.commands.ScrapyCommand.exitcode` (int): Process exit code to + set when the command completes (default: ``0``). **Methods you must override:** -* :meth:`~scrapy.commands.ScrapyCommand.syntax`: Return command syntax (preferably one-line, without command name) -* :meth:`~scrapy.commands.ScrapyCommand.short_desc`: Return a short description of the command -* :meth:`~scrapy.commands.ScrapyCommand.run`: Main entry point for command execution (must implement) +* :meth:`~scrapy.commands.ScrapyCommand.short_desc`: Return a short description + of the command. +* :meth:`~scrapy.commands.ScrapyCommand.run`: Main entry point for the command + execution. **Methods you can override:** -* :meth:`~scrapy.commands.ScrapyCommand.long_desc`: Return a detailed description (can contain newlines) -* :meth:`~scrapy.commands.ScrapyCommand.help`: Return extensive help text (can contain newlines) -* :meth:`~scrapy.commands.ScrapyCommand.add_options`: Add command-specific options to argument parser -* :meth:`~scrapy.commands.ScrapyCommand.process_options`: Process parsed command-line options +* :meth:`~scrapy.commands.ScrapyCommand.syntax`: Return command syntax + (preferably one-line, without command name). +* :meth:`~scrapy.commands.ScrapyCommand.long_desc`: Return a detailed command + description. +* :meth:`~scrapy.commands.ScrapyCommand.add_options`: Add command-specific + options to the argument parser. +* :meth:`~scrapy.commands.ScrapyCommand.process_options`: Process parsed + command-line options and set settings before + :attr:`~scrapy.commands.ScrapyCommand.crawler_process` is instantiated. **Example custom command:** diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 5456d2684..ab22b9de2 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -60,16 +60,12 @@ class ScrapyCommand(ABC): self._crawler: Crawler = crawler def syntax(self) -> str: - """ - Command syntax (preferably one-line). Do not include command name. - """ + """Command syntax (preferably one-line). Do not include command name.""" return "" @abstractmethod def short_desc(self) -> str: - """ - A short description of the command - """ + """A short description of the command.""" return "" def long_desc(self) -> str: @@ -88,9 +84,7 @@ class ScrapyCommand(ABC): return self.long_desc() def add_options(self, parser: argparse.ArgumentParser) -> None: - """ - Populate option parse with options available for this command - """ + """Populate the option parser with the options available for this command.""" assert self.settings is not None group = parser.add_argument_group(title="Global Options") group.add_argument( @@ -124,6 +118,7 @@ class ScrapyCommand(ABC): group.add_argument("--pdb", action="store_true", help="enable pdb on failure") def process_options(self, args: list[str], opts: argparse.Namespace) -> None: + """Set settings based on the command line options.""" assert self.settings is not None try: self.settings.setdict(arglist_to_dict(opts.set), priority="cmdline") @@ -153,9 +148,7 @@ class ScrapyCommand(ABC): @abstractmethod def run(self, args: list[str], opts: argparse.Namespace) -> None: - """ - Entry point for running commands - """ + """Entry point for running commands.""" raise NotImplementedError