diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index ee2c3a3cd..cc94bdab2 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -645,25 +645,88 @@ Custom project commands ======================= You can also add your custom project commands by using the -:setting:`COMMANDS_MODULE` setting. See the Scrapy commands in -`scrapy/commands`_ for examples on how to implement your commands. +:setting:`COMMANDS_MODULE` setting. This allows you to create project-specific +commands that are automatically discovered and made available through the +``scrapy`` command-line tool. -.. _scrapy/commands: https://github.com/scrapy/scrapy/tree/master/scrapy/commands -.. setting:: COMMANDS_MODULE +Creating custom commands +------------------------ -COMMANDS_MODULE ---------------- +To create a custom command, inherit from the :class:`~scrapy.commands.ScrapyCommand` class +and implement the required methods. This allows you to extend Scrapy's command-line +interface with your own functionality, such as project-specific utilities, data +processing tools, or deployment helpers. -Default: ``''`` (empty string) +When you create a custom command, you define its behavior by setting class attributes +and overriding specific methods. Here's what you need to know: -A module to use for looking up custom Scrapy commands. This is used to add custom -commands for your Scrapy project. +**Attributes you can set:** -Example: +* :attr:`~scrapy.commands.ScrapyCommand.requires_project` (bool): If ``True``, + the command only runs inside a Scrapy project (default: ``False``). +* :attr:`~scrapy.commands.ScrapyCommand.requires_crawler_process` (bool): If + ``True``, a :class:`~scrapy.crawler.AsyncCrawlerProcess` or + :class:`~scrapy.crawler.CrawlerProcess` instance will be created by Scrapy + when the command runs and made available in the + :attr:`~scrapy.commands.ScrapyCommand.crawler_process` attribute (default: + ``True``). +* :attr:`~scrapy.commands.ScrapyCommand.default_settings` (dict): Settings that + will override the default ones when running this command (default: ``{}``). +* :attr:`~scrapy.commands.ScrapyCommand.exitcode` (int): Process exit code to + set when the command completes (default: ``0``). + +**Methods you must override:** + +* :meth:`~scrapy.commands.ScrapyCommand.short_desc`: Return a short description + of the command. +* :meth:`~scrapy.commands.ScrapyCommand.run`: Main entry point for the command + execution. + +**Methods you can override:** + +* :meth:`~scrapy.commands.ScrapyCommand.syntax`: Return command syntax + (preferably one-line, without command name). +* :meth:`~scrapy.commands.ScrapyCommand.long_desc`: Return a detailed command + description. +* :meth:`~scrapy.commands.ScrapyCommand.add_options`: Add command-specific + options to the argument parser. +* :meth:`~scrapy.commands.ScrapyCommand.process_options`: Process parsed + command-line options and set settings before + :attr:`~scrapy.commands.ScrapyCommand.crawler_process` is instantiated. + +**Example custom command:** .. code-block:: python - COMMANDS_MODULE = "mybot.commands" + from scrapy.commands import ScrapyCommand + import argparse + + + class MyCustomCommand(ScrapyCommand): + requires_project = True + + def syntax(self): + return "[options] " + + def short_desc(self): + return "Run my custom command" + + def add_options(self, parser): + super().add_options(parser) + parser.add_argument("--my-option", help="My custom option") + + def run(self, args, opts): + # Command implementation here + spider_name = args[0] if args else None + print(f"Running custom command for spider: {spider_name}") + +For real examples, see the built-in Scrapy commands in the `scrapy/commands`_ directory. + +.. _scrapy/commands: https://github.com/scrapy/scrapy/tree/master/scrapy/commands + +.. autoclass:: scrapy.commands.ScrapyCommand + :members: + :undoc-members: .. _Deploying your project: https://scrapyd.readthedocs.io/en/latest/deploy.html @@ -690,3 +753,19 @@ The following example adds ``my_command`` command: ], }, ) + +.. setting:: COMMANDS_MODULE + +COMMANDS_MODULE +--------------- + +Default: ``''`` (empty string) + +A module to use for looking up custom Scrapy commands. This is used to add custom +commands for your Scrapy project. + +Example: + +.. code-block:: python + + COMMANDS_MODULE = "mybot.commands" diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index d9c919db9..ab22b9de2 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -27,6 +27,8 @@ if TYPE_CHECKING: class ScrapyCommand(ABC): + """Base class for all Scrapy commands.""" + requires_project: bool = False requires_crawler_process: bool = True crawler_process: CrawlerProcessBase | None = None # set in scrapy.cmdline @@ -58,16 +60,12 @@ class ScrapyCommand(ABC): self._crawler: Crawler = crawler def syntax(self) -> str: - """ - Command syntax (preferably one-line). Do not include command name. - """ + """Command syntax (preferably one-line). Do not include command name.""" return "" @abstractmethod def short_desc(self) -> str: - """ - A short description of the command - """ + """A short description of the command.""" return "" def long_desc(self) -> str: @@ -86,9 +84,7 @@ class ScrapyCommand(ABC): return self.long_desc() def add_options(self, parser: argparse.ArgumentParser) -> None: - """ - Populate option parse with options available for this command - """ + """Populate the option parser with the options available for this command.""" assert self.settings is not None group = parser.add_argument_group(title="Global Options") group.add_argument( @@ -122,6 +118,7 @@ class ScrapyCommand(ABC): group.add_argument("--pdb", action="store_true", help="enable pdb on failure") def process_options(self, args: list[str], opts: argparse.Namespace) -> None: + """Set settings based on the command line options.""" assert self.settings is not None try: self.settings.setdict(arglist_to_dict(opts.set), priority="cmdline") @@ -151,9 +148,7 @@ class ScrapyCommand(ABC): @abstractmethod def run(self, args: list[str], opts: argparse.Namespace) -> None: - """ - Entry point for running commands - """ + """Entry point for running commands.""" raise NotImplementedError