diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 50da4593a..0ecb94205 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -603,6 +603,50 @@ bench Run a quick benchmark test. :ref:`benchmarking`. +.. command:: complete + +complete +-------- + +* Syntax: ``scrapy complete `` +* Requires project: *no* + +.. versionadded:: VERSION + +Print a shell completion script. See :ref:`completion`. + +.. _completion: + +Shell completion +================ + +.. versionadded:: VERSION + +The :command:`complete` command prints a completion script for Bash, fish or +Zsh. + +For Bash, add the following to your ``~/.bashrc``: + +.. code-block:: bash + + eval "$(scrapy complete bash)" + +For Zsh, add the following to your ``~/.zshrc``, after ``compinit`` is called: + +.. code-block:: bash + + eval "$(scrapy complete zsh)" + +For fish: + +.. code-block:: bash + + scrapy complete fish > ~/.config/fish/completions/scrapy.fish + +Completion candidates are computed by Scrapy every time you press :kbd:`Tab`, +so they cover the spiders, settings and templates of the project you are in, as +well as any :ref:`custom command `. + .. _topics-commands-crawlerprocess: Commands that run a crawl @@ -644,6 +688,8 @@ In this case you should set the :setting:`FORCE_CRAWLER_PROCESS` setting to ``True`` (at the project level or via the command line) so that Scrapy uses :class:`~scrapy.crawler.CrawlerProcess` which supports all reactors. +.. _custom-commands: + Custom project commands ======================= @@ -651,6 +697,35 @@ You can also add your custom project commands by using the :setting:`COMMANDS_MODULE` setting. See the Scrapy commands in `scrapy/commands`_ for examples on how to implement your commands. +To make a command support :ref:`shell completion `, override +:meth:`~scrapy.commands.ScrapyCommand.complete_argument`, +:meth:`~scrapy.commands.ScrapyCommand.complete_option` or both: + +.. code-block:: python + + from collections.abc import Iterable + + from scrapy.commands import ScrapyCommand + + + class Command(ScrapyCommand): + def complete_argument(self, args: list[str]) -> Iterable[str]: + return ["report", "summary"] if not args else [] + + def complete_option(self, dest: str) -> Iterable[str]: + if dest == "format": + return ["csv", "json"] + return super().complete_option(dest) + +Candidates are filtered by the word being typed, and options that define +``choices`` are completed without any code on your side. + +.. autoclass:: scrapy.commands.ScrapyCommand + + .. automethod:: complete_argument + + .. automethod:: complete_option + .. _scrapy/commands: https://github.com/scrapy/scrapy/tree/master/scrapy/commands .. setting:: COMMANDS_MODULE diff --git a/extras/scrapy_bash_completion b/extras/scrapy_bash_completion deleted file mode 100644 index 07a7d88c5..000000000 --- a/extras/scrapy_bash_completion +++ /dev/null @@ -1,20 +0,0 @@ -# bash completion for the Scrapy command-line tool - -_scrapy_completion() { - local cmd cur commands spiders - cmd=${COMP_WORDS[1]} - cur=${COMP_WORDS[2]} - case "$cmd" in - crawl|edit|check) - spiders=$(scrapy list 2>/dev/null) || spiders="" - COMPREPLY=(${COMPREPLY[@]:-} $(compgen -W "$spiders" -- "$cur")) - ;; - *) - if [ $COMP_CWORD -eq 1 ]; then - commands="check crawl edit fetch genspider list parse runspider settings shell startproject version view" - COMPREPLY=(${COMPREPLY[@]:-} $(compgen -W "$commands" -- "$cmd")) - fi - ;; - esac -} -complete -F _scrapy_completion -o default scrapy diff --git a/extras/scrapy_zsh_completion b/extras/scrapy_zsh_completion deleted file mode 100644 index 82eb77cc0..000000000 --- a/extras/scrapy_zsh_completion +++ /dev/null @@ -1,211 +0,0 @@ -#compdef scrapy -_scrapy() { - local context state state_descr line - local ret=1 - typeset -A opt_args - _arguments \ - "(- 1 *)"{-h,--help}"[Help]" \ - "1: :->command" \ - "*:: :->args" && ret=0 - - case $state in - command) - _scrapy_cmds - ;; - args) - case $words[1] in - (bench) - _scrapy_glb_opts - ;; - (fetch) - local options=( - '--headers[print response HTTP headers instead of body]' - '--no-redirect[do not handle HTTP 3xx status codes and print response as-is]' - '--spider=[use this spider]:spider:_scrapy_spiders' - '1::URL:_httpie_urls' - ) - _scrapy_glb_opts $options - ;; - (genspider) - local options=( - {'(--list)-l','(-l)--list'}'[List available templates]' - {'(--edit)-e','(-e)--edit'}'[Edit spider after creating it]' - '--force[If the spider already exists, overwrite it with the template]' - {'(--dump)-d','(-d)--dump='}'[Dump template to standard output]:template:(basic crawl csvfeed xmlfeed)' - {'(--template)-t','(-t)--template='}'[Uses a custom template]:template:(basic crawl csvfeed xmlfeed)' - '1:name:(NAME)' - '2:domain:_httpie_urls' - ) - _scrapy_glb_opts $options - ;; - (runspider) - local options=( - {'(--output)-o','(-o)--output='}'[dump scraped items into FILE (use - for stdout)]:file:_files' - '*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)' - '1:spider file:_files -g \*.py' - ) - _scrapy_glb_opts $options - ;; - (settings) - local options=( - '--get=[print raw setting value]:option:(SETTING)' - '--getbool=[print setting value, interpreted as a boolean]:option:(SETTING)' - '--getint=[print setting value, interpreted as an integer]:option:(SETTING)' - '--getfloat=[print setting value, interpreted as a float]:option:(SETTING)' - '--getlist=[print setting value, interpreted as a list]:option:(SETTING)' - ) - _scrapy_glb_opts $options - ;; - (shell) - local options=( - '-c[evaluate the code in the shell, print the result and exit]:code:(CODE)' - '--no-redirect[do not handle HTTP 3xx status codes and print response as-is]' - '--spider=[use this spider]:spider:_scrapy_spiders' - '::file:_files -g \*.html' - '::URL:_httpie_urls' - ) - _scrapy_glb_opts $options - ;; - (startproject) - local options=( - '1:name:(NAME)' - '2:dir:_dir_list' - ) - _scrapy_glb_opts $options - ;; - (version) - local options=( - {'(--verbose)-v','(-v)--verbose'}'[also display twisted/python/platform info (useful for bug reports)]' - ) - _scrapy_glb_opts $options - ;; - (view) - local options=( - '--no-redirect[do not handle HTTP 3xx status codes and print response as-is]' - '--spider=[use this spider]:spider:_scrapy_spiders' - '1:URL:_httpie_urls' - ) - _scrapy_glb_opts $options - ;; - (check) - local options=( - {'(--list)-l','(-l)--list'}'[only list contracts, without checking them]' - {'(--verbose)-v','(-v)--verbose'}'[print contract tests for all spiders]' - '1:spider:_scrapy_spiders' - ) - _scrapy_glb_opts $options - ;; - (crawl) - local options=( - {'(--output)-o','(-o)--output='}'[dump scraped items into FILE (use - for stdout)]:file:_files' - '*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)' - '1:spider:_scrapy_spiders' - ) - _scrapy_glb_opts $options - ;; - (edit) - local options=( - '1:spider:_scrapy_spiders' - ) - _scrapy_glb_opts $options - ;; - (list) - _scrapy_glb_opts - ;; - (parse) - local options=( - '*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)' - '--spider=[use this spider without looking for one]:spider:_scrapy_spiders' - '--pipelines[process items through pipelines]' - "--nolinks[don't show links to follow (extracted requests)]" - "--noitems[don't show scraped items]" - '--nocolour[avoid using pygments to colorize the output]' - {'(--rules)-r','(-r)--rules'}'[use CrawlSpider rules to discover the callback]' - {'(--callback)-c','(-c)--callback'}'[use this callback for parsing, instead looking for a callback]:callback:(CALLBACK)' - {'(--meta)-m','(-m)--meta='}'[inject extra meta into the Request, it must be a valid raw json string]:meta:(META)' - '--cbkwargs=[inject extra callback kwargs into the Request, it must be a valid raw json string]:arguments:(CBKWARGS)' - {'(--depth)-d','(-d)--depth='}'[maximum depth for parsing requests (default: 1)]:depth:(DEPTH)' - {'(--verbose)-v','(-v)--verbose'}'[print each depth level one by one]' - '1:URL:_httpie_urls' - ) - _scrapy_glb_opts $options - ;; - esac - ;; - esac - - return ret -} - -_scrapy_cmds() { - local -a commands project_commands - commands=( - 'bench:Run quick benchmark test' - 'fetch:Fetch a URL using the Scrapy downloader' - 'genspider:Generate new spider using pre-defined templates' - 'runspider:Run a self-contained spider (without creating a project)' - 'settings:Get settings values' - 'shell:Interactive scraping console' - 'startproject:Create new project' - 'version:Print Scrapy version' - 'view:Open URL in browser, as seen by Scrapy' - ) - project_commands=( - 'check:Check spider contracts' - 'crawl:Run a spider' - 'edit:Edit spider' - 'list:List available spiders' - 'parse:Parse URL (using its spider) and print the results' - ) - if [[ $(scrapy -h | grep -s "no active project") == "" ]]; then - commands=(${commands[@]} ${project_commands[@]}) - fi - _describe -t common-commands 'common commands' commands && ret=0 -} - -_scrapy_glb_opts() { - local -a options - options=( - '(- *)'{-h,--help}'[show this help message and exit]' - '(--nolog)--logfile=[log file. if omitted stderr will be used]:file:_files' - '--pidfile=[write process ID to FILE]:file:_files' - '--profile=[write python cProfile stats to FILE]:file:_files' - {'(--loglevel --nolog)-L','(-L --nolog)--loglevel='}'[log level (default: INFO)]:log level:(DEBUG INFO WARN ERROR)' - '(-L --loglevel --logfile)--nolog[disable logging completely]' - '--pdb[enable pdb on failure]' - '*'{-s,--set=}'[set/override setting (may be repeated)]:value pair:(NAME=VALUE)' - ) - options=(${options[@]} "$@") - _arguments -A "-*" $options && ret=0 -} - -_httpie_urls() { - - local ret=1 - - if ! [[ -prefix [-+.a-z0-9]#:// ]]; then - local expl - compset -S '[^:/]*' && compstate[to_end]='' - _wanted url-schemas expl 'URL schema' compadd -S '' http:// https:// && ret=0 - else - _urls && ret=0 - fi - - return $ret - -} - -_scrapy_spiders() { - - local ret=1 - - if [[ $(scrapy -h | grep -s "no active project") == "" ]]; then - compadd -S '' $(scrapy list) && ret=0 - else - compadd -S '' SPIDER && ret=0 - fi - - return $ret -} - -_scrapy $@ diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index e6d5ff96a..45a5485ab 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -150,6 +150,21 @@ def _print_unknown_command( print('Use "scrapy" to see available commands') +def _build_parser( + cmd: ScrapyCommand, cmdname: str, settings: Settings +) -> ScrapyArgumentParser: + parser = ScrapyArgumentParser( + formatter_class=ScrapyHelpFormatter, + usage=f"scrapy {cmdname} {cmd.syntax()}", + conflict_handler="resolve", + description=cmd.long_desc(), + ) + settings.setdict(cmd.default_settings, priority="command") + cmd.settings = settings + cmd.add_options(parser) + return parser + + def _run_print_help( parser: argparse.ArgumentParser, func: Callable[_P, None], @@ -191,15 +206,7 @@ def execute(argv: list[str] | None = None, settings: Settings | None = None) -> sys.exit(2) cmd = cmds[cmdname] - parser = ScrapyArgumentParser( - formatter_class=ScrapyHelpFormatter, - usage=f"scrapy {cmdname} {cmd.syntax()}", - conflict_handler="resolve", - description=cmd.long_desc(), - ) - settings.setdict(cmd.default_settings, priority="command") - cmd.settings = settings - cmd.add_options(parser) + parser = _build_parser(cmd, cmdname, settings) opts, args = parser.parse_known_args(args=argv[1:]) _run_print_help(parser, cmd.process_options, args, opts) diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index d9c919db9..57ce8467f 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -15,6 +15,7 @@ from typing import TYPE_CHECKING, Any, ClassVar from twisted.python import failure from scrapy.exceptions import ScrapyDeprecationWarning, UsageError +from scrapy.spiderloader import get_spider_loader from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli from scrapy.utils.deprecate import method_is_overridden from scrapy.utils.python import global_object_name @@ -121,6 +122,39 @@ class ScrapyCommand(ABC): ) group.add_argument("--pdb", action="store_true", help="enable pdb on failure") + def complete_argument(self, args: list[str]) -> Iterable[str]: + """Return shell completion candidates for the positional argument + being typed, where *args* are the positional arguments typed before + it. + + .. versionadded:: VERSION + + See :ref:`completion`. + """ + return () + + def complete_option(self, dest: str) -> Iterable[str]: + """Return shell completion candidates for the value of the option + being typed, identified by *dest*, i.e. the name under which its value + is available in the ``opts`` parameter of ``run()``. + + .. versionadded:: VERSION + + See :ref:`completion`. + """ + if dest == "loglevel": + return ["CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG"] + return () + + def _spider_names(self) -> list[str]: + assert self.settings is not None + try: + return sorted(get_spider_loader(self.settings).list()) + except Exception: + # Completion runs on incomplete command lines and outside + # projects, where loading spiders is expected to fail. + return [] + def process_options(self, args: list[str], opts: argparse.Namespace) -> None: assert self.settings is not None try: diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index 2113c19d2..03b0b9080 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -1,7 +1,7 @@ import argparse import time from collections import defaultdict -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Iterable from typing import Any, ClassVar from unittest import TextTestResult as _TextTestResult from unittest import TextTestRunner @@ -52,6 +52,9 @@ class Command(ScrapyCommand): def short_desc(self) -> str: return "Check spider contracts" + def complete_argument(self, args: list[str]) -> Iterable[str]: + return () if args else self._spider_names() + def add_options(self, parser: argparse.ArgumentParser) -> None: super().add_options(parser) parser.add_argument( diff --git a/scrapy/commands/complete.py b/scrapy/commands/complete.py new file mode 100644 index 000000000..5130a8f38 --- /dev/null +++ b/scrapy/commands/complete.py @@ -0,0 +1,167 @@ +from __future__ import annotations + +import argparse +from typing import TYPE_CHECKING, Any, ClassVar + +from scrapy.cmdline import _build_parser, _get_commands_dict +from scrapy.commands import ScrapyCommand +from scrapy.exceptions import UsageError +from scrapy.utils.project import inside_project + +if TYPE_CHECKING: + from collections.abc import Iterable, Iterator + + from scrapy.settings import Settings + + +_BASH = """\ +_scrapy_completion() { + local IFS=$'\\n' + COMPREPLY=($(scrapy complete -- "${COMP_WORDS[@]:1:COMP_CWORD}" 2>/dev/null | cut -f1)) +} +complete -o default -F _scrapy_completion scrapy +""" + +_ZSH = """\ +#compdef scrapy +_scrapy() { + local -a candidates + candidates=(${(f)"$(scrapy complete -- "${(@)words[2,CURRENT]}" 2>/dev/null)"}) + if (( $#candidates )); then + candidates=("${(@)candidates//$'\\t'/:}") + _describe -t scrapy scrapy candidates + else + _default + fi +} +compdef _scrapy scrapy +""" + +_FISH = """\ +function __scrapy_complete + set -l tokens (commandline -opc) + set -e tokens[1] + set -l current (commandline -ct) + set -l candidates (scrapy complete -- $tokens "$current" 2>/dev/null) + if set -q candidates[1] + printf '%s\\n' $candidates + else + __fish_complete_path "$current" + end +end +complete -c scrapy -f -a '(__scrapy_complete)' +""" + +_SCRIPTS = {"bash": _BASH, "fish": _FISH, "zsh": _ZSH} + + +def _positional_args(args: list[str], options: dict[str, argparse.Action]) -> list[str]: + positional = [] + index = 0 + while index < len(args): + arg = args[index] + action = options.get(arg) + if action is not None: + index += 1 if action.nargs == 0 else 2 + elif arg.startswith("-"): + index += 1 + else: + positional.append(arg) + index += 1 + return positional + + +def _iter_candidates( + settings: Settings, prefix: str, typed: list[str] +) -> Iterator[tuple[str, str]]: + """Yield ``(value, description)`` pairs for the word being completed, + where *prefix* is that word and *typed* are the words before it.""" + cmds = _get_commands_dict(settings, inside_project()) + if not typed: + for name, command in sorted(cmds.items()): + yield name, command.short_desc() + return + cmdname, args = typed[0], typed[1:] + cmd = cmds.get(cmdname) + if cmd is None: + return + parser = _build_parser(cmd, cmdname, settings) + # argparse offers no public access to the arguments of a parser. + actions = parser._actions + options = { + option_string: action + for action in actions + for option_string in action.option_strings + } + previous = options.get(args[-1]) if args else None + if previous is not None and previous.nargs != 0: + values: Iterable[Any] = previous.choices or cmd.complete_option(previous.dest) + yield from ((str(value), "") for value in values) + elif prefix.startswith("-"): + for action in actions: + if action.help == argparse.SUPPRESS: + continue + for option_string in action.option_strings: + yield option_string, action.help or "" + else: + for value in cmd.complete_argument(_positional_args(args, options)): + yield value, "" + + +def _split_equal_signs(words: list[str]) -> tuple[list[str], str]: + """Return *words* with ``--option=value`` split into two words, along with + the ``--option=`` part of the word being completed, which shells that keep + it as a single word expect back in every candidate. + + Bash splits on ``=`` on its own, into a separate word. + """ + if words and words[-1] == "=": + words = [*words[:-1], ""] + words = [ + word + for index, word in enumerate(words) + if not (word == "=" and index and words[index - 1].startswith("-")) + ] + if words and words[-1].startswith("-") and "=" in words[-1]: + option, _, value = words[-1].partition("=") + return [*words[:-1], option, value], f"{option}=" + return words, "" + + +def _candidates(settings: Settings, words: list[str]) -> Iterator[str]: + words, inline = _split_equal_signs(words) + prefix = words[-1] if words else "" + for value, description in _iter_candidates(settings, prefix, words[:-1]): + if value.startswith(prefix): + candidate = f"{inline}{value}" + yield f"{candidate}\t{description}" if description else candidate + + +class Command(ScrapyCommand): + requires_crawler_process = False + default_settings: ClassVar[dict[str, Any]] = {"LOG_ENABLED": False} + + def syntax(self) -> str: + return "" + + def short_desc(self) -> str: + return "Print a shell completion script" + + def long_desc(self) -> str: + return ( + "Print a completion script for the given shell, to be installed as " + "described in the Scrapy documentation." + ) + + def complete_argument(self, args: list[str]) -> Iterable[str]: + return () if args else _SCRIPTS + + def run(self, args: list[str], opts: argparse.Namespace) -> None: + assert self.settings is not None + if args and args[0] == "--": + for candidate in _candidates(self.settings, args[1:]): + print(candidate) + elif len(args) == 1 and args[0] in _SCRIPTS: + print(_SCRIPTS[args[0]], end="") + else: + raise UsageError diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index 4e086e057..f991a6f23 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -7,6 +7,7 @@ from scrapy.exceptions import UsageError if TYPE_CHECKING: import argparse + from collections.abc import Iterable class Command(BaseRunSpiderCommand): @@ -18,6 +19,9 @@ class Command(BaseRunSpiderCommand): def short_desc(self) -> str: return "Run a spider of the current project, by name" + def complete_argument(self, args: list[str]) -> Iterable[str]: + return () if args else self._spider_names() + def run(self, args: list[str], opts: argparse.Namespace) -> None: if len(args) < 1: raise UsageError diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py index fa75cb09c..2c99b6981 100644 --- a/scrapy/commands/edit.py +++ b/scrapy/commands/edit.py @@ -13,6 +13,7 @@ from scrapy.spiderloader import get_spider_loader if TYPE_CHECKING: import argparse + from collections.abc import Iterable def _edit_file(editor: str, file_path: str | os.PathLike[str]) -> int: @@ -42,6 +43,9 @@ class Command(ScrapyCommand): " variable or else the EDITOR setting" ) + def complete_argument(self, args: list[str]) -> Iterable[str]: + return () if args else self._spider_names() + def _err(self, msg: str) -> None: sys.stderr.write(msg + os.linesep) self.exitcode = 1 diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index 0b8311efb..42a8f3747 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -14,7 +14,7 @@ from scrapy.utils.spider import DefaultSpider, spidercls_for_request if TYPE_CHECKING: from argparse import ArgumentParser - from collections.abc import AsyncIterator + from collections.abc import AsyncIterator, Iterable from scrapy import Spider @@ -32,6 +32,11 @@ class Command(ScrapyCommand): " to stdout. You may want to use --nolog to disable logging" ) + def complete_option(self, dest: str) -> Iterable[str]: + if dest == "spider": + return self._spider_names() + return super().complete_option(dest) + def add_options(self, parser: ArgumentParser) -> None: super().add_options(parser) parser.add_argument("--spider", dest="spider", help="use this spider") diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 52f9cd4b0..1b845e452 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -17,6 +17,7 @@ from scrapy.utils.template import render_templatefile, string_camelcase if TYPE_CHECKING: import argparse import os + from collections.abc import Iterable def sanitize_module_name(module_name: str) -> str: @@ -53,6 +54,11 @@ class Command(ScrapyCommand): def short_desc(self) -> str: return "Generate new spider using pre-defined templates" + def complete_option(self, dest: str) -> Iterable[str]: + if dest in {"dump", "template"}: + return self._template_names() + return super().complete_option(dest) + def add_options(self, parser: argparse.ArgumentParser) -> None: super().add_options(parser) parser.add_argument( @@ -180,14 +186,17 @@ class Command(ScrapyCommand): ) return None + def _template_names(self) -> list[str]: + return [ + file.stem + for file in sorted(Path(self.templates_dir).iterdir()) + if file.suffix == ".tmpl" + ] + def _list_templates(self) -> None: print( "Available templates:\n", - "\n".join( - f" {file.stem}" - for file in sorted(Path(self.templates_dir).iterdir()) - if file.suffix == ".tmpl" - ), + "\n".join(f" {name}" for name in self._template_names()), ) def _spider_exists(self, name: str) -> bool: diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 51caed57f..b09d93448 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -51,6 +51,11 @@ class Command(BaseRunSpiderCommand): def short_desc(self) -> str: return "Parse URL (using its spider) and print the results" + def complete_option(self, dest: str) -> Iterable[str]: + if dest == "spider": + return self._spider_names() + return super().complete_option(dest) + def add_options(self, parser: argparse.ArgumentParser) -> None: super().add_options(parser) parser.add_argument( diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py index 26a97ccbb..70eccb915 100644 --- a/scrapy/commands/settings.py +++ b/scrapy/commands/settings.py @@ -1,5 +1,6 @@ import argparse import json +from collections.abc import Iterable from typing import Any, ClassVar from scrapy.commands import ScrapyCommand @@ -16,6 +17,12 @@ class Command(ScrapyCommand): def short_desc(self) -> str: return "Get settings values" + def complete_option(self, dest: str) -> Iterable[str]: + if dest.startswith("get"): + assert self.settings is not None + return sorted(self.settings) + return super().complete_option(dest) + def add_options(self, parser: argparse.ArgumentParser) -> None: super().add_options(parser) parser.add_argument( diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 19138ffd0..4c7c1933d 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -20,6 +20,7 @@ from scrapy.utils.url import guess_scheme if TYPE_CHECKING: from argparse import ArgumentParser, Namespace + from collections.abc import Iterable from scrapy import Spider @@ -43,6 +44,11 @@ class Command(ScrapyCommand): "Use ./file.html syntax or full path for local file." ) + def complete_option(self, dest: str) -> Iterable[str]: + if dest == "spider": + return self._spider_names() + return super().complete_option(dest) + def add_options(self, parser: ArgumentParser) -> None: super().add_options(parser) parser.add_argument( diff --git a/tests/test_command_complete.py b/tests/test_command_complete.py new file mode 100644 index 000000000..8af00ef86 --- /dev/null +++ b/tests/test_command_complete.py @@ -0,0 +1,189 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +from tests.utils.bases.commands import TestProjectBase +from tests.utils.cmdline import proc + +if TYPE_CHECKING: + from pathlib import Path + + +def candidates(*words: str, cwd: Path | None = None) -> list[str]: + """Return the completion values offered for the last of *words*.""" + returncode, out, err = proc("complete", "--", *words, cwd=cwd) + assert returncode == 0, err + return [line.partition("\t")[0] for line in out.splitlines()] + + +class TestScripts: + @pytest.mark.parametrize("shell", ["bash", "fish", "zsh"]) + def test_output(self, shell: str) -> None: + returncode, out, err = proc("complete", shell) + assert returncode == 0, err + assert "scrapy complete --" in out + + @pytest.mark.parametrize("args", [(), ("csh",), ("bash", "zsh")]) + def test_usage_error(self, args: tuple[str, ...]) -> None: + returncode, out, _ = proc("complete", *args) + assert returncode == 2 + assert "scrapy complete " in out + + +class TestCommandNames: + def test_all(self) -> None: + values = candidates("") + assert "complete" in values + assert "startproject" in values + + def test_prefix(self) -> None: + assert candidates("ver") == ["version"] + + def test_descriptions(self) -> None: + _, out, _ = proc("complete", "--", "version") + assert out.strip() == "version\tPrint Scrapy version" + + def test_unknown_command(self) -> None: + assert candidates("nosuchcommand", "") == [] + + +class TestOptions: + def test_names(self) -> None: + values = candidates("version", "-") + assert "-v" in values + assert "--nolog" in values + + def test_name_prefix(self) -> None: + assert candidates("runspider", "--overw") == ["--overwrite-output"] + + def test_suppressed_names(self) -> None: + assert "--headers" not in candidates("view", "--") + + def test_value(self) -> None: + assert candidates("version", "-L", "") == [ + "CRITICAL", + "ERROR", + "WARNING", + "INFO", + "DEBUG", + ] + + @pytest.mark.parametrize( + ("words", "expected"), + [ + # Bash splits ``--loglevel=DE`` into separate words. + (("--loglevel", "=", "DE"), ["DEBUG"]), + (("--loglevel", "="), ["CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG"]), + # fish and Zsh keep it as one word, and replace it as one word. + (("--loglevel=DE",), ["--loglevel=DEBUG"]), + ], + ) + def test_value_after_equal_sign( + self, words: tuple[str, ...], expected: list[str] + ) -> None: + assert candidates("version", *words) == expected + + def test_value_without_candidates(self) -> None: + assert candidates("version", "--logfile", "") == [] + + def test_templates(self) -> None: + assert "crawl" in candidates("genspider", "-t", "") + + def test_settings(self) -> None: + assert candidates("settings", "--getbool", "COOKIES_EN") == ["COOKIES_ENABLED"] + + +class TestArguments(TestProjectBase): + @pytest.fixture + def spiders_path(self, proj_path: Path) -> Path: + spiders = proj_path / self.project_name / "spiders" + for name in ("alpha", "beta"): + (spiders / f"{name}.py").write_text( + f"import scrapy\n\n\nclass {name.title()}Spider(scrapy.Spider):\n" + f" name = {name!r}\n", + encoding="utf-8", + ) + return proj_path + + @pytest.mark.parametrize("command", ["check", "crawl", "edit"]) + def test_spiders(self, command: str, spiders_path: Path) -> None: + assert candidates(command, "", cwd=spiders_path) == ["alpha", "beta"] + + def test_spiders_prefix(self, spiders_path: Path) -> None: + assert candidates("crawl", "al", cwd=spiders_path) == ["alpha"] + + def test_spiders_as_option_value(self, spiders_path: Path) -> None: + assert candidates("parse", "--spider", "b", cwd=spiders_path) == ["beta"] + + def test_spiders_after_options(self, spiders_path: Path) -> None: + assert candidates("crawl", "-L", "INFO", "", cwd=spiders_path) == [ + "alpha", + "beta", + ] + + def test_no_second_spider(self, spiders_path: Path) -> None: + assert candidates("crawl", "alpha", "", cwd=spiders_path) == [] + + def test_outside_project(self) -> None: + assert candidates("fetch", "--spider", "") == [] + + +class TestCustomCommand(TestProjectBase): + command_code = """ +from collections.abc import Iterable + +from scrapy.commands import ScrapyCommand + + +class Command(ScrapyCommand): + requires_crawler_process = False + + def short_desc(self): + return "A custom command" + + def add_options(self, parser): + super().add_options(parser) + parser.add_argument("--flavor", choices=["salty", "sweet"]) + parser.add_argument("--color") + + def complete_argument(self, args: list[str]) -> Iterable[str]: + return ["first", "second"][len(args):] + + def complete_option(self, dest: str) -> Iterable[str]: + if dest == "color": + return ["red", "green"] + return super().complete_option(dest) + + def run(self, args, opts): + pass +""" + + @pytest.fixture + def command_path(self, proj_path: Path) -> Path: + proj_mod_path = proj_path / self.project_name + commands = proj_mod_path / "commands" + commands.mkdir() + (commands / "__init__.py").touch() + (commands / "custom.py").write_text(self.command_code, encoding="utf-8") + self._append_settings( + proj_mod_path, f"COMMANDS_MODULE = '{self.project_name}.commands'\n" + ) + return proj_path + + def test_name(self, command_path: Path) -> None: + assert candidates("cust", cwd=command_path) == ["custom"] + + def test_argument(self, command_path: Path) -> None: + assert candidates("custom", "", cwd=command_path) == ["first", "second"] + assert candidates("custom", "first", "", cwd=command_path) == ["second"] + + def test_option_choices(self, command_path: Path) -> None: + assert candidates("custom", "--flavor", "s", cwd=command_path) == [ + "salty", + "sweet", + ] + + def test_option(self, command_path: Path) -> None: + assert candidates("custom", "--color", "", cwd=command_path) == ["red", "green"]