mirror of https://github.com/scrapy/scrapy.git
Merge 19f3345737 into ad43bf0c56
This commit is contained in:
Adrian
GitHub
commit
93491335e8
|
|
@ -603,6 +603,50 @@ bench
|
|||
|
||||
Run a quick benchmark test. :ref:`benchmarking`.
|
||||
|
||||
.. command:: complete
|
||||
|
||||
complete
|
||||
--------
|
||||
|
||||
* Syntax: ``scrapy complete <bash|fish|zsh>``
|
||||
* Requires project: *no*
|
||||
|
||||
.. versionadded:: VERSION
|
||||
|
||||
Print a shell completion script. See :ref:`completion`.
|
||||
|
||||
.. _completion:
|
||||
|
||||
Shell completion
|
||||
================
|
||||
|
||||
.. versionadded:: VERSION
|
||||
|
||||
The :command:`complete` command prints a completion script for Bash, fish or
|
||||
Zsh.
|
||||
|
||||
For Bash, add the following to your ``~/.bashrc``:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
eval "$(scrapy complete bash)"
|
||||
|
||||
For Zsh, add the following to your ``~/.zshrc``, after ``compinit`` is called:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
eval "$(scrapy complete zsh)"
|
||||
|
||||
For fish:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
scrapy complete fish > ~/.config/fish/completions/scrapy.fish
|
||||
|
||||
Completion candidates are computed by Scrapy every time you press :kbd:`Tab`,
|
||||
so they cover the spiders, settings and templates of the project you are in, as
|
||||
well as any :ref:`custom command <custom-commands>`.
|
||||
|
||||
.. _topics-commands-crawlerprocess:
|
||||
|
||||
Commands that run a crawl
|
||||
|
|
@ -644,6 +688,8 @@ In this case you should set the :setting:`FORCE_CRAWLER_PROCESS` setting to
|
|||
``True`` (at the project level or via the command line) so that Scrapy uses
|
||||
:class:`~scrapy.crawler.CrawlerProcess` which supports all reactors.
|
||||
|
||||
.. _custom-commands:
|
||||
|
||||
Custom project commands
|
||||
=======================
|
||||
|
||||
|
|
@ -651,6 +697,35 @@ You can also add your custom project commands by using the
|
|||
:setting:`COMMANDS_MODULE` setting. See the Scrapy commands in
|
||||
`scrapy/commands`_ for examples on how to implement your commands.
|
||||
|
||||
To make a command support :ref:`shell completion <completion>`, override
|
||||
:meth:`~scrapy.commands.ScrapyCommand.complete_argument`,
|
||||
:meth:`~scrapy.commands.ScrapyCommand.complete_option` or both:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
from scrapy.commands import ScrapyCommand
|
||||
|
||||
|
||||
class Command(ScrapyCommand):
|
||||
def complete_argument(self, args: list[str]) -> Iterable[str]:
|
||||
return ["report", "summary"] if not args else []
|
||||
|
||||
def complete_option(self, dest: str) -> Iterable[str]:
|
||||
if dest == "format":
|
||||
return ["csv", "json"]
|
||||
return super().complete_option(dest)
|
||||
|
||||
Candidates are filtered by the word being typed, and options that define
|
||||
``choices`` are completed without any code on your side.
|
||||
|
||||
.. autoclass:: scrapy.commands.ScrapyCommand
|
||||
|
||||
.. automethod:: complete_argument
|
||||
|
||||
.. automethod:: complete_option
|
||||
|
||||
.. _scrapy/commands: https://github.com/scrapy/scrapy/tree/master/scrapy/commands
|
||||
.. setting:: COMMANDS_MODULE
|
||||
|
||||
|
|
|
|||
|
|
@ -1,20 +0,0 @@
|
|||
# bash completion for the Scrapy command-line tool
|
||||
|
||||
_scrapy_completion() {
|
||||
local cmd cur commands spiders
|
||||
cmd=${COMP_WORDS[1]}
|
||||
cur=${COMP_WORDS[2]}
|
||||
case "$cmd" in
|
||||
crawl|edit|check)
|
||||
spiders=$(scrapy list 2>/dev/null) || spiders=""
|
||||
COMPREPLY=(${COMPREPLY[@]:-} $(compgen -W "$spiders" -- "$cur"))
|
||||
;;
|
||||
*)
|
||||
if [ $COMP_CWORD -eq 1 ]; then
|
||||
commands="check crawl edit fetch genspider list parse runspider settings shell startproject version view"
|
||||
COMPREPLY=(${COMPREPLY[@]:-} $(compgen -W "$commands" -- "$cmd"))
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
}
|
||||
complete -F _scrapy_completion -o default scrapy
|
||||
|
|
@ -1,211 +0,0 @@
|
|||
#compdef scrapy
|
||||
_scrapy() {
|
||||
local context state state_descr line
|
||||
local ret=1
|
||||
typeset -A opt_args
|
||||
_arguments \
|
||||
"(- 1 *)"{-h,--help}"[Help]" \
|
||||
"1: :->command" \
|
||||
"*:: :->args" && ret=0
|
||||
|
||||
case $state in
|
||||
command)
|
||||
_scrapy_cmds
|
||||
;;
|
||||
args)
|
||||
case $words[1] in
|
||||
(bench)
|
||||
_scrapy_glb_opts
|
||||
;;
|
||||
(fetch)
|
||||
local options=(
|
||||
'--headers[print response HTTP headers instead of body]'
|
||||
'--no-redirect[do not handle HTTP 3xx status codes and print response as-is]'
|
||||
'--spider=[use this spider]:spider:_scrapy_spiders'
|
||||
'1::URL:_httpie_urls'
|
||||
)
|
||||
_scrapy_glb_opts $options
|
||||
;;
|
||||
(genspider)
|
||||
local options=(
|
||||
{'(--list)-l','(-l)--list'}'[List available templates]'
|
||||
{'(--edit)-e','(-e)--edit'}'[Edit spider after creating it]'
|
||||
'--force[If the spider already exists, overwrite it with the template]'
|
||||
{'(--dump)-d','(-d)--dump='}'[Dump template to standard output]:template:(basic crawl csvfeed xmlfeed)'
|
||||
{'(--template)-t','(-t)--template='}'[Uses a custom template]:template:(basic crawl csvfeed xmlfeed)'
|
||||
'1:name:(NAME)'
|
||||
'2:domain:_httpie_urls'
|
||||
)
|
||||
_scrapy_glb_opts $options
|
||||
;;
|
||||
(runspider)
|
||||
local options=(
|
||||
{'(--output)-o','(-o)--output='}'[dump scraped items into FILE (use - for stdout)]:file:_files'
|
||||
'*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)'
|
||||
'1:spider file:_files -g \*.py'
|
||||
)
|
||||
_scrapy_glb_opts $options
|
||||
;;
|
||||
(settings)
|
||||
local options=(
|
||||
'--get=[print raw setting value]:option:(SETTING)'
|
||||
'--getbool=[print setting value, interpreted as a boolean]:option:(SETTING)'
|
||||
'--getint=[print setting value, interpreted as an integer]:option:(SETTING)'
|
||||
'--getfloat=[print setting value, interpreted as a float]:option:(SETTING)'
|
||||
'--getlist=[print setting value, interpreted as a list]:option:(SETTING)'
|
||||
)
|
||||
_scrapy_glb_opts $options
|
||||
;;
|
||||
(shell)
|
||||
local options=(
|
||||
'-c[evaluate the code in the shell, print the result and exit]:code:(CODE)'
|
||||
'--no-redirect[do not handle HTTP 3xx status codes and print response as-is]'
|
||||
'--spider=[use this spider]:spider:_scrapy_spiders'
|
||||
'::file:_files -g \*.html'
|
||||
'::URL:_httpie_urls'
|
||||
)
|
||||
_scrapy_glb_opts $options
|
||||
;;
|
||||
(startproject)
|
||||
local options=(
|
||||
'1:name:(NAME)'
|
||||
'2:dir:_dir_list'
|
||||
)
|
||||
_scrapy_glb_opts $options
|
||||
;;
|
||||
(version)
|
||||
local options=(
|
||||
{'(--verbose)-v','(-v)--verbose'}'[also display twisted/python/platform info (useful for bug reports)]'
|
||||
)
|
||||
_scrapy_glb_opts $options
|
||||
;;
|
||||
(view)
|
||||
local options=(
|
||||
'--no-redirect[do not handle HTTP 3xx status codes and print response as-is]'
|
||||
'--spider=[use this spider]:spider:_scrapy_spiders'
|
||||
'1:URL:_httpie_urls'
|
||||
)
|
||||
_scrapy_glb_opts $options
|
||||
;;
|
||||
(check)
|
||||
local options=(
|
||||
{'(--list)-l','(-l)--list'}'[only list contracts, without checking them]'
|
||||
{'(--verbose)-v','(-v)--verbose'}'[print contract tests for all spiders]'
|
||||
'1:spider:_scrapy_spiders'
|
||||
)
|
||||
_scrapy_glb_opts $options
|
||||
;;
|
||||
(crawl)
|
||||
local options=(
|
||||
{'(--output)-o','(-o)--output='}'[dump scraped items into FILE (use - for stdout)]:file:_files'
|
||||
'*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)'
|
||||
'1:spider:_scrapy_spiders'
|
||||
)
|
||||
_scrapy_glb_opts $options
|
||||
;;
|
||||
(edit)
|
||||
local options=(
|
||||
'1:spider:_scrapy_spiders'
|
||||
)
|
||||
_scrapy_glb_opts $options
|
||||
;;
|
||||
(list)
|
||||
_scrapy_glb_opts
|
||||
;;
|
||||
(parse)
|
||||
local options=(
|
||||
'*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)'
|
||||
'--spider=[use this spider without looking for one]:spider:_scrapy_spiders'
|
||||
'--pipelines[process items through pipelines]'
|
||||
"--nolinks[don't show links to follow (extracted requests)]"
|
||||
"--noitems[don't show scraped items]"
|
||||
'--nocolour[avoid using pygments to colorize the output]'
|
||||
{'(--rules)-r','(-r)--rules'}'[use CrawlSpider rules to discover the callback]'
|
||||
{'(--callback)-c','(-c)--callback'}'[use this callback for parsing, instead looking for a callback]:callback:(CALLBACK)'
|
||||
{'(--meta)-m','(-m)--meta='}'[inject extra meta into the Request, it must be a valid raw json string]:meta:(META)'
|
||||
'--cbkwargs=[inject extra callback kwargs into the Request, it must be a valid raw json string]:arguments:(CBKWARGS)'
|
||||
{'(--depth)-d','(-d)--depth='}'[maximum depth for parsing requests (default: 1)]:depth:(DEPTH)'
|
||||
{'(--verbose)-v','(-v)--verbose'}'[print each depth level one by one]'
|
||||
'1:URL:_httpie_urls'
|
||||
)
|
||||
_scrapy_glb_opts $options
|
||||
;;
|
||||
esac
|
||||
;;
|
||||
esac
|
||||
|
||||
return ret
|
||||
}
|
||||
|
||||
_scrapy_cmds() {
|
||||
local -a commands project_commands
|
||||
commands=(
|
||||
'bench:Run quick benchmark test'
|
||||
'fetch:Fetch a URL using the Scrapy downloader'
|
||||
'genspider:Generate new spider using pre-defined templates'
|
||||
'runspider:Run a self-contained spider (without creating a project)'
|
||||
'settings:Get settings values'
|
||||
'shell:Interactive scraping console'
|
||||
'startproject:Create new project'
|
||||
'version:Print Scrapy version'
|
||||
'view:Open URL in browser, as seen by Scrapy'
|
||||
)
|
||||
project_commands=(
|
||||
'check:Check spider contracts'
|
||||
'crawl:Run a spider'
|
||||
'edit:Edit spider'
|
||||
'list:List available spiders'
|
||||
'parse:Parse URL (using its spider) and print the results'
|
||||
)
|
||||
if [[ $(scrapy -h | grep -s "no active project") == "" ]]; then
|
||||
commands=(${commands[@]} ${project_commands[@]})
|
||||
fi
|
||||
_describe -t common-commands 'common commands' commands && ret=0
|
||||
}
|
||||
|
||||
_scrapy_glb_opts() {
|
||||
local -a options
|
||||
options=(
|
||||
'(- *)'{-h,--help}'[show this help message and exit]'
|
||||
'(--nolog)--logfile=[log file. if omitted stderr will be used]:file:_files'
|
||||
'--pidfile=[write process ID to FILE]:file:_files'
|
||||
'--profile=[write python cProfile stats to FILE]:file:_files'
|
||||
{'(--loglevel --nolog)-L','(-L --nolog)--loglevel='}'[log level (default: INFO)]:log level:(DEBUG INFO WARN ERROR)'
|
||||
'(-L --loglevel --logfile)--nolog[disable logging completely]'
|
||||
'--pdb[enable pdb on failure]'
|
||||
'*'{-s,--set=}'[set/override setting (may be repeated)]:value pair:(NAME=VALUE)'
|
||||
)
|
||||
options=(${options[@]} "$@")
|
||||
_arguments -A "-*" $options && ret=0
|
||||
}
|
||||
|
||||
_httpie_urls() {
|
||||
|
||||
local ret=1
|
||||
|
||||
if ! [[ -prefix [-+.a-z0-9]#:// ]]; then
|
||||
local expl
|
||||
compset -S '[^:/]*' && compstate[to_end]=''
|
||||
_wanted url-schemas expl 'URL schema' compadd -S '' http:// https:// && ret=0
|
||||
else
|
||||
_urls && ret=0
|
||||
fi
|
||||
|
||||
return $ret
|
||||
|
||||
}
|
||||
|
||||
_scrapy_spiders() {
|
||||
|
||||
local ret=1
|
||||
|
||||
if [[ $(scrapy -h | grep -s "no active project") == "" ]]; then
|
||||
compadd -S '' $(scrapy list) && ret=0
|
||||
else
|
||||
compadd -S '' SPIDER && ret=0
|
||||
fi
|
||||
|
||||
return $ret
|
||||
}
|
||||
|
||||
_scrapy $@
|
||||
|
|
@ -150,6 +150,21 @@ def _print_unknown_command(
|
|||
print('Use "scrapy" to see available commands')
|
||||
|
||||
|
||||
def _build_parser(
|
||||
cmd: ScrapyCommand, cmdname: str, settings: Settings
|
||||
) -> ScrapyArgumentParser:
|
||||
parser = ScrapyArgumentParser(
|
||||
formatter_class=ScrapyHelpFormatter,
|
||||
usage=f"scrapy {cmdname} {cmd.syntax()}",
|
||||
conflict_handler="resolve",
|
||||
description=cmd.long_desc(),
|
||||
)
|
||||
settings.setdict(cmd.default_settings, priority="command")
|
||||
cmd.settings = settings
|
||||
cmd.add_options(parser)
|
||||
return parser
|
||||
|
||||
|
||||
def _run_print_help(
|
||||
parser: argparse.ArgumentParser,
|
||||
func: Callable[_P, None],
|
||||
|
|
@ -191,15 +206,7 @@ def execute(argv: list[str] | None = None, settings: Settings | None = None) ->
|
|||
sys.exit(2)
|
||||
|
||||
cmd = cmds[cmdname]
|
||||
parser = ScrapyArgumentParser(
|
||||
formatter_class=ScrapyHelpFormatter,
|
||||
usage=f"scrapy {cmdname} {cmd.syntax()}",
|
||||
conflict_handler="resolve",
|
||||
description=cmd.long_desc(),
|
||||
)
|
||||
settings.setdict(cmd.default_settings, priority="command")
|
||||
cmd.settings = settings
|
||||
cmd.add_options(parser)
|
||||
parser = _build_parser(cmd, cmdname, settings)
|
||||
opts, args = parser.parse_known_args(args=argv[1:])
|
||||
_run_print_help(parser, cmd.process_options, args, opts)
|
||||
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ from typing import TYPE_CHECKING, Any, ClassVar
|
|||
from twisted.python import failure
|
||||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning, UsageError
|
||||
from scrapy.spiderloader import get_spider_loader
|
||||
from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
|
||||
from scrapy.utils.deprecate import method_is_overridden
|
||||
from scrapy.utils.python import global_object_name
|
||||
|
|
@ -121,6 +122,39 @@ class ScrapyCommand(ABC):
|
|||
)
|
||||
group.add_argument("--pdb", action="store_true", help="enable pdb on failure")
|
||||
|
||||
def complete_argument(self, args: list[str]) -> Iterable[str]:
|
||||
"""Return shell completion candidates for the positional argument
|
||||
being typed, where *args* are the positional arguments typed before
|
||||
it.
|
||||
|
||||
.. versionadded:: VERSION
|
||||
|
||||
See :ref:`completion`.
|
||||
"""
|
||||
return ()
|
||||
|
||||
def complete_option(self, dest: str) -> Iterable[str]:
|
||||
"""Return shell completion candidates for the value of the option
|
||||
being typed, identified by *dest*, i.e. the name under which its value
|
||||
is available in the ``opts`` parameter of ``run()``.
|
||||
|
||||
.. versionadded:: VERSION
|
||||
|
||||
See :ref:`completion`.
|
||||
"""
|
||||
if dest == "loglevel":
|
||||
return ["CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG"]
|
||||
return ()
|
||||
|
||||
def _spider_names(self) -> list[str]:
|
||||
assert self.settings is not None
|
||||
try:
|
||||
return sorted(get_spider_loader(self.settings).list())
|
||||
except Exception:
|
||||
# Completion runs on incomplete command lines and outside
|
||||
# projects, where loading spiders is expected to fail.
|
||||
return []
|
||||
|
||||
def process_options(self, args: list[str], opts: argparse.Namespace) -> None:
|
||||
assert self.settings is not None
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import argparse
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from collections.abc import AsyncIterator
|
||||
from collections.abc import AsyncIterator, Iterable
|
||||
from typing import Any, ClassVar
|
||||
from unittest import TextTestResult as _TextTestResult
|
||||
from unittest import TextTestRunner
|
||||
|
|
@ -52,6 +52,9 @@ class Command(ScrapyCommand):
|
|||
def short_desc(self) -> str:
|
||||
return "Check spider contracts"
|
||||
|
||||
def complete_argument(self, args: list[str]) -> Iterable[str]:
|
||||
return () if args else self._spider_names()
|
||||
|
||||
def add_options(self, parser: argparse.ArgumentParser) -> None:
|
||||
super().add_options(parser)
|
||||
parser.add_argument(
|
||||
|
|
|
|||
|
|
@ -0,0 +1,167 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from typing import TYPE_CHECKING, Any, ClassVar
|
||||
|
||||
from scrapy.cmdline import _build_parser, _get_commands_dict
|
||||
from scrapy.commands import ScrapyCommand
|
||||
from scrapy.exceptions import UsageError
|
||||
from scrapy.utils.project import inside_project
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterable, Iterator
|
||||
|
||||
from scrapy.settings import Settings
|
||||
|
||||
|
||||
_BASH = """\
|
||||
_scrapy_completion() {
|
||||
local IFS=$'\\n'
|
||||
COMPREPLY=($(scrapy complete -- "${COMP_WORDS[@]:1:COMP_CWORD}" 2>/dev/null | cut -f1))
|
||||
}
|
||||
complete -o default -F _scrapy_completion scrapy
|
||||
"""
|
||||
|
||||
_ZSH = """\
|
||||
#compdef scrapy
|
||||
_scrapy() {
|
||||
local -a candidates
|
||||
candidates=(${(f)"$(scrapy complete -- "${(@)words[2,CURRENT]}" 2>/dev/null)"})
|
||||
if (( $#candidates )); then
|
||||
candidates=("${(@)candidates//$'\\t'/:}")
|
||||
_describe -t scrapy scrapy candidates
|
||||
else
|
||||
_default
|
||||
fi
|
||||
}
|
||||
compdef _scrapy scrapy
|
||||
"""
|
||||
|
||||
_FISH = """\
|
||||
function __scrapy_complete
|
||||
set -l tokens (commandline -opc)
|
||||
set -e tokens[1]
|
||||
set -l current (commandline -ct)
|
||||
set -l candidates (scrapy complete -- $tokens "$current" 2>/dev/null)
|
||||
if set -q candidates[1]
|
||||
printf '%s\\n' $candidates
|
||||
else
|
||||
__fish_complete_path "$current"
|
||||
end
|
||||
end
|
||||
complete -c scrapy -f -a '(__scrapy_complete)'
|
||||
"""
|
||||
|
||||
_SCRIPTS = {"bash": _BASH, "fish": _FISH, "zsh": _ZSH}
|
||||
|
||||
|
||||
def _positional_args(args: list[str], options: dict[str, argparse.Action]) -> list[str]:
|
||||
positional = []
|
||||
index = 0
|
||||
while index < len(args):
|
||||
arg = args[index]
|
||||
action = options.get(arg)
|
||||
if action is not None:
|
||||
index += 1 if action.nargs == 0 else 2
|
||||
elif arg.startswith("-"):
|
||||
index += 1
|
||||
else:
|
||||
positional.append(arg)
|
||||
index += 1
|
||||
return positional
|
||||
|
||||
|
||||
def _iter_candidates(
|
||||
settings: Settings, prefix: str, typed: list[str]
|
||||
) -> Iterator[tuple[str, str]]:
|
||||
"""Yield ``(value, description)`` pairs for the word being completed,
|
||||
where *prefix* is that word and *typed* are the words before it."""
|
||||
cmds = _get_commands_dict(settings, inside_project())
|
||||
if not typed:
|
||||
for name, command in sorted(cmds.items()):
|
||||
yield name, command.short_desc()
|
||||
return
|
||||
cmdname, args = typed[0], typed[1:]
|
||||
cmd = cmds.get(cmdname)
|
||||
if cmd is None:
|
||||
return
|
||||
parser = _build_parser(cmd, cmdname, settings)
|
||||
# argparse offers no public access to the arguments of a parser.
|
||||
actions = parser._actions
|
||||
options = {
|
||||
option_string: action
|
||||
for action in actions
|
||||
for option_string in action.option_strings
|
||||
}
|
||||
previous = options.get(args[-1]) if args else None
|
||||
if previous is not None and previous.nargs != 0:
|
||||
values: Iterable[Any] = previous.choices or cmd.complete_option(previous.dest)
|
||||
yield from ((str(value), "") for value in values)
|
||||
elif prefix.startswith("-"):
|
||||
for action in actions:
|
||||
if action.help == argparse.SUPPRESS:
|
||||
continue
|
||||
for option_string in action.option_strings:
|
||||
yield option_string, action.help or ""
|
||||
else:
|
||||
for value in cmd.complete_argument(_positional_args(args, options)):
|
||||
yield value, ""
|
||||
|
||||
|
||||
def _split_equal_signs(words: list[str]) -> tuple[list[str], str]:
|
||||
"""Return *words* with ``--option=value`` split into two words, along with
|
||||
the ``--option=`` part of the word being completed, which shells that keep
|
||||
it as a single word expect back in every candidate.
|
||||
|
||||
Bash splits on ``=`` on its own, into a separate word.
|
||||
"""
|
||||
if words and words[-1] == "=":
|
||||
words = [*words[:-1], ""]
|
||||
words = [
|
||||
word
|
||||
for index, word in enumerate(words)
|
||||
if not (word == "=" and index and words[index - 1].startswith("-"))
|
||||
]
|
||||
if words and words[-1].startswith("-") and "=" in words[-1]:
|
||||
option, _, value = words[-1].partition("=")
|
||||
return [*words[:-1], option, value], f"{option}="
|
||||
return words, ""
|
||||
|
||||
|
||||
def _candidates(settings: Settings, words: list[str]) -> Iterator[str]:
|
||||
words, inline = _split_equal_signs(words)
|
||||
prefix = words[-1] if words else ""
|
||||
for value, description in _iter_candidates(settings, prefix, words[:-1]):
|
||||
if value.startswith(prefix):
|
||||
candidate = f"{inline}{value}"
|
||||
yield f"{candidate}\t{description}" if description else candidate
|
||||
|
||||
|
||||
class Command(ScrapyCommand):
|
||||
requires_crawler_process = False
|
||||
default_settings: ClassVar[dict[str, Any]] = {"LOG_ENABLED": False}
|
||||
|
||||
def syntax(self) -> str:
|
||||
return "<bash|fish|zsh>"
|
||||
|
||||
def short_desc(self) -> str:
|
||||
return "Print a shell completion script"
|
||||
|
||||
def long_desc(self) -> str:
|
||||
return (
|
||||
"Print a completion script for the given shell, to be installed as "
|
||||
"described in the Scrapy documentation."
|
||||
)
|
||||
|
||||
def complete_argument(self, args: list[str]) -> Iterable[str]:
|
||||
return () if args else _SCRIPTS
|
||||
|
||||
def run(self, args: list[str], opts: argparse.Namespace) -> None:
|
||||
assert self.settings is not None
|
||||
if args and args[0] == "--":
|
||||
for candidate in _candidates(self.settings, args[1:]):
|
||||
print(candidate)
|
||||
elif len(args) == 1 and args[0] in _SCRIPTS:
|
||||
print(_SCRIPTS[args[0]], end="")
|
||||
else:
|
||||
raise UsageError
|
||||
|
|
@ -7,6 +7,7 @@ from scrapy.exceptions import UsageError
|
|||
|
||||
if TYPE_CHECKING:
|
||||
import argparse
|
||||
from collections.abc import Iterable
|
||||
|
||||
|
||||
class Command(BaseRunSpiderCommand):
|
||||
|
|
@ -18,6 +19,9 @@ class Command(BaseRunSpiderCommand):
|
|||
def short_desc(self) -> str:
|
||||
return "Run a spider of the current project, by name"
|
||||
|
||||
def complete_argument(self, args: list[str]) -> Iterable[str]:
|
||||
return () if args else self._spider_names()
|
||||
|
||||
def run(self, args: list[str], opts: argparse.Namespace) -> None:
|
||||
if len(args) < 1:
|
||||
raise UsageError
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ from scrapy.spiderloader import get_spider_loader
|
|||
|
||||
if TYPE_CHECKING:
|
||||
import argparse
|
||||
from collections.abc import Iterable
|
||||
|
||||
|
||||
def _edit_file(editor: str, file_path: str | os.PathLike[str]) -> int:
|
||||
|
|
@ -42,6 +43,9 @@ class Command(ScrapyCommand):
|
|||
" variable or else the EDITOR setting"
|
||||
)
|
||||
|
||||
def complete_argument(self, args: list[str]) -> Iterable[str]:
|
||||
return () if args else self._spider_names()
|
||||
|
||||
def _err(self, msg: str) -> None:
|
||||
sys.stderr.write(msg + os.linesep)
|
||||
self.exitcode = 1
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ from scrapy.utils.spider import DefaultSpider, spidercls_for_request
|
|||
|
||||
if TYPE_CHECKING:
|
||||
from argparse import ArgumentParser
|
||||
from collections.abc import AsyncIterator
|
||||
from collections.abc import AsyncIterator, Iterable
|
||||
|
||||
from scrapy import Spider
|
||||
|
||||
|
|
@ -32,6 +32,11 @@ class Command(ScrapyCommand):
|
|||
" to stdout. You may want to use --nolog to disable logging"
|
||||
)
|
||||
|
||||
def complete_option(self, dest: str) -> Iterable[str]:
|
||||
if dest == "spider":
|
||||
return self._spider_names()
|
||||
return super().complete_option(dest)
|
||||
|
||||
def add_options(self, parser: ArgumentParser) -> None:
|
||||
super().add_options(parser)
|
||||
parser.add_argument("--spider", dest="spider", help="use this spider")
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ from scrapy.utils.template import render_templatefile, string_camelcase
|
|||
if TYPE_CHECKING:
|
||||
import argparse
|
||||
import os
|
||||
from collections.abc import Iterable
|
||||
|
||||
|
||||
def sanitize_module_name(module_name: str) -> str:
|
||||
|
|
@ -53,6 +54,11 @@ class Command(ScrapyCommand):
|
|||
def short_desc(self) -> str:
|
||||
return "Generate new spider using pre-defined templates"
|
||||
|
||||
def complete_option(self, dest: str) -> Iterable[str]:
|
||||
if dest in {"dump", "template"}:
|
||||
return self._template_names()
|
||||
return super().complete_option(dest)
|
||||
|
||||
def add_options(self, parser: argparse.ArgumentParser) -> None:
|
||||
super().add_options(parser)
|
||||
parser.add_argument(
|
||||
|
|
@ -180,14 +186,17 @@ class Command(ScrapyCommand):
|
|||
)
|
||||
return None
|
||||
|
||||
def _template_names(self) -> list[str]:
|
||||
return [
|
||||
file.stem
|
||||
for file in sorted(Path(self.templates_dir).iterdir())
|
||||
if file.suffix == ".tmpl"
|
||||
]
|
||||
|
||||
def _list_templates(self) -> None:
|
||||
print(
|
||||
"Available templates:\n",
|
||||
"\n".join(
|
||||
f" {file.stem}"
|
||||
for file in sorted(Path(self.templates_dir).iterdir())
|
||||
if file.suffix == ".tmpl"
|
||||
),
|
||||
"\n".join(f" {name}" for name in self._template_names()),
|
||||
)
|
||||
|
||||
def _spider_exists(self, name: str) -> bool:
|
||||
|
|
|
|||
|
|
@ -51,6 +51,11 @@ class Command(BaseRunSpiderCommand):
|
|||
def short_desc(self) -> str:
|
||||
return "Parse URL (using its spider) and print the results"
|
||||
|
||||
def complete_option(self, dest: str) -> Iterable[str]:
|
||||
if dest == "spider":
|
||||
return self._spider_names()
|
||||
return super().complete_option(dest)
|
||||
|
||||
def add_options(self, parser: argparse.ArgumentParser) -> None:
|
||||
super().add_options(parser)
|
||||
parser.add_argument(
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import argparse
|
||||
import json
|
||||
from collections.abc import Iterable
|
||||
from typing import Any, ClassVar
|
||||
|
||||
from scrapy.commands import ScrapyCommand
|
||||
|
|
@ -16,6 +17,12 @@ class Command(ScrapyCommand):
|
|||
def short_desc(self) -> str:
|
||||
return "Get settings values"
|
||||
|
||||
def complete_option(self, dest: str) -> Iterable[str]:
|
||||
if dest.startswith("get"):
|
||||
assert self.settings is not None
|
||||
return sorted(self.settings)
|
||||
return super().complete_option(dest)
|
||||
|
||||
def add_options(self, parser: argparse.ArgumentParser) -> None:
|
||||
super().add_options(parser)
|
||||
parser.add_argument(
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ from scrapy.utils.url import guess_scheme
|
|||
|
||||
if TYPE_CHECKING:
|
||||
from argparse import ArgumentParser, Namespace
|
||||
from collections.abc import Iterable
|
||||
|
||||
from scrapy import Spider
|
||||
|
||||
|
|
@ -42,6 +43,11 @@ class Command(ScrapyCommand):
|
|||
"Use ./file.html syntax or full path for local file."
|
||||
)
|
||||
|
||||
def complete_option(self, dest: str) -> Iterable[str]:
|
||||
if dest == "spider":
|
||||
return self._spider_names()
|
||||
return super().complete_option(dest)
|
||||
|
||||
def add_options(self, parser: ArgumentParser) -> None:
|
||||
super().add_options(parser)
|
||||
parser.add_argument(
|
||||
|
|
|
|||
|
|
@ -0,0 +1,217 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.utils.bases.commands import TestProjectBase
|
||||
from tests.utils.cmdline import proc
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def candidates(*words: str, cwd: Path | None = None) -> list[str]:
|
||||
"""Return the completion values offered for the last of *words*."""
|
||||
returncode, out, err = proc("complete", "--", *words, cwd=cwd)
|
||||
assert returncode == 0, err
|
||||
return [line.partition("\t")[0] for line in out.splitlines()]
|
||||
|
||||
|
||||
class TestScripts:
|
||||
@pytest.mark.parametrize("shell", ["bash", "fish", "zsh"])
|
||||
def test_output(self, shell: str) -> None:
|
||||
returncode, out, err = proc("complete", shell)
|
||||
assert returncode == 0, err
|
||||
assert "scrapy complete --" in out
|
||||
|
||||
@pytest.mark.parametrize("args", [(), ("csh",), ("bash", "zsh")])
|
||||
def test_usage_error(self, args: tuple[str, ...]) -> None:
|
||||
returncode, out, _ = proc("complete", *args)
|
||||
assert returncode == 2
|
||||
assert "scrapy complete <bash|fish|zsh>" in out
|
||||
|
||||
def test_shell_names(self) -> None:
|
||||
assert candidates("complete", "") == ["bash", "fish", "zsh"]
|
||||
|
||||
def test_no_second_shell_name(self) -> None:
|
||||
assert candidates("complete", "bash", "") == []
|
||||
|
||||
|
||||
class TestCommandNames:
|
||||
def test_all(self) -> None:
|
||||
values = candidates("")
|
||||
assert "complete" in values
|
||||
assert "startproject" in values
|
||||
|
||||
def test_prefix(self) -> None:
|
||||
assert candidates("ver") == ["version"]
|
||||
|
||||
def test_descriptions(self) -> None:
|
||||
_, out, _ = proc("complete", "--", "version")
|
||||
assert out.strip() == "version\tPrint Scrapy version"
|
||||
|
||||
def test_unknown_command(self) -> None:
|
||||
assert candidates("nosuchcommand", "") == []
|
||||
|
||||
|
||||
class TestOptions(TestProjectBase):
|
||||
def test_names(self) -> None:
|
||||
values = candidates("version", "-")
|
||||
assert "-v" in values
|
||||
assert "--nolog" in values
|
||||
|
||||
def test_name_prefix(self) -> None:
|
||||
assert candidates("runspider", "--overw") == ["--overwrite-output"]
|
||||
|
||||
def test_suppressed_names(self) -> None:
|
||||
assert "--headers" not in candidates("view", "--")
|
||||
|
||||
def test_value(self) -> None:
|
||||
assert candidates("version", "-L", "") == [
|
||||
"CRITICAL",
|
||||
"ERROR",
|
||||
"WARNING",
|
||||
"INFO",
|
||||
"DEBUG",
|
||||
]
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("words", "expected"),
|
||||
[
|
||||
# Bash splits ``--loglevel=DE`` into separate words.
|
||||
(("--loglevel", "=", "DE"), ["DEBUG"]),
|
||||
(("--loglevel", "="), ["CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG"]),
|
||||
# fish and Zsh keep it as one word, and replace it as one word.
|
||||
(("--loglevel=DE",), ["--loglevel=DEBUG"]),
|
||||
],
|
||||
)
|
||||
def test_value_after_equal_sign(
|
||||
self, words: tuple[str, ...], expected: list[str]
|
||||
) -> None:
|
||||
assert candidates("version", *words) == expected
|
||||
|
||||
def test_value_without_candidates(self) -> None:
|
||||
assert candidates("version", "--logfile", "") == []
|
||||
|
||||
def test_templates(self) -> None:
|
||||
assert "crawl" in candidates("genspider", "-t", "")
|
||||
|
||||
def test_settings(self) -> None:
|
||||
assert candidates("settings", "--getbool", "COOKIES_EN") == ["COOKIES_ENABLED"]
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"command", ["fetch", "genspider", "parse", "settings", "shell"]
|
||||
)
|
||||
def test_shared_value(self, command: str, proj_path: Path) -> None:
|
||||
assert candidates(command, "-L", "DE", cwd=proj_path) == ["DEBUG"]
|
||||
|
||||
|
||||
class TestArguments(TestProjectBase):
|
||||
@pytest.fixture
|
||||
def spiders_path(self, proj_path: Path) -> Path:
|
||||
spiders = proj_path / self.project_name / "spiders"
|
||||
for name in ("alpha", "beta"):
|
||||
(spiders / f"{name}.py").write_text(
|
||||
f"import scrapy\n\n\nclass {name.title()}Spider(scrapy.Spider):\n"
|
||||
f" name = {name!r}\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return proj_path
|
||||
|
||||
@pytest.mark.parametrize("command", ["check", "crawl", "edit"])
|
||||
def test_spiders(self, command: str, spiders_path: Path) -> None:
|
||||
assert candidates(command, "", cwd=spiders_path) == ["alpha", "beta"]
|
||||
|
||||
def test_no_arguments(self) -> None:
|
||||
assert candidates("version", "") == []
|
||||
|
||||
def test_spiders_after_inline_option_value(self, spiders_path: Path) -> None:
|
||||
assert candidates("crawl", "--output=items.json", "", cwd=spiders_path) == [
|
||||
"alpha",
|
||||
"beta",
|
||||
]
|
||||
|
||||
def test_broken_spiders(self, spiders_path: Path) -> None:
|
||||
(spiders_path / self.project_name / "spiders" / "broken.py").write_text(
|
||||
"class Broken(", encoding="utf-8"
|
||||
)
|
||||
assert candidates("crawl", "", cwd=spiders_path) == []
|
||||
|
||||
def test_spiders_prefix(self, spiders_path: Path) -> None:
|
||||
assert candidates("crawl", "al", cwd=spiders_path) == ["alpha"]
|
||||
|
||||
@pytest.mark.parametrize("command", ["fetch", "parse", "shell"])
|
||||
def test_spiders_as_option_value(self, command: str, spiders_path: Path) -> None:
|
||||
assert candidates(command, "--spider", "b", cwd=spiders_path) == ["beta"]
|
||||
|
||||
def test_spiders_after_options(self, spiders_path: Path) -> None:
|
||||
assert candidates("crawl", "-L", "INFO", "", cwd=spiders_path) == [
|
||||
"alpha",
|
||||
"beta",
|
||||
]
|
||||
|
||||
def test_no_second_spider(self, spiders_path: Path) -> None:
|
||||
assert candidates("crawl", "alpha", "", cwd=spiders_path) == []
|
||||
|
||||
def test_outside_project(self) -> None:
|
||||
assert candidates("fetch", "--spider", "") == []
|
||||
|
||||
|
||||
class TestCustomCommand(TestProjectBase):
|
||||
command_code = """
|
||||
from collections.abc import Iterable
|
||||
|
||||
from scrapy.commands import ScrapyCommand
|
||||
|
||||
|
||||
class Command(ScrapyCommand):
|
||||
requires_crawler_process = False
|
||||
|
||||
def short_desc(self):
|
||||
return "A custom command"
|
||||
|
||||
def add_options(self, parser):
|
||||
super().add_options(parser)
|
||||
parser.add_argument("--flavor", choices=["salty", "sweet"])
|
||||
parser.add_argument("--color")
|
||||
|
||||
def complete_argument(self, args: list[str]) -> Iterable[str]:
|
||||
return ["first", "second"][len(args):]
|
||||
|
||||
def complete_option(self, dest: str) -> Iterable[str]:
|
||||
if dest == "color":
|
||||
return ["red", "green"]
|
||||
return super().complete_option(dest)
|
||||
|
||||
def run(self, args, opts):
|
||||
pass
|
||||
"""
|
||||
|
||||
@pytest.fixture
|
||||
def command_path(self, proj_path: Path) -> Path:
|
||||
proj_mod_path = proj_path / self.project_name
|
||||
commands = proj_mod_path / "commands"
|
||||
commands.mkdir()
|
||||
(commands / "__init__.py").touch()
|
||||
(commands / "custom.py").write_text(self.command_code, encoding="utf-8")
|
||||
self._append_settings(
|
||||
proj_mod_path, f"COMMANDS_MODULE = '{self.project_name}.commands'\n"
|
||||
)
|
||||
return proj_path
|
||||
|
||||
def test_name(self, command_path: Path) -> None:
|
||||
assert candidates("cust", cwd=command_path) == ["custom"]
|
||||
|
||||
def test_argument(self, command_path: Path) -> None:
|
||||
assert candidates("custom", "", cwd=command_path) == ["first", "second"]
|
||||
assert candidates("custom", "first", "", cwd=command_path) == ["second"]
|
||||
|
||||
def test_option_choices(self, command_path: Path) -> None:
|
||||
assert candidates("custom", "--flavor", "s", cwd=command_path) == [
|
||||
"salty",
|
||||
"sweet",
|
||||
]
|
||||
|
||||
def test_option(self, command_path: Path) -> None:
|
||||
assert candidates("custom", "--color", "", cwd=command_path) == ["red", "green"]
|
||||
Loading…
Reference in New Issue