Full typing for scrapy/cmdline.py.

scrapy/cmdline.py

@@ -4,18 +4,22 @@ import inspect
 import os
 import sys
 from importlib.metadata import entry_points
+from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Type
 
 import scrapy
 from scrapy.commands import BaseRunSpiderCommand, ScrapyCommand, ScrapyHelpFormatter
 from scrapy.crawler import CrawlerProcess
 from scrapy.exceptions import UsageError
+from scrapy.settings import BaseSettings, Settings
 from scrapy.utils.misc import walk_modules
 from scrapy.utils.project import get_project_settings, inside_project
 from scrapy.utils.python import garbage_collect
 
 
 class ScrapyArgumentParser(argparse.ArgumentParser):
-    def _parse_optional(self, arg_string):
+    def _parse_optional(
+        self, arg_string: str
+    ) -> Optional[Tuple[Optional[argparse.Action], str, Optional[str]]]:
         # if starts with -: it means that is a parameter not a argument
         if arg_string[:2] == "-:":
             return None
@@ -23,7 +27,7 @@ class ScrapyArgumentParser(argparse.ArgumentParser):
         return super()._parse_optional(arg_string)
 
 
-def _iter_command_classes(module_name):
+def _iter_command_classes(module_name: str) -> Iterable[Type[ScrapyCommand]]:
     # TODO: add `name` attribute to commands and merge this function with
     # scrapy.utils.spider.iter_spider_classes
     for module in walk_modules(module_name):
@@ -37,8 +41,8 @@ def _iter_command_classes(module_name):
                 yield obj
 
 
-def _get_commands_from_module(module, inproject):
-    d = {}
+def _get_commands_from_module(module: str, inproject: bool) -> Dict[str, ScrapyCommand]:
+    d: Dict[str, ScrapyCommand] = {}
     for cmd in _iter_command_classes(module):
         if inproject or not cmd.requires_project:
             cmdname = cmd.__module__.split(".")[-1]
@@ -46,8 +50,10 @@ def _get_commands_from_module(module, inproject):
     return d
 
 
-def _get_commands_from_entry_points(inproject, group="scrapy.commands"):
-    cmds = {}
+def _get_commands_from_entry_points(
+    inproject: bool, group: str = "scrapy.commands"
+) -> Dict[str, ScrapyCommand]:
+    cmds: Dict[str, ScrapyCommand] = {}
     if sys.version_info >= (3, 10):
         eps = entry_points(group=group)
     else:
@@ -61,7 +67,9 @@ def _get_commands_from_entry_points(inproject, group="scrapy.commands"):
     return cmds
 
 
-def _get_commands_dict(settings, inproject):
+def _get_commands_dict(
+    settings: BaseSettings, inproject: bool
+) -> Dict[str, ScrapyCommand]:
     cmds = _get_commands_from_module("scrapy.commands", inproject)
     cmds.update(_get_commands_from_entry_points(inproject))
     cmds_module = settings["COMMANDS_MODULE"]
@@ -70,16 +78,17 @@ def _get_commands_dict(settings, inproject):
     return cmds
 
 
-def _pop_command_name(argv):
+def _pop_command_name(argv: List[str]) -> Optional[str]:
     i = 0
     for arg in argv[1:]:
         if not arg.startswith("-"):
             del argv[i]
             return arg
         i += 1
+    return None
 
 
-def _print_header(settings, inproject):
+def _print_header(settings: BaseSettings, inproject: bool) -> None:
     version = scrapy.__version__
     if inproject:
         print(f"Scrapy {version} - active project: {settings['BOT_NAME']}\n")
@@ -88,7 +97,7 @@ def _print_header(settings, inproject):
         print(f"Scrapy {version} - no active project\n")
 
 
-def _print_commands(settings, inproject):
+def _print_commands(settings: BaseSettings, inproject: bool) -> None:
     _print_header(settings, inproject)
     print("Usage:")
     print("  scrapy <command> [options] [args]\n")
@@ -103,13 +112,17 @@ def _print_commands(settings, inproject):
     print('Use "scrapy <command> -h" to see more info about a command')
 
 
-def _print_unknown_command(settings, cmdname, inproject):
+def _print_unknown_command(
+    settings: BaseSettings, cmdname: str, inproject: bool
+) -> None:
     _print_header(settings, inproject)
     print(f"Unknown command: {cmdname}\n")
     print('Use "scrapy" to see available commands')
 
 
-def _run_print_help(parser, func, *a, **kw):
+def _run_print_help(
+    parser: argparse.ArgumentParser, func: Callable, *a: Any, **kw: Any
+) -> None:
     try:
         func(*a, **kw)
     except UsageError as e:
@@ -120,7 +133,9 @@ def _run_print_help(parser, func, *a, **kw):
         sys.exit(2)
 
 
-def execute(argv=None, settings=None):
+def execute(
+    argv: Optional[List[str]] = None, settings: Optional[Settings] = None
+) -> None:
     if argv is None:
         argv = sys.argv
 
@@ -162,14 +177,16 @@ def execute(argv=None, settings=None):
     sys.exit(cmd.exitcode)
 
 
-def _run_command(cmd, args, opts):
+def _run_command(cmd: ScrapyCommand, args: List[str], opts: argparse.Namespace) -> None:
     if opts.profile:
         _run_command_profiled(cmd, args, opts)
     else:
         cmd.run(args, opts)
 
 
-def _run_command_profiled(cmd, args, opts):
+def _run_command_profiled(
+    cmd: ScrapyCommand, args: List[str], opts: argparse.Namespace
+) -> None:
     if opts.profile:
         sys.stderr.write(f"scrapy: writing cProfile stats to {opts.profile!r}\n")
     loc = locals()