Merge pull request #6003 from wRAR/typing-utils-2

Typing for scrapy/utils, second pass

scrapy/commands/__init__.py

@@ -4,7 +4,7 @@ Base class for Scrapy commands
 import argparse
 import os
 from pathlib import Path
-from typing import Any, Dict, Optional
+from typing import Any, Dict, List, Optional
 
 from twisted.python import failure
 
@@ -116,7 +116,7 @@ class ScrapyCommand:
         if opts.pdb:
             failure.startDebugMode()
 
-    def run(self, args, opts):
+    def run(self, args: List[str], opts: argparse.Namespace) -> None:
         """
         Entry point for running commands
         """

scrapy/commands/fetch.py

@@ -1,7 +1,10 @@
 import sys
+from argparse import Namespace
+from typing import List, Type
 
 from w3lib.url import is_url
 
+from scrapy import Spider
 from scrapy.commands import ScrapyCommand
 from scrapy.exceptions import UsageError
 from scrapy.http import Request
@@ -57,7 +60,7 @@ class Command(ScrapyCommand):
     def _print_bytes(self, bytes_):
         sys.stdout.buffer.write(bytes_ + b"\n")
 
-    def run(self, args, opts):
+    def run(self, args: List[str], opts: Namespace) -> None:
         if len(args) != 1 or not is_url(args[0]):
             raise UsageError()
         request = Request(
@@ -73,7 +76,8 @@ class Command(ScrapyCommand):
         else:
             request.meta["handle_httpstatus_all"] = True
 
-        spidercls = DefaultSpider
+        spidercls: Type[Spider] = DefaultSpider
+        assert self.crawler_process
         spider_loader = self.crawler_process.spider_loader
         if opts.spider:
             spidercls = spider_loader.load(opts.spider)

scrapy/commands/shell.py

@@ -3,8 +3,11 @@ Scrapy Shell
 
 See documentation in docs/topics/shell.rst
 """
+from argparse import Namespace
 from threading import Thread
+from typing import List, Type
 
+from scrapy import Spider
 from scrapy.commands import ScrapyCommand
 from scrapy.http import Request
 from scrapy.shell import Shell
@@ -54,15 +57,16 @@ class Command(ScrapyCommand):
         """
         pass
 
-    def run(self, args, opts):
+    def run(self, args: List[str], opts: Namespace) -> None:
         url = args[0] if args else None
         if url:
             # first argument may be a local file
             url = guess_scheme(url)
 
+        assert self.crawler_process
         spider_loader = self.crawler_process.spider_loader
 
-        spidercls = DefaultSpider
+        spidercls: Type[Spider] = DefaultSpider
         if opts.spider:
             spidercls = spider_loader.load(opts.spider)
         elif url:

scrapy/contracts/__init__.py

@@ -2,7 +2,8 @@ import re
 import sys
 from functools import wraps
 from inspect import getmembers
-from typing import Dict
+from types import CoroutineType
+from typing import AsyncGenerator, Dict
 from unittest import TestCase
 
 from scrapy.http import Request
@@ -37,7 +38,10 @@ class Contract:
                 else:
                     results.addSuccess(self.testcase_pre)
                 finally:
-                    return list(iterate_spider_output(cb(response, **cb_kwargs)))
+                    cb_result = cb(response, **cb_kwargs)
+                    if isinstance(cb_result, (AsyncGenerator, CoroutineType)):
+                        raise TypeError("Contracts don't support async callbacks")
+                    return list(iterate_spider_output(cb_result))
 
             request.callback = wrapper
 
@@ -49,7 +53,10 @@ class Contract:
 
             @wraps(cb)
             def wrapper(response, **cb_kwargs):
-                output = list(iterate_spider_output(cb(response, **cb_kwargs)))
+                cb_result = cb(response, **cb_kwargs)
+                if isinstance(cb_result, (AsyncGenerator, CoroutineType)):
+                    raise TypeError("Contracts don't support async callbacks")
+                output = list(iterate_spider_output(cb_result))
                 try:
                     results.startTest(self.testcase_post)
                     self.post_process(output)

scrapy/exceptions.py

@@ -4,6 +4,7 @@ Scrapy core exceptions
 These exceptions are documented in docs/topics/exceptions.rst. Please don't add
 new exceptions here without documenting them there.
 """
+from typing import Any
 
 # Internal
 
@@ -77,7 +78,7 @@ class NotSupported(Exception):
 class UsageError(Exception):
     """To indicate a command-line usage error"""
 
-    def __init__(self, *a, **kw):
+    def __init__(self, *a: Any, **kw: Any):
         self.print_help = kw.pop("print_help", True)
         super().__init__(*a, **kw)
 

scrapy/extensions/feedexport.py

@@ -11,7 +11,7 @@ import warnings
 from datetime import datetime
 from pathlib import Path, PureWindowsPath
 from tempfile import NamedTemporaryFile
-from typing import IO, Any, Callable, List, Optional, Tuple, Union
+from typing import IO, Any, Callable, Dict, List, Optional, Tuple, Union
 from urllib.parse import unquote, urlparse
 
 from twisted.internet import defer, threads
@@ -282,15 +282,23 @@ class GCSFeedStorage(BlockingFeedStorage):
 
 
 class FTPFeedStorage(BlockingFeedStorage):
-    def __init__(self, uri, use_active_mode=False, *, feed_options=None):
+    def __init__(
+        self,
+        uri: str,
+        use_active_mode: bool = False,
+        *,
+        feed_options: Optional[Dict[str, Any]] = None,
+    ):
         u = urlparse(uri)
-        self.host = u.hostname
-        self.port = int(u.port or "21")
-        self.username = u.username
-        self.password = unquote(u.password or "")
-        self.path = u.path
-        self.use_active_mode = use_active_mode
-        self.overwrite = not feed_options or feed_options.get("overwrite", True)
+        if not u.hostname:
+            raise ValueError(f"Got a storage URI without a hostname: {uri}")
+        self.host: str = u.hostname
+        self.port: int = int(u.port or "21")
+        self.username: str = u.username or ""
+        self.password: str = unquote(u.password or "")
+        self.path: str = u.path
+        self.use_active_mode: bool = use_active_mode
+        self.overwrite: bool = not feed_options or feed_options.get("overwrite", True)
 
     @classmethod
     def from_crawler(cls, crawler, uri, *, feed_options=None):

scrapy/http/response/text.py

@@ -4,10 +4,11 @@ discovering (through HTTP headers) to base Response class.
 
 See documentation in docs/topics/request-response.rst
 """
+from __future__ import annotations
 
 import json
 from contextlib import suppress
-from typing import Generator, Tuple
+from typing import TYPE_CHECKING, Any, Generator, Optional, Tuple
 from urllib.parse import urljoin
 
 import parsel
@@ -25,6 +26,9 @@ from scrapy.http.response import Response
 from scrapy.utils.python import memoizemethod_noargs, to_unicode
 from scrapy.utils.response import get_base_url
 
+if TYPE_CHECKING:
+    from scrapy.selector import Selector
+
 _NONE = object()
 
 
@@ -34,11 +38,11 @@ class TextResponse(Response):
 
     attributes: Tuple[str, ...] = Response.attributes + ("encoding",)
 
-    def __init__(self, *args, **kwargs):
+    def __init__(self, *args: Any, **kwargs: Any):
         self._encoding = kwargs.pop("encoding", None)
-        self._cached_benc = None
-        self._cached_ubody = None
-        self._cached_selector = None
+        self._cached_benc: Optional[str] = None
+        self._cached_ubody: Optional[str] = None
+        self._cached_selector: Optional[Selector] = None
         super().__init__(*args, **kwargs)
 
     def _set_url(self, url):
@@ -82,7 +86,7 @@ class TextResponse(Response):
         return self._cached_decoded_json
 
     @property
-    def text(self):
+    def text(self) -> str:
         """Body as unicode"""
         # access self.encoding before _cached_ubody to make sure
         # _body_inferred_encoding is called

scrapy/selector/unified.py

@@ -1,10 +1,11 @@
 """
 XPath selectors based on lxml
 """
+from typing import Any, Optional, Type, Union
 
 from parsel import Selector as _ParselSelector
 
-from scrapy.http import HtmlResponse, XmlResponse
+from scrapy.http import HtmlResponse, TextResponse, XmlResponse
 from scrapy.utils.python import to_bytes
 from scrapy.utils.trackref import object_ref
 
@@ -13,14 +14,14 @@ __all__ = ["Selector", "SelectorList"]
 _NOT_SET = object()
 
 
-def _st(response, st):
+def _st(response: Optional[TextResponse], st: Optional[str]) -> str:
     if st is None:
         return "xml" if isinstance(response, XmlResponse) else "html"
     return st
 
 
-def _response_from_text(text, st):
-    rt = XmlResponse if st == "xml" else HtmlResponse
+def _response_from_text(text: Union[str, bytes], st: Optional[str]) -> TextResponse:
+    rt: Type[TextResponse] = XmlResponse if st == "xml" else HtmlResponse
     return rt(url="about:blank", encoding="utf-8", body=to_bytes(text, "utf-8"))
 
 
@@ -65,7 +66,14 @@ class Selector(_ParselSelector, object_ref):
     __slots__ = ["response"]
     selectorlist_cls = SelectorList
 
-    def __init__(self, response=None, text=None, type=None, root=_NOT_SET, **kwargs):
+    def __init__(
+        self,
+        response: Optional[TextResponse] = None,
+        text: Optional[str] = None,
+        type: Optional[str] = None,
+        root: Optional[Any] = _NOT_SET,
+        **kwargs: Any,
+    ):
         if response is not None and text is not None:
             raise ValueError(
                 f"{self.__class__.__name__}.__init__() received "

scrapy/spiderloader.py

@@ -1,11 +1,12 @@
 import traceback
 import warnings
 from collections import defaultdict
+from types import ModuleType
 from typing import DefaultDict, Dict, List, Tuple, Type
 
 from zope.interface import implementer
 
-from scrapy import Spider
+from scrapy import Request, Spider
 from scrapy.interfaces import ISpiderLoader
 from scrapy.settings import BaseSettings
 from scrapy.utils.misc import walk_modules
@@ -26,7 +27,7 @@ class SpiderLoader:
         self._found: DefaultDict[str, List[Tuple[str, str]]] = defaultdict(list)
         self._load_all_spiders()
 
-    def _check_name_duplicates(self):
+    def _check_name_duplicates(self) -> None:
         dupes = []
         for name, locations in self._found.items():
             dupes.extend(
@@ -45,12 +46,12 @@ class SpiderLoader:
                 category=UserWarning,
             )
 
-    def _load_spiders(self, module):
+    def _load_spiders(self, module: ModuleType) -> None:
         for spcls in iter_spider_classes(module):
             self._found[spcls.name].append((module.__name__, spcls.__name__))
             self._spiders[spcls.name] = spcls
 
-    def _load_all_spiders(self):
+    def _load_all_spiders(self) -> None:
         for name in self.spider_modules:
             try:
                 for module in walk_modules(name):
@@ -81,7 +82,7 @@ class SpiderLoader:
         except KeyError:
             raise KeyError(f"Spider not found: {spider_name}")
 
-    def find_by_request(self, request):
+    def find_by_request(self, request: Request) -> List[str]:
         """
         Return the list of spider names that can handle the given request.
         """

scrapy/spiders/__init__.py

@@ -82,7 +82,7 @@ class Spider(object_ref):
         settings.setdict(cls.custom_settings or {}, priority="spider")
 
     @classmethod
-    def handles_request(cls, request):
+    def handles_request(cls, request: Request) -> bool:
         return url_is_from_spider(request.url, cls)
 
     @staticmethod

scrapy/utils/conf.py

@@ -5,7 +5,18 @@ import warnings
 from configparser import ConfigParser
 from operator import itemgetter
 from pathlib import Path
-from typing import Any, Dict, List, Optional, Union
+from typing import (
+    Any,
+    Callable,
+    Collection,
+    Dict,
+    Iterable,
+    List,
+    Mapping,
+    MutableMapping,
+    Optional,
+    Union,
+)
 
 from scrapy.exceptions import ScrapyDeprecationWarning, UsageError
 from scrapy.settings import BaseSettings
@@ -13,17 +24,21 @@ from scrapy.utils.deprecate import update_classpath
 from scrapy.utils.python import without_none_values
 
 
-def build_component_list(compdict, custom=None, convert=update_classpath):
+def build_component_list(
+    compdict: MutableMapping[Any, Any],
+    custom: Any = None,
+    convert: Callable[[Any], Any] = update_classpath,
+) -> List[Any]:
     """Compose a component list from a { class: order } dictionary."""
 
-    def _check_components(complist):
+    def _check_components(complist: Collection[Any]) -> None:
         if len({convert(c) for c in complist}) != len(complist):
             raise ValueError(
                 f"Some paths in {complist!r} convert to the same object, "
                 "please update your settings"
             )
 
-    def _map_keys(compdict):
+    def _map_keys(compdict: Mapping[Any, Any]) -> Union[BaseSettings, Dict[Any, Any]]:
         if isinstance(compdict, BaseSettings):
             compbs = BaseSettings()
             for k, v in compdict.items():
@@ -41,7 +56,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath):
         _check_components(compdict)
         return {convert(k): v for k, v in compdict.items()}
 
-    def _validate_values(compdict):
+    def _validate_values(compdict: Mapping[Any, Any]) -> None:
         """Fail if a value in the components dict is not a real number or None."""
         for name, value in compdict.items():
             if value is not None and not isinstance(value, numbers.Real):
@@ -60,7 +75,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath):
         )
         if isinstance(custom, (list, tuple)):
             _check_components(custom)
-            return type(custom)(convert(c) for c in custom)
+            return type(custom)(convert(c) for c in custom)  # type: ignore[return-value]
         compdict.update(custom)
 
     _validate_values(compdict)
@@ -68,7 +83,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath):
     return [k for k, v in sorted(compdict.items(), key=itemgetter(1))]
 
 
-def arglist_to_dict(arglist):
+def arglist_to_dict(arglist: List[str]) -> Dict[str, str]:
     """Convert a list of arguments like ['arg1=val1', 'arg2=val2', ...] to a
     dict
     """
@@ -91,7 +106,7 @@ def closest_scrapy_cfg(
     return closest_scrapy_cfg(path.parent, path)
 
 
-def init_env(project="default", set_syspath=True):
+def init_env(project: str = "default", set_syspath: bool = True) -> None:
     """Initialize environment to use command-line tool from inside a project
     dir. This sets the Scrapy settings module and modifies the Python path to
     be able to locate the project module.
@@ -106,7 +121,7 @@ def init_env(project="default", set_syspath=True):
             sys.path.append(projdir)
 
 
-def get_config(use_closest=True):
+def get_config(use_closest: bool = True) -> ConfigParser:
     """Get Scrapy config file as a ConfigParser"""
     sources = get_sources(use_closest)
     cfg = ConfigParser()
@@ -114,7 +129,7 @@ def get_config(use_closest=True):
     return cfg
 
 
-def get_sources(use_closest=True) -> List[str]:
+def get_sources(use_closest: bool = True) -> List[str]:
     xdg_config_home = (
         os.environ.get("XDG_CONFIG_HOME") or Path("~/.config").expanduser()
     )
@@ -129,7 +144,9 @@ def get_sources(use_closest=True) -> List[str]:
     return sources
 
 
-def feed_complete_default_values_from_settings(feed, settings):
+def feed_complete_default_values_from_settings(
+    feed: Dict[str, Any], settings: BaseSettings
+) -> Dict[str, Any]:
     out = feed.copy()
     out.setdefault("batch_item_count", settings.getint("FEED_EXPORT_BATCH_ITEM_COUNT"))
     out.setdefault("encoding", settings["FEED_EXPORT_ENCODING"])
@@ -145,21 +162,21 @@ def feed_complete_default_values_from_settings(feed, settings):
 
 
 def feed_process_params_from_cli(
-    settings,
+    settings: BaseSettings,
     output: List[str],
-    output_format=None,
+    output_format: Optional[str] = None,
     overwrite_output: Optional[List[str]] = None,
-):
+) -> Dict[str, Dict[str, Any]]:
     """
     Receives feed export params (from the 'crawl' or 'runspider' commands),
     checks for inconsistencies in their quantities and returns a dictionary
     suitable to be used as the FEEDS setting.
     """
-    valid_output_formats = without_none_values(
+    valid_output_formats: Iterable[str] = without_none_values(
         settings.getwithbase("FEED_EXPORTERS")
     ).keys()
 
-    def check_valid_format(output_format):
+    def check_valid_format(output_format: str) -> None:
         if output_format not in valid_output_formats:
             raise UsageError(
                 f"Unrecognized output format '{output_format}'. "

scrapy/utils/defer.py

@@ -5,11 +5,13 @@ import asyncio
 import inspect
 from asyncio import Future
 from functools import wraps
+from types import CoroutineType
 from typing import (
     Any,
     AsyncGenerator,
     AsyncIterable,
     AsyncIterator,
+    Awaitable,
     Callable,
     Coroutine,
     Dict,
@@ -19,8 +21,10 @@ from typing import (
     List,
     Optional,
     Tuple,
+    TypeVar,
     Union,
     cast,
+    overload,
 )
 
 from twisted.internet import defer
@@ -186,9 +190,7 @@ class _AsyncCooperatorAdapter(Iterator):
     def _call_anext(self) -> None:
         # This starts waiting for the next result from aiterator.
         # If aiterator is exhausted, _errback will be called.
-        self.anext_deferred = cast(
-            Deferred, deferred_from_coro(self.aiterator.__anext__())
-        )
+        self.anext_deferred = deferred_from_coro(self.aiterator.__anext__())
         self.anext_deferred.addCallbacks(self._callback, self._errback)
 
     def __next__(self) -> Deferred:
@@ -297,7 +299,21 @@ async def aiter_errback(
             errback(failure.Failure(), *a, **kw)
 
 
-def deferred_from_coro(o: Any) -> Any:
+_CT = TypeVar("_CT", bound=Union[Awaitable, CoroutineType, Future])
+_T = TypeVar("_T")
+
+
+@overload
+def deferred_from_coro(o: _CT) -> Deferred:
+    ...
+
+
+@overload
+def deferred_from_coro(o: _T) -> _T:
+    ...
+
+
+def deferred_from_coro(o: _T) -> Union[Deferred, _T]:
     """Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine"""
     if isinstance(o, Deferred):
         return o

scrapy/utils/ftp.py

@@ -1,9 +1,10 @@
 import posixpath
 from ftplib import FTP, error_perm
 from posixpath import dirname
+from typing import IO
 
 
-def ftp_makedirs_cwd(ftp, path, first_call=True):
+def ftp_makedirs_cwd(ftp: FTP, path: str, first_call: bool = True) -> None:
     """Set the current directory of the FTP connection given in the ``ftp``
     argument (as a ftplib.FTP object), creating all parent directories if they
     don't exist. The ftplib.FTP object must be already connected and logged in.
@@ -18,8 +19,16 @@ def ftp_makedirs_cwd(ftp, path, first_call=True):
 
 
 def ftp_store_file(
-    *, path, file, host, port, username, password, use_active_mode=False, overwrite=True
-):
+    *,
+    path: str,
+    file: IO,
+    host: str,
+    port: int,
+    username: str,
+    password: str,
+    use_active_mode: bool = False,
+    overwrite: bool = True,
+) -> None:
     """Opens a FTP connection with passed credentials,sets current directory
     to the directory extracted from given path, then uploads the file to server
     """

scrapy/utils/iterators.py

@@ -2,15 +2,34 @@ import csv
 import logging
 import re
 from io import StringIO
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    Dict,
+    Generator,
+    Iterable,
+    List,
+    Literal,
+    Optional,
+    Union,
+    cast,
+    overload,
+)
 
 from scrapy.http import Response, TextResponse
 from scrapy.selector import Selector
 from scrapy.utils.python import re_rsearch, to_unicode
 
+if TYPE_CHECKING:
+    from lxml._types import SupportsReadClose
+
 logger = logging.getLogger(__name__)
 
 
-def xmliter(obj, nodename):
+def xmliter(
+    obj: Union[Response, str, bytes], nodename: str
+) -> Generator[Selector, Any, None]:
     """Return a iterator of Selector's over all nodes of a XML document,
        given the name of the node to iterate. Useful for parsing XML feeds.
 
@@ -27,20 +46,22 @@ def xmliter(obj, nodename):
     NAMESPACE_RE = re.compile(r"((xmlns[:A-Za-z]*)=[^>\s]+)", re.S)
     text = _body_or_str(obj)
 
-    document_header = re.search(DOCUMENT_HEADER_RE, text)
-    document_header = document_header.group().strip() if document_header else ""
+    document_header_match = re.search(DOCUMENT_HEADER_RE, text)
+    document_header = (
+        document_header_match.group().strip() if document_header_match else ""
+    )
     header_end_idx = re_rsearch(HEADER_END_RE, text)
     header_end = text[header_end_idx[1] :].strip() if header_end_idx else ""
-    namespaces = {}
+    namespaces: Dict[str, str] = {}
     if header_end:
         for tagname in reversed(re.findall(END_TAG_RE, header_end)):
+            assert header_end_idx
             tag = re.search(
                 rf"<\s*{tagname}.*?xmlns[:=][^>]*>", text[: header_end_idx[1]], re.S
             )
             if tag:
-                namespaces.update(
-                    reversed(x) for x in re.findall(NAMESPACE_RE, tag.group())
-                )
+                for x in re.findall(NAMESPACE_RE, tag.group()):
+                    namespaces[x[1]] = x[0]
 
     r = re.compile(rf"<{nodename_patt}[\s>].*?</{nodename_patt}>", re.DOTALL)
     for match in r.finditer(text):
@@ -54,12 +75,19 @@ def xmliter(obj, nodename):
         yield Selector(text=nodetext, type="xml")
 
 
-def xmliter_lxml(obj, nodename, namespace=None, prefix="x"):
+def xmliter_lxml(
+    obj: Union[Response, str, bytes],
+    nodename: str,
+    namespace: Optional[str] = None,
+    prefix: str = "x",
+) -> Generator[Selector, Any, None]:
     from lxml import etree
 
     reader = _StreamReader(obj)
     tag = f"{{{namespace}}}{nodename}" if namespace else nodename
-    iterable = etree.iterparse(reader, tag=tag, encoding=reader.encoding)
+    iterable = etree.iterparse(
+        cast("SupportsReadClose[bytes]", reader), tag=tag, encoding=reader.encoding
+    )
     selxpath = "//" + (f"{prefix}:{nodename}" if namespace else nodename)
     for _, node in iterable:
         nodetext = etree.tostring(node, encoding="unicode")
@@ -71,30 +99,46 @@ def xmliter_lxml(obj, nodename, namespace=None, prefix="x"):
 
 
 class _StreamReader:
-    def __init__(self, obj):
-        self._ptr = 0
-        if isinstance(obj, Response):
+    def __init__(self, obj: Union[Response, str, bytes]):
+        self._ptr: int = 0
+        self._text: Union[str, bytes]
+        if isinstance(obj, TextResponse):
             self._text, self.encoding = obj.body, obj.encoding
+        elif isinstance(obj, Response):
+            self._text, self.encoding = obj.body, "utf-8"
         else:
             self._text, self.encoding = obj, "utf-8"
-        self._is_unicode = isinstance(self._text, str)
+        self._is_unicode: bool = isinstance(self._text, str)
+        self._is_first_read: bool = True
 
-    def read(self, n=65535):
-        self.read = self._read_unicode if self._is_unicode else self._read_string
-        return self.read(n).lstrip()
+    def read(self, n: int = 65535) -> bytes:
+        method: Callable[[int], bytes] = (
+            self._read_unicode if self._is_unicode else self._read_string
+        )
+        result = method(n)
+        if self._is_first_read:
+            self._is_first_read = False
+            result = result.lstrip()
+        return result
 
-    def _read_string(self, n=65535):
+    def _read_string(self, n: int = 65535) -> bytes:
         s, e = self._ptr, self._ptr + n
         self._ptr = e
-        return self._text[s:e]
+        return cast(bytes, self._text)[s:e]
 
-    def _read_unicode(self, n=65535):
+    def _read_unicode(self, n: int = 65535) -> bytes:
         s, e = self._ptr, self._ptr + n
         self._ptr = e
-        return self._text[s:e].encode("utf-8")
+        return cast(str, self._text)[s:e].encode("utf-8")
 
 
-def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):
+def csviter(
+    obj: Union[Response, str, bytes],
+    delimiter: Optional[str] = None,
+    headers: Optional[List[str]] = None,
+    encoding: Optional[str] = None,
+    quotechar: Optional[str] = None,
+) -> Generator[Dict[str, str], Any, None]:
     """Returns an iterator of dictionaries from the given csv object
 
     obj can be:
@@ -112,12 +156,12 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):
 
     encoding = obj.encoding if isinstance(obj, TextResponse) else encoding or "utf-8"
 
-    def row_to_unicode(row_):
+    def row_to_unicode(row_: Iterable) -> List[str]:
         return [to_unicode(field, encoding) for field in row_]
 
     lines = StringIO(_body_or_str(obj, unicode=True))
 
-    kwargs = {}
+    kwargs: Dict[str, Any] = {}
     if delimiter:
         kwargs["delimiter"] = delimiter
     if quotechar:
@@ -147,7 +191,24 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):
         yield dict(zip(headers, row))
 
 
-def _body_or_str(obj, unicode=True):
+@overload
+def _body_or_str(obj: Union[Response, str, bytes]) -> str:
+    ...
+
+
+@overload
+def _body_or_str(obj: Union[Response, str, bytes], unicode: Literal[True]) -> str:
+    ...
+
+
+@overload
+def _body_or_str(obj: Union[Response, str, bytes], unicode: Literal[False]) -> bytes:
+    ...
+
+
+def _body_or_str(
+    obj: Union[Response, str, bytes], unicode: bool = True
+) -> Union[str, bytes]:
     expected_types = (Response, str, bytes)
     if not isinstance(obj, expected_types):
         expected_types_str = " or ".join(t.__name__ for t in expected_types)
@@ -156,10 +217,10 @@ def _body_or_str(obj, unicode=True):
         )
     if isinstance(obj, Response):
         if not unicode:
-            return obj.body
+            return cast(bytes, obj.body)
         if isinstance(obj, TextResponse):
             return obj.text
-        return obj.body.decode("utf-8")
+        return cast(bytes, obj.body).decode("utf-8")
     if isinstance(obj, str):
         return obj if unicode else obj.encode("utf-8")
     return obj.decode("utf-8") if unicode else obj

scrapy/utils/job.py

@@ -5,7 +5,7 @@ from scrapy.settings import BaseSettings
 
 
 def job_dir(settings: BaseSettings) -> Optional[str]:
-    path = settings["JOBDIR"]
+    path: str = settings["JOBDIR"]
     if path and not Path(path).exists():
         Path(path).mkdir(parents=True)
     return path

scrapy/utils/log.py

@@ -1,8 +1,11 @@
+from __future__ import annotations
+
 import logging
 import sys
 import warnings
 from logging.config import dictConfig
-from typing import Tuple
+from types import TracebackType
+from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Type, Union, cast
 
 from twisted.python import log as twisted_log
 from twisted.python.failure import Failure
@@ -12,13 +15,25 @@ from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.settings import Settings
 from scrapy.utils.versions import scrapy_components_versions
 
+if TYPE_CHECKING:
+    from scrapy.crawler import Crawler
+
 logger = logging.getLogger(__name__)
 
 
-def failure_to_exc_info(failure: Failure):
+def failure_to_exc_info(
+    failure: Failure,
+) -> Optional[Tuple[Type[BaseException], BaseException, Optional[TracebackType]]]:
     """Extract exc_info from Failure instances"""
     if isinstance(failure, Failure):
-        return (failure.type, failure.value, failure.getTracebackObject())
+        assert failure.type
+        assert failure.value
+        return (
+            failure.type,
+            failure.value,
+            cast(Optional[TracebackType], failure.getTracebackObject()),
+        )
+    return None
 
 
 class TopLevelFormatter(logging.Filter):
@@ -33,10 +48,10 @@ class TopLevelFormatter(logging.Filter):
     ``loggers`` list where it should act.
     """
 
-    def __init__(self, loggers=None):
-        self.loggers = loggers or []
+    def __init__(self, loggers: Optional[List[str]] = None):
+        self.loggers: List[str] = loggers or []
 
-    def filter(self, record):
+    def filter(self, record: logging.LogRecord) -> bool:
         if any(record.name.startswith(logger + ".") for logger in self.loggers):
             record.name = record.name.split(".", 1)[0]
         return True
@@ -62,7 +77,9 @@ DEFAULT_LOGGING = {
 }
 
 
-def configure_logging(settings=None, install_root_handler=True):
+def configure_logging(
+    settings: Union[Settings, dict, None] = None, install_root_handler: bool = True
+) -> None:
     """
     Initialize logging defaults for Scrapy.
 
@@ -99,13 +116,16 @@ def configure_logging(settings=None, install_root_handler=True):
         settings = Settings(settings)
 
     if settings.getbool("LOG_STDOUT"):
-        sys.stdout = StreamLogger(logging.getLogger("stdout"))
+        sys.stdout = StreamLogger(logging.getLogger("stdout"))  # type: ignore[assignment]
 
     if install_root_handler:
         install_scrapy_root_handler(settings)
 
 
-def install_scrapy_root_handler(settings):
+_scrapy_root_handler: Optional[logging.Handler] = None
+
+
+def install_scrapy_root_handler(settings: Settings) -> None:
     global _scrapy_root_handler
 
     if (
@@ -118,16 +138,14 @@ def install_scrapy_root_handler(settings):
     logging.root.addHandler(_scrapy_root_handler)
 
 
-def get_scrapy_root_handler():
+def get_scrapy_root_handler() -> Optional[logging.Handler]:
     return _scrapy_root_handler
 
 
-_scrapy_root_handler = None
-
-
-def _get_handler(settings):
+def _get_handler(settings: Settings) -> logging.Handler:
     """Return a log handler object according to settings"""
     filename = settings.get("LOG_FILE")
+    handler: logging.Handler
     if filename:
         mode = "a" if settings.getbool("LOG_FILE_APPEND") else "w"
         encoding = settings.get("LOG_ENCODING")
@@ -181,16 +199,16 @@ class StreamLogger:
         https://www.electricmonk.nl/log/2011/08/14/redirect-stdout-and-stderr-to-a-logger-in-python/
     """
 
-    def __init__(self, logger, log_level=logging.INFO):
-        self.logger = logger
-        self.log_level = log_level
-        self.linebuf = ""
+    def __init__(self, logger: logging.Logger, log_level: int = logging.INFO):
+        self.logger: logging.Logger = logger
+        self.log_level: int = log_level
+        self.linebuf: str = ""
 
-    def write(self, buf):
+    def write(self, buf: str) -> None:
         for line in buf.rstrip().splitlines():
             self.logger.log(self.log_level, line.rstrip())
 
-    def flush(self):
+    def flush(self) -> None:
         for h in self.logger.handlers:
             h.flush()
 
@@ -198,11 +216,11 @@ class StreamLogger:
 class LogCounterHandler(logging.Handler):
     """Record log levels count into a crawler stats"""
 
-    def __init__(self, crawler, *args, **kwargs):
+    def __init__(self, crawler: Crawler, *args: Any, **kwargs: Any):
         super().__init__(*args, **kwargs)
-        self.crawler = crawler
+        self.crawler: Crawler = crawler
 
-    def emit(self, record):
+    def emit(self, record: logging.LogRecord) -> None:
         sname = f"log_count/{record.levelname}"
         self.crawler.stats.inc_value(sname)
 

scrapy/utils/ossignal.py

@@ -1,6 +1,13 @@
 import signal
+from types import FrameType
+from typing import Any, Callable, Dict, Optional, Union
 
-signal_names = {}
+# copy of _HANDLER from typeshed/stdlib/signal.pyi
+SignalHandlerT = Union[
+    Callable[[int, Optional[FrameType]], Any], int, signal.Handlers, None
+]
+
+signal_names: Dict[int, str] = {}
 for signame in dir(signal):
     if signame.startswith("SIG") and not signame.startswith("SIG_"):
         signum = getattr(signal, signame)
@@ -8,7 +15,9 @@ for signame in dir(signal):
             signal_names[signum] = signame
 
 
-def install_shutdown_handlers(function, override_sigint=True):
+def install_shutdown_handlers(
+    function: SignalHandlerT, override_sigint: bool = True
+) -> None:
     """Install the given function as a signal handler for all common shutdown
     signals (such as SIGINT, SIGTERM, etc). If override_sigint is ``False`` the
     SIGINT handler won't be install if there is already a handler in place

scrapy/utils/project.py

@@ -11,7 +11,7 @@ ENVVAR = "SCRAPY_SETTINGS_MODULE"
 DATADIR_CFG_SECTION = "datadir"
 
 
-def inside_project():
+def inside_project() -> bool:
     scrapy_module = os.environ.get(ENVVAR)
     if scrapy_module:
         try:
@@ -25,7 +25,7 @@ def inside_project():
     return bool(closest_scrapy_cfg())
 
 
-def project_data_dir(project="default") -> str:
+def project_data_dir(project: str = "default") -> str:
     """Return the current project data dir, creating it if it doesn't exist"""
     if not inside_project():
         raise NotConfigured("Not inside a project")
@@ -44,7 +44,7 @@ def project_data_dir(project="default") -> str:
     return str(d)
 
 
-def data_path(path: str, createdir=False) -> str:
+def data_path(path: str, createdir: bool = False) -> str:
     """
     Return the given path joined with the .scrapy data directory.
     If given an absolute path, return it unmodified.
@@ -60,7 +60,7 @@ def data_path(path: str, createdir=False) -> str:
     return str(path_obj)
 
 
-def get_project_settings():
+def get_project_settings() -> Settings:
     if ENVVAR not in os.environ:
         project = os.environ.get("SCRAPY_PROJECT", "default")
         init_env(project)

scrapy/utils/response.py

@@ -42,8 +42,7 @@ def get_meta_refresh(
     """Parse the http-equiv refresh parameter from the given response"""
     if response not in _metaref_cache:
         text = response.text[0:4096]
-        # a w3lib typing bug here, fixed in https://github.com/scrapy/w3lib/pull/211
-        _metaref_cache[response] = html.get_meta_refresh(  # type: ignore[assignment]
+        _metaref_cache[response] = html.get_meta_refresh(
             text, response.url, response.encoding, ignore_tags=ignore_tags
         )
     return _metaref_cache[response]

scrapy/utils/signal.py

@@ -23,7 +23,10 @@ logger = logging.getLogger(__name__)
 
 
 def send_catch_log(
-    signal=Any, sender=Anonymous, *arguments, **named
+    signal: TypingAny = Any,
+    sender: TypingAny = Anonymous,
+    *arguments: TypingAny,
+    **named: TypingAny
 ) -> List[Tuple[TypingAny, TypingAny]]:
     """Like pydispatcher.robust.sendRobust but it also logs errors and returns
     Failures instead of exceptions.
@@ -65,13 +68,18 @@ def send_catch_log(
     return responses
 
 
-def send_catch_log_deferred(signal=Any, sender=Anonymous, *arguments, **named):
+def send_catch_log_deferred(
+    signal: TypingAny = Any,
+    sender: TypingAny = Anonymous,
+    *arguments: TypingAny,
+    **named: TypingAny
+) -> Deferred:
     """Like send_catch_log but supports returning deferreds on signal handlers.
     Returns a deferred that gets fired once all signal handlers deferreds were
     fired.
     """
 
-    def logerror(failure, recv):
+    def logerror(failure: Failure, recv: Any) -> Failure:
         if dont_log is None or not isinstance(failure.value, dont_log):
             logger.error(
                 "Error caught on signal handler: %(receiver)s",
@@ -96,7 +104,7 @@ def send_catch_log_deferred(signal=Any, sender=Anonymous, *arguments, **named):
     return d
 
 
-def disconnect_all(signal=Any, sender=Any):
+def disconnect_all(signal: TypingAny = Any, sender: TypingAny = Any) -> None:
     """Disconnect all signal handlers. Useful for cleaning up after running
     tests
     """

scrapy/utils/sitemap.py

@@ -4,7 +4,7 @@ Module for processing Sitemaps.
 Note: The main purpose of this module is to provide support for the
 SitemapSpider, its API is subject to change without notice.
 """
-
+from typing import Any, Dict, Generator, Iterator, Optional
 from urllib.parse import urljoin
 
 import lxml.etree
@@ -14,7 +14,7 @@ class Sitemap:
     """Class to parse Sitemap (type=urlset) and Sitemap Index
     (type=sitemapindex) files"""
 
-    def __init__(self, xmltext):
+    def __init__(self, xmltext: str):
         xmlp = lxml.etree.XMLParser(
             recover=True, remove_comments=True, resolve_entities=False
         )
@@ -22,9 +22,9 @@ class Sitemap:
         rt = self._root.tag
         self.type = self._root.tag.split("}", 1)[1] if "}" in rt else rt
 
-    def __iter__(self):
+    def __iter__(self) -> Iterator[Dict[str, Any]]:
         for elem in self._root.getchildren():
-            d = {}
+            d: Dict[str, Any] = {}
             for el in elem.getchildren():
                 tag = el.tag
                 name = tag.split("}", 1)[1] if "}" in tag else tag
@@ -39,11 +39,13 @@ class Sitemap:
                 yield d
 
 
-def sitemap_urls_from_robots(robots_text, base_url=None):
+def sitemap_urls_from_robots(
+    robots_text: str, base_url: Optional[str] = None
+) -> Generator[str, Any, None]:
     """Return an iterator over all sitemap urls contained in the given
     robots.txt file
     """
     for line in robots_text.splitlines():
         if line.lstrip().lower().startswith("sitemap:"):
             url = line.split(":", 1)[1].strip()
-            yield urljoin(base_url, url)
+            yield urljoin(base_url or "", url)

scrapy/utils/spider.py

@@ -1,14 +1,54 @@
+from __future__ import annotations
+
 import inspect
 import logging
+from types import CoroutineType, ModuleType
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    AsyncGenerator,
+    Generator,
+    Iterable,
+    Literal,
+    Optional,
+    Type,
+    TypeVar,
+    Union,
+    overload,
+)
 
+from twisted.internet.defer import Deferred
+
+from scrapy import Request
 from scrapy.spiders import Spider
 from scrapy.utils.defer import deferred_from_coro
 from scrapy.utils.misc import arg_to_iter
 
+if TYPE_CHECKING:
+    from scrapy.spiderloader import SpiderLoader
+
 logger = logging.getLogger(__name__)
 
+_T = TypeVar("_T")
 
-def iterate_spider_output(result):
+
+# https://stackoverflow.com/questions/60222982
+@overload
+def iterate_spider_output(result: AsyncGenerator) -> AsyncGenerator:  # type: ignore[misc]
+    ...
+
+
+@overload
+def iterate_spider_output(result: CoroutineType) -> Deferred:
+    ...
+
+
+@overload
+def iterate_spider_output(result: _T) -> Iterable:
+    ...
+
+
+def iterate_spider_output(result: Any) -> Union[Iterable, AsyncGenerator, Deferred]:
     if inspect.isasyncgen(result):
         return result
     if inspect.iscoroutine(result):
@@ -18,7 +58,7 @@ def iterate_spider_output(result):
     return arg_to_iter(deferred_from_coro(result))
 
 
-def iter_spider_classes(module):
+def iter_spider_classes(module: ModuleType) -> Generator[Type[Spider], Any, None]:
     """Return an iterator over all spider classes defined in the given module
     that can be instantiated (i.e. which have name)
     """
@@ -36,9 +76,46 @@ def iter_spider_classes(module):
             yield obj
 
 
+@overload
 def spidercls_for_request(
-    spider_loader, request, default_spidercls=None, log_none=False, log_multiple=False
-):
+    spider_loader: SpiderLoader,
+    request: Request,
+    default_spidercls: Type[Spider],
+    log_none: bool = ...,
+    log_multiple: bool = ...,
+) -> Type[Spider]:
+    ...
+
+
+@overload
+def spidercls_for_request(
+    spider_loader: SpiderLoader,
+    request: Request,
+    default_spidercls: Literal[None],
+    log_none: bool = ...,
+    log_multiple: bool = ...,
+) -> Optional[Type[Spider]]:
+    ...
+
+
+@overload
+def spidercls_for_request(
+    spider_loader: SpiderLoader,
+    request: Request,
+    *,
+    log_none: bool = ...,
+    log_multiple: bool = ...,
+) -> Optional[Type[Spider]]:
+    ...
+
+
+def spidercls_for_request(
+    spider_loader: SpiderLoader,
+    request: Request,
+    default_spidercls: Optional[Type[Spider]] = None,
+    log_none: bool = False,
+    log_multiple: bool = False,
+) -> Optional[Type[Spider]]:
     """Return a spider class that handles the given Request.
 
     This will look for the spiders that can handle the given request (using

scrapy/utils/ssl.py

@@ -1,4 +1,4 @@
-from typing import Any, Optional, cast
+from typing import Any, Optional
 
 import OpenSSL._util as pyOpenSSLutil
 import OpenSSL.SSL
@@ -58,9 +58,6 @@ def get_temp_key_info(ssl_object: Any) -> Optional[str]:
 
 
 def get_openssl_version() -> str:
-    # https://github.com/python/typeshed/issues/10024
-    system_openssl_bytes = cast(
-        bytes, OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION)
-    )
+    system_openssl_bytes = OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION)
     system_openssl = system_openssl_bytes.decode("ascii", errors="replace")
     return f"{OpenSSL.version.__version__} ({system_openssl})"

scrapy/utils/testproc.py

@@ -1,7 +1,13 @@
+from __future__ import annotations
+
 import os
 import sys
+from typing import Iterable, Optional, Tuple, cast
 
-from twisted.internet import defer, protocol
+from twisted.internet.defer import Deferred
+from twisted.internet.error import ProcessTerminated
+from twisted.internet.protocol import ProcessProtocol
+from twisted.python.failure import Failure
 
 
 class ProcessTest:
@@ -9,7 +15,12 @@ class ProcessTest:
     prefix = [sys.executable, "-m", "scrapy.cmdline"]
     cwd = os.getcwd()  # trial chdirs to temp dir
 
-    def execute(self, args, check_code=True, settings=None):
+    def execute(
+        self,
+        args: Iterable[str],
+        check_code: bool = True,
+        settings: Optional[str] = None,
+    ) -> Deferred:
         from twisted.internet import reactor
 
         env = os.environ.copy()
@@ -21,29 +32,31 @@ class ProcessTest:
         reactor.spawnProcess(pp, cmd[0], cmd, env=env, path=self.cwd)
         return pp.deferred
 
-    def _process_finished(self, pp, cmd, check_code):
+    def _process_finished(
+        self, pp: TestProcessProtocol, cmd: str, check_code: bool
+    ) -> Tuple[int, bytes, bytes]:
         if pp.exitcode and check_code:
             msg = f"process {cmd} exit with code {pp.exitcode}"
-            msg += f"\n>>> stdout <<<\n{pp.out}"
+            msg += f"\n>>> stdout <<<\n{pp.out.decode()}"
             msg += "\n"
-            msg += f"\n>>> stderr <<<\n{pp.err}"
+            msg += f"\n>>> stderr <<<\n{pp.err.decode()}"
             raise RuntimeError(msg)
-        return pp.exitcode, pp.out, pp.err
+        return cast(int, pp.exitcode), pp.out, pp.err
 
 
-class TestProcessProtocol(protocol.ProcessProtocol):
-    def __init__(self):
-        self.deferred = defer.Deferred()
-        self.out = b""
-        self.err = b""
-        self.exitcode = None
+class TestProcessProtocol(ProcessProtocol):
+    def __init__(self) -> None:
+        self.deferred: Deferred = Deferred()
+        self.out: bytes = b""
+        self.err: bytes = b""
+        self.exitcode: Optional[int] = None
 
-    def outReceived(self, data):
+    def outReceived(self, data: bytes) -> None:
         self.out += data
 
-    def errReceived(self, data):
+    def errReceived(self, data: bytes) -> None:
         self.err += data
 
-    def processEnded(self, status):
-        self.exitcode = status.value.exitCode
+    def processEnded(self, status: Failure) -> None:
+        self.exitcode = cast(ProcessTerminated, status.value).exitCode
         self.deferred.callback(self)

scrapy/utils/trackref.py

@@ -12,9 +12,14 @@ alias to object in that case).
 from collections import defaultdict
 from operator import itemgetter
 from time import time
-from typing import DefaultDict
+from typing import TYPE_CHECKING, Any, DefaultDict, Iterable
 from weakref import WeakKeyDictionary
 
+if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
+
+
 NoneType = type(None)
 live_refs: DefaultDict[type, WeakKeyDictionary] = defaultdict(WeakKeyDictionary)
 
@@ -24,13 +29,14 @@ class object_ref:
 
     __slots__ = ()
 
-    def __new__(cls, *args, **kwargs):
+    def __new__(cls, *args: Any, **kwargs: Any) -> "Self":
         obj = object.__new__(cls)
         live_refs[cls][obj] = time()
         return obj
 
 
-def format_live_refs(ignore=NoneType):
+# using Any as it's hard to type type(None)
+def format_live_refs(ignore: Any = NoneType) -> str:
     """Return a tabular representation of tracked objects"""
     s = "Live References\n\n"
     now = time()
@@ -44,12 +50,12 @@ def format_live_refs(ignore=NoneType):
     return s
 
 
-def print_live_refs(*a, **kw):
+def print_live_refs(*a: Any, **kw: Any) -> None:
     """Print tracked objects"""
     print(format_live_refs(*a, **kw))
 
 
-def get_oldest(class_name):
+def get_oldest(class_name: str) -> Any:
     """Get the oldest object for a specific class name"""
     for cls, wdict in live_refs.items():
         if cls.__name__ == class_name:
@@ -58,8 +64,9 @@ def get_oldest(class_name):
             return min(wdict.items(), key=itemgetter(1))[0]
 
 
-def iter_all(class_name):
+def iter_all(class_name: str) -> Iterable[Any]:
     """Iterate over all objects of the same class by its class name"""
     for cls, wdict in live_refs.items():
         if cls.__name__ == class_name:
             return wdict.keys()
+    return []

tests/test_contracts.py

@@ -63,6 +63,13 @@ class TestSpider(Spider):
         """
         return Request("http://scrapy.org", callback=self.returns_item)
 
+    async def returns_request_async(self, response):
+        """async method which returns request
+        @url http://scrapy.org
+        @returns requests 1
+        """
+        return Request("http://scrapy.org", callback=self.returns_item)
+
     def returns_item(self, response):
         """method which returns item
         @url http://scrapy.org
@@ -337,6 +344,14 @@ class ContractsManagerTest(unittest.TestCase):
         request.callback(response)
         self.should_fail()
 
+    def test_returns_async(self):
+        spider = TestSpider()
+        response = ResponseMock()
+
+        request = self.conman.from_method(spider.returns_request_async, self.results)
+        request.callback(response)
+        self.should_error()
+
     def test_scrapes(self):
         spider = TestSpider()
         response = ResponseMock()

tests/test_feedexport.py

@@ -2978,8 +2978,8 @@ class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase):
 
     def test_init(self):
         settings_dict = {
-            "FEED_URI": "file:///tmp/foobar",
-            "FEED_STORAGES": {"file": FTPFeedStorageWithoutFeedOptions},
+            "FEED_URI": "ftp://localhost/foo",
+            "FEED_STORAGES": {"ftp": FTPFeedStorageWithoutFeedOptions},
         }
         with pytest.warns(
             ScrapyDeprecationWarning,
@@ -3000,8 +3000,8 @@ class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase):
 
     def test_from_crawler(self):
         settings_dict = {
-            "FEED_URI": "file:///tmp/foobar",
-            "FEED_STORAGES": {"file": FTPFeedStorageWithoutFeedOptionsWithFromCrawler},
+            "FEED_URI": "ftp://localhost/foo",
+            "FEED_STORAGES": {"ftp": FTPFeedStorageWithoutFeedOptionsWithFromCrawler},
         }
         with pytest.warns(
             ScrapyDeprecationWarning,

tox.ini

@@ -41,6 +41,8 @@ deps =
     types-Pygments==2.15.0.2
     types-pyOpenSSL==23.2.0.2
     types-setuptools==68.0.0.3
+    # 2.1.2 fixes a typing bug: https://github.com/scrapy/w3lib/pull/211
+    w3lib >= 2.1.2
 commands =
     mypy {posargs: scrapy tests}