From d31829b72f6238a92a42c0990953d5056e8f5778 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 14 Jul 2023 23:13:15 +0400
Subject: [PATCH 01/14] More typing for scrapy/utils.

---
 scrapy/exceptions.py            |  3 +-
 scrapy/extensions/feedexport.py | 25 +++++++++++------
 scrapy/utils/conf.py            | 49 ++++++++++++++++++++++-----------
 scrapy/utils/ftp.py             | 15 ++++++++--
 scrapy/utils/job.py             |  2 +-
 scrapy/utils/ossignal.py        |  3 +-
 scrapy/utils/project.py         |  8 +++---
 scrapy/utils/sitemap.py         | 14 ++++++----
 scrapy/utils/ssl.py             |  7 ++---
 scrapy/utils/trackref.py        | 19 +++++++++----
 10 files changed, 93 insertions(+), 52 deletions(-)

diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py
index fedd02805..6e83e4a00 100644
--- a/scrapy/exceptions.py
+++ b/scrapy/exceptions.py
@@ -4,6 +4,7 @@ Scrapy core exceptions
 These exceptions are documented in docs/topics/exceptions.rst. Please don't add
 new exceptions here without documenting them there.
 """
+from typing import Any
 
 # Internal
 
@@ -77,7 +78,7 @@ class NotSupported(Exception):
 class UsageError(Exception):
     """To indicate a command-line usage error"""
 
-    def __init__(self, *a, **kw):
+    def __init__(self, *a: Any, **kw: Any):
         self.print_help = kw.pop("print_help", True)
         super().__init__(*a, **kw)
 
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index c8022ff57..2bbcaf3ad 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -11,7 +11,7 @@ import warnings
 from datetime import datetime
 from pathlib import Path, PureWindowsPath
 from tempfile import NamedTemporaryFile
-from typing import IO, Any, Callable, List, Optional, Tuple, Union
+from typing import IO, Any, Callable, Dict, List, Optional, Tuple, Union
 from urllib.parse import unquote, urlparse
 
 from twisted.internet import defer, threads
@@ -282,15 +282,22 @@ class GCSFeedStorage(BlockingFeedStorage):
 
 
 class FTPFeedStorage(BlockingFeedStorage):
-    def __init__(self, uri, use_active_mode=False, *, feed_options=None):
+    def __init__(
+        self,
+        uri: str,
+        use_active_mode: bool = False,
+        *,
+        feed_options: Optional[Dict[str, Any]] = None,
+    ):
         u = urlparse(uri)
-        self.host = u.hostname
-        self.port = int(u.port or "21")
-        self.username = u.username
-        self.password = unquote(u.password or "")
-        self.path = u.path
-        self.use_active_mode = use_active_mode
-        self.overwrite = not feed_options or feed_options.get("overwrite", True)
+        assert u.hostname
+        self.host: str = u.hostname
+        self.port: int = int(u.port or "21")
+        self.username: str = u.username or ""
+        self.password: str = unquote(u.password or "")
+        self.path: str = u.path
+        self.use_active_mode: bool = use_active_mode
+        self.overwrite: bool = not feed_options or feed_options.get("overwrite", True)
 
     @classmethod
     def from_crawler(cls, crawler, uri, *, feed_options=None):
diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
index 1889f7571..641dfa4a2 100644
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -5,7 +5,18 @@ import warnings
 from configparser import ConfigParser
 from operator import itemgetter
 from pathlib import Path
-from typing import Any, Dict, List, Optional, Union
+from typing import (
+    Any,
+    Callable,
+    Collection,
+    Dict,
+    Iterable,
+    List,
+    Mapping,
+    MutableMapping,
+    Optional,
+    Union,
+)
 
 from scrapy.exceptions import ScrapyDeprecationWarning, UsageError
 from scrapy.settings import BaseSettings
@@ -13,17 +24,21 @@ from scrapy.utils.deprecate import update_classpath
 from scrapy.utils.python import without_none_values
 
 
-def build_component_list(compdict, custom=None, convert=update_classpath):
+def build_component_list(
+    compdict: MutableMapping[Any, Any],
+    custom: Any = None,
+    convert: Callable[[Any], Any] = update_classpath,
+) -> List[Any]:
     """Compose a component list from a { class: order } dictionary."""
 
-    def _check_components(complist):
+    def _check_components(complist: Collection[Any]) -> None:
         if len({convert(c) for c in complist}) != len(complist):
             raise ValueError(
                 f"Some paths in {complist!r} convert to the same object, "
                 "please update your settings"
             )
 
-    def _map_keys(compdict):
+    def _map_keys(compdict: Mapping[Any, Any]) -> Union[BaseSettings, Dict[Any, Any]]:
         if isinstance(compdict, BaseSettings):
             compbs = BaseSettings()
             for k, v in compdict.items():
@@ -41,7 +56,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath):
         _check_components(compdict)
         return {convert(k): v for k, v in compdict.items()}
 
-    def _validate_values(compdict):
+    def _validate_values(compdict: Mapping[Any, Any]) -> None:
         """Fail if a value in the components dict is not a real number or None."""
         for name, value in compdict.items():
             if value is not None and not isinstance(value, numbers.Real):
@@ -60,7 +75,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath):
         )
         if isinstance(custom, (list, tuple)):
             _check_components(custom)
-            return type(custom)(convert(c) for c in custom)
+            return type(custom)(convert(c) for c in custom)  # type: ignore[return-value]
         compdict.update(custom)
 
     _validate_values(compdict)
@@ -68,7 +83,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath):
     return [k for k, v in sorted(compdict.items(), key=itemgetter(1))]
 
 
-def arglist_to_dict(arglist):
+def arglist_to_dict(arglist: List[str]) -> Dict[str, str]:
     """Convert a list of arguments like ['arg1=val1', 'arg2=val2', ...] to a
     dict
     """
@@ -91,7 +106,7 @@ def closest_scrapy_cfg(
     return closest_scrapy_cfg(path.parent, path)
 
 
-def init_env(project="default", set_syspath=True):
+def init_env(project: str = "default", set_syspath: bool = True) -> None:
     """Initialize environment to use command-line tool from inside a project
     dir. This sets the Scrapy settings module and modifies the Python path to
     be able to locate the project module.
@@ -106,7 +121,7 @@ def init_env(project="default", set_syspath=True):
             sys.path.append(projdir)
 
 
-def get_config(use_closest=True):
+def get_config(use_closest: bool = True) -> ConfigParser:
     """Get Scrapy config file as a ConfigParser"""
     sources = get_sources(use_closest)
     cfg = ConfigParser()
@@ -114,7 +129,7 @@ def get_config(use_closest=True):
     return cfg
 
 
-def get_sources(use_closest=True) -> List[str]:
+def get_sources(use_closest: bool = True) -> List[str]:
     xdg_config_home = (
         os.environ.get("XDG_CONFIG_HOME") or Path("~/.config").expanduser()
     )
@@ -129,7 +144,9 @@ def get_sources(use_closest=True) -> List[str]:
     return sources
 
 
-def feed_complete_default_values_from_settings(feed, settings):
+def feed_complete_default_values_from_settings(
+    feed: Dict[str, Any], settings: BaseSettings
+) -> Dict[str, Any]:
     out = feed.copy()
     out.setdefault("batch_item_count", settings.getint("FEED_EXPORT_BATCH_ITEM_COUNT"))
     out.setdefault("encoding", settings["FEED_EXPORT_ENCODING"])
@@ -145,21 +162,21 @@ def feed_complete_default_values_from_settings(feed, settings):
 
 
 def feed_process_params_from_cli(
-    settings,
+    settings: BaseSettings,
     output: List[str],
-    output_format=None,
+    output_format: Optional[str] = None,
     overwrite_output: Optional[List[str]] = None,
-):
+) -> Dict[str, Dict[str, Any]]:
     """
     Receives feed export params (from the 'crawl' or 'runspider' commands),
     checks for inconsistencies in their quantities and returns a dictionary
     suitable to be used as the FEEDS setting.
     """
-    valid_output_formats = without_none_values(
+    valid_output_formats: Iterable[str] = without_none_values(
         settings.getwithbase("FEED_EXPORTERS")
     ).keys()
 
-    def check_valid_format(output_format):
+    def check_valid_format(output_format: str) -> None:
         if output_format not in valid_output_formats:
             raise UsageError(
                 f"Unrecognized output format '{output_format}'. "
diff --git a/scrapy/utils/ftp.py b/scrapy/utils/ftp.py
index 6bf6e9195..c77681a53 100644
--- a/scrapy/utils/ftp.py
+++ b/scrapy/utils/ftp.py
@@ -1,9 +1,10 @@
 import posixpath
 from ftplib import FTP, error_perm
 from posixpath import dirname
+from typing import IO
 
 
-def ftp_makedirs_cwd(ftp, path, first_call=True):
+def ftp_makedirs_cwd(ftp: FTP, path: str, first_call: bool = True) -> None:
     """Set the current directory of the FTP connection given in the ``ftp``
     argument (as a ftplib.FTP object), creating all parent directories if they
     don't exist. The ftplib.FTP object must be already connected and logged in.
@@ -18,8 +19,16 @@ def ftp_makedirs_cwd(ftp, path, first_call=True):
 
 
 def ftp_store_file(
-    *, path, file, host, port, username, password, use_active_mode=False, overwrite=True
-):
+    *,
+    path: str,
+    file: IO,
+    host: str,
+    port: int,
+    username: str,
+    password: str,
+    use_active_mode: bool = False,
+    overwrite: bool = True,
+) -> None:
     """Opens a FTP connection with passed credentials,sets current directory
     to the directory extracted from given path, then uploads the file to server
     """
diff --git a/scrapy/utils/job.py b/scrapy/utils/job.py
index 858affc03..c49f7d758 100644
--- a/scrapy/utils/job.py
+++ b/scrapy/utils/job.py
@@ -5,7 +5,7 @@ from scrapy.settings import BaseSettings
 
 
 def job_dir(settings: BaseSettings) -> Optional[str]:
-    path = settings["JOBDIR"]
+    path: str = settings["JOBDIR"]
     if path and not Path(path).exists():
         Path(path).mkdir(parents=True)
     return path
diff --git a/scrapy/utils/ossignal.py b/scrapy/utils/ossignal.py
index 7646264a8..f835a2221 100644
--- a/scrapy/utils/ossignal.py
+++ b/scrapy/utils/ossignal.py
@@ -1,4 +1,5 @@
 import signal
+from typing import Callable
 
 signal_names = {}
 for signame in dir(signal):
@@ -8,7 +9,7 @@ for signame in dir(signal):
             signal_names[signum] = signame
 
 
-def install_shutdown_handlers(function, override_sigint=True):
+def install_shutdown_handlers(function: Callable, override_sigint: bool = True) -> None:
     """Install the given function as a signal handler for all common shutdown
     signals (such as SIGINT, SIGTERM, etc). If override_sigint is ``False`` the
     SIGINT handler won't be install if there is already a handler in place
diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py
index 652b74759..a2c224b90 100644
--- a/scrapy/utils/project.py
+++ b/scrapy/utils/project.py
@@ -11,7 +11,7 @@ ENVVAR = "SCRAPY_SETTINGS_MODULE"
 DATADIR_CFG_SECTION = "datadir"
 
 
-def inside_project():
+def inside_project() -> bool:
     scrapy_module = os.environ.get(ENVVAR)
     if scrapy_module:
         try:
@@ -25,7 +25,7 @@ def inside_project():
     return bool(closest_scrapy_cfg())
 
 
-def project_data_dir(project="default") -> str:
+def project_data_dir(project: str = "default") -> str:
     """Return the current project data dir, creating it if it doesn't exist"""
     if not inside_project():
         raise NotConfigured("Not inside a project")
@@ -44,7 +44,7 @@ def project_data_dir(project="default") -> str:
     return str(d)
 
 
-def data_path(path: str, createdir=False) -> str:
+def data_path(path: str, createdir: bool = False) -> str:
     """
     Return the given path joined with the .scrapy data directory.
     If given an absolute path, return it unmodified.
@@ -60,7 +60,7 @@ def data_path(path: str, createdir=False) -> str:
     return str(path_obj)
 
 
-def get_project_settings():
+def get_project_settings() -> Settings:
     if ENVVAR not in os.environ:
         project = os.environ.get("SCRAPY_PROJECT", "default")
         init_env(project)
diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py
index 2622c2775..3d2ecc9a7 100644
--- a/scrapy/utils/sitemap.py
+++ b/scrapy/utils/sitemap.py
@@ -4,7 +4,7 @@ Module for processing Sitemaps.
 Note: The main purpose of this module is to provide support for the
 SitemapSpider, its API is subject to change without notice.
 """
-
+from typing import Any, Dict, Generator, Iterator, Optional
 from urllib.parse import urljoin
 
 import lxml.etree
@@ -14,7 +14,7 @@ class Sitemap:
     """Class to parse Sitemap (type=urlset) and Sitemap Index
     (type=sitemapindex) files"""
 
-    def __init__(self, xmltext):
+    def __init__(self, xmltext: str):
         xmlp = lxml.etree.XMLParser(
             recover=True, remove_comments=True, resolve_entities=False
         )
@@ -22,9 +22,9 @@ class Sitemap:
         rt = self._root.tag
         self.type = self._root.tag.split("}", 1)[1] if "}" in rt else rt
 
-    def __iter__(self):
+    def __iter__(self) -> Iterator[Dict[str, Any]]:
         for elem in self._root.getchildren():
-            d = {}
+            d: Dict[str, Any] = {}
             for el in elem.getchildren():
                 tag = el.tag
                 name = tag.split("}", 1)[1] if "}" in tag else tag
@@ -39,11 +39,13 @@ class Sitemap:
                 yield d
 
 
-def sitemap_urls_from_robots(robots_text, base_url=None):
+def sitemap_urls_from_robots(
+    robots_text: str, base_url: Optional[str] = None
+) -> Generator[str, Any, None]:
     """Return an iterator over all sitemap urls contained in the given
     robots.txt file
     """
     for line in robots_text.splitlines():
         if line.lstrip().lower().startswith("sitemap:"):
             url = line.split(":", 1)[1].strip()
-            yield urljoin(base_url, url)
+            yield urljoin(base_url or "", url)
diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py
index 03ae4ba9e..d520ef809 100644
--- a/scrapy/utils/ssl.py
+++ b/scrapy/utils/ssl.py
@@ -1,4 +1,4 @@
-from typing import Any, Optional, cast
+from typing import Any, Optional
 
 import OpenSSL._util as pyOpenSSLutil
 import OpenSSL.SSL
@@ -58,9 +58,6 @@ def get_temp_key_info(ssl_object: Any) -> Optional[str]:
 
 
 def get_openssl_version() -> str:
-    # https://github.com/python/typeshed/issues/10024
-    system_openssl_bytes = cast(
-        bytes, OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION)
-    )
+    system_openssl_bytes = OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION)
     system_openssl = system_openssl_bytes.decode("ascii", errors="replace")
     return f"{OpenSSL.version.__version__} ({system_openssl})"
diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py
index 01b980c93..9ff9a273f 100644
--- a/scrapy/utils/trackref.py
+++ b/scrapy/utils/trackref.py
@@ -12,9 +12,14 @@ alias to object in that case).
 from collections import defaultdict
 from operator import itemgetter
 from time import time
-from typing import DefaultDict
+from typing import TYPE_CHECKING, Any, DefaultDict, Iterable
 from weakref import WeakKeyDictionary
 
+if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
+
+
 NoneType = type(None)
 live_refs: DefaultDict[type, WeakKeyDictionary] = defaultdict(WeakKeyDictionary)
 
@@ -24,13 +29,14 @@ class object_ref:
 
     __slots__ = ()
 
-    def __new__(cls, *args, **kwargs):
+    def __new__(cls, *args: Any, **kwargs: Any) -> "Self":
         obj = object.__new__(cls)
         live_refs[cls][obj] = time()
         return obj
 
 
-def format_live_refs(ignore=NoneType):
+# using Any as it's hard to type type(None)
+def format_live_refs(ignore: Any = NoneType) -> str:
     """Return a tabular representation of tracked objects"""
     s = "Live References\n\n"
     now = time()
@@ -44,12 +50,12 @@ def format_live_refs(ignore=NoneType):
     return s
 
 
-def print_live_refs(*a, **kw):
+def print_live_refs(*a: Any, **kw: Any) -> None:
     """Print tracked objects"""
     print(format_live_refs(*a, **kw))
 
 
-def get_oldest(class_name):
+def get_oldest(class_name: str) -> Any:
     """Get the oldest object for a specific class name"""
     for cls, wdict in live_refs.items():
         if cls.__name__ == class_name:
@@ -58,8 +64,9 @@ def get_oldest(class_name):
             return min(wdict.items(), key=itemgetter(1))[0]
 
 
-def iter_all(class_name):
+def iter_all(class_name: str) -> Iterable[Any]:
     """Iterate over all objects of the same class by its class name"""
     for cls, wdict in live_refs.items():
         if cls.__name__ == class_name:
             return wdict.keys()
+    return []

From d015329d759dda72586b856ee92d787dc730f06e Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 31 Jul 2023 01:54:28 +0400
Subject: [PATCH 02/14] Add more typing for scrapy/utils/log.py.

---
 scrapy/utils/log.py | 60 +++++++++++++++++++++++++++++----------------
 1 file changed, 39 insertions(+), 21 deletions(-)

diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index 2ce4725f4..2013bfc43 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -1,8 +1,11 @@
+from __future__ import annotations
+
 import logging
 import sys
 import warnings
 from logging.config import dictConfig
-from typing import Tuple
+from types import TracebackType
+from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Type, Union, cast
 
 from twisted.python import log as twisted_log
 from twisted.python.failure import Failure
@@ -12,13 +15,25 @@ from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.settings import Settings
 from scrapy.utils.versions import scrapy_components_versions
 
+if TYPE_CHECKING:
+    from scrapy.crawler import Crawler
+
 logger = logging.getLogger(__name__)
 
 
-def failure_to_exc_info(failure: Failure):
+def failure_to_exc_info(
+    failure: Failure,
+) -> Optional[Tuple[Type[BaseException], BaseException, Optional[TracebackType]]]:
     """Extract exc_info from Failure instances"""
     if isinstance(failure, Failure):
-        return (failure.type, failure.value, failure.getTracebackObject())
+        assert failure.type
+        assert failure.value
+        return (
+            failure.type,
+            failure.value,
+            cast(Optional[TracebackType], failure.getTracebackObject()),
+        )
+    return None
 
 
 class TopLevelFormatter(logging.Filter):
@@ -33,10 +48,10 @@ class TopLevelFormatter(logging.Filter):
     ``loggers`` list where it should act.
     """
 
-    def __init__(self, loggers=None):
-        self.loggers = loggers or []
+    def __init__(self, loggers: Optional[List[str]] = None):
+        self.loggers: List[str] = loggers or []
 
-    def filter(self, record):
+    def filter(self, record: logging.LogRecord) -> bool:
         if any(record.name.startswith(logger + ".") for logger in self.loggers):
             record.name = record.name.split(".", 1)[0]
         return True
@@ -62,7 +77,9 @@ DEFAULT_LOGGING = {
 }
 
 
-def configure_logging(settings=None, install_root_handler=True):
+def configure_logging(
+    settings: Union[Settings, dict, None] = None, install_root_handler: bool = True
+) -> None:
     """
     Initialize logging defaults for Scrapy.
 
@@ -99,13 +116,13 @@ def configure_logging(settings=None, install_root_handler=True):
         settings = Settings(settings)
 
     if settings.getbool("LOG_STDOUT"):
-        sys.stdout = StreamLogger(logging.getLogger("stdout"))
+        sys.stdout = StreamLogger(logging.getLogger("stdout"))  # type: ignore[assignment]
 
     if install_root_handler:
         install_scrapy_root_handler(settings)
 
 
-def install_scrapy_root_handler(settings):
+def install_scrapy_root_handler(settings: Settings) -> None:
     global _scrapy_root_handler
 
     if (
@@ -118,16 +135,17 @@ def install_scrapy_root_handler(settings):
     logging.root.addHandler(_scrapy_root_handler)
 
 
-def get_scrapy_root_handler():
+def get_scrapy_root_handler() -> Optional[logging.Handler]:
     return _scrapy_root_handler
 
 
-_scrapy_root_handler = None
+_scrapy_root_handler: Optional[logging.Handler] = None
 
 
-def _get_handler(settings):
+def _get_handler(settings: Settings) -> logging.Handler:
     """Return a log handler object according to settings"""
     filename = settings.get("LOG_FILE")
+    handler: logging.Handler
     if filename:
         mode = "a" if settings.getbool("LOG_FILE_APPEND") else "w"
         encoding = settings.get("LOG_ENCODING")
@@ -181,16 +199,16 @@ class StreamLogger:
         https://www.electricmonk.nl/log/2011/08/14/redirect-stdout-and-stderr-to-a-logger-in-python/
     """
 
-    def __init__(self, logger, log_level=logging.INFO):
-        self.logger = logger
-        self.log_level = log_level
-        self.linebuf = ""
+    def __init__(self, logger: logging.Logger, log_level: int = logging.INFO):
+        self.logger: logging.Logger = logger
+        self.log_level: int = log_level
+        self.linebuf: str = ""
 
-    def write(self, buf):
+    def write(self, buf: str) -> None:
         for line in buf.rstrip().splitlines():
             self.logger.log(self.log_level, line.rstrip())
 
-    def flush(self):
+    def flush(self) -> None:
         for h in self.logger.handlers:
             h.flush()
 
@@ -198,11 +216,11 @@ class StreamLogger:
 class LogCounterHandler(logging.Handler):
     """Record log levels count into a crawler stats"""
 
-    def __init__(self, crawler, *args, **kwargs):
+    def __init__(self, crawler: Crawler, *args: Any, **kwargs: Any):
         super().__init__(*args, **kwargs)
-        self.crawler = crawler
+        self.crawler: Crawler = crawler
 
-    def emit(self, record):
+    def emit(self, record: logging.LogRecord) -> None:
         sname = f"log_count/{record.levelname}"
         self.crawler.stats.inc_value(sname)
 

From c43798cb9bee99ccf96047f3dfcc6debb65973d4 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 2 Aug 2023 22:15:37 +0400
Subject: [PATCH 03/14] More typing for scrapy/utils/defer.py and
 scrapy/utils/spider.py.

---
 scrapy/spiderloader.py     |  9 +++++----
 scrapy/spiders/__init__.py |  2 +-
 scrapy/utils/defer.py      | 24 ++++++++++++++++++++----
 scrapy/utils/spider.py     | 31 +++++++++++++++++++++++++++----
 4 files changed, 53 insertions(+), 13 deletions(-)

diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py
index ea5a26e77..13d6f9f87 100644
--- a/scrapy/spiderloader.py
+++ b/scrapy/spiderloader.py
@@ -1,11 +1,12 @@
 import traceback
 import warnings
 from collections import defaultdict
+from types import ModuleType
 from typing import DefaultDict, Dict, List, Tuple, Type
 
 from zope.interface import implementer
 
-from scrapy import Spider
+from scrapy import Request, Spider
 from scrapy.interfaces import ISpiderLoader
 from scrapy.settings import BaseSettings
 from scrapy.utils.misc import walk_modules
@@ -45,12 +46,12 @@ class SpiderLoader:
                 category=UserWarning,
             )
 
-    def _load_spiders(self, module):
+    def _load_spiders(self, module: ModuleType) -> None:
         for spcls in iter_spider_classes(module):
             self._found[spcls.name].append((module.__name__, spcls.__name__))
             self._spiders[spcls.name] = spcls
 
-    def _load_all_spiders(self):
+    def _load_all_spiders(self) -> None:
         for name in self.spider_modules:
             try:
                 for module in walk_modules(name):
@@ -81,7 +82,7 @@ class SpiderLoader:
         except KeyError:
             raise KeyError(f"Spider not found: {spider_name}")
 
-    def find_by_request(self, request):
+    def find_by_request(self, request: Request) -> List[str]:
         """
         Return the list of spider names that can handle the given request.
         """
diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py
index 3502f8b27..388439f4f 100644
--- a/scrapy/spiders/__init__.py
+++ b/scrapy/spiders/__init__.py
@@ -82,7 +82,7 @@ class Spider(object_ref):
         settings.setdict(cls.custom_settings or {}, priority="spider")
 
     @classmethod
-    def handles_request(cls, request):
+    def handles_request(cls, request: Request) -> bool:
         return url_is_from_spider(request.url, cls)
 
     @staticmethod
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index 03f026ce9..bf3c5ef5b 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -5,11 +5,13 @@ import asyncio
 import inspect
 from asyncio import Future
 from functools import wraps
+from types import CoroutineType
 from typing import (
     Any,
     AsyncGenerator,
     AsyncIterable,
     AsyncIterator,
+    Awaitable,
     Callable,
     Coroutine,
     Dict,
@@ -19,8 +21,10 @@ from typing import (
     List,
     Optional,
     Tuple,
+    TypeVar,
     Union,
     cast,
+    overload,
 )
 
 from twisted.internet import defer
@@ -186,9 +190,7 @@ class _AsyncCooperatorAdapter(Iterator):
     def _call_anext(self) -> None:
         # This starts waiting for the next result from aiterator.
         # If aiterator is exhausted, _errback will be called.
-        self.anext_deferred = cast(
-            Deferred, deferred_from_coro(self.aiterator.__anext__())
-        )
+        self.anext_deferred = deferred_from_coro(self.aiterator.__anext__())
         self.anext_deferred.addCallbacks(self._callback, self._errback)
 
     def __next__(self) -> Deferred:
@@ -297,7 +299,21 @@ async def aiter_errback(
             errback(failure.Failure(), *a, **kw)
 
 
-def deferred_from_coro(o: Any) -> Any:
+_CT = TypeVar("_CT", bound=Union[Awaitable, CoroutineType, Future])
+_T = TypeVar("_T")
+
+
+@overload
+def deferred_from_coro(o: _CT) -> Deferred:
+    ...
+
+
+@overload
+def deferred_from_coro(o: _T) -> _T:
+    ...
+
+
+def deferred_from_coro(o: _T) -> Union[Deferred, _T]:
     """Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine"""
     if isinstance(o, Deferred):
         return o
diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py
index 86449eeb2..3228eda49 100644
--- a/scrapy/utils/spider.py
+++ b/scrapy/utils/spider.py
@@ -1,14 +1,33 @@
+from __future__ import annotations
+
 import inspect
 import logging
+from types import ModuleType
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    AsyncIterable,
+    Generator,
+    Iterable,
+    Optional,
+    Type,
+    Union,
+)
 
+from twisted.internet.defer import Deferred
+
+from scrapy import Request
 from scrapy.spiders import Spider
 from scrapy.utils.defer import deferred_from_coro
 from scrapy.utils.misc import arg_to_iter
 
+if TYPE_CHECKING:
+    from scrapy.spiderloader import SpiderLoader
+
 logger = logging.getLogger(__name__)
 
 
-def iterate_spider_output(result):
+def iterate_spider_output(result: Any) -> Union[Iterable, AsyncIterable, Deferred]:
     if inspect.isasyncgen(result):
         return result
     if inspect.iscoroutine(result):
@@ -18,7 +37,7 @@ def iterate_spider_output(result):
     return arg_to_iter(deferred_from_coro(result))
 
 
-def iter_spider_classes(module):
+def iter_spider_classes(module: ModuleType) -> Generator[Type[Spider], Any, None]:
     """Return an iterator over all spider classes defined in the given module
     that can be instantiated (i.e. which have name)
     """
@@ -37,8 +56,12 @@ def iter_spider_classes(module):
 
 
 def spidercls_for_request(
-    spider_loader, request, default_spidercls=None, log_none=False, log_multiple=False
-):
+    spider_loader: SpiderLoader,
+    request: Request,
+    default_spidercls: Optional[Type[Spider]] = None,
+    log_none: bool = False,
+    log_multiple: bool = False,
+) -> Optional[Type[Spider]]:
     """Return a spider class that handles the given Request.
 
     This will look for the spiders that can handle the given request (using

From d1f87e4f088c0757dd833d2a0841be0b830deb57 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 2 Aug 2023 23:11:15 +0400
Subject: [PATCH 04/14] More typing for scrapy/utils/iterators.py.

---
 scrapy/http/response/text.py  |   6 +-
 scrapy/selector/unified.py    |  18 ++++--
 scrapy/utils/iterators.py     | 106 ++++++++++++++++++++++++++--------
 tests/test_utils_iterators.py |   7 ++-
 4 files changed, 103 insertions(+), 34 deletions(-)

diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 5289f014a..f228e11c1 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -7,7 +7,7 @@ See documentation in docs/topics/request-response.rst
 
 import json
 from contextlib import suppress
-from typing import Generator, Tuple
+from typing import Generator, Optional, Tuple
 from urllib.parse import urljoin
 
 import parsel
@@ -37,7 +37,7 @@ class TextResponse(Response):
     def __init__(self, *args, **kwargs):
         self._encoding = kwargs.pop("encoding", None)
         self._cached_benc = None
-        self._cached_ubody = None
+        self._cached_ubody: Optional[str] = None
         self._cached_selector = None
         super().__init__(*args, **kwargs)
 
@@ -82,7 +82,7 @@ class TextResponse(Response):
         return self._cached_decoded_json
 
     @property
-    def text(self):
+    def text(self) -> str:
         """Body as unicode"""
         # access self.encoding before _cached_ubody to make sure
         # _body_inferred_encoding is called
diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py
index caff79e9c..863fb6032 100644
--- a/scrapy/selector/unified.py
+++ b/scrapy/selector/unified.py
@@ -1,10 +1,11 @@
 """
 XPath selectors based on lxml
 """
+from typing import Any, Optional, Type, Union
 
 from parsel import Selector as _ParselSelector
 
-from scrapy.http import HtmlResponse, XmlResponse
+from scrapy.http import HtmlResponse, TextResponse, XmlResponse
 from scrapy.utils.python import to_bytes
 from scrapy.utils.trackref import object_ref
 
@@ -13,14 +14,14 @@ __all__ = ["Selector", "SelectorList"]
 _NOT_SET = object()
 
 
-def _st(response, st):
+def _st(response: Optional[TextResponse], st: Optional[str]) -> str:
     if st is None:
         return "xml" if isinstance(response, XmlResponse) else "html"
     return st
 
 
-def _response_from_text(text, st):
-    rt = XmlResponse if st == "xml" else HtmlResponse
+def _response_from_text(text: Union[str, bytes], st: Optional[str]) -> TextResponse:
+    rt: Type[TextResponse] = XmlResponse if st == "xml" else HtmlResponse
     return rt(url="about:blank", encoding="utf-8", body=to_bytes(text, "utf-8"))
 
 
@@ -65,7 +66,14 @@ class Selector(_ParselSelector, object_ref):
     __slots__ = ["response"]
     selectorlist_cls = SelectorList
 
-    def __init__(self, response=None, text=None, type=None, root=_NOT_SET, **kwargs):
+    def __init__(
+        self,
+        response: Optional[TextResponse] = None,
+        text: Optional[str] = None,
+        type: Optional[str] = None,
+        root: Optional[Any] = _NOT_SET,
+        **kwargs: Any,
+    ):
         if response is not None and text is not None:
             raise ValueError(
                 f"{self.__class__.__name__}.__init__() received "
diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py
index 170055d5e..58850b843 100644
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@ -1,16 +1,37 @@
+from __future__ import annotations
+
 import csv
 import logging
 import re
 from io import StringIO
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    Dict,
+    Generator,
+    Iterable,
+    List,
+    Literal,
+    Optional,
+    Union,
+    cast,
+    overload,
+)
 
 from scrapy.http import Response, TextResponse
 from scrapy.selector import Selector
 from scrapy.utils.python import re_rsearch, to_unicode
 
+if TYPE_CHECKING:
+    from lxml._types import SupportsReadClose
+
 logger = logging.getLogger(__name__)
 
 
-def xmliter(obj, nodename):
+def xmliter(
+    obj: Union[Response, str, bytes], nodename: str
+) -> Generator[Selector, Any, None]:
     """Return a iterator of Selector's over all nodes of a XML document,
        given the name of the node to iterate. Useful for parsing XML feeds.
 
@@ -27,20 +48,22 @@ def xmliter(obj, nodename):
     NAMESPACE_RE = re.compile(r"((xmlns[:A-Za-z]*)=[^>\s]+)", re.S)
     text = _body_or_str(obj)
 
-    document_header = re.search(DOCUMENT_HEADER_RE, text)
-    document_header = document_header.group().strip() if document_header else ""
+    document_header_match = re.search(DOCUMENT_HEADER_RE, text)
+    document_header = (
+        document_header_match.group().strip() if document_header_match else ""
+    )
     header_end_idx = re_rsearch(HEADER_END_RE, text)
     header_end = text[header_end_idx[1] :].strip() if header_end_idx else ""
-    namespaces = {}
+    namespaces: Dict[str, str] = {}
     if header_end:
         for tagname in reversed(re.findall(END_TAG_RE, header_end)):
+            assert header_end_idx
             tag = re.search(
                 rf"<\s*{tagname}.*?xmlns[:=][^>]*>", text[: header_end_idx[1]], re.S
             )
             if tag:
-                namespaces.update(
-                    reversed(x) for x in re.findall(NAMESPACE_RE, tag.group())
-                )
+                for x in re.findall(NAMESPACE_RE, tag.group()):
+                    namespaces[x[1]] = x[0]
 
     r = re.compile(rf"<{nodename_patt}[\s>].*?</{nodename_patt}>", re.DOTALL)
     for match in r.finditer(text):
@@ -54,12 +77,19 @@ def xmliter(obj, nodename):
         yield Selector(text=nodetext, type="xml")
 
 
-def xmliter_lxml(obj, nodename, namespace=None, prefix="x"):
+def xmliter_lxml(
+    obj: Union[TextResponse, str, bytes],
+    nodename: str,
+    namespace: Optional[str] = None,
+    prefix: str = "x",
+) -> Generator[Selector, Any, None]:
     from lxml import etree
 
     reader = _StreamReader(obj)
     tag = f"{{{namespace}}}{nodename}" if namespace else nodename
-    iterable = etree.iterparse(reader, tag=tag, encoding=reader.encoding)
+    iterable = etree.iterparse(
+        cast(SupportsReadClose[bytes], reader), tag=tag, encoding=reader.encoding
+    )
     selxpath = "//" + (f"{prefix}:{nodename}" if namespace else nodename)
     for _, node in iterable:
         nodetext = etree.tostring(node, encoding="unicode")
@@ -71,30 +101,39 @@ def xmliter_lxml(obj, nodename, namespace=None, prefix="x"):
 
 
 class _StreamReader:
-    def __init__(self, obj):
-        self._ptr = 0
-        if isinstance(obj, Response):
+    def __init__(self, obj: Union[TextResponse, str, bytes]):
+        self._ptr: int = 0
+        self._text: Union[str, bytes]
+        if isinstance(obj, TextResponse):
             self._text, self.encoding = obj.body, obj.encoding
         else:
             self._text, self.encoding = obj, "utf-8"
-        self._is_unicode = isinstance(self._text, str)
+        self._is_unicode: bool = isinstance(self._text, str)
 
-    def read(self, n=65535):
-        self.read = self._read_unicode if self._is_unicode else self._read_string
+    def read(self, n: int = 65535) -> bytes:
+        self.read: Callable[[int], bytes] = (  # type: ignore[method-assign]
+            self._read_unicode if self._is_unicode else self._read_string
+        )
         return self.read(n).lstrip()
 
-    def _read_string(self, n=65535):
+    def _read_string(self, n: int = 65535) -> bytes:
         s, e = self._ptr, self._ptr + n
         self._ptr = e
-        return self._text[s:e]
+        return cast(bytes, self._text)[s:e]
 
-    def _read_unicode(self, n=65535):
+    def _read_unicode(self, n: int = 65535) -> bytes:
         s, e = self._ptr, self._ptr + n
         self._ptr = e
-        return self._text[s:e].encode("utf-8")
+        return cast(str, self._text)[s:e].encode("utf-8")
 
 
-def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):
+def csviter(
+    obj: Union[Response, str, bytes],
+    delimiter: Optional[str] = None,
+    headers: Optional[List[str]] = None,
+    encoding: Optional[str] = None,
+    quotechar: Optional[str] = None,
+) -> Generator[Dict[str, str], Any, None]:
     """Returns an iterator of dictionaries from the given csv object
 
     obj can be:
@@ -112,12 +151,12 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):
 
     encoding = obj.encoding if isinstance(obj, TextResponse) else encoding or "utf-8"
 
-    def row_to_unicode(row_):
+    def row_to_unicode(row_: Iterable) -> List[str]:
         return [to_unicode(field, encoding) for field in row_]
 
     lines = StringIO(_body_or_str(obj, unicode=True))
 
-    kwargs = {}
+    kwargs: Dict[str, Any] = {}
     if delimiter:
         kwargs["delimiter"] = delimiter
     if quotechar:
@@ -147,7 +186,24 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):
         yield dict(zip(headers, row))
 
 
-def _body_or_str(obj, unicode=True):
+@overload
+def _body_or_str(obj: Union[Response, str, bytes]) -> str:
+    ...
+
+
+@overload
+def _body_or_str(obj: Union[Response, str, bytes], unicode: Literal[True]) -> str:
+    ...
+
+
+@overload
+def _body_or_str(obj: Union[Response, str, bytes], unicode: Literal[False]) -> bytes:
+    ...
+
+
+def _body_or_str(
+    obj: Union[Response, str, bytes], unicode: bool = True
+) -> Union[str, bytes]:
     expected_types = (Response, str, bytes)
     if not isinstance(obj, expected_types):
         expected_types_str = " or ".join(t.__name__ for t in expected_types)
@@ -156,10 +212,10 @@ def _body_or_str(obj, unicode=True):
         )
     if isinstance(obj, Response):
         if not unicode:
-            return obj.body
+            return cast(bytes, obj.body)
         if isinstance(obj, TextResponse):
             return obj.text
-        return obj.body.decode("utf-8")
+        return cast(bytes, obj.body).decode("utf-8")
     if isinstance(obj, str):
         return obj if unicode else obj.encode("utf-8")
     return obj.decode("utf-8") if unicode else obj
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index 3598fa0bb..5dfd7e7ac 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -1,13 +1,18 @@
+from typing import Callable, Iterable, Union
+
 from pytest import mark
 from twisted.trial import unittest
 
+from scrapy import Selector
 from scrapy.http import Response, TextResponse, XmlResponse
 from scrapy.utils.iterators import _body_or_str, csviter, xmliter, xmliter_lxml
 from tests import get_testdata
 
 
 class XmliterTestCase(unittest.TestCase):
-    xmliter = staticmethod(xmliter)
+    xmliter: Callable[
+        [Union[TextResponse, str, bytes], str], Iterable[Selector]
+    ] = staticmethod(xmliter)
 
     def test_xmliter(self):
         body = b"""

From 9fe662d856a6b2496379585143aa2f4d023039f8 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 2 Aug 2023 23:42:59 +0400
Subject: [PATCH 05/14] Add typing for scrapy/utils/testproc.py.

---
 scrapy/utils/testproc.py | 45 ++++++++++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 16 deletions(-)

diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py
index 5f9bdef37..5f7a7db14 100644
--- a/scrapy/utils/testproc.py
+++ b/scrapy/utils/testproc.py
@@ -1,7 +1,13 @@
+from __future__ import annotations
+
 import os
 import sys
+from typing import Iterable, Optional, Tuple, cast
 
-from twisted.internet import defer, protocol
+from twisted.internet.defer import Deferred
+from twisted.internet.error import ProcessTerminated
+from twisted.internet.protocol import ProcessProtocol
+from twisted.python.failure import Failure
 
 
 class ProcessTest:
@@ -9,7 +15,12 @@ class ProcessTest:
     prefix = [sys.executable, "-m", "scrapy.cmdline"]
     cwd = os.getcwd()  # trial chdirs to temp dir
 
-    def execute(self, args, check_code=True, settings=None):
+    def execute(
+        self,
+        args: Iterable[str],
+        check_code: bool = True,
+        settings: Optional[str] = None,
+    ) -> Deferred:
         from twisted.internet import reactor
 
         env = os.environ.copy()
@@ -21,29 +32,31 @@ class ProcessTest:
         reactor.spawnProcess(pp, cmd[0], cmd, env=env, path=self.cwd)
         return pp.deferred
 
-    def _process_finished(self, pp, cmd, check_code):
+    def _process_finished(
+        self, pp: TestProcessProtocol, cmd: str, check_code: bool
+    ) -> Tuple[int, bytes, bytes]:
         if pp.exitcode and check_code:
             msg = f"process {cmd} exit with code {pp.exitcode}"
-            msg += f"\n>>> stdout <<<\n{pp.out}"
+            msg += f"\n>>> stdout <<<\n{pp.out.decode()}"
             msg += "\n"
-            msg += f"\n>>> stderr <<<\n{pp.err}"
+            msg += f"\n>>> stderr <<<\n{pp.err.decode()}"
             raise RuntimeError(msg)
-        return pp.exitcode, pp.out, pp.err
+        return cast(int, pp.exitcode), pp.out, pp.err
 
 
-class TestProcessProtocol(protocol.ProcessProtocol):
-    def __init__(self):
-        self.deferred = defer.Deferred()
-        self.out = b""
-        self.err = b""
-        self.exitcode = None
+class TestProcessProtocol(ProcessProtocol):
+    def __init__(self) -> None:
+        self.deferred: Deferred = Deferred()
+        self.out: bytes = b""
+        self.err: bytes = b""
+        self.exitcode: Optional[int] = None
 
-    def outReceived(self, data):
+    def outReceived(self, data: bytes) -> None:
         self.out += data
 
-    def errReceived(self, data):
+    def errReceived(self, data: bytes) -> None:
         self.err += data
 
-    def processEnded(self, status):
-        self.exitcode = status.value.exitCode
+    def processEnded(self, status: Failure) -> None:
+        self.exitcode = cast(ProcessTerminated, status.value).exitCode
         self.deferred.callback(self)

From 66bad1150cf0a72b39d1af8a63d7b4eb7c1f42fe Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 3 Aug 2023 00:08:28 +0400
Subject: [PATCH 06/14] Add more typing for scrapy/utils/signal.py.

---
 scrapy/utils/signal.py | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py
index 9e7ddd827..21a12a19e 100644
--- a/scrapy/utils/signal.py
+++ b/scrapy/utils/signal.py
@@ -23,7 +23,10 @@ logger = logging.getLogger(__name__)
 
 
 def send_catch_log(
-    signal=Any, sender=Anonymous, *arguments, **named
+    signal: TypingAny = Any,
+    sender: TypingAny = Anonymous,
+    *arguments: TypingAny,
+    **named: TypingAny
 ) -> List[Tuple[TypingAny, TypingAny]]:
     """Like pydispatcher.robust.sendRobust but it also logs errors and returns
     Failures instead of exceptions.
@@ -65,13 +68,18 @@ def send_catch_log(
     return responses
 
 
-def send_catch_log_deferred(signal=Any, sender=Anonymous, *arguments, **named):
+def send_catch_log_deferred(
+    signal: TypingAny = Any,
+    sender: TypingAny = Anonymous,
+    *arguments: TypingAny,
+    **named: TypingAny
+) -> Deferred:
     """Like send_catch_log but supports returning deferreds on signal handlers.
     Returns a deferred that gets fired once all signal handlers deferreds were
     fired.
     """
 
-    def logerror(failure, recv):
+    def logerror(failure: Failure, recv: Any) -> Failure:
         if dont_log is None or not isinstance(failure.value, dont_log):
             logger.error(
                 "Error caught on signal handler: %(receiver)s",
@@ -96,7 +104,7 @@ def send_catch_log_deferred(signal=Any, sender=Anonymous, *arguments, **named):
     return d
 
 
-def disconnect_all(signal=Any, sender=Any):
+def disconnect_all(signal: TypingAny = Any, sender: TypingAny = Any) -> None:
     """Disconnect all signal handlers. Useful for cleaning up after running
     tests
     """

From 518e56046e45d38c505db9e2bc00677ad08ac58c Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sun, 6 Aug 2023 17:28:34 +0400
Subject: [PATCH 07/14] Check for async callbacks in contracts.

---
 scrapy/contracts/__init__.py | 13 ++++++++++---
 tests/test_contracts.py      | 15 +++++++++++++++
 2 files changed, 25 insertions(+), 3 deletions(-)

diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py
index 86098edca..1ec2a0234 100644
--- a/scrapy/contracts/__init__.py
+++ b/scrapy/contracts/__init__.py
@@ -2,7 +2,8 @@ import re
 import sys
 from functools import wraps
 from inspect import getmembers
-from typing import Dict
+from types import CoroutineType
+from typing import AsyncGenerator, Dict
 from unittest import TestCase
 
 from scrapy.http import Request
@@ -37,7 +38,10 @@ class Contract:
                 else:
                     results.addSuccess(self.testcase_pre)
                 finally:
-                    return list(iterate_spider_output(cb(response, **cb_kwargs)))
+                    cb_result = cb(response, **cb_kwargs)
+                    if isinstance(cb_result, (AsyncGenerator, CoroutineType)):
+                        raise TypeError("Contracts don't support async callbacks")
+                    return list(iterate_spider_output(cb_result))
 
             request.callback = wrapper
 
@@ -49,7 +53,10 @@ class Contract:
 
             @wraps(cb)
             def wrapper(response, **cb_kwargs):
-                output = list(iterate_spider_output(cb(response, **cb_kwargs)))
+                cb_result = cb(response, **cb_kwargs)
+                if isinstance(cb_result, (AsyncGenerator, CoroutineType)):
+                    raise TypeError("Contracts don't support async callbacks")
+                output = list(iterate_spider_output(cb_result))
                 try:
                     results.startTest(self.testcase_post)
                     self.post_process(output)
diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index 813927fc5..1459e0b5f 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -63,6 +63,13 @@ class TestSpider(Spider):
         """
         return Request("http://scrapy.org", callback=self.returns_item)
 
+    async def returns_request_async(self, response):
+        """async method which returns request
+        @url http://scrapy.org
+        @returns requests 1
+        """
+        return Request("http://scrapy.org", callback=self.returns_item)
+
     def returns_item(self, response):
         """method which returns item
         @url http://scrapy.org
@@ -337,6 +344,14 @@ class ContractsManagerTest(unittest.TestCase):
         request.callback(response)
         self.should_fail()
 
+    def test_returns_async(self):
+        spider = TestSpider()
+        response = ResponseMock()
+
+        request = self.conman.from_method(spider.returns_request_async, self.results)
+        request.callback(response)
+        self.should_error()
+
     def test_scrapes(self):
         spider = TestSpider()
         response = ResponseMock()

From e2adec629b63e9d7735efd58b4353dcfe7ab2863 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sun, 6 Aug 2023 17:31:11 +0400
Subject: [PATCH 08/14] Fix regressions in typing.

---
 scrapy/commands/__init__.py  |  4 +--
 scrapy/commands/fetch.py     |  8 +++--
 scrapy/commands/shell.py     |  8 +++--
 scrapy/http/response/text.py | 12 +++++---
 scrapy/spiderloader.py       |  2 +-
 scrapy/utils/ossignal.py     | 14 +++++++--
 scrapy/utils/spider.py       | 60 ++++++++++++++++++++++++++++++++++--
 7 files changed, 91 insertions(+), 17 deletions(-)

diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
index 9baee3a48..2aa569cdd 100644
--- a/scrapy/commands/__init__.py
+++ b/scrapy/commands/__init__.py
@@ -4,7 +4,7 @@ Base class for Scrapy commands
 import argparse
 import os
 from pathlib import Path
-from typing import Any, Dict, Optional
+from typing import Any, Dict, List, Optional
 
 from twisted.python import failure
 
@@ -116,7 +116,7 @@ class ScrapyCommand:
         if opts.pdb:
             failure.startDebugMode()
 
-    def run(self, args, opts):
+    def run(self, args: List[str], opts: argparse.Namespace) -> None:
         """
         Entry point for running commands
         """
diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py
index 1359e445f..cdb7ad4ae 100644
--- a/scrapy/commands/fetch.py
+++ b/scrapy/commands/fetch.py
@@ -1,7 +1,10 @@
 import sys
+from argparse import Namespace
+from typing import List, Type
 
 from w3lib.url import is_url
 
+from scrapy import Spider
 from scrapy.commands import ScrapyCommand
 from scrapy.exceptions import UsageError
 from scrapy.http import Request
@@ -57,7 +60,7 @@ class Command(ScrapyCommand):
     def _print_bytes(self, bytes_):
         sys.stdout.buffer.write(bytes_ + b"\n")
 
-    def run(self, args, opts):
+    def run(self, args: List[str], opts: Namespace) -> None:
         if len(args) != 1 or not is_url(args[0]):
             raise UsageError()
         request = Request(
@@ -73,7 +76,8 @@ class Command(ScrapyCommand):
         else:
             request.meta["handle_httpstatus_all"] = True
 
-        spidercls = DefaultSpider
+        spidercls: Type[Spider] = DefaultSpider
+        assert self.crawler_process
         spider_loader = self.crawler_process.spider_loader
         if opts.spider:
             spidercls = spider_loader.load(opts.spider)
diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py
index 63c23d04c..0a5e61f7a 100644
--- a/scrapy/commands/shell.py
+++ b/scrapy/commands/shell.py
@@ -3,8 +3,11 @@ Scrapy Shell
 
 See documentation in docs/topics/shell.rst
 """
+from argparse import Namespace
 from threading import Thread
+from typing import List, Type
 
+from scrapy import Spider
 from scrapy.commands import ScrapyCommand
 from scrapy.http import Request
 from scrapy.shell import Shell
@@ -54,15 +57,16 @@ class Command(ScrapyCommand):
         """
         pass
 
-    def run(self, args, opts):
+    def run(self, args: List[str], opts: Namespace) -> None:
         url = args[0] if args else None
         if url:
             # first argument may be a local file
             url = guess_scheme(url)
 
+        assert self.crawler_process
         spider_loader = self.crawler_process.spider_loader
 
-        spidercls = DefaultSpider
+        spidercls: Type[Spider] = DefaultSpider
         if opts.spider:
             spidercls = spider_loader.load(opts.spider)
         elif url:
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index f228e11c1..7fc54b5d3 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -4,10 +4,11 @@ discovering (through HTTP headers) to base Response class.
 
 See documentation in docs/topics/request-response.rst
 """
+from __future__ import annotations
 
 import json
 from contextlib import suppress
-from typing import Generator, Optional, Tuple
+from typing import TYPE_CHECKING, Any, Generator, Optional, Tuple
 from urllib.parse import urljoin
 
 import parsel
@@ -25,6 +26,9 @@ from scrapy.http.response import Response
 from scrapy.utils.python import memoizemethod_noargs, to_unicode
 from scrapy.utils.response import get_base_url
 
+if TYPE_CHECKING:
+    from scrapy.selector import Selector
+
 _NONE = object()
 
 
@@ -34,11 +38,11 @@ class TextResponse(Response):
 
     attributes: Tuple[str, ...] = Response.attributes + ("encoding",)
 
-    def __init__(self, *args, **kwargs):
+    def __init__(self, *args: Any, **kwargs: Any):
         self._encoding = kwargs.pop("encoding", None)
-        self._cached_benc = None
+        self._cached_benc: Optional[str] = None
         self._cached_ubody: Optional[str] = None
-        self._cached_selector = None
+        self._cached_selector: Optional[Selector] = None
         super().__init__(*args, **kwargs)
 
     def _set_url(self, url):
diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py
index 13d6f9f87..f6bb93ddc 100644
--- a/scrapy/spiderloader.py
+++ b/scrapy/spiderloader.py
@@ -27,7 +27,7 @@ class SpiderLoader:
         self._found: DefaultDict[str, List[Tuple[str, str]]] = defaultdict(list)
         self._load_all_spiders()
 
-    def _check_name_duplicates(self):
+    def _check_name_duplicates(self) -> None:
         dupes = []
         for name, locations in self._found.items():
             dupes.extend(
diff --git a/scrapy/utils/ossignal.py b/scrapy/utils/ossignal.py
index f835a2221..2334ea792 100644
--- a/scrapy/utils/ossignal.py
+++ b/scrapy/utils/ossignal.py
@@ -1,7 +1,13 @@
 import signal
-from typing import Callable
+from types import FrameType
+from typing import Any, Callable, Dict, Optional, Union
 
-signal_names = {}
+# copy of _HANDLER from typeshed/stdlib/signal.pyi
+SignalHandlerT = Union[
+    Callable[[int, Optional[FrameType]], Any], int, signal.Handlers, None
+]
+
+signal_names: Dict[int, str] = {}
 for signame in dir(signal):
     if signame.startswith("SIG") and not signame.startswith("SIG_"):
         signum = getattr(signal, signame)
@@ -9,7 +15,9 @@ for signame in dir(signal):
             signal_names[signum] = signame
 
 
-def install_shutdown_handlers(function: Callable, override_sigint: bool = True) -> None:
+def install_shutdown_handlers(
+    function: SignalHandlerT, override_sigint: bool = True
+) -> None:
     """Install the given function as a signal handler for all common shutdown
     signals (such as SIGINT, SIGTERM, etc). If override_sigint is ``False`` the
     SIGINT handler won't be install if there is already a handler in place
diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py
index 3228eda49..704df8657 100644
--- a/scrapy/utils/spider.py
+++ b/scrapy/utils/spider.py
@@ -2,16 +2,19 @@ from __future__ import annotations
 
 import inspect
 import logging
-from types import ModuleType
+from types import CoroutineType, ModuleType
 from typing import (
     TYPE_CHECKING,
     Any,
-    AsyncIterable,
+    AsyncGenerator,
     Generator,
     Iterable,
+    Literal,
     Optional,
     Type,
+    TypeVar,
     Union,
+    overload,
 )
 
 from twisted.internet.defer import Deferred
@@ -26,8 +29,26 @@ if TYPE_CHECKING:
 
 logger = logging.getLogger(__name__)
 
+_T = TypeVar("_T")
 
-def iterate_spider_output(result: Any) -> Union[Iterable, AsyncIterable, Deferred]:
+
+# https://stackoverflow.com/questions/60222982
+@overload
+def iterate_spider_output(result: AsyncGenerator) -> AsyncGenerator:  # type: ignore[misc]
+    ...
+
+
+@overload
+def iterate_spider_output(result: CoroutineType) -> Deferred:
+    ...
+
+
+@overload
+def iterate_spider_output(result: _T) -> Iterable:
+    ...
+
+
+def iterate_spider_output(result: Any) -> Union[Iterable, AsyncGenerator, Deferred]:
     if inspect.isasyncgen(result):
         return result
     if inspect.iscoroutine(result):
@@ -55,6 +76,39 @@ def iter_spider_classes(module: ModuleType) -> Generator[Type[Spider], Any, None
             yield obj
 
 
+@overload
+def spidercls_for_request(
+    spider_loader: SpiderLoader,
+    request: Request,
+    default_spidercls: Type[Spider],
+    log_none: bool = ...,
+    log_multiple: bool = ...,
+) -> Type[Spider]:
+    ...
+
+
+@overload
+def spidercls_for_request(
+    spider_loader: SpiderLoader,
+    request: Request,
+    default_spidercls: Literal[None],
+    log_none: bool = ...,
+    log_multiple: bool = ...,
+) -> Optional[Type[Spider]]:
+    ...
+
+
+@overload
+def spidercls_for_request(
+    spider_loader: SpiderLoader,
+    request: Request,
+    *,
+    log_none: bool = ...,
+    log_multiple: bool = ...,
+) -> Optional[Type[Spider]]:
+    ...
+
+
 def spidercls_for_request(
     spider_loader: SpiderLoader,
     request: Request,

From f5f593e5f5e2b0c216e9d6fd41f4260f70c74d34 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sun, 6 Aug 2023 17:46:28 +0400
Subject: [PATCH 09/14] Remove a workaround for a w3lib typing bug.

---
 scrapy/utils/response.py | 3 +--
 tox.ini                  | 2 ++
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py
index 794678c48..c540d6278 100644
--- a/scrapy/utils/response.py
+++ b/scrapy/utils/response.py
@@ -42,8 +42,7 @@ def get_meta_refresh(
     """Parse the http-equiv refresh parameter from the given response"""
     if response not in _metaref_cache:
         text = response.text[0:4096]
-        # a w3lib typing bug here, fixed in https://github.com/scrapy/w3lib/pull/211
-        _metaref_cache[response] = html.get_meta_refresh(  # type: ignore[assignment]
+        _metaref_cache[response] = html.get_meta_refresh(
             text, response.url, response.encoding, ignore_tags=ignore_tags
         )
     return _metaref_cache[response]
diff --git a/tox.ini b/tox.ini
index ef7dd5854..8b2d207c7 100644
--- a/tox.ini
+++ b/tox.ini
@@ -41,6 +41,8 @@ deps =
     types-Pygments==2.15.0.1
     types-pyOpenSSL==23.2.0.1
     types-setuptools==68.0.0.1
+    # 2.1.2 fixes a typing bug: https://github.com/scrapy/w3lib/pull/211
+    w3lib >= 2.1.2
 commands =
     mypy {posargs: scrapy tests}
 

From 471281d29e5c7b8e293f59ac3c3330ecda687d19 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sun, 6 Aug 2023 23:05:02 +0400
Subject: [PATCH 10/14] Fixes for scrapy/utils/iterators.py typing.

---
 scrapy/utils/iterators.py     | 11 ++++++-----
 tests/test_utils_iterators.py |  7 +------
 2 files changed, 7 insertions(+), 11 deletions(-)

diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py
index 58850b843..40af68dec 100644
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@ -1,5 +1,3 @@
-from __future__ import annotations
-
 import csv
 import logging
 import re
@@ -78,7 +76,7 @@ def xmliter(
 
 
 def xmliter_lxml(
-    obj: Union[TextResponse, str, bytes],
+    obj: Union[Response, str, bytes],
     nodename: str,
     namespace: Optional[str] = None,
     prefix: str = "x",
@@ -87,8 +85,9 @@ def xmliter_lxml(
 
     reader = _StreamReader(obj)
     tag = f"{{{namespace}}}{nodename}" if namespace else nodename
+    # technically, etree.iterparse only needs .read() AFAICS, but this is how it's typed
     iterable = etree.iterparse(
-        cast(SupportsReadClose[bytes], reader), tag=tag, encoding=reader.encoding
+        cast("SupportsReadClose[bytes]", reader), tag=tag, encoding=reader.encoding
     )
     selxpath = "//" + (f"{prefix}:{nodename}" if namespace else nodename)
     for _, node in iterable:
@@ -101,11 +100,13 @@ def xmliter_lxml(
 
 
 class _StreamReader:
-    def __init__(self, obj: Union[TextResponse, str, bytes]):
+    def __init__(self, obj: Union[Response, str, bytes]):
         self._ptr: int = 0
         self._text: Union[str, bytes]
         if isinstance(obj, TextResponse):
             self._text, self.encoding = obj.body, obj.encoding
+        elif isinstance(obj, Response):
+            self._text, self.encoding = obj.body, "utf-8"
         else:
             self._text, self.encoding = obj, "utf-8"
         self._is_unicode: bool = isinstance(self._text, str)
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index 5dfd7e7ac..3598fa0bb 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -1,18 +1,13 @@
-from typing import Callable, Iterable, Union
-
 from pytest import mark
 from twisted.trial import unittest
 
-from scrapy import Selector
 from scrapy.http import Response, TextResponse, XmlResponse
 from scrapy.utils.iterators import _body_or_str, csviter, xmliter, xmliter_lxml
 from tests import get_testdata
 
 
 class XmliterTestCase(unittest.TestCase):
-    xmliter: Callable[
-        [Union[TextResponse, str, bytes], str], Iterable[Selector]
-    ] = staticmethod(xmliter)
+    xmliter = staticmethod(xmliter)
 
     def test_xmliter(self):
         body = b"""

From 644a71bfd40be3ce8a465ad49891d34c47516f56 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 7 Aug 2023 00:17:52 +0400
Subject: [PATCH 11/14] Use ftp:// URLs in FTP tests.

---
 tests/test_feedexport.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 46bd5733a..42fa25b1d 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -2978,8 +2978,8 @@ class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase):
 
     def test_init(self):
         settings_dict = {
-            "FEED_URI": "file:///tmp/foobar",
-            "FEED_STORAGES": {"file": FTPFeedStorageWithoutFeedOptions},
+            "FEED_URI": "ftp://localhost/foo",
+            "FEED_STORAGES": {"ftp": FTPFeedStorageWithoutFeedOptions},
         }
         with pytest.warns(
             ScrapyDeprecationWarning,
@@ -3000,8 +3000,8 @@ class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase):
 
     def test_from_crawler(self):
         settings_dict = {
-            "FEED_URI": "file:///tmp/foobar",
-            "FEED_STORAGES": {"file": FTPFeedStorageWithoutFeedOptionsWithFromCrawler},
+            "FEED_URI": "ftp://localhost/foo",
+            "FEED_STORAGES": {"ftp": FTPFeedStorageWithoutFeedOptionsWithFromCrawler},
         }
         with pytest.warns(
             ScrapyDeprecationWarning,

From 23af21491d526072873ab7ec3d1429560861964f Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 7 Aug 2023 00:21:06 +0400
Subject: [PATCH 12/14] Move definitions around to woark around a pypy3.8 bug.

---
 scrapy/utils/log.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index 2013bfc43..0d17f6153 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -122,6 +122,9 @@ def configure_logging(
         install_scrapy_root_handler(settings)
 
 
+_scrapy_root_handler: Optional[logging.Handler] = None
+
+
 def install_scrapy_root_handler(settings: Settings) -> None:
     global _scrapy_root_handler
 
@@ -139,9 +142,6 @@ def get_scrapy_root_handler() -> Optional[logging.Handler]:
     return _scrapy_root_handler
 
 
-_scrapy_root_handler: Optional[logging.Handler] = None
-
-
 def _get_handler(settings: Settings) -> logging.Handler:
     """Return a log handler object according to settings"""
     filename = settings.get("LOG_FILE")

From 53539483c32dec537064628c0c6e407eb8ce1c2f Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 7 Aug 2023 15:13:20 +0400
Subject: [PATCH 13/14] Refactor _StreamReader.read().

---
 scrapy/utils/iterators.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py
index 40af68dec..baf92681a 100644
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@ -110,12 +110,17 @@ class _StreamReader:
         else:
             self._text, self.encoding = obj, "utf-8"
         self._is_unicode: bool = isinstance(self._text, str)
+        self._is_first_read: bool = True
 
     def read(self, n: int = 65535) -> bytes:
-        self.read: Callable[[int], bytes] = (  # type: ignore[method-assign]
+        method: Callable[[int], bytes] = (
             self._read_unicode if self._is_unicode else self._read_string
         )
-        return self.read(n).lstrip()
+        result = method(n)
+        if self._is_first_read:
+            self._is_first_read = False
+            result = result.lstrip()
+        return result
 
     def _read_string(self, n: int = 65535) -> bytes:
         s, e = self._ptr, self._ptr + n

From 8050257c1495fe405767dcd37c4db0f6f29c38aa Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 9 Aug 2023 23:17:32 +0400
Subject: [PATCH 14/14] Small cleanup.

---
 scrapy/extensions/feedexport.py | 3 ++-
 scrapy/utils/iterators.py       | 1 -
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 2bbcaf3ad..c1b77f4fb 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -290,7 +290,8 @@ class FTPFeedStorage(BlockingFeedStorage):
         feed_options: Optional[Dict[str, Any]] = None,
     ):
         u = urlparse(uri)
-        assert u.hostname
+        if not u.hostname:
+            raise ValueError(f"Got a storage URI without a hostname: {uri}")
         self.host: str = u.hostname
         self.port: int = int(u.port or "21")
         self.username: str = u.username or ""
diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py
index baf92681a..03d779afb 100644
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@ -85,7 +85,6 @@ def xmliter_lxml(
 
     reader = _StreamReader(obj)
     tag = f"{{{namespace}}}{nodename}" if namespace else nodename
-    # technically, etree.iterparse only needs .read() AFAICS, but this is how it's typed
     iterable = etree.iterparse(
         cast("SupportsReadClose[bytes]", reader), tag=tag, encoding=reader.encoding
     )