Merge remote-tracking branch 'origin/master' into async-s3

docs/topics/downloader-middleware.rst

@@ -564,6 +564,10 @@ defines the methods described below.
 
       Return response if present in cache, or ``None`` otherwise.
 
+      If this method raises an exception, e.g. because the cache entry is
+      corrupted, the middleware logs a warning and handles the request as a
+      cache miss.
+
       :param spider: the spider which generated the request
       :type spider: :class:`~scrapy.Spider` object
 

docs/topics/practices.rst

@@ -533,8 +533,7 @@ Here are some tips to keep in mind when dealing with these kinds of sites:
 * if possible, use `Common Crawl`_ to fetch pages, instead of hitting the sites
   directly
 * use a pool of rotating IPs. For example, the free `Tor project`_ or paid
-  services like `ProxyMesh`_. An open source alternative is `scrapoxy`_, a
-  super proxy that you can attach your own proxies to.
+  services like `ProxyMesh`_.
 * for HTTPS websites, if blocking appears related to TLS behavior, consider
   adjusting the :setting:`DOWNLOAD_TLS_MIN_VERSION` and
   :setting:`DOWNLOAD_TLS_MAX_VERSION` settings, since some websites may respond
@@ -559,5 +558,4 @@ projects that detects common mistakes and anti-patterns.
 .. _ProxyMesh: https://proxymesh.com/
 .. _Common Crawl: https://commoncrawl.org/
 .. _testspiders: https://github.com/scrapinghub/testspiders
-.. _scrapoxy: https://scrapoxy.io/
 .. _Zyte API: https://docs.zyte.com/zyte-api/get-started.html

docs/topics/request-response.rst

@@ -260,6 +260,8 @@ Request objects
 
     .. automethod:: from_curl
 
+    .. automethod:: to_curl
+
     .. automethod:: to_dict
 
 

pyproject.toml

@@ -178,13 +178,6 @@ module = [
     "tests.test_squeues",
     "tests.test_squeues_request",
     "tests.test_stats",
-    "tests.test_utils_datatypes",
-    "tests.test_utils_decorators",
-    "tests.test_utils_defer",
-    "tests.test_utils_deprecate",
-    "tests.test_utils_misc.test_return_with_argument_inside_generator",
-    "tests.test_utils_python",
-    "tests.test_utils_request",
     "tests.utils.bases.http_request",
     "tests.utils.bases.http_response",
     "tests.utils.bases.spider",

scrapy/cmdline.py

@@ -225,13 +225,11 @@ def _run_command(cmd: ScrapyCommand, args: list[str], opts: argparse.Namespace)
 def _run_command_profiled(
     cmd: ScrapyCommand, args: list[str], opts: argparse.Namespace
 ) -> None:
-    if opts.profile:
-        sys.stderr.write(f"scrapy: writing cProfile stats to {opts.profile!r}\n")
+    sys.stderr.write(f"scrapy: writing cProfile stats to {opts.profile!r}\n")
     loc = locals()
     p = cProfile.Profile()
     p.runctx("cmd.run(args, opts)", globals(), loc)
-    if opts.profile:
-        p.dump_stats(opts.profile)
+    p.dump_stats(opts.profile)
 
 
 if __name__ == "__main__":

scrapy/commands/parse.py

@@ -346,6 +346,8 @@ class Command(BaseRunSpiderCommand):
                 self.first_response = response
 
             cb = self._get_callback(spider=spider, opts=opts, response=response)
+            assert response.request
+            response.request.callback = cb
 
             # parse items and requests
             depth: int = response.meta["_depth"]

scrapy/core/downloader/handlers/ftp.py

@@ -126,5 +126,4 @@ class FTPDownloadHandler(BaseDownloadHandler):
         headers = {"local filename": protocol.filename or b"", "size": protocol.size}
         body = protocol.filename or protocol.body.read()
         respcls = responsetypes.from_args(url=request.url, body=body)
-        # hints for Headers-related types may need to be fixed to not use AnyStr
-        return respcls(url=request.url, status=200, body=body, headers=headers)  # type: ignore[arg-type]
+        return respcls(url=request.url, status=200, body=body, headers=headers)

scrapy/downloadermiddlewares/httpcache.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import logging
 from email.utils import formatdate
 from typing import TYPE_CHECKING
 
@@ -28,6 +29,9 @@ if TYPE_CHECKING:
     from scrapy.statscollectors import StatsCollector
 
 
+logger = logging.getLogger(__name__)
+
+
 class HttpCacheMiddleware:
     DOWNLOAD_EXCEPTIONS = (
         ConnectionDone,
@@ -77,9 +81,20 @@ class HttpCacheMiddleware:
             return None
 
         # Look for cached response and check if expired
-        cachedresponse: Response | None = self.storage.retrieve_response(
-            self.crawler.spider, request
-        )
+        cachedresponse: Response | None
+        try:
+            cachedresponse = self.storage.retrieve_response(
+                self.crawler.spider, request
+            )
+        except Exception:
+            self.stats.inc_value("httpcache/retrieve_error")
+            logger.warning(
+                f"Could not read the cache entry for {request}, treating it as a "
+                f"cache miss.",
+                exc_info=True,
+                extra={"spider": self.crawler.spider},
+            )
+            cachedresponse = None
         if cachedresponse is None:
             self.stats.inc_value("httpcache/miss")
             if self.ignore_missing:

scrapy/extensions/feedexport.py

@@ -13,7 +13,7 @@ import re
 import sys
 import warnings
 from abc import ABC, abstractmethod
-from collections.abc import Callable, Coroutine
+from collections.abc import Callable
 from datetime import datetime, timezone
 from pathlib import Path, PureWindowsPath
 from tempfile import NamedTemporaryFile
@@ -501,7 +501,7 @@ class FeedExporter:
         self.feeds = {}
         self.slots: list[FeedSlot] = []
         self.filters: dict[str, ItemFilter] = {}
-        self._pending_close_coros: list[Coroutine[Any, Any, None]] = []
+        self._pending_close_tasks: list[asyncio.Task[None] | Deferred[None]] = []
 
         if not self.settings["FEEDS"] and not self.settings["FEED_URI"]:
             raise NotConfigured
@@ -565,23 +565,44 @@ class FeedExporter:
             )
 
     async def close_spider(self, spider: Spider) -> None:
-        self._pending_close_coros.extend(
-            self._close_slot(slot, spider) for slot in self.slots
-        )
+        for slot in self.slots:
+            self._schedule_slot_close(slot, spider)
 
-        if self._pending_close_coros:
+        if self._pending_close_tasks:
             if is_asyncio_available():
                 await asyncio.wait(
-                    [asyncio.create_task(coro) for coro in self._pending_close_coros]
+                    cast("list[asyncio.Task[None]]", list(self._pending_close_tasks))
                 )
             else:
                 await DeferredList(
-                    deferred_from_coro(coro) for coro in self._pending_close_coros
+                    cast("list[Deferred[None]]", list(self._pending_close_tasks))
                 )
 
         # Send FEED_EXPORTER_CLOSED signal
         await self.crawler.signals.send_catch_log_async(signals.feed_exporter_closed)
 
+    def _schedule_slot_close(
+        self, slot: FeedSlot, spider: Spider
+    ) -> asyncio.Task[None] | Deferred[None]:
+        """Start closing the slot without waiting for it to finish, keeping
+        track of the pending work so that it can be awaited in
+        :meth:`close_spider` if it hasn't finished by then."""
+        aw: asyncio.Task[None] | Deferred[None]
+        coro = self._close_slot(slot, spider)
+        if is_asyncio_available():
+            aw = asyncio.create_task(coro)
+            self._pending_close_tasks.append(aw)
+            aw.add_done_callback(self._pending_close_tasks.remove)
+        else:
+            aw = deferred_from_coro(coro)
+            self._pending_close_tasks.append(aw)
+            aw.addBoth(self._untrack_pending_close_task, aw)
+        return aw
+
+    def _untrack_pending_close_task(self, result: Any, aw: Deferred[None]) -> Any:
+        self._pending_close_tasks.remove(aw)
+        return result
+
     @staticmethod
     def _get_file(slot_: FeedSlot) -> IO[bytes]:
         assert slot_.file
@@ -678,7 +699,7 @@ class FeedExporter:
                 uri_params = self._get_uri_params(
                     spider, self.feeds[slot.uri_template]["uri_params"], slot
                 )
-                self._pending_close_coros.append(self._close_slot(slot, spider))
+                self._schedule_slot_close(slot, spider)
                 slots.append(
                     self._start_new_batch(
                         batch_id=slot.batch_id + 1,

scrapy/http/headers.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from collections.abc import Mapping
-from typing import TYPE_CHECKING, Any, AnyStr, TypeAlias, cast
+from typing import TYPE_CHECKING, Any, TypeAlias, cast
 
 from w3lib.http import headers_dict_to_raw
 
@@ -25,14 +25,20 @@ class Headers(CaselessDict):
 
     def __init__(
         self,
-        seq: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,
+        seq: Mapping[str, Any]
+        | Mapping[bytes, Any]
+        | Iterable[tuple[str | bytes, Any]]
+        | None = None,
         encoding: str = "utf-8",
     ):
         self.encoding: str = encoding
         super().__init__(seq)
 
     def update(  # type: ignore[override]
-        self, seq: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]]
+        self,
+        seq: Mapping[str, Any]
+        | Mapping[bytes, Any]
+        | Iterable[tuple[str | bytes, Any]],
     ) -> None:
         seq = seq.items() if isinstance(seq, Mapping) else seq
         iseq: dict[bytes, list[bytes]] = {}
@@ -40,7 +46,7 @@ class Headers(CaselessDict):
             iseq.setdefault(self.normkey(k), []).extend(self.normvalue(v))
         super().update(iseq)
 
-    def normkey(self, key: AnyStr) -> bytes:  # type: ignore[override]
+    def normkey(self, key: str | bytes) -> bytes:
         """Normalize key to bytes"""
         return self._tobytes(key.title())
 
@@ -67,19 +73,19 @@ class Headers(CaselessDict):
             return str(x).encode(self.encoding)
         raise TypeError(f"Unsupported value type: {type(x)}")
 
-    def __getitem__(self, key: AnyStr) -> bytes | None:
+    def __getitem__(self, key: str | bytes) -> bytes | None:
         try:
             return cast("list[bytes]", super().__getitem__(key))[-1]
         except IndexError:
             return None
 
-    def get(self, key: AnyStr, def_val: Any = None) -> bytes | None:
+    def get(self, key: str | bytes, def_val: Any = None) -> bytes | None:
         try:
             return cast("list[bytes]", super().get(key, def_val))[-1]
         except IndexError:
             return None
 
-    def getlist(self, key: AnyStr, def_val: Any = None) -> list[bytes]:
+    def getlist(self, key: str | bytes, def_val: Any = None) -> list[bytes]:
         try:
             return cast("list[bytes]", super().__getitem__(key))
         except KeyError:
@@ -87,15 +93,15 @@ class Headers(CaselessDict):
                 return self.normvalue(def_val)
             return []
 
-    def setlist(self, key: AnyStr, list_: Iterable[_RawValue]) -> None:
+    def setlist(self, key: str | bytes, list_: Iterable[_RawValue]) -> None:
         self[key] = list_
 
     def setlistdefault(
-        self, key: AnyStr, default_list: Iterable[_RawValue] = ()
+        self, key: str | bytes, default_list: Iterable[_RawValue] = ()
     ) -> Any:
         return self.setdefault(key, default_list)
 
-    def appendlist(self, key: AnyStr, value: Iterable[_RawValue]) -> None:
+    def appendlist(self, key: str | bytes, value: Iterable[_RawValue]) -> None:
         lst = self.getlist(key)
         lst.extend(self.normvalue(value))
         self[key] = lst

scrapy/http/request/__init__.py

@@ -11,7 +11,6 @@ import inspect
 from typing import (
     TYPE_CHECKING,
     Any,
-    AnyStr,
     Concatenate,
     NoReturn,
     TypeAlias,
@@ -125,7 +124,10 @@ class Request(object_ref):
         url: str,
         callback: CallbackT | None = None,
         method: str = "GET",
-        headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,
+        headers: Mapping[str, Any]
+        | Mapping[bytes, Any]
+        | Iterable[tuple[str | bytes, Any]]
+        | None = None,
         body: bytes | str | None = None,
         cookies: CookiesT | None = None,
         meta: dict[str, Any] | None = None,
@@ -310,7 +312,11 @@ class Request(object_ref):
 
     @headers.setter
     def headers(
-        self, value: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None
+        self,
+        value: Mapping[str, Any]
+        | Mapping[bytes, Any]
+        | Iterable[tuple[str | bytes, Any]]
+        | None,
     ) -> None:
         if isinstance(value, Headers):
             self._headers = value
@@ -381,6 +387,20 @@ class Request(object_ref):
         request_kwargs.update(kwargs)
         return cls(**request_kwargs)
 
+    def to_curl(self) -> str:
+        """Return a string with a `cURL <https://curl.se/>`_ command equivalent
+        to this request.
+
+        Inverse of :meth:`from_curl`. See also
+        :func:`scrapy.utils.request.request_to_curl`.
+
+        .. versionadded:: VERSION
+        """
+        # Imported here to avoid a circular import.
+        from scrapy.utils.request import request_to_curl  # noqa: PLC0415
+
+        return request_to_curl(self)
+
     def to_dict(self, *, spider: scrapy.Spider | None = None) -> dict[str, Any]:
         """Return a dictionary containing the Request's data.
 

scrapy/http/response/__init__.py

@@ -7,7 +7,7 @@ See documentation in docs/topics/request-response.rst
 
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Any, AnyStr, TypeVar, overload
+from typing import TYPE_CHECKING, Any, TypeVar, overload
 from urllib.parse import urljoin
 
 from scrapy.exceptions import NotSupported
@@ -72,7 +72,10 @@ class Response(object_ref):
         self,
         url: str,
         status: int = 200,
-        headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,
+        headers: Mapping[str, Any]
+        | Mapping[bytes, Any]
+        | Iterable[tuple[str | bytes, Any]]
+        | None = None,
         body: bytes = b"",
         flags: list[str] | None = None,
         request: Request | None = None,
@@ -145,7 +148,11 @@ class Response(object_ref):
 
     @headers.setter
     def headers(
-        self, value: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None
+        self,
+        value: Mapping[str, Any]
+        | Mapping[bytes, Any]
+        | Iterable[tuple[str | bytes, Any]]
+        | None,
     ) -> None:
         if isinstance(value, Headers):
             self._headers = value
@@ -222,7 +229,10 @@ class Response(object_ref):
         url: str | Link,
         callback: CallbackT | None = None,
         method: str = "GET",
-        headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,
+        headers: Mapping[str, Any]
+        | Mapping[bytes, Any]
+        | Iterable[tuple[str | bytes, Any]]
+        | None = None,
         body: bytes | str | None = None,
         cookies: CookiesT | None = None,
         meta: dict[str, Any] | None = None,
@@ -272,7 +282,10 @@ class Response(object_ref):
         urls: Iterable[str | Link],
         callback: CallbackT | None = None,
         method: str = "GET",
-        headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,
+        headers: Mapping[str, Any]
+        | Mapping[bytes, Any]
+        | Iterable[tuple[str | bytes, Any]]
+        | None = None,
         body: bytes | str | None = None,
         cookies: CookiesT | None = None,
         meta: dict[str, Any] | None = None,

scrapy/http/response/text.py

@@ -9,7 +9,7 @@ from __future__ import annotations
 
 import json
 from contextlib import suppress
-from typing import TYPE_CHECKING, Any, AnyStr, cast
+from typing import TYPE_CHECKING, Any, cast
 from urllib.parse import urljoin
 
 import parsel
@@ -170,7 +170,10 @@ class TextResponse(Response):
         url: str | Link | parsel.Selector,
         callback: CallbackT | None = None,
         method: str = "GET",
-        headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,
+        headers: Mapping[str, Any]
+        | Mapping[bytes, Any]
+        | Iterable[tuple[str | bytes, Any]]
+        | None = None,
         body: bytes | str | None = None,
         cookies: CookiesT | None = None,
         meta: dict[str, Any] | None = None,
@@ -223,7 +226,10 @@ class TextResponse(Response):
         urls: Iterable[str | Link] | parsel.SelectorList[Any] | None = None,
         callback: CallbackT | None = None,
         method: str = "GET",
-        headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,
+        headers: Mapping[str, Any]
+        | Mapping[bytes, Any]
+        | Iterable[tuple[str | bytes, Any]]
+        | None = None,
         body: bytes | str | None = None,
         cookies: CookiesT | None = None,
         meta: dict[str, Any] | None = None,

scrapy/mail.py

@@ -2,6 +2,8 @@
 Mail sending helpers
 """
 
+# pragma: no file cover
+
 from __future__ import annotations
 
 import logging

scrapy/spiders/feed.py

@@ -9,7 +9,7 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING, Any
 
-from scrapy.exceptions import NotConfigured, NotSupported
+from scrapy.exceptions import NotSupported
 from scrapy.http import Response, TextResponse
 from scrapy.selector import Selector
 from scrapy.spiders import Spider
@@ -76,11 +76,6 @@ class XMLFeedSpider(Spider):
             yield from self.process_results(response, ret)
 
     def _parse(self, response: Response, **kwargs: Any) -> Any:
-        if not hasattr(self, "parse_node"):
-            raise NotConfigured(
-                "You must define parse_node method in order to scrape this XML feed"
-            )
-
         response = self.adapt_response(response)
         nodes: Iterable[Selector]
         if self.iterator == "iternodes":
@@ -158,9 +153,5 @@ class CSVFeedSpider(Spider):
             yield from self.process_results(response, ret)
 
     def _parse(self, response: Response, **kwargs: Any) -> Any:
-        if not hasattr(self, "parse_row"):
-            raise NotConfigured(
-                "You must define parse_row method in order to scrape this CSV feed"
-            )
         response = self.adapt_response(response)
         return self.parse_rows(response)

scrapy/utils/datatypes.py

@@ -11,12 +11,12 @@ import warnings
 import weakref
 from collections import OrderedDict
 from collections.abc import Mapping
-from typing import TYPE_CHECKING, Any, AnyStr, TypeVar, cast
+from typing import TYPE_CHECKING, Any, TypeVar, cast
 
 from scrapy.exceptions import ScrapyDeprecationWarning
 
 if TYPE_CHECKING:
-    from collections.abc import Iterable, Sequence
+    from collections.abc import Container, Iterable
 
     # typing.Self requires Python 3.11
     from typing_extensions import Self
@@ -44,22 +44,25 @@ class CaselessDict(dict):  # type: ignore[type-arg]
 
     def __init__(
         self,
-        seq: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,
+        seq: Mapping[str, Any]
+        | Mapping[bytes, Any]
+        | Iterable[tuple[str | bytes, Any]]
+        | None = None,
     ):
         super().__init__()
         if seq:
             self.update(seq)
 
-    def __getitem__(self, key: AnyStr) -> Any:
+    def __getitem__(self, key: str | bytes) -> Any:
         return dict.__getitem__(self, self.normkey(key))
 
-    def __setitem__(self, key: AnyStr, value: Any) -> None:
+    def __setitem__(self, key: str | bytes, value: Any) -> None:
         dict.__setitem__(self, self.normkey(key), self.normvalue(value))
 
-    def __delitem__(self, key: AnyStr) -> None:
+    def __delitem__(self, key: str | bytes) -> None:
         dict.__delitem__(self, self.normkey(key))
 
-    def __contains__(self, key: AnyStr) -> bool:  # type: ignore[override]
+    def __contains__(self, key: str | bytes) -> bool:  # type: ignore[override]
         return dict.__contains__(self, self.normkey(key))
 
     has_key = __contains__
@@ -69,7 +72,7 @@ class CaselessDict(dict):  # type: ignore[type-arg]
 
     copy = __copy__
 
-    def normkey(self, key: AnyStr) -> AnyStr:
+    def normkey(self, key: str | bytes) -> str | bytes:
         """Method to normalize dictionary key access"""
         return key.lower()
 
@@ -77,23 +80,28 @@ class CaselessDict(dict):  # type: ignore[type-arg]
         """Method to normalize values prior to be set"""
         return value
 
-    def get(self, key: AnyStr, def_val: Any = None) -> Any:
+    def get(self, key: str | bytes, def_val: Any = None) -> Any:
         return dict.get(self, self.normkey(key), self.normvalue(def_val))
 
-    def setdefault(self, key: AnyStr, def_val: Any = None) -> Any:
+    def setdefault(self, key: str | bytes, def_val: Any = None) -> Any:
         return dict.setdefault(self, self.normkey(key), self.normvalue(def_val))
 
     # doesn't fully implement MutableMapping.update()
-    def update(self, seq: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]]) -> None:  # type: ignore[override]
+    def update(  # type: ignore[override]
+        self,
+        seq: Mapping[str, Any]
+        | Mapping[bytes, Any]
+        | Iterable[tuple[str | bytes, Any]],
+    ) -> None:
         seq = seq.items() if isinstance(seq, Mapping) else seq
         iseq = ((self.normkey(k), self.normvalue(v)) for k, v in seq)
         super().update(iseq)
 
     @classmethod
-    def fromkeys(cls, keys: Iterable[AnyStr], value: Any = None) -> Self:  # type: ignore[override]
-        return cls((k, value) for k in keys)  # type: ignore[misc]
+    def fromkeys(cls, keys: Iterable[str | bytes], value: Any = None) -> Self:  # type: ignore[override]
+        return cls((k, value) for k in keys)
 
-    def pop(self, key: AnyStr, *args: Any) -> Any:
+    def pop(self, key: str | bytes, *args: Any) -> Any:
         return dict.pop(self, self.normkey(key), *args)
 
 
@@ -132,6 +140,22 @@ class CaseInsensitiveDict(collections.UserDict[str | bytes, Any]):
     def __repr__(self) -> str:
         return f"<{self.__class__.__name__}: {super().__repr__()}>"
 
+    # UserDict.copy() shallow-copies the instance, which would share self._keys
+    # between the copy and the original.
+    def __copy__(self) -> Self:
+        new = self.__class__()
+        new.data = self.data.copy()
+        new._keys = self._keys.copy()
+        return new
+
+    copy = __copy__
+
+    # UserDict.__ior__ updates self.data directly, which would leave self._keys
+    # out of date.
+    def __ior__(self, other: Any) -> Self:  # type: ignore[override,misc]
+        self.update(other)
+        return self
+
     def _normkey(self, key: str | bytes) -> str | bytes:
         return key
 
@@ -189,8 +213,8 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary[_KT, _VT | None]):
 class SequenceExclude:
     """Object to test if an item is NOT within some sequence."""
 
-    def __init__(self, seq: Sequence[Any]):
-        self.seq: Sequence[Any] = seq
+    def __init__(self, seq: Container[Any]):
+        self.seq: Container[Any] = seq
 
     def __contains__(self, item: Any) -> bool:
         return item not in self.seq

scrapy/utils/decorators.py

@@ -19,9 +19,19 @@ _T = TypeVar("_T")
 _P = ParamSpec("_P")
 
 
+@overload
+def deprecated(use_instead: Callable[_P, _T]) -> Callable[_P, _T]: ...
+
+
+@overload
 def deprecated(
-    use_instead: Any = None,
-) -> Callable[[Callable[_P, _T]], Callable[_P, _T]]:
+    use_instead: str | None = None,
+) -> Callable[[Callable[_P, _T]], Callable[_P, _T]]: ...
+
+
+def deprecated(
+    use_instead: Callable[_P, _T] | str | None = None,
+) -> Callable[_P, _T] | Callable[[Callable[_P, _T]], Callable[_P, _T]]:
     """This is a decorator which can be used to mark functions
     as deprecated. It will result in a warning being emitted
     when the function is used."""
@@ -38,8 +48,9 @@ def deprecated(
         return wrapped
 
     if callable(use_instead):
-        deco = deco(use_instead)
+        func = use_instead
         use_instead = None
+        return deco(func)
     return deco
 
 

tests/spiders.py

@@ -38,6 +38,35 @@ class MockServerSpider(Spider):
         self.is_secure = is_secure
 
 
+class RawResponseSpider(MockServerSpider):
+    """Base class for spiders that fetch a response built by the test itself.
+
+    Subclasses return the body from :meth:`raw_body` and request
+    :attr:`raw_url`, which the mock server answers with that body verbatim
+    under :attr:`content_type`. This lets tests reach parsing code that only
+    a specific kind of response triggers while still going through a regular
+    crawl, instead of calling internal parsing methods directly.
+    """
+
+    name = "raw_response"
+    content_type = "text/plain"
+
+    def raw_body(self) -> str:
+        raise NotImplementedError
+
+    @property
+    def raw_url(self) -> str:
+        assert self.mockserver
+        raw = (
+            "HTTP/1.1 200 OK\r\n"
+            f"Content-Type: {self.content_type}\r\n"
+            "Connection: close\r\n"
+            "\r\n"
+            f"{self.raw_body()}"
+        )
+        return self.mockserver.url("/raw?" + urlencode({"raw": raw}))
+
+
 class MetaSpider(MockServerSpider):
     name = "meta"
 
@@ -496,6 +525,23 @@ class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod):
         self.logger.info("[errback] status %i", failure.value.response.status)
 
 
+class CrawlSpiderWithoutErrback(CrawlSpiderWithParseMethod):
+    name = "crawl_spider_without_errback"
+
+    async def start(self):
+        test_body = b"""
+        <html>
+            <head><title>Page title</title></head>
+            <body>
+                <p><a href="/status?n=200">Item 200</a></p>  <!-- callback -->
+                <p><a href="/status?n=404">Item 404</a></p>  <!-- failure, no errback -->
+            </body>
+        </html>
+        """
+        url = self.mockserver.url("/alpayload")
+        yield Request(url, method="POST", body=test_body)
+
+
 class CrawlSpiderWithProcessRequestCallbackKeywordArguments(CrawlSpiderWithParseMethod):
     name = "crawl_spider_with_process_request_cb_kwargs"
     rules = (

tests/test_command_parse.py

@@ -126,6 +126,32 @@ class MySpider(scrapy.Spider):
         else:
             self.logger.debug('It Does Not Work :(')
 
+class RetryRequestSpider(BaseSpider):
+    name = 'retry_request'
+
+    def parse(self, response):
+        if response.meta.get('retried'):
+            yield {{'retried': True}}
+            return
+        response.meta['retried'] = True
+        yield response.request.replace(dont_filter=True)
+
+class CustomCallbackRetryRequestSpider(BaseSpider):
+    name = 'retry_request_custom_callback'
+
+    def parse(self, response):
+        yield response.request.replace(
+            callback=self.parse_retry,
+            dont_filter=True,
+        )
+
+    def parse_retry(self, response):
+        if response.meta.get('retried'):
+            yield {{'retried_with_custom_callback': True}}
+            return
+        response.meta['retried'] = True
+        yield response.request.replace(dont_filter=True)
+
 class MyGoodCrawlSpider(CrawlSpider):
     name = 'goodcrawl{self.spider_name}'
 
@@ -381,6 +407,36 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
         )
         assert "[{}, {'foo': 'bar'}]" in out
 
+    def test_retry_response_request(
+        self, proj_path: Path, mockserver: MockServer
+    ) -> None:
+        _, out, stderr = proc(
+            "parse",
+            "--spider",
+            "retry_request",
+            "-d",
+            "2",
+            mockserver.url("/html"),
+            cwd=proj_path,
+        )
+        assert "RecursionError" not in stderr
+        assert "{'retried': True}" in out
+
+    def test_retry_response_request_with_custom_callback(
+        self, proj_path: Path, mockserver: MockServer
+    ) -> None:
+        _, out, stderr = proc(
+            "parse",
+            "--spider",
+            "retry_request_custom_callback",
+            "-d",
+            "3",
+            mockserver.url("/html"),
+            cwd=proj_path,
+        )
+        assert "RecursionError" not in stderr
+        assert "{'retried_with_custom_callback': True}" in out
+
     def test_wrong_callback_passed(
         self, proj_path: Path, mockserver: MockServer
     ) -> None:

tests/test_commands.py

@@ -3,14 +3,12 @@ from __future__ import annotations
 import argparse
 import json
 import sys
-from io import StringIO
 from typing import TYPE_CHECKING
-from unittest import mock
 
 import pytest
 
 import scrapy
-from scrapy.cmdline import _pop_command_name, _print_unknown_command_msg
+from scrapy.cmdline import _pop_command_name, execute
 from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter, view
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.settings import Settings
@@ -153,12 +151,6 @@ class MySpider(scrapy.Spider):
 
         self._append_settings(proj_mod_path, "LOG_LEVEL = 'DEBUG'\n")
 
-    @staticmethod
-    def _append_settings(proj_mod_path: Path, text: str) -> None:
-        """Add text to the end of the project settings.py."""
-        with (proj_mod_path / "settings.py").open("a", encoding="utf-8") as f:
-            f.write(text)
-
     @staticmethod
     def _replace_custom_settings(
         proj_mod_path: Path, spider_name: str, text: str
@@ -347,23 +339,223 @@ class TestMiscCommands(TestProjectBase):
         subdir.mkdir(exist_ok=True)
         assert call("list", cwd=subdir) == 0
 
-    def test_command_not_found(self) -> None:
-        na_msg = """
-The list command is not available from this location.
-These commands are only available from within a project: check, crawl, edit, list, parse.
-"""
-        not_found_msg = """
-Unknown command: abc
-"""
-        params = [
-            ("list", False, na_msg),
-            ("abc", False, not_found_msg),
-            ("abc", True, not_found_msg),
-        ]
-        for cmdname, inproject, message in params:
-            with mock.patch("sys.stdout", new=StringIO()) as out:
-                _print_unknown_command_msg(Settings(), cmdname, inproject)
-                assert out.getvalue().strip() == message.strip()
+
+class TestCommandListing(TestProjectBase):
+    """Tests for the command list that ``scrapy`` prints when called without a
+    command name."""
+
+    def test_outside_project(self) -> None:
+        returncode, out, err = proc()
+        assert returncode == 0, err
+        assert f"Scrapy {scrapy.__version__} - no active project" in out
+        assert "Available commands:" in out
+        assert "Create new project" in out
+        assert "More commands available when run from project directory" in out
+        assert 'Use "scrapy <command> -h" to see more info about a command' in out
+
+    def test_inside_project(self, proj_path: Path) -> None:
+        returncode, out, err = proc(cwd=proj_path)
+        assert returncode == 0, err
+        assert (
+            f"Scrapy {scrapy.__version__} - active project: {self.project_name}" in out
+        )
+        assert "List available spiders" in out
+        assert "More commands available when run from project directory" not in out
+
+
+class TestUnknownCommand(TestProjectBase):
+    def test_outside_project(self) -> None:
+        returncode, out, err = proc("abc")
+        assert returncode == 2, err
+        assert f"Scrapy {scrapy.__version__} - no active project" in out
+        assert "Unknown command: abc" in out
+        assert 'Use "scrapy" to see available commands' in out
+
+    def test_inside_project(self, proj_path: Path) -> None:
+        returncode, out, err = proc("abc", cwd=proj_path)
+        assert returncode == 2, err
+        assert (
+            f"Scrapy {scrapy.__version__} - active project: {self.project_name}" in out
+        )
+        assert "Unknown command: abc" in out
+
+    def test_project_only_command_outside_project(self) -> None:
+        returncode, out, err = proc("list")
+        assert returncode == 2, err
+        assert "The list command is not available from this location." in out
+        assert (
+            "These commands are only available from within a project: "
+            "check, crawl, edit, list, parse." in out
+        )
+
+
+class TestCommandsModule(TestProjectBase):
+    """Tests for commands defined in the module of the COMMANDS_MODULE setting."""
+
+    @pytest.fixture
+    def proj_path_with_commands(self, proj_path: Path) -> Path:
+        commands_path = proj_path / self.project_name / "commands"
+        commands_path.mkdir()
+        (commands_path / "__init__.py").touch()
+        (commands_path / "mycmd.py").write_text(
+            """
+from scrapy.commands import ScrapyCommand
+
+
+class Command(ScrapyCommand):
+    requires_crawler_process = False
+
+    def short_desc(self):
+        return "My custom command"
+
+    def run(self, args, opts):
+        print("My custom command ran")
+""",
+            encoding="utf-8",
+        )
+        (commands_path / "helpcmd.py").write_text(
+            """
+from scrapy.commands import ScrapyCommand
+from scrapy.exceptions import UsageError
+
+
+class Command(ScrapyCommand):
+    requires_crawler_process = False
+
+    def short_desc(self):
+        return "My command that asks for its help message"
+
+    def run(self, args, opts):
+        raise UsageError
+""",
+            encoding="utf-8",
+        )
+        (commands_path / "silentcmd.py").write_text(
+            """
+from scrapy.commands import ScrapyCommand
+from scrapy.exceptions import UsageError
+
+
+class Command(ScrapyCommand):
+    requires_crawler_process = False
+
+    def short_desc(self):
+        return "My command that fails silently"
+
+    def run(self, args, opts):
+        raise UsageError(print_help=False)
+""",
+            encoding="utf-8",
+        )
+        self._append_settings(
+            proj_path / self.project_name,
+            f'\nCOMMANDS_MODULE = "{self.project_name}.commands"\n',
+        )
+        return proj_path
+
+    def test_listed(self, proj_path_with_commands: Path) -> None:
+        returncode, out, err = proc(cwd=proj_path_with_commands)
+        assert returncode == 0, err
+        assert "My custom command" in out
+
+    def test_run(self, proj_path_with_commands: Path) -> None:
+        returncode, out, err = proc("mycmd", cwd=proj_path_with_commands)
+        assert returncode == 0, err
+        assert "My custom command ran" in out
+
+    def test_usage_error(self, proj_path_with_commands: Path) -> None:
+        """A message-less UsageError makes the help message be printed."""
+        returncode, out, err = proc("helpcmd", cwd=proj_path_with_commands)
+        assert returncode == 2, err
+        assert "scrapy helpcmd" in out
+
+    def test_usage_error_without_help(self, proj_path_with_commands: Path) -> None:
+        """A message-less UsageError with print_help disabled prints nothing."""
+        returncode, out, err = proc("silentcmd", cwd=proj_path_with_commands)
+        assert returncode == 2, err
+        assert not out
+
+
+class TestEntryPointCommands:
+    """Tests for commands defined in the scrapy.commands entry point group."""
+
+    @staticmethod
+    def _write_dist(path: Path, entry_point: str) -> None:
+        """Write into *path* a package with a command and a function, and the
+        metadata of an installed distribution that declares *entry_point* in
+        the scrapy.commands entry point group.
+
+        Since ``python -m scrapy.cmdline`` puts the current working directory
+        in the import path, running it with *path* as the working directory
+        makes Scrapy find that entry point.
+        """
+        package_path = path / "mycmds"
+        package_path.mkdir()
+        (package_path / "__init__.py").touch()
+        (package_path / "mycmd.py").write_text(
+            """
+from scrapy.commands import ScrapyCommand
+
+
+class Command(ScrapyCommand):
+    requires_crawler_process = False
+
+    def short_desc(self):
+        return "My entry point command"
+
+    def run(self, args, opts):
+        print("My entry point command ran")
+
+
+def not_a_command():
+    pass
+""",
+            encoding="utf-8",
+        )
+        dist_info_path = path / "mycmds-1.0.dist-info"
+        dist_info_path.mkdir()
+        (dist_info_path / "METADATA").write_text(
+            "Metadata-Version: 2.1\nName: mycmds\nVersion: 1.0\n", encoding="utf-8"
+        )
+        (dist_info_path / "entry_points.txt").write_text(
+            f"[scrapy.commands]\n{entry_point}\n", encoding="utf-8"
+        )
+
+    def test_listed(self, tmp_path: Path) -> None:
+        self._write_dist(tmp_path, "mycmd = mycmds.mycmd:Command")
+        returncode, out, err = proc(cwd=tmp_path)
+        assert returncode == 0, err
+        assert "My entry point command" in out
+
+    def test_run(self, tmp_path: Path) -> None:
+        self._write_dist(tmp_path, "mycmd = mycmds.mycmd:Command")
+        returncode, out, err = proc("mycmd", cwd=tmp_path)
+        assert returncode == 0, err
+        assert "My entry point command ran" in out
+
+    def test_not_a_class(self, tmp_path: Path) -> None:
+        self._write_dist(tmp_path, "mycmd = mycmds.mycmd:not_a_command")
+        returncode, _, err = proc("version", cwd=tmp_path)
+        assert returncode == 1
+        assert "ValueError: Invalid entry point mycmd" in err
+
+
+class TestExecute:
+    """Tests for calls to scrapy.cmdline.execute() from Python code, which the
+    command line does not cover."""
+
+    def test_argv(self, capsys: pytest.CaptureFixture[str]) -> None:
+        with pytest.raises(SystemExit) as exc_info:
+            execute(["scrapy", "version"])
+        assert exc_info.value.code == 0
+        assert scrapy.__version__ in capsys.readouterr().out
+
+    def test_settings(self, capsys: pytest.CaptureFixture[str]) -> None:
+        settings = Settings()
+        with pytest.raises(SystemExit) as exc_info:
+            execute(["scrapy", "settings", "--get", "BOT_NAME"], settings=settings)
+        assert exc_info.value.code == 0
+        assert capsys.readouterr().out.strip() == "scrapybot"
 
 
 class TestBenchCommand:

tests/test_crawl.py

@@ -41,6 +41,7 @@ from tests.spiders import (
     CrawlSpiderWithAsyncCallback,
     CrawlSpiderWithAsyncGeneratorCallback,
     CrawlSpiderWithErrback,
+    CrawlSpiderWithoutErrback,
     CrawlSpiderWithParseMethod,
     CrawlSpiderWithProcessRequestCallbackKeywordArguments,
     DelaySpider,
@@ -505,6 +506,21 @@ class TestCrawlSpider:
         assert "[errback] status 500" in caplog.text
         assert "[errback] status 501" in caplog.text
 
+    @coroutine_test
+    async def test_crawlspider_without_errback(
+        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+    ) -> None:
+        crawler = get_crawler(CrawlSpiderWithoutErrback)
+        with caplog.at_level(logging.INFO):
+            await crawler.crawl_async(mockserver=mockserver)
+
+        # The failing request (404) is followed by a rule without an errback,
+        # so the failure is dropped silently and the crawl finishes normally.
+        assert "[parse] status 200 (foo: None)" in caplog.text
+        assert "[errback]" not in caplog.text
+        assert crawler.stats
+        assert crawler.stats.get_value("downloader/response_status_count/404") == 1
+
     @coroutine_test
     async def test_crawlspider_process_request_cb_kwargs(
         self, caplog: pytest.LogCaptureFixture, mockserver: MockServer

tests/test_downloadermiddleware_httpcache.py

@@ -1,10 +1,12 @@
 from __future__ import annotations
 
 import email.utils
+import logging
 import shutil
 import tempfile
 import time
 from contextlib import contextmanager
+from pathlib import Path
 from typing import TYPE_CHECKING, Any
 from unittest import mock
 
@@ -93,6 +95,12 @@ class TestBase:
 class StorageTestMixin:
     """Mixin containing storage-specific test methods."""
 
+    def _corrupt_cache_entry(
+        self, storage: Any, spider: Spider, request: Request
+    ) -> None:
+        """Make the cache entry of *request* unreadable for *storage*."""
+        raise NotImplementedError
+
     def test_storage(self):
         with self._storage(HTTPCACHE_EXPIRATION_SECS=1) as (storage, crawler):
             request2 = self.request.copy()
@@ -115,6 +123,38 @@ class StorageTestMixin:
             with mock.patch("scrapy.extensions.httpcache.time", return_value=future):
                 assert storage.retrieve_response(crawler.spider, self.request)
 
+    def test_corrupted_cache_entry_is_a_miss(self, caplog):
+        with self._middleware() as mw:
+            spider = mw.crawler.spider
+            mw.storage.store_response(spider, self.request, self.response)
+            self._corrupt_cache_entry(mw.storage, spider, self.request)
+
+            caplog.clear()
+            with caplog.at_level(logging.WARNING):
+                assert mw.process_request(self.request) is None
+
+            assert "treating it as a cache miss" in caplog.text
+            assert mw.crawler.stats.get_value("httpcache/retrieve_error") == 1
+            assert mw.crawler.stats.get_value("httpcache/miss") == 1
+
+            # Storing the response again replaces the corrupted cache entry.
+            mw.storage.store_response(spider, self.request, self.response)
+            self.assertEqualResponse(
+                self.response, mw.storage.retrieve_response(spider, self.request)
+            )
+
+    def test_corrupted_cache_entry_ignore_missing(self):
+        with self._middleware(HTTPCACHE_IGNORE_MISSING=True) as mw:
+            spider = mw.crawler.spider
+            mw.storage.store_response(spider, self.request, self.response)
+            self._corrupt_cache_entry(mw.storage, spider, self.request)
+
+            with pytest.raises(IgnoreRequest):
+                mw.process_request(self.request)
+
+            assert mw.crawler.stats.get_value("httpcache/retrieve_error") == 1
+            assert mw.crawler.stats.get_value("httpcache/ignore") == 1
+
     def test_storage_no_content_type_header(self):
         """Test that the response body is used to get the right response class
         even if there is no Content-Type header"""
@@ -556,29 +596,43 @@ class RFC2616PolicyTestMixin(PolicyTestMixin):
 # Concrete test classes that combine storage and policy mixins
 
 
-class TestFilesystemStorageWithDummyPolicy(
-    TestBase, StorageTestMixin, DummyPolicyTestMixin
-):
+class FilesystemStorageTestMixin(StorageTestMixin):
     storage_class = "scrapy.extensions.httpcache.FilesystemCacheStorage"
+
+    def _corrupt_cache_entry(self, storage, spider, request) -> None:
+        rpath = Path(storage._get_request_path(spider, request))
+        (rpath / "response_body").unlink()
+
+
+class DbmStorageTestMixin(StorageTestMixin):
+    storage_class = "scrapy.extensions.httpcache.DbmCacheStorage"
+
+    def _corrupt_cache_entry(self, storage, spider, request) -> None:
+        key = storage._fingerprinter.fingerprint(request).hex()
+        storage.db[f"{key}_data"] = b"not a pickle"
+
+
+class TestFilesystemStorageWithDummyPolicy(
+    TestBase, FilesystemStorageTestMixin, DummyPolicyTestMixin
+):
     policy_class = "scrapy.extensions.httpcache.DummyPolicy"
 
 
 class TestFilesystemStorageWithRFC2616Policy(
-    TestBase, StorageTestMixin, RFC2616PolicyTestMixin
+    TestBase, FilesystemStorageTestMixin, RFC2616PolicyTestMixin
 ):
-    storage_class = "scrapy.extensions.httpcache.FilesystemCacheStorage"
     policy_class = "scrapy.extensions.httpcache.RFC2616Policy"
 
 
-class TestDbmStorageWithDummyPolicy(TestBase, StorageTestMixin, DummyPolicyTestMixin):
-    storage_class = "scrapy.extensions.httpcache.DbmCacheStorage"
+class TestDbmStorageWithDummyPolicy(
+    TestBase, DbmStorageTestMixin, DummyPolicyTestMixin
+):
     policy_class = "scrapy.extensions.httpcache.DummyPolicy"
 
 
 class TestDbmStorageWithRFC2616Policy(
-    TestBase, StorageTestMixin, RFC2616PolicyTestMixin
+    TestBase, DbmStorageTestMixin, RFC2616PolicyTestMixin
 ):
-    storage_class = "scrapy.extensions.httpcache.DbmCacheStorage"
     policy_class = "scrapy.extensions.httpcache.RFC2616Policy"
 
 
@@ -599,3 +653,8 @@ class TestFilesystemStorageGzipWithDummyPolicy(TestFilesystemStorageWithDummyPol
     def _get_settings(self, **new_settings) -> dict[str, Any]:
         new_settings.setdefault("HTTPCACHE_GZIP", True)
         return super()._get_settings(**new_settings)
+
+    def _corrupt_cache_entry(self, storage, spider, request) -> None:
+        # A spider killed while writing a gzip file leaves it truncated.
+        body_path = Path(storage._get_request_path(spider, request), "response_body")
+        body_path.write_bytes(body_path.read_bytes()[:-5])

tests/test_feedexport_batch.py

@@ -211,6 +211,47 @@ class TestBatchDeliveries(TestFeedExportBase):
         header = MyItem.fields.keys()
         await self.assertExported(items, header, rows, settings=settings)
 
+    @coroutine_test
+    async def test_batch_delivered_when_full(self):
+        """Full batches must be finalized and delivered as soon as they are
+        full, instead of when the spider closes."""
+        dir_path = self._random_temp_filename()
+        batch1_path = Path(dir_path, "1.json")
+        mockserver_url = self.mockserver.url("/")
+        batch1_contents: list[bytes | None] = []
+
+        class TestSpider(scrapy.Spider):
+            name = "testspider"
+            start_urls = [mockserver_url]
+
+            def parse(self, response):
+                yield {"foo": "bar1"}
+                yield {"foo": "bar2"}
+                yield scrapy.Request(
+                    mockserver_url, callback=self.parse2, dont_filter=True
+                )
+
+            def parse2(self, response):
+                # the first batch was full after the second item, so it must
+                # have been delivered by now
+                batch1_contents.append(
+                    batch1_path.read_bytes() if batch1_path.exists() else None
+                )
+                yield {"foo": "bar3"}
+
+        settings = {
+            "FEEDS": {
+                build_url(dir_path / "%(batch_id)d.json"): {"format": "json"},
+            },
+            "FEED_EXPORT_BATCH_ITEM_COUNT": 2,
+        }
+        crawler = get_crawler(TestSpider, settings)
+        await crawler.crawl_async()
+
+        assert batch1_contents, "the second request was not processed"
+        assert batch1_contents[0] is not None, "batch 1 was not stored during the crawl"
+        assert json.loads(batch1_contents[0]) == [{"foo": "bar1"}, {"foo": "bar2"}]
+
     def test_wrong_path(self):
         """If path is without %(batch_time)s and %(batch_id) an exception must be raised"""
         settings = {

tests/test_pipeline_files.py

@@ -33,7 +33,7 @@ from scrapy.pipelines.files import (
     GCSFilesStore,
     S3FilesStore,
 )
-from scrapy.pipelines.media import MediaPipeline, _MediaRequestFiltered
+from scrapy.pipelines.media import _MediaRequestFiltered
 from scrapy.settings import Settings
 from scrapy.utils.asyncio import call_later
 from scrapy.utils.defer import maybe_deferred_to_future
@@ -43,11 +43,7 @@ from tests.mockserver.ftp import MockFTPServer
 from tests.utils.decorators import coroutine_test, inline_callbacks_test
 
 from .utils.cloud import mock_google_cloud_storage
-from .utils.media_pipelines import mocked_download_func
-
-# required by persist_file() and stat_file(), but as some stores don't use the argument
-# we can pass this singleton to keep type hints correct
-DUMMY_SPIDER_INFO = MediaPipeline.SpiderInfo(DefaultSpider())
+from .utils.media_pipelines import DUMMY_SPIDER_INFO, mocked_download_func
 
 
 def get_ftp_content_and_delete(
@@ -94,16 +90,19 @@ class DeferredFSFilesStore(FSFilesStore):
 class TestFilesPipeline:
     def setup_method(self):
         self.tempdir = mkdtemp()
-        settings_dict = {"FILES_STORE": self.tempdir}
-        crawler = get_crawler(DefaultSpider, settings_dict=settings_dict)
-        crawler.spider = crawler._create_spider()
-        crawler.engine = MagicMock(download_async=mocked_download_func)
-        self.pipeline = FilesPipeline.from_crawler(crawler)
-        self.pipeline.open_spider()
+        self.pipeline = self._create_pipeline(FilesPipeline)
 
     def teardown_method(self):
         rmtree(self.tempdir)
 
+    def _create_pipeline(self, pipeline_cls: type[FilesPipeline]) -> FilesPipeline:
+        crawler = get_crawler(DefaultSpider, {"FILES_STORE": self.tempdir})
+        crawler.spider = crawler._create_spider()
+        crawler.engine = MagicMock(download_async=mocked_download_func)
+        pipeline = pipeline_cls.from_crawler(crawler)
+        pipeline.open_spider()
+        return pipeline
+
     @coroutine_test
     async def test_close_spider_closes_store(self):
         """close_spider() closes the store when it exposes a close() method."""
@@ -268,6 +267,107 @@ class TestFilesPipeline:
         assert result["files"][0]["checksum"] != "abc"
         assert result["files"][0]["status"] == "cached"
 
+    @coroutine_test
+    async def test_file_stat_without_last_modified(self) -> None:
+        """A stat result without a last modification time forces a download."""
+        item_url = "http://example.com/file4.pdf"
+        item = _create_item_with_files(item_url)
+        with (
+            mock.patch.object(FilesPipeline, "inc_stats", return_value=True),
+            mock.patch.object(
+                FSFilesStore, "stat_file", return_value={"checksum": "abc"}
+            ),
+            mock.patch.object(
+                FilesPipeline,
+                "get_media_requests",
+                return_value=[_prepare_request_object(item_url)],
+            ),
+        ):
+            result = await self.pipeline.process_item(item)
+        assert result["files"][0]["checksum"] != "abc"
+        assert result["files"][0]["status"] == "downloaded"
+
+    @coroutine_test
+    async def test_file_empty_content(self, caplog: pytest.LogCaptureFixture) -> None:
+        item_url = "http://example.com/empty.pdf"
+        item = _create_item_with_files(item_url)
+        request = Request(
+            item_url, meta={"response": Response(item_url, status=200, body=b"")}
+        )
+        with (
+            caplog.at_level(logging.WARNING),
+            mock.patch.object(
+                FilesPipeline, "get_media_requests", return_value=[request]
+            ),
+        ):
+            result = await self.pipeline.process_item(item)
+        assert result["files"] == []
+        assert "File (empty-content): Empty file from" in caplog.text
+
+    @coroutine_test
+    async def test_file_downloaded_file_exception(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
+        """A FileException from file_downloaded() is logged as a warning and
+        kept as is."""
+
+        class FailingFilesPipeline(FilesPipeline):
+            def file_downloaded(self, response, request, info, *, item=None):
+                raise FileException("boom")
+
+        item_url = "http://example.com/file5.pdf"
+        item = _create_item_with_files(item_url)
+        pipeline = self._create_pipeline(FailingFilesPipeline)
+        with (
+            caplog.at_level(logging.WARNING),
+            mock.patch.object(
+                FilesPipeline,
+                "get_media_requests",
+                return_value=[_prepare_request_object(item_url)],
+            ),
+        ):
+            result = await pipeline.process_item(item)
+        assert result["files"] == []
+        records = [
+            r for r in caplog.records if "Error processing file" in r.getMessage()
+        ]
+        assert len(records) == 1
+        assert records[0].levelname == "WARNING"
+        assert "boom" in records[0].getMessage()
+
+    @coroutine_test
+    async def test_file_downloaded_unknown_error(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
+        """Any other exception from file_downloaded() is logged as an error and
+        reported as a FileException."""
+
+        class FailingFilesPipeline(FilesPipeline):
+            def file_downloaded(self, response, request, info, *, item=None):
+                raise RuntimeError("boom")
+
+        item_url = "http://example.com/file6.pdf"
+        item = _create_item_with_files(item_url)
+        pipeline = self._create_pipeline(FailingFilesPipeline)
+        with (
+            caplog.at_level(logging.WARNING),
+            mock.patch.object(
+                FilesPipeline,
+                "get_media_requests",
+                return_value=[_prepare_request_object(item_url)],
+            ),
+        ):
+            result = await pipeline.process_item(item)
+        assert result["files"] == []
+        records = [
+            r for r in caplog.records if "Error processing file" in r.getMessage()
+        ]
+        assert len(records) == 1
+        assert records[0].levelname == "ERROR"
+        exc_info = records[0].exc_info
+        assert exc_info is not None
+        assert exc_info[0] is RuntimeError
+
     @coroutine_test
     async def test_async_store(self) -> None:
         """Test that async persist_file() works and is awaited."""
@@ -662,9 +762,24 @@ class TestFilesPipelineCustomSettings:
         request = Request("http://example.com/image01.jpg")
         assert pipeline.file_path(request) == Path("subdir/image01.jpg")
 
-    def test_files_store_constructor_with_pathlike_object(self, tmp_path):
-        fs_store = FSFilesStore(tmp_path)
-        assert fs_store.basedir == str(tmp_path)
+
+class TestFSFilesStore:
+    def test_constructor_with_pathlike_object(self, tmp_path: Path) -> None:
+        assert FSFilesStore(tmp_path).basedir == str(tmp_path)
+
+    def test_constructor_with_uri(self, tmp_path: Path) -> None:
+        assert FSFilesStore(f"file://{tmp_path}").basedir == str(tmp_path)
+
+    def test_stat_file(self, tmp_path: Path) -> None:
+        store = FSFilesStore(tmp_path)
+        store.persist_file("full/filename", BytesIO(b"data"), DUMMY_SPIDER_INFO)
+        stat = store.stat_file("full/filename", DUMMY_SPIDER_INFO)
+        assert stat["checksum"] == "8d777f385d3dfec8815d20f7496026dc"
+        assert stat["last_modified"] == pytest.approx(time.time(), abs=60)
+
+    def test_stat_missing_file(self, tmp_path: Path) -> None:
+        store = FSFilesStore(tmp_path)
+        assert store.stat_file("full/filename", DUMMY_SPIDER_INFO) == {}
 
 
 @pytest.mark.requires_botocore
@@ -712,6 +827,60 @@ class TestS3FilesStore:
             # The call to read does not happen with Stubber
             assert buffer.method_calls == [mock.call.seek(0)]
 
+    @inline_callbacks_test
+    def test_persist_without_headers(self, monkeypatch):
+        """Without custom headers only the default ones are sent."""
+        bucket = "mybucket"
+        key = "export.csv"
+        buffer = mock.MagicMock()
+
+        store = S3FilesStore(f"s3://{bucket}/{key}")
+        monkeypatch.setattr(store, "_use_async", lambda: False)
+        from botocore.stub import Stubber  # noqa: PLC0415
+
+        with Stubber(store.s3_client) as stub:
+            stub.add_response(
+                "put_object",
+                expected_params={
+                    "ACL": S3FilesStore.POLICY,
+                    "Body": buffer,
+                    "Bucket": bucket,
+                    "CacheControl": S3FilesStore.HEADERS["Cache-Control"],
+                    "Key": key,
+                    "Metadata": {},
+                },
+                service_response={},
+            )
+
+            yield store.persist_file("", buffer, info=DUMMY_SPIDER_INFO)
+
+            stub.assert_no_pending_responses()
+
+    def test_missing_botocore(self):
+        with (
+            mock.patch(
+                "scrapy.pipelines.files.is_botocore_available", return_value=False
+            ),
+            pytest.raises(NotConfigured, match="missing botocore library"),
+        ):
+            S3FilesStore("s3://mybucket/key")
+
+    def test_wrong_uri_scheme(self):
+        with pytest.raises(
+            ValueError,
+            match=re.escape(
+                "Incorrect URI scheme in ftp://mybucket/key, expected 's3'"
+            ),
+        ):
+            S3FilesStore("ftp://mybucket/key")
+
+    def test_unsupported_header(self):
+        store = S3FilesStore("s3://mybucket/key")
+        with pytest.raises(
+            TypeError, match='Header "X-Custom" is not supported by botocore'
+        ):
+            store._headers_to_botocore_kwargs({"X-Custom": "value"})
+
     @inline_callbacks_test
     def test_stat(self, monkeypatch):
         """The blocking botocore client is used when asyncio/aiobotocore support
@@ -1020,6 +1189,28 @@ class TestFTPFileStore:
             )
         assert data == content
 
+    @inline_callbacks_test
+    def test_persist_active_mode(self, monkeypatch: pytest.MonkeyPatch):
+        data = b"active mode"
+        path = "full/filename"
+        monkeypatch.setattr(FTPFilesStore, "FTP_USERNAME", "anonymous")
+        monkeypatch.setattr(FTPFilesStore, "FTP_PASSWORD", "guest")
+        monkeypatch.setattr(FTPFilesStore, "USE_ACTIVE_MODE", True)
+        with MockFTPServer() as ftp_server:
+            store = FTPFilesStore(ftp_server.url("/"))
+            yield store.persist_file(path, BytesIO(data), info=DUMMY_SPIDER_INFO)
+            stat = yield store.stat_file(path, info=DUMMY_SPIDER_INFO)
+        assert stat["checksum"] == "ff1575649a39a27c13faa0d37c84bab3"
+
+    def test_wrong_uri_scheme(self):
+        with pytest.raises(
+            ValueError,
+            match=re.escape(
+                "Incorrect URI scheme in http://example.com/, expected 'ftp'"
+            ),
+        ):
+            FTPFilesStore("http://example.com/")
+
 
 class ItemWithFiles(Item):
     file_urls = Field()

tests/test_pipeline_images.py

@@ -3,20 +3,26 @@ from __future__ import annotations
 import dataclasses
 import io
 import random
+import sys
 from abc import ABC, abstractmethod
+from pathlib import Path
 from shutil import rmtree
 from tempfile import mkdtemp
+from types import SimpleNamespace
 from typing import Any
 
 import attr
 import pytest
 from itemadapter import ItemAdapter
 
+from scrapy.exceptions import NotConfigured
 from scrapy.http import Request, Response
 from scrapy.item import Field, Item
-from scrapy.pipelines.files import GCSFilesStore, S3FilesStore
+from scrapy.pipelines.files import GCSFilesStore, S3FilesStore, _md5sum
 from scrapy.pipelines.images import ImageException, ImagesPipeline
 from scrapy.utils.test import get_crawler
+from tests.utils.decorators import coroutine_test
+from tests.utils.media_pipelines import DUMMY_SPIDER_INFO
 
 try:
     from PIL import Image
@@ -40,6 +46,11 @@ class TestImagesPipeline:
     def teardown_method(self):
         rmtree(self.tempdir)
 
+    def test_missing_pillow(self, monkeypatch: pytest.MonkeyPatch) -> None:
+        monkeypatch.setitem(sys.modules, "PIL", None)
+        with pytest.raises(NotConfigured, match="requires installing Pillow"):
+            ImagesPipeline(self.tempdir, crawler=get_crawler())
+
     def test_file_path(self):
         file_path = self.pipeline.file_path
         assert (
@@ -197,6 +208,25 @@ class TestImagesPipeline:
         assert path == "full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg"
         assert new_im.getpixel((0, 0)) == (255, 0, 0)
 
+    @coroutine_test
+    async def test_image_downloaded(self) -> None:
+        """The image and its thumbnails are stored, and the checksum of the
+        full-size image is returned."""
+        self.pipeline.thumbs = {"small": (20, 20)}
+        _, buf = _create_image("JPEG", "RGB", (50, 50), (0, 0, 0))
+        url = "https://dev.mydeco.com/mydeco.gif"
+        response = Response(url=url, body=buf.getvalue())
+
+        checksum = await self.pipeline.image_downloaded(
+            response, Request(url=url), DUMMY_SPIDER_INFO
+        )
+
+        buf.seek(0)
+        assert checksum == _md5sum(buf)
+        name = "3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg"
+        assert Path(self.tempdir, "full", name).read_bytes() == buf.getvalue()
+        assert Path(self.tempdir, "thumbs", "small", name).exists()
+
     def test_convert_image(self):
         SIZE = (100, 100)
         # straight forward case: RGB and JPEG
@@ -230,6 +260,24 @@ class TestImagesPipeline:
         assert converted.mode == "RGB"
         assert converted.getcolors() == [(10000, (205, 230, 255))]
 
+    def test_convert_image_legacy_resampling_filter(
+        self, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        """Pillow older than 9.1.0 has Image.ANTIALIAS instead of
+        Image.Resampling.LANCZOS."""
+        # Image.LANCZOS is the only spelling that exists in every supported
+        # Pillow version, but Pillow defines it dynamically, hence the ignore.
+        monkeypatch.setattr(
+            self.pipeline,
+            "_Image",
+            SimpleNamespace(ANTIALIAS=Image.LANCZOS),  # type: ignore[attr-defined]
+        )
+        im, buf = _create_image("JPEG", "RGB", (100, 100), (0, 127, 255))
+
+        thumbnail, _ = self.pipeline.convert_image(im, size=(10, 25), response_body=buf)
+
+        assert thumbnail.size == (10, 10)
+
     @pytest.mark.parametrize(
         "bad_type",
         [
@@ -581,7 +629,7 @@ class TestImagesPipelineCustomSettings:
             GCSFilesStore.POLICY = old_policy
 
 
-def _create_image(format_, *a, **kw):
+def _create_image(format_: str, *a: Any, **kw: Any) -> tuple[Image.Image, io.BytesIO]:
     buf = io.BytesIO()
     Image.new(*a, **kw).save(buf, format_)
     buf.seek(0)

tests/test_pipeline_media.py

@@ -319,6 +319,42 @@ class TestMediaPipeline(TestBaseMediaPipeline):
         assert self.fingerprint(req1) == self.fingerprint(req2)
         assert new_item["results"] == [(True, {})]
 
+    @coroutine_test
+    async def test_failures_are_cached_across_multiple_items(self):
+        self.pipe.LOG_FAILED_RESULTS = False
+        exc = Exception("foo")
+        req1 = Request("http://url1", meta={"response": exc})
+        new_item = await self.pipe.process_item({"requests": req1})
+        assert new_item["results"][0][1].value is exc
+
+        # rsp2 is ignored, the cached failure must be reused because request
+        # fingerprints are the same
+        req2 = Request(
+            req1.url, meta={"response": Response("http://donot.download.me")}
+        )
+        new_item = await self.pipe.process_item({"requests": req2})
+        assert new_item["results"][0][0] is False
+        assert new_item["results"][0][1].value is exc
+        assert self.pipe._mockcalled.count("media_to_download") == 1
+
+    @coroutine_test
+    async def test_cached_failure_calls_errback(self):
+        """The errback of a request is called for a cached failure as well."""
+        self.pipe.LOG_FAILED_RESULTS = False
+        exc = Exception("foo")
+        await self.pipe.process_item(
+            {"requests": Request("http://url1", meta={"response": exc})}
+        )
+
+        def errback(failure):
+            self.pipe._mockcalled.append("request_errback")
+            return {"recovered": failure.value}
+
+        req = Request("http://url1", errback=errback)
+        new_item = await self.pipe.process_item({"requests": req})
+        assert new_item["results"] == [(True, {"recovered": exc})]
+        assert self.pipe._mockcalled.count("request_errback") == 1
+
     @coroutine_test
     async def test_results_are_cached_for_requests_of_single_item(self):
         rsp1 = Response("http://url1")
@@ -472,6 +508,30 @@ class TestBuildFromCrawler:
         assert pipe._from_crawler_called
 
 
+class MediaFailedNonePipeline(MockedMediaPipeline):
+    def media_failed(self, failure, request, info):
+        self._mockcalled.append("media_failed")
+
+
+class TestMediaFailedNone(TestBaseMediaPipeline):
+    """Test what happens when media_failed() neither raises an exception nor
+    returns a failure."""
+
+    pipeline_class = MediaFailedNonePipeline
+
+    @coroutine_test
+    async def test_result_none(self):
+        req = Request("http://url1", meta={"response": Exception("foo")})
+        new_item = await self.pipe.process_item({"requests": req})
+        assert new_item["results"] == [(True, None)]
+        assert self.pipe._mockcalled == [
+            "get_media_requests",
+            "media_to_download",
+            "media_failed",
+            "item_completed",
+        ]
+
+
 class MediaFailedFailurePipeline(MockedMediaPipeline):
     def media_failed(self, failure, request, info):
         self._mockcalled.append("media_failed")

tests/test_spider.py

@@ -1,11 +1,26 @@
 from __future__ import annotations
 
+from typing import TYPE_CHECKING
+
 import pytest
 
-from scrapy.http import Response, TextResponse, XmlResponse
+from scrapy.http import Request, Response, TextResponse, XmlResponse
 from scrapy.spiders import CSVFeedSpider, Spider, XMLFeedSpider
 from tests import get_testdata
+from tests.spiders import RawResponseSpider
 from tests.utils.bases.spider import TestSpiderBase
+from tests.utils.crawl import crawl_items
+from tests.utils.decorators import coroutine_test
+
+if TYPE_CHECKING:
+    from tests.mockserver.http import MockServer
+
+
+class RawFeedSpider(RawResponseSpider):
+    content_type = "text/xml"
+
+    async def start(self):
+        yield Request(self.raw_url)
 
 
 class TestSpider(TestSpiderBase):
@@ -60,6 +75,89 @@ class TestXMLFeedSpider(TestSpiderBase):
                 },
             ], iterator
 
+    @coroutine_test
+    async def test_parse_node_uses_parse_item(self, mockserver: MockServer):
+        # parse_node falls back to parse_item for backward compatibility.
+        class _Spider(RawFeedSpider, self.spider_class):  # type: ignore[name-defined,misc]
+            itertag = "item"
+
+            def raw_body(self):
+                return "<items><item><id>1</id></item></items>"
+
+            def parse_item(self, response, selector):
+                return {"id": selector.xpath("id/text()").get()}
+
+        items, _ = await crawl_items(_Spider, mockserver)
+        assert items == [{"id": "1"}]
+
+    @coroutine_test
+    async def test_parse_node_not_defined(self, mockserver: MockServer):
+        class _Spider(RawFeedSpider, self.spider_class):  # type: ignore[name-defined,misc]
+            itertag = "item"
+
+            def raw_body(self):
+                return "<items><item><id>1</id></item></items>"
+
+        items, crawler = await crawl_items(_Spider, mockserver)
+        assert items == []
+        assert crawler.stats
+        assert crawler.stats.get_value("spider_exceptions/NotImplementedError") == 1
+
+    @coroutine_test
+    async def test_html_iterator(self, mockserver: MockServer):
+        class _Spider(RawFeedSpider, self.spider_class):  # type: ignore[name-defined,misc]
+            iterator = "html"
+            itertag = "item"
+            content_type = "text/html"
+
+            def raw_body(self):
+                return (
+                    "<html><body><item><id>1</id></item>"
+                    "<item><id>2</id></item></body></html>"
+                )
+
+            def parse_node(self, response, selector):
+                return {"id": selector.xpath("id/text()").get()}
+
+        items, _ = await crawl_items(_Spider, mockserver)
+        assert items == [{"id": "1"}, {"id": "2"}]
+
+    @coroutine_test
+    async def test_unsupported_iterator(self, mockserver: MockServer):
+        class _Spider(RawFeedSpider, self.spider_class):  # type: ignore[name-defined,misc]
+            iterator = "unsupported"
+
+            def raw_body(self):
+                return "<items><item/></items>"
+
+            def parse_node(self, response, selector):
+                return {}
+
+        items, crawler = await crawl_items(_Spider, mockserver)
+        assert items == []
+        assert crawler.stats
+        assert crawler.stats.get_value("spider_exceptions/NotSupported") == 1
+
+    @pytest.mark.parametrize("feed_iterator", ["xml", "html"])
+    @coroutine_test
+    async def test_non_text_response(self, feed_iterator: str, mockserver: MockServer):
+        # The xml and html iterators require a text response.
+        class _Spider(RawFeedSpider, self.spider_class):  # type: ignore[name-defined,misc]
+            content_type = "application/octet-stream"
+            iterator = feed_iterator
+
+            def raw_body(self):
+                # A binary (non-text) body, so the response is a plain Response.
+                return "\x00\x01\x02\x03"
+
+            def parse_node(self, response, selector):
+                return {}
+
+        items, crawler = await crawl_items(_Spider, mockserver)
+        assert items == []
+        assert crawler.stats
+        assert crawler.stats.get_value("spider_exceptions/ValueError") == 1
+
 
 class TestCSVFeedSpider(TestSpiderBase):
     spider_class = CSVFeedSpider
@@ -81,6 +179,36 @@ class TestCSVFeedSpider(TestSpiderBase):
         assert rows[0] == {"id": "1", "name": "alpha", "value": "foobar"}
         assert len(rows) == 4
 
+    @coroutine_test
+    async def test_parse(self, mockserver: MockServer):
+        class _Spider(RawFeedSpider, self.spider_class):  # type: ignore[name-defined,misc]
+            content_type = "text/csv"
+            delimiter = ","
+            quotechar = "'"
+
+            def raw_body(self):
+                return get_testdata("feeds", "feed-sample6.csv").decode()
+
+            def parse_row(self, response, row):
+                return row
+
+        items, _ = await crawl_items(_Spider, mockserver)
+        assert items[0] == {"id": "1", "name": "alpha", "value": "foobar"}
+        assert len(items) == 4
+
+    @coroutine_test
+    async def test_parse_row_not_defined(self, mockserver: MockServer):
+        class _Spider(RawFeedSpider, self.spider_class):  # type: ignore[name-defined,misc]
+            content_type = "text/csv"
+
+            def raw_body(self):
+                return "id\n1\n"
+
+        items, crawler = await crawl_items(_Spider, mockserver)
+        assert items == []
+        assert crawler.stats
+        assert crawler.stats.get_value("spider_exceptions/NotImplementedError") == 1
+
 
 class TestNoParseMethodSpider:
     spider_class = Spider

tests/test_spider_crawl.py

@@ -12,6 +12,7 @@ from scrapy.linkextractors import LinkExtractor
 from scrapy.spiders import CrawlSpider, Rule, Spider
 from scrapy.utils.test import get_crawler
 from tests.utils.bases.spider import TestSpiderBase
+from tests.utils.decorators import coroutine_test
 
 
 class TestCrawlSpider(TestSpiderBase):
@@ -293,6 +294,48 @@ class TestCrawlSpider(TestSpiderBase):
                 TextResponse(spider.start_urls, body=b""), None, None
             )
 
+    @coroutine_test
+    async def test_parse_with_rules_without_callback(self):
+        response = HtmlResponse(
+            "http://example.org/somepage/index.html", body=self.test_body
+        )
+
+        class _CrawlSpider(CrawlSpider):
+            name = "test"
+            allowed_domains = ["example.org"]
+            rules = (Rule(),)
+
+        spider = _CrawlSpider.from_crawler(get_crawler(_CrawlSpider))
+        results = [
+            r async for r in spider.parse_with_rules(response, None, {}, follow=True)
+        ]
+        assert [r.url for r in results] == [
+            "http://example.org/somepage/item/12.html",
+            "http://example.org/about.html",
+            "http://example.org/nofollow.html",
+        ]
+
+    @coroutine_test
+    async def test_parse_with_rules_without_following(self):
+        response = HtmlResponse(
+            "http://example.org/somepage/index.html", body=self.test_body
+        )
+        item = {"name": "item"}
+
+        class _CrawlSpider(CrawlSpider):
+            name = "test"
+            allowed_domains = ["example.org"]
+            rules = (Rule(),)
+
+        spider = _CrawlSpider.from_crawler(get_crawler(_CrawlSpider))
+        results = [
+            r
+            async for r in spider.parse_with_rules(
+                response, lambda response: [item], {}, follow=False
+            )
+        ]
+        assert results == [item]
+
 
 class TestDeprecation:
     def test_crawl_spider(self):

tests/test_spider_sitemap.py

@@ -6,6 +6,7 @@ from datetime import datetime
 from io import BytesIO
 from logging import WARNING
 from pathlib import Path
+from typing import TYPE_CHECKING
 
 import pytest
 
@@ -13,9 +14,30 @@ from scrapy.http import HtmlResponse, Request, Response, TextResponse, XmlRespon
 from scrapy.spiders import SitemapSpider
 from scrapy.utils.test import get_crawler
 from tests import tests_datadir
+from tests.spiders import RawResponseSpider
 from tests.utils.bases.spider import TestSpiderBase
+from tests.utils.crawl import crawl_items
 from tests.utils.decorators import coroutine_test
 
+if TYPE_CHECKING:
+    from tests.mockserver.http import MockServer
+
+
+class RawSitemapSpider(RawResponseSpider):
+    """Feeds :meth:`raw_body` to :class:`~scrapy.spiders.SitemapSpider` as a
+    sitemap, so that it is fetched and followed through a regular crawl.
+
+    Subclasses build the document in :meth:`raw_body`, typically using
+    :attr:`mockserver` to point ``<loc>`` entries at real endpoints.
+    """
+
+    content_type = "application/xml"
+
+    async def start(self):
+        self.sitemap_urls = [self.raw_url]
+        async for request in super().start():
+            yield request
+
 
 class TestSitemapSpider(TestSpiderBase):
     spider_class = SitemapSpider
@@ -253,6 +275,46 @@ Sitemap: /sitemap-relative-url.xml
         urls = [req.url for req in spider._parse_sitemap(r)]
         assert urls == result
 
+    @coroutine_test
+    async def test_sitemap_rules_with_callable(self, mockserver: MockServer):
+        # A sitemap_rules entry may hold a callable instead of a method name.
+        def parse_item(response):
+            yield {"url": response.url}
+
+        class _Spider(RawSitemapSpider, self.spider_class):  # type: ignore[name-defined,misc]
+            sitemap_rules = [("", parse_item)]
+
+            def raw_body(self):
+                loc = self.mockserver.url("/text")
+                return (
+                    '<?xml version="1.0" encoding="UTF-8"?>'
+                    '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">'
+                    f"<url><loc>{loc}</loc></url>"
+                    "</urlset>"
+                )
+
+        items, _ = await crawl_items(_Spider, mockserver)
+        assert items == [{"url": mockserver.url("/text")}]
+
+    @coroutine_test
+    async def test_sitemap_empty_loc(self, mockserver: MockServer):
+        class _Spider(RawSitemapSpider, self.spider_class):  # type: ignore[name-defined,misc]
+            def parse(self, response):
+                yield {"url": response.url}
+
+            def raw_body(self):
+                loc = self.mockserver.url("/text")
+                return (
+                    '<?xml version="1.0" encoding="UTF-8"?>'
+                    '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">'
+                    "<url><loc></loc></url>"
+                    f"<url><loc>{loc}</loc></url>"
+                    "</urlset>"
+                )
+
+        items, _ = await crawl_items(_Spider, mockserver)
+        assert items == [{"url": mockserver.url("/text")}]
+
     def test_parse_sitemap_empty_body(self, caplog: pytest.LogCaptureFixture) -> None:
         r = XmlResponse(url="http://www.example.com/sitemap.xml", body=b"")
         spider = self.spider_class("example.com")

tests/test_utils_asyncgen.py

@@ -1,16 +1,18 @@
+from __future__ import annotations
+
 from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
 from tests.utils.decorators import coroutine_test
 
 
-class TestAsyncgenUtils:
-    @coroutine_test
-    async def test_as_async_generator(self):
-        ag = as_async_generator(range(42))
-        results = [i async for i in ag]
-        assert results == list(range(42))
+@coroutine_test
+async def test_as_async_generator():
+    ag = as_async_generator(range(42))
+    results = [i async for i in ag]
+    assert results == list(range(42))
 
-    @coroutine_test
-    async def test_collect_asyncgen(self):
-        ag = as_async_generator(range(42))
-        results = await collect_asyncgen(ag)
-        assert results == list(range(42))
+
+@coroutine_test
+async def test_collect_asyncgen():
+    ag = as_async_generator(range(42))
+    results = await collect_asyncgen(ag)
+    assert results == list(range(42))

tests/test_utils_asyncio.py

@@ -20,11 +20,10 @@ if TYPE_CHECKING:
     from collections.abc import AsyncGenerator
 
 
-class TestAsyncio:
-    @coroutine_test
-    async def test_is_asyncio_available(self, reactor_pytest: str) -> None:
-        # the result should depend only on the pytest --reactor argument
-        assert is_asyncio_available() == (reactor_pytest != "default")
+@coroutine_test
+async def test_is_asyncio_available(reactor_pytest: str) -> None:
+    # the result should depend only on the pytest --reactor argument
+    assert is_asyncio_available() == (reactor_pytest != "default")
 
 
 @pytest.mark.only_asyncio

tests/test_utils_curl.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import warnings
 from typing import Any
 

tests/test_utils_datatypes.py

@@ -1,7 +1,9 @@
+from __future__ import annotations
+
 import copy
 from abc import ABC, abstractmethod
 from collections.abc import Iterator, Mapping, MutableMapping
-from typing import Any
+from typing import Any, Generic, TypeVar
 
 import pytest
 
@@ -16,11 +18,13 @@ from scrapy.utils.datatypes import (
 )
 from scrapy.utils.python import garbage_collect
 
+_DictT = TypeVar("_DictT", bound="CaselessDict | CaseInsensitiveDict")
 
-class TestCaseInsensitiveDictBase(ABC):
+
+class TestCaseInsensitiveDictBase(ABC, Generic[_DictT]):
     @property
     @abstractmethod
-    def dict_class(self) -> type[MutableMapping[str, Any]]:
+    def dict_class(self) -> type[_DictT]:
         raise NotImplementedError
 
     def test_init_dict(self):
@@ -36,17 +40,17 @@ class TestCaseInsensitiveDictBase(ABC):
         assert d["black"] == 3
 
     def test_init_mapping(self):
-        class MyMapping(Mapping):
-            def __init__(self, **kwargs):
+        class MyMapping(Mapping[str, int]):
+            def __init__(self, **kwargs: int) -> None:
                 self._d = kwargs
 
-            def __getitem__(self, key):
+            def __getitem__(self, key: str) -> int:
                 return self._d[key]
 
-            def __iter__(self):
+            def __iter__(self) -> Iterator[str]:
                 return iter(self._d)
 
-            def __len__(self):
+            def __len__(self) -> int:
                 return len(self._d)
 
         seq = MyMapping(red=1, black=3)
@@ -55,23 +59,23 @@ class TestCaseInsensitiveDictBase(ABC):
         assert d["black"] == 3
 
     def test_init_mutable_mapping(self):
-        class MyMutableMapping(MutableMapping):
-            def __init__(self, **kwargs):
+        class MyMutableMapping(MutableMapping[str, int]):
+            def __init__(self, **kwargs: int) -> None:
                 self._d = kwargs
 
-            def __getitem__(self, key):
+            def __getitem__(self, key: str) -> int:
                 return self._d[key]
 
-            def __setitem__(self, key, value):
+            def __setitem__(self, key: str, value: int) -> None:
                 self._d[key] = value
 
-            def __delitem__(self, key):
+            def __delitem__(self, key: str) -> None:
                 del self._d[key]
 
-            def __iter__(self):
+            def __iter__(self) -> Iterator[str]:
                 return iter(self._d)
 
-            def __len__(self):
+            def __len__(self) -> int:
                 return len(self._d)
 
         seq = MyMutableMapping(red=1, black=3)
@@ -149,7 +153,7 @@ class TestCaseInsensitiveDictBase(ABC):
             d.pop("A")
 
     def test_normkey(self):
-        class MyDict(self.dict_class):
+        class MyDict(self.dict_class):  # type: ignore[misc,name-defined]
             def _normkey(self, key):
                 return key.title()
 
@@ -160,7 +164,7 @@ class TestCaseInsensitiveDictBase(ABC):
         assert list(d.keys()) == ["Key-One"]
 
     def test_normvalue(self):
-        class MyDict(self.dict_class):
+        class MyDict(self.dict_class):  # type: ignore[misc,name-defined]
             def _normvalue(self, value):
                 if value is not None:
                     return value + 1
@@ -204,9 +208,18 @@ class TestCaseInsensitiveDictBase(ABC):
         assert h1.get("header1") == h3.get("header1")
         assert h1.get("header1") == h3.get("HEADER1")
 
+    def test_copy_is_independent(self):
+        h1 = self.dict_class({"header1": "value1", "header2": "value2"})
+        for h2 in (copy.copy(h1), h1.copy()):
+            del h2["header1"]
+            h2["header3"] = "value3"
+            assert "header1" in h1
+            assert "header3" not in h1
+            assert dict(h1) == {"header1": "value1", "header2": "value2"}
 
-class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase):
-    dict_class = CaseInsensitiveDict  # type: ignore[assignment]
+
+class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase[CaseInsensitiveDict]):
+    dict_class = CaseInsensitiveDict
 
     def test_repr(self):
         d1 = self.dict_class({"foo": "bar"})
@@ -220,9 +233,31 @@ class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase):
         assert isinstance(iterkeys, Iterator)
         assert list(iterkeys) == ["AsDf", "FoO"]
 
+    def test_copy_keeps_values(self):
+        class MyDict(self.dict_class):  # type: ignore[misc,name-defined]
+            def _normvalue(self, value):
+                return value + 1
+
+        d = MyDict({"key": 1})
+        for copied in (copy.copy(d), d.copy()):
+            assert copied["key"] == 2
+
+    def test_ior(self):
+        d = self.dict_class({"header1": "value1"})
+        d |= {"HEADER1": "value2", "header2": "value3"}
+        assert len(d) == 2
+        assert d["HeAdEr1"] == "value2"
+        assert d["HeAdEr2"] == "value3"
+
+    def test_ior_mapping(self):
+        d = self.dict_class({"header1": "value1"})
+        d |= self.dict_class({"HEADER1": "value2"})
+        assert len(d) == 1
+        assert d["HeAdEr1"] == "value2"
+
 
 @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
-class TestCaselessDict(TestCaseInsensitiveDictBase):
+class TestCaselessDict(TestCaseInsensitiveDictBase[CaselessDict]):
     dict_class = CaselessDict
 
     def test_deprecation_message(self):
@@ -288,7 +323,7 @@ class TestSequenceExclude:
 
 class TestLocalCache:
     def test_cache_with_limit(self):
-        cache = LocalCache(limit=2)
+        cache: LocalCache[str, int] = LocalCache(limit=2)
         cache["a"] = 1
         cache["b"] = 2
         cache["c"] = 3
@@ -301,7 +336,7 @@ class TestLocalCache:
 
     def test_cache_without_limit(self):
         maximum = 10**4
-        cache = LocalCache()
+        cache: LocalCache[str, int] = LocalCache()
         for x in range(maximum):
             cache[str(x)] = x
         assert len(cache) == maximum
@@ -310,7 +345,7 @@ class TestLocalCache:
             assert cache[str(x)] == x
 
     def test_cache_with_zero_limit(self):
-        cache = LocalCache(limit=0)
+        cache: LocalCache[str, int] = LocalCache(limit=0)
         cache["a"] = 1
         cache["b"] = 2
         cache["c"] = 3
@@ -322,7 +357,9 @@ class TestLocalCache:
 
 class TestLocalWeakReferencedCache:
     def test_cache_with_limit(self):
-        cache = LocalWeakReferencedCache(limit=2)
+        cache: LocalWeakReferencedCache[Request, int] = LocalWeakReferencedCache(
+            limit=2
+        )
         r1 = Request("https://example.org")
         r2 = Request("https://example.com")
         r3 = Request("https://example.net")
@@ -344,7 +381,7 @@ class TestLocalWeakReferencedCache:
         assert len(cache) == 1
 
     def test_cache_non_weak_referenceable_objects(self):
-        cache = LocalWeakReferencedCache()
+        cache: LocalWeakReferencedCache[Any, int] = LocalWeakReferencedCache()
         k1 = None
         k2 = 1
         k3 = [1, 2, 3]
@@ -358,7 +395,7 @@ class TestLocalWeakReferencedCache:
 
     def test_cache_without_limit(self):
         maximum = 10**4
-        cache = LocalWeakReferencedCache()
+        cache: LocalWeakReferencedCache[Request, int] = LocalWeakReferencedCache()
         refs = []
         for x in range(maximum):
             refs.append(Request(f"https://example.org/{x}"))

tests/test_utils_decorators.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import warnings
+from typing import TYPE_CHECKING
 
 import pytest
 from twisted.internet.defer import Deferred
@@ -10,11 +11,14 @@ from scrapy.utils.decorators import _warn_spider_arg, deprecated, inthread
 from scrapy.utils.defer import maybe_deferred_to_future
 from tests.utils.decorators import coroutine_test
 
+if TYPE_CHECKING:
+    from collections.abc import AsyncGenerator
+
 
 class TestDeprecated:
     def test_warns_and_still_calls(self):
         @deprecated()
-        def add(a, b):
+        def add(a: int, b: int) -> int:
             return a + b
 
         with pytest.warns(
@@ -26,7 +30,7 @@ class TestDeprecated:
 
     def test_use_instead_in_message(self):
         @deprecated(use_instead="other_function")
-        def old():
+        def old() -> None:
             return None
 
         with pytest.warns(
@@ -37,7 +41,7 @@ class TestDeprecated:
 
     def test_applied_without_parentheses(self):
         @deprecated
-        def square(x):
+        def square(x: int) -> int:
             return x * x
 
         with pytest.warns(
@@ -65,7 +69,7 @@ class TestInthread:
 class TestWarnSpiderArg:
     def test_sync_warns_with_spider_arg(self):
         @_warn_spider_arg
-        def parse(response, spider=None):
+        def parse(response: str, spider: str | None = None) -> str:
             return response
 
         with pytest.warns(
@@ -75,7 +79,7 @@ class TestWarnSpiderArg:
 
     def test_sync_no_warning_without_spider_arg(self):
         @_warn_spider_arg
-        def parse(response, spider=None):
+        def parse(response: str, spider: str | None = None) -> str:
             return response
 
         with warnings.catch_warnings():
@@ -85,7 +89,7 @@ class TestWarnSpiderArg:
     @coroutine_test
     async def test_async_warns_with_spider_arg(self):
         @_warn_spider_arg
-        async def parse(response, spider=None):
+        async def parse(response: str, spider: str | None = None) -> str:
             return response
 
         with pytest.warns(
@@ -96,7 +100,9 @@ class TestWarnSpiderArg:
     @coroutine_test
     async def test_asyncgen_warns_with_spider_arg(self):
         @_warn_spider_arg
-        async def parse(response, spider=None):
+        async def parse(
+            response: str, spider: str | None = None
+        ) -> AsyncGenerator[str]:
             yield response
 
         with pytest.warns(

tests/test_utils_defer.py

@@ -24,6 +24,8 @@ from tests.utils.decorators import coroutine_test, inline_callbacks_test
 if TYPE_CHECKING:
     from collections.abc import AsyncGenerator, Awaitable, Callable, Generator
 
+    from twisted.python.failure import Failure
+
 
 @pytest.mark.requires_reactor  # mustbe_deferred() requires a reactor
 @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
@@ -70,7 +72,7 @@ class TestIterErrback:
         def itergood() -> Generator[int, None, None]:
             yield from range(10)
 
-        errors = []
+        errors: list[Failure] = []
         out = list(iter_errback(itergood(), errors.append))
         assert out == list(range(10))
         assert not errors
@@ -82,7 +84,7 @@ class TestIterErrback:
                     1 / 0
                 yield x
 
-        errors = []
+        errors: list[Failure] = []
         out = list(iter_errback(iterbad(), errors.append))
         assert out == [0, 1, 2, 3, 4]
         assert len(errors) == 1
@@ -96,7 +98,7 @@ class TestAiterErrback:
             for x in range(10):
                 yield x
 
-        errors = []
+        errors: list[Failure] = []
         out = await collect_asyncgen(aiter_errback(itergood(), errors.append))
         assert out == list(range(10))
         assert not errors
@@ -109,7 +111,7 @@ class TestAiterErrback:
                     1 / 0
                 yield x
 
-        errors = []
+        errors: list[Failure] = []
         out = await collect_asyncgen(aiter_errback(iterbad(), errors.append))
         assert out == [0, 1, 2, 3, 4]
         assert len(errors) == 1
@@ -202,7 +204,7 @@ class TestParallelAsync:
         for length in [20, 50, 100]:
             parallel_count = [0]
             max_parallel_count = [0]
-            results = []
+            results: list[int] = []
             ait = self.get_async_iterable(length)
             dl = parallel_async(
                 ait,
@@ -222,7 +224,7 @@ class TestParallelAsync:
         for length in [20, 50, 100]:
             parallel_count = [0]
             max_parallel_count = [0]
-            results = []
+            results: list[int] = []
             ait = self.get_async_iterable_with_delays(length)
             dl = parallel_async(
                 ait,
@@ -240,7 +242,7 @@ class TestParallelAsync:
 
 class TestDeferredFromCoro:
     def test_deferred(self):
-        d = Deferred()
+        d: Deferred[None] = Deferred()
         result = deferred_from_coro(d)
         assert isinstance(result, Deferred)
         assert result is d
@@ -274,7 +276,7 @@ class TestDeferredFromCoro:
     @pytest.mark.only_asyncio
     @inline_callbacks_test
     def test_future(self):
-        future = Future()
+        future: Future[int] = Future()
         result = deferred_from_coro(future)
         assert isinstance(result, Deferred)
         future.set_result(42)
@@ -324,7 +326,7 @@ class TestDeferredFFromCoroF:
 class TestDeferredToFuture:
     @coroutine_test
     async def test_deferred(self):
-        d = Deferred()
+        d: Deferred[int] = Deferred()
         result = deferred_to_future(d)
         assert isinstance(result, Future)
         d.callback(42)
@@ -359,7 +361,7 @@ class TestDeferredToFuture:
 class TestMaybeDeferredToFutureAsyncio:
     @coroutine_test
     async def test_deferred(self):
-        d = Deferred()
+        d: Deferred[int] = Deferred()
         result = maybe_deferred_to_future(d)
         assert isinstance(result, Future)
         d.callback(42)
@@ -394,7 +396,7 @@ class TestMaybeDeferredToFutureAsyncio:
 class TestMaybeDeferredToFutureNotAsyncio:
     @coroutine_test
     async def test_deferred(self):
-        d = Deferred()
+        d: Deferred[int] = Deferred()
         result = maybe_deferred_to_future(d)
         assert isinstance(result, Deferred)
         assert result is d

tests/test_utils_deprecate.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import inspect
 import warnings
 from unittest import mock
@@ -38,7 +40,7 @@ class TestWarnWhenSubclassed:
         )
         with pytest.warns(MyWarning, match=msg) as w:
 
-            class UserClass(Deprecated):
+            class UserClass(Deprecated):  # type: ignore[misc, valid-type]
                 pass
 
         assert w[0].lineno == inspect.getsourcelines(UserClass)[1]
@@ -57,7 +59,7 @@ class TestWarnWhenSubclassed:
             match=r"UserClass inherits from deprecated class bar\.OldClass, please inherit from foo\.NewClass",
         ):
 
-            class UserClass(Deprecated):
+            class UserClass(Deprecated):  # type: ignore[misc, valid-type]
                 pass
 
         with pytest.warns(
@@ -76,7 +78,7 @@ class TestWarnWhenSubclassed:
             match="UserClass inherits from deprecated class",
         ):
 
-            class UserClass(Deprecated):
+            class UserClass(Deprecated):  # type: ignore[misc, valid-type]
                 pass
 
         with warnings.catch_warnings():
@@ -95,16 +97,16 @@ class TestWarnWhenSubclassed:
             match="UserClass inherits from deprecated class",
         ):
 
-            class UserClass(Deprecated):
+            class UserClass(Deprecated):  # type: ignore[misc, valid-type]
                 pass
 
         with warnings.catch_warnings():
             warnings.simplefilter("error", MyWarning)
 
-            class FooClass(Deprecated):
+            class FooClass(Deprecated):  # type: ignore[misc, valid-type]
                 pass
 
-            class BarClass(Deprecated):
+            class BarClass(Deprecated):  # type: ignore[misc, valid-type]
                 pass
 
     def test_warning_on_instance(self):
@@ -112,22 +114,20 @@ class TestWarnWhenSubclassed:
             "Deprecated", NewName, warn_category=MyWarning
         )
 
-        with pytest.warns(MyWarning) as w:
-            _, lineno = Deprecated(), inspect.getlineno(inspect.currentframe())
-
-        w = [x for x in w if x.category is MyWarning]
+        with pytest.warns(
+            MyWarning,
+            match=r"tests\.test_utils_deprecate\.Deprecated is deprecated, "
+            r"instantiate tests\.test_utils_deprecate\.NewName instead\.",
+        ) as w:
+            _, lineno = Deprecated(), inspect.getlineno(inspect.currentframe())  # type: ignore[arg-type]
         assert len(w) == 1
-        assert (
-            str(w[0].message) == "tests.test_utils_deprecate.Deprecated is deprecated, "
-            "instantiate tests.test_utils_deprecate.NewName instead."
-        )
         assert w[0].lineno == lineno
 
         # ignore subclassing warnings
         with warnings.catch_warnings():
             warnings.simplefilter("ignore", MyWarning)
 
-            class UserClass(Deprecated):
+            class UserClass(Deprecated):  # type: ignore[misc, valid-type]
                 pass
 
         with warnings.catch_warnings():
@@ -141,7 +141,7 @@ class TestWarnWhenSubclassed:
             match=r"UserClass2 inherits from deprecated class tests\.test_utils_deprecate\.Deprecated, please inherit from tests\.test_utils_deprecate\.NewName",
         ):
 
-            class UserClass2(Deprecated):
+            class UserClass2(Deprecated):  # type: ignore[misc, valid-type]
                 pass
 
     def test_issubclass(self):
@@ -155,10 +155,10 @@ class TestWarnWhenSubclassed:
             class UpdatedUserClass1a(NewName):
                 pass
 
-            class OutdatedUserClass1(DeprecatedName):
+            class OutdatedUserClass1(DeprecatedName):  # type: ignore[misc, valid-type]
                 pass
 
-            class OutdatedUserClass1a(DeprecatedName):
+            class OutdatedUserClass1a(DeprecatedName):  # type: ignore[misc, valid-type]
                 pass
 
             class UnrelatedClass:
@@ -174,7 +174,7 @@ class TestWarnWhenSubclassed:
         assert not issubclass(OutdatedUserClass1a, OutdatedUserClass1)
 
         with pytest.raises(TypeError):
-            issubclass(object(), DeprecatedName)
+            issubclass(object(), DeprecatedName)  # type: ignore[arg-type]
 
     def test_isinstance(self):
         with warnings.catch_warnings():
@@ -187,10 +187,10 @@ class TestWarnWhenSubclassed:
             class UpdatedUserClass2a(NewName):
                 pass
 
-            class OutdatedUserClass2(DeprecatedName):
+            class OutdatedUserClass2(DeprecatedName):  # type: ignore[misc, valid-type]
                 pass
 
-            class OutdatedUserClass2a(DeprecatedName):
+            class OutdatedUserClass2a(DeprecatedName):  # type: ignore[misc, valid-type]
                 pass
 
             class UnrelatedClass:
@@ -211,7 +211,7 @@ class TestWarnWhenSubclassed:
             warnings.simplefilter("ignore", ScrapyDeprecationWarning)
             Deprecated = create_deprecated_class("Deprecated", NewName, {"foo": "bar"})
 
-        assert Deprecated.foo == "bar"
+        assert Deprecated.foo == "bar"  # type: ignore[attr-defined]
 
     def test_deprecate_a_class_with_custom_metaclass(self):
         Meta1 = type("Meta1", (type,), {})
@@ -242,7 +242,7 @@ class TestWarnWhenSubclassed:
             match=r"UserClass inherits from deprecated class tests\.test_utils_deprecate\.AlsoDeprecated, please inherit from foo\.Bar",
         ):
 
-            class UserClass(AlsoDeprecated):
+            class UserClass(AlsoDeprecated):  # type: ignore[misc, valid-type]
                 pass
 
     def test_inspect_stack(self):

tests/test_utils_display.py

@@ -31,13 +31,13 @@ plain_string = "{'a': 1}"
 
 @mock.patch("sys.platform", "linux")
 @mock.patch("sys.stdout.isatty")
-def test_pformat(isatty):
+def test_pformat(isatty: mock.Mock) -> None:
     isatty.return_value = True
     assert pformat(value) in colorized_strings
 
 
 @mock.patch("sys.stdout.isatty")
-def test_pformat_dont_colorize(isatty):
+def test_pformat_dont_colorize(isatty: mock.Mock) -> None:
     isatty.return_value = True
     assert pformat(value, colorize=False) == plain_string
 
@@ -49,7 +49,7 @@ def test_pformat_not_tty():
 @mock.patch("sys.platform", "win32")
 @mock.patch("platform.version")
 @mock.patch("sys.stdout.isatty")
-def test_pformat_old_windows(isatty, version):
+def test_pformat_old_windows(isatty: mock.Mock, version: mock.Mock) -> None:
     isatty.return_value = True
     version.return_value = "10.0.14392"
     assert pformat(value) in colorized_strings
@@ -59,7 +59,9 @@ def test_pformat_old_windows(isatty, version):
 @mock.patch("scrapy.utils.display._enable_windows_terminal_processing")
 @mock.patch("platform.version")
 @mock.patch("sys.stdout.isatty")
-def test_pformat_windows_no_terminal_processing(isatty, version, terminal_processing):
+def test_pformat_windows_no_terminal_processing(
+    isatty: mock.Mock, version: mock.Mock, terminal_processing: mock.Mock
+) -> None:
     isatty.return_value = True
     version.return_value = "10.0.14393"
     terminal_processing.return_value = False
@@ -70,7 +72,9 @@ def test_pformat_windows_no_terminal_processing(isatty, version, terminal_proces
 @mock.patch("scrapy.utils.display._enable_windows_terminal_processing")
 @mock.patch("platform.version")
 @mock.patch("sys.stdout.isatty")
-def test_pformat_windows(isatty, version, terminal_processing):
+def test_pformat_windows(
+    isatty: mock.Mock, version: mock.Mock, terminal_processing: mock.Mock
+) -> None:
     isatty.return_value = True
     version.return_value = "10.0.14393"
     terminal_processing.return_value = True
@@ -79,7 +83,7 @@ def test_pformat_windows(isatty, version, terminal_processing):
 
 @mock.patch("sys.platform", "linux")
 @mock.patch("sys.stdout.isatty")
-def test_pformat_no_pygments(isatty):
+def test_pformat_no_pygments(isatty: mock.Mock) -> None:
     isatty.return_value = True
 
     real_import = builtins.__import__

tests/test_utils_gz.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from gzip import BadGzipFile
 from pathlib import Path
 

tests/test_utils_httpobj.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from urllib.parse import urlparse
 
 from scrapy.http import Request

tests/test_utils_misc/__init__.py

@@ -20,150 +20,157 @@ from scrapy.utils.misc import (
 )
 
 
-class TestUtilsMisc:
-    def test_load_object_class(self):
-        obj = load_object(Field)
-        assert obj is Field
-        obj = load_object("scrapy.item.Field")
-        assert obj is Field
+def test_load_object_class() -> None:
+    obj = load_object(Field)
+    assert obj is Field
+    obj = load_object("scrapy.item.Field")
+    assert obj is Field
 
-    def test_load_object_function(self):
-        obj = load_object(load_object)
-        assert obj is load_object
-        obj = load_object("scrapy.utils.misc.load_object")
-        assert obj is load_object
 
-    def test_load_object_exceptions(self):
-        with pytest.raises(ImportError):
-            load_object("nomodule999.mod.function")
-        with pytest.raises(NameError):
-            load_object("scrapy.utils.misc.load_object999")
-        with pytest.raises(TypeError):
-            load_object({})  # type: ignore[arg-type]
+def test_load_object_function() -> None:
+    obj = load_object(load_object)
+    assert obj is load_object
+    obj = load_object("scrapy.utils.misc.load_object")
+    assert obj is load_object
 
-    def test_walk_modules(self):
-        mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules")
+
+def test_load_object_exceptions() -> None:
+    with pytest.raises(ImportError):
+        load_object("nomodule999.mod.function")
+    with pytest.raises(NameError):
+        load_object("scrapy.utils.misc.load_object999")
+    with pytest.raises(TypeError):
+        load_object({})  # type: ignore[arg-type]
+
+
+def test_walk_modules() -> None:
+    mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules")
+    expected = [
+        "tests.test_utils_misc.test_walk_modules",
+        "tests.test_utils_misc.test_walk_modules.mod",
+        "tests.test_utils_misc.test_walk_modules.mod.mod0",
+        "tests.test_utils_misc.test_walk_modules.mod1",
+    ]
+    assert {m.__name__ for m in mods} == set(expected)
+
+    mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules.mod")
+    expected = [
+        "tests.test_utils_misc.test_walk_modules.mod",
+        "tests.test_utils_misc.test_walk_modules.mod.mod0",
+    ]
+    assert {m.__name__ for m in mods} == set(expected)
+
+    mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules.mod1")
+    expected = [
+        "tests.test_utils_misc.test_walk_modules.mod1",
+    ]
+    assert {m.__name__ for m in mods} == set(expected)
+
+    with pytest.raises(ImportError):
+        for _ in walk_modules_iter("nomodule999"):
+            pass
+    with (
+        pytest.raises(ImportError),
+        pytest.warns(
+            ScrapyDeprecationWarning,
+            match="The scrapy.utils.misc.walk_modules function is deprecated and will be "
+            "removed in a future version of Scrapy. "
+            "Use scrapy.utils.misc.walk_modules_iter instead.",
+        ),
+    ):
+        walk_modules("nomodule999")
+
+
+def test_walk_modules_egg() -> None:
+    egg = str(Path(__file__).parent / "test.egg")
+    sys.path.append(egg)
+    try:
+        mods = walk_modules_iter("testegg")
         expected = [
-            "tests.test_utils_misc.test_walk_modules",
-            "tests.test_utils_misc.test_walk_modules.mod",
-            "tests.test_utils_misc.test_walk_modules.mod.mod0",
-            "tests.test_utils_misc.test_walk_modules.mod1",
+            "testegg.spiders",
+            "testegg.spiders.a",
+            "testegg.spiders.b",
+            "testegg",
         ]
         assert {m.__name__ for m in mods} == set(expected)
+    finally:
+        sys.path.remove(egg)
 
-        mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules.mod")
-        expected = [
-            "tests.test_utils_misc.test_walk_modules.mod",
-            "tests.test_utils_misc.test_walk_modules.mod.mod0",
-        ]
-        assert {m.__name__ for m in mods} == set(expected)
 
-        mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules.mod1")
-        expected = [
-            "tests.test_utils_misc.test_walk_modules.mod1",
-        ]
-        assert {m.__name__ for m in mods} == set(expected)
+def test_arg_to_iter() -> None:
+    class TestItem(Item):
+        name = Field()
 
-        with pytest.raises(ImportError):
-            for _ in walk_modules_iter("nomodule999"):
-                pass
-        with (
-            pytest.raises(ImportError),
-            pytest.warns(
-                ScrapyDeprecationWarning,
-                match="The scrapy.utils.misc.walk_modules function is deprecated and will be "
-                "removed in a future version of Scrapy. "
-                "Use scrapy.utils.misc.walk_modules_iter instead.",
-            ),
-        ):
-            walk_modules("nomodule999")
+    assert hasattr(arg_to_iter(None), "__iter__")
+    assert hasattr(arg_to_iter(100), "__iter__")
+    assert hasattr(arg_to_iter("lala"), "__iter__")
+    assert hasattr(arg_to_iter([1, 2, 3]), "__iter__")
+    assert hasattr(arg_to_iter(c for c in "abcd"), "__iter__")
 
-    def test_walk_modules_egg(self):
-        egg = str(Path(__file__).parent / "test.egg")
-        sys.path.append(egg)
-        try:
-            mods = walk_modules_iter("testegg")
-            expected = [
-                "testegg.spiders",
-                "testegg.spiders.a",
-                "testegg.spiders.b",
-                "testegg",
-            ]
-            assert {m.__name__ for m in mods} == set(expected)
-        finally:
-            sys.path.remove(egg)
+    assert not list(arg_to_iter(None))
+    assert list(arg_to_iter("lala")) == ["lala"]
+    assert list(arg_to_iter(100)) == [100]
+    assert list(arg_to_iter(c for c in "abc")) == ["a", "b", "c"]
+    assert list(arg_to_iter([1, 2, 3])) == [1, 2, 3]
+    assert list(arg_to_iter({"a": 1})) == [{"a": 1}]
+    assert list(arg_to_iter(TestItem(name="john"))) == [TestItem(name="john")]
 
-    def test_arg_to_iter(self):
-        class TestItem(Item):
-            name = Field()
 
-        assert hasattr(arg_to_iter(None), "__iter__")
-        assert hasattr(arg_to_iter(100), "__iter__")
-        assert hasattr(arg_to_iter("lala"), "__iter__")
-        assert hasattr(arg_to_iter([1, 2, 3]), "__iter__")
-        assert hasattr(arg_to_iter(c for c in "abcd"), "__iter__")
+def test_build_from_crawler() -> None:
+    crawler = mock.MagicMock(spec_set=["settings"])
+    args = (True, 100.0)
+    kwargs = {"key": "val"}
 
-        assert not list(arg_to_iter(None))
-        assert list(arg_to_iter("lala")) == ["lala"]
-        assert list(arg_to_iter(100)) == [100]
-        assert list(arg_to_iter(c for c in "abc")) == ["a", "b", "c"]
-        assert list(arg_to_iter([1, 2, 3])) == [1, 2, 3]
-        assert list(arg_to_iter({"a": 1})) == [{"a": 1}]
-        assert list(arg_to_iter(TestItem(name="john"))) == [TestItem(name="john")]
+    def _test_with_crawler(mock: mock.MagicMock, crawler: mock.MagicMock) -> None:
+        build_from_crawler(mock, crawler, *args, **kwargs)
+        if hasattr(mock, "from_crawler"):
+            mock.from_crawler.assert_called_once_with(crawler, *args, **kwargs)
+            assert mock.call_count == 0
+        else:
+            mock.assert_called_once_with(*args, **kwargs)
 
-    def test_build_from_crawler(self):
-        crawler = mock.MagicMock(spec_set=["settings"])
-        args = (True, 100.0)
-        kwargs = {"key": "val"}
+    # Check usage of correct constructor using 2 mocks:
+    #   1. with no alternative constructors
+    #   2. with from_crawler() constructor
+    spec_sets = (
+        ["__qualname__"],
+        ["__qualname__", "from_crawler"],
+    )
+    for specs in spec_sets:
+        m = mock.MagicMock(spec_set=specs)
+        _test_with_crawler(m, crawler)
+        m.reset_mock()
 
-        def _test_with_crawler(mock: mock.MagicMock, crawler: mock.MagicMock) -> None:
-            build_from_crawler(mock, crawler, *args, **kwargs)
-            if hasattr(mock, "from_crawler"):
-                mock.from_crawler.assert_called_once_with(crawler, *args, **kwargs)
-                assert mock.call_count == 0
-            else:
-                mock.assert_called_once_with(*args, **kwargs)
+    # Check adoption of crawler
+    m = mock.MagicMock(spec_set=["__qualname__", "from_crawler"])
+    m.from_crawler.return_value = None
+    with pytest.raises(TypeError):
+        build_from_crawler(m, crawler, *args, **kwargs)
 
-        # Check usage of correct constructor using 2 mocks:
-        #   1. with no alternative constructors
-        #   2. with from_crawler() constructor
-        spec_sets = (
-            ["__qualname__"],
-            ["__qualname__", "from_crawler"],
-        )
-        for specs in spec_sets:
-            m = mock.MagicMock(spec_set=specs)
-            _test_with_crawler(m, crawler)
-            m.reset_mock()
 
-        # Check adoption of crawler
-        m = mock.MagicMock(spec_set=["__qualname__", "from_crawler"])
-        m.from_crawler.return_value = None
-        with pytest.raises(TypeError):
-            build_from_crawler(m, crawler, *args, **kwargs)
+def test_set_environ() -> None:
+    assert os.environ.get("some_test_environ") is None
+    with set_environ(some_test_environ="test_value"):
+        assert os.environ.get("some_test_environ") == "test_value"
+    assert os.environ.get("some_test_environ") is None
 
-    def test_set_environ(self):
-        assert os.environ.get("some_test_environ") is None
-        with set_environ(some_test_environ="test_value"):
-            assert os.environ.get("some_test_environ") == "test_value"
-        assert os.environ.get("some_test_environ") is None
+    os.environ["some_test_environ"] = "test"
+    assert os.environ.get("some_test_environ") == "test"
+    with set_environ(some_test_environ="test_value"):
+        assert os.environ.get("some_test_environ") == "test_value"
+    assert os.environ.get("some_test_environ") == "test"
 
-        os.environ["some_test_environ"] = "test"
-        assert os.environ.get("some_test_environ") == "test"
-        with set_environ(some_test_environ="test_value"):
-            assert os.environ.get("some_test_environ") == "test_value"
-        assert os.environ.get("some_test_environ") == "test"
 
-    def test_rel_has_nofollow(self):
-        assert rel_has_nofollow("ugc nofollow") is True
-        assert rel_has_nofollow("ugc,nofollow") is True
-        assert rel_has_nofollow("ugc") is False
-        assert rel_has_nofollow("nofollow") is True
-        assert rel_has_nofollow("nofollowfoo") is False
-        assert rel_has_nofollow("foonofollow") is False
-        assert rel_has_nofollow("ugc,  ,  nofollow") is True
-        # rel attribute values are ASCII case-insensitive per the HTML spec
-        assert rel_has_nofollow("NoFollow") is True
-        assert rel_has_nofollow("NOFOLLOW") is True
-        assert rel_has_nofollow("UGC NoFollow") is True
-        assert rel_has_nofollow("ugc,NoFollow") is True
+def test_rel_has_nofollow() -> None:
+    assert rel_has_nofollow("ugc nofollow") is True
+    assert rel_has_nofollow("ugc,nofollow") is True
+    assert rel_has_nofollow("ugc") is False
+    assert rel_has_nofollow("nofollow") is True
+    assert rel_has_nofollow("nofollowfoo") is False
+    assert rel_has_nofollow("foonofollow") is False
+    assert rel_has_nofollow("ugc,  ,  nofollow") is True
+    # rel attribute values are ASCII case-insensitive per the HTML spec
+    assert rel_has_nofollow("NoFollow") is True
+    assert rel_has_nofollow("NOFOLLOW") is True
+    assert rel_has_nofollow("UGC NoFollow") is True
+    assert rel_has_nofollow("ugc,NoFollow") is True

tests/test_utils_misc/test_return_with_argument_inside_generator.py

@@ -1,5 +1,8 @@
+from __future__ import annotations
+
 import warnings
 from functools import partial
+from typing import TYPE_CHECKING, Any
 from unittest import mock
 
 import pytest
@@ -9,6 +12,11 @@ from scrapy.utils.misc import (
     warn_on_generator_with_return_value,
 )
 
+if TYPE_CHECKING:
+    from collections.abc import Generator
+
+    from scrapy import Spider
+
 
 def _indentation_error(*args, **kwargs):
     raise IndentationError
@@ -35,244 +43,239 @@ https://example.org
     yield url
 
 
-def generator_that_returns_stuff():
+def generator_that_returns_stuff() -> Generator[int, None, int]:
     yield 1
     yield 2
     return 3
 
 
-class TestUtilsMisc:
-    @pytest.fixture
-    def mock_spider(self):
-        class MockSettings:
-            def __init__(self, settings_dict=None):
-                self.settings_dict = settings_dict or {
-                    "WARN_ON_GENERATOR_RETURN_VALUE": True
-                }
+@pytest.fixture
+def mock_spider() -> Spider:
+    class MockSettings:
+        def __init__(self, settings_dict: dict[str, Any] | None = None):
+            self.settings_dict = settings_dict or {
+                "WARN_ON_GENERATOR_RETURN_VALUE": True
+            }
 
-            def getbool(self, name, default=False):
-                return self.settings_dict.get(name, default)
+        def getbool(self, name, default=False):
+            return self.settings_dict.get(name, default)
 
-        class MockSpider:
-            def __init__(self):
-                self.settings = MockSettings()
+    class MockSpider:
+        def __init__(self) -> None:
+            self.settings = MockSettings()
 
-        return MockSpider()
+    return MockSpider()  # type: ignore[return-value]
 
-    def test_generators_return_something(self, mock_spider):
-        def f1():
-            yield 1
-            return 2
 
-        def g1():
-            yield 1
-            return "asdf"
+def test_generators_return_something(mock_spider):
+    def f1():
+        yield 1
+        return 2
 
-        def h1():
-            yield 1
+    def g1():
+        yield 1
+        return "asdf"
 
-            def helper():
-                return 0
+    def h1():
+        yield 1
 
-            yield helper()
-            return 2
+        def helper() -> int:
+            return 0
 
-        def i1():
-            """
-            docstring
-            """
-            url = """
-https://example.org
+        yield helper()
+        return 2
+
+    def i1():
         """
-            yield url
-            return 1
-
-        assert is_generator_with_return_value(top_level_return_something)
-        assert is_generator_with_return_value(f1)
-        assert is_generator_with_return_value(g1)
-        assert is_generator_with_return_value(h1)
-        assert is_generator_with_return_value(i1)
-
-        with pytest.warns(
-            UserWarning,
-            match='The "MockSpider.top_level_return_something" method is a generator',
-        ):
-            warn_on_generator_with_return_value(mock_spider, top_level_return_something)
-        with pytest.warns(
-            UserWarning, match='The "MockSpider.f1" method is a generator'
-        ):
-            warn_on_generator_with_return_value(mock_spider, f1)
-        with pytest.warns(
-            UserWarning, match='The "MockSpider.g1" method is a generator'
-        ):
-            warn_on_generator_with_return_value(mock_spider, g1)
-        with pytest.warns(
-            UserWarning, match='The "MockSpider.h1" method is a generator'
-        ):
-            warn_on_generator_with_return_value(mock_spider, h1)
-        with pytest.warns(
-            UserWarning, match='The "MockSpider.i1" method is a generator'
-        ):
-            warn_on_generator_with_return_value(mock_spider, i1)
-
-    def test_generators_return_none(self, mock_spider):
-        def f2():
-            yield 1
-
-        def g2():
-            yield 1
-
-        def h2():
-            yield 1
-
-        def i2():
-            yield 1
-            yield from generator_that_returns_stuff()
-
-        def j2():
-            yield 1
-
-            def helper():
-                return 0
-
-            yield helper()
-
-        def k2():
-            """
-            docstring
-            """
-            url = """
-https://example.org
+        docstring
         """
-            yield url
-
-        def l2():
-            return
-
-        assert not is_generator_with_return_value(top_level_return_none)
-        assert not is_generator_with_return_value(f2)
-        assert not is_generator_with_return_value(g2)
-        assert not is_generator_with_return_value(h2)
-        assert not is_generator_with_return_value(i2)
-        assert not is_generator_with_return_value(j2)  # not recursive
-        assert not is_generator_with_return_value(k2)  # not recursive
-        assert not is_generator_with_return_value(l2)
-
-        with warnings.catch_warnings():
-            warnings.simplefilter("error", UserWarning)
-            warn_on_generator_with_return_value(mock_spider, top_level_return_none)
-            warn_on_generator_with_return_value(mock_spider, f2)
-            warn_on_generator_with_return_value(mock_spider, g2)
-            warn_on_generator_with_return_value(mock_spider, h2)
-            warn_on_generator_with_return_value(mock_spider, i2)
-            warn_on_generator_with_return_value(mock_spider, j2)
-            warn_on_generator_with_return_value(mock_spider, k2)
-            warn_on_generator_with_return_value(mock_spider, l2)
-
-    def test_generators_return_none_with_decorator(self, mock_spider):
-        def decorator(func):
-            def inner_func():
-                func()
-
-            return inner_func
-
-        @decorator
-        def f3():
-            yield 1
-
-        @decorator
-        def g3():
-            yield 1
-
-        @decorator
-        def h3():
-            yield 1
-
-        @decorator
-        def i3():
-            yield 1
-            yield from generator_that_returns_stuff()
-
-        @decorator
-        def j3():
-            yield 1
-
-            def helper():
-                return 0
-
-            yield helper()
-
-        @decorator
-        def k3():
-            """
-            docstring
-            """
-            url = """
+        url = """
 https://example.org
+    """
+        yield url
+        return 1
+
+    assert is_generator_with_return_value(top_level_return_something)
+    assert is_generator_with_return_value(f1)
+    assert is_generator_with_return_value(g1)
+    assert is_generator_with_return_value(h1)
+    assert is_generator_with_return_value(i1)
+
+    with pytest.warns(
+        UserWarning,
+        match='The "MockSpider.top_level_return_something" method is a generator',
+    ):
+        warn_on_generator_with_return_value(mock_spider, top_level_return_something)
+    with pytest.warns(UserWarning, match='The "MockSpider.f1" method is a generator'):
+        warn_on_generator_with_return_value(mock_spider, f1)
+    with pytest.warns(UserWarning, match='The "MockSpider.g1" method is a generator'):
+        warn_on_generator_with_return_value(mock_spider, g1)
+    with pytest.warns(UserWarning, match='The "MockSpider.h1" method is a generator'):
+        warn_on_generator_with_return_value(mock_spider, h1)
+    with pytest.warns(UserWarning, match='The "MockSpider.i1" method is a generator'):
+        warn_on_generator_with_return_value(mock_spider, i1)
+
+
+def test_generators_return_none(mock_spider):
+    def f2():
+        yield 1
+
+    def g2():
+        yield 1
+
+    def h2():
+        yield 1
+
+    def i2():
+        yield 1
+        yield from generator_that_returns_stuff()
+
+    def j2():
+        yield 1
+
+        def helper() -> int:
+            return 0
+
+        yield helper()
+
+    def k2():
         """
-            yield url
+        docstring
+        """
+        url = """
+https://example.org
+    """
+        yield url
 
-        @decorator
-        def l3():
-            return
+    def l2():
+        return
 
-        assert not is_generator_with_return_value(top_level_return_none)
-        assert not is_generator_with_return_value(f3)
-        assert not is_generator_with_return_value(g3)
-        assert not is_generator_with_return_value(h3)
-        assert not is_generator_with_return_value(i3)
-        assert not is_generator_with_return_value(j3)  # not recursive
-        assert not is_generator_with_return_value(k3)  # not recursive
-        assert not is_generator_with_return_value(l3)
+    assert not is_generator_with_return_value(top_level_return_none)
+    assert not is_generator_with_return_value(f2)
+    assert not is_generator_with_return_value(g2)
+    assert not is_generator_with_return_value(h2)
+    assert not is_generator_with_return_value(i2)
+    assert not is_generator_with_return_value(j2)  # not recursive
+    assert not is_generator_with_return_value(k2)  # not recursive
+    assert not is_generator_with_return_value(l2)
 
-        with warnings.catch_warnings():
-            warnings.simplefilter("error", UserWarning)
-            warn_on_generator_with_return_value(mock_spider, top_level_return_none)
-            warn_on_generator_with_return_value(mock_spider, f3)
-            warn_on_generator_with_return_value(mock_spider, g3)
-            warn_on_generator_with_return_value(mock_spider, h3)
-            warn_on_generator_with_return_value(mock_spider, i3)
-            warn_on_generator_with_return_value(mock_spider, j3)
-            warn_on_generator_with_return_value(mock_spider, k3)
-            warn_on_generator_with_return_value(mock_spider, l3)
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", UserWarning)
+        warn_on_generator_with_return_value(mock_spider, top_level_return_none)
+        warn_on_generator_with_return_value(mock_spider, f2)
+        warn_on_generator_with_return_value(mock_spider, g2)
+        warn_on_generator_with_return_value(mock_spider, h2)
+        warn_on_generator_with_return_value(mock_spider, i2)
+        warn_on_generator_with_return_value(mock_spider, j2)
+        warn_on_generator_with_return_value(mock_spider, k2)
+        warn_on_generator_with_return_value(mock_spider, l2)
 
-    @mock.patch(
-        "scrapy.utils.misc.is_generator_with_return_value", new=_indentation_error
-    )
-    def test_indentation_error(self, mock_spider):
-        with pytest.warns(UserWarning, match="Unable to determine"):
-            warn_on_generator_with_return_value(mock_spider, top_level_return_none)
 
-    def test_partial(self):
-        def cb(arg1, arg2):
-            yield {}
+def test_generators_return_none_with_decorator(mock_spider):
+    def decorator(func):
+        def inner_func():
+            func()
 
-        partial_cb = partial(cb, arg1=42)
-        assert not is_generator_with_return_value(partial_cb)
+        return inner_func
 
-    def test_warn_on_generator_with_return_value_settings_disabled(self):
-        class MockSettings:
-            def __init__(self, settings_dict=None):
-                self.settings_dict = settings_dict or {}
+    @decorator
+    def f3():
+        yield 1
 
-            def getbool(self, name, default=False):
-                return self.settings_dict.get(name, default)
+    @decorator
+    def g3():
+        yield 1
 
-        class MockSpider:
-            def __init__(self):
-                self.settings = MockSettings({"WARN_ON_GENERATOR_RETURN_VALUE": False})
+    @decorator
+    def h3():
+        yield 1
 
-        spider = MockSpider()
+    @decorator
+    def i3():
+        yield 1
+        yield from generator_that_returns_stuff()
 
-        def gen_with_return():
-            yield 1
-            return "value"
+    @decorator
+    def j3():
+        yield 1
 
-        with warnings.catch_warnings():
-            warnings.simplefilter("error", UserWarning)
-            warn_on_generator_with_return_value(spider, gen_with_return)
+        def helper() -> int:
+            return 0
 
-        spider.settings.settings_dict["WARN_ON_GENERATOR_RETURN_VALUE"] = True
+        yield helper()
 
-        with pytest.warns(UserWarning, match="is a generator"):
-            warn_on_generator_with_return_value(spider, gen_with_return)
+    @decorator
+    def k3():
+        """
+        docstring
+        """
+        url = """
+https://example.org
+    """
+        yield url
+
+    @decorator
+    def l3():
+        return
+
+    assert not is_generator_with_return_value(top_level_return_none)
+    assert not is_generator_with_return_value(f3)
+    assert not is_generator_with_return_value(g3)
+    assert not is_generator_with_return_value(h3)
+    assert not is_generator_with_return_value(i3)
+    assert not is_generator_with_return_value(j3)  # not recursive
+    assert not is_generator_with_return_value(k3)  # not recursive
+    assert not is_generator_with_return_value(l3)
+
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", UserWarning)
+        warn_on_generator_with_return_value(mock_spider, top_level_return_none)
+        warn_on_generator_with_return_value(mock_spider, f3)
+        warn_on_generator_with_return_value(mock_spider, g3)
+        warn_on_generator_with_return_value(mock_spider, h3)
+        warn_on_generator_with_return_value(mock_spider, i3)
+        warn_on_generator_with_return_value(mock_spider, j3)
+        warn_on_generator_with_return_value(mock_spider, k3)
+        warn_on_generator_with_return_value(mock_spider, l3)
+
+
+@mock.patch("scrapy.utils.misc.is_generator_with_return_value", new=_indentation_error)
+def test_indentation_error(mock_spider):
+    with pytest.warns(UserWarning, match="Unable to determine"):
+        warn_on_generator_with_return_value(mock_spider, top_level_return_none)
+
+
+def test_partial() -> None:
+    def cb(arg1, arg2):
+        yield {}
+
+    partial_cb = partial(cb, arg1=42)
+    assert not is_generator_with_return_value(partial_cb)
+
+
+def test_warn_on_generator_with_return_value_settings_disabled() -> None:
+    class MockSettings:
+        def __init__(self, settings_dict: dict[str, Any] | None = None):
+            self.settings_dict = settings_dict or {}
+
+        def getbool(self, name, default=False):
+            return self.settings_dict.get(name, default)
+
+    class MockSpider:
+        def __init__(self) -> None:
+            self.settings = MockSettings({"WARN_ON_GENERATOR_RETURN_VALUE": False})
+
+    spider = MockSpider()
+
+    def gen_with_return():
+        yield 1
+        return "value"
+
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", UserWarning)
+        warn_on_generator_with_return_value(spider, gen_with_return)  # type: ignore[arg-type]
+
+    spider.settings.settings_dict["WARN_ON_GENERATOR_RETURN_VALUE"] = True
+
+    with pytest.warns(UserWarning, match="is a generator"):
+        warn_on_generator_with_return_value(spider, gen_with_return)  # type: ignore[arg-type]

tests/test_utils_project.py

@@ -1,14 +1,20 @@
+from __future__ import annotations
+
 import os
 from pathlib import Path
+from typing import TYPE_CHECKING
 
 import pytest
 
 from scrapy.utils.misc import set_environ
 from scrapy.utils.project import data_path, get_project_settings
 
+if TYPE_CHECKING:
+    from collections.abc import Generator
+
 
 @pytest.fixture
-def proj_path(tmp_path):
+def proj_path(tmp_path: Path) -> Generator[Path]:
     prev_dir = Path.cwd()
     project_dir = tmp_path
 
@@ -21,7 +27,7 @@ def proj_path(tmp_path):
         os.chdir(prev_dir)
 
 
-def test_data_path_outside_project():
+def test_data_path_outside_project() -> None:
     assert str(Path(".scrapy", "somepath")) == data_path("somepath")
     abspath = str(Path(os.path.sep, "absolute", "path"))
     assert abspath == data_path(abspath)

tests/test_utils_python.py

@@ -22,8 +22,7 @@ from scrapy.utils.python import (
 from tests.utils.decorators import coroutine_test
 
 if TYPE_CHECKING:
-    from collections.abc import Iterable, Mapping
-
+    from collections.abc import AsyncIterator, Iterable, Mapping
 
 _KT = TypeVar("_KT")
 _VT = TypeVar("_VT")
@@ -31,22 +30,22 @@ _VT = TypeVar("_VT")
 
 class TestMutableAsyncChain:
     @staticmethod
-    async def g1():
+    async def g1() -> AsyncIterator[int]:
         for i in range(3):
             yield i
 
     @staticmethod
-    async def g2():
+    async def g2() -> AsyncIterator[int]:
         return
         yield
 
     @staticmethod
-    async def g3():
+    async def g3() -> AsyncIterator[int]:
         for i in range(7, 10):
             yield i
 
     @staticmethod
-    async def g4():
+    async def g4() -> AsyncIterator[int]:
         for i in range(3, 5):
             yield i
         1 / 0
@@ -85,7 +84,7 @@ class TestToUnicode:
 
     def test_converting_a_strange_object_should_raise_type_error(self):
         with pytest.raises(TypeError):
-            to_unicode(423)
+            to_unicode(423)  # type: ignore[arg-type]
 
     def test_errors_argument(self):
         assert to_unicode(b"a\xedb", "utf-8", errors="replace") == "a\ufffdb"
@@ -103,7 +102,7 @@ class TestToBytes:
 
     def test_converting_a_strange_object_should_raise_type_error(self):
         with pytest.raises(TypeError):
-            to_bytes(pytest)
+            to_bytes(pytest)  # type: ignore[arg-type]
 
     def test_errors_argument(self):
         assert to_bytes("a\ufffdb", "latin-1", errors="replace") == b"a?b"
@@ -112,10 +111,10 @@ class TestToBytes:
 def test_memoizemethod_noargs():
     class A:
         @memoizemethod_noargs
-        def cached(self):
+        def cached(self) -> object:
             return object()
 
-        def noncached(self):
+        def noncached(self) -> object:
             return object()
 
     a = A()
@@ -150,7 +149,7 @@ def test_get_func_args():
         pass
 
     class A:
-        def __init__(self, a, b, c):
+        def __init__(self, a: int, b: int, c: int):
             pass
 
         def method(self, a, b, c):

tests/test_utils_reactor.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import asyncio
 
 import pytest

tests/test_utils_request.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 import json
 from hashlib import sha1
-from typing import Any
+from typing import TYPE_CHECKING, Any, Protocol
 from weakref import WeakKeyDictionary
 
 import pytest
@@ -17,6 +17,9 @@ from scrapy.utils.request import (
 )
 from scrapy.utils.test import get_crawler
 
+if TYPE_CHECKING:
+    from collections.abc import Iterable
+
 
 @pytest.mark.parametrize(
     ("r", "expected"),
@@ -56,8 +59,18 @@ def test_request_httprepr_for_non_http_request(r: Request) -> None:
     request_httprepr(r)
 
 
+class _FingerprintFunction(Protocol):
+    def __call__(
+        self,
+        request: Request,
+        *,
+        include_headers: Iterable[bytes | str] | None = None,
+        keep_fragments: bool = False,
+    ) -> bytes: ...
+
+
 class TestFingerprint:
-    function: staticmethod[[Request], bytes] = staticmethod(fingerprint)
+    function: _FingerprintFunction = staticmethod(fingerprint)
     cache: (
         WeakKeyDictionary[
             Request, dict[tuple[tuple[bytes, ...] | None, bool, bool], bytes]
@@ -261,6 +274,7 @@ class TestRequestFingerprinter:
     def test_fingerprint(self):
         crawler = get_crawler()
         request = Request("https://example.com")
+        assert crawler.request_fingerprinter
         assert crawler.request_fingerprinter.fingerprint(request) == fingerprint(
             request
         )
@@ -277,6 +291,7 @@ class TestCustomRequestFingerprinter:
         }
         crawler = get_crawler(settings_dict=settings)
 
+        assert crawler.request_fingerprinter
         r1 = Request("http://www.example.com", headers={"X-ID": "1"})
         fp1 = crawler.request_fingerprinter.fingerprint(r1)
         r2 = Request("http://www.example.com", headers={"X-ID": "2"})
@@ -285,9 +300,9 @@ class TestCustomRequestFingerprinter:
 
     def test_dont_canonicalize(self):
         class RequestFingerprinter:
-            cache = WeakKeyDictionary()
+            cache: WeakKeyDictionary[Request, bytes] = WeakKeyDictionary()
 
-            def fingerprint(self, request):
+            def fingerprint(self, request: Request) -> bytes:
                 if request not in self.cache:
                     fp = sha1()
                     fp.update(to_bytes(request.url))
@@ -299,6 +314,7 @@ class TestCustomRequestFingerprinter:
         }
         crawler = get_crawler(settings_dict=settings)
 
+        assert crawler.request_fingerprinter
         r1 = Request("http://www.example.com?a=1&a=2")
         fp1 = crawler.request_fingerprinter.fingerprint(r1)
         r2 = Request("http://www.example.com?a=2&a=1")
@@ -317,6 +333,7 @@ class TestCustomRequestFingerprinter:
         }
         crawler = get_crawler(settings_dict=settings)
 
+        assert crawler.request_fingerprinter
         r1 = Request("http://www.example.com")
         fp1 = crawler.request_fingerprinter.fingerprint(r1)
         r2 = Request("http://www.example.com", meta={"fingerprint": "a"})
@@ -348,6 +365,7 @@ class TestCustomRequestFingerprinter:
         }
         crawler = get_crawler(settings_dict=settings)
 
+        assert crawler.request_fingerprinter
         request = Request("http://www.example.com")
         fingerprint = crawler.request_fingerprinter.fingerprint(request)
         assert fingerprint == settings["FINGERPRINT"]
@@ -457,3 +475,14 @@ class TestRequestToCurl:
             " --data-raw '{\"foo\": \"bar\"}' --cookie 'foo=1'"
         )
         self._test_request(request_object, expected_curl_command)
+
+    def test_request_to_curl_method(self) -> None:
+        request_object = Request(
+            "https://www.httpbin.org/post",
+            method="POST",
+            body=json.dumps({"foo": "bar"}),
+        )
+        expected_curl_command = (
+            'curl -X POST https://www.httpbin.org/post --data-raw \'{"foo": "bar"}\''
+        )
+        assert request_object.to_curl() == expected_curl_command

tests/test_utils_response.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from pathlib import Path
 from time import process_time
 from urllib.parse import urlparse
@@ -15,7 +17,7 @@ from scrapy.utils.response import (
 )
 
 
-def _read_browser_output(burl: str):
+def _read_browser_output(burl: str) -> bytes:
     path = urlparse(burl).path
     if not path or not Path(path).exists():
         path = burl.replace("file://", "")
@@ -224,7 +226,7 @@ def test_open_in_browser_redos_head():
         (b"<!-- <head>fake</head> --><head>real</head>", b"<head>real</head>"),
     ],
 )
-def test_remove_html_comments(input_body, output_body):
+def test_remove_html_comments(input_body: bytes, output_body: bytes) -> None:
     assert _remove_html_comments(input_body) == output_body
 
 

tests/test_utils_serialize.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import dataclasses
 import datetime
 import json

tests/test_utils_signal.py

@@ -21,9 +21,6 @@ from tests.utils.decorators import coroutine_test
 if TYPE_CHECKING:
     from collections.abc import Callable
 
-if TYPE_CHECKING:
-    from collections.abc import Callable
-
 
 class TestSendCatchLog:
     # whether the function being tested returns exceptions or failures

tests/test_utils_sitemap.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import pytest
 
 from scrapy.exceptions import ScrapyDeprecationWarning

tests/test_utils_template.py

@@ -1,7 +1,14 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
 from scrapy.utils.template import render_templatefile
 
+if TYPE_CHECKING:
+    from pathlib import Path
 
-def test_simple_render(tmp_path):
+
+def test_simple_render(tmp_path: Path) -> None:
     context = {"project_name": "proj", "name": "spi", "classname": "TheSpider"}
     template = "from ${project_name}.spiders.${name} import ${classname}"
     rendered = "from proj.spiders.spi import TheSpider"

tests/test_utils_trackref.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import sys
 from io import StringIO
 from unittest import mock
@@ -46,13 +48,13 @@ Bar                                 1   oldest: 0s ago
 
 
 @mock.patch("sys.stdout", new_callable=StringIO)
-def test_print_live_refs_empty(stdout):
+def test_print_live_refs_empty(stdout: StringIO) -> None:
     trackref.print_live_refs()
     assert stdout.getvalue() == "Live References\n\n\n"
 
 
 @mock.patch("sys.stdout", new_callable=StringIO)
-def test_print_live_refs_with_objects(stdout):
+def test_print_live_refs_with_objects(stdout: StringIO) -> None:
     o1 = Foo()  # noqa: F841
     trackref.print_live_refs()
     assert (

tests/test_utils_url.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import pytest
 
 from scrapy.linkextractors import IGNORED_EXTENSIONS
@@ -191,7 +193,7 @@ def test_guess_scheme(url: str, expected: str):
         ),
     ],
 )
-def test_guess_scheme_skipped(url: str, expected: str, reason: str):
+def test_guess_scheme_skipped(url: str, expected: str, reason: str) -> None:
     pytest.skip(reason)
 
 

tests/utils/bases/commands.py

@@ -32,3 +32,9 @@ class TestProjectBase:
         proj_path = tmp_path / self.project_name
         copytree(_proj_path_cached, proj_path)
         return proj_path
+
+    @staticmethod
+    def _append_settings(proj_mod_path: Path, text: str) -> None:
+        """Add text to the end of the project settings.py."""
+        with (proj_mod_path / "settings.py").open("a", encoding="utf-8") as f:
+            f.write(text)

tests/utils/bases/http_request.py

@@ -6,6 +6,7 @@ import pytest
 
 from scrapy.http import Headers, Request
 from scrapy.http.request import NO_CALLBACK
+from scrapy.utils.request import request_to_curl
 
 
 class TestRequestBase(ABC):
@@ -488,3 +489,11 @@ class TestRequestBase(ABC):
                 'curl -X PATCH "http://example.org" --foo -z',
                 ignore_unknown_options=False,
             )
+
+    def test_to_curl(self):
+        # Note: more curated tests regarding curl conversion are in
+        # `test_utils_request.py`
+        r = self.request_class(
+            "http://www.example.com/", method="POST", body=b"foo=bar"
+        )
+        assert r.to_curl() == request_to_curl(r)

tests/utils/crawl.py

@@ -0,0 +1,27 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from scrapy import signals
+from scrapy.utils.test import get_crawler
+
+if TYPE_CHECKING:
+    from scrapy.crawler import Crawler
+    from scrapy.spiders import Spider
+    from tests.mockserver.http import MockServer
+
+
+async def crawl_items(
+    spider_cls: type[Spider], mockserver: MockServer, **kwargs: Any
+) -> tuple[list[Any], Crawler]:
+    """Run *spider_cls* against *mockserver* and return the scraped items along
+    with the crawler, which gives tests access to the resulting stats."""
+    items: list[Any] = []
+
+    def collect(item: Any) -> None:
+        items.append(item)
+
+    crawler = get_crawler(spider_cls)
+    crawler.signals.connect(collect, signals.item_scraped)
+    await crawler.crawl_async(mockserver=mockserver, **kwargs)
+    return items, crawler

tests/utils/media_pipelines.py

@@ -3,6 +3,12 @@ from __future__ import annotations
 from typing import Any
 
 from scrapy.http.request import NO_CALLBACK, Request
+from scrapy.pipelines.media import MediaPipeline
+from scrapy.utils.spider import DefaultSpider
+
+# required by persist_file() and stat_file(), but as some stores don't use the argument
+# we can pass this singleton to keep type hints correct
+DUMMY_SPIDER_INFO = MediaPipeline.SpiderInfo(DefaultSpider())
 
 
 async def mocked_download_func(request: Request) -> Any:

tests_typing/test_http_request.mypy-testing

@@ -16,7 +16,7 @@ class MyRequest2(Request):
 
 @pytest.mark.mypy_testing
 def mypy_test_headers() -> None:
-    Request("data:,", headers=1)  # E: Argument "headers" to "Request" has incompatible type "int"; expected "Mapping[str, Any] | Iterable[tuple[str, Any]] | None"
+    Request("data:,", headers=1)  # E: Argument "headers" to "Request" has incompatible type "int"; expected "Mapping[str, Any] | Mapping[bytes, Any] | Iterable[tuple[str | bytes, Any]] | None"
     Request("data:,", headers=None)
     Request("data:,", headers={})
     Request("data:,", headers=[])

tests_typing/test_http_response.mypy-testing

@@ -7,7 +7,7 @@ from scrapy.http import HtmlResponse, Response, TextResponse
 
 @pytest.mark.mypy_testing
 def mypy_test_headers() -> None:
-    Response("data:,", headers=1)  # E: Argument "headers" to "Response" has incompatible type "int"; expected "Mapping[str, Any] | Iterable[tuple[str, Any]] | None"
+    Response("data:,", headers=1)  # E: Argument "headers" to "Response" has incompatible type "int"; expected "Mapping[str, Any] | Mapping[bytes, Any] | Iterable[tuple[str | bytes, Any]] | None"
     Response("data:,", headers=None)
     Response("data:,", headers={})
     Response("data:,", headers=[])