Merge remote-tracking branch 'origin/master' into faster-shutdown-2

conftest.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import os
 from importlib.util import find_spec
 from pathlib import Path
 from typing import TYPE_CHECKING
@@ -135,5 +136,6 @@ def pytest_runtest_setup(item):
         pytest.skip("mitmdump is not available")
 
 
-# Generate localhost certificate files, needed by some tests
-generate_keys()
+# Generate localhost certificate files, needed by some tests (but only once if xdist is used)
+if "PYTEST_XDIST_WORKER" not in os.environ:
+    generate_keys()

docs/faq.rst

@@ -136,7 +136,7 @@ middleware with a :ref:`custom downloader middleware
 <topics-downloader-middleware-custom>` that requires less memory. For example:
 
 -   If your domain names are similar enough, use your own regular expression
-    instead joining the strings in :attr:`~scrapy.Spider.allowed_domains` into
+    instead of joining the strings in :attr:`~scrapy.Spider.allowed_domains` into
     a complex regular expression.
 
 -   If you can meet the installation requirements, use pyre2_ instead of
@@ -332,8 +332,8 @@ section of the site (which varies each time). In that case, the credentials to
 log in would be settings, while the url of the section to scrape would be a
 spider argument.
 
-I'm scraping a XML document and my XPath selector doesn't return any items
---------------------------------------------------------------------------
+I'm scraping an XML document and my XPath selector doesn't return any items
+---------------------------------------------------------------------------
 
 You may need to remove namespaces. See :ref:`removing-namespaces`.
 

docs/topics/contracts.rst

@@ -30,43 +30,15 @@ You can use the following contracts:
 
 .. module:: scrapy.contracts.default
 
-.. class:: UrlContract
+.. autoclass:: UrlContract
 
-    This contract (``@url``) sets the sample URL used when checking other
-    contract conditions for this spider. This contract is mandatory. All
-    callbacks lacking this contract are ignored when running the checks::
+.. autoclass:: CallbackKeywordArgumentsContract
 
-    @url url
+.. autoclass:: MetadataContract
 
-.. class:: CallbackKeywordArgumentsContract
+.. autoclass:: ReturnsContract
 
-    This contract (``@cb_kwargs``) sets the :attr:`cb_kwargs <scrapy.Request.cb_kwargs>`
-    attribute for the sample request. It must be a valid JSON dictionary.
-    ::
-
-    @cb_kwargs {"arg1": "value1", "arg2": "value2", ...}
-
-.. class:: MetadataContract
-
-    This contract (``@meta``) sets the :attr:`meta <scrapy.Request.meta>`
-    attribute for the sample request. It must be a valid JSON dictionary.
-    ::
-
-    @meta {"arg1": "value1", "arg2": "value2", ...}
-
-.. class:: ReturnsContract
-
-    This contract (``@returns``) sets lower and upper bounds for the items and
-    requests returned by the spider. The upper bound is optional::
-
-    @returns item(s)|request(s) [min [max]]
-
-.. class:: ScrapesContract
-
-    This contract (``@scrapes``) checks that all the items returned by the
-    callback have the specified fields::
-
-    @scrapes field_1 field_2 ...
+.. autoclass:: ScrapesContract
 
 Use the :command:`check` command to run the contract checks.
 
@@ -89,30 +61,16 @@ override three methods:
 
 .. module:: scrapy.contracts
 
-.. class:: Contract(method, *args)
+.. autoclass:: Contract
 
-    :param method: callback function to which the contract is associated
-    :type method: collections.abc.Callable
+    .. automethod:: adjust_request_args
 
-    :param args: list of arguments passed into the docstring (whitespace
-        separated)
-    :type args: list
-
-    .. method:: Contract.adjust_request_args(args)
-
-        This receives a ``dict`` as an argument containing default arguments
-        for request object. :class:`~scrapy.Request` is used by default,
-        but this can be changed with the ``request_cls`` attribute.
-        If multiple contracts in chain have this attribute defined, the last one is used.
-
-        Must return the same or a modified version of it.
-
-    .. method:: Contract.pre_process(response)
+    .. method:: pre_process(response)
 
         This allows hooking in various checks on the response received from the
         sample request, before it's being passed to the callback.
 
-    .. method:: Contract.post_process(output)
+    .. method:: post_process(output)
 
         This allows processing the output of the callback. Iterators are
         converted to lists before being passed to this hook.

docs/topics/leaks.rst

@@ -187,30 +187,13 @@ scrapy.utils.trackref module
 
 Here are the functions available in the :mod:`~scrapy.utils.trackref` module.
 
-.. class:: object_ref
+.. autoclass:: object_ref
 
-    Inherit from this class if you want to track live
-    instances with the ``trackref`` module.
+.. autofunction:: print_live_refs(ignore=NoneType)
 
-.. function:: print_live_refs(ignore=NoneType)
+.. autofunction:: get_oldest
 
-    Print a report of live references, grouped by class name.
-
-    :param ignore: if given, all objects from the specified class (or tuple of
-        classes) will be ignored.
-    :type ignore: type or tuple
-
-.. function:: get_oldest(class_name)
-
-    Return the oldest object alive with the given class name, or ``None`` if
-    none is found. Use :func:`print_live_refs` first to get a list of all
-    tracked live objects per class name.
-
-.. function:: iter_all(class_name)
-
-    Return an iterator over all objects alive with the given class name. Use
-    :func:`print_live_refs` first to get a list of all tracked live objects
-    per class name.
+.. autofunction:: iter_all
 
 .. skip: end
 

docs/topics/request-response.rst

@@ -342,6 +342,8 @@ Other functions related to requests
 
 .. autofunction:: scrapy.utils.request.request_from_dict
 
+.. autofunction:: scrapy.utils.httpobj.urlparse_cached
+
 
 .. _topics-request-response-ref-request-callback-arguments:
 

pyproject.toml

@@ -136,7 +136,6 @@ module = [
     "tests.test_downloadermiddleware_redirect_metarefresh",
     "tests.test_downloadermiddleware_retry",
     "tests.test_downloadermiddleware_robotstxt",
-    "tests.test_downloadermiddleware_stats",
     "tests.test_downloaderslotssettings",
     "tests.test_dupefilters",
     "tests.test_engine_loop",
@@ -154,11 +153,9 @@ module = [
     "tests.test_http_response",
     "tests.test_http_response_text",
     "tests.test_item",
-    "tests.test_link",
     "tests.test_linkextractors",
     "tests.test_loader",
     "tests.test_logformatter",
-    "tests.test_logstats",
     "tests.test_mail",
     "tests.test_pipeline_crawl",
     "tests.test_pipeline_files",
@@ -188,6 +185,9 @@ module = [
     "tests.test_utils_misc.test_return_with_argument_inside_generator",
     "tests.test_utils_python",
     "tests.test_utils_request",
+    "tests.utils.bases.http_request",
+    "tests.utils.bases.http_response",
+    "tests.utils.bases.spider",
 ]
 check_untyped_defs = false
 
@@ -225,7 +225,6 @@ module = [
     "pyftpdlib.*",
     "pytest_twisted",
     "robotexclusionrulesparser",
-    "testfixtures",
     "zope.interface.*",
 ]
 ignore_missing_imports = true

scrapy/contracts/__init__.py

@@ -22,7 +22,16 @@ if TYPE_CHECKING:
 
 
 class Contract:
-    """Abstract class for contracts"""
+    """Base class for :ref:`custom contracts <topics-contracts>`.
+
+    *method* is the callback function to which the contract is associated.
+
+    *args* is the list of arguments passed into the docstring, separated by
+    whitespace.
+
+    Subclasses may override :meth:`adjust_request_args`, and define a
+    ``pre_process`` method or a ``post_process`` method, or both.
+    """
 
     request_cls: type[Request] | None = None
     name: str
@@ -90,6 +99,13 @@ class Contract:
         return request
 
     def adjust_request_args(self, args: dict[str, Any]) -> dict[str, Any]:
+        """Receive a ``dict`` with the default arguments for the sample request
+        and return it, either unmodified or with changes.
+
+        :class:`~scrapy.Request` is used by default, but this can be changed
+        with the ``request_cls`` attribute. If multiple contracts in the chain
+        define this attribute, the last one is used.
+        """
         return args
 
 

scrapy/contracts/default.py

@@ -15,8 +15,15 @@ if TYPE_CHECKING:
 
 # contracts
 class UrlContract(Contract):
-    """Contract to set the url of the request (mandatory)
-    @url http://scrapy.org
+    """Sets (``@url``) the sample URL used when checking the other contract
+    conditions of a callback.
+
+    This contract is mandatory: callbacks lacking it are ignored when running
+    the checks.
+
+    .. code-block:: none
+
+        @url url
     """
 
     name = "url"
@@ -27,10 +34,14 @@ class UrlContract(Contract):
 
 
 class CallbackKeywordArgumentsContract(Contract):
-    """Contract to set the keyword arguments for the request.
-    The value should be a JSON-encoded dictionary, e.g.:
+    """Sets (``@cb_kwargs``) the :attr:`cb_kwargs <scrapy.Request.cb_kwargs>`
+    attribute of the sample request.
 
-    @cb_kwargs {"arg1": "some value"}
+    Its value must be a valid JSON dictionary.
+
+    .. code-block:: none
+
+        @cb_kwargs {"arg1": "value1", "arg2": "value2", ...}
     """
 
     name = "cb_kwargs"
@@ -41,10 +52,14 @@ class CallbackKeywordArgumentsContract(Contract):
 
 
 class MetadataContract(Contract):
-    """Contract to set metadata arguments for the request.
-    The value should be JSON-encoded dictionary, e.g.:
+    """Sets (``@meta``) the :attr:`meta <scrapy.Request.meta>` attribute of the
+    sample request.
 
-    @meta {"arg1": "some value"}
+    Its value must be a valid JSON dictionary.
+
+    .. code-block:: none
+
+        @meta {"arg1": "value1", "arg2": "value2", ...}
     """
 
     name = "meta"
@@ -55,16 +70,29 @@ class MetadataContract(Contract):
 
 
 class ReturnsContract(Contract):
-    """Contract to check the output of a callback
+    """Sets (``@returns``) lower and upper bounds for the items and requests
+    returned by a callback.
 
-    general form:
-    @returns request(s)/item(s) [min=1 [max]]
+    The upper bound is optional:
 
-    e.g.:
-    @returns request
-    @returns request 2
-    @returns request 2 10
-    @returns request 0 10
+    .. code-block:: none
+
+        @returns item(s)|request(s) [min [max]]
+
+    For example:
+
+    .. code-block:: none
+
+        @returns request
+        @returns request 2
+        @returns request 2 10
+        @returns request 0 10
+
+    Set both bounds to the same value to require an exact number:
+
+    .. code-block:: none
+
+        @returns request 2 2
     """
 
     name = "returns"
@@ -115,8 +143,12 @@ class ReturnsContract(Contract):
 
 
 class ScrapesContract(Contract):
-    """Contract to check presence of fields in scraped items
-    @scrapes page_name page_body
+    """Checks (``@scrapes``) that all items returned by a callback have the
+    specified fields.
+
+    .. code-block:: none
+
+        @scrapes field_1 field_2 ...
     """
 
     name = "scrapes"

scrapy/core/engine.py

@@ -498,16 +498,17 @@ class ExecutionEngine:
         """
         if self.spider is None:
             raise RuntimeError(f"No open spider to crawl: {request}")
-        try:
-            response_or_request = await maybe_deferred_to_future(
-                self._download(request)
-            )
-        finally:
-            assert self._slot is not None
-            self._slot.remove_request(request)
-        if isinstance(response_or_request, Request):
-            return await self.download_async(response_or_request)
-        return response_or_request
+        while True:
+            try:
+                response_or_request = await maybe_deferred_to_future(
+                    self._download(request)
+                )
+            finally:
+                assert self._slot is not None
+                self._slot.remove_request(request)
+            if not isinstance(response_or_request, Request):
+                return response_or_request
+            request = response_or_request
 
     @inlineCallbacks
     def _download(

scrapy/core/http2/stream.py

@@ -315,7 +315,7 @@ class Stream:
             0, self.metadata["remaining_content_length"]
         )
 
-        # End the stream if no more data needs to be send
+        # End the stream if no more data needs to be sent
         if self.metadata["remaining_content_length"] == 0:
             self._protocol.conn.end_stream(self.stream_id)
 

scrapy/downloadermiddlewares/offsite.py

@@ -61,7 +61,7 @@ class OffsiteMiddleware:
             )
             self.stats.inc_value("offsite/domains")
         self.stats.inc_value("offsite/filtered")
-        raise IgnoreRequest
+        raise IgnoreRequest(f"Filtered offsite request to {domain!r}")
 
     def should_follow(self, request: Request, spider: Spider) -> bool:
         regex = self.host_regex

scrapy/pipelines/files.py

@@ -27,7 +27,15 @@ from twisted.internet.defer import Deferred, maybeDeferred
 from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarning
 from scrapy.http import Request, Response
 from scrapy.http.request import NO_CALLBACK
-from scrapy.pipelines.media import FileInfo, FileInfoOrError, MediaPipeline
+from scrapy.pipelines.media import (
+    FileException as FileException,  # noqa: PLC0414  # re-exported for backward compatibility
+)
+from scrapy.pipelines.media import (
+    FileInfo,
+    FileInfoOrError,
+    MediaPipeline,
+    _MediaRequestFiltered,
+)
 from scrapy.utils.asyncio import run_in_thread
 from scrapy.utils.boto import is_botocore_available
 from scrapy.utils.datatypes import CaseInsensitiveDict
@@ -75,10 +83,6 @@ def _md5sum(file: IO[bytes]) -> str:
     return m.hexdigest()
 
 
-class FileException(Exception):
-    """General media error exception"""
-
-
 class StatInfo(TypedDict, total=False):
     checksum: str
     last_modified: float
@@ -597,20 +601,20 @@ class FilesPipeline(MediaPipeline):
     def media_failed(
         self, failure: Failure, request: Request, info: MediaPipeline.SpiderInfo
     ) -> NoReturn:
-        if not isinstance(failure.value, IgnoreRequest):
-            referer = referer_str(request)
-            logger.warning(
-                "File (unknown-error): Error downloading %(medianame)s from "
-                "%(request)s referred in <%(referer)s>: %(exception)s",
-                {
-                    "medianame": self.MEDIA_NAME,
-                    "request": request,
-                    "referer": referer,
-                    "exception": failure.value,
-                },
+        referer = referer_str(request)
+        if isinstance(failure.value, IgnoreRequest):
+            logger.debug(
+                f"File (filtered): Not downloading {self.MEDIA_NAME} from "
+                f"{request} referred in <{referer}>: {failure.value}",
                 extra={"spider": info.spider},
             )
+            raise _MediaRequestFiltered(str(failure.value)) from failure.value
 
+        logger.warning(
+            f"File (unknown-error): Error downloading {self.MEDIA_NAME} from "
+            f"{request} referred in <{referer}>: {failure.value}",
+            extra={"spider": info.spider},
+        )
         raise FileException
 
     async def media_downloaded(

scrapy/pipelines/media.py

@@ -55,6 +55,25 @@ FileInfoOrError: TypeAlias = (
 logger = logging.getLogger(__name__)
 
 
+class FileException(Exception):
+    """General media error exception"""
+
+
+class _MediaRequestFiltered(FileException):
+    """Raised internally by media pipelines when a media request is filtered
+    out (e.g. as an offsite request) instead of being downloaded.
+
+    It is a subclass of :exc:`FileException` for backward compatibility, but
+    unlike an actual download error it is logged at the ``DEBUG`` level and
+    without a traceback, since filtering a request is expected behavior rather
+    than an error.
+    """
+
+
+def _media_request_filtered(failure: Failure) -> bool:
+    return isinstance(failure.value, _MediaRequestFiltered)
+
+
 class MediaPipeline(ABC):
     LOG_FAILED_RESULTS: bool = True
 
@@ -193,7 +212,8 @@ class MediaPipeline(ABC):
                 result = await self._check_media_to_download(request, info, item=item)
         except Exception:
             result = Failure()
-            logger.exception(result)
+            if not _media_request_filtered(result):
+                logger.exception(result)
         self._cache_result_and_execute_waiters(result, fp, info)
         return await maybe_deferred_to_future(wad)  # it must return wad at last
 
@@ -304,6 +324,8 @@ class MediaPipeline(ABC):
             for ok, value in results:
                 if not ok:
                     assert isinstance(value, Failure)
+                    if _media_request_filtered(value):
+                        continue
                     logger.error(
                         "%(class)s found errors processing %(item)s",
                         {"class": self.__class__.__name__, "item": item},

scrapy/resolver.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING, Any
 
+import attr
 from twisted.internet import defer
 from twisted.internet.base import ReactorBase, ThreadedResolver
 from twisted.internet.interfaces import (
@@ -70,6 +71,12 @@ class CachingThreadedResolver(ThreadedResolver):
         return result
 
 
+def _address_with_port(address: IAddress, port: int) -> IAddress:
+    if getattr(address, "port", port) == port:
+        return address
+    return attr.evolve(address, port=port)
+
+
 @implementer(IHostResolution)
 class HostResolution:
     def __init__(self, name: str):
@@ -97,7 +104,11 @@ class _CachingResolutionReceiver:
     def resolutionComplete(self) -> None:
         self.resolutionReceiver.resolutionComplete()
         if self.addresses:
-            dnscache[self.hostName] = self.addresses
+            # Name resolution does not depend on the port, so cache entries are
+            # kept port-agnostic and the requested port is set on cache hits.
+            dnscache[self.hostName] = [
+                _address_with_port(address, 0) for address in self.addresses
+            ]
 
 
 @implementer(IHostnameResolver)
@@ -142,7 +153,7 @@ class CachingHostnameResolver:
                 transportSemantics,
             )
         resolutionReceiver.resolutionBegan(HostResolution(hostName))
-        for addr in addresses:
-            resolutionReceiver.addressResolved(addr)
+        for address in addresses:
+            resolutionReceiver.addressResolved(_address_with_port(address, portNumber))
         resolutionReceiver.resolutionComplete()
         return resolutionReceiver

scrapy/utils/httpobj.py

@@ -16,8 +16,15 @@ _urlparse_cache: WeakKeyDictionary[Request | Response, ParseResult] = (
 
 
 def urlparse_cached(request_or_response: Request | Response) -> ParseResult:
-    """Return urlparse.urlparse caching the result, where the argument can be a
-    Request or Response object
+    """Return the result of parsing the URL of *request_or_response*, a
+    :class:`~scrapy.Request` or :class:`~scrapy.http.Response` object, with
+    :func:`urllib.parse.urlparse`.
+
+    The result is cached, using a :class:`weakref.WeakKeyDictionary` keyed on
+    *request_or_response*, so that the URL of a given object is parsed only
+    once. Prefer this function over calling :func:`urllib.parse.urlparse` on
+    ``request_or_response.url`` directly when the same URL may be parsed more
+    than once.
     """
     if request_or_response not in _urlparse_cache:
         _urlparse_cache[request_or_response] = urlparse(request_or_response.url)

scrapy/utils/trackref.py

@@ -34,7 +34,8 @@ live_refs: defaultdict[type, WeakKeyDictionary[object, float]] = defaultdict(
 
 
 class object_ref:
-    """Inherit from this class to a keep a record of live instances"""
+    """Inherit from this class if you want to track live instances with the
+    ``trackref`` module."""
 
     __slots__ = ()
 
@@ -60,12 +61,19 @@ def format_live_refs(ignore: Any = NoneType) -> str:
 
 
 def print_live_refs(*a: Any, **kw: Any) -> None:
-    """Print tracked objects"""
+    """Print a report of live references, grouped by class name.
+
+    :param ignore: if given, all objects from the specified class (or tuple of
+        classes) will be ignored.
+    :type ignore: type or tuple
+    """
     print(format_live_refs(*a, **kw))
 
 
 def get_oldest(class_name: str) -> Any:
-    """Get the oldest object for a specific class name"""
+    """Return the oldest object alive with the given class name, or ``None`` if
+    none is found. Use :func:`print_live_refs` first to get a list of all
+    tracked live objects per class name."""
     for cls, wdict in live_refs.items():
         if cls.__name__ == class_name:
             if not wdict:
@@ -75,7 +83,9 @@ def get_oldest(class_name: str) -> Any:
 
 
 def iter_all(class_name: str) -> Iterable[Any]:
-    """Iterate over all objects of the same class by its class name"""
+    """Return an iterator over all objects alive with the given class name. Use
+    :func:`print_live_refs` first to get a list of all tracked live objects per
+    class name."""
     for cls, wdict in live_refs.items():
         if cls.__name__ == class_name:
             return wdict.keys()

tests/mockserver/http_base.py

@@ -20,6 +20,9 @@ if TYPE_CHECKING:
 
     from twisted.web import resource
 
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
+
 
 class BaseMockServer(ABC):
     listen_http: bool = True
@@ -39,13 +42,14 @@ class BaseMockServer(ABC):
         self.http_port: int | None = None
         self.https_port: int | None = None
 
-    def __enter__(self):
+    def __enter__(self) -> Self:
         self.proc = Popen(
             [sys.executable, "-u", "-m", self.module_name, *self.get_additional_args()],
             stdout=PIPE,
             env=get_script_run_env(),
             text=True,
         )
+        assert self.proc.stdout is not None
         if self.listen_http:
             http_address = self.proc.stdout.readline().strip()
             http_parsed = urlparse(http_address)
@@ -56,7 +60,7 @@ class BaseMockServer(ABC):
             self.https_port = https_parsed.port
         return self
 
-    def __exit__(self, exc_type, exc_value, traceback):
+    def __exit__(self, exc_type, exc_value, traceback) -> None:
         if self.proc:
             self.proc.kill()
             self.proc.communicate()

tests/test_command_check.py

@@ -7,7 +7,7 @@ from unittest import TestCase
 from unittest.mock import MagicMock, Mock, PropertyMock, call, patch
 
 from scrapy.commands.check import Command, TextTestResult
-from tests.utils.base_commands import TestProjectBase
+from tests.utils.bases.commands import TestProjectBase
 from tests.utils.cmdline import proc
 
 if TYPE_CHECKING:

tests/test_command_crawl.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING
 
-from tests.utils.base_commands import TestProjectBase
+from tests.utils.bases.commands import TestProjectBase
 from tests.utils.cmdline import proc
 
 if TYPE_CHECKING:

tests/test_command_genspider.py

@@ -6,7 +6,7 @@ from pathlib import Path
 
 import pytest
 
-from tests.utils.base_commands import TestProjectBase
+from tests.utils.bases.commands import TestProjectBase
 from tests.utils.cmdline import call, proc, write_recording_editor
 
 

tests/test_command_parse.py

@@ -8,7 +8,7 @@ import pytest
 
 from scrapy.commands import parse
 from scrapy.settings import Settings
-from tests.utils.base_commands import TestProjectBase
+from tests.utils.bases.commands import TestProjectBase
 from tests.utils.cmdline import call, proc
 
 if TYPE_CHECKING:

tests/test_commands.py

@@ -15,7 +15,7 @@ from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter, view
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.settings import Settings
 from scrapy.utils.reactor import _asyncio_reactor_path
-from tests.utils.base_commands import TestProjectBase
+from tests.utils.bases.commands import TestProjectBase
 from tests.utils.cmdline import call, proc, write_recording_editor
 
 if TYPE_CHECKING:

tests/test_downloader_handler_httpx.py

@@ -15,7 +15,7 @@ from scrapy.core.downloader.handlers._httpx import (
     HttpxDownloadHandler,
 )
 from scrapy.exceptions import DownloadFailedError
-from tests.test_downloader_handlers_http_base import (
+from tests.utils.bases.download_handlers_http import (
     TestHttpBase,
     TestHttpProxyBase,
     TestHttpsBase,

tests/test_downloader_handler_twisted_http11.py

@@ -11,7 +11,7 @@ from scrapy import Spider
 from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler
 from scrapy.crawler import Crawler
 from scrapy.exceptions import NotConfigured
-from tests.test_downloader_handlers_http_base import (
+from tests.utils.bases.download_handlers_http import (
     TestHttpBase,
     TestHttpProxyBase,
     TestHttpsBase,

tests/test_downloader_handler_twisted_http2.py

@@ -2,18 +2,18 @@
 
 from __future__ import annotations
 
+import logging
 import sys
 from typing import TYPE_CHECKING, Any
 
 import pytest
-from testfixtures import LogCapture
 from twisted.web.http import H2_ENABLED
 
 from scrapy import Spider
 from scrapy.crawler import Crawler
 from scrapy.exceptions import DownloadFailedError, NotConfigured
 from scrapy.http import Request
-from tests.test_downloader_handlers_http_base import (
+from tests.utils.bases.download_handlers_http import (
     TestHttpProxyBase,
     TestHttpsBase,
     TestHttpsCustomCiphersBase,
@@ -130,24 +130,24 @@ class TestHttp2(H2DownloadHandlerMixin, TestHttpsBase):
         assert response.text == custom_content_length
 
     @coroutine_test
-    async def test_custom_content_length_bad(self, mockserver: MockServer) -> None:
+    async def test_custom_content_length_bad(
+        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+    ) -> None:
         request = Request(mockserver.url("/contentlength", is_secure=self.is_secure))
         actual_content_length = str(len(request.body))
         bad_content_length = str(len(request.body) + 1)
         request.headers["Content-Length"] = bad_content_length
         async with self.get_dh() as download_handler:
-            with LogCapture() as log:
+            with caplog.at_level(logging.DEBUG):
                 response = await download_handler.download_request(request)
         assert response.text == actual_content_length
-        log.check_present(
-            (
-                "scrapy.core.http2.stream",
-                "WARNING",
-                f"Ignoring bad Content-Length header "
-                f"{bad_content_length!r} of request {request}, sending "
-                f"{actual_content_length!r} instead",
-            )
-        )
+        assert (
+            "scrapy.core.http2.stream",
+            logging.WARNING,
+            f"Ignoring bad Content-Length header "
+            f"{bad_content_length!r} of request {request}, sending "
+            f"{actual_content_length!r} instead",
+        ) in caplog.record_tuples
 
     @coroutine_test
     async def test_data_loss_handling(self, mockserver: MockServer) -> None:

tests/test_downloadermiddleware_cookies.py

@@ -2,7 +2,6 @@ import logging
 from collections.abc import Iterable
 
 import pytest
-from testfixtures import LogCapture
 
 from scrapy.downloadermiddlewares.cookies import CookiesMiddleware
 from scrapy.downloadermiddlewares.defaultheaders import DefaultHeadersMiddleware
@@ -111,14 +110,15 @@ class TestCookiesMiddleware:
             CookiesMiddleware,
         )
 
-    def test_setting_enabled_cookies_debug(self):
+    def test_setting_enabled_cookies_debug(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         crawler = get_crawler(settings_dict={"COOKIES_DEBUG": True})
         mw = CookiesMiddleware.from_crawler(crawler)
-        with LogCapture(
-            "scrapy.downloadermiddlewares.cookies",
-            propagate=False,
-            level=logging.DEBUG,
-        ) as log:
+        caplog.clear()
+        with caplog.at_level(
+            logging.DEBUG, logger="scrapy.downloadermiddlewares.cookies"
+        ):
             req = Request("http://scrapytest.org/")
             res = Response(
                 "http://scrapytest.org/", headers={"Set-Cookie": "C1=value1; path=/"}
@@ -127,43 +127,44 @@ class TestCookiesMiddleware:
             req2 = Request("http://scrapytest.org/sub1/")
             mw.process_request(req2)
 
-            log.check(
-                (
-                    "scrapy.downloadermiddlewares.cookies",
-                    "DEBUG",
-                    "Received cookies from: <200 http://scrapytest.org/>\n"
-                    "Set-Cookie: C1=value1; path=/\n",
-                ),
-                (
-                    "scrapy.downloadermiddlewares.cookies",
-                    "DEBUG",
-                    "Sending cookies to: <GET http://scrapytest.org/sub1/>\n"
-                    "Cookie: C1=value1\n",
-                ),
-            )
+        assert caplog.record_tuples == [
+            (
+                "scrapy.downloadermiddlewares.cookies",
+                logging.DEBUG,
+                "Received cookies from: <200 http://scrapytest.org/>\n"
+                "Set-Cookie: C1=value1; path=/\n",
+            ),
+            (
+                "scrapy.downloadermiddlewares.cookies",
+                logging.DEBUG,
+                "Sending cookies to: <GET http://scrapytest.org/sub1/>\n"
+                "Cookie: C1=value1\n",
+            ),
+        ]
 
-    def test_debug_no_cookies(self):
+    def test_debug_no_cookies(self, caplog: pytest.LogCaptureFixture) -> None:
         crawler = get_crawler(settings_dict={"COOKIES_DEBUG": True})
         mw = CookiesMiddleware.from_crawler(crawler)
-        with LogCapture(
-            "scrapy.downloadermiddlewares.cookies",
-            propagate=False,
-            level=logging.DEBUG,
-        ) as log:
+        caplog.clear()
+        with caplog.at_level(
+            logging.DEBUG, logger="scrapy.downloadermiddlewares.cookies"
+        ):
             req = Request("http://scrapytest.org/")
             res = Response("http://scrapytest.org/")  # no Set-Cookie header
             mw.process_response(req, res)
             mw.process_request(req)  # no cookies to send either
-            log.check()  # no log output since cl is empty in both cases
+        # no log output since cl is empty in both cases
+        assert caplog.record_tuples == []
 
-    def test_setting_disabled_cookies_debug(self):
+    def test_setting_disabled_cookies_debug(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         crawler = get_crawler(settings_dict={"COOKIES_DEBUG": False})
         mw = CookiesMiddleware.from_crawler(crawler)
-        with LogCapture(
-            "scrapy.downloadermiddlewares.cookies",
-            propagate=False,
-            level=logging.DEBUG,
-        ) as log:
+        caplog.clear()
+        with caplog.at_level(
+            logging.DEBUG, logger="scrapy.downloadermiddlewares.cookies"
+        ):
             req = Request("http://scrapytest.org/")
             res = Response(
                 "http://scrapytest.org/", headers={"Set-Cookie": "C1=value1; path=/"}
@@ -172,7 +173,7 @@ class TestCookiesMiddleware:
             req2 = Request("http://scrapytest.org/sub1/")
             mw.process_request(req2)
 
-            log.check()
+        assert caplog.record_tuples == []
 
     def test_do_not_break_on_non_utf8_header(self):
         req = Request("http://scrapytest.org/")
@@ -420,44 +421,41 @@ class TestCookiesMiddleware:
         assert self.mw.process_request(req3) is None
         self.assertCookieValEqual(req3.headers["Cookie"], b"a=\xc3\xa1")
 
-    def test_invalid_cookies(self):
+    def test_invalid_cookies(self, caplog: pytest.LogCaptureFixture) -> None:
         """
         Invalid cookies are logged as warnings and discarded
         """
-        with LogCapture(
-            "scrapy.utils.request",
-            propagate=False,
-            level=logging.INFO,
-        ) as lc:
+        caplog.clear()
+        with caplog.at_level(logging.INFO, logger="scrapy.utils.request"):
             cookies1 = [{"value": "bar"}, {"name": "key", "value": "value1"}]
-            req1 = Request("http://example.org/1", cookies=cookies1)
+            req1 = Request("http://example.org/1", cookies=cookies1)  # type: ignore[arg-type]
             assert self.mw.process_request(req1) is None
             cookies2 = [{"name": "foo"}, {"name": "key", "value": "value2"}]
-            req2 = Request("http://example.org/2", cookies=cookies2)
+            req2 = Request("http://example.org/2", cookies=cookies2)  # type: ignore[arg-type]
             assert self.mw.process_request(req2) is None
             cookies3 = [{"name": "foo", "value": None}, {"name": "key", "value": ""}]
-            req3 = Request("http://example.org/3", cookies=cookies3)
+            req3 = Request("http://example.org/3", cookies=cookies3)  # type: ignore[arg-type]
             assert self.mw.process_request(req3) is None
-            lc.check(
-                (
-                    "scrapy.utils.request",
-                    "WARNING",
-                    "Invalid cookie found in request <GET http://example.org/1>:"
-                    " {'value': 'bar', 'secure': False} ('name' is missing)",
-                ),
-                (
-                    "scrapy.utils.request",
-                    "WARNING",
-                    "Invalid cookie found in request <GET http://example.org/2>:"
-                    " {'name': 'foo', 'secure': False} ('value' is missing)",
-                ),
-                (
-                    "scrapy.utils.request",
-                    "WARNING",
-                    "Invalid cookie found in request <GET http://example.org/3>:"
-                    " {'name': 'foo', 'value': None, 'secure': False} ('value' is missing)",
-                ),
-            )
+        assert caplog.record_tuples == [
+            (
+                "scrapy.utils.request",
+                logging.WARNING,
+                "Invalid cookie found in request <GET http://example.org/1>:"
+                " {'value': 'bar', 'secure': False} ('name' is missing)",
+            ),
+            (
+                "scrapy.utils.request",
+                logging.WARNING,
+                "Invalid cookie found in request <GET http://example.org/2>:"
+                " {'name': 'foo', 'secure': False} ('value' is missing)",
+            ),
+            (
+                "scrapy.utils.request",
+                logging.WARNING,
+                "Invalid cookie found in request <GET http://example.org/3>:"
+                " {'name': 'foo', 'value': None, 'secure': False} ('value' is missing)",
+            ),
+        ]
         self.assertCookieValEqual(req1.headers["Cookie"], "key=value1")
         self.assertCookieValEqual(req2.headers["Cookie"], "key=value2")
         self.assertCookieValEqual(req3.headers["Cookie"], "key=")

tests/test_downloadermiddleware_httpcompression.py

@@ -5,7 +5,6 @@ from logging import WARNING
 from pathlib import Path
 
 import pytest
-from testfixtures import LogCapture
 from w3lib.encoding import resolve_encoding
 
 from scrapy.downloadermiddlewares.httpcompression import (
@@ -75,7 +74,7 @@ class TestHttpCompression:
         self.mw = HttpCompressionMiddleware.from_crawler(self.crawler)
         self.crawler.stats.open_spider()
 
-    def _getresponse(self, coding):
+    def _getresponse(self, coding: str) -> Response:
         if coding not in FORMAT:
             raise ValueError
 
@@ -169,29 +168,28 @@ class TestHttpCompression:
         self.assertStatsEqual("httpcompression/response_count", 1)
         self.assertStatsEqual("httpcompression/response_bytes", 74837)
 
-    def test_process_response_br_unsupported(self):
+    def test_process_response_br_unsupported(self, caplog: pytest.LogCaptureFixture):
         if find_spec("brotli") is not None or find_spec("brotlicffi") is not None:
             pytest.skip("Requires not having brotli support")
         response = self._getresponse("br")
         request = response.request
         assert response.headers["Content-Encoding"] == b"br"
-        with LogCapture(
-            "scrapy.downloadermiddlewares.httpcompression",
-            propagate=False,
-            level=WARNING,
-        ) as log:
+        caplog.clear()
+        with caplog.at_level(
+            WARNING, logger="scrapy.downloadermiddlewares.httpcompression"
+        ):
             newresponse = self.mw.process_response(request, response)
-        log.check(
+        assert caplog.record_tuples == [
             (
                 "scrapy.downloadermiddlewares.httpcompression",
-                "WARNING",
+                WARNING,
                 (
                     "HttpCompressionMiddleware cannot decode the response for "
                     "http://scrapytest.org/ from unsupported encoding(s) 'br'. "
                     "You need to install brotli or brotlicffi >= 1.2.0 to decode 'br'."
                 ),
             ),
-        )
+        ]
         assert newresponse is not response
         assert newresponse.headers.getlist("Content-Encoding") == [b"br"]
 
@@ -214,29 +212,28 @@ class TestHttpCompression:
             assert newresponse.body.startswith(b"<!DOCTYPE")
             assert "Content-Encoding" not in newresponse.headers
 
-    def test_process_response_zstd_unsupported(self):
+    def test_process_response_zstd_unsupported(self, caplog: pytest.LogCaptureFixture):
         if find_spec("zstandard") is not None:
             pytest.skip("Requires not having zstandard support")
         response = self._getresponse("zstd-static-content-size")
         request = response.request
         assert response.headers["Content-Encoding"] == b"zstd"
-        with LogCapture(
-            "scrapy.downloadermiddlewares.httpcompression",
-            propagate=False,
-            level=WARNING,
-        ) as log:
+        caplog.clear()
+        with caplog.at_level(
+            WARNING, logger="scrapy.downloadermiddlewares.httpcompression"
+        ):
             newresponse = self.mw.process_response(request, response)
-        log.check(
+        assert caplog.record_tuples == [
             (
                 "scrapy.downloadermiddlewares.httpcompression",
-                "WARNING",
+                WARNING,
                 (
                     "HttpCompressionMiddleware cannot decode the response for"
                     " http://scrapytest.org/ from unsupported encoding(s) 'zstd'."
                     " You need to install zstandard to decode 'zstd'."
                 ),
             ),
-        )
+        ]
         assert newresponse is not response
         assert newresponse.headers.getlist("Content-Encoding") == [b"zstd"]
 
@@ -291,26 +288,27 @@ class TestHttpCompression:
         assert "Content-Encoding" not in newresponse.headers
         assert newresponse.body.startswith(b"<!DOCTYPE")
 
-    def test_multi_compression_single_header_invalid_compression(self):
+    def test_multi_compression_single_header_invalid_compression(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         response = self._getresponse("gzip-deflate")
         response.headers["Content-Encoding"] = [b"gzip, foo, deflate"]
         request = response.request
-        with LogCapture(
-            "scrapy.downloadermiddlewares.httpcompression",
-            propagate=False,
-            level=WARNING,
-        ) as log:
+        caplog.clear()
+        with caplog.at_level(
+            WARNING, logger="scrapy.downloadermiddlewares.httpcompression"
+        ):
             newresponse = self.mw.process_response(request, response)
-        log.check(
+        assert caplog.record_tuples == [
             (
                 "scrapy.downloadermiddlewares.httpcompression",
-                "WARNING",
+                WARNING,
                 (
                     "HttpCompressionMiddleware cannot decode the response for"
                     " http://scrapytest.org/ from unsupported encoding(s) 'gzip,foo'."
                 ),
             ),
-        )
+        ]
         assert newresponse is not response
         assert newresponse.headers.getlist("Content-Encoding") == [b"gzip", b"foo"]
 
@@ -629,7 +627,9 @@ class TestHttpCompression:
 
         self._test_compression_bomb_request_meta("zstd")
 
-    def _test_download_warnsize_setting(self, compression_id):
+    def _test_download_warnsize_setting(
+        self, caplog: pytest.LogCaptureFixture, compression_id: str
+    ) -> None:
         settings = {"DOWNLOAD_WARNSIZE": 10_000_000}
         crawler = get_crawler(Spider, settings_dict=settings)
         spider = crawler._create_spider("scrapytest.org")
@@ -637,41 +637,51 @@ class TestHttpCompression:
         mw.open_spider(spider)
         response = self._getresponse(f"bomb-{compression_id}")
 
-        with LogCapture(
-            "scrapy.downloadermiddlewares.httpcompression",
-            propagate=False,
-            level=WARNING,
-        ) as log:
+        assert response.request
+        caplog.clear()
+        with caplog.at_level(
+            WARNING, logger="scrapy.downloadermiddlewares.httpcompression"
+        ):
             mw.process_response(response.request, response)
-        log.check(
+        assert caplog.record_tuples == [
             (
                 "scrapy.downloadermiddlewares.httpcompression",
-                "WARNING",
+                WARNING,
                 (
                     "<200 http://scrapytest.org/> body size after "
                     "decompression (11511612 B) is larger than the download "
                     "warning size (10000000 B)."
                 ),
             ),
-        )
+        ]
 
-    def test_download_warnsize_setting_br(self):
+    def test_download_warnsize_setting_br(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         _skip_if_no_br()
 
-        self._test_download_warnsize_setting("br")
+        self._test_download_warnsize_setting(caplog, "br")
 
-    def test_download_warnsize_setting_deflate(self):
-        self._test_download_warnsize_setting("deflate")
+    def test_download_warnsize_setting_deflate(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
+        self._test_download_warnsize_setting(caplog, "deflate")
 
-    def test_download_warnsize_setting_gzip(self):
-        self._test_download_warnsize_setting("gzip")
+    def test_download_warnsize_setting_gzip(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
+        self._test_download_warnsize_setting(caplog, "gzip")
 
-    def test_download_warnsize_setting_zstd(self):
+    def test_download_warnsize_setting_zstd(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         _skip_if_no_zstd()
 
-        self._test_download_warnsize_setting("zstd")
+        self._test_download_warnsize_setting(caplog, "zstd")
 
-    def _test_download_warnsize_spider_attr(self, compression_id):
+    def _test_download_warnsize_spider_attr(
+        self, caplog: pytest.LogCaptureFixture, compression_id: str
+    ) -> None:
         class DownloadWarnSizeSpider(Spider):
             download_warnsize = 10_000_000
 
@@ -681,45 +691,55 @@ class TestHttpCompression:
         mw.open_spider(spider)
         response = self._getresponse(f"bomb-{compression_id}")
 
-        with LogCapture(
-            "scrapy.downloadermiddlewares.httpcompression",
-            propagate=False,
-            level=WARNING,
-        ) as log:
+        assert response.request
+        caplog.clear()
+        with caplog.at_level(
+            WARNING, logger="scrapy.downloadermiddlewares.httpcompression"
+        ):
             mw.process_response(response.request, response)
-        log.check(
+        assert caplog.record_tuples == [
             (
                 "scrapy.downloadermiddlewares.httpcompression",
-                "WARNING",
+                WARNING,
                 (
                     "<200 http://scrapytest.org/> body size after "
                     "decompression (11511612 B) is larger than the download "
                     "warning size (10000000 B)."
                 ),
             ),
-        )
+        ]
 
     @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
-    def test_download_warnsize_spider_attr_br(self):
+    def test_download_warnsize_spider_attr_br(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         _skip_if_no_br()
 
-        self._test_download_warnsize_spider_attr("br")
+        self._test_download_warnsize_spider_attr(caplog, "br")
 
     @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
-    def test_download_warnsize_spider_attr_deflate(self):
-        self._test_download_warnsize_spider_attr("deflate")
+    def test_download_warnsize_spider_attr_deflate(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
+        self._test_download_warnsize_spider_attr(caplog, "deflate")
 
     @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
-    def test_download_warnsize_spider_attr_gzip(self):
-        self._test_download_warnsize_spider_attr("gzip")
+    def test_download_warnsize_spider_attr_gzip(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
+        self._test_download_warnsize_spider_attr(caplog, "gzip")
 
     @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
-    def test_download_warnsize_spider_attr_zstd(self):
+    def test_download_warnsize_spider_attr_zstd(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         _skip_if_no_zstd()
 
-        self._test_download_warnsize_spider_attr("zstd")
+        self._test_download_warnsize_spider_attr(caplog, "zstd")
 
-    def _test_download_warnsize_request_meta(self, compression_id):
+    def _test_download_warnsize_request_meta(
+        self, caplog: pytest.LogCaptureFixture, compression_id: str
+    ) -> None:
         crawler = get_crawler(Spider)
         spider = crawler._create_spider("scrapytest.org")
         mw = HttpCompressionMiddleware.from_crawler(crawler)
@@ -727,39 +747,47 @@ class TestHttpCompression:
         response = self._getresponse(f"bomb-{compression_id}")
         response.meta["download_warnsize"] = 10_000_000
 
-        with LogCapture(
-            "scrapy.downloadermiddlewares.httpcompression",
-            propagate=False,
-            level=WARNING,
-        ) as log:
+        assert response.request
+        caplog.clear()
+        with caplog.at_level(
+            WARNING, logger="scrapy.downloadermiddlewares.httpcompression"
+        ):
             mw.process_response(response.request, response)
-        log.check(
+        assert caplog.record_tuples == [
             (
                 "scrapy.downloadermiddlewares.httpcompression",
-                "WARNING",
+                WARNING,
                 (
                     "<200 http://scrapytest.org/> body size after "
                     "decompression (11511612 B) is larger than the download "
                     "warning size (10000000 B)."
                 ),
             ),
-        )
+        ]
 
-    def test_download_warnsize_request_meta_br(self):
+    def test_download_warnsize_request_meta_br(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         _skip_if_no_br()
 
-        self._test_download_warnsize_request_meta("br")
+        self._test_download_warnsize_request_meta(caplog, "br")
 
-    def test_download_warnsize_request_meta_deflate(self):
-        self._test_download_warnsize_request_meta("deflate")
+    def test_download_warnsize_request_meta_deflate(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
+        self._test_download_warnsize_request_meta(caplog, "deflate")
 
-    def test_download_warnsize_request_meta_gzip(self):
-        self._test_download_warnsize_request_meta("gzip")
+    def test_download_warnsize_request_meta_gzip(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
+        self._test_download_warnsize_request_meta(caplog, "gzip")
 
-    def test_download_warnsize_request_meta_zstd(self):
+    def test_download_warnsize_request_meta_zstd(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         _skip_if_no_zstd()
 
-        self._test_download_warnsize_request_meta("zstd")
+        self._test_download_warnsize_request_meta(caplog, "zstd")
 
     def _get_truncated_response(self, compression_id):
         crawler = get_crawler(Spider)

tests/test_downloadermiddleware_offsite.py

@@ -1,3 +1,5 @@
+import re
+
 import pytest
 
 from scrapy import Request, Spider
@@ -231,3 +233,15 @@ def test_repeated_offsite_domain():
         mw.process_request(req2)
     assert crawler.stats.get_value("offsite/domains") == 1  # not incremented again
     assert crawler.stats.get_value("offsite/filtered") == 2
+
+
+def test_ignore_request_reason():
+    crawler = get_crawler(Spider)
+    crawler.spider = crawler._create_spider(name="a", allowed_domains=["example.com"])
+    mw = OffsiteMiddleware.from_crawler(crawler)
+    mw.spider_opened(crawler.spider)
+    request = Request("http://other.org/1")
+    with pytest.raises(
+        IgnoreRequest, match=re.escape("Filtered offsite request to 'other.org'")
+    ):
+        mw.process_request(request)

tests/test_downloadermiddleware_redirect.py

@@ -16,11 +16,11 @@ from scrapy.spiders import Spider
 from scrapy.utils.misc import build_from_crawler
 from scrapy.utils.spider import DefaultSpider
 from scrapy.utils.test import get_crawler
-from tests.test_downloadermiddleware_redirect_base import Base
+from tests.utils.bases.redirect import TestRedirectBase
 from tests.utils.redirect import REDIRECT_SCHEME_CASES, SCHEME_PARAMS
 
 
-class TestRedirectMiddleware(Base.Test):
+class TestRedirectMiddleware(TestRedirectBase):
     mwcls = RedirectMiddleware
     reason = 302
 

tests/test_downloadermiddleware_redirect_base.py

@@ -1,984 +0,0 @@
-from __future__ import annotations
-
-import pytest
-
-from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware
-from scrapy.exceptions import IgnoreRequest
-from scrapy.http import Request, Response
-from scrapy.utils.misc import set_environ
-from scrapy.utils.test import get_crawler
-
-
-class Base:
-    class Test:
-        def test_priority_adjust(self):
-            req = Request("http://a.example")
-            rsp = self.get_response(req, "http://a.example/redirected")
-            req2 = self.mw.process_response(req, rsp)
-            assert req2.priority > req.priority
-
-        def test_dont_redirect(self):
-            url = "http://www.example.com/301"
-            url2 = "http://www.example.com/redirected"
-            req = Request(url, meta={"dont_redirect": True})
-            rsp = self.get_response(req, url2)
-
-            r = self.mw.process_response(req, rsp)
-            assert isinstance(r, Response)
-            assert r is rsp
-
-            # Test that it redirects when dont_redirect is False
-            req = Request(url, meta={"dont_redirect": False})
-            rsp = self.get_response(req, url2)
-
-            r = self.mw.process_response(req, rsp)
-            assert isinstance(r, Request)
-
-        def test_post(self):
-            url = "http://www.example.com/302"
-            url2 = "http://www.example.com/redirected2"
-            req = Request(
-                url,
-                method="POST",
-                body="test",
-                headers={"Content-Type": "text/plain", "Content-length": "4"},
-            )
-            rsp = self.get_response(req, url2)
-
-            req2 = self.mw.process_response(req, rsp)
-            assert isinstance(req2, Request)
-            assert req2.url == url2
-            assert req2.method == "GET"
-            assert "Content-Type" not in req2.headers, (
-                "Content-Type header must not be present in redirected request"
-            )
-            assert "Content-Length" not in req2.headers, (
-                "Content-Length header must not be present in redirected request"
-            )
-            assert not req2.body, f"Redirected body must be empty, not '{req2.body}'"
-
-        def test_max_redirect_times(self):
-            self.mw.max_redirect_times = 1
-            req = Request("http://a.example/302")
-            rsp = self.get_response(req, "/redirected")
-
-            req = self.mw.process_response(req, rsp)
-            assert isinstance(req, Request)
-            assert "redirect_times" in req.meta
-            assert req.meta["redirect_times"] == 1
-            with pytest.raises(IgnoreRequest):
-                self.mw.process_response(req, rsp)
-
-        def test_ttl(self):
-            self.mw.max_redirect_times = 100
-            req = Request("http://a.example/302", meta={"redirect_ttl": 1})
-            rsp = self.get_response(req, "/a")
-
-            req = self.mw.process_response(req, rsp)
-            assert isinstance(req, Request)
-            with pytest.raises(IgnoreRequest):
-                self.mw.process_response(req, rsp)
-
-        def test_redirect_urls(self):
-            req1 = Request("http://a.example/first")
-            rsp1 = self.get_response(req1, "/redirected")
-            req2 = self.mw.process_response(req1, rsp1)
-            rsp2 = self.get_response(req2, "/redirected2")
-            req3 = self.mw.process_response(req2, rsp2)
-
-            assert req2.url == "http://a.example/redirected"
-            assert req2.meta["redirect_urls"] == ["http://a.example/first"]
-            assert req3.url == "http://a.example/redirected2"
-            assert req3.meta["redirect_urls"] == [
-                "http://a.example/first",
-                "http://a.example/redirected",
-            ]
-
-        def test_redirect_reasons(self):
-            req1 = Request("http://a.example/first")
-            rsp1 = self.get_response(req1, "/redirected1")
-            req2 = self.mw.process_response(req1, rsp1)
-            rsp2 = self.get_response(req2, "/redirected2")
-            req3 = self.mw.process_response(req2, rsp2)
-            assert req2.meta["redirect_reasons"] == [self.reason]
-            assert req3.meta["redirect_reasons"] == [self.reason, self.reason]
-
-        def test_cross_origin_header_dropping(self):
-            safe_headers = {"A": "B"}
-            cookie_header = {"Cookie": "a=b"}
-            authorization_header = {"Authorization": "Bearer 123456"}
-
-            original_request = Request(
-                "https://example.com",
-                headers={**safe_headers, **cookie_header, **authorization_header},
-            )
-
-            # Redirects to the same origin (same scheme, same domain, same port)
-            # keep all headers.
-            internal_response = self.get_response(
-                original_request, "https://example.com/a"
-            )
-            internal_redirect_request = self.mw.process_response(
-                original_request, internal_response
-            )
-            assert isinstance(internal_redirect_request, Request)
-            assert original_request.headers == internal_redirect_request.headers
-
-            # Redirects to the same origin (same scheme, same domain, same port)
-            # keep all headers also when the scheme is http.
-            http_request = Request(
-                "http://example.com",
-                headers={**safe_headers, **cookie_header, **authorization_header},
-            )
-            http_response = self.get_response(http_request, "http://example.com/a")
-            http_redirect_request = self.mw.process_response(
-                http_request, http_response
-            )
-            assert isinstance(http_redirect_request, Request)
-            assert http_request.headers == http_redirect_request.headers
-
-            # For default ports, whether the port is explicit or implicit does not
-            # affect the outcome, it is still the same origin.
-            to_explicit_port_response = self.get_response(
-                original_request, "https://example.com:443/a"
-            )
-            to_explicit_port_redirect_request = self.mw.process_response(
-                original_request, to_explicit_port_response
-            )
-            assert isinstance(to_explicit_port_redirect_request, Request)
-            assert original_request.headers == to_explicit_port_redirect_request.headers
-
-            # For default ports, whether the port is explicit or implicit does not
-            # affect the outcome, it is still the same origin.
-            to_implicit_port_response = self.get_response(
-                original_request, "https://example.com/a"
-            )
-            to_implicit_port_redirect_request = self.mw.process_response(
-                original_request, to_implicit_port_response
-            )
-            assert isinstance(to_implicit_port_redirect_request, Request)
-            assert original_request.headers == to_implicit_port_redirect_request.headers
-
-            # A port change drops the Authorization header because the origin
-            # changes, but keeps the Cookie header because the domain remains the
-            # same.
-            different_port_response = self.get_response(
-                original_request, "https://example.com:8080/a"
-            )
-            different_port_redirect_request = self.mw.process_response(
-                original_request, different_port_response
-            )
-            assert isinstance(different_port_redirect_request, Request)
-            assert {
-                **safe_headers,
-                **cookie_header,
-            } == different_port_redirect_request.headers.to_unicode_dict()
-
-            # A domain change drops both the Authorization and the Cookie header.
-            external_response = self.get_response(
-                original_request, "https://example.org/a"
-            )
-            external_redirect_request = self.mw.process_response(
-                original_request, external_response
-            )
-            assert isinstance(external_redirect_request, Request)
-            assert safe_headers == external_redirect_request.headers.to_unicode_dict()
-
-            # A scheme upgrade (http → https) drops the Authorization header
-            # because the origin changes, but keeps the Cookie header because the
-            # domain remains the same.
-            upgrade_response = self.get_response(http_request, "https://example.com/a")
-            upgrade_redirect_request = self.mw.process_response(
-                http_request, upgrade_response
-            )
-            assert isinstance(upgrade_redirect_request, Request)
-            assert {
-                **safe_headers,
-                **cookie_header,
-            } == upgrade_redirect_request.headers.to_unicode_dict()
-
-            # A scheme downgrade (https → http) drops the Authorization header
-            # because the origin changes, and the Cookie header because its value
-            # cannot indicate whether the cookies were secure (HTTPS-only) or not.
-            #
-            # Note: If the Cookie header is set by the cookie management
-            # middleware, as recommended in the docs, the dropping of Cookie on
-            # scheme downgrade is not an issue, because the cookie management
-            # middleware will add again the Cookie header to the new request if
-            # appropriate.
-            downgrade_response = self.get_response(
-                original_request, "http://example.com/a"
-            )
-            downgrade_redirect_request = self.mw.process_response(
-                original_request, downgrade_response
-            )
-            assert isinstance(downgrade_redirect_request, Request)
-            assert safe_headers == downgrade_redirect_request.headers.to_unicode_dict()
-
-        def test_meta_proxy_http_absolute(self):
-            crawler = get_crawler()
-            redirect_mw = self.mwcls.from_crawler(crawler)
-            proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
-
-            meta = {"proxy": "https://a:@a.example"}
-            request1 = Request("http://example.com", meta=meta)
-            proxy_mw.process_request(request1)
-
-            assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request1.meta["_auth_proxy"] == "https://a.example"
-            assert request1.meta["proxy"] == "https://a.example"
-
-            response1 = self.get_response(request1, "http://example.com")
-            request2 = redirect_mw.process_response(request1, response1)
-
-            assert isinstance(request2, Request)
-            assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request2.meta["_auth_proxy"] == "https://a.example"
-            assert request2.meta["proxy"] == "https://a.example"
-
-            proxy_mw.process_request(request2)
-
-            assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request2.meta["_auth_proxy"] == "https://a.example"
-            assert request2.meta["proxy"] == "https://a.example"
-
-            response2 = self.get_response(request2, "http://example.com")
-            request3 = redirect_mw.process_response(request2, response2)
-
-            assert isinstance(request3, Request)
-            assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request3.meta["_auth_proxy"] == "https://a.example"
-            assert request3.meta["proxy"] == "https://a.example"
-
-            proxy_mw.process_request(request3)
-
-            assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request3.meta["_auth_proxy"] == "https://a.example"
-            assert request3.meta["proxy"] == "https://a.example"
-
-        def test_meta_proxy_http_relative(self):
-            crawler = get_crawler()
-            redirect_mw = self.mwcls.from_crawler(crawler)
-            proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
-
-            meta = {"proxy": "https://a:@a.example"}
-            request1 = Request("http://example.com", meta=meta)
-            proxy_mw.process_request(request1)
-
-            assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request1.meta["_auth_proxy"] == "https://a.example"
-            assert request1.meta["proxy"] == "https://a.example"
-
-            response1 = self.get_response(request1, "/a")
-            request2 = redirect_mw.process_response(request1, response1)
-
-            assert isinstance(request2, Request)
-            assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request2.meta["_auth_proxy"] == "https://a.example"
-            assert request2.meta["proxy"] == "https://a.example"
-
-            proxy_mw.process_request(request2)
-
-            assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request2.meta["_auth_proxy"] == "https://a.example"
-            assert request2.meta["proxy"] == "https://a.example"
-
-            response2 = self.get_response(request2, "/a")
-            request3 = redirect_mw.process_response(request2, response2)
-
-            assert isinstance(request3, Request)
-            assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request3.meta["_auth_proxy"] == "https://a.example"
-            assert request3.meta["proxy"] == "https://a.example"
-
-            proxy_mw.process_request(request3)
-
-            assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request3.meta["_auth_proxy"] == "https://a.example"
-            assert request3.meta["proxy"] == "https://a.example"
-
-        def test_meta_proxy_https_absolute(self):
-            crawler = get_crawler()
-            redirect_mw = self.mwcls.from_crawler(crawler)
-            proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
-
-            meta = {"proxy": "https://a:@a.example"}
-            request1 = Request("https://example.com", meta=meta)
-            proxy_mw.process_request(request1)
-
-            assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request1.meta["_auth_proxy"] == "https://a.example"
-            assert request1.meta["proxy"] == "https://a.example"
-
-            response1 = self.get_response(request1, "https://example.com")
-            request2 = redirect_mw.process_response(request1, response1)
-
-            assert isinstance(request2, Request)
-            assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request2.meta["_auth_proxy"] == "https://a.example"
-            assert request2.meta["proxy"] == "https://a.example"
-
-            proxy_mw.process_request(request2)
-
-            assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request2.meta["_auth_proxy"] == "https://a.example"
-            assert request2.meta["proxy"] == "https://a.example"
-
-            response2 = self.get_response(request2, "https://example.com")
-            request3 = redirect_mw.process_response(request2, response2)
-
-            assert isinstance(request3, Request)
-            assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request3.meta["_auth_proxy"] == "https://a.example"
-            assert request3.meta["proxy"] == "https://a.example"
-
-            proxy_mw.process_request(request3)
-
-            assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request3.meta["_auth_proxy"] == "https://a.example"
-            assert request3.meta["proxy"] == "https://a.example"
-
-        def test_meta_proxy_https_relative(self):
-            crawler = get_crawler()
-            redirect_mw = self.mwcls.from_crawler(crawler)
-            proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
-
-            meta = {"proxy": "https://a:@a.example"}
-            request1 = Request("https://example.com", meta=meta)
-            proxy_mw.process_request(request1)
-
-            assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request1.meta["_auth_proxy"] == "https://a.example"
-            assert request1.meta["proxy"] == "https://a.example"
-
-            response1 = self.get_response(request1, "/a")
-            request2 = redirect_mw.process_response(request1, response1)
-
-            assert isinstance(request2, Request)
-            assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request2.meta["_auth_proxy"] == "https://a.example"
-            assert request2.meta["proxy"] == "https://a.example"
-
-            proxy_mw.process_request(request2)
-
-            assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request2.meta["_auth_proxy"] == "https://a.example"
-            assert request2.meta["proxy"] == "https://a.example"
-
-            response2 = self.get_response(request2, "/a")
-            request3 = redirect_mw.process_response(request2, response2)
-
-            assert isinstance(request3, Request)
-            assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request3.meta["_auth_proxy"] == "https://a.example"
-            assert request3.meta["proxy"] == "https://a.example"
-
-            proxy_mw.process_request(request3)
-
-            assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request3.meta["_auth_proxy"] == "https://a.example"
-            assert request3.meta["proxy"] == "https://a.example"
-
-        def test_meta_proxy_http_to_https(self):
-            crawler = get_crawler()
-            redirect_mw = self.mwcls.from_crawler(crawler)
-            proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
-
-            meta = {"proxy": "https://a:@a.example"}
-            request1 = Request("http://example.com", meta=meta)
-            proxy_mw.process_request(request1)
-
-            assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request1.meta["_auth_proxy"] == "https://a.example"
-            assert request1.meta["proxy"] == "https://a.example"
-
-            response1 = self.get_response(request1, "https://example.com")
-            request2 = redirect_mw.process_response(request1, response1)
-
-            assert isinstance(request2, Request)
-            assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request2.meta["_auth_proxy"] == "https://a.example"
-            assert request2.meta["proxy"] == "https://a.example"
-
-            proxy_mw.process_request(request2)
-
-            assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request2.meta["_auth_proxy"] == "https://a.example"
-            assert request2.meta["proxy"] == "https://a.example"
-
-            response2 = self.get_response(request2, "http://example.com")
-            request3 = redirect_mw.process_response(request2, response2)
-
-            assert isinstance(request3, Request)
-            assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request3.meta["_auth_proxy"] == "https://a.example"
-            assert request3.meta["proxy"] == "https://a.example"
-
-            proxy_mw.process_request(request3)
-
-            assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request3.meta["_auth_proxy"] == "https://a.example"
-            assert request3.meta["proxy"] == "https://a.example"
-
-        def test_meta_proxy_https_to_http(self):
-            crawler = get_crawler()
-            redirect_mw = self.mwcls.from_crawler(crawler)
-            proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
-
-            meta = {"proxy": "https://a:@a.example"}
-            request1 = Request("https://example.com", meta=meta)
-            proxy_mw.process_request(request1)
-
-            assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request1.meta["_auth_proxy"] == "https://a.example"
-            assert request1.meta["proxy"] == "https://a.example"
-
-            response1 = self.get_response(request1, "http://example.com")
-            request2 = redirect_mw.process_response(request1, response1)
-
-            assert isinstance(request2, Request)
-            assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request2.meta["_auth_proxy"] == "https://a.example"
-            assert request2.meta["proxy"] == "https://a.example"
-
-            proxy_mw.process_request(request2)
-
-            assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request2.meta["_auth_proxy"] == "https://a.example"
-            assert request2.meta["proxy"] == "https://a.example"
-
-            response2 = self.get_response(request2, "https://example.com")
-            request3 = redirect_mw.process_response(request2, response2)
-
-            assert isinstance(request3, Request)
-            assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request3.meta["_auth_proxy"] == "https://a.example"
-            assert request3.meta["proxy"] == "https://a.example"
-
-            proxy_mw.process_request(request3)
-
-            assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request3.meta["_auth_proxy"] == "https://a.example"
-            assert request3.meta["proxy"] == "https://a.example"
-
-        def test_system_proxy_http_absolute(self):
-            crawler = get_crawler()
-            redirect_mw = self.mwcls.from_crawler(crawler)
-            env = {
-                "http_proxy": "https://a:@a.example",
-            }
-            with set_environ(**env):
-                proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
-
-            request1 = Request("http://example.com")
-            proxy_mw.process_request(request1)
-
-            assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request1.meta["_auth_proxy"] == "https://a.example"
-            assert request1.meta["proxy"] == "https://a.example"
-
-            response1 = self.get_response(request1, "http://example.com")
-            request2 = redirect_mw.process_response(request1, response1)
-
-            assert isinstance(request2, Request)
-            assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request2.meta["_auth_proxy"] == "https://a.example"
-            assert request2.meta["proxy"] == "https://a.example"
-
-            proxy_mw.process_request(request2)
-
-            assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request2.meta["_auth_proxy"] == "https://a.example"
-            assert request2.meta["proxy"] == "https://a.example"
-
-            response2 = self.get_response(request2, "http://example.com")
-            request3 = redirect_mw.process_response(request2, response2)
-
-            assert isinstance(request3, Request)
-            assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request3.meta["_auth_proxy"] == "https://a.example"
-            assert request3.meta["proxy"] == "https://a.example"
-
-            proxy_mw.process_request(request3)
-
-            assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request3.meta["_auth_proxy"] == "https://a.example"
-            assert request3.meta["proxy"] == "https://a.example"
-
-        def test_system_proxy_http_relative(self):
-            crawler = get_crawler()
-            redirect_mw = self.mwcls.from_crawler(crawler)
-            env = {
-                "http_proxy": "https://a:@a.example",
-            }
-            with set_environ(**env):
-                proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
-
-            request1 = Request("http://example.com")
-            proxy_mw.process_request(request1)
-
-            assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request1.meta["_auth_proxy"] == "https://a.example"
-            assert request1.meta["proxy"] == "https://a.example"
-
-            response1 = self.get_response(request1, "/a")
-            request2 = redirect_mw.process_response(request1, response1)
-
-            assert isinstance(request2, Request)
-            assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request2.meta["_auth_proxy"] == "https://a.example"
-            assert request2.meta["proxy"] == "https://a.example"
-
-            proxy_mw.process_request(request2)
-
-            assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request2.meta["_auth_proxy"] == "https://a.example"
-            assert request2.meta["proxy"] == "https://a.example"
-
-            response2 = self.get_response(request2, "/a")
-            request3 = redirect_mw.process_response(request2, response2)
-
-            assert isinstance(request3, Request)
-            assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request3.meta["_auth_proxy"] == "https://a.example"
-            assert request3.meta["proxy"] == "https://a.example"
-
-            proxy_mw.process_request(request3)
-
-            assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request3.meta["_auth_proxy"] == "https://a.example"
-            assert request3.meta["proxy"] == "https://a.example"
-
-        def test_system_proxy_https_absolute(self):
-            crawler = get_crawler()
-            redirect_mw = self.mwcls.from_crawler(crawler)
-            env = {
-                "https_proxy": "https://a:@a.example",
-            }
-            with set_environ(**env):
-                proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
-
-            request1 = Request("https://example.com")
-            proxy_mw.process_request(request1)
-
-            assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request1.meta["_auth_proxy"] == "https://a.example"
-            assert request1.meta["proxy"] == "https://a.example"
-
-            response1 = self.get_response(request1, "https://example.com")
-            request2 = redirect_mw.process_response(request1, response1)
-
-            assert isinstance(request2, Request)
-            assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request2.meta["_auth_proxy"] == "https://a.example"
-            assert request2.meta["proxy"] == "https://a.example"
-
-            proxy_mw.process_request(request2)
-
-            assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request2.meta["_auth_proxy"] == "https://a.example"
-            assert request2.meta["proxy"] == "https://a.example"
-
-            response2 = self.get_response(request2, "https://example.com")
-            request3 = redirect_mw.process_response(request2, response2)
-
-            assert isinstance(request3, Request)
-            assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request3.meta["_auth_proxy"] == "https://a.example"
-            assert request3.meta["proxy"] == "https://a.example"
-
-            proxy_mw.process_request(request3)
-
-            assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request3.meta["_auth_proxy"] == "https://a.example"
-            assert request3.meta["proxy"] == "https://a.example"
-
-        def test_system_proxy_https_relative(self):
-            crawler = get_crawler()
-            redirect_mw = self.mwcls.from_crawler(crawler)
-            env = {
-                "https_proxy": "https://a:@a.example",
-            }
-            with set_environ(**env):
-                proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
-
-            request1 = Request("https://example.com")
-            proxy_mw.process_request(request1)
-
-            assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request1.meta["_auth_proxy"] == "https://a.example"
-            assert request1.meta["proxy"] == "https://a.example"
-
-            response1 = self.get_response(request1, "/a")
-            request2 = redirect_mw.process_response(request1, response1)
-
-            assert isinstance(request2, Request)
-            assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request2.meta["_auth_proxy"] == "https://a.example"
-            assert request2.meta["proxy"] == "https://a.example"
-
-            proxy_mw.process_request(request2)
-
-            assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request2.meta["_auth_proxy"] == "https://a.example"
-            assert request2.meta["proxy"] == "https://a.example"
-
-            response2 = self.get_response(request2, "/a")
-            request3 = redirect_mw.process_response(request2, response2)
-
-            assert isinstance(request3, Request)
-            assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request3.meta["_auth_proxy"] == "https://a.example"
-            assert request3.meta["proxy"] == "https://a.example"
-
-            proxy_mw.process_request(request3)
-
-            assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request3.meta["_auth_proxy"] == "https://a.example"
-            assert request3.meta["proxy"] == "https://a.example"
-
-        def test_system_proxy_proxied_http_to_proxied_https(self):
-            crawler = get_crawler()
-            redirect_mw = self.mwcls.from_crawler(crawler)
-            env = {
-                "http_proxy": "https://a:@a.example",
-                "https_proxy": "https://b:@b.example",
-            }
-            with set_environ(**env):
-                proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
-
-            request1 = Request("http://example.com")
-            proxy_mw.process_request(request1)
-
-            assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request1.meta["_auth_proxy"] == "https://a.example"
-            assert request1.meta["proxy"] == "https://a.example"
-
-            response1 = self.get_response(request1, "https://example.com")
-            request2 = redirect_mw.process_response(request1, response1)
-
-            assert isinstance(request2, Request)
-            assert "Proxy-Authorization" not in request2.headers
-            assert "_auth_proxy" not in request2.meta
-            assert "proxy" not in request2.meta
-
-            proxy_mw.process_request(request2)
-
-            assert request2.headers["Proxy-Authorization"] == b"Basic Yjo="
-            assert request2.meta["_auth_proxy"] == "https://b.example"
-            assert request2.meta["proxy"] == "https://b.example"
-
-            response2 = self.get_response(request2, "http://example.com")
-            request3 = redirect_mw.process_response(request2, response2)
-
-            assert isinstance(request3, Request)
-            assert "Proxy-Authorization" not in request3.headers
-            assert "_auth_proxy" not in request3.meta
-            assert "proxy" not in request3.meta
-
-            proxy_mw.process_request(request3)
-
-            assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request3.meta["_auth_proxy"] == "https://a.example"
-            assert request3.meta["proxy"] == "https://a.example"
-
-        def test_system_proxy_proxied_http_to_unproxied_https(self):
-            crawler = get_crawler()
-            redirect_mw = self.mwcls.from_crawler(crawler)
-            env = {
-                "http_proxy": "https://a:@a.example",
-            }
-            with set_environ(**env):
-                proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
-
-            request1 = Request("http://example.com")
-            proxy_mw.process_request(request1)
-
-            assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request1.meta["_auth_proxy"] == "https://a.example"
-            assert request1.meta["proxy"] == "https://a.example"
-
-            response1 = self.get_response(request1, "https://example.com")
-            request2 = redirect_mw.process_response(request1, response1)
-
-            assert isinstance(request2, Request)
-            assert "Proxy-Authorization" not in request2.headers
-            assert "_auth_proxy" not in request2.meta
-            assert "proxy" not in request2.meta
-
-            proxy_mw.process_request(request2)
-
-            assert "Proxy-Authorization" not in request2.headers
-            assert "_auth_proxy" not in request2.meta
-            assert "proxy" not in request2.meta
-
-            response2 = self.get_response(request2, "http://example.com")
-            request3 = redirect_mw.process_response(request2, response2)
-
-            assert isinstance(request3, Request)
-            assert "Proxy-Authorization" not in request3.headers
-            assert "_auth_proxy" not in request3.meta
-            assert "proxy" not in request3.meta
-
-            proxy_mw.process_request(request3)
-
-            assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request3.meta["_auth_proxy"] == "https://a.example"
-            assert request3.meta["proxy"] == "https://a.example"
-
-        def test_system_proxy_unproxied_http_to_proxied_https(self):
-            crawler = get_crawler()
-            redirect_mw = self.mwcls.from_crawler(crawler)
-            env = {
-                "https_proxy": "https://b:@b.example",
-            }
-            with set_environ(**env):
-                proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
-
-            request1 = Request("http://example.com")
-            proxy_mw.process_request(request1)
-
-            assert "Proxy-Authorization" not in request1.headers
-            assert "_auth_proxy" not in request1.meta
-            assert "proxy" not in request1.meta
-
-            response1 = self.get_response(request1, "https://example.com")
-            request2 = redirect_mw.process_response(request1, response1)
-
-            assert isinstance(request2, Request)
-            assert "Proxy-Authorization" not in request2.headers
-            assert "_auth_proxy" not in request2.meta
-            assert "proxy" not in request2.meta
-
-            proxy_mw.process_request(request2)
-
-            assert request2.headers["Proxy-Authorization"] == b"Basic Yjo="
-            assert request2.meta["_auth_proxy"] == "https://b.example"
-            assert request2.meta["proxy"] == "https://b.example"
-
-            response2 = self.get_response(request2, "http://example.com")
-            request3 = redirect_mw.process_response(request2, response2)
-
-            assert isinstance(request3, Request)
-            assert "Proxy-Authorization" not in request3.headers
-            assert "_auth_proxy" not in request3.meta
-            assert "proxy" not in request3.meta
-
-            proxy_mw.process_request(request3)
-
-            assert "Proxy-Authorization" not in request3.headers
-            assert "_auth_proxy" not in request3.meta
-            assert "proxy" not in request3.meta
-
-        def test_system_proxy_unproxied_http_to_unproxied_https(self):
-            crawler = get_crawler()
-            redirect_mw = self.mwcls.from_crawler(crawler)
-            proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
-
-            request1 = Request("http://example.com")
-            proxy_mw.process_request(request1)
-
-            assert "Proxy-Authorization" not in request1.headers
-            assert "_auth_proxy" not in request1.meta
-            assert "proxy" not in request1.meta
-
-            response1 = self.get_response(request1, "https://example.com")
-            request2 = redirect_mw.process_response(request1, response1)
-
-            assert isinstance(request2, Request)
-            assert "Proxy-Authorization" not in request2.headers
-            assert "_auth_proxy" not in request2.meta
-            assert "proxy" not in request2.meta
-
-            proxy_mw.process_request(request2)
-
-            assert "Proxy-Authorization" not in request2.headers
-            assert "_auth_proxy" not in request2.meta
-            assert "proxy" not in request2.meta
-
-            response2 = self.get_response(request2, "http://example.com")
-            request3 = redirect_mw.process_response(request2, response2)
-
-            assert isinstance(request3, Request)
-            assert "Proxy-Authorization" not in request3.headers
-            assert "_auth_proxy" not in request3.meta
-            assert "proxy" not in request3.meta
-
-            proxy_mw.process_request(request3)
-
-            assert "Proxy-Authorization" not in request3.headers
-            assert "_auth_proxy" not in request3.meta
-            assert "proxy" not in request3.meta
-
-        def test_system_proxy_proxied_https_to_proxied_http(self):
-            crawler = get_crawler()
-            redirect_mw = self.mwcls.from_crawler(crawler)
-            env = {
-                "http_proxy": "https://a:@a.example",
-                "https_proxy": "https://b:@b.example",
-            }
-            with set_environ(**env):
-                proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
-
-            request1 = Request("https://example.com")
-            proxy_mw.process_request(request1)
-
-            assert request1.headers["Proxy-Authorization"] == b"Basic Yjo="
-            assert request1.meta["_auth_proxy"] == "https://b.example"
-            assert request1.meta["proxy"] == "https://b.example"
-
-            response1 = self.get_response(request1, "http://example.com")
-            request2 = redirect_mw.process_response(request1, response1)
-
-            assert isinstance(request2, Request)
-            assert "Proxy-Authorization" not in request2.headers
-            assert "_auth_proxy" not in request2.meta
-            assert "proxy" not in request2.meta
-
-            proxy_mw.process_request(request2)
-
-            assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request2.meta["_auth_proxy"] == "https://a.example"
-            assert request2.meta["proxy"] == "https://a.example"
-
-            response2 = self.get_response(request2, "https://example.com")
-            request3 = redirect_mw.process_response(request2, response2)
-
-            assert isinstance(request3, Request)
-            assert "Proxy-Authorization" not in request3.headers
-            assert "_auth_proxy" not in request3.meta
-            assert "proxy" not in request3.meta
-
-            proxy_mw.process_request(request3)
-
-            assert request3.headers["Proxy-Authorization"] == b"Basic Yjo="
-            assert request3.meta["_auth_proxy"] == "https://b.example"
-            assert request3.meta["proxy"] == "https://b.example"
-
-        def test_system_proxy_proxied_https_to_unproxied_http(self):
-            crawler = get_crawler()
-            redirect_mw = self.mwcls.from_crawler(crawler)
-            env = {
-                "https_proxy": "https://b:@b.example",
-            }
-            with set_environ(**env):
-                proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
-
-            request1 = Request("https://example.com")
-            proxy_mw.process_request(request1)
-
-            assert request1.headers["Proxy-Authorization"] == b"Basic Yjo="
-            assert request1.meta["_auth_proxy"] == "https://b.example"
-            assert request1.meta["proxy"] == "https://b.example"
-
-            response1 = self.get_response(request1, "http://example.com")
-            request2 = redirect_mw.process_response(request1, response1)
-
-            assert isinstance(request2, Request)
-            assert "Proxy-Authorization" not in request2.headers
-            assert "_auth_proxy" not in request2.meta
-            assert "proxy" not in request2.meta
-
-            proxy_mw.process_request(request2)
-
-            assert "Proxy-Authorization" not in request2.headers
-            assert "_auth_proxy" not in request2.meta
-            assert "proxy" not in request2.meta
-
-            response2 = self.get_response(request2, "https://example.com")
-            request3 = redirect_mw.process_response(request2, response2)
-
-            assert isinstance(request3, Request)
-            assert "Proxy-Authorization" not in request3.headers
-            assert "_auth_proxy" not in request3.meta
-            assert "proxy" not in request3.meta
-
-            proxy_mw.process_request(request3)
-
-            assert request3.headers["Proxy-Authorization"] == b"Basic Yjo="
-            assert request3.meta["_auth_proxy"] == "https://b.example"
-            assert request3.meta["proxy"] == "https://b.example"
-
-        def test_system_proxy_unproxied_https_to_proxied_http(self):
-            crawler = get_crawler()
-            redirect_mw = self.mwcls.from_crawler(crawler)
-            env = {
-                "http_proxy": "https://a:@a.example",
-            }
-            with set_environ(**env):
-                proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
-
-            request1 = Request("https://example.com")
-            proxy_mw.process_request(request1)
-
-            assert "Proxy-Authorization" not in request1.headers
-            assert "_auth_proxy" not in request1.meta
-            assert "proxy" not in request1.meta
-
-            response1 = self.get_response(request1, "http://example.com")
-            request2 = redirect_mw.process_response(request1, response1)
-
-            assert isinstance(request2, Request)
-            assert "Proxy-Authorization" not in request2.headers
-            assert "_auth_proxy" not in request2.meta
-            assert "proxy" not in request2.meta
-
-            proxy_mw.process_request(request2)
-
-            assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request2.meta["_auth_proxy"] == "https://a.example"
-            assert request2.meta["proxy"] == "https://a.example"
-
-            response2 = self.get_response(request2, "https://example.com")
-            request3 = redirect_mw.process_response(request2, response2)
-
-            assert isinstance(request3, Request)
-            assert "Proxy-Authorization" not in request3.headers
-            assert "_auth_proxy" not in request3.meta
-            assert "proxy" not in request3.meta
-
-            proxy_mw.process_request(request3)
-
-            assert "Proxy-Authorization" not in request3.headers
-            assert "_auth_proxy" not in request3.meta
-            assert "proxy" not in request3.meta
-
-        def test_system_proxy_unproxied_https_to_unproxied_http(self):
-            crawler = get_crawler()
-            redirect_mw = self.mwcls.from_crawler(crawler)
-            proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
-
-            request1 = Request("https://example.com")
-            proxy_mw.process_request(request1)
-
-            assert "Proxy-Authorization" not in request1.headers
-            assert "_auth_proxy" not in request1.meta
-            assert "proxy" not in request1.meta
-
-            response1 = self.get_response(request1, "http://example.com")
-            request2 = redirect_mw.process_response(request1, response1)
-
-            assert isinstance(request2, Request)
-            assert "Proxy-Authorization" not in request2.headers
-            assert "_auth_proxy" not in request2.meta
-            assert "proxy" not in request2.meta
-
-            proxy_mw.process_request(request2)
-
-            assert "Proxy-Authorization" not in request2.headers
-            assert "_auth_proxy" not in request2.meta
-            assert "proxy" not in request2.meta
-
-            response2 = self.get_response(request2, "https://example.com")
-            request3 = redirect_mw.process_response(request2, response2)
-
-            assert isinstance(request3, Request)
-            assert "Proxy-Authorization" not in request3.headers
-            assert "_auth_proxy" not in request3.meta
-            assert "proxy" not in request3.meta
-
-            proxy_mw.process_request(request3)
-
-            assert "Proxy-Authorization" not in request3.headers
-            assert "_auth_proxy" not in request3.meta
-            assert "proxy" not in request3.meta

tests/test_downloadermiddleware_redirect_metarefresh.py

@@ -12,7 +12,7 @@ from scrapy.http import HtmlResponse, Request, Response
 from scrapy.spiders import Spider
 from scrapy.utils.misc import build_from_crawler
 from scrapy.utils.test import get_crawler
-from tests.test_downloadermiddleware_redirect_base import Base
+from tests.utils.bases.redirect import TestRedirectBase
 from tests.utils.redirect import (
     HTTP_SCHEMES,
     NON_HTTP_SCHEMES,
@@ -26,7 +26,7 @@ def meta_refresh_body(url, interval=5):
     return html.encode("utf-8")
 
 
-class TestMetaRefreshMiddleware(Base.Test):
+class TestMetaRefreshMiddleware(TestRedirectBase):
     mwcls = MetaRefreshMiddleware
     reason = "meta refresh"
 

tests/test_downloadermiddleware_retry.py

@@ -1,7 +1,9 @@
+from __future__ import annotations
+
 import logging
+from typing import Any, cast
 
 import pytest
-from testfixtures import LogCapture
 from twisted.internet.error import ConnectError, ConnectionDone, ConnectionLost
 
 from scrapy.downloadermiddlewares.retry import RetryMiddleware, get_retry_request
@@ -85,7 +87,7 @@ class TestRetry:
         )
         assert self.crawler.stats.get_value("retry/count") == 2
 
-    def test_give_up_log_level_setting(self):
+    def test_give_up_log_level_setting(self, caplog: pytest.LogCaptureFixture) -> None:
         crawler = get_crawler(
             DefaultSpider, settings_dict={"RETRY_GIVE_UP_LOG_LEVEL": "WARNING"}
         )
@@ -94,29 +96,25 @@ class TestRetry:
         mw.max_retry_times = 0
         req = Request("http://example.com/503")
         rsp = Response("http://example.com/503", body=b"", status=503)
-        with LogCapture() as log:
+        with caplog.at_level(logging.WARNING):
             assert mw.process_response(req, rsp) is rsp
-        log.check_present(
-            (
-                "scrapy.downloadermiddlewares.retry",
-                "WARNING",
-                f"Gave up retrying {req} (failed 1 times): 503 Service Unavailable",
-            )
-        )
+        assert (
+            "scrapy.downloadermiddlewares.retry",
+            logging.WARNING,
+            f"Gave up retrying {req} (failed 1 times): 503 Service Unavailable",
+        ) in caplog.record_tuples
 
-    def test_give_up_log_level_meta(self):
+    def test_give_up_log_level_meta(self, caplog: pytest.LogCaptureFixture) -> None:
         self.mw.max_retry_times = 0
         req = Request("http://example.com/503", meta={"give_up_log_level": "WARNING"})
         rsp = Response("http://example.com/503", body=b"", status=503)
-        with LogCapture() as log:
+        with caplog.at_level(logging.WARNING):
             assert self.mw.process_response(req, rsp) is rsp
-        log.check_present(
-            (
-                "scrapy.downloadermiddlewares.retry",
-                "WARNING",
-                f"Gave up retrying {req} (failed 1 times): 503 Service Unavailable",
-            )
-        )
+        assert (
+            "scrapy.downloadermiddlewares.retry",
+            logging.WARNING,
+            f"Gave up retrying {req} (failed 1 times): 503 Service Unavailable",
+        ) in caplog.record_tuples
 
     def test_twistederrors(self):
         exceptions = [
@@ -294,14 +292,15 @@ class TestMaxRetryTimes:
 
 
 class TestGetRetryRequest:
-    def get_spider(self, settings=None):
+    @staticmethod
+    def get_spider(settings: dict[str, Any] | None = None) -> Spider:
         crawler = get_crawler(Spider, settings or {})
         return crawler._create_spider("foo")
 
-    def test_basic_usage(self):
+    def test_basic_usage(self, caplog: pytest.LogCaptureFixture) -> None:
         request = Request("https://example.com")
         spider = self.get_spider()
-        with LogCapture() as log:
+        with caplog.at_level(logging.DEBUG):
             new_request = get_retry_request(
                 request,
                 spider=spider,
@@ -313,44 +312,42 @@ class TestGetRetryRequest:
         assert new_request.meta["retry_times"] == expected_retry_times
         assert new_request.priority == -1
         expected_reason = "unspecified"
+        assert spider.crawler.stats
         for stat in ("retry/count", f"retry/reason_count/{expected_reason}"):
             assert spider.crawler.stats.get_value(stat) == 1
-        log.check_present(
-            (
-                "scrapy.downloadermiddlewares.retry",
-                "DEBUG",
-                f"Retrying {request} (failed {expected_retry_times} times): "
-                f"{expected_reason}",
-            )
-        )
+        assert (
+            "scrapy.downloadermiddlewares.retry",
+            logging.DEBUG,
+            f"Retrying {request} (failed {expected_retry_times} times): "
+            f"{expected_reason}",
+        ) in caplog.record_tuples
 
-    def test_max_retries_reached(self):
+    def test_max_retries_reached(self, caplog: pytest.LogCaptureFixture) -> None:
         request = Request("https://example.com")
         spider = self.get_spider()
         max_retry_times = 0
-        with LogCapture() as log:
+        with caplog.at_level(logging.DEBUG):
             new_request = get_retry_request(
                 request,
                 spider=spider,
                 max_retry_times=max_retry_times,
             )
         assert new_request is None
+        assert spider.crawler.stats
         assert spider.crawler.stats.get_value("retry/max_reached") == 1
         failure_count = max_retry_times + 1
         expected_reason = "unspecified"
-        log.check_present(
-            (
-                "scrapy.downloadermiddlewares.retry",
-                "ERROR",
-                f"Gave up retrying {request} (failed {failure_count} times): "
-                f"{expected_reason}",
-            )
-        )
+        assert (
+            "scrapy.downloadermiddlewares.retry",
+            logging.ERROR,
+            f"Gave up retrying {request} (failed {failure_count} times): "
+            f"{expected_reason}",
+        ) in caplog.record_tuples
 
-    def test_one_retry(self):
+    def test_one_retry(self, caplog: pytest.LogCaptureFixture) -> None:
         request = Request("https://example.com")
         spider = self.get_spider()
-        with LogCapture() as log:
+        with caplog.at_level(logging.DEBUG):
             new_request = get_retry_request(
                 request,
                 spider=spider,
@@ -363,28 +360,31 @@ class TestGetRetryRequest:
         assert new_request.meta["retry_times"] == expected_retry_times
         assert new_request.priority == -1
         expected_reason = "unspecified"
+        assert spider.crawler.stats
         for stat in ("retry/count", f"retry/reason_count/{expected_reason}"):
             assert spider.crawler.stats.get_value(stat) == 1
-        log.check_present(
-            (
-                "scrapy.downloadermiddlewares.retry",
-                "DEBUG",
-                f"Retrying {request} (failed {expected_retry_times} times): "
-                f"{expected_reason}",
-            )
-        )
+        assert (
+            "scrapy.downloadermiddlewares.retry",
+            logging.DEBUG,
+            f"Retrying {request} (failed {expected_retry_times} times): "
+            f"{expected_reason}",
+        ) in caplog.record_tuples
 
-    def test_two_retries(self):
+    def test_two_retries(self, caplog: pytest.LogCaptureFixture) -> None:
         spider = self.get_spider()
         request = Request("https://example.com")
         new_request = request
         max_retry_times = 2
         for index in range(max_retry_times):
-            with LogCapture() as log:
-                new_request = get_retry_request(
-                    new_request,
-                    spider=spider,
-                    max_retry_times=max_retry_times,
+            caplog.clear()
+            with caplog.at_level(logging.DEBUG):
+                new_request = cast(
+                    "Request",
+                    get_retry_request(
+                        new_request,
+                        spider=spider,
+                        max_retry_times=max_retry_times,
+                    ),
                 )
             assert isinstance(new_request, Request)
             assert new_request != request
@@ -393,36 +393,37 @@ class TestGetRetryRequest:
             assert new_request.meta["retry_times"] == expected_retry_times
             assert new_request.priority == -expected_retry_times
             expected_reason = "unspecified"
+            assert spider.crawler.stats
             for stat in ("retry/count", f"retry/reason_count/{expected_reason}"):
                 value = spider.crawler.stats.get_value(stat)
                 assert value == expected_retry_times
-            log.check_present(
-                (
-                    "scrapy.downloadermiddlewares.retry",
-                    "DEBUG",
-                    f"Retrying {request} (failed {expected_retry_times} times): "
-                    f"{expected_reason}",
-                )
-            )
+            assert (
+                "scrapy.downloadermiddlewares.retry",
+                logging.DEBUG,
+                f"Retrying {request} (failed {expected_retry_times} times): "
+                f"{expected_reason}",
+            ) in caplog.record_tuples
 
-        with LogCapture() as log:
-            new_request = get_retry_request(
-                new_request,
-                spider=spider,
-                max_retry_times=max_retry_times,
+        caplog.clear()
+        with caplog.at_level(logging.DEBUG):
+            new_request = cast(
+                "Request",
+                get_retry_request(
+                    new_request,
+                    spider=spider,
+                    max_retry_times=max_retry_times,
+                ),
             )
         assert new_request is None
         assert spider.crawler.stats.get_value("retry/max_reached") == 1
         failure_count = max_retry_times + 1
         expected_reason = "unspecified"
-        log.check_present(
-            (
-                "scrapy.downloadermiddlewares.retry",
-                "ERROR",
-                f"Gave up retrying {request} (failed {failure_count} times): "
-                f"{expected_reason}",
-            )
-        )
+        assert (
+            "scrapy.downloadermiddlewares.retry",
+            logging.ERROR,
+            f"Gave up retrying {request} (failed {failure_count} times): "
+            f"{expected_reason}",
+        ) in caplog.record_tuples
 
     def test_no_spider(self):
         request = Request("https://example.com")
@@ -483,238 +484,231 @@ class TestGetRetryRequest:
         )
         assert new_request.priority == priority_adjust
 
-    def test_log_extra_retry_success(self):
+    def test_log_extra_retry_success(self, caplog: pytest.LogCaptureFixture) -> None:
         request = Request("https://example.com")
         spider = self.get_spider()
-        with LogCapture(attributes=("spider",)) as log:
+        with caplog.at_level(logging.DEBUG):
             get_retry_request(
                 request,
                 spider=spider,
             )
-        log.check_present(spider)
+        assert any(getattr(r, "spider", None) is spider for r in caplog.records)
 
-    def test_log_extra_retries_exceeded(self):
+    def test_log_extra_retries_exceeded(self, caplog: pytest.LogCaptureFixture) -> None:
         request = Request("https://example.com")
         spider = self.get_spider()
-        with LogCapture(attributes=("spider",)) as log:
+        with caplog.at_level(logging.DEBUG):
             get_retry_request(
                 request,
                 spider=spider,
                 max_retry_times=0,
             )
-        log.check_present(spider)
+        assert any(getattr(r, "spider", None) is spider for r in caplog.records)
 
-    def test_reason_string(self):
+    def test_reason_string(self, caplog: pytest.LogCaptureFixture) -> None:
         request = Request("https://example.com")
         spider = self.get_spider()
         expected_reason = "because"
-        with LogCapture() as log:
+        with caplog.at_level(logging.DEBUG):
             get_retry_request(
                 request,
                 spider=spider,
                 reason=expected_reason,
             )
         expected_retry_times = 1
+        assert spider.crawler.stats
         for stat in ("retry/count", f"retry/reason_count/{expected_reason}"):
             assert spider.crawler.stats.get_value(stat) == 1
-        log.check_present(
-            (
-                "scrapy.downloadermiddlewares.retry",
-                "DEBUG",
-                f"Retrying {request} (failed {expected_retry_times} times): "
-                f"{expected_reason}",
-            )
-        )
+        assert (
+            "scrapy.downloadermiddlewares.retry",
+            logging.DEBUG,
+            f"Retrying {request} (failed {expected_retry_times} times): "
+            f"{expected_reason}",
+        ) in caplog.record_tuples
 
-    def test_reason_builtin_exception(self):
+    def test_reason_builtin_exception(self, caplog: pytest.LogCaptureFixture) -> None:
         request = Request("https://example.com")
         spider = self.get_spider()
         expected_reason = NotImplementedError()
         expected_reason_string = "builtins.NotImplementedError"
-        with LogCapture() as log:
+        with caplog.at_level(logging.DEBUG):
             get_retry_request(
                 request,
                 spider=spider,
                 reason=expected_reason,
             )
         expected_retry_times = 1
+        assert spider.crawler.stats
         stat = spider.crawler.stats.get_value(
             f"retry/reason_count/{expected_reason_string}"
         )
         assert stat == 1
-        log.check_present(
-            (
-                "scrapy.downloadermiddlewares.retry",
-                "DEBUG",
-                f"Retrying {request} (failed {expected_retry_times} times): "
-                f"{expected_reason}",
-            )
-        )
+        assert (
+            "scrapy.downloadermiddlewares.retry",
+            logging.DEBUG,
+            f"Retrying {request} (failed {expected_retry_times} times): "
+            f"{expected_reason}",
+        ) in caplog.record_tuples
 
-    def test_reason_builtin_exception_class(self):
+    def test_reason_builtin_exception_class(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         request = Request("https://example.com")
         spider = self.get_spider()
         expected_reason = NotImplementedError
         expected_reason_string = "builtins.NotImplementedError"
-        with LogCapture() as log:
+        with caplog.at_level(logging.DEBUG):
             get_retry_request(
                 request,
                 spider=spider,
                 reason=expected_reason,
             )
         expected_retry_times = 1
+        assert spider.crawler.stats
         stat = spider.crawler.stats.get_value(
             f"retry/reason_count/{expected_reason_string}"
         )
         assert stat == 1
-        log.check_present(
-            (
-                "scrapy.downloadermiddlewares.retry",
-                "DEBUG",
-                f"Retrying {request} (failed {expected_retry_times} times): "
-                f"{expected_reason}",
-            )
-        )
+        assert (
+            "scrapy.downloadermiddlewares.retry",
+            logging.DEBUG,
+            f"Retrying {request} (failed {expected_retry_times} times): "
+            f"{expected_reason}",
+        ) in caplog.record_tuples
 
-    def test_reason_custom_exception(self):
+    def test_reason_custom_exception(self, caplog: pytest.LogCaptureFixture) -> None:
         request = Request("https://example.com")
         spider = self.get_spider()
         expected_reason = IgnoreRequest()
         expected_reason_string = "scrapy.exceptions.IgnoreRequest"
-        with LogCapture() as log:
+        with caplog.at_level(logging.DEBUG):
             get_retry_request(
                 request,
                 spider=spider,
                 reason=expected_reason,
             )
         expected_retry_times = 1
+        assert spider.crawler.stats
         stat = spider.crawler.stats.get_value(
             f"retry/reason_count/{expected_reason_string}"
         )
         assert stat == 1
-        log.check_present(
-            (
-                "scrapy.downloadermiddlewares.retry",
-                "DEBUG",
-                f"Retrying {request} (failed {expected_retry_times} times): "
-                f"{expected_reason}",
-            )
-        )
+        assert (
+            "scrapy.downloadermiddlewares.retry",
+            logging.DEBUG,
+            f"Retrying {request} (failed {expected_retry_times} times): "
+            f"{expected_reason}",
+        ) in caplog.record_tuples
 
-    def test_reason_custom_exception_class(self):
+    def test_reason_custom_exception_class(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         request = Request("https://example.com")
         spider = self.get_spider()
         expected_reason = IgnoreRequest
         expected_reason_string = "scrapy.exceptions.IgnoreRequest"
-        with LogCapture() as log:
+        with caplog.at_level(logging.DEBUG):
             get_retry_request(
                 request,
                 spider=spider,
                 reason=expected_reason,
             )
         expected_retry_times = 1
+        assert spider.crawler.stats
         stat = spider.crawler.stats.get_value(
             f"retry/reason_count/{expected_reason_string}"
         )
         assert stat == 1
-        log.check_present(
-            (
-                "scrapy.downloadermiddlewares.retry",
-                "DEBUG",
-                f"Retrying {request} (failed {expected_retry_times} times): "
-                f"{expected_reason}",
-            )
-        )
+        assert (
+            "scrapy.downloadermiddlewares.retry",
+            logging.DEBUG,
+            f"Retrying {request} (failed {expected_retry_times} times): "
+            f"{expected_reason}",
+        ) in caplog.record_tuples
 
-    def test_custom_logger(self):
+    def test_custom_logger(self, caplog: pytest.LogCaptureFixture) -> None:
         logger = logging.getLogger("custom-logger")
         request = Request("https://example.com")
         spider = self.get_spider()
         expected_reason = "because"
-        with LogCapture() as log:
+        with caplog.at_level(logging.DEBUG):
             get_retry_request(
                 request,
                 spider=spider,
                 reason=expected_reason,
                 logger=logger,
             )
-        log.check_present(
-            (
-                "custom-logger",
-                "DEBUG",
-                f"Retrying {request} (failed 1 times): {expected_reason}",
-            )
-        )
+        assert (
+            "custom-logger",
+            logging.DEBUG,
+            f"Retrying {request} (failed 1 times): {expected_reason}",
+        ) in caplog.record_tuples
 
-    def test_give_up_log_level_default(self):
+    def test_give_up_log_level_default(self, caplog: pytest.LogCaptureFixture) -> None:
         request = Request("https://example.com")
         spider = self.get_spider()
-        with LogCapture() as log:
+        with caplog.at_level(logging.ERROR):
             get_retry_request(
                 request,
                 spider=spider,
                 max_retry_times=0,
             )
-        log.check_present(
-            (
-                "scrapy.downloadermiddlewares.retry",
-                "ERROR",
-                f"Gave up retrying {request} (failed 1 times): unspecified",
-            )
-        )
+        assert (
+            "scrapy.downloadermiddlewares.retry",
+            logging.ERROR,
+            f"Gave up retrying {request} (failed 1 times): unspecified",
+        ) in caplog.record_tuples
 
-    def test_give_up_log_level_argument_name(self):
+    def test_give_up_log_level_argument_name(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         request = Request("https://example.com")
         spider = self.get_spider()
-        with LogCapture() as log:
+        with caplog.at_level(logging.WARNING):
             get_retry_request(
                 request,
                 spider=spider,
                 max_retry_times=0,
                 give_up_log_level="WARNING",
             )
-        log.check_present(
-            (
-                "scrapy.downloadermiddlewares.retry",
-                "WARNING",
-                f"Gave up retrying {request} (failed 1 times): unspecified",
-            )
-        )
+        assert (
+            "scrapy.downloadermiddlewares.retry",
+            logging.WARNING,
+            f"Gave up retrying {request} (failed 1 times): unspecified",
+        ) in caplog.record_tuples
 
-    def test_give_up_log_level_argument_number(self):
+    def test_give_up_log_level_argument_number(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         request = Request("https://example.com")
         spider = self.get_spider()
-        with LogCapture() as log:
+        with caplog.at_level(logging.WARNING):
             get_retry_request(
                 request,
                 spider=spider,
                 max_retry_times=0,
                 give_up_log_level=logging.WARNING,
             )
-        log.check_present(
-            (
-                "scrapy.downloadermiddlewares.retry",
-                "WARNING",
-                f"Gave up retrying {request} (failed 1 times): unspecified",
-            )
-        )
+        assert (
+            "scrapy.downloadermiddlewares.retry",
+            logging.WARNING,
+            f"Gave up retrying {request} (failed 1 times): unspecified",
+        ) in caplog.record_tuples
 
-    def test_give_up_log_level_setting(self):
+    def test_give_up_log_level_setting(self, caplog: pytest.LogCaptureFixture) -> None:
         request = Request("https://example.com")
         spider = self.get_spider({"RETRY_GIVE_UP_LOG_LEVEL": "WARNING"})
-        with LogCapture() as log:
+        with caplog.at_level(logging.WARNING):
             get_retry_request(
                 request,
                 spider=spider,
                 max_retry_times=0,
             )
-        log.check_present(
-            (
-                "scrapy.downloadermiddlewares.retry",
-                "WARNING",
-                f"Gave up retrying {request} (failed 1 times): unspecified",
-            )
-        )
+        assert (
+            "scrapy.downloadermiddlewares.retry",
+            logging.WARNING,
+            f"Gave up retrying {request} (failed 1 times): unspecified",
+        ) in caplog.record_tuples
 
     def test_give_up_log_level_invalid(self):
         request = Request("https://example.com")

tests/test_downloadermiddleware_stats.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import pytest
 
 from scrapy.downloadermiddlewares.stats import DownloaderStats, get_header_size
@@ -12,8 +14,9 @@ class MyException(Exception):
 
 
 class TestDownloaderStats:
-    def setup_method(self):
+    def setup_method(self) -> None:
         self.crawler = get_crawler(Spider)
+        assert self.crawler.stats is not None
         self.mw = DownloaderStats(self.crawler.stats)
 
         self.crawler.stats.open_spider()
@@ -21,20 +24,21 @@ class TestDownloaderStats:
         self.req = Request("http://scrapytest.org")
         self.res = Response("http://scrapytest.org", status=400)
 
-    def assertStatsEqual(self, key, value):
+    def assertStatsEqual(self, key: str, value: object) -> None:
+        assert self.crawler.stats is not None
         assert self.crawler.stats.get_value(key) == value, str(
             self.crawler.stats.get_stats()
         )
 
-    def test_process_request(self):
+    def test_process_request(self) -> None:
         self.mw.process_request(self.req)
         self.assertStatsEqual("downloader/request_count", 1)
 
-    def test_process_response(self):
+    def test_process_response(self) -> None:
         self.mw.process_response(self.req, self.res)
         self.assertStatsEqual("downloader/response_count", 1)
 
-    def test_process_exception(self):
+    def test_process_exception(self) -> None:
         self.mw.process_exception(self.req, MyException())
         self.assertStatsEqual("downloader/exception_count", 1)
         self.assertStatsEqual(
@@ -42,14 +46,17 @@ class TestDownloaderStats:
             1,
         )
 
-    def test_from_crawler_not_configured(self):
+    def test_from_crawler_not_configured(self) -> None:
         crawler = get_crawler(Spider, {"DOWNLOADER_STATS": False})
         with pytest.raises(NotConfigured):
             DownloaderStats.from_crawler(crawler)
 
-    def teardown_method(self):
+    def teardown_method(self) -> None:
+        assert self.crawler.stats is not None
         self.crawler.stats.close_spider()
 
 
-def test_get_header_size_non_list_value():
-    assert get_header_size({"Content-Type": "text/html"}) == 0
+def test_get_header_size_non_list_value() -> None:
+    # Deliberately passing a non-list/tuple header value to make sure
+    # get_header_size() degrades gracefully instead of raising.
+    assert get_header_size({"Content-Type": "text/html"}) == 0  # type: ignore[dict-item]

tests/test_dupefilters.py

@@ -1,11 +1,14 @@
+from __future__ import annotations
+
 import hashlib
+import logging
 import shutil
 import sys
 import tempfile
 from pathlib import Path
+from typing import TYPE_CHECKING, Any
 
 import pytest
-from testfixtures import LogCapture
 
 from scrapy.core.scheduler import Scheduler
 from scrapy.dupefilters import BaseDupeFilter, RFPDupeFilter
@@ -15,8 +18,16 @@ from scrapy.utils.python import to_bytes
 from scrapy.utils.test import get_crawler
 from tests.spiders import SimpleSpider
 
+if TYPE_CHECKING:
+    from scrapy.crawler import Crawler
 
-def _get_dupefilter(*, crawler=None, settings=None, open_=True):
+
+def _get_dupefilter(
+    *,
+    crawler: Crawler | None = None,
+    settings: dict[str, Any] | None = None,
+    open_: bool = True,
+) -> BaseDupeFilter:
     if crawler is None:
         crawler = get_crawler(settings_dict=settings)
     scheduler = Scheduler.from_crawler(crawler)
@@ -151,108 +162,71 @@ class TestRFPDupeFilter:
         finally:
             shutil.rmtree(path)
 
-    def test_log(self):
-        with LogCapture() as log:
-            settings = {
-                "DUPEFILTER_DEBUG": False,
-                "DUPEFILTER_CLASS": FromCrawlerRFPDupeFilter,
-            }
-            crawler = get_crawler(SimpleSpider, settings_dict=settings)
-            spider = SimpleSpider.from_crawler(crawler)
-            dupefilter = _get_dupefilter(crawler=crawler)
+    def test_log(self, caplog: pytest.LogCaptureFixture) -> None:
+        settings = {
+            "DUPEFILTER_DEBUG": False,
+            "DUPEFILTER_CLASS": FromCrawlerRFPDupeFilter,
+        }
+        crawler = get_crawler(SimpleSpider, settings_dict=settings)
+        spider = SimpleSpider.from_crawler(crawler)
+        dupefilter = _get_dupefilter(crawler=crawler)
 
-            r1 = Request("http://scrapytest.org/index.html")
-            r2 = Request("http://scrapytest.org/index.html")
+        r1 = Request("http://scrapytest.org/index.html")
+        r2 = Request("http://scrapytest.org/index.html")
 
+        with caplog.at_level(logging.DEBUG):
             dupefilter.log(r1, spider)
             dupefilter.log(r2, spider)
 
-            assert crawler.stats.get_value("dupefilter/filtered") == 2
-            log.check_present(
-                (
-                    "scrapy.dupefilters",
-                    "DEBUG",
-                    "Filtered duplicate request: <GET http://scrapytest.org/index.html> - no more"
-                    " duplicates will be shown (see DUPEFILTER_DEBUG to show all duplicates)",
-                )
-            )
+        assert crawler.stats
+        assert crawler.stats.get_value("dupefilter/filtered") == 2
+        assert (
+            "scrapy.dupefilters",
+            logging.DEBUG,
+            "Filtered duplicate request: <GET http://scrapytest.org/index.html> - no more"
+            " duplicates will be shown (see DUPEFILTER_DEBUG to show all duplicates)",
+        ) in caplog.record_tuples
 
-            dupefilter.close("finished")
+        dupefilter.close("finished")
 
-    def test_log_debug(self):
-        with LogCapture() as log:
-            settings = {
-                "DUPEFILTER_DEBUG": True,
-                "DUPEFILTER_CLASS": FromCrawlerRFPDupeFilter,
-            }
-            crawler = get_crawler(SimpleSpider, settings_dict=settings)
-            spider = SimpleSpider.from_crawler(crawler)
-            dupefilter = _get_dupefilter(crawler=crawler)
+    @pytest.mark.parametrize("df", [None, FromCrawlerRFPDupeFilter])
+    def test_log_debug(
+        self, caplog: pytest.LogCaptureFixture, df: type[BaseDupeFilter] | None
+    ) -> None:
+        settings: dict[str, Any] = {
+            "DUPEFILTER_DEBUG": True,
+        }
+        if df:
+            settings["DUPEFILTER_CLASS"] = df
+        crawler = get_crawler(SimpleSpider, settings_dict=settings)
+        spider = SimpleSpider.from_crawler(crawler)
+        dupefilter = _get_dupefilter(crawler=crawler)
 
-            r1 = Request("http://scrapytest.org/index.html")
-            r2 = Request(
-                "http://scrapytest.org/index.html",
-                headers={"Referer": "http://scrapytest.org/INDEX.html"},
-            )
+        r1 = Request("http://scrapytest.org/index.html")
+        r2 = Request(
+            "http://scrapytest.org/index.html",
+            headers={"Referer": "http://scrapytest.org/INDEX.html"},
+        )
 
+        with caplog.at_level(logging.DEBUG):
             dupefilter.log(r1, spider)
             dupefilter.log(r2, spider)
 
-            assert crawler.stats.get_value("dupefilter/filtered") == 2
-            log.check_present(
-                (
-                    "scrapy.dupefilters",
-                    "DEBUG",
-                    "Filtered duplicate request: <GET http://scrapytest.org/index.html> (referer: None)",
-                )
-            )
-            log.check_present(
-                (
-                    "scrapy.dupefilters",
-                    "DEBUG",
-                    "Filtered duplicate request: <GET http://scrapytest.org/index.html>"
-                    " (referer: http://scrapytest.org/INDEX.html)",
-                )
-            )
+        assert crawler.stats
+        assert crawler.stats.get_value("dupefilter/filtered") == 2
+        assert (
+            "scrapy.dupefilters",
+            logging.DEBUG,
+            "Filtered duplicate request: <GET http://scrapytest.org/index.html> (referer: None)",
+        ) in caplog.record_tuples
+        assert (
+            "scrapy.dupefilters",
+            logging.DEBUG,
+            "Filtered duplicate request: <GET http://scrapytest.org/index.html>"
+            " (referer: http://scrapytest.org/INDEX.html)",
+        ) in caplog.record_tuples
 
-            dupefilter.close("finished")
-
-    def test_log_debug_default_dupefilter(self):
-        with LogCapture() as log:
-            settings = {
-                "DUPEFILTER_DEBUG": True,
-            }
-            crawler = get_crawler(SimpleSpider, settings_dict=settings)
-            spider = SimpleSpider.from_crawler(crawler)
-            dupefilter = _get_dupefilter(crawler=crawler)
-
-            r1 = Request("http://scrapytest.org/index.html")
-            r2 = Request(
-                "http://scrapytest.org/index.html",
-                headers={"Referer": "http://scrapytest.org/INDEX.html"},
-            )
-
-            dupefilter.log(r1, spider)
-            dupefilter.log(r2, spider)
-
-            assert crawler.stats.get_value("dupefilter/filtered") == 2
-            log.check_present(
-                (
-                    "scrapy.dupefilters",
-                    "DEBUG",
-                    "Filtered duplicate request: <GET http://scrapytest.org/index.html> (referer: None)",
-                )
-            )
-            log.check_present(
-                (
-                    "scrapy.dupefilters",
-                    "DEBUG",
-                    "Filtered duplicate request: <GET http://scrapytest.org/index.html>"
-                    " (referer: http://scrapytest.org/INDEX.html)",
-                )
-            )
-
-            dupefilter.close("finished")
+        dupefilter.close("finished")
 
 
 class TestBaseDupeFilter:

tests/test_engine.py

@@ -1,20 +1,13 @@
 from __future__ import annotations
 
 import asyncio
-import re
+import logging
 import subprocess
 import sys
-from collections import defaultdict
-from dataclasses import dataclass
 from typing import TYPE_CHECKING, Any
 from unittest.mock import AsyncMock, Mock, patch
-from urllib.parse import urlparse
 
-import attr
 import pytest
-from itemadapter import ItemAdapter
-from pydispatch import dispatcher
-from testfixtures import LogCapture
 from twisted.internet import defer
 from twisted.python.failure import Failure
 
@@ -22,21 +15,25 @@ from scrapy import signals
 from scrapy.core.engine import ExecutionEngine, _Slot
 from scrapy.core.scheduler import BaseScheduler
 from scrapy.exceptions import CloseSpider, DownloadCancelledError, IgnoreRequest
-from scrapy.http import Headers, Request, Response
-from scrapy.item import Field, Item
-from scrapy.linkextractors import LinkExtractor
+from scrapy.http import Request
 from scrapy.spiders import Spider
 from scrapy.utils.defer import (
     _schedule_coro,
     deferred_from_coro,
     maybe_deferred_to_future,
 )
-from scrapy.utils.signal import disconnect_all
 from scrapy.utils.spider import DefaultSpider
 from scrapy.utils.test import get_crawler
-from tests import get_testdata
 from tests.utils import async_sleep
+from tests.utils.bases.engine import TestEngineBase
 from tests.utils.decorators import coroutine_test, inline_callbacks_test
+from tests.utils.engine import (
+    AttrsItemsSpider,
+    CrawlerRun,
+    DataClassItemsSpider,
+    DictItemsSpider,
+    MySpider,
+)
 
 if TYPE_CHECKING:
     from collections.abc import AsyncIterator
@@ -44,72 +41,12 @@ if TYPE_CHECKING:
     from tests.mockserver.http import MockServer
 
 
-class MyItem(Item):
-    name = Field()
-    url = Field()
-    price = Field()
-
-
-@attr.s
-class AttrsItem:
-    name = attr.ib(default="")
-    url = attr.ib(default="")
-    price = attr.ib(default=0)
-
-
-@dataclass
-class DataClassItem:
-    name: str = ""
-    url: str = ""
-    price: int = 0
-
-
-class MySpider(Spider):
-    name = "scrapytest.org"
-
-    itemurl_re = re.compile(r"item\d+.html")
-    name_re = re.compile(r"<h1>(.*?)</h1>", re.MULTILINE)
-    price_re = re.compile(r">Price: \$(.*?)<", re.MULTILINE)
-
-    item_cls: type = MyItem
-
-    def parse(self, response):
-        xlink = LinkExtractor()
-        itemre = re.compile(self.itemurl_re)
-        for link in xlink.extract_links(response):
-            if itemre.search(link.url):
-                yield Request(url=link.url, callback=self.parse_item)
-
-    def parse_item(self, response):
-        adapter = ItemAdapter(self.item_cls())
-        m = self.name_re.search(response.text)
-        if m:
-            adapter["name"] = m.group(1)
-        adapter["url"] = response.url
-        m = self.price_re.search(response.text)
-        if m:
-            adapter["price"] = m.group(1)
-        return adapter.item
-
-
 class DupeFilterSpider(MySpider):
     async def start(self):
         for url in self.start_urls:
             yield Request(url)  # no dont_filter=True
 
 
-class DictItemsSpider(MySpider):
-    item_cls = dict
-
-
-class AttrsItemsSpider(MySpider):
-    item_cls = AttrsItem
-
-
-class DataClassItemsSpider(MySpider):
-    item_cls = DataClassItem
-
-
 class ItemZeroDivisionErrorSpider(MySpider):
     custom_settings = {
         "ITEM_PIPELINES": {
@@ -130,253 +67,6 @@ class ChangeCloseReasonSpider(MySpider):
         raise CloseSpider(reason="custom_reason")
 
 
-class CrawlerRun:
-    """A class to run the crawler and keep track of events occurred"""
-
-    def __init__(self, spider_class: type[Spider]):
-        self.respplug: list[tuple[Response, Spider]] = []
-        self.reqplug: list[tuple[Request, Spider]] = []
-        self.reqdropped: list[tuple[Request, Spider]] = []
-        self.reqreached: list[tuple[Request, Spider]] = []
-        self.itemerror: list[tuple[Any, Response, Spider, Failure]] = []
-        self.itemresp: list[tuple[Any, Response]] = []
-        self.headers: dict[Request, Headers] = {}
-        self.bytes: defaultdict[Request, list[bytes]] = defaultdict(list)
-        self.signals_caught: dict[Any, dict[str, Any]] = {}
-        self.spider_class = spider_class
-
-    async def run(self, mockserver: MockServer) -> None:
-        self.mockserver = mockserver
-
-        start_urls = [
-            self.geturl("/static/"),
-            self.geturl("/redirect"),
-            self.geturl("/redirect"),  # duplicate
-            self.geturl("/numbers"),
-        ]
-
-        for name, signal in vars(signals).items():
-            if not name.startswith("_"):
-                dispatcher.connect(self.record_signal, signal)
-
-        self.crawler = get_crawler(self.spider_class)
-        self.crawler.signals.connect(self.item_scraped, signals.item_scraped)
-        self.crawler.signals.connect(self.item_error, signals.item_error)
-        self.crawler.signals.connect(self.headers_received, signals.headers_received)
-        self.crawler.signals.connect(self.bytes_received, signals.bytes_received)
-        self.crawler.signals.connect(self.request_scheduled, signals.request_scheduled)
-        self.crawler.signals.connect(self.request_dropped, signals.request_dropped)
-        self.crawler.signals.connect(
-            self.request_reached, signals.request_reached_downloader
-        )
-        self.crawler.signals.connect(
-            self.response_downloaded, signals.response_downloaded
-        )
-        self.crawler.crawl(start_urls=start_urls)
-
-        self.deferred: defer.Deferred[None] = defer.Deferred()
-        dispatcher.connect(self.stop, signals.engine_stopped)
-        await maybe_deferred_to_future(self.deferred)
-
-    async def stop(self):
-        for name, signal in vars(signals).items():
-            if not name.startswith("_"):
-                disconnect_all(signal)
-        self.deferred.callback(None)
-        await self.crawler.stop_async()
-
-    def geturl(self, path: str) -> str:
-        return self.mockserver.url(path)
-
-    def getpath(self, url: str) -> str:
-        u = urlparse(url)
-        return u.path
-
-    def item_error(
-        self, item: Any, response: Response, spider: Spider, failure: Failure
-    ) -> None:
-        self.itemerror.append((item, response, spider, failure))
-
-    def item_scraped(self, item: Any, spider: Spider, response: Response) -> None:
-        self.itemresp.append((item, response))
-
-    def headers_received(
-        self, headers: Headers, body_length: int, request: Request, spider: Spider
-    ) -> None:
-        self.headers[request] = headers
-
-    def bytes_received(self, data: bytes, request: Request, spider: Spider) -> None:
-        self.bytes[request].append(data)
-
-    def request_scheduled(self, request: Request, spider: Spider) -> None:
-        self.reqplug.append((request, spider))
-
-    def request_reached(self, request: Request, spider: Spider) -> None:
-        self.reqreached.append((request, spider))
-
-    def request_dropped(self, request: Request, spider: Spider) -> None:
-        self.reqdropped.append((request, spider))
-
-    def response_downloaded(self, response: Response, spider: Spider) -> None:
-        self.respplug.append((response, spider))
-
-    def record_signal(self, *args: Any, **kwargs: Any) -> None:
-        """Record a signal and its parameters"""
-        signalargs = kwargs.copy()
-        sig = signalargs.pop("signal")
-        signalargs.pop("sender", None)
-        self.signals_caught[sig] = signalargs
-
-
-class TestEngineBase:
-    @staticmethod
-    def _assert_visited_urls(run: CrawlerRun) -> None:
-        must_be_visited = [
-            "/static/",
-            "/redirect",
-            "/redirected",
-            "/static/item1.html",
-            "/static/item2.html",
-            "/static/item999.html",
-        ]
-        urls_visited = {rp[0].url for rp in run.respplug}
-        urls_expected = {run.geturl(p) for p in must_be_visited}
-        assert urls_expected <= urls_visited, (
-            f"URLs not visited: {list(urls_expected - urls_visited)}"
-        )
-
-    @staticmethod
-    def _assert_scheduled_requests(run: CrawlerRun, count: int) -> None:
-        assert len(run.reqplug) == count
-
-        paths_expected = [
-            "/static/item999.html",
-            "/static/item2.html",
-            "/static/item1.html",
-        ]
-
-        urls_requested = {rq[0].url for rq in run.reqplug}
-        urls_expected = {run.geturl(p) for p in paths_expected}
-        assert urls_expected <= urls_requested
-        scheduled_requests_count = len(run.reqplug)
-        dropped_requests_count = len(run.reqdropped)
-        responses_count = len(run.respplug)
-        assert scheduled_requests_count == dropped_requests_count + responses_count
-        assert len(run.reqreached) == responses_count
-
-    @staticmethod
-    def _assert_dropped_requests(run: CrawlerRun) -> None:
-        assert len(run.reqdropped) == 1
-
-    @staticmethod
-    def _assert_downloaded_responses(run: CrawlerRun, count: int) -> None:
-        # response tests
-        assert len(run.respplug) == count
-        assert len(run.reqreached) == count
-
-        for response, _ in run.respplug:
-            if run.getpath(response.url) == "/static/item999.html":
-                assert response.status == 404
-            if run.getpath(response.url) == "/redirect":
-                assert response.status == 302
-
-    @staticmethod
-    def _assert_items_error(run: CrawlerRun) -> None:
-        assert len(run.itemerror) == 2
-        for item, response, spider, failure in run.itemerror:
-            assert failure.value.__class__ is ZeroDivisionError
-            assert spider == run.crawler.spider
-
-            assert item["url"] == response.url
-            if "item1.html" in item["url"]:
-                assert item["name"] == "Item 1 name"
-                assert item["price"] == "100"
-            if "item2.html" in item["url"]:
-                assert item["name"] == "Item 2 name"
-                assert item["price"] == "200"
-
-    @staticmethod
-    def _assert_scraped_items(run: CrawlerRun) -> None:
-        assert len(run.itemresp) == 2
-        for item_, response in run.itemresp:
-            item = ItemAdapter(item_)
-            assert item["url"] == response.url
-            if "item1.html" in item["url"]:
-                assert item["name"] == "Item 1 name"
-                assert item["price"] == "100"
-            if "item2.html" in item["url"]:
-                assert item["name"] == "Item 2 name"
-                assert item["price"] == "200"
-
-    @staticmethod
-    def _assert_headers_received(run: CrawlerRun) -> None:
-        for headers in run.headers.values():
-            assert b"Server" in headers
-            assert headers[b"Server"]
-            assert b"TwistedWeb" in headers[b"Server"]
-            assert b"Date" in headers
-            assert b"Content-Type" in headers
-
-    @staticmethod
-    def _assert_bytes_received(run: CrawlerRun) -> None:
-        assert len(run.bytes) == 9
-        for request, data in run.bytes.items():
-            joined_data = b"".join(data)
-            if run.getpath(request.url) == "/static/":
-                assert joined_data == get_testdata("test_site", "index.html")
-            elif run.getpath(request.url) == "/static/item1.html":
-                assert joined_data == get_testdata("test_site", "item1.html")
-            elif run.getpath(request.url) == "/static/item2.html":
-                assert joined_data == get_testdata("test_site", "item2.html")
-            elif run.getpath(request.url) == "/redirected":
-                assert joined_data == b"Redirected here"
-            elif run.getpath(request.url) == "/redirect":
-                assert (
-                    joined_data == b"\n<html>\n"
-                    b"    <head>\n"
-                    b'        <meta http-equiv="refresh" content="0;URL=/redirected">\n'
-                    b"    </head>\n"
-                    b'    <body bgcolor="#FFFFFF" text="#000000">\n'
-                    b'    <a href="/redirected">click here</a>\n'
-                    b"    </body>\n"
-                    b"</html>\n"
-                )
-            elif run.getpath(request.url) == "/static/item999.html":
-                assert (
-                    joined_data == b"\n<html>\n"
-                    b"  <head><title>404 - No Such Resource</title></head>\n"
-                    b"  <body>\n"
-                    b"    <h1>No Such Resource</h1>\n"
-                    b"    <p>File not found.</p>\n"
-                    b"  </body>\n"
-                    b"</html>\n"
-                )
-            elif run.getpath(request.url) == "/numbers":
-                # signal was fired multiple times
-                assert len(data) > 1
-                # bytes were received in order
-                numbers = [str(x).encode("utf8") for x in range(2**18)]
-                assert joined_data == b"".join(numbers)
-
-    @staticmethod
-    def _assert_signals_caught(run: CrawlerRun) -> None:
-        assert signals.engine_started in run.signals_caught
-        assert signals.engine_stopped in run.signals_caught
-        assert signals.spider_opened in run.signals_caught
-        assert signals.spider_idle in run.signals_caught
-        assert signals.spider_closed in run.signals_caught
-        assert signals.headers_received in run.signals_caught
-
-        assert {"spider": run.crawler.spider} == run.signals_caught[
-            signals.spider_opened
-        ]
-        assert {"spider": run.crawler.spider} == run.signals_caught[signals.spider_idle]
-        assert {
-            "spider": run.crawler.spider,
-            "reason": "finished",
-        } == run.signals_caught[signals.spider_closed]
-
-
 class TestEngine(TestEngineBase):
     @coroutine_test
     async def test_crawler(self, mockserver: MockServer) -> None:
@@ -527,8 +217,10 @@ class TestEngine(TestEngineBase):
             await asyncio.gather(e.start_async(), e.start_async())
         await e.stop_async()
 
-    @inline_callbacks_test
-    def test_start_request_processing_exception(self):
+    @coroutine_test
+    async def test_start_request_processing_exception(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         class BadRequestFingerprinter:
             def fingerprint(self, request):
                 raise ValueError  # to make Scheduler.enqueue_request() fail
@@ -542,10 +234,10 @@ class TestEngine(TestEngineBase):
         crawler = get_crawler(
             SimpleSpider, {"REQUEST_FINGERPRINTER_CLASS": BadRequestFingerprinter}
         )
-        with LogCapture() as log:
-            yield crawler.crawl()
-        assert "Error while processing requests from start()" in str(log)
-        assert "Spider closed (shutdown)" in str(log)
+        with caplog.at_level(logging.DEBUG):
+            await crawler.crawl_async()
+        assert "Error while processing requests from start()" in caplog.text
+        assert "Spider closed (shutdown)" in caplog.text
 
     def test_short_timeout(self):
         args = (

tests/test_engine_download.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import sys
 from unittest.mock import Mock, call
 
 import pytest
@@ -73,6 +74,25 @@ class TestEngineDownloadAsync:
             [call(original_request), call(redirect_request)]
         )
 
+    @coroutine_test
+    async def test_download_async_many_redirects(self, engine):
+        """A long chain of requests being replaced by new ones is handled
+        iteratively, without hitting the recursion limit."""
+        count = sys.getrecursionlimit() * 2
+        requests = [Request(f"http://example.com/{i}") for i in range(count)]
+        final_response = Response("http://example.com/final", body=b"done")
+        engine.downloader.fetch.side_effect = [
+            *(defer.succeed(request) for request in requests[1:]),
+            defer.succeed(final_response),
+        ]
+        engine.spider = Mock()
+        engine._slot.add_request = Mock()
+        engine._slot.remove_request = Mock()
+
+        result = await self._download(engine, requests[0])
+        assert result == final_response
+        assert engine.downloader.fetch.call_count == count
+
     @coroutine_test
     async def test_download_async_no_spider(self, engine):
         """Test async download attempt when no spider is available."""

tests/test_engine_stop_download_bytes.py

@@ -3,15 +3,15 @@ from __future__ import annotations
 from typing import TYPE_CHECKING
 
 from scrapy.exceptions import StopDownload
-from tests.test_engine import (
+from tests.utils.bases.engine import TestEngineBase
+from tests.utils.decorators import coroutine_test
+from tests.utils.engine import (
     AttrsItemsSpider,
     CrawlerRun,
     DataClassItemsSpider,
     DictItemsSpider,
     MySpider,
-    TestEngineBase,
 )
-from tests.utils.decorators import coroutine_test
 
 if TYPE_CHECKING:
     import pytest

tests/test_engine_stop_download_headers.py

@@ -3,15 +3,15 @@ from __future__ import annotations
 from typing import TYPE_CHECKING
 
 from scrapy.exceptions import StopDownload
-from tests.test_engine import (
+from tests.utils.bases.engine import TestEngineBase
+from tests.utils.decorators import coroutine_test
+from tests.utils.engine import (
     AttrsItemsSpider,
     CrawlerRun,
     DataClassItemsSpider,
     DictItemsSpider,
     MySpider,
-    TestEngineBase,
 )
-from tests.utils.decorators import coroutine_test
 
 if TYPE_CHECKING:
     import pytest

tests/test_feedexport.py

@@ -2,21 +2,17 @@ from __future__ import annotations
 
 import csv
 import json
+import logging
 import marshal
 import pickle
-import random
-import shutil
 import tempfile
-from abc import ABC, abstractmethod
 from logging import getLogger
 from pathlib import Path
-from string import ascii_letters, digits
 from typing import IO, TYPE_CHECKING, Any
 from unittest import mock
 
 import lxml.etree
 import pytest
-from testfixtures import LogCapture
 from w3lib.url import file_uri_to_path
 
 import scrapy
@@ -32,8 +28,8 @@ from scrapy.extensions.feedexport import (
 )
 from scrapy.utils.python import to_unicode
 from scrapy.utils.test import get_crawler
-from tests.mockserver.http import MockServer
 from tests.spiders import ItemSpider
+from tests.utils.bases.feedexport import TestFeedExportBase
 from tests.utils.decorators import coroutine_test, inline_callbacks_test
 from tests.utils.feedexport import MyItem, MyItem2, path_to_url, printf_escape
 
@@ -99,141 +95,6 @@ class LogOnStoreFileStorage:
         file.close()
 
 
-class TestFeedExportBase(ABC):
-    mockserver: MockServer
-
-    def _random_temp_filename(self, inter_dir="") -> Path:
-        chars = [random.choice(ascii_letters + digits) for _ in range(15)]
-        filename = "".join(chars)
-        return Path(self.temp_dir, inter_dir, filename)
-
-    @classmethod
-    def setup_class(cls):
-        cls.mockserver = MockServer()
-        cls.mockserver.__enter__()
-
-    @classmethod
-    def teardown_class(cls):
-        cls.mockserver.__exit__(None, None, None)
-
-    def setup_method(self):
-        self.temp_dir = tempfile.mkdtemp()
-
-    def teardown_method(self):
-        shutil.rmtree(self.temp_dir, ignore_errors=True)
-
-    async def exported_data(
-        self, items: Iterable[Any], settings: dict[str, Any]
-    ) -> dict[str, Any]:
-        """
-        Return exported data which a spider yielding ``items`` would return.
-        """
-
-        class TestSpider(scrapy.Spider):
-            name = "testspider"
-
-            def parse(self, response):
-                yield from items
-
-        return await self.run_and_export(TestSpider, settings)
-
-    async def exported_no_data(self, settings: dict[str, Any]) -> dict[str, Any]:
-        """
-        Return exported data which a spider yielding no ``items`` would return.
-        """
-
-        class TestSpider(scrapy.Spider):
-            name = "testspider"
-
-            def parse(self, response):
-                pass
-
-        return await self.run_and_export(TestSpider, settings)
-
-    async def assertExported(
-        self,
-        items: Iterable[Any],
-        header: Iterable[str],
-        rows: Iterable[dict[str, Any]],
-        settings: dict[str, Any] | None = None,
-    ) -> None:
-        await self.assertExportedCsv(items, header, rows, settings)
-        await self.assertExportedJsonLines(items, rows, settings)
-        await self.assertExportedXml(items, rows, settings)
-        await self.assertExportedPickle(items, rows, settings)
-        await self.assertExportedMarshal(items, rows, settings)
-        await self.assertExportedMultiple(items, rows, settings)
-
-    async def assertExportedCsv(  # noqa: B027
-        self,
-        items: Iterable[Any],
-        header: Iterable[str],
-        rows: Iterable[dict[str, Any]],
-        settings: dict[str, Any] | None = None,
-    ) -> None:
-        pass
-
-    async def assertExportedJsonLines(  # noqa: B027
-        self,
-        items: Iterable[Any],
-        rows: Iterable[dict[str, Any]],
-        settings: dict[str, Any] | None = None,
-    ) -> None:
-        pass
-
-    async def assertExportedXml(  # noqa: B027
-        self,
-        items: Iterable[Any],
-        rows: Iterable[dict[str, Any]],
-        settings: dict[str, Any] | None = None,
-    ) -> None:
-        pass
-
-    async def assertExportedMultiple(  # noqa: B027
-        self,
-        items: Iterable[Any],
-        rows: Iterable[dict[str, Any]],
-        settings: dict[str, Any] | None = None,
-    ) -> None:
-        pass
-
-    async def assertExportedPickle(  # noqa: B027
-        self,
-        items: Iterable[Any],
-        rows: Iterable[dict[str, Any]],
-        settings: dict[str, Any] | None = None,
-    ) -> None:
-        pass
-
-    async def assertExportedMarshal(  # noqa: B027
-        self,
-        items: Iterable[Any],
-        rows: Iterable[dict[str, Any]],
-        settings: dict[str, Any] | None = None,
-    ) -> None:
-        pass
-
-    @abstractmethod
-    async def run_and_export(
-        self, spider_cls: type[Spider], settings: dict[str, Any]
-    ) -> dict[str, Any]:
-        pass
-
-    def _load_until_eof(
-        self, data: bytes, load_func: Callable[[IO[bytes]], Any]
-    ) -> list[Any]:
-        result: list[Any] = []
-        with tempfile.TemporaryFile() as temp:
-            temp.write(data)
-            temp.seek(0)
-            while True:
-                try:
-                    result.append(load_func(temp))
-                except EOFError:
-                    break
-        return result
-
-
 class InstrumentedFeedSlot(FeedSlot):
     """Instrumented FeedSlot subclass for keeping track of calls to
     start_exporting and finish_exporting."""
@@ -702,7 +563,9 @@ class TestFeedExport(TestFeedExportBase):
             assert expctd == data[fmt]
 
     @coroutine_test
-    async def test_export_no_items_multiple_feeds(self):
+    async def test_export_no_items_multiple_feeds(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         """Make sure that `storage.store` is not called."""
         settings = {
             "FEEDS": {
@@ -714,10 +577,10 @@ class TestFeedExport(TestFeedExportBase):
             "FEED_STORE_EMPTY": False,
         }
 
-        with LogCapture() as log:
+        with caplog.at_level(logging.INFO):
             await self.exported_no_data(settings)
 
-        assert str(log).count("Storage.store is called") == 0
+        assert caplog.text.count("Storage.store is called") == 0
 
     @coroutine_test
     async def test_export_multiple_item_classes(self):
@@ -1219,7 +1082,9 @@ class TestFeedExport(TestFeedExportBase):
         assert data["csv"] == b""
 
     @coroutine_test
-    async def test_multiple_feeds_success_logs_blocking_feed_storage(self):
+    async def test_multiple_feeds_success_logs_blocking_feed_storage(
+        self, caplog: pytest.LogCaptureFixture
+    ):
         settings = {
             "FEEDS": {
                 self._random_temp_filename(): {"format": "json"},
@@ -1232,14 +1097,16 @@ class TestFeedExport(TestFeedExportBase):
             {"foo": "bar1", "baz": ""},
             {"foo": "bar2", "baz": "quux"},
         ]
-        with LogCapture() as log:
+        with caplog.at_level(logging.DEBUG):
             await self.exported_data(items, settings)
 
         for fmt in ["json", "xml", "csv"]:
-            assert f"Stored {fmt} feed (2 items)" in str(log)
+            assert f"Stored {fmt} feed (2 items)" in caplog.text
 
     @coroutine_test
-    async def test_multiple_feeds_failing_logs_blocking_feed_storage(self):
+    async def test_multiple_feeds_failing_logs_blocking_feed_storage(
+        self, caplog: pytest.LogCaptureFixture
+    ):
         settings = {
             "FEEDS": {
                 self._random_temp_filename(): {"format": "json"},
@@ -1252,11 +1119,11 @@ class TestFeedExport(TestFeedExportBase):
             {"foo": "bar1", "baz": ""},
             {"foo": "bar2", "baz": "quux"},
         ]
-        with LogCapture() as log:
+        with caplog.at_level(logging.DEBUG):
             await self.exported_data(items, settings)
 
         for fmt in ["json", "xml", "csv"]:
-            assert f"Error storing {fmt} feed (2 items)" in str(log)
+            assert f"Error storing {fmt} feed (2 items)" in caplog.text
 
     @coroutine_test
     async def test_extend_kwargs(self):

tests/test_feedexport_batch.py

@@ -21,7 +21,7 @@ from scrapy.settings import Settings
 from scrapy.utils.python import to_unicode
 from scrapy.utils.test import get_crawler
 from tests.spiders import ItemSpider
-from tests.test_feedexport import TestFeedExportBase
+from tests.utils.bases.feedexport import TestFeedExportBase
 from tests.utils.decorators import coroutine_test, inline_callbacks_test
 from tests.utils.feedexport import MyItem
 

tests/test_feedexport_postprocess.py

@@ -13,7 +13,7 @@ from typing import TYPE_CHECKING, Any
 import pytest
 
 from scrapy.utils.test import get_crawler
-from tests.test_feedexport import TestFeedExportBase
+from tests.utils.bases.feedexport import TestFeedExportBase
 from tests.utils.decorators import coroutine_test
 from tests.utils.feedexport import path_to_url, printf_escape
 

tests/test_feedexport_storages.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import logging
 import os
 import string
 import tempfile
@@ -10,7 +11,6 @@ from unittest import mock
 from urllib.parse import quote
 
 import pytest
-from testfixtures import LogCapture
 from w3lib.url import path_to_file_uri
 
 import scrapy
@@ -415,23 +415,21 @@ class TestS3FeedStorage:
         acl = storage.s3_client.upload_fileobj.call_args[1]["ExtraArgs"]["ACL"]
         assert acl == "custom-acl"
 
-    def test_overwrite_default(self):
-        with LogCapture() as log:
-            S3FeedStorage(
-                "s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl"
-            )
-        assert "S3 does not support appending to files" not in str(log)
+    def test_overwrite_default(self, caplog: pytest.LogCaptureFixture) -> None:
+        S3FeedStorage(
+            "s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl"
+        )
+        assert "S3 does not support appending to files" not in caplog.text
 
-    def test_overwrite_false(self):
-        with LogCapture() as log:
-            S3FeedStorage(
-                "s3://mybucket/export.csv",
-                "access_key",
-                "secret_key",
-                "custom-acl",
-                feed_options={"overwrite": False},
-            )
-        assert "S3 does not support appending to files" in str(log)
+    def test_overwrite_false(self, caplog: pytest.LogCaptureFixture) -> None:
+        S3FeedStorage(
+            "s3://mybucket/export.csv",
+            "access_key",
+            "secret_key",
+            "custom-acl",
+            feed_options={"overwrite": False},
+        )
+        assert "S3 does not support appending to files" in caplog.text
 
 
 class TestGCSFeedStorage:
@@ -505,20 +503,20 @@ class TestGCSFeedStorage:
             blob_mock.upload_from_file.assert_called_once_with(f, predefined_acl=acl)
             f.close.assert_called_once_with()
 
-    def test_overwrite_default(self):
-        with LogCapture() as log:
+    def test_overwrite_default(self, caplog: pytest.LogCaptureFixture):
+        with caplog.at_level(logging.DEBUG):
             GCSFeedStorage("gs://mybucket/export.csv", "myproject-123", "custom-acl")
-        assert "GCS does not support appending to files" not in str(log)
+        assert "GCS does not support appending to files" not in caplog.text
 
-    def test_overwrite_false(self):
-        with LogCapture() as log:
+    def test_overwrite_false(self, caplog: pytest.LogCaptureFixture):
+        with caplog.at_level(logging.DEBUG):
             GCSFeedStorage(
                 "gs://mybucket/export.csv",
                 "myproject-123",
                 "custom-acl",
                 feed_options={"overwrite": False},
             )
-        assert "GCS does not support appending to files" in str(log)
+        assert "GCS does not support appending to files" in caplog.text
 
 
 class TestStdoutFeedStorage:
@@ -530,17 +528,18 @@ class TestStdoutFeedStorage:
         storage.store(file)
         assert out.getvalue() == b"content"
 
-    def test_overwrite_default(self):
-        with LogCapture() as log:
+    def test_overwrite_default(self, caplog: pytest.LogCaptureFixture):
+        with caplog.at_level(logging.DEBUG):
             StdoutFeedStorage("stdout:")
         assert (
             "Standard output (stdout) storage does not support overwriting"
-            not in str(log)
+            not in caplog.text
         )
 
-    def test_overwrite_true(self):
-        with LogCapture() as log:
+    def test_overwrite_true(self, caplog: pytest.LogCaptureFixture):
+        with caplog.at_level(logging.DEBUG):
             StdoutFeedStorage("stdout:", feed_options={"overwrite": True})
-        assert "Standard output (stdout) storage does not support overwriting" in str(
-            log
+        assert (
+            "Standard output (stdout) storage does not support overwriting"
+            in caplog.text
         )

tests/test_http_request.py

@@ -1,493 +1,18 @@
-import warnings
 import xmlrpc.client
-from typing import Any
 
 import pytest
 
-from scrapy.http import Headers, Request, XmlRpcRequest
-from scrapy.http.request import NO_CALLBACK
+from scrapy import Request
+from scrapy.http import XmlRpcRequest
 from scrapy.utils.python import to_bytes
+from tests.utils.bases.http_request import TestRequestBase
 
 
-class TestRequest:
+class TestRequest(TestRequestBase):
     request_class = Request
-    default_method = "GET"
-    default_headers: dict[bytes, list[bytes]] = {}
-    default_meta: dict[str, Any] = {}
 
-    def test_init(self):
-        # Request requires url in the __init__ method
-        with pytest.raises(TypeError):
-            self.request_class()
 
-        # url argument must be basestring
-        with pytest.raises(TypeError):
-            self.request_class(123)
-
-        # priority argument must be an integer
-        with pytest.raises(TypeError, match="Request priority not an integer"):
-            self.request_class("http://www.example.com", priority="1")
-
-        r = self.request_class("http://www.example.com")
-        assert isinstance(r.url, str)
-        assert r.url == "http://www.example.com"
-        assert r.method == self.default_method
-
-        assert isinstance(r.headers, Headers)
-        assert r.headers == self.default_headers
-        assert r.meta == self.default_meta
-
-        meta = {"lala": "lolo"}
-        headers = {b"caca": b"coco"}
-        r = self.request_class(
-            "http://www.example.com", meta=meta, headers=headers, body="a body"
-        )
-
-        assert r.meta is not meta
-        assert r.meta == meta
-        assert r.headers is not headers
-        assert r.headers[b"caca"] == b"coco"
-
-    def test_url_scheme(self):
-        # This test passes by not raising any (ValueError) exception
-        self.request_class("http://example.org")
-        self.request_class("https://example.org")
-        self.request_class("s3://example.org")
-        self.request_class("ftp://example.org")
-        self.request_class("about:config")
-        self.request_class("data:,Hello%2C%20World!")
-
-    def test_url_no_scheme(self):
-        msg = "Missing scheme in request url:"
-        with pytest.raises(ValueError, match=msg):
-            self.request_class("foo")
-        with pytest.raises(ValueError, match=msg):
-            self.request_class("/foo/")
-        with pytest.raises(ValueError, match=msg):
-            self.request_class("/foo:bar")
-
-    def test_headers(self):
-        # Different ways of setting headers attribute
-        url = "http://www.scrapy.org"
-        headers = {b"Accept": "gzip", b"Custom-Header": "nothing to tell you"}
-        r = self.request_class(url=url, headers=headers)
-        p = self.request_class(url=url, headers=r.headers)
-
-        assert r.headers == p.headers
-        assert r.headers is not headers
-        assert p.headers is not r.headers
-
-        # headers must not be unicode
-        h = Headers({"key1": "val1", "key2": "val2"})
-        h["newkey"] = "newval"
-        for k, v in h.items():
-            assert isinstance(k, bytes)
-            for s in v:
-                assert isinstance(s, bytes)
-
-    def test_eq(self):
-        url = "http://www.scrapy.org"
-        r1 = self.request_class(url=url)
-        r2 = self.request_class(url=url)
-        assert r1 != r2
-
-        set_ = set()
-        set_.add(r1)
-        set_.add(r2)
-        assert len(set_) == 2
-
-    def test_url(self):
-        r = self.request_class(url="http://www.scrapy.org/path")
-        assert r.url == "http://www.scrapy.org/path"
-
-    def test_url_quoting(self):
-        r = self.request_class(url="http://www.scrapy.org/blank%20space")
-        assert r.url == "http://www.scrapy.org/blank%20space"
-        r = self.request_class(url="http://www.scrapy.org/blank space")
-        assert r.url == "http://www.scrapy.org/blank%20space"
-
-    def test_url_encoding(self):
-        r = self.request_class(url="http://www.scrapy.org/price/£")
-        assert r.url == "http://www.scrapy.org/price/%C2%A3"
-
-    def test_url_encoding_other(self):
-        # encoding affects only query part of URI, not path
-        # path part should always be UTF-8 encoded before percent-escaping
-        r = self.request_class(url="http://www.scrapy.org/price/£", encoding="utf-8")
-        assert r.url == "http://www.scrapy.org/price/%C2%A3"
-
-        r = self.request_class(url="http://www.scrapy.org/price/£", encoding="latin1")
-        assert r.url == "http://www.scrapy.org/price/%C2%A3"
-
-    def test_url_encoding_query(self):
-        r1 = self.request_class(url="http://www.scrapy.org/price/£?unit=µ")
-        assert r1.url == "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5"
-
-        # should be same as above
-        r2 = self.request_class(
-            url="http://www.scrapy.org/price/£?unit=µ", encoding="utf-8"
-        )
-        assert r2.url == "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5"
-
-    def test_url_encoding_query_latin1(self):
-        # encoding is used for encoding query-string before percent-escaping;
-        # path is still UTF-8 encoded before percent-escaping
-        r3 = self.request_class(
-            url="http://www.scrapy.org/price/µ?currency=£", encoding="latin1"
-        )
-        assert r3.url == "http://www.scrapy.org/price/%C2%B5?currency=%A3"
-
-    def test_url_encoding_nonutf8_untouched(self):
-        # percent-escaping sequences that do not match valid UTF-8 sequences
-        # should be kept untouched (just upper-cased perhaps)
-        #
-        # See https://datatracker.ietf.org/doc/html/rfc3987#section-3.2
-        #
-        # "Conversions from URIs to IRIs MUST NOT use any character encoding
-        # other than UTF-8 in steps 3 and 4, even if it might be possible to
-        # guess from the context that another character encoding than UTF-8 was
-        # used in the URI.  For example, the URI
-        # "http://www.example.org/r%E9sum%E9.html" might with some guessing be
-        # interpreted to contain two e-acute characters encoded as iso-8859-1.
-        # It must not be converted to an IRI containing these e-acute
-        # characters.  Otherwise, in the future the IRI will be mapped to
-        # "http://www.example.org/r%C3%A9sum%C3%A9.html", which is a different
-        # URI from "http://www.example.org/r%E9sum%E9.html".
-        r1 = self.request_class(url="http://www.scrapy.org/price/%a3")
-        assert r1.url == "http://www.scrapy.org/price/%a3"
-
-        r2 = self.request_class(url="http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3")
-        assert r2.url == "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3"
-
-        r3 = self.request_class(url="http://www.scrapy.org/résumé/%a3")
-        assert r3.url == "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3"
-
-        r4 = self.request_class(url="http://www.example.org/r%E9sum%E9.html")
-        assert r4.url == "http://www.example.org/r%E9sum%E9.html"
-
-    def test_url_verbatim(self):
-        r = self.request_class(
-            url="http://www.scrapy.org/price/£",
-            meta={"verbatim_url": True},
-        )
-        assert r.url == "http://www.scrapy.org/price/£"
-
-        r = self.request_class(
-            url="http://www.scrapy.org/blank space",
-            meta={"verbatim_url": True},
-        )
-        assert r.url == "http://www.scrapy.org/blank space"
-
-    def test_body(self):
-        r1 = self.request_class(url="http://www.example.com/")
-        assert r1.body == b""
-
-        r2 = self.request_class(url="http://www.example.com/", body=b"")
-        assert isinstance(r2.body, bytes)
-        assert r2.encoding == "utf-8"  # default encoding
-
-        r3 = self.request_class(
-            url="http://www.example.com/", body="Price: \xa3100", encoding="utf-8"
-        )
-        assert isinstance(r3.body, bytes)
-        assert r3.body == b"Price: \xc2\xa3100"
-
-        r4 = self.request_class(
-            url="http://www.example.com/", body="Price: \xa3100", encoding="latin1"
-        )
-        assert isinstance(r4.body, bytes)
-        assert r4.body == b"Price: \xa3100"
-
-    def test_copy(self):
-        """Test Request copy"""
-
-        def somecallback():
-            pass
-
-        r1 = self.request_class(
-            "http://www.example.com",
-            flags=["f1", "f2"],
-            callback=somecallback,
-            errback=somecallback,
-        )
-        r1.meta["foo"] = "bar"
-        r1.cb_kwargs["key"] = "value"
-        r2 = r1.copy()
-
-        # make sure callbaclks are copied
-        assert r1.callback is somecallback
-        assert r1.errback is somecallback
-        assert r2.callback is r1.callback
-        assert r2.errback is r1.errback
-
-        # make sure flags list is shallow copied
-        assert r1.flags is not r2.flags, "flags must be a shallow copy, not identical"
-        assert r1.flags == r2.flags
-
-        # make sure cb_kwargs dict is shallow copied
-        assert r1.cb_kwargs is not r2.cb_kwargs, (
-            "cb_kwargs must be a shallow copy, not identical"
-        )
-        assert r1.cb_kwargs == r2.cb_kwargs
-
-        # make sure meta dict is shallow copied
-        assert r1.meta is not r2.meta, "meta must be a shallow copy, not identical"
-        assert r1.meta == r2.meta
-
-        # make sure headers attribute is shallow copied
-        assert r1.headers is not r2.headers, (
-            "headers must be a shallow copy, not identical"
-        )
-        assert r1.headers == r2.headers
-        assert r1.encoding == r2.encoding
-        assert r1.dont_filter == r2.dont_filter
-
-        # Request.body can be identical since it's an immutable object (str)
-
-    def test_copy_inherited_classes(self):
-        """Test Request children copies preserve their class"""
-
-        class CustomRequest(self.request_class):
-            pass
-
-        r1 = CustomRequest("http://www.example.com")
-        r2 = r1.copy()
-
-        assert isinstance(r2, CustomRequest)
-
-    def test_replace(self):
-        """Test Request.replace() method"""
-        r1 = self.request_class("http://www.example.com", method="GET")
-        hdrs = Headers(r1.headers)
-        hdrs[b"key"] = b"value"
-        r2 = r1.replace(method="POST", body="New body", headers=hdrs)
-        assert r1.url == r2.url
-        assert (r1.method, r2.method) == ("GET", "POST")
-        assert (r1.body, r2.body) == (b"", b"New body")
-        assert (r1.headers, r2.headers) == (self.default_headers, hdrs)
-
-        # Empty attributes (which may fail if not compared properly)
-        r3 = self.request_class(
-            "http://www.example.com", meta={"a": 1}, dont_filter=True
-        )
-        r4 = r3.replace(
-            url="http://www.example.com/2", body=b"", meta={}, dont_filter=False
-        )
-        assert r4.url == "http://www.example.com/2"
-        assert r4.body == b""
-        assert r4.meta == {}
-        assert r4.dont_filter is False
-
-        # the cls argument allows changing the resulting class
-        custom_request_cls = type("CustomRequest", (self.request_class,), {})
-        r5 = r1.replace(cls=custom_request_cls)
-        assert isinstance(r5, custom_request_cls)
-        assert r5.url == r1.url
-
-    def test_method_always_str(self):
-        r = self.request_class("http://www.example.com", method="POST")
-        assert isinstance(r.method, str)
-
-    def test_immutable_attributes(self):
-        r = self.request_class("http://example.com")
-        with pytest.raises(AttributeError):
-            r.url = "http://example2.com"
-        with pytest.raises(AttributeError):
-            r.body = "xxx"
-
-    def test_callback_and_errback(self):
-        def a_function():
-            pass
-
-        r1 = self.request_class("http://example.com")
-        assert r1.callback is None
-        assert r1.errback is None
-
-        r2 = self.request_class("http://example.com", callback=a_function)
-        assert r2.callback is a_function
-        assert r2.errback is None
-
-        r3 = self.request_class("http://example.com", errback=a_function)
-        assert r3.callback is None
-        assert r3.errback is a_function
-
-        r4 = self.request_class(
-            url="http://example.com",
-            callback=a_function,
-            errback=a_function,
-        )
-        assert r4.callback is a_function
-        assert r4.errback is a_function
-
-        r5 = self.request_class(
-            url="http://example.com",
-            callback=NO_CALLBACK,
-            errback=NO_CALLBACK,
-        )
-        assert r5.callback is NO_CALLBACK
-        assert r5.errback is NO_CALLBACK
-
-    def test_callback_and_errback_type(self):
-        with pytest.raises(TypeError):
-            self.request_class("http://example.com", callback="a_function")
-        with pytest.raises(TypeError):
-            self.request_class("http://example.com", errback="a_function")
-        with pytest.raises(TypeError):
-            self.request_class(
-                url="http://example.com",
-                callback="a_function",
-                errback="a_function",
-            )
-
-    def test_setters(self):
-        request = self.request_class("http://example.com")
-
-        request.flags = ["f1"]
-        assert request.flags == ["f1"]
-
-        request.cookies = {"sid": "1"}
-        assert request.cookies == {"sid": "1"}
-
-        headers = Headers({b"X-Test": b"1"})
-        request.headers = headers
-        assert request._headers is headers
-        request.headers = {b"A": b"b"}
-        assert isinstance(request.headers, Headers)
-        assert request._headers[b"A"] == b"b"
-
-    def test_setter_mutable_lazy_loading(self):
-        """Mutable attributes are set internally to None only until they are
-        read, then they always return the same falsy instance of the
-        corresponding mutable structure.
-
-        Setting them to None causes the next read to return a different object.
-        """
-
-        request = self.request_class("http://example.com")
-
-        assert request._flags is None
-        assert request.flags == []
-        assert request.flags is request.flags
-        assert request._flags == []
-        original_flags = request.flags
-        request.flags = None
-        assert request._flags is None
-        assert request.flags == []
-        assert request.flags is not original_flags
-
-        assert request._cookies is None
-        assert request.cookies == {}
-        assert request.cookies is request.cookies
-        assert request._cookies == {}
-        original_cookies = request.cookies
-        request.cookies = None
-        assert request._cookies is None
-        assert request.cookies == {}
-        assert request.cookies is not original_cookies
-
-        if self.default_headers:
-            assert request._headers == self.default_headers
-            assert request._headers is not self.default_headers
-            assert request.headers == self.default_headers
-        else:
-            assert request._headers is None
-            assert request.headers == {}
-        assert request.headers is request.headers
-        assert isinstance(request.headers, Headers)
-        assert isinstance(request._headers, Headers)
-        original_headers = request.headers
-        request.headers = None
-        assert request._headers is None
-        assert request.headers == {}
-        assert request._headers == {}
-        assert request.headers is not original_headers
-
-    def test_no_callback(self):
-        with pytest.raises(RuntimeError):
-            NO_CALLBACK()
-
-    def test_from_curl(self):
-        # Note: more curated tests regarding curl conversion are in
-        # `test_utils_curl.py`
-        curl_command = (
-            "curl 'http://httpbin.org/post' -X POST -H 'Cookie: _gauges_unique"
-            "_year=1; _gauges_unique=1; _gauges_unique_month=1; _gauges_unique"
-            "_hour=1; _gauges_unique_day=1' -H 'Origin: http://httpbin.org' -H"
-            " 'Accept-Encoding: gzip, deflate' -H 'Accept-Language: en-US,en;q"
-            "=0.9,ru;q=0.8,es;q=0.7' -H 'Upgrade-Insecure-Requests: 1' -H 'Use"
-            "r-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTM"
-            "L, like Gecko) Ubuntu Chromium/62.0.3202.75 Chrome/62.0.3202.75 S"
-            "afari/537.36' -H 'Content-Type: application /x-www-form-urlencode"
-            "d' -H 'Accept: text/html,application/xhtml+xml,application/xml;q="
-            "0.9,image/webp,image/apng,*/*;q=0.8' -H 'Cache-Control: max-age=0"
-            "' -H 'Referer: http://httpbin.org/forms/post' -H 'Connection: kee"
-            "p-alive' --data 'custname=John+Smith&custtel=500&custemail=jsmith"
-            "%40example.org&size=small&topping=cheese&topping=onion&delivery=1"
-            "2%3A15&comments=' --compressed"
-        )
-        r = self.request_class.from_curl(curl_command)
-        assert r.method == "POST"
-        assert r.url == "http://httpbin.org/post"
-        assert (
-            r.body == b"custname=John+Smith&custtel=500&custemail=jsmith%40"
-            b"example.org&size=small&topping=cheese&topping=onion"
-            b"&delivery=12%3A15&comments="
-        )
-        assert r.cookies == {
-            "_gauges_unique_year": "1",
-            "_gauges_unique": "1",
-            "_gauges_unique_month": "1",
-            "_gauges_unique_hour": "1",
-            "_gauges_unique_day": "1",
-        }
-        assert r.headers == {
-            b"Origin": [b"http://httpbin.org"],
-            b"Accept-Encoding": [b"gzip, deflate"],
-            b"Accept-Language": [b"en-US,en;q=0.9,ru;q=0.8,es;q=0.7"],
-            b"Upgrade-Insecure-Requests": [b"1"],
-            b"User-Agent": [
-                b"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537."
-                b"36 (KHTML, like Gecko) Ubuntu Chromium/62.0.3202"
-                b".75 Chrome/62.0.3202.75 Safari/537.36"
-            ],
-            b"Content-Type": [b"application /x-www-form-urlencoded"],
-            b"Accept": [
-                b"text/html,application/xhtml+xml,application/xml;q=0."
-                b"9,image/webp,image/apng,*/*;q=0.8"
-            ],
-            b"Cache-Control": [b"max-age=0"],
-            b"Referer": [b"http://httpbin.org/forms/post"],
-            b"Connection": [b"keep-alive"],
-        }
-
-    def test_from_curl_with_kwargs(self):
-        r = self.request_class.from_curl(
-            'curl -X PATCH "http://example.org"', method="POST", meta={"key": "value"}
-        )
-        assert r.method == "POST"
-        assert r.meta == {"key": "value"}
-
-    def test_from_curl_ignore_unknown_options(self):
-        # By default: it works and ignores the unknown options: --foo and -z
-        with warnings.catch_warnings():  # avoid warning when executing tests
-            warnings.filterwarnings(
-                "ignore", category=UserWarning, message="Unrecognized options:"
-            )
-            r = self.request_class.from_curl(
-                'curl -X DELETE "http://example.org" --foo -z',
-            )
-        assert r.method == "DELETE"
-
-        # If `ignore_unknown_options` is set to `False` it raises an error with
-        # the unknown options: --foo and -z
-        with pytest.raises(ValueError, match="Unrecognized options:"):
-            self.request_class.from_curl(
-                'curl -X PATCH "http://example.org" --foo -z',
-                ignore_unknown_options=False,
-            )
-
-
-class TestXmlRpcRequest(TestRequest):
+class TestXmlRpcRequest(TestRequestBase):
     request_class = XmlRpcRequest
     default_method = "POST"
     default_headers = {b"Content-Type": [b"text/xml"]}

tests/test_http_request_form.py

@@ -10,7 +10,7 @@ from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import FormRequest, HtmlResponse
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.python import to_unicode
-from tests.test_http_request import TestRequest
+from tests.utils.bases.http_request import TestRequestBase
 
 
 def _buildresponse(body, **kwargs):
@@ -31,7 +31,7 @@ def _qs(req, encoding="utf-8", to_unicode=False):
 # FormRequest.from_response() is deprecated in favor of form2request, so the
 # many tests below that exercise it ignore the resulting deprecation warning.
 @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
-class TestFormRequest(TestRequest):
+class TestFormRequest(TestRequestBase):
     request_class = FormRequest
 
     def assertQueryEqual(self, first, second, msg=None):

tests/test_http_request_json.py

@@ -8,10 +8,10 @@ import pytest
 
 from scrapy.http import JsonRequest
 from scrapy.utils.python import to_bytes
-from tests.test_http_request import TestRequest
+from tests.utils.bases.http_request import TestRequestBase
 
 
-class TestJsonRequest(TestRequest):
+class TestJsonRequest(TestRequestBase):
     request_class = JsonRequest
     default_method = "GET"
     default_headers = {

tests/test_http_response.py

@@ -1,413 +1,8 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING
-
-import pytest
-from w3lib.encoding import resolve_encoding
-
-from scrapy.exceptions import NotSupported
-from scrapy.http import Headers, Request, Response
-from scrapy.link import Link
-from scrapy.utils._deps_compat import W3LIB_STRIPS_URLS
-from tests import get_testdata
-
-if TYPE_CHECKING:
-    from collections.abc import Iterable
+from scrapy.http import Response
+from tests.utils.bases.http_response import TestResponseBase
 
 
-class TestResponse:
+class TestResponse(TestResponseBase):
     response_class = Response
-
-    def test_init(self):
-        # Response requires url in the constructor
-        with pytest.raises(TypeError):
-            self.response_class()
-        assert isinstance(
-            self.response_class("http://example.com/"), self.response_class
-        )
-        with pytest.raises(TypeError):
-            self.response_class(b"http://example.com")
-        with pytest.raises(TypeError):
-            self.response_class(url="http://example.com", body={})
-        # body can be str or None
-        assert isinstance(
-            self.response_class("http://example.com/", body=b""),
-            self.response_class,
-        )
-        assert isinstance(
-            self.response_class("http://example.com/", body=b"body"),
-            self.response_class,
-        )
-        # test presence of all optional parameters
-        assert isinstance(
-            self.response_class(
-                "http://example.com/", body=b"", headers={}, status=200
-            ),
-            self.response_class,
-        )
-
-        r = self.response_class("http://www.example.com")
-        assert isinstance(r.url, str)
-        assert r.url == "http://www.example.com"
-        assert r.status == 200
-
-        assert isinstance(r.headers, Headers)
-        assert not r.headers
-
-        headers = {"foo": "bar"}
-        body = b"a body"
-        r = self.response_class("http://www.example.com", headers=headers, body=body)
-
-        assert r.headers is not headers
-        assert r.headers[b"foo"] == b"bar"
-
-        r = self.response_class("http://www.example.com", status=301)
-        assert r.status == 301
-        r = self.response_class("http://www.example.com", status="301")
-        assert r.status == 301
-        with pytest.raises(ValueError, match=r"invalid literal for int\(\)"):
-            self.response_class("http://example.com", status="lala200")
-
-    def test_copy(self):
-        """Test Response copy"""
-
-        r1 = self.response_class("http://www.example.com", body=b"Some body")
-        r1.flags.append("cached")
-        r2 = r1.copy()
-
-        assert r1.status == r2.status
-        assert r1.body == r2.body
-
-        # make sure flags list is shallow copied
-        assert r1.flags is not r2.flags, "flags must be a shallow copy, not identical"
-        assert r1.flags == r2.flags
-
-        # make sure headers attribute is shallow copied
-        assert r1.headers is not r2.headers, (
-            "headers must be a shallow copy, not identical"
-        )
-        assert r1.headers == r2.headers
-
-    def test_copy_meta(self):
-        req = Request("http://www.example.com")
-        req.meta["foo"] = "bar"
-        r1 = self.response_class(
-            "http://www.example.com", body=b"Some body", request=req
-        )
-        assert r1.meta is req.meta
-
-    def test_copy_cb_kwargs(self):
-        req = Request("http://www.example.com")
-        req.cb_kwargs["foo"] = "bar"
-        r1 = self.response_class(
-            "http://www.example.com", body=b"Some body", request=req
-        )
-        assert r1.cb_kwargs is req.cb_kwargs
-
-    def test_unavailable_meta(self):
-        r1 = self.response_class("http://www.example.com", body=b"Some body")
-        with pytest.raises(AttributeError, match=r"Response\.meta not available"):
-            r1.meta
-
-    def test_unavailable_cb_kwargs(self):
-        r1 = self.response_class("http://www.example.com", body=b"Some body")
-        with pytest.raises(AttributeError, match=r"Response\.cb_kwargs not available"):
-            r1.cb_kwargs
-
-    def test_copy_inherited_classes(self):
-        """Test Response children copies preserve their class"""
-
-        class CustomResponse(self.response_class):
-            pass
-
-        r1 = CustomResponse("http://www.example.com")
-        r2 = r1.copy()
-
-        assert isinstance(r2, CustomResponse)
-
-    def test_replace(self):
-        """Test Response.replace() method"""
-        hdrs = Headers({"key": "value"})
-        r1 = self.response_class("http://www.example.com")
-        r2 = r1.replace(status=301, body=b"New body", headers=hdrs)
-        assert r1.body == b""
-        assert r1.url == r2.url
-        assert (r1.status, r2.status) == (200, 301)
-        assert (r1.body, r2.body) == (b"", b"New body")
-        assert (r1.headers, r2.headers) == ({}, hdrs)
-
-        # Empty attributes (which may fail if not compared properly)
-        r3 = self.response_class("http://www.example.com", flags=["cached"])
-        r4 = r3.replace(body=b"", flags=[])
-        assert r4.body == b""
-        assert not r4.flags
-
-    def _assert_response_values(self, response, encoding, body):
-        if isinstance(body, str):
-            body_unicode = body
-            body_bytes = body.encode(encoding)
-        else:
-            body_unicode = body.decode(encoding)
-            body_bytes = body
-
-        assert isinstance(response.body, bytes)
-        assert isinstance(response.text, str)
-        self._assert_response_encoding(response, encoding)
-        assert response.body == body_bytes
-        assert response.text == body_unicode
-
-    def _assert_response_encoding(self, response, encoding):
-        assert response.encoding == resolve_encoding(encoding)
-
-    def test_immutable_attributes(self):
-        r = self.response_class("http://example.com")
-        with pytest.raises(AttributeError):
-            r.url = "http://example2.com"
-        with pytest.raises(AttributeError):
-            r.body = "xxx"
-
-    def test_setter_mutable_lazy_loading(self):
-        """Mutable attributes are set internally to None only until they are
-        read, then they always return the same falsy instance of the
-        corresponding mutable structure.
-
-        Setting them to None causes the next read to return a different object.
-        """
-
-        response = self.response_class("http://example.com")
-
-        response.request = Request("http://example.com")
-
-        assert response._flags is None
-        assert response.flags == []
-        assert response.flags is response.flags
-        assert response._flags == []
-        original_flags = response.flags
-        response.flags = None
-        assert response._flags is None
-        assert response.flags == []
-        assert response.flags is not original_flags
-
-        assert response._headers is None
-        assert response.headers == {}
-        assert response.headers is response.headers
-        assert isinstance(response.headers, Headers)
-        assert isinstance(response._headers, Headers)
-        original_headers = response.headers
-        response.headers = None
-        assert response._headers is None
-        assert response.headers == {}
-        assert response._headers == {}
-        assert response.headers is not original_headers
-
-    def test_setters(self):
-        response = self.response_class("http://example.com")
-
-        response.flags = ["f1"]
-        assert response.flags == ["f1"]
-
-        headers = Headers({b"X-Test": b"1"})
-        response.headers = headers
-        assert response._headers is headers
-        response.headers = {b"A": b"b"}
-        assert isinstance(response.headers, Headers)
-        assert response._headers[b"A"] == b"b"
-
-    def test_urljoin(self):
-        """Test urljoin shortcut (only for existence, since behavior equals urljoin)"""
-        joined = self.response_class("http://www.example.com").urljoin("/test")
-        absolute = "http://www.example.com/test"
-        assert joined == absolute
-
-    def test_shortcut_attributes(self):
-        r = self.response_class("http://example.com", body=b"hello")
-        if self.response_class == Response:
-            msg = "Response content isn't text"
-            with pytest.raises(AttributeError, match=msg):
-                r.text
-            with pytest.raises(NotSupported, match=msg):
-                r.css("body")
-            with pytest.raises(NotSupported, match=msg):
-                r.xpath("//body")
-            with pytest.raises(NotSupported, match=msg):
-                r.jmespath("body")
-        else:
-            r.text
-            r.css("body")
-            r.xpath("//body")
-
-    # Response.follow
-
-    def test_follow_url_absolute(self):
-        self._assert_followed_url("http://foo.example.com", "http://foo.example.com")
-
-    def test_follow_url_relative(self):
-        self._assert_followed_url("foo", "http://example.com/foo")
-
-    def test_follow_link(self):
-        self._assert_followed_url(
-            Link("http://example.com/foo"), "http://example.com/foo"
-        )
-
-    def test_follow_None_url(self):
-        r = self.response_class("http://example.com")
-        with pytest.raises(ValueError, match="url can't be None"):
-            r.follow(None)
-
-    def test_follow_None_encoding(self):
-        r = self.response_class("http://example.com")
-        with pytest.raises(ValueError, match="encoding can't be None"):
-            r.follow("foo", encoding=None)
-
-    @pytest.mark.xfail(
-        not W3LIB_STRIPS_URLS,
-        reason="https://github.com/scrapy/w3lib/pull/207",
-        strict=True,
-    )
-    def test_follow_whitespace_url(self):
-        self._assert_followed_url("foo ", "http://example.com/foo")
-
-    @pytest.mark.xfail(
-        not W3LIB_STRIPS_URLS,
-        reason="https://github.com/scrapy/w3lib/pull/207",
-        strict=True,
-    )
-    def test_follow_whitespace_link(self):
-        self._assert_followed_url(
-            Link("http://example.com/foo "), "http://example.com/foo"
-        )
-
-    def test_follow_flags(self):
-        res = self.response_class("http://example.com/")
-        fol = res.follow("http://example.com/", flags=["cached", "allowed"])
-        assert fol.flags == ["cached", "allowed"]
-
-    # Response.follow_all
-
-    def test_follow_all_absolute(self):
-        url_list = [
-            "http://example.org",
-            "http://www.example.org",
-            "http://example.com",
-            "http://www.example.com",
-        ]
-        self._assert_followed_all_urls(url_list, url_list)
-
-    def test_follow_all_relative(self):
-        relative = ["foo", "bar", "foo/bar", "bar/foo"]
-        absolute = [
-            "http://example.com/foo",
-            "http://example.com/bar",
-            "http://example.com/foo/bar",
-            "http://example.com/bar/foo",
-        ]
-        self._assert_followed_all_urls(relative, absolute)
-
-    def test_follow_all_links(self):
-        absolute = [
-            "http://example.com/foo",
-            "http://example.com/bar",
-            "http://example.com/foo/bar",
-            "http://example.com/bar/foo",
-        ]
-        links = map(Link, absolute)
-        self._assert_followed_all_urls(links, absolute)
-
-    def test_follow_all_empty(self):
-        r = self.response_class("http://example.com")
-        assert not list(r.follow_all([]))
-
-    def test_follow_all_invalid(self):
-        r = self.response_class("http://example.com")
-        if self.response_class == Response:
-            with pytest.raises(TypeError):
-                list(r.follow_all(urls=None))
-            with pytest.raises(TypeError):
-                list(r.follow_all(urls=12345))
-            with pytest.raises(ValueError, match="url can't be None"):
-                list(r.follow_all(urls=[None]))
-        else:
-            with pytest.raises(
-                ValueError, match="Please supply exactly one of the following arguments"
-            ):
-                list(r.follow_all(urls=None))
-            with pytest.raises(TypeError):
-                list(r.follow_all(urls=12345))
-            with pytest.raises(ValueError, match="url can't be None"):
-                list(r.follow_all(urls=[None]))
-
-    @pytest.mark.xfail(
-        not W3LIB_STRIPS_URLS,
-        reason="https://github.com/scrapy/w3lib/pull/207",
-        strict=True,
-    )
-    def test_follow_all_whitespace(self):
-        relative = ["foo ", "bar ", "foo/bar ", "bar/foo "]
-        absolute = [
-            "http://example.com/foo",
-            "http://example.com/bar",
-            "http://example.com/foo/bar",
-            "http://example.com/bar/foo",
-        ]
-        self._assert_followed_all_urls(relative, absolute)
-
-    @pytest.mark.xfail(
-        not W3LIB_STRIPS_URLS,
-        reason="https://github.com/scrapy/w3lib/pull/207",
-        strict=True,
-    )
-    def test_follow_all_whitespace_links(self):
-        absolute = [
-            "http://example.com/foo ",
-            "http://example.com/bar ",
-            "http://example.com/foo/bar ",
-            "http://example.com/bar/foo ",
-        ]
-        links = [Link(u) for u in absolute]
-        expected = [u.strip() for u in absolute]
-        self._assert_followed_all_urls(links, expected)
-
-    def test_follow_all_flags(self):
-        re = self.response_class("http://www.example.com/")
-        urls = [
-            "http://www.example.com/",
-            "http://www.example.com/2",
-            "http://www.example.com/foo",
-        ]
-        fol = re.follow_all(urls, flags=["cached", "allowed"])
-        for req in fol:
-            assert req.flags == ["cached", "allowed"]
-
-    def _assert_followed_url(
-        self,
-        follow_obj: str | Link,
-        target_url: str,
-        response: Response | None = None,
-        encoding: str | None = None,
-    ) -> None:
-        if response is None:
-            response = self._links_response()
-        req = response.follow(follow_obj)
-        assert req.url == target_url
-        if encoding is not None:
-            assert req.encoding == encoding
-
-    def _assert_followed_all_urls(
-        self,
-        follow_obj: Iterable[str | Link],
-        target_urls: Iterable[str],
-        response: Response | None = None,
-    ) -> None:
-        if response is None:
-            response = self._links_response()
-        followed = response.follow_all(follow_obj)
-        for req, target in zip(followed, target_urls, strict=True):
-            assert req.url == target
-
-    def _links_response(self) -> Response:
-        body = get_testdata("link_extractor", "linkextractor.html")
-        return self.response_class("http://example.com/index", body=body)
-
-    def _links_response_no_href(self) -> Response:
-        body = get_testdata("link_extractor", "linkextractor_no_href.html")
-        return self.response_class("http://example.com/index", body=body)

tests/test_http_response_text.py

@@ -8,10 +8,10 @@ import pytest
 from scrapy.http import HtmlResponse, TextResponse, XmlResponse
 from scrapy.selector import Selector
 from scrapy.utils.python import to_unicode
-from tests.test_http_response import TestResponse
+from tests.utils.bases.http_response import TestResponseBase
 
 
-class TestTextResponse(TestResponse):
+class TestTextResponse(TestResponseBase):
     response_class = TextResponse
 
     def test_follow_None_encoding(self):

tests/test_link.py

@@ -1,18 +1,20 @@
+from __future__ import annotations
+
 import pytest
 
 from scrapy.link import Link
 
 
 class TestLink:
-    def _assert_same_links(self, link1, link2):
+    def _assert_same_links(self, link1: Link, link2: Link) -> None:
         assert link1 == link2
         assert hash(link1) == hash(link2)
 
-    def _assert_different_links(self, link1, link2):
+    def _assert_different_links(self, link1: Link, link2: Link) -> None:
         assert link1 != link2
         assert hash(link1) != hash(link2)
 
-    def test_eq_and_hash(self):
+    def test_eq_and_hash(self) -> None:
         l1 = Link("http://www.example.com")
         l2 = Link("http://www.example.com/other")
         l3 = Link("http://www.example.com")
@@ -45,17 +47,17 @@ class TestLink:
         self._assert_different_links(l7, l9)
         self._assert_different_links(l7, l10)
 
-    def test_repr(self):
+    def test_repr(self) -> None:
         l1 = Link(
             "http://www.example.com", text="test", fragment="something", nofollow=True
         )
         l2 = eval(repr(l1))
         self._assert_same_links(l1, l2)
 
-    def test_bytes_url(self):
+    def test_bytes_url(self) -> None:
         with pytest.raises(TypeError):
-            Link(b"http://www.example.com/\xc2\xa3")
+            Link(b"http://www.example.com/\xc2\xa3")  # type: ignore[arg-type]
 
-    def test_eq_non_link(self):
+    def test_eq_non_link(self) -> None:
         url = "http://example.com"
         assert Link(url) != url

tests/test_logformatter.py

@@ -1,7 +1,9 @@
+from __future__ import annotations
+
 import logging
+from typing import TYPE_CHECKING
 
 import pytest
-from testfixtures import LogCapture
 from twisted.python.failure import Failure
 
 from scrapy.exceptions import DropItem
@@ -10,9 +12,11 @@ from scrapy.item import Field, Item
 from scrapy.logformatter import LogFormatter
 from scrapy.spiders import Spider
 from scrapy.utils.test import get_crawler
-from tests.mockserver.http import MockServer
 from tests.spiders import ItemSpider
-from tests.utils.decorators import inline_callbacks_test
+from tests.utils.decorators import coroutine_test
+
+if TYPE_CHECKING:
+    from tests.mockserver.http import MockServer
 
 
 class CustomItem(Item):
@@ -254,15 +258,6 @@ class DropSomeItemsPipeline:
 
 
 class TestShowOrSkipMessages:
-    @classmethod
-    def setup_class(cls):
-        cls.mockserver = MockServer()
-        cls.mockserver.__enter__()
-
-    @classmethod
-    def teardown_class(cls):
-        cls.mockserver.__exit__(None, None, None)
-
     def setup_method(self):
         self.base_settings = {
             "LOG_LEVEL": "DEBUG",
@@ -271,22 +266,26 @@ class TestShowOrSkipMessages:
             },
         }
 
-    @inline_callbacks_test
-    def test_show_messages(self):
+    @coroutine_test
+    async def test_show_messages(
+        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+    ) -> None:
         crawler = get_crawler(ItemSpider, self.base_settings)
-        with LogCapture() as lc:
-            yield crawler.crawl(mockserver=self.mockserver)
-        assert "Scraped from <200 http://127.0.0.1:" in str(lc)
-        assert "Crawled (200) <GET http://127.0.0.1:" in str(lc)
-        assert "Dropped: Ignoring item" in str(lc)
+        with caplog.at_level(logging.DEBUG):
+            await crawler.crawl_async(mockserver=mockserver)
+        assert "Scraped from <200 http://127.0.0.1:" in caplog.text
+        assert "Crawled (200) <GET http://127.0.0.1:" in caplog.text
+        assert "Dropped: Ignoring item" in caplog.text
 
-    @inline_callbacks_test
-    def test_skip_messages(self):
+    @coroutine_test
+    async def test_skip_messages(
+        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+    ) -> None:
         settings = self.base_settings.copy()
         settings["LOG_FORMATTER"] = SkipMessagesLogFormatter
         crawler = get_crawler(ItemSpider, settings)
-        with LogCapture() as lc:
-            yield crawler.crawl(mockserver=self.mockserver)
-        assert "Scraped from <200 http://127.0.0.1:" not in str(lc)
-        assert "Crawled (200) <GET http://127.0.0.1:" not in str(lc)
-        assert "Dropped: Ignoring item" not in str(lc)
+        with caplog.at_level(logging.DEBUG):
+            await crawler.crawl_async(mockserver=mockserver)
+        assert "Scraped from <200 http://127.0.0.1:" not in caplog.text
+        assert "Crawled (200) <GET http://127.0.0.1:" not in caplog.text
+        assert "Dropped: Ignoring item" not in caplog.text

tests/test_logstats.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from datetime import datetime
 
 import pytest
@@ -9,16 +11,17 @@ from tests.utils.decorators import coroutine_test
 
 
 class TestLogStats:
-    def setup_method(self):
+    def setup_method(self) -> None:
         self.crawler = get_crawler(SimpleSpider)
         self.spider = self.crawler._create_spider("spidey")
+        assert self.crawler.stats is not None
         self.stats = self.crawler.stats
 
         self.stats.set_value("response_received_count", 4802)
         self.stats.set_value("item_scraped_count", 3201)
 
     @coroutine_test
-    async def test_stats_calculations(self):
+    async def test_stats_calculations(self) -> None:
         logstats = LogStats.from_crawler(self.crawler)
 
         with pytest.raises(AttributeError):
@@ -56,7 +59,7 @@ class TestLogStats:
         assert self.stats.get_value("responses_per_minute") == 172.9
         assert self.stats.get_value("items_per_minute") == 116.4
 
-    def test_stats_calculations_no_time(self):
+    def test_stats_calculations_no_time(self) -> None:
         """The stat values should be None since the start and finish time are
         not available.
         """
@@ -65,7 +68,7 @@ class TestLogStats:
         assert self.stats.get_value("responses_per_minute") is None
         assert self.stats.get_value("items_per_minute") is None
 
-    def test_stats_calculation_no_elapsed_time(self):
+    def test_stats_calculation_no_elapsed_time(self) -> None:
         """The stat values should be None since the elapsed time is 0."""
         logstats = LogStats.from_crawler(self.crawler)
         self.stats.set_value("start_time", datetime.fromtimestamp(1655100172))

tests/test_pipeline_crawl.py

@@ -1,12 +1,12 @@
 from __future__ import annotations
 
+import logging
 import shutil
 from pathlib import Path
 from tempfile import mkdtemp
 from typing import TYPE_CHECKING, Any
 
 import pytest
-from testfixtures import LogCapture
 from w3lib.url import add_or_replace_parameter
 
 from scrapy import Spider, signals
@@ -14,7 +14,7 @@ from scrapy.utils.misc import load_object
 from scrapy.utils.test import get_crawler
 from tests.mockserver.http import MockServer
 from tests.spiders import SimpleSpider
-from tests.utils.decorators import inline_callbacks_test
+from tests.utils.decorators import coroutine_test
 
 if TYPE_CHECKING:
     from scrapy.crawler import Crawler
@@ -58,6 +58,8 @@ class RedirectedMediaDownloadSpider(MediaDownloadSpider):
 
 
 class TestFileDownloadCrawl:
+    mockserver: MockServer
+
     pipeline_class = "scrapy.pipelines.files.FilesPipeline"
     store_setting_key = "FILES_STORE"
     media_key = "files"
@@ -102,7 +104,7 @@ class TestFileDownloadCrawl:
         crawler.signals.connect(self._on_item_scraped, signals.item_scraped)
         return crawler
 
-    def _assert_files_downloaded(self, items, logs):
+    def _assert_files_downloaded(self, items: list[Any], logs: str) -> None:
         assert len(items) == 1
         assert self.media_key in items[0]
 
@@ -125,13 +127,16 @@ class TestFileDownloadCrawl:
             for i in item[self.media_key]:
                 assert (self.tmpmediastore / i["path"]).exists()
 
-    def _assert_files_download_failure(self, crawler, items, code, logs):
+    def _assert_files_download_failure(
+        self, crawler: Crawler, items: list[Any], code: int, logs: str
+    ) -> None:
         # check that the item does NOT have the "images/files" field populated
         assert len(items) == 1
         assert self.media_key in items[0]
         assert not items[0][self.media_key]
 
         # check that there was 1 successful fetch and 3 other responses with non-200 code
+        assert crawler.stats
         assert crawler.stats.get_value("downloader/request_method_count/GET") == 4
         assert crawler.stats.get_value("downloader/response_count") == 4
         assert crawler.stats.get_value("downloader/response_status_count/200") == 1
@@ -144,62 +149,71 @@ class TestFileDownloadCrawl:
         # check that no files were written to the media store
         assert not list(self.tmpmediastore.iterdir())
 
-    @inline_callbacks_test
-    def test_download_media(self):
+    @coroutine_test
+    async def test_download_media(self, caplog: pytest.LogCaptureFixture) -> None:
         crawler = self._create_crawler(MediaDownloadSpider)
-        with LogCapture() as log:
-            yield crawler.crawl(
+        with caplog.at_level(logging.DEBUG):
+            await crawler.crawl_async(
                 self.mockserver.url("/static/files/images/"),
                 media_key=self.media_key,
                 media_urls_key=self.media_urls_key,
             )
-        self._assert_files_downloaded(self.items, str(log))
+        self._assert_files_downloaded(self.items, caplog.text)
 
-    @inline_callbacks_test
-    def test_download_media_wrong_urls(self):
+    @coroutine_test
+    async def test_download_media_wrong_urls(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         crawler = self._create_crawler(BrokenLinksMediaDownloadSpider)
-        with LogCapture() as log:
-            yield crawler.crawl(
+        with caplog.at_level(logging.DEBUG):
+            await crawler.crawl_async(
                 self.mockserver.url("/static/files/images/"),
                 media_key=self.media_key,
                 media_urls_key=self.media_urls_key,
             )
-        self._assert_files_download_failure(crawler, self.items, 404, str(log))
+        self._assert_files_download_failure(crawler, self.items, 404, caplog.text)
 
-    @inline_callbacks_test
-    def test_download_media_redirected_default_failure(self):
+    @coroutine_test
+    async def test_download_media_redirected_default_failure(
+        self, caplog: pytest.LogCaptureFixture
+    ):
         crawler = self._create_crawler(RedirectedMediaDownloadSpider)
-        with LogCapture() as log:
-            yield crawler.crawl(
+        with caplog.at_level(logging.DEBUG):
+            await crawler.crawl_async(
                 self.mockserver.url("/static/files/images/"),
                 media_key=self.media_key,
                 media_urls_key=self.media_urls_key,
                 mockserver=self.mockserver,
             )
-        self._assert_files_download_failure(crawler, self.items, 302, str(log))
+        self._assert_files_download_failure(crawler, self.items, 302, caplog.text)
 
-    @inline_callbacks_test
-    def test_download_media_redirected_allowed(self):
+    @coroutine_test
+    async def test_download_media_redirected_allowed(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         settings = {
             **self.settings,
             "MEDIA_ALLOW_REDIRECTS": True,
         }
         crawler = self._create_crawler(RedirectedMediaDownloadSpider, settings)
-        with LogCapture() as log:
-            yield crawler.crawl(
+        with caplog.at_level(logging.DEBUG):
+            await crawler.crawl_async(
                 self.mockserver.url("/static/files/images/"),
                 media_key=self.media_key,
                 media_urls_key=self.media_urls_key,
                 mockserver=self.mockserver,
             )
-        self._assert_files_downloaded(self.items, str(log))
+        self._assert_files_downloaded(self.items, caplog.text)
+        assert crawler.stats
         assert crawler.stats.get_value("downloader/response_status_count/302") == 3
 
-    @inline_callbacks_test
-    def test_download_media_file_path_error(self):
+    @coroutine_test
+    async def test_download_media_file_path_error(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         cls = load_object(self.pipeline_class)
 
-        class ExceptionRaisingMediaPipeline(cls):
+        class ExceptionRaisingMediaPipeline(cls):  # type: ignore[misc,valid-type]
             def file_path(self, request, response=None, info=None, *, item=None):
                 return 1 / 0
 
@@ -208,14 +222,14 @@ class TestFileDownloadCrawl:
             "ITEM_PIPELINES": {ExceptionRaisingMediaPipeline: 1},
         }
         crawler = self._create_crawler(MediaDownloadSpider, settings)
-        with LogCapture() as log:
-            yield crawler.crawl(
+        with caplog.at_level(logging.DEBUG):
+            await crawler.crawl_async(
                 self.mockserver.url("/static/files/images/"),
                 media_key=self.media_key,
                 media_urls_key=self.media_urls_key,
                 mockserver=self.mockserver,
             )
-        assert "ZeroDivisionError" in str(log)
+        assert "ZeroDivisionError" in caplog.text
 
 
 pillow_available: bool

tests/test_pipeline_files.py

@@ -2,6 +2,7 @@ import base64
 import dataclasses
 import logging
 import random
+import re
 import time
 from abc import ABC, abstractmethod
 from datetime import datetime
@@ -19,18 +20,20 @@ import attr
 import pytest
 from itemadapter import ItemAdapter
 from twisted.internet.defer import Deferred
+from twisted.python.failure import Failure
 
-from scrapy.exceptions import NotConfigured
+from scrapy.exceptions import IgnoreRequest, NotConfigured
 from scrapy.http import Request, Response
 from scrapy.item import Field, Item
 from scrapy.pipelines.files import (
+    FileException,
     FilesPipeline,
     FSFilesStore,
     FTPFilesStore,
     GCSFilesStore,
     S3FilesStore,
 )
-from scrapy.pipelines.media import MediaPipeline
+from scrapy.pipelines.media import MediaPipeline, _MediaRequestFiltered
 from scrapy.settings import Settings
 from scrapy.utils.asyncio import call_later
 from scrapy.utils.defer import maybe_deferred_to_future
@@ -39,8 +42,8 @@ from scrapy.utils.test import get_crawler
 from tests.mockserver.ftp import MockFTPServer
 from tests.utils.decorators import coroutine_test, inline_callbacks_test
 
-from .test_pipeline_media import _mocked_download_func
 from .utils.cloud import mock_google_cloud_storage
+from .utils.media_pipelines import mocked_download_func
 
 # required by persist_file() and stat_file(), but as some stores don't use the argument
 # we can pass this singleton to keep type hints correct
@@ -94,7 +97,7 @@ class TestFilesPipeline:
         settings_dict = {"FILES_STORE": self.tempdir}
         crawler = get_crawler(DefaultSpider, settings_dict=settings_dict)
         crawler.spider = crawler._create_spider()
-        crawler.engine = MagicMock(download_async=_mocked_download_func)
+        crawler.engine = MagicMock(download_async=mocked_download_func)
         self.pipeline = FilesPipeline.from_crawler(crawler)
         self.pipeline.open_spider()
 
@@ -290,6 +293,71 @@ class TestFilesPipeline:
         request = Request("http://example.com")
         assert file_path(request, item=item) == "full/path-to-store-file"
 
+    def test_media_failed_filtered_request(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
+        """A filtered media request (IgnoreRequest) is reported as a
+        _MediaRequestFiltered exception and logged at the DEBUG level, instead
+        of as a download error with a traceback."""
+        request = Request("http://example.com/file.pdf")
+        reason = "Filtered offsite request to 'example.com'"
+        failure = Failure(IgnoreRequest(reason))
+
+        with (
+            caplog.at_level(logging.DEBUG),
+            pytest.raises(_MediaRequestFiltered, match=re.escape(reason)),
+        ):
+            self.pipeline.media_failed(failure, request, self.pipeline.spiderinfo)
+
+        assert len(caplog.records) == 1
+        record = caplog.records[0]
+        assert record.levelname == "DEBUG"
+        assert record.exc_info is None
+        assert reason in record.getMessage()
+
+    def test_media_failed_download_error(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
+        """A genuine download error is reported as a FileException and logged as
+        a warning."""
+        request = Request("http://example.com/file.pdf")
+        failure = Failure(Exception("boom"))
+
+        with caplog.at_level(logging.WARNING), pytest.raises(FileException):
+            self.pipeline.media_failed(failure, request, self.pipeline.spiderinfo)
+
+        assert len(caplog.records) == 1
+        assert caplog.records[0].levelname == "WARNING"
+
+    @coroutine_test
+    async def test_process_item_filtered_request(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
+        """A filtered (e.g. offsite) media request is processed as a failed
+        result without being logged as an error with a traceback."""
+        item_url = "http://example.com/file.pdf"
+        item = _create_item_with_files(item_url)
+        request = Request(
+            item_url,
+            meta={
+                "response": IgnoreRequest("Filtered offsite request to 'example.com'")
+            },
+        )
+        with (
+            caplog.at_level(logging.DEBUG),
+            mock.patch.object(
+                FilesPipeline, "get_media_requests", return_value=[request]
+            ),
+        ):
+            result = await self.pipeline.process_item(item)
+
+        assert result["files"] == []
+        assert not any(r.levelname in ("WARNING", "ERROR") for r in caplog.records)
+        assert any(
+            "Filtered offsite request to 'example.com'" in r.getMessage()
+            for r in caplog.records
+        )
+
     @pytest.mark.parametrize(
         "bad_type",
         [

tests/test_pipeline_media.py

@@ -1,33 +1,23 @@
 from __future__ import annotations
 
+import logging
 from unittest.mock import MagicMock
 
 import pytest
-from testfixtures import LogCapture
 from twisted.python.failure import Failure
 
 from scrapy import signals
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Request, Response
-from scrapy.http.request import NO_CALLBACK
 from scrapy.pipelines.files import FileException
-from scrapy.pipelines.media import MediaPipeline
+from scrapy.pipelines.media import MediaPipeline, _MediaRequestFiltered
 from scrapy.utils.defer import _defer_sleep_async
 from scrapy.utils.log import failure_to_exc_info
 from scrapy.utils.signal import disconnect_all
 from scrapy.utils.spider import DefaultSpider
 from scrapy.utils.test import get_crawler
 from tests.utils.decorators import coroutine_test
-
-
-async def _mocked_download_func(request):
-    assert request.callback is NO_CALLBACK
-    response = request.meta.get("response")
-    if callable(response):
-        response = await response()
-    if isinstance(response, Exception):
-        raise response
-    return response
+from tests.utils.media_pipelines import mocked_download_func
 
 
 class UserDefinedPipeline(MediaPipeline):
@@ -54,7 +44,7 @@ class TestBaseMediaPipeline:
     def setup_method(self):
         crawler = get_crawler(DefaultSpider, self.settings)
         crawler.spider = crawler._create_spider()
-        crawler.engine = MagicMock(download_async=_mocked_download_func)
+        crawler.engine = MagicMock(download_async=mocked_download_func)
         self.pipe = self.pipeline_class.from_crawler(crawler)
         self.pipe.open_spider()
         self.info = self.pipe.spiderinfo
@@ -138,7 +128,7 @@ class TestBaseMediaPipeline:
         context = getattr(info.downloaded[fp].value, "__context__", None)
         assert context is None
 
-    def test_default_item_completed(self):
+    def test_default_item_completed(self, caplog: pytest.LogCaptureFixture) -> None:
         item = {"name": "name"}
         assert self.pipe.item_completed([], item, self.info) is item
 
@@ -146,21 +136,35 @@ class TestBaseMediaPipeline:
         fail = Failure(Exception())
         results = [(True, 1), (False, fail)]
 
-        with LogCapture() as log:
-            new_item = self.pipe.item_completed(results, item, self.info)
-
+        caplog.clear()
+        new_item = self.pipe.item_completed(results, item, self.info)
         assert new_item is item
-        assert len(log.records) == 1
-        record = log.records[0]
+        assert len(caplog.records) == 1
+        record = caplog.records[0]
         assert record.levelname == "ERROR"
         assert record.exc_info == failure_to_exc_info(fail)
 
         # disable failure logging and check again
+        caplog.clear()
         self.pipe.LOG_FAILED_RESULTS = False
-        with LogCapture() as log:
-            new_item = self.pipe.item_completed(results, item, self.info)
+        new_item = self.pipe.item_completed(results, item, self.info)
         assert new_item is item
-        assert len(log.records) == 0
+        assert len(caplog.records) == 0
+
+    def test_item_completed_filtered_request_not_logged(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
+        """Filtered media requests (e.g. offsite ones) are not logged as errors
+        by item_completed(), as they are not download errors."""
+        item = {"name": "name"}
+        fail = Failure(_MediaRequestFiltered("Filtered offsite request"))
+        results = [(True, 1), (False, fail)]
+
+        with caplog.at_level(logging.DEBUG):
+            new_item = self.pipe.item_completed(results, item, self.info)
+
+        assert new_item is item
+        assert len(caplog.records) == 0
 
     @coroutine_test
     async def test_default_process_item(self):

tests/test_pqueues.py

@@ -11,7 +11,7 @@ from scrapy.spiders import Spider
 from scrapy.squeues import FifoMemoryQueue, PickleFifoDiskQueue
 from scrapy.utils.misc import build_from_crawler, load_object
 from scrapy.utils.test import get_crawler
-from tests.test_scheduler import MockDownloader
+from tests.utils.downloader import MockDownloader
 
 
 class TestPriorityQueue:

tests/test_request_attribute_binding.py

@@ -1,11 +1,17 @@
-from testfixtures import LogCapture
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING
 
 from scrapy import Request, signals
 from scrapy.http.response import Response
 from scrapy.utils.test import get_crawler
 from tests.mockserver.http import MockServer
 from tests.spiders import SingleRequestSpider
-from tests.utils.decorators import inline_callbacks_test
+from tests.utils.decorators import coroutine_test, inline_callbacks_test
+
+if TYPE_CHECKING:
+    import pytest
 
 OVERRIDDEN_URL = "https://example.org"
 
@@ -63,6 +69,8 @@ class AlternativeCallbacksMiddleware:
 
 
 class TestCrawl:
+    mockserver: MockServer
+
     @classmethod
     def setup_class(cls):
         cls.mockserver = MockServer()
@@ -107,8 +115,10 @@ class TestCrawl:
         assert failure.request.url == url
         assert isinstance(failure.value, ZeroDivisionError)
 
-    @inline_callbacks_test
-    def test_downloader_middleware_override_request_in_process_response(self):
+    @coroutine_test
+    async def test_downloader_middleware_override_request_in_process_response(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         """
         Downloader middleware which returns a response with an specific 'request' attribute.
 
@@ -133,22 +143,21 @@ class TestCrawl:
         )
         crawler.signals.connect(signal_handler, signal=signals.response_received)
 
-        with LogCapture() as log:
-            yield crawler.crawl(seed=url, mockserver=self.mockserver)
+        with caplog.at_level(logging.DEBUG):
+            await crawler.crawl_async(seed=url, mockserver=self.mockserver)
 
+        assert isinstance(crawler.spider, SingleRequestSpider)
         response = crawler.spider.meta["responses"][0]
         assert response.request.url == OVERRIDDEN_URL
 
         assert signal_params["response"].url == url
         assert signal_params["request"].url == OVERRIDDEN_URL
 
-        log.check_present(
-            (
-                "scrapy.core.engine",
-                "DEBUG",
-                f"Crawled (200) <GET {OVERRIDDEN_URL}> (referer: None)",
-            ),
-        )
+        assert (
+            "scrapy.core.engine",
+            logging.DEBUG,
+            f"Crawled (200) <GET {OVERRIDDEN_URL}> (referer: None)",
+        ) in caplog.record_tuples
 
     @inline_callbacks_test
     def test_downloader_middleware_override_in_process_exception(self):
@@ -196,8 +205,10 @@ class TestCrawl:
         assert response.body == b"Caught ZeroDivisionError"
         assert response.request.url == url
 
-    @inline_callbacks_test
-    def test_downloader_middleware_alternative_callback(self):
+    @coroutine_test
+    async def test_downloader_middleware_alternative_callback(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         """
         Downloader middleware which returns a response with a
         specific 'request' attribute, with an alternative callback
@@ -211,14 +222,11 @@ class TestCrawl:
             },
         )
 
-        with LogCapture() as log:
-            url = self.mockserver.url("/status?n=200")
-            yield crawler.crawl(seed=url, mockserver=self.mockserver)
-
-        log.check_present(
-            (
-                "alternative_callbacks_spider",
-                "INFO",
-                "alt_callback was invoked with foo=bar",
-            ),
-        )
+        url = self.mockserver.url("/status?n=200")
+        with caplog.at_level(logging.INFO):
+            await crawler.crawl_async(seed=url, mockserver=self.mockserver)
+        assert (
+            "alternative_callbacks_spider",
+            logging.INFO,
+            "alt_callback was invoked with foo=bar",
+        ) in caplog.record_tuples

tests/test_request_cb_kwargs.py

@@ -1,10 +1,17 @@
-from testfixtures import LogCapture
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING
 
 from scrapy.http import Request
 from scrapy.utils.test import get_crawler
-from tests.mockserver.http import MockServer
 from tests.spiders import MockServerSpider
-from tests.utils.decorators import inline_callbacks_test
+from tests.utils.decorators import coroutine_test
+
+if TYPE_CHECKING:
+    import pytest
+
+    from tests.mockserver.http import MockServer
 
 
 class InjectArgumentsDownloaderMiddleware:
@@ -147,33 +154,32 @@ class KeywordArgumentsSpider(MockServerSpider):
 
 
 class TestCallbackKeywordArguments:
-    @classmethod
-    def setup_class(cls):
-        cls.mockserver = MockServer()
-        cls.mockserver.__enter__()
-
-    @classmethod
-    def teardown_class(cls):
-        cls.mockserver.__exit__(None, None, None)
-
-    @inline_callbacks_test
-    def test_callback_kwargs(self):
+    @coroutine_test
+    async def test_callback_kwargs(
+        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+    ) -> None:
         crawler = get_crawler(KeywordArgumentsSpider)
-        with LogCapture() as log:
-            yield crawler.crawl(mockserver=self.mockserver)
+        with caplog.at_level(logging.ERROR):
+            await crawler.crawl_async(mockserver=mockserver)
+        assert isinstance(crawler.spider, KeywordArgumentsSpider)
         assert all(crawler.spider.checks)
+        assert crawler.stats
         assert len(crawler.spider.checks) == crawler.stats.get_value("boolean_checks")
         # check exceptions for argument mismatch
         exceptions = {}
-        for line in log.records:
+        for line in caplog.records:
             for key in ("takes_less", "takes_more"):
                 if key in line.getMessage():
                     exceptions[key] = line
-        assert exceptions["takes_less"].exc_info[0] is TypeError
-        assert str(exceptions["takes_less"].exc_info[1]).endswith(
+        takes_less_exc_info = exceptions["takes_less"].exc_info
+        assert takes_less_exc_info is not None
+        assert takes_less_exc_info[0] is TypeError
+        assert str(takes_less_exc_info[1]).endswith(
             "parse_takes_less() got an unexpected keyword argument 'number'"
-        ), "Exception message: " + str(exceptions["takes_less"].exc_info[1])
-        assert exceptions["takes_more"].exc_info[0] is TypeError
-        assert str(exceptions["takes_more"].exc_info[1]).endswith(
+        )
+        takes_more_exc_info = exceptions["takes_more"].exc_info
+        assert takes_more_exc_info is not None
+        assert takes_more_exc_info[0] is TypeError
+        assert str(takes_more_exc_info[1]).endswith(
             "parse_takes_more() missing 1 required positional argument: 'other'"
-        ), "Exception message: " + str(exceptions["takes_more"].exc_info[1])
+        )

tests/test_resolver.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 from unittest.mock import Mock
 
 import pytest
+from twisted.internet.address import IPv4Address, IPv6Address
 
 from scrapy.resolver import CachingHostnameResolver, CachingThreadedResolver, dnscache
 from scrapy.utils.defer import maybe_deferred_to_future
@@ -55,11 +56,77 @@ def test_caching_hostname_resolver_no_addresses_not_cached():
     assert "example.com" not in dnscache
 
 
+def test_caching_hostname_resolver_cached_addresses_have_no_port():
+    def fake_resolve(receiver, *_):
+        receiver.resolutionBegan(Mock())
+        receiver.addressResolved(IPv4Address("TCP", "1.2.3.4", 80))
+        receiver.addressResolved(IPv6Address("TCP", "::1", 80))
+        receiver.resolutionComplete()
+        return receiver
+
+    reactor = Mock()
+    reactor.nameResolver.resolveHostName.side_effect = fake_resolve
+
+    receiver = Mock()
+    resolver = CachingHostnameResolver(reactor, cache_size=10)
+    resolver.resolveHostName(receiver, "example.com", portNumber=80)
+
+    # The port requested on a cache miss is passed through unchanged, but it is
+    # not part of what gets cached.
+    resolved_ports = [
+        call.args[0].port for call in receiver.addressResolved.call_args_list
+    ]
+    assert resolved_ports == [80, 80]
+    assert [address.port for address in dnscache["example.com"]] == [0, 0]
+
+
+def test_caching_hostname_resolver_cache_hit_without_port():
+    cached_addresses = [
+        IPv4Address("TCP", "1.2.3.4", 0),
+        IPv6Address("TCP", "::1", 0),
+    ]
+    dnscache["example.com"] = cached_addresses
+
+    receiver = Mock()
+    resolver = CachingHostnameResolver(Mock(), cache_size=10)
+    resolver.resolveHostName(receiver, "example.com")
+
+    # Cached addresses already use the requested port, so they are reused as is.
+    resolved_addresses = [
+        call.args[0] for call in receiver.addressResolved.call_args_list
+    ]
+    assert all(
+        resolved is cached
+        for resolved, cached in zip(resolved_addresses, cached_addresses, strict=True)
+    )
+
+
+def test_caching_hostname_resolver_cache_hit_uses_requested_port():
+    dnscache["example.com"] = [
+        IPv4Address("TCP", "1.2.3.4", 0),
+        IPv6Address("TCP", "::1", 0),
+    ]
+
+    receiver = Mock()
+    resolver = CachingHostnameResolver(Mock(), cache_size=10)
+    resolver.resolveHostName(receiver, "example.com", portNumber=443)
+
+    resolved_addresses = [
+        call.args[0] for call in receiver.addressResolved.call_args_list
+    ]
+    assert resolved_addresses == [
+        IPv4Address("TCP", "1.2.3.4", 443),
+        IPv6Address("TCP", "::1", 443),
+    ]
+    # The cached addresses must not be mutated in place.
+    assert [address.port for address in dnscache["example.com"]] == [0, 0]
+
+
 def test_caching_hostname_resolver_dnscache_disabled_rejects_storage():
 
     def fake_resolve(receiver, *_):
         receiver.resolutionBegan(Mock())
-        receiver.addressResolved(Mock())
+        receiver.addressResolved(IPv4Address("TCP", "1.2.3.4", 80))
         receiver.resolutionComplete()
         return receiver
 

tests/test_scheduler.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import warnings
 from abc import ABC, abstractmethod
 from contextlib import AbstractAsyncContextManager, asynccontextmanager
-from typing import TYPE_CHECKING, Any, NamedTuple, cast
+from typing import TYPE_CHECKING
 from unittest.mock import Mock
 
 import pytest
@@ -15,43 +15,17 @@ from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Request
 from scrapy.spiders import Spider
 from scrapy.utils.defer import ensure_awaitable
-from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.misc import load_object
 from scrapy.utils.test import get_crawler
 from tests.mockserver.http import MockServer
 from tests.utils.decorators import coroutine_test, inline_callbacks_test
+from tests.utils.downloader import MockDownloader
 
 if TYPE_CHECKING:
     from collections.abc import AsyncGenerator
     from pathlib import Path
 
 
-class MockSlot(NamedTuple):
-    active: list[Any]
-
-
-class MockDownloader:
-    def __init__(self) -> None:
-        self.slots: dict[str, MockSlot] = {}
-
-    def get_slot_key(self, request: Request) -> str:
-        if Downloader.DOWNLOAD_SLOT in request.meta:
-            return cast("str", request.meta[Downloader.DOWNLOAD_SLOT])
-
-        return urlparse_cached(request).hostname or ""
-
-    def increment(self, slot_key: str) -> None:
-        slot = self.slots.setdefault(slot_key, MockSlot(active=[]))
-        slot.active.append(1)
-
-    def decrement(self, slot_key: str) -> None:
-        slot = self.slots[slot_key]
-        slot.active.pop()
-
-    def close(self) -> None:
-        pass
-
-
 class MockCrawler(Crawler):
     def __init__(self, priority_queue_cls: str, jobdir: Path | None):
         settings = {

tests/test_scheduler_base.py

@@ -1,9 +1,10 @@
 from __future__ import annotations
 
+import logging
+from typing import TYPE_CHECKING
 from urllib.parse import urljoin
 
 import pytest
-from testfixtures import LogCapture
 from twisted.internet import defer
 
 from scrapy.core.scheduler import BaseScheduler
@@ -12,8 +13,10 @@ from scrapy.spiders import Spider
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.request import fingerprint
 from scrapy.utils.test import get_crawler
-from tests.mockserver.http import MockServer
-from tests.utils.decorators import inline_callbacks_test
+from tests.utils.decorators import coroutine_test, inline_callbacks_test
+
+if TYPE_CHECKING:
+    from tests.mockserver.http import MockServer
 
 PATHS = ["/a", "/b", "/c"]
 URLS = [urljoin("https://example.org", p) for p in PATHS]
@@ -147,18 +150,19 @@ class TestSimpleScheduler(InterfaceCheckMixin):
 class TestMinimalSchedulerCrawl:
     scheduler_cls = MinimalScheduler
 
-    @inline_callbacks_test
-    def test_crawl(self):
-        with MockServer() as mockserver:
-            settings = {
-                "SCHEDULER": self.scheduler_cls,
-            }
-            with LogCapture() as log:
-                crawler = get_crawler(PathsSpider, settings)
-                yield crawler.crawl(mockserver)
-            for path in PATHS:
-                assert f"{{'path': '{path}'}}" in str(log)
-            assert f"'item_scraped_count': {len(PATHS)}" in str(log)
+    @coroutine_test
+    async def test_crawl(
+        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+    ) -> None:
+        settings = {
+            "SCHEDULER": self.scheduler_cls,
+        }
+        with caplog.at_level(logging.DEBUG):
+            crawler = get_crawler(PathsSpider, settings)
+            await crawler.crawl_async(mockserver)
+        for path in PATHS:
+            assert f"{{'path': '{path}'}}" in caplog.text
+        assert f"'item_scraped_count': {len(PATHS)}" in caplog.text
 
 
 class TestSimpleSchedulerCrawl(TestMinimalSchedulerCrawl):

tests/test_spider.py

@@ -1,133 +1,18 @@
 from __future__ import annotations
 
-from typing import Any
-from unittest import mock
-
 import pytest
-from testfixtures import LogCapture
 
-from scrapy import signals
-from scrapy.crawler import Crawler
-from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Response, TextResponse, XmlResponse
-from scrapy.settings import Settings
 from scrapy.spiders import CSVFeedSpider, Spider, XMLFeedSpider
-from scrapy.utils.test import get_crawler, get_reactor_settings
 from tests import get_testdata
-from tests.utils.decorators import inline_callbacks_test
+from tests.utils.bases.spider import TestSpiderBase
 
 
-class TestSpider:
+class TestSpider(TestSpiderBase):
     spider_class = Spider
 
-    def test_base_spider(self):
-        spider = self.spider_class("example.com")
-        assert spider.name == "example.com"
-        assert spider.start_urls == []
 
-    def test_spider_args(self):
-        """``__init__`` method arguments are assigned to spider attributes"""
-        spider = self.spider_class("example.com", foo="bar")
-        assert spider.foo == "bar"
-
-    def test_spider_without_name(self):
-        """``__init__`` raises when the name is not provided."""
-        msg = "must have a name"
-        with pytest.raises(ValueError, match=msg):
-            self.spider_class()
-        with pytest.raises(ValueError, match=msg):
-            self.spider_class(somearg="foo")
-
-    def test_from_crawler_crawler_and_settings_population(self):
-        crawler = get_crawler()
-        spider = self.spider_class.from_crawler(crawler, "example.com")
-        assert hasattr(spider, "crawler")
-        assert spider.crawler is crawler
-        assert hasattr(spider, "settings")
-        assert spider.settings is crawler.settings
-
-    def test_from_crawler_init_call(self):
-        with mock.patch.object(
-            self.spider_class, "__init__", return_value=None
-        ) as mock_init:
-            self.spider_class.from_crawler(get_crawler(), "example.com", foo="bar")
-            mock_init.assert_called_once_with("example.com", foo="bar")
-
-    def test_closed_signal_call(self):
-        class TestSpider(self.spider_class):
-            closed_called = False
-
-            def closed(self, reason):
-                self.closed_called = True
-
-        crawler = get_crawler()
-        spider = TestSpider.from_crawler(crawler, "example.com")
-        crawler.signals.send_catch_log(signal=signals.spider_opened, spider=spider)
-        crawler.signals.send_catch_log(
-            signal=signals.spider_closed, spider=spider, reason=None
-        )
-        assert spider.closed_called
-
-    def test_update_settings(self):
-        spider_settings = {"TEST1": "spider", "TEST2": "spider"}
-        project_settings = {"TEST1": "project", "TEST3": "project"}
-        self.spider_class.custom_settings = spider_settings
-        settings = Settings(project_settings, priority="project")
-
-        self.spider_class.update_settings(settings)
-        assert settings.get("TEST1") == "spider"
-        assert settings.get("TEST2") == "spider"
-        assert settings.get("TEST3") == "project"
-
-    @inline_callbacks_test
-    def test_settings_in_from_crawler(self):
-        spider_settings = {"TEST1": "spider", "TEST2": "spider"}
-        project_settings = {
-            "TEST1": "project",
-            "TEST3": "project",
-            **get_reactor_settings(),
-        }
-
-        class TestSpider(self.spider_class):
-            name = "test"
-            custom_settings = spider_settings
-
-            @classmethod
-            def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any):
-                spider = super().from_crawler(crawler, *args, **kwargs)
-                spider.settings.set("TEST1", "spider_instance", priority="spider")
-                return spider
-
-        crawler = Crawler(TestSpider, project_settings)
-        assert crawler.settings.get("TEST1") == "spider"
-        assert crawler.settings.get("TEST2") == "spider"
-        assert crawler.settings.get("TEST3") == "project"
-        yield crawler.crawl()
-        assert crawler.settings.get("TEST1") == "spider_instance"
-
-    def test_logger(self):
-        spider = self.spider_class("example.com")
-        with LogCapture() as lc:
-            spider.logger.info("test log msg")
-        lc.check(("example.com", "INFO", "test log msg"))
-
-        record = lc.records[0]
-        assert "spider" in record.__dict__
-        assert record.spider is spider
-
-    def test_log(self):
-        spider = self.spider_class("example.com")
-        with (
-            mock.patch("scrapy.spiders.Spider.logger") as mock_logger,
-            pytest.warns(
-                ScrapyDeprecationWarning, match=r"Spider.log\(\) is deprecated"
-            ),
-        ):
-            spider.log("test log msg", "INFO")
-        mock_logger.log.assert_called_once_with("INFO", "test log msg")
-
-
-class TestXMLFeedSpider(TestSpider):
+class TestXMLFeedSpider(TestSpiderBase):
     spider_class = XMLFeedSpider
 
     def test_register_namespace(self):
@@ -176,7 +61,7 @@ class TestXMLFeedSpider(TestSpider):
             ], iterator
 
 
-class TestCSVFeedSpider(TestSpider):
+class TestCSVFeedSpider(TestSpiderBase):
     spider_class = CSVFeedSpider
 
     def test_parse_rows(self):

tests/test_spider_crawl.py

@@ -11,10 +11,10 @@ from scrapy.http import HtmlResponse, Request, TextResponse
 from scrapy.linkextractors import LinkExtractor
 from scrapy.spiders import CrawlSpider, Rule, Spider
 from scrapy.utils.test import get_crawler
-from tests.test_spider import TestSpider
+from tests.utils.bases.spider import TestSpiderBase
 
 
-class TestCrawlSpider(TestSpider):
+class TestCrawlSpider(TestSpiderBase):
     test_body = b"""<html><head><title>Page title</title></head>
     <body>
     <p><a href="item/12.html">Item 12</a></p>

tests/test_spider_sitemap.py

@@ -8,17 +8,16 @@ from logging import WARNING
 from pathlib import Path
 
 import pytest
-from testfixtures import LogCapture
 
 from scrapy.http import HtmlResponse, Request, Response, TextResponse, XmlResponse
 from scrapy.spiders import SitemapSpider
 from scrapy.utils.test import get_crawler
 from tests import tests_datadir
-from tests.test_spider import TestSpider
+from tests.utils.bases.spider import TestSpiderBase
 from tests.utils.decorators import coroutine_test
 
 
-class TestSitemapSpider(TestSpider):
+class TestSitemapSpider(TestSpiderBase):
     spider_class = SitemapSpider
 
     BODY = b"SITEMAP"
@@ -254,22 +253,23 @@ Sitemap: /sitemap-relative-url.xml
         urls = [req.url for req in spider._parse_sitemap(r)]
         assert urls == result
 
-    def test_parse_sitemap_empty_body(self):
+    def test_parse_sitemap_empty_body(self, caplog: pytest.LogCaptureFixture) -> None:
         r = XmlResponse(url="http://www.example.com/sitemap.xml", body=b"")
         spider = self.spider_class("example.com")
 
-        with LogCapture() as lc:
+        caplog.clear()
+        with caplog.at_level(WARNING):
             results = list(spider._parse_sitemap(r))
 
         assert not results
 
-        lc.check(
+        assert caplog.record_tuples == [
             (
                 "scrapy.spiders.sitemap",
-                "WARNING",
+                WARNING,
                 "Ignoring invalid sitemap: <200 http://www.example.com/sitemap.xml>",
             )
-        )
+        ]
 
     def test_parse_sitemap_not_sitemap(self):
         body = b"""<?xml version="1.0" encoding="UTF-8"?>
@@ -342,7 +342,7 @@ Sitemap: /sitemap-relative-url.xml
         response = Response(url="https://example.com", body=body, request=request)
         assert spider._get_sitemap_body(response) is None
 
-    def test_download_warnsize_setting(self):
+    def test_download_warnsize_setting(self, caplog: pytest.LogCaptureFixture) -> None:
         settings = {"DOWNLOAD_WARNSIZE": 10_000_000}
         crawler = get_crawler(settings_dict=settings)
         spider = self.spider_class.from_crawler(crawler, "example.com")
@@ -350,25 +350,26 @@ Sitemap: /sitemap-relative-url.xml
         body = body_path.read_bytes()
         request = Request(url="https://example.com")
         response = Response(url="https://example.com", body=body, request=request)
-        with LogCapture(
-            "scrapy.spiders.sitemap", propagate=False, level=WARNING
-        ) as log:
+        caplog.clear()
+        with caplog.at_level(WARNING, logger="scrapy.spiders.sitemap"):
             spider._get_sitemap_body(response)
-        log.check(
+        assert caplog.record_tuples == [
             (
                 "scrapy.spiders.sitemap",
-                "WARNING",
+                WARNING,
                 (
                     "<200 https://example.com> body size after decompression "
                     "(11511612 B) is larger than the download warning size "
                     "(10000000 B)."
                 ),
             ),
-        )
+        ]
 
     @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
-    def test_download_warnsize_spider_attr(self):
-        class DownloadWarnSizeSpider(self.spider_class):
+    def test_download_warnsize_spider_attr(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
+        class DownloadWarnSizeSpider(self.spider_class):  # type: ignore[name-defined,misc]
             download_warnsize = 10_000_000
 
         crawler = get_crawler()
@@ -379,23 +380,24 @@ Sitemap: /sitemap-relative-url.xml
             url="https://example.com", meta={"download_warnsize": 10_000_000}
         )
         response = Response(url="https://example.com", body=body, request=request)
-        with LogCapture(
-            "scrapy.spiders.sitemap", propagate=False, level=WARNING
-        ) as log:
+        caplog.clear()
+        with caplog.at_level(WARNING, logger="scrapy.spiders.sitemap"):
             spider._get_sitemap_body(response)
-        log.check(
+        assert caplog.record_tuples == [
             (
                 "scrapy.spiders.sitemap",
-                "WARNING",
+                WARNING,
                 (
                     "<200 https://example.com> body size after decompression "
                     "(11511612 B) is larger than the download warning size "
                     "(10000000 B)."
                 ),
             ),
-        )
+        ]
 
-    def test_download_warnsize_request_meta(self):
+    def test_download_warnsize_request_meta(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         crawler = get_crawler()
         spider = self.spider_class.from_crawler(crawler, "example.com")
         body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin")
@@ -404,21 +406,20 @@ Sitemap: /sitemap-relative-url.xml
             url="https://example.com", meta={"download_warnsize": 10_000_000}
         )
         response = Response(url="https://example.com", body=body, request=request)
-        with LogCapture(
-            "scrapy.spiders.sitemap", propagate=False, level=WARNING
-        ) as log:
+        caplog.clear()
+        with caplog.at_level(WARNING, logger="scrapy.spiders.sitemap"):
             spider._get_sitemap_body(response)
-        log.check(
+        assert caplog.record_tuples == [
             (
                 "scrapy.spiders.sitemap",
-                "WARNING",
+                WARNING,
                 (
                     "<200 https://example.com> body size after decompression "
                     "(11511612 B) is larger than the download warning size "
                     "(10000000 B)."
                 ),
             ),
-        )
+        ]
 
     @coroutine_test
     async def test_sitemap_urls(self):

tests/test_spidermiddleware_output_chain.py

@@ -1,10 +1,16 @@
-from testfixtures import LogCapture
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING
 
 from scrapy import Request, Spider
 from scrapy.utils.test import get_crawler
 from tests.mockserver.http import MockServer
 from tests.utils.decorators import coroutine_test
 
+if TYPE_CHECKING:
+    import pytest
+
 
 class _BaseSpiderMiddleware:
     def __init__(self, crawler):
@@ -250,113 +256,132 @@ class TestSpiderMiddleware:
     def teardown_class(cls):
         cls.mockserver.__exit__(None, None, None)
 
-    async def crawl_log(self, spider: type[Spider]) -> LogCapture:
+    async def crawl_log(
+        self, spider: type[Spider], caplog: pytest.LogCaptureFixture
+    ) -> str:
         crawler = get_crawler(spider)
-        with LogCapture() as log:
+        caplog.clear()
+        with caplog.at_level(logging.DEBUG):
             await crawler.crawl_async(mockserver=self.mockserver)
-        return log
+        return caplog.text
 
     @coroutine_test
-    async def test_recovery(self):
+    async def test_recovery(self, caplog: pytest.LogCaptureFixture) -> None:
         """
         (0) Recover from an exception in a spider callback. The final item count should be 3
         (one yielded from the callback method before the exception is raised, one directly
         from the recovery middleware and one from the spider when processing the request that
         was enqueued from the recovery middleware)
         """
-        log = await self.crawl_log(RecoverySpider)
-        assert "Middleware: TabError exception caught" in str(log)
-        assert str(log).count("Middleware: TabError exception caught") == 1
-        assert "'item_scraped_count': 3" in str(log)
+        log = await self.crawl_log(RecoverySpider, caplog)
+        assert "Middleware: TabError exception caught" in log
+        assert log.count("Middleware: TabError exception caught") == 1
+        assert "'item_scraped_count': 3" in log
 
     @coroutine_test
-    async def test_recovery_asyncgen(self):
+    async def test_recovery_asyncgen(self, caplog: pytest.LogCaptureFixture) -> None:
         """
         Same as test_recovery but with an async callback.
         """
-        log = await self.crawl_log(RecoveryAsyncGenSpider)
-        assert "Middleware: TabError exception caught" in str(log)
-        assert str(log).count("Middleware: TabError exception caught") == 1
-        assert "'item_scraped_count': 3" in str(log)
+        log = await self.crawl_log(RecoveryAsyncGenSpider, caplog)
+        assert "Middleware: TabError exception caught" in log
+        assert log.count("Middleware: TabError exception caught") == 1
+        assert "'item_scraped_count': 3" in log
 
     @coroutine_test
-    async def test_process_spider_input_without_errback(self):
+    async def test_process_spider_input_without_errback(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         """
         (1.1) An exception from the process_spider_input chain should be caught by the
         process_spider_exception chain from the start if the Request has no errback
         """
-        log1 = await self.crawl_log(ProcessSpiderInputSpiderWithoutErrback)
-        assert "Middleware: will raise IndexError" in str(log1)
-        assert "Middleware: IndexError exception caught" in str(log1)
+        log1 = await self.crawl_log(ProcessSpiderInputSpiderWithoutErrback, caplog)
+        assert "Middleware: will raise IndexError" in log1
+        assert "Middleware: IndexError exception caught" in log1
 
     @coroutine_test
-    async def test_process_spider_input_with_errback(self):
+    async def test_process_spider_input_with_errback(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         """
         (1.2) An exception from the process_spider_input chain should not be caught by the
         process_spider_exception chain if the Request has an errback
         """
-        log1 = await self.crawl_log(ProcessSpiderInputSpiderWithErrback)
-        assert "Middleware: IndexError exception caught" not in str(log1)
-        assert "Middleware: will raise IndexError" in str(log1)
-        assert "Got a Failure on the Request errback" in str(log1)
-        assert "{'from': 'errback'}" in str(log1)
-        assert "{'from': 'callback'}" not in str(log1)
-        assert "'item_scraped_count': 1" in str(log1)
+        log1 = await self.crawl_log(ProcessSpiderInputSpiderWithErrback, caplog)
+        assert "Middleware: IndexError exception caught" not in log1
+        assert "Middleware: will raise IndexError" in log1
+        assert "Got a Failure on the Request errback" in log1
+        assert "{'from': 'errback'}" in log1
+        assert "{'from': 'callback'}" not in log1
+        assert "'item_scraped_count': 1" in log1
 
     @coroutine_test
-    async def test_generator_callback(self):
+    async def test_generator_callback(self, caplog: pytest.LogCaptureFixture) -> None:
         """
         (2) An exception from a spider callback (returning a generator) should
         be caught by the process_spider_exception chain. Items yielded before the
         exception is raised should be processed normally.
         """
-        log2 = await self.crawl_log(GeneratorCallbackSpider)
-        assert "Middleware: ImportError exception caught" in str(log2)
-        assert "'item_scraped_count': 2" in str(log2)
+        log2 = await self.crawl_log(GeneratorCallbackSpider, caplog)
+        assert "Middleware: ImportError exception caught" in log2
+        assert "'item_scraped_count': 2" in log2
 
     @coroutine_test
-    async def test_async_generator_callback(self):
+    async def test_async_generator_callback(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         """
         Same as test_generator_callback but with an async callback.
         """
-        log2 = await self.crawl_log(AsyncGeneratorCallbackSpider)
-        assert "Middleware: ImportError exception caught" in str(log2)
-        assert "'item_scraped_count': 2" in str(log2)
+        log2 = await self.crawl_log(AsyncGeneratorCallbackSpider, caplog)
+        assert "Middleware: ImportError exception caught" in log2
+        assert "'item_scraped_count': 2" in log2
 
     @coroutine_test
-    async def test_generator_callback_right_after_callback(self):
+    async def test_generator_callback_right_after_callback(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         """
         (2.1) Special case of (2): Exceptions should be caught
         even if the middleware is placed right after the spider
         """
-        log21 = await self.crawl_log(GeneratorCallbackSpiderMiddlewareRightAfterSpider)
-        assert "Middleware: ImportError exception caught" in str(log21)
-        assert "'item_scraped_count': 2" in str(log21)
+        log21 = await self.crawl_log(
+            GeneratorCallbackSpiderMiddlewareRightAfterSpider, caplog
+        )
+        assert "Middleware: ImportError exception caught" in log21
+        assert "'item_scraped_count': 2" in log21
 
     @coroutine_test
-    async def test_not_a_generator_callback(self):
+    async def test_not_a_generator_callback(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         """
         (3) An exception from a spider callback (returning a list) should
         be caught by the process_spider_exception chain. No items should be processed.
         """
-        log3 = await self.crawl_log(NotGeneratorCallbackSpider)
-        assert "Middleware: ZeroDivisionError exception caught" in str(log3)
-        assert "item_scraped_count" not in str(log3)
+        log3 = await self.crawl_log(NotGeneratorCallbackSpider, caplog)
+        assert "Middleware: ZeroDivisionError exception caught" in log3
+        assert "item_scraped_count" not in log3
 
     @coroutine_test
-    async def test_not_a_generator_callback_right_after_callback(self):
+    async def test_not_a_generator_callback_right_after_callback(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         """
         (3.1) Special case of (3): Exceptions should be caught
         even if the middleware is placed right after the spider
         """
         log31 = await self.crawl_log(
-            NotGeneratorCallbackSpiderMiddlewareRightAfterSpider
+            NotGeneratorCallbackSpiderMiddlewareRightAfterSpider, caplog
         )
-        assert "Middleware: ZeroDivisionError exception caught" in str(log31)
-        assert "item_scraped_count" not in str(log31)
+        assert "Middleware: ZeroDivisionError exception caught" in log31
+        assert "item_scraped_count" not in log31
 
     @coroutine_test
-    async def test_generator_output_chain(self):
+    async def test_generator_output_chain(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         """
         (4) An exception from a middleware's process_spider_output method should be sent
         to the process_spider_exception method from the next middleware in the chain.
@@ -365,23 +390,23 @@ class TestSpiderMiddleware:
         The final item count should be 2 (one from the spider callback and one from the
         process_spider_exception chain)
         """
-        log4 = await self.crawl_log(GeneratorOutputChainSpider)
-        assert "'item_scraped_count': 2" in str(log4)
+        log4 = await self.crawl_log(GeneratorOutputChainSpider, caplog)
+        assert "'item_scraped_count': 2" in log4
         assert (
             "GeneratorRecoverMiddleware.process_spider_exception: LookupError caught"
-            in str(log4)
+            in log4
         )
         assert (
             "GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: LookupError caught"
-            in str(log4)
+            in log4
         )
         assert (
             "GeneratorFailMiddleware.process_spider_exception: LookupError caught"
-            not in str(log4)
+            not in log4
         )
         assert (
             "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: LookupError caught"
-            not in str(log4)
+            not in log4
         )
         item_from_callback = {
             "processed": [
@@ -398,6 +423,6 @@ class TestSpiderMiddleware:
                 "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_output",
             ]
         }
-        assert str(item_from_callback) in str(log4)
-        assert str(item_recovered) in str(log4)
-        assert "parse-second-item" not in str(log4)
+        assert str(item_from_callback) in log4
+        assert str(item_recovered) in log4
+        assert "parse-second-item" not in log4

tests/test_spidermiddleware_process_start.py

@@ -5,7 +5,6 @@ import pytest
 from scrapy import Spider, signals
 from scrapy.utils.defer import maybe_deferred_to_future
 from scrapy.utils.test import get_crawler
-from tests.test_spider_start import SLEEP_SECONDS
 
 from .utils import twisted_sleep
 from .utils.decorators import coroutine_test
@@ -14,6 +13,8 @@ ITEM_A = {"id": "a"}
 ITEM_B = {"id": "b"}
 ITEM_C = {"id": "c"}
 
+SLEEP_SECONDS = 0.1
+
 
 class AsyncioSleepSpiderMiddleware:
     async def process_start(self, start):

tests/test_utils_log.py

@@ -8,7 +8,6 @@ from io import StringIO
 from typing import TYPE_CHECKING, Any
 
 import pytest
-from testfixtures import LogCapture
 from twisted.python.failure import Failure
 
 from scrapy.utils.log import (
@@ -107,15 +106,15 @@ class TestLogCounterHandler:
 
 
 class TestStreamLogger:
-    def test_redirect(self):
+    def test_redirect(self, caplog: pytest.LogCaptureFixture) -> None:
         logger = logging.getLogger("test")
         logger.setLevel(logging.WARNING)
         old_stdout = sys.stdout
         sys.stdout = StreamLogger(logger, logging.ERROR)
 
-        with LogCapture() as log:
-            print("test log msg")
-        log.check(("test", "ERROR", "test log msg"))
+        caplog.clear()
+        print("test log msg")
+        assert caplog.record_tuples == [("test", logging.ERROR, "test log msg")]
 
         sys.stdout = old_stdout
 

tests/test_utils_signal.py

@@ -1,23 +1,25 @@
 from __future__ import annotations
 
 import asyncio
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
 
 import pytest
 from pydispatch import dispatcher
-from testfixtures import LogCapture
 from twisted.internet import defer
 from twisted.python.failure import Failure
 
 from scrapy.utils.asyncio import call_later
-from scrapy.utils.defer import deferred_from_coro
+from scrapy.utils.defer import deferred_from_coro, ensure_awaitable
 from scrapy.utils.signal import (
     send_catch_log,
     send_catch_log_async,
     send_catch_log_deferred,
 )
 from scrapy.utils.test import get_from_asyncio_queue
-from tests.utils.decorators import inline_callbacks_test
+from tests.utils.decorators import coroutine_test
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
 
 if TYPE_CHECKING:
     from collections.abc import Callable
@@ -27,25 +29,22 @@ class TestSendCatchLog:
     # whether the function being tested returns exceptions or failures
     returns_exceptions: bool = False
 
-    @inline_callbacks_test
-    def test_send_catch_log(self):
+    @coroutine_test
+    async def test_send_catch_log(self, caplog: pytest.LogCaptureFixture) -> None:
         test_signal = object()
         handlers_called: set[Callable[..., None]] = set()
 
         dispatcher.connect(self.error_handler, signal=test_signal)
         dispatcher.connect(self.ok_handler, signal=test_signal)
-        with LogCapture() as log:
-            result = yield defer.maybeDeferred(
-                self._get_result,
-                test_signal,
-                arg="test",
-                handlers_called=handlers_called,
-            )
+        caplog.clear()
+        result = await ensure_awaitable(
+            self._get_result(test_signal, arg="test", handlers_called=handlers_called)
+        )
 
         assert self.error_handler in handlers_called
         assert self.ok_handler in handlers_called
-        assert len(log.records) == 1
-        record = log.records[0]
+        assert len(caplog.records) == 1
+        record = caplog.records[0]
         assert "error_handler" in record.getMessage()
         assert record.levelname == "ERROR"
         assert result[0][0] == self.error_handler  # pylint: disable=comparison-with-callable
@@ -57,7 +56,7 @@ class TestSendCatchLog:
         dispatcher.disconnect(self.error_handler, signal=test_signal)
         dispatcher.disconnect(self.ok_handler, signal=test_signal)
 
-    def _get_result(self, signal, *a, **kw):
+    def _get_result(self, signal: Any, *a: Any, **kw: Any) -> Any:
         return send_catch_log(signal, *a, **kw)
 
     def error_handler(self, arg, handlers_called):
@@ -72,7 +71,7 @@ class TestSendCatchLog:
 
 @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
 class TestSendCatchLogDeferred(TestSendCatchLog):
-    def _get_result(self, signal, *a, **kw):
+    def _get_result(self, signal: Any, *a: Any, **kw: Any) -> Any:
         return send_catch_log_deferred(signal, *a, **kw)
 
 
@@ -137,14 +136,15 @@ class TestSendCatchLogAsyncAsyncio(TestSendCatchLogAsync):
 
 
 class TestSendCatchLog2:
-    def test_error_logged_if_deferred_not_supported(self):
+    def test_error_logged_if_deferred_not_supported(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         def test_handler():
             return defer.Deferred()
 
         test_signal = object()
         dispatcher.connect(test_handler, test_signal)
-        with LogCapture() as log:
-            send_catch_log(test_signal)
-        assert len(log.records) == 1
-        assert "Cannot return deferreds from signal handler" in str(log)
+        send_catch_log(test_signal)
+        assert len(caplog.records) == 1
+        assert "Cannot return deferreds from signal handler" in caplog.text
         dispatcher.disconnect(test_handler, test_signal)

tests/utils/bases/download_handlers_http.py

@@ -1,5 +1,3 @@
-"""Base classes for HTTP download handler tests."""
-
 from __future__ import annotations
 
 import gzip

tests/utils/bases/engine.py

@@ -0,0 +1,160 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from itemadapter import ItemAdapter
+
+from scrapy import signals
+from tests import get_testdata
+
+if TYPE_CHECKING:
+    from tests.utils.engine import CrawlerRun
+
+
+class TestEngineBase:
+    @staticmethod
+    def _assert_visited_urls(run: CrawlerRun) -> None:
+        must_be_visited = [
+            "/static/",
+            "/redirect",
+            "/redirected",
+            "/static/item1.html",
+            "/static/item2.html",
+            "/static/item999.html",
+        ]
+        urls_visited = {rp[0].url for rp in run.respplug}
+        urls_expected = {run.geturl(p) for p in must_be_visited}
+        assert urls_expected <= urls_visited, (
+            f"URLs not visited: {list(urls_expected - urls_visited)}"
+        )
+
+    @staticmethod
+    def _assert_scheduled_requests(run: CrawlerRun, count: int) -> None:
+        assert len(run.reqplug) == count
+
+        paths_expected = [
+            "/static/item999.html",
+            "/static/item2.html",
+            "/static/item1.html",
+        ]
+
+        urls_requested = {rq[0].url for rq in run.reqplug}
+        urls_expected = {run.geturl(p) for p in paths_expected}
+        assert urls_expected <= urls_requested
+        scheduled_requests_count = len(run.reqplug)
+        dropped_requests_count = len(run.reqdropped)
+        responses_count = len(run.respplug)
+        assert scheduled_requests_count == dropped_requests_count + responses_count
+        assert len(run.reqreached) == responses_count
+
+    @staticmethod
+    def _assert_dropped_requests(run: CrawlerRun) -> None:
+        assert len(run.reqdropped) == 1
+
+    @staticmethod
+    def _assert_downloaded_responses(run: CrawlerRun, count: int) -> None:
+        # response tests
+        assert len(run.respplug) == count
+        assert len(run.reqreached) == count
+
+        for response, _ in run.respplug:
+            if run.getpath(response.url) == "/static/item999.html":
+                assert response.status == 404
+            if run.getpath(response.url) == "/redirect":
+                assert response.status == 302
+
+    @staticmethod
+    def _assert_items_error(run: CrawlerRun) -> None:
+        assert len(run.itemerror) == 2
+        for item, response, spider, failure in run.itemerror:
+            assert failure.value.__class__ is ZeroDivisionError
+            assert spider == run.crawler.spider
+
+            assert item["url"] == response.url
+            if "item1.html" in item["url"]:
+                assert item["name"] == "Item 1 name"
+                assert item["price"] == "100"
+            if "item2.html" in item["url"]:
+                assert item["name"] == "Item 2 name"
+                assert item["price"] == "200"
+
+    @staticmethod
+    def _assert_scraped_items(run: CrawlerRun) -> None:
+        assert len(run.itemresp) == 2
+        for item_, response in run.itemresp:
+            item = ItemAdapter(item_)
+            assert item["url"] == response.url
+            if "item1.html" in item["url"]:
+                assert item["name"] == "Item 1 name"
+                assert item["price"] == "100"
+            if "item2.html" in item["url"]:
+                assert item["name"] == "Item 2 name"
+                assert item["price"] == "200"
+
+    @staticmethod
+    def _assert_headers_received(run: CrawlerRun) -> None:
+        for headers in run.headers.values():
+            assert b"Server" in headers
+            assert headers[b"Server"]
+            assert b"TwistedWeb" in headers[b"Server"]
+            assert b"Date" in headers
+            assert b"Content-Type" in headers
+
+    @staticmethod
+    def _assert_bytes_received(run: CrawlerRun) -> None:
+        assert len(run.bytes) == 9
+        for request, data in run.bytes.items():
+            joined_data = b"".join(data)
+            if run.getpath(request.url) == "/static/":
+                assert joined_data == get_testdata("test_site", "index.html")
+            elif run.getpath(request.url) == "/static/item1.html":
+                assert joined_data == get_testdata("test_site", "item1.html")
+            elif run.getpath(request.url) == "/static/item2.html":
+                assert joined_data == get_testdata("test_site", "item2.html")
+            elif run.getpath(request.url) == "/redirected":
+                assert joined_data == b"Redirected here"
+            elif run.getpath(request.url) == "/redirect":
+                assert (
+                    joined_data == b"\n<html>\n"
+                    b"    <head>\n"
+                    b'        <meta http-equiv="refresh" content="0;URL=/redirected">\n'
+                    b"    </head>\n"
+                    b'    <body bgcolor="#FFFFFF" text="#000000">\n'
+                    b'    <a href="/redirected">click here</a>\n'
+                    b"    </body>\n"
+                    b"</html>\n"
+                )
+            elif run.getpath(request.url) == "/static/item999.html":
+                assert (
+                    joined_data == b"\n<html>\n"
+                    b"  <head><title>404 - No Such Resource</title></head>\n"
+                    b"  <body>\n"
+                    b"    <h1>No Such Resource</h1>\n"
+                    b"    <p>File not found.</p>\n"
+                    b"  </body>\n"
+                    b"</html>\n"
+                )
+            elif run.getpath(request.url) == "/numbers":
+                # signal was fired multiple times
+                assert len(data) > 1
+                # bytes were received in order
+                numbers = [str(x).encode("utf8") for x in range(2**18)]
+                assert joined_data == b"".join(numbers)
+
+    @staticmethod
+    def _assert_signals_caught(run: CrawlerRun) -> None:
+        assert signals.engine_started in run.signals_caught
+        assert signals.engine_stopped in run.signals_caught
+        assert signals.spider_opened in run.signals_caught
+        assert signals.spider_idle in run.signals_caught
+        assert signals.spider_closed in run.signals_caught
+        assert signals.headers_received in run.signals_caught
+
+        assert {"spider": run.crawler.spider} == run.signals_caught[
+            signals.spider_opened
+        ]
+        assert {"spider": run.crawler.spider} == run.signals_caught[signals.spider_idle]
+        assert {
+            "spider": run.crawler.spider,
+            "reason": "finished",
+        } == run.signals_caught[signals.spider_closed]

tests/utils/bases/feedexport.py

@@ -0,0 +1,151 @@
+from __future__ import annotations
+
+import random
+import shutil
+import tempfile
+from abc import ABC, abstractmethod
+from pathlib import Path
+from string import ascii_letters, digits
+from typing import IO, TYPE_CHECKING, Any
+
+import scrapy
+from scrapy import Spider
+from tests.mockserver.http import MockServer
+
+if TYPE_CHECKING:
+    from collections.abc import Callable, Iterable
+
+
+class TestFeedExportBase(ABC):
+    mockserver: MockServer
+
+    def _random_temp_filename(self, inter_dir="") -> Path:
+        chars = [random.choice(ascii_letters + digits) for _ in range(15)]
+        filename = "".join(chars)
+        return Path(self.temp_dir, inter_dir, filename)
+
+    @classmethod
+    def setup_class(cls):
+        cls.mockserver = MockServer()
+        cls.mockserver.__enter__()  # pylint: disable=unnecessary-dunder-call
+
+    @classmethod
+    def teardown_class(cls):
+        cls.mockserver.__exit__(None, None, None)
+
+    def setup_method(self):
+        self.temp_dir = tempfile.mkdtemp()
+
+    def teardown_method(self):
+        shutil.rmtree(self.temp_dir, ignore_errors=True)
+
+    async def exported_data(
+        self, items: Iterable[Any], settings: dict[str, Any]
+    ) -> dict[str, Any]:
+        """
+        Return exported data which a spider yielding ``items`` would return.
+        """
+
+        class TestSpider(scrapy.Spider):
+            name = "testspider"
+
+            def parse(self, response):
+                yield from items
+
+        return await self.run_and_export(TestSpider, settings)
+
+    async def exported_no_data(self, settings: dict[str, Any]) -> dict[str, Any]:
+        """
+        Return exported data which a spider yielding no ``items`` would return.
+        """
+
+        class TestSpider(scrapy.Spider):
+            name = "testspider"
+
+            def parse(self, response):
+                pass
+
+        return await self.run_and_export(TestSpider, settings)
+
+    async def assertExported(
+        self,
+        items: Iterable[Any],
+        header: Iterable[str],
+        rows: Iterable[dict[str, Any]],
+        settings: dict[str, Any] | None = None,
+    ) -> None:
+        await self.assertExportedCsv(items, header, rows, settings)
+        await self.assertExportedJsonLines(items, rows, settings)
+        await self.assertExportedXml(items, rows, settings)
+        await self.assertExportedPickle(items, rows, settings)
+        await self.assertExportedMarshal(items, rows, settings)
+        await self.assertExportedMultiple(items, rows, settings)
+
+    async def assertExportedCsv(  # noqa: B027
+        self,
+        items: Iterable[Any],
+        header: Iterable[str],
+        rows: Iterable[dict[str, Any]],
+        settings: dict[str, Any] | None = None,
+    ) -> None:
+        pass
+
+    async def assertExportedJsonLines(  # noqa: B027
+        self,
+        items: Iterable[Any],
+        rows: Iterable[dict[str, Any]],
+        settings: dict[str, Any] | None = None,
+    ) -> None:
+        pass
+
+    async def assertExportedXml(  # noqa: B027
+        self,
+        items: Iterable[Any],
+        rows: Iterable[dict[str, Any]],
+        settings: dict[str, Any] | None = None,
+    ) -> None:
+        pass
+
+    async def assertExportedMultiple(  # noqa: B027
+        self,
+        items: Iterable[Any],
+        rows: Iterable[dict[str, Any]],
+        settings: dict[str, Any] | None = None,
+    ) -> None:
+        pass
+
+    async def assertExportedPickle(  # noqa: B027
+        self,
+        items: Iterable[Any],
+        rows: Iterable[dict[str, Any]],
+        settings: dict[str, Any] | None = None,
+    ) -> None:
+        pass
+
+    async def assertExportedMarshal(  # noqa: B027
+        self,
+        items: Iterable[Any],
+        rows: Iterable[dict[str, Any]],
+        settings: dict[str, Any] | None = None,
+    ) -> None:
+        pass
+
+    @abstractmethod
+    async def run_and_export(
+        self, spider_cls: type[Spider], settings: dict[str, Any]
+    ) -> dict[str, Any]:
+        pass
+
+    def _load_until_eof(
+        self, data: bytes, load_func: Callable[[IO[bytes]], Any]
+    ) -> list[Any]:
+        result: list[Any] = []
+        with tempfile.TemporaryFile() as temp:
+            temp.write(data)
+            temp.seek(0)
+            while True:
+                try:
+                    result.append(load_func(temp))
+                except EOFError:
+                    break
+        return result

tests/utils/bases/http_request.py

@@ -0,0 +1,490 @@
+import warnings
+from abc import ABC, abstractmethod
+from typing import Any
+
+import pytest
+
+from scrapy.http import Headers, Request
+from scrapy.http.request import NO_CALLBACK
+
+
+class TestRequestBase(ABC):
+    default_method = "GET"
+    default_headers: dict[bytes, list[bytes]] = {}
+    default_meta: dict[str, Any] = {}
+
+    @property
+    @abstractmethod
+    def request_class(self) -> type[Request]:
+        raise NotImplementedError
+
+    def test_init(self):
+        # Request requires url in the __init__ method
+        with pytest.raises(TypeError):
+            self.request_class()
+
+        # url argument must be basestring
+        with pytest.raises(TypeError):
+            self.request_class(123)
+
+        # priority argument must be an integer
+        with pytest.raises(TypeError, match="Request priority not an integer"):
+            self.request_class("http://www.example.com", priority="1")
+
+        r = self.request_class("http://www.example.com")
+        assert isinstance(r.url, str)
+        assert r.url == "http://www.example.com"
+        assert r.method == self.default_method
+
+        assert isinstance(r.headers, Headers)
+        assert r.headers == self.default_headers
+        assert r.meta == self.default_meta
+
+        meta = {"lala": "lolo"}
+        headers = {b"caca": b"coco"}
+        r = self.request_class(
+            "http://www.example.com", meta=meta, headers=headers, body="a body"
+        )
+
+        assert r.meta is not meta
+        assert r.meta == meta
+        assert r.headers is not headers
+        assert r.headers[b"caca"] == b"coco"
+
+    def test_url_scheme(self):
+        # This test passes by not raising any (ValueError) exception
+        self.request_class("http://example.org")
+        self.request_class("https://example.org")
+        self.request_class("s3://example.org")
+        self.request_class("ftp://example.org")
+        self.request_class("about:config")
+        self.request_class("data:,Hello%2C%20World!")
+
+    def test_url_no_scheme(self):
+        msg = "Missing scheme in request url:"
+        with pytest.raises(ValueError, match=msg):
+            self.request_class("foo")
+        with pytest.raises(ValueError, match=msg):
+            self.request_class("/foo/")
+        with pytest.raises(ValueError, match=msg):
+            self.request_class("/foo:bar")
+
+    def test_headers(self):
+        # Different ways of setting headers attribute
+        url = "http://www.scrapy.org"
+        headers = {b"Accept": "gzip", b"Custom-Header": "nothing to tell you"}
+        r = self.request_class(url=url, headers=headers)
+        p = self.request_class(url=url, headers=r.headers)
+
+        assert r.headers == p.headers
+        assert r.headers is not headers
+        assert p.headers is not r.headers
+
+        # headers must not be unicode
+        h = Headers({"key1": "val1", "key2": "val2"})
+        h["newkey"] = "newval"
+        for k, v in h.items():
+            assert isinstance(k, bytes)
+            for s in v:
+                assert isinstance(s, bytes)
+
+    def test_eq(self):
+        url = "http://www.scrapy.org"
+        r1 = self.request_class(url=url)
+        r2 = self.request_class(url=url)
+        assert r1 != r2
+
+        set_ = set()
+        set_.add(r1)
+        set_.add(r2)
+        assert len(set_) == 2
+
+    def test_url(self):
+        r = self.request_class(url="http://www.scrapy.org/path")
+        assert r.url == "http://www.scrapy.org/path"
+
+    def test_url_quoting(self):
+        r = self.request_class(url="http://www.scrapy.org/blank%20space")
+        assert r.url == "http://www.scrapy.org/blank%20space"
+        r = self.request_class(url="http://www.scrapy.org/blank space")
+        assert r.url == "http://www.scrapy.org/blank%20space"
+
+    def test_url_encoding(self):
+        r = self.request_class(url="http://www.scrapy.org/price/£")
+        assert r.url == "http://www.scrapy.org/price/%C2%A3"
+
+    def test_url_encoding_other(self):
+        # encoding affects only query part of URI, not path
+        # path part should always be UTF-8 encoded before percent-escaping
+        r = self.request_class(url="http://www.scrapy.org/price/£", encoding="utf-8")
+        assert r.url == "http://www.scrapy.org/price/%C2%A3"
+
+        r = self.request_class(url="http://www.scrapy.org/price/£", encoding="latin1")
+        assert r.url == "http://www.scrapy.org/price/%C2%A3"
+
+    def test_url_encoding_query(self):
+        r1 = self.request_class(url="http://www.scrapy.org/price/£?unit=µ")
+        assert r1.url == "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5"
+
+        # should be same as above
+        r2 = self.request_class(
+            url="http://www.scrapy.org/price/£?unit=µ", encoding="utf-8"
+        )
+        assert r2.url == "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5"
+
+    def test_url_encoding_query_latin1(self):
+        # encoding is used for encoding query-string before percent-escaping;
+        # path is still UTF-8 encoded before percent-escaping
+        r3 = self.request_class(
+            url="http://www.scrapy.org/price/µ?currency=£", encoding="latin1"
+        )
+        assert r3.url == "http://www.scrapy.org/price/%C2%B5?currency=%A3"
+
+    def test_url_encoding_nonutf8_untouched(self):
+        # percent-escaping sequences that do not match valid UTF-8 sequences
+        # should be kept untouched (just upper-cased perhaps)
+        #
+        # See https://datatracker.ietf.org/doc/html/rfc3987#section-3.2
+        #
+        # "Conversions from URIs to IRIs MUST NOT use any character encoding
+        # other than UTF-8 in steps 3 and 4, even if it might be possible to
+        # guess from the context that another character encoding than UTF-8 was
+        # used in the URI.  For example, the URI
+        # "http://www.example.org/r%E9sum%E9.html" might with some guessing be
+        # interpreted to contain two e-acute characters encoded as iso-8859-1.
+        # It must not be converted to an IRI containing these e-acute
+        # characters.  Otherwise, in the future the IRI will be mapped to
+        # "http://www.example.org/r%C3%A9sum%C3%A9.html", which is a different
+        # URI from "http://www.example.org/r%E9sum%E9.html".
+        r1 = self.request_class(url="http://www.scrapy.org/price/%a3")
+        assert r1.url == "http://www.scrapy.org/price/%a3"
+
+        r2 = self.request_class(url="http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3")
+        assert r2.url == "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3"
+
+        r3 = self.request_class(url="http://www.scrapy.org/résumé/%a3")
+        assert r3.url == "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3"
+
+        r4 = self.request_class(url="http://www.example.org/r%E9sum%E9.html")
+        assert r4.url == "http://www.example.org/r%E9sum%E9.html"
+
+    def test_url_verbatim(self):
+        r = self.request_class(
+            url="http://www.scrapy.org/price/£",
+            meta={"verbatim_url": True},
+        )
+        assert r.url == "http://www.scrapy.org/price/£"
+
+        r = self.request_class(
+            url="http://www.scrapy.org/blank space",
+            meta={"verbatim_url": True},
+        )
+        assert r.url == "http://www.scrapy.org/blank space"
+
+    def test_body(self):
+        r1 = self.request_class(url="http://www.example.com/")
+        assert r1.body == b""
+
+        r2 = self.request_class(url="http://www.example.com/", body=b"")
+        assert isinstance(r2.body, bytes)
+        assert r2.encoding == "utf-8"  # default encoding
+
+        r3 = self.request_class(
+            url="http://www.example.com/", body="Price: \xa3100", encoding="utf-8"
+        )
+        assert isinstance(r3.body, bytes)
+        assert r3.body == b"Price: \xc2\xa3100"
+
+        r4 = self.request_class(
+            url="http://www.example.com/", body="Price: \xa3100", encoding="latin1"
+        )
+        assert isinstance(r4.body, bytes)
+        assert r4.body == b"Price: \xa3100"
+
+    def test_copy(self):
+        """Test Request copy"""
+
+        def somecallback():
+            pass
+
+        r1 = self.request_class(
+            "http://www.example.com",
+            flags=["f1", "f2"],
+            callback=somecallback,
+            errback=somecallback,
+        )
+        r1.meta["foo"] = "bar"
+        r1.cb_kwargs["key"] = "value"
+        r2 = r1.copy()
+
+        # make sure callbaclks are copied
+        assert r1.callback is somecallback
+        assert r1.errback is somecallback
+        assert r2.callback is r1.callback
+        assert r2.errback is r1.errback
+
+        # make sure flags list is shallow copied
+        assert r1.flags is not r2.flags, "flags must be a shallow copy, not identical"
+        assert r1.flags == r2.flags
+
+        # make sure cb_kwargs dict is shallow copied
+        assert r1.cb_kwargs is not r2.cb_kwargs, (
+            "cb_kwargs must be a shallow copy, not identical"
+        )
+        assert r1.cb_kwargs == r2.cb_kwargs
+
+        # make sure meta dict is shallow copied
+        assert r1.meta is not r2.meta, "meta must be a shallow copy, not identical"
+        assert r1.meta == r2.meta
+
+        # make sure headers attribute is shallow copied
+        assert r1.headers is not r2.headers, (
+            "headers must be a shallow copy, not identical"
+        )
+        assert r1.headers == r2.headers
+        assert r1.encoding == r2.encoding
+        assert r1.dont_filter == r2.dont_filter
+
+        # Request.body can be identical since it's an immutable object (str)
+
+    def test_copy_inherited_classes(self):
+        """Test Request children copies preserve their class"""
+
+        class CustomRequest(self.request_class):
+            pass
+
+        r1 = CustomRequest("http://www.example.com")
+        r2 = r1.copy()
+
+        assert isinstance(r2, CustomRequest)
+
+    def test_replace(self):
+        """Test Request.replace() method"""
+        r1 = self.request_class("http://www.example.com", method="GET")
+        hdrs = Headers(r1.headers)
+        hdrs[b"key"] = b"value"
+        r2 = r1.replace(method="POST", body="New body", headers=hdrs)
+        assert r1.url == r2.url
+        assert (r1.method, r2.method) == ("GET", "POST")
+        assert (r1.body, r2.body) == (b"", b"New body")
+        assert (r1.headers, r2.headers) == (self.default_headers, hdrs)
+
+        # Empty attributes (which may fail if not compared properly)
+        r3 = self.request_class(
+            "http://www.example.com", meta={"a": 1}, dont_filter=True
+        )
+        r4 = r3.replace(
+            url="http://www.example.com/2", body=b"", meta={}, dont_filter=False
+        )
+        assert r4.url == "http://www.example.com/2"
+        assert r4.body == b""
+        assert r4.meta == {}
+        assert r4.dont_filter is False
+
+        # the cls argument allows changing the resulting class
+        custom_request_cls = type("CustomRequest", (self.request_class,), {})
+        r5 = r1.replace(cls=custom_request_cls)
+        assert isinstance(r5, custom_request_cls)
+        assert r5.url == r1.url
+
+    def test_method_always_str(self):
+        r = self.request_class("http://www.example.com", method="POST")
+        assert isinstance(r.method, str)
+
+    def test_immutable_attributes(self):
+        r = self.request_class("http://example.com")
+        with pytest.raises(AttributeError):
+            r.url = "http://example2.com"
+        with pytest.raises(AttributeError):
+            r.body = "xxx"
+
+    def test_callback_and_errback(self):
+        def a_function():
+            pass
+
+        r1 = self.request_class("http://example.com")
+        assert r1.callback is None
+        assert r1.errback is None
+
+        r2 = self.request_class("http://example.com", callback=a_function)
+        assert r2.callback is a_function
+        assert r2.errback is None
+
+        r3 = self.request_class("http://example.com", errback=a_function)
+        assert r3.callback is None
+        assert r3.errback is a_function
+
+        r4 = self.request_class(
+            url="http://example.com",
+            callback=a_function,
+            errback=a_function,
+        )
+        assert r4.callback is a_function
+        assert r4.errback is a_function
+
+        r5 = self.request_class(
+            url="http://example.com",
+            callback=NO_CALLBACK,
+            errback=NO_CALLBACK,
+        )
+        assert r5.callback is NO_CALLBACK
+        assert r5.errback is NO_CALLBACK
+
+    def test_callback_and_errback_type(self):
+        with pytest.raises(TypeError):
+            self.request_class("http://example.com", callback="a_function")
+        with pytest.raises(TypeError):
+            self.request_class("http://example.com", errback="a_function")
+        with pytest.raises(TypeError):
+            self.request_class(
+                url="http://example.com",
+                callback="a_function",
+                errback="a_function",
+            )
+
+    def test_setters(self):
+        request = self.request_class("http://example.com")
+
+        request.flags = ["f1"]
+        assert request.flags == ["f1"]
+
+        request.cookies = {"sid": "1"}
+        assert request.cookies == {"sid": "1"}
+
+        headers = Headers({b"X-Test": b"1"})
+        request.headers = headers
+        assert request._headers is headers
+        request.headers = {b"A": b"b"}
+        assert isinstance(request.headers, Headers)
+        assert request._headers[b"A"] == b"b"
+
+    def test_setter_mutable_lazy_loading(self):
+        """Mutable attributes are set internally to None only until they are
+        read, then they always return the same falsy instance of the
+        corresponding mutable structure.
+
+        Setting them to None causes the next read to return a different object.
+        """
+
+        request = self.request_class("http://example.com")
+
+        assert request._flags is None
+        assert request.flags == []
+        assert request.flags is request.flags
+        assert request._flags == []
+        original_flags = request.flags
+        request.flags = None
+        assert request._flags is None
+        assert request.flags == []
+        assert request.flags is not original_flags
+
+        assert request._cookies is None
+        assert request.cookies == {}
+        assert request.cookies is request.cookies
+        assert request._cookies == {}
+        original_cookies = request.cookies
+        request.cookies = None
+        assert request._cookies is None
+        assert request.cookies == {}
+        assert request.cookies is not original_cookies
+
+        if self.default_headers:
+            assert request._headers == self.default_headers
+            assert request._headers is not self.default_headers
+            assert request.headers == self.default_headers
+        else:
+            assert request._headers is None
+            assert request.headers == {}
+        assert request.headers is request.headers
+        assert isinstance(request.headers, Headers)
+        assert isinstance(request._headers, Headers)
+        original_headers = request.headers
+        request.headers = None
+        assert request._headers is None
+        assert request.headers == {}
+        assert request._headers == {}
+        assert request.headers is not original_headers
+
+    def test_no_callback(self):
+        with pytest.raises(RuntimeError):
+            NO_CALLBACK()
+
+    def test_from_curl(self):
+        # Note: more curated tests regarding curl conversion are in
+        # `test_utils_curl.py`
+        curl_command = (
+            "curl 'http://httpbin.org/post' -X POST -H 'Cookie: _gauges_unique"
+            "_year=1; _gauges_unique=1; _gauges_unique_month=1; _gauges_unique"
+            "_hour=1; _gauges_unique_day=1' -H 'Origin: http://httpbin.org' -H"
+            " 'Accept-Encoding: gzip, deflate' -H 'Accept-Language: en-US,en;q"
+            "=0.9,ru;q=0.8,es;q=0.7' -H 'Upgrade-Insecure-Requests: 1' -H 'Use"
+            "r-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTM"
+            "L, like Gecko) Ubuntu Chromium/62.0.3202.75 Chrome/62.0.3202.75 S"
+            "afari/537.36' -H 'Content-Type: application /x-www-form-urlencode"
+            "d' -H 'Accept: text/html,application/xhtml+xml,application/xml;q="
+            "0.9,image/webp,image/apng,*/*;q=0.8' -H 'Cache-Control: max-age=0"
+            "' -H 'Referer: http://httpbin.org/forms/post' -H 'Connection: kee"
+            "p-alive' --data 'custname=John+Smith&custtel=500&custemail=jsmith"
+            "%40example.org&size=small&topping=cheese&topping=onion&delivery=1"
+            "2%3A15&comments=' --compressed"
+        )
+        r = self.request_class.from_curl(curl_command)
+        assert r.method == "POST"
+        assert r.url == "http://httpbin.org/post"
+        assert (
+            r.body == b"custname=John+Smith&custtel=500&custemail=jsmith%40"
+            b"example.org&size=small&topping=cheese&topping=onion"
+            b"&delivery=12%3A15&comments="
+        )
+        assert r.cookies == {
+            "_gauges_unique_year": "1",
+            "_gauges_unique": "1",
+            "_gauges_unique_month": "1",
+            "_gauges_unique_hour": "1",
+            "_gauges_unique_day": "1",
+        }
+        assert r.headers == {
+            b"Origin": [b"http://httpbin.org"],
+            b"Accept-Encoding": [b"gzip, deflate"],
+            b"Accept-Language": [b"en-US,en;q=0.9,ru;q=0.8,es;q=0.7"],
+            b"Upgrade-Insecure-Requests": [b"1"],
+            b"User-Agent": [
+                b"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537."
+                b"36 (KHTML, like Gecko) Ubuntu Chromium/62.0.3202"
+                b".75 Chrome/62.0.3202.75 Safari/537.36"
+            ],
+            b"Content-Type": [b"application /x-www-form-urlencoded"],
+            b"Accept": [
+                b"text/html,application/xhtml+xml,application/xml;q=0."
+                b"9,image/webp,image/apng,*/*;q=0.8"
+            ],
+            b"Cache-Control": [b"max-age=0"],
+            b"Referer": [b"http://httpbin.org/forms/post"],
+            b"Connection": [b"keep-alive"],
+        }
+
+    def test_from_curl_with_kwargs(self):
+        r = self.request_class.from_curl(
+            'curl -X PATCH "http://example.org"', method="POST", meta={"key": "value"}
+        )
+        assert r.method == "POST"
+        assert r.meta == {"key": "value"}
+
+    def test_from_curl_ignore_unknown_options(self):
+        # By default: it works and ignores the unknown options: --foo and -z
+        with warnings.catch_warnings():  # avoid warning when executing tests
+            warnings.filterwarnings(
+                "ignore", category=UserWarning, message="Unrecognized options:"
+            )
+            r = self.request_class.from_curl(
+                'curl -X DELETE "http://example.org" --foo -z',
+            )
+        assert r.method == "DELETE"
+
+        # If `ignore_unknown_options` is set to `False` it raises an error with
+        # the unknown options: --foo and -z
+        with pytest.raises(ValueError, match="Unrecognized options:"):
+            self.request_class.from_curl(
+                'curl -X PATCH "http://example.org" --foo -z',
+                ignore_unknown_options=False,
+            )

tests/utils/bases/http_response.py

@@ -0,0 +1,417 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING
+
+import pytest
+from w3lib.encoding import resolve_encoding
+
+from scrapy.exceptions import NotSupported
+from scrapy.http import Headers, Request, Response
+from scrapy.link import Link
+from scrapy.utils._deps_compat import W3LIB_STRIPS_URLS
+from tests import get_testdata
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable
+
+
+class TestResponseBase(ABC):
+    @property
+    @abstractmethod
+    def response_class(self) -> type[Response]:
+        raise NotImplementedError
+
+    def test_init(self):
+        # Response requires url in the constructor
+        with pytest.raises(TypeError):
+            self.response_class()
+        assert isinstance(
+            self.response_class("http://example.com/"), self.response_class
+        )
+        with pytest.raises(TypeError):
+            self.response_class(b"http://example.com")
+        with pytest.raises(TypeError):
+            self.response_class(url="http://example.com", body={})
+        # body can be str or None
+        assert isinstance(
+            self.response_class("http://example.com/", body=b""),
+            self.response_class,
+        )
+        assert isinstance(
+            self.response_class("http://example.com/", body=b"body"),
+            self.response_class,
+        )
+        # test presence of all optional parameters
+        assert isinstance(
+            self.response_class(
+                "http://example.com/", body=b"", headers={}, status=200
+            ),
+            self.response_class,
+        )
+
+        r = self.response_class("http://www.example.com")
+        assert isinstance(r.url, str)
+        assert r.url == "http://www.example.com"
+        assert r.status == 200
+
+        assert isinstance(r.headers, Headers)
+        assert not r.headers
+
+        headers = {"foo": "bar"}
+        body = b"a body"
+        r = self.response_class("http://www.example.com", headers=headers, body=body)
+
+        assert r.headers is not headers
+        assert r.headers[b"foo"] == b"bar"
+
+        r = self.response_class("http://www.example.com", status=301)
+        assert r.status == 301
+        r = self.response_class("http://www.example.com", status="301")
+        assert r.status == 301
+        with pytest.raises(ValueError, match=r"invalid literal for int\(\)"):
+            self.response_class("http://example.com", status="lala200")
+
+    def test_copy(self):
+        """Test Response copy"""
+
+        r1 = self.response_class("http://www.example.com", body=b"Some body")
+        r1.flags.append("cached")
+        r2 = r1.copy()
+
+        assert r1.status == r2.status
+        assert r1.body == r2.body
+
+        # make sure flags list is shallow copied
+        assert r1.flags is not r2.flags, "flags must be a shallow copy, not identical"
+        assert r1.flags == r2.flags
+
+        # make sure headers attribute is shallow copied
+        assert r1.headers is not r2.headers, (
+            "headers must be a shallow copy, not identical"
+        )
+        assert r1.headers == r2.headers
+
+    def test_copy_meta(self):
+        req = Request("http://www.example.com")
+        req.meta["foo"] = "bar"
+        r1 = self.response_class(
+            "http://www.example.com", body=b"Some body", request=req
+        )
+        assert r1.meta is req.meta
+
+    def test_copy_cb_kwargs(self):
+        req = Request("http://www.example.com")
+        req.cb_kwargs["foo"] = "bar"
+        r1 = self.response_class(
+            "http://www.example.com", body=b"Some body", request=req
+        )
+        assert r1.cb_kwargs is req.cb_kwargs
+
+    def test_unavailable_meta(self):
+        r1 = self.response_class("http://www.example.com", body=b"Some body")
+        with pytest.raises(AttributeError, match=r"Response\.meta not available"):
+            r1.meta  # pylint: disable=pointless-statement
+
+    def test_unavailable_cb_kwargs(self):
+        r1 = self.response_class("http://www.example.com", body=b"Some body")
+        with pytest.raises(AttributeError, match=r"Response\.cb_kwargs not available"):
+            r1.cb_kwargs  # pylint: disable=pointless-statement
+
+    def test_copy_inherited_classes(self):
+        """Test Response children copies preserve their class"""
+
+        class CustomResponse(self.response_class):
+            pass
+
+        r1 = CustomResponse("http://www.example.com")
+        r2 = r1.copy()
+
+        assert isinstance(r2, CustomResponse)
+
+    def test_replace(self):
+        """Test Response.replace() method"""
+        hdrs = Headers({"key": "value"})
+        r1 = self.response_class("http://www.example.com")
+        r2 = r1.replace(status=301, body=b"New body", headers=hdrs)
+        assert r1.body == b""
+        assert r1.url == r2.url
+        assert (r1.status, r2.status) == (200, 301)
+        assert (r1.body, r2.body) == (b"", b"New body")
+        assert (r1.headers, r2.headers) == ({}, hdrs)
+
+        # Empty attributes (which may fail if not compared properly)
+        r3 = self.response_class("http://www.example.com", flags=["cached"])
+        r4 = r3.replace(body=b"", flags=[])
+        assert r4.body == b""
+        assert not r4.flags
+
+    def _assert_response_values(self, response, encoding, body):
+        if isinstance(body, str):
+            body_unicode = body
+            body_bytes = body.encode(encoding)
+        else:
+            body_unicode = body.decode(encoding)
+            body_bytes = body
+
+        assert isinstance(response.body, bytes)
+        assert isinstance(response.text, str)
+        self._assert_response_encoding(response, encoding)
+        assert response.body == body_bytes
+        assert response.text == body_unicode
+
+    def _assert_response_encoding(self, response, encoding):
+        assert response.encoding == resolve_encoding(encoding)
+
+    def test_immutable_attributes(self):
+        r = self.response_class("http://example.com")
+        with pytest.raises(AttributeError):
+            r.url = "http://example2.com"
+        with pytest.raises(AttributeError):
+            r.body = "xxx"
+
+    def test_setter_mutable_lazy_loading(self):
+        """Mutable attributes are set internally to None only until they are
+        read, then they always return the same falsy instance of the
+        corresponding mutable structure.
+
+        Setting them to None causes the next read to return a different object.
+        """
+
+        response = self.response_class("http://example.com")
+
+        response.request = Request("http://example.com")
+
+        assert response._flags is None
+        assert response.flags == []
+        assert response.flags is response.flags
+        assert response._flags == []
+        original_flags = response.flags
+        response.flags = None
+        assert response._flags is None
+        assert response.flags == []
+        assert response.flags is not original_flags
+
+        assert response._headers is None
+        assert response.headers == {}
+        assert response.headers is response.headers
+        assert isinstance(response.headers, Headers)
+        assert isinstance(response._headers, Headers)
+        original_headers = response.headers
+        response.headers = None
+        assert response._headers is None
+        assert response.headers == {}
+        assert response._headers == {}
+        assert response.headers is not original_headers
+
+    def test_setters(self):
+        response = self.response_class("http://example.com")
+
+        response.flags = ["f1"]
+        assert response.flags == ["f1"]
+
+        headers = Headers({b"X-Test": b"1"})
+        response.headers = headers
+        assert response._headers is headers
+        response.headers = {b"A": b"b"}
+        assert isinstance(response.headers, Headers)
+        assert response._headers[b"A"] == b"b"
+
+    def test_urljoin(self):
+        """Test urljoin shortcut (only for existence, since behavior equals urljoin)"""
+        joined = self.response_class("http://www.example.com").urljoin("/test")
+        absolute = "http://www.example.com/test"
+        assert joined == absolute
+
+    def test_shortcut_attributes(self):
+        r = self.response_class("http://example.com", body=b"hello")
+        if self.response_class == Response:
+            msg = "Response content isn't text"
+            with pytest.raises(AttributeError, match=msg):
+                r.text  # pylint: disable=pointless-statement
+            with pytest.raises(NotSupported, match=msg):
+                r.css("body")
+            with pytest.raises(NotSupported, match=msg):
+                r.xpath("//body")
+            with pytest.raises(NotSupported, match=msg):
+                r.jmespath("body")
+        else:
+            r.text  # pylint: disable=pointless-statement
+            r.css("body")
+            r.xpath("//body")
+
+    # Response.follow
+
+    def test_follow_url_absolute(self):
+        self._assert_followed_url("http://foo.example.com", "http://foo.example.com")
+
+    def test_follow_url_relative(self):
+        self._assert_followed_url("foo", "http://example.com/foo")
+
+    def test_follow_link(self):
+        self._assert_followed_url(
+            Link("http://example.com/foo"), "http://example.com/foo"
+        )
+
+    def test_follow_None_url(self):
+        r = self.response_class("http://example.com")
+        with pytest.raises(ValueError, match="url can't be None"):
+            r.follow(None)
+
+    def test_follow_None_encoding(self):
+        r = self.response_class("http://example.com")
+        with pytest.raises(ValueError, match="encoding can't be None"):
+            r.follow("foo", encoding=None)
+
+    @pytest.mark.xfail(
+        not W3LIB_STRIPS_URLS,
+        reason="https://github.com/scrapy/w3lib/pull/207",
+        strict=True,
+    )
+    def test_follow_whitespace_url(self):
+        self._assert_followed_url("foo ", "http://example.com/foo")
+
+    @pytest.mark.xfail(
+        not W3LIB_STRIPS_URLS,
+        reason="https://github.com/scrapy/w3lib/pull/207",
+        strict=True,
+    )
+    def test_follow_whitespace_link(self):
+        self._assert_followed_url(
+            Link("http://example.com/foo "), "http://example.com/foo"
+        )
+
+    def test_follow_flags(self):
+        res = self.response_class("http://example.com/")
+        fol = res.follow("http://example.com/", flags=["cached", "allowed"])
+        assert fol.flags == ["cached", "allowed"]
+
+    # Response.follow_all
+
+    def test_follow_all_absolute(self):
+        url_list = [
+            "http://example.org",
+            "http://www.example.org",
+            "http://example.com",
+            "http://www.example.com",
+        ]
+        self._assert_followed_all_urls(url_list, url_list)
+
+    def test_follow_all_relative(self):
+        relative = ["foo", "bar", "foo/bar", "bar/foo"]
+        absolute = [
+            "http://example.com/foo",
+            "http://example.com/bar",
+            "http://example.com/foo/bar",
+            "http://example.com/bar/foo",
+        ]
+        self._assert_followed_all_urls(relative, absolute)
+
+    def test_follow_all_links(self):
+        absolute = [
+            "http://example.com/foo",
+            "http://example.com/bar",
+            "http://example.com/foo/bar",
+            "http://example.com/bar/foo",
+        ]
+        links = map(Link, absolute)
+        self._assert_followed_all_urls(links, absolute)
+
+    def test_follow_all_empty(self):
+        r = self.response_class("http://example.com")
+        assert not list(r.follow_all([]))
+
+    def test_follow_all_invalid(self):
+        r = self.response_class("http://example.com")
+        if self.response_class == Response:
+            with pytest.raises(TypeError):
+                list(r.follow_all(urls=None))
+            with pytest.raises(TypeError):
+                list(r.follow_all(urls=12345))
+            with pytest.raises(ValueError, match="url can't be None"):
+                list(r.follow_all(urls=[None]))
+        else:
+            with pytest.raises(
+                ValueError, match="Please supply exactly one of the following arguments"
+            ):
+                list(r.follow_all(urls=None))
+            with pytest.raises(TypeError):
+                list(r.follow_all(urls=12345))
+            with pytest.raises(ValueError, match="url can't be None"):
+                list(r.follow_all(urls=[None]))
+
+    @pytest.mark.xfail(
+        not W3LIB_STRIPS_URLS,
+        reason="https://github.com/scrapy/w3lib/pull/207",
+        strict=True,
+    )
+    def test_follow_all_whitespace(self):
+        relative = ["foo ", "bar ", "foo/bar ", "bar/foo "]
+        absolute = [
+            "http://example.com/foo",
+            "http://example.com/bar",
+            "http://example.com/foo/bar",
+            "http://example.com/bar/foo",
+        ]
+        self._assert_followed_all_urls(relative, absolute)
+
+    @pytest.mark.xfail(
+        not W3LIB_STRIPS_URLS,
+        reason="https://github.com/scrapy/w3lib/pull/207",
+        strict=True,
+    )
+    def test_follow_all_whitespace_links(self):
+        absolute = [
+            "http://example.com/foo ",
+            "http://example.com/bar ",
+            "http://example.com/foo/bar ",
+            "http://example.com/bar/foo ",
+        ]
+        links = [Link(u) for u in absolute]
+        expected = [u.strip() for u in absolute]
+        self._assert_followed_all_urls(links, expected)
+
+    def test_follow_all_flags(self):
+        re = self.response_class("http://www.example.com/")
+        urls = [
+            "http://www.example.com/",
+            "http://www.example.com/2",
+            "http://www.example.com/foo",
+        ]
+        fol = re.follow_all(urls, flags=["cached", "allowed"])
+        for req in fol:
+            assert req.flags == ["cached", "allowed"]
+
+    def _assert_followed_url(
+        self,
+        follow_obj: str | Link,
+        target_url: str,
+        response: Response | None = None,
+        encoding: str | None = None,
+    ) -> None:
+        if response is None:
+            response = self._links_response()
+        req = response.follow(follow_obj)
+        assert req.url == target_url
+        if encoding is not None:
+            assert req.encoding == encoding
+
+    def _assert_followed_all_urls(
+        self,
+        follow_obj: Iterable[str | Link],
+        target_urls: Iterable[str],
+        response: Response | None = None,
+    ) -> None:
+        if response is None:
+            response = self._links_response()
+        followed = response.follow_all(follow_obj)
+        for req, target in zip(followed, target_urls, strict=True):
+            assert req.url == target
+
+    def _links_response(self) -> Response:
+        body = get_testdata("link_extractor", "linkextractor.html")
+        return self.response_class("http://example.com/index", body=body)
+
+    def _links_response_no_href(self) -> Response:
+        body = get_testdata("link_extractor", "linkextractor_no_href.html")
+        return self.response_class("http://example.com/index", body=body)

tests/utils/bases/redirect.py

@@ -0,0 +1,988 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Any
+
+import pytest
+
+from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware
+from scrapy.exceptions import IgnoreRequest
+from scrapy.http import Request, Response
+from scrapy.utils.misc import set_environ
+from scrapy.utils.test import get_crawler
+
+
+class TestRedirectBase(ABC):
+    mwcls: type[Any]
+    mw: Any
+    reason: int | str
+
+    @abstractmethod
+    def get_response(
+        self, request: Request, location: str, status: int = 302
+    ) -> Response:
+        raise NotImplementedError
+
+    def test_priority_adjust(self):
+        req = Request("http://a.example")
+        rsp = self.get_response(req, "http://a.example/redirected")
+        req2 = self.mw.process_response(req, rsp)
+        assert req2.priority > req.priority
+
+    def test_dont_redirect(self):
+        url = "http://www.example.com/301"
+        url2 = "http://www.example.com/redirected"
+        req = Request(url, meta={"dont_redirect": True})
+        rsp = self.get_response(req, url2)
+
+        r = self.mw.process_response(req, rsp)
+        assert isinstance(r, Response)
+        assert r is rsp
+
+        # Test that it redirects when dont_redirect is False
+        req = Request(url, meta={"dont_redirect": False})
+        rsp = self.get_response(req, url2)
+
+        r = self.mw.process_response(req, rsp)
+        assert isinstance(r, Request)
+
+    def test_post(self):
+        url = "http://www.example.com/302"
+        url2 = "http://www.example.com/redirected2"
+        req = Request(
+            url,
+            method="POST",
+            body="test",
+            headers={"Content-Type": "text/plain", "Content-length": "4"},
+        )
+        rsp = self.get_response(req, url2)
+
+        req2 = self.mw.process_response(req, rsp)
+        assert isinstance(req2, Request)
+        assert req2.url == url2
+        assert req2.method == "GET"
+        assert "Content-Type" not in req2.headers, (
+            "Content-Type header must not be present in redirected request"
+        )
+        assert "Content-Length" not in req2.headers, (
+            "Content-Length header must not be present in redirected request"
+        )
+        assert not req2.body, f"Redirected body must be empty, not '{req2.body!r}'"
+
+    def test_max_redirect_times(self):
+        self.mw.max_redirect_times = 1
+        req = Request("http://a.example/302")
+        rsp = self.get_response(req, "/redirected")
+
+        req = self.mw.process_response(req, rsp)
+        assert isinstance(req, Request)
+        assert "redirect_times" in req.meta
+        assert req.meta["redirect_times"] == 1
+        with pytest.raises(IgnoreRequest):
+            self.mw.process_response(req, rsp)
+
+    def test_ttl(self):
+        self.mw.max_redirect_times = 100
+        req = Request("http://a.example/302", meta={"redirect_ttl": 1})
+        rsp = self.get_response(req, "/a")
+
+        req = self.mw.process_response(req, rsp)
+        assert isinstance(req, Request)
+        with pytest.raises(IgnoreRequest):
+            self.mw.process_response(req, rsp)
+
+    def test_redirect_urls(self):
+        req1 = Request("http://a.example/first")
+        rsp1 = self.get_response(req1, "/redirected")
+        req2 = self.mw.process_response(req1, rsp1)
+        rsp2 = self.get_response(req2, "/redirected2")
+        req3 = self.mw.process_response(req2, rsp2)
+
+        assert req2.url == "http://a.example/redirected"
+        assert req2.meta["redirect_urls"] == ["http://a.example/first"]
+        assert req3.url == "http://a.example/redirected2"
+        assert req3.meta["redirect_urls"] == [
+            "http://a.example/first",
+            "http://a.example/redirected",
+        ]
+
+    def test_redirect_reasons(self):
+        req1 = Request("http://a.example/first")
+        rsp1 = self.get_response(req1, "/redirected1")
+        req2 = self.mw.process_response(req1, rsp1)
+        rsp2 = self.get_response(req2, "/redirected2")
+        req3 = self.mw.process_response(req2, rsp2)
+        assert req2.meta["redirect_reasons"] == [self.reason]
+        assert req3.meta["redirect_reasons"] == [self.reason, self.reason]
+
+    def test_cross_origin_header_dropping(self):
+        safe_headers = {"A": "B"}
+        cookie_header = {"Cookie": "a=b"}
+        authorization_header = {"Authorization": "Bearer 123456"}
+
+        original_request = Request(
+            "https://example.com",
+            headers={**safe_headers, **cookie_header, **authorization_header},
+        )
+
+        # Redirects to the same origin (same scheme, same domain, same port)
+        # keep all headers.
+        internal_response = self.get_response(original_request, "https://example.com/a")
+        internal_redirect_request = self.mw.process_response(
+            original_request, internal_response
+        )
+        assert isinstance(internal_redirect_request, Request)
+        assert original_request.headers == internal_redirect_request.headers
+
+        # Redirects to the same origin (same scheme, same domain, same port)
+        # keep all headers also when the scheme is http.
+        http_request = Request(
+            "http://example.com",
+            headers={**safe_headers, **cookie_header, **authorization_header},
+        )
+        http_response = self.get_response(http_request, "http://example.com/a")
+        http_redirect_request = self.mw.process_response(http_request, http_response)
+        assert isinstance(http_redirect_request, Request)
+        assert http_request.headers == http_redirect_request.headers
+
+        # For default ports, whether the port is explicit or implicit does not
+        # affect the outcome, it is still the same origin.
+        to_explicit_port_response = self.get_response(
+            original_request, "https://example.com:443/a"
+        )
+        to_explicit_port_redirect_request = self.mw.process_response(
+            original_request, to_explicit_port_response
+        )
+        assert isinstance(to_explicit_port_redirect_request, Request)
+        assert original_request.headers == to_explicit_port_redirect_request.headers
+
+        # For default ports, whether the port is explicit or implicit does not
+        # affect the outcome, it is still the same origin.
+        to_implicit_port_response = self.get_response(
+            original_request, "https://example.com/a"
+        )
+        to_implicit_port_redirect_request = self.mw.process_response(
+            original_request, to_implicit_port_response
+        )
+        assert isinstance(to_implicit_port_redirect_request, Request)
+        assert original_request.headers == to_implicit_port_redirect_request.headers
+
+        # A port change drops the Authorization header because the origin
+        # changes, but keeps the Cookie header because the domain remains the
+        # same.
+        different_port_response = self.get_response(
+            original_request, "https://example.com:8080/a"
+        )
+        different_port_redirect_request = self.mw.process_response(
+            original_request, different_port_response
+        )
+        assert isinstance(different_port_redirect_request, Request)
+        assert {
+            **safe_headers,
+            **cookie_header,
+        } == different_port_redirect_request.headers.to_unicode_dict()
+
+        # A domain change drops both the Authorization and the Cookie header.
+        external_response = self.get_response(original_request, "https://example.org/a")
+        external_redirect_request = self.mw.process_response(
+            original_request, external_response
+        )
+        assert isinstance(external_redirect_request, Request)
+        assert safe_headers == external_redirect_request.headers.to_unicode_dict()
+
+        # A scheme upgrade (http → https) drops the Authorization header
+        # because the origin changes, but keeps the Cookie header because the
+        # domain remains the same.
+        upgrade_response = self.get_response(http_request, "https://example.com/a")
+        upgrade_redirect_request = self.mw.process_response(
+            http_request, upgrade_response
+        )
+        assert isinstance(upgrade_redirect_request, Request)
+        assert {
+            **safe_headers,
+            **cookie_header,
+        } == upgrade_redirect_request.headers.to_unicode_dict()
+
+        # A scheme downgrade (https → http) drops the Authorization header
+        # because the origin changes, and the Cookie header because its value
+        # cannot indicate whether the cookies were secure (HTTPS-only) or not.
+        #
+        # Note: If the Cookie header is set by the cookie management
+        # middleware, as recommended in the docs, the dropping of Cookie on
+        # scheme downgrade is not an issue, because the cookie management
+        # middleware will add again the Cookie header to the new request if
+        # appropriate.
+        downgrade_response = self.get_response(original_request, "http://example.com/a")
+        downgrade_redirect_request = self.mw.process_response(
+            original_request, downgrade_response
+        )
+        assert isinstance(downgrade_redirect_request, Request)
+        assert safe_headers == downgrade_redirect_request.headers.to_unicode_dict()
+
+    def test_meta_proxy_http_absolute(self):
+        crawler = get_crawler()
+        redirect_mw = self.mwcls.from_crawler(crawler)
+        proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
+
+        meta = {"proxy": "https://a:@a.example"}
+        request1 = Request("http://example.com", meta=meta)
+        proxy_mw.process_request(request1)
+
+        assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request1.meta["_auth_proxy"] == "https://a.example"
+        assert request1.meta["proxy"] == "https://a.example"
+
+        response1 = self.get_response(request1, "http://example.com")
+        request2 = redirect_mw.process_response(request1, response1)
+
+        assert isinstance(request2, Request)
+        assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request2.meta["_auth_proxy"] == "https://a.example"
+        assert request2.meta["proxy"] == "https://a.example"
+
+        proxy_mw.process_request(request2)
+
+        assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request2.meta["_auth_proxy"] == "https://a.example"
+        assert request2.meta["proxy"] == "https://a.example"
+
+        response2 = self.get_response(request2, "http://example.com")
+        request3 = redirect_mw.process_response(request2, response2)
+
+        assert isinstance(request3, Request)
+        assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request3.meta["_auth_proxy"] == "https://a.example"
+        assert request3.meta["proxy"] == "https://a.example"
+
+        proxy_mw.process_request(request3)
+
+        assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request3.meta["_auth_proxy"] == "https://a.example"
+        assert request3.meta["proxy"] == "https://a.example"
+
+    def test_meta_proxy_http_relative(self):
+        crawler = get_crawler()
+        redirect_mw = self.mwcls.from_crawler(crawler)
+        proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
+
+        meta = {"proxy": "https://a:@a.example"}
+        request1 = Request("http://example.com", meta=meta)
+        proxy_mw.process_request(request1)
+
+        assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request1.meta["_auth_proxy"] == "https://a.example"
+        assert request1.meta["proxy"] == "https://a.example"
+
+        response1 = self.get_response(request1, "/a")
+        request2 = redirect_mw.process_response(request1, response1)
+
+        assert isinstance(request2, Request)
+        assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request2.meta["_auth_proxy"] == "https://a.example"
+        assert request2.meta["proxy"] == "https://a.example"
+
+        proxy_mw.process_request(request2)
+
+        assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request2.meta["_auth_proxy"] == "https://a.example"
+        assert request2.meta["proxy"] == "https://a.example"
+
+        response2 = self.get_response(request2, "/a")
+        request3 = redirect_mw.process_response(request2, response2)
+
+        assert isinstance(request3, Request)
+        assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request3.meta["_auth_proxy"] == "https://a.example"
+        assert request3.meta["proxy"] == "https://a.example"
+
+        proxy_mw.process_request(request3)
+
+        assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request3.meta["_auth_proxy"] == "https://a.example"
+        assert request3.meta["proxy"] == "https://a.example"
+
+    def test_meta_proxy_https_absolute(self):
+        crawler = get_crawler()
+        redirect_mw = self.mwcls.from_crawler(crawler)
+        proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
+
+        meta = {"proxy": "https://a:@a.example"}
+        request1 = Request("https://example.com", meta=meta)
+        proxy_mw.process_request(request1)
+
+        assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request1.meta["_auth_proxy"] == "https://a.example"
+        assert request1.meta["proxy"] == "https://a.example"
+
+        response1 = self.get_response(request1, "https://example.com")
+        request2 = redirect_mw.process_response(request1, response1)
+
+        assert isinstance(request2, Request)
+        assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request2.meta["_auth_proxy"] == "https://a.example"
+        assert request2.meta["proxy"] == "https://a.example"
+
+        proxy_mw.process_request(request2)
+
+        assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request2.meta["_auth_proxy"] == "https://a.example"
+        assert request2.meta["proxy"] == "https://a.example"
+
+        response2 = self.get_response(request2, "https://example.com")
+        request3 = redirect_mw.process_response(request2, response2)
+
+        assert isinstance(request3, Request)
+        assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request3.meta["_auth_proxy"] == "https://a.example"
+        assert request3.meta["proxy"] == "https://a.example"
+
+        proxy_mw.process_request(request3)
+
+        assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request3.meta["_auth_proxy"] == "https://a.example"
+        assert request3.meta["proxy"] == "https://a.example"
+
+    def test_meta_proxy_https_relative(self):
+        crawler = get_crawler()
+        redirect_mw = self.mwcls.from_crawler(crawler)
+        proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
+
+        meta = {"proxy": "https://a:@a.example"}
+        request1 = Request("https://example.com", meta=meta)
+        proxy_mw.process_request(request1)
+
+        assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request1.meta["_auth_proxy"] == "https://a.example"
+        assert request1.meta["proxy"] == "https://a.example"
+
+        response1 = self.get_response(request1, "/a")
+        request2 = redirect_mw.process_response(request1, response1)
+
+        assert isinstance(request2, Request)
+        assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request2.meta["_auth_proxy"] == "https://a.example"
+        assert request2.meta["proxy"] == "https://a.example"
+
+        proxy_mw.process_request(request2)
+
+        assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request2.meta["_auth_proxy"] == "https://a.example"
+        assert request2.meta["proxy"] == "https://a.example"
+
+        response2 = self.get_response(request2, "/a")
+        request3 = redirect_mw.process_response(request2, response2)
+
+        assert isinstance(request3, Request)
+        assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request3.meta["_auth_proxy"] == "https://a.example"
+        assert request3.meta["proxy"] == "https://a.example"
+
+        proxy_mw.process_request(request3)
+
+        assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request3.meta["_auth_proxy"] == "https://a.example"
+        assert request3.meta["proxy"] == "https://a.example"
+
+    def test_meta_proxy_http_to_https(self):
+        crawler = get_crawler()
+        redirect_mw = self.mwcls.from_crawler(crawler)
+        proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
+
+        meta = {"proxy": "https://a:@a.example"}
+        request1 = Request("http://example.com", meta=meta)
+        proxy_mw.process_request(request1)
+
+        assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request1.meta["_auth_proxy"] == "https://a.example"
+        assert request1.meta["proxy"] == "https://a.example"
+
+        response1 = self.get_response(request1, "https://example.com")
+        request2 = redirect_mw.process_response(request1, response1)
+
+        assert isinstance(request2, Request)
+        assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request2.meta["_auth_proxy"] == "https://a.example"
+        assert request2.meta["proxy"] == "https://a.example"
+
+        proxy_mw.process_request(request2)
+
+        assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request2.meta["_auth_proxy"] == "https://a.example"
+        assert request2.meta["proxy"] == "https://a.example"
+
+        response2 = self.get_response(request2, "http://example.com")
+        request3 = redirect_mw.process_response(request2, response2)
+
+        assert isinstance(request3, Request)
+        assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request3.meta["_auth_proxy"] == "https://a.example"
+        assert request3.meta["proxy"] == "https://a.example"
+
+        proxy_mw.process_request(request3)
+
+        assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request3.meta["_auth_proxy"] == "https://a.example"
+        assert request3.meta["proxy"] == "https://a.example"
+
+    def test_meta_proxy_https_to_http(self):
+        crawler = get_crawler()
+        redirect_mw = self.mwcls.from_crawler(crawler)
+        proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
+
+        meta = {"proxy": "https://a:@a.example"}
+        request1 = Request("https://example.com", meta=meta)
+        proxy_mw.process_request(request1)
+
+        assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request1.meta["_auth_proxy"] == "https://a.example"
+        assert request1.meta["proxy"] == "https://a.example"
+
+        response1 = self.get_response(request1, "http://example.com")
+        request2 = redirect_mw.process_response(request1, response1)
+
+        assert isinstance(request2, Request)
+        assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request2.meta["_auth_proxy"] == "https://a.example"
+        assert request2.meta["proxy"] == "https://a.example"
+
+        proxy_mw.process_request(request2)
+
+        assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request2.meta["_auth_proxy"] == "https://a.example"
+        assert request2.meta["proxy"] == "https://a.example"
+
+        response2 = self.get_response(request2, "https://example.com")
+        request3 = redirect_mw.process_response(request2, response2)
+
+        assert isinstance(request3, Request)
+        assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request3.meta["_auth_proxy"] == "https://a.example"
+        assert request3.meta["proxy"] == "https://a.example"
+
+        proxy_mw.process_request(request3)
+
+        assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request3.meta["_auth_proxy"] == "https://a.example"
+        assert request3.meta["proxy"] == "https://a.example"
+
+    def test_system_proxy_http_absolute(self):
+        crawler = get_crawler()
+        redirect_mw = self.mwcls.from_crawler(crawler)
+        env = {
+            "http_proxy": "https://a:@a.example",
+        }
+        with set_environ(**env):
+            proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
+
+        request1 = Request("http://example.com")
+        proxy_mw.process_request(request1)
+
+        assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request1.meta["_auth_proxy"] == "https://a.example"
+        assert request1.meta["proxy"] == "https://a.example"
+
+        response1 = self.get_response(request1, "http://example.com")
+        request2 = redirect_mw.process_response(request1, response1)
+
+        assert isinstance(request2, Request)
+        assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request2.meta["_auth_proxy"] == "https://a.example"
+        assert request2.meta["proxy"] == "https://a.example"
+
+        proxy_mw.process_request(request2)
+
+        assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request2.meta["_auth_proxy"] == "https://a.example"
+        assert request2.meta["proxy"] == "https://a.example"
+
+        response2 = self.get_response(request2, "http://example.com")
+        request3 = redirect_mw.process_response(request2, response2)
+
+        assert isinstance(request3, Request)
+        assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request3.meta["_auth_proxy"] == "https://a.example"
+        assert request3.meta["proxy"] == "https://a.example"
+
+        proxy_mw.process_request(request3)
+
+        assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request3.meta["_auth_proxy"] == "https://a.example"
+        assert request3.meta["proxy"] == "https://a.example"
+
+    def test_system_proxy_http_relative(self):
+        crawler = get_crawler()
+        redirect_mw = self.mwcls.from_crawler(crawler)
+        env = {
+            "http_proxy": "https://a:@a.example",
+        }
+        with set_environ(**env):
+            proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
+
+        request1 = Request("http://example.com")
+        proxy_mw.process_request(request1)
+
+        assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request1.meta["_auth_proxy"] == "https://a.example"
+        assert request1.meta["proxy"] == "https://a.example"
+
+        response1 = self.get_response(request1, "/a")
+        request2 = redirect_mw.process_response(request1, response1)
+
+        assert isinstance(request2, Request)
+        assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request2.meta["_auth_proxy"] == "https://a.example"
+        assert request2.meta["proxy"] == "https://a.example"
+
+        proxy_mw.process_request(request2)
+
+        assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request2.meta["_auth_proxy"] == "https://a.example"
+        assert request2.meta["proxy"] == "https://a.example"
+
+        response2 = self.get_response(request2, "/a")
+        request3 = redirect_mw.process_response(request2, response2)
+
+        assert isinstance(request3, Request)
+        assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request3.meta["_auth_proxy"] == "https://a.example"
+        assert request3.meta["proxy"] == "https://a.example"
+
+        proxy_mw.process_request(request3)
+
+        assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request3.meta["_auth_proxy"] == "https://a.example"
+        assert request3.meta["proxy"] == "https://a.example"
+
+    def test_system_proxy_https_absolute(self):
+        crawler = get_crawler()
+        redirect_mw = self.mwcls.from_crawler(crawler)
+        env = {
+            "https_proxy": "https://a:@a.example",
+        }
+        with set_environ(**env):
+            proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
+
+        request1 = Request("https://example.com")
+        proxy_mw.process_request(request1)
+
+        assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request1.meta["_auth_proxy"] == "https://a.example"
+        assert request1.meta["proxy"] == "https://a.example"
+
+        response1 = self.get_response(request1, "https://example.com")
+        request2 = redirect_mw.process_response(request1, response1)
+
+        assert isinstance(request2, Request)
+        assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request2.meta["_auth_proxy"] == "https://a.example"
+        assert request2.meta["proxy"] == "https://a.example"
+
+        proxy_mw.process_request(request2)
+
+        assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request2.meta["_auth_proxy"] == "https://a.example"
+        assert request2.meta["proxy"] == "https://a.example"
+
+        response2 = self.get_response(request2, "https://example.com")
+        request3 = redirect_mw.process_response(request2, response2)
+
+        assert isinstance(request3, Request)
+        assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request3.meta["_auth_proxy"] == "https://a.example"
+        assert request3.meta["proxy"] == "https://a.example"
+
+        proxy_mw.process_request(request3)
+
+        assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request3.meta["_auth_proxy"] == "https://a.example"
+        assert request3.meta["proxy"] == "https://a.example"
+
+    def test_system_proxy_https_relative(self):
+        crawler = get_crawler()
+        redirect_mw = self.mwcls.from_crawler(crawler)
+        env = {
+            "https_proxy": "https://a:@a.example",
+        }
+        with set_environ(**env):
+            proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
+
+        request1 = Request("https://example.com")
+        proxy_mw.process_request(request1)
+
+        assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request1.meta["_auth_proxy"] == "https://a.example"
+        assert request1.meta["proxy"] == "https://a.example"
+
+        response1 = self.get_response(request1, "/a")
+        request2 = redirect_mw.process_response(request1, response1)
+
+        assert isinstance(request2, Request)
+        assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request2.meta["_auth_proxy"] == "https://a.example"
+        assert request2.meta["proxy"] == "https://a.example"
+
+        proxy_mw.process_request(request2)
+
+        assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request2.meta["_auth_proxy"] == "https://a.example"
+        assert request2.meta["proxy"] == "https://a.example"
+
+        response2 = self.get_response(request2, "/a")
+        request3 = redirect_mw.process_response(request2, response2)
+
+        assert isinstance(request3, Request)
+        assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request3.meta["_auth_proxy"] == "https://a.example"
+        assert request3.meta["proxy"] == "https://a.example"
+
+        proxy_mw.process_request(request3)
+
+        assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request3.meta["_auth_proxy"] == "https://a.example"
+        assert request3.meta["proxy"] == "https://a.example"
+
+    def test_system_proxy_proxied_http_to_proxied_https(self):
+        crawler = get_crawler()
+        redirect_mw = self.mwcls.from_crawler(crawler)
+        env = {
+            "http_proxy": "https://a:@a.example",
+            "https_proxy": "https://b:@b.example",
+        }
+        with set_environ(**env):
+            proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
+
+        request1 = Request("http://example.com")
+        proxy_mw.process_request(request1)
+
+        assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request1.meta["_auth_proxy"] == "https://a.example"
+        assert request1.meta["proxy"] == "https://a.example"
+
+        response1 = self.get_response(request1, "https://example.com")
+        request2 = redirect_mw.process_response(request1, response1)
+
+        assert isinstance(request2, Request)
+        assert "Proxy-Authorization" not in request2.headers
+        assert "_auth_proxy" not in request2.meta
+        assert "proxy" not in request2.meta
+
+        proxy_mw.process_request(request2)
+
+        assert request2.headers["Proxy-Authorization"] == b"Basic Yjo="
+        assert request2.meta["_auth_proxy"] == "https://b.example"
+        assert request2.meta["proxy"] == "https://b.example"
+
+        response2 = self.get_response(request2, "http://example.com")
+        request3 = redirect_mw.process_response(request2, response2)
+
+        assert isinstance(request3, Request)
+        assert "Proxy-Authorization" not in request3.headers
+        assert "_auth_proxy" not in request3.meta
+        assert "proxy" not in request3.meta
+
+        proxy_mw.process_request(request3)
+
+        assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request3.meta["_auth_proxy"] == "https://a.example"
+        assert request3.meta["proxy"] == "https://a.example"
+
+    def test_system_proxy_proxied_http_to_unproxied_https(self):
+        crawler = get_crawler()
+        redirect_mw = self.mwcls.from_crawler(crawler)
+        env = {
+            "http_proxy": "https://a:@a.example",
+        }
+        with set_environ(**env):
+            proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
+
+        request1 = Request("http://example.com")
+        proxy_mw.process_request(request1)
+
+        assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request1.meta["_auth_proxy"] == "https://a.example"
+        assert request1.meta["proxy"] == "https://a.example"
+
+        response1 = self.get_response(request1, "https://example.com")
+        request2 = redirect_mw.process_response(request1, response1)
+
+        assert isinstance(request2, Request)
+        assert "Proxy-Authorization" not in request2.headers
+        assert "_auth_proxy" not in request2.meta
+        assert "proxy" not in request2.meta
+
+        proxy_mw.process_request(request2)
+
+        assert "Proxy-Authorization" not in request2.headers
+        assert "_auth_proxy" not in request2.meta
+        assert "proxy" not in request2.meta
+
+        response2 = self.get_response(request2, "http://example.com")
+        request3 = redirect_mw.process_response(request2, response2)
+
+        assert isinstance(request3, Request)
+        assert "Proxy-Authorization" not in request3.headers
+        assert "_auth_proxy" not in request3.meta
+        assert "proxy" not in request3.meta
+
+        proxy_mw.process_request(request3)
+
+        assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request3.meta["_auth_proxy"] == "https://a.example"
+        assert request3.meta["proxy"] == "https://a.example"
+
+    def test_system_proxy_unproxied_http_to_proxied_https(self):
+        crawler = get_crawler()
+        redirect_mw = self.mwcls.from_crawler(crawler)
+        env = {
+            "https_proxy": "https://b:@b.example",
+        }
+        with set_environ(**env):
+            proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
+
+        request1 = Request("http://example.com")
+        proxy_mw.process_request(request1)
+
+        assert "Proxy-Authorization" not in request1.headers
+        assert "_auth_proxy" not in request1.meta
+        assert "proxy" not in request1.meta
+
+        response1 = self.get_response(request1, "https://example.com")
+        request2 = redirect_mw.process_response(request1, response1)
+
+        assert isinstance(request2, Request)
+        assert "Proxy-Authorization" not in request2.headers
+        assert "_auth_proxy" not in request2.meta
+        assert "proxy" not in request2.meta
+
+        proxy_mw.process_request(request2)
+
+        assert request2.headers["Proxy-Authorization"] == b"Basic Yjo="
+        assert request2.meta["_auth_proxy"] == "https://b.example"
+        assert request2.meta["proxy"] == "https://b.example"
+
+        response2 = self.get_response(request2, "http://example.com")
+        request3 = redirect_mw.process_response(request2, response2)
+
+        assert isinstance(request3, Request)
+        assert "Proxy-Authorization" not in request3.headers
+        assert "_auth_proxy" not in request3.meta
+        assert "proxy" not in request3.meta
+
+        proxy_mw.process_request(request3)
+
+        assert "Proxy-Authorization" not in request3.headers
+        assert "_auth_proxy" not in request3.meta
+        assert "proxy" not in request3.meta
+
+    def test_system_proxy_unproxied_http_to_unproxied_https(self):
+        crawler = get_crawler()
+        redirect_mw = self.mwcls.from_crawler(crawler)
+        proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
+
+        request1 = Request("http://example.com")
+        proxy_mw.process_request(request1)
+
+        assert "Proxy-Authorization" not in request1.headers
+        assert "_auth_proxy" not in request1.meta
+        assert "proxy" not in request1.meta
+
+        response1 = self.get_response(request1, "https://example.com")
+        request2 = redirect_mw.process_response(request1, response1)
+
+        assert isinstance(request2, Request)
+        assert "Proxy-Authorization" not in request2.headers
+        assert "_auth_proxy" not in request2.meta
+        assert "proxy" not in request2.meta
+
+        proxy_mw.process_request(request2)
+
+        assert "Proxy-Authorization" not in request2.headers
+        assert "_auth_proxy" not in request2.meta
+        assert "proxy" not in request2.meta
+
+        response2 = self.get_response(request2, "http://example.com")
+        request3 = redirect_mw.process_response(request2, response2)
+
+        assert isinstance(request3, Request)
+        assert "Proxy-Authorization" not in request3.headers
+        assert "_auth_proxy" not in request3.meta
+        assert "proxy" not in request3.meta
+
+        proxy_mw.process_request(request3)
+
+        assert "Proxy-Authorization" not in request3.headers
+        assert "_auth_proxy" not in request3.meta
+        assert "proxy" not in request3.meta
+
+    def test_system_proxy_proxied_https_to_proxied_http(self):
+        crawler = get_crawler()
+        redirect_mw = self.mwcls.from_crawler(crawler)
+        env = {
+            "http_proxy": "https://a:@a.example",
+            "https_proxy": "https://b:@b.example",
+        }
+        with set_environ(**env):
+            proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
+
+        request1 = Request("https://example.com")
+        proxy_mw.process_request(request1)
+
+        assert request1.headers["Proxy-Authorization"] == b"Basic Yjo="
+        assert request1.meta["_auth_proxy"] == "https://b.example"
+        assert request1.meta["proxy"] == "https://b.example"
+
+        response1 = self.get_response(request1, "http://example.com")
+        request2 = redirect_mw.process_response(request1, response1)
+
+        assert isinstance(request2, Request)
+        assert "Proxy-Authorization" not in request2.headers
+        assert "_auth_proxy" not in request2.meta
+        assert "proxy" not in request2.meta
+
+        proxy_mw.process_request(request2)
+
+        assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request2.meta["_auth_proxy"] == "https://a.example"
+        assert request2.meta["proxy"] == "https://a.example"
+
+        response2 = self.get_response(request2, "https://example.com")
+        request3 = redirect_mw.process_response(request2, response2)
+
+        assert isinstance(request3, Request)
+        assert "Proxy-Authorization" not in request3.headers
+        assert "_auth_proxy" not in request3.meta
+        assert "proxy" not in request3.meta
+
+        proxy_mw.process_request(request3)
+
+        assert request3.headers["Proxy-Authorization"] == b"Basic Yjo="
+        assert request3.meta["_auth_proxy"] == "https://b.example"
+        assert request3.meta["proxy"] == "https://b.example"
+
+    def test_system_proxy_proxied_https_to_unproxied_http(self):
+        crawler = get_crawler()
+        redirect_mw = self.mwcls.from_crawler(crawler)
+        env = {
+            "https_proxy": "https://b:@b.example",
+        }
+        with set_environ(**env):
+            proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
+
+        request1 = Request("https://example.com")
+        proxy_mw.process_request(request1)
+
+        assert request1.headers["Proxy-Authorization"] == b"Basic Yjo="
+        assert request1.meta["_auth_proxy"] == "https://b.example"
+        assert request1.meta["proxy"] == "https://b.example"
+
+        response1 = self.get_response(request1, "http://example.com")
+        request2 = redirect_mw.process_response(request1, response1)
+
+        assert isinstance(request2, Request)
+        assert "Proxy-Authorization" not in request2.headers
+        assert "_auth_proxy" not in request2.meta
+        assert "proxy" not in request2.meta
+
+        proxy_mw.process_request(request2)
+
+        assert "Proxy-Authorization" not in request2.headers
+        assert "_auth_proxy" not in request2.meta
+        assert "proxy" not in request2.meta
+
+        response2 = self.get_response(request2, "https://example.com")
+        request3 = redirect_mw.process_response(request2, response2)
+
+        assert isinstance(request3, Request)
+        assert "Proxy-Authorization" not in request3.headers
+        assert "_auth_proxy" not in request3.meta
+        assert "proxy" not in request3.meta
+
+        proxy_mw.process_request(request3)
+
+        assert request3.headers["Proxy-Authorization"] == b"Basic Yjo="
+        assert request3.meta["_auth_proxy"] == "https://b.example"
+        assert request3.meta["proxy"] == "https://b.example"
+
+    def test_system_proxy_unproxied_https_to_proxied_http(self):
+        crawler = get_crawler()
+        redirect_mw = self.mwcls.from_crawler(crawler)
+        env = {
+            "http_proxy": "https://a:@a.example",
+        }
+        with set_environ(**env):
+            proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
+
+        request1 = Request("https://example.com")
+        proxy_mw.process_request(request1)
+
+        assert "Proxy-Authorization" not in request1.headers
+        assert "_auth_proxy" not in request1.meta
+        assert "proxy" not in request1.meta
+
+        response1 = self.get_response(request1, "http://example.com")
+        request2 = redirect_mw.process_response(request1, response1)
+
+        assert isinstance(request2, Request)
+        assert "Proxy-Authorization" not in request2.headers
+        assert "_auth_proxy" not in request2.meta
+        assert "proxy" not in request2.meta
+
+        proxy_mw.process_request(request2)
+
+        assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request2.meta["_auth_proxy"] == "https://a.example"
+        assert request2.meta["proxy"] == "https://a.example"
+
+        response2 = self.get_response(request2, "https://example.com")
+        request3 = redirect_mw.process_response(request2, response2)
+
+        assert isinstance(request3, Request)
+        assert "Proxy-Authorization" not in request3.headers
+        assert "_auth_proxy" not in request3.meta
+        assert "proxy" not in request3.meta
+
+        proxy_mw.process_request(request3)
+
+        assert "Proxy-Authorization" not in request3.headers
+        assert "_auth_proxy" not in request3.meta
+        assert "proxy" not in request3.meta
+
+    def test_system_proxy_unproxied_https_to_unproxied_http(self):
+        crawler = get_crawler()
+        redirect_mw = self.mwcls.from_crawler(crawler)
+        proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
+
+        request1 = Request("https://example.com")
+        proxy_mw.process_request(request1)
+
+        assert "Proxy-Authorization" not in request1.headers
+        assert "_auth_proxy" not in request1.meta
+        assert "proxy" not in request1.meta
+
+        response1 = self.get_response(request1, "http://example.com")
+        request2 = redirect_mw.process_response(request1, response1)
+
+        assert isinstance(request2, Request)
+        assert "Proxy-Authorization" not in request2.headers
+        assert "_auth_proxy" not in request2.meta
+        assert "proxy" not in request2.meta
+
+        proxy_mw.process_request(request2)
+
+        assert "Proxy-Authorization" not in request2.headers
+        assert "_auth_proxy" not in request2.meta
+        assert "proxy" not in request2.meta
+
+        response2 = self.get_response(request2, "https://example.com")
+        request3 = redirect_mw.process_response(request2, response2)
+
+        assert isinstance(request3, Request)
+        assert "Proxy-Authorization" not in request3.headers
+        assert "_auth_proxy" not in request3.meta
+        assert "proxy" not in request3.meta
+
+        proxy_mw.process_request(request3)
+
+        assert "Proxy-Authorization" not in request3.headers
+        assert "_auth_proxy" not in request3.meta
+        assert "proxy" not in request3.meta

tests/utils/bases/spider.py

@@ -0,0 +1,131 @@
+from __future__ import annotations
+
+import logging
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING, Any
+from unittest import mock
+
+import pytest
+
+from scrapy import signals
+from scrapy.crawler import Crawler
+from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.settings import Settings
+from scrapy.utils.test import get_crawler, get_reactor_settings
+from tests.utils.decorators import inline_callbacks_test
+
+if TYPE_CHECKING:
+    from scrapy.spiders import Spider
+
+
+class TestSpiderBase(ABC):
+    @property
+    @abstractmethod
+    def spider_class(self) -> type[Spider]:
+        raise NotImplementedError
+
+    def test_base_spider(self):
+        spider = self.spider_class("example.com")
+        assert spider.name == "example.com"
+        assert spider.start_urls == []
+
+    def test_spider_args(self):
+        """``__init__`` method arguments are assigned to spider attributes"""
+        spider = self.spider_class("example.com", foo="bar")
+        assert spider.foo == "bar"
+
+    def test_spider_without_name(self):
+        """``__init__`` raises when the name is not provided."""
+        msg = "must have a name"
+        with pytest.raises(ValueError, match=msg):
+            self.spider_class()
+        with pytest.raises(ValueError, match=msg):
+            self.spider_class(somearg="foo")
+
+    def test_from_crawler_crawler_and_settings_population(self):
+        crawler = get_crawler()
+        spider = self.spider_class.from_crawler(crawler, "example.com")
+        assert hasattr(spider, "crawler")
+        assert spider.crawler is crawler
+        assert hasattr(spider, "settings")
+        assert spider.settings is crawler.settings
+
+    def test_from_crawler_init_call(self):
+        with mock.patch.object(
+            self.spider_class, "__init__", return_value=None
+        ) as mock_init:
+            self.spider_class.from_crawler(get_crawler(), "example.com", foo="bar")
+            mock_init.assert_called_once_with("example.com", foo="bar")
+
+    def test_closed_signal_call(self):
+        class TestSpider(self.spider_class):
+            closed_called = False
+
+            def closed(self, reason):
+                self.closed_called = True
+
+        crawler = get_crawler()
+        spider = TestSpider.from_crawler(crawler, "example.com")
+        crawler.signals.send_catch_log(signal=signals.spider_opened, spider=spider)
+        crawler.signals.send_catch_log(
+            signal=signals.spider_closed, spider=spider, reason=None
+        )
+        assert spider.closed_called
+
+    def test_update_settings(self):
+        spider_settings = {"TEST1": "spider", "TEST2": "spider"}
+        project_settings = {"TEST1": "project", "TEST3": "project"}
+        self.spider_class.custom_settings = spider_settings
+        settings = Settings(project_settings, priority="project")
+
+        self.spider_class.update_settings(settings)
+        assert settings.get("TEST1") == "spider"
+        assert settings.get("TEST2") == "spider"
+        assert settings.get("TEST3") == "project"
+
+    @inline_callbacks_test
+    def test_settings_in_from_crawler(self):
+        spider_settings = {"TEST1": "spider", "TEST2": "spider"}
+        project_settings = {
+            "TEST1": "project",
+            "TEST3": "project",
+            **get_reactor_settings(),
+        }
+
+        class TestSpider(self.spider_class):
+            name = "test"
+            custom_settings = spider_settings
+
+            @classmethod
+            def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any):
+                spider = super().from_crawler(crawler, *args, **kwargs)
+                spider.settings.set("TEST1", "spider_instance", priority="spider")
+                return spider
+
+        crawler = Crawler(TestSpider, project_settings)
+        assert crawler.settings.get("TEST1") == "spider"
+        assert crawler.settings.get("TEST2") == "spider"
+        assert crawler.settings.get("TEST3") == "project"
+        yield crawler.crawl()
+        assert crawler.settings.get("TEST1") == "spider_instance"
+
+    def test_logger(self, caplog: pytest.LogCaptureFixture) -> None:
+        spider = self.spider_class("example.com")
+        caplog.clear()
+        with caplog.at_level(logging.INFO):
+            spider.logger.info("test log msg")
+        assert caplog.record_tuples == [("example.com", logging.INFO, "test log msg")]
+
+        record = caplog.records[0]
+        assert getattr(record, "spider", None) is spider
+
+    def test_log(self):
+        spider = self.spider_class("example.com")
+        with (
+            mock.patch("scrapy.spiders.Spider.logger") as mock_logger,
+            pytest.warns(
+                ScrapyDeprecationWarning, match=r"Spider.log\(\) is deprecated"
+            ),
+        ):
+            spider.log("test log msg", "INFO")
+        mock_logger.log.assert_called_once_with("INFO", "test log msg")

tests/utils/downloader.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, NamedTuple, cast
+
+from scrapy.core.downloader import Downloader
+from scrapy.utils.httpobj import urlparse_cached
+
+if TYPE_CHECKING:
+    from scrapy.http import Request
+
+
+class MockSlot(NamedTuple):
+    active: list[Any]
+
+
+class MockDownloader:
+    def __init__(self) -> None:
+        self.slots: dict[str, MockSlot] = {}
+
+    def get_slot_key(self, request: Request) -> str:
+        if Downloader.DOWNLOAD_SLOT in request.meta:
+            return cast("str", request.meta[Downloader.DOWNLOAD_SLOT])
+
+        return urlparse_cached(request).hostname or ""
+
+    def increment(self, slot_key: str) -> None:
+        slot = self.slots.setdefault(slot_key, MockSlot(active=[]))
+        slot.active.append(1)
+
+    def decrement(self, slot_key: str) -> None:
+        slot = self.slots[slot_key]
+        slot.active.pop()
+
+    def close(self) -> None:
+        pass

tests/utils/engine.py

@@ -0,0 +1,184 @@
+from __future__ import annotations
+
+import re
+from collections import defaultdict
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any
+from urllib.parse import urlparse
+
+import attr
+from itemadapter import ItemAdapter
+from pydispatch import dispatcher
+from twisted.internet import defer
+
+from scrapy import signals
+from scrapy.http import Headers, Request, Response
+from scrapy.item import Field, Item
+from scrapy.linkextractors import LinkExtractor
+from scrapy.spiders import Spider
+from scrapy.utils.defer import maybe_deferred_to_future
+from scrapy.utils.signal import disconnect_all
+from scrapy.utils.test import get_crawler
+
+if TYPE_CHECKING:
+    from twisted.python.failure import Failure
+
+    from tests.mockserver.http import MockServer
+
+
+class MyItem(Item):
+    name = Field()
+    url = Field()
+    price = Field()
+
+
+@attr.s
+class AttrsItem:
+    name = attr.ib(default="")
+    url = attr.ib(default="")
+    price = attr.ib(default=0)
+
+
+@dataclass
+class DataClassItem:
+    name: str = ""
+    url: str = ""
+    price: int = 0
+
+
+class MySpider(Spider):
+    name = "scrapytest.org"
+
+    itemurl_re = re.compile(r"item\d+.html")
+    name_re = re.compile(r"<h1>(.*?)</h1>", re.MULTILINE)
+    price_re = re.compile(r">Price: \$(.*?)<", re.MULTILINE)
+
+    item_cls: type = MyItem
+
+    def parse(self, response):
+        xlink = LinkExtractor()
+        itemre = re.compile(self.itemurl_re)
+        for link in xlink.extract_links(response):
+            if itemre.search(link.url):
+                yield Request(url=link.url, callback=self.parse_item)
+
+    def parse_item(self, response):
+        adapter = ItemAdapter(self.item_cls())
+        m = self.name_re.search(response.text)
+        if m:
+            adapter["name"] = m.group(1)
+        adapter["url"] = response.url
+        m = self.price_re.search(response.text)
+        if m:
+            adapter["price"] = m.group(1)
+        return adapter.item
+
+
+class DictItemsSpider(MySpider):
+    item_cls = dict
+
+
+class AttrsItemsSpider(MySpider):
+    item_cls = AttrsItem
+
+
+class DataClassItemsSpider(MySpider):
+    item_cls = DataClassItem
+
+
+class CrawlerRun:
+    """A class to run the crawler and keep track of events occurred"""
+
+    def __init__(self, spider_class: type[Spider]):
+        self.respplug: list[tuple[Response, Spider]] = []
+        self.reqplug: list[tuple[Request, Spider]] = []
+        self.reqdropped: list[tuple[Request, Spider]] = []
+        self.reqreached: list[tuple[Request, Spider]] = []
+        self.itemerror: list[tuple[Any, Response, Spider, Failure]] = []
+        self.itemresp: list[tuple[Any, Response]] = []
+        self.headers: dict[Request, Headers] = {}
+        self.bytes: defaultdict[Request, list[bytes]] = defaultdict(list)
+        self.signals_caught: dict[Any, dict[str, Any]] = {}
+        self.spider_class = spider_class
+
+    async def run(self, mockserver: MockServer) -> None:
+        self.mockserver = mockserver
+
+        start_urls = [
+            self.geturl("/static/"),
+            self.geturl("/redirect"),
+            self.geturl("/redirect"),  # duplicate
+            self.geturl("/numbers"),
+        ]
+
+        for name, signal in vars(signals).items():
+            if not name.startswith("_"):
+                dispatcher.connect(self.record_signal, signal)
+
+        self.crawler = get_crawler(self.spider_class)
+        self.crawler.signals.connect(self.item_scraped, signals.item_scraped)
+        self.crawler.signals.connect(self.item_error, signals.item_error)
+        self.crawler.signals.connect(self.headers_received, signals.headers_received)
+        self.crawler.signals.connect(self.bytes_received, signals.bytes_received)
+        self.crawler.signals.connect(self.request_scheduled, signals.request_scheduled)
+        self.crawler.signals.connect(self.request_dropped, signals.request_dropped)
+        self.crawler.signals.connect(
+            self.request_reached, signals.request_reached_downloader
+        )
+        self.crawler.signals.connect(
+            self.response_downloaded, signals.response_downloaded
+        )
+        self.crawler.crawl(start_urls=start_urls)
+
+        self.deferred: defer.Deferred[None] = defer.Deferred()
+        dispatcher.connect(self.stop, signals.engine_stopped)
+        await maybe_deferred_to_future(self.deferred)
+
+    async def stop(self):
+        for name, signal in vars(signals).items():
+            if not name.startswith("_"):
+                disconnect_all(signal)
+        self.deferred.callback(None)
+        await self.crawler.stop_async()
+
+    def geturl(self, path: str) -> str:
+        return self.mockserver.url(path)
+
+    def getpath(self, url: str) -> str:
+        u = urlparse(url)
+        return u.path
+
+    def item_error(
+        self, item: Any, response: Response, spider: Spider, failure: Failure
+    ) -> None:
+        self.itemerror.append((item, response, spider, failure))
+
+    def item_scraped(self, item: Any, spider: Spider, response: Response) -> None:
+        self.itemresp.append((item, response))
+
+    def headers_received(
+        self, headers: Headers, body_length: int, request: Request, spider: Spider
+    ) -> None:
+        self.headers[request] = headers
+
+    def bytes_received(self, data: bytes, request: Request, spider: Spider) -> None:
+        self.bytes[request].append(data)
+
+    def request_scheduled(self, request: Request, spider: Spider) -> None:
+        self.reqplug.append((request, spider))
+
+    def request_reached(self, request: Request, spider: Spider) -> None:
+        self.reqreached.append((request, spider))
+
+    def request_dropped(self, request: Request, spider: Spider) -> None:
+        self.reqdropped.append((request, spider))
+
+    def response_downloaded(self, response: Response, spider: Spider) -> None:
+        self.respplug.append((response, spider))
+
+    def record_signal(self, *args: Any, **kwargs: Any) -> None:
+        """Record a signal and its parameters"""
+        signalargs = kwargs.copy()
+        sig = signalargs.pop("signal")
+        signalargs.pop("sender", None)
+        self.signals_caught[sig] = signalargs

tests/utils/media_pipelines.py

@@ -0,0 +1,15 @@
+from __future__ import annotations
+
+from typing import Any
+
+from scrapy.http.request import NO_CALLBACK, Request
+
+
+async def mocked_download_func(request: Request) -> Any:
+    assert request.callback is NO_CALLBACK
+    response = request.meta.get("response")
+    if callable(response):
+        response = await response()
+    if isinstance(response, Exception):
+        raise response
+    return response

tox.ini

@@ -44,7 +44,6 @@ deps =
     pytest-cov >= 7.0.0
     pytest-xdist
     sybil >= 1.3.0  # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422
-    testfixtures
     pytest-twisted >= 1.14.3
 
 [testenv]