Merge remote-tracking branch 'scrapy/master' into integrate-mime

docs/contributing.rst

@@ -178,7 +178,7 @@ Scrapy:
 * We use `black <https://black.readthedocs.io/en/stable/>`_ for code formatting.
   There is a hook in the pre-commit config
   that will automatically format your code before every commit. You can also
-  run black manually with ``tox -e black``.
+  run black manually with ``tox -e pre-commit``.
 
 * Don't put your name in the code you contribute; git provides enough
   metadata to identify author of the code.

docs/faq.rst

@@ -405,6 +405,23 @@ or :class:`~scrapy.signals.headers_received` signals and raising a
 :ref:`topics-stop-response-download` topic for additional information and examples.
 
 
+.. _faq-blank-request:
+
+How can I make a blank request?
+-------------------------------
+
+.. code-block:: python
+    
+    from scrapy import Request
+
+
+    blank_request = Request("data:,")
+
+In this case, the URL is set to a data URI scheme. Data URLs allow you to include data 
+in-line in web pages as if they were external resources. The "data:" scheme with an empty 
+content (",") essentially creates a request to a data URL without any specific content.
+
+
 Running ``runspider`` I get ``error: No spider found in file: <filename>``
 --------------------------------------------------------------------------
 

docs/news.rst

@@ -62,6 +62,9 @@ Deprecation removals
     1.0.0, use :attr:`CrawlerRunner.spider_loader
     <scrapy.crawler.CrawlerRunner.spider_loader>` instead. (:issue:`6010`)
 
+-   The :func:`scrapy.utils.response.response_httprepr` function, deprecated in
+    Scrapy 2.6.0, has now been removed. (:issue:`6111`)
+
 Deprecations
 ~~~~~~~~~~~~
 
@@ -1157,6 +1160,9 @@ Deprecations
         Instead, call :meth:`~scrapy.core.engine.ExecutionEngine.open_spider`
         first to set the :class:`~scrapy.Spider` object.
 
+-   :func:`scrapy.utils.response.response_httprepr` is now deprecated.
+    (:issue:`4972`)
+
 
 New features
 ~~~~~~~~~~~~

docs/requirements.txt

@@ -1,4 +1,4 @@
-sphinx==5.0.2
+sphinx==6.2.1
-sphinx-hoverxref==1.1.1
+sphinx-hoverxref==1.3.0
-sphinx-notfound-page==0.8
+sphinx-notfound-page==1.0.0
-sphinx-rtd-theme==1.0.0
+sphinx-rtd-theme==2.0.0

docs/topics/request-response.rst

@@ -469,60 +469,6 @@ import path.
 
 .. autoclass:: scrapy.utils.request.RequestFingerprinter
 
-
-.. setting:: REQUEST_FINGERPRINTER_IMPLEMENTATION
-
-REQUEST_FINGERPRINTER_IMPLEMENTATION
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. versionadded:: 2.7
-
-Default: ``'2.6'``
-
-Determines which request fingerprinting algorithm is used by the default
-request fingerprinter class (see :setting:`REQUEST_FINGERPRINTER_CLASS`).
-
-Possible values are:
-
--   ``'2.6'`` (default)
-
-    This implementation uses the same request fingerprinting algorithm as
-    Scrapy 2.6 and earlier versions.
-
-    Even though this is the default value for backward compatibility reasons,
-    it is a deprecated value.
-
--   ``'2.7'``
-
-    This implementation was introduced in Scrapy 2.7 to fix an issue of the
-    previous implementation.
-
-    New projects should use this value. The :command:`startproject` command
-    sets this value in the generated ``settings.py`` file.
-
-If you are using the default value (``'2.6'``) for this setting, and you are
-using Scrapy components where changing the request fingerprinting algorithm
-would cause undesired results, you need to carefully decide when to change the
-value of this setting, or switch the :setting:`REQUEST_FINGERPRINTER_CLASS`
-setting to a custom request fingerprinter class that implements the 2.6 request
-fingerprinting algorithm and does not log this warning (
-:ref:`2.6-request-fingerprinter` includes an example implementation of such a
-class).
-
-Scenarios where changing the request fingerprinting algorithm may cause
-undesired results include, for example, using the HTTP cache middleware (see
-:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`).
-Changing the request fingerprinting algorithm would invalidate the current
-cache, requiring you to redownload all requests again.
-
-Otherwise, set :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` to ``'2.7'`` in
-your settings to switch already to the request fingerprinting implementation
-that will be the only request fingerprinting implementation available in a
-future version of Scrapy, and remove the deprecation warning triggered by using
-the default value (``'2.6'``).
-
-
-.. _2.6-request-fingerprinter:
 .. _custom-request-fingerprinter:
 
 Writing your own request fingerprinter

scrapy/commands/parse.py

@@ -1,3 +1,4 @@
+import functools
 import inspect
 import json
 import logging
@@ -251,39 +252,40 @@ class Command(BaseRunSpiderCommand):
 
         return scraped_data
 
+    def _get_callback(self, *, spider, opts, response=None):
+        cb = None
+        if response:
+            cb = response.meta["_callback"]
+        if not cb:
+            if opts.callback:
+                cb = opts.callback
+            elif response and opts.rules and self.first_response == response:
+                cb = self.get_callback_from_rules(spider, response)
+                if not cb:
+                    raise ValueError(
+                        f"Cannot find a rule that matches {response.url!r} in spider: "
+                        f"{spider.name}"
+                    )
+            else:
+                cb = "parse"
+
+        if not callable(cb):
+            cb_method = getattr(spider, cb, None)
+            if callable(cb_method):
+                cb = cb_method
+            else:
+                raise ValueError(
+                    f"Cannot find callback {cb!r} in spider: {spider.name}"
+                )
+        return cb
+
     def prepare_request(self, spider, request, opts):
         def callback(response, **cb_kwargs):
             # memorize first request
             if not self.first_response:
                 self.first_response = response
 
-            # determine real callback
+            cb = self._get_callback(spider=spider, opts=opts, response=response)
-            cb = response.meta["_callback"]
-            if not cb:
-                if opts.callback:
-                    cb = opts.callback
-                elif opts.rules and self.first_response == response:
-                    cb = self.get_callback_from_rules(spider, response)
-
-                    if not cb:
-                        logger.error(
-                            "Cannot find a rule that matches %(url)r in spider: %(spider)s",
-                            {"url": response.url, "spider": spider.name},
-                        )
-                        return
-                else:
-                    cb = "parse"
-
-            if not callable(cb):
-                cb_method = getattr(spider, cb, None)
-                if callable(cb_method):
-                    cb = cb_method
-                else:
-                    logger.error(
-                        "Cannot find callback %(callback)r in spider: %(spider)s",
-                        {"callback": cb, "spider": spider.name},
-                    )
-                    return
 
             # parse items and requests
             depth = response.meta["_depth"]
@@ -303,6 +305,9 @@ class Command(BaseRunSpiderCommand):
 
         request.meta["_depth"] = 1
         request.meta["_callback"] = request.callback
+        if not request.callback and not opts.rules:
+            cb = self._get_callback(spider=spider, opts=opts)
+            functools.update_wrapper(callback, cb)
         request.callback = callback
         return request
 

scrapy/core/spidermw.py

@@ -103,8 +103,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
     ) -> Union[Generator, AsyncGenerator]:
         def process_sync(iterable: Iterable) -> Generator:
             try:
-                for r in iterable:
+                yield from iterable
-                    yield r
             except Exception as ex:
                 exception_result = self._process_spider_exception(
                     response, spider, Failure(ex), exception_processor_index

scrapy/crawler.py

@@ -178,6 +178,48 @@ class Crawler:
             assert self.engine
             yield maybeDeferred(self.engine.stop)
 
+    @staticmethod
+    def _get_component(component_class, components):
+        for component in components:
+            if isinstance(component, component_class):
+                return component
+        return None
+
+    def get_addon(self, cls):
+        return self._get_component(cls, self.addons.addons)
+
+    def get_downloader_middleware(self, cls):
+        if not self.engine:
+            raise RuntimeError(
+                "Crawler.get_downloader_middleware() can only be called after "
+                "the crawl engine has been created."
+            )
+        return self._get_component(cls, self.engine.downloader.middleware.middlewares)
+
+    def get_extension(self, cls):
+        if not self.extensions:
+            raise RuntimeError(
+                "Crawler.get_extension() can only be called after the "
+                "extension manager has been created."
+            )
+        return self._get_component(cls, self.extensions.middlewares)
+
+    def get_item_pipeline(self, cls):
+        if not self.engine:
+            raise RuntimeError(
+                "Crawler.get_item_pipeline() can only be called after the "
+                "crawl engine has been created."
+            )
+        return self._get_component(cls, self.engine.scraper.itemproc.middlewares)
+
+    def get_spider_middleware(self, cls):
+        if not self.engine:
+            raise RuntimeError(
+                "Crawler.get_spider_middleware() can only be called after the "
+                "crawl engine has been created."
+            )
+        return self._get_component(cls, self.engine.scraper.spidermw.middlewares)
+
 
 class CrawlerRunner:
     """

scrapy/extensions/memusage.py

@@ -128,9 +128,9 @@ class MemoryUsage:
     def _send_report(self, rcpts, subject):
         """send notification mail with some additional useful info"""
         stats = self.crawler.stats
-        s = f"Memory usage at engine startup : {stats.get_value('memusage/startup')/1024/1024}M\r\n"
+        s = f"Memory usage at engine startup : {stats.get_value('memusage/startup') / 1024 / 1024}M\r\n"
-        s += f"Maximum memory usage          : {stats.get_value('memusage/max')/1024/1024}M\r\n"
+        s += f"Maximum memory usage          : {stats.get_value('memusage/max') / 1024 / 1024}M\r\n"
-        s += f"Current memory usage          : {self.get_virtual_size()/1024/1024}M\r\n"
+        s += f"Current memory usage          : {self.get_virtual_size() / 1024 / 1024}M\r\n"
 
         s += (
             "ENGINE STATUS ------------------------------------------------------- \r\n"

scrapy/pipelines/files.py

@@ -8,7 +8,6 @@ import functools
 import hashlib
 import logging
 import mimetypes
-import os
 import time
 from collections import defaultdict
 from contextlib import suppress
@@ -66,7 +65,7 @@ class FSFilesStore:
         absolute_path = self._get_filesystem_path(path)
         try:
             last_modified = absolute_path.stat().st_mtime
-        except os.error:
+        except OSError:
             return {}
 
         with absolute_path.open("rb") as f:
@@ -340,7 +339,9 @@ class FilesPipeline(MediaPipeline):
     DEFAULT_FILES_URLS_FIELD = "file_urls"
     DEFAULT_FILES_RESULT_FIELD = "files"
 
-    def __init__(self, store_uri, download_func=None, settings=None):
+    def __init__(
+        self, store_uri: Union[str, PathLike], download_func=None, settings=None
+    ):
         store_uri = _to_string(store_uri)
         if not store_uri:
             raise NotConfigured

scrapy/pipelines/images.py

@@ -8,7 +8,8 @@ import hashlib
 import warnings
 from contextlib import suppress
 from io import BytesIO
-from typing import Dict, Tuple
+from os import PathLike
+from typing import Dict, Tuple, Union
 
 from itemadapter import ItemAdapter
 
@@ -53,7 +54,9 @@ class ImagesPipeline(FilesPipeline):
     DEFAULT_IMAGES_URLS_FIELD = "image_urls"
     DEFAULT_IMAGES_RESULT_FIELD = "images"
 
-    def __init__(self, store_uri, download_func=None, settings=None):
+    def __init__(
+        self, store_uri: Union[str, PathLike], download_func=None, settings=None
+    ):
         try:
             from PIL import Image
 

scrapy/settings/default_settings.py

@@ -260,7 +260,7 @@ REFERER_ENABLED = True
 REFERRER_POLICY = "scrapy.spidermiddlewares.referer.DefaultReferrerPolicy"
 
 REQUEST_FINGERPRINTER_CLASS = "scrapy.utils.request.RequestFingerprinter"
-REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.6"
+REQUEST_FINGERPRINTER_IMPLEMENTATION = "SENTINEL"
 
 RETRY_ENABLED = True
 RETRY_TIMES = 2  # initial response + 2 retries = 3 requests

scrapy/spiders/crawl.py

@@ -131,8 +131,7 @@ class CrawlSpider(Spider):
     def _handle_failure(self, failure, errback):
         if errback:
             results = errback(failure) or ()
-            for request_or_item in iterate_spider_output(results):
+            yield from iterate_spider_output(results)
-                yield request_or_item
 
     def _compile_rules(self):
         self._rules = []

scrapy/spiders/feed.py

@@ -58,8 +58,7 @@ class XMLFeedSpider(Spider):
 
         for selector in nodes:
             ret = iterate_spider_output(self.parse_node(response, selector))
-            for result_item in self.process_results(response, ret):
+            yield from self.process_results(response, ret)
-                yield result_item
 
     def _parse(self, response, **kwargs):
         if not hasattr(self, "parse_node"):
@@ -133,8 +132,7 @@ class CSVFeedSpider(Spider):
             response, self.delimiter, self.headers, quotechar=self.quotechar
         ):
             ret = iterate_spider_output(self.parse_row(response, row))
-            for result_item in self.process_results(response, ret):
+            yield from self.process_results(response, ret)
-                yield result_item
 
     def _parse(self, response, **kwargs):
         if not hasattr(self, "parse_row"):

scrapy/spiders/sitemap.py

@@ -33,8 +33,7 @@ class SitemapSpider(Spider):
         attributes, for example, you can filter locs with lastmod greater
         than a given date (see docs).
         """
-        for entry in entries:
+        yield from entries
-            yield entry
 
     def _parse_sitemap(self, response):
         if response.url.endswith("/robots.txt"):

scrapy/templates/project/module/settings.py.tmpl

@@ -88,6 +88,5 @@ ROBOTSTXT_OBEY = True
 #HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage"
 
 # Set settings whose default value is deprecated to a future-proof value
-REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.7"
 TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
 FEED_EXPORT_ENCODING = "utf-8"

scrapy/utils/python.py

@@ -59,8 +59,7 @@ def iflatten(x: Iterable) -> Iterable:
     Similar to ``.flatten()``, but returns iterator instead"""
     for el in x:
         if is_listlike(el):
-            for el_ in iflatten(el):
+            yield from iflatten(el)
-                yield el_
         else:
             yield el
 

scrapy/utils/request.py

@@ -34,9 +34,6 @@ from scrapy.utils.python import to_bytes, to_unicode
 if TYPE_CHECKING:
     from scrapy.crawler import Crawler
 
-_deprecated_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]"
-_deprecated_fingerprint_cache = WeakKeyDictionary()
-
 
 def _serialize_headers(
     headers: Iterable[bytes], request: Request
@@ -44,122 +41,7 @@ def _serialize_headers(
     for header in headers:
         if header in request.headers:
             yield header
-            for value in request.headers.getlist(header):
+            yield from request.headers.getlist(header)
-                yield value
-
-
-def request_fingerprint(
-    request: Request,
-    include_headers: Optional[Iterable[Union[bytes, str]]] = None,
-    keep_fragments: bool = False,
-) -> str:
-    """
-    Return the request fingerprint as an hexadecimal string.
-
-    The request fingerprint is a hash that uniquely identifies the resource the
-    request points to. For example, take the following two urls:
-
-    http://www.example.com/query?id=111&cat=222
-    http://www.example.com/query?cat=222&id=111
-
-    Even though those are two different URLs both point to the same resource
-    and are equivalent (i.e. they should return the same response).
-
-    Another example are cookies used to store session ids. Suppose the
-    following page is only accessible to authenticated users:
-
-    http://www.example.com/members/offers.html
-
-    Lots of sites use a cookie to store the session id, which adds a random
-    component to the HTTP Request and thus should be ignored when calculating
-    the fingerprint.
-
-    For this reason, request headers are ignored by default when calculating
-    the fingerprint. If you want to include specific headers use the
-    include_headers argument, which is a list of Request headers to include.
-
-    Also, servers usually ignore fragments in urls when handling requests,
-    so they are also ignored by default when calculating the fingerprint.
-    If you want to include them, set the keep_fragments argument to True
-    (for instance when handling requests with a headless browser).
-    """
-    if include_headers or keep_fragments:
-        message = (
-            "Call to deprecated function "
-            "scrapy.utils.request.request_fingerprint().\n"
-            "\n"
-            "If you are using this function in a Scrapy component because you "
-            "need a non-default fingerprinting algorithm, and you are OK "
-            "with that non-default fingerprinting algorithm being used by "
-            "all Scrapy components and not just the one calling this "
-            "function, use crawler.request_fingerprinter.fingerprint() "
-            "instead in your Scrapy component (you can get the crawler "
-            "object from the 'from_crawler' class method), and use the "
-            "'REQUEST_FINGERPRINTER_CLASS' setting to configure your "
-            "non-default fingerprinting algorithm.\n"
-            "\n"
-            "Otherwise, consider using the "
-            "scrapy.utils.request.fingerprint() function instead.\n"
-            "\n"
-            "If you switch to 'fingerprint()', or assign the "
-            "'REQUEST_FINGERPRINTER_CLASS' setting a class that uses "
-            "'fingerprint()', the generated fingerprints will not only be "
-            "bytes instead of a string, but they will also be different from "
-            "those generated by 'request_fingerprint()'. Before you switch, "
-            "make sure that you understand the consequences of this (e.g. "
-            "cache invalidation) and are OK with them; otherwise, consider "
-            "implementing your own function which returns the same "
-            "fingerprints as the deprecated 'request_fingerprint()' function."
-        )
-    else:
-        message = (
-            "Call to deprecated function "
-            "scrapy.utils.request.request_fingerprint().\n"
-            "\n"
-            "If you are using this function in a Scrapy component, and you "
-            "are OK with users of your component changing the fingerprinting "
-            "algorithm through settings, use "
-            "crawler.request_fingerprinter.fingerprint() instead in your "
-            "Scrapy component (you can get the crawler object from the "
-            "'from_crawler' class method).\n"
-            "\n"
-            "Otherwise, consider using the "
-            "scrapy.utils.request.fingerprint() function instead.\n"
-            "\n"
-            "Either way, the resulting fingerprints will be returned as "
-            "bytes, not as a string, and they will also be different from "
-            "those generated by 'request_fingerprint()'. Before you switch, "
-            "make sure that you understand the consequences of this (e.g. "
-            "cache invalidation) and are OK with them; otherwise, consider "
-            "implementing your own function which returns the same "
-            "fingerprints as the deprecated 'request_fingerprint()' function."
-        )
-    warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2)
-    processed_include_headers: Optional[Tuple[bytes, ...]] = None
-    if include_headers:
-        processed_include_headers = tuple(
-            to_bytes(h.lower()) for h in sorted(include_headers)
-        )
-    cache = _deprecated_fingerprint_cache.setdefault(request, {})
-    cache_key = (processed_include_headers, keep_fragments)
-    if cache_key not in cache:
-        fp = hashlib.sha1()
-        fp.update(to_bytes(request.method))
-        fp.update(
-            to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments))
-        )
-        fp.update(request.body or b"")
-        if processed_include_headers:
-            for part in _serialize_headers(processed_include_headers, request):
-                fp.update(part)
-        cache[cache_key] = fp.hexdigest()
-    return cache[cache_key]
-
-
-def _request_fingerprint_as_bytes(*args: Any, **kwargs: Any) -> bytes:
-    with warnings.catch_warnings():
-        warnings.simplefilter("ignore")
-        return bytes.fromhex(request_fingerprint(*args, **kwargs))
 
 
 _fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]"
@@ -259,33 +141,15 @@ class RequestFingerprinter:
                 "REQUEST_FINGERPRINTER_IMPLEMENTATION"
             )
         else:
-            implementation = "2.6"
+            implementation = "SENTINEL"
-        if implementation == "2.6":
+
+        if implementation != "SENTINEL":
             message = (
-                "'2.6' is a deprecated value for the "
+                "'REQUEST_FINGERPRINTER_IMPLEMENTATION' is a deprecated setting.\n"
-                "'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting.\n"
+                "And it will be removed in future version of Scrapy."
-                "\n"
-                "It is also the default value. In other words, it is normal "
-                "to get this warning if you have not defined a value for the "
-                "'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting. This is so "
-                "for backward compatibility reasons, but it will change in a "
-                "future version of Scrapy.\n"
-                "\n"
-                "See the documentation of the "
-                "'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting for "
-                "information on how to handle this deprecation."
             )
             warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2)
-            self._fingerprint = _request_fingerprint_as_bytes
+        self._fingerprint = fingerprint
-        elif implementation == "2.7":
-            self._fingerprint = fingerprint
-        else:
-            raise ValueError(
-                f"Got an invalid value on setting "
-                f"'REQUEST_FINGERPRINTER_IMPLEMENTATION': "
-                f"{implementation!r}. Valid values are '2.6' (deprecated) "
-                f"and '2.7'."
-            )
 
     def fingerprint(self, request: Request) -> bytes:
         return self._fingerprint(request)

scrapy/utils/test.py

@@ -86,8 +86,6 @@ def get_crawler(
 
     # Set by default settings that prevent deprecation warnings.
     settings: Dict[str, Any] = {}
-    if prevent_warnings:
-        settings["REQUEST_FINGERPRINTER_IMPLEMENTATION"] = "2.7"
     settings.update(settings_dict or {})
     runner = CrawlerRunner(settings)
     crawler = runner.create_crawler(spidercls or TestSpider)

tests/spiders.py

@@ -301,8 +301,7 @@ class BrokenStartRequestsSpider(FollowAllSpider):
 
     def parse(self, response):
         self.seedsseen.append(response.meta.get("seed"))
-        for req in super().parse(response):
+        yield from super().parse(response)
-            yield req
 
 
 class SingleRequestSpider(MetaSpider):

tests/test_command_check.py

@@ -16,11 +16,11 @@ import scrapy
 
 class CheckSpider(scrapy.Spider):
     name = '{self.spider_name}'
-    start_urls = ['http://toscrape.com']
+    start_urls = ['data:,']
 
     def parse(self, response, **cb_kwargs):
         \"\"\"
-        @url http://toscrape.com
+        @url data:,
         {contracts}
         \"\"\"
         {parse_def}

tests/test_command_parse.py

@@ -78,9 +78,21 @@ class AsyncDefAsyncioGenExcSpider(scrapy.Spider):
             if i > 5:
                 raise ValueError("Stopping the processing")
 
+class CallbackSignatureDownloaderMiddleware:
+    def process_request(self, request, spider):
+        from inspect import signature
+        spider.logger.debug(f"request.callback signature: {{signature(request.callback)}}")
+
+
 class MySpider(scrapy.Spider):
     name = '{self.spider_name}'
 
+    custom_settings = {{
+        "DOWNLOADER_MIDDLEWARES": {{
+            CallbackSignatureDownloaderMiddleware: 0,
+        }}
+    }}
+
     def parse(self, response):
         if getattr(self, 'test_arg', None):
             self.logger.debug('It Works!')
@@ -220,7 +232,11 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
                 self.url("/html"),
             ]
         )
-        self.assertIn("DEBUG: It Works!", _textmode(stderr))
+        log = _textmode(stderr)
+        self.assertIn("DEBUG: It Works!", log)
+        self.assertIn(
+            "DEBUG: request.callback signature: (response, foo=None, key=None)", log
+        )
 
     @defer.inlineCallbacks
     def test_request_without_meta(self):

tests/test_crawler.py

@@ -12,12 +12,13 @@ import pytest
 from packaging.version import parse as parse_version
 from pexpect.popen_spawn import PopenSpawn
 from pytest import mark, raises
-from twisted.internet import defer
+from twisted.internet.defer import Deferred, inlineCallbacks
 from twisted.trial import unittest
 from w3lib import __version__ as w3lib_version
 from zope.interface.exceptions import MultipleInvalid
 
 import scrapy
+from scrapy import Spider
 from scrapy.crawler import Crawler, CrawlerProcess, CrawlerRunner
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.extensions import telnet
@@ -29,6 +30,19 @@ from scrapy.utils.spider import DefaultSpider
 from scrapy.utils.test import get_crawler
 from tests.mockserver import MockServer, get_mockserver_env
 
+# To prevent warnings.
+BASE_SETTINGS = {
+    "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
+}
+
+
+def get_raw_crawler(spidercls=None, settings_dict=None):
+    """get_crawler alternative that only calls the __init__ method of the
+    crawler."""
+    settings = Settings()
+    settings.setdict(settings_dict or {})
+    return Crawler(spidercls or DefaultSpider, settings)
+
 
 class BaseCrawlerTest(unittest.TestCase):
     def assertOptionIsDefault(self, settings, key):
@@ -39,7 +53,7 @@ class BaseCrawlerTest(unittest.TestCase):
 class CrawlerTestCase(BaseCrawlerTest):
     def test_populate_spidercls_settings(self):
         spider_settings = {"TEST1": "spider", "TEST2": "spider"}
-        project_settings = {"TEST1": "project", "TEST3": "project"}
+        project_settings = {**BASE_SETTINGS, "TEST1": "project", "TEST3": "project"}
 
         class CustomSettingsSpider(DefaultSpider):
             custom_settings = spider_settings
@@ -71,9 +85,9 @@ class CrawlerTestCase(BaseCrawlerTest):
         with raises(ValueError):
             Crawler(DefaultSpider())
 
-    @defer.inlineCallbacks
+    @inlineCallbacks
     def test_crawler_crawl_twice_deprecated(self):
-        crawler = Crawler(NoRequestsSpider)
+        crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS)
         yield crawler.crawl()
         with pytest.warns(
             ScrapyDeprecationWarning,
@@ -81,6 +95,358 @@ class CrawlerTestCase(BaseCrawlerTest):
         ):
             yield crawler.crawl()
 
+    def test_get_addon(self):
+        class ParentAddon:
+            pass
+
+        class TrackingAddon(ParentAddon):
+            instances = []
+
+            def __init__(self):
+                TrackingAddon.instances.append(self)
+
+            def update_settings(self, settings):
+                pass
+
+        settings = {
+            **BASE_SETTINGS,
+            "ADDONS": {
+                TrackingAddon: 0,
+            },
+        }
+        crawler = get_crawler(settings_dict=settings)
+        self.assertEqual(len(TrackingAddon.instances), 1)
+        expected = TrackingAddon.instances[-1]
+
+        addon = crawler.get_addon(TrackingAddon)
+        self.assertEqual(addon, expected)
+
+        addon = crawler.get_addon(DefaultSpider)
+        self.assertIsNone(addon)
+
+        addon = crawler.get_addon(ParentAddon)
+        self.assertEqual(addon, expected)
+
+        class ChildAddon(TrackingAddon):
+            pass
+
+        addon = crawler.get_addon(ChildAddon)
+        self.assertIsNone(addon)
+
+    @inlineCallbacks
+    def test_get_downloader_middleware(self):
+        class ParentDownloaderMiddleware:
+            pass
+
+        class TrackingDownloaderMiddleware(ParentDownloaderMiddleware):
+            instances = []
+
+            def __init__(self):
+                TrackingDownloaderMiddleware.instances.append(self)
+
+        class MySpider(Spider):
+            name = "myspider"
+
+            @classmethod
+            def from_crawler(cls, crawler):
+                return cls(crawler=crawler)
+
+            def __init__(self, crawler):
+                self.crawler = crawler
+
+            def start_requests(self):
+                MySpider.result = crawler.get_downloader_middleware(MySpider.cls)
+                return
+                yield
+
+        settings = {
+            **BASE_SETTINGS,
+            "DOWNLOADER_MIDDLEWARES": {
+                TrackingDownloaderMiddleware: 0,
+            },
+        }
+
+        crawler = get_raw_crawler(MySpider, settings)
+        MySpider.cls = TrackingDownloaderMiddleware
+        yield crawler.crawl()
+        self.assertEqual(len(TrackingDownloaderMiddleware.instances), 1)
+        self.assertEqual(MySpider.result, TrackingDownloaderMiddleware.instances[-1])
+
+        crawler = get_raw_crawler(MySpider, settings)
+        MySpider.cls = DefaultSpider
+        yield crawler.crawl()
+        self.assertIsNone(MySpider.result)
+
+        crawler = get_raw_crawler(MySpider, settings)
+        MySpider.cls = ParentDownloaderMiddleware
+        yield crawler.crawl()
+        self.assertEqual(MySpider.result, TrackingDownloaderMiddleware.instances[-1])
+
+        class ChildDownloaderMiddleware(TrackingDownloaderMiddleware):
+            pass
+
+        crawler = get_raw_crawler(MySpider, settings)
+        MySpider.cls = ChildDownloaderMiddleware
+        yield crawler.crawl()
+        self.assertIsNone(MySpider.result)
+
+    def test_get_downloader_middleware_not_crawling(self):
+        crawler = get_raw_crawler(settings_dict=BASE_SETTINGS)
+        self.assertRaises(
+            RuntimeError, crawler.get_downloader_middleware, DefaultSpider
+        )
+
+    @inlineCallbacks
+    def test_get_downloader_middleware_no_engine(self):
+        class MySpider(Spider):
+            name = "myspider"
+
+            @classmethod
+            def from_crawler(cls, crawler):
+                try:
+                    crawler.get_downloader_middleware(DefaultSpider)
+                except Exception as e:
+                    MySpider.result = e
+                    raise
+
+        crawler = get_raw_crawler(MySpider, BASE_SETTINGS)
+        with raises(RuntimeError):
+            yield crawler.crawl()
+
+    @inlineCallbacks
+    def test_get_extension(self):
+        class ParentExtension:
+            pass
+
+        class TrackingExtension(ParentExtension):
+            instances = []
+
+            def __init__(self):
+                TrackingExtension.instances.append(self)
+
+        class MySpider(Spider):
+            name = "myspider"
+
+            @classmethod
+            def from_crawler(cls, crawler):
+                return cls(crawler=crawler)
+
+            def __init__(self, crawler):
+                self.crawler = crawler
+
+            def start_requests(self):
+                MySpider.result = crawler.get_extension(MySpider.cls)
+                return
+                yield
+
+        settings = {
+            **BASE_SETTINGS,
+            "EXTENSIONS": {
+                TrackingExtension: 0,
+            },
+        }
+
+        crawler = get_raw_crawler(MySpider, settings)
+        MySpider.cls = TrackingExtension
+        yield crawler.crawl()
+        self.assertEqual(len(TrackingExtension.instances), 1)
+        self.assertEqual(MySpider.result, TrackingExtension.instances[-1])
+
+        crawler = get_raw_crawler(MySpider, settings)
+        MySpider.cls = DefaultSpider
+        yield crawler.crawl()
+        self.assertIsNone(MySpider.result)
+
+        crawler = get_raw_crawler(MySpider, settings)
+        MySpider.cls = ParentExtension
+        yield crawler.crawl()
+        self.assertEqual(MySpider.result, TrackingExtension.instances[-1])
+
+        class ChildExtension(TrackingExtension):
+            pass
+
+        crawler = get_raw_crawler(MySpider, settings)
+        MySpider.cls = ChildExtension
+        yield crawler.crawl()
+        self.assertIsNone(MySpider.result)
+
+    def test_get_extension_not_crawling(self):
+        crawler = get_raw_crawler(settings_dict=BASE_SETTINGS)
+        self.assertRaises(RuntimeError, crawler.get_extension, DefaultSpider)
+
+    @inlineCallbacks
+    def test_get_extension_no_engine(self):
+        class MySpider(Spider):
+            name = "myspider"
+
+            @classmethod
+            def from_crawler(cls, crawler):
+                try:
+                    crawler.get_extension(DefaultSpider)
+                except Exception as e:
+                    MySpider.result = e
+                    raise
+
+        crawler = get_raw_crawler(MySpider, BASE_SETTINGS)
+        with raises(RuntimeError):
+            yield crawler.crawl()
+
+    @inlineCallbacks
+    def test_get_item_pipeline(self):
+        class ParentItemPipeline:
+            pass
+
+        class TrackingItemPipeline(ParentItemPipeline):
+            instances = []
+
+            def __init__(self):
+                TrackingItemPipeline.instances.append(self)
+
+        class MySpider(Spider):
+            name = "myspider"
+
+            @classmethod
+            def from_crawler(cls, crawler):
+                return cls(crawler=crawler)
+
+            def __init__(self, crawler):
+                self.crawler = crawler
+
+            def start_requests(self):
+                MySpider.result = crawler.get_item_pipeline(MySpider.cls)
+                return
+                yield
+
+        settings = {
+            **BASE_SETTINGS,
+            "ITEM_PIPELINES": {
+                TrackingItemPipeline: 0,
+            },
+        }
+
+        crawler = get_raw_crawler(MySpider, settings)
+        MySpider.cls = TrackingItemPipeline
+        yield crawler.crawl()
+        self.assertEqual(len(TrackingItemPipeline.instances), 1)
+        self.assertEqual(MySpider.result, TrackingItemPipeline.instances[-1])
+
+        crawler = get_raw_crawler(MySpider, settings)
+        MySpider.cls = DefaultSpider
+        yield crawler.crawl()
+        self.assertIsNone(MySpider.result)
+
+        crawler = get_raw_crawler(MySpider, settings)
+        MySpider.cls = ParentItemPipeline
+        yield crawler.crawl()
+        self.assertEqual(MySpider.result, TrackingItemPipeline.instances[-1])
+
+        class ChildItemPipeline(TrackingItemPipeline):
+            pass
+
+        crawler = get_raw_crawler(MySpider, settings)
+        MySpider.cls = ChildItemPipeline
+        yield crawler.crawl()
+        self.assertIsNone(MySpider.result)
+
+    def test_get_item_pipeline_not_crawling(self):
+        crawler = get_raw_crawler(settings_dict=BASE_SETTINGS)
+        self.assertRaises(RuntimeError, crawler.get_item_pipeline, DefaultSpider)
+
+    @inlineCallbacks
+    def test_get_item_pipeline_no_engine(self):
+        class MySpider(Spider):
+            name = "myspider"
+
+            @classmethod
+            def from_crawler(cls, crawler):
+                try:
+                    crawler.get_item_pipeline(DefaultSpider)
+                except Exception as e:
+                    MySpider.result = e
+                    raise
+
+        crawler = get_raw_crawler(MySpider, BASE_SETTINGS)
+        with raises(RuntimeError):
+            yield crawler.crawl()
+
+    @inlineCallbacks
+    def test_get_spider_middleware(self):
+        class ParentSpiderMiddleware:
+            pass
+
+        class TrackingSpiderMiddleware(ParentSpiderMiddleware):
+            instances = []
+
+            def __init__(self):
+                TrackingSpiderMiddleware.instances.append(self)
+
+        class MySpider(Spider):
+            name = "myspider"
+
+            @classmethod
+            def from_crawler(cls, crawler):
+                return cls(crawler=crawler)
+
+            def __init__(self, crawler):
+                self.crawler = crawler
+
+            def start_requests(self):
+                MySpider.result = crawler.get_spider_middleware(MySpider.cls)
+                return
+                yield
+
+        settings = {
+            **BASE_SETTINGS,
+            "SPIDER_MIDDLEWARES": {
+                TrackingSpiderMiddleware: 0,
+            },
+        }
+
+        crawler = get_raw_crawler(MySpider, settings)
+        MySpider.cls = TrackingSpiderMiddleware
+        yield crawler.crawl()
+        self.assertEqual(len(TrackingSpiderMiddleware.instances), 1)
+        self.assertEqual(MySpider.result, TrackingSpiderMiddleware.instances[-1])
+
+        crawler = get_raw_crawler(MySpider, settings)
+        MySpider.cls = DefaultSpider
+        yield crawler.crawl()
+        self.assertIsNone(MySpider.result)
+
+        crawler = get_raw_crawler(MySpider, settings)
+        MySpider.cls = ParentSpiderMiddleware
+        yield crawler.crawl()
+        self.assertEqual(MySpider.result, TrackingSpiderMiddleware.instances[-1])
+
+        class ChildSpiderMiddleware(TrackingSpiderMiddleware):
+            pass
+
+        crawler = get_raw_crawler(MySpider, settings)
+        MySpider.cls = ChildSpiderMiddleware
+        yield crawler.crawl()
+        self.assertIsNone(MySpider.result)
+
+    def test_get_spider_middleware_not_crawling(self):
+        crawler = get_raw_crawler(settings_dict=BASE_SETTINGS)
+        self.assertRaises(RuntimeError, crawler.get_spider_middleware, DefaultSpider)
+
+    @inlineCallbacks
+    def test_get_spider_middleware_no_engine(self):
+        class MySpider(Spider):
+            name = "myspider"
+
+            @classmethod
+            def from_crawler(cls, crawler):
+                try:
+                    crawler.get_spider_middleware(DefaultSpider)
+                except Exception as e:
+                    MySpider.result = e
+                    raise
+
+        crawler = get_raw_crawler(MySpider, BASE_SETTINGS)
+        with raises(RuntimeError):
+            yield crawler.crawl()
+
 
 class SpiderSettingsTestCase(unittest.TestCase):
     def test_spider_custom_settings(self):
@@ -223,20 +589,20 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
     def _runner(self):
         return CrawlerRunner({"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7"})
 
-    @defer.inlineCallbacks
+    @inlineCallbacks
     def test_crawler_runner_bootstrap_successful(self):
         runner = self._runner()
         yield runner.crawl(NoRequestsSpider)
         self.assertFalse(runner.bootstrap_failed)
 
-    @defer.inlineCallbacks
+    @inlineCallbacks
     def test_crawler_runner_bootstrap_successful_for_several(self):
         runner = self._runner()
         yield runner.crawl(NoRequestsSpider)
         yield runner.crawl(NoRequestsSpider)
         self.assertFalse(runner.bootstrap_failed)
 
-    @defer.inlineCallbacks
+    @inlineCallbacks
     def test_crawler_runner_bootstrap_failed(self):
         runner = self._runner()
 
@@ -249,7 +615,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
 
         self.assertTrue(runner.bootstrap_failed)
 
-    @defer.inlineCallbacks
+    @inlineCallbacks
     def test_crawler_runner_bootstrap_failed_for_several(self):
         runner = self._runner()
 
@@ -264,7 +630,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
 
         self.assertTrue(runner.bootstrap_failed)
 
-    @defer.inlineCallbacks
+    @inlineCallbacks
     def test_crawler_runner_asyncio_enabled_true(self):
         if self.reactor_pytest == "asyncio":
             CrawlerRunner(
@@ -531,7 +897,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
         p.expect_exact("Spider closed (shutdown)")
         p.wait()
 
-    @defer.inlineCallbacks
+    @inlineCallbacks
     def test_shutdown_forced(self):
         from twisted.internet import reactor
 
@@ -543,7 +909,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
         p.kill(sig)
         p.expect_exact("shutting down gracefully")
         # sending the second signal too fast often causes problems
-        d = defer.Deferred()
+        d = Deferred()
         reactor.callLater(0.1, d.callback, None)
         yield d
         p.kill(sig)

tests/test_extension_periodic_log.py

@@ -67,7 +67,7 @@ def extension(settings=None):
 
 class TestPeriodicLog(unittest.TestCase):
     def test_extension_enabled(self):
-        # Expected that settings for this extension loaded succesfully
+        # Expected that settings for this extension loaded successfully
         # And on certain conditions - extension raising NotConfigured
 
         # "PERIODIC_LOG_STATS": True -> set to {"enabled": True}

tests/test_feedexport.py

@@ -673,8 +673,7 @@ class FeedExportTestBase(ABC, unittest.TestCase):
             name = "testspider"
 
             def parse(self, response):
-                for item in items:
+                yield from items
-                    yield item
 
         data = yield self.run_and_export(TestSpider, settings)
         return data
@@ -2300,7 +2299,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                     content[feed["format"]].append(file.read_bytes())
         finally:
             self.tearDown()
-        defer.returnValue(content)
+        return content
 
     @defer.inlineCallbacks
     def assertExportedJsonLines(self, items, rows, settings=None):
@@ -2696,8 +2695,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
             name = "testspider"
 
             def parse(self, response):
-                for item in items:
+                yield from items
-                    yield item
 
         with MockServer() as server:
             TestSpider.start_urls = [server.url("/")]

tests/test_proxy_connect.py

@@ -31,6 +31,7 @@ sys.exit(mitmdump())
         self.proc = Popen(
             [
                 sys.executable,
+                "-u",
                 "-c",
                 script,
                 "--listen-host",
@@ -46,7 +47,7 @@ sys.exit(mitmdump())
             stdout=PIPE,
         )
         line = self.proc.stdout.readline().decode("utf-8")
-        host_port = re.search(r"listening at http://([^:]+:\d+)", line).group(1)
+        host_port = re.search(r"listening at (?:http://)?([^:]+:\d+)", line).group(1)
         address = f"http://{self.auth_user}:{self.auth_pass}@{host_port}"
         return address
 

tests/test_spider.py

@@ -317,8 +317,7 @@ class CrawlSpiderTest(SpiderTest):
             rules = (Rule(LinkExtractor(), process_links="dummy_process_links"),)
 
             def dummy_process_links(self, links):
-                for link in links:
+                yield from links
-                    yield link
 
         spider = _CrawlSpider()
         output = list(spider._requests_to_follow(response))

tests/test_spidermiddleware.py

@@ -170,8 +170,7 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase):
 
 class ProcessSpiderOutputSimpleMiddleware:
     def process_spider_output(self, response, result, spider):
-        for r in result:
+        yield from result
-            yield r
 
 
 class ProcessSpiderOutputAsyncGenMiddleware:
@@ -182,8 +181,7 @@ class ProcessSpiderOutputAsyncGenMiddleware:
 
 class ProcessSpiderOutputUniversalMiddleware:
     def process_spider_output(self, response, result, spider):
-        for r in result:
+        yield from result
-            yield r
 
     async def process_spider_output_async(self, response, result, spider):
         async for r in result:
@@ -324,8 +322,7 @@ class ProcessSpiderOutputInvalidResult(BaseAsyncSpiderMiddlewareTestCase):
 
 class ProcessStartRequestsSimpleMiddleware:
     def process_start_requests(self, start_requests, spider):
-        for r in start_requests:
+        yield from start_requests
-            yield r
 
 
 class ProcessStartRequestsSimple(BaseAsyncSpiderMiddlewareTestCase):

tests/test_utils_defer.py

@@ -107,8 +107,7 @@ class DeferUtilsTest(unittest.TestCase):
 class IterErrbackTest(unittest.TestCase):
     def test_iter_errback_good(self):
         def itergood():
-            for x in range(10):
+            yield from range(10)
-                yield x
 
         errors = []
         out = list(iter_errback(itergood(), errors.append))

tests/test_utils_iterators.py

@@ -524,6 +524,6 @@ class TestHelper(unittest.TestCase):
 
     def _assert_type_and_value(self, a, b, obj):
         self.assertTrue(
-            type(a) is type(b), f"Got {type(a)}, expected {type(b)} for { obj!r}"
+            type(a) is type(b), f"Got {type(a)}, expected {type(b)} for {obj!r}"
         )
         self.assertEqual(a, b)

tests/test_utils_request.py

@@ -2,22 +2,15 @@ import json
 import unittest
 import warnings
 from hashlib import sha1
-from typing import Dict, Mapping, Optional, Tuple, Union
+from typing import Dict, Optional, Tuple, Union
 from weakref import WeakKeyDictionary
 
-import pytest
-from w3lib.url import canonicalize_url
-
 from scrapy.http import Request
-from scrapy.utils.deprecate import ScrapyDeprecationWarning
 from scrapy.utils.python import to_bytes
 from scrapy.utils.request import (
-    _deprecated_fingerprint_cache,
     _fingerprint_cache,
-    _request_fingerprint_as_bytes,
     fingerprint,
     request_authenticate,
-    request_fingerprint,
     request_httprepr,
     request_to_curl,
 )
@@ -233,168 +226,6 @@ class FingerprintTest(unittest.TestCase):
         self.assertEqual(actual, expected)
 
 
-class RequestFingerprintTest(FingerprintTest):
-    function = staticmethod(request_fingerprint)
-    cache = _deprecated_fingerprint_cache
-    known_hashes: Tuple[Tuple[Request, Union[bytes, str], Dict], ...] = (
-        (
-            Request("http://example.org"),
-            "b2e5245ef826fd9576c93bd6e392fce3133fab62",
-            {},
-        ),
-        (
-            Request("https://example.org"),
-            "bd10a0a89ea32cdee77917320f1309b0da87e892",
-            {},
-        ),
-        (
-            Request("https://example.org?a"),
-            "2fb7d48ae02f04b749f40caa969c0bc3c43204ce",
-            {},
-        ),
-        (
-            Request("https://example.org?a=b"),
-            "42e5fe149b147476e3f67ad0670c57b4cc57856a",
-            {},
-        ),
-        (
-            Request("https://example.org?a=b&a"),
-            "d23a9787cb56c6375c2cae4453c5a8c634526942",
-            {},
-        ),
-        (
-            Request("https://example.org?a=b&a=c"),
-            "9a18a7a8552a9182b7f1e05d33876409e421e5c5",
-            {},
-        ),
-        (
-            Request("https://example.org", method="POST"),
-            "ba20a80cb5c5ca460021ceefb3c2467b2bfd1bc6",
-            {},
-        ),
-        (
-            Request("https://example.org", body=b"a"),
-            "4bb136e54e715a4ea7a9dd1101831765d33f2d60",
-            {},
-        ),
-        (
-            Request("https://example.org", method="POST", body=b"a"),
-            "6c6595374a304b293be762f7b7be3f54e9947c65",
-            {},
-        ),
-        (
-            Request("https://example.org#a", headers={"A": b"B"}),
-            "bd10a0a89ea32cdee77917320f1309b0da87e892",
-            {},
-        ),
-        (
-            Request("https://example.org#a", headers={"A": b"B"}),
-            "515b633cb3ca502a33a9d8c890e889ec1e425e65",
-            {"include_headers": ["A"]},
-        ),
-        (
-            Request("https://example.org#a", headers={"A": b"B"}),
-            "505c96e7da675920dfef58725e8c957dfdb38f47",
-            {"keep_fragments": True},
-        ),
-        (
-            Request("https://example.org#a", headers={"A": b"B"}),
-            "d6f673cdcb661b7970c2b9a00ee63e87d1e2e5da",
-            {"include_headers": ["A"], "keep_fragments": True},
-        ),
-        (
-            Request("https://example.org/ab"),
-            "4e2870fee58582d6f81755e9b8fdefe3cba0c951",
-            {},
-        ),
-        (
-            Request("https://example.org/a", body=b"b"),
-            "4e2870fee58582d6f81755e9b8fdefe3cba0c951",
-            {},
-        ),
-    )
-
-    def setUp(self) -> None:
-        warnings.simplefilter("ignore", ScrapyDeprecationWarning)
-
-    def tearDown(self) -> None:
-        warnings.simplefilter("default", ScrapyDeprecationWarning)
-
-    @pytest.mark.xfail(reason="known bug kept for backward compatibility", strict=True)
-    def test_part_separation(self):
-        super().test_part_separation()
-
-
-class RequestFingerprintDeprecationTest(unittest.TestCase):
-    def test_deprecation_default_parameters(self):
-        with pytest.warns(ScrapyDeprecationWarning) as warnings:
-            request_fingerprint(Request("http://www.example.com"))
-        messages = [str(warning.message) for warning in warnings]
-        self.assertTrue(
-            any("Call to deprecated function" in message for message in messages)
-        )
-        self.assertFalse(any("non-default" in message for message in messages))
-
-    def test_deprecation_non_default_parameters(self):
-        with pytest.warns(ScrapyDeprecationWarning) as warnings:
-            request_fingerprint(Request("http://www.example.com"), keep_fragments=True)
-        messages = [str(warning.message) for warning in warnings]
-        self.assertTrue(
-            any("Call to deprecated function" in message for message in messages)
-        )
-        self.assertTrue(any("non-default" in message for message in messages))
-
-
-class RequestFingerprintAsBytesTest(FingerprintTest):
-    function = staticmethod(_request_fingerprint_as_bytes)
-    cache = _deprecated_fingerprint_cache
-    known_hashes = RequestFingerprintTest.known_hashes
-
-    def test_caching(self):
-        r1 = Request("http://www.example.com/hnnoticiaj1.aspx?78160,199")
-        self.assertEqual(
-            self.function(r1), bytes.fromhex(self.cache[r1][self.default_cache_key])
-        )
-
-    @pytest.mark.xfail(reason="known bug kept for backward compatibility", strict=True)
-    def test_part_separation(self):
-        super().test_part_separation()
-
-    def test_hashes(self):
-        actual = [
-            self.function(request, **kwargs) for request, _, kwargs in self.known_hashes
-        ]
-        expected = [
-            bytes.fromhex(_fingerprint) for _, _fingerprint, _ in self.known_hashes
-        ]
-        self.assertEqual(actual, expected)
-
-
-_fingerprint_cache_2_6: Mapping[Request, Tuple[None, bool]] = WeakKeyDictionary()
-
-
-def request_fingerprint_2_6(request, include_headers=None, keep_fragments=False):
-    if include_headers:
-        include_headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers))
-    cache = _fingerprint_cache_2_6.setdefault(request, {})
-    cache_key = (include_headers, keep_fragments)
-    if cache_key not in cache:
-        fp = sha1()
-        fp.update(to_bytes(request.method))
-        fp.update(
-            to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments))
-        )
-        fp.update(request.body or b"")
-        if include_headers:
-            for hdr in include_headers:
-                if hdr in request.headers:
-                    fp.update(hdr)
-                    for v in request.headers.getlist(hdr):
-                        fp.update(v)
-        cache[cache_key] = fp.hexdigest()
-    return cache[cache_key]
-
-
 REQUEST_OBJECTS_TO_TEST = (
     Request("http://www.example.com/"),
     Request("http://www.example.com/query?id=111&cat=222"),
@@ -424,94 +255,16 @@ REQUEST_OBJECTS_TO_TEST = (
 )
 
 
-class BackwardCompatibilityTestCase(unittest.TestCase):
-    def test_function_backward_compatibility(self):
-        include_headers_to_test = (
-            None,
-            ["Accept-Language"],
-            ["accept-language", "sessionid"],
-            ["SESSIONID", "Accept-Language"],
-        )
-        for request_object in REQUEST_OBJECTS_TO_TEST:
-            for include_headers in include_headers_to_test:
-                for keep_fragments in (False, True):
-                    with warnings.catch_warnings():
-                        warnings.simplefilter("ignore")
-                        fp = request_fingerprint(
-                            request_object,
-                            include_headers=include_headers,
-                            keep_fragments=keep_fragments,
-                        )
-                    old_fp = request_fingerprint_2_6(
-                        request_object,
-                        include_headers=include_headers,
-                        keep_fragments=keep_fragments,
-                    )
-                    self.assertEqual(fp, old_fp)
-
-    def test_component_backward_compatibility(self):
-        for request_object in REQUEST_OBJECTS_TO_TEST:
-            with warnings.catch_warnings():
-                warnings.simplefilter("ignore")
-                crawler = get_crawler(prevent_warnings=False)
-                fp = crawler.request_fingerprinter.fingerprint(request_object)
-            old_fp = request_fingerprint_2_6(request_object)
-            self.assertEqual(fp.hex(), old_fp)
-
-    def test_custom_component_backward_compatibility(self):
-        """Tests that the backward-compatible request fingerprinting class featured
-        in the documentation is indeed backward compatible and does not cause a
-        warning to be logged."""
-
-        class RequestFingerprinter:
-            cache = WeakKeyDictionary()
-
-            def fingerprint(self, request):
-                if request not in self.cache:
-                    fp = sha1()
-                    fp.update(to_bytes(request.method))
-                    fp.update(to_bytes(canonicalize_url(request.url)))
-                    fp.update(request.body or b"")
-                    self.cache[request] = fp.digest()
-                return self.cache[request]
-
-        for request_object in REQUEST_OBJECTS_TO_TEST:
-            with warnings.catch_warnings() as logged_warnings:
-                settings = {
-                    "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter,
-                }
-                crawler = get_crawler(settings_dict=settings)
-                fp = crawler.request_fingerprinter.fingerprint(request_object)
-            old_fp = request_fingerprint_2_6(request_object)
-            self.assertEqual(fp.hex(), old_fp)
-            self.assertFalse(logged_warnings)
-
-
 class RequestFingerprinterTestCase(unittest.TestCase):
     def test_default_implementation(self):
-        with warnings.catch_warnings(record=True) as logged_warnings:
+        crawler = get_crawler()
-            crawler = get_crawler(prevent_warnings=False)
         request = Request("https://example.com")
         self.assertEqual(
             crawler.request_fingerprinter.fingerprint(request),
-            _request_fingerprint_as_bytes(request),
+            fingerprint(request),
         )
-        self.assertTrue(logged_warnings)
 
     def test_deprecated_implementation(self):
-        settings = {
-            "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.6",
-        }
-        with warnings.catch_warnings(record=True) as logged_warnings:
-            crawler = get_crawler(settings_dict=settings)
-        request = Request("https://example.com")
-        self.assertEqual(
-            crawler.request_fingerprinter.fingerprint(request),
-            _request_fingerprint_as_bytes(request),
-        )
-        self.assertTrue(logged_warnings)
-
-    def test_recommended_implementation(self):
         settings = {
             "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
         }
@@ -522,14 +275,7 @@ class RequestFingerprinterTestCase(unittest.TestCase):
             crawler.request_fingerprinter.fingerprint(request),
             fingerprint(request),
         )
-        self.assertFalse(logged_warnings)
+        self.assertTrue(logged_warnings)
-
-    def test_unknown_implementation(self):
-        settings = {
-            "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.5",
-        }
-        with self.assertRaises(ValueError):
-            get_crawler(settings_dict=settings)
 
 
 class CustomRequestFingerprinterTestCase(unittest.TestCase):

tox.ini

@@ -11,11 +11,7 @@ minversion = 1.7.0
 deps =
     -rtests/requirements.txt
     # mitmproxy does not support PyPy
-    # Python 3.9+ requires mitmproxy >= 5.3.0
+    mitmproxy; implementation_name != 'pypy'
-    # mitmproxy >= 5.3.0 requires h2 >= 4.0, Twisted 21.2 requires h2 < 4.0
-    #mitmproxy >= 5.3.0; python_version >= '3.9' and implementation_name != 'pypy'
-    # The tests hang with mitmproxy 8.0.0: https://github.com/scrapy/scrapy/issues/5454
-    mitmproxy >= 4.0.4, < 8; python_version < '3.9' and implementation_name != 'pypy'
 passenv =
     S3_TEST_FILE_URI
     AWS_ACCESS_KEY_ID
@@ -88,7 +84,7 @@ deps =
     xtractmime==0.2.0
     -rtests/requirements.txt
 
-    # mitmproxy 4.0.4+ requires upgrading some of the pinned dependencies
+    # mitmproxy 8.0.0 requires upgrading some of the pinned dependencies
     # above, hence we do not install it in pinned environments at the moment
 setenv =
     _SCRAPY_PINNED=true