From 24634f1bb236f72a1a7b73900f8e3b524f7717b5 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Wed, 31 Jan 2024 12:29:43 -0300 Subject: [PATCH] Attend PR comments --- docs/topics/request-response.rst | 30 ------------------- scrapy/settings/default_settings.py | 2 +- .../templates/project/module/settings.py.tmpl | 1 - scrapy/utils/request.py | 16 ++++++++-- scrapy/utils/test.py | 3 -- tests/test_utils_request.py | 14 +++++++++ 6 files changed, 28 insertions(+), 38 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 6dbcb4584..67fc0c6e9 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -469,36 +469,6 @@ import path. .. autoclass:: scrapy.utils.request.RequestFingerprinter - -.. setting:: REQUEST_FINGERPRINTER_IMPLEMENTATION - -REQUEST_FINGERPRINTER_IMPLEMENTATION -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. versionadded:: 2.7 - -Default: ``'2.7'`` - -Determines which request fingerprinting algorithm is used by the default -request fingerprinter class (see :setting:`REQUEST_FINGERPRINTER_CLASS`). - -Possible value is: - -- ``'2.7'`` - - This implementation was introduced in Scrapy 2.7 to fix an issue of the - previous implementation. - - New projects should use this value. The :command:`startproject` command - sets this value in the generated ``settings.py`` file. - -Scenarios where changing the request fingerprinting algorithm may cause -undesired results include, for example, using the HTTP cache middleware (see -:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`). -Changing the request fingerprinting algorithm would invalidate the current -cache, requiring you to redownload all requests again. - - .. _custom-request-fingerprinter: Writing your own request fingerprinter diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 02494bad0..49ab1b5ef 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -260,7 +260,7 @@ REFERER_ENABLED = True REFERRER_POLICY = "scrapy.spidermiddlewares.referer.DefaultReferrerPolicy" REQUEST_FINGERPRINTER_CLASS = "scrapy.utils.request.RequestFingerprinter" -REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.7" +REQUEST_FINGERPRINTER_IMPLEMENTATION = "SENTINEL" RETRY_ENABLED = True RETRY_TIMES = 2 # initial response + 2 retries = 3 requests diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index ecb1e5e5c..b4779e555 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -88,6 +88,5 @@ ROBOTSTXT_OBEY = True #HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage" # Set settings whose default value is deprecated to a future-proof value -REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.7" TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor" FEED_EXPORT_ENCODING = "utf-8" diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index b230cd214..068e5bdcb 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -5,6 +5,7 @@ scrapy.http.Request objects import hashlib import json +import warnings from typing import ( TYPE_CHECKING, Any, @@ -25,6 +26,7 @@ from w3lib.http import basic_auth_header from w3lib.url import canonicalize_url from scrapy import Request, Spider +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import load_object from scrapy.utils.python import to_bytes, to_unicode @@ -139,14 +141,22 @@ class RequestFingerprinter: "REQUEST_FINGERPRINTER_IMPLEMENTATION" ) else: - implementation = "2.7" - if implementation == "2.7": + implementation = "SENTINEL" + + if implementation == "SENTINEL": + self._fingerprint = fingerprint + elif implementation == "2.7": + message = ( + "'REQUEST_FINGERPRINTER_IMPLEMENTATION' is a deprecated setting.\n" + "And it will be removed in future version of Scrapy." + ) + warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2) self._fingerprint = fingerprint else: raise ValueError( f"Got an invalid value on setting " f"'REQUEST_FINGERPRINTER_IMPLEMENTATION': " - f"{implementation!r}. Valid value is '2.7'." + f"{implementation!r}. Valid values are '2.7' and 'SENTINEL'." ) def fingerprint(self, request: Request) -> bytes: diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 709e0b00d..c6a31cacf 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -76,7 +76,6 @@ class TestSpider(Spider): def get_crawler( spidercls: Optional[Type[Spider]] = None, settings_dict: Optional[Dict[str, Any]] = None, - prevent_warnings: bool = True, ) -> Crawler: """Return an unconfigured Crawler object. If settings_dict is given, it will be used to populate the crawler settings with a project level @@ -86,8 +85,6 @@ def get_crawler( # Set by default settings that prevent deprecation warnings. settings: Dict[str, Any] = {} - if prevent_warnings: - settings["REQUEST_FINGERPRINTER_IMPLEMENTATION"] = "2.7" settings.update(settings_dict or {}) runner = CrawlerRunner(settings) crawler = runner.create_crawler(spidercls or TestSpider) diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index 68f6eb045..c0c44875e 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -1,5 +1,6 @@ import json import unittest +import warnings from hashlib import sha1 from typing import Dict, Optional, Tuple, Union from weakref import WeakKeyDictionary @@ -263,6 +264,19 @@ class RequestFingerprinterTestCase(unittest.TestCase): fingerprint(request), ) + def test_deprecated_implementation(self): + settings = { + "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", + } + with warnings.catch_warnings(record=True) as logged_warnings: + crawler = get_crawler(settings_dict=settings) + request = Request("https://example.com") + self.assertEqual( + crawler.request_fingerprinter.fingerprint(request), + fingerprint(request), + ) + self.assertTrue(logged_warnings) + def test_unknown_implementation(self): settings = { "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.5",