diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index d6b3585e2..02494bad0 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -260,7 +260,7 @@ REFERER_ENABLED = True REFERRER_POLICY = "scrapy.spidermiddlewares.referer.DefaultReferrerPolicy" REQUEST_FINGERPRINTER_CLASS = "scrapy.utils.request.RequestFingerprinter" -REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.6" +REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.7" RETRY_ENABLED = True RETRY_TIMES = 2 # initial response + 2 retries = 3 requests diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index cea1bc727..b230cd214 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -5,7 +5,6 @@ scrapy.http.Request objects import hashlib import json -import warnings from typing import ( TYPE_CHECKING, Any, @@ -26,7 +25,6 @@ from w3lib.http import basic_auth_header from w3lib.url import canonicalize_url from scrapy import Request, Spider -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import load_object from scrapy.utils.python import to_bytes, to_unicode @@ -34,9 +32,6 @@ from scrapy.utils.python import to_bytes, to_unicode if TYPE_CHECKING: from scrapy.crawler import Crawler -_deprecated_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]" -_deprecated_fingerprint_cache = WeakKeyDictionary() - def _serialize_headers( headers: Iterable[bytes], request: Request @@ -47,120 +42,6 @@ def _serialize_headers( yield from request.headers.getlist(header) -def request_fingerprint( - request: Request, - include_headers: Optional[Iterable[Union[bytes, str]]] = None, - keep_fragments: bool = False, -) -> str: - """ - Return the request fingerprint as an hexadecimal string. - - The request fingerprint is a hash that uniquely identifies the resource the - request points to. For example, take the following two urls: - - http://www.example.com/query?id=111&cat=222 - http://www.example.com/query?cat=222&id=111 - - Even though those are two different URLs both point to the same resource - and are equivalent (i.e. they should return the same response). - - Another example are cookies used to store session ids. Suppose the - following page is only accessible to authenticated users: - - http://www.example.com/members/offers.html - - Lots of sites use a cookie to store the session id, which adds a random - component to the HTTP Request and thus should be ignored when calculating - the fingerprint. - - For this reason, request headers are ignored by default when calculating - the fingerprint. If you want to include specific headers use the - include_headers argument, which is a list of Request headers to include. - - Also, servers usually ignore fragments in urls when handling requests, - so they are also ignored by default when calculating the fingerprint. - If you want to include them, set the keep_fragments argument to True - (for instance when handling requests with a headless browser). - """ - if include_headers or keep_fragments: - message = ( - "Call to deprecated function " - "scrapy.utils.request.request_fingerprint().\n" - "\n" - "If you are using this function in a Scrapy component because you " - "need a non-default fingerprinting algorithm, and you are OK " - "with that non-default fingerprinting algorithm being used by " - "all Scrapy components and not just the one calling this " - "function, use crawler.request_fingerprinter.fingerprint() " - "instead in your Scrapy component (you can get the crawler " - "object from the 'from_crawler' class method), and use the " - "'REQUEST_FINGERPRINTER_CLASS' setting to configure your " - "non-default fingerprinting algorithm.\n" - "\n" - "Otherwise, consider using the " - "scrapy.utils.request.fingerprint() function instead.\n" - "\n" - "If you switch to 'fingerprint()', or assign the " - "'REQUEST_FINGERPRINTER_CLASS' setting a class that uses " - "'fingerprint()', the generated fingerprints will not only be " - "bytes instead of a string, but they will also be different from " - "those generated by 'request_fingerprint()'. Before you switch, " - "make sure that you understand the consequences of this (e.g. " - "cache invalidation) and are OK with them; otherwise, consider " - "implementing your own function which returns the same " - "fingerprints as the deprecated 'request_fingerprint()' function." - ) - else: - message = ( - "Call to deprecated function " - "scrapy.utils.request.request_fingerprint().\n" - "\n" - "If you are using this function in a Scrapy component, and you " - "are OK with users of your component changing the fingerprinting " - "algorithm through settings, use " - "crawler.request_fingerprinter.fingerprint() instead in your " - "Scrapy component (you can get the crawler object from the " - "'from_crawler' class method).\n" - "\n" - "Otherwise, consider using the " - "scrapy.utils.request.fingerprint() function instead.\n" - "\n" - "Either way, the resulting fingerprints will be returned as " - "bytes, not as a string, and they will also be different from " - "those generated by 'request_fingerprint()'. Before you switch, " - "make sure that you understand the consequences of this (e.g. " - "cache invalidation) and are OK with them; otherwise, consider " - "implementing your own function which returns the same " - "fingerprints as the deprecated 'request_fingerprint()' function." - ) - warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2) - processed_include_headers: Optional[Tuple[bytes, ...]] = None - if include_headers: - processed_include_headers = tuple( - to_bytes(h.lower()) for h in sorted(include_headers) - ) - cache = _deprecated_fingerprint_cache.setdefault(request, {}) - cache_key = (processed_include_headers, keep_fragments) - if cache_key not in cache: - fp = hashlib.sha1() - fp.update(to_bytes(request.method)) - fp.update( - to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments)) - ) - fp.update(request.body or b"") - if processed_include_headers: - for part in _serialize_headers(processed_include_headers, request): - fp.update(part) - cache[cache_key] = fp.hexdigest() - return cache[cache_key] - - -def _request_fingerprint_as_bytes(*args: Any, **kwargs: Any) -> bytes: - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - return bytes.fromhex(request_fingerprint(*args, **kwargs)) - - _fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]" _fingerprint_cache = WeakKeyDictionary() @@ -258,32 +139,14 @@ class RequestFingerprinter: "REQUEST_FINGERPRINTER_IMPLEMENTATION" ) else: - implementation = "2.6" - if implementation == "2.6": - message = ( - "'2.6' is a deprecated value for the " - "'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting.\n" - "\n" - "It is also the default value. In other words, it is normal " - "to get this warning if you have not defined a value for the " - "'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting. This is so " - "for backward compatibility reasons, but it will change in a " - "future version of Scrapy.\n" - "\n" - "See the documentation of the " - "'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting for " - "information on how to handle this deprecation." - ) - warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2) - self._fingerprint = _request_fingerprint_as_bytes - elif implementation == "2.7": + implementation = "2.7" + if implementation == "2.7": self._fingerprint = fingerprint else: raise ValueError( f"Got an invalid value on setting " f"'REQUEST_FINGERPRINTER_IMPLEMENTATION': " - f"{implementation!r}. Valid values are '2.6' (deprecated) " - f"and '2.7'." + f"{implementation!r}. Valid value is '2.7'." ) def fingerprint(self, request: Request) -> bytes: diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index e6d1abe3f..f6bc9ba6f 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -1,23 +1,15 @@ import json import unittest -import warnings from hashlib import sha1 -from typing import Dict, Mapping, Optional, Tuple, Union +from typing import Dict, Optional, Tuple, Union from weakref import WeakKeyDictionary -import pytest -from w3lib.url import canonicalize_url - from scrapy.http import Request -from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.python import to_bytes from scrapy.utils.request import ( - _deprecated_fingerprint_cache, _fingerprint_cache, - _request_fingerprint_as_bytes, fingerprint, request_authenticate, - request_fingerprint, request_httprepr, request_to_curl, ) @@ -233,168 +225,6 @@ class FingerprintTest(unittest.TestCase): self.assertEqual(actual, expected) -class RequestFingerprintTest(FingerprintTest): - function = staticmethod(request_fingerprint) - cache = _deprecated_fingerprint_cache - known_hashes: Tuple[Tuple[Request, Union[bytes, str], Dict], ...] = ( - ( - Request("http://example.org"), - "b2e5245ef826fd9576c93bd6e392fce3133fab62", - {}, - ), - ( - Request("https://example.org"), - "bd10a0a89ea32cdee77917320f1309b0da87e892", - {}, - ), - ( - Request("https://example.org?a"), - "2fb7d48ae02f04b749f40caa969c0bc3c43204ce", - {}, - ), - ( - Request("https://example.org?a=b"), - "42e5fe149b147476e3f67ad0670c57b4cc57856a", - {}, - ), - ( - Request("https://example.org?a=b&a"), - "d23a9787cb56c6375c2cae4453c5a8c634526942", - {}, - ), - ( - Request("https://example.org?a=b&a=c"), - "9a18a7a8552a9182b7f1e05d33876409e421e5c5", - {}, - ), - ( - Request("https://example.org", method="POST"), - "ba20a80cb5c5ca460021ceefb3c2467b2bfd1bc6", - {}, - ), - ( - Request("https://example.org", body=b"a"), - "4bb136e54e715a4ea7a9dd1101831765d33f2d60", - {}, - ), - ( - Request("https://example.org", method="POST", body=b"a"), - "6c6595374a304b293be762f7b7be3f54e9947c65", - {}, - ), - ( - Request("https://example.org#a", headers={"A": b"B"}), - "bd10a0a89ea32cdee77917320f1309b0da87e892", - {}, - ), - ( - Request("https://example.org#a", headers={"A": b"B"}), - "515b633cb3ca502a33a9d8c890e889ec1e425e65", - {"include_headers": ["A"]}, - ), - ( - Request("https://example.org#a", headers={"A": b"B"}), - "505c96e7da675920dfef58725e8c957dfdb38f47", - {"keep_fragments": True}, - ), - ( - Request("https://example.org#a", headers={"A": b"B"}), - "d6f673cdcb661b7970c2b9a00ee63e87d1e2e5da", - {"include_headers": ["A"], "keep_fragments": True}, - ), - ( - Request("https://example.org/ab"), - "4e2870fee58582d6f81755e9b8fdefe3cba0c951", - {}, - ), - ( - Request("https://example.org/a", body=b"b"), - "4e2870fee58582d6f81755e9b8fdefe3cba0c951", - {}, - ), - ) - - def setUp(self) -> None: - warnings.simplefilter("ignore", ScrapyDeprecationWarning) - - def tearDown(self) -> None: - warnings.simplefilter("default", ScrapyDeprecationWarning) - - @pytest.mark.xfail(reason="known bug kept for backward compatibility", strict=True) - def test_part_separation(self): - super().test_part_separation() - - -class RequestFingerprintDeprecationTest(unittest.TestCase): - def test_deprecation_default_parameters(self): - with pytest.warns(ScrapyDeprecationWarning) as warnings: - request_fingerprint(Request("http://www.example.com")) - messages = [str(warning.message) for warning in warnings] - self.assertTrue( - any("Call to deprecated function" in message for message in messages) - ) - self.assertFalse(any("non-default" in message for message in messages)) - - def test_deprecation_non_default_parameters(self): - with pytest.warns(ScrapyDeprecationWarning) as warnings: - request_fingerprint(Request("http://www.example.com"), keep_fragments=True) - messages = [str(warning.message) for warning in warnings] - self.assertTrue( - any("Call to deprecated function" in message for message in messages) - ) - self.assertTrue(any("non-default" in message for message in messages)) - - -class RequestFingerprintAsBytesTest(FingerprintTest): - function = staticmethod(_request_fingerprint_as_bytes) - cache = _deprecated_fingerprint_cache - known_hashes = RequestFingerprintTest.known_hashes - - def test_caching(self): - r1 = Request("http://www.example.com/hnnoticiaj1.aspx?78160,199") - self.assertEqual( - self.function(r1), bytes.fromhex(self.cache[r1][self.default_cache_key]) - ) - - @pytest.mark.xfail(reason="known bug kept for backward compatibility", strict=True) - def test_part_separation(self): - super().test_part_separation() - - def test_hashes(self): - actual = [ - self.function(request, **kwargs) for request, _, kwargs in self.known_hashes - ] - expected = [ - bytes.fromhex(_fingerprint) for _, _fingerprint, _ in self.known_hashes - ] - self.assertEqual(actual, expected) - - -_fingerprint_cache_2_6: Mapping[Request, Tuple[None, bool]] = WeakKeyDictionary() - - -def request_fingerprint_2_6(request, include_headers=None, keep_fragments=False): - if include_headers: - include_headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers)) - cache = _fingerprint_cache_2_6.setdefault(request, {}) - cache_key = (include_headers, keep_fragments) - if cache_key not in cache: - fp = sha1() - fp.update(to_bytes(request.method)) - fp.update( - to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments)) - ) - fp.update(request.body or b"") - if include_headers: - for hdr in include_headers: - if hdr in request.headers: - fp.update(hdr) - for v in request.headers.getlist(hdr): - fp.update(v) - cache[cache_key] = fp.hexdigest() - return cache[cache_key] - - REQUEST_OBJECTS_TO_TEST = ( Request("http://www.example.com/"), Request("http://www.example.com/query?id=111&cat=222"), @@ -424,105 +254,17 @@ REQUEST_OBJECTS_TO_TEST = ( ) -class BackwardCompatibilityTestCase(unittest.TestCase): - def test_function_backward_compatibility(self): - include_headers_to_test = ( - None, - ["Accept-Language"], - ["accept-language", "sessionid"], - ["SESSIONID", "Accept-Language"], - ) - for request_object in REQUEST_OBJECTS_TO_TEST: - for include_headers in include_headers_to_test: - for keep_fragments in (False, True): - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - fp = request_fingerprint( - request_object, - include_headers=include_headers, - keep_fragments=keep_fragments, - ) - old_fp = request_fingerprint_2_6( - request_object, - include_headers=include_headers, - keep_fragments=keep_fragments, - ) - self.assertEqual(fp, old_fp) - - def test_component_backward_compatibility(self): - for request_object in REQUEST_OBJECTS_TO_TEST: - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - crawler = get_crawler(prevent_warnings=False) - fp = crawler.request_fingerprinter.fingerprint(request_object) - old_fp = request_fingerprint_2_6(request_object) - self.assertEqual(fp.hex(), old_fp) - - def test_custom_component_backward_compatibility(self): - """Tests that the backward-compatible request fingerprinting class featured - in the documentation is indeed backward compatible and does not cause a - warning to be logged.""" - - class RequestFingerprinter: - cache = WeakKeyDictionary() - - def fingerprint(self, request): - if request not in self.cache: - fp = sha1() - fp.update(to_bytes(request.method)) - fp.update(to_bytes(canonicalize_url(request.url))) - fp.update(request.body or b"") - self.cache[request] = fp.digest() - return self.cache[request] - - for request_object in REQUEST_OBJECTS_TO_TEST: - with warnings.catch_warnings() as logged_warnings: - settings = { - "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter, - } - crawler = get_crawler(settings_dict=settings) - fp = crawler.request_fingerprinter.fingerprint(request_object) - old_fp = request_fingerprint_2_6(request_object) - self.assertEqual(fp.hex(), old_fp) - self.assertFalse(logged_warnings) - - class RequestFingerprinterTestCase(unittest.TestCase): def test_default_implementation(self): - with warnings.catch_warnings(record=True) as logged_warnings: - crawler = get_crawler(prevent_warnings=False) - request = Request("https://example.com") - self.assertEqual( - crawler.request_fingerprinter.fingerprint(request), - _request_fingerprint_as_bytes(request), - ) - self.assertTrue(logged_warnings) - - def test_deprecated_implementation(self): - settings = { - "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.6", - } - with warnings.catch_warnings(record=True) as logged_warnings: - crawler = get_crawler(settings_dict=settings) - request = Request("https://example.com") - self.assertEqual( - crawler.request_fingerprinter.fingerprint(request), - _request_fingerprint_as_bytes(request), - ) - self.assertTrue(logged_warnings) - - def test_recommended_implementation(self): settings = { "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", } - with warnings.catch_warnings(record=True) as logged_warnings: - crawler = get_crawler(settings_dict=settings) + crawler = get_crawler(settings_dict=settings) request = Request("https://example.com") self.assertEqual( crawler.request_fingerprinter.fingerprint(request), fingerprint(request), ) - self.assertFalse(logged_warnings) def test_unknown_implementation(self): settings = {