RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Cleanup deprecated fingerprint code in scrapy.utils.request

This commit is contained in:
Laerte Pereira 2024-01-31 10:05:50 -03:00
parent 4229048255
commit 2487e3cc03
3 changed files with 6 additions and 401 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
scrapy/settings/default_settings.py
View File

@ -260,7 +260,7 @@ REFERER_ENABLED = True
REFERRER_POLICY = "scrapy.spidermiddlewares.referer.DefaultReferrerPolicy"
REQUEST_FINGERPRINTER_CLASS = "scrapy.utils.request.RequestFingerprinter"
REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.6"
REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.7"
RETRY_ENABLED = True
RETRY_TIMES = 2 # initial response + 2 retries = 3 requests

143
scrapy/utils/request.py
View File

@ -5,7 +5,6 @@ scrapy.http.Request objects
import hashlib
import json
import warnings
from typing import (
TYPE_CHECKING,
Any,
@ -26,7 +25,6 @@ from w3lib.http import basic_auth_header
from w3lib.url import canonicalize_url
from scrapy import Request, Spider
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import load_object
from scrapy.utils.python import to_bytes, to_unicode
@ -34,9 +32,6 @@ from scrapy.utils.python import to_bytes, to_unicode
if TYPE_CHECKING:
from scrapy.crawler import Crawler
_deprecated_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]"
_deprecated_fingerprint_cache = WeakKeyDictionary()
def _serialize_headers(
headers: Iterable[bytes], request: Request
@ -47,120 +42,6 @@ def _serialize_headers(
yield from request.headers.getlist(header)
def request_fingerprint(
request: Request,
include_headers: Optional[Iterable[Union[bytes, str]]] = None,
keep_fragments: bool = False,
) -> str:
"""
Return the request fingerprint as an hexadecimal string.
The request fingerprint is a hash that uniquely identifies the resource the
request points to. For example, take the following two urls:
http://www.example.com/query?id=111&cat=222
http://www.example.com/query?cat=222&id=111
Even though those are two different URLs both point to the same resource
and are equivalent (i.e. they should return the same response).
Another example are cookies used to store session ids. Suppose the
following page is only accessible to authenticated users:
http://www.example.com/members/offers.html
Lots of sites use a cookie to store the session id, which adds a random
component to the HTTP Request and thus should be ignored when calculating
the fingerprint.
For this reason, request headers are ignored by default when calculating
the fingerprint. If you want to include specific headers use the
include_headers argument, which is a list of Request headers to include.
Also, servers usually ignore fragments in urls when handling requests,
so they are also ignored by default when calculating the fingerprint.
If you want to include them, set the keep_fragments argument to True
(for instance when handling requests with a headless browser).
"""
if include_headers or keep_fragments:
message = (
"Call to deprecated function "
"scrapy.utils.request.request_fingerprint().\n"
"\n"
"If you are using this function in a Scrapy component because you "
"need a non-default fingerprinting algorithm, and you are OK "
"with that non-default fingerprinting algorithm being used by "
"all Scrapy components and not just the one calling this "
"function, use crawler.request_fingerprinter.fingerprint() "
"instead in your Scrapy component (you can get the crawler "
"object from the 'from_crawler' class method), and use the "
"'REQUEST_FINGERPRINTER_CLASS' setting to configure your "
"non-default fingerprinting algorithm.\n"
"\n"
"Otherwise, consider using the "
"scrapy.utils.request.fingerprint() function instead.\n"
"\n"
"If you switch to 'fingerprint()', or assign the "
"'REQUEST_FINGERPRINTER_CLASS' setting a class that uses "
"'fingerprint()', the generated fingerprints will not only be "
"bytes instead of a string, but they will also be different from "
"those generated by 'request_fingerprint()'. Before you switch, "
"make sure that you understand the consequences of this (e.g. "
"cache invalidation) and are OK with them; otherwise, consider "
"implementing your own function which returns the same "
"fingerprints as the deprecated 'request_fingerprint()' function."
)
else:
message = (
"Call to deprecated function "
"scrapy.utils.request.request_fingerprint().\n"
"\n"
"If you are using this function in a Scrapy component, and you "
"are OK with users of your component changing the fingerprinting "
"algorithm through settings, use "
"crawler.request_fingerprinter.fingerprint() instead in your "
"Scrapy component (you can get the crawler object from the "
"'from_crawler' class method).\n"
"\n"
"Otherwise, consider using the "
"scrapy.utils.request.fingerprint() function instead.\n"
"\n"
"Either way, the resulting fingerprints will be returned as "
"bytes, not as a string, and they will also be different from "
"those generated by 'request_fingerprint()'. Before you switch, "
"make sure that you understand the consequences of this (e.g. "
"cache invalidation) and are OK with them; otherwise, consider "
"implementing your own function which returns the same "
"fingerprints as the deprecated 'request_fingerprint()' function."
)
warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2)
processed_include_headers: Optional[Tuple[bytes, ...]] = None
if include_headers:
processed_include_headers = tuple(
to_bytes(h.lower()) for h in sorted(include_headers)
)
cache = _deprecated_fingerprint_cache.setdefault(request, {})
cache_key = (processed_include_headers, keep_fragments)
if cache_key not in cache:
fp = hashlib.sha1()
fp.update(to_bytes(request.method))
fp.update(
to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments))
)
fp.update(request.body or b"")
if processed_include_headers:
for part in _serialize_headers(processed_include_headers, request):
fp.update(part)
cache[cache_key] = fp.hexdigest()
return cache[cache_key]
def _request_fingerprint_as_bytes(*args: Any, **kwargs: Any) -> bytes:
with warnings.catch_warnings():
warnings.simplefilter("ignore")
return bytes.fromhex(request_fingerprint(*args, **kwargs))
_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]"
_fingerprint_cache = WeakKeyDictionary()
@ -258,32 +139,14 @@ class RequestFingerprinter:
"REQUEST_FINGERPRINTER_IMPLEMENTATION"
)
else:
implementation = "2.6"
if implementation == "2.6":
message = (
"'2.6' is a deprecated value for the "
"'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting.\n"
"\n"
"It is also the default value. In other words, it is normal "
"to get this warning if you have not defined a value for the "
"'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting. This is so "
"for backward compatibility reasons, but it will change in a "
"future version of Scrapy.\n"
"\n"
"See the documentation of the "
"'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting for "
"information on how to handle this deprecation."
)
warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2)
self._fingerprint = _request_fingerprint_as_bytes
elif implementation == "2.7":
implementation = "2.7"
if implementation == "2.7":
self._fingerprint = fingerprint
else:
raise ValueError(
f"Got an invalid value on setting "
f"'REQUEST_FINGERPRINTER_IMPLEMENTATION': "
f"{implementation!r}. Valid values are '2.6' (deprecated) "
f"and '2.7'."
f"{implementation!r}. Valid value is '2.7'."
)
def fingerprint(self, request: Request) -> bytes:

262
tests/test_utils_request.py
View File

@ -1,23 +1,15 @@
import json
import unittest
import warnings
from hashlib import sha1
from typing import Dict, Mapping, Optional, Tuple, Union
from typing import Dict, Optional, Tuple, Union
from weakref import WeakKeyDictionary
import pytest
from w3lib.url import canonicalize_url
from scrapy.http import Request
from scrapy.utils.deprecate import ScrapyDeprecationWarning
from scrapy.utils.python import to_bytes
from scrapy.utils.request import (
_deprecated_fingerprint_cache,
_fingerprint_cache,
_request_fingerprint_as_bytes,
fingerprint,
request_authenticate,
request_fingerprint,
request_httprepr,
request_to_curl,
)
@ -233,168 +225,6 @@ class FingerprintTest(unittest.TestCase):
self.assertEqual(actual, expected)
class RequestFingerprintTest(FingerprintTest):
function = staticmethod(request_fingerprint)
cache = _deprecated_fingerprint_cache
known_hashes: Tuple[Tuple[Request, Union[bytes, str], Dict], ...] = (
(
Request("http://example.org"),
"b2e5245ef826fd9576c93bd6e392fce3133fab62",
{},
),
(
Request("https://example.org"),
"bd10a0a89ea32cdee77917320f1309b0da87e892",
{},
),
(
Request("https://example.org?a"),
"2fb7d48ae02f04b749f40caa969c0bc3c43204ce",
{},
),
(
Request("https://example.org?a=b"),
"42e5fe149b147476e3f67ad0670c57b4cc57856a",
{},
),
(
Request("https://example.org?a=b&a"),
"d23a9787cb56c6375c2cae4453c5a8c634526942",
{},
),
(
Request("https://example.org?a=b&a=c"),
"9a18a7a8552a9182b7f1e05d33876409e421e5c5",
{},
),
(
Request("https://example.org", method="POST"),
"ba20a80cb5c5ca460021ceefb3c2467b2bfd1bc6",
{},
),
(
Request("https://example.org", body=b"a"),
"4bb136e54e715a4ea7a9dd1101831765d33f2d60",
{},
),
(
Request("https://example.org", method="POST", body=b"a"),
"6c6595374a304b293be762f7b7be3f54e9947c65",
{},
),
(
Request("https://example.org#a", headers={"A": b"B"}),
"bd10a0a89ea32cdee77917320f1309b0da87e892",
{},
),
(
Request("https://example.org#a", headers={"A": b"B"}),
"515b633cb3ca502a33a9d8c890e889ec1e425e65",
{"include_headers": ["A"]},
),
(
Request("https://example.org#a", headers={"A": b"B"}),
"505c96e7da675920dfef58725e8c957dfdb38f47",
{"keep_fragments": True},
),
(
Request("https://example.org#a", headers={"A": b"B"}),
"d6f673cdcb661b7970c2b9a00ee63e87d1e2e5da",
{"include_headers": ["A"], "keep_fragments": True},
),
(
Request("https://example.org/ab"),
"4e2870fee58582d6f81755e9b8fdefe3cba0c951",
{},
),
(
Request("https://example.org/a", body=b"b"),
"4e2870fee58582d6f81755e9b8fdefe3cba0c951",
{},
),
)
def setUp(self) -> None:
warnings.simplefilter("ignore", ScrapyDeprecationWarning)
def tearDown(self) -> None:
warnings.simplefilter("default", ScrapyDeprecationWarning)
@pytest.mark.xfail(reason="known bug kept for backward compatibility", strict=True)
def test_part_separation(self):
super().test_part_separation()
class RequestFingerprintDeprecationTest(unittest.TestCase):
def test_deprecation_default_parameters(self):
with pytest.warns(ScrapyDeprecationWarning) as warnings:
request_fingerprint(Request("http://www.example.com"))
messages = [str(warning.message) for warning in warnings]
self.assertTrue(
any("Call to deprecated function" in message for message in messages)
)
self.assertFalse(any("non-default" in message for message in messages))
def test_deprecation_non_default_parameters(self):
with pytest.warns(ScrapyDeprecationWarning) as warnings:
request_fingerprint(Request("http://www.example.com"), keep_fragments=True)
messages = [str(warning.message) for warning in warnings]
self.assertTrue(
any("Call to deprecated function" in message for message in messages)
)
self.assertTrue(any("non-default" in message for message in messages))
class RequestFingerprintAsBytesTest(FingerprintTest):
function = staticmethod(_request_fingerprint_as_bytes)
cache = _deprecated_fingerprint_cache
known_hashes = RequestFingerprintTest.known_hashes
def test_caching(self):
r1 = Request("http://www.example.com/hnnoticiaj1.aspx?78160,199")
self.assertEqual(
self.function(r1), bytes.fromhex(self.cache[r1][self.default_cache_key])
)
@pytest.mark.xfail(reason="known bug kept for backward compatibility", strict=True)
def test_part_separation(self):
super().test_part_separation()
def test_hashes(self):
actual = [
self.function(request, **kwargs) for request, _, kwargs in self.known_hashes
]
expected = [
bytes.fromhex(_fingerprint) for _, _fingerprint, _ in self.known_hashes
]
self.assertEqual(actual, expected)
_fingerprint_cache_2_6: Mapping[Request, Tuple[None, bool]] = WeakKeyDictionary()
def request_fingerprint_2_6(request, include_headers=None, keep_fragments=False):
if include_headers:
include_headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers))
cache = _fingerprint_cache_2_6.setdefault(request, {})
cache_key = (include_headers, keep_fragments)
if cache_key not in cache:
fp = sha1()
fp.update(to_bytes(request.method))
fp.update(
to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments))
)
fp.update(request.body or b"")
if include_headers:
for hdr in include_headers:
if hdr in request.headers:
fp.update(hdr)
for v in request.headers.getlist(hdr):
fp.update(v)
cache[cache_key] = fp.hexdigest()
return cache[cache_key]
REQUEST_OBJECTS_TO_TEST = (
Request("http://www.example.com/"),
Request("http://www.example.com/query?id=111&cat=222"),
@ -424,105 +254,17 @@ REQUEST_OBJECTS_TO_TEST = (
)
class BackwardCompatibilityTestCase(unittest.TestCase):
def test_function_backward_compatibility(self):
include_headers_to_test = (
None,
["Accept-Language"],
["accept-language", "sessionid"],
["SESSIONID", "Accept-Language"],
)
for request_object in REQUEST_OBJECTS_TO_TEST:
for include_headers in include_headers_to_test:
for keep_fragments in (False, True):
with warnings.catch_warnings():
warnings.simplefilter("ignore")
fp = request_fingerprint(
request_object,
include_headers=include_headers,
keep_fragments=keep_fragments,
)
old_fp = request_fingerprint_2_6(
request_object,
include_headers=include_headers,
keep_fragments=keep_fragments,
)
self.assertEqual(fp, old_fp)
def test_component_backward_compatibility(self):
for request_object in REQUEST_OBJECTS_TO_TEST:
with warnings.catch_warnings():
warnings.simplefilter("ignore")
crawler = get_crawler(prevent_warnings=False)
fp = crawler.request_fingerprinter.fingerprint(request_object)
old_fp = request_fingerprint_2_6(request_object)
self.assertEqual(fp.hex(), old_fp)
def test_custom_component_backward_compatibility(self):
"""Tests that the backward-compatible request fingerprinting class featured
in the documentation is indeed backward compatible and does not cause a
warning to be logged."""
class RequestFingerprinter:
cache = WeakKeyDictionary()
def fingerprint(self, request):
if request not in self.cache:
fp = sha1()
fp.update(to_bytes(request.method))
fp.update(to_bytes(canonicalize_url(request.url)))
fp.update(request.body or b"")
self.cache[request] = fp.digest()
return self.cache[request]
for request_object in REQUEST_OBJECTS_TO_TEST:
with warnings.catch_warnings() as logged_warnings:
settings = {
"REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter,
}
crawler = get_crawler(settings_dict=settings)
fp = crawler.request_fingerprinter.fingerprint(request_object)
old_fp = request_fingerprint_2_6(request_object)
self.assertEqual(fp.hex(), old_fp)
self.assertFalse(logged_warnings)
class RequestFingerprinterTestCase(unittest.TestCase):
def test_default_implementation(self):
with warnings.catch_warnings(record=True) as logged_warnings:
crawler = get_crawler(prevent_warnings=False)
request = Request("https://example.com")
self.assertEqual(
crawler.request_fingerprinter.fingerprint(request),
_request_fingerprint_as_bytes(request),
)
self.assertTrue(logged_warnings)
def test_deprecated_implementation(self):
settings = {
"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.6",
}
with warnings.catch_warnings(record=True) as logged_warnings:
crawler = get_crawler(settings_dict=settings)
request = Request("https://example.com")
self.assertEqual(
crawler.request_fingerprinter.fingerprint(request),
_request_fingerprint_as_bytes(request),
)
self.assertTrue(logged_warnings)
def test_recommended_implementation(self):
settings = {
"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
}
with warnings.catch_warnings(record=True) as logged_warnings:
crawler = get_crawler(settings_dict=settings)
crawler = get_crawler(settings_dict=settings)
request = Request("https://example.com")
self.assertEqual(
crawler.request_fingerprinter.fingerprint(request),
fingerprint(request),
)
self.assertFalse(logged_warnings)
def test_unknown_implementation(self):
settings = {