Simplify the logic

This commit is contained in:
Laerte Pereira 2024-01-31 12:53:08 -03:00
parent 24634f1bb2
commit 7001193c80
3 changed files with 5 additions and 17 deletions

View File

@ -143,21 +143,13 @@ class RequestFingerprinter:
else:
implementation = "SENTINEL"
if implementation == "SENTINEL":
self._fingerprint = fingerprint
elif implementation == "2.7":
if implementation != "SENTINEL":
message = (
"'REQUEST_FINGERPRINTER_IMPLEMENTATION' is a deprecated setting.\n"
"And it will be removed in future version of Scrapy."
)
warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2)
self._fingerprint = fingerprint
else:
raise ValueError(
f"Got an invalid value on setting "
f"'REQUEST_FINGERPRINTER_IMPLEMENTATION': "
f"{implementation!r}. Valid values are '2.7' and 'SENTINEL'."
)
self._fingerprint = fingerprint
def fingerprint(self, request: Request) -> bytes:
return self._fingerprint(request)

View File

@ -76,6 +76,7 @@ class TestSpider(Spider):
def get_crawler(
spidercls: Optional[Type[Spider]] = None,
settings_dict: Optional[Dict[str, Any]] = None,
prevent_warnings: bool = True,
) -> Crawler:
"""Return an unconfigured Crawler object. If settings_dict is given, it
will be used to populate the crawler settings with a project level
@ -85,6 +86,8 @@ def get_crawler(
# Set by default settings that prevent deprecation warnings.
settings: Dict[str, Any] = {}
if prevent_warnings:
pass
settings.update(settings_dict or {})
runner = CrawlerRunner(settings)
crawler = runner.create_crawler(spidercls or TestSpider)

View File

@ -277,13 +277,6 @@ class RequestFingerprinterTestCase(unittest.TestCase):
)
self.assertTrue(logged_warnings)
def test_unknown_implementation(self):
settings = {
"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.5",
}
with self.assertRaises(ValueError):
get_crawler(settings_dict=settings)
class CustomRequestFingerprinterTestCase(unittest.TestCase):
def test_include_headers(self):