mirror of https://github.com/scrapy/scrapy.git
Simplify the logic
This commit is contained in:
parent
24634f1bb2
commit
7001193c80
|
|
@ -143,21 +143,13 @@ class RequestFingerprinter:
|
|||
else:
|
||||
implementation = "SENTINEL"
|
||||
|
||||
if implementation == "SENTINEL":
|
||||
self._fingerprint = fingerprint
|
||||
elif implementation == "2.7":
|
||||
if implementation != "SENTINEL":
|
||||
message = (
|
||||
"'REQUEST_FINGERPRINTER_IMPLEMENTATION' is a deprecated setting.\n"
|
||||
"And it will be removed in future version of Scrapy."
|
||||
)
|
||||
warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2)
|
||||
self._fingerprint = fingerprint
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Got an invalid value on setting "
|
||||
f"'REQUEST_FINGERPRINTER_IMPLEMENTATION': "
|
||||
f"{implementation!r}. Valid values are '2.7' and 'SENTINEL'."
|
||||
)
|
||||
self._fingerprint = fingerprint
|
||||
|
||||
def fingerprint(self, request: Request) -> bytes:
|
||||
return self._fingerprint(request)
|
||||
|
|
|
|||
|
|
@ -76,6 +76,7 @@ class TestSpider(Spider):
|
|||
def get_crawler(
|
||||
spidercls: Optional[Type[Spider]] = None,
|
||||
settings_dict: Optional[Dict[str, Any]] = None,
|
||||
prevent_warnings: bool = True,
|
||||
) -> Crawler:
|
||||
"""Return an unconfigured Crawler object. If settings_dict is given, it
|
||||
will be used to populate the crawler settings with a project level
|
||||
|
|
@ -85,6 +86,8 @@ def get_crawler(
|
|||
|
||||
# Set by default settings that prevent deprecation warnings.
|
||||
settings: Dict[str, Any] = {}
|
||||
if prevent_warnings:
|
||||
pass
|
||||
settings.update(settings_dict or {})
|
||||
runner = CrawlerRunner(settings)
|
||||
crawler = runner.create_crawler(spidercls or TestSpider)
|
||||
|
|
|
|||
|
|
@ -277,13 +277,6 @@ class RequestFingerprinterTestCase(unittest.TestCase):
|
|||
)
|
||||
self.assertTrue(logged_warnings)
|
||||
|
||||
def test_unknown_implementation(self):
|
||||
settings = {
|
||||
"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.5",
|
||||
}
|
||||
with self.assertRaises(ValueError):
|
||||
get_crawler(settings_dict=settings)
|
||||
|
||||
|
||||
class CustomRequestFingerprinterTestCase(unittest.TestCase):
|
||||
def test_include_headers(self):
|
||||
|
|
|
|||
Loading…
Reference in New Issue