import json import unittest import warnings from hashlib import sha1 from typing import Dict, Mapping, Optional, Tuple, Union from weakref import WeakKeyDictionary import pytest from w3lib.url import canonicalize_url from scrapy.http import Request from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.python import to_bytes from scrapy.utils.request import ( _deprecated_fingerprint_cache, _fingerprint_cache, _request_fingerprint_as_bytes, fingerprint, request_authenticate, request_fingerprint, request_httprepr, request_to_curl, ) from scrapy.utils.test import get_crawler class UtilsRequestTest(unittest.TestCase): def test_request_authenticate(self): r = Request("http://www.example.com") request_authenticate(r, "someuser", "somepass") self.assertEqual(r.headers["Authorization"], b"Basic c29tZXVzZXI6c29tZXBhc3M=") def test_request_httprepr(self): r1 = Request("http://www.example.com") self.assertEqual( request_httprepr(r1), b"GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n" ) r1 = Request("http://www.example.com/some/page.html?arg=1") self.assertEqual( request_httprepr(r1), b"GET /some/page.html?arg=1 HTTP/1.1\r\nHost: www.example.com\r\n\r\n", ) r1 = Request( "http://www.example.com", method="POST", headers={"Content-type": b"text/html"}, body=b"Some body", ) self.assertEqual( request_httprepr(r1), b"POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body", ) def test_request_httprepr_for_non_http_request(self): # the representation is not important but it must not fail. request_httprepr(Request("file:///tmp/foo.txt")) request_httprepr(Request("ftp://localhost/tmp/foo.txt")) class FingerprintTest(unittest.TestCase): maxDiff = None function: staticmethod = staticmethod(fingerprint) cache: Union[ "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]", "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]", ] = _fingerprint_cache default_cache_key = (None, False) known_hashes: Tuple[Tuple[Request, Union[bytes, str], Dict], ...] = ( ( Request("http://example.org"), b"xs\xd7\x0c3uj\x15\xfe\xd7d\x9b\xa9\t\xe0d\xbf\x9cXD", {}, ), ( Request("https://example.org"), b"\xc04\x85P,\xaa\x91\x06\xf8t\xb4\xbd*\xd9\xe9\x8a:m\xc3l", {}, ), ( Request("https://example.org?a"), b"G\xad\xb8Ck\x19\x1c\xed\x838,\x01\xc4\xde;\xee\xa5\x94a\x0c", {}, ), ( Request("https://example.org?a=b"), b"\x024MYb\x8a\xc2\x1e\xbc>\xd6\xac*\xda\x9cF\xc1r\x7f\x17", {}, ), ( Request("https://example.org?a=b&a"), b"t+\xe8*\xfb\x84\xe3v\x1a}\x88p\xc0\xccB\xd7\x9d\xfez\x96", {}, ), ( Request("https://example.org?a=b&a=c"), b"\xda\x1ec\xd0\x9c\x08s`\xb4\x9b\xe2\xb6R\xf8k\xef\xeaQG\xef", {}, ), ( Request("https://example.org", method="POST"), b"\x9d\xcdA\x0fT\x02:\xca\xa0}\x90\xda\x05B\xded\x8aN7\x1d", {}, ), ( Request("https://example.org", body=b"a"), b"\xc34z>\xd8\x99\x8b\xda7\x05r\x99I\xa8\xa0x;\xa41_", {}, ), ( Request("https://example.org", method="POST", body=b"a"), b"5`\xe2y4\xd0\x9d\xee\xe0\xbatw\x87Q\xe8O\xd78\xfc\xe7", {}, ), ( Request("https://example.org#a", headers={"A": b"B"}), b"\xc04\x85P,\xaa\x91\x06\xf8t\xb4\xbd*\xd9\xe9\x8a:m\xc3l", {}, ), ( Request("https://example.org#a", headers={"A": b"B"}), b"]\xc7\x1f\xf2\xafG2\xbc\xa4\xfa\x99\n33\xda\x18\x94\x81U.", {"include_headers": ["A"]}, ), ( Request("https://example.org#a", headers={"A": b"B"}), b"<\x1a\xeb\x85y\xdeW\xfb\xdcq\x88\xee\xaf\x17\xdd\x0c\xbfH\x18\x1f", {"keep_fragments": True}, ), ( Request("https://example.org#a", headers={"A": b"B"}), b"\xc1\xef~\x94\x9bS\xc1\x83\t\xdcz8\x9f\xdc{\x11\x16I.\x11", {"include_headers": ["A"], "keep_fragments": True}, ), ( Request("https://example.org/ab"), b"N\xe5l\xb8\x12@iw\xe2\xf3\x1bp\xea\xffp!u\xe2\x8a\xc6", {}, ), ( Request("https://example.org/a", body=b"b"), b"_NOv\xbco$6\xfcW\x9f\xb24g\x9f\xbb\xdd\xa82\xc5", {}, ), ) def test_query_string_key_order(self): r1 = Request("http://www.example.com/query?id=111&cat=222") r2 = Request("http://www.example.com/query?cat=222&id=111") self.assertEqual(self.function(r1), self.function(r1)) self.assertEqual(self.function(r1), self.function(r2)) def test_query_string_key_without_value(self): r1 = Request("http://www.example.com/hnnoticiaj1.aspx?78132,199") r2 = Request("http://www.example.com/hnnoticiaj1.aspx?78160,199") self.assertNotEqual(self.function(r1), self.function(r2)) def test_caching(self): r1 = Request("http://www.example.com/hnnoticiaj1.aspx?78160,199") self.assertEqual(self.function(r1), self.cache[r1][self.default_cache_key]) def test_header(self): r1 = Request("http://www.example.com/members/offers.html") r2 = Request("http://www.example.com/members/offers.html") r2.headers["SESSIONID"] = b"somehash" self.assertEqual(self.function(r1), self.function(r2)) def test_headers(self): r1 = Request("http://www.example.com/") r2 = Request("http://www.example.com/") r2.headers["Accept-Language"] = b"en" r3 = Request("http://www.example.com/") r3.headers["Accept-Language"] = b"en" r3.headers["SESSIONID"] = b"somehash" self.assertEqual(self.function(r1), self.function(r2), self.function(r3)) self.assertEqual( self.function(r1), self.function(r1, include_headers=["Accept-Language"]) ) self.assertNotEqual( self.function(r1), self.function(r2, include_headers=["Accept-Language"]) ) self.assertEqual( self.function(r3, include_headers=["accept-language", "sessionid"]), self.function(r3, include_headers=["SESSIONID", "Accept-Language"]), ) def test_fragment(self): r1 = Request("http://www.example.com/test.html") r2 = Request("http://www.example.com/test.html#fragment") self.assertEqual(self.function(r1), self.function(r2)) self.assertEqual(self.function(r1), self.function(r1, keep_fragments=True)) self.assertNotEqual(self.function(r2), self.function(r2, keep_fragments=True)) self.assertNotEqual(self.function(r1), self.function(r2, keep_fragments=True)) def test_method_and_body(self): r1 = Request("http://www.example.com") r2 = Request("http://www.example.com", method="POST") r3 = Request("http://www.example.com", method="POST", body=b"request body") self.assertNotEqual(self.function(r1), self.function(r2)) self.assertNotEqual(self.function(r2), self.function(r3)) def test_request_replace(self): # cached fingerprint must be cleared on request copy r1 = Request("http://www.example.com") fp1 = self.function(r1) r2 = r1.replace(url="http://www.example.com/other") fp2 = self.function(r2) self.assertNotEqual(fp1, fp2) def test_part_separation(self): # An old implementation used to serialize request data in a way that # would put the body right after the URL. r1 = Request("http://www.example.com/foo") fp1 = self.function(r1) r2 = Request("http://www.example.com/f", body=b"oo") fp2 = self.function(r2) self.assertNotEqual(fp1, fp2) def test_hashes(self): """Test hardcoded hashes, to make sure future changes to not introduce backward incompatibilities.""" actual = [ self.function(request, **kwargs) for request, _, kwargs in self.known_hashes ] expected = [_fingerprint for _, _fingerprint, _ in self.known_hashes] self.assertEqual(actual, expected) class RequestFingerprintTest(FingerprintTest): function = staticmethod(request_fingerprint) cache = _deprecated_fingerprint_cache known_hashes: Tuple[Tuple[Request, Union[bytes, str], Dict], ...] = ( ( Request("http://example.org"), "b2e5245ef826fd9576c93bd6e392fce3133fab62", {}, ), ( Request("https://example.org"), "bd10a0a89ea32cdee77917320f1309b0da87e892", {}, ), ( Request("https://example.org?a"), "2fb7d48ae02f04b749f40caa969c0bc3c43204ce", {}, ), ( Request("https://example.org?a=b"), "42e5fe149b147476e3f67ad0670c57b4cc57856a", {}, ), ( Request("https://example.org?a=b&a"), "d23a9787cb56c6375c2cae4453c5a8c634526942", {}, ), ( Request("https://example.org?a=b&a=c"), "9a18a7a8552a9182b7f1e05d33876409e421e5c5", {}, ), ( Request("https://example.org", method="POST"), "ba20a80cb5c5ca460021ceefb3c2467b2bfd1bc6", {}, ), ( Request("https://example.org", body=b"a"), "4bb136e54e715a4ea7a9dd1101831765d33f2d60", {}, ), ( Request("https://example.org", method="POST", body=b"a"), "6c6595374a304b293be762f7b7be3f54e9947c65", {}, ), ( Request("https://example.org#a", headers={"A": b"B"}), "bd10a0a89ea32cdee77917320f1309b0da87e892", {}, ), ( Request("https://example.org#a", headers={"A": b"B"}), "515b633cb3ca502a33a9d8c890e889ec1e425e65", {"include_headers": ["A"]}, ), ( Request("https://example.org#a", headers={"A": b"B"}), "505c96e7da675920dfef58725e8c957dfdb38f47", {"keep_fragments": True}, ), ( Request("https://example.org#a", headers={"A": b"B"}), "d6f673cdcb661b7970c2b9a00ee63e87d1e2e5da", {"include_headers": ["A"], "keep_fragments": True}, ), ( Request("https://example.org/ab"), "4e2870fee58582d6f81755e9b8fdefe3cba0c951", {}, ), ( Request("https://example.org/a", body=b"b"), "4e2870fee58582d6f81755e9b8fdefe3cba0c951", {}, ), ) def setUp(self) -> None: warnings.simplefilter("ignore", ScrapyDeprecationWarning) def tearDown(self) -> None: warnings.simplefilter("default", ScrapyDeprecationWarning) @pytest.mark.xfail(reason="known bug kept for backward compatibility", strict=True) def test_part_separation(self): super().test_part_separation() class RequestFingerprintDeprecationTest(unittest.TestCase): def test_deprecation_default_parameters(self): with pytest.warns(ScrapyDeprecationWarning) as warnings: request_fingerprint(Request("http://www.example.com")) messages = [str(warning.message) for warning in warnings] self.assertTrue( any("Call to deprecated function" in message for message in messages) ) self.assertFalse(any("non-default" in message for message in messages)) def test_deprecation_non_default_parameters(self): with pytest.warns(ScrapyDeprecationWarning) as warnings: request_fingerprint(Request("http://www.example.com"), keep_fragments=True) messages = [str(warning.message) for warning in warnings] self.assertTrue( any("Call to deprecated function" in message for message in messages) ) self.assertTrue(any("non-default" in message for message in messages)) class RequestFingerprintAsBytesTest(FingerprintTest): function = staticmethod(_request_fingerprint_as_bytes) cache = _deprecated_fingerprint_cache known_hashes = RequestFingerprintTest.known_hashes def test_caching(self): r1 = Request("http://www.example.com/hnnoticiaj1.aspx?78160,199") self.assertEqual( self.function(r1), bytes.fromhex(self.cache[r1][self.default_cache_key]) ) @pytest.mark.xfail(reason="known bug kept for backward compatibility", strict=True) def test_part_separation(self): super().test_part_separation() def test_hashes(self): actual = [ self.function(request, **kwargs) for request, _, kwargs in self.known_hashes ] expected = [ bytes.fromhex(_fingerprint) for _, _fingerprint, _ in self.known_hashes ] self.assertEqual(actual, expected) _fingerprint_cache_2_6: Mapping[Request, Tuple[None, bool]] = WeakKeyDictionary() def request_fingerprint_2_6(request, include_headers=None, keep_fragments=False): if include_headers: include_headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers)) cache = _fingerprint_cache_2_6.setdefault(request, {}) cache_key = (include_headers, keep_fragments) if cache_key not in cache: fp = sha1() fp.update(to_bytes(request.method)) fp.update( to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments)) ) fp.update(request.body or b"") if include_headers: for hdr in include_headers: if hdr in request.headers: fp.update(hdr) for v in request.headers.getlist(hdr): fp.update(v) cache[cache_key] = fp.hexdigest() return cache[cache_key] REQUEST_OBJECTS_TO_TEST = ( Request("http://www.example.com/"), Request("http://www.example.com/query?id=111&cat=222"), Request("http://www.example.com/query?cat=222&id=111"), Request("http://www.example.com/hnnoticiaj1.aspx?78132,199"), Request("http://www.example.com/hnnoticiaj1.aspx?78160,199"), Request("http://www.example.com/members/offers.html"), Request( "http://www.example.com/members/offers.html", headers={"SESSIONID": b"somehash"}, ), Request( "http://www.example.com/", headers={"Accept-Language": b"en"}, ), Request( "http://www.example.com/", headers={ "Accept-Language": b"en", "SESSIONID": b"somehash", }, ), Request("http://www.example.com/test.html"), Request("http://www.example.com/test.html#fragment"), Request("http://www.example.com", method="POST"), Request("http://www.example.com", method="POST", body=b"request body"), ) class BackwardCompatibilityTestCase(unittest.TestCase): def test_function_backward_compatibility(self): include_headers_to_test = ( None, ["Accept-Language"], ["accept-language", "sessionid"], ["SESSIONID", "Accept-Language"], ) for request_object in REQUEST_OBJECTS_TO_TEST: for include_headers in include_headers_to_test: for keep_fragments in (False, True): with warnings.catch_warnings(): warnings.simplefilter("ignore") fp = request_fingerprint( request_object, include_headers=include_headers, keep_fragments=keep_fragments, ) old_fp = request_fingerprint_2_6( request_object, include_headers=include_headers, keep_fragments=keep_fragments, ) self.assertEqual(fp, old_fp) def test_component_backward_compatibility(self): for request_object in REQUEST_OBJECTS_TO_TEST: with warnings.catch_warnings(): warnings.simplefilter("ignore") crawler = get_crawler(prevent_warnings=False) fp = crawler.request_fingerprinter.fingerprint(request_object) old_fp = request_fingerprint_2_6(request_object) self.assertEqual(fp.hex(), old_fp) def test_custom_component_backward_compatibility(self): """Tests that the backward-compatible request fingerprinting class featured in the documentation is indeed backward compatible and does not cause a warning to be logged.""" class RequestFingerprinter: cache = WeakKeyDictionary() def fingerprint(self, request): if request not in self.cache: fp = sha1() fp.update(to_bytes(request.method)) fp.update(to_bytes(canonicalize_url(request.url))) fp.update(request.body or b"") self.cache[request] = fp.digest() return self.cache[request] for request_object in REQUEST_OBJECTS_TO_TEST: with warnings.catch_warnings() as logged_warnings: settings = { "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter, } crawler = get_crawler(settings_dict=settings) fp = crawler.request_fingerprinter.fingerprint(request_object) old_fp = request_fingerprint_2_6(request_object) self.assertEqual(fp.hex(), old_fp) self.assertFalse(logged_warnings) class RequestFingerprinterTestCase(unittest.TestCase): def test_default_implementation(self): with warnings.catch_warnings(record=True) as logged_warnings: crawler = get_crawler(prevent_warnings=False) request = Request("https://example.com") self.assertEqual( crawler.request_fingerprinter.fingerprint(request), _request_fingerprint_as_bytes(request), ) self.assertTrue(logged_warnings) def test_deprecated_implementation(self): settings = { "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.6", } with warnings.catch_warnings(record=True) as logged_warnings: crawler = get_crawler(settings_dict=settings) request = Request("https://example.com") self.assertEqual( crawler.request_fingerprinter.fingerprint(request), _request_fingerprint_as_bytes(request), ) self.assertTrue(logged_warnings) def test_recommended_implementation(self): settings = { "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", } with warnings.catch_warnings(record=True) as logged_warnings: crawler = get_crawler(settings_dict=settings) request = Request("https://example.com") self.assertEqual( crawler.request_fingerprinter.fingerprint(request), fingerprint(request), ) self.assertFalse(logged_warnings) def test_unknown_implementation(self): settings = { "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.5", } with self.assertRaises(ValueError): get_crawler(settings_dict=settings) class CustomRequestFingerprinterTestCase(unittest.TestCase): def test_include_headers(self): class RequestFingerprinter: def fingerprint(self, request): return fingerprint(request, include_headers=["X-ID"]) settings = { "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter, } crawler = get_crawler(settings_dict=settings) r1 = Request("http://www.example.com", headers={"X-ID": "1"}) fp1 = crawler.request_fingerprinter.fingerprint(r1) r2 = Request("http://www.example.com", headers={"X-ID": "2"}) fp2 = crawler.request_fingerprinter.fingerprint(r2) self.assertNotEqual(fp1, fp2) def test_dont_canonicalize(self): class RequestFingerprinter: cache = WeakKeyDictionary() def fingerprint(self, request): if request not in self.cache: fp = sha1() fp.update(to_bytes(request.url)) self.cache[request] = fp.digest() return self.cache[request] settings = { "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter, } crawler = get_crawler(settings_dict=settings) r1 = Request("http://www.example.com?a=1&a=2") fp1 = crawler.request_fingerprinter.fingerprint(r1) r2 = Request("http://www.example.com?a=2&a=1") fp2 = crawler.request_fingerprinter.fingerprint(r2) self.assertNotEqual(fp1, fp2) def test_meta(self): class RequestFingerprinter: def fingerprint(self, request): if "fingerprint" in request.meta: return request.meta["fingerprint"] return fingerprint(request) settings = { "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter, } crawler = get_crawler(settings_dict=settings) r1 = Request("http://www.example.com") fp1 = crawler.request_fingerprinter.fingerprint(r1) r2 = Request("http://www.example.com", meta={"fingerprint": "a"}) fp2 = crawler.request_fingerprinter.fingerprint(r2) r3 = Request("http://www.example.com", meta={"fingerprint": "a"}) fp3 = crawler.request_fingerprinter.fingerprint(r3) r4 = Request("http://www.example.com", meta={"fingerprint": "b"}) fp4 = crawler.request_fingerprinter.fingerprint(r4) self.assertNotEqual(fp1, fp2) self.assertNotEqual(fp1, fp4) self.assertNotEqual(fp2, fp4) self.assertEqual(fp2, fp3) def test_from_crawler(self): class RequestFingerprinter: @classmethod def from_crawler(cls, crawler): return cls(crawler) def __init__(self, crawler): self._fingerprint = crawler.settings["FINGERPRINT"] def fingerprint(self, request): return self._fingerprint settings = { "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter, "FINGERPRINT": b"fingerprint", } crawler = get_crawler(settings_dict=settings) request = Request("http://www.example.com") fingerprint = crawler.request_fingerprinter.fingerprint(request) self.assertEqual(fingerprint, settings["FINGERPRINT"]) def test_from_settings(self): class RequestFingerprinter: @classmethod def from_settings(cls, settings): return cls(settings) def __init__(self, settings): self._fingerprint = settings["FINGERPRINT"] def fingerprint(self, request): return self._fingerprint settings = { "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter, "FINGERPRINT": b"fingerprint", } crawler = get_crawler(settings_dict=settings) request = Request("http://www.example.com") fingerprint = crawler.request_fingerprinter.fingerprint(request) self.assertEqual(fingerprint, settings["FINGERPRINT"]) def test_from_crawler_and_settings(self): class RequestFingerprinter: # This method is ignored due to the presence of from_crawler @classmethod def from_settings(cls, settings): return cls(settings) @classmethod def from_crawler(cls, crawler): return cls(crawler) def __init__(self, crawler): self._fingerprint = crawler.settings["FINGERPRINT"] def fingerprint(self, request): return self._fingerprint settings = { "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter, "FINGERPRINT": b"fingerprint", } crawler = get_crawler(settings_dict=settings) request = Request("http://www.example.com") fingerprint = crawler.request_fingerprinter.fingerprint(request) self.assertEqual(fingerprint, settings["FINGERPRINT"]) class RequestToCurlTest(unittest.TestCase): def _test_request(self, request_object, expected_curl_command): curl_command = request_to_curl(request_object) self.assertEqual(curl_command, expected_curl_command) def test_get(self): request_object = Request("https://www.example.com") expected_curl_command = "curl -X GET https://www.example.com" self._test_request(request_object, expected_curl_command) def test_post(self): request_object = Request( "https://www.httpbin.org/post", method="POST", body=json.dumps({"foo": "bar"}), ) expected_curl_command = ( 'curl -X POST https://www.httpbin.org/post --data-raw \'{"foo": "bar"}\'' ) self._test_request(request_object, expected_curl_command) def test_headers(self): request_object = Request( "https://www.httpbin.org/post", method="POST", headers={"Content-Type": "application/json", "Accept": "application/json"}, body=json.dumps({"foo": "bar"}), ) expected_curl_command = ( "curl -X POST https://www.httpbin.org/post" ' --data-raw \'{"foo": "bar"}\'' " -H 'Content-Type: application/json' -H 'Accept: application/json'" ) self._test_request(request_object, expected_curl_command) def test_cookies_dict(self): request_object = Request( "https://www.httpbin.org/post", method="POST", cookies={"foo": "bar"}, body=json.dumps({"foo": "bar"}), ) expected_curl_command = ( "curl -X POST https://www.httpbin.org/post" " --data-raw '{\"foo\": \"bar\"}' --cookie 'foo=bar'" ) self._test_request(request_object, expected_curl_command) def test_cookies_list(self): request_object = Request( "https://www.httpbin.org/post", method="POST", cookies=[{"foo": "bar"}], body=json.dumps({"foo": "bar"}), ) expected_curl_command = ( "curl -X POST https://www.httpbin.org/post" " --data-raw '{\"foo\": \"bar\"}' --cookie 'foo=bar'" ) self._test_request(request_object, expected_curl_command) if __name__ == "__main__": unittest.main()