mirror of https://github.com/scrapy/scrapy.git
757 lines
28 KiB
Python
757 lines
28 KiB
Python
import json
|
|
import warnings
|
|
from hashlib import sha1
|
|
from typing import Dict, Mapping, Optional, Tuple, Union
|
|
from weakref import WeakKeyDictionary
|
|
|
|
import pytest
|
|
from twisted.internet.defer import inlineCallbacks
|
|
from twisted.trial import unittest
|
|
from w3lib.url import canonicalize_url
|
|
|
|
from scrapy.http import Request
|
|
from scrapy.utils.deprecate import ScrapyDeprecationWarning
|
|
from scrapy.utils.python import to_bytes
|
|
from scrapy.utils.request import (
|
|
_deprecated_fingerprint_cache,
|
|
_fingerprint_cache,
|
|
_request_fingerprint_as_bytes,
|
|
fingerprint,
|
|
request_authenticate,
|
|
request_fingerprint,
|
|
request_httprepr,
|
|
request_to_curl,
|
|
)
|
|
from scrapy.utils.test import get_crawler
|
|
from tests.spiders import NoRequestsSpider
|
|
|
|
|
|
class UtilsRequestTest(unittest.TestCase):
|
|
def test_request_authenticate(self):
|
|
r = Request("http://www.example.com")
|
|
request_authenticate(r, "someuser", "somepass")
|
|
self.assertEqual(r.headers["Authorization"], b"Basic c29tZXVzZXI6c29tZXBhc3M=")
|
|
|
|
def test_request_httprepr(self):
|
|
r1 = Request("http://www.example.com")
|
|
self.assertEqual(
|
|
request_httprepr(r1), b"GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n"
|
|
)
|
|
|
|
r1 = Request("http://www.example.com/some/page.html?arg=1")
|
|
self.assertEqual(
|
|
request_httprepr(r1),
|
|
b"GET /some/page.html?arg=1 HTTP/1.1\r\nHost: www.example.com\r\n\r\n",
|
|
)
|
|
|
|
r1 = Request(
|
|
"http://www.example.com",
|
|
method="POST",
|
|
headers={"Content-type": b"text/html"},
|
|
body=b"Some body",
|
|
)
|
|
self.assertEqual(
|
|
request_httprepr(r1),
|
|
b"POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body",
|
|
)
|
|
|
|
def test_request_httprepr_for_non_http_request(self):
|
|
# the representation is not important but it must not fail.
|
|
request_httprepr(Request("file:///tmp/foo.txt"))
|
|
request_httprepr(Request("ftp://localhost/tmp/foo.txt"))
|
|
|
|
|
|
class FingerprintTest(unittest.TestCase):
|
|
maxDiff = None
|
|
|
|
function: staticmethod = staticmethod(fingerprint)
|
|
cache: Union[
|
|
"WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]",
|
|
"WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]",
|
|
] = _fingerprint_cache
|
|
default_cache_key = (None, False)
|
|
known_hashes: Tuple[Tuple[Request, Union[bytes, str], Dict], ...] = (
|
|
(
|
|
Request("http://example.org"),
|
|
b"xs\xd7\x0c3uj\x15\xfe\xd7d\x9b\xa9\t\xe0d\xbf\x9cXD",
|
|
{},
|
|
),
|
|
(
|
|
Request("https://example.org"),
|
|
b"\xc04\x85P,\xaa\x91\x06\xf8t\xb4\xbd*\xd9\xe9\x8a:m\xc3l",
|
|
{},
|
|
),
|
|
(
|
|
Request("https://example.org?a"),
|
|
b"G\xad\xb8Ck\x19\x1c\xed\x838,\x01\xc4\xde;\xee\xa5\x94a\x0c",
|
|
{},
|
|
),
|
|
(
|
|
Request("https://example.org?a=b"),
|
|
b"\x024MYb\x8a\xc2\x1e\xbc>\xd6\xac*\xda\x9cF\xc1r\x7f\x17",
|
|
{},
|
|
),
|
|
(
|
|
Request("https://example.org?a=b&a"),
|
|
b"t+\xe8*\xfb\x84\xe3v\x1a}\x88p\xc0\xccB\xd7\x9d\xfez\x96",
|
|
{},
|
|
),
|
|
(
|
|
Request("https://example.org?a=b&a=c"),
|
|
b"\xda\x1ec\xd0\x9c\x08s`\xb4\x9b\xe2\xb6R\xf8k\xef\xeaQG\xef",
|
|
{},
|
|
),
|
|
(
|
|
Request("https://example.org", method="POST"),
|
|
b"\x9d\xcdA\x0fT\x02:\xca\xa0}\x90\xda\x05B\xded\x8aN7\x1d",
|
|
{},
|
|
),
|
|
(
|
|
Request("https://example.org", body=b"a"),
|
|
b"\xc34z>\xd8\x99\x8b\xda7\x05r\x99I\xa8\xa0x;\xa41_",
|
|
{},
|
|
),
|
|
(
|
|
Request("https://example.org", method="POST", body=b"a"),
|
|
b"5`\xe2y4\xd0\x9d\xee\xe0\xbatw\x87Q\xe8O\xd78\xfc\xe7",
|
|
{},
|
|
),
|
|
(
|
|
Request("https://example.org#a", headers={"A": b"B"}),
|
|
b"\xc04\x85P,\xaa\x91\x06\xf8t\xb4\xbd*\xd9\xe9\x8a:m\xc3l",
|
|
{},
|
|
),
|
|
(
|
|
Request("https://example.org#a", headers={"A": b"B"}),
|
|
b"]\xc7\x1f\xf2\xafG2\xbc\xa4\xfa\x99\n33\xda\x18\x94\x81U.",
|
|
{"include_headers": ["A"]},
|
|
),
|
|
(
|
|
Request("https://example.org#a", headers={"A": b"B"}),
|
|
b"<\x1a\xeb\x85y\xdeW\xfb\xdcq\x88\xee\xaf\x17\xdd\x0c\xbfH\x18\x1f",
|
|
{"keep_fragments": True},
|
|
),
|
|
(
|
|
Request("https://example.org#a", headers={"A": b"B"}),
|
|
b"\xc1\xef~\x94\x9bS\xc1\x83\t\xdcz8\x9f\xdc{\x11\x16I.\x11",
|
|
{"include_headers": ["A"], "keep_fragments": True},
|
|
),
|
|
(
|
|
Request("https://example.org/ab"),
|
|
b"N\xe5l\xb8\x12@iw\xe2\xf3\x1bp\xea\xffp!u\xe2\x8a\xc6",
|
|
{},
|
|
),
|
|
(
|
|
Request("https://example.org/a", body=b"b"),
|
|
b"_NOv\xbco$6\xfcW\x9f\xb24g\x9f\xbb\xdd\xa82\xc5",
|
|
{},
|
|
),
|
|
)
|
|
|
|
def test_query_string_key_order(self):
|
|
r1 = Request("http://www.example.com/query?id=111&cat=222")
|
|
r2 = Request("http://www.example.com/query?cat=222&id=111")
|
|
self.assertEqual(self.function(r1), self.function(r1))
|
|
self.assertEqual(self.function(r1), self.function(r2))
|
|
|
|
def test_query_string_key_without_value(self):
|
|
r1 = Request("http://www.example.com/hnnoticiaj1.aspx?78132,199")
|
|
r2 = Request("http://www.example.com/hnnoticiaj1.aspx?78160,199")
|
|
self.assertNotEqual(self.function(r1), self.function(r2))
|
|
|
|
def test_caching(self):
|
|
r1 = Request("http://www.example.com/hnnoticiaj1.aspx?78160,199")
|
|
self.assertEqual(self.function(r1), self.cache[r1][self.default_cache_key])
|
|
|
|
def test_header(self):
|
|
r1 = Request("http://www.example.com/members/offers.html")
|
|
r2 = Request("http://www.example.com/members/offers.html")
|
|
r2.headers["SESSIONID"] = b"somehash"
|
|
self.assertEqual(self.function(r1), self.function(r2))
|
|
|
|
def test_headers(self):
|
|
r1 = Request("http://www.example.com/")
|
|
r2 = Request("http://www.example.com/")
|
|
r2.headers["Accept-Language"] = b"en"
|
|
r3 = Request("http://www.example.com/")
|
|
r3.headers["Accept-Language"] = b"en"
|
|
r3.headers["SESSIONID"] = b"somehash"
|
|
|
|
self.assertEqual(self.function(r1), self.function(r2), self.function(r3))
|
|
|
|
self.assertEqual(
|
|
self.function(r1), self.function(r1, include_headers=["Accept-Language"])
|
|
)
|
|
|
|
self.assertNotEqual(
|
|
self.function(r1), self.function(r2, include_headers=["Accept-Language"])
|
|
)
|
|
|
|
self.assertEqual(
|
|
self.function(r3, include_headers=["accept-language", "sessionid"]),
|
|
self.function(r3, include_headers=["SESSIONID", "Accept-Language"]),
|
|
)
|
|
|
|
def test_fragment(self):
|
|
r1 = Request("http://www.example.com/test.html")
|
|
r2 = Request("http://www.example.com/test.html#fragment")
|
|
self.assertEqual(self.function(r1), self.function(r2))
|
|
self.assertEqual(self.function(r1), self.function(r1, keep_fragments=True))
|
|
self.assertNotEqual(self.function(r2), self.function(r2, keep_fragments=True))
|
|
self.assertNotEqual(self.function(r1), self.function(r2, keep_fragments=True))
|
|
|
|
def test_method_and_body(self):
|
|
r1 = Request("http://www.example.com")
|
|
r2 = Request("http://www.example.com", method="POST")
|
|
r3 = Request("http://www.example.com", method="POST", body=b"request body")
|
|
|
|
self.assertNotEqual(self.function(r1), self.function(r2))
|
|
self.assertNotEqual(self.function(r2), self.function(r3))
|
|
|
|
def test_request_replace(self):
|
|
# cached fingerprint must be cleared on request copy
|
|
r1 = Request("http://www.example.com")
|
|
fp1 = self.function(r1)
|
|
r2 = r1.replace(url="http://www.example.com/other")
|
|
fp2 = self.function(r2)
|
|
self.assertNotEqual(fp1, fp2)
|
|
|
|
def test_part_separation(self):
|
|
# An old implementation used to serialize request data in a way that
|
|
# would put the body right after the URL.
|
|
r1 = Request("http://www.example.com/foo")
|
|
fp1 = self.function(r1)
|
|
r2 = Request("http://www.example.com/f", body=b"oo")
|
|
fp2 = self.function(r2)
|
|
self.assertNotEqual(fp1, fp2)
|
|
|
|
def test_hashes(self):
|
|
"""Test hardcoded hashes, to make sure future changes to not introduce
|
|
backward incompatibilities."""
|
|
actual = [
|
|
self.function(request, **kwargs) for request, _, kwargs in self.known_hashes
|
|
]
|
|
expected = [_fingerprint for _, _fingerprint, _ in self.known_hashes]
|
|
self.assertEqual(actual, expected)
|
|
|
|
|
|
class RequestFingerprintTest(FingerprintTest):
|
|
function = staticmethod(request_fingerprint)
|
|
cache = _deprecated_fingerprint_cache
|
|
known_hashes: Tuple[Tuple[Request, Union[bytes, str], Dict], ...] = (
|
|
(
|
|
Request("http://example.org"),
|
|
"b2e5245ef826fd9576c93bd6e392fce3133fab62",
|
|
{},
|
|
),
|
|
(
|
|
Request("https://example.org"),
|
|
"bd10a0a89ea32cdee77917320f1309b0da87e892",
|
|
{},
|
|
),
|
|
(
|
|
Request("https://example.org?a"),
|
|
"2fb7d48ae02f04b749f40caa969c0bc3c43204ce",
|
|
{},
|
|
),
|
|
(
|
|
Request("https://example.org?a=b"),
|
|
"42e5fe149b147476e3f67ad0670c57b4cc57856a",
|
|
{},
|
|
),
|
|
(
|
|
Request("https://example.org?a=b&a"),
|
|
"d23a9787cb56c6375c2cae4453c5a8c634526942",
|
|
{},
|
|
),
|
|
(
|
|
Request("https://example.org?a=b&a=c"),
|
|
"9a18a7a8552a9182b7f1e05d33876409e421e5c5",
|
|
{},
|
|
),
|
|
(
|
|
Request("https://example.org", method="POST"),
|
|
"ba20a80cb5c5ca460021ceefb3c2467b2bfd1bc6",
|
|
{},
|
|
),
|
|
(
|
|
Request("https://example.org", body=b"a"),
|
|
"4bb136e54e715a4ea7a9dd1101831765d33f2d60",
|
|
{},
|
|
),
|
|
(
|
|
Request("https://example.org", method="POST", body=b"a"),
|
|
"6c6595374a304b293be762f7b7be3f54e9947c65",
|
|
{},
|
|
),
|
|
(
|
|
Request("https://example.org#a", headers={"A": b"B"}),
|
|
"bd10a0a89ea32cdee77917320f1309b0da87e892",
|
|
{},
|
|
),
|
|
(
|
|
Request("https://example.org#a", headers={"A": b"B"}),
|
|
"515b633cb3ca502a33a9d8c890e889ec1e425e65",
|
|
{"include_headers": ["A"]},
|
|
),
|
|
(
|
|
Request("https://example.org#a", headers={"A": b"B"}),
|
|
"505c96e7da675920dfef58725e8c957dfdb38f47",
|
|
{"keep_fragments": True},
|
|
),
|
|
(
|
|
Request("https://example.org#a", headers={"A": b"B"}),
|
|
"d6f673cdcb661b7970c2b9a00ee63e87d1e2e5da",
|
|
{"include_headers": ["A"], "keep_fragments": True},
|
|
),
|
|
(
|
|
Request("https://example.org/ab"),
|
|
"4e2870fee58582d6f81755e9b8fdefe3cba0c951",
|
|
{},
|
|
),
|
|
(
|
|
Request("https://example.org/a", body=b"b"),
|
|
"4e2870fee58582d6f81755e9b8fdefe3cba0c951",
|
|
{},
|
|
),
|
|
)
|
|
|
|
def setUp(self) -> None:
|
|
warnings.simplefilter("ignore", ScrapyDeprecationWarning)
|
|
|
|
def tearDown(self) -> None:
|
|
warnings.simplefilter("default", ScrapyDeprecationWarning)
|
|
|
|
@pytest.mark.xfail(reason="known bug kept for backward compatibility", strict=True)
|
|
def test_part_separation(self):
|
|
super().test_part_separation()
|
|
|
|
|
|
class RequestFingerprintDeprecationTest(unittest.TestCase):
|
|
def test_deprecation_default_parameters(self):
|
|
with pytest.warns(ScrapyDeprecationWarning) as warnings:
|
|
request_fingerprint(Request("http://www.example.com"))
|
|
messages = [str(warning.message) for warning in warnings]
|
|
self.assertTrue(
|
|
any("Call to deprecated function" in message for message in messages)
|
|
)
|
|
self.assertFalse(any("non-default" in message for message in messages))
|
|
|
|
def test_deprecation_non_default_parameters(self):
|
|
with pytest.warns(ScrapyDeprecationWarning) as warnings:
|
|
request_fingerprint(Request("http://www.example.com"), keep_fragments=True)
|
|
messages = [str(warning.message) for warning in warnings]
|
|
self.assertTrue(
|
|
any("Call to deprecated function" in message for message in messages)
|
|
)
|
|
self.assertTrue(any("non-default" in message for message in messages))
|
|
|
|
|
|
class RequestFingerprintAsBytesTest(FingerprintTest):
|
|
function = staticmethod(_request_fingerprint_as_bytes)
|
|
cache = _deprecated_fingerprint_cache
|
|
known_hashes = RequestFingerprintTest.known_hashes
|
|
|
|
def test_caching(self):
|
|
r1 = Request("http://www.example.com/hnnoticiaj1.aspx?78160,199")
|
|
self.assertEqual(
|
|
self.function(r1), bytes.fromhex(self.cache[r1][self.default_cache_key])
|
|
)
|
|
|
|
@pytest.mark.xfail(reason="known bug kept for backward compatibility", strict=True)
|
|
def test_part_separation(self):
|
|
super().test_part_separation()
|
|
|
|
def test_hashes(self):
|
|
actual = [
|
|
self.function(request, **kwargs) for request, _, kwargs in self.known_hashes
|
|
]
|
|
expected = [
|
|
bytes.fromhex(_fingerprint) for _, _fingerprint, _ in self.known_hashes
|
|
]
|
|
self.assertEqual(actual, expected)
|
|
|
|
|
|
_fingerprint_cache_2_6: Mapping[Request, Tuple[None, bool]] = WeakKeyDictionary()
|
|
|
|
|
|
def request_fingerprint_2_6(request, include_headers=None, keep_fragments=False):
|
|
if include_headers:
|
|
include_headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers))
|
|
cache = _fingerprint_cache_2_6.setdefault(request, {})
|
|
cache_key = (include_headers, keep_fragments)
|
|
if cache_key not in cache:
|
|
fp = sha1()
|
|
fp.update(to_bytes(request.method))
|
|
fp.update(
|
|
to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments))
|
|
)
|
|
fp.update(request.body or b"")
|
|
if include_headers:
|
|
for hdr in include_headers:
|
|
if hdr in request.headers:
|
|
fp.update(hdr)
|
|
for v in request.headers.getlist(hdr):
|
|
fp.update(v)
|
|
cache[cache_key] = fp.hexdigest()
|
|
return cache[cache_key]
|
|
|
|
|
|
REQUEST_OBJECTS_TO_TEST = (
|
|
Request("http://www.example.com/"),
|
|
Request("http://www.example.com/query?id=111&cat=222"),
|
|
Request("http://www.example.com/query?cat=222&id=111"),
|
|
Request("http://www.example.com/hnnoticiaj1.aspx?78132,199"),
|
|
Request("http://www.example.com/hnnoticiaj1.aspx?78160,199"),
|
|
Request("http://www.example.com/members/offers.html"),
|
|
Request(
|
|
"http://www.example.com/members/offers.html",
|
|
headers={"SESSIONID": b"somehash"},
|
|
),
|
|
Request(
|
|
"http://www.example.com/",
|
|
headers={"Accept-Language": b"en"},
|
|
),
|
|
Request(
|
|
"http://www.example.com/",
|
|
headers={
|
|
"Accept-Language": b"en",
|
|
"SESSIONID": b"somehash",
|
|
},
|
|
),
|
|
Request("http://www.example.com/test.html"),
|
|
Request("http://www.example.com/test.html#fragment"),
|
|
Request("http://www.example.com", method="POST"),
|
|
Request("http://www.example.com", method="POST", body=b"request body"),
|
|
)
|
|
|
|
|
|
class BackwardCompatibilityTestCase(unittest.TestCase):
|
|
def test_function_backward_compatibility(self):
|
|
include_headers_to_test = (
|
|
None,
|
|
["Accept-Language"],
|
|
["accept-language", "sessionid"],
|
|
["SESSIONID", "Accept-Language"],
|
|
)
|
|
for request_object in REQUEST_OBJECTS_TO_TEST:
|
|
for include_headers in include_headers_to_test:
|
|
for keep_fragments in (False, True):
|
|
with warnings.catch_warnings():
|
|
warnings.simplefilter("ignore")
|
|
fp = request_fingerprint(
|
|
request_object,
|
|
include_headers=include_headers,
|
|
keep_fragments=keep_fragments,
|
|
)
|
|
old_fp = request_fingerprint_2_6(
|
|
request_object,
|
|
include_headers=include_headers,
|
|
keep_fragments=keep_fragments,
|
|
)
|
|
self.assertEqual(fp, old_fp)
|
|
|
|
@inlineCallbacks
|
|
def test_component_backward_compatibility(self):
|
|
for request_object in REQUEST_OBJECTS_TO_TEST:
|
|
with warnings.catch_warnings():
|
|
warnings.simplefilter("ignore")
|
|
crawler = get_crawler(NoRequestsSpider, prevent_warnings=False)
|
|
yield crawler.crawl()
|
|
fp = crawler.request_fingerprinter.fingerprint(request_object)
|
|
old_fp = request_fingerprint_2_6(request_object)
|
|
self.assertEqual(fp.hex(), old_fp)
|
|
|
|
@inlineCallbacks
|
|
def test_custom_component_backward_compatibility(self):
|
|
"""Tests that the backward-compatible request fingerprinting class featured
|
|
in the documentation is indeed backward compatible and does not cause a
|
|
warning to be logged."""
|
|
|
|
class RequestFingerprinter:
|
|
cache = WeakKeyDictionary()
|
|
|
|
def fingerprint(self, request):
|
|
if request not in self.cache:
|
|
fp = sha1()
|
|
fp.update(to_bytes(request.method))
|
|
fp.update(to_bytes(canonicalize_url(request.url)))
|
|
fp.update(request.body or b"")
|
|
self.cache[request] = fp.digest()
|
|
return self.cache[request]
|
|
|
|
for request_object in REQUEST_OBJECTS_TO_TEST:
|
|
with warnings.catch_warnings() as logged_warnings:
|
|
settings = {
|
|
"REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter,
|
|
}
|
|
crawler = get_crawler(NoRequestsSpider, settings_dict=settings)
|
|
yield crawler.crawl()
|
|
fp = crawler.request_fingerprinter.fingerprint(request_object)
|
|
old_fp = request_fingerprint_2_6(request_object)
|
|
self.assertEqual(fp.hex(), old_fp)
|
|
self.assertFalse(logged_warnings)
|
|
|
|
|
|
class RequestFingerprinterTestCase(unittest.TestCase):
|
|
@inlineCallbacks
|
|
def test_default_implementation(self):
|
|
with warnings.catch_warnings(record=True) as logged_warnings:
|
|
crawler = get_crawler(NoRequestsSpider, prevent_warnings=False)
|
|
yield crawler.crawl()
|
|
request = Request("https://example.com")
|
|
self.assertEqual(
|
|
crawler.request_fingerprinter.fingerprint(request),
|
|
_request_fingerprint_as_bytes(request),
|
|
)
|
|
self.assertTrue(logged_warnings)
|
|
|
|
@inlineCallbacks
|
|
def test_deprecated_implementation(self):
|
|
settings = {
|
|
"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.6",
|
|
}
|
|
with warnings.catch_warnings(record=True) as logged_warnings:
|
|
crawler = get_crawler(NoRequestsSpider, settings_dict=settings)
|
|
yield crawler.crawl()
|
|
request = Request("https://example.com")
|
|
self.assertEqual(
|
|
crawler.request_fingerprinter.fingerprint(request),
|
|
_request_fingerprint_as_bytes(request),
|
|
)
|
|
self.assertTrue(logged_warnings)
|
|
|
|
@inlineCallbacks
|
|
def test_recommended_implementation(self):
|
|
settings = {
|
|
"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
|
|
}
|
|
with warnings.catch_warnings(record=True) as logged_warnings:
|
|
crawler = get_crawler(NoRequestsSpider, settings_dict=settings)
|
|
yield crawler.crawl()
|
|
request = Request("https://example.com")
|
|
self.assertEqual(
|
|
crawler.request_fingerprinter.fingerprint(request),
|
|
fingerprint(request),
|
|
)
|
|
self.assertFalse(logged_warnings)
|
|
|
|
@inlineCallbacks
|
|
def test_unknown_implementation(self):
|
|
settings = {
|
|
"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.5",
|
|
}
|
|
with self.assertRaises(ValueError):
|
|
crawler = get_crawler(NoRequestsSpider, settings_dict=settings)
|
|
yield crawler.crawl()
|
|
|
|
|
|
class CustomRequestFingerprinterTestCase(unittest.TestCase):
|
|
@inlineCallbacks
|
|
def test_include_headers(self):
|
|
class RequestFingerprinter:
|
|
def fingerprint(self, request):
|
|
return fingerprint(request, include_headers=["X-ID"])
|
|
|
|
settings = {
|
|
"REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter,
|
|
}
|
|
crawler = get_crawler(NoRequestsSpider, settings_dict=settings)
|
|
yield crawler.crawl()
|
|
|
|
r1 = Request("http://www.example.com", headers={"X-ID": "1"})
|
|
fp1 = crawler.request_fingerprinter.fingerprint(r1)
|
|
r2 = Request("http://www.example.com", headers={"X-ID": "2"})
|
|
fp2 = crawler.request_fingerprinter.fingerprint(r2)
|
|
self.assertNotEqual(fp1, fp2)
|
|
|
|
@inlineCallbacks
|
|
def test_dont_canonicalize(self):
|
|
class RequestFingerprinter:
|
|
cache = WeakKeyDictionary()
|
|
|
|
def fingerprint(self, request):
|
|
if request not in self.cache:
|
|
fp = sha1()
|
|
fp.update(to_bytes(request.url))
|
|
self.cache[request] = fp.digest()
|
|
return self.cache[request]
|
|
|
|
settings = {
|
|
"REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter,
|
|
}
|
|
crawler = get_crawler(NoRequestsSpider, settings_dict=settings)
|
|
yield crawler.crawl()
|
|
|
|
r1 = Request("http://www.example.com?a=1&a=2")
|
|
fp1 = crawler.request_fingerprinter.fingerprint(r1)
|
|
r2 = Request("http://www.example.com?a=2&a=1")
|
|
fp2 = crawler.request_fingerprinter.fingerprint(r2)
|
|
self.assertNotEqual(fp1, fp2)
|
|
|
|
@inlineCallbacks
|
|
def test_meta(self):
|
|
class RequestFingerprinter:
|
|
def fingerprint(self, request):
|
|
if "fingerprint" in request.meta:
|
|
return request.meta["fingerprint"]
|
|
return fingerprint(request)
|
|
|
|
settings = {
|
|
"REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter,
|
|
}
|
|
crawler = get_crawler(NoRequestsSpider, settings_dict=settings)
|
|
yield crawler.crawl()
|
|
|
|
r1 = Request("http://www.example.com")
|
|
fp1 = crawler.request_fingerprinter.fingerprint(r1)
|
|
r2 = Request("http://www.example.com", meta={"fingerprint": "a"})
|
|
fp2 = crawler.request_fingerprinter.fingerprint(r2)
|
|
r3 = Request("http://www.example.com", meta={"fingerprint": "a"})
|
|
fp3 = crawler.request_fingerprinter.fingerprint(r3)
|
|
r4 = Request("http://www.example.com", meta={"fingerprint": "b"})
|
|
fp4 = crawler.request_fingerprinter.fingerprint(r4)
|
|
self.assertNotEqual(fp1, fp2)
|
|
self.assertNotEqual(fp1, fp4)
|
|
self.assertNotEqual(fp2, fp4)
|
|
self.assertEqual(fp2, fp3)
|
|
|
|
@inlineCallbacks
|
|
def test_from_crawler(self):
|
|
class RequestFingerprinter:
|
|
@classmethod
|
|
def from_crawler(cls, crawler):
|
|
return cls(crawler)
|
|
|
|
def __init__(self, crawler):
|
|
self._fingerprint = crawler.settings["FINGERPRINT"]
|
|
|
|
def fingerprint(self, request):
|
|
return self._fingerprint
|
|
|
|
settings = {
|
|
"REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter,
|
|
"FINGERPRINT": b"fingerprint",
|
|
}
|
|
crawler = get_crawler(NoRequestsSpider, settings_dict=settings)
|
|
yield crawler.crawl()
|
|
|
|
request = Request("http://www.example.com")
|
|
fingerprint = crawler.request_fingerprinter.fingerprint(request)
|
|
self.assertEqual(fingerprint, settings["FINGERPRINT"])
|
|
|
|
@inlineCallbacks
|
|
def test_from_settings(self):
|
|
class RequestFingerprinter:
|
|
@classmethod
|
|
def from_settings(cls, settings):
|
|
return cls(settings)
|
|
|
|
def __init__(self, settings):
|
|
self._fingerprint = settings["FINGERPRINT"]
|
|
|
|
def fingerprint(self, request):
|
|
return self._fingerprint
|
|
|
|
settings = {
|
|
"REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter,
|
|
"FINGERPRINT": b"fingerprint",
|
|
}
|
|
crawler = get_crawler(NoRequestsSpider, settings_dict=settings)
|
|
yield crawler.crawl()
|
|
|
|
request = Request("http://www.example.com")
|
|
fingerprint = crawler.request_fingerprinter.fingerprint(request)
|
|
self.assertEqual(fingerprint, settings["FINGERPRINT"])
|
|
|
|
@inlineCallbacks
|
|
def test_from_crawler_and_settings(self):
|
|
class RequestFingerprinter:
|
|
# This method is ignored due to the presence of from_crawler
|
|
@classmethod
|
|
def from_settings(cls, settings):
|
|
return cls(settings)
|
|
|
|
@classmethod
|
|
def from_crawler(cls, crawler):
|
|
return cls(crawler)
|
|
|
|
def __init__(self, crawler):
|
|
self._fingerprint = crawler.settings["FINGERPRINT"]
|
|
|
|
def fingerprint(self, request):
|
|
return self._fingerprint
|
|
|
|
settings = {
|
|
"REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter,
|
|
"FINGERPRINT": b"fingerprint",
|
|
}
|
|
crawler = get_crawler(NoRequestsSpider, settings_dict=settings)
|
|
yield crawler.crawl()
|
|
|
|
request = Request("http://www.example.com")
|
|
fingerprint = crawler.request_fingerprinter.fingerprint(request)
|
|
self.assertEqual(fingerprint, settings["FINGERPRINT"])
|
|
|
|
|
|
class RequestToCurlTest(unittest.TestCase):
|
|
def _test_request(self, request_object, expected_curl_command):
|
|
curl_command = request_to_curl(request_object)
|
|
self.assertEqual(curl_command, expected_curl_command)
|
|
|
|
def test_get(self):
|
|
request_object = Request("https://www.example.com")
|
|
expected_curl_command = "curl -X GET https://www.example.com"
|
|
self._test_request(request_object, expected_curl_command)
|
|
|
|
def test_post(self):
|
|
request_object = Request(
|
|
"https://www.httpbin.org/post",
|
|
method="POST",
|
|
body=json.dumps({"foo": "bar"}),
|
|
)
|
|
expected_curl_command = (
|
|
'curl -X POST https://www.httpbin.org/post --data-raw \'{"foo": "bar"}\''
|
|
)
|
|
self._test_request(request_object, expected_curl_command)
|
|
|
|
def test_headers(self):
|
|
request_object = Request(
|
|
"https://www.httpbin.org/post",
|
|
method="POST",
|
|
headers={"Content-Type": "application/json", "Accept": "application/json"},
|
|
body=json.dumps({"foo": "bar"}),
|
|
)
|
|
expected_curl_command = (
|
|
"curl -X POST https://www.httpbin.org/post"
|
|
' --data-raw \'{"foo": "bar"}\''
|
|
" -H 'Content-Type: application/json' -H 'Accept: application/json'"
|
|
)
|
|
self._test_request(request_object, expected_curl_command)
|
|
|
|
def test_cookies_dict(self):
|
|
request_object = Request(
|
|
"https://www.httpbin.org/post",
|
|
method="POST",
|
|
cookies={"foo": "bar"},
|
|
body=json.dumps({"foo": "bar"}),
|
|
)
|
|
expected_curl_command = (
|
|
"curl -X POST https://www.httpbin.org/post"
|
|
" --data-raw '{\"foo\": \"bar\"}' --cookie 'foo=bar'"
|
|
)
|
|
self._test_request(request_object, expected_curl_command)
|
|
|
|
def test_cookies_list(self):
|
|
request_object = Request(
|
|
"https://www.httpbin.org/post",
|
|
method="POST",
|
|
cookies=[{"foo": "bar"}],
|
|
body=json.dumps({"foo": "bar"}),
|
|
)
|
|
expected_curl_command = (
|
|
"curl -X POST https://www.httpbin.org/post"
|
|
" --data-raw '{\"foo\": \"bar\"}' --cookie 'foo=bar'"
|
|
)
|
|
self._test_request(request_object, expected_curl_command)
|