From 0007676e8d5210f8d0d05ffb12610ca780a3c5ea Mon Sep 17 00:00:00 2001 From: Adrian Date: Fri, 26 Jun 2026 09:21:12 +0200 Subject: [PATCH] Improve test coverage for http/ (#7672) * Improve test coverage for http/ * Use isinstance() instead of type() --- tests/test_http_cookies.py | 53 +++++++++++++++++++++++++++++++- tests/test_http_headers.py | 1 + tests/test_http_request.py | 10 ++++++ tests/test_http_request_form.py | 3 ++ tests/test_http_response.py | 5 +++ tests/test_http_response_text.py | 7 +++++ 6 files changed, 78 insertions(+), 1 deletion(-) diff --git a/tests/test_http_cookies.py b/tests/test_http_cookies.py index 660b76d08..ce9296764 100644 --- a/tests/test_http_cookies.py +++ b/tests/test_http_cookies.py @@ -1,8 +1,59 @@ +from http.cookiejar import DefaultCookiePolicy + from scrapy.http import Request, Response -from scrapy.http.cookies import WrappedRequest, WrappedResponse +from scrapy.http.cookies import CookieJar, WrappedRequest, WrappedResponse from scrapy.utils.httpobj import urlparse_cached +class TestCookieJar: + def setup_method(self): + self.jar = CookieJar() + self.request = Request("http://example.com/") + self.response = Response( + "http://example.com/", + headers={"Set-Cookie": "name=value; Domain=example.com; Path=/"}, + ) + + def test_extract_cookies(self): + assert len(self.jar) == 0 + self.jar.extract_cookies(self.response, self.request) + assert len(self.jar) == 1 + cookie = next(iter(self.jar)) + assert cookie.name == "name" + assert cookie.value == "value" + assert ".example.com" in self.jar._cookies + + def test_make_cookies_and_set_cookie(self): + cookies = self.jar.make_cookies(self.response, self.request) + assert len(cookies) == 1 + jar = CookieJar() + for cookie in cookies: + jar.set_cookie(cookie) + assert len(jar) == 1 + + def test_clear(self): + self.jar.extract_cookies(self.response, self.request) + assert len(self.jar) == 1 + self.jar.clear() + assert len(self.jar) == 0 + + def test_clear_session_cookies(self): + self.jar.extract_cookies(self.response, self.request) + assert len(self.jar) == 1 + self.jar.clear_session_cookies() + assert len(self.jar) == 0 + + def test_set_policy(self): + policy = DefaultCookiePolicy() + self.jar.set_policy(policy) + assert self.jar.jar._policy is policy + + def test_check_expired_frequency(self): + jar = CookieJar(check_expired_frequency=1) + jar.add_cookie_header(self.request) + assert jar.processed == 1 + + class TestWrappedRequest: def setup_method(self): self.request = Request( diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py index 243aa6afe..aff3562e3 100644 --- a/tests/test_http_headers.py +++ b/tests/test_http_headers.py @@ -140,6 +140,7 @@ class TestHeaders: h1["foo"] = "bar" h1["foo"] = None h1.setdefault("foo", "bar") + assert h1["foo"] is None assert h1.get("foo") is None assert h1.getlist("foo") == [] diff --git a/tests/test_http_request.py b/tests/test_http_request.py index e22689d49..1941b826f 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -24,6 +24,10 @@ class TestRequest: with pytest.raises(TypeError): self.request_class(123) + # priority argument must be an integer + with pytest.raises(TypeError, match="Request priority not an integer"): + self.request_class("http://www.example.com", priority="1") + r = self.request_class("http://www.example.com") assert isinstance(r.url, str) assert r.url == "http://www.example.com" @@ -274,6 +278,12 @@ class TestRequest: assert r4.meta == {} assert r4.dont_filter is False + # the cls argument allows changing the resulting class + custom_request_cls = type("CustomRequest", (self.request_class,), {}) + r5 = r1.replace(cls=custom_request_cls) + assert isinstance(r5, custom_request_cls) + assert r5.url == r1.url + def test_method_always_str(self): r = self.request_class("http://www.example.com", method="POST") assert isinstance(r.method, str) diff --git a/tests/test_http_request_form.py b/tests/test_http_request_form.py index a4f87d50d..c667e48a1 100644 --- a/tests/test_http_request_form.py +++ b/tests/test_http_request_form.py @@ -294,6 +294,9 @@ class TestFormRequest(TestRequest): assert request.method == "GET" request = FormRequest.from_response(response, method="POST") assert request.method == "POST" + # an explicit method=None skips form-method normalization + request = FormRequest.from_response(response, method=None) + assert request.method == "NONE" def test_from_response_override_url(self): response = _buildresponse( diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 079c547c7..a8ea4920e 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -254,6 +254,11 @@ class TestResponse: with pytest.raises(ValueError, match="url can't be None"): r.follow(None) + def test_follow_None_encoding(self): + r = self.response_class("http://example.com") + with pytest.raises(ValueError, match="encoding can't be None"): + r.follow("foo", encoding=None) + @pytest.mark.xfail( not W3LIB_STRIPS_URLS, reason="https://github.com/scrapy/w3lib/pull/207", diff --git a/tests/test_http_response_text.py b/tests/test_http_response_text.py index 4b3fa2302..507fd1864 100644 --- a/tests/test_http_response_text.py +++ b/tests/test_http_response_text.py @@ -14,6 +14,13 @@ from tests.test_http_response import TestResponse class TestTextResponse(TestResponse): response_class = TextResponse + def test_follow_None_encoding(self): + # unlike the base Response, TextResponse.follow() falls back to the + # response encoding when encoding is None instead of raising + r = self.response_class("http://example.com", body=b"hello", encoding="cp1252") + req = r.follow("foo", encoding=None) + assert req.encoding == "cp1252" + def test_replace(self): super().test_replace() r1 = self.response_class(