Improve test coverage for http/ (#7672)

* Improve test coverage for http/

* Use isinstance() instead of type()

tests/test_http_cookies.py

@@ -1,8 +1,59 @@
+from http.cookiejar import DefaultCookiePolicy
+
 from scrapy.http import Request, Response
-from scrapy.http.cookies import WrappedRequest, WrappedResponse
+from scrapy.http.cookies import CookieJar, WrappedRequest, WrappedResponse
 from scrapy.utils.httpobj import urlparse_cached
 
 
+class TestCookieJar:
+    def setup_method(self):
+        self.jar = CookieJar()
+        self.request = Request("http://example.com/")
+        self.response = Response(
+            "http://example.com/",
+            headers={"Set-Cookie": "name=value; Domain=example.com; Path=/"},
+        )
+
+    def test_extract_cookies(self):
+        assert len(self.jar) == 0
+        self.jar.extract_cookies(self.response, self.request)
+        assert len(self.jar) == 1
+        cookie = next(iter(self.jar))
+        assert cookie.name == "name"
+        assert cookie.value == "value"
+        assert ".example.com" in self.jar._cookies
+
+    def test_make_cookies_and_set_cookie(self):
+        cookies = self.jar.make_cookies(self.response, self.request)
+        assert len(cookies) == 1
+        jar = CookieJar()
+        for cookie in cookies:
+            jar.set_cookie(cookie)
+        assert len(jar) == 1
+
+    def test_clear(self):
+        self.jar.extract_cookies(self.response, self.request)
+        assert len(self.jar) == 1
+        self.jar.clear()
+        assert len(self.jar) == 0
+
+    def test_clear_session_cookies(self):
+        self.jar.extract_cookies(self.response, self.request)
+        assert len(self.jar) == 1
+        self.jar.clear_session_cookies()
+        assert len(self.jar) == 0
+
+    def test_set_policy(self):
+        policy = DefaultCookiePolicy()
+        self.jar.set_policy(policy)
+        assert self.jar.jar._policy is policy
+
+    def test_check_expired_frequency(self):
+        jar = CookieJar(check_expired_frequency=1)
+        jar.add_cookie_header(self.request)
+        assert jar.processed == 1
+
+
 class TestWrappedRequest:
     def setup_method(self):
         self.request = Request(

tests/test_http_headers.py

@@ -140,6 +140,7 @@ class TestHeaders:
         h1["foo"] = "bar"
         h1["foo"] = None
         h1.setdefault("foo", "bar")
+        assert h1["foo"] is None
         assert h1.get("foo") is None
         assert h1.getlist("foo") == []
 

tests/test_http_request.py

@@ -24,6 +24,10 @@ class TestRequest:
         with pytest.raises(TypeError):
             self.request_class(123)
 
+        # priority argument must be an integer
+        with pytest.raises(TypeError, match="Request priority not an integer"):
+            self.request_class("http://www.example.com", priority="1")
+
         r = self.request_class("http://www.example.com")
         assert isinstance(r.url, str)
         assert r.url == "http://www.example.com"
@@ -274,6 +278,12 @@ class TestRequest:
         assert r4.meta == {}
         assert r4.dont_filter is False
 
+        # the cls argument allows changing the resulting class
+        custom_request_cls = type("CustomRequest", (self.request_class,), {})
+        r5 = r1.replace(cls=custom_request_cls)
+        assert isinstance(r5, custom_request_cls)
+        assert r5.url == r1.url
+
     def test_method_always_str(self):
         r = self.request_class("http://www.example.com", method="POST")
         assert isinstance(r.method, str)

tests/test_http_request_form.py

@@ -294,6 +294,9 @@ class TestFormRequest(TestRequest):
         assert request.method == "GET"
         request = FormRequest.from_response(response, method="POST")
         assert request.method == "POST"
+        # an explicit method=None skips form-method normalization
+        request = FormRequest.from_response(response, method=None)
+        assert request.method == "NONE"
 
     def test_from_response_override_url(self):
         response = _buildresponse(

tests/test_http_response.py

@@ -254,6 +254,11 @@ class TestResponse:
         with pytest.raises(ValueError, match="url can't be None"):
             r.follow(None)
 
+    def test_follow_None_encoding(self):
+        r = self.response_class("http://example.com")
+        with pytest.raises(ValueError, match="encoding can't be None"):
+            r.follow("foo", encoding=None)
+
     @pytest.mark.xfail(
         not W3LIB_STRIPS_URLS,
         reason="https://github.com/scrapy/w3lib/pull/207",

tests/test_http_response_text.py

@@ -14,6 +14,13 @@ from tests.test_http_response import TestResponse
 class TestTextResponse(TestResponse):
     response_class = TextResponse
 
+    def test_follow_None_encoding(self):
+        # unlike the base Response, TextResponse.follow() falls back to the
+        # response encoding when encoding is None instead of raising
+        r = self.response_class("http://example.com", body=b"hello", encoding="cp1252")
+        req = r.follow("foo", encoding=None)
+        assert req.encoding == "cp1252"
+
     def test_replace(self):
         super().test_replace()
         r1 = self.response_class(