Improve test coverage for downloader middlewares

tests/test_downloadermiddleware_httpcache.py

@@ -135,6 +135,7 @@ class PolicyTestMixin:
     def test_dont_cache(self):
         with self._middleware() as mw:
             self.request.meta["dont_cache"] = True
+            assert mw.process_request(self.request) is None
             mw.process_response(self.request, self.response)
             assert mw.storage.retrieve_response(mw.crawler.spider, self.request) is None
 

tests/test_downloadermiddleware_redirect.py

@@ -4,6 +4,7 @@ from unittest.mock import MagicMock
 import pytest
 
 from scrapy.downloadermiddlewares.redirect import RedirectMiddleware
+from scrapy.exceptions import NotConfigured
 from scrapy.http import Request, Response
 from scrapy.spidermiddlewares.referer import (
     POLICY_NO_REFERRER,
@@ -265,6 +266,16 @@ class TestRedirectMiddleware(Base.Test):
         assert isinstance(req2, Request)
         assert req2.url == "http://www.example.com/redirected#frag"
 
+    def test_redirect_target_has_fragment(self):
+        url = "http://www.example.com/302#original"
+        url2 = "http://www.example.com/redirected#target"
+        req = Request(url)
+        rsp = Response(url, headers={"Location": url2}, status=302)
+
+        req2 = self.mw.process_response(req, rsp)
+        assert isinstance(req2, Request)
+        assert req2.url == "http://www.example.com/redirected#target"
+
     def test_redirect_302_head(self):
         url = "http://www.example.com/302"
         url2 = "http://www.example.com/redirected2"
@@ -458,3 +469,9 @@ def test_warning_subclass(caplog):
     assert (
         "(if defined in your code base) to override the handle_referer() method"
     ) in caplog.text
+
+
+def test_not_configured():
+    crawler = get_crawler(DefaultSpider, {"REDIRECT_ENABLED": False})
+    with pytest.raises(NotConfigured):
+        RedirectMiddleware.from_crawler(crawler)

tests/test_downloadermiddleware_redirect_metarefresh.py

@@ -7,6 +7,7 @@ from unittest.mock import MagicMock
 import pytest
 
 from scrapy.downloadermiddlewares.redirect import MetaRefreshMiddleware
+from scrapy.exceptions import NotConfigured
 from scrapy.http import HtmlResponse, Request, Response
 from scrapy.spiders import Spider
 from scrapy.utils.misc import build_from_crawler
@@ -157,3 +158,9 @@ def test_warning_meta_refresh_middleware(caplog):
         "replace scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware "
         "with a subclass that overrides the handle_referer() method"
     ) in caplog.text
+
+
+def test_not_configured():
+    crawler = get_crawler(Spider, {"METAREFRESH_ENABLED": False})
+    with pytest.raises(NotConfigured):
+        MetaRefreshMiddleware.from_crawler(crawler)

tests/test_downloadermiddleware_stats.py

@@ -1,4 +1,7 @@
-from scrapy.downloadermiddlewares.stats import DownloaderStats
+import pytest
+
+from scrapy.downloadermiddlewares.stats import DownloaderStats, get_header_size
+from scrapy.exceptions import NotConfigured
 from scrapy.http import Request, Response
 from scrapy.spiders import Spider
 from scrapy.utils.test import get_crawler
@@ -39,5 +42,14 @@ class TestDownloaderStats:
             1,
         )
 
+    def test_from_crawler_not_configured(self):
+        crawler = get_crawler(Spider, {"DOWNLOADER_STATS": False})
+        with pytest.raises(NotConfigured):
+            DownloaderStats.from_crawler(crawler)
+
     def teardown_method(self):
         self.crawler.stats.close_spider()
+
+
+def test_get_header_size_non_list_value():
+    assert get_header_size({"Content-Type": "text/html"}) == 0