Address warnings in tests. (#7637)

scrapy/contracts/__init__.py

@@ -51,6 +51,8 @@ class Contract:
                     results.addSuccess(self.testcase_pre)
                 cb_result = cb(response, **cb_kwargs)
                 if isinstance(cb_result, (AsyncGenerator, CoroutineType)):
+                    if isinstance(cb_result, CoroutineType):
+                        cb_result.close()
                     raise TypeError("Contracts don't support async callbacks")
                 return list(cast("Iterable[Any]", iterate_spider_output(cb_result)))
 
@@ -67,6 +69,8 @@ class Contract:
             def wrapper(response: Response, **cb_kwargs: Any) -> list[Any]:
                 cb_result = cb(response, **cb_kwargs)
                 if isinstance(cb_result, (AsyncGenerator, CoroutineType)):
+                    if isinstance(cb_result, CoroutineType):
+                        cb_result.close()
                     raise TypeError("Contracts don't support async callbacks")
                 output = list(cast("Iterable[Any]", iterate_spider_output(cb_result)))
                 try:

scrapy/utils/misc.py

@@ -135,14 +135,9 @@ def walk_modules(path: str) -> list[ModuleType]:  # pragma: no cover
     return list(walk_modules_iter(path))
 
 
-def md5sum(file: IO[bytes]) -> str:
+def md5sum(file: IO[bytes]) -> str:  # pragma: no cover
     """Calculate the md5 checksum of a file-like object without reading its
-    whole content in memory.
+    whole content in memory."""
-
-    >>> from io import BytesIO
-    >>> md5sum(BytesIO(b'file content to hash'))
-    '784406af91dd5a54fbb9c84c2236595a'
-    """
     warnings.warn(
         (
             "The scrapy.utils.misc.md5sum function is deprecated and will be "

tests/ignores.txt

@@ -1 +1,3 @@
+scrapy/core/downloader/handlers/http.py
 scrapy/extensions/statsmailer.py
+scrapy/mail.py

tests/test_crawl.py

@@ -715,6 +715,9 @@ class TestCrawlSpider:
         assert isinstance(crawler.spider, SingleRequestSpider)
         assert crawler.spider.meta["responses"][0].certificate is None
 
+    @pytest.mark.filterwarnings(
+        r"ignore:.*You should use cryptography's X\.509 APIs:DeprecationWarning"
+    )
     @pytest.mark.parametrize(
         "url",
         [

tests/test_downloader_handlers_http_base.py

@@ -1148,6 +1148,9 @@ class TestHttpWithCrawlerBase(ABC):
         reason = crawler.spider.meta["close_reason"]  # type: ignore[attr-defined]
         assert reason == "finished"
 
+    @pytest.mark.filterwarnings(
+        r"ignore:.*You should use cryptography's X\.509 APIs:DeprecationWarning"
+    )
     @coroutine_test
     async def test_response_ssl_certificate(self, mockserver: MockServer) -> None:
         if not self.is_secure:

tests/test_extension_statsmailer.py

@@ -1,20 +1,27 @@
+import warnings
 from unittest.mock import MagicMock
 
 import pytest
 
 from scrapy import signals
-from scrapy.exceptions import NotConfigured
+from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
 from scrapy.signalmanager import SignalManager
 from scrapy.statscollectors import StatsCollector
 from scrapy.utils.spider import DefaultSpider
 
-pytestmark = pytest.mark.filterwarnings(
+with warnings.catch_warnings():
-    "ignore:The scrapy.extensions.statsmailer module is deprecated:scrapy.exceptions.ScrapyDeprecationWarning",
+    warnings.filterwarnings(
-    "ignore:The scrapy.mail module is deprecated:scrapy.exceptions.ScrapyDeprecationWarning",
+        "ignore",
-)
+        r"The scrapy\.extensions\.statsmailer module is deprecated",
-
+        ScrapyDeprecationWarning,
-from scrapy.extensions import statsmailer  # noqa: E402
+    )
-from scrapy.mail import MailSender  # noqa: E402
+    warnings.filterwarnings(
+        "ignore",
+        r"The scrapy\.mail module is deprecated",
+        ScrapyDeprecationWarning,
+    )
+    from scrapy.extensions import statsmailer
+    from scrapy.mail import MailSender
 
 
 @pytest.fixture

tests/test_http2_client_protocol.py

@@ -669,6 +669,9 @@ class TestHttps2ClientProtocol:
             response = await make_request(client, request)
             assert response.status == status
 
+    @pytest.mark.filterwarnings(
+        r"ignore:.*You should use cryptography's X\.509 APIs:DeprecationWarning"
+    )
     @deferred_f_from_coro_f
     async def test_response_has_correct_certificate_ip_address(
         self,

tests/test_spider_sitemap.py

@@ -2,7 +2,6 @@ from __future__ import annotations
 
 import gzip
 import re
-import warnings
 from datetime import datetime
 from io import BytesIO
 from logging import WARNING
@@ -429,9 +428,7 @@ Sitemap: /sitemap-relative-url.xml
 
         crawler = get_crawler(TestSpider)
         spider = TestSpider.from_crawler(crawler)
-        with warnings.catch_warnings():
+        requests = [request async for request in spider.start()]
-            warnings.simplefilter("error")
-            requests = [request async for request in spider.start()]
 
         assert len(requests) == 1
         request = requests[0]

tests/test_spider_start.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import warnings
 from asyncio import sleep
 from typing import Any
 
@@ -45,9 +44,7 @@ class TestMain:
             async def parse(self, response):
                 yield ITEM_A
 
-        with warnings.catch_warnings():
+        await self._test_spider(TestSpider, [ITEM_A])
-            warnings.simplefilter("error")
-            await self._test_spider(TestSpider, [ITEM_A])
 
     @coroutine_test
     async def test_start(self):
@@ -57,9 +54,7 @@ class TestMain:
             async def start(self):
                 yield ITEM_A
 
-        with warnings.catch_warnings():
+        await self._test_spider(TestSpider, [ITEM_A])
-            warnings.simplefilter("error")
-            await self._test_spider(TestSpider, [ITEM_A])
 
     @coroutine_test
     async def test_start_subclass(self):
@@ -70,9 +65,7 @@ class TestMain:
         class TestSpider(BaseSpider):
             name = "test"
 
-        with warnings.catch_warnings():
+        await self._test_spider(TestSpider, [ITEM_A])
-            warnings.simplefilter("error")
-            await self._test_spider(TestSpider, [ITEM_A])
 
     async def _test_start(self, start_, expected_items=None):
         class TestSpider(Spider):

tests/test_spidermiddleware_process_start.py

@@ -1,4 +1,3 @@
-import warnings
 from asyncio import sleep
 
 import pytest
@@ -76,9 +75,7 @@ class TestMain:
 
     @coroutine_test
     async def test_modern_mw_modern_spider(self):
-        with warnings.catch_warnings():
+        await self._test_wrap(ModernWrapSpiderMiddleware, ModernWrapSpider)
-            warnings.simplefilter("error")
-            await self._test_wrap(ModernWrapSpiderMiddleware, ModernWrapSpider)
 
     async def _test_sleep(self, spider_middlewares):
         class TestSpider(Spider):

tests/test_utils_project.py

@@ -1,5 +1,4 @@
 import os
-import warnings
 from pathlib import Path
 
 import pytest
@@ -41,11 +40,8 @@ class TestGetProjectSettings:
         envvars = {
             "SCRAPY_SETTINGS_MODULE": value,
         }
-        with warnings.catch_warnings():
+        with set_environ(**envvars):
-            warnings.simplefilter("error")
+            settings = get_project_settings()
-            with set_environ(**envvars):
-                settings = get_project_settings()
-
         assert settings.get("SETTINGS_MODULE") == value
 
     def test_invalid_envvar(self):