diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index c0da7dfa4..ebbdf1b98 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -51,6 +51,8 @@ class Contract: results.addSuccess(self.testcase_pre) cb_result = cb(response, **cb_kwargs) if isinstance(cb_result, (AsyncGenerator, CoroutineType)): + if isinstance(cb_result, CoroutineType): + cb_result.close() raise TypeError("Contracts don't support async callbacks") return list(cast("Iterable[Any]", iterate_spider_output(cb_result))) @@ -67,6 +69,8 @@ class Contract: def wrapper(response: Response, **cb_kwargs: Any) -> list[Any]: cb_result = cb(response, **cb_kwargs) if isinstance(cb_result, (AsyncGenerator, CoroutineType)): + if isinstance(cb_result, CoroutineType): + cb_result.close() raise TypeError("Contracts don't support async callbacks") output = list(cast("Iterable[Any]", iterate_spider_output(cb_result))) try: diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 47568e656..20e7cb381 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -135,14 +135,9 @@ def walk_modules(path: str) -> list[ModuleType]: # pragma: no cover return list(walk_modules_iter(path)) -def md5sum(file: IO[bytes]) -> str: +def md5sum(file: IO[bytes]) -> str: # pragma: no cover """Calculate the md5 checksum of a file-like object without reading its - whole content in memory. - - >>> from io import BytesIO - >>> md5sum(BytesIO(b'file content to hash')) - '784406af91dd5a54fbb9c84c2236595a' - """ + whole content in memory.""" warnings.warn( ( "The scrapy.utils.misc.md5sum function is deprecated and will be " diff --git a/tests/ignores.txt b/tests/ignores.txt index 94edcf186..3717bbc95 100644 --- a/tests/ignores.txt +++ b/tests/ignores.txt @@ -1 +1,3 @@ +scrapy/core/downloader/handlers/http.py scrapy/extensions/statsmailer.py +scrapy/mail.py diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 85d98f847..66b15bfdd 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -715,6 +715,9 @@ class TestCrawlSpider: assert isinstance(crawler.spider, SingleRequestSpider) assert crawler.spider.meta["responses"][0].certificate is None + @pytest.mark.filterwarnings( + r"ignore:.*You should use cryptography's X\.509 APIs:DeprecationWarning" + ) @pytest.mark.parametrize( "url", [ diff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.py index 2ae9206a0..c4a8193c8 100644 --- a/tests/test_downloader_handlers_http_base.py +++ b/tests/test_downloader_handlers_http_base.py @@ -1148,6 +1148,9 @@ class TestHttpWithCrawlerBase(ABC): reason = crawler.spider.meta["close_reason"] # type: ignore[attr-defined] assert reason == "finished" + @pytest.mark.filterwarnings( + r"ignore:.*You should use cryptography's X\.509 APIs:DeprecationWarning" + ) @coroutine_test async def test_response_ssl_certificate(self, mockserver: MockServer) -> None: if not self.is_secure: diff --git a/tests/test_extension_statsmailer.py b/tests/test_extension_statsmailer.py index 28db389c3..4d208a1ea 100644 --- a/tests/test_extension_statsmailer.py +++ b/tests/test_extension_statsmailer.py @@ -1,20 +1,27 @@ +import warnings from unittest.mock import MagicMock import pytest from scrapy import signals -from scrapy.exceptions import NotConfigured +from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.signalmanager import SignalManager from scrapy.statscollectors import StatsCollector from scrapy.utils.spider import DefaultSpider -pytestmark = pytest.mark.filterwarnings( - "ignore:The scrapy.extensions.statsmailer module is deprecated:scrapy.exceptions.ScrapyDeprecationWarning", - "ignore:The scrapy.mail module is deprecated:scrapy.exceptions.ScrapyDeprecationWarning", -) - -from scrapy.extensions import statsmailer # noqa: E402 -from scrapy.mail import MailSender # noqa: E402 +with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + r"The scrapy\.extensions\.statsmailer module is deprecated", + ScrapyDeprecationWarning, + ) + warnings.filterwarnings( + "ignore", + r"The scrapy\.mail module is deprecated", + ScrapyDeprecationWarning, + ) + from scrapy.extensions import statsmailer + from scrapy.mail import MailSender @pytest.fixture diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 28f306e31..3c1347fd3 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -669,6 +669,9 @@ class TestHttps2ClientProtocol: response = await make_request(client, request) assert response.status == status + @pytest.mark.filterwarnings( + r"ignore:.*You should use cryptography's X\.509 APIs:DeprecationWarning" + ) @deferred_f_from_coro_f async def test_response_has_correct_certificate_ip_address( self, diff --git a/tests/test_spider_sitemap.py b/tests/test_spider_sitemap.py index 57c209615..0af99ab6d 100644 --- a/tests/test_spider_sitemap.py +++ b/tests/test_spider_sitemap.py @@ -2,7 +2,6 @@ from __future__ import annotations import gzip import re -import warnings from datetime import datetime from io import BytesIO from logging import WARNING @@ -429,9 +428,7 @@ Sitemap: /sitemap-relative-url.xml crawler = get_crawler(TestSpider) spider = TestSpider.from_crawler(crawler) - with warnings.catch_warnings(): - warnings.simplefilter("error") - requests = [request async for request in spider.start()] + requests = [request async for request in spider.start()] assert len(requests) == 1 request = requests[0] diff --git a/tests/test_spider_start.py b/tests/test_spider_start.py index aef2093ac..9257e0232 100644 --- a/tests/test_spider_start.py +++ b/tests/test_spider_start.py @@ -1,6 +1,5 @@ from __future__ import annotations -import warnings from asyncio import sleep from typing import Any @@ -45,9 +44,7 @@ class TestMain: async def parse(self, response): yield ITEM_A - with warnings.catch_warnings(): - warnings.simplefilter("error") - await self._test_spider(TestSpider, [ITEM_A]) + await self._test_spider(TestSpider, [ITEM_A]) @coroutine_test async def test_start(self): @@ -57,9 +54,7 @@ class TestMain: async def start(self): yield ITEM_A - with warnings.catch_warnings(): - warnings.simplefilter("error") - await self._test_spider(TestSpider, [ITEM_A]) + await self._test_spider(TestSpider, [ITEM_A]) @coroutine_test async def test_start_subclass(self): @@ -70,9 +65,7 @@ class TestMain: class TestSpider(BaseSpider): name = "test" - with warnings.catch_warnings(): - warnings.simplefilter("error") - await self._test_spider(TestSpider, [ITEM_A]) + await self._test_spider(TestSpider, [ITEM_A]) async def _test_start(self, start_, expected_items=None): class TestSpider(Spider): diff --git a/tests/test_spidermiddleware_process_start.py b/tests/test_spidermiddleware_process_start.py index 21df73a65..67ce3a920 100644 --- a/tests/test_spidermiddleware_process_start.py +++ b/tests/test_spidermiddleware_process_start.py @@ -1,4 +1,3 @@ -import warnings from asyncio import sleep import pytest @@ -76,9 +75,7 @@ class TestMain: @coroutine_test async def test_modern_mw_modern_spider(self): - with warnings.catch_warnings(): - warnings.simplefilter("error") - await self._test_wrap(ModernWrapSpiderMiddleware, ModernWrapSpider) + await self._test_wrap(ModernWrapSpiderMiddleware, ModernWrapSpider) async def _test_sleep(self, spider_middlewares): class TestSpider(Spider): diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py index 20a3d940c..5333a55cb 100644 --- a/tests/test_utils_project.py +++ b/tests/test_utils_project.py @@ -1,5 +1,4 @@ import os -import warnings from pathlib import Path import pytest @@ -41,11 +40,8 @@ class TestGetProjectSettings: envvars = { "SCRAPY_SETTINGS_MODULE": value, } - with warnings.catch_warnings(): - warnings.simplefilter("error") - with set_environ(**envvars): - settings = get_project_settings() - + with set_environ(**envvars): + settings = get_project_settings() assert settings.get("SETTINGS_MODULE") == value def test_invalid_envvar(self):