From ccfa052fa19f712355fb17b863e8ff77f34ff3ac Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 13 Feb 2026 21:08:06 +0300 Subject: [PATCH] Enable in-process HTTP tests without a reactor. (#7254) --- conftest.py | 5 +--- pyproject.toml | 1 - scrapy/utils/test.py | 13 +++++++--- tests/spiders.py | 18 ++++++-------- tests/test_closespider.py | 3 --- tests/test_contracts.py | 1 - tests/test_core_scraper.py | 5 ++-- tests/test_crawl.py | 27 +++++++++++++++------ tests/test_downloader_handlers.py | 9 ++++--- tests/test_downloaderslotssettings.py | 2 -- tests/test_engine.py | 4 --- tests/test_engine_loop.py | 1 - tests/test_engine_stop_download_bytes.py | 5 ++-- tests/test_engine_stop_download_headers.py | 5 ++-- tests/test_feedexport.py | 4 +-- tests/test_logformatter.py | 1 - tests/test_pipeline_crawl.py | 1 - tests/test_pipelines.py | 4 --- tests/test_proxy_connect.py | 1 - tests/test_request_attribute_binding.py | 2 -- tests/test_request_cb_kwargs.py | 2 -- tests/test_scheduler.py | 1 - tests/test_scheduler_base.py | 1 - tests/test_signals.py | 1 - tests/test_spidermiddleware_httperror.py | 1 - tests/test_spidermiddleware_output_chain.py | 2 -- tox.ini | 2 ++ 27 files changed, 51 insertions(+), 71 deletions(-) diff --git a/conftest.py b/conftest.py index 26e0434ba..d49901a7c 100644 --- a/conftest.py +++ b/conftest.py @@ -93,10 +93,7 @@ def pytest_runtest_setup(item): # Skip tests based on reactor markers reactor = item.config.getoption("--reactor") - if ( - item.get_closest_marker("requires_reactor") - or item.get_closest_marker("requires_http_handler") - ) and reactor == "none": + if item.get_closest_marker("requires_reactor") and reactor == "none": pytest.skip('This test is only run when the --reactor value is not "none"') if item.get_closest_marker("only_asyncio") and reactor not in {"asyncio", "none"}: diff --git a/pyproject.toml b/pyproject.toml index 22e3f7b22..7cb83c483 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -230,7 +230,6 @@ markers = [ "only_asyncio: marks tests that require the asyncio loop to be used", "only_not_asyncio: marks tests that require the asyncio loop to not be used", "requires_reactor: marks tests that require a reactor", - "requires_http_handler: marks tests that require a HTTP handler", "requires_uvloop: marks tests as only enabled when uvloop is known to be working", "requires_botocore: marks tests that need botocore (but not boto3)", "requires_boto3: marks tests that need botocore and boto3", diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index e60ae18db..75f357d50 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -131,8 +131,8 @@ def get_reactor_settings() -> dict[str, Any]: settings["TWISTED_ENABLED"] = False settings["DOWNLOAD_HANDLERS"] = { "ftp": None, - "http": None, - "https": None, + "http": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler", + "https": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler", } return settings @@ -208,7 +208,14 @@ def mock_google_cloud_storage() -> tuple[Any, Any, Any]: # pragma: no cover return (client_mock, bucket_mock, blob_mock) -def get_web_client_agent_req(url: str) -> Deferred[TxResponse]: +def get_web_client_agent_req(url: str) -> Deferred[TxResponse]: # pragma: no cover + warnings.warn( + "The get_web_client_agent_req() function is deprecated" + " and will be removed in a future version of Scrapy.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + from twisted.internet import reactor agent = Agent(reactor) diff --git a/tests/spiders.py b/tests/spiders.py index de2e64242..79565738d 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -18,7 +18,7 @@ from scrapy.linkextractors import LinkExtractor from scrapy.spiders import Spider from scrapy.spiders.crawl import CrawlSpider, Rule from scrapy.utils.defer import deferred_to_future, maybe_deferred_to_future -from scrapy.utils.test import get_from_asyncio_queue, get_web_client_agent_req +from scrapy.utils.test import get_from_asyncio_queue class MockServerSpider(Spider): @@ -199,28 +199,24 @@ class AsyncDefDeferredDirectSpider(SimpleSpider): name = "asyncdef_deferred_direct" async def parse(self, response): - resp = await get_web_client_agent_req(self.mockserver.url("/status?n=200")) - yield {"code": resp.code} + await defer.succeed(None) + yield {"code": 200} class AsyncDefDeferredWrappedSpider(SimpleSpider): name = "asyncdef_deferred_wrapped" async def parse(self, response): - resp = await deferred_to_future( - get_web_client_agent_req(self.mockserver.url("/status?n=200")) - ) - yield {"code": resp.code} + await deferred_to_future(defer.succeed(None)) + yield {"code": 200} class AsyncDefDeferredMaybeWrappedSpider(SimpleSpider): name = "asyncdef_deferred_wrapped" async def parse(self, response): - resp = await maybe_deferred_to_future( - get_web_client_agent_req(self.mockserver.url("/status?n=200")) - ) - yield {"code": resp.code} + await maybe_deferred_to_future(defer.succeed(None)) + yield {"code": 200} class AsyncDefAsyncioGenSpider(SimpleSpider): diff --git a/tests/test_closespider.py b/tests/test_closespider.py index a6fd6dd19..9a7447e16 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -1,5 +1,3 @@ -import pytest - from scrapy.utils.test import get_crawler from tests.mockserver.http import MockServer from tests.spiders import ( @@ -12,7 +10,6 @@ from tests.spiders import ( from tests.utils.decorators import inline_callbacks_test -@pytest.mark.requires_http_handler class TestCloseSpider: @classmethod def setup_class(cls): diff --git a/tests/test_contracts.py b/tests/test_contracts.py index f1678f912..a35ef7010 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -501,7 +501,6 @@ class TestContractsManager: assert not self.results.failures assert self.results.errors - @pytest.mark.requires_http_handler @inline_callbacks_test def test_same_url(self): class TestSameUrlSpider(Spider): diff --git a/tests/test_core_scraper.py b/tests/test_core_scraper.py index f4ef5ec5d..1b8ba08ab 100644 --- a/tests/test_core_scraper.py +++ b/tests/test_core_scraper.py @@ -2,17 +2,16 @@ from __future__ import annotations from typing import TYPE_CHECKING -import pytest - from scrapy.utils.test import get_crawler from tests.spiders import SimpleSpider from tests.utils.decorators import coroutine_test if TYPE_CHECKING: + import pytest + from tests.mockserver.http import MockServer -@pytest.mark.requires_http_handler @coroutine_test async def test_scraper_exception( mockserver: MockServer, diff --git a/tests/test_crawl.py b/tests/test_crawl.py index b206cfffa..7e9e029a1 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -14,11 +14,11 @@ from twisted.internet.ssl import Certificate from twisted.python.failure import Failure from scrapy import Spider, signals -from scrapy.crawler import CrawlerRunner +from scrapy.crawler import AsyncCrawlerRunner, CrawlerRunner from scrapy.exceptions import CloseSpider, ScrapyDeprecationWarning, StopDownload from scrapy.http import Request from scrapy.http.response import Response -from scrapy.utils.defer import maybe_deferred_to_future +from scrapy.utils.defer import ensure_awaitable, maybe_deferred_to_future from scrapy.utils.engine import format_engine_status, get_engine_status from scrapy.utils.python import to_unicode from scrapy.utils.test import get_crawler, get_reactor_settings @@ -61,7 +61,6 @@ if TYPE_CHECKING: from scrapy.statscollectors import StatsCollector -@pytest.mark.requires_http_handler # easier than marking many individual tests class TestCrawl: mockserver: MockServer @@ -402,9 +401,15 @@ with multiples lines ) assert "Got response 200" in str(log) - @inline_callbacks_test - def test_crawl_multiple(self, caplog: pytest.LogCaptureFixture): - runner = CrawlerRunner(get_reactor_settings()) + @coroutine_test + async def test_crawl_multiple(self, caplog: pytest.LogCaptureFixture) -> None: + settings_dict = get_reactor_settings() + runner_cls = ( + CrawlerRunner + if settings_dict.get("TWISTED_ENABLED", True) + else AsyncCrawlerRunner + ) + runner = runner_cls(settings_dict) runner.crawl( SimpleSpider, self.mockserver.url("/status?n=200"), @@ -417,7 +422,7 @@ with multiples lines ) with caplog.at_level(logging.DEBUG): - yield runner.join() + await ensure_awaitable(runner.join()) self._assert_retried(caplog.text) assert "Got response 200" in caplog.text @@ -429,7 +434,6 @@ with multiples lines assert "NotSupported: Unsupported URL scheme 'foo'" in caplog.text -@pytest.mark.requires_http_handler class TestCrawlSpider: mockserver: MockServer @@ -636,6 +640,11 @@ class TestCrawlSpider: yield crawler.crawl(seed=url, mockserver=self.mockserver) assert crawler.spider.meta["responses"][0].certificate is None + @pytest.mark.xfail( + 'config.getoption("--reactor") == "none"', + reason="Not implemented in HttpxDownloadHandler", + strict=True, + ) @pytest.mark.parametrize( "url", [ @@ -643,6 +652,7 @@ class TestCrawlSpider: pytest.param( "/status?n=200", marks=pytest.mark.xfail( + 'config.getoption("--reactor") != "none"', reason="With HTTP11DownloadHandler, responses with no body are returned early and contain no certificate", strict=True, ), @@ -669,6 +679,7 @@ class TestCrawlSpider: pytest.param( "/status?n=200", marks=pytest.mark.xfail( + 'config.getoption("--reactor") != "none"', reason="With HTTP11DownloadHandler, responses with no body are returned early and contain no ip_address", strict=True, ), diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 4589e667f..a8e635707 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -93,10 +93,11 @@ class TestLoad: crawler = get_crawler(settings_dict={"DOWNLOAD_HANDLERS": handlers}) dh = DownloadHandlers(crawler) assert "scheme" not in dh._schemes - for scheme in handlers: # force load handlers - dh._get_handler(scheme) + assert dh._get_handler("scheme") is None assert "scheme" not in dh._handlers assert "scheme" in dh._notconfigured + # get the handler again to cover the code that gets it from dh._notconfigured + assert dh._get_handler("scheme") is None def test_lazy_handlers(self): handlers = {"scheme": DummyLazyDH} @@ -108,8 +109,8 @@ class TestLoad: dh = DownloadHandlers(crawler) assert "scheme" in dh._schemes assert "scheme" not in dh._handlers - for scheme in handlers: # force load lazy handler - dh._get_handler(scheme) + handler = dh._get_handler("scheme") # force load lazy handler + assert handler assert "scheme" in dh._handlers assert "scheme" not in dh._notconfigured diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index 5ddb1cdfa..5c31f68bc 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -67,7 +67,6 @@ class TestCrawl: def setup_method(self): self.runner = CrawlerRunner() - @pytest.mark.requires_http_handler @inline_callbacks_test def test_delay(self): crawler = get_crawler(DownloaderSlotsSettingsTestSpider) @@ -129,7 +128,6 @@ def test_get_slot_deprecated_spider_arg(): assert slot1 == slot2 -@pytest.mark.requires_http_handler @pytest.mark.parametrize( "priority_queue_class", [ diff --git a/tests/test_engine.py b/tests/test_engine.py index c77e3c5c9..4573c5001 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -373,7 +373,6 @@ class TestEngineBase: class TestEngine(TestEngineBase): - @pytest.mark.requires_http_handler @coroutine_test async def test_crawler(self, mockserver: MockServer) -> None: for spider in ( @@ -391,7 +390,6 @@ class TestEngine(TestEngineBase): self._assert_signals_caught(run) self._assert_bytes_received(run) - @pytest.mark.requires_http_handler @coroutine_test async def test_crawler_dupefilter(self, mockserver: MockServer) -> None: run = CrawlerRun(DupeFilterSpider) @@ -399,14 +397,12 @@ class TestEngine(TestEngineBase): self._assert_scheduled_requests(run, count=8) self._assert_dropped_requests(run) - @pytest.mark.requires_http_handler @coroutine_test async def test_crawler_itemerror(self, mockserver: MockServer) -> None: run = CrawlerRun(ItemZeroDivisionErrorSpider) await run.run(mockserver) self._assert_items_error(run) - @pytest.mark.requires_http_handler @coroutine_test async def test_crawler_change_close_reason_on_idle( self, mockserver: MockServer diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py index 9bdbfcc56..115deb821 100644 --- a/tests/test_engine_loop.py +++ b/tests/test_engine_loop.py @@ -332,7 +332,6 @@ class TestRequestSendOrder: # Examples from the “Start requests” section of the documentation about # spiders. - @pytest.mark.requires_http_handler @coroutine_test async def test_lazy(self): start_nums = [1, 2, 4] diff --git a/tests/test_engine_stop_download_bytes.py b/tests/test_engine_stop_download_bytes.py index 970fffbc9..091485781 100644 --- a/tests/test_engine_stop_download_bytes.py +++ b/tests/test_engine_stop_download_bytes.py @@ -2,8 +2,6 @@ from __future__ import annotations from typing import TYPE_CHECKING -import pytest - from scrapy.exceptions import StopDownload from tests.test_engine import ( AttrsItemsSpider, @@ -16,6 +14,8 @@ from tests.test_engine import ( from tests.utils.decorators import coroutine_test if TYPE_CHECKING: + import pytest + from tests.mockserver.http import MockServer @@ -26,7 +26,6 @@ class BytesReceivedCrawlerRun(CrawlerRun): class TestBytesReceivedEngine(TestEngineBase): - @pytest.mark.requires_http_handler @coroutine_test async def test_crawler( self, mockserver: MockServer, caplog: pytest.LogCaptureFixture diff --git a/tests/test_engine_stop_download_headers.py b/tests/test_engine_stop_download_headers.py index e0008a53a..8ebc948e2 100644 --- a/tests/test_engine_stop_download_headers.py +++ b/tests/test_engine_stop_download_headers.py @@ -2,8 +2,6 @@ from __future__ import annotations from typing import TYPE_CHECKING -import pytest - from scrapy.exceptions import StopDownload from tests.test_engine import ( AttrsItemsSpider, @@ -16,6 +14,8 @@ from tests.test_engine import ( from tests.utils.decorators import coroutine_test if TYPE_CHECKING: + import pytest + from tests.mockserver.http import MockServer @@ -26,7 +26,6 @@ class HeadersReceivedCrawlerRun(CrawlerRun): class TestHeadersReceivedEngine(TestEngineBase): - @pytest.mark.requires_http_handler @coroutine_test async def test_crawler( self, mockserver: MockServer, caplog: pytest.LogCaptureFixture diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 3a9c37042..1ca5f21f0 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -849,7 +849,6 @@ class ExceptionJsonItemExporter(JsonItemExporter): raise RuntimeError("foo") -@pytest.mark.requires_http_handler class TestFeedExport(TestFeedExportBase): async def run_and_export( self, spider_cls: type[Spider], settings: dict[str, Any] @@ -1703,6 +1702,7 @@ class TestFeedExport(TestFeedExportBase): data = await self.exported_no_data(settings) assert data["csv"] == b"" + @pytest.mark.requires_reactor # needs a reactor for BlockingFeedStorage @coroutine_test async def test_multiple_feeds_success_logs_blocking_feed_storage(self): settings = { @@ -1834,7 +1834,6 @@ class TestFeedExport(TestFeedExportBase): assert not Storage.file_was_closed -@pytest.mark.requires_http_handler class TestFeedPostProcessedExports(TestFeedExportBase): items = [{"foo": "bar"}] expected = b"foo\r\nbar\r\n" @@ -2353,7 +2352,6 @@ class TestFeedPostProcessedExports(TestFeedExportBase): assert result == expected -@pytest.mark.requires_http_handler class TestBatchDeliveries(TestFeedExportBase): _file_mark = "_%(batch_time)s_#%(batch_id)02d_" diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index 66544a508..9806315b4 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -253,7 +253,6 @@ class DropSomeItemsPipeline: self.drop = True -@pytest.mark.requires_http_handler class TestShowOrSkipMessages: @classmethod def setup_class(cls): diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index acc07f48b..0376d27c4 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -57,7 +57,6 @@ class RedirectedMediaDownloadSpider(MediaDownloadSpider): ) -@pytest.mark.requires_http_handler class TestFileDownloadCrawl: pipeline_class = "scrapy.pipelines.files.FilesPipeline" store_setting_key = "FILES_STORE" diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py index fb1fccfc7..753fcd42e 100644 --- a/tests/test_pipelines.py +++ b/tests/test_pipelines.py @@ -127,7 +127,6 @@ class ItemSpider(Spider): return {"field": 42} -@pytest.mark.requires_http_handler class TestPipeline: def _on_item_scraped(self, item): assert isinstance(item, dict) @@ -266,7 +265,6 @@ class TestCustomPipelineManager: ): itemproc.process_item({}, crawler.spider) - @pytest.mark.requires_http_handler @coroutine_test async def test_integration_recommended(self, mockserver: MockServer) -> None: class CustomPipelineManager(ItemPipelineManager): @@ -293,7 +291,6 @@ class TestCustomPipelineManager: assert len(items) == 1 - @pytest.mark.requires_http_handler @coroutine_test async def test_integration_no_async_subclass(self, mockserver: MockServer) -> None: class CustomPipelineManager(ItemPipelineManager): @@ -352,7 +349,6 @@ class TestCustomPipelineManager: assert len(items) == 1 - @pytest.mark.requires_http_handler @coroutine_test async def test_integration_no_async_not_subclass( self, mockserver: MockServer diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index c78c882cf..912075401 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -61,7 +61,6 @@ def _wrong_credentials(proxy_url): return urlunsplit(bad_auth_proxy) -@pytest.mark.requires_http_handler @pytest.mark.requires_mitmproxy class TestProxyConnect: @classmethod diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.py index 66dd48737..38d56e9bd 100644 --- a/tests/test_request_attribute_binding.py +++ b/tests/test_request_attribute_binding.py @@ -1,4 +1,3 @@ -import pytest from testfixtures import LogCapture from scrapy import Request, signals @@ -63,7 +62,6 @@ class AlternativeCallbacksMiddleware: return response.replace(request=new_request) -@pytest.mark.requires_http_handler class TestCrawl: @classmethod def setup_class(cls): diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index c8629df00..8d3977452 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -1,4 +1,3 @@ -import pytest from testfixtures import LogCapture from scrapy.http import Request @@ -149,7 +148,6 @@ class KeywordArgumentsSpider(MockServerSpider): self.crawler.stats.inc_value("boolean_checks", 1) -@pytest.mark.requires_http_handler class TestCallbackKeywordArguments: @classmethod def setup_class(cls): diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 873be0b8f..0d2d8be93 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -369,7 +369,6 @@ class TestIntegrationWithDownloaderAwareInMemory: }, ) - @pytest.mark.requires_http_handler @inline_callbacks_test def test_integration_downloader_aware_priority_queue(self): with MockServer() as mockserver: diff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.py index 6234bef86..176fefbbf 100644 --- a/tests/test_scheduler_base.py +++ b/tests/test_scheduler_base.py @@ -144,7 +144,6 @@ class TestSimpleScheduler(InterfaceCheckMixin): assert close_result == "close" -@pytest.mark.requires_http_handler class TestMinimalSchedulerCrawl: scheduler_cls = MinimalScheduler diff --git a/tests/test_signals.py b/tests/test_signals.py index faf24a129..f5f13b992 100644 --- a/tests/test_signals.py +++ b/tests/test_signals.py @@ -50,7 +50,6 @@ class TestMockServer: item = await get_from_asyncio_queue(item) self.items.append(item) - @pytest.mark.requires_http_handler @pytest.mark.only_asyncio @inline_callbacks_test def test_simple_pipeline(self): diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index 8dc6ff300..b15f68dd0 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -191,7 +191,6 @@ class TestHttpErrorMiddlewareHandleAll: mw.process_spider_input(res402) -@pytest.mark.requires_http_handler class TestHttpErrorMiddlewareIntegrational: @classmethod def setup_class(cls): diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index 80f5828c9..8cf08dd94 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -1,4 +1,3 @@ -import pytest from testfixtures import LogCapture from scrapy import Request, Spider @@ -319,7 +318,6 @@ class NotGeneratorOutputChainSpider(Spider): # ================================================================================ -@pytest.mark.requires_http_handler class TestSpiderMiddleware: mockserver: MockServer diff --git a/tox.ini b/tox.ini index 213ecbef6..7656cb017 100644 --- a/tox.ini +++ b/tox.ini @@ -168,6 +168,7 @@ commands = [testenv:no-reactor] deps = {[testenv]deps} + httpx pytest-asyncio commands = {[testenv]commands} -p no:twisted --reactor=none @@ -183,6 +184,7 @@ setenv = basepython = {[pinned]basepython} deps = {[testenv:pinned]deps} + httpx==0.26.0 pytest-asyncio commands = {[pinned]commands} -p no:twisted --reactor=none setenv =