Enable in-process HTTP tests without a reactor. (#7254)

conftest.py

@@ -93,10 +93,7 @@ def pytest_runtest_setup(item):
     # Skip tests based on reactor markers
     reactor = item.config.getoption("--reactor")
 
-    if (
+    if item.get_closest_marker("requires_reactor") and reactor == "none":
-        item.get_closest_marker("requires_reactor")
-        or item.get_closest_marker("requires_http_handler")
-    ) and reactor == "none":
         pytest.skip('This test is only run when the --reactor value is not "none"')
 
     if item.get_closest_marker("only_asyncio") and reactor not in {"asyncio", "none"}:

pyproject.toml

@@ -230,7 +230,6 @@ markers = [
     "only_asyncio: marks tests that require the asyncio loop to be used",
     "only_not_asyncio: marks tests that require the asyncio loop to not be used",
     "requires_reactor: marks tests that require a reactor",
-    "requires_http_handler: marks tests that require a HTTP handler",
     "requires_uvloop: marks tests as only enabled when uvloop is known to be working",
     "requires_botocore: marks tests that need botocore (but not boto3)",
     "requires_boto3: marks tests that need botocore and boto3",

scrapy/utils/test.py

@@ -131,8 +131,8 @@ def get_reactor_settings() -> dict[str, Any]:
         settings["TWISTED_ENABLED"] = False
         settings["DOWNLOAD_HANDLERS"] = {
             "ftp": None,
-            "http": None,
+            "http": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler",
-            "https": None,
+            "https": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler",
         }
     return settings
 
@@ -208,7 +208,14 @@ def mock_google_cloud_storage() -> tuple[Any, Any, Any]:  # pragma: no cover
     return (client_mock, bucket_mock, blob_mock)
 
 
-def get_web_client_agent_req(url: str) -> Deferred[TxResponse]:
+def get_web_client_agent_req(url: str) -> Deferred[TxResponse]:  # pragma: no cover
+    warnings.warn(
+        "The get_web_client_agent_req() function is deprecated"
+        " and will be removed in a future version of Scrapy.",
+        category=ScrapyDeprecationWarning,
+        stacklevel=2,
+    )
+
     from twisted.internet import reactor
 
     agent = Agent(reactor)

tests/spiders.py

@@ -18,7 +18,7 @@ from scrapy.linkextractors import LinkExtractor
 from scrapy.spiders import Spider
 from scrapy.spiders.crawl import CrawlSpider, Rule
 from scrapy.utils.defer import deferred_to_future, maybe_deferred_to_future
-from scrapy.utils.test import get_from_asyncio_queue, get_web_client_agent_req
+from scrapy.utils.test import get_from_asyncio_queue
 
 
 class MockServerSpider(Spider):
@@ -199,28 +199,24 @@ class AsyncDefDeferredDirectSpider(SimpleSpider):
     name = "asyncdef_deferred_direct"
 
     async def parse(self, response):
-        resp = await get_web_client_agent_req(self.mockserver.url("/status?n=200"))
+        await defer.succeed(None)
-        yield {"code": resp.code}
+        yield {"code": 200}
 
 
 class AsyncDefDeferredWrappedSpider(SimpleSpider):
     name = "asyncdef_deferred_wrapped"
 
     async def parse(self, response):
-        resp = await deferred_to_future(
+        await deferred_to_future(defer.succeed(None))
-            get_web_client_agent_req(self.mockserver.url("/status?n=200"))
+        yield {"code": 200}
-        )
-        yield {"code": resp.code}
 
 
 class AsyncDefDeferredMaybeWrappedSpider(SimpleSpider):
     name = "asyncdef_deferred_wrapped"
 
     async def parse(self, response):
-        resp = await maybe_deferred_to_future(
+        await maybe_deferred_to_future(defer.succeed(None))
-            get_web_client_agent_req(self.mockserver.url("/status?n=200"))
+        yield {"code": 200}
-        )
-        yield {"code": resp.code}
 
 
 class AsyncDefAsyncioGenSpider(SimpleSpider):

tests/test_closespider.py

@@ -1,5 +1,3 @@
-import pytest
-
 from scrapy.utils.test import get_crawler
 from tests.mockserver.http import MockServer
 from tests.spiders import (
@@ -12,7 +10,6 @@ from tests.spiders import (
 from tests.utils.decorators import inline_callbacks_test
 
 
-@pytest.mark.requires_http_handler
 class TestCloseSpider:
     @classmethod
     def setup_class(cls):

tests/test_contracts.py

@@ -501,7 +501,6 @@ class TestContractsManager:
         assert not self.results.failures
         assert self.results.errors
 
-    @pytest.mark.requires_http_handler
     @inline_callbacks_test
     def test_same_url(self):
         class TestSameUrlSpider(Spider):

tests/test_core_scraper.py

@@ -2,17 +2,16 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING
 
-import pytest
-
 from scrapy.utils.test import get_crawler
 from tests.spiders import SimpleSpider
 from tests.utils.decorators import coroutine_test
 
 if TYPE_CHECKING:
+    import pytest
+
     from tests.mockserver.http import MockServer
 
 
-@pytest.mark.requires_http_handler
 @coroutine_test
 async def test_scraper_exception(
     mockserver: MockServer,

tests/test_crawl.py

@@ -14,11 +14,11 @@ from twisted.internet.ssl import Certificate
 from twisted.python.failure import Failure
 
 from scrapy import Spider, signals
-from scrapy.crawler import CrawlerRunner
+from scrapy.crawler import AsyncCrawlerRunner, CrawlerRunner
 from scrapy.exceptions import CloseSpider, ScrapyDeprecationWarning, StopDownload
 from scrapy.http import Request
 from scrapy.http.response import Response
-from scrapy.utils.defer import maybe_deferred_to_future
+from scrapy.utils.defer import ensure_awaitable, maybe_deferred_to_future
 from scrapy.utils.engine import format_engine_status, get_engine_status
 from scrapy.utils.python import to_unicode
 from scrapy.utils.test import get_crawler, get_reactor_settings
@@ -61,7 +61,6 @@ if TYPE_CHECKING:
     from scrapy.statscollectors import StatsCollector
 
 
-@pytest.mark.requires_http_handler  # easier than marking many individual tests
 class TestCrawl:
     mockserver: MockServer
 
@@ -402,9 +401,15 @@ with multiples lines
             )
         assert "Got response 200" in str(log)
 
-    @inline_callbacks_test
+    @coroutine_test
-    def test_crawl_multiple(self, caplog: pytest.LogCaptureFixture):
+    async def test_crawl_multiple(self, caplog: pytest.LogCaptureFixture) -> None:
-        runner = CrawlerRunner(get_reactor_settings())
+        settings_dict = get_reactor_settings()
+        runner_cls = (
+            CrawlerRunner
+            if settings_dict.get("TWISTED_ENABLED", True)
+            else AsyncCrawlerRunner
+        )
+        runner = runner_cls(settings_dict)
         runner.crawl(
             SimpleSpider,
             self.mockserver.url("/status?n=200"),
@@ -417,7 +422,7 @@ with multiples lines
         )
 
         with caplog.at_level(logging.DEBUG):
-            yield runner.join()
+            await ensure_awaitable(runner.join())
 
         self._assert_retried(caplog.text)
         assert "Got response 200" in caplog.text
@@ -429,7 +434,6 @@ with multiples lines
         assert "NotSupported: Unsupported URL scheme 'foo'" in caplog.text
 
 
-@pytest.mark.requires_http_handler
 class TestCrawlSpider:
     mockserver: MockServer
 
@@ -636,6 +640,11 @@ class TestCrawlSpider:
         yield crawler.crawl(seed=url, mockserver=self.mockserver)
         assert crawler.spider.meta["responses"][0].certificate is None
 
+    @pytest.mark.xfail(
+        'config.getoption("--reactor") == "none"',
+        reason="Not implemented in HttpxDownloadHandler",
+        strict=True,
+    )
     @pytest.mark.parametrize(
         "url",
         [
@@ -643,6 +652,7 @@ class TestCrawlSpider:
             pytest.param(
                 "/status?n=200",
                 marks=pytest.mark.xfail(
+                    'config.getoption("--reactor") != "none"',
                     reason="With HTTP11DownloadHandler, responses with no body are returned early and contain no certificate",
                     strict=True,
                 ),
@@ -669,6 +679,7 @@ class TestCrawlSpider:
             pytest.param(
                 "/status?n=200",
                 marks=pytest.mark.xfail(
+                    'config.getoption("--reactor") != "none"',
                     reason="With HTTP11DownloadHandler, responses with no body are returned early and contain no ip_address",
                     strict=True,
                 ),

tests/test_downloader_handlers.py

@@ -93,10 +93,11 @@ class TestLoad:
         crawler = get_crawler(settings_dict={"DOWNLOAD_HANDLERS": handlers})
         dh = DownloadHandlers(crawler)
         assert "scheme" not in dh._schemes
-        for scheme in handlers:  # force load handlers
+        assert dh._get_handler("scheme") is None
-            dh._get_handler(scheme)
         assert "scheme" not in dh._handlers
         assert "scheme" in dh._notconfigured
+        # get the handler again to cover the code that gets it from dh._notconfigured
+        assert dh._get_handler("scheme") is None
 
     def test_lazy_handlers(self):
         handlers = {"scheme": DummyLazyDH}
@@ -108,8 +109,8 @@ class TestLoad:
             dh = DownloadHandlers(crawler)
         assert "scheme" in dh._schemes
         assert "scheme" not in dh._handlers
-        for scheme in handlers:  # force load lazy handler
+        handler = dh._get_handler("scheme")  # force load lazy handler
-            dh._get_handler(scheme)
+        assert handler
         assert "scheme" in dh._handlers
         assert "scheme" not in dh._notconfigured
 

tests/test_downloaderslotssettings.py

@@ -67,7 +67,6 @@ class TestCrawl:
     def setup_method(self):
         self.runner = CrawlerRunner()
 
-    @pytest.mark.requires_http_handler
     @inline_callbacks_test
     def test_delay(self):
         crawler = get_crawler(DownloaderSlotsSettingsTestSpider)
@@ -129,7 +128,6 @@ def test_get_slot_deprecated_spider_arg():
     assert slot1 == slot2
 
 
-@pytest.mark.requires_http_handler
 @pytest.mark.parametrize(
     "priority_queue_class",
     [

tests/test_engine.py

@@ -373,7 +373,6 @@ class TestEngineBase:
 
 
 class TestEngine(TestEngineBase):
-    @pytest.mark.requires_http_handler
     @coroutine_test
     async def test_crawler(self, mockserver: MockServer) -> None:
         for spider in (
@@ -391,7 +390,6 @@ class TestEngine(TestEngineBase):
             self._assert_signals_caught(run)
             self._assert_bytes_received(run)
 
-    @pytest.mark.requires_http_handler
     @coroutine_test
     async def test_crawler_dupefilter(self, mockserver: MockServer) -> None:
         run = CrawlerRun(DupeFilterSpider)
@@ -399,14 +397,12 @@ class TestEngine(TestEngineBase):
         self._assert_scheduled_requests(run, count=8)
         self._assert_dropped_requests(run)
 
-    @pytest.mark.requires_http_handler
     @coroutine_test
     async def test_crawler_itemerror(self, mockserver: MockServer) -> None:
         run = CrawlerRun(ItemZeroDivisionErrorSpider)
         await run.run(mockserver)
         self._assert_items_error(run)
 
-    @pytest.mark.requires_http_handler
     @coroutine_test
     async def test_crawler_change_close_reason_on_idle(
         self, mockserver: MockServer

tests/test_engine_loop.py

@@ -332,7 +332,6 @@ class TestRequestSendOrder:
     # Examples from the “Start requests” section of the documentation about
     # spiders.
 
-    @pytest.mark.requires_http_handler
     @coroutine_test
     async def test_lazy(self):
         start_nums = [1, 2, 4]

tests/test_engine_stop_download_bytes.py

@@ -2,8 +2,6 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING
 
-import pytest
-
 from scrapy.exceptions import StopDownload
 from tests.test_engine import (
     AttrsItemsSpider,
@@ -16,6 +14,8 @@ from tests.test_engine import (
 from tests.utils.decorators import coroutine_test
 
 if TYPE_CHECKING:
+    import pytest
+
     from tests.mockserver.http import MockServer
 
 
@@ -26,7 +26,6 @@ class BytesReceivedCrawlerRun(CrawlerRun):
 
 
 class TestBytesReceivedEngine(TestEngineBase):
-    @pytest.mark.requires_http_handler
     @coroutine_test
     async def test_crawler(
         self, mockserver: MockServer, caplog: pytest.LogCaptureFixture

tests/test_engine_stop_download_headers.py

@@ -2,8 +2,6 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING
 
-import pytest
-
 from scrapy.exceptions import StopDownload
 from tests.test_engine import (
     AttrsItemsSpider,
@@ -16,6 +14,8 @@ from tests.test_engine import (
 from tests.utils.decorators import coroutine_test
 
 if TYPE_CHECKING:
+    import pytest
+
     from tests.mockserver.http import MockServer
 
 
@@ -26,7 +26,6 @@ class HeadersReceivedCrawlerRun(CrawlerRun):
 
 
 class TestHeadersReceivedEngine(TestEngineBase):
-    @pytest.mark.requires_http_handler
     @coroutine_test
     async def test_crawler(
         self, mockserver: MockServer, caplog: pytest.LogCaptureFixture

tests/test_feedexport.py

@@ -849,7 +849,6 @@ class ExceptionJsonItemExporter(JsonItemExporter):
         raise RuntimeError("foo")
 
 
-@pytest.mark.requires_http_handler
 class TestFeedExport(TestFeedExportBase):
     async def run_and_export(
         self, spider_cls: type[Spider], settings: dict[str, Any]
@@ -1703,6 +1702,7 @@ class TestFeedExport(TestFeedExportBase):
         data = await self.exported_no_data(settings)
         assert data["csv"] == b""
 
+    @pytest.mark.requires_reactor  # needs a reactor for BlockingFeedStorage
     @coroutine_test
     async def test_multiple_feeds_success_logs_blocking_feed_storage(self):
         settings = {
@@ -1834,7 +1834,6 @@ class TestFeedExport(TestFeedExportBase):
         assert not Storage.file_was_closed
 
 
-@pytest.mark.requires_http_handler
 class TestFeedPostProcessedExports(TestFeedExportBase):
     items = [{"foo": "bar"}]
     expected = b"foo\r\nbar\r\n"
@@ -2353,7 +2352,6 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
             assert result == expected
 
 
-@pytest.mark.requires_http_handler
 class TestBatchDeliveries(TestFeedExportBase):
     _file_mark = "_%(batch_time)s_#%(batch_id)02d_"
 

tests/test_logformatter.py

@@ -253,7 +253,6 @@ class DropSomeItemsPipeline:
         self.drop = True
 
 
-@pytest.mark.requires_http_handler
 class TestShowOrSkipMessages:
     @classmethod
     def setup_class(cls):

tests/test_pipeline_crawl.py

@@ -57,7 +57,6 @@ class RedirectedMediaDownloadSpider(MediaDownloadSpider):
         )
 
 
-@pytest.mark.requires_http_handler
 class TestFileDownloadCrawl:
     pipeline_class = "scrapy.pipelines.files.FilesPipeline"
     store_setting_key = "FILES_STORE"

tests/test_pipelines.py

@@ -127,7 +127,6 @@ class ItemSpider(Spider):
         return {"field": 42}
 
 
-@pytest.mark.requires_http_handler
 class TestPipeline:
     def _on_item_scraped(self, item):
         assert isinstance(item, dict)
@@ -266,7 +265,6 @@ class TestCustomPipelineManager:
         ):
             itemproc.process_item({}, crawler.spider)
 
-    @pytest.mark.requires_http_handler
     @coroutine_test
     async def test_integration_recommended(self, mockserver: MockServer) -> None:
         class CustomPipelineManager(ItemPipelineManager):
@@ -293,7 +291,6 @@ class TestCustomPipelineManager:
 
         assert len(items) == 1
 
-    @pytest.mark.requires_http_handler
     @coroutine_test
     async def test_integration_no_async_subclass(self, mockserver: MockServer) -> None:
         class CustomPipelineManager(ItemPipelineManager):
@@ -352,7 +349,6 @@ class TestCustomPipelineManager:
 
         assert len(items) == 1
 
-    @pytest.mark.requires_http_handler
     @coroutine_test
     async def test_integration_no_async_not_subclass(
         self, mockserver: MockServer

tests/test_proxy_connect.py

@@ -61,7 +61,6 @@ def _wrong_credentials(proxy_url):
     return urlunsplit(bad_auth_proxy)
 
 
-@pytest.mark.requires_http_handler
 @pytest.mark.requires_mitmproxy
 class TestProxyConnect:
     @classmethod

tests/test_request_attribute_binding.py

@@ -1,4 +1,3 @@
-import pytest
 from testfixtures import LogCapture
 
 from scrapy import Request, signals
@@ -63,7 +62,6 @@ class AlternativeCallbacksMiddleware:
         return response.replace(request=new_request)
 
 
-@pytest.mark.requires_http_handler
 class TestCrawl:
     @classmethod
     def setup_class(cls):

tests/test_request_cb_kwargs.py

@@ -1,4 +1,3 @@
-import pytest
 from testfixtures import LogCapture
 
 from scrapy.http import Request
@@ -149,7 +148,6 @@ class KeywordArgumentsSpider(MockServerSpider):
         self.crawler.stats.inc_value("boolean_checks", 1)
 
 
-@pytest.mark.requires_http_handler
 class TestCallbackKeywordArguments:
     @classmethod
     def setup_class(cls):

tests/test_scheduler.py

@@ -369,7 +369,6 @@ class TestIntegrationWithDownloaderAwareInMemory:
             },
         )
 
-    @pytest.mark.requires_http_handler
     @inline_callbacks_test
     def test_integration_downloader_aware_priority_queue(self):
         with MockServer() as mockserver:

tests/test_scheduler_base.py

@@ -144,7 +144,6 @@ class TestSimpleScheduler(InterfaceCheckMixin):
         assert close_result == "close"
 
 
-@pytest.mark.requires_http_handler
 class TestMinimalSchedulerCrawl:
     scheduler_cls = MinimalScheduler
 

tests/test_signals.py

@@ -50,7 +50,6 @@ class TestMockServer:
         item = await get_from_asyncio_queue(item)
         self.items.append(item)
 
-    @pytest.mark.requires_http_handler
     @pytest.mark.only_asyncio
     @inline_callbacks_test
     def test_simple_pipeline(self):

tests/test_spidermiddleware_httperror.py

@@ -191,7 +191,6 @@ class TestHttpErrorMiddlewareHandleAll:
         mw.process_spider_input(res402)
 
 
-@pytest.mark.requires_http_handler
 class TestHttpErrorMiddlewareIntegrational:
     @classmethod
     def setup_class(cls):

tests/test_spidermiddleware_output_chain.py

@@ -1,4 +1,3 @@
-import pytest
 from testfixtures import LogCapture
 
 from scrapy import Request, Spider
@@ -319,7 +318,6 @@ class NotGeneratorOutputChainSpider(Spider):
 
 
 # ================================================================================
-@pytest.mark.requires_http_handler
 class TestSpiderMiddleware:
     mockserver: MockServer
 

tox.ini

@@ -168,6 +168,7 @@ commands =
 [testenv:no-reactor]
 deps =
     {[testenv]deps}
+    httpx
     pytest-asyncio
 commands =
     {[testenv]commands} -p no:twisted --reactor=none
@@ -183,6 +184,7 @@ setenv =
 basepython = {[pinned]basepython}
 deps =
     {[testenv:pinned]deps}
+    httpx==0.26.0
     pytest-asyncio
 commands = {[pinned]commands} -p no:twisted --reactor=none
 setenv =