From 56dee203e97aafb92e25e724d7bbbfbf4f940d3e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 23 Jul 2026 13:29:22 +0500 Subject: [PATCH] Use httpx2 as a preferred alternative to httpx. (#7762) --- conftest.py | 2 +- docs/topics/asyncio.rst | 4 ++-- docs/topics/download-handlers.rst | 4 ++-- pyproject.toml | 2 +- scrapy/core/downloader/handlers/_httpx.py | 23 +++++++++++++---------- tests/test_downloader_handler_httpx.py | 4 +++- tox.ini | 10 +++++----- 7 files changed, 27 insertions(+), 22 deletions(-) diff --git a/conftest.py b/conftest.py index 1bfb002cf..b8a9dc19e 100644 --- a/conftest.py +++ b/conftest.py @@ -50,7 +50,7 @@ if not H2_ENABLED: ) ) -if not find_spec("httpx"): +if find_spec("httpx2") is None and find_spec("httpx") is None: collect_ignore.append("scrapy/core/downloader/handlers/_httpx.py") diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 83e40656b..63c217e93 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -153,8 +153,8 @@ Using Scrapy without a Twisted reactor .. note:: As the Twisted download handlers cannot be used without a reactor, the default download handler in this mode is :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`. You - will need to additionally install the ``httpx`` library to use it, unless - you switch to some different handler. + will need to additionally install the :ref:`httpx ` extra to use + it, unless you switch to some different handler. It's possible to use Scrapy without installing a Twisted reactor at all, by setting the :setting:`TWISTED_REACTOR_ENABLED` setting to ``False``. In this diff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rst index 33cbf185e..34ab4f105 100644 --- a/docs/topics/download-handlers.rst +++ b/docs/topics/download-handlers.rst @@ -291,9 +291,9 @@ HttpxDownloadHandler This handler supports ``http://host/path`` and ``https://host/path`` URLs and uses the HTTP/1.1 or HTTP/2 protocol for them. -It's implemented using the httpx_ library. +It's implemented using the httpx2_ library. -.. _httpx: https://www.python-httpx.org/ +.. _httpx2: https://httpx2.pydantic.dev/ If you want to use this handler you need to replace the default ones for the ``http`` and ``https`` schemes: diff --git a/pyproject.toml b/pyproject.toml index 81d697c12..4220b12ed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,7 +67,7 @@ brotli = [ "brotlicffi>=1.2.0.0; implementation_name == 'pypy'", ] gcs = ["google-cloud-storage>=1.29.0"] -httpx = ["httpx[http2,socks]>=0.27.1"] +httpx = ["httpx2[http2,socks]>=2.0.0"] images = ["Pillow>=8.3.2"] ipython = ["ipython>=7.1.0"] ptpython = ["ptpython>=2.0.1"] diff --git a/scrapy/core/downloader/handlers/_httpx.py b/scrapy/core/downloader/handlers/_httpx.py index 9b54b44d8..d5a4e9fcd 100644 --- a/scrapy/core/downloader/handlers/_httpx.py +++ b/scrapy/core/downloader/handlers/_httpx.py @@ -30,7 +30,7 @@ from ._base_streaming import BaseStreamingDownloadHandler, _BaseResponseArgs if TYPE_CHECKING: from collections.abc import AsyncIterator - from httpcore import AsyncNetworkStream + from httpcore2 import AsyncNetworkStream from scrapy import Request from scrapy.crawler import Crawler @@ -39,8 +39,11 @@ if TYPE_CHECKING: HAS_SOCKS = HAS_HTTP2 = False try: - import httpx -except ImportError: + try: + import httpx2 as httpx + except ImportError: # pragma: no cover + import httpx # type: ignore[import-not-found,no-redef] +except ImportError: # pragma: no cover httpx = None # type: ignore[assignment] else: # a small hack to avoid importing these optional extras unconditionally @@ -84,7 +87,7 @@ class HttpxDownloadHandler(_Base): self._enable_h2: bool = crawler.settings.getbool("HTTPX_HTTP2_ENABLED") if self._enable_h2 and not HAS_HTTP2: # pragma: no cover raise NotConfigured( - f"HTTP/2 support in {type(self).__name__} requires the 'httpx[http2]' extra to be installed." + f"HTTP/2 support in {type(self).__name__} requires the 'httpx2[http2]' extra to be installed." ) self._ssl_context: ssl.SSLContext = _make_ssl_context(crawler.settings) self._bind_host: str | None = self._get_bind_address_host() @@ -96,7 +99,7 @@ class HttpxDownloadHandler(_Base): ) self._default_client: httpx.AsyncClient = self._make_client() - # httpx doesn't support per-request proxies: https://github.com/encode/httpx/discussions/3183, + # httpx2 doesn't support per-request proxies: https://github.com/pydantic/httpx2/issues/818, # so we keep a pool of clients per proxy URL. LRU eviction can be added here if needed. self._proxy_clients: dict[str, httpx.AsyncClient] = {} @@ -104,7 +107,7 @@ class HttpxDownloadHandler(_Base): def _check_deps_installed() -> None: if httpx is None: # pragma: no cover raise NotConfigured( - "HttpxDownloadHandler requires the httpx library to be installed." + "HttpxDownloadHandler requires the httpx2 library to be installed." ) def _make_client(self, proxy_url: str | None = None) -> httpx.AsyncClient: @@ -128,7 +131,7 @@ class HttpxDownloadHandler(_Base): proxy=proxy, ), ) - # https://github.com/encode/httpx/discussions/1566 + # https://github.com/pydantic/httpx2/issues/368 for header_name in ("accept", "accept-encoding", "user-agent"): client.headers.pop(header_name, None) return client @@ -149,7 +152,7 @@ class HttpxDownloadHandler(_Base): proxy = self._extract_proxy_url_with_creds(request) if proxy and proxy.startswith("socks") and not HAS_SOCKS: # pragma: no cover raise ValueError( - f"SOCKS proxy support in {type(self).__name__} requires the 'httpx[socks]' extra to be installed." + f"SOCKS proxy support in {type(self).__name__} requires the 'httpx2[socks]' extra to be installed." ) client = self._get_client(proxy) headers = self._request_headers(request).to_tuple_list() @@ -175,7 +178,7 @@ class HttpxDownloadHandler(_Base): raise DownloadConnectionRefusedError(str(e)) from e except httpx.ProxyError as e: raise DownloadConnectionRefusedError(str(e)) from e - except DOWNLOAD_FAILED_EXCEPTIONS as e: + except DOWNLOAD_FAILED_EXCEPTIONS as e: # pylint: disable=catching-non-exception raise DownloadFailedError(str(e)) from e @staticmethod @@ -218,7 +221,7 @@ class HttpxDownloadHandler(_Base): def _log_tls_info(self, response: httpx.Response, request: Request) -> None: network_stream: AsyncNetworkStream = response.extensions["network_stream"] extra_ssl_object = network_stream.get_extra_info("ssl_object") - if isinstance(extra_ssl_object, ssl.SSLObject): + if isinstance(extra_ssl_object, ssl.SSLObject): # pragma: no branch _log_sslobj_debug_info(extra_ssl_object) async def close(self) -> None: diff --git a/tests/test_downloader_handler_httpx.py b/tests/test_downloader_handler_httpx.py index fdacad963..fd50248a7 100644 --- a/tests/test_downloader_handler_httpx.py +++ b/tests/test_downloader_handler_httpx.py @@ -3,6 +3,7 @@ from __future__ import annotations import sys +from importlib.util import find_spec from typing import TYPE_CHECKING, Any, ClassVar import pytest @@ -37,7 +38,8 @@ if TYPE_CHECKING: pytestmark = pytest.mark.only_asyncio -pytest.importorskip("httpx") +if find_spec("httpx2") is None and find_spec("httpx") is None: + pytest.skip("Neither httpx2 nor httpx are installed", allow_module_level=True) class HttpxDownloadHandlerMixin: diff --git a/tox.ini b/tox.ini index b2f3b0db1..d35fe91db 100644 --- a/tox.ini +++ b/tox.ini @@ -36,7 +36,7 @@ minversion = 1.7.0 deps = attrs coverage >= 7.10.6 - httpx + httpx2 pexpect >= 4.8.0 pyftpdlib >= 2.0.1 pygments @@ -75,7 +75,7 @@ deps = boto3-stubs[s3]==1.43.41 botocore-stubs==1.43.14 h2==4.3.0 - httpx==0.28.1 + httpx2==2.7.0 itemadapter==0.13.1 ptpython==3.0.32 # newer ones require newer Python @@ -136,7 +136,7 @@ deps = Twisted==21.7.0 cryptography==37.0.0 cssselect==0.9.1 - httpx==0.27.1 + httpx2==2.0.0 itemadapter==0.1.0 lxml==4.6.4 parsel==1.5.0 @@ -171,7 +171,7 @@ deps = brotli >= 1.2.0; implementation_name != "pypy" # optional for HTTP compress downloader middleware tests brotlicffi >= 1.2.0.0; implementation_name == "pypy" # optional for HTTP compress downloader middleware tests google-cloud-storage - httpx[http2,socks] + httpx2[http2,socks] ipython ptpython # optional for shell wrapper tests robotexclusionrulesparser @@ -189,7 +189,7 @@ deps = brotli==1.2.0; implementation_name != "pypy" brotlicffi==1.2.0.0; implementation_name == "pypy" google-cloud-storage==1.29.0 - httpx[http2,socks]==0.27.1 + httpx2[http2,socks]==2.0.0 ipython==7.1.0 ptpython==2.0.1 robotexclusionrulesparser==1.6.2