Use httpx2 as a preferred alternative to httpx. (#7762)

conftest.py

@@ -50,7 +50,7 @@ if not H2_ENABLED:
         )
     )
 
-if not find_spec("httpx"):
+if find_spec("httpx2") is None and find_spec("httpx") is None:
     collect_ignore.append("scrapy/core/downloader/handlers/_httpx.py")
 
 

docs/topics/asyncio.rst

@@ -153,8 +153,8 @@ Using Scrapy without a Twisted reactor
 .. note:: As the Twisted download handlers cannot be used without a reactor,
     the default download handler in this mode is
     :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`. You
-    will need to additionally install the ``httpx`` library to use it, unless
-    you switch to some different handler.
+    will need to additionally install the :ref:`httpx <extras>` extra to use
+    it, unless you switch to some different handler.
 
 It's possible to use Scrapy without installing a Twisted reactor at all, by
 setting the :setting:`TWISTED_REACTOR_ENABLED` setting to ``False``. In this

docs/topics/download-handlers.rst

@@ -291,9 +291,9 @@ HttpxDownloadHandler
 This handler supports ``http://host/path`` and ``https://host/path`` URLs and
 uses the HTTP/1.1 or HTTP/2 protocol for them.
 
-It's implemented using the httpx_ library.
+It's implemented using the httpx2_ library.
 
-.. _httpx: https://www.python-httpx.org/
+.. _httpx2: https://httpx2.pydantic.dev/
 
 If you want to use this handler you need to replace the default ones for the
 ``http`` and ``https`` schemes:

pyproject.toml

@@ -67,7 +67,7 @@ brotli = [
     "brotlicffi>=1.2.0.0; implementation_name == 'pypy'",
 ]
 gcs = ["google-cloud-storage>=1.29.0"]
-httpx = ["httpx[http2,socks]>=0.27.1"]
+httpx = ["httpx2[http2,socks]>=2.0.0"]
 images = ["Pillow>=8.3.2"]
 ipython = ["ipython>=7.1.0"]
 ptpython = ["ptpython>=2.0.1"]

scrapy/core/downloader/handlers/_httpx.py

@@ -30,7 +30,7 @@ from ._base_streaming import BaseStreamingDownloadHandler, _BaseResponseArgs
 if TYPE_CHECKING:
     from collections.abc import AsyncIterator
 
-    from httpcore import AsyncNetworkStream
+    from httpcore2 import AsyncNetworkStream
 
     from scrapy import Request
     from scrapy.crawler import Crawler
@@ -39,8 +39,11 @@ if TYPE_CHECKING:
 HAS_SOCKS = HAS_HTTP2 = False
 
 try:
-    import httpx
-except ImportError:
+    try:
+        import httpx2 as httpx
+    except ImportError:  # pragma: no cover
+        import httpx  # type: ignore[import-not-found,no-redef]
+except ImportError:  # pragma: no cover
     httpx = None  # type: ignore[assignment]
 else:
     # a small hack to avoid importing these optional extras unconditionally
@@ -84,7 +87,7 @@ class HttpxDownloadHandler(_Base):
         self._enable_h2: bool = crawler.settings.getbool("HTTPX_HTTP2_ENABLED")
         if self._enable_h2 and not HAS_HTTP2:  # pragma: no cover
             raise NotConfigured(
-                f"HTTP/2 support in {type(self).__name__} requires the 'httpx[http2]' extra to be installed."
+                f"HTTP/2 support in {type(self).__name__} requires the 'httpx2[http2]' extra to be installed."
             )
         self._ssl_context: ssl.SSLContext = _make_ssl_context(crawler.settings)
         self._bind_host: str | None = self._get_bind_address_host()
@@ -96,7 +99,7 @@ class HttpxDownloadHandler(_Base):
         )
 
         self._default_client: httpx.AsyncClient = self._make_client()
-        # httpx doesn't support per-request proxies: https://github.com/encode/httpx/discussions/3183,
+        # httpx2 doesn't support per-request proxies: https://github.com/pydantic/httpx2/issues/818,
         # so we keep a pool of clients per proxy URL. LRU eviction can be added here if needed.
         self._proxy_clients: dict[str, httpx.AsyncClient] = {}
 
@@ -104,7 +107,7 @@ class HttpxDownloadHandler(_Base):
     def _check_deps_installed() -> None:
         if httpx is None:  # pragma: no cover
             raise NotConfigured(
-                "HttpxDownloadHandler requires the httpx library to be installed."
+                "HttpxDownloadHandler requires the httpx2 library to be installed."
             )
 
     def _make_client(self, proxy_url: str | None = None) -> httpx.AsyncClient:
@@ -128,7 +131,7 @@ class HttpxDownloadHandler(_Base):
                 proxy=proxy,
             ),
         )
-        # https://github.com/encode/httpx/discussions/1566
+        # https://github.com/pydantic/httpx2/issues/368
         for header_name in ("accept", "accept-encoding", "user-agent"):
             client.headers.pop(header_name, None)
         return client
@@ -149,7 +152,7 @@ class HttpxDownloadHandler(_Base):
         proxy = self._extract_proxy_url_with_creds(request)
         if proxy and proxy.startswith("socks") and not HAS_SOCKS:  # pragma: no cover
             raise ValueError(
-                f"SOCKS proxy support in {type(self).__name__} requires the 'httpx[socks]' extra to be installed."
+                f"SOCKS proxy support in {type(self).__name__} requires the 'httpx2[socks]' extra to be installed."
             )
         client = self._get_client(proxy)
         headers = self._request_headers(request).to_tuple_list()
@@ -175,7 +178,7 @@ class HttpxDownloadHandler(_Base):
             raise DownloadConnectionRefusedError(str(e)) from e
         except httpx.ProxyError as e:
             raise DownloadConnectionRefusedError(str(e)) from e
-        except DOWNLOAD_FAILED_EXCEPTIONS as e:
+        except DOWNLOAD_FAILED_EXCEPTIONS as e:  # pylint: disable=catching-non-exception
             raise DownloadFailedError(str(e)) from e
 
     @staticmethod
@@ -218,7 +221,7 @@ class HttpxDownloadHandler(_Base):
     def _log_tls_info(self, response: httpx.Response, request: Request) -> None:
         network_stream: AsyncNetworkStream = response.extensions["network_stream"]
         extra_ssl_object = network_stream.get_extra_info("ssl_object")
-        if isinstance(extra_ssl_object, ssl.SSLObject):
+        if isinstance(extra_ssl_object, ssl.SSLObject):  # pragma: no branch
             _log_sslobj_debug_info(extra_ssl_object)
 
     async def close(self) -> None:

tests/test_downloader_handler_httpx.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import sys
+from importlib.util import find_spec
 from typing import TYPE_CHECKING, Any, ClassVar
 
 import pytest
@@ -37,7 +38,8 @@ if TYPE_CHECKING:
 
 pytestmark = pytest.mark.only_asyncio
 
-pytest.importorskip("httpx")
+if find_spec("httpx2") is None and find_spec("httpx") is None:
+    pytest.skip("Neither httpx2 nor httpx are installed", allow_module_level=True)
 
 
 class HttpxDownloadHandlerMixin:

tox.ini

@@ -36,7 +36,7 @@ minversion = 1.7.0
 deps =
     attrs
     coverage >= 7.10.6
-    httpx
+    httpx2
     pexpect >= 4.8.0
     pyftpdlib >= 2.0.1
     pygments
@@ -75,7 +75,7 @@ deps =
     boto3-stubs[s3]==1.43.41
     botocore-stubs==1.43.14
     h2==4.3.0
-    httpx==0.28.1
+    httpx2==2.7.0
     itemadapter==0.13.1
     ptpython==3.0.32
     # newer ones require newer Python
@@ -136,7 +136,7 @@ deps =
     Twisted==21.7.0
     cryptography==37.0.0
     cssselect==0.9.1
-    httpx==0.27.1
+    httpx2==2.0.0
     itemadapter==0.1.0
     lxml==4.6.4
     parsel==1.5.0
@@ -171,7 +171,7 @@ deps =
     brotli >= 1.2.0; implementation_name != "pypy"  # optional for HTTP compress downloader middleware tests
     brotlicffi >= 1.2.0.0; implementation_name == "pypy"  # optional for HTTP compress downloader middleware tests
     google-cloud-storage
-    httpx[http2,socks]
+    httpx2[http2,socks]
     ipython
     ptpython  # optional for shell wrapper tests
     robotexclusionrulesparser
@@ -189,7 +189,7 @@ deps =
     brotli==1.2.0; implementation_name != "pypy"
     brotlicffi==1.2.0.0; implementation_name == "pypy"
     google-cloud-storage==1.29.0
-    httpx[http2,socks]==0.27.1
+    httpx2[http2,socks]==2.0.0
     ipython==7.1.0
     ptpython==2.0.1
     robotexclusionrulesparser==1.6.2