Add support for HTTP/2 and for SOCKS proxies to HttpxDownloadHandler, improve handler docs (#7575)

* Add support for HTTP/2 and SOCKS proxies to HttpxDownloadHandler.

* Update the docs.

* Trim the tables.

* Restore lost wording.

* Handlers docs improvements and fixes.

conftest.py

@@ -99,6 +99,19 @@ def mitm_proxy_server_https(monkeypatch: pytest.MonkeyPatch) -> Generator[MitmPr
         proxy.stop()
 
 
+@pytest.fixture  # function scope because it modifies os.environ
+def socks5_proxy_server(monkeypatch: pytest.MonkeyPatch) -> Generator[MitmProxy]:
+    proxy = MitmProxy(mode="socks5")
+    url = proxy.start()
+    monkeypatch.setenv("http_proxy", url)
+    monkeypatch.setenv("https_proxy", url)
+
+    try:
+        yield proxy
+    finally:
+        proxy.stop()
+
+
 @pytest.fixture(scope="session")
 def reactor_pytest(request) -> str:
     return request.config.getoption("--reactor")

docs/faq.rst

@@ -82,10 +82,18 @@ to steal from us!
 Does Scrapy work with HTTP proxies?
 -----------------------------------
 
-Yes. Support for HTTP proxies is provided (since Scrapy 0.8) through the HTTP
+Yes. Support for HTTP proxies is provided through the HTTP Proxy downloader
-Proxy downloader middleware. See
+middleware. See
 :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware`.
 
+Does Scrapy work with SOCKS proxies?
+------------------------------------
+
+Yes, when using
+:class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`. See
+:class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` and the
+handler documentation.
+
 How can I scrape an item with attributes in different pages?
 ------------------------------------------------------------
 
@@ -360,7 +368,10 @@ method for this purpose. For example:
 Does Scrapy support IPv6 addresses?
 -----------------------------------
 
-Yes, by setting :setting:`TWISTED_DNS_RESOLVER` to ``scrapy.resolver.CachingHostnameResolver``.
+Yes, but when using
+:class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` or
+:class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` you need to
+set :setting:`TWISTED_DNS_RESOLVER` to ``scrapy.resolver.CachingHostnameResolver``.
 Note that by doing so, you lose the ability to set a specific timeout for DNS requests
 (the value of the :setting:`DNS_TIMEOUT` setting is ignored).
 

docs/topics/download-handlers.rst

@@ -130,44 +130,39 @@ these exceptions.
 
 .. _download-handlers-ref:
 
-Built-in download handlers reference
+Built-in HTTP download handlers reference
-====================================
+=========================================
 
-DataURIDownloadHandler
+Scrapy ships several handlers for HTTP and HTTPS requests. While all of them
-----------------------
+support basic features, they may differ in support of specific Scrapy features
+and settings and HTTP protocol features. See the documentation of specific
+handlers and specific settings for more information. Additionally, as the
+underlying HTTP client implementations differ between handlers, the behavior of
+specific websites may be different when doing the same Scrapy requests but
+using different handlers.
 
-.. autoclass:: scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler
+Here is a comparison of some features of the built-in HTTP handlers, see the
+individual handler docs for more differences:
 
-| Supported scheme: ``data``.
+================== ================= ===================== ====================
-| Lazy: no.
+Feature            H2DownloadHandler HTTP11DownloadHandler HttpxDownloadHandler
+================== ================= ===================== ====================
+Requires asyncio   No                No                    Yes
+Requires a reactor Yes               Yes                   No
+HTTP/1.1           No                Yes                   Yes
+HTTP/2             Yes               No                    Yes
+TLS implementation ``cryptography``  ``cryptography``      Stdlib ``ssl``
+HTTP proxies       No                Yes                   Yes
+SOCKS proxies      No                No                    Yes
+================== ================= ===================== ====================
 
-This handler supports RFC 2397 ``data:content/type;base64,`` data URIs.
+You can find additional HTTP download handlers in the
+scrapy-download-handlers-incubator_ package. This package is made by the Scrapy
+developers and contains experimental handlers that may be included in some
+later Scrapy version but can already be used. Please refer to the documentation
+of this package for more information.
 
-FileDownloadHandler
+.. _scrapy-download-handlers-incubator: https://github.com/scrapy-plugins/scrapy-download-handlers-incubator
--------------------
-
-.. autoclass:: scrapy.core.downloader.handlers.file.FileDownloadHandler
-
-| Supported scheme: ``file``.
-| Lazy: no.
-
-This handler supports ``file:///path`` local file URIs. It doesn't
-support remote files.
-
-FTPDownloadHandler
-------------------
-
-.. autoclass:: scrapy.core.downloader.handlers.ftp.FTPDownloadHandler
-
-| Supported scheme: ``ftp``.
-| Lazy: no.
-
-This handler supports ``ftp://host/path`` FTP URIs.
-
-It's implemented using :mod:`twisted.protocols.ftp`.
-
-.. note::
-    This handler is not supported when :setting:`TWISTED_REACTOR_ENABLED` is ``False``.
 
 .. _twisted-http2-handler:
 
@@ -177,7 +172,9 @@ H2DownloadHandler
 .. autoclass:: scrapy.core.downloader.handlers.http2.H2DownloadHandler
 
 | Supported scheme: ``https``.
-| Lazy: yes.
+| :ref:`Lazy <lazy-download-handlers>`: yes.
+| :ref:`Requires asyncio support <using-asyncio>`: no.
+| :ref:`Requires a Twisted reactor <asyncio-without-reactor>`: yes.
 
 This handler supports ``https://host/path`` URLs and uses the HTTP/2 protocol
 for them.
@@ -196,63 +193,96 @@ If you want to use this handler you need to replace the default one for the
         "https": "scrapy.core.downloader.handlers.http2.H2DownloadHandler",
     }
 
+Features and limitations
+^^^^^^^^^^^^^^^^^^^^^^^^
+
 .. warning::
 
     This handler is experimental, and not yet recommended for production
     environments. Future Scrapy versions may introduce related changes without
     a deprecation period or warning.
 
-.. note::
+=========================== ================================================
+HTTP proxies                No (not implemented)
+SOCKS proxies               No (not supported by the library)
+HTTP/2                      Yes
+``response.certificate``    :class:`twisted.internet.ssl.Certificate` object
+Per-request ``bindaddress`` Yes
+TLS implementation          ``pyOpenSSL``/``cryptography``
+=========================== ================================================
 
-    Known limitations of the HTTP/2 implementation in this handler include:
+Other limitations:
 
-    -   No support for proxies.
+-   No support for HTTP/1.1.
 
-    -   No support for HTTP/2 Cleartext (h2c), since no major browser supports
+-   IPv6 support requires setting :setting:`TWISTED_DNS_RESOLVER`
-        HTTP/2 unencrypted (refer `http2 faq`_).
+    to ``scrapy.resolver.CachingHostnameResolver``.
 
-    -   No setting to specify a maximum `frame size`_ larger than the default
+-   No support for the :signal:`bytes_received` and :signal:`headers_received`
-        value, 16384. Connections to servers that send a larger frame will
+    signals.
-        fail.
 
-    -   No support for `server pushes`_, which are ignored.
+Known limitations of the HTTP/2 support:
 
-    -   No support for the :signal:`bytes_received` and
+-   No support for HTTP/2 Cleartext (h2c), since no major browser supports
-        :signal:`headers_received` signals.
+    HTTP/2 unencrypted (refer `http2 faq`_).
+
+-   No setting to specify a maximum `frame size`_ larger than the default
+    value, 16384. Connections to servers that send a larger frame will fail.
+
+-   No support for `server pushes`_, which are ignored.
 
 .. _frame size: https://datatracker.ietf.org/doc/html/rfc7540#section-4.2
 .. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption
 .. _server pushes: https://datatracker.ietf.org/doc/html/rfc7540#section-8.2
 
-.. note::
-    This handler is not supported when :setting:`TWISTED_REACTOR_ENABLED` is ``False``.
-
 HTTP11DownloadHandler
 ---------------------
 
 .. autoclass:: scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler
 
 | Supported schemes: ``http``, ``https``.
-| Lazy: no.
+| :ref:`Lazy <lazy-download-handlers>`: no.
+| :ref:`Requires asyncio support <using-asyncio>`: no.
+| :ref:`Requires a Twisted reactor <asyncio-without-reactor>`: yes.
 
 This handler supports ``http://host/path`` and ``https://host/path`` URLs and
 uses the HTTP/1.1 protocol for them.
 
 It's implemented using :mod:`twisted.web.client`.
 
-.. note::
+Features and limitations
-    This handler is not supported when :setting:`TWISTED_REACTOR_ENABLED` is ``False``.
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+=========================== ================================================
+HTTP proxies                Yes
+SOCKS proxies               No (not supported by the library)
+HTTP/2                      No (implemented as a separate handler)
+``response.certificate``    :class:`twisted.internet.ssl.Certificate` object
+Per-request ``bindaddress`` Yes
+TLS implementation          ``pyOpenSSL``/``cryptography``
+=========================== ================================================
+
+Other limitations:
+
+-   IPv6 support requires setting :setting:`TWISTED_DNS_RESOLVER`
+    to ``scrapy.resolver.CachingHostnameResolver``.
+
+-   HTTPS proxies to HTTPS destinations are not supported.
 
 HttpxDownloadHandler
 --------------------
 
+.. versionadded:: 2.15.0
+
 .. autoclass:: scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler
 
 | Supported schemes: ``http``, ``https``.
-| Lazy: no.
+| :ref:`Lazy <lazy-download-handlers>`: no.
+| :ref:`Requires asyncio support <using-asyncio>`: yes.
+| :ref:`Requires a Twisted reactor <asyncio-without-reactor>`: no.
 
 This handler supports ``http://host/path`` and ``https://host/path`` URLs and
-uses the HTTP/1.1 protocol for them.
+uses the HTTP/1.1 or HTTP/2 protocol for them.
 
 It's implemented using the ``httpx`` library and needs it to be installed.
 
@@ -266,28 +296,83 @@ If you want to use this handler you need to replace the default ones for the
         "https": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler",
     }
 
+Features and limitations
+^^^^^^^^^^^^^^^^^^^^^^^^
+
 .. warning::
 
     This handler is experimental, and not yet recommended for production
     environments. Future Scrapy versions may introduce related changes without
     a deprecation period or warning or even remove it altogether.
 
-.. note::
+=========================== =======================================
+HTTP proxies                Yes
+SOCKS proxies               Yes (SOCKS5; requires ``httpx[socks]``)
+HTTP/2                      Yes (requires ``httpx[http2]``)
+``response.certificate``    DER bytes
+Per-request ``bindaddress`` No (not supported by the library)
+TLS implementation          Standard library ``ssl``
+=========================== =======================================
 
-    As this handler is based on a different HTTP client implementation compared
+Other limitations:
-    to :class:`~.HTTP11DownloadHandler`, it's expected that its behavior on
-    some websites may be different. Additionally, these are the Scrapy features
-    that are explicitly not supported when using it:
 
-    - Per-request bind address support (the :reqmeta:`bindaddress` meta key).
+-   The handler creates a separate connection pool for each proxy URL (due to
-      The global :setting:`DOWNLOAD_BIND_ADDRESS` setting is supported but the
+    limitations of ``httpx``) which may lead to higher resource usage when
-      port number, if specified, will be ignored.
+    using proxy rotation.
 
-    - Settings specific to the Twisted networking or HTTP implementation, like
+.. setting:: HTTPX_HTTP2_ENABLED
-      :setting:`DNS_RESOLVER`.
 
-    - Using :ref:`non-asyncio reactors <disable-asyncio>` (``httpx`` requires
+HTTPX_HTTP2_ENABLED
-      ``asyncio``).
+^^^^^^^^^^^^^^^^^^^
+
+Default: ``False``
+
+Whether to enable HTTP/2 support in this handler. The ``httpx[http2]`` extra
+needs to be installed if you want to enable this setting.
+
+.. versionadded:: VERSION
+
+Built-in non-HTTP download handlers reference
+=============================================
+
+DataURIDownloadHandler
+----------------------
+
+.. autoclass:: scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler
+
+| Supported scheme: ``data``.
+| :ref:`Lazy <lazy-download-handlers>`: no.
+| :ref:`Requires asyncio support <using-asyncio>`: no.
+| :ref:`Requires a Twisted reactor <asyncio-without-reactor>`: no.
+
+This handler supports RFC 2397 ``data:content/type;base64,`` data URIs.
+
+FileDownloadHandler
+-------------------
+
+.. autoclass:: scrapy.core.downloader.handlers.file.FileDownloadHandler
+
+| Supported scheme: ``file``.
+| :ref:`Lazy <lazy-download-handlers>`: no.
+| :ref:`Requires asyncio support <using-asyncio>`: no.
+| :ref:`Requires a Twisted reactor <asyncio-without-reactor>`: no.
+
+This handler supports ``file:///path`` local file URIs. It doesn't
+support remote files.
+
+FTPDownloadHandler
+------------------
+
+.. autoclass:: scrapy.core.downloader.handlers.ftp.FTPDownloadHandler
+
+| Supported scheme: ``ftp``.
+| :ref:`Lazy <lazy-download-handlers>`: no.
+| :ref:`Requires asyncio support <using-asyncio>`: no.
+| :ref:`Requires a Twisted reactor <asyncio-without-reactor>`: yes.
+
+This handler supports ``ftp://host/path`` FTP URIs.
+
+It's implemented using :mod:`twisted.protocols.ftp`.
 
 S3DownloadHandler
 -----------------
@@ -295,7 +380,9 @@ S3DownloadHandler
 .. autoclass:: scrapy.core.downloader.handlers.s3.S3DownloadHandler
 
 | Supported scheme: ``s3``.
-| Lazy: yes.
+| :ref:`Lazy <lazy-download-handlers>`: yes.
+| :ref:`Requires asyncio support <using-asyncio>`: no.
+| :ref:`Requires a Twisted reactor <asyncio-without-reactor>`: no.
 
 This handler supports ``s3://bucket/path`` S3 URIs.
 

docs/topics/downloader-middleware.rst

@@ -745,8 +745,7 @@ HttpProxyMiddleware
     Handling of this meta key needs to be implemented inside the :ref:`download
     handler <topics-download-handlers>`, so it's not guaranteed to be supported
     by all 3rd-party handlers. It's currently unsupported by
-    :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` and
+    :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler`.
-    :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`.
 
 .. note::
 
@@ -758,6 +757,13 @@ HttpProxyMiddleware
     :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler`
     supports HTTPS proxies only for HTTP destinations.
 
+.. note::
+
+    If the download handler supports it, you can use a SOCKS proxy URL (e.g.
+    ``socks5://username:password@some_proxy_server:port``).
+    :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`
+    supports SOCKS proxies while other built-in handlers don't.
+
 HttpProxyMiddleware settings
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

scrapy/core/downloader/handlers/_httpx.py

@@ -5,6 +5,7 @@ from __future__ import annotations
 import ipaddress
 import ssl
 from contextlib import asynccontextmanager
+from socket import gaierror
 from typing import TYPE_CHECKING, ClassVar
 
 from scrapy.exceptions import (
@@ -17,6 +18,7 @@ from scrapy.exceptions import (
 )
 from scrapy.http import Headers
 from scrapy.utils._download_handlers import NullCookieJar
+from scrapy.utils.python import _iter_exc_causes
 from scrapy.utils.ssl import (
     _log_sslobj_debug_info,
     _make_insecure_ssl_ctx,
@@ -34,10 +36,35 @@ if TYPE_CHECKING:
     from scrapy.crawler import Crawler
 
 
+HAS_SOCKS = HAS_HTTP2 = False
+
 try:
     import httpx
 except ImportError:
     httpx = None  # type: ignore[assignment]
+else:
+    # a small hack to avoid importing these optional extras unconditionally
+
+    DOWNLOAD_FAILED_EXCEPTIONS: tuple[type[BaseException], ...] = (
+        httpx.RequestError,
+        httpx.InvalidURL,
+    )
+
+    try:
+        import h2.exceptions
+
+        HAS_HTTP2 = True
+        DOWNLOAD_FAILED_EXCEPTIONS += (h2.exceptions.InvalidBodyLengthError,)
+    except ImportError:  # pragma: no cover
+        pass
+
+    try:
+        import socksio.exceptions
+
+        HAS_SOCKS = True
+        DOWNLOAD_FAILED_EXCEPTIONS += (socksio.exceptions.ProtocolError,)
+    except ImportError:  # pragma: no cover
+        pass
 
 
 if TYPE_CHECKING:
@@ -54,6 +81,11 @@ class HttpxDownloadHandler(_Base):
         self._verify_certificates: bool = crawler.settings.getbool(
             "DOWNLOAD_VERIFY_CERTIFICATES"
         )
+        self._enable_h2: bool = crawler.settings.getbool("HTTPX_HTTP2_ENABLED")
+        if self._enable_h2 and not HAS_HTTP2:  # pragma: no cover
+            raise NotConfigured(
+                f"HTTP/2 support in {type(self).__name__} requires the 'httpx[http2]' extra to be installed."
+            )
         self._ssl_context: ssl.SSLContext = _make_ssl_context(crawler.settings)
         self._bind_host: str | None = self._get_bind_address_host()
         self._limits: httpx.Limits = httpx.Limits(
@@ -90,6 +122,7 @@ class HttpxDownloadHandler(_Base):
             transport=httpx.AsyncHTTPTransport(
                 verify=self._ssl_context,
                 local_address=self._bind_host,
+                http2=self._enable_h2,
                 limits=self._limits,
                 trust_env=False,
                 proxy=proxy,
@@ -113,7 +146,13 @@ class HttpxDownloadHandler(_Base):
     async def _make_request(
         self, request: Request, timeout: float
     ) -> AsyncIterator[httpx.Response]:
-        client = self._get_client(self._extract_proxy_url_with_creds(request))
+        proxy = self._extract_proxy_url_with_creds(request)
+        if proxy and proxy.startswith("socks") and not HAS_SOCKS:  # pragma: no cover
+            raise ValueError(
+                f"SOCKS proxy support in {type(self).__name__} requires the 'httpx[socks]' extra to be installed."
+            )
+        client = self._get_client(proxy)
+
         try:
             async with client.stream(
                 request.method,
@@ -130,18 +169,12 @@ class HttpxDownloadHandler(_Base):
         except httpx.UnsupportedProtocol as e:
             raise UnsupportedURLSchemeError(str(e)) from e
         except httpx.ConnectError as e:
-            error_message = str(e)
+            if any(isinstance(c, gaierror) for c in _iter_exc_causes(e)):
-            if (
+                raise CannotResolveHostError(str(e)) from e
-                "Name or service not known" in error_message
-                or "getaddrinfo failed" in error_message
-                or "nodename nor servname" in error_message
-                or "Temporary failure in name resolution" in error_message
-            ):
-                raise CannotResolveHostError(error_message) from e
             raise DownloadConnectionRefusedError(str(e)) from e
         except httpx.ProxyError as e:
             raise DownloadConnectionRefusedError(str(e)) from e
-        except (httpx.NetworkError, httpx.RemoteProtocolError) as e:
+        except DOWNLOAD_FAILED_EXCEPTIONS as e:
             raise DownloadFailedError(str(e)) from e
 
     @staticmethod

scrapy/settings/default_settings.py

@@ -109,6 +109,7 @@ __all__ = [
     "HTTPCACHE_STORAGE",
     "HTTPPROXY_AUTH_ENCODING",
     "HTTPPROXY_ENABLED",
+    "HTTPX_HTTP2_ENABLED",
     "IMAGES_STORE_GCS_ACL",
     "IMAGES_STORE_S3_ACL",
     "ITEM_PIPELINES",
@@ -382,6 +383,8 @@ HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage"
 HTTPPROXY_ENABLED = True
 HTTPPROXY_AUTH_ENCODING = "latin-1"
 
+HTTPX_HTTP2_ENABLED = False
+
 IMAGES_STORE_GCS_ACL = ""
 IMAGES_STORE_S3_ACL = "private"
 

scrapy/utils/python.py

@@ -359,3 +359,13 @@ def _looks_like_import_path(value: str) -> bool:
     if any(part == "" for part in parts):
         return False
     return all(part.isidentifier() for part in parts)
+
+
+def _iter_exc_causes(exc: BaseException) -> Iterable[BaseException]:
+    """Iterate over the exception causes/contexts."""
+    seen: set[int] = set()
+    cur: BaseException | None = exc
+    while cur is not None and id(cur) not in seen:
+        seen.add(id(cur))
+        yield cur
+        cur = cur.__cause__ or cur.__context__

tests/mockserver/mitm_proxy.py

@@ -11,6 +11,9 @@ class MitmProxy:
     auth_user = "scrapy"
     auth_pass = "scrapy"
 
+    def __init__(self, mode: str | None = None) -> None:
+        self.mode = mode
+
     def start(self) -> str:
         script = """
 import sys
@@ -32,6 +35,8 @@ sys.exit(mitmdump())
             "-s",
             str(Path(__file__).with_name("mitm_proxy_addon.py")),
         ]
+        if self.mode:
+            args += ["--mode", self.mode]
         self.proc: Popen[str] = Popen(
             [
                 sys.executable,
@@ -44,12 +49,13 @@ sys.exit(mitmdump())
             text=True,
         )
         assert self.proc.stdout is not None
+        scheme = "socks5" if self.mode == "socks5" else "http"
         line = ""
         for line in self.proc.stdout:
-            m = re.search(r"listening at (?:http://)?([^:]+:\d+)", line)
+            m = re.search(r"listening at (?:\w+://)?([^:]+:\d+)", line)
             if m:
                 host_port = m.group(1)
-                return f"http://{self.auth_user}:{self.auth_pass}@{host_port}"
+                return f"{scheme}://{self.auth_user}:{self.auth_pass}@{host_port}"
         self.stop()
         raise RuntimeError(f"Failed to parse mitmdump output: {line}")
 

tests/test_downloader_handler_httpx.py

@@ -3,11 +3,17 @@
 from __future__ import annotations
 
 import sys
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, ClassVar
 
 import pytest
 
 from scrapy import Request
+from scrapy.core.downloader.handlers._httpx import (
+    HAS_HTTP2,
+    HAS_SOCKS,
+    HttpxDownloadHandler,
+)
+from scrapy.exceptions import DownloadFailedError
 from tests.test_downloader_handlers_http_base import (
     TestHttpBase,
     TestHttpProxyBase,
@@ -37,10 +43,6 @@ pytest.importorskip("httpx")
 class HttpxDownloadHandlerMixin:
     @property
     def download_handler_cls(self) -> type[DownloadHandlerProtocol]:
-        from scrapy.core.downloader.handlers._httpx import (  # noqa: PLC0415
-            HttpxDownloadHandler,
-        )
-
         return HttpxDownloadHandler
 
     @property
@@ -83,6 +85,30 @@ class TestHttps(HttpxDownloadHandlerMixin, TestHttpsBase):
         pass
 
 
+@pytest.mark.skipif(not HAS_HTTP2, reason="No HTTP/2 support in HttpxDownloadHandler")
+class TestHttp2(TestHttps):
+    http2 = True
+    handler_supports_http2_dataloss = False
+
+    default_handler_settings: ClassVar[dict[str, Any]] = {
+        "HTTPX_HTTP2_ENABLED": True,
+    }
+
+    @coroutine_test
+    async def test_protocol(self, mockserver: MockServer) -> None:
+        request = Request(mockserver.url("/host", is_secure=self.is_secure))
+        async with self.get_dh() as download_handler:
+            response = await download_handler.download_request(request)
+        assert response.protocol == "HTTP/2"
+
+    @coroutine_test
+    async def test_data_loss_handling(self, mockserver: MockServer) -> None:
+        request = Request(mockserver.url("/broken", is_secure=self.is_secure))
+        async with self.get_dh() as download_handler:
+            with pytest.raises(DownloadFailedError):
+                await download_handler.download_request(request)
+
+
 class TestSimpleHttps(HttpxDownloadHandlerMixin, TestSimpleHttpsBase):
     pass
 
@@ -127,7 +153,7 @@ class TestHttpsProxy(TestHttpProxy):
 
 @pytest.mark.requires_mitmproxy
 class TestMitmProxy(HttpxDownloadHandlerMixin, TestMitmProxyBase):
-    pass
+    handler_supports_socks = HAS_SOCKS
 
 
 @pytest.mark.requires_internet

tests/test_downloader_handlers_http_base.py

@@ -39,7 +39,7 @@ from scrapy.utils.misc import build_from_crawler
 from scrapy.utils.spider import DefaultSpider
 from scrapy.utils.test import get_crawler
 from tests import NON_EXISTING_RESOLVABLE
-from tests.mockserver.mitm_proxy import MitmProxy, wrong_credentials
+from tests.mockserver.mitm_proxy import wrong_credentials
 from tests.mockserver.proxy_echo import ProxyEchoMockServer
 from tests.mockserver.simple_https import SimpleMockServer
 from tests.spiders import (
@@ -73,6 +73,7 @@ class TestHttpBase(ABC):
     handler_supports_http2_dataloss: bool = True
     # default headers added by the underlying library that cannot be suppressed
     always_present_req_headers: ClassVar[frozenset[str]] = frozenset()
+    default_handler_settings: ClassVar[dict[str, Any]] = {}
 
     @property
     @abstractmethod
@@ -83,6 +84,10 @@ class TestHttpBase(ABC):
     async def get_dh(
         self, settings_dict: dict[str, Any] | None = None
     ) -> AsyncGenerator[DownloadHandlerProtocol]:
+        settings_dict = {
+            **self.default_handler_settings,
+            **(settings_dict or {}),
+        }
         crawler = get_crawler(DefaultSpider, settings_dict)
         crawler.spider = crawler._create_spider()
         dh = build_from_crawler(self.download_handler_cls, crawler)
@@ -339,9 +344,7 @@ class TestHttpBase(ABC):
 
     @coroutine_test
     async def test_timeout_download_from_spider_server_hangs(
-        self,
+        self, mockserver: MockServer, reactor_pytest: str
-        mockserver: MockServer,
-        reactor_pytest: str,
     ) -> None:
         if reactor_pytest == "asyncio" and sys.platform == "win32":
             # https://twistedmatrix.com/trac/ticket/10279
@@ -1317,6 +1320,7 @@ class TestHttpProxyBase(ABC):
 class TestMitmProxyBase(ABC):
     # whether the handler supports HTTPS proxies with HTTPS destinations
     handler_supports_tls_in_tls: bool = True
+    handler_supports_socks: bool = False
 
     @property
     @abstractmethod
@@ -1326,13 +1330,10 @@ class TestMitmProxyBase(ABC):
     @pytest.mark.parametrize(
         "https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"]
     )
+    @pytest.mark.usefixtures("mitm_proxy_server")
     @coroutine_test
     async def test_http_proxy(
-        self,
+        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer, https_dest: bool
-        caplog: pytest.LogCaptureFixture,
-        mockserver: MockServer,
-        mitm_proxy_server: MitmProxy,
-        https_dest: bool,
     ) -> None:
         """HTTP proxy, HTTP or HTTPS destination."""
         crawler = get_crawler(SingleRequestSpider, self.settings_dict)
@@ -1347,13 +1348,10 @@ class TestMitmProxyBase(ABC):
     @pytest.mark.parametrize(
         "https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"]
     )
+    @pytest.mark.usefixtures("mitm_proxy_server_https")
     @coroutine_test
     async def test_https_proxy(
-        self,
+        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer, https_dest: bool
-        caplog: pytest.LogCaptureFixture,
-        mockserver: MockServer,
-        mitm_proxy_server_https: MitmProxy,
-        https_dest: bool,
     ) -> None:
         """HTTPS proxy, HTTP or HTTPS destination."""
         if https_dest and not self.handler_supports_tls_in_tls:
@@ -1370,13 +1368,13 @@ class TestMitmProxyBase(ABC):
     @pytest.mark.parametrize(
         "https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"]
     )
+    @pytest.mark.usefixtures("mitm_proxy_server")
     @coroutine_test
     async def test_http_proxy_auth_error(
         self,
         caplog: pytest.LogCaptureFixture,
         monkeypatch: pytest.MonkeyPatch,
         mockserver: MockServer,
-        mitm_proxy_server: MitmProxy,
         https_dest: bool,
     ) -> None:
         """HTTP proxy, HTTP or HTTPS destination, wrong proxy creds."""
@@ -1394,13 +1392,10 @@ class TestMitmProxyBase(ABC):
     @pytest.mark.parametrize(
         "https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"]
     )
+    @pytest.mark.usefixtures("mitm_proxy_server")
     @coroutine_test
     async def test_dont_leak_proxy_authorization_header(
-        self,
+        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer, https_dest: bool
-        caplog: pytest.LogCaptureFixture,
-        mockserver: MockServer,
-        mitm_proxy_server: MitmProxy,
-        https_dest: bool,
     ) -> None:
         """HTTP proxy, HTTP or HTTPS destination. Check that the auth header
         is not sent to the destination."""
@@ -1414,6 +1409,49 @@ class TestMitmProxyBase(ABC):
         echo = json.loads(crawler.spider.meta["responses"][0].text)
         assert "Proxy-Authorization" not in echo["headers"]
 
+    @pytest.mark.parametrize(
+        "https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"]
+    )
+    @pytest.mark.usefixtures("socks5_proxy_server")
+    @coroutine_test
+    async def test_download_with_socks_proxy(
+        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer, https_dest: bool
+    ) -> None:
+        """SOCKS5 proxy, HTTP or HTTPS destination."""
+        if not self.handler_supports_socks:
+            pytest.skip("SOCKS proxies are not supported")
+        crawler = get_crawler(SingleRequestSpider, self.settings_dict)
+        with caplog.at_level(logging.DEBUG):
+            await crawler.crawl_async(
+                seed=mockserver.url("/status?n=200", is_secure=https_dest)
+            )
+        assert isinstance(crawler.spider, SingleRequestSpider)
+        self._assert_got_response_code(200, caplog.text)
+        self._assert_headers(crawler.spider.meta["responses"][0].headers, https_dest)
+
+    @pytest.mark.parametrize(
+        "https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"]
+    )
+    @pytest.mark.usefixtures("socks5_proxy_server")
+    @coroutine_test
+    async def test_socks_proxy_auth_error(
+        self,
+        caplog: pytest.LogCaptureFixture,
+        monkeypatch: pytest.MonkeyPatch,
+        mockserver: MockServer,
+        https_dest: bool,
+    ) -> None:
+        if not self.handler_supports_socks:
+            pytest.skip("SOCKS proxies are not supported")
+        envvar = "https_proxy" if https_dest else "http_proxy"
+        monkeypatch.setenv(envvar, wrong_credentials(os.environ[envvar]))
+        crawler = get_crawler(SimpleSpider, self.settings_dict)
+        with caplog.at_level(logging.DEBUG):
+            await crawler.crawl_async(
+                mockserver.url("/status?n=200", is_secure=https_dest)
+            )
+        assert "DownloadConnectionRefusedError" in caplog.text
+
     @staticmethod
     def _assert_headers(headers: Headers, https_dest: bool) -> None:
         assert b"X-Via-Mitmproxy" in headers

tox.ini

@@ -60,6 +60,7 @@ deps =
     ipython==8.39.0
     pyOpenSSL==26.2.0
     pytest==9.0.3
+    socksio==1.0.0
     types-Pygments==2.20.0.20260508
     types-defusedxml==0.7.0.20260504
     types-lxml==2026.2.16
@@ -148,7 +149,7 @@ deps =
     brotli >= 1.2.0; implementation_name != "pypy"  # optional for HTTP compress downloader middleware tests
     brotlicffi >= 1.2.0.0; implementation_name == "pypy"  # optional for HTTP compress downloader middleware tests
     google-cloud-storage
-    httpx
+    httpx[http2,socks]
     ipython
     robotexclusionrulesparser
     uvloop; platform_system != "Windows" and implementation_name != "pypy"
@@ -304,5 +305,6 @@ deps =
     {[testenv]deps}
     # mitmproxy does not support PyPy
     mitmproxy; implementation_name != "pypy"
+    httpx[http2,socks]
 commands =
     pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=botocore.junit.xml -o junit_family=legacy} -m requires_mitmproxy