RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add HttpxDownloader. (#7239)

This commit is contained in:
Andrey Rakhmatullin 2026-02-12 11:30:44 +03:00 committed by GitHub
parent 0cfc4e4386
commit fc30c47f38
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 579 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
conftest.py
View File

@ -52,6 +52,11 @@ if not H2_ENABLED:
)
)
try:
import httpx # noqa: F401
except ImportError:
collect_ignore.append("scrapy/core/downloader/handlers/_httpx.py")
def pytest_addoption(parser, pluginmanager):
if pluginmanager.hasplugin("twisted"):

13
docs/news.rst
View File

@ -3,6 +3,17 @@
Release notes
=============
Scrapy VERSION (unreleased)
---------------------------
New features
~~~~~~~~~~~~
- Added
:meth:`Headers.to_tuple_list() <scrapy.http.headers.Headers.to_tuple_list>`
that returns headers as a list of ``(key, value)`` tuples.
(:issue:`7239`)
.. _release-2.14.1:
Scrapy 2.14.1 (2026-01-12)
@ -2997,7 +3008,7 @@ Bug fixes
that does not match the asyncio event loop actually installed
(:issue:`5529`).
- Fixed :meth:`Headers.getlist <scrapy.http.headers.Headers.getlist>`
- Fixed :meth:`Headers.getlist() <scrapy.http.headers.Headers.getlist>`
returning only the last header (:issue:`5515`, :issue:`5526`).
- Fixed :class:`LinkExtractor

269
scrapy/core/downloader/handlers/_httpx.py Normal file
View File

@ -0,0 +1,269 @@
"""``httpx``-based HTTP(S) download handler. Currently not recommended for production use."""
from __future__ import annotations
import ipaddress
import logging
import ssl
from http.cookiejar import Cookie, CookieJar
from io import BytesIO
from typing import TYPE_CHECKING, Any, NoReturn, TypedDict
import httpx
from scrapy import Request, signals
from scrapy.exceptions import (
CannotResolveHostError,
DownloadCancelledError,
DownloadConnectionRefusedError,
DownloadFailedError,
DownloadTimeoutError,
NotConfigured,
ResponseDataLossError,
UnsupportedURLSchemeError,
)
from scrapy.http import Headers, Response
from scrapy.utils._download_handlers import (
BaseHttpDownloadHandler,
check_stop_download,
get_dataloss_msg,
get_maxsize_msg,
get_warnsize_msg,
make_response,
)
from scrapy.utils.asyncio import is_asyncio_available
from scrapy.utils.ssl import _log_sslobj_debug_info, _make_ssl_context
if TYPE_CHECKING:
from contextlib import AbstractAsyncContextManager
from http.client import HTTPResponse
from ipaddress import IPv4Address, IPv6Address
from urllib.request import Request as ULRequest
from httpcore import AsyncNetworkStream
from scrapy.crawler import Crawler
logger = logging.getLogger(__name__)
class _BaseResponseArgs(TypedDict):
status: int
url: str
headers: Headers
ip_address: IPv4Address | IPv6Address
protocol: str
# workaround for (and from) https://github.com/encode/httpx/issues/2992
class _NullCookieJar(CookieJar): # pragma: no cover
"""A CookieJar that rejects all cookies."""
def extract_cookies(self, response: HTTPResponse, request: ULRequest) -> None:
pass
def set_cookie(self, cookie: Cookie) -> None:
pass
class HttpxDownloadHandler(BaseHttpDownloadHandler):
_DEFAULT_CONNECT_TIMEOUT = 10
def __init__(self, crawler: Crawler):
# we don't run extra-deps tests with the non-asyncio reactor
if not is_asyncio_available(): # pragma: no cover
raise NotConfigured(
f"{type(self).__name__} requires the asyncio support. Make"
f" sure that you have either enabled the asyncio Twisted"
f" reactor in the TWISTED_REACTOR setting or disabled the"
f" TWISTED_ENABLED setting. See the asyncio documentation"
f" of Scrapy for more information."
)
super().__init__(crawler)
logger.warning(
"HttpxDownloadHandler is experimental and is not recommented for production use."
)
self._tls_verbose_logging: bool = self.crawler.settings.getbool(
"DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING"
)
self._client = httpx.AsyncClient(
verify=_make_ssl_context(crawler.settings), cookies=_NullCookieJar()
)
async def download_request(self, request: Request) -> Response:
self._warn_unsupported_meta(request.meta)
timeout: float = request.meta.get(
"download_timeout", self._DEFAULT_CONNECT_TIMEOUT
)
try:
async with self._get_httpx_response(request, timeout) as httpx_response:
return await self._read_response(httpx_response, request)
except httpx.TimeoutException as e:
raise DownloadTimeoutError(
f"Getting {request.url} took longer than {timeout} seconds."
) from e
except httpx.UnsupportedProtocol as e:
raise UnsupportedURLSchemeError(str(e)) from e
except httpx.ConnectError as e:
if "Name or service not known" in str(e) or "getaddrinfo failed" in str(e):
raise CannotResolveHostError(str(e)) from e
raise DownloadConnectionRefusedError(str(e)) from e
except httpx.NetworkError as e:
raise DownloadFailedError(str(e)) from e
except httpx.RemoteProtocolError as e:
raise DownloadFailedError(str(e)) from e
def _warn_unsupported_meta(self, meta: dict[str, Any]) -> None:
if meta.get("bindaddress"):
# configurable only per-client:
# https://github.com/encode/httpx/issues/755#issuecomment-2746121794
logger.error(
f"The 'bindaddress' request meta key is not supported by"
f" {type(self).__name__} and will be ignored."
)
if meta.get("proxy"):
# configurable only per-client:
# https://github.com/encode/httpx/issues/486
logger.error(
f"The 'proxy' request meta key is not supported by"
f" {type(self).__name__} and will be ignored."
)
def _get_httpx_response(
self, request: Request, timeout: float
) -> AbstractAsyncContextManager[httpx.Response]:
return self._client.stream(
request.method,
request.url,
content=request.body,
headers=request.headers.to_tuple_list(),
timeout=timeout,
)
async def _read_response(
self, httpx_response: httpx.Response, request: Request
) -> Response:
maxsize: int = request.meta.get("download_maxsize", self._default_maxsize)
warnsize: int = request.meta.get("download_warnsize", self._default_warnsize)
content_length = httpx_response.headers.get("Content-Length")
expected_size = int(content_length) if content_length is not None else None
if maxsize and expected_size and expected_size > maxsize:
self._cancel_maxsize(expected_size, maxsize, request, expected=True)
reached_warnsize = False
if warnsize and expected_size and expected_size > warnsize:
reached_warnsize = True
logger.warning(
get_warnsize_msg(expected_size, warnsize, request, expected=True)
)
headers = Headers(httpx_response.headers.multi_items())
network_stream: AsyncNetworkStream = httpx_response.extensions["network_stream"]
make_response_base_args: _BaseResponseArgs = {
"status": httpx_response.status_code,
"url": request.url,
"headers": headers,
"ip_address": self._get_server_ip(network_stream),
"protocol": httpx_response.http_version,
}
self._log_tls_info(network_stream)
if stop_download := check_stop_download(
signals.headers_received,
self.crawler,
request,
headers=headers,
body_length=expected_size,
):
return make_response(
**make_response_base_args,
stop_download=stop_download,
)
response_body = BytesIO()
bytes_received = 0
try:
async for chunk in httpx_response.aiter_raw():
response_body.write(chunk)
bytes_received += len(chunk)
if stop_download := check_stop_download(
signals.bytes_received, self.crawler, request, data=chunk
):
return make_response(
**make_response_base_args,
body=response_body.getvalue(),
stop_download=stop_download,
)
if maxsize and bytes_received > maxsize:
response_body.truncate(0)
self._cancel_maxsize(
bytes_received, maxsize, request, expected=False
)
if warnsize and bytes_received > warnsize and not reached_warnsize:
reached_warnsize = True
logger.warning(
get_warnsize_msg(
bytes_received, warnsize, request, expected=False
)
)
except httpx.RemoteProtocolError as e:
# special handling of the dataloss case
if (
"peer closed connection without sending complete message body"
not in str(e)
):
raise
fail_on_dataloss: bool = request.meta.get(
"download_fail_on_dataloss", self._fail_on_dataloss
)
if not fail_on_dataloss:
return make_response(
**make_response_base_args,
body=response_body.getvalue(),
flags=["dataloss"],
)
self._log_dataloss_warning(request.url)
raise ResponseDataLossError(str(e)) from e
return make_response(
**make_response_base_args,
body=response_body.getvalue(),
)
@staticmethod
def _get_server_ip(network_stream: AsyncNetworkStream) -> IPv4Address | IPv6Address:
extra_server_addr = network_stream.get_extra_info("server_addr")
return ipaddress.ip_address(extra_server_addr[0])
def _log_tls_info(self, network_stream: AsyncNetworkStream) -> None:
if not self._tls_verbose_logging:
return
extra_ssl_object = network_stream.get_extra_info("ssl_object")
if isinstance(extra_ssl_object, ssl.SSLObject):
_log_sslobj_debug_info(extra_ssl_object)
def _log_dataloss_warning(self, url: str) -> None:
if self._fail_on_dataloss_warned:
return
logger.warning(get_dataloss_msg(url))
self._fail_on_dataloss_warned = True
@staticmethod
def _cancel_maxsize(
size: int, limit: int, request: Request, *, expected: bool
) -> NoReturn:
warning_msg = get_maxsize_msg(size, limit, request, expected=expected)
logger.warning(warning_msg)
raise DownloadCancelledError(warning_msg)
async def close(self):
await self._client.aclose()

11
scrapy/http/headers.py
View File

@ -124,6 +124,17 @@ class Headers(CaselessDict):
for key, value in self.items()
)
def to_tuple_list(self) -> list[tuple[str, str]]:
"""Return headers as a list of ``(key, value)`` tuples.
Multiple values are represented as multiple tuples with the same key.
"""
return [
(key.decode(self.encoding), value.decode(self.encoding))
for key, values in self.items()
for value in values
]
def __copy__(self) -> Self:
return self.__class__(self)

6
scrapy/utils/log.py
View File

@ -72,6 +72,12 @@ DEFAULT_LOGGING = {
"hpack": {
"level": "ERROR",
},
"httpcore": {
"level": "ERROR",
},
"httpx": {
"level": "WARNING",
},
"scrapy": {
"level": "DEBUG",
},

51
scrapy/utils/ssl.py
View File

@ -1,5 +1,7 @@
from __future__ import annotations
import logging
import ssl
from typing import TYPE_CHECKING, Any
import OpenSSL._util as pyOpenSSLutil
@ -11,6 +13,54 @@ from scrapy.utils.python import to_unicode
if TYPE_CHECKING:
from OpenSSL.crypto import X509Name
from scrapy.settings import BaseSettings
logger = logging.getLogger(__name__)
# stdlib ssl module utils
# possible documented values for DOWNLOADER_CLIENT_TLS_METHOD
_STDLIB_PROTOCOL_MAP = {
"TLS": ssl.PROTOCOL_TLS_CLIENT,
"TLSv1.0": ssl.PROTOCOL_TLSv1,
"TLSv1.1": ssl.PROTOCOL_TLSv1_1,
"TLSv1.2": ssl.PROTOCOL_TLSv1_2,
}
def _make_ssl_context(settings: BaseSettings) -> ssl.SSLContext:
"""Create an :class:`ssl.SSLContext` instance according to the settings.
It's intended to be used in an HTTPS download handler.
"""
method_setting: str = settings["DOWNLOADER_CLIENT_TLS_METHOD"]
if method_setting not in _STDLIB_PROTOCOL_MAP:
raise ValueError(f"Unsupported TLS method: {method_setting}")
ciphers_setting: str | None = settings["DOWNLOADER_CLIENT_TLS_CIPHERS"]
ctx = ssl.SSLContext(_STDLIB_PROTOCOL_MAP[method_setting])
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
if ciphers_setting:
ctx.set_ciphers(ciphers_setting)
return ctx
def _log_sslobj_debug_info(sslobj: ssl.SSLObject) -> None:
cipher = sslobj.cipher()
logger.debug(
f"SSL connection to {sslobj.server_hostname}"
f" using protocol {sslobj.version()},"
f" cipher {cipher[0] if cipher else None}"
)
# The peer certificate is unavailable on SSLObject unless peer
# certificate verification is enabled, which we don't want.
# pyOpenSSL utils
def ffi_buf_to_string(buf: Any) -> str:
return to_unicode(pyOpenSSLutil.ffi.string(buf))
@ -22,7 +72,6 @@ def x509name_to_string(x509name: X509Name) -> str:
pyOpenSSLutil.lib.X509_NAME_oneline(
x509name._name, result_buffer, len(result_buffer)
)
return ffi_buf_to_string(result_buffer)

10
tests/spiders.py
View File

@ -22,9 +22,10 @@ from scrapy.utils.test import get_from_asyncio_queue, get_web_client_agent_req
class MockServerSpider(Spider):
def __init__(self, mockserver=None, *args, **kwargs):
def __init__(self, *args, mockserver=None, is_secure=False, **kwargs):
super().__init__(*args, **kwargs)
self.mockserver = mockserver
self.is_secure = is_secure
class MetaSpider(MockServerSpider):
@ -516,7 +517,7 @@ class BytesReceivedCallbackSpider(MetaSpider):
async def start(self):
body = b"a" * self.full_response_length
url = self.mockserver.url("/alpayload")
url = self.mockserver.url("/alpayload", is_secure=self.is_secure)
yield Request(url, method="POST", body=body, errback=self.errback)
def parse(self, response):
@ -544,7 +545,10 @@ class HeadersReceivedCallbackSpider(MetaSpider):
return spider
async def start(self):
yield Request(self.mockserver.url("/status"), errback=self.errback)
yield Request(
self.mockserver.url("/status", is_secure=self.is_secure),
errback=self.errback,
)
def parse(self, response):
self.meta["response"] = response

127
tests/test_downloader_handler_httpx.py Normal file
View File

@ -0,0 +1,127 @@
"""Tests for scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler."""
from __future__ import annotations
from typing import TYPE_CHECKING, Any
import pytest
from scrapy import Request
from tests.test_downloader_handlers_http_base import (
TestHttp11Base,
TestHttpProxyBase,
TestHttps11Base,
TestHttpsCustomCiphersBase,
TestHttpsInvalidDNSIdBase,
TestHttpsInvalidDNSPatternBase,
TestHttpsWrongHostnameBase,
TestHttpWithCrawlerBase,
TestSimpleHttpsBase,
)
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
from scrapy.core.downloader.handlers import DownloadHandlerProtocol
from tests.mockserver.http import MockServer
pytest.importorskip("httpx")
class HttpxDownloadHandlerMixin:
@property
def download_handler_cls(self) -> type[DownloadHandlerProtocol]:
# the import will fail if httpx is not installed
from scrapy.core.downloader.handlers._httpx import ( # noqa: PLC0415
HttpxDownloadHandler,
)
return HttpxDownloadHandler
class TestHttp11(HttpxDownloadHandlerMixin, TestHttp11Base):
@coroutine_test
async def test_unsupported_bindaddress(
self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
) -> None:
meta = {"bindaddress": "127.0.0.2"}
request = Request(mockserver.url("/text"), meta=meta)
async with self.get_dh() as download_handler:
response = await download_handler.download_request(request)
assert response.body == b"Works"
assert (
"The 'bindaddress' request meta key is not supported by HttpxDownloadHandler"
in caplog.text
)
@coroutine_test
async def test_unsupported_proxy(
self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
) -> None:
meta = {"proxy": "127.0.0.2"}
request = Request(mockserver.url("/text"), meta=meta)
async with self.get_dh() as download_handler:
response = await download_handler.download_request(request)
assert response.body == b"Works"
assert (
"The 'proxy' request meta key is not supported by HttpxDownloadHandler"
in caplog.text
)
class TestHttps11(HttpxDownloadHandlerMixin, TestHttps11Base):
tls_log_message = "SSL connection to 127.0.0.1 using protocol TLSv1.3, cipher"
class TestSimpleHttps(HttpxDownloadHandlerMixin, TestSimpleHttpsBase):
pass
class Https11WrongHostnameTestCase(
HttpxDownloadHandlerMixin, TestHttpsWrongHostnameBase
):
pass
class Https11InvalidDNSId(HttpxDownloadHandlerMixin, TestHttpsInvalidDNSIdBase):
pass
class Https11InvalidDNSPattern(
HttpxDownloadHandlerMixin, TestHttpsInvalidDNSPatternBase
):
pass
class Https11CustomCiphers(HttpxDownloadHandlerMixin, TestHttpsCustomCiphersBase):
pass
class TestHttp11WithCrawler(TestHttpWithCrawlerBase):
@property
def settings_dict(self) -> dict[str, Any] | None:
return {
"DOWNLOAD_HANDLERS": {
"http": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler",
"https": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler",
}
}
class TestHttps11WithCrawler(TestHttp11WithCrawler):
is_secure = True
@pytest.mark.skip(reason="response.certificate is not implemented")
@coroutine_test
async def test_response_ssl_certificate(self, mockserver: MockServer) -> None:
pass
@pytest.mark.skip(reason="Proxy support is not implemented yet")
class TestHttp11Proxy(HttpxDownloadHandlerMixin, TestHttpProxyBase):
pass
@pytest.mark.skip(reason="Proxy support is not implemented yet")
class TestHttps11Proxy(HttpxDownloadHandlerMixin, TestHttpProxyBase):
is_secure = True

12
tests/test_downloader_handler_twisted_http2.py
View File

@ -189,6 +189,18 @@ class TestHttp2WithCrawler(TestHttpWithCrawlerBase):
is_secure = True
def test_bytes_received_stop_download_callback(self) -> None: # type: ignore[override]
pytest.skip("bytes_received support is not implemented")
def test_bytes_received_stop_download_errback(self) -> None: # type: ignore[override]
pytest.skip("bytes_received support is not implemented")
def test_headers_received_stop_download_callback(self) -> None: # type: ignore[override]
pytest.skip("headers_received support is not implemented")
def test_headers_received_stop_download_errback(self) -> None: # type: ignore[override]
pytest.skip("headers_received support is not implemented")
class TestHttps2Proxy(H2DownloadHandlerMixin, TestHttpProxyBase):
is_secure = True

78
tests/test_downloader_handlers_http_base.py
View File

@ -16,6 +16,7 @@ from urllib.parse import urlparse
import pytest
from twisted.internet.ssl import Certificate
from twisted.python.failure import Failure
from scrapy.exceptions import (
CannotResolveHostError,
@ -24,6 +25,7 @@ from scrapy.exceptions import (
DownloadFailedError,
DownloadTimeoutError,
ResponseDataLossError,
StopDownload,
UnsupportedURLSchemeError,
)
from scrapy.http import Headers, HtmlResponse, Request, Response, TextResponse
@ -34,7 +36,13 @@ from scrapy.utils.test import get_crawler
from tests import NON_EXISTING_RESOLVABLE
from tests.mockserver.proxy_echo import ProxyEchoMockServer
from tests.mockserver.simple_https import SimpleMockServer
from tests.spiders import SingleRequestSpider
from tests.spiders import (
BytesReceivedCallbackSpider,
BytesReceivedErrbackSpider,
HeadersReceivedCallbackSpider,
HeadersReceivedErrbackSpider,
SingleRequestSpider,
)
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
@ -832,6 +840,74 @@ class TestHttpWithCrawlerBase(ABC):
assert isinstance(ip_address, IPv4Address)
assert str(ip_address) == gethostbyname(expected_netloc)
@coroutine_test
async def test_bytes_received_stop_download_callback(
self, mockserver: MockServer
) -> None:
# copy of TestCrawl.test_bytes_received_stop_download_callback()
crawler = get_crawler(BytesReceivedCallbackSpider, self.settings_dict)
await crawler.crawl_async(mockserver=mockserver, is_secure=self.is_secure)
assert isinstance(crawler.spider, BytesReceivedCallbackSpider)
assert crawler.spider.meta.get("failure") is None
assert isinstance(crawler.spider.meta["response"], Response)
assert crawler.spider.meta["response"].body == crawler.spider.meta.get(
"bytes_received"
)
assert (
len(crawler.spider.meta["response"].body)
< crawler.spider.full_response_length
)
@coroutine_test
async def test_bytes_received_stop_download_errback(
self, mockserver: MockServer
) -> None:
# copy of TestCrawl.test_bytes_received_stop_download_errback()
crawler = get_crawler(BytesReceivedErrbackSpider, self.settings_dict)
await crawler.crawl_async(mockserver=mockserver, is_secure=self.is_secure)
assert isinstance(crawler.spider, BytesReceivedErrbackSpider)
assert crawler.spider.meta.get("response") is None
assert isinstance(crawler.spider.meta["failure"], Failure)
assert isinstance(crawler.spider.meta["failure"].value, StopDownload)
assert isinstance(crawler.spider.meta["failure"].value.response, Response)
assert crawler.spider.meta[
"failure"
].value.response.body == crawler.spider.meta.get("bytes_received")
assert (
len(crawler.spider.meta["failure"].value.response.body)
< crawler.spider.full_response_length
)
@coroutine_test
async def test_headers_received_stop_download_callback(
self, mockserver: MockServer
) -> None:
# copy of TestCrawl.test_headers_received_stop_download_callback()
crawler = get_crawler(HeadersReceivedCallbackSpider, self.settings_dict)
await crawler.crawl_async(mockserver=mockserver, is_secure=self.is_secure)
assert isinstance(crawler.spider, HeadersReceivedCallbackSpider)
assert crawler.spider.meta.get("failure") is None
assert isinstance(crawler.spider.meta["response"], Response)
assert crawler.spider.meta["response"].headers == crawler.spider.meta.get(
"headers_received"
)
@coroutine_test
async def test_headers_received_stop_download_errback(
self, mockserver: MockServer
) -> None:
# copy of TestCrawl.test_headers_received_stop_download_errback()
crawler = get_crawler(HeadersReceivedErrbackSpider, self.settings_dict)
await crawler.crawl_async(mockserver=mockserver, is_secure=self.is_secure)
assert isinstance(crawler.spider, HeadersReceivedErrbackSpider)
assert crawler.spider.meta.get("response") is None
assert isinstance(crawler.spider.meta["failure"], Failure)
assert isinstance(crawler.spider.meta["failure"].value, StopDownload)
assert isinstance(crawler.spider.meta["failure"].value.response, Response)
assert crawler.spider.meta[
"failure"
].value.response.headers == crawler.spider.meta.get("headers_received")
class TestHttpProxyBase(ABC):
is_secure = False

3
tox.ini
View File

@ -48,6 +48,7 @@ deps =
types-Pygments==2.19.0.20250809
botocore-stubs==1.40.59
boto3-stubs[s3]==1.40.59
httpx==0.28.1
itemadapter==0.12.2
Protego==0.5.0
w3lib==2.3.1
@ -134,6 +135,7 @@ deps =
brotli >= 1.2.0; implementation_name != "pypy" # optional for HTTP compress downloader middleware tests
brotlicffi >= 1.2.0.0; implementation_name == "pypy" # optional for HTTP compress downloader middleware tests
google-cloud-storage
httpx
ipython
robotexclusionrulesparser
uvloop; platform_system != "Windows" and implementation_name != "pypy"
@ -149,6 +151,7 @@ deps =
brotli==1.2.0; implementation_name != "pypy"
brotlicffi==1.2.0.0; implementation_name == "pypy"
google-cloud-storage==1.29.0
httpx==0.26.0
ipython==7.1.0
robotexclusionrulesparser==1.6.2
uvloop==0.16.0; platform_system != "Windows" and implementation_name != "pypy"