diff --git a/conftest.py b/conftest.py index 50925a311..26e0434ba 100644 --- a/conftest.py +++ b/conftest.py @@ -52,6 +52,11 @@ if not H2_ENABLED: ) ) +try: + import httpx # noqa: F401 +except ImportError: + collect_ignore.append("scrapy/core/downloader/handlers/_httpx.py") + def pytest_addoption(parser, pluginmanager): if pluginmanager.hasplugin("twisted"): diff --git a/docs/news.rst b/docs/news.rst index bab2e5ff9..ff8e6fb08 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,17 @@ Release notes ============= +Scrapy VERSION (unreleased) +--------------------------- + +New features +~~~~~~~~~~~~ + +- Added + :meth:`Headers.to_tuple_list() ` + that returns headers as a list of ``(key, value)`` tuples. + (:issue:`7239`) + .. _release-2.14.1: Scrapy 2.14.1 (2026-01-12) @@ -2997,7 +3008,7 @@ Bug fixes that does not match the asyncio event loop actually installed (:issue:`5529`). -- Fixed :meth:`Headers.getlist ` +- Fixed :meth:`Headers.getlist() ` returning only the last header (:issue:`5515`, :issue:`5526`). - Fixed :class:`LinkExtractor diff --git a/scrapy/core/downloader/handlers/_httpx.py b/scrapy/core/downloader/handlers/_httpx.py new file mode 100644 index 000000000..1f5cf007e --- /dev/null +++ b/scrapy/core/downloader/handlers/_httpx.py @@ -0,0 +1,269 @@ +"""``httpx``-based HTTP(S) download handler. Currently not recommended for production use.""" + +from __future__ import annotations + +import ipaddress +import logging +import ssl +from http.cookiejar import Cookie, CookieJar +from io import BytesIO +from typing import TYPE_CHECKING, Any, NoReturn, TypedDict + +import httpx + +from scrapy import Request, signals +from scrapy.exceptions import ( + CannotResolveHostError, + DownloadCancelledError, + DownloadConnectionRefusedError, + DownloadFailedError, + DownloadTimeoutError, + NotConfigured, + ResponseDataLossError, + UnsupportedURLSchemeError, +) +from scrapy.http import Headers, Response +from scrapy.utils._download_handlers import ( + BaseHttpDownloadHandler, + check_stop_download, + get_dataloss_msg, + get_maxsize_msg, + get_warnsize_msg, + make_response, +) +from scrapy.utils.asyncio import is_asyncio_available +from scrapy.utils.ssl import _log_sslobj_debug_info, _make_ssl_context + +if TYPE_CHECKING: + from contextlib import AbstractAsyncContextManager + from http.client import HTTPResponse + from ipaddress import IPv4Address, IPv6Address + from urllib.request import Request as ULRequest + + from httpcore import AsyncNetworkStream + + from scrapy.crawler import Crawler + + +logger = logging.getLogger(__name__) + + +class _BaseResponseArgs(TypedDict): + status: int + url: str + headers: Headers + ip_address: IPv4Address | IPv6Address + protocol: str + + +# workaround for (and from) https://github.com/encode/httpx/issues/2992 +class _NullCookieJar(CookieJar): # pragma: no cover + """A CookieJar that rejects all cookies.""" + + def extract_cookies(self, response: HTTPResponse, request: ULRequest) -> None: + pass + + def set_cookie(self, cookie: Cookie) -> None: + pass + + +class HttpxDownloadHandler(BaseHttpDownloadHandler): + _DEFAULT_CONNECT_TIMEOUT = 10 + + def __init__(self, crawler: Crawler): + # we don't run extra-deps tests with the non-asyncio reactor + if not is_asyncio_available(): # pragma: no cover + raise NotConfigured( + f"{type(self).__name__} requires the asyncio support. Make" + f" sure that you have either enabled the asyncio Twisted" + f" reactor in the TWISTED_REACTOR setting or disabled the" + f" TWISTED_ENABLED setting. See the asyncio documentation" + f" of Scrapy for more information." + ) + super().__init__(crawler) + logger.warning( + "HttpxDownloadHandler is experimental and is not recommented for production use." + ) + self._tls_verbose_logging: bool = self.crawler.settings.getbool( + "DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING" + ) + self._client = httpx.AsyncClient( + verify=_make_ssl_context(crawler.settings), cookies=_NullCookieJar() + ) + + async def download_request(self, request: Request) -> Response: + self._warn_unsupported_meta(request.meta) + + timeout: float = request.meta.get( + "download_timeout", self._DEFAULT_CONNECT_TIMEOUT + ) + + try: + async with self._get_httpx_response(request, timeout) as httpx_response: + return await self._read_response(httpx_response, request) + except httpx.TimeoutException as e: + raise DownloadTimeoutError( + f"Getting {request.url} took longer than {timeout} seconds." + ) from e + except httpx.UnsupportedProtocol as e: + raise UnsupportedURLSchemeError(str(e)) from e + except httpx.ConnectError as e: + if "Name or service not known" in str(e) or "getaddrinfo failed" in str(e): + raise CannotResolveHostError(str(e)) from e + raise DownloadConnectionRefusedError(str(e)) from e + except httpx.NetworkError as e: + raise DownloadFailedError(str(e)) from e + except httpx.RemoteProtocolError as e: + raise DownloadFailedError(str(e)) from e + + def _warn_unsupported_meta(self, meta: dict[str, Any]) -> None: + if meta.get("bindaddress"): + # configurable only per-client: + # https://github.com/encode/httpx/issues/755#issuecomment-2746121794 + logger.error( + f"The 'bindaddress' request meta key is not supported by" + f" {type(self).__name__} and will be ignored." + ) + if meta.get("proxy"): + # configurable only per-client: + # https://github.com/encode/httpx/issues/486 + logger.error( + f"The 'proxy' request meta key is not supported by" + f" {type(self).__name__} and will be ignored." + ) + + def _get_httpx_response( + self, request: Request, timeout: float + ) -> AbstractAsyncContextManager[httpx.Response]: + return self._client.stream( + request.method, + request.url, + content=request.body, + headers=request.headers.to_tuple_list(), + timeout=timeout, + ) + + async def _read_response( + self, httpx_response: httpx.Response, request: Request + ) -> Response: + maxsize: int = request.meta.get("download_maxsize", self._default_maxsize) + warnsize: int = request.meta.get("download_warnsize", self._default_warnsize) + + content_length = httpx_response.headers.get("Content-Length") + expected_size = int(content_length) if content_length is not None else None + if maxsize and expected_size and expected_size > maxsize: + self._cancel_maxsize(expected_size, maxsize, request, expected=True) + + reached_warnsize = False + if warnsize and expected_size and expected_size > warnsize: + reached_warnsize = True + logger.warning( + get_warnsize_msg(expected_size, warnsize, request, expected=True) + ) + + headers = Headers(httpx_response.headers.multi_items()) + network_stream: AsyncNetworkStream = httpx_response.extensions["network_stream"] + + make_response_base_args: _BaseResponseArgs = { + "status": httpx_response.status_code, + "url": request.url, + "headers": headers, + "ip_address": self._get_server_ip(network_stream), + "protocol": httpx_response.http_version, + } + + self._log_tls_info(network_stream) + + if stop_download := check_stop_download( + signals.headers_received, + self.crawler, + request, + headers=headers, + body_length=expected_size, + ): + return make_response( + **make_response_base_args, + stop_download=stop_download, + ) + + response_body = BytesIO() + bytes_received = 0 + try: + async for chunk in httpx_response.aiter_raw(): + response_body.write(chunk) + bytes_received += len(chunk) + + if stop_download := check_stop_download( + signals.bytes_received, self.crawler, request, data=chunk + ): + return make_response( + **make_response_base_args, + body=response_body.getvalue(), + stop_download=stop_download, + ) + + if maxsize and bytes_received > maxsize: + response_body.truncate(0) + self._cancel_maxsize( + bytes_received, maxsize, request, expected=False + ) + + if warnsize and bytes_received > warnsize and not reached_warnsize: + reached_warnsize = True + logger.warning( + get_warnsize_msg( + bytes_received, warnsize, request, expected=False + ) + ) + except httpx.RemoteProtocolError as e: + # special handling of the dataloss case + if ( + "peer closed connection without sending complete message body" + not in str(e) + ): + raise + fail_on_dataloss: bool = request.meta.get( + "download_fail_on_dataloss", self._fail_on_dataloss + ) + if not fail_on_dataloss: + return make_response( + **make_response_base_args, + body=response_body.getvalue(), + flags=["dataloss"], + ) + self._log_dataloss_warning(request.url) + raise ResponseDataLossError(str(e)) from e + + return make_response( + **make_response_base_args, + body=response_body.getvalue(), + ) + + @staticmethod + def _get_server_ip(network_stream: AsyncNetworkStream) -> IPv4Address | IPv6Address: + extra_server_addr = network_stream.get_extra_info("server_addr") + return ipaddress.ip_address(extra_server_addr[0]) + + def _log_tls_info(self, network_stream: AsyncNetworkStream) -> None: + if not self._tls_verbose_logging: + return + extra_ssl_object = network_stream.get_extra_info("ssl_object") + if isinstance(extra_ssl_object, ssl.SSLObject): + _log_sslobj_debug_info(extra_ssl_object) + + def _log_dataloss_warning(self, url: str) -> None: + if self._fail_on_dataloss_warned: + return + logger.warning(get_dataloss_msg(url)) + self._fail_on_dataloss_warned = True + + @staticmethod + def _cancel_maxsize( + size: int, limit: int, request: Request, *, expected: bool + ) -> NoReturn: + warning_msg = get_maxsize_msg(size, limit, request, expected=expected) + logger.warning(warning_msg) + raise DownloadCancelledError(warning_msg) + + async def close(self): + await self._client.aclose() diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index f3f60bffb..34d4ec6f2 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -124,6 +124,17 @@ class Headers(CaselessDict): for key, value in self.items() ) + def to_tuple_list(self) -> list[tuple[str, str]]: + """Return headers as a list of ``(key, value)`` tuples. + + Multiple values are represented as multiple tuples with the same key. + """ + return [ + (key.decode(self.encoding), value.decode(self.encoding)) + for key, values in self.items() + for value in values + ] + def __copy__(self) -> Self: return self.__class__(self) diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 6d3bb1db9..07a9e5ff7 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -72,6 +72,12 @@ DEFAULT_LOGGING = { "hpack": { "level": "ERROR", }, + "httpcore": { + "level": "ERROR", + }, + "httpx": { + "level": "WARNING", + }, "scrapy": { "level": "DEBUG", }, diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py index 7d46cbd4f..0062da52b 100644 --- a/scrapy/utils/ssl.py +++ b/scrapy/utils/ssl.py @@ -1,5 +1,7 @@ from __future__ import annotations +import logging +import ssl from typing import TYPE_CHECKING, Any import OpenSSL._util as pyOpenSSLutil @@ -11,6 +13,54 @@ from scrapy.utils.python import to_unicode if TYPE_CHECKING: from OpenSSL.crypto import X509Name + from scrapy.settings import BaseSettings + +logger = logging.getLogger(__name__) + + +# stdlib ssl module utils + +# possible documented values for DOWNLOADER_CLIENT_TLS_METHOD +_STDLIB_PROTOCOL_MAP = { + "TLS": ssl.PROTOCOL_TLS_CLIENT, + "TLSv1.0": ssl.PROTOCOL_TLSv1, + "TLSv1.1": ssl.PROTOCOL_TLSv1_1, + "TLSv1.2": ssl.PROTOCOL_TLSv1_2, +} + + +def _make_ssl_context(settings: BaseSettings) -> ssl.SSLContext: + """Create an :class:`ssl.SSLContext` instance according to the settings. + + It's intended to be used in an HTTPS download handler. + """ + + method_setting: str = settings["DOWNLOADER_CLIENT_TLS_METHOD"] + if method_setting not in _STDLIB_PROTOCOL_MAP: + raise ValueError(f"Unsupported TLS method: {method_setting}") + ciphers_setting: str | None = settings["DOWNLOADER_CLIENT_TLS_CIPHERS"] + + ctx = ssl.SSLContext(_STDLIB_PROTOCOL_MAP[method_setting]) + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + if ciphers_setting: + ctx.set_ciphers(ciphers_setting) + return ctx + + +def _log_sslobj_debug_info(sslobj: ssl.SSLObject) -> None: + cipher = sslobj.cipher() + logger.debug( + f"SSL connection to {sslobj.server_hostname}" + f" using protocol {sslobj.version()}," + f" cipher {cipher[0] if cipher else None}" + ) + # The peer certificate is unavailable on SSLObject unless peer + # certificate verification is enabled, which we don't want. + + +# pyOpenSSL utils + def ffi_buf_to_string(buf: Any) -> str: return to_unicode(pyOpenSSLutil.ffi.string(buf)) @@ -22,7 +72,6 @@ def x509name_to_string(x509name: X509Name) -> str: pyOpenSSLutil.lib.X509_NAME_oneline( x509name._name, result_buffer, len(result_buffer) ) - return ffi_buf_to_string(result_buffer) diff --git a/tests/spiders.py b/tests/spiders.py index 43cdcaef3..de2e64242 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -22,9 +22,10 @@ from scrapy.utils.test import get_from_asyncio_queue, get_web_client_agent_req class MockServerSpider(Spider): - def __init__(self, mockserver=None, *args, **kwargs): + def __init__(self, *args, mockserver=None, is_secure=False, **kwargs): super().__init__(*args, **kwargs) self.mockserver = mockserver + self.is_secure = is_secure class MetaSpider(MockServerSpider): @@ -516,7 +517,7 @@ class BytesReceivedCallbackSpider(MetaSpider): async def start(self): body = b"a" * self.full_response_length - url = self.mockserver.url("/alpayload") + url = self.mockserver.url("/alpayload", is_secure=self.is_secure) yield Request(url, method="POST", body=body, errback=self.errback) def parse(self, response): @@ -544,7 +545,10 @@ class HeadersReceivedCallbackSpider(MetaSpider): return spider async def start(self): - yield Request(self.mockserver.url("/status"), errback=self.errback) + yield Request( + self.mockserver.url("/status", is_secure=self.is_secure), + errback=self.errback, + ) def parse(self, response): self.meta["response"] = response diff --git a/tests/test_downloader_handler_httpx.py b/tests/test_downloader_handler_httpx.py new file mode 100644 index 000000000..ad6c6d3c7 --- /dev/null +++ b/tests/test_downloader_handler_httpx.py @@ -0,0 +1,127 @@ +"""Tests for scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +import pytest + +from scrapy import Request +from tests.test_downloader_handlers_http_base import ( + TestHttp11Base, + TestHttpProxyBase, + TestHttps11Base, + TestHttpsCustomCiphersBase, + TestHttpsInvalidDNSIdBase, + TestHttpsInvalidDNSPatternBase, + TestHttpsWrongHostnameBase, + TestHttpWithCrawlerBase, + TestSimpleHttpsBase, +) +from tests.utils.decorators import coroutine_test + +if TYPE_CHECKING: + from scrapy.core.downloader.handlers import DownloadHandlerProtocol + from tests.mockserver.http import MockServer + + +pytest.importorskip("httpx") + + +class HttpxDownloadHandlerMixin: + @property + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + # the import will fail if httpx is not installed + from scrapy.core.downloader.handlers._httpx import ( # noqa: PLC0415 + HttpxDownloadHandler, + ) + + return HttpxDownloadHandler + + +class TestHttp11(HttpxDownloadHandlerMixin, TestHttp11Base): + @coroutine_test + async def test_unsupported_bindaddress( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + meta = {"bindaddress": "127.0.0.2"} + request = Request(mockserver.url("/text"), meta=meta) + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + assert response.body == b"Works" + assert ( + "The 'bindaddress' request meta key is not supported by HttpxDownloadHandler" + in caplog.text + ) + + @coroutine_test + async def test_unsupported_proxy( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + meta = {"proxy": "127.0.0.2"} + request = Request(mockserver.url("/text"), meta=meta) + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + assert response.body == b"Works" + assert ( + "The 'proxy' request meta key is not supported by HttpxDownloadHandler" + in caplog.text + ) + + +class TestHttps11(HttpxDownloadHandlerMixin, TestHttps11Base): + tls_log_message = "SSL connection to 127.0.0.1 using protocol TLSv1.3, cipher" + + +class TestSimpleHttps(HttpxDownloadHandlerMixin, TestSimpleHttpsBase): + pass + + +class Https11WrongHostnameTestCase( + HttpxDownloadHandlerMixin, TestHttpsWrongHostnameBase +): + pass + + +class Https11InvalidDNSId(HttpxDownloadHandlerMixin, TestHttpsInvalidDNSIdBase): + pass + + +class Https11InvalidDNSPattern( + HttpxDownloadHandlerMixin, TestHttpsInvalidDNSPatternBase +): + pass + + +class Https11CustomCiphers(HttpxDownloadHandlerMixin, TestHttpsCustomCiphersBase): + pass + + +class TestHttp11WithCrawler(TestHttpWithCrawlerBase): + @property + def settings_dict(self) -> dict[str, Any] | None: + return { + "DOWNLOAD_HANDLERS": { + "http": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler", + "https": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler", + } + } + + +class TestHttps11WithCrawler(TestHttp11WithCrawler): + is_secure = True + + @pytest.mark.skip(reason="response.certificate is not implemented") + @coroutine_test + async def test_response_ssl_certificate(self, mockserver: MockServer) -> None: + pass + + +@pytest.mark.skip(reason="Proxy support is not implemented yet") +class TestHttp11Proxy(HttpxDownloadHandlerMixin, TestHttpProxyBase): + pass + + +@pytest.mark.skip(reason="Proxy support is not implemented yet") +class TestHttps11Proxy(HttpxDownloadHandlerMixin, TestHttpProxyBase): + is_secure = True diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.py index df19646ff..1b9dc21e7 100644 --- a/tests/test_downloader_handler_twisted_http2.py +++ b/tests/test_downloader_handler_twisted_http2.py @@ -189,6 +189,18 @@ class TestHttp2WithCrawler(TestHttpWithCrawlerBase): is_secure = True + def test_bytes_received_stop_download_callback(self) -> None: # type: ignore[override] + pytest.skip("bytes_received support is not implemented") + + def test_bytes_received_stop_download_errback(self) -> None: # type: ignore[override] + pytest.skip("bytes_received support is not implemented") + + def test_headers_received_stop_download_callback(self) -> None: # type: ignore[override] + pytest.skip("headers_received support is not implemented") + + def test_headers_received_stop_download_errback(self) -> None: # type: ignore[override] + pytest.skip("headers_received support is not implemented") + class TestHttps2Proxy(H2DownloadHandlerMixin, TestHttpProxyBase): is_secure = True diff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.py index bc7807b4d..8e9fa0ae1 100644 --- a/tests/test_downloader_handlers_http_base.py +++ b/tests/test_downloader_handlers_http_base.py @@ -16,6 +16,7 @@ from urllib.parse import urlparse import pytest from twisted.internet.ssl import Certificate +from twisted.python.failure import Failure from scrapy.exceptions import ( CannotResolveHostError, @@ -24,6 +25,7 @@ from scrapy.exceptions import ( DownloadFailedError, DownloadTimeoutError, ResponseDataLossError, + StopDownload, UnsupportedURLSchemeError, ) from scrapy.http import Headers, HtmlResponse, Request, Response, TextResponse @@ -34,7 +36,13 @@ from scrapy.utils.test import get_crawler from tests import NON_EXISTING_RESOLVABLE from tests.mockserver.proxy_echo import ProxyEchoMockServer from tests.mockserver.simple_https import SimpleMockServer -from tests.spiders import SingleRequestSpider +from tests.spiders import ( + BytesReceivedCallbackSpider, + BytesReceivedErrbackSpider, + HeadersReceivedCallbackSpider, + HeadersReceivedErrbackSpider, + SingleRequestSpider, +) from tests.utils.decorators import coroutine_test if TYPE_CHECKING: @@ -832,6 +840,74 @@ class TestHttpWithCrawlerBase(ABC): assert isinstance(ip_address, IPv4Address) assert str(ip_address) == gethostbyname(expected_netloc) + @coroutine_test + async def test_bytes_received_stop_download_callback( + self, mockserver: MockServer + ) -> None: + # copy of TestCrawl.test_bytes_received_stop_download_callback() + crawler = get_crawler(BytesReceivedCallbackSpider, self.settings_dict) + await crawler.crawl_async(mockserver=mockserver, is_secure=self.is_secure) + assert isinstance(crawler.spider, BytesReceivedCallbackSpider) + assert crawler.spider.meta.get("failure") is None + assert isinstance(crawler.spider.meta["response"], Response) + assert crawler.spider.meta["response"].body == crawler.spider.meta.get( + "bytes_received" + ) + assert ( + len(crawler.spider.meta["response"].body) + < crawler.spider.full_response_length + ) + + @coroutine_test + async def test_bytes_received_stop_download_errback( + self, mockserver: MockServer + ) -> None: + # copy of TestCrawl.test_bytes_received_stop_download_errback() + crawler = get_crawler(BytesReceivedErrbackSpider, self.settings_dict) + await crawler.crawl_async(mockserver=mockserver, is_secure=self.is_secure) + assert isinstance(crawler.spider, BytesReceivedErrbackSpider) + assert crawler.spider.meta.get("response") is None + assert isinstance(crawler.spider.meta["failure"], Failure) + assert isinstance(crawler.spider.meta["failure"].value, StopDownload) + assert isinstance(crawler.spider.meta["failure"].value.response, Response) + assert crawler.spider.meta[ + "failure" + ].value.response.body == crawler.spider.meta.get("bytes_received") + assert ( + len(crawler.spider.meta["failure"].value.response.body) + < crawler.spider.full_response_length + ) + + @coroutine_test + async def test_headers_received_stop_download_callback( + self, mockserver: MockServer + ) -> None: + # copy of TestCrawl.test_headers_received_stop_download_callback() + crawler = get_crawler(HeadersReceivedCallbackSpider, self.settings_dict) + await crawler.crawl_async(mockserver=mockserver, is_secure=self.is_secure) + assert isinstance(crawler.spider, HeadersReceivedCallbackSpider) + assert crawler.spider.meta.get("failure") is None + assert isinstance(crawler.spider.meta["response"], Response) + assert crawler.spider.meta["response"].headers == crawler.spider.meta.get( + "headers_received" + ) + + @coroutine_test + async def test_headers_received_stop_download_errback( + self, mockserver: MockServer + ) -> None: + # copy of TestCrawl.test_headers_received_stop_download_errback() + crawler = get_crawler(HeadersReceivedErrbackSpider, self.settings_dict) + await crawler.crawl_async(mockserver=mockserver, is_secure=self.is_secure) + assert isinstance(crawler.spider, HeadersReceivedErrbackSpider) + assert crawler.spider.meta.get("response") is None + assert isinstance(crawler.spider.meta["failure"], Failure) + assert isinstance(crawler.spider.meta["failure"].value, StopDownload) + assert isinstance(crawler.spider.meta["failure"].value.response, Response) + assert crawler.spider.meta[ + "failure" + ].value.response.headers == crawler.spider.meta.get("headers_received") + class TestHttpProxyBase(ABC): is_secure = False diff --git a/tox.ini b/tox.ini index 83cd06561..3be564502 100644 --- a/tox.ini +++ b/tox.ini @@ -48,6 +48,7 @@ deps = types-Pygments==2.19.0.20250809 botocore-stubs==1.40.59 boto3-stubs[s3]==1.40.59 + httpx==0.28.1 itemadapter==0.12.2 Protego==0.5.0 w3lib==2.3.1 @@ -134,6 +135,7 @@ deps = brotli >= 1.2.0; implementation_name != "pypy" # optional for HTTP compress downloader middleware tests brotlicffi >= 1.2.0.0; implementation_name == "pypy" # optional for HTTP compress downloader middleware tests google-cloud-storage + httpx ipython robotexclusionrulesparser uvloop; platform_system != "Windows" and implementation_name != "pypy" @@ -149,6 +151,7 @@ deps = brotli==1.2.0; implementation_name != "pypy" brotlicffi==1.2.0.0; implementation_name == "pypy" google-cloud-storage==1.29.0 + httpx==0.26.0 ipython==7.1.0 robotexclusionrulesparser==1.6.2 uvloop==0.16.0; platform_system != "Windows" and implementation_name != "pypy"