diff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rst index 1e95864c6..96cf46c35 100644 --- a/docs/topics/download-handlers.rst +++ b/docs/topics/download-handlers.rst @@ -206,8 +206,6 @@ If you want to use this handler you need to replace the default one for the Known limitations of the HTTP/2 implementation in this handler include: - - No support for proxies. - - No support for HTTP/2 Cleartext (h2c), since no major browser supports HTTP/2 unencrypted (refer `http2 faq`_). diff --git a/tests/test_downloader_handler_twisted_http11.py b/tests/test_downloader_handler_twisted_http11.py index ed742e55f..ad06f2a23 100644 --- a/tests/test_downloader_handler_twisted_http11.py +++ b/tests/test_downloader_handler_twisted_http11.py @@ -93,6 +93,10 @@ class TestHttpProxy(HTTP11DownloadHandlerMixin, TestHttpProxyBase): pass +class TestHttpsProxy(HTTP11DownloadHandlerMixin, TestHttpProxyBase): + is_secure = True + + @pytest.mark.requires_mitmproxy class TestMitmProxy(HTTP11DownloadHandlerMixin, TestMitmProxyBase): # not implemented diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.py index 60b276166..acecd2e43 100644 --- a/tests/test_downloader_handler_twisted_http2.py +++ b/tests/test_downloader_handler_twisted_http2.py @@ -10,8 +10,13 @@ from twisted.web.http import H2_ENABLED from scrapy import Spider from scrapy.crawler import Crawler -from scrapy.exceptions import DownloadFailedError, NotConfigured +from scrapy.exceptions import ( + DownloadFailedError, + NotConfigured, + UnsupportedURLSchemeError, +) from scrapy.http import Request +from scrapy.utils.defer import maybe_deferred_to_future from tests.test_downloader_handlers_http_base import ( TestHttpProxyBase, TestHttpsBase, @@ -27,6 +32,7 @@ from tests.utils.decorators import coroutine_test if TYPE_CHECKING: from scrapy.core.downloader.handlers import DownloadHandlerProtocol from tests.mockserver.http import MockServer + from tests.mockserver.proxy_echo import ProxyEchoMockServer pytestmark = [ @@ -187,12 +193,32 @@ class TestHttp2WithCrawler(H2DownloadHandlerMixin, TestHttpWithCrawlerBase): pytest.skip("headers_received support is not implemented") -@pytest.mark.skip(reason="Proxy support is not implemented yet") class TestHttp2Proxy(H2DownloadHandlerMixin, TestHttpProxyBase): is_secure = True + expected_http_proxy_request_body = b"/" + + @coroutine_test + async def test_download_with_proxy_https_timeout( + self, proxy_mockserver: ProxyEchoMockServer + ) -> None: + with pytest.raises(NotImplementedError): + await maybe_deferred_to_future( + super().test_download_with_proxy_https_timeout(proxy_mockserver) # type: ignore[arg-type] + ) + + @coroutine_test + async def test_download_with_proxy_without_http_scheme( + self, proxy_mockserver: ProxyEchoMockServer + ) -> None: + with pytest.raises(UnsupportedURLSchemeError): + await maybe_deferred_to_future( + super().test_download_with_proxy_without_http_scheme(proxy_mockserver) # type: ignore[arg-type] + ) @pytest.mark.skip(reason="Proxy support is not implemented yet") @pytest.mark.requires_mitmproxy class TestMitmProxy(H2DownloadHandlerMixin, TestMitmProxyBase): - pass + handler_supports_http = False + # not implemented + handler_supports_tls_in_tls = False diff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.py index 31239e1d8..88b1101b7 100644 --- a/tests/test_downloader_handlers_http_base.py +++ b/tests/test_downloader_handlers_http_base.py @@ -1152,6 +1152,8 @@ class TestHttpProxyBase(ABC): class TestMitmProxyBase(ABC): # whether the handler supports HTTPS proxies with HTTPS destinations handler_supports_tls_in_tls: bool = True + # whether the handler supports HTTP destinations + handler_supports_http: bool = True @property @abstractmethod @@ -1170,6 +1172,8 @@ class TestMitmProxyBase(ABC): https_dest: bool, ) -> None: """HTTP proxy, HTTP or HTTPS destination.""" + if not https_dest and not self.handler_supports_http: + pytest.skip("HTTP destinations are not supported") crawler = get_crawler(SingleRequestSpider, self.settings_dict) with caplog.at_level(logging.DEBUG): await crawler.crawl_async( @@ -1191,6 +1195,8 @@ class TestMitmProxyBase(ABC): https_dest: bool, ) -> None: """HTTPS proxy, HTTP or HTTPS destination.""" + if not https_dest and not self.handler_supports_http: + pytest.skip("HTTP destinations are not supported") if https_dest and not self.handler_supports_tls_in_tls: pytest.skip("HTTPS proxy to HTTPS destination is not supported") crawler = get_crawler(SingleRequestSpider, self.settings_dict) @@ -1215,6 +1221,8 @@ class TestMitmProxyBase(ABC): https_dest: bool, ) -> None: """HTTP proxy, HTTP or HTTPS destination, wrong proxy creds.""" + if not https_dest and not self.handler_supports_http: + pytest.skip("HTTP destinations are not supported") envvar = "https_proxy" if https_dest else "http_proxy" monkeypatch.setenv(envvar, wrong_credentials(os.environ[envvar])) crawler = get_crawler(SimpleSpider, self.settings_dict) @@ -1239,6 +1247,8 @@ class TestMitmProxyBase(ABC): ) -> None: """HTTP proxy, HTTP or HTTPS destination. Check that the auth header is not sent to the destination.""" + if not https_dest and not self.handler_supports_http: + pytest.skip("HTTP destinations are not supported") request = Request(mockserver.url("/echo", is_secure=https_dest)) crawler = get_crawler(SingleRequestSpider, self.settings_dict) with caplog.at_level(logging.DEBUG):