diff --git a/conftest.py b/conftest.py index fcbf59426..1674086ec 100644 --- a/conftest.py +++ b/conftest.py @@ -99,6 +99,19 @@ def mitm_proxy_server_https(monkeypatch: pytest.MonkeyPatch) -> Generator[MitmPr proxy.stop() +@pytest.fixture # function scope because it modifies os.environ +def socks5_proxy_server(monkeypatch: pytest.MonkeyPatch) -> Generator[MitmProxy]: + proxy = MitmProxy(mode="socks5") + url = proxy.start() + monkeypatch.setenv("http_proxy", url) + monkeypatch.setenv("https_proxy", url) + + try: + yield proxy + finally: + proxy.stop() + + @pytest.fixture(scope="session") def reactor_pytest(request) -> str: return request.config.getoption("--reactor") diff --git a/docs/faq.rst b/docs/faq.rst index 87adbfa4b..df90122f5 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -82,10 +82,18 @@ to steal from us! Does Scrapy work with HTTP proxies? ----------------------------------- -Yes. Support for HTTP proxies is provided (since Scrapy 0.8) through the HTTP -Proxy downloader middleware. See +Yes. Support for HTTP proxies is provided through the HTTP Proxy downloader +middleware. See :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware`. +Does Scrapy work with SOCKS proxies? +------------------------------------ + +Yes, when using +:class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`. See +:class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` and the +handler documentation. + How can I scrape an item with attributes in different pages? ------------------------------------------------------------ @@ -360,7 +368,10 @@ method for this purpose. For example: Does Scrapy support IPv6 addresses? ----------------------------------- -Yes, by setting :setting:`TWISTED_DNS_RESOLVER` to ``scrapy.resolver.CachingHostnameResolver``. +Yes, but when using +:class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` or +:class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` you need to +set :setting:`TWISTED_DNS_RESOLVER` to ``scrapy.resolver.CachingHostnameResolver``. Note that by doing so, you lose the ability to set a specific timeout for DNS requests (the value of the :setting:`DNS_TIMEOUT` setting is ignored). diff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rst index 84711e5af..888bfaf08 100644 --- a/docs/topics/download-handlers.rst +++ b/docs/topics/download-handlers.rst @@ -130,44 +130,39 @@ these exceptions. .. _download-handlers-ref: -Built-in download handlers reference -==================================== +Built-in HTTP download handlers reference +========================================= -DataURIDownloadHandler ----------------------- +Scrapy ships several handlers for HTTP and HTTPS requests. While all of them +support basic features, they may differ in support of specific Scrapy features +and settings and HTTP protocol features. See the documentation of specific +handlers and specific settings for more information. Additionally, as the +underlying HTTP client implementations differ between handlers, the behavior of +specific websites may be different when doing the same Scrapy requests but +using different handlers. -.. autoclass:: scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler +Here is a comparison of some features of the built-in HTTP handlers, see the +individual handler docs for more differences: -| Supported scheme: ``data``. -| Lazy: no. +================== ================= ===================== ==================== +Feature H2DownloadHandler HTTP11DownloadHandler HttpxDownloadHandler +================== ================= ===================== ==================== +Requires asyncio No No Yes +Requires a reactor Yes Yes No +HTTP/1.1 No Yes Yes +HTTP/2 Yes No Yes +TLS implementation ``cryptography`` ``cryptography`` Stdlib ``ssl`` +HTTP proxies No Yes Yes +SOCKS proxies No No Yes +================== ================= ===================== ==================== -This handler supports RFC 2397 ``data:content/type;base64,`` data URIs. +You can find additional HTTP download handlers in the +scrapy-download-handlers-incubator_ package. This package is made by the Scrapy +developers and contains experimental handlers that may be included in some +later Scrapy version but can already be used. Please refer to the documentation +of this package for more information. -FileDownloadHandler -------------------- - -.. autoclass:: scrapy.core.downloader.handlers.file.FileDownloadHandler - -| Supported scheme: ``file``. -| Lazy: no. - -This handler supports ``file:///path`` local file URIs. It doesn't -support remote files. - -FTPDownloadHandler ------------------- - -.. autoclass:: scrapy.core.downloader.handlers.ftp.FTPDownloadHandler - -| Supported scheme: ``ftp``. -| Lazy: no. - -This handler supports ``ftp://host/path`` FTP URIs. - -It's implemented using :mod:`twisted.protocols.ftp`. - -.. note:: - This handler is not supported when :setting:`TWISTED_REACTOR_ENABLED` is ``False``. +.. _scrapy-download-handlers-incubator: https://github.com/scrapy-plugins/scrapy-download-handlers-incubator .. _twisted-http2-handler: @@ -177,7 +172,9 @@ H2DownloadHandler .. autoclass:: scrapy.core.downloader.handlers.http2.H2DownloadHandler | Supported scheme: ``https``. -| Lazy: yes. +| :ref:`Lazy `: yes. +| :ref:`Requires asyncio support `: no. +| :ref:`Requires a Twisted reactor `: yes. This handler supports ``https://host/path`` URLs and uses the HTTP/2 protocol for them. @@ -196,63 +193,96 @@ If you want to use this handler you need to replace the default one for the "https": "scrapy.core.downloader.handlers.http2.H2DownloadHandler", } +Features and limitations +^^^^^^^^^^^^^^^^^^^^^^^^ + .. warning:: This handler is experimental, and not yet recommended for production environments. Future Scrapy versions may introduce related changes without a deprecation period or warning. -.. note:: +=========================== ================================================ +HTTP proxies No (not implemented) +SOCKS proxies No (not supported by the library) +HTTP/2 Yes +``response.certificate`` :class:`twisted.internet.ssl.Certificate` object +Per-request ``bindaddress`` Yes +TLS implementation ``pyOpenSSL``/``cryptography`` +=========================== ================================================ - Known limitations of the HTTP/2 implementation in this handler include: +Other limitations: - - No support for proxies. +- No support for HTTP/1.1. - - No support for HTTP/2 Cleartext (h2c), since no major browser supports - HTTP/2 unencrypted (refer `http2 faq`_). +- IPv6 support requires setting :setting:`TWISTED_DNS_RESOLVER` + to ``scrapy.resolver.CachingHostnameResolver``. - - No setting to specify a maximum `frame size`_ larger than the default - value, 16384. Connections to servers that send a larger frame will - fail. +- No support for the :signal:`bytes_received` and :signal:`headers_received` + signals. - - No support for `server pushes`_, which are ignored. +Known limitations of the HTTP/2 support: - - No support for the :signal:`bytes_received` and - :signal:`headers_received` signals. +- No support for HTTP/2 Cleartext (h2c), since no major browser supports + HTTP/2 unencrypted (refer `http2 faq`_). + +- No setting to specify a maximum `frame size`_ larger than the default + value, 16384. Connections to servers that send a larger frame will fail. + +- No support for `server pushes`_, which are ignored. .. _frame size: https://datatracker.ietf.org/doc/html/rfc7540#section-4.2 .. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption .. _server pushes: https://datatracker.ietf.org/doc/html/rfc7540#section-8.2 -.. note:: - This handler is not supported when :setting:`TWISTED_REACTOR_ENABLED` is ``False``. - HTTP11DownloadHandler --------------------- .. autoclass:: scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler | Supported schemes: ``http``, ``https``. -| Lazy: no. +| :ref:`Lazy `: no. +| :ref:`Requires asyncio support `: no. +| :ref:`Requires a Twisted reactor `: yes. This handler supports ``http://host/path`` and ``https://host/path`` URLs and uses the HTTP/1.1 protocol for them. It's implemented using :mod:`twisted.web.client`. -.. note:: - This handler is not supported when :setting:`TWISTED_REACTOR_ENABLED` is ``False``. +Features and limitations +^^^^^^^^^^^^^^^^^^^^^^^^ + +=========================== ================================================ +HTTP proxies Yes +SOCKS proxies No (not supported by the library) +HTTP/2 No (implemented as a separate handler) +``response.certificate`` :class:`twisted.internet.ssl.Certificate` object +Per-request ``bindaddress`` Yes +TLS implementation ``pyOpenSSL``/``cryptography`` +=========================== ================================================ + +Other limitations: + +- IPv6 support requires setting :setting:`TWISTED_DNS_RESOLVER` + to ``scrapy.resolver.CachingHostnameResolver``. + +- HTTPS proxies to HTTPS destinations are not supported. HttpxDownloadHandler -------------------- +.. versionadded:: 2.15.0 + .. autoclass:: scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler | Supported schemes: ``http``, ``https``. -| Lazy: no. +| :ref:`Lazy `: no. +| :ref:`Requires asyncio support `: yes. +| :ref:`Requires a Twisted reactor `: no. This handler supports ``http://host/path`` and ``https://host/path`` URLs and -uses the HTTP/1.1 protocol for them. +uses the HTTP/1.1 or HTTP/2 protocol for them. It's implemented using the ``httpx`` library and needs it to be installed. @@ -266,28 +296,83 @@ If you want to use this handler you need to replace the default ones for the "https": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler", } +Features and limitations +^^^^^^^^^^^^^^^^^^^^^^^^ + .. warning:: This handler is experimental, and not yet recommended for production environments. Future Scrapy versions may introduce related changes without a deprecation period or warning or even remove it altogether. -.. note:: +=========================== ======================================= +HTTP proxies Yes +SOCKS proxies Yes (SOCKS5; requires ``httpx[socks]``) +HTTP/2 Yes (requires ``httpx[http2]``) +``response.certificate`` DER bytes +Per-request ``bindaddress`` No (not supported by the library) +TLS implementation Standard library ``ssl`` +=========================== ======================================= - As this handler is based on a different HTTP client implementation compared - to :class:`~.HTTP11DownloadHandler`, it's expected that its behavior on - some websites may be different. Additionally, these are the Scrapy features - that are explicitly not supported when using it: +Other limitations: - - Per-request bind address support (the :reqmeta:`bindaddress` meta key). - The global :setting:`DOWNLOAD_BIND_ADDRESS` setting is supported but the - port number, if specified, will be ignored. +- The handler creates a separate connection pool for each proxy URL (due to + limitations of ``httpx``) which may lead to higher resource usage when + using proxy rotation. - - Settings specific to the Twisted networking or HTTP implementation, like - :setting:`DNS_RESOLVER`. +.. setting:: HTTPX_HTTP2_ENABLED - - Using :ref:`non-asyncio reactors ` (``httpx`` requires - ``asyncio``). +HTTPX_HTTP2_ENABLED +^^^^^^^^^^^^^^^^^^^ + +Default: ``False`` + +Whether to enable HTTP/2 support in this handler. The ``httpx[http2]`` extra +needs to be installed if you want to enable this setting. + +.. versionadded:: VERSION + +Built-in non-HTTP download handlers reference +============================================= + +DataURIDownloadHandler +---------------------- + +.. autoclass:: scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler + +| Supported scheme: ``data``. +| :ref:`Lazy `: no. +| :ref:`Requires asyncio support `: no. +| :ref:`Requires a Twisted reactor `: no. + +This handler supports RFC 2397 ``data:content/type;base64,`` data URIs. + +FileDownloadHandler +------------------- + +.. autoclass:: scrapy.core.downloader.handlers.file.FileDownloadHandler + +| Supported scheme: ``file``. +| :ref:`Lazy `: no. +| :ref:`Requires asyncio support `: no. +| :ref:`Requires a Twisted reactor `: no. + +This handler supports ``file:///path`` local file URIs. It doesn't +support remote files. + +FTPDownloadHandler +------------------ + +.. autoclass:: scrapy.core.downloader.handlers.ftp.FTPDownloadHandler + +| Supported scheme: ``ftp``. +| :ref:`Lazy `: no. +| :ref:`Requires asyncio support `: no. +| :ref:`Requires a Twisted reactor `: yes. + +This handler supports ``ftp://host/path`` FTP URIs. + +It's implemented using :mod:`twisted.protocols.ftp`. S3DownloadHandler ----------------- @@ -295,7 +380,9 @@ S3DownloadHandler .. autoclass:: scrapy.core.downloader.handlers.s3.S3DownloadHandler | Supported scheme: ``s3``. -| Lazy: yes. +| :ref:`Lazy `: yes. +| :ref:`Requires asyncio support `: no. +| :ref:`Requires a Twisted reactor `: no. This handler supports ``s3://bucket/path`` S3 URIs. diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 91ee38b85..8cb29deff 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -745,8 +745,7 @@ HttpProxyMiddleware Handling of this meta key needs to be implemented inside the :ref:`download handler `, so it's not guaranteed to be supported by all 3rd-party handlers. It's currently unsupported by - :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` and - :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`. + :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler`. .. note:: @@ -758,6 +757,13 @@ HttpProxyMiddleware :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` supports HTTPS proxies only for HTTP destinations. +.. note:: + + If the download handler supports it, you can use a SOCKS proxy URL (e.g. + ``socks5://username:password@some_proxy_server:port``). + :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler` + supports SOCKS proxies while other built-in handlers don't. + HttpProxyMiddleware settings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/scrapy/core/downloader/handlers/_httpx.py b/scrapy/core/downloader/handlers/_httpx.py index 43e1cd965..c960fc152 100644 --- a/scrapy/core/downloader/handlers/_httpx.py +++ b/scrapy/core/downloader/handlers/_httpx.py @@ -5,6 +5,7 @@ from __future__ import annotations import ipaddress import ssl from contextlib import asynccontextmanager +from socket import gaierror from typing import TYPE_CHECKING, ClassVar from scrapy.exceptions import ( @@ -17,6 +18,7 @@ from scrapy.exceptions import ( ) from scrapy.http import Headers from scrapy.utils._download_handlers import NullCookieJar +from scrapy.utils.python import _iter_exc_causes from scrapy.utils.ssl import ( _log_sslobj_debug_info, _make_insecure_ssl_ctx, @@ -34,10 +36,35 @@ if TYPE_CHECKING: from scrapy.crawler import Crawler +HAS_SOCKS = HAS_HTTP2 = False + try: import httpx except ImportError: httpx = None # type: ignore[assignment] +else: + # a small hack to avoid importing these optional extras unconditionally + + DOWNLOAD_FAILED_EXCEPTIONS: tuple[type[BaseException], ...] = ( + httpx.RequestError, + httpx.InvalidURL, + ) + + try: + import h2.exceptions + + HAS_HTTP2 = True + DOWNLOAD_FAILED_EXCEPTIONS += (h2.exceptions.InvalidBodyLengthError,) + except ImportError: # pragma: no cover + pass + + try: + import socksio.exceptions + + HAS_SOCKS = True + DOWNLOAD_FAILED_EXCEPTIONS += (socksio.exceptions.ProtocolError,) + except ImportError: # pragma: no cover + pass if TYPE_CHECKING: @@ -54,6 +81,11 @@ class HttpxDownloadHandler(_Base): self._verify_certificates: bool = crawler.settings.getbool( "DOWNLOAD_VERIFY_CERTIFICATES" ) + self._enable_h2: bool = crawler.settings.getbool("HTTPX_HTTP2_ENABLED") + if self._enable_h2 and not HAS_HTTP2: # pragma: no cover + raise NotConfigured( + f"HTTP/2 support in {type(self).__name__} requires the 'httpx[http2]' extra to be installed." + ) self._ssl_context: ssl.SSLContext = _make_ssl_context(crawler.settings) self._bind_host: str | None = self._get_bind_address_host() self._limits: httpx.Limits = httpx.Limits( @@ -90,6 +122,7 @@ class HttpxDownloadHandler(_Base): transport=httpx.AsyncHTTPTransport( verify=self._ssl_context, local_address=self._bind_host, + http2=self._enable_h2, limits=self._limits, trust_env=False, proxy=proxy, @@ -113,7 +146,13 @@ class HttpxDownloadHandler(_Base): async def _make_request( self, request: Request, timeout: float ) -> AsyncIterator[httpx.Response]: - client = self._get_client(self._extract_proxy_url_with_creds(request)) + proxy = self._extract_proxy_url_with_creds(request) + if proxy and proxy.startswith("socks") and not HAS_SOCKS: # pragma: no cover + raise ValueError( + f"SOCKS proxy support in {type(self).__name__} requires the 'httpx[socks]' extra to be installed." + ) + client = self._get_client(proxy) + try: async with client.stream( request.method, @@ -130,18 +169,12 @@ class HttpxDownloadHandler(_Base): except httpx.UnsupportedProtocol as e: raise UnsupportedURLSchemeError(str(e)) from e except httpx.ConnectError as e: - error_message = str(e) - if ( - "Name or service not known" in error_message - or "getaddrinfo failed" in error_message - or "nodename nor servname" in error_message - or "Temporary failure in name resolution" in error_message - ): - raise CannotResolveHostError(error_message) from e + if any(isinstance(c, gaierror) for c in _iter_exc_causes(e)): + raise CannotResolveHostError(str(e)) from e raise DownloadConnectionRefusedError(str(e)) from e except httpx.ProxyError as e: raise DownloadConnectionRefusedError(str(e)) from e - except (httpx.NetworkError, httpx.RemoteProtocolError) as e: + except DOWNLOAD_FAILED_EXCEPTIONS as e: raise DownloadFailedError(str(e)) from e @staticmethod diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 909a5fd5c..7d5612026 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -109,6 +109,7 @@ __all__ = [ "HTTPCACHE_STORAGE", "HTTPPROXY_AUTH_ENCODING", "HTTPPROXY_ENABLED", + "HTTPX_HTTP2_ENABLED", "IMAGES_STORE_GCS_ACL", "IMAGES_STORE_S3_ACL", "ITEM_PIPELINES", @@ -382,6 +383,8 @@ HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage" HTTPPROXY_ENABLED = True HTTPPROXY_AUTH_ENCODING = "latin-1" +HTTPX_HTTP2_ENABLED = False + IMAGES_STORE_GCS_ACL = "" IMAGES_STORE_S3_ACL = "private" diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 2d40e8555..00cf818b7 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -359,3 +359,13 @@ def _looks_like_import_path(value: str) -> bool: if any(part == "" for part in parts): return False return all(part.isidentifier() for part in parts) + + +def _iter_exc_causes(exc: BaseException) -> Iterable[BaseException]: + """Iterate over the exception causes/contexts.""" + seen: set[int] = set() + cur: BaseException | None = exc + while cur is not None and id(cur) not in seen: + seen.add(id(cur)) + yield cur + cur = cur.__cause__ or cur.__context__ diff --git a/tests/mockserver/mitm_proxy.py b/tests/mockserver/mitm_proxy.py index 1ec5a79b5..56620f84e 100644 --- a/tests/mockserver/mitm_proxy.py +++ b/tests/mockserver/mitm_proxy.py @@ -11,6 +11,9 @@ class MitmProxy: auth_user = "scrapy" auth_pass = "scrapy" + def __init__(self, mode: str | None = None) -> None: + self.mode = mode + def start(self) -> str: script = """ import sys @@ -32,6 +35,8 @@ sys.exit(mitmdump()) "-s", str(Path(__file__).with_name("mitm_proxy_addon.py")), ] + if self.mode: + args += ["--mode", self.mode] self.proc: Popen[str] = Popen( [ sys.executable, @@ -44,12 +49,13 @@ sys.exit(mitmdump()) text=True, ) assert self.proc.stdout is not None + scheme = "socks5" if self.mode == "socks5" else "http" line = "" for line in self.proc.stdout: - m = re.search(r"listening at (?:http://)?([^:]+:\d+)", line) + m = re.search(r"listening at (?:\w+://)?([^:]+:\d+)", line) if m: host_port = m.group(1) - return f"http://{self.auth_user}:{self.auth_pass}@{host_port}" + return f"{scheme}://{self.auth_user}:{self.auth_pass}@{host_port}" self.stop() raise RuntimeError(f"Failed to parse mitmdump output: {line}") diff --git a/tests/test_downloader_handler_httpx.py b/tests/test_downloader_handler_httpx.py index e7a2ac0ba..fdacad963 100644 --- a/tests/test_downloader_handler_httpx.py +++ b/tests/test_downloader_handler_httpx.py @@ -3,11 +3,17 @@ from __future__ import annotations import sys -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, ClassVar import pytest from scrapy import Request +from scrapy.core.downloader.handlers._httpx import ( + HAS_HTTP2, + HAS_SOCKS, + HttpxDownloadHandler, +) +from scrapy.exceptions import DownloadFailedError from tests.test_downloader_handlers_http_base import ( TestHttpBase, TestHttpProxyBase, @@ -37,10 +43,6 @@ pytest.importorskip("httpx") class HttpxDownloadHandlerMixin: @property def download_handler_cls(self) -> type[DownloadHandlerProtocol]: - from scrapy.core.downloader.handlers._httpx import ( # noqa: PLC0415 - HttpxDownloadHandler, - ) - return HttpxDownloadHandler @property @@ -83,6 +85,30 @@ class TestHttps(HttpxDownloadHandlerMixin, TestHttpsBase): pass +@pytest.mark.skipif(not HAS_HTTP2, reason="No HTTP/2 support in HttpxDownloadHandler") +class TestHttp2(TestHttps): + http2 = True + handler_supports_http2_dataloss = False + + default_handler_settings: ClassVar[dict[str, Any]] = { + "HTTPX_HTTP2_ENABLED": True, + } + + @coroutine_test + async def test_protocol(self, mockserver: MockServer) -> None: + request = Request(mockserver.url("/host", is_secure=self.is_secure)) + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + assert response.protocol == "HTTP/2" + + @coroutine_test + async def test_data_loss_handling(self, mockserver: MockServer) -> None: + request = Request(mockserver.url("/broken", is_secure=self.is_secure)) + async with self.get_dh() as download_handler: + with pytest.raises(DownloadFailedError): + await download_handler.download_request(request) + + class TestSimpleHttps(HttpxDownloadHandlerMixin, TestSimpleHttpsBase): pass @@ -127,7 +153,7 @@ class TestHttpsProxy(TestHttpProxy): @pytest.mark.requires_mitmproxy class TestMitmProxy(HttpxDownloadHandlerMixin, TestMitmProxyBase): - pass + handler_supports_socks = HAS_SOCKS @pytest.mark.requires_internet diff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.py index e67129abf..0e1ff07c9 100644 --- a/tests/test_downloader_handlers_http_base.py +++ b/tests/test_downloader_handlers_http_base.py @@ -39,7 +39,7 @@ from scrapy.utils.misc import build_from_crawler from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler from tests import NON_EXISTING_RESOLVABLE -from tests.mockserver.mitm_proxy import MitmProxy, wrong_credentials +from tests.mockserver.mitm_proxy import wrong_credentials from tests.mockserver.proxy_echo import ProxyEchoMockServer from tests.mockserver.simple_https import SimpleMockServer from tests.spiders import ( @@ -73,6 +73,7 @@ class TestHttpBase(ABC): handler_supports_http2_dataloss: bool = True # default headers added by the underlying library that cannot be suppressed always_present_req_headers: ClassVar[frozenset[str]] = frozenset() + default_handler_settings: ClassVar[dict[str, Any]] = {} @property @abstractmethod @@ -83,6 +84,10 @@ class TestHttpBase(ABC): async def get_dh( self, settings_dict: dict[str, Any] | None = None ) -> AsyncGenerator[DownloadHandlerProtocol]: + settings_dict = { + **self.default_handler_settings, + **(settings_dict or {}), + } crawler = get_crawler(DefaultSpider, settings_dict) crawler.spider = crawler._create_spider() dh = build_from_crawler(self.download_handler_cls, crawler) @@ -339,9 +344,7 @@ class TestHttpBase(ABC): @coroutine_test async def test_timeout_download_from_spider_server_hangs( - self, - mockserver: MockServer, - reactor_pytest: str, + self, mockserver: MockServer, reactor_pytest: str ) -> None: if reactor_pytest == "asyncio" and sys.platform == "win32": # https://twistedmatrix.com/trac/ticket/10279 @@ -1317,6 +1320,7 @@ class TestHttpProxyBase(ABC): class TestMitmProxyBase(ABC): # whether the handler supports HTTPS proxies with HTTPS destinations handler_supports_tls_in_tls: bool = True + handler_supports_socks: bool = False @property @abstractmethod @@ -1326,13 +1330,10 @@ class TestMitmProxyBase(ABC): @pytest.mark.parametrize( "https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"] ) + @pytest.mark.usefixtures("mitm_proxy_server") @coroutine_test async def test_http_proxy( - self, - caplog: pytest.LogCaptureFixture, - mockserver: MockServer, - mitm_proxy_server: MitmProxy, - https_dest: bool, + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer, https_dest: bool ) -> None: """HTTP proxy, HTTP or HTTPS destination.""" crawler = get_crawler(SingleRequestSpider, self.settings_dict) @@ -1347,13 +1348,10 @@ class TestMitmProxyBase(ABC): @pytest.mark.parametrize( "https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"] ) + @pytest.mark.usefixtures("mitm_proxy_server_https") @coroutine_test async def test_https_proxy( - self, - caplog: pytest.LogCaptureFixture, - mockserver: MockServer, - mitm_proxy_server_https: MitmProxy, - https_dest: bool, + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer, https_dest: bool ) -> None: """HTTPS proxy, HTTP or HTTPS destination.""" if https_dest and not self.handler_supports_tls_in_tls: @@ -1370,13 +1368,13 @@ class TestMitmProxyBase(ABC): @pytest.mark.parametrize( "https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"] ) + @pytest.mark.usefixtures("mitm_proxy_server") @coroutine_test async def test_http_proxy_auth_error( self, caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch, mockserver: MockServer, - mitm_proxy_server: MitmProxy, https_dest: bool, ) -> None: """HTTP proxy, HTTP or HTTPS destination, wrong proxy creds.""" @@ -1394,13 +1392,10 @@ class TestMitmProxyBase(ABC): @pytest.mark.parametrize( "https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"] ) + @pytest.mark.usefixtures("mitm_proxy_server") @coroutine_test async def test_dont_leak_proxy_authorization_header( - self, - caplog: pytest.LogCaptureFixture, - mockserver: MockServer, - mitm_proxy_server: MitmProxy, - https_dest: bool, + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer, https_dest: bool ) -> None: """HTTP proxy, HTTP or HTTPS destination. Check that the auth header is not sent to the destination.""" @@ -1414,6 +1409,49 @@ class TestMitmProxyBase(ABC): echo = json.loads(crawler.spider.meta["responses"][0].text) assert "Proxy-Authorization" not in echo["headers"] + @pytest.mark.parametrize( + "https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"] + ) + @pytest.mark.usefixtures("socks5_proxy_server") + @coroutine_test + async def test_download_with_socks_proxy( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer, https_dest: bool + ) -> None: + """SOCKS5 proxy, HTTP or HTTPS destination.""" + if not self.handler_supports_socks: + pytest.skip("SOCKS proxies are not supported") + crawler = get_crawler(SingleRequestSpider, self.settings_dict) + with caplog.at_level(logging.DEBUG): + await crawler.crawl_async( + seed=mockserver.url("/status?n=200", is_secure=https_dest) + ) + assert isinstance(crawler.spider, SingleRequestSpider) + self._assert_got_response_code(200, caplog.text) + self._assert_headers(crawler.spider.meta["responses"][0].headers, https_dest) + + @pytest.mark.parametrize( + "https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"] + ) + @pytest.mark.usefixtures("socks5_proxy_server") + @coroutine_test + async def test_socks_proxy_auth_error( + self, + caplog: pytest.LogCaptureFixture, + monkeypatch: pytest.MonkeyPatch, + mockserver: MockServer, + https_dest: bool, + ) -> None: + if not self.handler_supports_socks: + pytest.skip("SOCKS proxies are not supported") + envvar = "https_proxy" if https_dest else "http_proxy" + monkeypatch.setenv(envvar, wrong_credentials(os.environ[envvar])) + crawler = get_crawler(SimpleSpider, self.settings_dict) + with caplog.at_level(logging.DEBUG): + await crawler.crawl_async( + mockserver.url("/status?n=200", is_secure=https_dest) + ) + assert "DownloadConnectionRefusedError" in caplog.text + @staticmethod def _assert_headers(headers: Headers, https_dest: bool) -> None: assert b"X-Via-Mitmproxy" in headers diff --git a/tox.ini b/tox.ini index cc916caea..602394761 100644 --- a/tox.ini +++ b/tox.ini @@ -60,6 +60,7 @@ deps = ipython==8.39.0 pyOpenSSL==26.2.0 pytest==9.0.3 + socksio==1.0.0 types-Pygments==2.20.0.20260508 types-defusedxml==0.7.0.20260504 types-lxml==2026.2.16 @@ -148,7 +149,7 @@ deps = brotli >= 1.2.0; implementation_name != "pypy" # optional for HTTP compress downloader middleware tests brotlicffi >= 1.2.0.0; implementation_name == "pypy" # optional for HTTP compress downloader middleware tests google-cloud-storage - httpx + httpx[http2,socks] ipython robotexclusionrulesparser uvloop; platform_system != "Windows" and implementation_name != "pypy" @@ -304,5 +305,6 @@ deps = {[testenv]deps} # mitmproxy does not support PyPy mitmproxy; implementation_name != "pypy" + httpx[http2,socks] commands = pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=botocore.junit.xml -o junit_family=legacy} -m requires_mitmproxy