From dac7387b8b48580f298da12af11782663d015bb5 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Sat, 8 Aug 2026 19:14:59 +0200 Subject: [PATCH 1/2] Add SSLKEYLOGFILE support --- docs/topics/debug.rst | 20 +++++++++++++++ scrapy/core/downloader/contextfactory.py | 22 ++++++++++++++-- scrapy/utils/ssl.py | 28 +++++++++++++++++++++ tests/test_core_downloader.py | 21 ++++++++++++++++ tests/utils/bases/download_handlers_http.py | 23 +++++++++++++++++ 5 files changed, 112 insertions(+), 2 deletions(-) diff --git a/docs/topics/debug.rst b/docs/topics/debug.rst index 988e37bbd..e398e0a1d 100644 --- a/docs/topics/debug.rst +++ b/docs/topics/debug.rst @@ -153,6 +153,26 @@ available in all future runs should they be necessary again: For more information, check the :ref:`topics-logging` section. +.. _debug-tls: + +Decrypting TLS traffic +====================== + +Scrapy writes the session keys of its HTTPS connections to the file that the +``SSLKEYLOGFILE`` environment variable points to, using the `NSS key log +format`_ that traffic analysis tools such as Wireshark understand. + +.. versionadded:: VERSION + +.. code-block:: shell + + SSLKEYLOGFILE=/tmp/sslkeylog scrapy crawl myspider + +.. _NSS key log format: https://firefox-source-docs.mozilla.org/security/nss/legacy/key_log_format/index.html + +.. warning:: Anyone who can read the key log file can decrypt the traffic of + the connections recorded in it, including any credentials that they carry. + .. _debug-vscode: Visual Studio Code diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index ef948997d..a599a8e03 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -24,7 +24,12 @@ from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils._deps_compat import TWISTED_TLS_NEW_IMPL from scrapy.utils.deprecate import create_deprecated_class from scrapy.utils.misc import build_from_crawler, load_object -from scrapy.utils.ssl import _get_cert_options_version_kwargs, _get_tls_version_limits +from scrapy.utils.ssl import ( + _get_cert_options_version_kwargs, + _get_keylog_filename, + _get_tls_version_limits, + _set_keylog_callback, +) if TYPE_CHECKING: from twisted.internet._sslverify import ClientTLSOptions @@ -154,10 +159,22 @@ class _ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): self._get_context(), # type: ignore[arg-type] ) # Otherwise use the normal Twisted function. - return optionsForClientTLS( # type: ignore[no-any-return] + creator = optionsForClientTLS( hostname=hostname.decode("ascii"), extraCertificateOptions=self._get_cert_options_kwargs(), ) + if _get_keylog_filename(): + # This creator builds its own certificate options, so its context + # doesn't come from _ScrapyCertificateOptions. + _set_keylog_callback(_get_creator_context(creator)) + return cast("ClientTLSOptions", creator) + + +def _get_creator_context(creator: Any) -> SSL.Context: + """Return the context that *creator* uses for its connections.""" + if TWISTED_TLS_NEW_IMPL: + return cast("SSL.Context", creator._createConnection.__self__.getContext()) + return cast("SSL.Context", creator._ctx) ScrapyClientContextFactory = create_deprecated_class( @@ -264,6 +281,7 @@ class _ScrapyCertificateOptions(CertificateOptions): else: ctx = super()._makeContext() ctx.set_options(0x4) # OP_LEGACY_SERVER_CONNECT + _set_keylog_callback(ctx) return ctx diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py index 828a99e23..61c71cd6d 100644 --- a/scrapy/utils/ssl.py +++ b/scrapy/utils/ssl.py @@ -1,8 +1,11 @@ from __future__ import annotations import logging +import os import ssl +import sys import warnings +from pathlib import Path from typing import TYPE_CHECKING, Any, TypedDict, TypeVar import OpenSSL._util as pyOpenSSLutil @@ -53,6 +56,13 @@ def _get_tls_version_limits( ) +def _get_keylog_filename() -> str | None: + """Return the path where TLS session keys should be logged, or ``None``.""" + if sys.flags.ignore_environment: + return None + return os.environ.get("SSLKEYLOGFILE") + + # stdlib ssl module utils _STDLIB_VERSION_MAP: dict[str, ssl.TLSVersion] = { @@ -89,6 +99,8 @@ def _make_ssl_context(settings: BaseSettings) -> ssl.SSLContext: ctx.maximum_version = tls_max_ver if ciphers_setting: ctx.set_ciphers(ciphers_setting) + if keylog_filename := _get_keylog_filename(): + ctx.keylog_filename = keylog_filename return ctx @@ -101,6 +113,8 @@ def _make_insecure_ssl_ctx() -> ssl.SSLContext: ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) ctx.check_hostname = False ctx.verify_mode = ssl.CERT_NONE + if keylog_filename := _get_keylog_filename(): + ctx.keylog_filename = keylog_filename return ctx @@ -121,6 +135,20 @@ def _log_sslobj_debug_info(sslobj: ssl.SSLObject) -> None: # pyOpenSSL utils +def _set_keylog_callback(ctx: OpenSSL.SSL.Context) -> None: + """Make *ctx* write TLS session keys to the key log file, if enabled.""" + keylog_filename = _get_keylog_filename() + if not keylog_filename: + return + keylog_path = Path(keylog_filename) + + def write_keylog_line(connection: OpenSSL.SSL.Connection, line: bytes) -> None: + with keylog_path.open("ab") as f: + f.write(line + b"\n") + + ctx.set_keylog_callback(write_keylog_line) + + def _ffi_buf_to_string(buf: Any) -> str: return to_unicode(pyOpenSSLutil.ffi.string(buf)) diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py index 912c0450b..3f4e7cd5a 100644 --- a/tests/test_core_downloader.py +++ b/tests/test_core_downloader.py @@ -36,6 +36,8 @@ from tests.mockserver.utils import ssl_context_factory from tests.utils.decorators import coroutine_test if TYPE_CHECKING: + from pathlib import Path + from twisted.internet.defer import Deferred from twisted.internet.interfaces import IListeningPort from twisted.web.iweb import IBodyProducer @@ -174,6 +176,25 @@ class TestContextFactory(TestContextFactoryBase): ) factory.creatorForNetloc(b"website.tld", 443) + @pytest.mark.parametrize("verify_certs", [False, True]) + def test_keylog( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + verify_certs: bool, + ) -> None: + """The key log callback should be set with and without verification.""" + monkeypatch.setenv("SSLKEYLOGFILE", str(tmp_path / "keylog")) + crawler = get_crawler( + settings_dict={"DOWNLOAD_VERIFY_CERTIFICATES": verify_certs} + ) + factory: _ScrapyClientContextFactory = _load_context_factory_from_settings( + crawler + ) + creator = factory.creatorForNetloc(b"website.tld", 443) + conn = creator.clientConnectionForTLS(self._get_dummy_protocol()) + assert conn.get_context()._keylog_callback is not None + def test_ctx_flags(self, factory: _ScrapyClientContextFactory) -> None: """The context should have the expected flags set.""" creator = factory.creatorForNetloc(b"website.tld", 443) diff --git a/tests/utils/bases/download_handlers_http.py b/tests/utils/bases/download_handlers_http.py index e44f9bcb8..09c8f02d7 100644 --- a/tests/utils/bases/download_handlers_http.py +++ b/tests/utils/bases/download_handlers_http.py @@ -55,6 +55,7 @@ from tests.utils.decorators import coroutine_test if TYPE_CHECKING: from collections.abc import AsyncGenerator, Generator + from pathlib import Path from scrapy.core.downloader.handlers import DownloadHandlerProtocol from tests.mockserver.http import MockServer @@ -862,6 +863,28 @@ class TestHttpsBase(TestHttpBase): assert response.body == b"Works" assert self.tls_log_message in caplog.text + @coroutine_test + async def test_keylog( + self, + mockserver: MockServer, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + ) -> None: + keylog_file = tmp_path / "keylog" + monkeypatch.setenv("SSLKEYLOGFILE", str(keylog_file)) + request = Request(mockserver.url("/text", is_secure=self.is_secure)) + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + assert response.body == b"Works" + # The stdlib writes a comment line at the top of the file. + lines = [ + line + for line in keylog_file.read_text().splitlines() + if not line.startswith("#") + ] + assert lines + assert all(len(line.split()) == 3 for line in lines) + @coroutine_test async def test_verify_certs_deprecated(self, mockserver: MockServer) -> None: request = Request(mockserver.url("/text", is_secure=self.is_secure)) From 6e29c1792bdae641d5ce386c17b8cb0172f887b7 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Sat, 8 Aug 2026 20:11:39 +0200 Subject: [PATCH 2/2] Complete test coverage --- tests/test_core_downloader.py | 16 ++++++++++++++++ tests/test_downloader_handler_httpx.py | 19 +++++++++++++++++++ tests/utils/cmdline.py | 6 ++++-- 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py index 3f4e7cd5a..9f86336fe 100644 --- a/tests/test_core_downloader.py +++ b/tests/test_core_downloader.py @@ -33,6 +33,7 @@ from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler from tests.mockserver.http_resources import PayloadResource, put_child from tests.mockserver.utils import ssl_context_factory +from tests.utils.cmdline import proc from tests.utils.decorators import coroutine_test if TYPE_CHECKING: @@ -43,6 +44,7 @@ if TYPE_CHECKING: from twisted.web.iweb import IBodyProducer from scrapy.http import Response + from tests.mockserver.http import MockServer class TestSlot: @@ -195,6 +197,20 @@ class TestContextFactory(TestContextFactoryBase): conn = creator.clientConnectionForTLS(self._get_dummy_protocol()) assert conn.get_context()._keylog_callback is not None + def test_keylog_ignore_environment( + self, + monkeypatch: pytest.MonkeyPatch, + mockserver: MockServer, + tmp_path: Path, + ) -> None: + keylog_file = tmp_path / "keylog" + monkeypatch.setenv("SSLKEYLOGFILE", str(keylog_file)) + _, out, err = proc( + "fetch", mockserver.url("/text", is_secure=True), python_args=("-E",) + ) + assert out.strip() == "Works", err + assert not keylog_file.exists() + def test_ctx_flags(self, factory: _ScrapyClientContextFactory) -> None: """The context should have the expected flags set.""" creator = factory.creatorForNetloc(b"website.tld", 443) diff --git a/tests/test_downloader_handler_httpx.py b/tests/test_downloader_handler_httpx.py index 976daacaf..a5fa15f2e 100644 --- a/tests/test_downloader_handler_httpx.py +++ b/tests/test_downloader_handler_httpx.py @@ -34,8 +34,11 @@ from tests.utils.bases.download_handlers_http import ( from tests.utils.decorators import coroutine_test if TYPE_CHECKING: + from pathlib import Path + from scrapy.core.downloader.handlers import DownloadHandlerProtocol from tests.mockserver.http import MockServer + from tests.mockserver.proxy_echo import ProxyEchoMockServer pytestmark = pytest.mark.only_asyncio @@ -154,6 +157,22 @@ class TestHttpProxy(HttpxDownloadHandlerMixin, TestHttpProxyBase): class TestHttpsProxy(TestHttpProxy): is_secure = True + @coroutine_test + async def test_keylog( + self, + monkeypatch: pytest.MonkeyPatch, + proxy_mockserver: ProxyEchoMockServer, + tmp_path: Path, + ) -> None: + keylog_file = tmp_path / "keylog" + monkeypatch.setenv("SSLKEYLOGFILE", str(keylog_file)) + http_proxy = proxy_mockserver.url("", is_secure=True) + request = Request("http://example.com", meta={"proxy": http_proxy}) + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + assert response.body == self.expected_http_proxy_request_body + assert keylog_file.read_text() + @pytest.mark.requires_mitmproxy class TestMitmProxy(HttpxDownloadHandlerMixin, TestMitmProxyBase): diff --git a/tests/utils/cmdline.py b/tests/utils/cmdline.py index 095cb17a7..064a19b5a 100644 --- a/tests/utils/cmdline.py +++ b/tests/utils/cmdline.py @@ -23,8 +23,10 @@ def call(*args: str, **popen_kwargs: Any) -> int: ) -def proc(*args: str, **popen_kwargs: Any) -> tuple[int, str, str]: - args = (sys.executable, "-m", "scrapy.cmdline", *args) +def proc( + *args: str, python_args: tuple[str, ...] = (), **popen_kwargs: Any +) -> tuple[int, str, str]: + args = (sys.executable, *python_args, "-m", "scrapy.cmdline", *args) try: p = subprocess.run( args,