From 8eb545867e0a0e66f5ff6d8b95f2139b2bda07c2 Mon Sep 17 00:00:00 2001 From: Shun Kakinoki Date: Mon, 10 Aug 2026 21:21:37 +0800 Subject: [PATCH] feat(providers): allow custom prompt cache key opt-in --- agent/agent_init.py | 54 +++++++++++++++++ agent/chat_completion_helpers.py | 3 + agent/transports/chat_completions.py | 5 +- hermes_cli/config.py | 10 +++- .../test_custom_provider_prompt_cache_key.py | 59 +++++++++++++++++++ .../agent/transports/test_chat_completions.py | 14 +++++ tests/hermes_cli/test_config.py | 3 + .../test_runtime_provider_resolution.py | 17 ++++++ tests/run_agent/test_run_agent.py | 12 ++++ website/docs/integrations/providers.md | 18 +++++- 10 files changed, 191 insertions(+), 4 deletions(-) create mode 100644 tests/agent/test_custom_provider_prompt_cache_key.py diff --git a/agent/agent_init.py b/agent/agent_init.py index 797c18c6dfcdc..c98dd55a2c1d7 100644 --- a/agent/agent_init.py +++ b/agent/agent_init.py @@ -456,6 +456,52 @@ def _merge_custom_provider_extra_body(agent, custom_providers: List[Dict[str, An agent.request_overrides = overrides +def _custom_provider_supports_prompt_cache_key_for_agent( + *, + provider: str, + model: str, + base_url: str, + custom_providers: List[Dict[str, Any]], +) -> bool: + """Resolve the explicit cache-key capability for the active endpoint.""" + provider_norm = (provider or "").strip().lower() + if provider_norm == "custom": + provider_key_filter = "" + elif provider_norm.startswith("custom:"): + provider_key_filter = provider_norm.split(":", 1)[1].strip() + else: + return False + + target_url = _normalized_custom_base_url(base_url) + if not target_url: + return False + + fallback: Optional[bool] = None + for entry in custom_providers or []: + if not isinstance(entry, dict): + continue + if provider_key_filter: + entry_keys = { + str(entry.get("provider_key", "") or "").strip().lower(), + str(entry.get("name", "") or "").strip().lower(), + } + if provider_key_filter not in entry_keys: + continue + if _normalized_custom_base_url(entry.get("base_url")) != target_url: + continue + capability = entry.get("supports_prompt_cache_key") + if not isinstance(capability, bool): + continue + has_model_scope = bool(entry.get("model") or entry.get("models")) + if has_model_scope: + if _custom_provider_model_matches(model, entry): + return capability + elif fallback is None: + fallback = capability + + return bool(fallback) + + def init_agent( agent, base_url: str = None, @@ -2359,6 +2405,14 @@ def init_agent( # compression model context-length detection needs the same list). agent._custom_providers = _custom_providers _merge_custom_provider_extra_body(agent, _custom_providers) + agent._supports_prompt_cache_key = ( + _custom_provider_supports_prompt_cache_key_for_agent( + provider=agent.provider, + model=agent.model, + base_url=agent.base_url, + custom_providers=_custom_providers, + ) + ) # Check custom_providers per-model context_length if _config_context_length is None and _custom_providers: diff --git a/agent/chat_completion_helpers.py b/agent/chat_completion_helpers.py index 6a87c7c1e05ff..08affc58abe47 100644 --- a/agent/chat_completion_helpers.py +++ b/agent/chat_completion_helpers.py @@ -1550,6 +1550,9 @@ def build_api_kwargs(agent, api_messages: list, tools_for_api: list | None = Non request_overrides=agent.request_overrides, session_id=getattr(agent, "session_id", None), provider_profile=_profile, + supports_prompt_cache_key=bool( + getattr(agent, "_supports_prompt_cache_key", False) + ), ollama_num_ctx=agent._ollama_num_ctx, # Context forwarded to profile hooks: provider_preferences=_prefs or None, diff --git a/agent/transports/chat_completions.py b/agent/transports/chat_completions.py index 275a66db3299e..cbe67a5d13844 100644 --- a/agent/transports/chat_completions.py +++ b/agent/transports/chat_completions.py @@ -740,7 +740,10 @@ class ChatCompletionsTransport(ProviderTransport): api_kwargs, messages=sanitized, tools=api_kwargs.get("tools"), - supports_prompt_cache_key=bool(getattr(profile, "supports_prompt_cache_key", False)), + supports_prompt_cache_key=bool( + getattr(profile, "supports_prompt_cache_key", False) + or params.get("supports_prompt_cache_key") + ), session_id=params.get("session_id"), ) diff --git a/hermes_cli/config.py b/hermes_cli/config.py index 8424677d34daa..68c21ac0c0b90 100644 --- a/hermes_cli/config.py +++ b/hermes_cli/config.py @@ -1309,6 +1309,7 @@ def _normalize_custom_provider_entry( "defaultModel": "default_model", "contextLength": "context_length", "rateLimitDelay": "rate_limit_delay", + "supportsPromptCacheKey": "supports_prompt_cache_key", } # api_key_env is a documented snake_case alias for key_env (see # website/docs/guides/azure-foundry.md). Normalize it up front so the @@ -1326,7 +1327,7 @@ def _normalize_custom_provider_entry( "context_length", "rate_limit_delay", "request_timeout_seconds", "stale_timeout_seconds", "discover_models", "extra_body", "extra_headers", - "ssl_ca_cert", "ssl_verify", + "ssl_ca_cert", "ssl_verify", "supports_prompt_cache_key", } for camel, snake in _CAMEL_ALIASES.items(): if camel in entry and snake not in entry: @@ -1450,6 +1451,10 @@ def _normalize_custom_provider_entry( if isinstance(discover_models, bool): normalized["discover_models"] = discover_models + supports_prompt_cache_key = entry.get("supports_prompt_cache_key") + if isinstance(supports_prompt_cache_key, bool): + normalized["supports_prompt_cache_key"] = supports_prompt_cache_key + extra_body = entry.get("extra_body") if isinstance(extra_body, dict): normalized["extra_body"] = dict(extra_body) @@ -1497,6 +1502,7 @@ def _custom_provider_entry_to_provider_config( "context_length", "rate_limit_delay", "discover_models", + "supports_prompt_cache_key", "extra_body", "extra_headers", "ssl_ca_cert", @@ -1884,7 +1890,7 @@ _KNOWN_ROOT_KEYS = frozenset(DEFAULT_CONFIG.keys()) | _EXTRA_KNOWN_ROOT_KEYS _VALID_CUSTOM_PROVIDER_FIELDS = { "name", "base_url", "api_key", "api_mode", "model", "models", "context_length", "rate_limit_delay", "extra_body", - "ssl_ca_cert", "ssl_verify", + "ssl_ca_cert", "ssl_verify", "supports_prompt_cache_key", # key_env is read at runtime by runtime_provider.py and auxiliary_client.py # — include it here so the set accurately describes the supported schema. "key_env", diff --git a/tests/agent/test_custom_provider_prompt_cache_key.py b/tests/agent/test_custom_provider_prompt_cache_key.py new file mode 100644 index 0000000000000..843e4388dd6f4 --- /dev/null +++ b/tests/agent/test_custom_provider_prompt_cache_key.py @@ -0,0 +1,59 @@ +"""End-to-end config plumbing for custom-provider prompt cache keys.""" + +from agent.agent_init import ( + _custom_provider_supports_prompt_cache_key_for_agent, +) +from agent.transports.chat_completions import ChatCompletionsTransport +from hermes_cli.config import get_compatible_custom_providers +from providers import get_provider_profile + + +def test_custom_provider_capability_reaches_chat_completions_transport(): + config = { + "providers": { + "local-gateway": { + "api": "https://gateway.example.com/v1", + "transport": "chat_completions", + "models": {"deepseek-chat": {}}, + "supports_prompt_cache_key": True, + } + } + } + entries = get_compatible_custom_providers(config) + capability = _custom_provider_supports_prompt_cache_key_for_agent( + provider="custom:local-gateway", + model="deepseek-chat", + base_url="https://gateway.example.com/v1/", + custom_providers=entries, + ) + + kwargs = ChatCompletionsTransport().build_kwargs( + model="deepseek-chat", + messages=[{"role": "system", "content": "You are stable."}], + tools=[{"type": "function", "function": {"name": "lookup"}}], + session_id="session-local-gateway", + provider_profile=get_provider_profile("custom"), + supports_prompt_cache_key=capability, + ) + + assert capability is True + assert kwargs["prompt_cache_key"].startswith("pck_") + + +def test_custom_provider_capability_stays_scoped_to_matching_model(): + entries = [ + { + "provider_key": "local-gateway", + "name": "Local Gateway", + "base_url": "https://gateway.example.com/v1", + "models": {"deepseek-chat": {}}, + "supports_prompt_cache_key": True, + } + ] + + assert not _custom_provider_supports_prompt_cache_key_for_agent( + provider="custom:local-gateway", + model="strict-model", + base_url="https://gateway.example.com/v1", + custom_providers=entries, + ) diff --git a/tests/agent/transports/test_chat_completions.py b/tests/agent/transports/test_chat_completions.py index d401d6d958781..f609d7d16300f 100644 --- a/tests/agent/transports/test_chat_completions.py +++ b/tests/agent/transports/test_chat_completions.py @@ -647,6 +647,20 @@ class TestPromptCacheKeyCapability: assert body["prompt_cache_key"] == kwargs["prompt_cache_key"] + def test_explicit_capability_overrides_default_off_profile(self, transport): + from providers.base import ProviderProfile + + kwargs = transport.build_kwargs( + model="cache-model", + messages=self._messages(), + tools=self._tools(), + session_id="session-custom-gateway", + provider_profile=ProviderProfile(name="custom"), + supports_prompt_cache_key=True, + ) + + assert kwargs["prompt_cache_key"].startswith("pck_") + def test_openai_api_base_url_implies_capability(self, transport): """api.openai.com gets the key WITHOUT an explicit flag (exact host).""" kwargs = transport.build_kwargs( diff --git a/tests/hermes_cli/test_config.py b/tests/hermes_cli/test_config.py index c2db04cd96618..5ad4476157928 100644 --- a/tests/hermes_cli/test_config.py +++ b/tests/hermes_cli/test_config.py @@ -837,6 +837,7 @@ class TestCustomProviderCompatibility: "context_length": 262144, "rate_limit_delay": 0.25, "discover_models": False, + "supports_prompt_cache_key": True, "extra_body": { "chat_template_kwargs": {"enable_thinking": False} }, @@ -867,6 +868,7 @@ class TestCustomProviderCompatibility: assert provider["context_length"] == 262144 assert provider["rate_limit_delay"] == 0.25 assert provider["discover_models"] is False + assert provider["supports_prompt_cache_key"] is True assert provider["extra_body"] == { "chat_template_kwargs": {"enable_thinking": False} } @@ -880,6 +882,7 @@ class TestCustomProviderCompatibility: ) assert compatible_provider["models"] == model_map assert compatible_provider["key_env"] == "KIMI_CODING_API_KEY" + assert compatible_provider["supports_prompt_cache_key"] is True def test_providers_dict_resolves_at_runtime(self, tmp_path): """After migration deleted custom_providers, get_compatible_custom_providers diff --git a/tests/hermes_cli/test_runtime_provider_resolution.py b/tests/hermes_cli/test_runtime_provider_resolution.py index 1f90e3edb9c61..8d676e5ca80e9 100644 --- a/tests/hermes_cli/test_runtime_provider_resolution.py +++ b/tests/hermes_cli/test_runtime_provider_resolution.py @@ -1144,6 +1144,23 @@ class TestProviderEntryApiKeyEnvAlias: assert normalized is not None assert "extra_body" in _VALID_CUSTOM_PROVIDER_FIELDS assert normalized["extra_body"] == entry["extra_body"] + + def test_prompt_cache_key_capability_is_supported_schema(self): + from hermes_cli.config import ( + _VALID_CUSTOM_PROVIDER_FIELDS, + _normalize_custom_provider_entry, + ) + + entry = { + "name": "cache-gateway", + "base_url": "https://gateway.example.com/v1", + "supports_prompt_cache_key": True, + } + normalized = _normalize_custom_provider_entry(entry) + + assert normalized is not None + assert "supports_prompt_cache_key" in _VALID_CUSTOM_PROVIDER_FIELDS + assert normalized["supports_prompt_cache_key"] is True # ============================================================================= # Tencent TokenHub — API-key provider runtime resolution # ============================================================================= diff --git a/tests/run_agent/test_run_agent.py b/tests/run_agent/test_run_agent.py index 0c16a73ce7db9..a8133aed48089 100644 --- a/tests/run_agent/test_run_agent.py +++ b/tests/run_agent/test_run_agent.py @@ -1239,6 +1239,18 @@ class TestBuildApiKwargs: assert kwargs["messages"] is messages assert kwargs["timeout"] == 1800.0 + def test_custom_provider_cache_key_capability_reaches_transport(self, agent): + agent.provider = "custom" + agent._supports_prompt_cache_key = True + messages = [ + {"role": "system", "content": "You are stable."}, + {"role": "user", "content": "hi"}, + ] + + kwargs = agent._build_api_kwargs(messages) + + assert kwargs["prompt_cache_key"].startswith("pck_") + def test_explicit_request_local_tools_reach_native_transport(self, agent, monkeypatch): from agent.prompt_caching import build_prompt_cache_plan diff --git a/website/docs/integrations/providers.md b/website/docs/integrations/providers.md index a5b81f5660c6c..684cf7c9dee41 100644 --- a/website/docs/integrations/providers.md +++ b/website/docs/integrations/providers.md @@ -1271,7 +1271,7 @@ providers: transport: anthropic_messages # for Anthropic-compatible proxies ``` -Each entry accepts: `api` (the endpoint base URL — `base_url`/`url` are accepted aliases), `name` (optional display name; defaults to the dict key), `key_env` or inline `api_key`, `transport` (`chat_completions` / `anthropic_messages` / `codex_responses`), `default_model`, `models`, `context_length`, `discover_models`, `extra_body`, `extra_headers`, `ssl_ca_cert` / `ssl_verify`, and `enabled: false` to hide an entry without deleting it. +Each entry accepts: `api` (the endpoint base URL — `base_url`/`url` are accepted aliases), `name` (optional display name; defaults to the dict key), `key_env` or inline `api_key`, `transport` (`chat_completions` / `anthropic_messages` / `codex_responses`), `default_model`, `models`, `context_length`, `discover_models`, `supports_prompt_cache_key`, `extra_body`, `extra_headers`, `ssl_ca_cert` / `ssl_verify`, and `enabled: false` to hide an entry without deleting it. :::note Legacy format Older configs used a top-level `custom_providers:` list instead. It still works — Hermes reads both — and `hermes update` auto-migrates it to the `providers:` dict (config v12). Field names differ slightly in the dict format: legacy `model` is `default_model`, and legacy `api_mode` is `transport`. @@ -1289,6 +1289,22 @@ providers: reasoning_effort: high ``` +If the endpoint explicitly accepts OpenAI's `prompt_cache_key` request field, +opt it in with `supports_prompt_cache_key: true`. Hermes then derives a stable, +session-scoped key from the system prompt and tools instead of requiring a +hard-coded key: + +```yaml +providers: + local-gateway: + api: https://gateway.example.com/v1 + transport: chat_completions + supports_prompt_cache_key: true +``` + +Leave this off for endpoints that do not document the field; some strict +OpenAI-compatible servers reject unknown top-level request fields. + Use the shape your server documents. For example, vLLM Gemma deployments and some NVIDIA NIM endpoints expect `enable_thinking` under `chat_template_kwargs` instead of as a top-level `extra_body` field: ```yaml