From de82b802270158df1907957415f3ee8762f48a32 Mon Sep 17 00:00:00 2001 From: adavyas Date: Tue, 7 Apr 2026 12:11:26 -0400 Subject: [PATCH] Require explicit deriver custom instruction token cap --- README.md | 4 ++-- config.toml.example | 4 ++-- docs/v3/contributing/configuration.mdx | 4 ++-- src/config.py | 20 ++++++++++++-------- src/schemas/configuration.py | 2 +- tests/deriver/test_deriver_processing.py | 6 ++++++ tests/test_config.py | 11 +++++------ tests/test_schema_validations.py | 23 +++++++++++++++++++++++ tests/utils/test_config_helpers.py | 20 ++++++++++++++++++-- 9 files changed, 71 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index a582ca09..b7390607 100644 --- a/README.md +++ b/README.md @@ -429,13 +429,13 @@ Examples: - `AUTH_JWT_SECRET` - JWT secret key - `DIALECTIC_LEVELS__low__MODEL` - Model for low reasoning level - `DERIVER_PROVIDER` - Provider for background deriver -- `DERIVER_MAX_CUSTOM_INSTRUCTIONS_TOKENS` - Optional cap for `reasoning.custom_instructions` +- `DERIVER_MAX_CUSTOM_INSTRUCTIONS_TOKENS` - Explicit cap for `reasoning.custom_instructions` - `SUMMARY_PROVIDER` - Summary generation provider - `LOG_LEVEL` - Application log level - `METRICS_ENABLED` - Enable Prometheus metrics - `TELEMETRY_ENABLED` - Enable CloudEvents telemetry -When using `reasoning.custom_instructions`, keep in mind that those instructions count against the deriver input budget along with discussion/history. If `DERIVER_MAX_CUSTOM_INSTRUCTIONS_TOKENS` is unset, Honcho derives a conservative cap from `DERIVER_MAX_INPUT_TOKENS` and rejects oversized values instead of truncating them silently. +When using `reasoning.custom_instructions`, keep in mind that those instructions count against the deriver input budget along with discussion/history. `DERIVER_MAX_CUSTOM_INSTRUCTIONS_TOKENS` must be set explicitly; Honcho does not derive it from `DERIVER_MAX_INPUT_TOKENS`. If custom instructions are provided without that setting, validation fails with a config error instead of silently assuming a fallback. ### Configuration Priority diff --git a/config.toml.example b/config.toml.example index bc28ae7b..e99215e9 100644 --- a/config.toml.example +++ b/config.toml.example @@ -84,8 +84,8 @@ MAX_OUTPUT_TOKENS = 4096 THINKING_BUDGET_TOKENS = 1024 LOG_OBSERVATIONS = false MAX_INPUT_TOKENS = 23000 -# Optional override for reasoning.custom_instructions token budget. -# If unset, Honcho derives a conservative cap from MAX_INPUT_TOKENS. +# Explicit token budget for reasoning.custom_instructions. +# Honcho does not derive this from MAX_INPUT_TOKENS. MAX_CUSTOM_INSTRUCTIONS_TOKENS = 2048 WORKING_REPRESENTATION_MAX_OBSERVATIONS = 100 REPRESENTATION_BATCH_MAX_TOKENS = 1024 diff --git a/docs/v3/contributing/configuration.mdx b/docs/v3/contributing/configuration.mdx index a15f2527..ed4a4b7f 100644 --- a/docs/v3/contributing/configuration.mdx +++ b/docs/v3/contributing/configuration.mdx @@ -371,7 +371,7 @@ DERIVER_MODEL=gemini-2.5-flash-lite DERIVER_MAX_OUTPUT_TOKENS=4096 DERIVER_THINKING_BUDGET_TOKENS=1024 DERIVER_MAX_INPUT_TOKENS=23000 # Maximum input tokens for deriver -DERIVER_MAX_CUSTOM_INSTRUCTIONS_TOKENS=2048 # Optional cap for reasoning.custom_instructions +DERIVER_MAX_CUSTOM_INSTRUCTIONS_TOKENS=2048 # Explicit cap for reasoning.custom_instructions DERIVER_TEMPERATURE= # Optional temperature override (unset by default) # Backup provider (optional, must set both or neither) @@ -395,7 +395,7 @@ DERIVER_WORKING_REPRESENTATION_MAX_OBSERVATIONS=100 # Max observations stored DERIVER_REPRESENTATION_BATCH_MAX_TOKENS=1024 # Max tokens per batch (must be <= MAX_INPUT_TOKENS) ``` -`reasoning.custom_instructions` consumes the same deriver input budget as the discussion/history sent to the model. If `DERIVER_MAX_CUSTOM_INSTRUCTIONS_TOKENS` is unset, Honcho derives a conservative cap from `DERIVER_MAX_INPUT_TOKENS`. If the custom instructions exceed that cap, the request/configuration is rejected instead of being truncated. +`reasoning.custom_instructions` consumes the same deriver input budget as the discussion/history sent to the model. `DERIVER_MAX_CUSTOM_INSTRUCTIONS_TOKENS` must be configured explicitly; Honcho does not derive it from `DERIVER_MAX_INPUT_TOKENS`. If custom instructions are supplied without that setting, or if they exceed it, the request/configuration is rejected instead of being truncated. **Peer Card:** diff --git a/src/config.py b/src/config.py index d786eea1..6b8969d5 100644 --- a/src/config.py +++ b/src/config.py @@ -281,13 +281,14 @@ class DeriverSettings(BackupLLMSettingsMixin, HonchoSettings): @property def effective_max_custom_instructions_tokens(self) -> int: - """Resolve the custom instruction budget from explicit or derived settings.""" - if self.MAX_CUSTOM_INSTRUCTIONS_TOKENS is not None: - return self.MAX_CUSTOM_INSTRUCTIONS_TOKENS + """Return the explicit custom instruction budget.""" + if self.MAX_CUSTOM_INSTRUCTIONS_TOKENS is None: + raise ValueError( + "No value configured for DERIVER.MAX_CUSTOM_INSTRUCTIONS_TOKENS. " + "Set [deriver].MAX_CUSTOM_INSTRUCTIONS_TOKENS in config.toml." + ) - # Reserve most of the deriver input budget for discussion/history while - # still allowing moderately detailed custom instructions by default. - return max(1, min(2048, self.MAX_INPUT_TOKENS // 4)) + return self.MAX_CUSTOM_INSTRUCTIONS_TOKENS @model_validator(mode="after") def validate_batch_tokens_vs_context_limit(self): @@ -295,10 +296,13 @@ class DeriverSettings(BackupLLMSettingsMixin, HonchoSettings): raise ValueError( f"REPRESENTATION_BATCH_MAX_TOKENS ({self.REPRESENTATION_BATCH_MAX_TOKENS}) cannot exceed max deriver input tokens ({self.MAX_INPUT_TOKENS})" ) - if self.effective_max_custom_instructions_tokens > self.MAX_INPUT_TOKENS: + if ( + self.MAX_CUSTOM_INSTRUCTIONS_TOKENS is not None + and self.MAX_CUSTOM_INSTRUCTIONS_TOKENS > self.MAX_INPUT_TOKENS + ): raise ValueError( "MAX_CUSTOM_INSTRUCTIONS_TOKENS " - f"({self.effective_max_custom_instructions_tokens}) cannot exceed " + f"({self.MAX_CUSTOM_INSTRUCTIONS_TOKENS}) cannot exceed " f"max deriver input tokens ({self.MAX_INPUT_TOKENS})" ) return self diff --git a/src/schemas/configuration.py b/src/schemas/configuration.py index 6f6c6b89..80993d23 100644 --- a/src/schemas/configuration.py +++ b/src/schemas/configuration.py @@ -53,7 +53,7 @@ class ReasoningConfiguration(BaseModel): ) custom_instructions: str | None = Field( default=None, - description="Optional custom instructions for the reasoning system on this workspace/session/message. May be omitted or set to a blank string. Non-blank values are rejected if they exceed the configured deriver custom-instructions token budget.", + description="Optional custom instructions for the reasoning system on this workspace/session/message. May be omitted or set to a blank string. Non-blank values are rejected if they exceed the explicitly configured deriver custom-instructions token budget.", ) _validate_custom_instructions = field_validator( diff --git a/tests/deriver/test_deriver_processing.py b/tests/deriver/test_deriver_processing.py index a4cb9039..c3201e35 100644 --- a/tests/deriver/test_deriver_processing.py +++ b/tests/deriver/test_deriver_processing.py @@ -6,6 +6,7 @@ from unittest.mock import AsyncMock import pytest from src import models +from src.config import settings from src.deriver.deriver import process_representation_tasks_batch from src.schemas import ( ResolvedConfiguration, @@ -222,6 +223,11 @@ class TestCustomInstructions: "src.crud.representation.RepresentationManager.save_representation", AsyncMock(), ) + monkeypatch.setattr( + settings.DERIVER, + "MAX_CUSTOM_INSTRUCTIONS_TOKENS", + 100, + ) message = models.Message( id=1, diff --git a/tests/test_config.py b/tests/test_config.py index 266e90ef..07377e08 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -5,15 +5,14 @@ from src.config import DeriverSettings class TestDeriverSettings: - def test_derives_custom_instruction_token_limit_from_input_budget(self) -> None: + def test_requires_explicit_custom_instruction_token_limit(self) -> None: settings = DeriverSettings(MAX_INPUT_TOKENS=2000) - assert settings.effective_max_custom_instructions_tokens == 500 + with pytest.raises(ValueError) as exc_info: + _ = settings.effective_max_custom_instructions_tokens - def test_caps_derived_custom_instruction_token_limit(self) -> None: - settings = DeriverSettings(MAX_INPUT_TOKENS=23000) - - assert settings.effective_max_custom_instructions_tokens == 2048 + assert "MAX_CUSTOM_INSTRUCTIONS_TOKENS" in str(exc_info.value) + assert "config.toml" in str(exc_info.value) def test_uses_explicit_custom_instruction_token_limit(self) -> None: settings = DeriverSettings( diff --git a/tests/test_schema_validations.py b/tests/test_schema_validations.py index 6d56e3e4..a32e1aa2 100644 --- a/tests/test_schema_validations.py +++ b/tests/test_schema_validations.py @@ -256,6 +256,29 @@ class TestReasoningConfigurationValidation: assert session.configuration.reasoning is not None assert session.configuration.reasoning.custom_instructions == "" + def test_session_create_requires_explicit_custom_instruction_token_budget( + self, monkeypatch: pytest.MonkeyPatch + ): + monkeypatch.setattr( + settings.DERIVER, + "MAX_CUSTOM_INSTRUCTIONS_TOKENS", + None, + ) + + with pytest.raises(ValidationError) as exc_info: + SessionCreate( + name="test-session", + configuration={ + "reasoning": { + "enabled": True, + "custom_instructions": "focus on durable preferences", + } + }, + ) + + assert "MAX_CUSTOM_INSTRUCTIONS_TOKENS" in str(exc_info.value) + assert "config.toml" in str(exc_info.value) + def test_session_create_rejects_over_budget_custom_instructions( self, monkeypatch: pytest.MonkeyPatch ): diff --git a/tests/utils/test_config_helpers.py b/tests/utils/test_config_helpers.py index ece01f78..05265e5f 100644 --- a/tests/utils/test_config_helpers.py +++ b/tests/utils/test_config_helpers.py @@ -8,7 +8,15 @@ from src.utils.config_helpers import get_configuration class TestGetConfiguration: - def test_preserves_workspace_custom_instructions(self) -> None: + def test_preserves_workspace_custom_instructions( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr( + settings.DERIVER, + "MAX_CUSTOM_INSTRUCTIONS_TOKENS", + 100, + ) + workspace = models.Workspace( name="workspace-1", configuration={ @@ -24,7 +32,15 @@ class TestGetConfiguration: assert config.reasoning.enabled is True assert config.reasoning.custom_instructions == "Focus on durable preferences." - def test_message_custom_instructions_override_session_and_workspace(self) -> None: + def test_message_custom_instructions_override_session_and_workspace( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr( + settings.DERIVER, + "MAX_CUSTOM_INSTRUCTIONS_TOKENS", + 100, + ) + workspace = models.Workspace( name="workspace-1", configuration={"reasoning": {"custom_instructions": "workspace scope"}}