From 92c0a324eafe6702f6f1a3b850d7fadd1a3b0d01 Mon Sep 17 00:00:00 2001 From: adavyas Date: Thu, 23 Apr 2026 11:18:57 -0700 Subject: [PATCH] feat: wire deriver custom instructions on main --- README.md | 5 +- config.toml.example | 1 + docs/v3/contributing/configuration.mdx | 3 + src/config.py | 20 ++++++ src/deriver/deriver.py | 12 +++- src/deriver/prompts.py | 34 +++++++++++ src/schemas/configuration.py | 35 ++++++++++- src/utils/config_helpers.py | 5 +- tests/deriver/test_deriver_processing.py | 63 +++++++++++++++++++ tests/deriver/test_prompts.py | 34 +++++++++++ tests/test_config.py | 45 ++++++++++++++ tests/test_schema_validations.py | 64 +++++++++++++++++++ tests/utils/test_config_helpers.py | 78 ++++++++++++++++++++++++ 13 files changed, 392 insertions(+), 7 deletions(-) create mode 100644 tests/deriver/test_prompts.py create mode 100644 tests/test_config.py create mode 100644 tests/utils/test_config_helpers.py diff --git a/README.md b/README.md index 9a7cf541..ffb419fa 100644 --- a/README.md +++ b/README.md @@ -404,7 +404,7 @@ Then modify the values as needed. The TOML file is organized into sections: - `[auth]` - Authentication configuration - `[cache]` - Redis cache configuration - `[llm]` - LLM provider API keys and general settings -- `[deriver]` - Background worker settings and representation configuration +- `[deriver]` - Background worker settings and representation configuration. `reasoning.custom_instructions` is active for deriver. If non-blank custom instructions are provided and `MAX_CUSTOM_INSTRUCTIONS_TOKENS` is unset, validation fails. - `[peer_card]` - Peer card generation settings - `[dialectic]` - Dialectic API configuration with per-level reasoning settings - `[summary]` - Session summarization settings @@ -428,6 +428,7 @@ Examples: - `DB_CONNECTION_URI` - Database connection string - `AUTH_JWT_SECRET` - JWT secret key - `DERIVER_MODEL_CONFIG__TRANSPORT` - Transport for the background deriver +- `DERIVER_MAX_CUSTOM_INSTRUCTIONS_TOKENS` - Explicit prompt budget cap for deriver custom instructions - `SUMMARY_MODEL_CONFIG__MODEL` - Summary model override - `DIALECTIC_LEVELS__low__MODEL_CONFIG__MODEL` - Model for low reasoning level - `LOG_LEVEL` - Application log level @@ -449,6 +450,8 @@ This allows you to: - Override specific values with environment variables in production - Use `.env` files for local development without modifying config.toml +Non-blank `reasoning.custom_instructions` values fail validation if `DERIVER_MAX_CUSTOM_INSTRUCTIONS_TOKENS` / `[deriver].MAX_CUSTOM_INSTRUCTIONS_TOKENS` is unset, and they are rejected if they exceed the configured limit. + ### Example If you have this in `config.toml`: diff --git a/config.toml.example b/config.toml.example index 236f3402..affc0392 100644 --- a/config.toml.example +++ b/config.toml.example @@ -86,6 +86,7 @@ STALE_SESSION_TIMEOUT_MINUTES = 5 DEDUPLICATE = true LOG_OBSERVATIONS = false MAX_INPUT_TOKENS = 23000 +MAX_CUSTOM_INSTRUCTIONS_TOKENS = 1024 # Required for non-blank reasoning.custom_instructions; over-limit values fail validation WORKING_REPRESENTATION_MAX_OBSERVATIONS = 100 REPRESENTATION_BATCH_MAX_TOKENS = 1024 FLUSH_ENABLED = false # Bypass batch token threshold, process work immediately diff --git a/docs/v3/contributing/configuration.mdx b/docs/v3/contributing/configuration.mdx index 02916ab6..3fc72eb9 100644 --- a/docs/v3/contributing/configuration.mdx +++ b/docs/v3/contributing/configuration.mdx @@ -30,6 +30,8 @@ All config values map to environment variables: - `{KEY}` for app-level settings (e.g., `LOG_LEVEL` → `[app].LOG_LEVEL`) - Use `__` inside `{KEY}` for nested settings (e.g., `DIALECTIC_LEVELS__minimal__MODEL_CONFIG__TRANSPORT`, `DERIVER_MODEL_CONFIG__OVERRIDES__BASE_URL`) +Deriver uses `reasoning.custom_instructions` from workspace, session, and message configuration. If non-blank custom instructions are provided and `DERIVER_MAX_CUSTOM_INSTRUCTIONS_TOKENS` / `[deriver].MAX_CUSTOM_INSTRUCTIONS_TOKENS` is unset, validation fails. If the limit is set, non-blank values that exceed it are rejected during validation. + ## LLM Configuration The [Self-Hosting Guide](./self-hosting#llm-setup) covers the basic setup: either the built-in OpenAI defaults or one OpenAI-compatible endpoint/model for all features. This section covers recommended model tiers, using multiple providers, and per-feature tuning. @@ -530,6 +532,7 @@ DEFAULT_TTL_SECONDS = 300 [deriver] ENABLED = true WORKERS = 1 +MAX_CUSTOM_INSTRUCTIONS_TOKENS = 1024 [deriver.model_config] transport = "openai" diff --git a/src/config.py b/src/config.py index cae0c5de..fa647031 100644 --- a/src/config.py +++ b/src/config.py @@ -732,6 +732,9 @@ class DeriverSettings(HonchoSettings): LOG_OBSERVATIONS: bool = False MAX_INPUT_TOKENS: Annotated[int, Field(default=23000, gt=0, le=23000)] = 23000 + MAX_CUSTOM_INSTRUCTIONS_TOKENS: Annotated[ + int | None, Field(default=None, gt=0, le=23000) + ] = None # Maximum number of observations to return in working representation # This is applied to both explicit and deductive observations @@ -764,8 +767,25 @@ class DeriverSettings(HonchoSettings): raise ValueError( f"REPRESENTATION_BATCH_MAX_TOKENS ({self.REPRESENTATION_BATCH_MAX_TOKENS}) cannot exceed max deriver input tokens ({self.MAX_INPUT_TOKENS})" ) + if ( + self.MAX_CUSTOM_INSTRUCTIONS_TOKENS is not None + and self.MAX_CUSTOM_INSTRUCTIONS_TOKENS > self.MAX_INPUT_TOKENS + ): + raise ValueError( + f"MAX_CUSTOM_INSTRUCTIONS_TOKENS ({self.MAX_CUSTOM_INSTRUCTIONS_TOKENS}) " + + f"cannot exceed max deriver input tokens ({self.MAX_INPUT_TOKENS})" + ) return self + @property + def effective_max_custom_instructions_tokens(self) -> int: + if self.MAX_CUSTOM_INSTRUCTIONS_TOKENS is None: + raise ValueError( + "DERIVER.MAX_CUSTOM_INSTRUCTIONS_TOKENS is not set; set " + + "[deriver].MAX_CUSTOM_INSTRUCTIONS_TOKENS in config.toml" + ) + return self.MAX_CUSTOM_INSTRUCTIONS_TOKENS + class PeerCardSettings(HonchoSettings): model_config = SettingsConfigDict(env_prefix="PEER_CARD_", extra="ignore") # pyright: ignore diff --git a/src/deriver/deriver.py b/src/deriver/deriver.py index 1fcc5ad2..1bb720dc 100644 --- a/src/deriver/deriver.py +++ b/src/deriver/deriver.py @@ -22,7 +22,7 @@ from src.utils.formatting import format_new_turn_with_timestamp from src.utils.representation import PromptRepresentation, Representation from src.utils.tokens import track_deriver_input_tokens -from .prompts import estimate_minimal_deriver_prompt_tokens, minimal_deriver_prompt +from .prompts import estimate_deriver_prompt_tokens, minimal_deriver_prompt logger = logging.getLogger(__name__) @@ -78,6 +78,8 @@ async def process_representation_tasks_batch( if message_level_configuration.reasoning.enabled is False: return + custom_instructions = message_level_configuration.reasoning.custom_instructions + accumulate_metric( f"minimal_deriver_{latest_message.id}_{observed}", "starting_message_id", @@ -98,7 +100,7 @@ async def process_representation_tasks_batch( ) # Track token usage - count only tokens from messages being processed - prompt_tokens = estimate_minimal_deriver_prompt_tokens() + prompt_tokens = estimate_deriver_prompt_tokens(custom_instructions) queue_item_message_ids_set = set(queue_item_message_ids) messages_tokens = sum( msg.token_count for msg in messages if msg.id in queue_item_message_ids_set @@ -112,7 +114,11 @@ async def process_representation_tasks_batch( ) # Build prompt - prompt = minimal_deriver_prompt(peer_id=observed, messages=formatted_messages) + prompt = minimal_deriver_prompt( + peer_id=observed, + messages=formatted_messages, + custom_instructions=custom_instructions, + ) context_prep_duration = (time.perf_counter() - overall_start) * 1000 accumulate_metric( diff --git a/src/deriver/prompts.py b/src/deriver/prompts.py index 98ab501c..69cbac82 100644 --- a/src/deriver/prompts.py +++ b/src/deriver/prompts.py @@ -11,9 +11,23 @@ from inspect import cleandoc as c from src.utils.tokens import estimate_tokens +def _custom_instructions_section(custom_instructions: str | None) -> str: + """Render optional custom instructions for the deriver prompt.""" + if not custom_instructions or not custom_instructions.strip(): + return "" + + return c( + f""" + CUSTOM INSTRUCTIONS: + {custom_instructions.strip()} + """ + ) + + def minimal_deriver_prompt( peer_id: str, messages: str, + custom_instructions: str | None = None, ) -> str: """ Generate minimal prompt for fast observation extraction. @@ -25,6 +39,7 @@ def minimal_deriver_prompt( Returns: Formatted prompt string for observation extraction. """ + custom_instructions_section = _custom_instructions_section(custom_instructions) return c( f""" Analyze messages from {peer_id} to extract **explicit atomic facts** about them. @@ -45,6 +60,8 @@ EXAMPLES: - EXPLICIT: "I took my dog for a walk in NYC" → "{peer_id} has a dog", "{peer_id} lives in NYC" - EXPLICIT: "{peer_id} attended college" + general knowledge → "{peer_id} completed high school or equivalent" +{custom_instructions_section} + Messages to analyze: {messages} @@ -56,10 +73,27 @@ Messages to analyze: @cache def estimate_minimal_deriver_prompt_tokens() -> int: """Estimate base prompt tokens (cached).""" + return estimate_deriver_prompt_tokens(None) + + +def estimate_deriver_prompt_tokens(custom_instructions: str | None) -> int: + """Estimate deriver prompt tokens, including optional custom instructions.""" + if not custom_instructions or not custom_instructions.strip(): + try: + prompt = minimal_deriver_prompt( + peer_id="", + messages="", + custom_instructions=None, + ) + return estimate_tokens(prompt) + except Exception: + return 300 + try: prompt = minimal_deriver_prompt( peer_id="", messages="", + custom_instructions=custom_instructions, ) return estimate_tokens(prompt) except Exception: diff --git a/src/schemas/configuration.py b/src/schemas/configuration.py index 29180247..27e2da49 100644 --- a/src/schemas/configuration.py +++ b/src/schemas/configuration.py @@ -7,7 +7,10 @@ the fully-resolved variants used at runtime. from enum import Enum from typing import Any, Self, cast -from pydantic import BaseModel, ConfigDict, Field, model_validator +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + +from src.config import settings +from src.utils.tokens import estimate_tokens class DreamType(str, Enum): @@ -23,9 +26,14 @@ class ReasoningConfiguration(BaseModel): ) custom_instructions: str | None = Field( default=None, - description="TODO: currently unused. Custom instructions to use for the reasoning system on this workspace/session/message.", + description="Optional custom instructions for the reasoning system on this workspace/session/message. Non-blank values require an explicit deriver custom-instruction token cap and are rejected if they exceed it.", ) + @field_validator("custom_instructions") + @classmethod + def validate_custom_instructions(cls, value: str | None) -> str | None: + return _validate_custom_instructions_budget(value) + class PeerCardConfiguration(BaseModel): use: bool | None = Field( @@ -75,6 +83,23 @@ class DreamConfiguration(BaseModel): ) +def _validate_custom_instructions_budget( + custom_instructions: str | None, +) -> str | None: + if custom_instructions is None: + return None + if not custom_instructions.strip(): + return custom_instructions + + max_tokens = settings.DERIVER.effective_max_custom_instructions_tokens + if estimate_tokens(custom_instructions) > max_tokens: + raise ValueError( + f"custom_instructions exceeds DERIVER.MAX_CUSTOM_INSTRUCTIONS_TOKENS ({max_tokens} tokens)" + ) + + return custom_instructions + + class WorkspaceConfiguration(BaseModel): """ The set of options that can be in a workspace DB-level configuration dictionary. @@ -127,6 +152,12 @@ class MessageConfiguration(BaseModel): class ResolvedReasoningConfiguration(BaseModel): enabled: bool + custom_instructions: str | None = None + + @field_validator("custom_instructions") + @classmethod + def validate_custom_instructions(cls, value: str | None) -> str | None: + return _validate_custom_instructions_budget(value) class ResolvedPeerCardConfiguration(BaseModel): diff --git a/src/utils/config_helpers.py b/src/utils/config_helpers.py index 74099a62..b4afa078 100644 --- a/src/utils/config_helpers.py +++ b/src/utils/config_helpers.py @@ -101,7 +101,10 @@ def get_configuration( """ # Start with defaults config_dict: dict[str, Any] = { - "reasoning": {"enabled": settings.DERIVER.ENABLED}, + "reasoning": { + "enabled": settings.DERIVER.ENABLED, + "custom_instructions": None, + }, "peer_card": { "use": settings.PEER_CARD.ENABLED, "create": settings.PEER_CARD.ENABLED, diff --git a/tests/deriver/test_deriver_processing.py b/tests/deriver/test_deriver_processing.py index 5822f4d5..76e7f90c 100644 --- a/tests/deriver/test_deriver_processing.py +++ b/tests/deriver/test_deriver_processing.py @@ -70,6 +70,69 @@ class TestDeriverProcessing: assert kwargs["model_config"].stop_sequences == expected_config.stop_sequences assert "llm_settings" not in kwargs + async def test_process_representation_tasks_batch_passes_custom_instructions_into_prompt( + self, + ) -> None: + message = Mock( + id=1, + public_id="msg_1", + session_name="session-1", + workspace_name="workspace-1", + peer_name="alice", + content="hello", + token_count=5, + created_at=datetime.now(timezone.utc), + ) + configuration = Mock() + configuration.reasoning.enabled = True + configuration.reasoning.custom_instructions = ( + "Prefer explicit facts with dates." + ) + + mock_response = HonchoLLMCallResponse( + content=PromptRepresentation(explicit=[]), + input_tokens=10, + output_tokens=5, + finish_reasons=["STOP"], + ) + + with ( + patch( + "src.deriver.deriver.estimate_deriver_prompt_tokens", + return_value=123, + ) as mock_estimate_prompt_tokens, + patch( + "src.deriver.deriver.minimal_deriver_prompt", + return_value="prompt", + ) as mock_prompt, + patch( + "src.deriver.deriver.honcho_llm_call", + new_callable=AsyncMock, + return_value=mock_response, + ) as mock_llm_call, + ): + await process_representation_tasks_batch( + messages=[message], + message_level_configuration=configuration, + observers=["bob"], + observed="alice", + queue_item_message_ids=[1], + ) + + mock_estimate_prompt_tokens.assert_called_once_with( + "Prefer explicit facts with dates." + ) + mock_prompt.assert_called_once() + assert ( + mock_prompt.call_args.kwargs["custom_instructions"] + == "Prefer explicit facts with dates." + ) + + await_args = mock_llm_call.await_args + if await_args is None: + raise AssertionError("Expected deriver LLM call") + assert await_args.kwargs["prompt"] == "prompt" + async def test_work_unit_key_generation( self, sample_session_with_peers: tuple[models.Session, list[models.Peer]], diff --git a/tests/deriver/test_prompts.py b/tests/deriver/test_prompts.py new file mode 100644 index 00000000..4a1f0928 --- /dev/null +++ b/tests/deriver/test_prompts.py @@ -0,0 +1,34 @@ +from src.deriver.prompts import ( + estimate_deriver_prompt_tokens, + minimal_deriver_prompt, +) + + +def test_minimal_deriver_prompt_includes_custom_instructions_when_present() -> None: + prompt = minimal_deriver_prompt( + peer_id="alice", + messages="alice: hello", + custom_instructions="Prefer concrete timeline facts.", + ) + + assert "CUSTOM INSTRUCTIONS:" in prompt + assert "Prefer concrete timeline facts." in prompt + + +def test_minimal_deriver_prompt_omits_custom_instructions_when_absent() -> None: + prompt = minimal_deriver_prompt( + peer_id="alice", + messages="alice: hello", + custom_instructions=None, + ) + + assert "CUSTOM INSTRUCTIONS:" not in prompt + + +def test_estimate_deriver_prompt_tokens_increases_with_custom_instructions() -> None: + base_tokens = estimate_deriver_prompt_tokens(None) + custom_tokens = estimate_deriver_prompt_tokens( + "Prefer explicit facts with absolute dates and keep the subject precise." + ) + + assert custom_tokens > base_tokens diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 00000000..cc3b9258 --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,45 @@ +import pytest + +from src.config import ConfiguredModelSettings, DeriverSettings + + +def _make_deriver_settings( + *, + MAX_INPUT_TOKENS: int = 23000, + MAX_CUSTOM_INSTRUCTIONS_TOKENS: int | None = None, +) -> DeriverSettings: + return DeriverSettings( + MODEL_CONFIG=ConfiguredModelSettings( + model="gpt-5.4-mini", + transport="openai", + ), + MAX_INPUT_TOKENS=MAX_INPUT_TOKENS, + MAX_CUSTOM_INSTRUCTIONS_TOKENS=MAX_CUSTOM_INSTRUCTIONS_TOKENS, + ) + + +def test_effective_custom_instructions_tokens_requires_explicit_limit() -> None: + settings = _make_deriver_settings() + + with pytest.raises( + ValueError, + match=r"set \[deriver\]\.MAX_CUSTOM_INSTRUCTIONS_TOKENS in config\.toml", + ): + _ = settings.effective_max_custom_instructions_tokens + + +def test_effective_custom_instructions_tokens_uses_explicit_limit() -> None: + settings = _make_deriver_settings(MAX_CUSTOM_INSTRUCTIONS_TOKENS=2048) + + assert settings.effective_max_custom_instructions_tokens == 2048 + + +def test_custom_instructions_tokens_cannot_exceed_input_budget() -> None: + with pytest.raises( + ValueError, + match=r"MAX_CUSTOM_INSTRUCTIONS_TOKENS.*cannot exceed max deriver input tokens", + ): + _make_deriver_settings( + MAX_INPUT_TOKENS=1024, + MAX_CUSTOM_INSTRUCTIONS_TOKENS=2048, + ) diff --git a/tests/test_schema_validations.py b/tests/test_schema_validations.py index 281522a9..58016ad9 100644 --- a/tests/test_schema_validations.py +++ b/tests/test_schema_validations.py @@ -3,11 +3,13 @@ from typing import Any import pytest from pydantic import ValidationError +from src.config import settings from src.schemas import ( DocumentCreate, DocumentMetadata, MessageCreate, PeerCreate, + ReasoningConfiguration, ResolvedConfiguration, SessionCreate, WorkspaceCreate, @@ -203,3 +205,65 @@ class TestResolvedConfigurationMigration: ResolvedConfiguration.model_validate(payload) assert any(e["loc"] == ("reasoning",) for e in exc_info.value.errors()) + + +class TestReasoningCustomInstructionsValidation: + def test_reasoning_configuration_rejects_oversized_custom_instructions( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr( + settings.DERIVER, "MAX_CUSTOM_INSTRUCTIONS_TOKENS", 1, raising=False + ) + + with pytest.raises(ValidationError) as exc_info: + ReasoningConfiguration( + custom_instructions="repeat repeat repeat repeat repeat" + ) + + assert any( + error["loc"] == ("custom_instructions",) + for error in exc_info.value.errors() + ) + + def test_oversized_custom_instructions_are_rejected( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr( + settings.DERIVER, "MAX_CUSTOM_INSTRUCTIONS_TOKENS", 1, raising=False + ) + + payload = { + "reasoning": { + "enabled": True, + "custom_instructions": "repeat repeat repeat repeat repeat", + }, + "peer_card": {"use": True, "create": True}, + "summary": { + "enabled": True, + "messages_per_short_summary": 20, + "messages_per_long_summary": 60, + }, + "dream": {"enabled": False}, + } + + with pytest.raises(ValidationError) as exc_info: + ResolvedConfiguration.model_validate(payload) + + assert any( + error["loc"] == ("reasoning", "custom_instructions") + for error in exc_info.value.errors() + ) + + @pytest.mark.parametrize("custom_instructions", ["", " \n\t "]) + def test_blank_custom_instructions_do_not_require_token_cap( + self, monkeypatch: pytest.MonkeyPatch, custom_instructions: str + ) -> None: + monkeypatch.setattr( + settings.DERIVER, "MAX_CUSTOM_INSTRUCTIONS_TOKENS", None, raising=False + ) + + configuration = ReasoningConfiguration( + custom_instructions=custom_instructions + ) + + assert configuration.custom_instructions == custom_instructions diff --git a/tests/utils/test_config_helpers.py b/tests/utils/test_config_helpers.py new file mode 100644 index 00000000..7ad602f3 --- /dev/null +++ b/tests/utils/test_config_helpers.py @@ -0,0 +1,78 @@ +from typing import Any + +import pytest + +from src import models +from src.config import settings +from src.schemas import MessageConfiguration, ReasoningConfiguration +from src.utils.config_helpers import get_configuration + + +def _workspace(configuration: dict[str, Any]) -> models.Workspace: + return models.Workspace(name="workspace", configuration=configuration) + + +def _session(configuration: dict[str, Any]) -> models.Session: + return models.Session( + name="session", + workspace_name="workspace", + configuration=configuration, + ) + + +def test_preserves_workspace_custom_instructions( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + settings.DERIVER, "MAX_CUSTOM_INSTRUCTIONS_TOKENS", 100, raising=False + ) + + workspace = _workspace( + { + "reasoning": { + "custom_instructions": "Use the workspace-specific guidance.", + } + } + ) + + configuration = get_configuration(None, None, workspace) + + assert ( + configuration.reasoning.custom_instructions + == "Use the workspace-specific guidance." + ) + + +def test_message_custom_instructions_override_session_and_workspace( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + settings.DERIVER, "MAX_CUSTOM_INSTRUCTIONS_TOKENS", 100, raising=False + ) + + workspace = _workspace( + { + "reasoning": { + "custom_instructions": "Use the workspace-specific guidance.", + } + } + ) + session = _session( + { + "reasoning": { + "custom_instructions": "Use the session-specific guidance.", + } + } + ) + message = MessageConfiguration( + reasoning=ReasoningConfiguration( + custom_instructions="Use the message-specific guidance.", + ), + ) + + configuration = get_configuration(message, session, workspace) + + assert ( + configuration.reasoning.custom_instructions + == "Use the message-specific guidance." + )