diff --git a/.env.template b/.env.template index 123af642..71f1b969 100644 --- a/.env.template +++ b/.env.template @@ -112,7 +112,8 @@ LLM_OPENAI_API_KEY=your-api-key-here # DERIVER_DEDUPLICATE=true # DERIVER_MODEL_CONFIG__MAX_OUTPUT_TOKENS=4096 # DERIVER_LOG_OBSERVATIONS=false -# DERIVER_MAX_INPUT_TOKENS=23000 +# DERIVER_MAX_INPUT_TOKENS=25000 +# DERIVER_MAX_CUSTOM_INSTRUCTIONS_TOKENS=2000 # DERIVER_WORKING_REPRESENTATION_MAX_OBSERVATIONS=100 # DERIVER_REPRESENTATION_BATCH_MAX_TOKENS=1024 # DERIVER_FLUSH_ENABLED=false # Bypass batch token threshold, process work immediately diff --git a/README.md b/README.md index 81af98ad..9a7cf541 100644 --- a/README.md +++ b/README.md @@ -404,7 +404,7 @@ Then modify the values as needed. The TOML file is organized into sections: - `[auth]` - Authentication configuration - `[cache]` - Redis cache configuration - `[llm]` - LLM provider API keys and general settings -- `[deriver]` - Background worker settings and representation configuration. `reasoning.custom_instructions` is active for deriver. If non-blank custom instructions are provided and `MAX_CUSTOM_INSTRUCTIONS_TOKENS` is unset, validation fails. +- `[deriver]` - Background worker settings and representation configuration - `[peer_card]` - Peer card generation settings - `[dialectic]` - Dialectic API configuration with per-level reasoning settings - `[summary]` - Session summarization settings @@ -428,7 +428,6 @@ Examples: - `DB_CONNECTION_URI` - Database connection string - `AUTH_JWT_SECRET` - JWT secret key - `DERIVER_MODEL_CONFIG__TRANSPORT` - Transport for the background deriver -- `DERIVER_MAX_CUSTOM_INSTRUCTIONS_TOKENS` - Explicit prompt budget cap for deriver custom instructions (maximum supported value: `500`) - `SUMMARY_MODEL_CONFIG__MODEL` - Summary model override - `DIALECTIC_LEVELS__low__MODEL_CONFIG__MODEL` - Model for low reasoning level - `LOG_LEVEL` - Application log level @@ -450,8 +449,6 @@ This allows you to: - Override specific values with environment variables in production - Use `.env` files for local development without modifying config.toml -Non-blank `reasoning.custom_instructions` values fail validation if `DERIVER_MAX_CUSTOM_INSTRUCTIONS_TOKENS` / `[deriver].MAX_CUSTOM_INSTRUCTIONS_TOKENS` is unset, and they are rejected if they exceed the configured limit. - ### Example If you have this in `config.toml`: diff --git a/config.toml.example b/config.toml.example index d63b0de3..ba35d02a 100644 --- a/config.toml.example +++ b/config.toml.example @@ -85,8 +85,8 @@ STALE_SESSION_TIMEOUT_MINUTES = 5 # QUEUE_ERROR_RETENTION_SECONDS = 2592000 # 30 days DEDUPLICATE = true LOG_OBSERVATIONS = false -MAX_INPUT_TOKENS = 23000 -MAX_CUSTOM_INSTRUCTIONS_TOKENS = 500 # Required for non-blank reasoning.custom_instructions; max supported value is 500 +MAX_INPUT_TOKENS = 25000 +MAX_CUSTOM_INSTRUCTIONS_TOKENS = 2000 # Required for non-blank reasoning.custom_instructions; max supported value is 2000 WORKING_REPRESENTATION_MAX_OBSERVATIONS = 100 REPRESENTATION_BATCH_MAX_TOKENS = 1024 FLUSH_ENABLED = false # Bypass batch token threshold, process work immediately diff --git a/docs/v3/contributing/configuration.mdx b/docs/v3/contributing/configuration.mdx index 7eb26dee..02916ab6 100644 --- a/docs/v3/contributing/configuration.mdx +++ b/docs/v3/contributing/configuration.mdx @@ -30,8 +30,6 @@ All config values map to environment variables: - `{KEY}` for app-level settings (e.g., `LOG_LEVEL` → `[app].LOG_LEVEL`) - Use `__` inside `{KEY}` for nested settings (e.g., `DIALECTIC_LEVELS__minimal__MODEL_CONFIG__TRANSPORT`, `DERIVER_MODEL_CONFIG__OVERRIDES__BASE_URL`) -Deriver uses `reasoning.custom_instructions` from workspace, session, and message configuration. If non-blank custom instructions are provided and `DERIVER_MAX_CUSTOM_INSTRUCTIONS_TOKENS` / `[deriver].MAX_CUSTOM_INSTRUCTIONS_TOKENS` is unset, validation fails. If the limit is set, non-blank values that exceed it are rejected during validation. - ## LLM Configuration The [Self-Hosting Guide](./self-hosting#llm-setup) covers the basic setup: either the built-in OpenAI defaults or one OpenAI-compatible endpoint/model for all features. This section covers recommended model tiers, using multiple providers, and per-feature tuning. @@ -532,7 +530,6 @@ DEFAULT_TTL_SECONDS = 300 [deriver] ENABLED = true WORKERS = 1 -MAX_CUSTOM_INSTRUCTIONS_TOKENS = 500 [deriver.model_config] transport = "openai" diff --git a/src/config.py b/src/config.py index 39e38ac0..eb16ec55 100644 --- a/src/config.py +++ b/src/config.py @@ -731,9 +731,10 @@ class DeriverSettings(HonchoSettings): LOG_OBSERVATIONS: bool = False - MAX_INPUT_TOKENS: Annotated[int, Field(default=23000, gt=0, le=23000)] = 23000 + MAX_INPUT_TOKENS: Annotated[int, Field(default=25000, gt=0, le=25000)] = 25000 + # Optional so deployments must opt in to accepting non-blank custom instructions. MAX_CUSTOM_INSTRUCTIONS_TOKENS: Annotated[ - int | None, Field(default=None, gt=0, le=500) + int | None, Field(default=None, gt=0, le=2000) ] = None # Maximum number of observations to return in working representation diff --git a/src/deriver/prompts.py b/src/deriver/prompts.py index b19c9fa5..4be35b95 100644 --- a/src/deriver/prompts.py +++ b/src/deriver/prompts.py @@ -82,30 +82,24 @@ Messages to analyze: @cache def estimate_minimal_deriver_prompt_tokens() -> int: - """Estimate base prompt tokens (cached).""" - return estimate_deriver_prompt_tokens(None) + """Estimate the static minimal deriver prompt without custom instructions.""" + prompt = minimal_deriver_prompt( + peer_id="", + messages="", + custom_instructions=None, + ) + return estimate_tokens(prompt) def estimate_deriver_prompt_tokens(custom_instructions: str | None) -> int: - """Estimate deriver prompt tokens, including optional custom instructions.""" + """Estimate minimal deriver prompt tokens, including custom instructions if present.""" normalized_custom_instructions = _normalized_custom_instructions(custom_instructions) if normalized_custom_instructions is None: - try: - prompt = minimal_deriver_prompt( - peer_id="", - messages="", - custom_instructions=None, - ) - return estimate_tokens(prompt) - except Exception: - return 300 + return estimate_minimal_deriver_prompt_tokens() - try: - prompt = minimal_deriver_prompt( - peer_id="", - messages="", - custom_instructions=normalized_custom_instructions, - ) - return estimate_tokens(prompt) - except Exception: - return 300 + prompt = minimal_deriver_prompt( + peer_id="", + messages="", + custom_instructions=normalized_custom_instructions, + ) + return estimate_tokens(prompt) diff --git a/tests/deriver/test_prompts.py b/tests/deriver/test_prompts.py index 4a1f0928..c0f7db95 100644 --- a/tests/deriver/test_prompts.py +++ b/tests/deriver/test_prompts.py @@ -1,5 +1,10 @@ +from unittest.mock import patch + +import pytest + from src.deriver.prompts import ( estimate_deriver_prompt_tokens, + estimate_minimal_deriver_prompt_tokens, minimal_deriver_prompt, ) @@ -26,9 +31,23 @@ def test_minimal_deriver_prompt_omits_custom_instructions_when_absent() -> None: def test_estimate_deriver_prompt_tokens_increases_with_custom_instructions() -> None: - base_tokens = estimate_deriver_prompt_tokens(None) + base_tokens = estimate_minimal_deriver_prompt_tokens() custom_tokens = estimate_deriver_prompt_tokens( "Prefer explicit facts with absolute dates and keep the subject precise." ) assert custom_tokens > base_tokens + + +def test_estimate_deriver_prompt_tokens_propagates_token_estimation_errors() -> None: + estimate_minimal_deriver_prompt_tokens.cache_clear() + + with patch( + "src.deriver.prompts.estimate_tokens", + side_effect=RuntimeError("tokenizer unavailable"), + ): + with pytest.raises(RuntimeError, match="tokenizer unavailable"): + estimate_deriver_prompt_tokens(None) + + with pytest.raises(RuntimeError, match="tokenizer unavailable"): + estimate_deriver_prompt_tokens("Prefer concrete facts.") diff --git a/tests/test_config.py b/tests/test_config.py index e6d2a46f..6b38ba5b 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -5,7 +5,7 @@ from src.config import ConfiguredModelSettings, DeriverSettings def _make_deriver_settings( *, - MAX_INPUT_TOKENS: int = 23000, + MAX_INPUT_TOKENS: int = 25000, MAX_CUSTOM_INSTRUCTIONS_TOKENS: int | None = None, REPRESENTATION_BATCH_MAX_TOKENS: int = 1024, ) -> DeriverSettings: @@ -31,9 +31,16 @@ def test_effective_custom_instructions_tokens_requires_explicit_limit() -> None: def test_effective_custom_instructions_tokens_uses_explicit_limit() -> None: - settings = _make_deriver_settings(MAX_CUSTOM_INSTRUCTIONS_TOKENS=500) + settings = _make_deriver_settings(MAX_CUSTOM_INSTRUCTIONS_TOKENS=2000) - assert settings.effective_max_custom_instructions_tokens == 500 + assert settings.effective_max_custom_instructions_tokens == 2000 + + +def test_deriver_defaults_allow_larger_custom_instruction_budget() -> None: + settings = _make_deriver_settings(MAX_CUSTOM_INSTRUCTIONS_TOKENS=2000) + + assert settings.MAX_INPUT_TOKENS == 25000 + assert settings.effective_max_custom_instructions_tokens == 2000 def test_custom_instructions_tokens_cannot_exceed_input_budget() -> None: @@ -49,5 +56,5 @@ def test_custom_instructions_tokens_cannot_exceed_input_budget() -> None: def test_custom_instructions_tokens_cannot_exceed_supported_cap() -> None: - with pytest.raises(ValueError, match="less than or equal to 500"): - _make_deriver_settings(MAX_CUSTOM_INSTRUCTIONS_TOKENS=501) + with pytest.raises(ValueError, match="less than or equal to 2000"): + _make_deriver_settings(MAX_CUSTOM_INSTRUCTIONS_TOKENS=2001)