From 43d962d1609076f45e5b5edc9e937c76ada6b95b Mon Sep 17 00:00:00 2001 From: Eugene Eisenstein Date: Thu, 9 Jul 2026 10:42:52 -0400 Subject: [PATCH] rename to REPRESENTATION_BATCH_TARGET_INPUT_TOKENS --- .env.template | 2 +- config.toml.example | 2 +- docs/v3/contributing/configuration.mdx | 2 +- docs/v3/contributing/troubleshooting.mdx | 2 +- src/config.py | 10 +++++----- src/deriver/consumer.py | 2 +- src/deriver/deriver.py | 2 +- src/deriver/queue_manager.py | 2 +- src/telemetry/events/representation.py | 2 +- tests/bench/calculate_expected_events.py | 2 +- tests/deriver/test_queue_processing.py | 14 +++++++------- tests/test_config.py | 12 ++++++------ 12 files changed, 27 insertions(+), 27 deletions(-) diff --git a/.env.template b/.env.template index fbbd2a50..167fa857 100644 --- a/.env.template +++ b/.env.template @@ -132,7 +132,7 @@ LLM_OPENAI_API_KEY=your-api-key-here # DERIVER_MAX_CUSTOM_INSTRUCTIONS_TOKENS=2000 # DERIVER_WORKING_REPRESENTATION_MAX_OBSERVATIONS=100 # DERIVER_REPRESENTATION_BATCH_WORK_UNIT_TARGET_TOKENS=512 # Min tokens a work unit accumulates before the deriver claims it; 0 disables the gate -# DERIVER_REPRESENTATION_BATCH_LLM_MAX_TOKENS=1024 # Max context-window tokens per deriver LLM call +# DERIVER_REPRESENTATION_BATCH_TARGET_INPUT_TOKENS=1024 # Max context-window tokens per deriver LLM call # DERIVER_REPRESENTATION_BATCH_MAX_AGE_SECONDS=1800 # DERIVER_FLUSH_ENABLED=false # Bypass batch token threshold, process work immediately # DERIVER_MODEL_CONFIG__FALLBACK__MODEL= diff --git a/config.toml.example b/config.toml.example index 60dfb323..e64d90f0 100644 --- a/config.toml.example +++ b/config.toml.example @@ -110,7 +110,7 @@ MAX_INPUT_TOKENS = 25000 MAX_CUSTOM_INSTRUCTIONS_TOKENS = 2000 WORKING_REPRESENTATION_MAX_OBSERVATIONS = 100 REPRESENTATION_BATCH_WORK_UNIT_TARGET_TOKENS = 512 # Min tokens a work unit accumulates before the deriver claims it; 0 disables the gate -REPRESENTATION_BATCH_LLM_MAX_TOKENS = 1024 # Max context-window tokens per deriver LLM call +REPRESENTATION_BATCH_TARGET_INPUT_TOKENS = 1024 # Max context-window tokens per deriver LLM call REPRESENTATION_BATCH_MAX_AGE_SECONDS = 1800 FLUSH_ENABLED = false # Bypass batch token threshold, process work immediately diff --git a/docs/v3/contributing/configuration.mdx b/docs/v3/contributing/configuration.mdx index eeda7b7a..8f4fb443 100644 --- a/docs/v3/contributing/configuration.mdx +++ b/docs/v3/contributing/configuration.mdx @@ -409,7 +409,7 @@ DERIVER_DEDUPLICATE=true DERIVER_LOG_OBSERVATIONS=false DERIVER_WORKING_REPRESENTATION_MAX_OBSERVATIONS=100 DERIVER_REPRESENTATION_BATCH_WORK_UNIT_TARGET_TOKENS=512 -DERIVER_REPRESENTATION_BATCH_LLM_MAX_TOKENS=1024 +DERIVER_REPRESENTATION_BATCH_TARGET_INPUT_TOKENS=1024 DERIVER_REPRESENTATION_BATCH_MAX_AGE_SECONDS=1800 ``` diff --git a/docs/v3/contributing/troubleshooting.mdx b/docs/v3/contributing/troubleshooting.mdx index 7db76b34..d9b745a1 100644 --- a/docs/v3/contributing/troubleshooting.mdx +++ b/docs/v3/contributing/troubleshooting.mdx @@ -109,7 +109,7 @@ Messages are stored but no observations, summaries, or representations are being ```bash DERIVER_WORKERS=4 ``` -5. **Representation Batching** — By default the deriver buffers representation work until a work unit has accumulated enough tokens, set via `DERIVER_REPRESENTATION_BATCH_WORK_UNIT_TARGET_TOKENS` (`0` disables the accumulation gate). A separate setting, `DERIVER_REPRESENTATION_BATCH_LLM_MAX_TOKENS`, caps the conversation window fed to each deriver LLM call when draining a claimed work unit. Sub-threshold tails become eligible after `DERIVER_REPRESENTATION_BATCH_MAX_AGE_SECONDS` (default 1800 seconds), so quiet sessions eventually flush without disabling batching globally. Set the age to `0` for legacy behavior where sub-threshold tails wait indefinitely. See [token batching](/v3/documentation/core-concepts/reasoning#token-batching) for more details +5. **Representation Batching** — By default the deriver buffers representation work until a work unit has accumulated enough tokens, set via `DERIVER_REPRESENTATION_BATCH_WORK_UNIT_TARGET_TOKENS` (`0` disables the accumulation gate). A separate setting, `DERIVER_REPRESENTATION_BATCH_TARGET_INPUT_TOKENS`, caps the conversation window fed to each deriver LLM call when draining a claimed work unit. Sub-threshold tails become eligible after `DERIVER_REPRESENTATION_BATCH_MAX_AGE_SECONDS` (default 1800 seconds), so quiet sessions eventually flush without disabling batching globally. Set the age to `0` for legacy behavior where sub-threshold tails wait indefinitely. See [token batching](/v3/documentation/core-concepts/reasoning#token-batching) for more details ## Alternative Provider Issues diff --git a/src/config.py b/src/config.py index 977b02d7..a5cf9bf7 100644 --- a/src/config.py +++ b/src/config.py @@ -870,7 +870,7 @@ class DeriverSettings(HonchoSettings): # interleaved context) fed to a single deriver LLM call when draining a # claimed work unit. The first unprocessed message is always included, # even if it alone exceeds the cap. - REPRESENTATION_BATCH_LLM_MAX_TOKENS: Annotated[ + REPRESENTATION_BATCH_TARGET_INPUT_TOKENS: Annotated[ int, Field(default=1024, ge=128, le=16_384), ] = 1024 @@ -901,7 +901,7 @@ class DeriverSettings(HonchoSettings): The old single setting was split into REPRESENTATION_BATCH_WORK_UNIT_TARGET_TOKENS (claim gate) and - REPRESENTATION_BATCH_LLM_MAX_TOKENS (per-LLM-call window cap). + REPRESENTATION_BATCH_TARGET_INPUT_TOKENS (per-LLM-call window cap). `extra="ignore"` would otherwise silently drop the old key and revert both roles to defaults — an operator-hostile failure mode for a batching knob — so reject it loudly instead. @@ -912,15 +912,15 @@ class DeriverSettings(HonchoSettings): ) if legacy_in_data or "DERIVER_REPRESENTATION_BATCH_MAX_TOKENS" in os.environ: raise ValueError( - "REPRESENTATION_BATCH_MAX_TOKENS has been split into REPRESENTATION_BATCH_WORK_UNIT_TARGET_TOKENS (minimum tokens a work unit must accumulate before it is claimed) and REPRESENTATION_BATCH_LLM_MAX_TOKENS (token cap on the context window per deriver LLM call). Set those instead." + "REPRESENTATION_BATCH_MAX_TOKENS has been split into REPRESENTATION_BATCH_WORK_UNIT_TARGET_TOKENS (minimum tokens a work unit must accumulate before it is claimed) and REPRESENTATION_BATCH_TARGET_INPUT_TOKENS (token cap on the context window per deriver LLM call). Set those instead." ) return data # pyright: ignore[reportUnknownVariableType] @model_validator(mode="after") def validate_batch_tokens_vs_context_limit(self): - if self.REPRESENTATION_BATCH_LLM_MAX_TOKENS > self.MAX_INPUT_TOKENS: + if self.REPRESENTATION_BATCH_TARGET_INPUT_TOKENS > self.MAX_INPUT_TOKENS: raise ValueError( - f"REPRESENTATION_BATCH_LLM_MAX_TOKENS ({self.REPRESENTATION_BATCH_LLM_MAX_TOKENS}) cannot exceed max deriver input tokens ({self.MAX_INPUT_TOKENS})" + f"REPRESENTATION_BATCH_TARGET_INPUT_TOKENS ({self.REPRESENTATION_BATCH_TARGET_INPUT_TOKENS}) cannot exceed max deriver input tokens ({self.MAX_INPUT_TOKENS})" ) return self diff --git a/src/deriver/consumer.py b/src/deriver/consumer.py index 9a961df9..708d05f6 100644 --- a/src/deriver/consumer.py +++ b/src/deriver/consumer.py @@ -176,7 +176,7 @@ async def process_representation_batch( queue_item_message_ids: Message IDs from queue items hit_batch_token_cap: whether the queue batcher clamped this batch to fit was_flush_enabled: snapshot of DERIVER.FLUSH_ENABLED at fetch time - batch_max_tokens: DERIVER.REPRESENTATION_BATCH_LLM_MAX_TOKENS snapshot + batch_max_tokens: DERIVER.REPRESENTATION_BATCH_TARGET_INPUT_TOKENS snapshot """ if not messages or not messages[0]: logger.debug("process_representation_batch received no messages") diff --git a/src/deriver/deriver.py b/src/deriver/deriver.py index 93f18eb1..3e68d084 100644 --- a/src/deriver/deriver.py +++ b/src/deriver/deriver.py @@ -58,7 +58,7 @@ async def process_representation_tasks_batch( queue_item_message_ids: Message IDs from queue items being processed hit_batch_token_cap: queue batcher clamped this batch to fit was_flush_enabled: DERIVER.FLUSH_ENABLED snapshot at batch time - batch_max_tokens: DERIVER.REPRESENTATION_BATCH_LLM_MAX_TOKENS snapshot + batch_max_tokens: DERIVER.REPRESENTATION_BATCH_TARGET_INPUT_TOKENS snapshot """ if not messages: return diff --git a/src/deriver/queue_manager.py b/src/deriver/queue_manager.py index 145a6252..498372ed 100644 --- a/src/deriver/queue_manager.py +++ b/src/deriver/queue_manager.py @@ -818,7 +818,7 @@ class QueueManager: f"{task_type} tasks are not supported for get_queue_item_batch" ) - batch_max_tokens = settings.DERIVER.REPRESENTATION_BATCH_LLM_MAX_TOKENS + batch_max_tokens = settings.DERIVER.REPRESENTATION_BATCH_TARGET_INPUT_TOKENS was_flush_enabled = settings.DERIVER.FLUSH_ENABLED parsed_key = parse_work_unit_key(work_unit_key) messages_context: list[models.Message] = [] diff --git a/src/telemetry/events/representation.py b/src/telemetry/events/representation.py index dcad8a12..92db6806 100644 --- a/src/telemetry/events/representation.py +++ b/src/telemetry/events/representation.py @@ -104,7 +104,7 @@ class RepresentationCompletedEvent(BaseEvent): # Cap configuration + hit flags () batch_max_tokens: int = Field( default=0, - description="settings.DERIVER.REPRESENTATION_BATCH_LLM_MAX_TOKENS at fetch time", + description="settings.DERIVER.REPRESENTATION_BATCH_TARGET_INPUT_TOKENS at fetch time", ) max_input_tokens: int = Field( default=0, description="settings.DERIVER.MAX_INPUT_TOKENS at call time" diff --git a/tests/bench/calculate_expected_events.py b/tests/bench/calculate_expected_events.py index 45eea82b..b9696262 100644 --- a/tests/bench/calculate_expected_events.py +++ b/tests/bench/calculate_expected_events.py @@ -246,7 +246,7 @@ def calculate_question_events( # Calculate representation events # Each unique (session, observed) pair generates one representation event - # (assuming messages fit within REPRESENTATION_BATCH_LLM_MAX_TOKENS) + # (assuming messages fit within REPRESENTATION_BATCH_TARGET_INPUT_TOKENS) # When merge_sessions=True, all messages go into one session if merge_sessions: # One merged session = one representation event diff --git a/tests/deriver/test_queue_processing.py b/tests/deriver/test_queue_processing.py index e860f6cd..81dd0632 100644 --- a/tests/deriver/test_queue_processing.py +++ b/tests/deriver/test_queue_processing.py @@ -359,7 +359,7 @@ class TestQueueProcessing: peer = peers[0] # Create messages with token counts that exceed batch limit - limit = settings.DERIVER.REPRESENTATION_BATCH_LLM_MAX_TOKENS + limit = settings.DERIVER.REPRESENTATION_BATCH_TARGET_INPUT_TOKENS token_counts = [limit // 2, limit // 2, limit // 2] # Create and save messages to the database first @@ -479,7 +479,7 @@ class TestQueueProcessing: session, peers = sample_session_with_peers peer = peers[0] - cap = settings.DERIVER.REPRESENTATION_BATCH_LLM_MAX_TOKENS + cap = settings.DERIVER.REPRESENTATION_BATCH_TARGET_INPUT_TOKENS # M1 + M2 sum to exactly the cap; M3 pushes over it. After SQL, # messages_context = [M1, M2]; the cap is genuinely binding *on the @@ -581,7 +581,7 @@ class TestQueueProcessing: session, peers = sample_session_with_peers peer_a = peers[0] peer_b = peers[1] if len(peers) > 1 else peers[0] - cap = settings.DERIVER.REPRESENTATION_BATCH_LLM_MAX_TOKENS + cap = settings.DERIVER.REPRESENTATION_BATCH_TARGET_INPUT_TOKENS # Layout: 4 messages, ordered. # M1 (peer_a, queue, 200) @@ -745,7 +745,7 @@ class TestQueueProcessing: # Mock the token limit to 2000 for this test with patch.object( - settings.DERIVER, "REPRESENTATION_BATCH_LLM_MAX_TOKENS", 2000 + settings.DERIVER, "REPRESENTATION_BATCH_TARGET_INPUT_TOKENS", 2000 ): # Test alice's work unit alice_work_unit_key = alice_queue_items[0].work_unit_key @@ -922,7 +922,7 @@ class TestQueueProcessing: # Mock the token limit to 1500 for this test with patch.object( - settings.DERIVER, "REPRESENTATION_BATCH_LLM_MAX_TOKENS", 1500 + settings.DERIVER, "REPRESENTATION_BATCH_TARGET_INPUT_TOKENS", 1500 ): # Test alice's work unit # With per-work-unit anchoring + preceding context: @@ -1137,7 +1137,7 @@ class TestQueueProcessing: peer = peers[0] # Create messages where first message exceeds the batch limit - limit = settings.DERIVER.REPRESENTATION_BATCH_LLM_MAX_TOKENS + limit = settings.DERIVER.REPRESENTATION_BATCH_TARGET_INPUT_TOKENS token_counts = [limit + 1000, 100, 200] # First message way over limit # Create and save messages to the database first @@ -1254,7 +1254,7 @@ class TestQueueProcessing: peer = peers[0] # Create messages that test the exact boundary - limit = settings.DERIVER.REPRESENTATION_BATCH_LLM_MAX_TOKENS + limit = settings.DERIVER.REPRESENTATION_BATCH_TARGET_INPUT_TOKENS token_counts = [ limit // 2, limit // 2, diff --git a/tests/test_config.py b/tests/test_config.py index 707b6e8a..9b242e55 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -8,7 +8,7 @@ def _make_deriver_settings( MAX_INPUT_TOKENS: int = 25000, MAX_CUSTOM_INSTRUCTIONS_TOKENS: int = 2000, REPRESENTATION_BATCH_WORK_UNIT_TARGET_TOKENS: int = 512, - REPRESENTATION_BATCH_LLM_MAX_TOKENS: int = 1024, + REPRESENTATION_BATCH_TARGET_INPUT_TOKENS: int = 1024, REPRESENTATION_BATCH_MAX_AGE_SECONDS: int = 1800, ) -> DeriverSettings: return DeriverSettings( @@ -19,7 +19,7 @@ def _make_deriver_settings( MAX_INPUT_TOKENS=MAX_INPUT_TOKENS, MAX_CUSTOM_INSTRUCTIONS_TOKENS=MAX_CUSTOM_INSTRUCTIONS_TOKENS, REPRESENTATION_BATCH_WORK_UNIT_TARGET_TOKENS=REPRESENTATION_BATCH_WORK_UNIT_TARGET_TOKENS, - REPRESENTATION_BATCH_LLM_MAX_TOKENS=REPRESENTATION_BATCH_LLM_MAX_TOKENS, + REPRESENTATION_BATCH_TARGET_INPUT_TOKENS=REPRESENTATION_BATCH_TARGET_INPUT_TOKENS, REPRESENTATION_BATCH_MAX_AGE_SECONDS=REPRESENTATION_BATCH_MAX_AGE_SECONDS, ) @@ -68,18 +68,18 @@ def test_representation_batch_work_unit_target_rejects_negative_values() -> None def test_representation_batch_tokens_can_diverge() -> None: settings = _make_deriver_settings( REPRESENTATION_BATCH_WORK_UNIT_TARGET_TOKENS=4096, - REPRESENTATION_BATCH_LLM_MAX_TOKENS=1024, + REPRESENTATION_BATCH_TARGET_INPUT_TOKENS=1024, ) assert settings.REPRESENTATION_BATCH_WORK_UNIT_TARGET_TOKENS == 4096 - assert settings.REPRESENTATION_BATCH_LLM_MAX_TOKENS == 1024 + assert settings.REPRESENTATION_BATCH_TARGET_INPUT_TOKENS == 1024 -def test_representation_batch_llm_max_cannot_exceed_max_input_tokens() -> None: +def test_representation_batch_target_input_cannot_exceed_max_input_tokens() -> None: with pytest.raises(ValueError, match="cannot exceed max deriver input tokens"): _make_deriver_settings( MAX_INPUT_TOKENS=1000, - REPRESENTATION_BATCH_LLM_MAX_TOKENS=2048, + REPRESENTATION_BATCH_TARGET_INPUT_TOKENS=2048, )