RomanNum3ral
/
honcho
mirror of https://github.com/plastic-labs/honcho.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

rename to REPRESENTATION_BATCH_TARGET_INPUT_TOKENS

This commit is contained in:
Eugene Eisenstein 2026-07-09 10:42:52 -04:00
parent be26c859ad
commit 43d962d160
12 changed files with 27 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.env.template
View File

@ -132,7 +132,7 @@ LLM_OPENAI_API_KEY=your-api-key-here
# DERIVER_MAX_CUSTOM_INSTRUCTIONS_TOKENS=2000
# DERIVER_WORKING_REPRESENTATION_MAX_OBSERVATIONS=100
# DERIVER_REPRESENTATION_BATCH_WORK_UNIT_TARGET_TOKENS=512 # Min tokens a work unit accumulates before the deriver claims it; 0 disables the gate
# DERIVER_REPRESENTATION_BATCH_LLM_MAX_TOKENS=1024 # Max context-window tokens per deriver LLM call
# DERIVER_REPRESENTATION_BATCH_TARGET_INPUT_TOKENS=1024 # Max context-window tokens per deriver LLM call
# DERIVER_REPRESENTATION_BATCH_MAX_AGE_SECONDS=1800
# DERIVER_FLUSH_ENABLED=false # Bypass batch token threshold, process work immediately
# DERIVER_MODEL_CONFIG__FALLBACK__MODEL=

2
config.toml.example
View File

@ -110,7 +110,7 @@ MAX_INPUT_TOKENS = 25000
MAX_CUSTOM_INSTRUCTIONS_TOKENS = 2000
WORKING_REPRESENTATION_MAX_OBSERVATIONS = 100
REPRESENTATION_BATCH_WORK_UNIT_TARGET_TOKENS = 512 # Min tokens a work unit accumulates before the deriver claims it; 0 disables the gate
REPRESENTATION_BATCH_LLM_MAX_TOKENS = 1024 # Max context-window tokens per deriver LLM call
REPRESENTATION_BATCH_TARGET_INPUT_TOKENS = 1024 # Max context-window tokens per deriver LLM call
REPRESENTATION_BATCH_MAX_AGE_SECONDS = 1800
FLUSH_ENABLED = false # Bypass batch token threshold, process work immediately

2
docs/v3/contributing/configuration.mdx
View File

@ -409,7 +409,7 @@ DERIVER_DEDUPLICATE=true
DERIVER_LOG_OBSERVATIONS=false
DERIVER_WORKING_REPRESENTATION_MAX_OBSERVATIONS=100
DERIVER_REPRESENTATION_BATCH_WORK_UNIT_TARGET_TOKENS=512
DERIVER_REPRESENTATION_BATCH_LLM_MAX_TOKENS=1024
DERIVER_REPRESENTATION_BATCH_TARGET_INPUT_TOKENS=1024
DERIVER_REPRESENTATION_BATCH_MAX_AGE_SECONDS=1800
```

2
docs/v3/contributing/troubleshooting.mdx
View File

@ -109,7 +109,7 @@ Messages are stored but no observations, summaries, or representations are being
```bash
DERIVER_WORKERS=4
```
5. **Representation Batching** — By default the deriver buffers representation work until a work unit has accumulated enough tokens, set via `DERIVER_REPRESENTATION_BATCH_WORK_UNIT_TARGET_TOKENS` (`0` disables the accumulation gate). A separate setting, `DERIVER_REPRESENTATION_BATCH_LLM_MAX_TOKENS`, caps the conversation window fed to each deriver LLM call when draining a claimed work unit. Sub-threshold tails become eligible after `DERIVER_REPRESENTATION_BATCH_MAX_AGE_SECONDS` (default 1800 seconds), so quiet sessions eventually flush without disabling batching globally. Set the age to `0` for legacy behavior where sub-threshold tails wait indefinitely. See [token batching](/v3/documentation/core-concepts/reasoning#token-batching) for more details
5. **Representation Batching** — By default the deriver buffers representation work until a work unit has accumulated enough tokens, set via `DERIVER_REPRESENTATION_BATCH_WORK_UNIT_TARGET_TOKENS` (`0` disables the accumulation gate). A separate setting, `DERIVER_REPRESENTATION_BATCH_TARGET_INPUT_TOKENS`, caps the conversation window fed to each deriver LLM call when draining a claimed work unit. Sub-threshold tails become eligible after `DERIVER_REPRESENTATION_BATCH_MAX_AGE_SECONDS` (default 1800 seconds), so quiet sessions eventually flush without disabling batching globally. Set the age to `0` for legacy behavior where sub-threshold tails wait indefinitely. See [token batching](/v3/documentation/core-concepts/reasoning#token-batching) for more details
## Alternative Provider Issues

10
src/config.py
View File

@ -870,7 +870,7 @@ class DeriverSettings(HonchoSettings):
# interleaved context) fed to a single deriver LLM call when draining a
# claimed work unit. The first unprocessed message is always included,
# even if it alone exceeds the cap.
REPRESENTATION_BATCH_LLM_MAX_TOKENS: Annotated[
REPRESENTATION_BATCH_TARGET_INPUT_TOKENS: Annotated[
int,
Field(default=1024, ge=128, le=16_384),
] = 1024
@ -901,7 +901,7 @@ class DeriverSettings(HonchoSettings):
The old single setting was split into
REPRESENTATION_BATCH_WORK_UNIT_TARGET_TOKENS (claim gate) and
REPRESENTATION_BATCH_LLM_MAX_TOKENS (per-LLM-call window cap).
REPRESENTATION_BATCH_TARGET_INPUT_TOKENS (per-LLM-call window cap).
`extra="ignore"` would otherwise silently drop the old key and revert
both roles to defaults — an operator-hostile failure mode for a
batching knob — so reject it loudly instead.
@ -912,15 +912,15 @@ class DeriverSettings(HonchoSettings):
)
if legacy_in_data or "DERIVER_REPRESENTATION_BATCH_MAX_TOKENS" in os.environ:
raise ValueError(
"REPRESENTATION_BATCH_MAX_TOKENS has been split into REPRESENTATION_BATCH_WORK_UNIT_TARGET_TOKENS (minimum tokens a work unit must accumulate before it is claimed) and REPRESENTATION_BATCH_LLM_MAX_TOKENS (token cap on the context window per deriver LLM call). Set those instead."
"REPRESENTATION_BATCH_MAX_TOKENS has been split into REPRESENTATION_BATCH_WORK_UNIT_TARGET_TOKENS (minimum tokens a work unit must accumulate before it is claimed) and REPRESENTATION_BATCH_TARGET_INPUT_TOKENS (token cap on the context window per deriver LLM call). Set those instead."
)
return data # pyright: ignore[reportUnknownVariableType]
@model_validator(mode="after")
def validate_batch_tokens_vs_context_limit(self):
if self.REPRESENTATION_BATCH_LLM_MAX_TOKENS > self.MAX_INPUT_TOKENS:
if self.REPRESENTATION_BATCH_TARGET_INPUT_TOKENS > self.MAX_INPUT_TOKENS:
raise ValueError(
f"REPRESENTATION_BATCH_LLM_MAX_TOKENS ({self.REPRESENTATION_BATCH_LLM_MAX_TOKENS}) cannot exceed max deriver input tokens ({self.MAX_INPUT_TOKENS})"
f"REPRESENTATION_BATCH_TARGET_INPUT_TOKENS ({self.REPRESENTATION_BATCH_TARGET_INPUT_TOKENS}) cannot exceed max deriver input tokens ({self.MAX_INPUT_TOKENS})"
)
return self

2
src/deriver/consumer.py
View File

@ -176,7 +176,7 @@ async def process_representation_batch(
queue_item_message_ids: Message IDs from queue items
hit_batch_token_cap: whether the queue batcher clamped this batch to fit
was_flush_enabled: snapshot of DERIVER.FLUSH_ENABLED at fetch time
batch_max_tokens: DERIVER.REPRESENTATION_BATCH_LLM_MAX_TOKENS snapshot
batch_max_tokens: DERIVER.REPRESENTATION_BATCH_TARGET_INPUT_TOKENS snapshot
"""
if not messages or not messages[0]:
logger.debug("process_representation_batch received no messages")

2
src/deriver/deriver.py
View File

@ -58,7 +58,7 @@ async def process_representation_tasks_batch(
queue_item_message_ids: Message IDs from queue items being processed
hit_batch_token_cap: queue batcher clamped this batch to fit
was_flush_enabled: DERIVER.FLUSH_ENABLED snapshot at batch time
batch_max_tokens: DERIVER.REPRESENTATION_BATCH_LLM_MAX_TOKENS snapshot
batch_max_tokens: DERIVER.REPRESENTATION_BATCH_TARGET_INPUT_TOKENS snapshot
"""
if not messages:
return

2
src/deriver/queue_manager.py
View File

@ -818,7 +818,7 @@ class QueueManager:
f"{task_type} tasks are not supported for get_queue_item_batch"
)
batch_max_tokens = settings.DERIVER.REPRESENTATION_BATCH_LLM_MAX_TOKENS
batch_max_tokens = settings.DERIVER.REPRESENTATION_BATCH_TARGET_INPUT_TOKENS
was_flush_enabled = settings.DERIVER.FLUSH_ENABLED
parsed_key = parse_work_unit_key(work_unit_key)
messages_context: list[models.Message] = []

2
src/telemetry/events/representation.py
View File

@ -104,7 +104,7 @@ class RepresentationCompletedEvent(BaseEvent):
# Cap configuration + hit flags ()
batch_max_tokens: int = Field(
default=0,
description="settings.DERIVER.REPRESENTATION_BATCH_LLM_MAX_TOKENS at fetch time",
description="settings.DERIVER.REPRESENTATION_BATCH_TARGET_INPUT_TOKENS at fetch time",
)
max_input_tokens: int = Field(
default=0, description="settings.DERIVER.MAX_INPUT_TOKENS at call time"

2
tests/bench/calculate_expected_events.py
View File

@ -246,7 +246,7 @@ def calculate_question_events(
# Calculate representation events
# Each unique (session, observed) pair generates one representation event
# (assuming messages fit within REPRESENTATION_BATCH_LLM_MAX_TOKENS)
# (assuming messages fit within REPRESENTATION_BATCH_TARGET_INPUT_TOKENS)
# When merge_sessions=True, all messages go into one session
if merge_sessions:
# One merged session = one representation event

14
tests/deriver/test_queue_processing.py
View File

@ -359,7 +359,7 @@ class TestQueueProcessing:
peer = peers[0]
# Create messages with token counts that exceed batch limit
limit = settings.DERIVER.REPRESENTATION_BATCH_LLM_MAX_TOKENS
limit = settings.DERIVER.REPRESENTATION_BATCH_TARGET_INPUT_TOKENS
token_counts = [limit // 2, limit // 2, limit // 2]
# Create and save messages to the database first
@ -479,7 +479,7 @@ class TestQueueProcessing:
session, peers = sample_session_with_peers
peer = peers[0]
cap = settings.DERIVER.REPRESENTATION_BATCH_LLM_MAX_TOKENS
cap = settings.DERIVER.REPRESENTATION_BATCH_TARGET_INPUT_TOKENS
# M1 + M2 sum to exactly the cap; M3 pushes over it. After SQL,
# messages_context = [M1, M2]; the cap is genuinely binding *on the
@ -581,7 +581,7 @@ class TestQueueProcessing:
session, peers = sample_session_with_peers
peer_a = peers[0]
peer_b = peers[1] if len(peers) > 1 else peers[0]
cap = settings.DERIVER.REPRESENTATION_BATCH_LLM_MAX_TOKENS
cap = settings.DERIVER.REPRESENTATION_BATCH_TARGET_INPUT_TOKENS
# Layout: 4 messages, ordered.
# M1 (peer_a, queue, 200)
@ -745,7 +745,7 @@ class TestQueueProcessing:
# Mock the token limit to 2000 for this test
with patch.object(
settings.DERIVER, "REPRESENTATION_BATCH_LLM_MAX_TOKENS", 2000
settings.DERIVER, "REPRESENTATION_BATCH_TARGET_INPUT_TOKENS", 2000
):
# Test alice's work unit
alice_work_unit_key = alice_queue_items[0].work_unit_key
@ -922,7 +922,7 @@ class TestQueueProcessing:
# Mock the token limit to 1500 for this test
with patch.object(
settings.DERIVER, "REPRESENTATION_BATCH_LLM_MAX_TOKENS", 1500
settings.DERIVER, "REPRESENTATION_BATCH_TARGET_INPUT_TOKENS", 1500
):
# Test alice's work unit
# With per-work-unit anchoring + preceding context:
@ -1137,7 +1137,7 @@ class TestQueueProcessing:
peer = peers[0]
# Create messages where first message exceeds the batch limit
limit = settings.DERIVER.REPRESENTATION_BATCH_LLM_MAX_TOKENS
limit = settings.DERIVER.REPRESENTATION_BATCH_TARGET_INPUT_TOKENS
token_counts = [limit + 1000, 100, 200] # First message way over limit
# Create and save messages to the database first
@ -1254,7 +1254,7 @@ class TestQueueProcessing:
peer = peers[0]
# Create messages that test the exact boundary
limit = settings.DERIVER.REPRESENTATION_BATCH_LLM_MAX_TOKENS
limit = settings.DERIVER.REPRESENTATION_BATCH_TARGET_INPUT_TOKENS
token_counts = [
limit // 2,
limit // 2,

12
tests/test_config.py
View File

@ -8,7 +8,7 @@ def _make_deriver_settings(
MAX_INPUT_TOKENS: int = 25000,
MAX_CUSTOM_INSTRUCTIONS_TOKENS: int = 2000,
REPRESENTATION_BATCH_WORK_UNIT_TARGET_TOKENS: int = 512,
REPRESENTATION_BATCH_LLM_MAX_TOKENS: int = 1024,
REPRESENTATION_BATCH_TARGET_INPUT_TOKENS: int = 1024,
REPRESENTATION_BATCH_MAX_AGE_SECONDS: int = 1800,
) -> DeriverSettings:
return DeriverSettings(
@ -19,7 +19,7 @@ def _make_deriver_settings(
MAX_INPUT_TOKENS=MAX_INPUT_TOKENS,
MAX_CUSTOM_INSTRUCTIONS_TOKENS=MAX_CUSTOM_INSTRUCTIONS_TOKENS,
REPRESENTATION_BATCH_WORK_UNIT_TARGET_TOKENS=REPRESENTATION_BATCH_WORK_UNIT_TARGET_TOKENS,
REPRESENTATION_BATCH_LLM_MAX_TOKENS=REPRESENTATION_BATCH_LLM_MAX_TOKENS,
REPRESENTATION_BATCH_TARGET_INPUT_TOKENS=REPRESENTATION_BATCH_TARGET_INPUT_TOKENS,
REPRESENTATION_BATCH_MAX_AGE_SECONDS=REPRESENTATION_BATCH_MAX_AGE_SECONDS,
)
@ -68,18 +68,18 @@ def test_representation_batch_work_unit_target_rejects_negative_values() -> None
def test_representation_batch_tokens_can_diverge() -> None:
settings = _make_deriver_settings(
REPRESENTATION_BATCH_WORK_UNIT_TARGET_TOKENS=4096,
REPRESENTATION_BATCH_LLM_MAX_TOKENS=1024,
REPRESENTATION_BATCH_TARGET_INPUT_TOKENS=1024,
)
assert settings.REPRESENTATION_BATCH_WORK_UNIT_TARGET_TOKENS == 4096
assert settings.REPRESENTATION_BATCH_LLM_MAX_TOKENS == 1024
assert settings.REPRESENTATION_BATCH_TARGET_INPUT_TOKENS == 1024
def test_representation_batch_llm_max_cannot_exceed_max_input_tokens() -> None:
def test_representation_batch_target_input_cannot_exceed_max_input_tokens() -> None:
with pytest.raises(ValueError, match="cannot exceed max deriver input tokens"):
_make_deriver_settings(
MAX_INPUT_TOKENS=1000,
REPRESENTATION_BATCH_LLM_MAX_TOKENS=2048,
REPRESENTATION_BATCH_TARGET_INPUT_TOKENS=2048,
)