RomanNum3ral
/
honcho
mirror of https://github.com/plastic-labs/honcho.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fixes: update unified tests, bugs in dream scheduling (#322) 

* fix: swap to reasoning config in unified test files, handle new representation output format

* fix: resolve bugs in dream scheduling -- message from a peer should cancel all dreams observing them, and document counts should be checked at moment of dream, not in enqueue. clean up and test
This commit is contained in:
doria 2026-01-13 12:33:59 -05:00 committed by GitHub
parent 5978c670de
commit 817123de95
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
25 changed files with 733 additions and 285 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
src/config.py
View File

@ -492,9 +492,8 @@ class DreamSettings(BackupLLMSettingsMixin, HonchoSettings):
MIN_HOURS_BETWEEN_DREAMS: Annotated[int, Field(default=8, gt=0, le=72)] = 8
ENABLED_TYPES: list[str] = ["omni"]
# LLM settings for dream processing - upgraded for extended reasoning
PROVIDER: SupportedProviders = "anthropic"
MODEL: str = "claude-sonnet-4-20250514" # Upgraded from haiku for reasoning
MODEL: str = "claude-sonnet-4-20250514"
MAX_OUTPUT_TOKENS: Annotated[int, Field(default=16_384, gt=0, le=64_000)] = 16_384
THINKING_BUDGET_TOKENS: Annotated[int, Field(default=8192, gt=0, le=32_000)] = 8192
@ -506,17 +505,12 @@ class DreamSettings(BackupLLMSettingsMixin, HonchoSettings):
16_384
)
# Observation limits for orchestrated dreaming prescan
# Higher = more context for reasoning but slower prescan
PRESCAN_OBSERVATIONS_PER_LEVEL: Annotated[
int, Field(default=200, gt=0, le=1000)
] = 200
## NOTE: specialist models use the same provider as the main model
# Specialist model settings (OpenRouter format: provider/model)
# DeductionSpecialist: handles logical inference + temporal reasoning
DEDUCTION_MODEL: str = "anthropic/claude-haiku-4.5"
# InductionSpecialist: identifies patterns across observations
INDUCTION_MODEL: str = "anthropic/claude-haiku-4.5"
# Deduction Specialist: handles logical inference
DEDUCTION_MODEL: str = "claude-haiku-4-5"
# Induction Specialist: identifies patterns across observations
INDUCTION_MODEL: str = "claude-haiku-4-5"
# Surprisal-based sampling subsystem
SURPRISAL: SurprisalSettings = Field(default_factory=SurprisalSettings)

18
src/deriver/enqueue.py
View File

@ -8,7 +8,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
from src import crud, models, schemas
from src.config import settings
from src.dependencies import tracked_db
from src.dreamer.dream_scheduler import get_affected_dream_keys, get_dream_scheduler
from src.dreamer.dream_scheduler import get_dream_scheduler
from src.exceptions import ValidationException
from src.models import QueueItem
from src.schemas import MessageConfiguration, ResolvedConfiguration
@ -31,16 +31,20 @@ async def enqueue(payload: list[dict[str, Any]]) -> None:
payload: List of message payload dictionaries
"""
# Cancel any pending dreams for affected collections since user is active again
# Cancel any pending dreams for affected collections since user is active again.
# This cancels dreams for all collections where observed=peer_name, which covers
# both self-observation and peer-to-peer observation cases.
dream_scheduler = get_dream_scheduler()
if dream_scheduler and payload:
cancelled_dreams: set[str] = set()
for message in payload:
# Generate work unit keys for dreams that might be affected by this message
dream_keys: list[str] = get_affected_dream_keys(message)
for dream_key in dream_keys:
if await dream_scheduler.cancel_dream(dream_key):
cancelled_dreams.add(dream_key)
workspace_name = message.get("workspace_name")
peer_name = message.get("peer_name")
if workspace_name and peer_name:
cancelled = await dream_scheduler.cancel_dreams_for_observed(
workspace_name, peer_name
)
cancelled_dreams.update(cancelled)
if cancelled_dreams:
logger.info(

2
src/dreamer/__init__.py
View File

@ -1,12 +1,10 @@
from .dream_scheduler import (
check_and_schedule_dream,
get_affected_dream_keys,
get_dream_scheduler,
)
from .dreamer import process_dream
__all__ = [
"get_affected_dream_keys",
"get_dream_scheduler",
"check_and_schedule_dream",
"process_dream",

129
src/dreamer/dream_scheduler.py
View File

@ -2,7 +2,6 @@ import asyncio
import contextlib
from datetime import datetime, timezone
from logging import getLogger
from typing import Any
import sentry_sdk
from sqlalchemy import func, select
@ -31,39 +30,6 @@ def get_dream_scheduler() -> "DreamScheduler | None":
return _dream_scheduler
def get_affected_dream_keys(message: dict[str, Any]) -> list[str]:
"""
Get all work unit keys for dreams that might be affected by this message.
Args:
message: The message payload
Returns:
List of work unit keys that should have their dreams cancelled
"""
workspace_name = message.get("workspace_name")
peer_name = message.get("peer_name")
if not workspace_name or not peer_name:
return []
# Generate dream work unit keys for each enabled dream type
dream_keys: list[str] = []
for dream_type in settings.DREAM.ENABLED_TYPES:
dream_key = construct_work_unit_key(
workspace_name,
{
"task_type": "dream",
"observer": peer_name,
"observed": peer_name,
"dream_type": dream_type,
},
)
dream_keys.append(dream_key)
return dream_keys
class DreamScheduler:
_instance: "DreamScheduler | None" = None
_initialized: bool = False
@ -89,7 +55,6 @@ class DreamScheduler:
self,
work_unit_key: str,
workspace_name: str,
document_count: int,
delay_minutes: int,
dream_type: DreamType,
*,
@ -107,7 +72,6 @@ class DreamScheduler:
self._delayed_dream(
work_unit_key,
workspace_name,
document_count,
delay_minutes,
dream_type,
observer=observer,
@ -128,11 +92,42 @@ class DreamScheduler:
return True
return False
async def cancel_dreams_for_observed(
self, workspace_name: str, observed: str
) -> set[str]:
"""
Cancel all pending dreams where the observed peer matches.
This handles both self-observation (observer=observed) and peer-to-peer
observation (observer!=observed) dreams.
Args:
workspace_name: The workspace to match
observed: The observed peer name to match
Returns:
Set of work_unit_keys that were cancelled
"""
cancelled: set[str] = set()
# Collect keys to cancel (can't modify dict while iterating)
keys_to_cancel: list[str] = []
for work_unit_key in self.pending_dreams:
parsed = parse_work_unit_key(work_unit_key)
if parsed.workspace_name == workspace_name and parsed.observed == observed:
keys_to_cancel.append(work_unit_key)
# Cancel each matching dream
for key in keys_to_cancel:
if await self.cancel_dream(key):
cancelled.add(key)
return cancelled
async def _delayed_dream(
self,
work_unit_key: str,
workspace_name: str,
document_count: int,
delay_minutes: int,
dream_type: DreamType,
*,
@ -142,22 +137,13 @@ class DreamScheduler:
try:
await asyncio.sleep(delay_minutes * 60)
# Check if collection is still inactive before executing dream
if await self._should_execute_dream(
workspace_name, observer=observer, observed=observed
):
await self.execute_dream(
workspace_name,
document_count,
dream_type,
observer=observer,
observed=observed,
)
logger.info("Executed dream for %s", work_unit_key)
else:
logger.info(
"Skipping dream for %s - collection is active", work_unit_key
)
await self.execute_dream(
workspace_name,
dream_type,
observer=observer,
observed=observed,
)
logger.info("Executed dream for %s", work_unit_key)
except asyncio.CancelledError:
logger.debug("Dream task cancelled for %s", work_unit_key)
@ -166,33 +152,9 @@ class DreamScheduler:
if settings.SENTRY.ENABLED:
sentry_sdk.capture_exception(e)
async def _should_execute_dream(
self, workspace_name: str, *, observer: str, observed: str
) -> bool:
"""Check if the collection is inactive and should be dreamed upon."""
async with tracked_db("dream_activity_check") as db:
# Check for active queue sessions related to this collection
query = select(models.ActiveQueueSession)
result = await db.execute(query)
active_sessions = result.scalars().all()
# Look for any active work units that match this collection
for active_session in active_sessions:
parsed_key = parse_work_unit_key(active_session.work_unit_key)
if (
parsed_key.workspace_name == workspace_name
and parsed_key.observer == observer
and parsed_key.observed == observed
):
logger.debug("Collection is active, skipping dream")
return False
return True
async def execute_dream(
self,
workspace_name: str,
document_count: int,
dream_type: DreamType,
*,
observer: str,
@ -204,7 +166,7 @@ class DreamScheduler:
from src.deriver.enqueue import enqueue_dream
from src.utils.config_helpers import get_configuration
# Find the most recent session for this observer/observed pair
# Find the most recent session and get current document count
async with tracked_db("dream_session_lookup") as db:
stmt = (
select(models.Document.session_name)
@ -224,6 +186,14 @@ class DreamScheduler:
)
return
# Get current document count at execution time (not stale from scheduling)
count_stmt = select(func.count(models.Document.id)).where(
models.Document.workspace_name == workspace_name,
models.Document.observer == observer,
models.Document.observed == observed,
)
current_document_count = int(await db.scalar(count_stmt) or 0)
session = await crud.get_session(
db, workspace_name=workspace_name, session_name=session_name
)
@ -242,7 +212,7 @@ class DreamScheduler:
observer=observer,
observed=observed,
dream_type=dream_type,
document_count=document_count,
document_count=current_document_count,
session_name=session_name,
)
@ -346,7 +316,6 @@ async def check_and_schedule_dream(
await dream_scheduler.schedule_dream(
dream_work_unit_key,
collection.workspace_name,
current_document_count,
settings.DREAM.IDLE_TIMEOUT_MINUTES,
dream_type=DreamType(dream_type),
observer=collection.observer,

1
tests/dreamer/__init__.py Normal file
View File

@ -0,0 +1 @@
# Dream scheduler tests

460
tests/dreamer/test_dream_scheduler.py Normal file
View File

@ -0,0 +1,460 @@
"""Regression tests for dream scheduler bug fixes."""
from typing import Any
from unittest.mock import AsyncMock, patch
import pytest
from src.dreamer.dream_scheduler import DreamScheduler, set_dream_scheduler
from src.schemas import DreamType
from src.utils.work_unit import construct_work_unit_key
@pytest.fixture
def dream_scheduler():
"""Create a fresh DreamScheduler instance for each test."""
# Reset the singleton before each test
DreamScheduler.reset_singleton()
scheduler = DreamScheduler()
set_dream_scheduler(scheduler)
yield scheduler
# Cleanup
DreamScheduler.reset_singleton()
class TestCancelDreamsForObserved:
"""Regression tests for Bug #1: Peer-to-peer observation dreams not cancelled on activity.
Previously, when a message arrived from peer Bob, only the self-observation dream
(observer=Bob, observed=Bob) was cancelled. Peer-to-peer observation dreams
(observer=Alice, observed=Bob) were NOT cancelled, allowing dreams to fire
during active conversation.
"""
@pytest.mark.asyncio
async def test_cancel_self_observation_dream(self, dream_scheduler: DreamScheduler):
"""Cancelling dreams for observed peer should cancel self-observation dreams."""
workspace_name = "test_workspace"
peer_name = "bob"
# Schedule a self-observation dream (observer=bob, observed=bob)
work_unit_key = construct_work_unit_key(
workspace_name,
{
"task_type": "dream",
"observer": peer_name,
"observed": peer_name,
"dream_type": "omni",
},
)
with patch.object(dream_scheduler, "execute_dream", new_callable=AsyncMock):
await dream_scheduler.schedule_dream(
work_unit_key,
workspace_name,
delay_minutes=60,
dream_type=DreamType.OMNI,
observer=peer_name,
observed=peer_name,
)
# Verify dream is pending
assert work_unit_key in dream_scheduler.pending_dreams
# Cancel dreams for observed peer
cancelled = await dream_scheduler.cancel_dreams_for_observed(
workspace_name, peer_name
)
# Verify the dream was cancelled
assert work_unit_key in cancelled
assert work_unit_key not in dream_scheduler.pending_dreams
@pytest.mark.asyncio
async def test_cancel_peer_to_peer_observation_dream(
self, dream_scheduler: DreamScheduler
):
"""Cancelling dreams for observed peer should also cancel peer-to-peer dreams.
This is the core regression test for Bug #1: previously, if Alice was
observing Bob and Bob sent a message, the dream for (observer=Alice,
observed=Bob) would NOT be cancelled.
"""
workspace_name = "test_workspace"
observer = "alice" # Alice is watching Bob
observed = "bob" # Bob sends a message
# Schedule a peer-to-peer observation dream (observer=alice, observed=bob)
work_unit_key = construct_work_unit_key(
workspace_name,
{
"task_type": "dream",
"observer": observer,
"observed": observed,
"dream_type": "omni",
},
)
with patch.object(dream_scheduler, "execute_dream", new_callable=AsyncMock):
await dream_scheduler.schedule_dream(
work_unit_key,
workspace_name,
delay_minutes=60,
dream_type=DreamType.OMNI,
observer=observer,
observed=observed,
)
# Verify dream is pending
assert work_unit_key in dream_scheduler.pending_dreams
# When Bob sends a message, cancel all dreams where observed=bob
cancelled = await dream_scheduler.cancel_dreams_for_observed(
workspace_name, observed
)
# Verify the peer-to-peer dream was cancelled
assert work_unit_key in cancelled
assert work_unit_key not in dream_scheduler.pending_dreams
@pytest.mark.asyncio
async def test_cancel_multiple_observers_same_observed(
self, dream_scheduler: DreamScheduler
):
"""When observed peer sends a message, ALL dreams observing them should cancel."""
workspace_name = "test_workspace"
observed = "bob"
observers = ["alice", "charlie", "bob"] # Multiple observers including self
work_unit_keys: list[str] = []
with patch.object(dream_scheduler, "execute_dream", new_callable=AsyncMock):
for observer in observers:
work_unit_key = construct_work_unit_key(
workspace_name,
{
"task_type": "dream",
"observer": observer,
"observed": observed,
"dream_type": "omni",
},
)
work_unit_keys.append(work_unit_key)
await dream_scheduler.schedule_dream(
work_unit_key,
workspace_name,
delay_minutes=60,
dream_type=DreamType.OMNI,
observer=observer,
observed=observed,
)
# Verify all dreams are pending
assert len(dream_scheduler.pending_dreams) == 3
# Cancel all dreams where observed=bob
cancelled = await dream_scheduler.cancel_dreams_for_observed(
workspace_name, observed
)
# All three should be cancelled
assert len(cancelled) == 3
for key in work_unit_keys:
assert key in cancelled
assert len(dream_scheduler.pending_dreams) == 0
@pytest.mark.asyncio
async def test_does_not_cancel_dreams_for_different_observed(
self, dream_scheduler: DreamScheduler
):
"""Cancelling dreams for one observed peer should not affect others."""
workspace_name = "test_workspace"
# Dream for Alice observing Bob
key_alice_bob = construct_work_unit_key(
workspace_name,
{
"task_type": "dream",
"observer": "alice",
"observed": "bob",
"dream_type": "omni",
},
)
# Dream for Alice observing Charlie (should NOT be cancelled)
key_alice_charlie = construct_work_unit_key(
workspace_name,
{
"task_type": "dream",
"observer": "alice",
"observed": "charlie",
"dream_type": "omni",
},
)
with patch.object(dream_scheduler, "execute_dream", new_callable=AsyncMock):
await dream_scheduler.schedule_dream(
key_alice_bob,
workspace_name,
delay_minutes=60,
dream_type=DreamType.OMNI,
observer="alice",
observed="bob",
)
await dream_scheduler.schedule_dream(
key_alice_charlie,
workspace_name,
delay_minutes=60,
dream_type=DreamType.OMNI,
observer="alice",
observed="charlie",
)
assert len(dream_scheduler.pending_dreams) == 2
# Cancel only dreams where observed=bob
cancelled = await dream_scheduler.cancel_dreams_for_observed(
workspace_name, "bob"
)
# Only the bob dream should be cancelled
assert key_alice_bob in cancelled
assert key_alice_charlie not in cancelled
assert key_alice_charlie in dream_scheduler.pending_dreams
@pytest.mark.asyncio
async def test_does_not_cancel_dreams_for_different_workspace(
self, dream_scheduler: DreamScheduler
):
"""Cancelling dreams should be scoped to the correct workspace."""
observed = "bob"
key_ws1 = construct_work_unit_key(
"workspace1",
{
"task_type": "dream",
"observer": "alice",
"observed": observed,
"dream_type": "omni",
},
)
key_ws2 = construct_work_unit_key(
"workspace2",
{
"task_type": "dream",
"observer": "alice",
"observed": observed,
"dream_type": "omni",
},
)
with patch.object(dream_scheduler, "execute_dream", new_callable=AsyncMock):
await dream_scheduler.schedule_dream(
key_ws1,
"workspace1",
delay_minutes=60,
dream_type=DreamType.OMNI,
observer="alice",
observed=observed,
)
await dream_scheduler.schedule_dream(
key_ws2,
"workspace2",
delay_minutes=60,
dream_type=DreamType.OMNI,
observer="alice",
observed=observed,
)
# Cancel only in workspace1
cancelled = await dream_scheduler.cancel_dreams_for_observed(
"workspace1", observed
)
assert key_ws1 in cancelled
assert key_ws2 not in cancelled
assert key_ws2 in dream_scheduler.pending_dreams
class TestDocumentCountAtExecutionTime:
"""Regression tests for Bug #2: Stale document count used in metadata update.
Previously, the document count was captured when the dream was scheduled
(at check_and_schedule_dream time), then used 60 minutes later when the
dream actually executed. This caused incorrect metadata if documents were
added during the wait period.
Now, execute_dream queries the current document count at execution time.
"""
@pytest.mark.asyncio
async def test_execute_dream_queries_document_count_at_execution(
self, dream_scheduler: DreamScheduler
):
"""execute_dream should query current document count, not use a stale value.
This test verifies that execute_dream fetches the document count fresh
from the database at execution time rather than using a pre-captured value.
The key architectural change was:
- OLD: schedule_dream(document_count) -> _delayed_dream(document_count) -> execute_dream(document_count)
- NEW: schedule_dream() -> _delayed_dream() -> execute_dream() queries count internally
We verify this by mocking the database to return a specific count and
checking that enqueue_dream receives that count.
"""
from contextlib import asynccontextmanager
from unittest.mock import MagicMock
from src import models
from src.schemas import (
ResolvedConfiguration,
ResolvedDreamConfiguration,
ResolvedPeerCardConfiguration,
ResolvedReasoningConfiguration,
ResolvedSummaryConfiguration,
)
workspace_name = "test_workspace"
observer = "bob"
observed = "bob"
session_name = "test_session"
# The document count that the database will return
CURRENT_DOC_COUNT = 42
# Track what document_count is passed to enqueue_dream
captured_document_count: int | None = None
async def capture_enqueue_dream(
_ws_name: str,
observer: str, # pyright: ignore[reportUnusedParameter]
observed: str, # pyright: ignore[reportUnusedParameter]
dream_type: Any, # pyright: ignore[reportUnusedParameter]
document_count: int,
session_name: str, # pyright: ignore[reportUnusedParameter]
) -> None:
nonlocal captured_document_count
captured_document_count = document_count
# Create mock database session that returns our test data
mock_session = MagicMock()
mock_workspace = MagicMock(spec=models.Workspace)
mock_db_session = MagicMock(spec=models.Session)
# Mock scalar to return session_name for first call, document count for second
scalar_call_count = 0
async def mock_scalar(_stmt: Any) -> str | int:
nonlocal scalar_call_count
scalar_call_count += 1
if scalar_call_count == 1:
return session_name # First call gets session_name from documents
else:
return CURRENT_DOC_COUNT # Second call gets document count
mock_session.scalar = mock_scalar
@asynccontextmanager
async def mock_tracked_db(_: str | None = None):
yield mock_session
with (
patch(
"src.dreamer.dream_scheduler.tracked_db",
mock_tracked_db,
),
patch(
"src.deriver.enqueue.enqueue_dream",
side_effect=capture_enqueue_dream,
),
patch(
"src.crud.get_session",
return_value=mock_db_session,
),
patch(
"src.crud.get_workspace",
return_value=mock_workspace,
),
patch(
"src.utils.config_helpers.get_configuration",
return_value=ResolvedConfiguration(
reasoning=ResolvedReasoningConfiguration(enabled=True),
peer_card=ResolvedPeerCardConfiguration(use=True, create=True),
summary=ResolvedSummaryConfiguration(
enabled=True,
messages_per_short_summary=10,
messages_per_long_summary=20,
),
dream=ResolvedDreamConfiguration(enabled=True),
),
),
):
# Execute the dream
await dream_scheduler.execute_dream(
workspace_name,
DreamType.OMNI,
observer=observer,
observed=observed,
)
# Verify that execute_dream queried the document count (2 scalar calls)
assert (
scalar_call_count == 2
), "Should have queried session_name and document count"
# Verify that enqueue_dream received the CURRENT document count (42),
# proving that execute_dream queries the count at execution time
assert captured_document_count == CURRENT_DOC_COUNT
class TestEnqueueCancelsDreamsCorrectly:
"""Integration test verifying the full flow of message enqueue cancelling dreams."""
@pytest.mark.asyncio
async def test_enqueue_cancels_peer_to_peer_dreams(
self, dream_scheduler: DreamScheduler
):
"""When a message is enqueued, it should cancel all dreams for that observed peer."""
workspace_name = "test_workspace"
observed = "bob"
# Schedule dreams for multiple observers watching bob
keys: list[Any] = []
for observer in ["alice", "charlie", "bob"]:
key = construct_work_unit_key(
workspace_name,
{
"task_type": "dream",
"observer": observer,
"observed": observed,
"dream_type": "omni",
},
)
keys.append(key)
with patch.object(dream_scheduler, "execute_dream", new_callable=AsyncMock):
await dream_scheduler.schedule_dream(
key,
workspace_name,
delay_minutes=60,
dream_type=DreamType.OMNI,
observer=observer,
observed=observed,
)
assert len(dream_scheduler.pending_dreams) == 3
# Mock the database operations in enqueue
with patch("src.deriver.enqueue.tracked_db"):
# The enqueue function should cancel dreams via cancel_dreams_for_observed
# We just test that the scheduler method was called correctly
cancelled = await dream_scheduler.cancel_dreams_for_observed(
workspace_name, observed
)
# All three dreams should be cancelled
assert len(cancelled) == 3
assert len(dream_scheduler.pending_dreams) == 0

27
tests/unified/test_cases/config_deriver_hierarchy.json
View File

@ -1,10 +1,10 @@
{
"description": "Test deriver configuration hierarchy (Session > Workspace)",
"description": "Test reasoning configuration hierarchy (Session > Workspace)",
"steps": [
{
"step_type": "set_workspace_config",
"config": {
"deriver": {
"reasoning": {
"enabled": true
}
}
@ -13,7 +13,7 @@
"step_type": "create_session",
"session_id": "session_disabled",
"config": {
"deriver": {
"reasoning": {
"enabled": false
}
}
@ -43,18 +43,19 @@
"session_id": "session_disabled",
"assertions": [
{
"assertion_type": "json_match",
"key_value_pairs": {
"explicit": [],
"deductive": []
}
"assertion_type": "not_contains",
"text": "## Explicit"
},
{
"assertion_type": "not_contains",
"text": "## Deductive"
}
]
},
{
"step_type": "set_workspace_config",
"config": {
"deriver": {
"reasoning": {
"enabled": false
}
}
@ -63,7 +64,7 @@
"step_type": "create_session",
"session_id": "session_enabled",
"config": {
"deriver": {
"reasoning": {
"enabled": true
}
}
@ -93,9 +94,9 @@
"session_id": "session_enabled",
"assertions": [
{
"assertion_type": "llm_judge",
"prompt": "Check if there are explicit observations about testing or overriding in the representation.",
"pass_if": true
"assertion_type": "contains",
"text": "testing",
"case_sensitive": false
}
]
}

8
tests/unified/test_cases/config_message_positive_override.json
View File

@ -1,7 +1,7 @@
{
"description": "Test message-level deriver override",
"description": "Test message-level reasoning override",
"workspace_config": {
"deriver": {
"reasoning": {
"enabled": false
}
},
@ -18,7 +18,7 @@
"peer_id": "charlie",
"content": "I like turtles.",
"config": {
"deriver": {
"reasoning": {
"enabled": true
}
}
@ -27,7 +27,7 @@
"peer_id": "charlie",
"content": "I like squids.",
"config": {
"deriver": {
"reasoning": {
"enabled": false
}
}

2
tests/unified/test_cases/config_peercard_control.json
View File

@ -1,7 +1,7 @@
{
"description": "Test peer card generation control",
"workspace_config": {
"deriver": {
"reasoning": {
"enabled": true
},
"peer_card": {

4
tests/unified/test_cases/config_summary_control_deriver_off.json
View File

@ -1,7 +1,7 @@
{
"description": "Test summary generation control with deriver disabled",
"description": "Test summary generation control with reasoning disabled",
"workspace_config": {
"deriver": {
"reasoning": {
"enabled": false
},
"summary": {

19
tests/unified/test_cases/message_deriver_disabled.json
View File

@ -1,5 +1,5 @@
{
"description": "Test that disabling deriver prevents any derived data generation",
"description": "Test that disabling reasoning prevents any derived data generation",
"steps": [
{
"step_type": "create_session",
@ -13,7 +13,7 @@
"peer_id": "alice",
"content": "Message 1",
"config": {
"deriver": {
"reasoning": {
"enabled": false
}
}
@ -22,7 +22,7 @@
"peer_id": "alice",
"content": "Message 2",
"config": {
"deriver": {
"reasoning": {
"enabled": false
}
}
@ -31,7 +31,7 @@
"peer_id": "alice",
"content": "Message 3",
"config": {
"deriver": {
"reasoning": {
"enabled": false
}
}
@ -62,11 +62,12 @@
"session_id": "session_1",
"assertions": [
{
"assertion_type": "json_match",
"key_value_pairs": {
"explicit": [],
"deductive": []
}
"assertion_type": "not_contains",
"text": "## Explicit"
},
{
"assertion_type": "not_contains",
"text": "## Deductive"
}
]
},

44
tests/unified/test_cases/observation_2peer_both_observe_me_false.json
View File

@ -51,11 +51,12 @@
"session_id": "session_no_local_reps",
"assertions": [
{
"assertion_type": "json_match",
"key_value_pairs": {
"explicit": [],
"deductive": []
}
"assertion_type": "not_contains",
"text": "## Explicit"
},
{
"assertion_type": "not_contains",
"text": "## Deductive"
}
]
},
@ -67,11 +68,12 @@
"session_id": "session_no_local_reps",
"assertions": [
{
"assertion_type": "json_match",
"key_value_pairs": {
"explicit": [],
"deductive": []
}
"assertion_type": "not_contains",
"text": "## Explicit"
},
{
"assertion_type": "not_contains",
"text": "## Deductive"
}
]
},
@ -82,11 +84,12 @@
"session_id": "session_no_local_reps",
"assertions": [
{
"assertion_type": "json_match",
"key_value_pairs": {
"explicit": [],
"deductive": []
}
"assertion_type": "not_contains",
"text": "## Explicit"
},
{
"assertion_type": "not_contains",
"text": "## Deductive"
}
]
},
@ -97,11 +100,12 @@
"session_id": "session_no_local_reps",
"assertions": [
{
"assertion_type": "json_match",
"key_value_pairs": {
"explicit": [],
"deductive": []
}
"assertion_type": "not_contains",
"text": "## Explicit"
},
{
"assertion_type": "not_contains",
"text": "## Deductive"
}
]
}

34
tests/unified/test_cases/observation_2peer_default.json
View File

@ -51,11 +51,12 @@
"session_id": "session_default",
"assertions": [
{
"assertion_type": "json_match",
"key_value_pairs": {
"explicit": [],
"deductive": []
}
"assertion_type": "not_contains",
"text": "## Explicit"
},
{
"assertion_type": "not_contains",
"text": "## Deductive"
}
]
},
@ -67,11 +68,12 @@
"session_id": "session_default",
"assertions": [
{
"assertion_type": "json_match",
"key_value_pairs": {
"explicit": [],
"deductive": []
}
"assertion_type": "not_contains",
"text": "## Explicit"
},
{
"assertion_type": "not_contains",
"text": "## Deductive"
}
]
},
@ -82,9 +84,9 @@
"session_id": "session_default",
"assertions": [
{
"assertion_type": "llm_judge",
"prompt": "Check if there are facts about hiking and mountains in Alice's global representation.",
"pass_if": true
"assertion_type": "contains",
"text": "hiking",
"case_sensitive": false
}
]
},
@ -95,9 +97,9 @@
"session_id": "session_default",
"assertions": [
{
"assertion_type": "llm_judge",
"prompt": "Check if there are facts about swimming and beach in Bob's global representation.",
"pass_if": true
"assertion_type": "contains",
"text": "swimming",
"case_sensitive": false
}
]
}

22
tests/unified/test_cases/observation_2peer_observe_me_false_blocks_observation.json
View File

@ -51,11 +51,12 @@
"session_id": "session_no_observe_bob",
"assertions": [
{
"assertion_type": "json_match",
"key_value_pairs": {
"explicit": [],
"deductive": []
}
"assertion_type": "not_contains",
"text": "## Explicit"
},
{
"assertion_type": "not_contains",
"text": "## Deductive"
}
]
},
@ -66,11 +67,12 @@
"session_id": "session_no_observe_bob",
"assertions": [
{
"assertion_type": "json_match",
"key_value_pairs": {
"explicit": [],
"deductive": []
}
"assertion_type": "not_contains",
"text": "## Explicit"
},
{
"assertion_type": "not_contains",
"text": "## Deductive"
}
]
}

17
tests/unified/test_cases/observation_2peer_observe_me_false_but_can_still_observe_others.json
View File

@ -51,9 +51,9 @@
"session_id": "session_invisible_observer",
"assertions": [
{
"assertion_type": "llm_judge",
"prompt": "Check if Alice's local representation of Bob contains information about Bob working in retail management with a team of 20 people.",
"pass_if": true
"assertion_type": "contains",
"text": "retail",
"case_sensitive": false
}
]
},
@ -65,11 +65,12 @@
"session_id": "session_invisible_observer",
"assertions": [
{
"assertion_type": "json_match",
"key_value_pairs": {
"explicit": [],
"deductive": []
}
"assertion_type": "not_contains",
"text": "## Explicit"
},
{
"assertion_type": "not_contains",
"text": "## Deductive"
}
]
}

17
tests/unified/test_cases/observation_2peer_unidirectional_alice_observes_bob.json
View File

@ -51,9 +51,9 @@
"session_id": "session_uni_a_to_b",
"assertions": [
{
"assertion_type": "llm_judge",
"prompt": "Check if Alice's local representation of Bob contains information about Bob studying biology and genetics.",
"pass_if": true
"assertion_type": "contains",
"text": "biology",
"case_sensitive": false
}
]
},
@ -65,11 +65,12 @@
"session_id": "session_uni_a_to_b",
"assertions": [
{
"assertion_type": "json_match",
"key_value_pairs": {
"explicit": [],
"deductive": []
}
"assertion_type": "not_contains",
"text": "## Explicit"
},
{
"assertion_type": "not_contains",
"text": "## Deductive"
}
]
}

17
tests/unified/test_cases/observation_2peer_unidirectional_bob_observes_alice.json
View File

@ -51,9 +51,9 @@
"session_id": "session_uni_b_to_a",
"assertions": [
{
"assertion_type": "llm_judge",
"prompt": "Check if Bob's local representation of Alice contains information about Alice playing guitar in a band and playing rock and blues.",
"pass_if": true
"assertion_type": "contains",
"text": "guitar",
"case_sensitive": false
}
]
},
@ -65,11 +65,12 @@
"session_id": "session_uni_b_to_a",
"assertions": [
{
"assertion_type": "json_match",
"key_value_pairs": {
"explicit": [],
"deductive": []
}
"assertion_type": "not_contains",
"text": "## Explicit"
},
{
"assertion_type": "not_contains",
"text": "## Deductive"
}
]
}

46
tests/unified/test_cases/observation_3peer_multiple_observers_one_observed.json
View File

@ -59,9 +59,9 @@
"session_id": "session_all_watch_alice",
"assertions": [
{
"assertion_type": "llm_judge",
"prompt": "Check if Bob's local representation of Alice contains information about Alice being a professional dancer specializing in contemporary and ballet.",
"pass_if": true
"assertion_type": "contains",
"text": "dancer",
"case_sensitive": false
}
]
},
@ -73,9 +73,9 @@
"session_id": "session_all_watch_alice",
"assertions": [
{
"assertion_type": "llm_judge",
"prompt": "Check if Charlie's local representation of Alice contains information about Alice being a professional dancer who has been dancing for 15 years.",
"pass_if": true
"assertion_type": "contains",
"text": "dancer",
"case_sensitive": false
}
]
},
@ -87,11 +87,12 @@
"session_id": "session_all_watch_alice",
"assertions": [
{
"assertion_type": "json_match",
"key_value_pairs": {
"explicit": [],
"deductive": []
}
"assertion_type": "not_contains",
"text": "## Explicit"
},
{
"assertion_type": "not_contains",
"text": "## Deductive"
}
]
},
@ -103,11 +104,12 @@
"session_id": "session_all_watch_alice",
"assertions": [
{
"assertion_type": "json_match",
"key_value_pairs": {
"explicit": [],
"deductive": []
}
"assertion_type": "not_contains",
"text": "## Explicit"
},
{
"assertion_type": "not_contains",
"text": "## Deductive"
}
]
},
@ -118,9 +120,9 @@
"observed_peer_id": "alice",
"assertions": [
{
"assertion_type": "llm_judge",
"prompt": "Check if Bob has a peer card for Alice that contains relevant information about her.",
"pass_if": true
"assertion_type": "contains",
"text": "dancer",
"case_sensitive": false
}
]
},
@ -131,9 +133,9 @@
"observed_peer_id": "alice",
"assertions": [
{
"assertion_type": "llm_judge",
"prompt": "Check if Charlie has a peer card for Alice that contains relevant information about her.",
"pass_if": true
"assertion_type": "contains",
"text": "dancer",
"case_sensitive": false
}
]
},

34
tests/unified/test_cases/observation_3peer_one_observer_multiple_observed.json
View File

@ -59,9 +59,9 @@
"session_id": "session_alice_watches_all",
"assertions": [
{
"assertion_type": "llm_judge",
"prompt": "Check if Alice's local representation of Bob contains information about Bob training for a marathon and running 5 miles every morning.",
"pass_if": true
"assertion_type": "contains",
"text": "marathon",
"case_sensitive": false
}
]
},
@ -73,9 +73,9 @@
"session_id": "session_alice_watches_all",
"assertions": [
{
"assertion_type": "llm_judge",
"prompt": "Check if Alice's local representation of Charlie contains information about Charlie learning chess and practicing tactics daily.",
"pass_if": true
"assertion_type": "contains",
"text": "chess",
"case_sensitive": false
}
]
},
@ -87,11 +87,12 @@
"session_id": "session_alice_watches_all",
"assertions": [
{
"assertion_type": "json_match",
"key_value_pairs": {
"explicit": [],
"deductive": []
}
"assertion_type": "not_contains",
"text": "## Explicit"
},
{
"assertion_type": "not_contains",
"text": "## Deductive"
}
]
},
@ -103,11 +104,12 @@
"session_id": "session_alice_watches_all",
"assertions": [
{
"assertion_type": "json_match",
"key_value_pairs": {
"explicit": [],
"deductive": []
}
"assertion_type": "not_contains",
"text": "## Explicit"
},
{
"assertion_type": "not_contains",
"text": "## Deductive"
}
]
}

17
tests/unified/test_cases/observation_3peer_selective_observation.json
View File

@ -59,9 +59,9 @@
"session_id": "session_selective",
"assertions": [
{
"assertion_type": "llm_judge",
"prompt": "Check if Alice's local representation of Bob contains information about Bob being an architect who designs skyscrapers.",
"pass_if": true
"assertion_type": "contains",
"text": "architect",
"case_sensitive": false
}
]
},
@ -73,11 +73,12 @@
"session_id": "session_selective",
"assertions": [
{
"assertion_type": "json_match",
"key_value_pairs": {
"explicit": [],
"deductive": []
}
"assertion_type": "not_contains",
"text": "## Explicit"
},
{
"assertion_type": "not_contains",
"text": "## Deductive"
}
]
}

48
tests/unified/test_cases/observation_4peer_complex_matrix.json
View File

@ -1,7 +1,7 @@
{
"description": "Test complex 4-peer scenario with mixed observation patterns",
"workspace_config": {
"deriver": {
"reasoning": {
"enabled": true
}
},
@ -62,9 +62,9 @@
"session_id": "session_four_peer",
"assertions": [
{
"assertion_type": "llm_judge",
"prompt": "Check if Alice observes Bob (should contain lawyer info).",
"pass_if": true
"assertion_type": "contains",
"text": "lawyer",
"case_sensitive": false
}
]
},
@ -76,11 +76,12 @@
"session_id": "session_four_peer",
"assertions": [
{
"assertion_type": "json_match",
"key_value_pairs": {
"explicit": [],
"deductive": []
}
"assertion_type": "not_contains",
"text": "## Explicit"
},
{
"assertion_type": "not_contains",
"text": "## Deductive"
}
]
},
@ -92,9 +93,9 @@
"session_id": "session_four_peer",
"assertions": [
{
"assertion_type": "llm_judge",
"prompt": "Check if Bob observes Alice (should contain neurologist info).",
"pass_if": true
"assertion_type": "contains",
"text": "neurologist",
"case_sensitive": false
}
]
},
@ -106,9 +107,9 @@
"session_id": "session_four_peer",
"assertions": [
{
"assertion_type": "llm_judge",
"prompt": "Check if Charlie observes Alice (should contain neurologist info, even though Charlie has observe_me=false).",
"pass_if": true
"assertion_type": "contains",
"text": "neurologist",
"case_sensitive": false
}
]
},
@ -120,9 +121,9 @@
"session_id": "session_four_peer",
"assertions": [
{
"assertion_type": "llm_judge",
"prompt": "Check if Alice observes Diana (should contain pharmacist info).",
"pass_if": true
"assertion_type": "contains",
"text": "pharmacist",
"case_sensitive": false
}
]
},
@ -134,11 +135,12 @@
"session_id": "session_four_peer",
"assertions": [
{
"assertion_type": "json_match",
"key_value_pairs": {
"explicit": [],
"deductive": []
}
"assertion_type": "not_contains",
"text": "## Explicit"
},
{
"assertion_type": "not_contains",
"text": "## Deductive"
}
]
}

2
tests/unified/test_cases/observation_asymmetric_visibility.json
View File

@ -1,7 +1,7 @@
{
"description": "Test asymmetric visibility scenario from documentation - Alice lies to Bob but tells truth to Charlie",
"workspace_config": {
"deriver": {
"reasoning": {
"enabled": true
}
},

2
tests/unified/test_cases/observation_isolation_between_sessions.json
View File

@ -1,7 +1,7 @@
{
"description": "Test that local representations are isolated between sessions - same peers in different sessions have different local reps",
"workspace_config": {
"deriver": {
"reasoning": {
"enabled": true
}
},

15
tests/unified/test_cases/session_deriver_disabled.json
View File

@ -1,11 +1,11 @@
{
"description": "Test that disabling deriver prevents any derived data generation",
"description": "Test that disabling reasoning prevents any derived data generation",
"steps": [
{
"step_type": "create_session",
"session_id": "session_1",
"config": {
"deriver": {
"reasoning": {
"enabled": false
}
}
@ -52,11 +52,12 @@
"session_id": "session_1",
"assertions": [
{
"assertion_type": "json_match",
"key_value_pairs": {
"explicit": [],
"deductive": []
}
"assertion_type": "not_contains",
"text": "## Explicit"
},
{
"assertion_type": "not_contains",
"text": "## Deductive"
}
]
},

15
tests/unified/test_cases/workspace_deriver_disabled.json
View File

@ -1,7 +1,7 @@
{
"description": "Test that disabling deriver prevents any derived data generation",
"description": "Test that disabling reasoning prevents any derived data generation",
"workspace_config": {
"deriver": {
"reasoning": {
"enabled": false
}
},
@ -52,11 +52,12 @@
"session_id": "session_1",
"assertions": [
{
"assertion_type": "json_match",
"key_value_pairs": {
"explicit": [],
"deductive": []
}
"assertion_type": "not_contains",
"text": "## Explicit"
},
{
"assertion_type": "not_contains",
"text": "## Deductive"
}
]
},