fix(dialectic): make workspace chat search before it answers
The workspace agent's prefetch is an orientation overview — scale, active peers, their cards — not the corpus. `low` is the only reasoning level that explicitly sets TOOL_CHOICE="auto", so the model was free to skip tools entirely, and it did: every workspace_chat call in CI run 33662772219 made zero tool calls. It answered when the overview happened to carry the fact and otherwise wrote out the search it should have run, then asked the caller which option to take — at an endpoint with no caller to answer. Add a `_tool_choice` seam alongside `_select_tools` and override it on WorkspaceDialecticAgent to require a tool call. `execute_tool_loop` already relaxes "required"/"any" to "auto" after the first iteration, so this costs one search round rather than pinning the loop, and the model can still stop and synthesize. Any value a level configures other than None/"auto" passes through. The pair agent is unaffected: it prefetches the observations for its query and can legitimately answer from context alone. Also tell the workspace prompt it is non-interactive. It had "Do not narrate tool use" but never said the caller cannot reply, and three of the five traced responses ended in a menu of lookups. Unified subset goes 1/5 -> 5/5, and search_memory — the recall path that never once ran — now fires on 6 of 7 workspace queries. workspace_chat_scope is the notable one: its two not_contains assertions were passing vacuously because nothing was ever retrieved, and it now recalls the in-scope fact while still excluding the out-of-scope vault code. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
5d992bc65a
commit
473684407a
|
|
@ -14,7 +14,12 @@ from nanoid import generate as generate_nanoid
|
|||
from pydantic import BaseModel
|
||||
|
||||
from src import crud
|
||||
from src.config import ConfiguredModelSettings, ReasoningLevel, settings
|
||||
from src.config import (
|
||||
ConfiguredModelSettings,
|
||||
DialecticLevelSettings,
|
||||
ReasoningLevel,
|
||||
settings,
|
||||
)
|
||||
from src.dependencies import tracked_db
|
||||
from src.dialectic import prompts
|
||||
from src.embedding_client import embedding_client
|
||||
|
|
@ -139,6 +144,20 @@ class DialecticAgent:
|
|||
tools = [t for t in tools if t.get("name") != "get_reasoning_chain"]
|
||||
return tools
|
||||
|
||||
def _tool_choice(
|
||||
self, level_settings: DialecticLevelSettings
|
||||
) -> str | dict[str, Any] | None:
|
||||
"""Pick the tool_choice for this query.
|
||||
|
||||
Defaults to whatever the reasoning level configures. Subclasses override
|
||||
when the agent has no prefetched corpus to fall back on and so must
|
||||
search before it can answer. Forcing "required"/"any" here costs exactly
|
||||
one tool round rather than pinning the loop: `execute_tool_loop` relaxes
|
||||
it to "auto" after the first iteration so the model can still stop and
|
||||
synthesize.
|
||||
"""
|
||||
return level_settings.TOOL_CHOICE
|
||||
|
||||
async def _initialize_session_history(self) -> None:
|
||||
"""Fetch and inject session history into the system prompt if configured."""
|
||||
if self._session_history_initialized:
|
||||
|
|
@ -505,7 +524,7 @@ class DialecticAgent:
|
|||
prompt="", # Ignored since we pass messages
|
||||
max_tokens=max_tokens,
|
||||
tools=tools,
|
||||
tool_choice=level_settings.TOOL_CHOICE,
|
||||
tool_choice=self._tool_choice(level_settings),
|
||||
tool_executor=tool_executor,
|
||||
max_tool_iterations=level_settings.MAX_TOOL_ITERATIONS,
|
||||
messages=self.messages,
|
||||
|
|
@ -581,7 +600,7 @@ class DialecticAgent:
|
|||
stream=True,
|
||||
stream_final_only=True,
|
||||
tools=tools,
|
||||
tool_choice=level_settings.TOOL_CHOICE,
|
||||
tool_choice=self._tool_choice(level_settings),
|
||||
tool_executor=tool_executor,
|
||||
max_tool_iterations=level_settings.MAX_TOOL_ITERATIONS,
|
||||
messages=self.messages,
|
||||
|
|
|
|||
|
|
@ -396,7 +396,11 @@ If this query is restricted to a session or a set of sessions, message tools alr
|
|||
|
||||
4. **Attribute**. Every fact you state names the peer it is about. If it is a cross-peer view, also name whose model it came from. Example: "Alice is a violinist." / "From Bob's model of Alice, …"
|
||||
|
||||
5. **Synthesize**. Answer the question. Quote exact names, dates, and numbers. For aggregations, list findings per peer. Do not narrate tool use.
|
||||
5. **Synthesize**. Answer the question. Quote exact names, dates, and numbers. For aggregations, list findings per peer. Do not narrate tool use, and do not describe a search you did not run.
|
||||
|
||||
## NO CLARIFYING QUESTIONS
|
||||
|
||||
Your answer goes to a program, not to someone who can reply. No one will answer a question you ask, approve a plan you propose, or pick from options you offer — your response ends the exchange. So never ask which lookup to run, never lay out a plan and stop, never present a menu. Run the searches yourself and answer from what they return. Empty results are a complete answer; an unanswered question is not.
|
||||
|
||||
## NEVER FABRICATE
|
||||
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ from collections.abc import Callable
|
|||
from typing import Any
|
||||
|
||||
from src import crud
|
||||
from src.config import ReasoningLevel, settings
|
||||
from src.config import DialecticLevelSettings, ReasoningLevel, settings
|
||||
from src.dependencies import tracked_db
|
||||
from src.dialectic import prompts
|
||||
from src.dialectic.core import DialecticAgent
|
||||
|
|
@ -161,6 +161,30 @@ class WorkspaceDialecticAgent(DialecticAgent):
|
|||
tools = [t for t in tools if t.get("name") not in unscopable]
|
||||
return tools
|
||||
|
||||
def _tool_choice(
|
||||
self, level_settings: DialecticLevelSettings
|
||||
) -> str | dict[str, Any] | None:
|
||||
"""Require a tool call on the first turn.
|
||||
|
||||
The pair agent prefetches the observations relevant to its query, so it
|
||||
can legitimately answer from context alone. This agent's prefetch is an
|
||||
orientation overview — scale, active peers, their cards — not the corpus.
|
||||
Left free to skip tools, the model treats that overview as everything it
|
||||
has: it answers when the overview happens to carry the fact, and
|
||||
otherwise writes out the search it should have run and asks the caller
|
||||
which option to take. Workspace chat has no caller to answer, so that
|
||||
response is dead on arrival.
|
||||
|
||||
Recall is the job, so make the first search mandatory and let the loop
|
||||
relax to "auto" afterwards. Any other value a level configures is passed
|
||||
through untouched, so this only overrides the two cases that let the
|
||||
model opt out entirely.
|
||||
"""
|
||||
choice = level_settings.TOOL_CHOICE
|
||||
if choice is None or choice == "auto":
|
||||
return "required"
|
||||
return choice
|
||||
|
||||
async def _create_tool_executor(self) -> Callable[[str, dict[str, Any]], Any]:
|
||||
return await create_workspace_tool_executor(
|
||||
workspace_name=self.workspace_name,
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ import asyncio
|
|||
import json
|
||||
from collections.abc import Callable
|
||||
from contextlib import asynccontextmanager
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
|
|
@ -82,7 +82,7 @@ async def workspace_test_data(
|
|||
await db_session.flush()
|
||||
|
||||
# Create messages
|
||||
now = datetime.now(timezone.utc)
|
||||
now = datetime.now(UTC)
|
||||
messages: list[models.Message] = []
|
||||
for i in range(6):
|
||||
peer_name = [peer1.name, peer2.name, peer3.name][i % 3]
|
||||
|
|
@ -593,7 +593,7 @@ class TestSearchMemoryWorkspace:
|
|||
content="I really like programming in Python",
|
||||
seq_in_session=1,
|
||||
token_count=10,
|
||||
created_at=datetime.now(timezone.utc),
|
||||
created_at=datetime.now(UTC),
|
||||
)
|
||||
db_session.add(msg)
|
||||
await db_session.flush()
|
||||
|
|
@ -919,7 +919,7 @@ class TestGetObservationContextWorkspace:
|
|||
content="LEAKED_FROM_OTHER_SESSION",
|
||||
seq_in_session=messages[0].seq_in_session,
|
||||
token_count=10,
|
||||
created_at=datetime.now(timezone.utc),
|
||||
created_at=datetime.now(UTC),
|
||||
)
|
||||
db_session.add(leaked_message)
|
||||
await db_session.commit()
|
||||
|
|
@ -1253,3 +1253,63 @@ class TestWorkspaceChatPrompt:
|
|||
}
|
||||
assert agent.messages[0]["content"] == workspace_agent_system_prompt(offered)
|
||||
assert agent._prefetch_heading() == "Workspace overview (prefetched)" # pyright: ignore[reportPrivateUsage]
|
||||
|
||||
def test_forbids_clarifying_questions(self) -> None:
|
||||
"""The endpoint is non-interactive, so the prompt must say so.
|
||||
|
||||
Without this the model answers a recall query with a plan and a menu of
|
||||
lookups for a caller that cannot reply. The pair agent talks to a peer
|
||||
and is deliberately left alone.
|
||||
"""
|
||||
from src.dialectic.prompts import (
|
||||
agent_system_prompt,
|
||||
workspace_agent_system_prompt,
|
||||
)
|
||||
|
||||
prompt = workspace_agent_system_prompt()
|
||||
assert "NO CLARIFYING QUESTIONS" in prompt
|
||||
assert "NO CLARIFYING QUESTIONS" not in agent_system_prompt(
|
||||
"alice", "alice", None, None
|
||||
)
|
||||
|
||||
|
||||
class TestWorkspaceToolChoice:
|
||||
"""The workspace agent must search before it answers.
|
||||
|
||||
Its prefetch is an orientation overview, not the corpus, so a turn with no
|
||||
tool call ends the loop with whatever the overview happened to contain.
|
||||
"""
|
||||
|
||||
@pytest.mark.parametrize("level", ["minimal", "low", "medium", "high", "max"])
|
||||
def test_first_turn_requires_a_tool_call(self, level: str) -> None:
|
||||
from src.config import settings
|
||||
from src.dialectic.workspace import WorkspaceDialecticAgent
|
||||
|
||||
agent = WorkspaceDialecticAgent(workspace_name="w", reasoning_level=level) # pyright: ignore[reportArgumentType]
|
||||
level_settings = settings.DIALECTIC.LEVELS[level] # pyright: ignore[reportArgumentType]
|
||||
assert agent._tool_choice(level_settings) == "required" # pyright: ignore[reportPrivateUsage]
|
||||
|
||||
def test_pair_agent_keeps_the_configured_choice(self) -> None:
|
||||
from src.config import settings
|
||||
from src.dialectic.core import DialecticAgent
|
||||
|
||||
agent = DialecticAgent(
|
||||
workspace_name="w", session_name=None, observer="a", observed="a"
|
||||
)
|
||||
level_settings = settings.DIALECTIC.LEVELS["low"]
|
||||
assert (
|
||||
agent._tool_choice(level_settings) # pyright: ignore[reportPrivateUsage]
|
||||
== level_settings.TOOL_CHOICE
|
||||
)
|
||||
|
||||
def test_a_configured_non_auto_choice_is_passed_through(self) -> None:
|
||||
from src.config import DialecticLevelSettings, settings
|
||||
from src.dialectic.workspace import WorkspaceDialecticAgent
|
||||
|
||||
agent = WorkspaceDialecticAgent(workspace_name="w")
|
||||
pinned = DialecticLevelSettings(
|
||||
MODEL_CONFIG=settings.DIALECTIC.LEVELS["low"].MODEL_CONFIG,
|
||||
MAX_TOOL_ITERATIONS=5,
|
||||
TOOL_CHOICE="none",
|
||||
)
|
||||
assert agent._tool_choice(pinned) == "none" # pyright: ignore[reportPrivateUsage]
|
||||
|
|
|
|||
Loading…
Reference in New Issue