feat: phase 4: developer feedback and interview
This commit is contained in:
parent
97f3409280
commit
b04845c451
|
|
@ -452,3 +452,33 @@ async def store_introspection_report(
|
|||
logger.error(f"Failed to store introspection report: {e}")
|
||||
await db.rollback()
|
||||
# Don't re-raise - storing the report is secondary to generating it
|
||||
|
||||
|
||||
async def get_latest_introspection_report(
|
||||
db: AsyncSession,
|
||||
workspace_name: str,
|
||||
) -> IntrospectionReport | None:
|
||||
"""
|
||||
Retrieve the most recent introspection report for a workspace.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
workspace_name: Name of the workspace
|
||||
|
||||
Returns:
|
||||
The most recent IntrospectionReport, or None if not found
|
||||
"""
|
||||
stmt = (
|
||||
select(models.Document)
|
||||
.where(models.Document.workspace_name == workspace_name)
|
||||
.where(models.Document.observer == SYSTEM_OBSERVER)
|
||||
.where(models.Document.observed == INTROSPECTION_OBSERVED)
|
||||
.where(models.Document.deleted_at.is_(None))
|
||||
.order_by(models.Document.created_at.desc())
|
||||
.limit(1)
|
||||
)
|
||||
result = await db.execute(stmt)
|
||||
doc = result.scalar_one_or_none()
|
||||
if doc is None:
|
||||
return None
|
||||
return IntrospectionReport.model_validate(json.loads(doc.content))
|
||||
|
|
|
|||
|
|
@ -0,0 +1,303 @@
|
|||
"""
|
||||
Developer Feedback Channel for configuring Honcho's agent behavior.
|
||||
|
||||
This module provides a natural language interface for developers to configure
|
||||
workspace agent settings through conversation.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from src import crud
|
||||
from src.config import settings
|
||||
from src.schemas import (
|
||||
ConfigChange,
|
||||
FeedbackRequest,
|
||||
FeedbackResponse,
|
||||
IntrospectionReport,
|
||||
WorkspaceAgentConfig,
|
||||
)
|
||||
from src.utils.clients import honcho_llm_call
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
INTERVIEW_QUESTIONS = """Before I can help configure Honcho for your workspace, I'd like to understand your application better. Please tell me:
|
||||
|
||||
1. **What type of application are you building?** (e.g., journaling app, customer support bot, educational tutor, personal assistant, etc.)
|
||||
|
||||
2. **What aspects of your users do you want Honcho to focus on?** (e.g., emotions, preferences, technical skills, learning progress, goals, habits)
|
||||
|
||||
3. **How should the Dialectic API respond to questions about users?** (e.g., detailed analysis, brief summaries, specific focus areas)
|
||||
|
||||
4. **Are there any topics or patterns you want Honcho to explicitly ignore or avoid?**
|
||||
|
||||
Feel free to answer any or all of these questions, and I'll help configure your workspace accordingly."""
|
||||
|
||||
|
||||
def _is_simple_greeting(message: str) -> bool:
|
||||
"""Check if a message is a simple greeting or question that should trigger interview mode."""
|
||||
message_lower = message.lower().strip()
|
||||
|
||||
# Check length - short messages are more likely greetings
|
||||
if len(message) > 100:
|
||||
return False
|
||||
|
||||
# Common greetings and simple starts
|
||||
greeting_patterns = [
|
||||
r"^h(i|ello|ey)\b",
|
||||
r"^good (morning|afternoon|evening)",
|
||||
r"^what('s| is) up",
|
||||
r"^how('s| are) (it going|you|things)",
|
||||
r"^yo\b",
|
||||
r"^sup\b",
|
||||
r"^greetings",
|
||||
r"^howdy",
|
||||
r"^help$",
|
||||
r"^help me",
|
||||
r"^how do i",
|
||||
r"^what can you",
|
||||
r"^configure",
|
||||
r"^setup",
|
||||
r"^start",
|
||||
r"^begin",
|
||||
r"^get started",
|
||||
]
|
||||
|
||||
return any(re.match(pattern, message_lower) for pattern in greeting_patterns)
|
||||
|
||||
|
||||
def _config_is_empty(config: WorkspaceAgentConfig) -> bool:
|
||||
"""Check if config has no custom rules set."""
|
||||
return not config.deriver_rules.strip() and not config.dialectic_rules.strip()
|
||||
|
||||
|
||||
def build_feedback_prompt(
|
||||
message: str,
|
||||
current_config: WorkspaceAgentConfig,
|
||||
introspection_report: IntrospectionReport | None = None,
|
||||
) -> str:
|
||||
"""
|
||||
Build a prompt for the LLM to process developer feedback.
|
||||
|
||||
Args:
|
||||
message: The developer's feedback message
|
||||
current_config: Current workspace agent configuration
|
||||
introspection_report: Optional introspection report for context
|
||||
|
||||
Returns:
|
||||
A formatted prompt string for the LLM
|
||||
"""
|
||||
introspection_section = ""
|
||||
if introspection_report:
|
||||
introspection_section = f"""
|
||||
## Recent Introspection Report
|
||||
|
||||
**Performance Summary:** {introspection_report.performance_summary}
|
||||
|
||||
**Identified Issues:**
|
||||
{chr(10).join(f'- {issue}' for issue in introspection_report.identified_issues) if introspection_report.identified_issues else '(none)'}
|
||||
|
||||
**Suggestions:**
|
||||
{chr(10).join(f'- [{s.target}] {s.rationale} (confidence: {s.confidence})' for s in introspection_report.suggestions) if introspection_report.suggestions else '(none)'}
|
||||
|
||||
"""
|
||||
|
||||
return f"""You are a configuration assistant for Honcho, an AI memory infrastructure system.
|
||||
|
||||
A developer is interacting with the feedback channel to configure their workspace's agent behavior.
|
||||
|
||||
## Current Configuration
|
||||
|
||||
**Deriver Rules** (guides memory extraction):
|
||||
```
|
||||
{current_config.deriver_rules or "(empty - using defaults)"}
|
||||
```
|
||||
|
||||
**Dialectic Rules** (guides question answering):
|
||||
```
|
||||
{current_config.dialectic_rules or "(empty - using defaults)"}
|
||||
```
|
||||
{introspection_section}
|
||||
## Developer Message
|
||||
|
||||
{message}
|
||||
|
||||
## Your Task
|
||||
|
||||
1. **Understand the intent**: Is the developer asking a question, providing configuration instructions, or just chatting?
|
||||
|
||||
2. **Determine configuration changes**: Based on the message, decide if any configuration changes should be made:
|
||||
- `deriver_rules`: Controls what the memory extraction agent focuses on
|
||||
- `dialectic_rules`: Controls how the question-answering agent responds
|
||||
|
||||
3. **Be incremental**: When adding rules, PRESERVE existing rules unless the developer explicitly asks to replace them. Append new rules to existing ones.
|
||||
|
||||
4. **Respond helpfully**: Provide a clear, friendly response explaining what you understood and what changes (if any) you made.
|
||||
|
||||
## Response Format
|
||||
|
||||
Respond with a JSON object:
|
||||
```json
|
||||
{{
|
||||
"message": "Your response to the developer",
|
||||
"understood_intent": "Brief description of what you understood the developer wants",
|
||||
"changes": [
|
||||
{{
|
||||
"field": "deriver_rules" | "dialectic_rules",
|
||||
"new_value": "The complete new value for this field (including preserved old rules if applicable)"
|
||||
}}
|
||||
]
|
||||
}}
|
||||
```
|
||||
|
||||
If no changes are needed (e.g., the developer is asking a question), return an empty `changes` array.
|
||||
|
||||
Important:
|
||||
- Keep rules concise and actionable
|
||||
- Each rule should be on its own line for clarity
|
||||
- When adding to existing rules, put a newline between old and new rules
|
||||
- Be helpful and explain what the rules will do"""
|
||||
|
||||
|
||||
async def process_feedback(
|
||||
db: AsyncSession,
|
||||
workspace_name: str,
|
||||
request: FeedbackRequest,
|
||||
introspection_report: IntrospectionReport | None = None,
|
||||
) -> FeedbackResponse:
|
||||
"""
|
||||
Process developer feedback and update workspace configuration.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
workspace_name: Name of the workspace
|
||||
request: The feedback request
|
||||
introspection_report: Optional introspection report for context
|
||||
|
||||
Returns:
|
||||
FeedbackResponse with the result
|
||||
"""
|
||||
# Get current config
|
||||
current_config = await crud.get_workspace_agent_config(db, workspace_name)
|
||||
|
||||
# Check for interview mode: empty config + simple greeting
|
||||
if _config_is_empty(current_config) and _is_simple_greeting(request.message):
|
||||
logger.info(
|
||||
f"Feedback channel: Interview mode triggered for workspace {workspace_name}"
|
||||
)
|
||||
return FeedbackResponse(
|
||||
message=INTERVIEW_QUESTIONS,
|
||||
understood_intent="First-time setup - gathering information about the application",
|
||||
changes_made=[],
|
||||
current_config=current_config,
|
||||
)
|
||||
|
||||
# Build prompt and call LLM
|
||||
prompt = build_feedback_prompt(
|
||||
message=request.message,
|
||||
current_config=current_config,
|
||||
introspection_report=introspection_report,
|
||||
)
|
||||
|
||||
try:
|
||||
llm_response = await honcho_llm_call(
|
||||
llm_settings=settings.DREAM,
|
||||
prompt=prompt,
|
||||
max_tokens=4096,
|
||||
track_name="feedback_channel",
|
||||
json_mode=True,
|
||||
temperature=0.3,
|
||||
)
|
||||
|
||||
# Parse response
|
||||
response_text = llm_response.content
|
||||
try:
|
||||
response_data: dict[str, object] = json.loads(response_text)
|
||||
except json.JSONDecodeError as e:
|
||||
logger.error(f"Failed to parse feedback LLM response: {e}")
|
||||
return FeedbackResponse(
|
||||
message="I had trouble processing your request. Could you try rephrasing?",
|
||||
understood_intent="Error parsing response",
|
||||
changes_made=[],
|
||||
current_config=current_config,
|
||||
)
|
||||
|
||||
# Process changes
|
||||
changes_made: list[ConfigChange] = []
|
||||
raw_changes = response_data.get("changes", [])
|
||||
|
||||
if isinstance(raw_changes, list):
|
||||
for change_item in raw_changes:
|
||||
if not isinstance(change_item, dict):
|
||||
continue
|
||||
|
||||
change_dict: dict[str, object] = change_item
|
||||
field = str(change_dict.get("field", ""))
|
||||
new_value = str(change_dict.get("new_value", ""))
|
||||
|
||||
if field not in ("deriver_rules", "dialectic_rules"):
|
||||
continue
|
||||
|
||||
# Get previous value
|
||||
previous_value = (
|
||||
current_config.deriver_rules
|
||||
if field == "deriver_rules"
|
||||
else current_config.dialectic_rules
|
||||
)
|
||||
|
||||
# Skip if no actual change
|
||||
if previous_value == new_value:
|
||||
continue
|
||||
|
||||
changes_made.append(
|
||||
ConfigChange(
|
||||
field=field, # type: ignore[arg-type]
|
||||
previous_value=previous_value,
|
||||
new_value=new_value,
|
||||
)
|
||||
)
|
||||
|
||||
# Apply changes if any
|
||||
if changes_made:
|
||||
new_config = WorkspaceAgentConfig(
|
||||
deriver_rules=current_config.deriver_rules,
|
||||
dialectic_rules=current_config.dialectic_rules,
|
||||
)
|
||||
|
||||
for change in changes_made:
|
||||
if change.field == "deriver_rules":
|
||||
new_config.deriver_rules = change.new_value
|
||||
elif change.field == "dialectic_rules":
|
||||
new_config.dialectic_rules = change.new_value
|
||||
|
||||
await crud.set_workspace_agent_config(db, workspace_name, new_config)
|
||||
current_config = new_config
|
||||
|
||||
logger.info(
|
||||
f"Feedback channel: Applied {len(changes_made)} changes to workspace {workspace_name}"
|
||||
)
|
||||
|
||||
message = response_data.get("message", "Configuration updated.")
|
||||
understood_intent = response_data.get("understood_intent", "Processed feedback")
|
||||
|
||||
return FeedbackResponse(
|
||||
message=str(message),
|
||||
understood_intent=str(understood_intent),
|
||||
changes_made=changes_made,
|
||||
current_config=current_config,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Feedback channel LLM call failed: {e}")
|
||||
return FeedbackResponse(
|
||||
message="I encountered an error processing your feedback. Please try again.",
|
||||
understood_intent="Error during processing",
|
||||
changes_made=[],
|
||||
current_config=current_config,
|
||||
)
|
||||
|
|
@ -10,7 +10,9 @@ from src import crud, models, schemas
|
|||
from src.config import settings
|
||||
from src.dependencies import db
|
||||
from src.deriver.enqueue import enqueue_dream
|
||||
from src.dreamer.introspection import get_latest_introspection_report
|
||||
from src.exceptions import AuthenticationException
|
||||
from src.feedback import process_feedback
|
||||
from src.security import JWTParams, require_auth
|
||||
from src.telemetry.events import DeletionCompletedEvent, emit
|
||||
from src.utils.search import search
|
||||
|
|
@ -258,3 +260,33 @@ async def schedule_dream(
|
|||
observed,
|
||||
request.session_id,
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/{workspace_id}/feedback",
|
||||
response_model=schemas.FeedbackResponse,
|
||||
dependencies=[Depends(require_auth(workspace_name="workspace_id"))],
|
||||
)
|
||||
async def process_developer_feedback(
|
||||
workspace_id: str = Path(...),
|
||||
request: schemas.FeedbackRequest = Body(...),
|
||||
db: AsyncSession = db,
|
||||
):
|
||||
"""
|
||||
Process developer feedback and update workspace agent configuration.
|
||||
|
||||
This endpoint provides a natural language interface for developers to configure
|
||||
Honcho's agent behavior. Developers can give instructions, ask questions, and
|
||||
receive configuration updates - all via conversation.
|
||||
|
||||
The feedback channel supports:
|
||||
- First-time setup with interview questions
|
||||
- Incremental configuration updates
|
||||
- Questions about current configuration
|
||||
- Introspection-informed suggestions (when include_introspection=True)
|
||||
"""
|
||||
introspection_report = None
|
||||
if request.include_introspection:
|
||||
introspection_report = await get_latest_introspection_report(db, workspace_id)
|
||||
|
||||
return await process_feedback(db, workspace_id, request, introspection_report)
|
||||
|
|
|
|||
|
|
@ -83,6 +83,30 @@ class IntrospectionReport(BaseModel):
|
|||
signals: IntrospectionSignals
|
||||
|
||||
|
||||
class FeedbackRequest(BaseModel):
|
||||
"""Request to the developer feedback channel."""
|
||||
|
||||
message: str = Field(..., min_length=1, max_length=10000)
|
||||
include_introspection: bool = Field(default=False)
|
||||
|
||||
|
||||
class ConfigChange(BaseModel):
|
||||
"""A configuration change made by the feedback processor."""
|
||||
|
||||
field: Literal["deriver_rules", "dialectic_rules"]
|
||||
previous_value: str
|
||||
new_value: str
|
||||
|
||||
|
||||
class FeedbackResponse(BaseModel):
|
||||
"""Response from the developer feedback channel."""
|
||||
|
||||
message: str
|
||||
understood_intent: str
|
||||
changes_made: list[ConfigChange] = Field(default_factory=list)
|
||||
current_config: WorkspaceAgentConfig
|
||||
|
||||
|
||||
class ReconcilerType(str, Enum):
|
||||
"""Types of reconciler tasks that can be performed."""
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,514 @@
|
|||
"""Tests for the developer feedback channel (Phase 4 of Agentic FDE)."""
|
||||
|
||||
import json
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from src import crud, models
|
||||
from src.feedback import (
|
||||
INTERVIEW_QUESTIONS,
|
||||
_config_is_empty,
|
||||
_is_simple_greeting,
|
||||
build_feedback_prompt,
|
||||
process_feedback,
|
||||
)
|
||||
from src.schemas import (
|
||||
ConfigChange,
|
||||
FeedbackRequest,
|
||||
FeedbackResponse,
|
||||
IntrospectionReport,
|
||||
IntrospectionSignals,
|
||||
IntrospectionSuggestion,
|
||||
WorkspaceAgentConfig,
|
||||
)
|
||||
|
||||
|
||||
class TestFeedbackSchemas:
|
||||
"""Test the feedback-related Pydantic schemas."""
|
||||
|
||||
def test_feedback_request_basic(self):
|
||||
"""Test basic FeedbackRequest creation."""
|
||||
request = FeedbackRequest(message="Hello")
|
||||
assert request.message == "Hello"
|
||||
assert request.include_introspection is False
|
||||
|
||||
def test_feedback_request_with_introspection(self):
|
||||
"""Test FeedbackRequest with introspection enabled."""
|
||||
request = FeedbackRequest(
|
||||
message="Configure my workspace", include_introspection=True
|
||||
)
|
||||
assert request.message == "Configure my workspace"
|
||||
assert request.include_introspection is True
|
||||
|
||||
def test_feedback_request_validation(self):
|
||||
"""Test FeedbackRequest validation."""
|
||||
# Empty message should fail
|
||||
with pytest.raises(ValidationError):
|
||||
FeedbackRequest(message="")
|
||||
|
||||
# Very long message should fail
|
||||
with pytest.raises(ValidationError):
|
||||
FeedbackRequest(message="x" * 10001)
|
||||
|
||||
def test_config_change(self):
|
||||
"""Test ConfigChange schema."""
|
||||
change = ConfigChange(
|
||||
field="deriver_rules",
|
||||
previous_value="old rule",
|
||||
new_value="new rule",
|
||||
)
|
||||
assert change.field == "deriver_rules"
|
||||
assert change.previous_value == "old rule"
|
||||
assert change.new_value == "new rule"
|
||||
|
||||
def test_config_change_dialectic(self):
|
||||
"""Test ConfigChange for dialectic_rules."""
|
||||
change = ConfigChange(
|
||||
field="dialectic_rules",
|
||||
previous_value="",
|
||||
new_value="Be concise",
|
||||
)
|
||||
assert change.field == "dialectic_rules"
|
||||
|
||||
def test_feedback_response(self):
|
||||
"""Test FeedbackResponse schema."""
|
||||
response = FeedbackResponse(
|
||||
message="Configuration updated",
|
||||
understood_intent="Add focus on emotions",
|
||||
changes_made=[
|
||||
ConfigChange(
|
||||
field="deriver_rules",
|
||||
previous_value="",
|
||||
new_value="Focus on emotions",
|
||||
)
|
||||
],
|
||||
current_config=WorkspaceAgentConfig(deriver_rules="Focus on emotions"),
|
||||
)
|
||||
assert response.message == "Configuration updated"
|
||||
assert len(response.changes_made) == 1
|
||||
assert response.current_config.deriver_rules == "Focus on emotions"
|
||||
|
||||
|
||||
class TestHelperFunctions:
|
||||
"""Test helper functions for feedback processing."""
|
||||
|
||||
def test_is_simple_greeting_true_cases(self):
|
||||
"""Test cases that should be detected as simple greetings."""
|
||||
greetings = [
|
||||
"hello",
|
||||
"Hello",
|
||||
"HELLO",
|
||||
"hi",
|
||||
"Hi there",
|
||||
"hey",
|
||||
"Hey!",
|
||||
"good morning",
|
||||
"Good afternoon",
|
||||
"howdy",
|
||||
"help",
|
||||
"Help me",
|
||||
"how do i configure",
|
||||
"configure",
|
||||
"setup",
|
||||
"get started",
|
||||
]
|
||||
for greeting in greetings:
|
||||
assert _is_simple_greeting(greeting), f"Expected '{greeting}' to be a greeting"
|
||||
|
||||
def test_is_simple_greeting_false_cases(self):
|
||||
"""Test cases that should NOT be detected as simple greetings."""
|
||||
non_greetings = [
|
||||
"I'm building a journaling app and want to focus on emotions",
|
||||
"The deriver should extract technical facts only",
|
||||
"x" * 101, # Too long
|
||||
"Please configure the workspace to focus on user preferences",
|
||||
]
|
||||
for msg in non_greetings:
|
||||
assert not _is_simple_greeting(msg), f"Expected '{msg}' to NOT be a greeting"
|
||||
|
||||
def test_config_is_empty_true(self):
|
||||
"""Test detecting empty configuration."""
|
||||
config = WorkspaceAgentConfig()
|
||||
assert _config_is_empty(config)
|
||||
|
||||
config = WorkspaceAgentConfig(deriver_rules="", dialectic_rules="")
|
||||
assert _config_is_empty(config)
|
||||
|
||||
config = WorkspaceAgentConfig(deriver_rules=" ", dialectic_rules=" ")
|
||||
assert _config_is_empty(config)
|
||||
|
||||
def test_config_is_empty_false(self):
|
||||
"""Test detecting non-empty configuration."""
|
||||
config = WorkspaceAgentConfig(deriver_rules="Some rule")
|
||||
assert not _config_is_empty(config)
|
||||
|
||||
config = WorkspaceAgentConfig(dialectic_rules="Another rule")
|
||||
assert not _config_is_empty(config)
|
||||
|
||||
|
||||
class TestBuildFeedbackPrompt:
|
||||
"""Test the prompt building function."""
|
||||
|
||||
def test_basic_prompt(self):
|
||||
"""Test basic prompt without introspection."""
|
||||
prompt = build_feedback_prompt(
|
||||
message="Focus on emotions",
|
||||
current_config=WorkspaceAgentConfig(),
|
||||
)
|
||||
assert "Focus on emotions" in prompt
|
||||
assert "(empty - using defaults)" in prompt
|
||||
assert "Developer Message" in prompt
|
||||
|
||||
def test_prompt_with_existing_config(self):
|
||||
"""Test prompt includes existing configuration."""
|
||||
config = WorkspaceAgentConfig(
|
||||
deriver_rules="Extract technical facts",
|
||||
dialectic_rules="Be concise",
|
||||
)
|
||||
prompt = build_feedback_prompt(
|
||||
message="Add emotion tracking",
|
||||
current_config=config,
|
||||
)
|
||||
assert "Extract technical facts" in prompt
|
||||
assert "Be concise" in prompt
|
||||
|
||||
def test_prompt_with_introspection(self):
|
||||
"""Test prompt includes introspection report when provided."""
|
||||
import datetime
|
||||
|
||||
report = IntrospectionReport(
|
||||
workspace_name="test",
|
||||
generated_at=datetime.datetime.now(datetime.timezone.utc),
|
||||
performance_summary="Good performance overall",
|
||||
identified_issues=["High abstention rate"],
|
||||
suggestions=[
|
||||
IntrospectionSuggestion(
|
||||
target="deriver_rules",
|
||||
current_value="",
|
||||
suggested_value="Focus more",
|
||||
rationale="Would reduce abstentions",
|
||||
confidence="high",
|
||||
)
|
||||
],
|
||||
signals=IntrospectionSignals(),
|
||||
)
|
||||
|
||||
prompt = build_feedback_prompt(
|
||||
message="Help me improve",
|
||||
current_config=WorkspaceAgentConfig(),
|
||||
introspection_report=report,
|
||||
)
|
||||
assert "Good performance overall" in prompt
|
||||
assert "High abstention rate" in prompt
|
||||
assert "Would reduce abstentions" in prompt
|
||||
|
||||
|
||||
class TestProcessFeedback:
|
||||
"""Test the main feedback processing function."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_interview_mode_trigger(
|
||||
self,
|
||||
db_session: AsyncSession,
|
||||
sample_data: tuple[models.Workspace, models.Peer],
|
||||
):
|
||||
"""Test that interview mode is triggered for empty config + greeting."""
|
||||
workspace, _ = sample_data
|
||||
|
||||
request = FeedbackRequest(message="Hello")
|
||||
response = await process_feedback(db_session, workspace.name, request)
|
||||
|
||||
assert INTERVIEW_QUESTIONS in response.message
|
||||
assert "First-time setup" in response.understood_intent
|
||||
assert len(response.changes_made) == 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_interview_mode_not_triggered_with_config(
|
||||
self,
|
||||
db_session: AsyncSession,
|
||||
sample_data: tuple[models.Workspace, models.Peer],
|
||||
):
|
||||
"""Test that interview mode is NOT triggered when config exists."""
|
||||
workspace, _ = sample_data
|
||||
|
||||
# Set up existing config
|
||||
config = WorkspaceAgentConfig(deriver_rules="Existing rule")
|
||||
await crud.set_workspace_agent_config(db_session, workspace.name, config)
|
||||
|
||||
# Mock the LLM call
|
||||
mock_response = MagicMock()
|
||||
mock_response.content = json.dumps({
|
||||
"message": "I see you already have rules set up.",
|
||||
"understood_intent": "Greeting with existing config",
|
||||
"changes": [],
|
||||
})
|
||||
|
||||
with patch(
|
||||
"src.feedback.honcho_llm_call",
|
||||
new_callable=AsyncMock,
|
||||
return_value=mock_response,
|
||||
):
|
||||
request = FeedbackRequest(message="Hello")
|
||||
response = await process_feedback(db_session, workspace.name, request)
|
||||
|
||||
# Should NOT be interview mode
|
||||
assert INTERVIEW_QUESTIONS not in response.message
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_config_update(
|
||||
self,
|
||||
db_session: AsyncSession,
|
||||
sample_data: tuple[models.Workspace, models.Peer],
|
||||
):
|
||||
"""Test that config changes are applied correctly."""
|
||||
workspace, _ = sample_data
|
||||
|
||||
# Mock the LLM call to return a config change
|
||||
mock_response = MagicMock()
|
||||
mock_response.content = json.dumps({
|
||||
"message": "I've configured the workspace to focus on emotions.",
|
||||
"understood_intent": "Configure deriver for emotion tracking",
|
||||
"changes": [
|
||||
{
|
||||
"field": "deriver_rules",
|
||||
"new_value": "Focus on emotional content and feelings",
|
||||
}
|
||||
],
|
||||
})
|
||||
|
||||
with patch(
|
||||
"src.feedback.honcho_llm_call",
|
||||
new_callable=AsyncMock,
|
||||
return_value=mock_response,
|
||||
):
|
||||
request = FeedbackRequest(
|
||||
message="I'm building a journaling app, focus on emotions"
|
||||
)
|
||||
response = await process_feedback(db_session, workspace.name, request)
|
||||
|
||||
assert len(response.changes_made) == 1
|
||||
assert response.changes_made[0].field == "deriver_rules"
|
||||
assert "emotion" in response.changes_made[0].new_value.lower()
|
||||
|
||||
# Verify config was actually saved
|
||||
saved_config = await crud.get_workspace_agent_config(
|
||||
db_session, workspace.name
|
||||
)
|
||||
assert "emotion" in saved_config.deriver_rules.lower()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_question_no_changes(
|
||||
self,
|
||||
db_session: AsyncSession,
|
||||
sample_data: tuple[models.Workspace, models.Peer],
|
||||
):
|
||||
"""Test that questions don't result in config changes."""
|
||||
workspace, _ = sample_data
|
||||
|
||||
# Set up existing config
|
||||
config = WorkspaceAgentConfig(deriver_rules="Existing rule")
|
||||
await crud.set_workspace_agent_config(db_session, workspace.name, config)
|
||||
|
||||
# Mock LLM to return answer without changes
|
||||
mock_response = MagicMock()
|
||||
mock_response.content = json.dumps({
|
||||
"message": "Your current deriver rule is: 'Existing rule'",
|
||||
"understood_intent": "Question about current config",
|
||||
"changes": [],
|
||||
})
|
||||
|
||||
with patch(
|
||||
"src.feedback.honcho_llm_call",
|
||||
new_callable=AsyncMock,
|
||||
return_value=mock_response,
|
||||
):
|
||||
request = FeedbackRequest(message="What are my current rules?")
|
||||
response = await process_feedback(db_session, workspace.name, request)
|
||||
|
||||
assert len(response.changes_made) == 0
|
||||
# Config should be unchanged
|
||||
assert response.current_config.deriver_rules == "Existing rule"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_llm_error_handling(
|
||||
self,
|
||||
db_session: AsyncSession,
|
||||
sample_data: tuple[models.Workspace, models.Peer],
|
||||
):
|
||||
"""Test graceful handling of LLM errors."""
|
||||
workspace, _ = sample_data
|
||||
|
||||
# Set initial config
|
||||
config = WorkspaceAgentConfig(deriver_rules="Existing rule")
|
||||
await crud.set_workspace_agent_config(db_session, workspace.name, config)
|
||||
|
||||
with patch(
|
||||
"src.feedback.honcho_llm_call",
|
||||
new_callable=AsyncMock,
|
||||
side_effect=Exception("LLM service unavailable"),
|
||||
):
|
||||
request = FeedbackRequest(message="Update my config")
|
||||
response = await process_feedback(db_session, workspace.name, request)
|
||||
|
||||
# Should return error message
|
||||
assert "error" in response.message.lower()
|
||||
assert len(response.changes_made) == 0
|
||||
# Config should be unchanged
|
||||
assert response.current_config.deriver_rules == "Existing rule"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invalid_json_response(
|
||||
self,
|
||||
db_session: AsyncSession,
|
||||
sample_data: tuple[models.Workspace, models.Peer],
|
||||
):
|
||||
"""Test handling of invalid JSON from LLM."""
|
||||
workspace, _ = sample_data
|
||||
|
||||
# Set initial config
|
||||
config = WorkspaceAgentConfig(deriver_rules="Existing rule")
|
||||
await crud.set_workspace_agent_config(db_session, workspace.name, config)
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.content = "This is not valid JSON"
|
||||
|
||||
with patch(
|
||||
"src.feedback.honcho_llm_call",
|
||||
new_callable=AsyncMock,
|
||||
return_value=mock_response,
|
||||
):
|
||||
request = FeedbackRequest(message="Update my config")
|
||||
response = await process_feedback(db_session, workspace.name, request)
|
||||
|
||||
# Should return error message
|
||||
assert "trouble" in response.message.lower()
|
||||
assert len(response.changes_made) == 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_incremental_update(
|
||||
self,
|
||||
db_session: AsyncSession,
|
||||
sample_data: tuple[models.Workspace, models.Peer],
|
||||
):
|
||||
"""Test that new rules are added to existing rules."""
|
||||
workspace, _ = sample_data
|
||||
|
||||
# Set initial config
|
||||
config = WorkspaceAgentConfig(deriver_rules="Track emotions")
|
||||
await crud.set_workspace_agent_config(db_session, workspace.name, config)
|
||||
|
||||
# Mock LLM to append new rule
|
||||
mock_response = MagicMock()
|
||||
mock_response.content = json.dumps({
|
||||
"message": "Added goal tracking to existing rules.",
|
||||
"understood_intent": "Add goal tracking while preserving emotion tracking",
|
||||
"changes": [
|
||||
{
|
||||
"field": "deriver_rules",
|
||||
"new_value": "Track emotions\nAlso track goals and aspirations",
|
||||
}
|
||||
],
|
||||
})
|
||||
|
||||
with patch(
|
||||
"src.feedback.honcho_llm_call",
|
||||
new_callable=AsyncMock,
|
||||
return_value=mock_response,
|
||||
):
|
||||
request = FeedbackRequest(message="Also track goals")
|
||||
response = await process_feedback(db_session, workspace.name, request)
|
||||
|
||||
assert len(response.changes_made) == 1
|
||||
# Should contain both old and new rules
|
||||
assert "emotions" in response.changes_made[0].new_value.lower()
|
||||
assert "goals" in response.changes_made[0].new_value.lower()
|
||||
|
||||
|
||||
class TestFeedbackAPIEndpoint:
|
||||
"""Test the /feedback API endpoint via HTTP."""
|
||||
|
||||
def test_feedback_endpoint_basic(
|
||||
self,
|
||||
client,
|
||||
sample_data: tuple[models.Workspace, models.Peer],
|
||||
):
|
||||
"""Test basic feedback endpoint functionality."""
|
||||
workspace, _ = sample_data
|
||||
|
||||
# Mock the process_feedback function
|
||||
mock_response = FeedbackResponse(
|
||||
message="Configuration updated",
|
||||
understood_intent="Test intent",
|
||||
changes_made=[],
|
||||
current_config=WorkspaceAgentConfig(),
|
||||
)
|
||||
|
||||
with patch(
|
||||
"src.routers.workspaces.process_feedback",
|
||||
new_callable=AsyncMock,
|
||||
return_value=mock_response,
|
||||
):
|
||||
response = client.post(
|
||||
f"/v1/workspaces/{workspace.name}/feedback",
|
||||
json={"message": "Test message"},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["message"] == "Configuration updated"
|
||||
assert "current_config" in data
|
||||
|
||||
def test_feedback_endpoint_with_introspection(
|
||||
self,
|
||||
client,
|
||||
sample_data: tuple[models.Workspace, models.Peer],
|
||||
):
|
||||
"""Test feedback endpoint with introspection flag."""
|
||||
workspace, _ = sample_data
|
||||
|
||||
mock_response = FeedbackResponse(
|
||||
message="Used introspection data",
|
||||
understood_intent="Test intent",
|
||||
changes_made=[],
|
||||
current_config=WorkspaceAgentConfig(),
|
||||
)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"src.routers.workspaces.process_feedback",
|
||||
new_callable=AsyncMock,
|
||||
return_value=mock_response,
|
||||
) as mock_process,
|
||||
patch(
|
||||
"src.routers.workspaces.get_latest_introspection_report",
|
||||
new_callable=AsyncMock,
|
||||
return_value=None,
|
||||
) as mock_introspection,
|
||||
):
|
||||
response = client.post(
|
||||
f"/v1/workspaces/{workspace.name}/feedback",
|
||||
json={"message": "Help me", "include_introspection": True},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
# Verify introspection was fetched
|
||||
mock_introspection.assert_called_once()
|
||||
# Verify process_feedback was called
|
||||
mock_process.assert_called_once()
|
||||
|
||||
def test_feedback_endpoint_validation_error(
|
||||
self,
|
||||
client,
|
||||
sample_data: tuple[models.Workspace, models.Peer],
|
||||
):
|
||||
"""Test feedback endpoint with invalid input."""
|
||||
workspace, _ = sample_data
|
||||
|
||||
# Empty message should fail validation
|
||||
response = client.post(
|
||||
f"/v1/workspaces/{workspace.name}/feedback",
|
||||
json={"message": ""},
|
||||
)
|
||||
assert response.status_code == 422 # Validation error
|
||||
Loading…
Reference in New Issue