From 73e3297e7270af3ca118c82c1647fec8ef7acffc Mon Sep 17 00:00:00 2001 From: Benjamin McCormick Date: Thu, 29 Jan 2026 16:44:12 -0500 Subject: [PATCH] chore: type-checking cleanup, fix tests --- agentic_fde.md | 114 ++++++++++++ .../documentation/introduction/overview.mdx | 4 +- ...a8f2d4e6c9b1_add_dialectic_traces_table.py | 4 +- src/crud/dialectic_trace.py | 4 +- src/dialectic/core.py | 4 +- src/dreamer/introspection.py | 64 ++++--- src/feedback.py | 16 +- tests/alembic/revisions/__init__.py | 2 + ...a8f2d4e6c9b1_add_dialectic_traces_table.py | 54 ++++++ tests/integration/test_token_metrics.py | 12 ++ tests/routes/test_feedback.py | 91 +++++++++ tests/test_dialectic_trace.py | 20 +- tests/test_feedback.py | 174 +++++------------- tests/test_introspection.py | 34 ++-- 14 files changed, 400 insertions(+), 197 deletions(-) create mode 100644 agentic_fde.md create mode 100644 tests/alembic/revisions/test_a8f2d4e6c9b1_add_dialectic_traces_table.py create mode 100644 tests/routes/test_feedback.py diff --git a/agentic_fde.md b/agentic_fde.md new file mode 100644 index 00000000..1cb93ef4 --- /dev/null +++ b/agentic_fde.md @@ -0,0 +1,114 @@ +# Agentic FDE: Self-Adapting Honcho + +## Vision + +The software market is bifurcating: massive enterprise vs solopreneur vibe-coders. Enterprise requires human FDEs; vibe-coders won't pay for human help but also won't pay for one-size-fits-all SaaS. The solution: make Honcho itself an "Agentic FDE" that adapts to each developer's use case. + +Honcho observes usage patterns, engages in meta-cognition about developer goals, and adapts its behavior accordingly. A companion app needs emotional memory extraction and biographical recall. A coding agent needs preference/constraint extraction and should ignore stack traces. An email ingestion pipeline needs RAG, not conversation memory. + +The same primitives (workspaces, peers, sessions, messages, documents) can achieve any memory pattern - but the prompts and retrieval strategies must adapt. + +## Core Principles + +1. **Stable API, stable schema** - Honcho adapts its behavior, not its interface +2. **Developer feedback is highest priority** - Observed patterns can be overridden +3. **Two adaptation questions**: + - How should I handle the next marginal message? (deriver) + - How should I handle the next marginal .chat query? (dialectic) +4. **Constraints**: No touching deletion endpoints, workspace isolation, or the core reasoning model + +## The 5-Phase Plan + +### Phase 1: Instrumentation ✅ + +**Goal**: Log dialectic interactions so the dreamer can analyze performance. + +**Built**: + +- `DialecticTrace` model: workspace, session, observer, observed, query, retrieved_doc_ids, tool_calls, response, reasoning_level, duration, tokens, timestamps +- CRUD operations: `create_dialectic_trace()`, `get_dialectic_traces()`, `get_dialectic_trace_stats()` +- Abstention detection via regex patterns +- Integration: traces written at end of `DialecticAgent._log_response_metrics()` + +**Files**: `src/models.py`, `src/crud/dialectic_trace.py`, `src/dialectic/core.py`, `tests/test_dialectic_trace.py` + +### Phase 2: Prompt Injection Points ✅ + +**Goal**: Enable workspace-level prompt customization without changing default behavior. + +**Built**: + +- `WorkspaceAgentConfig` schema with `deriver_rules` and `dialectic_rules` fields +- Storage in `workspace.metadata["_agent_config"]` +- CRUD helpers: `get_workspace_agent_config()`, `set_workspace_agent_config()` +- Deriver prompt injection: `custom_rules` parameter in `minimal_deriver_prompt()` +- Dialectic prompt injection: `custom_rules` parameter in `agent_system_prompt()` +- Config threading through deriver and dialectic paths + +**Files**: `src/schemas.py`, `src/crud/workspace.py`, `src/deriver/prompts.py`, `src/deriver/deriver.py`, `src/dialectic/prompts.py`, `src/dialectic/core.py`, `src/dialectic/chat.py`, `tests/test_workspace_agent_config.py` + +### Phase 3: Meta-Cognitive Dreamer ✅ + +**Goal**: Dreamer analyzes logs and generates configuration suggestions. + +**Built**: + +- `DreamType.INTROSPECTION` enum value +- `IntrospectionSignals`, `IntrospectionSuggestion`, `IntrospectionReport` schemas +- `gather_introspection_context()` - collects dialectic stats, observation counts, peer/session patterns +- `build_introspection_prompt()` - formats signals for LLM analysis +- `run_introspection()` - calls LLM, parses structured suggestions +- `store_introspection_report()` - saves reports as documents in `_system`/`_introspection` collection +- `get_latest_introspection_report()` - retrieves most recent report +- Wired into `DreamType.INTROSPECTION` in orchestrator + +**Files**: `src/schemas.py`, `src/dreamer/introspection.py`, `src/dreamer/orchestrator.py`, `tests/test_introspection.py` + +### Phase 4: Developer Feedback Channel ✅ + +**Goal**: Developers can talk to Honcho about Honcho. + +**Built**: + +- `POST /workspaces/{id}/feedback` endpoint +- `FeedbackRequest`, `ConfigChange`, `FeedbackResponse` schemas +- `process_feedback()` - handles natural language feedback +- `build_feedback_prompt()` - formats context for LLM +- Interview mode: empty config + greeting triggers onboarding questions +- Incremental updates: preserves existing rules when adding new ones +- Introspection context: optionally includes latest report +- Uses `settings.DREAM` for LLM calls (not billed as dialectic) + +**Files**: `src/schemas.py`, `src/feedback.py`, `src/routers/workspaces.py`, `src/dreamer/introspection.py`, `tests/test_feedback.py` + +### Phase 5: Closed Loop (Future) + +**Goal**: Automatic adaptation with developer oversight. + +**To build**: + +- Dreamer introspection generates draft config changes +- Surfaces to developer via webhook or dashboard +- Developer approves/rejects/modifies +- Approved changes written to config +- Optional: `workspace.meta.auto_adapt = true` for brave workspaces + +## Summary Statistics + +| Phase | Lines Added | Test Coverage | +|-------|-------------|---------------| +| 1 | ~500 | 376 lines (13 tests) | +| 2 | ~200 | 246 lines (15 tests) | +| 3 | ~500 | 423 lines (10 tests) | +| 4 | ~350 | 514 lines (23 tests) | +| **Total** | **~2,950** | **1,559 lines (61 tests)** | + +## Testing Checklist + +- [ ] Run full test suite: `uv run pytest tests/` +- [ ] Test Phase 1: Create a dialectic query, verify trace is logged +- [ ] Test Phase 2: Set workspace agent config, verify rules appear in prompts +- [ ] Test Phase 3: Trigger introspection dream, verify report generated +- [ ] Test Phase 4: Submit feedback, verify config updated +- [ ] Test interview flow: New workspace + greeting triggers questions +- [ ] Test incremental updates: Existing rules preserved when adding new ones diff --git a/docs/v3/documentation/introduction/overview.mdx b/docs/v3/documentation/introduction/overview.mdx index 10ce0775..53cc2d48 100644 --- a/docs/v3/documentation/introduction/overview.mdx +++ b/docs/v3/documentation/introduction/overview.mdx @@ -65,7 +65,7 @@ Honcho has four storage primitives that work together: - **Workspaces** - Top-level containers that isolate different applications or environments - **Peers** - Any entity that persists but changes over time (users, agents, objects, and more) - **Sessions** - Interaction threads between peers with temporal boundaries -- **Messages** - Units of data that trigger reasoning (conversations, events, activity, documents, and more) +- **Messages** - Units of data that trigger reasoning (conversations, events, activity, documents, and more) When you write messages to Honcho, they're stored and processed in the background. Custom reasoning models perform formal logical [_reasoning_](/v3/documentation/core-concepts/reasoning) to generate conclusions about each peer. These conclusions are stored as [_representations_](/v3/documentation/core-concepts/representation) that you can query to provide rich context for your agents. @@ -100,4 +100,4 @@ Welcome to Honcho. We're excited to have you at the frontier of AI with us 🫡. Learn how Honcho reasons about data to build memory - \ No newline at end of file + diff --git a/migrations/versions/a8f2d4e6c9b1_add_dialectic_traces_table.py b/migrations/versions/a8f2d4e6c9b1_add_dialectic_traces_table.py index 82c21aac..8e1999a2 100644 --- a/migrations/versions/a8f2d4e6c9b1_add_dialectic_traces_table.py +++ b/migrations/versions/a8f2d4e6c9b1_add_dialectic_traces_table.py @@ -1,7 +1,7 @@ """add dialectic_traces table Revision ID: a8f2d4e6c9b1 -Revises: 7c0d9a4e3b1f +Revises: e4eba9cfaa6f Create Date: 2026-01-29 """ @@ -16,7 +16,7 @@ from migrations.utils import get_schema # revision identifiers, used by Alembic. revision: str = "a8f2d4e6c9b1" -down_revision: str | None = "7c0d9a4e3b1f" +down_revision: str | None = "e4eba9cfaa6f" branch_labels: str | Sequence[str] | None = None depends_on: str | Sequence[str] | None = None diff --git a/src/crud/dialectic_trace.py b/src/crud/dialectic_trace.py index e203cbbf..0a65ac61 100644 --- a/src/crud/dialectic_trace.py +++ b/src/crud/dialectic_trace.py @@ -23,7 +23,7 @@ ABSTENTION_PATTERNS = [ _ABSTENTION_REGEX = re.compile("|".join(ABSTENTION_PATTERNS), re.IGNORECASE) -def _is_abstention(response: str) -> bool: +def is_abstention(response: str) -> bool: """Check if a response indicates abstention from answering.""" return bool(_ABSTENTION_REGEX.search(response)) @@ -132,7 +132,7 @@ async def get_dialectic_trace_stats( responses_result = await db.execute(responses_stmt) responses = [r[0] for r in responses_result.all()] - abstention_count = sum(1 for r in responses if _is_abstention(r)) + abstention_count = sum(1 for r in responses if is_abstention(r)) abstention_rate = abstention_count / total_queries if total_queries > 0 else 0.0 return { diff --git a/src/dialectic/core.py b/src/dialectic/core.py index c59f1c49..28f7e964 100644 --- a/src/dialectic/core.py +++ b/src/dialectic/core.py @@ -46,7 +46,7 @@ logger = logging.getLogger(__name__) _DOC_ID_PATTERN = re.compile(r"\[id:([a-zA-Z0-9_-]+)\]") -def _extract_doc_ids_from_messages(messages: list[dict[str, str]]) -> list[str]: +def extract_doc_ids_from_messages(messages: list[dict[str, str]]) -> list[str]: """ Extract document IDs from tool_result messages. @@ -410,7 +410,7 @@ class DialecticAgent: ) # Persist dialectic trace for meta-cognitive analysis - retrieved_doc_ids = _extract_doc_ids_from_messages(self.messages) + retrieved_doc_ids = extract_doc_ids_from_messages(self.messages) trace = DialecticTraceCreate( workspace_name=self.workspace_name, session_name=self.session_name, diff --git a/src/dreamer/introspection.py b/src/dreamer/introspection.py index b87de109..3e41a870 100644 --- a/src/dreamer/introspection.py +++ b/src/dreamer/introspection.py @@ -10,6 +10,7 @@ from __future__ import annotations import datetime import json import logging +from typing import Any, cast from sqlalchemy import func, select from sqlalchemy.ext.asyncio import AsyncSession @@ -314,43 +315,48 @@ async def run_introspection( # Build suggestions list suggestions: list[IntrospectionSuggestion] = [] - raw_suggestions = response_data.get("suggestions", []) - if isinstance(raw_suggestions, list): - for raw_suggestion in raw_suggestions: - if isinstance(raw_suggestion, dict): - try: - # Validate target field - target_raw = raw_suggestion.get("target", "deriver_rules") - if target_raw not in ("deriver_rules", "dialectic_rules"): - target_raw = "deriver_rules" + raw_suggestions: list[dict[str, Any]] = cast( + list[dict[str, Any]], response_data.get("suggestions", []) + ) + for raw_suggestion in raw_suggestions: + try: + # Validate target field + target_raw = raw_suggestion.get("target", "deriver_rules") + if target_raw not in ("deriver_rules", "dialectic_rules"): + target_raw = "deriver_rules" - # Validate confidence field - confidence_raw = raw_suggestion.get("confidence", "low") - if confidence_raw not in ("high", "medium", "low"): - confidence_raw = "low" + # Validate confidence field + confidence_raw = raw_suggestion.get("confidence", "low") + if confidence_raw not in ("high", "medium", "low"): + confidence_raw = "low" - suggestions.append( - IntrospectionSuggestion( - target=target_raw, # type: ignore[arg-type] - current_value=str(raw_suggestion.get("current_value", "")), - suggested_value=str(raw_suggestion.get("suggested_value", "")), - rationale=str(raw_suggestion.get("rationale", "")), - confidence=confidence_raw, # type: ignore[arg-type] - ) - ) - except Exception as e: - logger.warning(f"Failed to parse suggestion: {e}") + suggestions.append( + IntrospectionSuggestion( + target=target_raw, # type: ignore[arg-type] + current_value=cast( + str, raw_suggestion.get("current_value", "") + ), + suggested_value=str(raw_suggestion.get("suggested_value", "")), + rationale=str(raw_suggestion.get("rationale", "")), + confidence=confidence_raw, # type: ignore[arg-type] + ) + ) + except Exception as e: + logger.warning(f"Failed to parse suggestion: {e}") # Extract fields with type coercion - performance_summary = response_data.get("performance_summary", "Analysis completed.") + performance_summary = response_data.get( + "performance_summary", "Analysis completed." + ) if not isinstance(performance_summary, str): performance_summary = "Analysis completed." - identified_issues_raw = response_data.get("identified_issues", []) + identified_issues_raw: list[Any] = cast( + list[Any], response_data.get("identified_issues", []) + ) identified_issues: list[str] = [] - if isinstance(identified_issues_raw, list): - for issue in identified_issues_raw: - identified_issues.append(str(issue)) + for issue in identified_issues_raw: + identified_issues.append(str(issue)) report = IntrospectionReport( workspace_name=workspace_name, diff --git a/src/feedback.py b/src/feedback.py index 65d57920..69d66ebd 100644 --- a/src/feedback.py +++ b/src/feedback.py @@ -10,6 +10,7 @@ from __future__ import annotations import json import logging import re +from typing import Any, cast from sqlalchemy.ext.asyncio import AsyncSession @@ -40,7 +41,7 @@ INTERVIEW_QUESTIONS = """Before I can help configure Honcho for your workspace, Feel free to answer any or all of these questions, and I'll help configure your workspace accordingly.""" -def _is_simple_greeting(message: str) -> bool: +def is_simple_greeting(message: str) -> bool: """Check if a message is a simple greeting or question that should trigger interview mode.""" message_lower = message.lower().strip() @@ -72,7 +73,7 @@ def _is_simple_greeting(message: str) -> bool: return any(re.match(pattern, message_lower) for pattern in greeting_patterns) -def _config_is_empty(config: WorkspaceAgentConfig) -> bool: +def config_is_empty(config: WorkspaceAgentConfig) -> bool: """Check if config has no custom rules set.""" return not config.deriver_rules.strip() and not config.dialectic_rules.strip() @@ -101,10 +102,10 @@ def build_feedback_prompt( **Performance Summary:** {introspection_report.performance_summary} **Identified Issues:** -{chr(10).join(f'- {issue}' for issue in introspection_report.identified_issues) if introspection_report.identified_issues else '(none)'} +{chr(10).join(f"- {issue}" for issue in introspection_report.identified_issues) if introspection_report.identified_issues else "(none)"} **Suggestions:** -{chr(10).join(f'- [{s.target}] {s.rationale} (confidence: {s.confidence})' for s in introspection_report.suggestions) if introspection_report.suggestions else '(none)'} +{chr(10).join(f"- [{s.target}] {s.rationale} (confidence: {s.confidence})" for s in introspection_report.suggestions) if introspection_report.suggestions else "(none)"} """ @@ -187,7 +188,7 @@ async def process_feedback( current_config = await crud.get_workspace_agent_config(db, workspace_name) # Check for interview mode: empty config + simple greeting - if _config_is_empty(current_config) and _is_simple_greeting(request.message): + if config_is_empty(current_config) and is_simple_greeting(request.message): logger.info( f"Feedback channel: Interview mode triggered for workspace {workspace_name}" ) @@ -233,10 +234,7 @@ async def process_feedback( raw_changes = response_data.get("changes", []) if isinstance(raw_changes, list): - for change_item in raw_changes: - if not isinstance(change_item, dict): - continue - + for change_item in cast(list[dict[str, Any]], raw_changes): change_dict: dict[str, object] = change_item field = str(change_dict.get("field", "")) new_value = str(change_dict.get("new_value", "")) diff --git a/tests/alembic/revisions/__init__.py b/tests/alembic/revisions/__init__.py index 77c15483..327752d8 100644 --- a/tests/alembic/revisions/__init__.py +++ b/tests/alembic/revisions/__init__.py @@ -16,6 +16,7 @@ from . import ( test_564ba40505c5_add_session_name_column_to_documents, test_917195d9b5e9_add_messageembedding_table, test_a1b2c3d4e5f6_initial_schema, + test_a8f2d4e6c9b1_add_dialectic_traces_table, test_b765d82110bd_change_metamessages_to_user_level_with_, test_b8183c5ffb48_codify_document_level_and_times_derived, test_baa22cad81e2_standardize_constraint_names, @@ -44,6 +45,7 @@ __all__ = [ "test_88b0fb10906f_add_webhooks_table", "test_917195d9b5e9_add_messageembedding_table", "test_a1b2c3d4e5f6_initial_schema", + "test_a8f2d4e6c9b1_add_dialectic_traces_table", "test_b765d82110bd_change_metamessages_to_user_level_with_", "test_b8183c5ffb48_codify_document_level_and_times_derived", "test_baa22cad81e2_standardize_constraint_names", diff --git a/tests/alembic/revisions/test_a8f2d4e6c9b1_add_dialectic_traces_table.py b/tests/alembic/revisions/test_a8f2d4e6c9b1_add_dialectic_traces_table.py new file mode 100644 index 00000000..68bec986 --- /dev/null +++ b/tests/alembic/revisions/test_a8f2d4e6c9b1_add_dialectic_traces_table.py @@ -0,0 +1,54 @@ +"""Hooks for revision a8f2d4e6c9b1 (add_dialectic_traces_table).""" + +from __future__ import annotations + +from tests.alembic.registry import register_after_upgrade, register_before_upgrade +from tests.alembic.verifier import MigrationVerifier + +INDEXES = ( + ("dialectic_traces", "ix_dialectic_traces_workspace_name"), + ("dialectic_traces", "ix_dialectic_traces_session_name"), + ("dialectic_traces", "ix_dialectic_traces_observer"), + ("dialectic_traces", "ix_dialectic_traces_observed"), + ("dialectic_traces", "ix_dialectic_traces_created_at"), +) + + +@register_before_upgrade("a8f2d4e6c9b1") +def prepare_dialectic_traces(verifier: MigrationVerifier) -> None: + """Assert dialectic_traces table does not exist before migration.""" + verifier.assert_table_exists("dialectic_traces", exists=False) + + +@register_after_upgrade("a8f2d4e6c9b1") +def verify_dialectic_traces_table(verifier: MigrationVerifier) -> None: + """Validate dialectic_traces table and indexes after migration.""" + verifier.assert_table_exists("dialectic_traces") + + # Verify columns exist with correct nullability + verifier.assert_column_exists("dialectic_traces", "id", nullable=False) + verifier.assert_column_exists("dialectic_traces", "workspace_name", nullable=False) + verifier.assert_column_exists("dialectic_traces", "session_name", nullable=True) + verifier.assert_column_exists("dialectic_traces", "observer", nullable=False) + verifier.assert_column_exists("dialectic_traces", "observed", nullable=False) + verifier.assert_column_exists("dialectic_traces", "query", nullable=False) + verifier.assert_column_exists( + "dialectic_traces", "retrieved_doc_ids", nullable=False + ) + verifier.assert_column_exists("dialectic_traces", "tool_calls", nullable=False) + verifier.assert_column_exists("dialectic_traces", "response", nullable=False) + verifier.assert_column_exists("dialectic_traces", "reasoning_level", nullable=False) + verifier.assert_column_exists( + "dialectic_traces", "total_duration_ms", nullable=False + ) + verifier.assert_column_exists("dialectic_traces", "input_tokens", nullable=False) + verifier.assert_column_exists("dialectic_traces", "output_tokens", nullable=False) + verifier.assert_column_exists("dialectic_traces", "created_at", nullable=False) + + # Verify indexes + verifier.assert_indexes_exist(INDEXES) + + # Verify foreign key constraint + verifier.assert_constraint_exists( + "dialectic_traces", "fk_dialectic_traces_workspace_name", "foreign_key" + ) diff --git a/tests/integration/test_token_metrics.py b/tests/integration/test_token_metrics.py index 31163474..847ec46e 100644 --- a/tests/integration/test_token_metrics.py +++ b/tests/integration/test_token_metrics.py @@ -255,6 +255,10 @@ class TestDeriverIngestionMetrics: "src.crud.representation.RepresentationManager.save_representation", new=AsyncMock(), ), + patch( + "src.deriver.deriver.crud.get_workspace_agent_config", + new=AsyncMock(return_value=schemas.WorkspaceAgentConfig()), + ), ): await process_representation_tasks_batch( messages=messages, @@ -312,6 +316,10 @@ class TestDeriverIngestionMetrics: "src.crud.representation.RepresentationManager.save_representation", new=AsyncMock(), ), + patch( + "src.deriver.deriver.crud.get_workspace_agent_config", + new=AsyncMock(return_value=schemas.WorkspaceAgentConfig()), + ), ): await process_representation_tasks_batch( messages=messages, @@ -369,6 +377,10 @@ class TestDeriverIngestionMetrics: "src.crud.representation.RepresentationManager.save_representation", new=AsyncMock(), ), + patch( + "src.deriver.deriver.crud.get_workspace_agent_config", + new=AsyncMock(return_value=schemas.WorkspaceAgentConfig()), + ), ): await process_representation_tasks_batch( messages=messages, diff --git a/tests/routes/test_feedback.py b/tests/routes/test_feedback.py new file mode 100644 index 00000000..8ddf589c --- /dev/null +++ b/tests/routes/test_feedback.py @@ -0,0 +1,91 @@ +"""API endpoint tests for the developer feedback channel.""" + +from unittest.mock import AsyncMock, patch + +from starlette.testclient import TestClient + +from src import models +from src.schemas import FeedbackResponse, WorkspaceAgentConfig + + +def test_feedback_endpoint_basic( + client: TestClient, + sample_data: tuple[models.Workspace, models.Peer], +): + """Test basic feedback endpoint functionality.""" + workspace, _ = sample_data + + # Mock the process_feedback function + mock_response = FeedbackResponse( + message="Configuration updated", + understood_intent="Test intent", + changes_made=[], + current_config=WorkspaceAgentConfig(), + ) + + with patch( + "src.routers.workspaces.process_feedback", + new_callable=AsyncMock, + return_value=mock_response, + ): + response = client.post( + f"/v3/workspaces/{workspace.name}/feedback", + json={"message": "Test message"}, + ) + assert response.status_code == 200 + data = response.json() + assert data["message"] == "Configuration updated" + assert "current_config" in data + + +def test_feedback_endpoint_with_introspection( + client: TestClient, + sample_data: tuple[models.Workspace, models.Peer], +): + """Test feedback endpoint with introspection flag.""" + workspace, _ = sample_data + + mock_response = FeedbackResponse( + message="Used introspection data", + understood_intent="Test intent", + changes_made=[], + current_config=WorkspaceAgentConfig(), + ) + + with ( + patch( + "src.routers.workspaces.process_feedback", + new_callable=AsyncMock, + return_value=mock_response, + ) as mock_process, + patch( + "src.routers.workspaces.get_latest_introspection_report", + new_callable=AsyncMock, + return_value=None, + ) as mock_introspection, + ): + response = client.post( + f"/v3/workspaces/{workspace.name}/feedback", + json={"message": "Help me", "include_introspection": True}, + ) + assert response.status_code == 200 + + # Verify introspection was fetched + mock_introspection.assert_called_once() + # Verify process_feedback was called + mock_process.assert_called_once() + + +def test_feedback_endpoint_validation_error( + client: TestClient, + sample_data: tuple[models.Workspace, models.Peer], +): + """Test feedback endpoint with invalid input.""" + workspace, _ = sample_data + + # Empty message should fail validation + response = client.post( + f"/v3/workspaces/{workspace.name}/feedback", + json={"message": ""}, + ) + assert response.status_code == 422 # Validation error diff --git a/tests/test_dialectic_trace.py b/tests/test_dialectic_trace.py index df45bafc..fe89b87d 100644 --- a/tests/test_dialectic_trace.py +++ b/tests/test_dialectic_trace.py @@ -7,7 +7,8 @@ from nanoid import generate as generate_nanoid from sqlalchemy.ext.asyncio import AsyncSession from src import crud, models -from src.crud.dialectic_trace import _is_abstention +from src.crud.dialectic_trace import is_abstention +from src.dialectic.core import extract_doc_ids_from_messages from src.schemas import DialecticTraceCreate @@ -296,7 +297,7 @@ class TestAbstentionDetection: ] for response in abstention_responses: - assert _is_abstention(response), f"Should detect abstention: {response}" + assert is_abstention(response), f"Should detect abstention: {response}" def test_non_abstention_responses(self): """Test that normal responses are not flagged as abstentions.""" @@ -309,9 +310,9 @@ class TestAbstentionDetection: ] for response in normal_responses: - assert not _is_abstention(response), ( - f"Should not detect abstention: {response}" - ) + assert not is_abstention( + response + ), f"Should not detect abstention: {response}" class TestDocIdExtraction: @@ -319,7 +320,6 @@ class TestDocIdExtraction: def test_extract_doc_ids_from_tool_results(self): """Test extracting document IDs from formatted tool results.""" - from src.dialectic.core import _extract_doc_ids_from_messages messages = [ {"role": "system", "content": "You are a helpful assistant."}, @@ -342,13 +342,12 @@ class TestDocIdExtraction: {"role": "assistant", "content": "Based on the observations..."}, ] - doc_ids = _extract_doc_ids_from_messages(messages) + doc_ids = extract_doc_ids_from_messages(messages) assert set(doc_ids) == {"abc123", "def456", "ghi789"} def test_extract_doc_ids_no_matches(self): """Test extraction when no document IDs are present.""" - from src.dialectic.core import _extract_doc_ids_from_messages messages = [ {"role": "system", "content": "You are a helpful assistant."}, @@ -356,13 +355,12 @@ class TestDocIdExtraction: {"role": "assistant", "content": "Hi there!"}, ] - doc_ids = _extract_doc_ids_from_messages(messages) + doc_ids = extract_doc_ids_from_messages(messages) assert doc_ids == [] def test_extract_doc_ids_duplicates_removed(self): """Test that duplicate IDs are deduplicated.""" - from src.dialectic.core import _extract_doc_ids_from_messages messages = [ { @@ -371,6 +369,6 @@ class TestDocIdExtraction: }, ] - doc_ids = _extract_doc_ids_from_messages(messages) + doc_ids = extract_doc_ids_from_messages(messages) assert doc_ids == ["abc123"] diff --git a/tests/test_feedback.py b/tests/test_feedback.py index 1526f33d..0ae3db2d 100644 --- a/tests/test_feedback.py +++ b/tests/test_feedback.py @@ -10,9 +10,9 @@ from sqlalchemy.ext.asyncio import AsyncSession from src import crud, models from src.feedback import ( INTERVIEW_QUESTIONS, - _config_is_empty, - _is_simple_greeting, build_feedback_prompt, + config_is_empty, + is_simple_greeting, process_feedback, ) from src.schemas import ( @@ -116,7 +116,9 @@ class TestHelperFunctions: "get started", ] for greeting in greetings: - assert _is_simple_greeting(greeting), f"Expected '{greeting}' to be a greeting" + assert is_simple_greeting( + greeting + ), f"Expected '{greeting}' to be a greeting" def test_is_simple_greeting_false_cases(self): """Test cases that should NOT be detected as simple greetings.""" @@ -127,26 +129,26 @@ class TestHelperFunctions: "Please configure the workspace to focus on user preferences", ] for msg in non_greetings: - assert not _is_simple_greeting(msg), f"Expected '{msg}' to NOT be a greeting" + assert not is_simple_greeting(msg), f"Expected '{msg}' to NOT be a greeting" def test_config_is_empty_true(self): """Test detecting empty configuration.""" config = WorkspaceAgentConfig() - assert _config_is_empty(config) + assert config_is_empty(config) config = WorkspaceAgentConfig(deriver_rules="", dialectic_rules="") - assert _config_is_empty(config) + assert config_is_empty(config) config = WorkspaceAgentConfig(deriver_rules=" ", dialectic_rules=" ") - assert _config_is_empty(config) + assert config_is_empty(config) def test_config_is_empty_false(self): """Test detecting non-empty configuration.""" config = WorkspaceAgentConfig(deriver_rules="Some rule") - assert not _config_is_empty(config) + assert not config_is_empty(config) config = WorkspaceAgentConfig(dialectic_rules="Another rule") - assert not _config_is_empty(config) + assert not config_is_empty(config) class TestBuildFeedbackPrompt: @@ -240,11 +242,13 @@ class TestProcessFeedback: # Mock the LLM call mock_response = MagicMock() - mock_response.content = json.dumps({ - "message": "I see you already have rules set up.", - "understood_intent": "Greeting with existing config", - "changes": [], - }) + mock_response.content = json.dumps( + { + "message": "I see you already have rules set up.", + "understood_intent": "Greeting with existing config", + "changes": [], + } + ) with patch( "src.feedback.honcho_llm_call", @@ -268,16 +272,18 @@ class TestProcessFeedback: # Mock the LLM call to return a config change mock_response = MagicMock() - mock_response.content = json.dumps({ - "message": "I've configured the workspace to focus on emotions.", - "understood_intent": "Configure deriver for emotion tracking", - "changes": [ - { - "field": "deriver_rules", - "new_value": "Focus on emotional content and feelings", - } - ], - }) + mock_response.content = json.dumps( + { + "message": "I've configured the workspace to focus on emotions.", + "understood_intent": "Configure deriver for emotion tracking", + "changes": [ + { + "field": "deriver_rules", + "new_value": "Focus on emotional content and feelings", + } + ], + } + ) with patch( "src.feedback.honcho_llm_call", @@ -314,11 +320,13 @@ class TestProcessFeedback: # Mock LLM to return answer without changes mock_response = MagicMock() - mock_response.content = json.dumps({ - "message": "Your current deriver rule is: 'Existing rule'", - "understood_intent": "Question about current config", - "changes": [], - }) + mock_response.content = json.dumps( + { + "message": "Your current deriver rule is: 'Existing rule'", + "understood_intent": "Question about current config", + "changes": [], + } + ) with patch( "src.feedback.honcho_llm_call", @@ -402,16 +410,18 @@ class TestProcessFeedback: # Mock LLM to append new rule mock_response = MagicMock() - mock_response.content = json.dumps({ - "message": "Added goal tracking to existing rules.", - "understood_intent": "Add goal tracking while preserving emotion tracking", - "changes": [ - { - "field": "deriver_rules", - "new_value": "Track emotions\nAlso track goals and aspirations", - } - ], - }) + mock_response.content = json.dumps( + { + "message": "Added goal tracking to existing rules.", + "understood_intent": "Add goal tracking while preserving emotion tracking", + "changes": [ + { + "field": "deriver_rules", + "new_value": "Track emotions\nAlso track goals and aspirations", + } + ], + } + ) with patch( "src.feedback.honcho_llm_call", @@ -427,88 +437,4 @@ class TestProcessFeedback: assert "goals" in response.changes_made[0].new_value.lower() -class TestFeedbackAPIEndpoint: - """Test the /feedback API endpoint via HTTP.""" - - def test_feedback_endpoint_basic( - self, - client, - sample_data: tuple[models.Workspace, models.Peer], - ): - """Test basic feedback endpoint functionality.""" - workspace, _ = sample_data - - # Mock the process_feedback function - mock_response = FeedbackResponse( - message="Configuration updated", - understood_intent="Test intent", - changes_made=[], - current_config=WorkspaceAgentConfig(), - ) - - with patch( - "src.routers.workspaces.process_feedback", - new_callable=AsyncMock, - return_value=mock_response, - ): - response = client.post( - f"/v1/workspaces/{workspace.name}/feedback", - json={"message": "Test message"}, - ) - assert response.status_code == 200 - data = response.json() - assert data["message"] == "Configuration updated" - assert "current_config" in data - - def test_feedback_endpoint_with_introspection( - self, - client, - sample_data: tuple[models.Workspace, models.Peer], - ): - """Test feedback endpoint with introspection flag.""" - workspace, _ = sample_data - - mock_response = FeedbackResponse( - message="Used introspection data", - understood_intent="Test intent", - changes_made=[], - current_config=WorkspaceAgentConfig(), - ) - - with ( - patch( - "src.routers.workspaces.process_feedback", - new_callable=AsyncMock, - return_value=mock_response, - ) as mock_process, - patch( - "src.routers.workspaces.get_latest_introspection_report", - new_callable=AsyncMock, - return_value=None, - ) as mock_introspection, - ): - response = client.post( - f"/v1/workspaces/{workspace.name}/feedback", - json={"message": "Help me", "include_introspection": True}, - ) - assert response.status_code == 200 - - # Verify introspection was fetched - mock_introspection.assert_called_once() - # Verify process_feedback was called - mock_process.assert_called_once() - - def test_feedback_endpoint_validation_error( - self, - client, - sample_data: tuple[models.Workspace, models.Peer], - ): - """Test feedback endpoint with invalid input.""" - workspace, _ = sample_data - - # Empty message should fail validation - response = client.post( - f"/v1/workspaces/{workspace.name}/feedback", - json={"message": ""}, - ) - assert response.status_code == 422 # Validation error +# Note: API endpoint tests are in tests/routes/test_feedback.py diff --git a/tests/test_introspection.py b/tests/test_introspection.py index a4e8ea2d..4a62d692 100644 --- a/tests/test_introspection.py +++ b/tests/test_introspection.py @@ -68,7 +68,9 @@ class TestGatherIntrospectionContext: observer=peer.name, observed=peer.name, query=f"What does the user like? Query {i}", - response="The user likes coffee." if i < 3 else "I don't have information.", + response="The user likes coffee." + if i < 3 + else "I don't have information.", reasoning_level="low", total_duration_ms=100.0 * (i + 1), input_tokens=100, @@ -190,19 +192,21 @@ class TestRunIntrospection: # Mock the LLM call mock_llm_response = MagicMock() - mock_llm_response.content = json.dumps({ - "performance_summary": "The workspace is performing well.", - "identified_issues": ["High abstention rate"], - "suggestions": [ - { - "target": "deriver_rules", - "current_value": "", - "suggested_value": "Focus on capturing user preferences", - "rationale": "Too many queries about preferences are being missed", - "confidence": "medium", - } - ], - }) + mock_llm_response.content = json.dumps( + { + "performance_summary": "The workspace is performing well.", + "identified_issues": ["High abstention rate"], + "suggestions": [ + { + "target": "deriver_rules", + "current_value": "", + "suggested_value": "Focus on capturing user preferences", + "rationale": "Too many queries about preferences are being missed", + "confidence": "medium", + } + ], + } + ) with ( patch( @@ -278,7 +282,6 @@ class TestStoreIntrospectionReport: sample_data: tuple[models.Workspace, models.Peer], ): """Test that storing a report creates a document in the reserved collection.""" - from src.crud.collection import get_or_create_collection as real_get_or_create_collection workspace, _ = sample_data # Capture workspace name before any potential session issues @@ -386,7 +389,6 @@ class TestIntrospectionDreamDispatch: @pytest.mark.asyncio async def test_introspection_dream_type_dispatch( self, - db_session: AsyncSession, sample_data: tuple[models.Workspace, models.Peer], ): """Test that INTROSPECTION dream type is properly dispatched."""