From 4112f949b4a4100381a2a57397f53afc8bb967b9 Mon Sep 17 00:00:00 2001 From: Praveen Kumar Sridhar Date: Sat, 11 Jul 2026 21:35:16 -0700 Subject: [PATCH] fix(deriver): document explicit observation JSON shape --- src/deriver/prompts.py | 4 +++ src/utils/representation.py | 7 ++++- tests/deriver/test_prompts.py | 39 ++++++++++++++++++++++++++ tests/llm/test_backends/test_openai.py | 32 +++++++++++++++++++++ 4 files changed, 81 insertions(+), 1 deletion(-) diff --git a/src/deriver/prompts.py b/src/deriver/prompts.py index 402bcae7..9a84a258 100644 --- a/src/deriver/prompts.py +++ b/src/deriver/prompts.py @@ -76,6 +76,10 @@ EXAMPLES (using `alice` as the target peer id): - EXPLICIT: "I took my dog for a walk in NYC" → "alice has a dog", "alice lives in NYC" - EXPLICIT: "alice attended college" + general knowledge → "alice completed high school or equivalent" +Respond with a json object in the following format: +{{"explicit": [{{"content": "fact 1"}}, {{"content": "fact 2"}}]}} +Each item in "explicit" must be an object with a "content" field, not a bare string. + {custom_instructions_section} Target peer: diff --git a/src/utils/representation.py b/src/utils/representation.py index 674b25bb..44789af7 100644 --- a/src/utils/representation.py +++ b/src/utils/representation.py @@ -114,7 +114,12 @@ class PromptRepresentation(BaseModel): """ explicit: list[ExplicitObservationBase] = Field( - description="Facts LITERALLY stated by the user - direct quotes or clear paraphrases only, no interpretation or inference. Example: ['The user is 25 years old', 'The user has a dog named Rover']", + description=( + "Facts LITERALLY stated by the user - direct quotes or clear " + "paraphrases only, no interpretation or inference. " + 'Example: [{"content": "The user is 25 years old"}, ' + '{"content": "The user has a dog named Rover"}]' + ), default_factory=list, ) diff --git a/tests/deriver/test_prompts.py b/tests/deriver/test_prompts.py index c0f7db95..14e05512 100644 --- a/tests/deriver/test_prompts.py +++ b/tests/deriver/test_prompts.py @@ -1,12 +1,27 @@ +from collections.abc import AsyncGenerator, Generator from unittest.mock import patch import pytest +from pydantic import ValidationError from src.deriver.prompts import ( estimate_deriver_prompt_tokens, estimate_minimal_deriver_prompt_tokens, minimal_deriver_prompt, ) +from src.utils.representation import PromptRepresentation + + +@pytest.fixture(autouse=True) +async def clean_queue_tables() -> AsyncGenerator[None, None]: + """Prompt-only tests do not need queue table cleanup.""" + yield + + +@pytest.fixture(autouse=True) +def mock_tracked_db() -> Generator[None, None, None]: + """Prompt-only tests do not need tracked_db patching.""" + yield def test_minimal_deriver_prompt_includes_custom_instructions_when_present() -> None: @@ -30,6 +45,30 @@ def test_minimal_deriver_prompt_omits_custom_instructions_when_absent() -> None: assert "CUSTOM INSTRUCTIONS:" not in prompt +def test_minimal_deriver_prompt_describes_explicit_json_object_shape() -> None: + prompt = minimal_deriver_prompt( + peer_id="alice", + messages="alice: I just had my 25th birthday", + custom_instructions=None, + ) + + assert '"explicit": [{"content": "fact 1"}' in prompt + assert 'Each item in "explicit" must be an object with a "content" field' in prompt + assert "not a bare string" in prompt + with pytest.raises(ValidationError): + PromptRepresentation.model_validate_json( + '{"explicit": ["alice is 25 years old"]}' + ) + assert PromptRepresentation.model_validate_json( + '{"explicit": [{"content": "alice is 25 years old"}]}' + ).explicit[0].content == "alice is 25 years old" + explicit_schema_description = PromptRepresentation.model_json_schema()["properties"][ + "explicit" + ]["description"] + assert '{"content": "The user is 25 years old"}' in explicit_schema_description + assert "['The user is 25 years old'" not in explicit_schema_description + + def test_estimate_deriver_prompt_tokens_increases_with_custom_instructions() -> None: base_tokens = estimate_minimal_deriver_prompt_tokens() custom_tokens = estimate_deriver_prompt_tokens( diff --git a/tests/llm/test_backends/test_openai.py b/tests/llm/test_backends/test_openai.py index 4270ebd8..dce7a184 100644 --- a/tests/llm/test_backends/test_openai.py +++ b/tests/llm/test_backends/test_openai.py @@ -813,6 +813,38 @@ async def test_structured_output_json_object_mode_request_shape() -> None: assert result.content.answer == "ok" +@pytest.mark.asyncio +async def test_structured_output_json_object_mode_prompt_representation_schema_shape() -> None: + """PromptRepresentation schema hints explicit observations as content objects.""" + client = Mock() + client.chat.completions.parse = AsyncMock() + client.chat.completions.create = AsyncMock( + return_value=_structured_create_return( + '{"explicit": [{"content": "alice likes coffee"}]}' + ) + ) + + backend = OpenAIBackend(client) + result = await backend.complete( + model="glm-4.6", + messages=[{"role": "user", "content": "Hello"}], + max_tokens=100, + response_format=PromptRepresentation, + extra_params={"structured_output_mode": "json_object"}, + ) + + call = _await_kwargs(client.chat.completions.create) + system_messages = [m for m in call["messages"] if m["role"] == "system"] + assert system_messages, "expected a system message carrying the schema" + system_content = system_messages[0]["content"] + schema = json.loads(system_content.split("JSON schema:\n", 1)[1]) + explicit_description = schema["properties"]["explicit"]["description"] + assert '{"content": "The user is 25 years old"}' in explicit_description + assert "['The user is 25 years old'" not in explicit_description + assert isinstance(result.content, PromptRepresentation) + assert result.content.explicit[0].content == "alice likes coffee" + + @pytest.mark.asyncio async def test_structured_output_json_object_mode_repairs_markdown() -> None: """A provider that ignores json_object and returns prose must not crash —