From 10772f991482989c792407481be8df5d58874eb1 Mon Sep 17 00:00:00 2001 From: foma-agent Date: Sat, 8 Aug 2026 08:34:48 -0700 Subject: [PATCH] fix(llm): surface invalid structured output --- src/llm/backends/anthropic.py | 20 ++-- src/llm/backends/gemini.py | 43 +++++++-- src/llm/backends/openai.py | 28 +++--- src/llm/structured_output.py | 111 ++++++++++++++++++---- tests/llm/test_backends/test_anthropic.py | 21 ++++ tests/llm/test_backends/test_gemini.py | 52 ++++++++++ tests/llm/test_backends/test_openai.py | 76 ++++++++++++++- tests/llm/test_structured_output.py | 87 +++++++++++++++++ tests/utils/test_length_finish_reason.py | 58 ++++++----- 9 files changed, 412 insertions(+), 84 deletions(-) create mode 100644 tests/llm/test_structured_output.py diff --git a/src/llm/backends/anthropic.py b/src/llm/backends/anthropic.py index 9f58af07..23a1128c 100644 --- a/src/llm/backends/anthropic.py +++ b/src/llm/backends/anthropic.py @@ -13,7 +13,12 @@ from src.llm.request_builder import ( apply_sdk_passthroughs, request_timeout_from_extra_params, ) -from src.llm.structured_output import repair_response_model_json, schema_instruction +from src.llm.structured_output import ( + StructuredOutputError, + repair_response_model_json, + schema_instruction, + validate_structured_output, +) class AnthropicBackend: @@ -270,12 +275,13 @@ class AnthropicBackend: if response_format is not None and not tool_calls: raw_content = f"{{{text_content}" if prefilled_json else text_content try: - if prefilled_json: - parsed_json = json.loads(raw_content) - content = response_format.model_validate(parsed_json) - else: - content = response_format.model_validate_json(raw_content) - except (json.JSONDecodeError, ValidationError, ValueError): + content = validate_structured_output(raw_content, response_format) + except ( + json.JSONDecodeError, + StructuredOutputError, + ValidationError, + ValueError, + ): content = repair_response_model_json( raw_content, response_format, diff --git a/src/llm/backends/gemini.py b/src/llm/backends/gemini.py index 3f6ade64..0c981c2c 100644 --- a/src/llm/backends/gemini.py +++ b/src/llm/backends/gemini.py @@ -1,11 +1,12 @@ from __future__ import annotations +import json from collections.abc import AsyncIterator from datetime import datetime, timedelta, timezone from typing import Any, ClassVar, cast from google.genai import types as genai_types -from pydantic import BaseModel +from pydantic import BaseModel, ValidationError from src.exceptions import LLMError, ValidationException from src.llm.backend import CompletionResult, StreamChunk, ToolCallResult @@ -19,7 +20,12 @@ from src.llm.request_builder import ( coerce_passthrough_mapping, request_timeout_from_extra_params, ) -from src.llm.structured_output import repair_response_model_json, schema_instruction +from src.llm.structured_output import ( + StructuredOutputError, + repair_response_model_json, + schema_instruction, + validate_structured_output, +) GEMINI_BLOCKED_FINISH_REASONS = { "SAFETY", @@ -398,12 +404,33 @@ class GeminiBackend: # fallback, failing the iteration. if response_format is not None and not tool_calls: parsed_response = getattr(response, "parsed", None) - if isinstance(parsed_response, response_format): - content = parsed_response - elif isinstance(parsed_response, dict): - content = response_format.model_validate(parsed_response) - elif isinstance(parsed_response, str): - content = response_format.model_validate_json(parsed_response) + if isinstance(parsed_response, response_format | dict | str): + try: + raw_text = "".join(text_parts) + validation_content = ( + raw_text + if isinstance(parsed_response, response_format) and raw_text + else cast(object, parsed_response) + ) + content = validate_structured_output( + validation_content, response_format + ) + except (json.JSONDecodeError, StructuredOutputError, ValidationError): + raw_content = ( + parsed_response + if isinstance(parsed_response, str) + else "".join(text_parts) + or json.dumps( + parsed_response.model_dump(mode="json", exclude_unset=True) + if isinstance(parsed_response, BaseModel) + else parsed_response + ) + ) + content = repair_response_model_json( + raw_content, + response_format, + model_name, + ) else: if finish_reason in GEMINI_BLOCKED_FINISH_REASONS: raise LLMError( diff --git a/src/llm/backends/openai.py b/src/llm/backends/openai.py index 672709ff..12e63b4a 100644 --- a/src/llm/backends/openai.py +++ b/src/llm/backends/openai.py @@ -21,6 +21,7 @@ from src.llm.structured_output import ( repair_response_model_json, validate_structured_output, ) +from src.utils.representation import PromptRepresentation logger = logging.getLogger(__name__) @@ -210,10 +211,8 @@ class OpenAIBackend: try: response = await self._client.chat.completions.parse(**params) except LengthFinishReasonError as exc: - # Truncated output: repair the partial content directly. repair - # handles empty/unrepairable JSON with its own model-aware fallback - # (PromptRepresentation -> empty, others -> raise), which differs - # from the parse-fallback terminal below, so it stays a direct call. + # Truncated output: repair the partial content directly. Invalid + # PromptRepresentation payloads raise a safe, digest-bearing error. truncated = exc.completion raw_content = truncated.choices[0].message.content or "" content = repair_response_model_json( @@ -249,12 +248,14 @@ class OpenAIBackend: return CompletionResult(content=fallback_content) parsed = response.choices[0].message.parsed if parsed is not None: - return self._normalize_response( - response, - content_override=validate_structured_output( - parsed, response_format - ), - ) + raw_content = response.choices[0].message.content + if raw_content and response_format is PromptRepresentation: + content = self._parse_or_repair_structured_content( + response, response_format, model, empty_on_missing=False + ) + else: + content = validate_structured_output(parsed, response_format) + return self._normalize_response(response, content_override=content) # parse() returned no model: repair raw content, surface a refusal, # or raise so the retry/fallback chain engages on a junk response. content = self._parse_or_repair_structured_content( @@ -538,12 +539,11 @@ class OpenAIBackend: message = response.choices[0].message raw_content = message.content or "" if raw_content: - # Fast path: clean JSON validates directly. Only fall back to the - # repair pipeline when validation fails — repair is comparatively - # expensive and silently degrades malformed input to an empty model. + # Fast path: clean, schema-relevant JSON validates directly. Route + # invalid or irrelevant payloads through the safe repair error path. try: return validate_structured_output(raw_content, response_format) - except (StructuredOutputError, ValidationError): + except (json.JSONDecodeError, StructuredOutputError, ValidationError): return repair_response_model_json(raw_content, response_format, model) refusal = getattr(message, "refusal", None) if refusal: diff --git a/src/llm/structured_output.py b/src/llm/structured_output.py index 37d6b982..53972ba0 100644 --- a/src/llm/structured_output.py +++ b/src/llm/structured_output.py @@ -1,6 +1,8 @@ from __future__ import annotations import json +from hashlib import sha256 +from typing import cast from pydantic import BaseModel, ValidationError @@ -31,43 +33,82 @@ def schema_instruction(response_format: type[BaseModel], *, tools_present: bool) def repair_response_model_json( raw_content: str, response_model: type[BaseModel], - _model: str, + model: str, ) -> BaseModel: """Repair truncated or malformed JSON and validate against the response model.""" + failure_class = "ValidationError" try: final = validate_and_repair_json(raw_content) - repaired_data = json.loads(final) + repaired_data = cast(object, json.loads(final)) + repaired_mapping = ( + cast(dict[str, object], repaired_data) + if isinstance(repaired_data, dict) + else None + ) + raw_is_empty_mapping = False + if response_model is PromptRepresentation and repaired_mapping == {}: + raw_is_empty_mapping = _starts_with_empty_json_object(raw_content) + if not raw_is_empty_mapping: + failure_class = "JSONDecodeError" - if ( - response_model is PromptRepresentation - and "deductive" in repaired_data - and isinstance(repaired_data["deductive"], list) + if response_model is PromptRepresentation and ( + repaired_mapping is None + or ( + not raw_is_empty_mapping + and not any( + field in repaired_mapping for field in response_model.model_fields + ) + ) ): - for item in repaired_data["deductive"]: + try: + json.loads(raw_content) + except json.JSONDecodeError: + failure_class = "JSONDecodeError" + else: + failure_class = "ValidationError" + final = "" + + deductive = repaired_mapping.get("deductive") if repaired_mapping else None + if response_model is PromptRepresentation and isinstance(deductive, list): + for item in cast(list[object], deductive): if isinstance(item, dict): - if "conclusion" not in item and "premises" in item: - if item["premises"]: - item["conclusion"] = ( - f"[Incomplete reasoning from premises: {item['premises'][0][:100]}...]" + item_mapping = cast(dict[str, object], item) + premises = item_mapping.get("premises") + if "conclusion" not in item_mapping and premises is not None: + if isinstance(premises, list) and premises: + premise_items = cast(list[object], premises) + item_mapping["conclusion"] = ( + "[Incomplete reasoning from premises: " + f"{str(premise_items[0])[:100]}...]" ) else: - item["conclusion"] = ( + item_mapping["conclusion"] = ( "[Incomplete reasoning - conclusion missing]" ) - if "premises" not in item: - item["premises"] = [] + if "premises" not in item_mapping: + item_mapping["premises"] = [] - final = json.dumps(repaired_data) - except (json.JSONDecodeError, KeyError, TypeError, ValueError): + if final: + final = json.dumps(repaired_data) + except (json.JSONDecodeError, KeyError, TypeError, ValueError) as exc: + failure_class = type(exc).__name__ final = "" try: return response_model.model_validate_json(final) except ValidationError: - if response_model is PromptRepresentation: - return PromptRepresentation(explicit=[]) - raise + if response_model is not PromptRepresentation: + raise + + payload = raw_content.encode("utf-8") + details = f"failure_class={failure_class} model={model}" + payload_details = ( + f"payload_bytes={len(payload)} payload_sha256={sha256(payload).hexdigest()}" + ) + raise StructuredOutputError( + f"PromptRepresentation structured output failed {details} {payload_details}" + ) from None def validate_structured_output( @@ -75,16 +116,48 @@ def validate_structured_output( response_model: type[BaseModel], ) -> BaseModel: if isinstance(content, response_model): + if not _is_schema_relevant(content, response_model): + raise StructuredOutputError("Structured output has no schema fields") return content if isinstance(content, str): + if response_model is PromptRepresentation: + parsed = cast(object, json.loads(content)) + if not _is_schema_relevant(parsed, response_model): + raise StructuredOutputError("Structured output has no schema fields") return response_model.model_validate_json(content) if isinstance(content, dict): + if not _is_schema_relevant(cast(object, content), response_model): + raise StructuredOutputError("Structured output has no schema fields") return response_model.model_validate(content) raise StructuredOutputError( f"Unsupported structured output payload: {type(content).__name__}" ) +def _is_schema_relevant(content: object, response_model: type[BaseModel]) -> bool: + if response_model is not PromptRepresentation: + return True + if isinstance(content, BaseModel): + return True + return isinstance(content, dict) and ( + not content or any(field in content for field in response_model.model_fields) + ) + + +def _starts_with_empty_json_object(raw_content: str) -> bool: + candidate = raw_content.strip() + if candidate.startswith("```"): + _, separator, candidate = candidate.partition("\n") + if not separator: + return False + candidate = candidate.lstrip() + try: + value, _ = json.JSONDecoder().raw_decode(candidate) + except json.JSONDecodeError: + return False + return isinstance(value, dict) and not value + + def empty_structured_output(response_model: type[BaseModel]) -> BaseModel: if response_model is PromptRepresentation: return PromptRepresentation(explicit=[]) diff --git a/tests/llm/test_backends/test_anthropic.py b/tests/llm/test_backends/test_anthropic.py index a226f626..9ef41152 100644 --- a/tests/llm/test_backends/test_anthropic.py +++ b/tests/llm/test_backends/test_anthropic.py @@ -7,6 +7,8 @@ from anthropic.types import TextBlock, ThinkingBlock, ToolUseBlock from pydantic import BaseModel from src.llm.backends.anthropic import AnthropicBackend +from src.llm.structured_output import StructuredOutputError +from src.utils.representation import PromptRepresentation @pytest.mark.asyncio @@ -357,6 +359,25 @@ async def test_anthropic_backend_repairs_malformed_structured_output( ] +@pytest.mark.asyncio +async def test_anthropic_backend_rejects_wrong_structured_output_keys() -> None: + client = _make_client([TextBlock(type="text", text='{"wrong": 1}')]) + + backend = AnthropicBackend(client) + with pytest.raises(StructuredOutputError) as exc_info: + await backend.complete( + model="claude-sonnet-4-5", + messages=[{"role": "user", "content": "Hello"}], + max_tokens=100, + response_format=PromptRepresentation, + ) + + message = str(exc_info.value) + assert "model=claude-sonnet-4-5" in message + assert "payload_sha256=" in message + assert "wrong" not in message + + @pytest.mark.asyncio async def test_anthropic_backend_skips_parsing_on_tool_call_turns( monkeypatch: pytest.MonkeyPatch, diff --git a/tests/llm/test_backends/test_gemini.py b/tests/llm/test_backends/test_gemini.py index 8295ab26..e063d17c 100644 --- a/tests/llm/test_backends/test_gemini.py +++ b/tests/llm/test_backends/test_gemini.py @@ -8,6 +8,8 @@ from pydantic import BaseModel from src.exceptions import LLMError, ValidationException from src.llm.backends.gemini import GeminiBackend from src.llm.caching import PromptCachePolicy, gemini_cache_store +from src.llm.structured_output import StructuredOutputError +from src.utils.representation import PromptRepresentation @pytest.mark.asyncio @@ -659,3 +661,53 @@ async def test_gemini_backend_structured_without_tools_uses_native_schema() -> N assert call["config"]["response_mime_type"] == "application/json" # No instruction injected on the tool-less path. assert call["contents"][-1]["parts"][-1]["text"] == "Hello" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "parsed", [{"wrong": 1}, '{"wrong": 1}', PromptRepresentation()] +) +async def test_gemini_backend_rejects_wrong_structured_output_keys( + parsed: dict[str, int] | str | PromptRepresentation, +) -> None: + client = Mock() + client.aio.models.generate_content = AsyncMock( + return_value=_gemini_response( + [SimpleNamespace(text='{"wrong": 1}')], + parsed=parsed, + ) + ) + + backend = GeminiBackend(client) + with pytest.raises(StructuredOutputError) as exc_info: + await backend.complete( + model="gemini-2.5-flash", + messages=[{"role": "user", "content": "Hello"}], + max_tokens=100, + response_format=PromptRepresentation, + ) + + message = str(exc_info.value) + assert "model=gemini-2.5-flash" in message + assert "payload_sha256=" in message + assert "wrong" not in message + + +@pytest.mark.asyncio +async def test_gemini_backend_accepts_schema_valid_empty_object() -> None: + client = Mock() + client.aio.models.generate_content = AsyncMock( + return_value=_gemini_response( + [SimpleNamespace(text="{}")], + parsed=PromptRepresentation(), + ) + ) + + result = await GeminiBackend(client).complete( + model="gemini-test", + messages=[{"role": "user", "content": "Hello"}], + max_tokens=100, + response_format=PromptRepresentation, + ) + + assert result.content == PromptRepresentation(explicit=[]) diff --git a/tests/llm/test_backends/test_openai.py b/tests/llm/test_backends/test_openai.py index fbd8e719..39d8b6b9 100644 --- a/tests/llm/test_backends/test_openai.py +++ b/tests/llm/test_backends/test_openai.py @@ -16,6 +16,7 @@ from src.llm.backends.openai import ( OpenAIBackend, _json_object_instruction, # pyright: ignore[reportPrivateUsage] ) +from src.llm.structured_output import StructuredOutputError from src.utils.representation import PromptRepresentation from src.utils.schema_conversion import json_response_schema_to_pydantic @@ -894,9 +895,8 @@ async def test_structured_output_json_object_mode_request_shape() -> None: @pytest.mark.asyncio -async def test_structured_output_json_object_mode_repairs_markdown() -> None: - """A provider that ignores json_object and returns prose must not crash — - PromptRepresentation repairs to an empty representation, not an exception.""" +async def test_structured_output_json_object_mode_rejects_markdown() -> None: + """PromptRepresentation prose is rejected so the caller can retry.""" client = Mock() client.chat.completions.parse = AsyncMock() client.chat.completions.create = AsyncMock( @@ -906,7 +906,25 @@ async def test_structured_output_json_object_mode_repairs_markdown() -> None: ) backend = OpenAIBackend(client) - result = await backend.complete( + with pytest.raises(StructuredOutputError): + await backend.complete( + model="glm-4.6", + messages=[{"role": "user", "content": "Hello"}], + max_tokens=100, + response_format=PromptRepresentation, + extra_params={"structured_output_mode": "json_object"}, + ) + + +@pytest.mark.asyncio +async def test_structured_output_json_object_mode_accepts_fenced_empty_object() -> None: + client = Mock() + client.chat.completions.parse = AsyncMock() + client.chat.completions.create = AsyncMock( + return_value=_structured_create_return("```json\n{}\n```") + ) + + result = await OpenAIBackend(client).complete( model="glm-4.6", messages=[{"role": "user", "content": "Hello"}], max_tokens=100, @@ -914,7 +932,55 @@ async def test_structured_output_json_object_mode_repairs_markdown() -> None: extra_params={"structured_output_mode": "json_object"}, ) - assert isinstance(result.content, PromptRepresentation) + assert result.content == PromptRepresentation(explicit=[]) + + +@pytest.mark.asyncio +async def test_structured_output_json_object_mode_rejects_wrong_keys() -> None: + client = Mock() + client.chat.completions.parse = AsyncMock() + client.chat.completions.create = AsyncMock( + return_value=_structured_create_return('{"wrong": 1}') + ) + + backend = OpenAIBackend(client) + with pytest.raises(StructuredOutputError) as exc_info: + await backend.complete( + model="glm-4.6", + messages=[{"role": "user", "content": "Hello"}], + max_tokens=100, + response_format=PromptRepresentation, + extra_params={"structured_output_mode": "json_object"}, + ) + + message = str(exc_info.value) + assert "model=glm-4.6" in message + assert "payload_sha256=" in message + assert "wrong" not in message + + +@pytest.mark.asyncio +async def test_structured_output_native_parse_rejects_defaulted_wrong_keys() -> None: + client = Mock() + client.chat.completions.parse = AsyncMock( + return_value=_structured_create_return( + '{"wrong": 1}', parsed=PromptRepresentation() + ) + ) + + backend = OpenAIBackend(client) + with pytest.raises(StructuredOutputError) as exc_info: + await backend.complete( + model="gpt-test", + messages=[{"role": "user", "content": "Hello"}], + max_tokens=100, + response_format=PromptRepresentation, + ) + + message = str(exc_info.value) + assert "model=gpt-test" in message + assert "payload_sha256=" in message + assert "wrong" not in message @pytest.mark.asyncio diff --git a/tests/llm/test_structured_output.py b/tests/llm/test_structured_output.py new file mode 100644 index 00000000..7f9e9ae5 --- /dev/null +++ b/tests/llm/test_structured_output.py @@ -0,0 +1,87 @@ +import hashlib +import traceback + +import pytest +from pydantic import BaseModel, ValidationError + +from src.llm.structured_output import ( + StructuredOutputError, + repair_response_model_json, + validate_structured_output, +) +from src.utils.representation import PromptRepresentation + + +class OtherResponse(BaseModel): + answer: str + + +def test_prompt_representation_malformed_json_raises_safe_error() -> None: + sentinel = "sentinel-secret-é" + payload = f"not json {sentinel} {{{{" + model = "test-model" + + with pytest.raises(StructuredOutputError) as exc_info: + repair_response_model_json(payload, PromptRepresentation, model) + + error = exc_info.value + rendered = "".join( + traceback.format_exception(type(error), error, error.__traceback__) + ) + assert "JSONDecodeError" in str(error) + assert model in str(error) + assert f"payload_bytes={len(payload.encode('utf-8'))}" in str(error) + assert hashlib.sha256(payload.encode()).hexdigest() in str(error) + assert payload not in rendered + assert sentinel not in rendered + assert error.__cause__ is None + assert error.__context__ is None + + +def test_prompt_representation_schema_irrelevant_json_raises() -> None: + with pytest.raises(StructuredOutputError, match="failure_class=ValidationError"): + repair_response_model_json('{"wrong": 1}', PromptRepresentation, "test-model") + + +def test_prompt_representation_explicit_empty_is_valid() -> None: + result = repair_response_model_json( + '{"explicit": []}', PromptRepresentation, "test-model" + ) + + assert result == PromptRepresentation(explicit=[]) + + +def test_prompt_representation_empty_object_is_valid() -> None: + expected = PromptRepresentation(explicit=[]) + + assert ( + repair_response_model_json("{}", PromptRepresentation, "test-model") == expected + ) + assert validate_structured_output("{}", PromptRepresentation) == expected + assert validate_structured_output(expected, PromptRepresentation) == expected + + +@pytest.mark.parametrize( + "payload", + ["```json\n{}\n```", "{}\n\nNothing to extract."], +) +def test_prompt_representation_wrapped_empty_object_is_valid(payload: str) -> None: + result = repair_response_model_json(payload, PromptRepresentation, "test-model") + + assert result == PromptRepresentation(explicit=[]) + + +def test_prompt_representation_truncated_explicit_json_is_repaired() -> None: + result = repair_response_model_json( + '{"explicit":[{"content":"prefers tabs', + PromptRepresentation, + "test-model", + ) + + assert isinstance(result, PromptRepresentation) + assert [item.content for item in result.explicit] == ["prefers tabs"] + + +def test_other_response_model_preserves_validation_error() -> None: + with pytest.raises(ValidationError): + repair_response_model_json("not json", OtherResponse, "test-model") diff --git a/tests/utils/test_length_finish_reason.py b/tests/utils/test_length_finish_reason.py index 69d3210c..d3ac340c 100644 --- a/tests/utils/test_length_finish_reason.py +++ b/tests/utils/test_length_finish_reason.py @@ -2,8 +2,8 @@ Tests for JSON repair handling across all providers in honcho_llm_call_inner, and Gemini thinking budget support. -Verifies that when an LLM hits the max token limit or returns malformed JSON, -the truncated output is repaired and returned instead of crashing. +Verifies that truncated JSON is repaired when possible and malformed structured +output raises a safe error. """ import json @@ -21,6 +21,7 @@ from openai.types.completion_usage import CompletionUsage from pydantic import BaseModel, ValidationError from src.llm import CLIENTS, HonchoLLMCallResponse, honcho_llm_call_inner +from src.llm.structured_output import StructuredOutputError from src.utils.representation import PromptRepresentation # --- Test models --- @@ -176,15 +177,17 @@ class TestOpenAILengthFinishReasonRepair: assert len(response.content.explicit) >= 1 assert response.finish_reasons == ["length"] - async def test_completely_broken_json_falls_back_to_empty(self) -> None: - """Completely unrepairable JSON should fall back to empty PromptRepresentation.""" + async def test_completely_broken_json_raises_safe_error(self) -> None: mock_client = AsyncMock(spec=AsyncOpenAI) mock_client.chat.completions.parse = _raise_length_error( "this is not json at all just random text" ) - with patch.dict(CLIENTS, {"openai": mock_client}): - response = await honcho_llm_call_inner( + with ( + patch.dict(CLIENTS, {"openai": mock_client}), + pytest.raises(StructuredOutputError), + ): + await honcho_llm_call_inner( provider="openai", model="test-model", prompt="Analyze messages", @@ -193,17 +196,15 @@ class TestOpenAILengthFinishReasonRepair: json_mode=True, ) - assert isinstance(response.content, PromptRepresentation) - assert response.content.explicit == [] - assert response.finish_reasons == ["length"] - - async def test_empty_content_falls_back_to_empty(self) -> None: - """Empty/null content should fall back to empty PromptRepresentation.""" + async def test_empty_truncated_content_raises_safe_error(self) -> None: mock_client = AsyncMock(spec=AsyncOpenAI) mock_client.chat.completions.parse = _raise_length_error("") - with patch.dict(CLIENTS, {"openai": mock_client}): - response = await honcho_llm_call_inner( + with ( + patch.dict(CLIENTS, {"openai": mock_client}), + pytest.raises(StructuredOutputError), + ): + await honcho_llm_call_inner( provider="openai", model="test-model", prompt="Analyze messages", @@ -212,9 +213,6 @@ class TestOpenAILengthFinishReasonRepair: json_mode=True, ) - assert isinstance(response.content, PromptRepresentation) - assert response.content.explicit == [] - async def test_non_prompt_representation_reraises_on_unfixable(self) -> None: """Non-PromptRepresentation with unrepairable JSON should raise ValidationError.""" mock_client = AsyncMock(spec=AsyncOpenAI) @@ -307,14 +305,16 @@ class TestAnthropicJsonRepair: assert isinstance(response.content, PromptRepresentation) assert len(response.content.explicit) >= 1 - async def test_broken_anthropic_response_falls_back_to_empty(self) -> None: - """Completely broken Anthropic JSON should fall back to empty PromptRepresentation.""" + async def test_broken_anthropic_response_raises_safe_error(self) -> None: mock_client = _make_anthropic_mock( "random gibberish that is not json", stop_reason="max_tokens" ) - with patch.dict(CLIENTS, {"anthropic": mock_client}): - response = await honcho_llm_call_inner( + with ( + patch.dict(CLIENTS, {"anthropic": mock_client}), + pytest.raises(StructuredOutputError), + ): + await honcho_llm_call_inner( provider="anthropic", model="claude-3-sonnet", prompt="Analyze messages", @@ -323,9 +323,6 @@ class TestAnthropicJsonRepair: json_mode=True, ) - assert isinstance(response.content, PromptRepresentation) - assert response.content.explicit == [] - async def test_non_prompt_representation_reraises(self) -> None: """Non-PromptRepresentation with broken JSON should raise.""" mock_client = _make_anthropic_mock("not json", stop_reason="max_tokens") @@ -380,8 +377,7 @@ class TestGeminiJsonRepair: assert isinstance(response.content, PromptRepresentation) assert len(response.content.explicit) == 2 - async def test_gemini_broken_text_falls_back_to_empty(self) -> None: - """Gemini with broken text and no parsed content should fall back.""" + async def test_gemini_broken_text_raises_safe_error(self) -> None: from google import genai mock_client = _make_gemini_mock( @@ -389,8 +385,11 @@ class TestGeminiJsonRepair: ) mock_client.__class__ = genai.Client # pyright: ignore[reportAttributeAccessIssue] - with patch.dict(CLIENTS, {"gemini": mock_client}): - response = await honcho_llm_call_inner( + with ( + patch.dict(CLIENTS, {"gemini": mock_client}), + pytest.raises(StructuredOutputError), + ): + await honcho_llm_call_inner( provider="gemini", model="gemini-2.5-flash", prompt="Analyze messages", @@ -399,9 +398,6 @@ class TestGeminiJsonRepair: json_mode=True, ) - assert isinstance(response.content, PromptRepresentation) - assert response.content.explicit == [] - # --------------------------------------------------------------------------- # Gemini thinking budget tests