diff --git a/.gitignore b/.gitignore index efb1f8ce..9fa90b3e 100644 --- a/.gitignore +++ b/.gitignore @@ -195,4 +195,4 @@ lancedb_data/ grafana-data/ # Claude Code addon stuff -.omc \ No newline at end of file +.omc diff --git a/docs/docs.json b/docs/docs.json index cbdd4e62..e5bb31e8 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -67,6 +67,7 @@ "v3/documentation/features/advanced/queue-status", "v3/documentation/features/advanced/search", "v3/documentation/features/advanced/using-filters", + "v3/documentation/features/advanced/structured-outputs", "v3/documentation/features/advanced/streaming-response", "v3/documentation/features/advanced/file-uploads" ] diff --git a/docs/v3/documentation/features/advanced/structured-outputs.mdx b/docs/v3/documentation/features/advanced/structured-outputs.mdx new file mode 100644 index 00000000..8e8f7e9a --- /dev/null +++ b/docs/v3/documentation/features/advanced/structured-outputs.mdx @@ -0,0 +1,239 @@ +--- +title: "Structured Outputs" +description: "Get chat endpoint answers as typed, machine-readable JSON" +icon: "brackets-curly" +--- + +By default, the [chat endpoint](/v3/documentation/features/chat) returns a free-form natural language answer. When your application needs machine-readable output, parsing that string yourself can be fragile and model-dependent. In this case you can use structured Dialectic outputs: pass a schema with your query, and the answer is guaranteed to conform to it. The agent still runs its full reasoning loop and only the final synthesized answer is formatted to your schema. + +## Basic Usage + +Pass a Pydantic model (Python) or Zod schema (TypeScript) as `response_format`, and the SDK returns a parsed, typed instance: + + +```python Python +from typing import Literal +from pydantic import BaseModel, Field +from honcho import Honcho + +class FoodPreference(BaseModel): + food: str + sentiment: Literal["loves", "likes", "neutral", "dislikes", "hates"] + confidence: float = Field(description="0-1, how certain the evidence is") + +class FoodPreferences(BaseModel): + preferences: list[FoodPreference] + summary: str + +honcho = Honcho() +peer = honcho.peer("user-123") + +result = peer.chat( + "What are this user's top 3 food preferences?", + response_format=FoodPreferences, +) + +# result is a FoodPreferences instance (or None if no relevant information) +if result: + for pref in result.preferences: + print(f"{pref.food}: {pref.sentiment} ({pref.confidence})") +``` + +```typescript TypeScript +import { z } from 'zod'; +import { Honcho } from '@honcho-ai/sdk'; + +const FoodPreferences = z.object({ + preferences: z.array(z.object({ + food: z.string(), + sentiment: z.enum(["loves", "likes", "neutral", "dislikes", "hates"]), + confidence: z.number(), + })), + summary: z.string(), +}); + +const honcho = new Honcho({}); +const peer = await honcho.peer("user-123"); + +const result = await peer.chat( + "What are this user's top 3 food preferences?", + { responseFormat: FoodPreferences }, +); + +// result is typed as z.infer (or null) +if (result) { + console.log(result.summary); +} +``` + + +## Using a Raw JSON Schema + +You can also pass a plain JSON Schema object instead of a Pydantic/Zod schema. In that case the SDK returns the answer as a JSON **string** and leaves parsing to you. This is also the shape the REST API accepts directly: + + +```python Python +result = peer.chat( + "What are this user's food preferences?", + response_format={ + "type": "object", + "properties": { + "foods": {"type": "array", "items": {"type": "string"}}, + }, + "required": ["foods"], + }, +) +# result is a JSON string, e.g. '{"foods": ["dark roast coffee", "sushi"]}' +``` + +```bash cURL +curl -X POST "$HONCHO_URL/v3/workspaces/my-app/peers/user-123/chat" \ + -H "Authorization: Bearer $HONCHO_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "query": "What are this user'\''s food preferences?", + "response_format": { + "type": "object", + "properties": { + "foods": { "type": "array", "items": { "type": "string" } } + }, + "required": ["foods"] + } + }' +``` + + +At the API level, `content` in the response is always a string. When `response_format` is set, it is a JSON-encoded object conforming to your schema. + +## Streaming + +`response_format` works with streaming. The stream emits the JSON answer incrementally as raw text chunks; the accumulated text is a valid JSON string once the stream completes. To enable streaming the SDKs cannot parse streamed responses for you, you parse the final string yourself: + + +```python Python +response_stream = peer.chat( + "What are this user's food preferences?", + stream=True, + response_format=FoodPreferences, +) + +chunks = [] +for chunk in response_stream.iter_text(): + chunks.append(chunk) + +result = FoodPreferences.model_validate_json("".join(chunks)) +``` + +```typescript TypeScript +const responseStream = await peer.chat( + "What are this user's food preferences?", + { stream: true, responseFormat: FoodPreferences }, +); + +let text = ""; +for await (const chunk of responseStream.iter_text()) { + text += chunk; +} + +const result = FoodPreferences.parse(JSON.parse(text)); +``` + + +## Supported Schema Subset + +Honcho supports a conservative subset of JSON Schema that enables the kind of Pydantic models used for structured LLM outputs. Schemas outside this subset are rejected with a `422` validation error before any reasoning runs. + +The root of the schema must be `"type": "object"`. + +| Construct | Support | +|-----------|---------| +| `string`, `number`, `integer`, `boolean`, `null` | Supported | +| `object` with `properties` (nested recursively) | Supported | +| `array` with `items` (missing `items` yields an untyped list) | Supported | +| `enum` of strings, integers, booleans, or null | Supported | +| `anyOf` / `oneOf` unions (a `null` member makes the field optional) | Supported | +| `type` given as a list (e.g. `["string", "null"]`) | Supported | +| `required`, `default`, `description` | Supported | +| Boolean `additionalProperties` | Accepted and ignored | +| `$ref` into root-level `$defs` / `definitions` | Supported — resolved by inlining (this is what Pydantic and Zod emit) | +| Recursive `$ref` (a definition that references itself, directly or indirectly) | Rejected (422) — the error will identify the cycle | +| Other `$ref` forms (external URLs, arbitrary JSON pointers) | Rejected (422) | +| `allOf`, `not`, `if` / `then` / `else` | Rejected (422) | +| `patternProperties`, schema-valued `additionalProperties` | Rejected (422) | + +Schemas may nest at most 20 levels deep and contain at most 500 total nodes. + + +Constraint keywords like `minItems`, `maxLength`, `minimum`, `pattern`, and `format` are passed through to the model as hints but are **not enforced server-side**. If you need hard guarantees on these, validate the returned object in your application. + + + +**Recursive schemas are not supported.** A self-referential Pydantic model (`Node.children: list[Node]`) or a recursive Zod schema (`z.lazy(...)`) produces a recursive `$ref`, which is rejected with a 422 naming the cycle. Restructure recursive shapes as explicit nesting with a fixed depth. + + +## Optional Fields and Unions + +Two distinct mechanisms control "optionality" in a raw JSON Schema: + +- **Omission** is controlled by `required`. A property not listed in `required` may be left out by the model entirely; the parsed answer will contain it as `null`. +- **Nullability** is controlled by the field's type. An `anyOf`/`oneOf` with a `{"type": "null"}` member (or the shorthand `"type": ["string", "null"]`) means the field's *value* may be `null` even when the field itself is required. + +`anyOf` and `oneOf` are treated identically: a plain union of the member schemas. Unions of non-null types (e.g. a string-or-integer field) are also supported. + +```json +{ + "type": "object", + "properties": { + "favorite_food": { "type": "string" }, + "dietary_restriction": { + "anyOf": [{ "type": "string" }, { "type": "null" }], + "description": "The user's dietary restriction, or null if none is known" + }, + "years_vegetarian": { "type": ["integer", "null"] }, + "confidence": { "type": "number" } + }, + "required": ["favorite_food", "dietary_restriction", "years_vegetarian"] +} +``` + +In this schema: + +- `favorite_food` is required and must be a string. +- `dietary_restriction` is required but **nullable**: the key is always present in the answer, and the model can answer `null` when it has no evidence. This is the recommended way to give the model an escape hatch (see [Best Practices](#model-uncertainty-explicitly)). +- `years_vegetarian` is the same thing written with the `type`-list shorthand (`["integer", "null"]` is equivalent to an `anyOf` of the two). +- `confidence` is not in `required`, so the model may omit it; if it does, the field comes back as `null`. + +If a property declares a `default`, that default is used whenever the model omits the field _even if_ the property is listed in `required`. + +Pydantic and Zod produce these shapes for you: `str | None` in Pydantic emits the `anyOf` form above, and `z.string().nullable()` does the same in Zod (`z.string().optional()` controls presence in `required`). + +## Error Handling + +| Condition | Result | +|-----------|--------| +| `response_format` is not a valid JSON Schema object | `422` validation error | +| Root type is not `"object"` | `422` validation error | +| Schema uses an unsupported construct | `422` identifying the construct and its path | +| Schema contains a recursive `$ref` | `422` identifying the cycle (e.g. `cycle: Node -> Node`) | +| Model fails to produce valid structured output after retries | `500`, same as any LLM failure | + +## How It Works + +Structured output constrains the final synthesis step. The reasoning itself works the same in both settings. + +1. The dialectic agent runs its normal tool loop in free-form text. It will search conclusions, grep messages, and traverse reasoning chains +2. Once the agent has gathered enough context, the final answer generation is constrained to your schema using the provider's native structured output support. +3. The conforming JSON is returned as the response `content` and parsed into a typed object by the SDK when you passed a Pydantic model or Zod schema. + +This means answer *quality* is unaffected by the schema: the agent reasons exactly as it would for a free-form answer, and reasoning levels (`minimal` through `max`) work the same way alongside `response_format`. + +## Best Practices + +### Add descriptions to your fields +Field `description`s are visible to the model when it formats the answer. `confidence: float` with a provided description of "score how certain the evidence is from 0-5" gets meaningfully better output than a bare field. + +### Model uncertainty explicitly +The chat endpoint returns `None`/`null` when it has no relevant information. With a schema, you can force an answer even when evidence is thin. To avoid hallucinations, consider including an escape hatch as an optional field, a `"confidence"` score, or an enum member like `"unknown"` so the model isn't forced to fabricate. + +### Keep schemas focused +A schema with three well-described fields outperforms one with twenty. If you need many distinct insights, consider making separate chat calls. diff --git a/docs/v3/documentation/features/chat.mdx b/docs/v3/documentation/features/chat.mdx index 7493e791..6aab6996 100644 --- a/docs/v3/documentation/features/chat.mdx +++ b/docs/v3/documentation/features/chat.mdx @@ -94,6 +94,43 @@ for await (const chunk of responseStream.iter_text()) { Streaming is useful for displaying real-time responses in chat interfaces or when asking complex questions that require longer answers. +## Structured Outputs + +When your application needs a machine-readable answer instead of prose, pass a schema as `response_format` and the answer is guaranteed to conform to it: + + +```python Python +from pydantic import BaseModel + +class OnboardingStatus(BaseModel): + completed: bool + remaining_steps: list[str] + +status = peer.chat( + "Has the user completed the onboarding flow?", + response_format=OnboardingStatus, +) +# status is a parsed OnboardingStatus instance +``` + +```typescript TypeScript +import { z } from 'zod'; + +const OnboardingStatus = z.object({ + completed: z.boolean(), + remainingSteps: z.array(z.string()), +}); + +const status = await peer.chat( + "Has the user completed the onboarding flow?", + { responseFormat: OnboardingStatus }, +); +// status is typed as z.infer +``` + + +The agent runs its full reasoning loop either way — only the final answer is formatted to your schema. See [Structured Outputs](/v3/documentation/features/advanced/structured-outputs) for the supported schema subset, streaming behavior, and best practices. + ## Integration Patterns ### Dynamic Prompt Enhancement diff --git a/sdks/python/src/honcho/aio.py b/sdks/python/src/honcho/aio.py index 2cf39ab1..51797c72 100644 --- a/sdks/python/src/honcho/aio.py +++ b/sdks/python/src/honcho/aio.py @@ -26,9 +26,9 @@ import logging import warnings from collections.abc import AsyncGenerator from datetime import datetime -from typing import TYPE_CHECKING, Any, ClassVar, Literal +from typing import TYPE_CHECKING, Any, ClassVar, Literal, overload -from pydantic import ConfigDict, Field, validate_call +from pydantic import BaseModel, ConfigDict, Field, validate_call from .api_types import ( ConclusionResponse, @@ -71,7 +71,7 @@ if TYPE_CHECKING: from .conclusions import ConclusionScope from .conclusions import ConclusionCreateParams -from .peer import Peer +from .peer import Peer, TResponseFormat, serialize_response_format from .session import Session logger = logging.getLogger(__name__) @@ -577,6 +577,30 @@ class PeerAio(AsyncMetadataConfigMixin): ) self._peer._configuration = configuration + @overload + async def chat( + self, + query: str, + *, + target: str | PeerBase | None = None, + session: str | SessionBase | None = None, + reasoning_level: Literal["minimal", "low", "medium", "high", "max"] + | None = None, + response_format: type[TResponseFormat], + ) -> TResponseFormat | None: ... + + @overload + async def chat( + self, + query: str, + *, + target: str | PeerBase | None = None, + session: str | SessionBase | None = None, + reasoning_level: Literal["minimal", "low", "medium", "high", "max"] + | None = None, + response_format: dict[str, Any] | None = None, + ) -> str | None: ... + @validate_call(config=ConfigDict(arbitrary_types_allowed=True)) async def chat( self, @@ -586,8 +610,14 @@ class PeerAio(AsyncMetadataConfigMixin): session: str | SessionBase | None = None, reasoning_level: Literal["minimal", "low", "medium", "high", "max"] | None = None, - ) -> str | None: - """Query the peer's representation asynchronously.""" + response_format: type[BaseModel] | dict[str, Any] | None = None, + ) -> BaseModel | str | None: + """Query the peer's representation asynchronously. + + See Peer.chat for parameter details. When response_format is a Pydantic + model class, the answer is parsed into an instance of it; when it is a + JSON Schema dict, the answer is a JSON string. + """ await self._peer._honcho._ensure_workspace_async() target_id = resolve_id(target) resolved_session_id = resolve_id(session) @@ -599,6 +629,9 @@ class PeerAio(AsyncMetadataConfigMixin): body["session_id"] = resolved_session_id if reasoning_level: body["reasoning_level"] = reasoning_level + response_format_schema = serialize_response_format(response_format) + if response_format_schema is not None: + body["response_format"] = response_format_schema data = await self._peer._honcho._async_http_client.post( routes.peer_chat(self._peer.workspace_id, self._peer.id), @@ -607,6 +640,8 @@ class PeerAio(AsyncMetadataConfigMixin): content = data.get("content") if not content: return None + if isinstance(response_format, type): + return response_format.model_validate_json(content) return content @validate_call(config=ConfigDict(arbitrary_types_allowed=True)) @@ -618,8 +653,14 @@ class PeerAio(AsyncMetadataConfigMixin): session: str | SessionBase | None = None, reasoning_level: Literal["minimal", "low", "medium", "high", "max"] | None = None, + response_format: type[BaseModel] | dict[str, Any] | None = None, ) -> AsyncDialecticStreamResponse: - """Query the peer's representation with streaming asynchronously.""" + """Query the peer's representation with streaming asynchronously. + + See Peer.chat_stream for parameter details. With response_format set, + chunks stay raw text that accumulates to a JSON string; parse it after + the stream completes. + """ await self._peer._honcho._ensure_workspace_async() target_id = resolve_id(target) resolved_session_id = resolve_id(session) @@ -631,6 +672,9 @@ class PeerAio(AsyncMetadataConfigMixin): body["session_id"] = resolved_session_id if reasoning_level: body["reasoning_level"] = reasoning_level + response_format_schema = serialize_response_format(response_format) + if response_format_schema is not None: + body["response_format"] = response_format_schema async def stream_response() -> AsyncGenerator[str, None]: async for content in parse_sse_astream( diff --git a/sdks/python/src/honcho/api_types.py b/sdks/python/src/honcho/api_types.py index 64ee7b65..f626c5a6 100644 --- a/sdks/python/src/honcho/api_types.py +++ b/sdks/python/src/honcho/api_types.py @@ -503,6 +503,7 @@ class DialecticParams(BaseModel): query: str = Field(min_length=1, max_length=10000) stream: bool = False reasoning_level: ReasoningLevel = "low" + response_format: dict[str, Any] | None = None class DialecticResponse(BaseModel): diff --git a/sdks/python/src/honcho/peer.py b/sdks/python/src/honcho/peer.py index f24357bb..e004db6a 100644 --- a/sdks/python/src/honcho/peer.py +++ b/sdks/python/src/honcho/peer.py @@ -7,9 +7,9 @@ import datetime import logging import warnings from collections.abc import Generator -from typing import TYPE_CHECKING, Any, Literal +from typing import TYPE_CHECKING, Any, Literal, TypeVar, overload -from pydantic import ConfigDict, Field, PrivateAttr, validate_call +from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, validate_call from .api_types import ( MessageCreateParams, @@ -38,6 +38,19 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +TResponseFormat = TypeVar("TResponseFormat", bound=BaseModel) + + +def serialize_response_format( + response_format: type[BaseModel] | dict[str, Any] | None, +) -> dict[str, Any] | None: + """Convert a chat response_format argument to a JSON Schema dict.""" + if response_format is None: + return None + if isinstance(response_format, type): + return response_format.model_json_schema() + return response_format + class Peer(PeerBase, MetadataConfigMixin): """ @@ -221,6 +234,30 @@ class Peer(PeerBase, MetadataConfigMixin): self._configuration = configuration # pyright: ignore[reportIncompatibleVariableOverride] self._created_at = created_at + @overload + def chat( + self, + query: str, + *, + target: str | PeerBase | None = None, + session: str | SessionBase | None = None, + reasoning_level: Literal["minimal", "low", "medium", "high", "max"] + | None = None, + response_format: type[TResponseFormat], + ) -> TResponseFormat | None: ... + + @overload + def chat( + self, + query: str, + *, + target: str | PeerBase | None = None, + session: str | SessionBase | None = None, + reasoning_level: Literal["minimal", "low", "medium", "high", "max"] + | None = None, + response_format: dict[str, Any] | None = None, + ) -> str | None: ... + @validate_call(config=ConfigDict(arbitrary_types_allowed=True)) def chat( self, @@ -230,7 +267,8 @@ class Peer(PeerBase, MetadataConfigMixin): session: str | SessionBase | None = None, reasoning_level: Literal["minimal", "low", "medium", "high", "max"] | None = None, - ) -> str | None: + response_format: type[BaseModel] | dict[str, Any] | None = None, + ) -> BaseModel | str | None: """ Query the peer's representation with a natural language question. @@ -249,9 +287,15 @@ class Peer(PeerBase, MetadataConfigMixin): ID string or a Session object. reasoning_level: Optional reasoning level for the query: "minimal", "low", "medium", "high", or "max". Defaults to "low" if not provided. + response_format: Optional structure for the answer. Pass a Pydantic + model class to get a parsed instance back, or a raw + JSON Schema dict (root type "object") to get the + answer as a JSON string. Returns: - Response string containing the answer, or None if no relevant information + Response string containing the answer (a JSON string when a schema + dict was given), a parsed model instance when a Pydantic model class + was given, or None if no relevant information. """ self._honcho._ensure_workspace() target_id = resolve_id(target) @@ -264,6 +308,9 @@ class Peer(PeerBase, MetadataConfigMixin): body["session_id"] = resolved_session_id if reasoning_level: body["reasoning_level"] = reasoning_level + response_format_schema = serialize_response_format(response_format) + if response_format_schema is not None: + body["response_format"] = response_format_schema data = self._honcho._http.post( routes.peer_chat(self.workspace_id, self.id), @@ -272,6 +319,8 @@ class Peer(PeerBase, MetadataConfigMixin): content = data.get("content") if not content: return None + if isinstance(response_format, type): + return response_format.model_validate_json(content) return content @validate_call(config=ConfigDict(arbitrary_types_allowed=True)) @@ -283,6 +332,7 @@ class Peer(PeerBase, MetadataConfigMixin): session: str | SessionBase | None = None, reasoning_level: Literal["minimal", "low", "medium", "high", "max"] | None = None, + response_format: type[BaseModel] | dict[str, Any] | None = None, ) -> DialecticStreamResponse: """ Query the peer's representation with a natural language question, streaming the response. @@ -302,6 +352,11 @@ class Peer(PeerBase, MetadataConfigMixin): ID string or a Session object. reasoning_level: Optional reasoning level for the query: "minimal", "low", "medium", "high", or "max". Defaults to "low" if not provided. + response_format: Optional structure for the answer: a Pydantic model + class or a JSON Schema dict (root type "object"). + Streamed chunks stay raw text that accumulates to a + JSON string; parse it yourself (e.g. with + Model.model_validate_json) once the stream completes. Returns: DialecticStreamResponse object that can be iterated over and provides final response @@ -317,6 +372,9 @@ class Peer(PeerBase, MetadataConfigMixin): body["session_id"] = resolved_session_id if reasoning_level: body["reasoning_level"] = reasoning_level + response_format_schema = serialize_response_format(response_format) + if response_format_schema is not None: + body["response_format"] = response_format_schema def stream_response() -> Generator[str, None, None]: yield from parse_sse_stream( diff --git a/sdks/typescript/__tests__/peer.test.ts b/sdks/typescript/__tests__/peer.test.ts index d8a3750c..fca85910 100644 --- a/sdks/typescript/__tests__/peer.test.ts +++ b/sdks/typescript/__tests__/peer.test.ts @@ -16,6 +16,7 @@ */ import { describe, test, expect, beforeAll, afterAll } from 'bun:test' +import { z } from 'zod' import { Honcho, Peer } from '../src' import { createTestClient, generateId, requireServer } from './setup' import { @@ -624,6 +625,44 @@ describe('Peer', () => { expect(response === null || typeof response === 'string').toBe(true) }) + test('chat with responseFormat as JSON schema object returns JSON string', async () => { + const peer = await client.peer('chat-rf-peer') + + const response = await peer.chat('What do you know?', { + responseFormat: { + type: 'object', + properties: { summary: { type: 'string' } }, + }, + }) + + expect(response === null || typeof response === 'string').toBe(true) + if (response !== null) { + expect(() => JSON.parse(response)).not.toThrow() + } + }) + + test('chat with responseFormat as Zod schema returns parsed object', async () => { + const peer = await client.peer('chat-rf-zod-peer') + const ResultSchema = z.object({ summary: z.string().optional() }) + + const response = await peer.chat('What do you know?', { + responseFormat: ResultSchema, + }) + + expect(response === null || typeof response === 'object').toBe(true) + }) + + test('chat with unsupported responseFormat is rejected by the server', async () => { + const peer = await client.peer('chat-rf-invalid-peer') + + // Non-object root is rejected with 422 + await expect( + peer.chat('What do you know?', { + responseFormat: { type: 'string' }, + }) + ).rejects.toThrow() + }) + // Streaming tests are in streaming.test.ts }) diff --git a/sdks/typescript/__tests__/validation.test.ts b/sdks/typescript/__tests__/validation.test.ts index 42d03cf7..97d71f71 100644 --- a/sdks/typescript/__tests__/validation.test.ts +++ b/sdks/typescript/__tests__/validation.test.ts @@ -5,7 +5,7 @@ */ import { describe, test, expect } from 'bun:test' -import { ZodError } from 'zod' +import { z, ZodError } from 'zod' import { ChatQuerySchema, ContextParamsSchema, @@ -67,6 +67,24 @@ describe('ChatQuerySchema', () => { } ) + test('responseFormat as plain JSON schema object is valid', () => { + const schema = { type: 'object', properties: { a: { type: 'string' } } } + const result = ChatQuerySchema.parse({ query: 'hello', responseFormat: schema }) + expect(result.responseFormat).toEqual(schema) + }) + + test('responseFormat as Zod schema instance is valid and passed through', () => { + const schema = z.object({ a: z.string() }) + const result = ChatQuerySchema.parse({ query: 'hello', responseFormat: schema }) + expect(result.responseFormat).toBe(schema) + }) + + test('responseFormat as a non-object throws', () => { + expect(() => + ChatQuerySchema.parse({ query: 'hello', responseFormat: 'not-a-schema' }) + ).toThrow(ZodError) + }) + // --- Missing required fields --- test('missing query throws', () => { diff --git a/sdks/typescript/src/peer.ts b/sdks/typescript/src/peer.ts index b4c1d756..adb67cb2 100644 --- a/sdks/typescript/src/peer.ts +++ b/sdks/typescript/src/peer.ts @@ -1,3 +1,4 @@ +import { ZodType, z } from 'zod' import { API_VERSION } from './api-version' import { ConclusionScope } from './conclusions' import type { HonchoHTTPClient } from './http/client' @@ -229,12 +230,29 @@ export class Peer { ) } + /** + * Convert a responseFormat option (Zod schema or raw JSON Schema object) + * to the JSON Schema dict the API expects. + */ + private static toResponseFormatSchema( + responseFormat: ZodType | Record | undefined + ): Record | undefined { + if (!responseFormat) { + return undefined + } + if (responseFormat instanceof ZodType) { + return z.toJSONSchema(responseFormat) as Record + } + return responseFormat + } + private async _chat(params: { query: string stream?: boolean target?: string session_id?: string reasoning_level?: string + response_format?: Record }): Promise { await this._ensureWorkspace() return this._http.post( @@ -248,6 +266,7 @@ export class Peer { target?: string session_id?: string reasoning_level?: string + response_format?: Record }): Promise { await this._ensureWorkspace() return this._http.stream( @@ -362,14 +381,33 @@ export class Peer { * }) * ``` */ + async chat( + query: string, + options: { + target?: string | Peer + session?: string | Session + reasoningLevel?: string + responseFormat: ZodType + } + ): Promise async chat( query: string, options?: { target?: string | Peer session?: string | Session reasoningLevel?: string + responseFormat?: Record } - ): Promise { + ): Promise + async chat( + query: string, + options?: { + target?: string | Peer + session?: string | Session + reasoningLevel?: string + responseFormat?: ZodType | Record + } + ): Promise { const targetId = options?.target ? typeof options.target === 'string' ? options.target @@ -386,18 +424,28 @@ export class Peer { target: targetId, session: resolvedSessionId, reasoningLevel: options?.reasoningLevel, + responseFormat: options?.responseFormat, }) + const zodSchema = + options?.responseFormat instanceof ZodType + ? options.responseFormat + : undefined + const response = await this._chat({ query: chatParams.query, stream: false, target: chatParams.target, session_id: chatParams.session, reasoning_level: chatParams.reasoningLevel, + response_format: Peer.toResponseFormatSchema(options?.responseFormat), }) if (!response.content) { return null } + if (zodSchema) { + return zodSchema.parse(JSON.parse(response.content)) + } return response.content } @@ -442,6 +490,7 @@ export class Peer { target?: string | Peer session?: string | Session reasoningLevel?: string + responseFormat?: ZodType | Record } ): Promise { const targetId = options?.target @@ -460,6 +509,7 @@ export class Peer { target: targetId, session: resolvedSessionId, reasoningLevel: options?.reasoningLevel, + responseFormat: options?.responseFormat, }) const response = await this._chatStream({ @@ -467,6 +517,7 @@ export class Peer { target: chatParams.target, session_id: chatParams.session, reasoning_level: chatParams.reasoningLevel, + response_format: Peer.toResponseFormatSchema(options?.responseFormat), }) return createDialecticStream(response) diff --git a/sdks/typescript/src/types/api.ts b/sdks/typescript/src/types/api.ts index dda25ad1..dbe4378d 100644 --- a/sdks/typescript/src/types/api.ts +++ b/sdks/typescript/src/types/api.ts @@ -74,6 +74,7 @@ export interface PeerChatParams { session_id?: string target?: string reasoning_level?: 'minimal' | 'low' | 'medium' | 'high' | 'max' + response_format?: Record } export interface PeerChatResponse { diff --git a/sdks/typescript/src/validation.ts b/sdks/typescript/src/validation.ts index fed1b10b..48255833 100644 --- a/sdks/typescript/src/validation.ts +++ b/sdks/typescript/src/validation.ts @@ -312,6 +312,11 @@ export const ChatQuerySchema = z reasoningLevel: z .enum(['minimal', 'low', 'medium', 'high', 'max']) .optional(), + // A Zod schema (checked first — it is itself an object) or a raw JSON + // Schema object describing the desired response structure. + responseFormat: z + .union([z.instanceof(z.ZodType), z.record(z.string(), z.unknown())]) + .optional(), }) .strict() diff --git a/src/dialectic/chat.py b/src/dialectic/chat.py index 9c118803..40d991b4 100644 --- a/src/dialectic/chat.py +++ b/src/dialectic/chat.py @@ -8,6 +8,8 @@ using the DialecticAgent. import logging from collections.abc import AsyncIterator +from pydantic import BaseModel + from src import crud, schemas from src.config import ReasoningLevel from src.dependencies import tracked_db @@ -24,6 +26,7 @@ async def agentic_chat( observer: str, observed: str, reasoning_level: ReasoningLevel = "low", + response_model: type[BaseModel] | None = None, ) -> str: """ Answer a query about a peer using the agentic dialectic. @@ -35,6 +38,8 @@ async def agentic_chat( observer: The peer making the query observed: The peer being queried about reasoning_level: Level of reasoning to apply + response_model: Optional Pydantic model the answer must conform to. + When set, the returned string is JSON matching the model's schema. Returns: The synthesized answer string @@ -79,7 +84,7 @@ async def agentic_chat( reasoning_level=reasoning_level, ) - return await agent.answer(query) + return await agent.answer(query, response_model=response_model) async def agentic_chat_stream( @@ -89,6 +94,7 @@ async def agentic_chat_stream( observer: str, observed: str, reasoning_level: ReasoningLevel = "low", + response_model: type[BaseModel] | None = None, ) -> AsyncIterator[str]: """ Stream an answer to a query about a peer using the agentic dialectic. @@ -100,6 +106,9 @@ async def agentic_chat_stream( observer: The peer making the query observed: The peer being queried about reasoning_level: Level of reasoning to apply + response_model: Optional Pydantic model the answer must conform to. + When set, the streamed text accumulates to JSON matching the + model's schema. Yields: Chunks of the response text as they are generated @@ -144,5 +153,5 @@ async def agentic_chat_stream( reasoning_level=reasoning_level, ) - async for chunk in agent.answer_stream(query): + async for chunk in agent.answer_stream(query, response_model=response_model): yield chunk diff --git a/src/dialectic/core.py b/src/dialectic/core.py index 6f5b17ab..757423e0 100644 --- a/src/dialectic/core.py +++ b/src/dialectic/core.py @@ -11,6 +11,7 @@ from collections.abc import AsyncIterator, Callable from typing import Any, cast from nanoid import generate as generate_nanoid +from pydantic import BaseModel from src import crud from src.config import ConfiguredModelSettings, ReasoningLevel, settings @@ -413,7 +414,9 @@ class DialecticAgent: ) ) - async def answer(self, query: str) -> str: + async def answer( + self, query: str, response_model: type[BaseModel] | None = None + ) -> str: """ Answer a query about the peer using agentic tool calling. @@ -424,6 +427,8 @@ class DialecticAgent: Args: query: The question to answer about the peer + response_model: Optional Pydantic model the final synthesis must + conform to. When set, the returned string is JSON. Returns: The synthesized answer string @@ -446,25 +451,38 @@ class DialecticAgent: else settings.DIALECTIC.MAX_OUTPUT_TOKENS ) - response: HonchoLLMCallResponse[str] = await honcho_llm_call( - model_config=_get_dialectic_level_model_config(self.reasoning_level), - prompt="", # Ignored since we pass messages - max_tokens=max_tokens, - tools=tools, - tool_choice=level_settings.TOOL_CHOICE, - tool_executor=tool_executor, - max_tool_iterations=level_settings.MAX_TOOL_ITERATIONS, - messages=self.messages, - max_input_tokens=settings.DIALECTIC.MAX_INPUT_TOKENS, - trace_name="dialectic_chat", - telemetry=self._telemetry_context(track_name="Dialectic Agent"), + # cast: `type[BaseModel] | None` matches neither the parsed nor the + # plain-text overload statically, so pyright resolves the stream + # overload — but without stream=True the call is non-streaming. + response = cast( # pyright: ignore[reportInvalidCast] + HonchoLLMCallResponse[Any], + await honcho_llm_call( + model_config=_get_dialectic_level_model_config(self.reasoning_level), + prompt="", # Ignored since we pass messages + max_tokens=max_tokens, + tools=tools, + tool_choice=level_settings.TOOL_CHOICE, + tool_executor=tool_executor, + max_tool_iterations=level_settings.MAX_TOOL_ITERATIONS, + messages=self.messages, + max_input_tokens=settings.DIALECTIC.MAX_INPUT_TOKENS, + trace_name="dialectic_chat", + telemetry=self._telemetry_context(track_name="Dialectic Agent"), + response_model=response_model, + ), ) + # With response_model, the backend parses content into a model + # instance; the API contract is a JSON string. + content = response.content + if isinstance(content, BaseModel): + content = content.model_dump_json(by_alias=True) + self._log_response_metrics( task_name=task_name, run_id=run_id, start_time=start_time, - response_content=response.content, + response_content=content, input_tokens=response.input_tokens, output_tokens=response.output_tokens, cache_read_input_tokens=response.cache_read_input_tokens, @@ -475,9 +493,11 @@ class DialecticAgent: hit_input_token_cap=response.hit_input_token_cap, ) - return response.content + return content - async def answer_stream(self, query: str) -> AsyncIterator[str]: + async def answer_stream( + self, query: str, response_model: type[BaseModel] | None = None + ) -> AsyncIterator[str]: """ Answer a query about the peer using agentic tool calling, streaming the response. @@ -488,6 +508,9 @@ class DialecticAgent: Args: query: The question to answer about the peer + response_model: Optional Pydantic model the final synthesis must + conform to. When set, the streamed text accumulates to JSON + (chunks are raw text; no parsing happens on the stream path). Yields: Chunks of the response text as they are generated @@ -526,6 +549,7 @@ class DialecticAgent: max_input_tokens=settings.DIALECTIC.MAX_INPUT_TOKENS, trace_name="dialectic_chat", telemetry=self._telemetry_context(track_name="Dialectic Agent Stream"), + response_model=response_model, ), ) diff --git a/src/llm/backends/openai.py b/src/llm/backends/openai.py index 6d13f1fc..d5d0ed73 100644 --- a/src/llm/backends/openai.py +++ b/src/llm/backends/openai.py @@ -2,8 +2,8 @@ from __future__ import annotations import json import logging +import weakref from collections.abc import AsyncIterator -from functools import cache from typing import Any, cast from openai import BadRequestError, LengthFinishReasonError @@ -22,22 +22,35 @@ from src.llm.structured_output import ( logger = logging.getLogger(__name__) -@cache +# The point of this being a WeakKeyDictionary, as opposed to a regular dict, is that it +# does not hold a reference to the keyed BaseModel so that when a dynamically created +# type is no longer referenced it becomes eligible for garbage collection. This avoids a +# memory leak. +_json_object_instruction_cache: weakref.WeakKeyDictionary[type[BaseModel], str] = ( + weakref.WeakKeyDictionary() +) + + def _json_object_instruction(response_format: type[BaseModel]) -> str: """Schema-injection instruction for json_object mode. The JSON schema is static per response_format class, so cache the serialized - instruction — the deriver issues one structured call per batch on the worker - hot path and would otherwise re-walk the schema + re-serialize it every call. + instruction — the deriver would otherwise re-walk the schema + re-serialize + it every call. """ + cached = _json_object_instruction_cache.get(response_format) + if cached is not None: + return cached # Some OpenAI-compatible providers enforce this JSON-object precondition with # a case-sensitive substring check, so include lowercase "json" explicitly. - return ( + instruction = ( "You must respond with a single JSON object (json) that conforms " "exactly to the following JSON schema. Do not include any text, " "markdown, or code fences outside the JSON object.\n\nJSON schema:\n" f"{json.dumps(response_format.model_json_schema())}" ) + _json_object_instruction_cache[response_format] = instruction + return instruction def _uses_max_completion_tokens(model: str) -> bool: diff --git a/src/routers/peers.py b/src/routers/peers.py index 90cdc1a5..b61bf442 100644 --- a/src/routers/peers.py +++ b/src/routers/peers.py @@ -10,6 +10,7 @@ from fastapi import APIRouter, Body, Depends, Path, Query, Response from fastapi.responses import StreamingResponse from fastapi_pagination import Page from fastapi_pagination.ext.sqlalchemy import apaginate +from pydantic import BaseModel from sqlalchemy.ext.asyncio import AsyncSession from src import crud, schemas @@ -18,10 +19,15 @@ from src.crud.session import is_peer_in_session from src.dependencies import db, read_db, tracked_db from src.dialectic.chat import agentic_chat, agentic_chat_stream from src.embedding_client import embedding_client -from src.exceptions import AuthenticationException, ResourceNotFoundException +from src.exceptions import ( + AuthenticationException, + ResourceNotFoundException, + ValidationException, +) from src.security import JWTParams, require_auth from src.telemetry import prometheus_metrics from src.telemetry.events import EmbeddingCallPurpose, GetContextEvent, emit +from src.utils.schema_conversion import json_response_schema_to_pydantic from src.utils.search import search from src.utils.types import embedding_call_purpose @@ -193,6 +199,14 @@ async def chat( ): raise AuthenticationException("JWT not permissioned for this resource") + # Convert the caller's JSON Schema so malformed schemas fail immediately with 422 + response_model: type[BaseModel] | None = None + if options.response_format is not None: + try: + response_model = json_response_schema_to_pydantic(options.response_format) + except ValueError as e: + raise ValidationException(f"Invalid response_format: {e}") from None + # Get or create the peer to ensure it exists async with tracked_db("peers.chat.get_or_create_peer") as peer_db: peers_result = await crud.get_or_create_peers( @@ -230,6 +244,7 @@ async def chat( observer=peer_id, observed=options.target if options.target is not None else peer_id, reasoning_level=options.reasoning_level, + response_model=response_model, ) ), media_type="text/event-stream", @@ -244,6 +259,7 @@ async def chat( # and it's answered from the omniscient Honcho perspective observed=options.target if options.target is not None else peer_id, reasoning_level=options.reasoning_level, + response_model=response_model, ) # Prometheus metrics diff --git a/src/schemas/api.py b/src/schemas/api.py index a276c4b7..ef3ebbc1 100644 --- a/src/schemas/api.py +++ b/src/schemas/api.py @@ -574,6 +574,16 @@ class DialecticOptions(BaseModel): default="low", description="Level of reasoning to apply: minimal, low, medium, high, or max", ) + response_format: dict[str, Any] | None = Field( + None, + description=( + "Optional JSON Schema (root type 'object') the response must conform" + " to. When provided, `content` is a JSON string matching this schema." + " Only a conservative subset of JSON Schema is supported; unsupported" + " schemas are rejected with 422. Constraint keywords (minItems, " + " maxLength, ...) are hints to the model, not enforced server-side." + ), + ) @field_validator("query", mode="after") @classmethod diff --git a/src/utils/schema_conversion.py b/src/utils/schema_conversion.py new file mode 100644 index 00000000..3e715f4b --- /dev/null +++ b/src/utils/schema_conversion.py @@ -0,0 +1,505 @@ +"""Convert caller-supplied JSON Schema objects into dynamic Pydantic models. + +Used by the dialectic chat endpoint's ``response_format`` option: the caller +sends a JSON Schema dict, and the resulting model is passed as +``response_model`` to ``honcho_llm_call()`` so providers return conforming +JSON. + +Only a conservative subset of JSON Schema is supported (see +``json_response_schema_to_pydantic``). Conversion doubles as validation: any +unsupported construct raises ``ValueError`` with the offending path, which the +router surfaces as a 422. +""" + +import re +from dataclasses import dataclass, field +from typing import Any, Literal, NoReturn, cast + +from pydantic import BaseModel, ConfigDict, Field, create_model + +# "$defs"/"definitions" are extracted at the root before the walk and are +# rejected anywhere else. "$ref" nodes are resolved by _resolve_ref before +# node validation, so they never reach this check. +_UNSUPPORTED_KEYS = ( + "$defs", + "definitions", + "allOf", + "not", + "if", + "then", + "else", + "patternProperties", +) + +_REF_PREFIXES = ("#/$defs/", "#/definitions/") + +_PRIMITIVE_TYPES: dict[str, Any] = { + "string": str, + "number": float, + "integer": int, + "boolean": bool, + "null": type(None), +} + +# Constraint keywords are forwarded to the model's json_schema_extra so the +# LLM sees them, but Pydantic does not enforce them (they are hints only). +_HINT_KEYS = ( + "minItems", + "maxItems", + "minLength", + "maxLength", + "minimum", + "maximum", + "exclusiveMinimum", + "exclusiveMaximum", + "multipleOf", + "pattern", + "format", + "minProperties", + "maxProperties", + "uniqueItems", +) + +_IDENTIFIER_RE = re.compile(r"^[A-Za-z][A-Za-z0-9_]*$") + + +@dataclass +class _Ctx: + """Mutable state shared across one conversion walk.""" + + max_depth: int + max_nodes: int + defs: dict[str, Any] = field(default_factory=dict) + used_names: set[str] = field(default_factory=set) + ref_stack: list[str] = field(default_factory=list) + node_count: int = 0 + + +def json_response_schema_to_pydantic( + schema: dict[str, Any], + *, + model_name: str = "ResponseFormat", + max_depth: int = 20, + max_nodes: int = 500, +) -> type[BaseModel]: + """Convert a JSON Schema dict (root type ``object``) into a Pydantic model. + + Supported constructs: primitive types (``string``/``number``/``integer``/ + ``boolean``/``null``), nested ``object`` with ``properties``, ``array`` + with ``items`` (missing ``items`` yields ``list[Any]``), ``enum`` of + strings/integers/booleans/null, ``anyOf``/``oneOf`` unions (a + ``{"type": "null"}`` member yields an optional), ``type`` given as a list, + ``required``, ``default``, and ``description``. A root-level ``$schema`` + key is ignored. Boolean ``additionalProperties`` is accepted and ignored; + extra keys in LLM output are silently dropped (``extra="ignore"``). + + ``$ref`` is supported for references of the form ``#/$defs/`` or + ``#/definitions/`` into root-level ``$defs``/``definitions`` (this + is what Pydantic's ``model_json_schema()`` and Zod's ``toJSONSchema`` + emit). References are resolved by inlining; sibling keys next to ``$ref`` + overlay the referenced definition (siblings win). Recursive references + are rejected — the error names the cycle. Unreferenced definitions are + ignored without validation. + + Constraint keywords (``minItems``, ``maxLength``, ``minimum``, + ``pattern``, ...) are passed through to the generated schema as hints but + are not enforced by Pydantic. + + Args: + schema: The JSON Schema object. Root must resolve to type ``object``. + model_name: Name for the generated root model class. + max_depth: Maximum nesting depth. Guard against excessive or + malicious schemas (e.g. pathologically deep nesting); exceeding + it raises ``ValueError``. + max_nodes: Maximum total nodes visited across the whole schema. + Guard against excessive or malicious schemas (e.g. enormous + property fan-out); exceeding it raises ``ValueError``. + + Returns: + A dynamically created Pydantic model class. + + Raises: + ValueError: If the schema is malformed or uses an unsupported + construct (``allOf``, ``not``, ``if``/``then``/``else``, + ``patternProperties``, schema-valued ``additionalProperties``, + boolean schemas, unknown types, a non-object root, a ``$ref`` + that is recursive, malformed, or targets an unknown definition, + or ``$defs``/``definitions`` anywhere but the root). The message + names the construct and its path. + """ + schema_obj: Any = schema + if not isinstance(schema_obj, dict): + raise ValueError("response_format must be a JSON Schema object") + + root = {k: v for k, v in schema.items() if k != "$schema"} + defs = _extract_defs(root) + root_type = root.get("type") + # A "$ref" root is allowed through here; the post-conversion check below + # still enforces that it resolves to an object. + is_object_root = root_type == "object" or ( + root_type is None and ("properties" in root or "$ref" in root) + ) + if not is_object_root: + raise ValueError("root schema must have type 'object'") + + ctx = _Ctx(max_depth=max_depth, max_nodes=max_nodes, defs=defs) + annotation = _convert_schema(root, "", model_name, ctx, depth=0) + # An object root always converts to a model class; this is a safety net. + if not (isinstance(annotation, type) and issubclass(annotation, BaseModel)): + raise ValueError("root schema must have type 'object'") + return annotation + + +def _fail(msg: str, path: str) -> NoReturn: + raise ValueError(f"{msg} at {path or 'root'}") + + +def _union(members: tuple[Any, ...]) -> Any: + """Build ``A | B | ...`` from a dynamic tuple of annotations.""" + result: Any = members[0] + for member in members[1:]: + result = result | member + return result + + +def _convert_schema( + raw_node: Any, path: str, name_hint: str, ctx: _Ctx, depth: int +) -> Any: + """Convert one schema node into a type annotation. + + Dispatch order matters: $ref resolution comes first (a ref node is + replaced by its target before anything else looks at it), then enum (a + value constraint) wins over unions, which win over "type"-based + conversion. + """ + if isinstance(raw_node, dict) and "$ref" in raw_node: + return _resolve_ref(cast(dict[str, Any], raw_node), path, ctx, depth) + + node = _validate_node(raw_node, path, ctx, depth) + + if "enum" in node: + return _convert_enum(node["enum"], path) + + if "anyOf" in node or "oneOf" in node: + return _convert_union(node, path, name_hint, ctx, depth) + + node_type = node.get("type") + if isinstance(node_type, list): + return _convert_type_list( + node, cast(list[Any], node_type), path, name_hint, ctx, depth + ) + + # Tolerate an omitted "type" when "properties" makes the intent clear. + if node_type is None and "properties" in node: + node_type = "object" + + if node_type == "object": + return _build_object_model(node, path, name_hint, ctx, depth) + + if node_type == "array": + return _convert_array(node, path, name_hint, ctx, depth) + + if node_type in _PRIMITIVE_TYPES: + return _PRIMITIVE_TYPES[node_type] + + if node_type is None: + _fail("schema has no recognizable type", path) + _fail(f"unsupported type '{node_type}'", path) + + +def _extract_defs(root: dict[str, Any]) -> dict[str, Any]: + """Pop root-level ``$defs``/``definitions`` and merge them into one + registry. Entries are validated lazily, when (and only when) referenced.""" + defs: dict[str, Any] = {} + for key in ("$defs", "definitions"): + raw = root.pop(key, None) + if raw is None: + continue + if not isinstance(raw, dict): + _fail(f"'{key}' must be an object", "") + for name, definition in cast(dict[Any, Any], raw).items(): + if not isinstance(name, str) or not name: + _fail(f"'{key}' definition names must be non-empty strings", "") + if name in defs: + _fail( + f"definition '{name}' appears in both '$defs' and 'definitions'", + "", + ) + defs[name] = definition + return defs + + +def _resolve_ref(node: dict[str, Any], path: str, ctx: _Ctx, depth: int) -> Any: + """Inline a ``$ref`` node: resolve the target definition, overlay any + sibling keys (siblings win), and convert the result in place. + + Only root-relative refs into ``$defs``/``definitions`` are supported. + Cycles are rejected — recursion cannot be inlined. The resolved node is + converted at the same depth (replacement semantics); the node budget in + ``_validate_node`` still counts every expansion, so a definition that is + referenced many times cannot blow up the walk. + """ + ref: Any = node["$ref"] + if not isinstance(ref, str): + _fail("'$ref' must be a string", path) + name: str | None = None + for prefix in _REF_PREFIXES: + if ref.startswith(prefix): + name = ref[len(prefix) :] + break + # "/", "~", and "%" would make the remainder a deeper or escaped JSON + # pointer (e.g. "#/$defs/a/b", "~1" escapes, %-encoding) rather than a + # plain definition name, so their presence means an unsupported form. + if not name or "/" in name or "~" in name or "%" in name: + _fail( + f"unsupported $ref '{ref}': only '#/$defs/' or " + + "'#/definitions/' references are supported", + path, + ) + if name not in ctx.defs: + _fail(f"$ref '{ref}' points to an unknown definition", path) + # ref_stack holds the definitions currently being expanded on this branch + # of the walk, so membership means the definition (transitively) + # references itself. Slicing from the first occurrence yields the cycle + # for the error message, e.g. stack [A, B] + name A -> "A -> B -> A". + if name in ctx.ref_stack: + cycle = " -> ".join([*ctx.ref_stack[ctx.ref_stack.index(name) :], name]) + _fail( + f"recursive $ref is not supported (cycle: {cycle}); " + + "restructure the schema so definitions do not reference themselves", + path, + ) + target: Any = ctx.defs[name] + siblings = {k: v for k, v in node.items() if k != "$ref"} + resolved: Any = target + if siblings and isinstance(target, dict): + resolved = {**cast(dict[str, Any], target), **siblings} + # Pop after converting (not just on success) so the stack tracks only the + # current branch: a diamond — the same definition referenced from two + # sibling nodes — is legitimate reuse, not a cycle. + ctx.ref_stack.append(name) + try: + return _convert_schema(resolved, path, name, ctx, depth) + finally: + ctx.ref_stack.pop() + + +def _validate_node(raw_node: Any, path: str, ctx: _Ctx, depth: int) -> dict[str, Any]: + """Enforce size budgets and node shape; reject unsupported constructs.""" + # node_count is cumulative across the whole walk; depth tracks only the + # current branch. + ctx.node_count += 1 + if ctx.node_count > ctx.max_nodes: + raise ValueError(f"schema exceeds the maximum of {ctx.max_nodes} nodes") + if depth > ctx.max_depth: + raise ValueError(f"schema nesting exceeds the maximum depth of {ctx.max_depth}") + if isinstance(raw_node, bool): + # A special case of the object requirement, with its own message: + # boolean schemas are legal JSON Schema, just deliberately unsupported. + _fail("boolean schemas are not supported", path) + if not isinstance(raw_node, dict): + _fail("schema must be an object", path) + node = cast(dict[str, Any], raw_node) + for key in _UNSUPPORTED_KEYS: + if key in node: + _fail(f"unsupported construct '{key}'", path) + if isinstance(node.get("additionalProperties"), dict): + _fail("additionalProperties with a schema is not supported", path) + return node + + +def _convert_union( + node: dict[str, Any], path: str, name_hint: str, ctx: _Ctx, depth: int +) -> Any: + """Convert anyOf/oneOf, treated identically: a plain union of the member + schemas (a {"type": "null"} member makes the union optional).""" + union_key = "anyOf" if "anyOf" in node else "oneOf" + members = node[union_key] + if not isinstance(members, list) or not members: + _fail(f"'{union_key}' must be a non-empty array", path) + converted = tuple( + _convert_schema( + member, + _child_path(path, f"{union_key}[{i}]"), + f"{name_hint}Option{i}", + ctx, + depth + 1, + ) + for i, member in enumerate(cast(list[Any], members)) + ) + return _union(converted) + + +def _convert_type_list( + node: dict[str, Any], + types: list[Any], + path: str, + name_hint: str, + ctx: _Ctx, + depth: int, +) -> Any: + """Convert the type: ["string", "null"] sugar — re-convert the node once + per entry (keeping sibling keys, at the same depth since it's the same + source node) and union the results.""" + if not types: + _fail("'type' array must not be empty", path) + variants = tuple( + _convert_schema( + {**{k: v for k, v in node.items() if k != "type"}, "type": t}, + path, + name_hint, + ctx, + depth, + ) + for t in types + ) + return _union(variants) + + +def _convert_array( + node: dict[str, Any], path: str, name_hint: str, ctx: _Ctx, depth: int +) -> Any: + """Convert an array schema into a list annotation.""" + items = node.get("items") + # A missing "items" constraint means any element type is allowed. + if items is None: + return list[Any] + item_annotation = _convert_schema( + items, _child_path(path, "items"), f"{name_hint}Item", ctx, depth + 1 + ) + return list[item_annotation] + + +def _convert_enum(raw_values: Any, path: str) -> Any: + if not isinstance(raw_values, list) or not raw_values: + _fail("'enum' must be a non-empty array", path) + literal_values: list[Any] = [] + # None is not Literal-legal, so collect it separately and union NoneType + # back in at the end (an all-null enum degenerates to NoneType). + has_null = False + for value in cast(list[Any], raw_values): + if value is None: + has_null = True + elif isinstance(value, str | int | bool): + literal_values.append(value) + else: + _fail("enum values must be strings, integers, booleans, or null", path) + if not literal_values: + return type(None) + annotation: Any = Literal[tuple(literal_values)] + return _union((annotation, type(None))) if has_null else annotation + + +def _build_object_model( + node: dict[str, Any], path: str, name_hint: str, ctx: _Ctx, depth: int +) -> type[BaseModel]: + raw_properties: Any = node.get("properties", {}) + if not isinstance(raw_properties, dict): + _fail("'properties' must be an object", path) + properties = cast(dict[Any, Any], raw_properties) + raw_required: Any = node.get("required", []) + if not isinstance(raw_required, list) or not all( + isinstance(entry, str) for entry in cast(list[Any], raw_required) + ): + _fail("'required' must be an array of strings", path) + # Entries naming properties that don't exist are tolerated; they just + # have no effect. + required = set(cast(list[str], raw_required)) + + fields: dict[str, tuple[Any, Any]] = {} + for prop_key, prop_schema in properties.items(): + if not isinstance(prop_key, str) or not prop_key: + _fail("property names must be non-empty strings", path) + annotation = _convert_schema( + prop_schema, + _child_path(path, f"properties.{prop_key}"), + f"{name_hint} {prop_key}", + ctx, + depth + 1, + ) + # Non-identifier keys ("my-key") get a sanitized field name, with the + # original key preserved as the alias for validation/serialization. + field_name = _field_name(prop_key, fields) + alias = prop_key if field_name != prop_key else None + fields[field_name] = _make_field( + cast(dict[str, Any], prop_schema) if isinstance(prop_schema, dict) else {}, + annotation, + is_required=prop_key in required, + alias=alias, + ) + + # create_model's overloads can't type dynamic **fields; the values are + # (annotation, FieldInfo) tuples, which is the documented calling form. + model: Any = create_model( # pyright: ignore[reportCallIssue, reportUnknownVariableType] + _unique_model_name(name_hint, ctx), + __config__=ConfigDict(extra="ignore", populate_by_name=True), + **fields, # pyright: ignore[reportArgumentType] + ) + return cast(type[BaseModel], model) + + +def _make_field( + prop_schema: dict[str, Any], + annotation: Any, + *, + is_required: bool, + alias: str | None, +) -> tuple[Any, Any]: + kwargs: dict[str, Any] = {} + if alias is not None: + kwargs["alias"] = alias + description = prop_schema.get("description") + if isinstance(description, str): + kwargs["description"] = description + hints = {key: prop_schema[key] for key in _HINT_KEYS if key in prop_schema} + if hints: + kwargs["json_schema_extra"] = hints + + # Precedence: an explicit default wins (even for required fields), then + # required, then optional — which widens to `T | None` defaulting to None. + if "default" in prop_schema: + return annotation, Field(default=prop_schema["default"], **kwargs) + if is_required: + return annotation, Field(**kwargs) + return annotation | None, Field(default=None, **kwargs) + + +def _child_path(path: str, segment: str) -> str: + return f"{path}.{segment}" if path else segment + + +def _field_name(prop_key: str, existing: dict[str, Any]) -> str: + """Return a valid, unique Python field name for a JSON property key. + + Keys that aren't valid identifiers (or would be Pydantic-private via a + leading underscore) are sanitized; the original key is preserved as the + field alias by the caller. + """ + if _IDENTIFIER_RE.match(prop_key) and prop_key not in existing: + return prop_key + sanitized = re.sub(r"[^A-Za-z0-9_]", "_", prop_key).lstrip("_") + if not sanitized or sanitized[0].isdigit(): + sanitized = f"field_{sanitized}" + candidate = sanitized + suffix = 2 + while candidate in existing: + candidate = f"{sanitized}_{suffix}" + suffix += 1 + return candidate + + +def _unique_model_name(name_hint: str, ctx: _Ctx) -> str: + # PascalCase the hint (e.g. "ResponseFormat address geo" -> "ResponseFormatAddressGeo"). + parts = re.split(r"[^A-Za-z0-9]+", name_hint) + name = "".join(part[:1].upper() + part[1:] for part in parts if part) + # Class names can't be empty or start with a digit. + if not name or name[0].isdigit(): + name = f"Model{name}" + # Distinct hints can sanitize to the same name; suffix _2, _3, ... to disambiguate. + candidate = name + suffix = 2 + while candidate in ctx.used_names: + candidate = f"{name}_{suffix}" + suffix += 1 + ctx.used_names.add(candidate) + return candidate diff --git a/tests/conftest.py b/tests/conftest.py index af35c999..b3697242 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -672,8 +672,15 @@ def mock_llm_call_functions(request: pytest.FixtureRequest): mock_short_summary.return_value = "Test short summary content" mock_long_summary.return_value = "Test long summary content" - # Mock agentic_chat to return a string (matching actual return type) - mock_agentic_chat.return_value = "Test dialectic response" + # Mock agentic_chat to return a string (matching actual return type). + # With a response_model (structured output) the real function returns + # a JSON string, so mirror that for SDK clients that parse content. + async def _agentic_chat_response(*_args: object, **kwargs: object) -> str: + if kwargs.get("response_model") is not None: + return "{}" + return "Test dialectic response" + + mock_agentic_chat.side_effect = _agentic_chat_response yield { "short_summary": mock_short_summary, diff --git a/tests/dialectic/test_structured_output.py b/tests/dialectic/test_structured_output.py new file mode 100644 index 00000000..10b32da8 --- /dev/null +++ b/tests/dialectic/test_structured_output.py @@ -0,0 +1,126 @@ +"""Tests for response_model threading through the DialecticAgent.""" + +import time +from unittest.mock import AsyncMock, patch + +import pytest +from pydantic import BaseModel + +from src.dialectic.core import DialecticAgent +from src.llm import ( + HonchoLLMCallResponse, + HonchoLLMCallStreamChunk, + StreamingResponseWithMetadata, +) + + +class FoodPreferences(BaseModel): + favorite: str + confidence: float + + +def _make_agent() -> DialecticAgent: + return DialecticAgent( + workspace_name="workspace", + session_name="session", + observer="observer", + observed="observed", + reasoning_level="low", + ) + + +def _patches(mock_llm_call: AsyncMock): + return ( + patch.object( + DialecticAgent, + "_prepare_query", + new=AsyncMock( + return_value=(AsyncMock(), "task", "run", time.perf_counter()) + ), + ), + patch.object(DialecticAgent, "_log_response_metrics"), + patch("src.dialectic.core.honcho_llm_call", new=mock_llm_call), + ) + + +@pytest.mark.asyncio +async def test_answer_passes_response_model_and_serializes() -> None: + """answer() threads response_model to the LLM call and serializes the + parsed model instance back to a JSON string.""" + agent = _make_agent() + parsed = FoodPreferences(favorite="sushi", confidence=0.9) + mock_llm_call = AsyncMock( + return_value=HonchoLLMCallResponse( + content=parsed, + input_tokens=10, + output_tokens=5, + finish_reasons=["stop"], + ) + ) + + p1, p2, p3 = _patches(mock_llm_call) + with p1, p2, p3: + result = await agent.answer("query", response_model=FoodPreferences) + + kwargs = mock_llm_call.await_args.kwargs # pyright: ignore + assert kwargs["response_model"] is FoodPreferences + assert isinstance(result, str) + assert FoodPreferences.model_validate_json(result) == parsed + + +@pytest.mark.asyncio +async def test_answer_without_response_model_returns_plain_text() -> None: + agent = _make_agent() + mock_llm_call = AsyncMock( + return_value=HonchoLLMCallResponse( + content="plain answer", + input_tokens=10, + output_tokens=5, + finish_reasons=["stop"], + ) + ) + + p1, p2, p3 = _patches(mock_llm_call) + with p1, p2, p3: + result = await agent.answer("query") + + assert result == "plain answer" + assert mock_llm_call.await_args.kwargs["response_model"] is None # pyright: ignore + + +@pytest.mark.asyncio +async def test_answer_stream_passes_response_model() -> None: + """answer_stream() threads response_model; chunks stay raw text.""" + agent = _make_agent() + + async def _stream(): + yield HonchoLLMCallStreamChunk(content='{"favorite":"sushi",') + yield HonchoLLMCallStreamChunk(content='"confidence":0.9}') + yield HonchoLLMCallStreamChunk(content="", is_done=True) + + mock_llm_call = AsyncMock( + return_value=StreamingResponseWithMetadata( + _stream(), + tool_calls_made=[], + input_tokens=10, + output_tokens=5, + cache_creation_input_tokens=0, + cache_read_input_tokens=0, + iterations=1, + ) + ) + + p1, p2, p3 = _patches(mock_llm_call) + with p1, p2, p3: + chunks = [ + chunk + async for chunk in agent.answer_stream( + "query", response_model=FoodPreferences + ) + ] + + kwargs = mock_llm_call.await_args.kwargs # pyright: ignore + assert kwargs["response_model"] is FoodPreferences + assert kwargs["stream_final_only"] is True + accumulated = "".join(chunks) + assert FoodPreferences.model_validate_json(accumulated).favorite == "sushi" diff --git a/tests/live_llm/test_live_structured_output_unions.py b/tests/live_llm/test_live_structured_output_unions.py new file mode 100644 index 00000000..bd69f8f1 --- /dev/null +++ b/tests/live_llm/test_live_structured_output_unions.py @@ -0,0 +1,162 @@ +"""Live coverage for union-bearing structured output without tools. + +The dialectic's final synthesis call carries response_format but tools=None +(see src/llm/tool_loop.py), and the model class is created dynamically from a +caller-supplied JSON Schema (src/utils/schema_conversion.py). That call shape +differs from test_live_tools_structured_output.py in one important way: with +no tools attached, the Gemini backend uses its NATIVE response_schema config +instead of injecting a schema instruction — and Gemini's response_schema +historically rejected anyOf. These tests drive a schema that exercises every +union-ish construct the converter supports (anyOf with null, a type list, an +enum, a $defs reference) through that exact call shape on all three +providers. +""" + +from __future__ import annotations + +from typing import Any + +import pytest +from pydantic import BaseModel, ValidationError + +from src.exceptions import LLMError +from src.llm.backend import CompletionResult +from src.llm.request_builder import execute_completion +from src.llm.structured_output import StructuredOutputError +from src.utils.schema_conversion import json_response_schema_to_pydantic + +from .conftest import make_backend, require_provider_key, wrap_async_method +from .model_matrix import LiveModelSpec, get_live_model_specs + +pytestmark = [pytest.mark.live_llm] + +_USER_FACTS_SCHEMA: dict[str, Any] = { + "type": "object", + "properties": { + "favorite_food": {"$ref": "#/$defs/Food"}, + "sentiment": {"enum": ["loves", "likes", "neutral", "dislikes", "hates"]}, + "years_vegetarian": {"type": ["integer", "null"]}, + "salient_fact": { + "anyOf": [{"type": "string"}, {"type": "null"}], + "description": "One short salient fact about the user, or null", + }, + }, + "required": ["favorite_food", "sentiment", "years_vegetarian"], + "$defs": { + "Food": { + "type": "object", + "properties": {"name": {"type": "string"}}, + "required": ["name"], + } + }, +} + +_PROMPT = ( + "The user said: 'I love sushi. I've been vegetarian for 3 years.' " + "Report the user's favorite food, their sentiment toward it, how many " + "years they have been vegetarian, and optionally one salient fact." +) + + +async def run_union_structured_flow(backend: Any, config: Any) -> CompletionResult: + """One no-tools turn that must return a schema-conforming answer. + + A fresh model class is created per call, matching how the dialectic + converts the caller's schema on every request. Retries mirror + test_live_tools_structured_output.py: empty/unparseable candidates are + absorbed by the executor's retry layer in production, but these tests + call the backend directly. + """ + response_model = json_response_schema_to_pydantic( + _USER_FACTS_SCHEMA, model_name="UserFactsReport" + ) + + result: CompletionResult | None = None + last_error: Exception | None = None + for _ in range(3): + try: + result = await execute_completion( + backend, + config, + messages=[{"role": "user", "content": _PROMPT}], + max_tokens=4096, + response_format=response_model, + ) + except (ValidationError, LLMError, StructuredOutputError) as exc: + last_error = exc + continue + break + if result is None: + raise AssertionError( + "union structured turn failed on all attempts" + ) from last_error + + content = result.content + assert isinstance(content, BaseModel), f"expected parsed model, got {content!r}" + # The model class is dynamic, so field access is untyped by construction. + report: Any = content + assert "sushi" in report.favorite_food.name.lower() + assert report.sentiment == "loves" + assert report.years_vegetarian == 3 + assert report.salient_fact is None or isinstance(report.salient_fact, str) + return result + + +@pytest.mark.asyncio +@pytest.mark.requires_anthropic +@pytest.mark.parametrize( + "model_spec", + get_live_model_specs(provider="anthropic", feature="structured_output"), + ids=lambda spec: spec.id, +) +async def test_live_anthropic_union_structured_output( + model_spec: LiveModelSpec, +) -> None: + require_provider_key(model_spec) + backend, config = make_backend(model_spec) + await run_union_structured_flow(backend, config) + + +@pytest.mark.asyncio +@pytest.mark.requires_openai +@pytest.mark.parametrize( + "model_spec", + get_live_model_specs(provider="openai", feature="structured_output"), + ids=lambda spec: spec.id, +) +async def test_live_openai_union_structured_output( + model_spec: LiveModelSpec, +) -> None: + require_provider_key(model_spec) + backend, config = make_backend(model_spec) + await run_union_structured_flow(backend, config) + + +@pytest.mark.asyncio +@pytest.mark.requires_gemini +@pytest.mark.parametrize( + "model_spec", + get_live_model_specs(provider="gemini", feature="structured_output"), + ids=lambda spec: spec.id, +) +async def test_live_gemini_union_structured_output( + model_spec: LiveModelSpec, + monkeypatch: pytest.MonkeyPatch, +) -> None: + require_provider_key(model_spec) + backend, config = make_backend(model_spec) + generate_calls = wrap_async_method( + monkeypatch, + backend._client.aio.models, + "generate_content", + ) + + await run_union_structured_flow(backend, config) + + # The point of this test: with no tools, Gemini must take the NATIVE + # response_schema path (the one that historically rejected anyOf), not + # the prompt-injection workaround used when tools are attached. + assert generate_calls + gen_config = generate_calls[-1]["kwargs"]["config"] + assert "response_schema" in gen_config + assert gen_config["response_mime_type"] == "application/json" diff --git a/tests/llm/test_backends/test_openai.py b/tests/llm/test_backends/test_openai.py index 89572f25..8567e034 100644 --- a/tests/llm/test_backends/test_openai.py +++ b/tests/llm/test_backends/test_openai.py @@ -1,4 +1,6 @@ +import gc import json +import weakref from collections.abc import AsyncIterator from types import SimpleNamespace from typing import Any @@ -10,8 +12,12 @@ from openai import BadRequestError from pydantic import BaseModel from src.exceptions import ValidationException -from src.llm.backends.openai import OpenAIBackend +from src.llm.backends.openai import ( + OpenAIBackend, + _json_object_instruction, # pyright: ignore[reportPrivateUsage] +) from src.utils.representation import PromptRepresentation +from src.utils.schema_conversion import json_response_schema_to_pydantic def _await_kwargs(mock_method: Any) -> dict[str, Any]: @@ -838,6 +844,70 @@ async def test_structured_output_json_object_mode_repairs_markdown() -> None: assert isinstance(result.content, PromptRepresentation) +@pytest.mark.asyncio +async def test_structured_output_json_object_mode_with_dynamic_model() -> None: + """json_object mode composes with a caller-supplied schema converted at + request time (the dialectic's response_format path): the generated class's + schema — unions included — is injected into the prompt and the JSON body + parses back through the class.""" + response_model = json_response_schema_to_pydantic( + { + "type": "object", + "properties": { + "answer": {"type": "string"}, + "years": {"type": ["integer", "null"]}, + }, + "required": ["answer", "years"], + } + ) + client = Mock() + client.chat.completions.parse = AsyncMock() + client.chat.completions.create = AsyncMock( + return_value=_structured_create_return('{"answer": "ok", "years": 3}') + ) + + backend = OpenAIBackend(client) + result = await backend.complete( + model="glm-4.6", + messages=[{"role": "user", "content": "Hello"}], + max_tokens=100, + response_format=response_model, + extra_params={"structured_output_mode": "json_object"}, + ) + + assert client.chat.completions.parse.await_count == 0 + call = _await_kwargs(client.chat.completions.create) + assert call["response_format"] == {"type": "json_object"} + system_messages = [m for m in call["messages"] if m["role"] == "system"] + assert system_messages, "expected a system message carrying the schema" + # The dynamic model's schema (union field included) made it into the prompt. + assert "years" in system_messages[0]["content"] + assert "anyOf" in system_messages[0]["content"] + + assert isinstance(result.content, response_model) + # The model class is dynamic, so field access is untyped by construction. + content: Any = result.content + assert content.answer == "ok" + assert content.years == 3 + + +def test_json_object_instruction_does_not_pin_dynamic_models() -> None: + """The instruction cache must hold its model-class keys weakly: the + dialectic creates a fresh response_format class per request (see + src/utils/schema_conversion.py), and a strong-keyed cache would grow by + one pinned class per structured chat call.""" + model = json_response_schema_to_pydantic( + {"type": "object", "properties": {"answer": {"type": "string"}}} + ) + first = _json_object_instruction(model) + assert _json_object_instruction(model) is first # cached while alive + + ref = weakref.ref(model) + del model + gc.collect() + assert ref() is None, "instruction cache must not keep dynamic classes alive" + + @pytest.mark.asyncio async def test_structured_output_json_object_empty_content_returns_empty() -> None: """An empty body with no refusal must produce a graceful empty result, not diff --git a/tests/routes/test_peers.py b/tests/routes/test_peers.py index 847c24e0..f4b54aa3 100644 --- a/tests/routes/test_peers.py +++ b/tests/routes/test_peers.py @@ -4,6 +4,7 @@ from typing import Any import pytest from fastapi.testclient import TestClient from nanoid import generate as generate_nanoid +from pydantic import BaseModel from sqlalchemy.ext.asyncio import AsyncSession from src import crud, models @@ -1270,3 +1271,114 @@ def test_set_peer_card(client: TestClient, sample_data: tuple[Workspace, Peer]): ) assert response.status_code == 200 assert response.json()["peer_card"] == target_card + + +FOOD_PREFS_SCHEMA = { + "type": "object", + "properties": { + "preferences": { + "type": "array", + "items": { + "type": "object", + "properties": { + "food": {"type": "string"}, + "sentiment": {"enum": ["loves", "likes", "dislikes"]}, + }, + "required": ["food", "sentiment"], + }, + }, + "summary": {"type": "string"}, + }, + "required": ["preferences", "summary"], +} + + +def test_chat_with_response_format( + client: TestClient, + sample_data: tuple[Workspace, Peer], + mock_llm_call_functions: dict[str, Any], +): + """A valid response_format converts to a Pydantic model and is passed to + the dialectic as response_model.""" + test_workspace, test_peer = sample_data + + response = client.post( + f"/v3/workspaces/{test_workspace.name}/peers/{test_peer.name}/chat", + json={ + "query": "What are this user's food preferences?", + "stream": False, + "response_format": FOOD_PREFS_SCHEMA, + }, + ) + assert response.status_code == 200 + assert "content" in response.json() + + kwargs = mock_llm_call_functions["agentic_chat"].await_args.kwargs + response_model = kwargs["response_model"] + assert isinstance(response_model, type) + assert issubclass(response_model, BaseModel) + # The converted model enforces the caller's schema. + instance = response_model.model_validate( + {"preferences": [{"food": "sushi", "sentiment": "loves"}], "summary": "s"} + ) + assert instance.summary == "s" # pyright: ignore + + +def test_chat_with_response_format_streaming( + client: TestClient, + sample_data: tuple[Workspace, Peer], + mock_llm_call_functions: dict[str, Any], +): + test_workspace, test_peer = sample_data + + response = client.post( + f"/v3/workspaces/{test_workspace.name}/peers/{test_peer.name}/chat", + json={ + "query": "What are this user's food preferences?", + "stream": True, + "response_format": FOOD_PREFS_SCHEMA, + }, + ) + assert response.status_code == 200 + assert "data:" in response.text + + kwargs = mock_llm_call_functions["agentic_chat_stream"].call_args.kwargs + response_model = kwargs["response_model"] + assert isinstance(response_model, type) + assert issubclass(response_model, BaseModel) + + +@pytest.mark.parametrize( + "bad_schema", + [ + {"type": "string"}, # non-object root + {"type": "object", "properties": {"a": {"$ref": "#/x"}}}, + {"type": "object", "properties": {"a": {"allOf": [{"type": "string"}]}}}, + { + "type": "object", + "properties": { + "m": {"type": "object", "additionalProperties": {"type": "string"}} + }, + }, + ], +) +def test_chat_with_invalid_response_format( + client: TestClient, + sample_data: tuple[Workspace, Peer], + mock_llm_call_functions: dict[str, Any], + bad_schema: dict[str, Any], +): + """Unsupported schemas are rejected with 422 before the dialectic runs.""" + test_workspace, test_peer = sample_data + + response = client.post( + f"/v3/workspaces/{test_workspace.name}/peers/{test_peer.name}/chat", + json={ + "query": "Hello?", + "stream": False, + "response_format": bad_schema, + }, + ) + assert response.status_code == 422 + assert "Invalid response_format" in response.json()["detail"] + mock_llm_call_functions["agentic_chat"].assert_not_awaited() diff --git a/tests/sdk/test_peer.py b/tests/sdk/test_peer.py index 258fb989..0c019d9b 100644 --- a/tests/sdk/test_peer.py +++ b/tests/sdk/test_peer.py @@ -2,6 +2,7 @@ from collections.abc import AsyncIterator, Iterator from unittest.mock import patch import pytest +from pydantic import BaseModel from sdks.python.src.honcho.client import Honcho from sdks.python.src.honcho.peer import Peer @@ -626,3 +627,166 @@ async def test_peer_representation_with_all_params( max_conclusions=5, ) assert isinstance(result, str) + + +class ChatFoodPreferences(BaseModel): + favorite: str + confidence: float + + +CHAT_SCHEMA_DICT = { + "type": "object", + "properties": {"items": {"type": "array", "items": {"type": "string"}}}, + "required": ["items"], +} + + +@pytest.mark.asyncio +async def test_peer_chat_response_format_pydantic(client_fixture: tuple[Honcho, str]): + """A Pydantic model class is sent as JSON Schema and the response content + is parsed back into a model instance.""" + honcho_client, client_type = client_fixture + content = '{"favorite": "sushi", "confidence": 0.9}' + + if client_type == "async": + peer = await honcho_client.aio.peer(id="test-rf-async-peer") + + async def mock_post(*_args: object, **_kwargs: object) -> dict[str, object]: + return {"content": content} + + with patch.object( + peer._honcho._async_http_client, # pyright: ignore[reportPrivateUsage] + "post", + side_effect=mock_post, + ) as mock: + result = await peer.aio.chat( + "What do I like?", response_format=ChatFoodPreferences + ) + else: + peer = honcho_client.peer(id="test-rf-peer") + with patch.object( + peer._honcho._http, # pyright: ignore[reportPrivateUsage] + "post", + return_value={"content": content}, + ) as mock: + result = peer.chat("What do I like?", response_format=ChatFoodPreferences) + + body = mock.call_args.kwargs["body"] + assert body["response_format"] == ChatFoodPreferences.model_json_schema() + assert isinstance(result, ChatFoodPreferences) + assert result.favorite == "sushi" + + +@pytest.mark.asyncio +async def test_peer_chat_response_format_dict(client_fixture: tuple[Honcho, str]): + """A raw JSON Schema dict is sent as-is and the response stays a string.""" + honcho_client, client_type = client_fixture + content = '{"items": ["sushi"]}' + + if client_type == "async": + peer = await honcho_client.aio.peer(id="test-rf-dict-async-peer") + + async def mock_post(*_args: object, **_kwargs: object) -> dict[str, object]: + return {"content": content} + + with patch.object( + peer._honcho._async_http_client, # pyright: ignore[reportPrivateUsage] + "post", + side_effect=mock_post, + ) as mock: + result = await peer.aio.chat( + "What do I like?", response_format=CHAT_SCHEMA_DICT + ) + else: + peer = honcho_client.peer(id="test-rf-dict-peer") + with patch.object( + peer._honcho._http, # pyright: ignore[reportPrivateUsage] + "post", + return_value={"content": content}, + ) as mock: + result = peer.chat("What do I like?", response_format=CHAT_SCHEMA_DICT) + + body = mock.call_args.kwargs["body"] + assert body["response_format"] == CHAT_SCHEMA_DICT + assert result == content + + +@pytest.mark.asyncio +async def test_peer_chat_response_format_empty_content( + client_fixture: tuple[Honcho, str], +): + """Empty/None content returns None even when a Pydantic class was given.""" + honcho_client, client_type = client_fixture + + if client_type == "async": + peer = await honcho_client.aio.peer(id="test-rf-empty-async-peer") + + async def mock_post(*_args: object, **_kwargs: object) -> dict[str, object]: + return {"content": None} + + with patch.object( + peer._honcho._async_http_client, # pyright: ignore[reportPrivateUsage] + "post", + side_effect=mock_post, + ): + result = await peer.aio.chat( + "What do I like?", response_format=ChatFoodPreferences + ) + else: + peer = honcho_client.peer(id="test-rf-empty-peer") + with patch.object( + peer._honcho._http, # pyright: ignore[reportPrivateUsage] + "post", + return_value={"content": None}, + ): + result = peer.chat("What do I like?", response_format=ChatFoodPreferences) + + assert result is None + + +@pytest.mark.asyncio +async def test_peer_chat_stream_response_format(client_fixture: tuple[Honcho, str]): + """chat_stream sends the schema in the body; chunks stay raw text.""" + honcho_client, client_type = client_fixture + + if client_type == "async": + peer = await honcho_client.aio.peer(id="test-rf-stream-async-peer") + + async def mock_astream( + *_args: object, **_kwargs: object + ) -> AsyncIterator[bytes]: + yield b'data: {"delta": {"content": "{\\"favorite\\":"}}\n' + yield b'data: {"delta": {"content": "\\"sushi\\",\\"confidence\\":0.9}"}}\n' + yield b'data: {"done": true}\n' + + with patch.object( + peer._honcho._async_http_client, # pyright: ignore[reportPrivateUsage] + "stream", + side_effect=mock_astream, + ) as mock: + result = await peer.aio.chat_stream( + "What do I like?", response_format=ChatFoodPreferences + ) + chunks = [chunk async for chunk in result] + else: + peer = honcho_client.peer(id="test-rf-stream-peer") + + def mock_stream(*_args: object, **_kwargs: object) -> Iterator[bytes]: + yield b'data: {"delta": {"content": "{\\"favorite\\":"}}\n' + yield b'data: {"delta": {"content": "\\"sushi\\",\\"confidence\\":0.9}"}}\n' + yield b'data: {"done": true}\n' + + with patch.object( + peer._honcho._http, # pyright: ignore[reportPrivateUsage] + "stream", + side_effect=mock_stream, + ) as mock: + result = peer.chat_stream( + "What do I like?", response_format=ChatFoodPreferences + ) + chunks = list(result) + + body = mock.call_args.kwargs["body"] + assert body["response_format"] == ChatFoodPreferences.model_json_schema() + accumulated = "".join(chunks) + assert ChatFoodPreferences.model_validate_json(accumulated).favorite == "sushi" diff --git a/tests/unified/runner.py b/tests/unified/runner.py index eaa78409..b99cd37a 100644 --- a/tests/unified/runner.py +++ b/tests/unified/runner.py @@ -357,6 +357,7 @@ class UnifiedTestExecutor: session=step.session_id, target=step.observed_peer_id, reasoning_level=step.reasoning_level, + response_format=step.response_format, ) return response diff --git a/tests/unified/schema.py b/tests/unified/schema.py index aa4c78ad..161d4f08 100644 --- a/tests/unified/schema.py +++ b/tests/unified/schema.py @@ -149,6 +149,9 @@ class QueryAction(TestStep): # for chat - reasoning level reasoning_level: ReasoningLevel | None = None + # for chat - optional JSON Schema the response must conform to + response_format: dict[str, Any] | None = None + assertions: list[ LLMJudgeAssertion | ContainsAssertion diff --git a/tests/unified/test_cases/dialectic_structured_output.json b/tests/unified/test_cases/dialectic_structured_output.json new file mode 100644 index 00000000..59cc1365 --- /dev/null +++ b/tests/unified/test_cases/dialectic_structured_output.json @@ -0,0 +1,135 @@ +{ + "description": "Dialectic chat with a response_format JSON Schema while the agent must use tools (reasoning off + enumeration question forces grep/search calls). Exercises the transport-layer combination of tool calling and structured output on every provider: OpenAI must avoid parse() for non-strict tools, Anthropic must skip the '{' prefill, Gemini must fall back to a schema instruction. The final answer must be a JSON string conforming to the schema.", + "workspace_config": {}, + "steps": [ + { + "step_type": "create_session", + "session_id": "structured_output_test", + "config": { + "reasoning": { + "enabled": false + } + }, + "peer_configs": { + "user": { + "observe_me": true, + "observe_others": false + }, + "assistant": { + "observe_me": false, + "observe_others": true + } + } + }, + { + "step_type": "add_messages", + "session_id": "structured_output_test", + "messages": [ + { + "peer_id": "user", + "content": "Monday I grabbed a $5 latte at Starbucks before my standup.", + "created_at": "2024-03-04T08:30:00" + }, + { + "peer_id": "assistant", + "content": "Nice, a classic way to start the week.", + "created_at": "2024-03-04T08:31:00" + }, + { + "peer_id": "user", + "content": "Tuesday I tried a $4 cold brew from Blue Bottle, really smooth.", + "created_at": "2024-03-05T09:15:00" + }, + { + "peer_id": "assistant", + "content": "Blue Bottle makes a solid cold brew.", + "created_at": "2024-03-05T09:16:00" + }, + { + "peer_id": "user", + "content": "Wednesday was a $6 oat-milk mocha at a little place downtown.", + "created_at": "2024-03-06T08:45:00" + }, + { + "peer_id": "assistant", + "content": "Oat milk mochas are underrated.", + "created_at": "2024-03-06T08:46:00" + }, + { + "peer_id": "user", + "content": "Thursday I skipped coffee and just had tea at home.", + "created_at": "2024-03-07T08:20:00" + }, + { + "peer_id": "assistant", + "content": "A calm morning, sounds good.", + "created_at": "2024-03-07T08:21:00" + }, + { + "peer_id": "user", + "content": "Friday I splurged on a $7 pour-over at the roastery near the office.", + "created_at": "2024-03-08T08:50:00" + }, + { + "peer_id": "assistant", + "content": "Ending the week strong!", + "created_at": "2024-03-08T08:51:00" + } + ] + }, + { + "step_type": "wait", + "target": "queue_empty", + "timeout": 180, + "flush": true + }, + { + "step_type": "query", + "description": "Global query + empty prefetch forces tool calls; response_format forces structured output on the same LLM calls", + "target": "chat", + "observer_peer_id": "assistant", + "observed_peer_id": "user", + "reasoning_level": "max", + "input": "How many separate coffees did I buy this week, and exactly how much did I spend in total across all of them?", + "response_format": { + "type": "object", + "properties": { + "coffee_count": { + "type": "integer", + "description": "How many separate coffees the user bought during the week" + }, + "total_spent_usd": { + "type": "number", + "description": "Total amount in US dollars the user spent on coffee" + }, + "purchases": { + "type": "array", + "items": { + "type": "string" + }, + "description": "One short entry per coffee purchase, including its price" + }, + "summary": { + "type": "string", + "description": "One-sentence answer to the question" + } + }, + "required": ["coffee_count", "total_spent_usd", "purchases", "summary"] + }, + "assertions": [ + { + "assertion_type": "json_match", + "key_value_pairs": { + "coffee_count": 4, + "total_spent_usd": 22 + } + }, + { + "assertion_type": "llm_judge", + "prompt": "The result must be a JSON object whose 'purchases' array enumerates the $5 latte, $4 cold brew, $6 mocha, and $7 pour-over (wording may vary), and whose 'summary' answers that the user bought 4 coffees for $22 total.", + "pass_if": true + } + ] + } + ] +} diff --git a/tests/utils/test_schema_conversion.py b/tests/utils/test_schema_conversion.py new file mode 100644 index 00000000..e6c607f4 --- /dev/null +++ b/tests/utils/test_schema_conversion.py @@ -0,0 +1,897 @@ +"""Unit tests for src/utils/schema_conversion.py.""" + +import json +import re +from typing import Any + +import pytest +from pydantic import BaseModel, ValidationError + +from src.utils.schema_conversion import json_response_schema_to_pydantic + + +def _object(properties: dict[str, Any], **extra: Any) -> dict[str, Any]: + return {"type": "object", "properties": properties, **extra} + + +class TestPrimitives: + def test_flat_object_with_primitives(self): + model = json_response_schema_to_pydantic( + _object( + { + "name": {"type": "string"}, + "age": {"type": "integer"}, + "score": {"type": "number"}, + "active": {"type": "boolean"}, + }, + required=["name", "age"], + ) + ) + instance = model.model_validate( + {"name": "ada", "age": 36, "score": 9.5, "active": True} + ) + assert instance.name == "ada" # pyright: ignore + assert instance.age == 36 # pyright: ignore + + def test_required_field_missing_fails(self): + model = json_response_schema_to_pydantic( + _object({"name": {"type": "string"}}, required=["name"]) + ) + with pytest.raises(ValidationError): + model.model_validate({}) + + def test_optional_field_defaults_to_none(self): + model = json_response_schema_to_pydantic( + _object({"nickname": {"type": "string"}}) + ) + instance = model.model_validate({}) + assert instance.nickname is None # pyright: ignore + + def test_default_value(self): + model = json_response_schema_to_pydantic( + _object({"count": {"type": "integer", "default": 3}}) + ) + assert model.model_validate({}).count == 3 # pyright: ignore + + def test_null_type(self): + model = json_response_schema_to_pydantic( + _object({"nothing": {"type": "null"}}, required=["nothing"]) + ) + assert model.model_validate({"nothing": None}).nothing is None # pyright: ignore + + def test_default_wins_over_required(self): + model = json_response_schema_to_pydantic( + _object({"count": {"type": "integer", "default": 3}}, required=["count"]) + ) + assert model.model_validate({}).count == 3 # pyright: ignore + + +class TestNesting: + def test_nested_object(self): + model = json_response_schema_to_pydantic( + _object( + { + "address": _object( + { + "city": {"type": "string"}, + "geo": _object( + {"lat": {"type": "number"}}, required=["lat"] + ), + }, + required=["city", "geo"], + ) + }, + required=["address"], + ) + ) + instance = model.model_validate( + {"address": {"city": "oakland", "geo": {"lat": 37.8}}} + ) + assert instance.address.geo.lat == 37.8 # pyright: ignore + + def test_array_of_objects(self): + model = json_response_schema_to_pydantic( + _object( + { + "items": { + "type": "array", + "items": _object( + {"food": {"type": "string"}}, required=["food"] + ), + } + }, + required=["items"], + ) + ) + instance = model.model_validate({"items": [{"food": "sushi"}]}) + assert instance.items[0].food == "sushi" # pyright: ignore + + def test_array_without_items_accepts_anything(self): + model = json_response_schema_to_pydantic( + _object({"stuff": {"type": "array"}}, required=["stuff"]) + ) + instance = model.model_validate({"stuff": [1, "two", {"three": 3}]}) + assert len(instance.stuff) == 3 # pyright: ignore + + def test_nested_model_name_collision(self): + # Two sibling objects whose name hints collide must not clash. + model = json_response_schema_to_pydantic( + _object( + { + "a": _object({"x b": _object({"v": {"type": "string"}})}), + "a_x": _object({"b": _object({"v": {"type": "integer"}})}), + } + ) + ) + instance = model.model_validate( + {"a": {"x b": {"v": "s"}}, "a_x": {"b": {"v": 1}}} + ) + assert instance.a_x.b.v == 1 # pyright: ignore + + +class TestEnumsAndUnions: + def test_string_enum(self): + model = json_response_schema_to_pydantic( + _object( + {"sentiment": {"enum": ["loves", "hates"]}}, + required=["sentiment"], + ) + ) + assert model.model_validate({"sentiment": "loves"}).sentiment == "loves" # pyright: ignore + with pytest.raises(ValidationError): + model.model_validate({"sentiment": "meh"}) + + def test_int_enum_and_null_member(self): + model = json_response_schema_to_pydantic( + _object({"level": {"enum": [1, 2, None]}}, required=["level"]) + ) + assert model.model_validate({"level": None}).level is None # pyright: ignore + assert model.model_validate({"level": 2}).level == 2 # pyright: ignore + + def test_invalid_enum_value_type(self): + with pytest.raises(ValueError, match="enum values"): + json_response_schema_to_pydantic(_object({"bad": {"enum": [[1]]}})) + + def test_anyof_with_null_is_optional(self): + model = json_response_schema_to_pydantic( + _object( + {"maybe": {"anyOf": [{"type": "string"}, {"type": "null"}]}}, + required=["maybe"], + ) + ) + assert model.model_validate({"maybe": None}).maybe is None # pyright: ignore + assert model.model_validate({"maybe": "x"}).maybe == "x" # pyright: ignore + + def test_oneof_union(self): + model = json_response_schema_to_pydantic( + _object( + {"value": {"oneOf": [{"type": "integer"}, {"type": "string"}]}}, + required=["value"], + ) + ) + assert model.model_validate({"value": 5}).value == 5 # pyright: ignore + + def test_type_list_form(self): + model = json_response_schema_to_pydantic( + _object({"name": {"type": ["string", "null"]}}, required=["name"]) + ) + assert model.model_validate({"name": None}).name is None # pyright: ignore + + def test_all_null_enum_degenerates_to_none(self): + model = json_response_schema_to_pydantic( + _object({"nothing": {"enum": [None]}}, required=["nothing"]) + ) + assert model.model_validate({"nothing": None}).nothing is None # pyright: ignore + with pytest.raises(ValidationError): + model.model_validate({"nothing": "x"}) + + def test_union_of_objects(self): + model = json_response_schema_to_pydantic( + _object( + { + "pet": { + "anyOf": [ + _object({"meows": {"type": "boolean"}}, required=["meows"]), + _object({"barks": {"type": "boolean"}}, required=["barks"]), + ] + } + }, + required=["pet"], + ) + ) + instance = model.model_validate({"pet": {"barks": True}}) + assert instance.pet.barks is True # pyright: ignore + + +class TestRejections: + @pytest.mark.parametrize( + "construct,schema", + [ + ("$defs", _object({"a": {"type": "object", "$defs": {}}})), + ("definitions", _object({"a": {"type": "object", "definitions": {}}})), + ("allOf", _object({"a": {"allOf": [{"type": "string"}]}})), + ("not", _object({"a": {"not": {"type": "string"}}})), + ("if", _object({"a": {"if": {"type": "string"}}})), + ( + "patternProperties", + _object({"a": {"type": "object", "patternProperties": {}}}), + ), + ], + ) + def test_unsupported_constructs(self, construct: str, schema: dict[str, Any]): + with pytest.raises(ValueError, match=re.escape(construct)): + json_response_schema_to_pydantic(schema) + + def test_error_message_includes_path(self): + with pytest.raises( + ValueError, match=r"unsupported \$ref '#/x'.*at properties\.address" + ): + json_response_schema_to_pydantic(_object({"address": {"$ref": "#/x"}})) + + def test_schema_valued_additional_properties(self): + with pytest.raises(ValueError, match="additionalProperties"): + json_response_schema_to_pydantic( + _object( + { + "map": { + "type": "object", + "additionalProperties": {"type": "string"}, + } + } + ) + ) + + def test_boolean_schema(self): + with pytest.raises(ValueError, match="boolean schemas"): + json_response_schema_to_pydantic(_object({"anything": True})) + + def test_unknown_type(self): + with pytest.raises(ValueError, match="unsupported type 'date'"): + json_response_schema_to_pydantic(_object({"when": {"type": "date"}})) + + +class TestRefs: + def test_ref_into_defs(self): + model = json_response_schema_to_pydantic( + _object( + {"address": {"$ref": "#/$defs/Address"}}, + required=["address"], + **{ + "$defs": { + "Address": _object( + {"city": {"type": "string"}}, required=["city"] + ) + } + }, + ) + ) + instance = model.model_validate({"address": {"city": "Berlin"}}) + assert instance.address.city == "Berlin" # pyright: ignore + + def test_ref_into_definitions_alias(self): + model = json_response_schema_to_pydantic( + _object( + {"item": {"$ref": "#/definitions/Item"}}, + required=["item"], + definitions={"Item": {"type": "string"}}, + ) + ) + assert model.model_validate({"item": "x"}).item == "x" # pyright: ignore + + def test_pydantic_nested_model_schema(self): + """The real-world motivation: model_json_schema() of a nested Pydantic + model emits $defs/$ref and must convert cleanly.""" + + class Preference(BaseModel): + food: str + confidence: float + + class Preferences(BaseModel): + preferences: list[Preference] + summary: str + + model = json_response_schema_to_pydantic(Preferences.model_json_schema()) + instance = model.model_validate( + { + "preferences": [{"food": "sushi", "confidence": 0.9}], + "summary": "likes sushi", + } + ) + assert instance.preferences[0].food == "sushi" # pyright: ignore + + def test_root_ref(self): + model = json_response_schema_to_pydantic( + { + "$ref": "#/$defs/Root", + "$defs": { + "Root": _object({"ok": {"type": "boolean"}}, required=["ok"]) + }, + } + ) + assert model.model_validate({"ok": True}).ok is True # pyright: ignore + + def test_ref_sibling_keys_overlay_target(self): + model = json_response_schema_to_pydantic( + _object( + {"count": {"$ref": "#/$defs/Count", "default": 3}}, + **{"$defs": {"Count": {"type": "integer"}}}, + ) + ) + assert model.model_validate({}).count == 3 # pyright: ignore + + def test_same_def_referenced_twice(self): + model = json_response_schema_to_pydantic( + _object( + { + "home": {"$ref": "#/$defs/Address"}, + "work": {"$ref": "#/$defs/Address"}, + }, + required=["home", "work"], + **{"$defs": {"Address": _object({"city": {"type": "string"}})}}, + ) + ) + instance = model.model_validate( + {"home": {"city": "Berlin"}, "work": {"city": "Kyiv"}} + ) + assert instance.work.city == "Kyiv" # pyright: ignore + + def test_chained_refs(self): + model = json_response_schema_to_pydantic( + _object( + {"a": {"$ref": "#/$defs/A"}}, + required=["a"], + **{"$defs": {"A": {"$ref": "#/$defs/B"}, "B": {"type": "string"}}}, + ) + ) + assert model.model_validate({"a": "x"}).a == "x" # pyright: ignore + + def test_unreferenced_invalid_def_is_ignored(self): + model = json_response_schema_to_pydantic( + _object( + {"name": {"type": "string"}}, + **{"$defs": {"Broken": {"allOf": [{"type": "string"}]}}}, + ) + ) + assert model.model_validate({"name": "x"}).name == "x" # pyright: ignore + + @pytest.mark.parametrize( + "ref", + ["#", "#/x", "#/$defs/a/b", "#/properties/a", "https://x.dev/s.json#/$defs/X"], + ) + def test_unsupported_ref_forms(self, ref: str): + with pytest.raises(ValueError, match=r"unsupported \$ref"): + json_response_schema_to_pydantic( + _object( + {"a": {"$ref": ref}}, + **{"$defs": {"a": {"type": "string"}}}, + ) + ) + + def test_unknown_definition(self): + with pytest.raises(ValueError, match="unknown definition"): + json_response_schema_to_pydantic( + _object({"a": {"$ref": "#/$defs/Missing"}}, **{"$defs": {}}) + ) + + def test_direct_recursion_rejected(self): + with pytest.raises(ValueError, match=r"recursive \$ref.*cycle: Node -> Node"): + json_response_schema_to_pydantic( + _object( + {"tree": {"$ref": "#/$defs/Node"}}, + **{ + "$defs": { + "Node": _object( + { + "children": { + "type": "array", + "items": {"$ref": "#/$defs/Node"}, + } + } + ) + } + }, + ) + ) + + def test_mutual_recursion_rejected(self): + with pytest.raises(ValueError, match=r"cycle: A -> B -> A"): + json_response_schema_to_pydantic( + _object( + {"a": {"$ref": "#/$defs/A"}}, + **{ + "$defs": { + "A": _object({"b": {"$ref": "#/$defs/B"}}), + "B": _object({"a": {"$ref": "#/$defs/A"}}), + } + }, + ) + ) + + def test_recursive_pydantic_model_rejected(self): + class Node(BaseModel): + value: str + children: list["Node"] = [] + + with pytest.raises(ValueError, match=r"recursive \$ref"): + json_response_schema_to_pydantic(Node.model_json_schema()) + + def test_ref_expansion_counts_against_node_budget(self): + """A doubling ref chain (billion laughs) is stopped by max_nodes.""" + defs = { + f"L{i}": _object( + { + "a": {"$ref": f"#/$defs/L{i + 1}"}, + "b": {"$ref": f"#/$defs/L{i + 1}"}, + } + ) + for i in range(10) + } + defs["L10"] = {"type": "string"} + with pytest.raises(ValueError, match="maximum of .* nodes"): + json_response_schema_to_pydantic( + _object({"root": {"$ref": "#/$defs/L0"}}, **{"$defs": defs}) + ) + + def test_duplicate_name_across_defs_and_definitions(self): + with pytest.raises(ValueError, match="appears in both"): + json_response_schema_to_pydantic( + _object( + {"a": {"$ref": "#/$defs/X"}}, + **{ + "$defs": {"X": {"type": "string"}}, + "definitions": {"X": {"type": "integer"}}, + }, + ) + ) + + def test_root_must_be_object(self): + with pytest.raises(ValueError, match="root schema"): + json_response_schema_to_pydantic({"type": "string"}) + + def test_root_must_be_dict(self): + with pytest.raises(ValueError, match="JSON Schema object"): + json_response_schema_to_pydantic(["not", "a", "schema"]) # pyright: ignore + + def test_no_recognizable_type(self): + with pytest.raises(ValueError, match="no recognizable type"): + json_response_schema_to_pydantic(_object({"mystery": {}})) + + def test_depth_limit(self): + schema: dict[str, Any] = {"type": "string"} + for _ in range(25): + schema = _object({"inner": schema}) + with pytest.raises(ValueError, match="maximum depth"): + json_response_schema_to_pydantic(schema) + + def test_node_limit(self): + schema = _object({f"field_{i}": {"type": "string"} for i in range(600)}) + with pytest.raises(ValueError, match="maximum of 500 nodes"): + json_response_schema_to_pydantic(schema) + + def test_property_schema_not_an_object(self): + with pytest.raises(ValueError, match="schema must be an object"): + json_response_schema_to_pydantic(_object({"a": "string"})) + + @pytest.mark.parametrize("members", [[], "not-a-list"]) + def test_malformed_anyof(self, members: Any): + with pytest.raises(ValueError, match="'anyOf' must be a non-empty array"): + json_response_schema_to_pydantic(_object({"a": {"anyOf": members}})) + + def test_empty_type_list(self): + with pytest.raises(ValueError, match="'type' array must not be empty"): + json_response_schema_to_pydantic(_object({"a": {"type": []}})) + + @pytest.mark.parametrize("values", [[], "loves"]) + def test_malformed_enum(self, values: Any): + with pytest.raises(ValueError, match="'enum' must be a non-empty array"): + json_response_schema_to_pydantic(_object({"a": {"enum": values}})) + + def test_properties_not_an_object(self): + with pytest.raises(ValueError, match="'properties' must be an object"): + json_response_schema_to_pydantic({"type": "object", "properties": []}) + + @pytest.mark.parametrize("required", ["a", [1]]) + def test_malformed_required(self, required: Any): + with pytest.raises(ValueError, match="'required' must be an array of strings"): + json_response_schema_to_pydantic( + _object({"a": {"type": "string"}}, required=required) + ) + + def test_empty_property_name(self): + with pytest.raises(ValueError, match="property names"): + json_response_schema_to_pydantic(_object({"": {"type": "string"}})) + + +class TestLenientAcceptance: + def test_additional_properties_false_ignored(self): + model = json_response_schema_to_pydantic( + _object( + {"known": {"type": "string"}}, + required=["known"], + additionalProperties=False, + ) + ) + instance = model.model_validate({"known": "x", "extra": "dropped"}) + assert instance.model_dump() == {"known": "x"} + + def test_root_dollar_schema_ignored(self): + model = json_response_schema_to_pydantic( + { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": {"a": {"type": "string"}}, + } + ) + assert issubclass(model, BaseModel) + + def test_empty_properties(self): + model = json_response_schema_to_pydantic({"type": "object", "properties": {}}) + assert model.model_validate({}).model_dump() == {} + + def test_missing_properties_with_object_type(self): + model = json_response_schema_to_pydantic({"type": "object"}) + assert model.model_validate({"anything": 1}).model_dump() == {} + + def test_missing_type_with_properties_treated_as_object(self): + model = json_response_schema_to_pydantic( + {"properties": {"a": {"type": "string"}}, "required": ["a"]} + ) + assert model.model_validate({"a": "x"}).a == "x" # pyright: ignore + + def test_required_naming_unknown_property_ignored(self): + model = json_response_schema_to_pydantic( + _object({"a": {"type": "string"}}, required=["a", "ghost"]) + ) + assert model.model_validate({"a": "x"}).a == "x" # pyright: ignore + + +class TestFieldMetadata: + def test_description_propagates(self): + model = json_response_schema_to_pydantic( + _object({"food": {"type": "string", "description": "A food item"}}) + ) + generated = model.model_json_schema() + assert generated["properties"]["food"]["description"] == "A food item" + + def test_constraint_hints_pass_through_unenforced(self): + model = json_response_schema_to_pydantic( + _object( + { + "tags": { + "type": "array", + "items": {"type": "string"}, + "maxItems": 3, + } + }, + required=["tags"], + ) + ) + generated = model.model_json_schema() + assert generated["properties"]["tags"]["maxItems"] == 3 + # Not enforced: more than maxItems still validates. + instance = model.model_validate({"tags": ["a", "b", "c", "d"]}) + assert len(instance.tags) == 4 # pyright: ignore + + def test_non_identifier_key_alias_round_trip(self): + model = json_response_schema_to_pydantic( + _object( + {"my-key": {"type": "string"}, "_private": {"type": "integer"}}, + required=["my-key"], + ) + ) + instance = model.model_validate({"my-key": "v", "_private": 7}) + dumped = instance.model_dump_json(by_alias=True) + assert '"my-key":"v"' in dumped + assert '"_private":7' in dumped + + def test_digit_leading_key_gets_field_prefix(self): + model = json_response_schema_to_pydantic( + _object({"123": {"type": "integer"}}, required=["123"]) + ) + instance = model.model_validate({"123": 7}) + assert instance.model_dump(by_alias=True) == {"123": 7} + + def test_sanitized_key_collision_round_trip(self): + # "my-key" sanitizes to "my_key", which then collides with the real + # "my_key" property; both must survive with their original JSON keys. + model = json_response_schema_to_pydantic( + _object( + {"my-key": {"type": "string"}, "my_key": {"type": "integer"}}, + required=["my-key", "my_key"], + ) + ) + instance = model.model_validate({"my-key": "v", "my_key": 7}) + assert instance.model_dump(by_alias=True) == {"my-key": "v", "my_key": 7} + + def test_digit_leading_model_name(self): + model = json_response_schema_to_pydantic( + _object({"a": {"type": "string"}}), model_name="123" + ) + assert model.__name__ == "Model123" + + +class TestZodCompatibility: + def test_zod4_tojsonschema_output_converts(self): + # Captured shape of zod 4's z.toJSONSchema() for + # z.object({ preferences: z.array(z.object({ food: z.string(), + # sentiment: z.enum(["loves","hates"]) })), summary: z.string(), + # note: z.string().optional() }) + schema = { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "preferences": { + "type": "array", + "items": { + "type": "object", + "properties": { + "food": {"type": "string"}, + "sentiment": { + "type": "string", + "enum": ["loves", "hates"], + }, + }, + "required": ["food", "sentiment"], + "additionalProperties": False, + }, + }, + "summary": {"type": "string"}, + "note": {"type": "string"}, + }, + "required": ["preferences", "summary"], + "additionalProperties": False, + } + model = json_response_schema_to_pydantic(schema) + instance = model.model_validate( + { + "preferences": [{"food": "sushi", "sentiment": "loves"}], + "summary": "likes sushi", + } + ) + assert instance.preferences[0].sentiment == "loves" # pyright: ignore + assert instance.note is None # pyright: ignore + + +class TestCustomGuardLimits: + def test_custom_max_depth(self): + schema: dict[str, Any] = {"type": "string"} + for _ in range(5): + schema = _object({"inner": schema}) + with pytest.raises(ValueError, match="maximum depth of 3"): + json_response_schema_to_pydantic(schema, max_depth=3) + + def test_custom_max_nodes(self): + schema = _object({f"f{i}": {"type": "string"} for i in range(20)}) + with pytest.raises(ValueError, match="maximum of 10 nodes"): + json_response_schema_to_pydantic(schema, max_nodes=10) + + def test_depth_exactly_at_limit_allowed(self): + # Leaf sits at depth == max_depth; only depth > max_depth must fail. + schema: dict[str, Any] = {"type": "string"} + for _ in range(3): + schema = _object({"inner": schema}) + model = json_response_schema_to_pydantic(schema, max_depth=3) + assert issubclass(model, BaseModel) + + +# The wiki spec's own request example (dialectic-enhancements §3.A.1). +SPEC_EXAMPLE_SCHEMA = _object( + { + "preferences": { + "type": "array", + "items": _object( + { + "food": {"type": "string"}, + "sentiment": { + "type": "string", + "enum": ["loves", "likes", "neutral", "dislikes", "hates"], + }, + "confidence": {"type": "number", "minimum": 0, "maximum": 1}, + }, + required=["food", "sentiment"], + ), + "maxItems": 3, + }, + "summary": {"type": "string"}, + }, + required=["preferences", "summary"], +) + + +class TestEndToEnd: + """Table tests running the full pipeline the server runs: convert the + caller's schema, validate a payload against the generated model, and + serialize it back with model_dump_json(by_alias=True).""" + + @pytest.mark.parametrize( + "schema,payload,expected", + [ + pytest.param( + SPEC_EXAMPLE_SCHEMA, + { + "preferences": [ + { + "food": "dark roast coffee", + "sentiment": "loves", + "confidence": 0.95, + }, + {"food": "sushi", "sentiment": "likes"}, + ], + "summary": "Coffee enthusiast.", + }, + { + "preferences": [ + { + "food": "dark roast coffee", + "sentiment": "loves", + "confidence": 0.95, + }, + {"food": "sushi", "sentiment": "likes", "confidence": None}, + ], + "summary": "Coffee enthusiast.", + }, + id="spec-example", + ), + pytest.param( + _object( + { + "user": _object( + { + "name": {"type": "string"}, + "location": _object( + { + "lat": {"type": "number"}, + "lon": {"type": "number"}, + }, + required=["lat", "lon"], + ), + }, + required=["name", "location"], + ) + }, + required=["user"], + ), + {"user": {"name": "ada", "location": {"lat": 37.8, "lon": -122.3}}}, + {"user": {"name": "ada", "location": {"lat": 37.8, "lon": -122.3}}}, + id="nested-three-levels", + ), + pytest.param( + _object( + {"my-key": {"type": "string"}, "first name": {"type": "string"}}, + required=["my-key"], + ), + {"my-key": "v", "first name": "Ada"}, + {"my-key": "v", "first name": "Ada"}, + id="alias-keys-round-trip", + ), + pytest.param( + _object( + { + "count": {"type": "integer", "default": 3}, + "tag": {"type": "string", "default": "none"}, + } + ), + {}, + {"count": 3, "tag": "none"}, + id="defaults-fill-omitted-fields", + ), + pytest.param( + _object( + { + "a": {"anyOf": [{"type": "string"}, {"type": "null"}]}, + "b": {"type": ["integer", "null"]}, + }, + required=["a", "b"], + ), + {"a": None, "b": 2}, + {"a": None, "b": 2}, + id="nullable-via-anyof-and-type-list", + ), + pytest.param( + _object( + {"value": {"oneOf": [{"type": "integer"}, {"type": "string"}]}}, + required=["value"], + ), + {"value": "five"}, + {"value": "five"}, + id="oneof-union-string-member", + ), + pytest.param( + _object({"level": {"enum": [1, 2, None]}}, required=["level"]), + {"level": None}, + {"level": None}, + id="enum-with-null-member", + ), + pytest.param( + _object({"stuff": {"type": "array"}}, required=["stuff"]), + {"stuff": [1, "two", {"three": 3}, None]}, + {"stuff": [1, "two", {"three": 3}, None]}, + id="array-without-items-accepts-anything", + ), + pytest.param( + _object( + { + "tags": { + "type": "array", + "items": {"type": "string"}, + "maxItems": 2, + } + }, + required=["tags"], + ), + {"tags": ["a", "b", "c", "d"]}, + {"tags": ["a", "b", "c", "d"]}, + id="constraint-hints-not-enforced", + ), + pytest.param( + _object({"known": {"type": "string"}}, required=["known"]), + {"known": "x", "hallucinated": "dropped"}, + {"known": "x"}, + id="extra-keys-dropped", + ), + pytest.param( + {"type": "object", "properties": {}}, + {}, + {}, + id="empty-object", + ), + ], + ) + def test_construct_validate_serialize( + self, + schema: dict[str, Any], + payload: dict[str, Any], + expected: dict[str, Any], + ): + model = json_response_schema_to_pydantic(schema) + instance = model.model_validate(payload) + # by_alias=True mirrors DialecticAgent.answer's serialization. + assert json.loads(instance.model_dump_json(by_alias=True)) == expected + + @pytest.mark.parametrize( + "schema,payload", + [ + pytest.param( + SPEC_EXAMPLE_SCHEMA, + {"preferences": [{"food": "sushi"}], "summary": "s"}, + id="missing-required-in-array-item", + ), + pytest.param( + SPEC_EXAMPLE_SCHEMA, + { + "preferences": [{"food": "sushi", "sentiment": "adores"}], + "summary": "s", + }, + id="invalid-enum-value", + ), + pytest.param( + SPEC_EXAMPLE_SCHEMA, + {"preferences": [{"food": "sushi", "sentiment": "likes"}]}, + id="missing-required-top-level", + ), + pytest.param( + _object( + {"user": _object({"name": {"type": "string"}}, required=["name"])}, + required=["user"], + ), + {"user": {}}, + id="missing-required-nested", + ), + pytest.param( + _object({"a": {"type": "string"}}, required=["a"]), + {"a": None}, + id="null-for-non-nullable", + ), + pytest.param( + _object({"n": {"type": "integer"}}, required=["n"]), + {"n": {"nested": "dict"}}, + id="wrong-type-for-integer", + ), + ], + ) + def test_rejects_nonconforming_payloads( + self, schema: dict[str, Any], payload: dict[str, Any] + ): + model = json_response_schema_to_pydantic(schema) + with pytest.raises(ValidationError): + model.model_validate(payload)