From 00b7da371bf121daf73755de911142b02df7cd0e Mon Sep 17 00:00:00 2001 From: Eugene Eisenstein Date: Thu, 3 Sep 2026 11:59:07 -0400 Subject: [PATCH] fix(mock-provider): honour include_usage, generate prefixItems tuples MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three fidelity gaps where the mock answered a request differently from the API it stands in for: - The usage chunk was emitted on every stream. The real API sends it only when stream_options.include_usage is set, so a caller that did not opt in had to skip a trailing chunk with an empty choices array. stream_options is now a typed model, which also rejects a non-boolean include_usage instead of reading it as truthy. - A fixed-length tuple is prefixItems with no items, which is what Pydantic emits for tuple[str, int]. Reading only items returned [], failing the minItems the same schema carries — the silent-empty failure schema_gen exists to avoid. - A zero or negative dimensions was silently replaced with 1536, answering a bad request with a plausible-looking vector rather than a 400. Three further deviations from JSON Schema are left in place and documented where they occur: allOf merges properties first-wins, oneOf is treated as anyOf, and string pattern is ignored. None is reachable from a Honcho response model — no model emits prefixItems or oneOf, and the only pattern constraints are on API request models — and each fix costs more than the unreachable path is worth. Co-Authored-By: Claude Opus 5 (1M context) --- src/mock_provider/chat.py | 19 +++- src/mock_provider/embeddings.py | 8 +- src/mock_provider/schema_gen.py | 40 ++++++- src/mock_provider/schemas.py | 19 +++- tests/mock_provider/test_mock_provider.py | 126 +++++++++++++++++++--- 5 files changed, 178 insertions(+), 34 deletions(-) diff --git a/src/mock_provider/chat.py b/src/mock_provider/chat.py index b8694d63..8c982473 100644 --- a/src/mock_provider/chat.py +++ b/src/mock_provider/chat.py @@ -127,8 +127,10 @@ def _created() -> int: async def _stream( - completion_id: str, model: str, content: str, usage: dict[str, int] + completion_id: str, model: str, content: str, usage: dict[str, int] | None ) -> AsyncIterator[bytes]: + """Stream ``content``, ending on a usage chunk when ``usage`` is given.""" + def chunk(payload: dict[str, Any]) -> bytes: return f"data: {json.dumps(payload)}\n\n".encode() @@ -164,9 +166,13 @@ async def _stream( "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], } ) - # The backend requests stream_options.include_usage and ends the stream on - # the usage chunk, so it must come last and must carry choices: []. - yield chunk({**base, "choices": [], "usage": usage}) + # The usage chunk is conditional: the real API emits it only when + # stream_options.include_usage is set, and ends the stream on it — so it + # must come last and must carry choices: []. Honcho's own backend always + # asks for it (_build_params in the OpenAI backend), but a caller that does + # not must not receive a chunk it never requested. + if usage is not None: + yield chunk({**base, "choices": [], "usage": usage}) yield b"data: [DONE]\n\n" @@ -178,8 +184,11 @@ async def chat_completions(body: ChatCompletionRequest) -> Any: completion_id = _completion_id(body) if body.stream: + include_usage = ( + body.stream_options is not None and body.stream_options.include_usage + ) return StreamingResponse( - _stream(completion_id, model, content, usage), + _stream(completion_id, model, content, usage if include_usage else None), media_type="text/event-stream", ) diff --git a/src/mock_provider/embeddings.py b/src/mock_provider/embeddings.py index 618ff7ba..8b9ee5eb 100644 --- a/src/mock_provider/embeddings.py +++ b/src/mock_provider/embeddings.py @@ -69,11 +69,9 @@ def _normalize_input( @router.post("/embeddings") async def embeddings(body: EmbeddingsRequest) -> Any: texts = _normalize_input(body.input) - dimensions = ( - body.dimensions - if body.dimensions and body.dimensions > 0 - else DEFAULT_DIMENSIONS - ) + # A non-positive width is rejected by the request model, so absent is the + # only case left to fill in. + dimensions = body.dimensions if body.dimensions is not None else DEFAULT_DIMENSIONS data: list[dict[str, Any]] = [] for index, text in enumerate(texts): diff --git a/src/mock_provider/schema_gen.py b/src/mock_provider/schema_gen.py index 65b5a739..36bcf4ae 100644 --- a/src/mock_provider/schema_gen.py +++ b/src/mock_provider/schema_gen.py @@ -97,6 +97,13 @@ def _merge_all_of(schema: dict[str, Any], root: dict[str, Any]) -> dict[str, Any if key == "properties": properties = as_dict(value) if properties is not None: + # First branch to define a property wins. Strictly, `allOf` + # requires every branch's constraints to apply, so a schema + # splitting `minimum` and `maximum` for one property across + # two branches generates a value satisfying only one of + # them. Not merged recursively because Pydantic's `allOf` is + # always a $ref plus sibling annotations — it never repeats + # a property key, let alone with conflicting constraints. existing = as_dict(merged.get("properties")) or {} merged["properties"] = {**properties, **existing} continue @@ -152,6 +159,12 @@ def _generate( if depth >= MAX_DEPTH and "default" in resolved: return resolved["default"] + # `oneOf` is treated as `anyOf`: a branch is picked without checking that + # the result matches only that one. A `oneOf` whose branches overlap can + # therefore yield a value matching several, which `oneOf` forbids. Enforcing + # the cardinality needs a full JSON Schema validator to test the candidate + # against every branch, and Pydantic emits `anyOf` for unions — never + # `oneOf` — so nothing Honcho sends reaches the distinction. for key in ("anyOf", "oneOf"): branches = as_list(resolved.get(key)) if branches: @@ -229,6 +242,17 @@ def _generate_object( def _generate_array( schema: dict[str, Any], root: dict[str, Any], path: str, depth: int ) -> list[Any]: + # A fixed-length tuple is `prefixItems` with no `items`, which is what + # Pydantic emits for `tuple[str, int]`. Reading only `items` would return [] + # for it and fail the minItems/maxItems the same schema carries. + prefix: list[Any] = [] + prefix_items = as_list(schema.get("prefixItems")) + if prefix_items is not None: + for index, entry in enumerate(prefix_items): + child = as_dict(entry) + if child is not None: + prefix.append(_generate(child, root, f"{path}[{index}]", depth + 1)) + items = as_dict(schema.get("items")) min_items = as_int(schema.get("minItems")) max_items = as_int(schema.get("maxItems")) @@ -240,10 +264,15 @@ def _generate_array( count = min(count, max_items) if depth >= MAX_DEPTH: count = min_items or 0 - if items is None or count <= 0: - return [] + if items is None: + return prefix - return [_generate(items, root, f"{path}[{i}]", depth + 1) for i in range(count)] + # `items` describes the positions after the prefix, so only the shortfall is + # filled homogeneously. + return prefix + [ + _generate(items, root, f"{path}[{len(prefix) + i}]", depth + 1) + for i in range(max(0, count - len(prefix))) + ] def _generate_string(schema: dict[str, Any], path: str) -> str: @@ -260,6 +289,11 @@ def _generate_string(schema: dict[str, Any], path: str) -> str: if fmt == "email": return "placeholder@mock.invalid" + # `pattern` is not honoured: this phrase fails any regex narrower than it, + # so a pattern-constrained string generates a value its own schema rejects. + # Satisfying an arbitrary regex needs a generator library, and no Honcho + # response model carries a `pattern` — the only ones in the codebase are on + # API request models, which are never sent as a response_format. value = _phrase(path) min_length = as_int(schema.get("minLength")) max_length = as_int(schema.get("maxLength")) diff --git a/src/mock_provider/schemas.py b/src/mock_provider/schemas.py index 3dfe39ab..e39fbc11 100644 --- a/src/mock_provider/schemas.py +++ b/src/mock_provider/schemas.py @@ -18,9 +18,9 @@ Two deliberate choices: from __future__ import annotations -from typing import Any, ClassVar, Literal +from typing import Annotated, Any, ClassVar, Literal -from pydantic import BaseModel, ConfigDict, StrictInt +from pydantic import BaseModel, ConfigDict, Field, StrictInt class MockRequest(BaseModel): @@ -36,13 +36,20 @@ class ChatMessage(MockRequest): content: Any = None +class StreamOptions(MockRequest): + # Typed rather than left as a dict because the usage chunk is conditional on + # it: `{"include_usage": "yes"}` must fail like the real API does rather than + # read as truthy and emit a chunk the caller never asked for. + include_usage: bool = False + + class ChatCompletionRequest(MockRequest): model: str | None = None messages: list[ChatMessage] = [] response_format: dict[str, Any] | None = None tools: list[dict[str, Any]] | None = None stream: bool = False - stream_options: dict[str, Any] | None = None + stream_options: StreamOptions | None = None class EmbeddingsRequest(MockRequest): @@ -52,7 +59,9 @@ class EmbeddingsRequest(MockRequest): input: str | list[str] | list[int] | list[list[int]] | None = None model: str | None = None # StrictInt because bool is an int subclass: a JSON `true` here would - # otherwise silently become a one-dimensional vector. - dimensions: StrictInt | None = None + # otherwise silently become a one-dimensional vector. gt=0 because the real + # API rejects a non-positive width, and substituting the default instead + # would answer a bad request with a plausible-looking vector. + dimensions: Annotated[StrictInt, Field(gt=0)] | None = None # The SDK omits this only when it wants base64, so absent means base64. encoding_format: Literal["float", "base64"] = "base64" diff --git a/tests/mock_provider/test_mock_provider.py b/tests/mock_provider/test_mock_provider.py index c08d122f..1104d71b 100644 --- a/tests/mock_provider/test_mock_provider.py +++ b/tests/mock_provider/test_mock_provider.py @@ -100,9 +100,9 @@ def test_deriver_response_model_round_trips() -> None: content = json.dumps(generate(schema)) representation = PromptRepresentation.model_validate_json(content) - assert ( - representation.explicit - ), "an empty explicit list is exactly the silent failure this mock avoids" + assert representation.explicit, ( + "an empty explicit list is exactly the silent failure this mock avoids" + ) def test_json_schema_response_is_never_prose(client: TestClient) -> None: @@ -279,6 +279,40 @@ def test_all_of_is_flattened() -> None: assert isinstance(result["b"], int) +def test_fixed_tuple_schema_round_trips() -> None: + """Pydantic emits a fixed tuple as `prefixItems` with no `items`. + + Reading only `items` yields [], which fails the minItems the same schema + carries — the silent-empty failure this module exists to avoid. + """ + + class Tupled(BaseModel): + pair: tuple[str, int] + + schema = Tupled.model_json_schema() + assert "prefixItems" in schema["properties"]["pair"] + + result = generate(schema) + assert isinstance(result["pair"], list) + Tupled.model_validate(result) + + +def test_prefix_items_are_followed_by_homogeneous_items() -> None: + """A variadic tuple constrains leading positions and the rest by `items`.""" + schema: dict[str, Any] = { + "type": "array", + "prefixItems": [{"type": "string"}, {"type": "integer"}], + "items": {"type": "boolean"}, + "minItems": 4, + } + result = generate(schema) + + assert len(result) == 4 + assert isinstance(result[0], str) + assert isinstance(result[1], int) + assert all(isinstance(value, bool) for value in result[2:]) + + def test_generation_is_stable_across_calls() -> None: assert generate(PROBE_SCHEMA) == generate(PROBE_SCHEMA) @@ -428,6 +462,20 @@ def test_boolean_dimensions_is_rejected_not_silently_coerced( assert response.status_code == 400 +@pytest.mark.parametrize("dimensions", [0, -1]) +def test_non_positive_dimensions_is_rejected_not_defaulted( + client: TestClient, dimensions: int +) -> None: + """Substituting 1536 would answer a bad request with a plausible vector.""" + response = client.post( + "/v1/embeddings", + json={"input": "hello", "dimensions": dimensions, "encoding_format": "float"}, + ) + + assert response.status_code == 400 + assert response.json()["error"]["type"] == "invalid_request_error" + + def test_unknown_fields_are_accepted(client: TestClient) -> None: """Validation must fire on wrong types, never on unrecognised parameters. @@ -479,18 +527,19 @@ def test_every_documented_input_shape_is_accepted( # --- streaming -------------------------------------------------------------- -def test_stream_emits_content_then_a_final_usage_chunk(client: TestClient) -> None: - """The backend ends the stream on the usage chunk, so it must come last.""" - with client.stream( - "POST", - "/v1/chat/completions", - json={ - "model": "mock-model", - "messages": [{"role": "user", "content": "stream please"}], - "stream": True, - "stream_options": {"include_usage": True}, - }, - ) as response: +def _stream_chunks( + client: TestClient, stream_options: dict[str, Any] | None = None +) -> list[dict[str, Any]]: + """The SSE payloads of a streaming completion, `[DONE]` asserted and dropped.""" + body: dict[str, Any] = { + "model": "mock-model", + "messages": [{"role": "user", "content": "stream please"}], + "stream": True, + } + if stream_options is not None: + body["stream_options"] = stream_options + + with client.stream("POST", "/v1/chat/completions", json=body) as response: assert response.status_code == 200 lines = [ line[len("data: ") :] @@ -499,7 +548,12 @@ def test_stream_emits_content_then_a_final_usage_chunk(client: TestClient) -> No ] assert lines[-1] == "[DONE]" - chunks = [json.loads(line) for line in lines[:-1]] + return [json.loads(line) for line in lines[:-1]] + + +def test_stream_emits_content_then_a_final_usage_chunk(client: TestClient) -> None: + """The backend ends the stream on the usage chunk, so it must come last.""" + chunks = _stream_chunks(client, {"include_usage": True}) content = "".join( chunk["choices"][0]["delta"].get("content", "") @@ -516,3 +570,43 @@ def test_stream_emits_content_then_a_final_usage_chunk(client: TestClient) -> No usage_chunk = chunks[-1] assert usage_chunk["usage"]["completion_tokens"] > 0 assert usage_chunk["choices"] == [] + + +@pytest.mark.parametrize( + "stream_options", + [None, {}, {"include_usage": False}], + ids=["absent", "empty", "false"], +) +def test_stream_without_include_usage_emits_no_usage_chunk( + client: TestClient, stream_options: dict[str, Any] | None +) -> None: + """The real API sends the usage chunk only when asked, so neither does this. + + A caller that did not opt in must not have to skip a trailing chunk with an + empty `choices` array. + """ + chunks = _stream_chunks(client, stream_options) + + assert all("usage" not in chunk for chunk in chunks) + assert chunks[-1]["choices"][0]["finish_reason"] == "stop" + + content = "".join( + chunk["choices"][0]["delta"].get("content", "") for chunk in chunks + ) + assert "[mock]" in content + + +def test_non_boolean_include_usage_is_rejected(client: TestClient) -> None: + """The usage chunk is conditional on this, so a wrong type must 400.""" + response = client.post( + "/v1/chat/completions", + json={ + "model": "mock-model", + "messages": [{"role": "user", "content": "x"}], + "stream": True, + "stream_options": {"include_usage": "definitely"}, + }, + ) + + assert response.status_code == 400 + assert response.json()["error"]["type"] == "invalid_request_error"