diff --git a/tests/live_llm/README.md b/tests/live_llm/README.md index 5cf39e4b..9590185e 100644 --- a/tests/live_llm/README.md +++ b/tests/live_llm/README.md @@ -63,8 +63,8 @@ export OPENROUTER_API_KEY="sk-or-v1-..." Coverage by provider: - Anthropic: structured output path, prompt caching metrics, thinking blocks, multi-turn tool replay -- OpenAI GPT-4 class: structured outputs, prompt caching -- OpenAI GPT-5 class (incl. gpt-5.x point-releases): structured outputs, prompt caching, `reasoning_effort`, `max_completion_tokens` routing +- OpenAI GPT-4 class: structured outputs, prompt caching, multi-turn tool replay (null `content` preserved) +- OpenAI GPT-5 class (incl. gpt-5.x point-releases): structured outputs, prompt caching, `reasoning_effort`, `max_completion_tokens` routing, multi-turn tool replay (null `content` preserved) - OpenAI transport → OpenRouter non-reasoning models (e.g. `inception/mercury-2`): non-chat / diffusion architectures must stay on `max_tokens`, no `reasoning_effort`, tool-calling parameter-schema compatibility is the canary for exotic OR-served providers - Gemini 2.5/3.0 classes: structured outputs, cached-content reuse, thought signatures, multi-turn tool replay - Gemini 3.1 class: thinking and tool replay coverage by default; structured-output/caching coverage should only be added once Google documents support for that path diff --git a/tests/live_llm/model_matrix.py b/tests/live_llm/model_matrix.py index 2478c6c3..3a18a84e 100644 --- a/tests/live_llm/model_matrix.py +++ b/tests/live_llm/model_matrix.py @@ -58,6 +58,7 @@ MODEL_FAMILIES: tuple[LiveModelFamily, ...] = ( default_models=("gpt-4.1",), supports_structured_output=True, supports_caching=True, + supports_tool_replay=True, docs_url="https://platform.openai.com/docs/models/gpt-4.1", ), LiveModelFamily( @@ -68,6 +69,7 @@ MODEL_FAMILIES: tuple[LiveModelFamily, ...] = ( supports_structured_output=True, supports_caching=True, supports_reasoning=True, + supports_tool_replay=True, docs_url="https://platform.openai.com/docs/models/gpt-5", ), # OpenAI-compatible transport → OpenRouter-served non-reasoning models. diff --git a/tests/live_llm/test_live_openai.py b/tests/live_llm/test_live_openai.py index 8b0544ec..b69818c6 100644 --- a/tests/live_llm/test_live_openai.py +++ b/tests/live_llm/test_live_openai.py @@ -2,10 +2,13 @@ from __future__ import annotations import pytest +from src.llm.history_adapters import OpenAIHistoryAdapter from src.llm.request_builder import execute_completion from .conftest import ( StructuredLiveResponse, + execute_local_tool, + favorite_prime_tools, make_backend, make_large_system_prompt, require_provider_key, @@ -30,6 +33,11 @@ _JSON_OBJECT_SPECS = tuple( for spec in get_live_model_specs(provider="openai") if spec.family == "openai_json_object" ) +_TOOL_REPLAY_SPECS = tuple( + spec + for spec in get_live_model_specs(provider="openai") + if spec.supports_tool_replay +) @pytest.mark.asyncio @@ -194,3 +202,83 @@ async def test_live_openai_json_object_structured_output( assert parse_calls == [] assert create_calls, "expected a chat.completions.create call" assert create_calls[0]["kwargs"]["response_format"] == {"type": "json_object"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model_spec", _TOOL_REPLAY_SPECS, ids=lambda spec: spec.id) +async def test_live_openai_tool_replay_preserves_null_content( + model_spec: LiveModelSpec, +) -> None: + """Tool-call turns with provider content=null must stay null through + normalize + history replay, and the continuation must still succeed.""" + require_provider_key(model_spec) + # Leave reasoning_effort unset: gpt-5.4 rejects function tools with any + # explicit reasoning_effort other than 'none' on /v1/chat/completions. + backend, config = make_backend(model_spec) + tools = favorite_prime_tools() + adapter = OpenAIHistoryAdapter() + + initial_messages = [ + { + "role": "user", + "content": ( + "Before answering, call the get_favorite_prime tool exactly once. " + "Do not answer with plain text on this turn. " + "After you receive the tool result, answer in one sentence that " + "includes the number and the word 'prime'." + ), + } + ] + + first = await execute_completion( + backend, + config, + messages=initial_messages, + max_tokens=1024, + tools=tools, + tool_choice="required", + ) + + assert first.tool_calls, "OpenAI should issue a tool call in the first turn" + raw_message = first.raw_response.choices[0].message + raw_content = raw_message.content + if raw_content is None: + assert first.content is None + else: + assert first.content == raw_content + + assistant_message = adapter.format_assistant_tool_message(first) + assert assistant_message["content"] is ( + first.content if isinstance(first.content, str) else None + ) + if raw_content is None: + assert assistant_message["content"] is None + + tool_call = first.tool_calls[0] + tool_result = execute_local_tool(tool_call.name, tool_call.input) + replay_messages = initial_messages + [ + assistant_message, + *adapter.format_tool_results( + [ + { + "tool_id": tool_call.id, + "tool_name": tool_call.name, + "result": tool_result, + } + ] + ), + ] + + second = await execute_completion( + backend, + config, + messages=replay_messages, + max_tokens=1024, + tools=tools, + tool_choice="auto", + ) + + assert not second.tool_calls, "continuation should answer without another tool call" + assert isinstance(second.content, str) + assert "13" in second.content + assert "prime" in second.content.lower()