feat: Add warning for empty or sentinel LLM responses

Detect and warn when model returns empty content or sentinel tokens
like <|endoftext|> without tool calls or refusals.

Changes:
- Add _warn_empty_response() helper method (10 lines)
- Call helper in get_response() and stream_response()
- Add 3 unit tests

Non-breaking: warning only, no change to control flow or return types.
This commit is contained in:
Rufino Cabrera 2025-12-16 12:05:14 +01:00
parent da1c6202e5
commit cd49201a7d
2 changed files with 172 additions and 1 deletions

View File

@ -517,6 +517,16 @@ class OpenAIChatCompletionsModel(Model):
def _non_null_or_not_given(self, value: Any) -> Any:
return value if value is not None else NOT_GIVEN
def _warn_empty_response(self, content: str | None, has_tool_calls: bool, has_refusal: bool) -> None:
"""Log warning for empty or sentinel responses (e.g., <|endoftext|>)."""
if has_tool_calls or has_refusal:
return
is_empty = not content or not str(content).strip()
is_sentinel = content and "<|endoftext|>" in str(content)
if is_empty or is_sentinel:
detail = "empty" if is_empty else f"sentinel ({str(content)[:50]})"
logger.warning(f"Model completed without output ({detail}). May indicate model-agent compatibility issue.")
async def get_response(
self,
system_instructions: str | None,
@ -1124,6 +1134,13 @@ class OpenAIChatCompletionsModel(Model):
if not hasattr(response, "cost"):
response.cost = None
# Warn if response is empty or contains sentinel token
self._warn_empty_response(
getattr(response.choices[0].message, "content", None),
bool(getattr(response.choices[0].message, "tool_calls", None)),
bool(getattr(response.choices[0].message, "refusal", None))
)
return ModelResponse(
output=items,
usage=usage,
@ -2198,6 +2215,17 @@ class OpenAIChatCompletionsModel(Model):
},
)
# Warn if streamed response is empty or contains sentinel token
text_content = ""
if state.text_content_index_and_output:
text_out = state.text_content_index_and_output[1]
text_content = getattr(text_out, 'text', '') or getattr(text_out, 'content', '')
self._warn_empty_response(
text_content,
bool(state.function_calls),
bool(state.refusal_content_index_and_output)
)
yield ResponseCompletedEvent(
response=final_response,
type="response.completed",

View File

@ -360,4 +360,147 @@ async def test_interaction_counter_single_turn_with_tool_calls(monkeypatch) -> N
)
# Counter should now be 2 (one increment per turn, not per item)
assert model.interaction_counter == 2
assert model.interaction_counter == 2
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_empty_response_warning(monkeypatch, caplog) -> None:
"""
When the model returns a response with empty content (no text, no tool calls, no refusal),
a warning should be logged indicating potential model-agent compatibility issues.
"""
import logging
# Create a response with empty content
msg = ChatCompletionMessage(role="assistant", content="")
choice = Choice(index=0, finish_reason="stop", message=msg)
chat = ChatCompletion(
id="resp-id",
created=0,
model="fake",
object="chat.completion",
choices=[choice],
usage=CompletionUsage(completion_tokens=0, prompt_tokens=5, total_tokens=5),
)
async def patched_fetch_response(self, *args, **kwargs):
return chat
monkeypatch.setattr(OpenAIChatCompletionsModel, "_fetch_response", patched_fetch_response)
model = OpenAIProvider(use_responses=False).get_model(cai_model)
# Enable logging capture
with caplog.at_level(logging.WARNING, logger="openai.agents"):
resp: ModelResponse = await model.get_response(
system_instructions=None,
input="Test",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
)
# Response should still be returned (non-breaking)
assert isinstance(resp, ModelResponse)
# Warning should have been logged
assert any("Model completed without output" in record.message for record in caplog.records)
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_sentinel_response_warning(monkeypatch, caplog) -> None:
"""
When the model returns a response with sentinel token (<|endoftext|>),
a warning should be logged indicating potential model-agent compatibility issues.
"""
import logging
# Create a response with sentinel token
msg = ChatCompletionMessage(role="assistant", content="<|endoftext|>")
choice = Choice(index=0, finish_reason="stop", message=msg)
chat = ChatCompletion(
id="resp-id",
created=0,
model="fake",
object="chat.completion",
choices=[choice],
usage=CompletionUsage(completion_tokens=1, prompt_tokens=5, total_tokens=6),
)
async def patched_fetch_response(self, *args, **kwargs):
return chat
monkeypatch.setattr(OpenAIChatCompletionsModel, "_fetch_response", patched_fetch_response)
model = OpenAIProvider(use_responses=False).get_model(cai_model)
# Enable logging capture
with caplog.at_level(logging.WARNING, logger="openai.agents"):
resp: ModelResponse = await model.get_response(
system_instructions=None,
input="Test",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
)
# Response should still be returned (non-breaking)
assert isinstance(resp, ModelResponse)
# Warning should have been logged with sentinel info
assert any("sentinel token" in record.message for record in caplog.records)
@pytest.mark.allow_call_model_methods
@pytest.mark.asyncio
async def test_no_warning_for_tool_calls(monkeypatch, caplog) -> None:
"""
When the model returns a response with tool calls (even without text content),
no warning should be logged since this is expected behavior.
"""
import logging
# Create a response with tool calls but no text content
tool_call = ChatCompletionMessageToolCall(
id="call_123",
type="function",
function=Function(name="test_tool", arguments='{"arg": "value"}'),
)
msg = ChatCompletionMessage(role="assistant", content=None, tool_calls=[tool_call])
choice = Choice(index=0, finish_reason="tool_calls", message=msg)
chat = ChatCompletion(
id="resp-id",
created=0,
model="fake",
object="chat.completion",
choices=[choice],
usage=CompletionUsage(completion_tokens=10, prompt_tokens=5, total_tokens=15),
)
async def patched_fetch_response(self, *args, **kwargs):
return chat
monkeypatch.setattr(OpenAIChatCompletionsModel, "_fetch_response", patched_fetch_response)
model = OpenAIProvider(use_responses=False).get_model(cai_model)
# Enable logging capture
with caplog.at_level(logging.WARNING, logger="openai.agents"):
resp: ModelResponse = await model.get_response(
system_instructions=None,
input="Test",
model_settings=ModelSettings(),
tools=[],
output_schema=None,
handoffs=[],
tracing=ModelTracing.DISABLED,
)
# Response should be returned
assert isinstance(resp, ModelResponse)
# No warning should be logged for tool calls
assert not any("Model completed without output" in record.message for record in caplog.records)