mirror of https://github.com/aliasrobotics/cai.git
feat: Add warning for empty or sentinel LLM responses
Detect and warn when model returns empty content or sentinel tokens like <|endoftext|> without tool calls or refusals. Changes: - Add _warn_empty_response() helper method (10 lines) - Call helper in get_response() and stream_response() - Add 3 unit tests Non-breaking: warning only, no change to control flow or return types.
This commit is contained in:
parent
da1c6202e5
commit
cd49201a7d
|
|
@ -517,6 +517,16 @@ class OpenAIChatCompletionsModel(Model):
|
|||
def _non_null_or_not_given(self, value: Any) -> Any:
|
||||
return value if value is not None else NOT_GIVEN
|
||||
|
||||
def _warn_empty_response(self, content: str | None, has_tool_calls: bool, has_refusal: bool) -> None:
|
||||
"""Log warning for empty or sentinel responses (e.g., <|endoftext|>)."""
|
||||
if has_tool_calls or has_refusal:
|
||||
return
|
||||
is_empty = not content or not str(content).strip()
|
||||
is_sentinel = content and "<|endoftext|>" in str(content)
|
||||
if is_empty or is_sentinel:
|
||||
detail = "empty" if is_empty else f"sentinel ({str(content)[:50]})"
|
||||
logger.warning(f"Model completed without output ({detail}). May indicate model-agent compatibility issue.")
|
||||
|
||||
async def get_response(
|
||||
self,
|
||||
system_instructions: str | None,
|
||||
|
|
@ -1124,6 +1134,13 @@ class OpenAIChatCompletionsModel(Model):
|
|||
if not hasattr(response, "cost"):
|
||||
response.cost = None
|
||||
|
||||
# Warn if response is empty or contains sentinel token
|
||||
self._warn_empty_response(
|
||||
getattr(response.choices[0].message, "content", None),
|
||||
bool(getattr(response.choices[0].message, "tool_calls", None)),
|
||||
bool(getattr(response.choices[0].message, "refusal", None))
|
||||
)
|
||||
|
||||
return ModelResponse(
|
||||
output=items,
|
||||
usage=usage,
|
||||
|
|
@ -2198,6 +2215,17 @@ class OpenAIChatCompletionsModel(Model):
|
|||
},
|
||||
)
|
||||
|
||||
# Warn if streamed response is empty or contains sentinel token
|
||||
text_content = ""
|
||||
if state.text_content_index_and_output:
|
||||
text_out = state.text_content_index_and_output[1]
|
||||
text_content = getattr(text_out, 'text', '') or getattr(text_out, 'content', '')
|
||||
self._warn_empty_response(
|
||||
text_content,
|
||||
bool(state.function_calls),
|
||||
bool(state.refusal_content_index_and_output)
|
||||
)
|
||||
|
||||
yield ResponseCompletedEvent(
|
||||
response=final_response,
|
||||
type="response.completed",
|
||||
|
|
|
|||
|
|
@ -360,4 +360,147 @@ async def test_interaction_counter_single_turn_with_tool_calls(monkeypatch) -> N
|
|||
)
|
||||
|
||||
# Counter should now be 2 (one increment per turn, not per item)
|
||||
assert model.interaction_counter == 2
|
||||
assert model.interaction_counter == 2
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_response_warning(monkeypatch, caplog) -> None:
|
||||
"""
|
||||
When the model returns a response with empty content (no text, no tool calls, no refusal),
|
||||
a warning should be logged indicating potential model-agent compatibility issues.
|
||||
"""
|
||||
import logging
|
||||
|
||||
# Create a response with empty content
|
||||
msg = ChatCompletionMessage(role="assistant", content="")
|
||||
choice = Choice(index=0, finish_reason="stop", message=msg)
|
||||
chat = ChatCompletion(
|
||||
id="resp-id",
|
||||
created=0,
|
||||
model="fake",
|
||||
object="chat.completion",
|
||||
choices=[choice],
|
||||
usage=CompletionUsage(completion_tokens=0, prompt_tokens=5, total_tokens=5),
|
||||
)
|
||||
|
||||
async def patched_fetch_response(self, *args, **kwargs):
|
||||
return chat
|
||||
|
||||
monkeypatch.setattr(OpenAIChatCompletionsModel, "_fetch_response", patched_fetch_response)
|
||||
model = OpenAIProvider(use_responses=False).get_model(cai_model)
|
||||
|
||||
# Enable logging capture
|
||||
with caplog.at_level(logging.WARNING, logger="openai.agents"):
|
||||
resp: ModelResponse = await model.get_response(
|
||||
system_instructions=None,
|
||||
input="Test",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
)
|
||||
|
||||
# Response should still be returned (non-breaking)
|
||||
assert isinstance(resp, ModelResponse)
|
||||
|
||||
# Warning should have been logged
|
||||
assert any("Model completed without output" in record.message for record in caplog.records)
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_sentinel_response_warning(monkeypatch, caplog) -> None:
|
||||
"""
|
||||
When the model returns a response with sentinel token (<|endoftext|>),
|
||||
a warning should be logged indicating potential model-agent compatibility issues.
|
||||
"""
|
||||
import logging
|
||||
|
||||
# Create a response with sentinel token
|
||||
msg = ChatCompletionMessage(role="assistant", content="<|endoftext|>")
|
||||
choice = Choice(index=0, finish_reason="stop", message=msg)
|
||||
chat = ChatCompletion(
|
||||
id="resp-id",
|
||||
created=0,
|
||||
model="fake",
|
||||
object="chat.completion",
|
||||
choices=[choice],
|
||||
usage=CompletionUsage(completion_tokens=1, prompt_tokens=5, total_tokens=6),
|
||||
)
|
||||
|
||||
async def patched_fetch_response(self, *args, **kwargs):
|
||||
return chat
|
||||
|
||||
monkeypatch.setattr(OpenAIChatCompletionsModel, "_fetch_response", patched_fetch_response)
|
||||
model = OpenAIProvider(use_responses=False).get_model(cai_model)
|
||||
|
||||
# Enable logging capture
|
||||
with caplog.at_level(logging.WARNING, logger="openai.agents"):
|
||||
resp: ModelResponse = await model.get_response(
|
||||
system_instructions=None,
|
||||
input="Test",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
)
|
||||
|
||||
# Response should still be returned (non-breaking)
|
||||
assert isinstance(resp, ModelResponse)
|
||||
|
||||
# Warning should have been logged with sentinel info
|
||||
assert any("sentinel token" in record.message for record in caplog.records)
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_warning_for_tool_calls(monkeypatch, caplog) -> None:
|
||||
"""
|
||||
When the model returns a response with tool calls (even without text content),
|
||||
no warning should be logged since this is expected behavior.
|
||||
"""
|
||||
import logging
|
||||
|
||||
# Create a response with tool calls but no text content
|
||||
tool_call = ChatCompletionMessageToolCall(
|
||||
id="call_123",
|
||||
type="function",
|
||||
function=Function(name="test_tool", arguments='{"arg": "value"}'),
|
||||
)
|
||||
msg = ChatCompletionMessage(role="assistant", content=None, tool_calls=[tool_call])
|
||||
choice = Choice(index=0, finish_reason="tool_calls", message=msg)
|
||||
chat = ChatCompletion(
|
||||
id="resp-id",
|
||||
created=0,
|
||||
model="fake",
|
||||
object="chat.completion",
|
||||
choices=[choice],
|
||||
usage=CompletionUsage(completion_tokens=10, prompt_tokens=5, total_tokens=15),
|
||||
)
|
||||
|
||||
async def patched_fetch_response(self, *args, **kwargs):
|
||||
return chat
|
||||
|
||||
monkeypatch.setattr(OpenAIChatCompletionsModel, "_fetch_response", patched_fetch_response)
|
||||
model = OpenAIProvider(use_responses=False).get_model(cai_model)
|
||||
|
||||
# Enable logging capture
|
||||
with caplog.at_level(logging.WARNING, logger="openai.agents"):
|
||||
resp: ModelResponse = await model.get_response(
|
||||
system_instructions=None,
|
||||
input="Test",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
)
|
||||
|
||||
# Response should be returned
|
||||
assert isinstance(resp, ModelResponse)
|
||||
|
||||
# No warning should be logged for tool calls
|
||||
assert not any("Model completed without output" in record.message for record in caplog.records)
|
||||
Loading…
Reference in New Issue