531 lines
17 KiB
Python
531 lines
17 KiB
Python
from types import SimpleNamespace
|
|
from typing import Any
|
|
from unittest.mock import AsyncMock, Mock
|
|
|
|
import pytest
|
|
from anthropic.types import TextBlock, ThinkingBlock, ToolUseBlock
|
|
from pydantic import BaseModel
|
|
|
|
from src.llm.backends.anthropic import AnthropicBackend
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_anthropic_backend_extracts_text_thinking_and_tool_calls() -> None:
|
|
client = Mock()
|
|
client.messages.create = AsyncMock(
|
|
return_value=SimpleNamespace(
|
|
content=[
|
|
ThinkingBlock(
|
|
type="thinking",
|
|
thinking="internal reasoning",
|
|
signature="sig_123",
|
|
),
|
|
TextBlock(type="text", text="Hello from Anthropic"),
|
|
ToolUseBlock(
|
|
type="tool_use",
|
|
id="tool_1",
|
|
name="search",
|
|
input={"query": "honcho"},
|
|
),
|
|
],
|
|
usage=SimpleNamespace(
|
|
input_tokens=10,
|
|
output_tokens=5,
|
|
cache_creation_input_tokens=3,
|
|
cache_read_input_tokens=2,
|
|
),
|
|
stop_reason="tool_use",
|
|
)
|
|
)
|
|
|
|
backend = AnthropicBackend(client)
|
|
result = await backend.complete(
|
|
model="claude-haiku-4-5",
|
|
messages=[
|
|
{"role": "system", "content": "System prompt"},
|
|
{"role": "user", "content": "Hello"},
|
|
],
|
|
max_tokens=100,
|
|
tools=[
|
|
{
|
|
"name": "search",
|
|
"description": "Search for information",
|
|
"input_schema": {
|
|
"type": "object",
|
|
"properties": {"query": {"type": "string"}},
|
|
},
|
|
}
|
|
],
|
|
thinking_budget_tokens=2048,
|
|
tool_choice="required",
|
|
)
|
|
|
|
assert result.content == "Hello from Anthropic"
|
|
assert result.thinking_content == "internal reasoning"
|
|
assert result.thinking_blocks == [
|
|
{
|
|
"type": "thinking",
|
|
"thinking": "internal reasoning",
|
|
"signature": "sig_123",
|
|
}
|
|
]
|
|
assert result.tool_calls[0].name == "search"
|
|
assert result.input_tokens == 15
|
|
assert result.output_tokens == 5
|
|
assert result.finish_reason == "tool_use"
|
|
|
|
await_args = client.messages.create.await_args
|
|
if await_args is None:
|
|
raise AssertionError("Expected Anthropic client call")
|
|
call = await_args.kwargs
|
|
assert call["model"] == "claude-haiku-4-5"
|
|
assert call["system"][0]["text"] == "System prompt"
|
|
assert call["thinking"] == {"type": "enabled", "budget_tokens": 2048}
|
|
assert call["tool_choice"] == {"type": "any"}
|
|
|
|
|
|
class StructuredResponse(BaseModel):
|
|
answer: str
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_anthropic_backend_skips_assistant_prefill_for_claude_4_models() -> None:
|
|
client = Mock()
|
|
client.messages.create = AsyncMock(
|
|
return_value=SimpleNamespace(
|
|
content=[TextBlock(type="text", text='{"answer":"ok"}')],
|
|
usage=SimpleNamespace(
|
|
input_tokens=10,
|
|
output_tokens=5,
|
|
cache_creation_input_tokens=0,
|
|
cache_read_input_tokens=0,
|
|
),
|
|
stop_reason="end_turn",
|
|
)
|
|
)
|
|
|
|
backend = AnthropicBackend(client)
|
|
result = await backend.complete(
|
|
model="claude-sonnet-4-5",
|
|
messages=[{"role": "user", "content": "Hello"}],
|
|
max_tokens=100,
|
|
response_format=StructuredResponse,
|
|
)
|
|
|
|
assert isinstance(result.content, StructuredResponse)
|
|
assert result.content.answer == "ok"
|
|
await_args = client.messages.create.await_args
|
|
if await_args is None:
|
|
raise AssertionError("Expected Anthropic client call")
|
|
call = await_args.kwargs
|
|
assert len(call["messages"]) == 1
|
|
assert call["messages"][0]["role"] == "user"
|
|
assert call["messages"][0]["content"].startswith("Hello\n\nRespond with valid JSON")
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_anthropic_backend_forwards_provider_params_passthroughs() -> None:
|
|
"""provider_params.extra_body/extra_headers/extra_query reach the Anthropic
|
|
SDK call as kwargs of the same name (the SDK's documented passthrough).
|
|
"""
|
|
client = Mock()
|
|
client.messages.create = AsyncMock(
|
|
return_value=SimpleNamespace(
|
|
content=[TextBlock(type="text", text="ok")],
|
|
usage=SimpleNamespace(
|
|
input_tokens=10,
|
|
output_tokens=5,
|
|
cache_creation_input_tokens=0,
|
|
cache_read_input_tokens=0,
|
|
),
|
|
stop_reason="end_turn",
|
|
)
|
|
)
|
|
|
|
backend = AnthropicBackend(client)
|
|
await backend.complete(
|
|
model="claude-haiku-4-5",
|
|
messages=[{"role": "user", "content": "Hello"}],
|
|
max_tokens=100,
|
|
extra_params={
|
|
"extra_body": {"anthropic_beta": ["context-1m-2025-01-15"]},
|
|
"extra_headers": {"X-Proxy-Route": "vertex"},
|
|
"extra_query": {"trace_id": "abc123"},
|
|
},
|
|
)
|
|
|
|
await_args = client.messages.create.await_args
|
|
if await_args is None:
|
|
raise AssertionError("Expected Anthropic client call")
|
|
call = await_args.kwargs
|
|
assert call["extra_body"] == {"anthropic_beta": ["context-1m-2025-01-15"]}
|
|
assert call["extra_headers"] == {"X-Proxy-Route": "vertex"}
|
|
assert call["extra_query"] == {"trace_id": "abc123"}
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_anthropic_backend_stream_forwards_provider_params_passthroughs() -> None:
|
|
"""The stream() path forwards provider_params passthroughs to the SDK the
|
|
same way complete() does — it has its own merge block, so cover it too.
|
|
"""
|
|
|
|
class _FakeStream:
|
|
async def __aenter__(self) -> "_FakeStream":
|
|
return self
|
|
|
|
async def __aexit__(self, *_: object) -> bool:
|
|
return False
|
|
|
|
def __aiter__(self) -> "_FakeStream":
|
|
return self
|
|
|
|
async def __anext__(self) -> object:
|
|
raise StopAsyncIteration
|
|
|
|
async def get_final_message(self) -> SimpleNamespace:
|
|
return SimpleNamespace(
|
|
usage=SimpleNamespace(output_tokens=5),
|
|
stop_reason="end_turn",
|
|
)
|
|
|
|
client = Mock()
|
|
client.messages.stream = Mock(return_value=_FakeStream())
|
|
|
|
backend = AnthropicBackend(client)
|
|
chunks = [
|
|
chunk
|
|
async for chunk in backend.stream(
|
|
model="claude-haiku-4-5",
|
|
messages=[{"role": "user", "content": "Hello"}],
|
|
max_tokens=100,
|
|
extra_params={
|
|
"extra_body": {"anthropic_beta": ["context-1m-2025-01-15"]},
|
|
"extra_headers": {"X-Proxy-Route": "vertex"},
|
|
"extra_query": {"trace_id": "abc123"},
|
|
},
|
|
)
|
|
]
|
|
|
|
assert chunks # the terminal is_done chunk is always emitted
|
|
call = client.messages.stream.call_args
|
|
if call is None:
|
|
raise AssertionError("Expected Anthropic stream call")
|
|
kwargs = call.kwargs
|
|
assert kwargs["extra_body"] == {"anthropic_beta": ["context-1m-2025-01-15"]}
|
|
assert kwargs["extra_headers"] == {"X-Proxy-Route": "vertex"}
|
|
assert kwargs["extra_query"] == {"trace_id": "abc123"}
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_anthropic_backend_ignores_thinking_effort() -> None:
|
|
client = Mock()
|
|
client.messages.create = AsyncMock(
|
|
return_value=SimpleNamespace(
|
|
content=[TextBlock(type="text", text="ok")],
|
|
usage=SimpleNamespace(
|
|
input_tokens=10,
|
|
output_tokens=5,
|
|
cache_creation_input_tokens=0,
|
|
cache_read_input_tokens=0,
|
|
),
|
|
stop_reason="end_turn",
|
|
)
|
|
)
|
|
|
|
backend = AnthropicBackend(client)
|
|
await backend.complete(
|
|
model="claude-haiku-4-5",
|
|
messages=[{"role": "user", "content": "Hello"}],
|
|
max_tokens=100,
|
|
thinking_effort="high",
|
|
)
|
|
|
|
await_args = client.messages.create.await_args
|
|
if await_args is None:
|
|
raise AssertionError("Expected Anthropic client call")
|
|
call = await_args.kwargs
|
|
assert "thinking" not in call
|
|
assert "reasoning_effort" not in call
|
|
|
|
|
|
def _make_client(content_blocks: list[Any]) -> Mock:
|
|
client = Mock()
|
|
client.messages.create = AsyncMock(
|
|
return_value=SimpleNamespace(
|
|
content=content_blocks,
|
|
usage=SimpleNamespace(
|
|
input_tokens=10,
|
|
output_tokens=5,
|
|
cache_creation_input_tokens=0,
|
|
cache_read_input_tokens=0,
|
|
),
|
|
stop_reason="end_turn",
|
|
)
|
|
)
|
|
return client
|
|
|
|
|
|
SEARCH_TOOL = {
|
|
"name": "search",
|
|
"description": "Search for information",
|
|
"input_schema": {
|
|
"type": "object",
|
|
"properties": {"query": {"type": "string"}},
|
|
},
|
|
}
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_anthropic_backend_no_prefill_when_tools_present() -> None:
|
|
"""With tools + response_format, the '{' prefill must be skipped and the
|
|
schema instruction must be conditional, so tool_use blocks stay reachable."""
|
|
client = _make_client([TextBlock(type="text", text='{"answer":"ok"}')])
|
|
|
|
backend = AnthropicBackend(client)
|
|
result = await backend.complete(
|
|
# claude-3-5 supports prefill, so only the tools guard prevents it here
|
|
model="claude-3-5-sonnet-latest",
|
|
messages=[{"role": "user", "content": "Hello"}],
|
|
max_tokens=100,
|
|
tools=[SEARCH_TOOL],
|
|
response_format=StructuredResponse,
|
|
)
|
|
|
|
assert isinstance(result.content, StructuredResponse)
|
|
call = client.messages.create.await_args.kwargs
|
|
assert call["messages"][-1]["role"] == "user" # no assistant '{' prefill
|
|
assert (
|
|
"If not responding with a tool call, respond with valid JSON"
|
|
in call["messages"][0]["content"]
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_anthropic_backend_prefill_unchanged_without_tools() -> None:
|
|
"""Tool-less structured calls keep the prefill + unconditional wording."""
|
|
client = _make_client([TextBlock(type="text", text='"answer":"ok"}')])
|
|
|
|
backend = AnthropicBackend(client)
|
|
result = await backend.complete(
|
|
model="claude-3-5-sonnet-latest",
|
|
messages=[{"role": "user", "content": "Hello"}],
|
|
max_tokens=100,
|
|
response_format=StructuredResponse,
|
|
)
|
|
|
|
assert isinstance(result.content, StructuredResponse)
|
|
call = client.messages.create.await_args.kwargs
|
|
assert call["messages"][-1] == {"role": "assistant", "content": "{"}
|
|
instruction = call["messages"][0]["content"]
|
|
assert "\n\nRespond with valid JSON matching this schema:" in instruction
|
|
assert "If not responding with a tool call" not in instruction
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_anthropic_backend_repairs_malformed_structured_output(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
"""Structured output that fails JSON parsing falls back to
|
|
repair_response_model_json, whose result becomes the response content."""
|
|
# After the '{' prefill is prepended this is still invalid JSON.
|
|
client = _make_client([TextBlock(type="text", text='"answer": not-json')])
|
|
|
|
repaired = StructuredResponse(answer="fixed")
|
|
repair_calls: list[tuple[str, type[BaseModel], str]] = []
|
|
|
|
def _fake_repair(
|
|
raw: str, response_format: type[BaseModel], model_name: str
|
|
) -> StructuredResponse:
|
|
repair_calls.append((raw, response_format, model_name))
|
|
return repaired
|
|
|
|
monkeypatch.setattr(
|
|
"src.llm.backends.anthropic.repair_response_model_json", _fake_repair
|
|
)
|
|
|
|
backend = AnthropicBackend(client)
|
|
result = await backend.complete(
|
|
model="claude-3-5-sonnet-latest",
|
|
messages=[{"role": "user", "content": "Hello"}],
|
|
max_tokens=100,
|
|
response_format=StructuredResponse,
|
|
)
|
|
|
|
assert result.content is repaired
|
|
assert repair_calls == [
|
|
('{"answer": not-json', StructuredResponse, "claude-3-5-sonnet-latest")
|
|
]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_anthropic_backend_skips_parsing_on_tool_call_turns(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
"""A tool-call response with response_format set must not attempt JSON
|
|
parsing — the repair fallback raises on the empty text of tool-call
|
|
turns, which would fail every intermediate tool iteration."""
|
|
client = _make_client(
|
|
[
|
|
ToolUseBlock(
|
|
type="tool_use",
|
|
id="tool_1",
|
|
name="search",
|
|
input={"query": "honcho"},
|
|
)
|
|
]
|
|
)
|
|
|
|
def _fail_repair(*_args: object, **_kwargs: object) -> None:
|
|
raise AssertionError("repair must not run for tool-call turns")
|
|
|
|
monkeypatch.setattr(
|
|
"src.llm.backends.anthropic.repair_response_model_json", _fail_repair
|
|
)
|
|
|
|
backend = AnthropicBackend(client)
|
|
result = await backend.complete(
|
|
model="claude-3-5-sonnet-latest",
|
|
messages=[{"role": "user", "content": "Hello"}],
|
|
max_tokens=100,
|
|
tools=[SEARCH_TOOL],
|
|
response_format=StructuredResponse,
|
|
)
|
|
|
|
assert result.content == "" # raw (empty) text, not a parsed model
|
|
assert result.tool_calls[0].name == "search"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_anthropic_backend_stream_no_prefill_when_tools_present() -> None:
|
|
"""The streaming path applies the same tools guard."""
|
|
|
|
class _FakeStream:
|
|
def __init__(self) -> None:
|
|
self._chunks: list[SimpleNamespace] = [
|
|
SimpleNamespace(
|
|
type="content_block_delta",
|
|
delta=SimpleNamespace(text='{"answer":"ok"}'),
|
|
)
|
|
]
|
|
|
|
async def __aenter__(self):
|
|
return self
|
|
|
|
async def __aexit__(self, *_args: object) -> bool:
|
|
return False
|
|
|
|
def __aiter__(self):
|
|
return self
|
|
|
|
async def __anext__(self):
|
|
if self._chunks:
|
|
return self._chunks.pop(0)
|
|
raise StopAsyncIteration
|
|
|
|
async def get_final_message(self):
|
|
return SimpleNamespace(
|
|
stop_reason="end_turn", usage=SimpleNamespace(output_tokens=5)
|
|
)
|
|
|
|
client = Mock()
|
|
client.messages.stream = Mock(return_value=_FakeStream())
|
|
|
|
backend = AnthropicBackend(client)
|
|
chunks = [
|
|
chunk
|
|
async for chunk in backend.stream(
|
|
model="claude-3-5-sonnet-latest",
|
|
messages=[{"role": "user", "content": "Hello"}],
|
|
max_tokens=100,
|
|
tools=[SEARCH_TOOL],
|
|
response_format=StructuredResponse,
|
|
)
|
|
]
|
|
|
|
assert chunks[0].content == '{"answer":"ok"}'
|
|
call = client.messages.stream.call_args.kwargs
|
|
assert call["messages"][-1]["role"] == "user" # no assistant '{' prefill
|
|
assert (
|
|
"If not responding with a tool call, respond with valid JSON"
|
|
in call["messages"][0]["content"]
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_anthropic_backend_passes_timeout_to_completion_request() -> None:
|
|
"""Anthropic completion requests receive per-request provider timeout."""
|
|
client = Mock()
|
|
client.messages.create = AsyncMock(
|
|
return_value=SimpleNamespace(
|
|
content=[TextBlock(type="text", text="ok")],
|
|
usage=SimpleNamespace(
|
|
input_tokens=10,
|
|
output_tokens=5,
|
|
cache_creation_input_tokens=0,
|
|
cache_read_input_tokens=0,
|
|
),
|
|
stop_reason="end_turn",
|
|
)
|
|
)
|
|
|
|
backend = AnthropicBackend(client)
|
|
await backend.complete(
|
|
model="claude-haiku-4-5",
|
|
messages=[{"role": "user", "content": "Hello"}],
|
|
max_tokens=100,
|
|
extra_params={"timeout": 45},
|
|
)
|
|
|
|
await_args = client.messages.create.await_args
|
|
if await_args is None:
|
|
raise AssertionError("Expected Anthropic create call")
|
|
assert await_args.kwargs["timeout"] == 45.0
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_anthropic_backend_passes_timeout_to_stream_request() -> None:
|
|
"""Anthropic stream requests receive per-request provider timeout."""
|
|
|
|
class FakeStream:
|
|
"""Minimal async stream manager for Anthropic streaming tests."""
|
|
|
|
async def __aenter__(self):
|
|
"""Return the stream object used by the backend."""
|
|
return self
|
|
|
|
async def __aexit__(self, *_args: object) -> bool:
|
|
"""Do not suppress stream errors."""
|
|
return False
|
|
|
|
def __aiter__(self):
|
|
"""Return the async iterator used by the backend."""
|
|
return self
|
|
|
|
async def __anext__(self):
|
|
"""End the fake stream immediately."""
|
|
raise StopAsyncIteration
|
|
|
|
async def get_final_message(self):
|
|
"""Return the final message required by the backend."""
|
|
return SimpleNamespace(
|
|
usage=SimpleNamespace(output_tokens=1),
|
|
stop_reason="end_turn",
|
|
)
|
|
|
|
client = Mock()
|
|
client.messages.stream = Mock(return_value=FakeStream())
|
|
|
|
backend = AnthropicBackend(client)
|
|
chunks = [
|
|
chunk
|
|
async for chunk in backend.stream(
|
|
model="claude-haiku-4-5",
|
|
messages=[{"role": "user", "content": "Hello"}],
|
|
max_tokens=100,
|
|
extra_params={"timeout": "60"},
|
|
)
|
|
]
|
|
|
|
assert chunks[-1].is_done is True
|
|
assert client.messages.stream.call_args.kwargs["timeout"] == 60.0
|