135 lines
4.5 KiB
JSON
135 lines
4.5 KiB
JSON
{
|
|
"description": "Dialectic chat with a response_format JSON Schema while the agent must use tools (reasoning off + enumeration question forces grep/search calls). Exercises the transport-layer combination of tool calling and structured output on every provider: OpenAI must avoid parse() for non-strict tools, Anthropic must skip the '{' prefill, Gemini must fall back to a schema instruction. The final answer must be a JSON string conforming to the schema.",
|
|
"workspace_config": {},
|
|
"steps": [
|
|
{
|
|
"step_type": "create_session",
|
|
"session_id": "structured_output_test",
|
|
"config": {
|
|
"reasoning": {
|
|
"enabled": false
|
|
}
|
|
},
|
|
"peer_configs": {
|
|
"user": {
|
|
"observe_me": true,
|
|
"observe_others": false
|
|
},
|
|
"assistant": {
|
|
"observe_me": false,
|
|
"observe_others": true
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"step_type": "add_messages",
|
|
"session_id": "structured_output_test",
|
|
"messages": [
|
|
{
|
|
"peer_id": "user",
|
|
"content": "Monday I grabbed a $5 latte at Starbucks before my standup.",
|
|
"created_at": "2024-03-04T08:30:00"
|
|
},
|
|
{
|
|
"peer_id": "assistant",
|
|
"content": "Nice, a classic way to start the week.",
|
|
"created_at": "2024-03-04T08:31:00"
|
|
},
|
|
{
|
|
"peer_id": "user",
|
|
"content": "Tuesday I tried a $4 cold brew from Blue Bottle, really smooth.",
|
|
"created_at": "2024-03-05T09:15:00"
|
|
},
|
|
{
|
|
"peer_id": "assistant",
|
|
"content": "Blue Bottle makes a solid cold brew.",
|
|
"created_at": "2024-03-05T09:16:00"
|
|
},
|
|
{
|
|
"peer_id": "user",
|
|
"content": "Wednesday was a $6 oat-milk mocha at a little place downtown.",
|
|
"created_at": "2024-03-06T08:45:00"
|
|
},
|
|
{
|
|
"peer_id": "assistant",
|
|
"content": "Oat milk mochas are underrated.",
|
|
"created_at": "2024-03-06T08:46:00"
|
|
},
|
|
{
|
|
"peer_id": "user",
|
|
"content": "Thursday I skipped coffee and just had tea at home.",
|
|
"created_at": "2024-03-07T08:20:00"
|
|
},
|
|
{
|
|
"peer_id": "assistant",
|
|
"content": "A calm morning, sounds good.",
|
|
"created_at": "2024-03-07T08:21:00"
|
|
},
|
|
{
|
|
"peer_id": "user",
|
|
"content": "Friday I splurged on a $7 pour-over at the roastery near the office.",
|
|
"created_at": "2024-03-08T08:50:00"
|
|
},
|
|
{
|
|
"peer_id": "assistant",
|
|
"content": "Ending the week strong!",
|
|
"created_at": "2024-03-08T08:51:00"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"step_type": "wait",
|
|
"target": "queue_empty",
|
|
"timeout": 180
|
|
},
|
|
{
|
|
"step_type": "query",
|
|
"description": "Global query + empty prefetch forces tool calls; response_format forces structured output on the same LLM calls",
|
|
"target": "chat",
|
|
"observer_peer_id": "assistant",
|
|
"observed_peer_id": "user",
|
|
"reasoning_level": "max",
|
|
"input": "How many separate coffees did I buy this week, and exactly how much did I spend in total across all of them?",
|
|
"response_format": {
|
|
"type": "object",
|
|
"properties": {
|
|
"coffee_count": {
|
|
"type": "integer",
|
|
"description": "How many separate coffees the user bought during the week"
|
|
},
|
|
"total_spent_usd": {
|
|
"type": "number",
|
|
"description": "Total amount in US dollars the user spent on coffee"
|
|
},
|
|
"purchases": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string"
|
|
},
|
|
"description": "One short entry per coffee purchase, including its price"
|
|
},
|
|
"summary": {
|
|
"type": "string",
|
|
"description": "One-sentence answer to the question"
|
|
}
|
|
},
|
|
"required": ["coffee_count", "total_spent_usd", "purchases", "summary"]
|
|
},
|
|
"assertions": [
|
|
{
|
|
"assertion_type": "json_match",
|
|
"key_value_pairs": {
|
|
"coffee_count": 4,
|
|
"total_spent_usd": 22
|
|
}
|
|
},
|
|
{
|
|
"assertion_type": "llm_judge",
|
|
"prompt": "The result must be a JSON object whose 'purchases' array enumerates the $5 latte, $4 cold brew, $6 mocha, and $7 pour-over (wording may vary), and whose 'summary' answers that the user bought 4 coffees for $22 total.",
|
|
"pass_if": true
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|