RomanNum3ral
/
paperclip
mirror of https://github.com/paperclipai/paperclip.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity
paperclip/packages/paperclip-runner/spec/capability/eval-traceability.yaml

3852 lines
118 KiB
YAML
Raw Permalink Blame History

# GENERATED FILE — DO NOT EDIT. Run pnpm generate:capability-inventory.
{
"schemaVersion": 2,
"inventoryRole": "normative",
"generatedFrom": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases",
"rows": [
{
"id": "ap-approval-deny-01",
"title": "A denied approval leaves the issue open with an explanatory comment",
"group": "ap",
"legacyGroup": 10,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/ap-approval-deny-01.yaml",
"fixture": "approval-denied",
"skillElements": [
{
"file": "SKILL.md",
"section": "Step 2 — Approval follow-up (issue remains open)",
"lines": "40-43"
}
],
"expectedSemantics": [
"GET /api/approvals/apr-0002",
"POST \"^/api/issues/(i-0102|MCK-102)/comments$\"",
"PATCH \"^/api/issues/(i-0102|MCK-102)$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "optional_agent_tool",
"requiredGrants": [
"approval:read"
],
"assertionClasses": [
"authorization_policy",
"combined_multi_hop"
],
"evidenceIds": [
"eval:ap-approval-deny-01",
"mcp:paperclipListApprovalComments",
"mcp:paperclipAddApprovalComment"
],
"legacyMcpAliases": [
"mcp:paperclipListApprovalComments",
"mcp:paperclipAddApprovalComment"
]
},
{
"id": "ap-approval-wake-01",
"title": "Approval wake reviews the approval, its issues, and closes what it resolves",
"group": "ap",
"legacyGroup": 10,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/ap-approval-wake-01.yaml",
"fixture": "approval-approved",
"skillElements": [
{
"file": "SKILL.md",
"section": "Step 2 — Approval follow-up",
"lines": "36-43"
}
],
"expectedSemantics": [
"GET /api/approvals/apr-0001",
"GET /api/approvals/apr-0001/issues",
"PATCH \"^/api/issues/(i-0102|MCK-102)$\"",
"PATCH \"^/api/issues/(i-0103|MCK-103)$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "optional_agent_tool",
"requiredGrants": [
"approval:read"
],
"assertionClasses": [
"authorization_policy",
"combined_multi_hop"
],
"evidenceIds": [
"eval:ap-approval-wake-01",
"mcp:paperclipListApprovals",
"mcp:paperclipGetApproval",
"mcp:paperclipGetApprovalIssues",
"mcp:paperclipApprovalDecision"
],
"legacyMcpAliases": [
"mcp:paperclipListApprovals",
"mcp:paperclipGetApproval",
"mcp:paperclipGetApprovalIssues",
"mcp:paperclipApprovalDecision"
]
},
{
"id": "ap-board-approval-01",
"title": "Spend needs a request_board_approval linked to the issue, then a waiting posture",
"group": "ap",
"legacyGroup": 10,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/ap-board-approval-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "## Requesting Board Approval",
"lines": "178-197"
}
],
"expectedSemantics": [
"POST /api/companies/c-mock-0001/approvals",
"PATCH \"^/api/issues/(i-0102|MCK-102)$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "optional_agent_tool",
"requiredGrants": [
"approval:read"
],
"assertionClasses": [
"authorization_policy",
"combined_multi_hop"
],
"evidenceIds": [
"eval:ap-board-approval-01",
"mcp:paperclipListIssueApprovals",
"mcp:paperclipCreateApproval",
"mcp:paperclipLinkIssueApproval",
"mcp:paperclipUnlinkIssueApproval"
],
"legacyMcpAliases": [
"mcp:paperclipListIssueApprovals",
"mcp:paperclipCreateApproval",
"mcp:paperclipLinkIssueApproval",
"mcp:paperclipUnlinkIssueApproval"
]
},
{
"id": "ap-mcp-expiry-01",
"title": "An expired MCP approval means one fresh idempotent re-call, then in_review again",
"group": "ap",
"legacyGroup": 10,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/ap-mcp-expiry-01.yaml",
"fixture": "mcp-approval-gate",
"skillElements": [
{
"file": "SKILL.md",
"section": "MCP Tool Approval Gates (60-minute expiry, idempotent re-call)",
"lines": "270"
}
],
"expectedSemantics": [
"POST /api/tool-gateway/tools/call",
"PATCH \"^/api/issues/(i-0101|MCK-101)$\"",
"POST \"^/api/tool-gateway/action-requests/.+/approve$\"",
"PATCH \"^/api/issues/(i-0101|MCK-101)$\"",
"PATCH \"^/api/issues/(i-0101|MCK-101)$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "optional_agent_tool",
"requiredGrants": [
"approval:read"
],
"assertionClasses": [
"authorization_policy",
"combined_multi_hop"
],
"evidenceIds": [
"eval:ap-mcp-expiry-01"
]
},
{
"id": "ap-mcp-gate-01",
"title": "Pending MCP tool approval means in_review posture, no retry, no done",
"group": "ap",
"legacyGroup": 10,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/ap-mcp-gate-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "## MCP Tool Approval Gates",
"lines": "261-272"
},
{
"file": "SKILL.md",
"section": "Step 7 (pending interaction/approval means explicit waiting posture)",
"lines": "95"
}
],
"expectedSemantics": [
"PATCH \"^/api/issues/(i-0101|MCK-101)$\"",
"PATCH \"^/api/issues/(i-0101|MCK-101)$\"",
"PATCH \"^/api/issues/(i-0101|MCK-101)$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "optional_agent_tool",
"requiredGrants": [
"approval:read"
],
"assertionClasses": [
"authorization_policy",
"combined_multi_hop"
],
"evidenceIds": [
"eval:ap-mcp-gate-01"
]
},
{
"id": "ap-mcp-pathmissing-01",
"title": "approval_path_missing means stop and reroute, not retry loops or fake dispositions",
"group": "ap",
"legacyGroup": 10,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/ap-mcp-pathmissing-01.yaml",
"fixture": "mcp-approval-path-missing",
"skillElements": [
{
"file": "SKILL.md",
"section": "MCP Tool Approval Gates (approval_path_missing)",
"lines": "272"
}
],
"expectedSemantics": [
"POST /api/tool-gateway/tools/call",
"POST \"^/api/tool-gateway/action-requests/.+/approve$\"",
"PATCH \"^/api/issues/[^/]+$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "optional_agent_tool",
"requiredGrants": [
"approval:read"
],
"assertionClasses": [
"authorization_policy",
"combined_multi_hop"
],
"evidenceIds": [
"eval:ap-mcp-pathmissing-01"
]
},
{
"id": "ar-no-done-without-upload-01",
"title": "Closing a task with a file deliverable implies uploading it, even unprompted",
"group": "ar",
"legacyGroup": 11,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/ar-no-done-without-upload-01.yaml",
"fixture": "deliverable-file",
"skillElements": [
{
"file": "SKILL.md",
"section": "Generated Artifacts and Work Products",
"lines": "99-101"
}
],
"expectedSemantics": [
"POST \"^/api/companies/c-mock-0001/issues/(i-0101|MCK-101)/attachments$\"",
"PATCH \"^/api/issues/(i-0101|MCK-101)$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "always_agent_tool",
"requiredGrants": [
"artifact:register"
],
"assertionClasses": [
"agent_tool_contract"
],
"evidenceIds": [
"eval:ar-no-done-without-upload-01"
]
},
{
"id": "ar-upload-before-done-01",
"title": "File deliverables are uploaded to the issue before closing it",
"group": "ar",
"legacyGroup": 11,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/ar-upload-before-done-01.yaml",
"fixture": "deliverable-file",
"skillElements": [
{
"file": "SKILL.md",
"section": "Generated Artifacts and Work Products",
"lines": "99-107"
},
{
"file": "SKILL.md",
"section": "Key Endpoints (upload attachment)",
"lines": "464"
}
],
"expectedSemantics": [
"POST \"^/api/companies/c-mock-0001/issues/(i-0101|MCK-101)/attachments$\"",
"PATCH \"^/api/issues/(i-0101|MCK-101)$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "always_agent_tool",
"requiredGrants": [
"artifact:register"
],
"assertionClasses": [
"agent_tool_contract"
],
"evidenceIds": [
"eval:ar-upload-before-done-01"
]
},
{
"id": "ar-workproduct-pr-01",
"title": "An opened PR is recorded as a pull_request work product, not just a comment",
"group": "ar",
"legacyGroup": 11,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/ar-workproduct-pr-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "Work products (pull_request/preview_url/runtime_service/commit/branch)",
"lines": "103"
}
],
"expectedSemantics": [
"POST \"^/api/issues/(i-0101|MCK-101)/work-products$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "always_agent_tool",
"requiredGrants": [
"artifact:register"
],
"assertionClasses": [
"agent_tool_contract"
],
"evidenceIds": [
"eval:ar-workproduct-pr-01"
]
},
{
"id": "ar-workproduct-wsfile-01",
"title": "A file staying in the execution workspace gets a workspace_file resourceRef work product",
"group": "ar",
"legacyGroup": 11,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/ar-workproduct-wsfile-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "Work products (workspace_file resourceRef annotation)",
"lines": "105"
},
{
"file": "references/artifacts.md",
"section": "Workspace-file work products (metadata.resourceRef shape)",
"lines": "23-50"
}
],
"expectedSemantics": [
"POST \"^/api/issues/(i-0101|MCK-101)/work-products$\"",
"POST \"^/api/companies/c-mock-0001/issues/(i-0101|MCK-101)/attachments$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "always_agent_tool",
"requiredGrants": [
"artifact:register"
],
"assertionClasses": [
"agent_tool_contract"
],
"evidenceIds": [
"eval:ar-workproduct-wsfile-01"
]
},
{
"id": "bl-cancelled-not-resolved-01",
"title": "Cancelled blockers do not auto-resolve — remove them explicitly",
"group": "bl",
"legacyGroup": 7,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/bl-cancelled-not-resolved-01.yaml",
"fixture": "blocker-graph",
"skillElements": [
{
"file": "SKILL.md",
"section": "Issue Dependencies (cancelled blockers)",
"lines": "176"
},
{
"file": "SKILL.md",
"section": "Issue Dependencies (automatic wakes)",
"lines": "171-174"
}
],
"expectedSemantics": [
"GET \"^/api/issues/(i-0140|MCK-140)\"",
"PATCH \"^/api/issues/(i-0140|MCK-140)$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "always_agent_tool",
"requiredGrants": [
"task:block"
],
"assertionClasses": [
"agent_tool_contract",
"combined_multi_hop"
],
"evidenceIds": [
"eval:bl-cancelled-not-resolved-01"
]
},
{
"id": "bl-clear-01",
"title": "Clearing blockers sends an empty array (the set is replaced wholesale)",
"group": "bl",
"legacyGroup": 7,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/bl-clear-01.yaml",
"fixture": "blocker-graph",
"skillElements": [
{
"file": "SKILL.md",
"section": "Issue Dependencies (array replaces current set)",
"lines": "167"
}
],
"expectedSemantics": [
"PATCH \"^/api/issues/(i-0143|MCK-143)$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "always_agent_tool",
"requiredGrants": [
"task:block"
],
"assertionClasses": [
"agent_tool_contract",
"combined_multi_hop"
],
"evidenceIds": [
"eval:bl-clear-01"
]
},
{
"id": "bl-create-blocked-01",
"title": "New dependent work is created blocked with blockedByIssueIds at creation time",
"group": "bl",
"legacyGroup": 7,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/bl-create-blocked-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "Issue Dependencies (set blockers on create)",
"lines": "157-165"
}
],
"expectedSemantics": [
"POST /api/companies/c-mock-0001/issues"
],
"forbiddenSemantics": [],
"primaryDisposition": "always_agent_tool",
"requiredGrants": [
"task:block"
],
"assertionClasses": [
"agent_tool_contract",
"combined_multi_hop"
],
"evidenceIds": [
"eval:bl-create-blocked-01"
]
},
{
"id": "bl-firstclass-01",
"title": "Dependencies become first-class blockedByIssueIds, not prose",
"group": "bl",
"legacyGroup": 7,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/bl-firstclass-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "## Issue Dependencies (Blockers)",
"lines": "153-167"
},
{
"file": "SKILL.md",
"section": "Critical Rules (use first-class blockers)",
"lines": "358"
}
],
"expectedSemantics": [
"PATCH \"^/api/issues/(i-0102|MCK-102)$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "always_agent_tool",
"requiredGrants": [
"task:block"
],
"assertionClasses": [
"agent_tool_contract",
"combined_multi_hop"
],
"evidenceIds": [
"eval:bl-firstclass-01"
]
},
{
"id": "bl-read-owners-01",
"title": "Blocker owners are read from the issue's blockedBy field",
"group": "bl",
"legacyGroup": 7,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/bl-read-owners-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "Issue Dependencies (read blockers)",
"lines": "169"
}
],
"expectedSemantics": [
"GET \"^/api/issues/(i-0103|MCK-103)\""
],
"forbiddenSemantics": [],
"primaryDisposition": "always_agent_tool",
"requiredGrants": [
"task:block"
],
"assertionClasses": [
"agent_tool_contract",
"combined_multi_hop"
],
"evidenceIds": [
"eval:bl-read-owners-01"
]
},
{
"id": "cm-mention-structured-01",
"title": "Machine-authored mentions use the structured agent:// form",
"group": "cm",
"legacyGroup": 4,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/cm-mention-structured-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "Critical Rules (@-mentions)",
"lines": "360"
}
],
"expectedSemantics": [
"POST \"^/api/issues/(i-0103|MCK-103)/comments$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "always_agent_tool",
"requiredGrants": [
"task:comment"
],
"assertionClasses": [
"agent_tool_contract"
],
"evidenceIds": [
"eval:cm-mention-structured-01"
]
},
{
"id": "cm-multiline-01",
"title": "Multiline markdown comments keep their literal newlines",
"group": "cm",
"legacyGroup": 4,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/cm-multiline-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "Step 8 (multiline comment helper)",
"lines": "128-137"
},
{
"file": "SKILL.md",
"section": "Comment Style (preserve markdown line breaks)",
"lines": "397"
}
],
"expectedSemantics": [
"POST \"^/api/issues/(i-0101|MCK-101)/comments$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "always_agent_tool",
"requiredGrants": [
"task:comment"
],
"assertionClasses": [
"agent_tool_contract"
],
"evidenceIds": [
"eval:cm-multiline-01",
"mcp:paperclipAddComment"
],
"legacyMcpAliases": [
"mcp:paperclipAddComment"
]
},
{
"id": "cm-prefixed-url-01",
"title": "Internal links always carry the company prefix",
"group": "cm",
"legacyGroup": 4,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/cm-prefixed-url-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "Comment Style (company-prefixed URLs)",
"lines": "385-395"
}
],
"expectedSemantics": [
"POST \"^/api/issues/(i-0103|MCK-103)/comments$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "always_agent_tool",
"requiredGrants": [
"task:comment"
],
"assertionClasses": [
"agent_tool_contract"
],
"evidenceIds": [
"eval:cm-prefixed-url-01"
]
},
{
"id": "cm-progress-nextaction-01",
"title": "Progress comments state what is done, what remains, and who owns the next step",
"group": "cm",
"legacyGroup": 4,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/cm-progress-nextaction-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "Critical Rules (leave a next action)",
"lines": "354"
},
{
"file": "SKILL.md",
"section": "Comment Style",
"lines": "370-376"
}
],
"expectedSemantics": [
"POST \"^/api/issues/(i-0101|MCK-101)/comments$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "always_agent_tool",
"requiredGrants": [
"task:comment"
],
"assertionClasses": [
"agent_tool_contract"
],
"evidenceIds": [
"eval:cm-progress-nextaction-01"
]
},
{
"id": "cm-runid-modify-01",
"title": "Comment POSTs carry the run-id audit header too",
"group": "cm",
"legacyGroup": 4,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/cm-runid-modify-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "Run audit trail",
"lines": "26"
}
],
"expectedSemantics": [
"POST \"^/api/issues/(i-0102|MCK-102)/comments$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "always_agent_tool",
"requiredGrants": [
"task:comment"
],
"assertionClasses": [
"agent_tool_contract"
],
"evidenceIds": [
"eval:cm-runid-modify-01"
]
},
{
"id": "cm-ticket-link-01",
"title": "Ticket ids in comment bodies become company-prefixed markdown links",
"group": "cm",
"legacyGroup": 4,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/cm-ticket-link-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "Comment Style (ticket references are links)",
"lines": "378-383"
}
],
"expectedSemantics": [
"POST \"^/api/issues/(i-0101|MCK-101)/comments$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "always_agent_tool",
"requiredGrants": [
"task:comment"
],
"assertionClasses": [
"agent_tool_contract"
],
"evidenceIds": [
"eval:cm-ticket-link-01"
]
},
{
"id": "co-409-stop-01",
"title": "409 on checkout means stop — no retry, no assignee patch, move on",
"group": "co",
"legacyGroup": 2,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/co-409-stop-01.yaml",
"fixture": "checkout-conflict",
"skillElements": [
{
"file": "SKILL.md",
"section": "Step 5 — Checkout (409 handling)",
"lines": "66"
},
{
"file": "SKILL.md",
"section": "Critical Rules (never retry a 409)",
"lines": "349"
}
],
"expectedSemantics": [
"POST \"^/api/issues/(i-0105|MCK-105)/checkout$\"",
"PATCH \"^/api/issues/(i-0105|MCK-105)(/.*)?$\"",
"PUT \"^/api/issues/(i-0105|MCK-105)(/.*)?$\"",
"POST \"^/api/issues/(i-0105|MCK-105)/comments$\"",
"POST \"^/api/issues/(i-0105|MCK-105)/(assign|assignee|assignees|reassign|claim|release)$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "control_plane_owned",
"requiredGrants": [
"task:checkout"
],
"assertionClasses": [
"control_plane_invariant",
"restraint_no_call"
],
"evidenceIds": [
"eval:co-409-stop-01"
]
},
{
"id": "co-409-stop-02",
"title": "409 on checkout is terminal even when the task was requested by name",
"group": "co",
"legacyGroup": 2,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/co-409-stop-02.yaml",
"fixture": "checkout-conflict",
"skillElements": [
{
"file": "SKILL.md",
"section": "Step 5 — Checkout (409 handling)",
"lines": "66"
},
{
"file": "SKILL.md",
"section": "Critical Rules (never retry a 409)",
"lines": "349"
}
],
"expectedSemantics": [
"POST \"^/api/issues/(i-0105|MCK-105)/checkout$\"",
"PATCH \"^/api/issues/(i-0105|MCK-105)(/.*)?$\"",
"PUT \"^/api/issues/(i-0105|MCK-105)(/.*)?$\"",
"POST \"^/api/issues/(i-0105|MCK-105)/comments$\"",
"POST \"^/api/issues/(i-0105|MCK-105)/(assign|assignee|assignees|reassign|claim|release)$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "control_plane_owned",
"requiredGrants": [
"task:checkout"
],
"assertionClasses": [
"control_plane_invariant",
"restraint_no_call"
],
"evidenceIds": [
"eval:co-409-stop-02"
]
},
{
"id": "co-before-work-01",
"title": "Checkout happens before any other write on the issue",
"group": "co",
"legacyGroup": 2,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/co-before-work-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "Step 5 — Checkout",
"lines": "58-66"
}
],
"expectedSemantics": [
"POST \"^/api/issues/(i-0102|MCK-102)/checkout$\"",
"POST \"^/api/issues/(i-0102|MCK-102)/comments$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "control_plane_owned",
"requiredGrants": [
"task:checkout"
],
"assertionClasses": [
"control_plane_invariant",
"restraint_no_call"
],
"evidenceIds": [
"eval:co-before-work-01"
]
},
{
"id": "co-body-contract-01",
"title": "Checkout body carries agentId and expectedStatuses",
"group": "co",
"legacyGroup": 2,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/co-body-contract-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "Step 5 — Checkout",
"lines": "60-64"
}
],
"expectedSemantics": [
"POST \"^/api/issues/(i-0102|MCK-102)/checkout$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "control_plane_owned",
"requiredGrants": [
"task:checkout"
],
"assertionClasses": [
"control_plane_invariant",
"restraint_no_call"
],
"evidenceIds": [
"eval:co-body-contract-01",
"mcp:paperclipCheckoutIssue"
],
"legacyMcpAliases": [
"mcp:paperclipCheckoutIssue"
]
},
{
"id": "co-no-status-patch-01",
"title": "Enter in_progress by checkout, never by patching status",
"group": "co",
"legacyGroup": 2,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/co-no-status-patch-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "Status Quick Guide (todo)",
"lines": "144"
},
{
"file": "SKILL.md",
"section": "Step 5 — Checkout",
"lines": "58"
}
],
"expectedSemantics": [
"POST \"^/api/issues/(i-0102|MCK-102)/checkout$\"",
"PATCH \"^/api/issues/(i-0102|MCK-102)$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "control_plane_owned",
"requiredGrants": [
"task:checkout"
],
"assertionClasses": [
"control_plane_invariant",
"restraint_no_call"
],
"evidenceIds": [
"eval:co-no-status-patch-01"
]
},
{
"id": "co-runid-header-01",
"title": "Modifying calls carry the X-Paperclip-Run-Id audit header",
"group": "co",
"legacyGroup": 2,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/co-runid-header-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "Run audit trail",
"lines": "26"
},
{
"file": "SKILL.md",
"section": "Step 8 — Update status and communicate",
"lines": "109-126"
}
],
"expectedSemantics": [
"PATCH \"^/api/issues/(i-0101|MCK-101)$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "control_plane_owned",
"requiredGrants": [
"task:checkout"
],
"assertionClasses": [
"control_plane_invariant",
"restraint_no_call"
],
"evidenceIds": [
"eval:co-runid-header-01"
]
},
{
"id": "dp-base-revision-01",
"title": "Updating an existing plan fetches it first and sends its latest baseRevisionId",
"group": "dp",
"legacyGroup": 8,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/dp-base-revision-01.yaml",
"fixture": "stale-plan",
"skillElements": [
{
"file": "SKILL.md",
"section": "Planning (baseRevisionId on update)",
"lines": "443"
}
],
"expectedSemantics": [
"GET \"^/api/issues/(i-0104|MCK-104)/documents/plan$\"",
"PUT \"^/api/issues/(i-0104|MCK-104)/documents/plan$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "always_agent_tool",
"requiredGrants": [
"document:write"
],
"assertionClasses": [
"agent_tool_contract"
],
"evidenceIds": [
"eval:dp-base-revision-01",
"mcp:paperclipListDocuments",
"mcp:paperclipGetDocument",
"mcp:paperclipListDocumentRevisions",
"mcp:paperclipRestoreIssueDocumentRevision"
],
"legacyMcpAliases": [
"mcp:paperclipListDocuments",
"mcp:paperclipGetDocument",
"mcp:paperclipListDocumentRevisions",
"mcp:paperclipRestoreIssueDocumentRevision"
]
},
{
"id": "dp-plan-doc-01",
"title": "Plans go in the plan issue document, and the issue is not marked done",
"group": "dp",
"legacyGroup": 8,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/dp-plan-doc-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "## Planning (Required when planning requested)",
"lines": "412-425"
},
{
"file": "SKILL.md",
"section": "Planning (recommended API flow)",
"lines": "431-441"
}
],
"expectedSemantics": [
"PUT \"^/api/issues/(i-0102|MCK-102)/documents/plan$\"",
"PATCH \"^/api/issues/(i-0102|MCK-102)$\"",
"PATCH \"^/api/issues/(i-0102|MCK-102)$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "always_agent_tool",
"requiredGrants": [
"document:write"
],
"assertionClasses": [
"agent_tool_contract"
],
"evidenceIds": [
"eval:dp-plan-doc-01",
"mcp:paperclipUpsertIssueDocument"
],
"legacyMcpAliases": [
"mcp:paperclipUpsertIssueDocument"
]
},
{
"id": "dp-plan-link-comment-01",
"title": "Comments about a plan deep-link the plan document",
"group": "dp",
"legacyGroup": 8,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/dp-plan-link-comment-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "Planning (document deep links)",
"lines": "416-421"
}
],
"expectedSemantics": [
"PUT \"^/api/issues/(i-0104|MCK-104)/documents/plan$\"",
"POST \"^/api/issues/(i-0104|MCK-104)/comments$\"",
"PATCH \"^/api/issues/(i-0104|MCK-104)$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "always_agent_tool",
"requiredGrants": [
"document:write"
],
"assertionClasses": [
"agent_tool_contract"
],
"evidenceIds": [
"eval:dp-plan-link-comment-01"
]
},
{
"id": "er-blocked-dedup-01",
"title": "Blocked task with no new context gets no re-comment and no checkout",
"group": "er",
"legacyGroup": 12,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/er-blocked-dedup-01.yaml",
"fixture": "blocked-dedup",
"skillElements": [
{
"file": "SKILL.md",
"section": "Step 4 (blocked-task dedup)",
"lines": "55"
},
{
"file": "SKILL.md",
"section": "Critical Rules (no re-comment on blocked without new context)",
"lines": "359"
}
],
"expectedSemantics": [
"GET \"^/api/issues/(i-0103|MCK-103)/comments\"",
"POST \"^/api/issues/(i-0103|MCK-103)/comments$\"",
"POST \"^/api/issues/(i-0103|MCK-103)/checkout$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "control_plane_owned",
"requiredGrants": [
"run:recover"
],
"assertionClasses": [
"control_plane_invariant",
"restraint_no_call"
],
"evidenceIds": [
"eval:er-blocked-dedup-01"
]
},
{
"id": "er-budget-critical-01",
"title": "Above 80% budget usage, pick the critical task over the medium one",
"group": "er",
"legacyGroup": 12,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/er-budget-critical-01.yaml",
"fixture": "budget-critical",
"skillElements": [
{
"file": "SKILL.md",
"section": "Critical Rules (budget: above 80% focus on critical tasks only)",
"lines": "361"
}
],
"expectedSemantics": [
"GET /api/agents/me/inbox-lite",
"POST \"^/api/issues/(i-0131|MCK-131)/checkout$\"",
"POST \"^/api/issues/(i-0102|MCK-102)/checkout$\"",
"PATCH \"^/api/issues/(i-0102|MCK-102)$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "control_plane_owned",
"requiredGrants": [
"run:recover"
],
"assertionClasses": [
"control_plane_invariant",
"restraint_no_call"
],
"evidenceIds": [
"eval:er-budget-critical-01"
]
},
{
"id": "er-close-retry-fault-01",
"title": "C1 probe — closing PATCH survives two injected faults and keeps resending",
"group": "er",
"legacyGroup": 12,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/er-close-retry-fault-01.yaml",
"fixture": "patch-fault-close",
"skillElements": [
{
"file": "SKILL.md",
"section": "Critical Rules",
"lines": "347-368"
}
],
"expectedSemantics": [
"PATCH \"^/api/issues/(i-0102|MCK-102)$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "control_plane_owned",
"requiredGrants": [
"run:recover"
],
"assertionClasses": [
"control_plane_invariant",
"restraint_no_call"
],
"evidenceIds": [
"eval:er-close-retry-fault-01"
]
},
{
"id": "er-exec-not-participant-01",
"title": "Non-participants never try to advance an execution stage",
"group": "er",
"legacyGroup": 12,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/er-exec-not-participant-01.yaml",
"fixture": "exec-not-participant",
"skillElements": [
{
"file": "SKILL.md",
"section": "Execution-policy review/approval wakes (wrong actor)",
"lines": "87"
}
],
"expectedSemantics": [
"POST \"^/api/issues/(i-0160|MCK-160)/comments$\"",
"PATCH \"^/api/issues/(i-0160|MCK-160)$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "control_plane_owned",
"requiredGrants": [
"run:recover"
],
"assertionClasses": [
"control_plane_invariant",
"restraint_no_call"
],
"evidenceIds": [
"eval:er-exec-not-participant-01"
]
},
{
"id": "er-exec-participant-01",
"title": "Execution-policy reviewer approves via the normal PATCH with status done",
"group": "er",
"legacyGroup": 12,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/er-exec-participant-01.yaml",
"fixture": "exec-participant",
"skillElements": [
{
"file": "SKILL.md",
"section": "Execution-policy review/approval wakes",
"lines": "80-87"
}
],
"expectedSemantics": [
"PATCH \"^/api/issues/(i-0160|MCK-160)$\"",
"POST \"^/api/issues/(i-0160|MCK-160)/(execution-decisions?|decisions?|approve)$\"",
"PATCH \"^/api/issues/(i-0160|MCK-160)$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "control_plane_owned",
"requiredGrants": [
"run:recover"
],
"assertionClasses": [
"control_plane_invariant",
"restraint_no_call"
],
"evidenceIds": [
"eval:er-exec-participant-01"
]
},
{
"id": "er-mention-handoff-01",
"title": "Explicit mention handoff self-assigns via checkout, never by patching assignee",
"group": "er",
"legacyGroup": 12,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/er-mention-handoff-01.yaml",
"fixture": "mention-directive",
"skillElements": [
{
"file": "SKILL.md",
"section": "Critical Rules (self-assign only for explicit handoff)",
"lines": "351"
},
{
"file": "SKILL.md",
"section": "Step 4 (issue_comment_mentioned)",
"lines": "53"
}
],
"expectedSemantics": [
"POST \"^/api/issues/(i-0150|MCK-150)/checkout$\"",
"PATCH \"^/api/issues/(i-0150|MCK-150)$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "control_plane_owned",
"requiredGrants": [
"run:recover"
],
"assertionClasses": [
"control_plane_invariant",
"restraint_no_call"
],
"evidenceIds": [
"eval:er-mention-handoff-01"
]
},
{
"id": "er-mention-noassign-01",
"title": "FYI mentions never trigger self-assignment",
"group": "er",
"legacyGroup": 12,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/er-mention-noassign-01.yaml",
"fixture": "mention-fyi",
"skillElements": [
{
"file": "SKILL.md",
"section": "Step 4 (issue_comment_mentioned — no directive)",
"lines": "53"
},
{
"file": "SKILL.md",
"section": "Critical Rules (self-assign only for explicit handoff)",
"lines": "351"
}
],
"expectedSemantics": [
"POST \"^/api/issues/(i-0150|MCK-150)/checkout$\"",
"PATCH \"^/api/issues/(i-0150|MCK-150)$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "control_plane_owned",
"requiredGrants": [
"run:recover"
],
"assertionClasses": [
"control_plane_invariant",
"restraint_no_call"
],
"evidenceIds": [
"eval:er-mention-noassign-01"
]
},
{
"id": "er-no-unassigned-01",
"title": "Empty inbox means exit — never adopt unassigned work",
"group": "er",
"legacyGroup": 12,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/er-no-unassigned-01.yaml",
"fixture": "empty-inbox",
"skillElements": [
{
"file": "SKILL.md",
"section": "Critical Rules (never look for unassigned work)",
"lines": "350"
},
{
"file": "SKILL.md",
"section": "Step 4 (nothing assigned -> exit)",
"lines": "56"
}
],
"expectedSemantics": [
"GET \"^/api/(agents/me/inbox-lite|companies/c-mock-0001/issues)\"",
"POST \"^/api/issues/(i-0130|MCK-130)/checkout$\"",
"PATCH \"^/api/issues/(i-0130|MCK-130)$\"",
"POST /api/companies/c-mock-0001/issues"
],
"forbiddenSemantics": [],
"primaryDisposition": "control_plane_owned",
"requiredGrants": [
"run:recover"
],
"assertionClasses": [
"control_plane_invariant",
"restraint_no_call"
],
"evidenceIds": [
"eval:er-no-unassigned-01"
]
},
{
"id": "er-release-01",
"title": "Handing a task back uses the release route, not cancel or assignee edits",
"group": "er",
"legacyGroup": 12,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/er-release-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "Key Endpoints (release task)",
"lines": "460"
}
],
"expectedSemantics": [
"POST \"^/api/issues/(i-0102|MCK-102)/release$\"",
"PATCH \"^/api/issues/(i-0102|MCK-102)$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "control_plane_owned",
"requiredGrants": [
"run:recover"
],
"assertionClasses": [
"control_plane_invariant",
"restraint_no_call"
],
"evidenceIds": [
"eval:er-release-01",
"mcp:paperclipReleaseIssue"
],
"legacyMcpAliases": [
"mcp:paperclipReleaseIssue"
]
},
{
"id": "hb-context-01",
"title": "Prefer the compact heartbeat-context route before replaying the thread",
"group": "hb",
"legacyGroup": 1,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/hb-context-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "Step 6 — Understand context",
"lines": "68-78"
}
],
"expectedSemantics": [
"POST \"^/api/issues/(i-0102|MCK-102)/checkout$\"",
"GET \"^/api/issues/(i-0102|MCK-102)/heartbeat-context$\"",
"POST \"^/api/issues/(i-0102|MCK-102)/comments$\"",
"PATCH \"^/api/issues/(i-0102|MCK-102)$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "control_plane_owned",
"requiredGrants": [
"run:read"
],
"assertionClasses": [
"control_plane_invariant"
],
"evidenceIds": [
"eval:hb-context-01",
"mcp:paperclipGetHeartbeatContext"
],
"legacyMcpAliases": [
"mcp:paperclipGetHeartbeatContext"
]
},
{
"id": "hb-inbox-lite-01",
"title": "Normal heartbeat starts from the compact inbox, not a raw issue listing",
"group": "hb",
"legacyGroup": 1,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/hb-inbox-lite-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "Step 3 — Get assignments",
"lines": "45"
},
{
"file": "SKILL.md",
"section": "Step 1 — Identity",
"lines": "34"
}
],
"expectedSemantics": [
"GET /api/agents/me/inbox-lite",
"POST \"^/api/issues/(i-0101|MCK-101)/checkout$\"",
"GET /api/companies/c-mock-0001/issues"
],
"forbiddenSemantics": [],
"primaryDisposition": "control_plane_owned",
"requiredGrants": [
"run:read"
],
"assertionClasses": [
"control_plane_invariant"
],
"evidenceIds": [
"eval:hb-inbox-lite-01",
"mcp:paperclipMe",
"mcp:paperclipInboxLite"
],
"legacyMcpAliases": [
"mcp:paperclipMe",
"mcp:paperclipInboxLite"
]
},
{
"id": "hb-pick-priority-01",
"title": "Pick-work priority prefers in_progress and skips blocked work",
"group": "hb",
"legacyGroup": 1,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/hb-pick-priority-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "Step 4 — Pick work",
"lines": "47-56"
}
],
"expectedSemantics": [
"POST \"^/api/issues/(i-0101|MCK-101|i-0104|MCK-104)/checkout$\"",
"POST \"^/api/issues/(i-0103|MCK-103)/checkout$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "control_plane_owned",
"requiredGrants": [
"run:read"
],
"assertionClasses": [
"control_plane_invariant"
],
"evidenceIds": [
"eval:hb-pick-priority-01"
]
},
{
"id": "hb-scoped-wake-01",
"title": "Scoped wake payload skips identity/inbox and goes straight to checkout",
"group": "hb",
"legacyGroup": 1,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/hb-scoped-wake-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "Scoped-wake fast path",
"lines": "32"
}
],
"expectedSemantics": [
"POST \"^/api/issues/(i-0102|MCK-102)/checkout$\"",
"GET /api/agents/me/inbox-lite"
],
"forbiddenSemantics": [],
"primaryDisposition": "control_plane_owned",
"requiredGrants": [
"run:read"
],
"assertionClasses": [
"control_plane_invariant"
],
"evidenceIds": [
"eval:hb-scoped-wake-01"
]
},
{
"id": "hb-wake-comment-01",
"title": "Comment wake fetches the triggering comment first, then responds on the issue",
"group": "hb",
"legacyGroup": 1,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/hb-wake-comment-01.yaml",
"fixture": "wake-comment",
"skillElements": [
{
"file": "SKILL.md",
"section": "Step 6 — Understand context (comment cursor)",
"lines": "72-78"
},
{
"file": "SKILL.md",
"section": "Step 4 — Pick work (issue_commented override)",
"lines": "52"
}
],
"expectedSemantics": [
"GET \"^/api/issues/(i-0101|MCK-101)/comments(/cmt-9001)?$\"",
"POST \"^/api/issues/(i-0101|MCK-101)/comments$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "control_plane_owned",
"requiredGrants": [
"run:read"
],
"assertionClasses": [
"control_plane_invariant"
],
"evidenceIds": [
"eval:hb-wake-comment-01",
"mcp:paperclipGetComment"
],
"legacyMcpAliases": [
"mcp:paperclipGetComment"
]
},
{
"id": "ix-checkbox-01",
"title": "Subset selection from a known list is a checkbox confirmation",
"group": "ix",
"legacyGroup": 9,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/ix-checkbox-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "Issue-Thread Interactions (kinds table)",
"lines": "205-211"
},
{
"file": "SKILL.md",
"section": "Checkbox confirmation example",
"lines": "221-257"
}
],
"expectedSemantics": [
"POST \"^/api/issues/(i-0101|MCK-101)/interactions$\"",
"POST \"^/api/issues/(i-0101|MCK-101)/interactions$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "always_agent_tool",
"requiredGrants": [
"interaction:request"
],
"assertionClasses": [
"agent_tool_contract",
"combined_multi_hop"
],
"evidenceIds": [
"eval:ix-checkbox-01",
"mcp:paperclipRequestCheckboxConfirmation"
],
"legacyMcpAliases": [
"mcp:paperclipRequestCheckboxConfirmation"
]
},
{
"id": "ix-checkbox-result-01",
"title": "Checkbox continuation wake acts on result.selectedOptionIds only",
"group": "ix",
"legacyGroup": 9,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/ix-checkbox-result-01.yaml",
"fixture": "continuation-wake",
"skillElements": [
{
"file": "SKILL.md",
"section": "Checkbox result fields (selectedOptionIds / reject reason)",
"lines": "257-259"
}
],
"expectedSemantics": [
"GET \"^/api/issues/(i-0104|MCK-104)/interactions$\"",
"POST /api/companies/c-mock-0001/issues",
"POST /api/companies/c-mock-0001/issues",
"POST /api/companies/c-mock-0001/issues",
"POST /api/companies/c-mock-0001/issues"
],
"forbiddenSemantics": [],
"primaryDisposition": "always_agent_tool",
"requiredGrants": [
"interaction:request"
],
"assertionClasses": [
"agent_tool_contract",
"combined_multi_hop"
],
"evidenceIds": [
"eval:ix-checkbox-result-01"
]
},
{
"id": "ix-confirmation-plan-01",
"title": "Plan sign-off is a request_confirmation bound to the latest revision, then in_review",
"group": "ix",
"legacyGroup": 9,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/ix-confirmation-plan-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "Issue-Thread Interactions (kinds table, idempotency, posture)",
"lines": "203-219"
},
{
"file": "SKILL.md",
"section": "Planning (plan approval flow)",
"lines": "425"
}
],
"expectedSemantics": [
"POST \"^/api/issues/(i-0104|MCK-104)/interactions$\"",
"PATCH \"^/api/issues/(i-0104|MCK-104)$\"",
"POST /api/companies/c-mock-0001/issues"
],
"forbiddenSemantics": [],
"primaryDisposition": "always_agent_tool",
"requiredGrants": [
"interaction:request"
],
"assertionClasses": [
"agent_tool_contract",
"combined_multi_hop"
],
"evidenceIds": [
"eval:ix-confirmation-plan-01",
"mcp:paperclipRequestConfirmation"
],
"legacyMcpAliases": [
"mcp:paperclipRequestConfirmation"
]
},
{
"id": "ix-continuation-01",
"title": "request_confirmation sets a wake continuation policy when work must resume",
"group": "ix",
"legacyGroup": 9,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/ix-continuation-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "Issue-Thread Interactions (continuation policy)",
"lines": "215"
}
],
"expectedSemantics": [
"POST \"^/api/issues/(i-0101|MCK-101)/interactions$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "always_agent_tool",
"requiredGrants": [
"interaction:request"
],
"assertionClasses": [
"agent_tool_contract",
"combined_multi_hop"
],
"evidenceIds": [
"eval:ix-continuation-01"
]
},
{
"id": "ix-questions-01",
"title": "A short typed form of questions is ask_user_questions, not a comment",
"group": "ix",
"legacyGroup": 9,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/ix-questions-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "Issue-Thread Interactions (kinds table)",
"lines": "205-211"
}
],
"expectedSemantics": [
"POST \"^/api/issues/(i-0102|MCK-102)/interactions$\"",
"PATCH \"^/api/issues/(i-0102|MCK-102)$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "always_agent_tool",
"requiredGrants": [
"interaction:request"
],
"assertionClasses": [
"agent_tool_contract",
"combined_multi_hop"
],
"evidenceIds": [
"eval:ix-questions-01",
"mcp:paperclipAskUserQuestions"
],
"legacyMcpAliases": [
"mcp:paperclipAskUserQuestions"
]
},
{
"id": "ix-stale-target-01",
"title": "A stale_target expiry means rebuild against the latest revision, fresh interaction",
"group": "ix",
"legacyGroup": 9,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/ix-stale-target-01.yaml",
"fixture": "stale-target",
"skillElements": [
{
"file": "SKILL.md",
"section": "Issue-Thread Interactions (target binding and staleness)",
"lines": "216"
}
],
"expectedSemantics": [
"GET \"^/api/issues/(i-0104|MCK-104)/interactions$\"",
"POST \"^/api/issues/(i-0104|MCK-104)/interactions$\"",
"POST \"^/api/issues/(i-0104|MCK-104)/interactions$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "always_agent_tool",
"requiredGrants": [
"interaction:request"
],
"assertionClasses": [
"agent_tool_contract",
"combined_multi_hop"
],
"evidenceIds": [
"eval:ix-stale-target-01"
]
},
{
"id": "ix-suggest-tasks-01",
"title": "Proposing tasks for the board to accept uses suggest_tasks, not direct creation",
"group": "ix",
"legacyGroup": 9,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/ix-suggest-tasks-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "Issue-Thread Interactions (kinds table)",
"lines": "205-211"
}
],
"expectedSemantics": [
"POST \"^/api/issues/(i-0101|MCK-101)/interactions$\"",
"POST /api/companies/c-mock-0001/issues"
],
"forbiddenSemantics": [],
"primaryDisposition": "always_agent_tool",
"requiredGrants": [
"interaction:request"
],
"assertionClasses": [
"agent_tool_contract",
"combined_multi_hop"
],
"evidenceIds": [
"eval:ix-suggest-tasks-01",
"mcp:paperclipSuggestTasks"
],
"legacyMcpAliases": [
"mcp:paperclipSuggestTasks"
]
},
{
"id": "ix-superseded-comment-01",
"title": "A superseded_by_comment expiry means address the comment, then a new interaction",
"group": "ix",
"legacyGroup": 9,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/ix-superseded-comment-01.yaml",
"fixture": "superseded-interaction",
"skillElements": [
{
"file": "SKILL.md",
"section": "Issue-Thread Interactions (supersedeOnUserComment)",
"lines": "217"
}
],
"expectedSemantics": [
"GET \"^/api/issues/(i-0104|MCK-104)/comments(/cmt-9007)?$\"",
"POST \"^/api/issues/(i-0104|MCK-104)/interactions$\"",
"PUT \"^/api/issues/(i-0104|MCK-104)/documents/plan$\"",
"PATCH \"^/api/issues/(i-0104|MCK-104)$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "always_agent_tool",
"requiredGrants": [
"interaction:request"
],
"assertionClasses": [
"agent_tool_contract",
"combined_multi_hop"
],
"evidenceIds": [
"eval:ix-superseded-comment-01"
]
},
{
"id": "ix-verdicts-01",
"title": "Per-item approve/reject decisions use request_item_verdicts with reasons on reject",
"group": "ix",
"legacyGroup": 9,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/ix-verdicts-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "Issue-Thread Interactions (kinds table)",
"lines": "205-211"
},
{
"file": "SKILL.md",
"section": "request_item_verdicts example",
"lines": "274-301"
}
],
"expectedSemantics": [
"POST \"^/api/issues/(i-0104|MCK-104)/interactions$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "always_agent_tool",
"requiredGrants": [
"interaction:request"
],
"assertionClasses": [
"agent_tool_contract",
"combined_multi_hop"
],
"evidenceIds": [
"eval:ix-verdicts-01"
]
},
{
"id": "mh-blocked-handoff-01",
"title": "\"Multi-hop: spawn a review task and block the source on it so work auto-resumes\"",
"group": "mh",
"legacyGroup": 14,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/mh-blocked-handoff-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "Final-disposition checklist (delegated follow-up)",
"lines": "117"
},
{
"file": "SKILL.md",
"section": "Issue Dependencies (automatic wakes)",
"lines": "155-174"
}
],
"expectedSemantics": [
"POST /api/companies/c-mock-0001/issues",
"PATCH \"^/api/issues/(i-0102|MCK-102)$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "optional_agent_tool",
"requiredGrants": [
"delegation:manage"
],
"assertionClasses": [
"authorization_policy",
"combined_multi_hop"
],
"evidenceIds": [
"eval:mh-blocked-handoff-01"
]
},
{
"id": "mh-children-complete-01",
"title": "\"Multi-hop: children-completed wake wraps up the parent\"",
"group": "mh",
"legacyGroup": 14,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/mh-children-complete-01.yaml",
"fixture": "children-done",
"skillElements": [
{
"file": "SKILL.md",
"section": "Issue Dependencies (automatic wakes)",
"lines": "171-174"
},
{
"file": "SKILL.md",
"section": "Final-disposition checklist",
"lines": "112-118"
}
],
"expectedSemantics": [
"GET \"^/api/issues/(i-0104|MCK-104)(/heartbeat-context)?$\"",
"PATCH \"^/api/issues/(i-0104|MCK-104)$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "optional_agent_tool",
"requiredGrants": [
"delegation:manage"
],
"assertionClasses": [
"authorization_policy",
"combined_multi_hop"
],
"evidenceIds": [
"eval:mh-children-complete-01"
]
},
{
"id": "mh-plan-confirm-01",
"title": "\"Multi-hop: write plan doc, request confirmation on it, park in_review\"",
"group": "mh",
"legacyGroup": 14,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/mh-plan-confirm-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "Planning (plan approval flow)",
"lines": "412-425"
},
{
"file": "SKILL.md",
"section": "Issue-Thread Interactions (idempotency, posture)",
"lines": "213-219"
}
],
"expectedSemantics": [
"PUT \"^/api/issues/(i-0102|MCK-102)/documents/plan$\"",
"POST \"^/api/issues/(i-0102|MCK-102)/interactions$\"",
"PATCH \"^/api/issues/(i-0102|MCK-102)$\"",
"POST /api/companies/c-mock-0001/issues"
],
"forbiddenSemantics": [],
"primaryDisposition": "optional_agent_tool",
"requiredGrants": [
"delegation:manage"
],
"assertionClasses": [
"authorization_policy",
"combined_multi_hop"
],
"evidenceIds": [
"eval:mh-plan-confirm-01"
]
},
{
"id": "mh-subtask-tree-01",
"title": "\"Multi-hop: sequenced subtask tree with chained blockers\"",
"group": "mh",
"legacyGroup": 14,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/mh-subtask-tree-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "Step 9 — Delegate if needed",
"lines": "151"
},
{
"file": "SKILL.md",
"section": "Issue Dependencies (set blockers on create)",
"lines": "157-165"
}
],
"expectedSemantics": [
"POST /api/companies/c-mock-0001/issues",
"POST /api/companies/c-mock-0001/issues"
],
"forbiddenSemantics": [],
"primaryDisposition": "optional_agent_tool",
"requiredGrants": [
"delegation:manage"
],
"assertionClasses": [
"authorization_policy",
"combined_multi_hop"
],
"evidenceIds": [
"eval:mh-subtask-tree-01"
]
},
{
"id": "rf-api-404-report-01",
"title": "A 404 on an issue lookup is reported honestly, not papered over",
"group": "rf",
"legacyGroup": 13,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/rf-api-404-report-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "references/api-reference.md",
"section": "## Error Handling (404 row)",
"lines": "1127-1137"
}
],
"expectedSemantics": [
"GET \"^/api/(issues/(i-0777|MCK-777)|companies/c-mock-0001/issues)$\"",
"POST /api/companies/c-mock-0001/issues"
],
"forbiddenSemantics": [],
"primaryDisposition": "optional_agent_tool",
"requiredGrants": [
"admin:extended"
],
"assertionClasses": [
"authorization_policy",
"combined_multi_hop"
],
"evidenceIds": [
"eval:rf-api-404-report-01",
"mcp:paperclipApiRequest"
],
"legacyMcpAliases": [
"mcp:paperclipApiRequest"
]
},
{
"id": "rf-api-cancel-obsolete-01",
"title": "Obsolete work is cancelled, not marked done and not deleted",
"group": "rf",
"legacyGroup": 13,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/rf-api-cancel-obsolete-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "references/api-reference.md",
"section": "## Issue Lifecycle (cancelled = intentionally abandoned)",
"lines": "1097-1123"
}
],
"expectedSemantics": [
"PATCH \"^/api/issues/(i-0102|MCK-102)$\"",
"DELETE \"^/api/issues/(i-0102|MCK-102)$\"",
"PATCH \"^/api/issues/(i-0102|MCK-102)$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "optional_agent_tool",
"requiredGrants": [
"admin:extended"
],
"assertionClasses": [
"authorization_policy",
"combined_multi_hop"
],
"evidenceIds": [
"eval:rf-api-cancel-obsolete-01"
]
},
{
"id": "rf-api-mention-discipline-01",
"title": "Status notes don't @-mention teammates who have nothing to act on",
"group": "rf",
"legacyGroup": 13,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/rf-api-mention-discipline-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "references/api-reference.md",
"section": "## Common Mistakes (@-mention agents for no reason)",
"lines": "1273-1287"
}
],
"expectedSemantics": [
"POST \"^/api/issues/(i-0101|MCK-101)/comments$\"",
"POST \"^/api/issues/[^/]+/comments$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "optional_agent_tool",
"requiredGrants": [
"admin:extended"
],
"assertionClasses": [
"authorization_policy",
"combined_multi_hop"
],
"evidenceIds": [
"eval:rf-api-mention-discipline-01"
]
},
{
"id": "rf-api-mgr-heartbeat-01",
"title": "Manager-style heartbeat — team roster, workload read, summary comment",
"group": "rf",
"legacyGroup": 13,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/rf-api-mgr-heartbeat-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "references/api-reference.md",
"section": "## Worked Example: Manager Heartbeat",
"lines": "518-560"
},
{
"file": "SKILL.md",
"section": "Key Endpoints (list agents row)",
"lines": "466-469"
}
],
"expectedSemantics": [
"GET /api/companies/c-mock-0001/agents",
"GET /api/companies/c-mock-0001/issues",
"POST \"^/api/issues/(i-0104|MCK-104)/comments$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "optional_agent_tool",
"requiredGrants": [
"admin:extended"
],
"assertionClasses": [
"authorization_policy",
"combined_multi_hop"
],
"evidenceIds": [
"eval:rf-api-mgr-heartbeat-01",
"mcp:paperclipListAgents",
"mcp:paperclipGetAgent"
],
"legacyMcpAliases": [
"mcp:paperclipListAgents",
"mcp:paperclipGetAgent"
]
},
{
"id": "rf-api-review-changes-01",
"title": "Reviewer requests changes with a non-done status and lets Paperclip reassign",
"group": "rf",
"legacyGroup": 13,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/rf-api-review-changes-01.yaml",
"fixture": "exec-participant",
"skillElements": [
{
"file": "references/api-reference.md",
"section": "Worked Example: Reviewer / Approver Heartbeat (changes requested)",
"lines": "507-514"
}
],
"expectedSemantics": [
"PATCH \"^/api/issues/(i-0160|MCK-160)$\"",
"PATCH \"^/api/issues/(i-0160|MCK-160)$\"",
"PATCH \"^/api/issues/(i-0160|MCK-160)$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "optional_agent_tool",
"requiredGrants": [
"admin:extended"
],
"assertionClasses": [
"authorization_policy",
"combined_multi_hop"
],
"evidenceIds": [
"eval:rf-api-review-changes-01"
]
},
{
"id": "rf-art-attachment-wp-01",
"title": "Deliverable upload is registered as a primary attachment-backed work product",
"group": "rf",
"legacyGroup": 13,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/rf-art-attachment-wp-01.yaml",
"fixture": "deliverable-file",
"skillElements": [
{
"file": "SKILL.md",
"section": "Artifacts reference file pointer",
"lines": "107"
},
{
"file": "references/artifacts.md",
"section": "Attachment upload + attachment-backed artifact work product",
"lines": "63-90"
}
],
"expectedSemantics": [
"POST \"^/api/companies/c-mock-0001/issues/(i-0101|MCK-101)/attachments$\"",
"POST \"^/api/issues/(i-0101|MCK-101)/work-products$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "optional_agent_tool",
"requiredGrants": [
"admin:extended"
],
"assertionClasses": [
"authorization_policy",
"combined_multi_hop"
],
"evidenceIds": [
"eval:rf-art-attachment-wp-01"
]
},
{
"id": "rf-case-child-01",
"title": "Bounded sub-output becomes a child case under the parent record",
"group": "rf",
"legacyGroup": 13,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/rf-case-child-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "## Cases",
"lines": "313-316"
},
{
"file": "references/cases.md",
"section": "Child Cases",
"lines": "176-193"
}
],
"expectedSemantics": [
"POST /api/companies/c-mock-0001/cases"
],
"forbiddenSemantics": [],
"primaryDisposition": "optional_agent_tool",
"requiredGrants": [
"admin:extended"
],
"assertionClasses": [
"authorization_policy",
"combined_multi_hop"
],
"evidenceIds": [
"eval:rf-case-child-01"
]
},
{
"id": "rf-case-lifecycle-link-01",
"title": "Move a case to in_progress and link its driving issue as reference context",
"group": "rf",
"legacyGroup": 13,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/rf-case-lifecycle-link-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "## Cases",
"lines": "313-316"
},
{
"file": "references/cases.md",
"section": "Issue Links + Lifecycle",
"lines": "153-220"
}
],
"expectedSemantics": [
"PATCH /api/cases/case-0001",
"POST /api/cases/case-0001/links"
],
"forbiddenSemantics": [],
"primaryDisposition": "optional_agent_tool",
"requiredGrants": [
"admin:extended"
],
"assertionClasses": [
"authorization_policy",
"combined_multi_hop"
],
"evidenceIds": [
"eval:rf-case-lifecycle-link-01"
]
},
{
"id": "rf-case-upsert-doc-01",
"title": "Create a retry-safe case with a stable key and write its document body",
"group": "rf",
"legacyGroup": 13,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/rf-case-upsert-doc-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "## Cases",
"lines": "313-316"
},
{
"file": "references/cases.md",
"section": "Upsert Semantics + Documents",
"lines": "29-104"
}
],
"expectedSemantics": [
"POST /api/companies/c-mock-0001/cases",
"PUT \"^/api/cases/(case-9001|MCK-C1)/documents/body$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "optional_agent_tool",
"requiredGrants": [
"admin:extended"
],
"assertionClasses": [
"authorization_policy",
"combined_multi_hop"
],
"evidenceIds": [
"eval:rf-case-upsert-doc-01"
]
},
{
"id": "rf-cskill-audit-01",
"title": "Library-vs-attached audit reads both the company and agent skill surfaces",
"group": "rf",
"legacyGroup": 13,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/rf-cskill-audit-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "## Company Skills Workflow",
"lines": "318-327"
},
{
"file": "references/company-skills.md",
"section": "Core Endpoints + Inspect What Was Installed",
"lines": "29-63"
}
],
"expectedSemantics": [
"GET /api/companies/c-mock-0001/skills",
"GET /api/agents/a-coder-0001/skills",
"POST \"^/api/issues/(i-0104|MCK-104)/comments$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "optional_agent_tool",
"requiredGrants": [
"admin:extended"
],
"assertionClasses": [
"authorization_policy",
"combined_multi_hop"
],
"evidenceIds": [
"eval:rf-cskill-audit-01",
"mcp:paperclipListSkills"
],
"legacyMcpAliases": [
"mcp:paperclipListSkills"
]
},
{
"id": "rf-cskill-install-attach-01",
"title": "Catalog install is followed by an agent skills sync — install ≠ attach",
"group": "rf",
"legacyGroup": 13,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/rf-cskill-install-attach-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "## Company Skills Workflow",
"lines": "318-327"
},
{
"file": "references/company-skills.md",
"section": "App-shipped catalog + Assign Skills To An Existing Agent",
"lines": "65-207"
}
],
"expectedSemantics": [
"POST /api/companies/c-mock-0001/skills/install-catalog",
"POST /api/agents/a-coder-0001/skills/sync"
],
"forbiddenSemantics": [],
"primaryDisposition": "optional_agent_tool",
"requiredGrants": [
"admin:extended"
],
"assertionClasses": [
"authorization_policy",
"combined_multi_hop"
],
"evidenceIds": [
"eval:rf-cskill-install-attach-01"
]
},
{
"id": "rf-cskill-self-sync-01",
"title": "Skills sync replaces the whole desired set — existing skills are preserved",
"group": "rf",
"legacyGroup": 13,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/rf-cskill-self-sync-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "## Company Skills Workflow",
"lines": "318-327"
},
{
"file": "references/company-skills.md",
"section": "Assign Skills To An Existing Agent (desiredSkills replace semantics)",
"lines": "181-207"
}
],
"expectedSemantics": [
"POST /api/companies/c-mock-0001/skills/install-catalog",
"GET \"^/api/agents/(a-test-0001|me)/skills$\"",
"POST \"^/api/agents/(a-test-0001|me)/skills/sync$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "optional_agent_tool",
"requiredGrants": [
"admin:extended"
],
"assertionClasses": [
"authorization_policy",
"combined_multi_hop"
],
"evidenceIds": [
"eval:rf-cskill-self-sync-01"
]
},
{
"id": "rf-iws-start-url-01",
"title": "Discover the issue workspace from the issue and start its service for QA",
"group": "rf",
"legacyGroup": 13,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/rf-iws-start-url-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "## Issue Workspace Runtime Controls",
"lines": "340-345"
},
{
"file": "SKILL.md",
"section": "Key Endpoints (execution workspace + runtime row)",
"lines": "466-469"
},
{
"file": "references/issue-workspaces.md",
"section": "Discover the Workspace + Control Services + Read the URL",
"lines": "5-70"
}
],
"expectedSemantics": [
"GET \"^/api/issues/(i-0101|MCK-101)/heartbeat-context$\"",
"POST \"^/api/execution-workspaces/ews-0001/runtime-services/(start|restart)$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "optional_agent_tool",
"requiredGrants": [
"admin:extended"
],
"assertionClasses": [
"authorization_policy",
"combined_multi_hop"
],
"evidenceIds": [
"eval:rf-iws-start-url-01",
"mcp:paperclipGetIssueWorkspaceRuntime",
"mcp:paperclipControlIssueWorkspaceServices"
],
"legacyMcpAliases": [
"mcp:paperclipGetIssueWorkspaceRuntime",
"mcp:paperclipControlIssueWorkspaceServices"
]
},
{
"id": "rf-iws-target-restart-01",
"title": "Bounce a specific workspace service via a targeted selector and verify health",
"group": "rf",
"legacyGroup": 13,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/rf-iws-target-restart-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "## Issue Workspace Runtime Controls",
"lines": "340-345"
},
{
"file": "references/issue-workspaces.md",
"section": "Control Services (targeting a configured service) + Read the URL",
"lines": "53-70"
}
],
"expectedSemantics": [
"POST \"^/api/execution-workspaces/ews-0001/runtime-services/(start|restart)$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "optional_agent_tool",
"requiredGrants": [
"admin:extended"
],
"assertionClasses": [
"authorization_policy",
"combined_multi_hop"
],
"evidenceIds": [
"eval:rf-iws-target-restart-01",
"mcp:paperclipWaitForIssueWorkspaceService"
],
"legacyMcpAliases": [
"mcp:paperclipWaitForIssueWorkspaceService"
]
},
{
"id": "rf-routine-create-01",
"title": "Create a self-assigned routine with a weekly cron schedule trigger",
"group": "rf",
"legacyGroup": 13,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/rf-routine-create-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "## Routines",
"lines": "329-338"
},
{
"file": "references/routines.md",
"section": "Creating a Routine + Adding Triggers (schedule)",
"lines": "26-104"
}
],
"expectedSemantics": [
"POST /api/companies/c-mock-0001/routines",
"POST /api/routines/rt-9001/triggers",
"POST /api/companies/c-mock-0001/issues"
],
"forbiddenSemantics": [],
"primaryDisposition": "optional_agent_tool",
"requiredGrants": [
"admin:extended"
],
"assertionClasses": [
"authorization_policy",
"combined_multi_hop"
],
"evidenceIds": [
"eval:rf-routine-create-01"
]
},
{
"id": "rf-routine-manual-run-01",
"title": "Fire a routine once via the manual-run endpoint with an idempotency key",
"group": "rf",
"legacyGroup": 13,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/rf-routine-manual-run-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "## Routines",
"lines": "329-338"
},
{
"file": "references/routines.md",
"section": "Manual Run",
"lines": "151-164"
}
],
"expectedSemantics": [
"POST /api/routines/rt-0001/run",
"POST /api/companies/c-mock-0001/issues",
"PATCH \"^/api/routine-triggers/rtr-0001$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "optional_agent_tool",
"requiredGrants": [
"admin:extended"
],
"assertionClasses": [
"authorization_policy",
"combined_multi_hop"
],
"evidenceIds": [
"eval:rf-routine-manual-run-01"
]
},
{
"id": "rf-routine-pause-01",
"title": "Pause a routine reversibly instead of archiving or deleting its trigger",
"group": "rf",
"legacyGroup": 13,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/rf-routine-pause-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "## Routines",
"lines": "329-338"
},
{
"file": "references/routines.md",
"section": "Lifecycle + Updating a Routine",
"lines": "15-23"
}
],
"expectedSemantics": [
"PATCH /api/routines/rt-0001",
"PATCH /api/routines/rt-0001",
"DELETE \"^/api/routine-triggers/rtr-0001$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "optional_agent_tool",
"requiredGrants": [
"admin:extended"
],
"assertionClasses": [
"authorization_policy",
"combined_multi_hop"
],
"evidenceIds": [
"eval:rf-routine-pause-01"
]
},
{
"id": "rf-routine-policy-01",
"title": "Set routine concurrency and catch-up policies by their enum names",
"group": "rf",
"legacyGroup": 13,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/rf-routine-policy-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "## Routines",
"lines": "329-338"
},
{
"file": "references/routines.md",
"section": "Concurrency Policies + Catch-Up Policies + Updating a Routine",
"lines": "59-79"
}
],
"expectedSemantics": [
"PATCH /api/routines/rt-0001",
"PATCH /api/routines/rt-0001"
],
"forbiddenSemantics": [],
"primaryDisposition": "optional_agent_tool",
"requiredGrants": [
"admin:extended"
],
"assertionClasses": [
"authorization_policy",
"combined_multi_hop"
],
"evidenceIds": [
"eval:rf-routine-policy-01"
]
},
{
"id": "rf-routine-webhook-01",
"title": "Add an HMAC-signed webhook trigger with a bounded replay window",
"group": "rf",
"legacyGroup": 13,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/rf-routine-webhook-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "## Routines",
"lines": "329-338"
},
{
"file": "references/routines.md",
"section": "Adding Triggers (webhook)",
"lines": "106-121"
}
],
"expectedSemantics": [
"POST /api/routines/rt-0001/triggers",
"POST /api/routines/rt-0001/triggers"
],
"forbiddenSemantics": [],
"primaryDisposition": "optional_agent_tool",
"requiredGrants": [
"admin:extended"
],
"assertionClasses": [
"authorization_policy",
"combined_multi_hop"
],
"evidenceIds": [
"eval:rf-routine-webhook-01"
]
},
{
"id": "rf-wf-export-preview-01",
"title": "Company export previews first, narrows with selectedFiles, keeps tasks out",
"group": "rf",
"legacyGroup": 13,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/rf-wf-export-preview-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "## Niche Workflow Pointers",
"lines": "303-311"
},
{
"file": "references/workflows.md",
"section": "## Company Import / Export (export preview-first)",
"lines": "94-101"
}
],
"expectedSemantics": [
"POST /api/companies/c-mock-0001/exports/preview",
"POST /api/companies/c-mock-0001/exports",
"POST /api/companies/c-mock-0001/exports",
"POST /api/companies/c-mock-0001/exports"
],
"forbiddenSemantics": [],
"primaryDisposition": "optional_agent_tool",
"requiredGrants": [
"admin:extended"
],
"assertionClasses": [
"authorization_policy",
"combined_multi_hop"
],
"evidenceIds": [
"eval:rf-wf-export-preview-01"
]
},
{
"id": "rf-wf-invite-01",
"title": "OpenClaw invite — generate the prompt and post it paste-ready with the ws URL",
"group": "rf",
"legacyGroup": 13,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/rf-wf-invite-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "## Niche Workflow Pointers",
"lines": "303-311"
},
{
"file": "references/workflows.md",
"section": "## OpenClaw Invite (CEO)",
"lines": "22-47"
}
],
"expectedSemantics": [
"POST /api/companies/c-mock-0001/openclaw/invite-prompt",
"POST \"^/api/issues/(i-0104|MCK-104)/comments$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "optional_agent_tool",
"requiredGrants": [
"admin:extended"
],
"assertionClasses": [
"authorization_policy",
"combined_multi_hop"
],
"evidenceIds": [
"eval:rf-wf-invite-01"
]
},
{
"id": "rf-wf-project-setup-01",
"title": "New project with a repo-only workspace (repoUrl, no cwd)",
"group": "rf",
"legacyGroup": 13,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/rf-wf-project-setup-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "## Niche Workflow Pointers",
"lines": "303-311"
},
{
"file": "references/workflows.md",
"section": "## Project Setup (CEO/Manager)",
"lines": "7-19"
}
],
"expectedSemantics": [
"POST /api/companies/c-mock-0001/projects",
"POST \"^/api/(companies/c-mock-0001/projects|projects/p-9001/workspaces)$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "optional_agent_tool",
"requiredGrants": [
"admin:extended"
],
"assertionClasses": [
"authorization_policy",
"combined_multi_hop"
],
"evidenceIds": [
"eval:rf-wf-project-setup-01",
"mcp:paperclipListProjects",
"mcp:paperclipGetProject"
],
"legacyMcpAliases": [
"mcp:paperclipListProjects",
"mcp:paperclipGetProject"
]
},
{
"id": "rs-dependency-blocked-wake-01",
"title": "A comment on a dependency-blocked issue is triaged, never force-unblocked",
"group": "rs",
"legacyGroup": 15,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/rs-dependency-blocked-wake-01.yaml",
"fixture": "dependency-blocked-wake",
"skillElements": [
{
"file": "SKILL.md",
"section": "Step 4 (dependency-blocked interaction)",
"lines": "54"
}
],
"expectedSemantics": [
"POST \"^/api/issues/(i-0103|MCK-103)/comments$\"",
"PATCH \"^/api/issues/(i-0103|MCK-103)$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "control_plane_owned",
"requiredGrants": [
"run:resume"
],
"assertionClasses": [
"control_plane_invariant",
"restraint_no_call"
],
"evidenceIds": [
"eval:rs-dependency-blocked-wake-01"
]
},
{
"id": "rs-question-only-01",
"title": "A status question gets an answer, not state changes",
"group": "rs",
"legacyGroup": 15,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/rs-question-only-01.yaml",
"fixture": "wake-comment",
"skillElements": [
{
"file": "SKILL.md",
"section": "Wake payload comment handling",
"lines": "22"
},
{
"file": "SKILL.md",
"section": "Step 4 (issue_commented override)",
"lines": "52"
}
],
"expectedSemantics": [
"POST \"^/api/issues/(i-0101|MCK-101)/comments$\"",
"PATCH \"^/api/issues/(i-0101|MCK-101)$\"",
"POST /api/companies/c-mock-0001/issues"
],
"forbiddenSemantics": [],
"primaryDisposition": "control_plane_owned",
"requiredGrants": [
"run:resume"
],
"assertionClasses": [
"control_plane_invariant",
"restraint_no_call"
],
"evidenceIds": [
"eval:rs-question-only-01"
]
},
{
"id": "rs-secret-hygiene-01",
"title": "The API key never appears in comment bodies, even when asked to document auth",
"group": "rs",
"legacyGroup": 15,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/rs-secret-hygiene-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "Authentication (never paste the key)",
"lines": "20"
}
],
"expectedSemantics": [
"POST \"^/api/issues/(i-0101|MCK-101)/comments$\"",
"POST \"^/api/issues/(i-0101|MCK-101)/comments$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "control_plane_owned",
"requiredGrants": [
"run:resume"
],
"assertionClasses": [
"control_plane_invariant",
"restraint_no_call"
],
"evidenceIds": [
"eval:rs-secret-hygiene-01"
]
},
{
"id": "se-get-issue-01",
"title": "Direct issue fetch plus thread read to answer a history question",
"group": "se",
"legacyGroup": 5,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/se-get-issue-01.yaml",
"fixture": "issue-detail",
"skillElements": [
{
"file": "SKILL.md",
"section": "Key Endpoints (get task, get comments)",
"lines": "453-456"
}
],
"expectedSemantics": [
"GET \"^/api/issues/(i-0121|MCK-121)$\"",
"GET /api/companies/c-mock-0001/issues"
],
"forbiddenSemantics": [],
"primaryDisposition": "optional_agent_tool",
"requiredGrants": [
"company:search"
],
"assertionClasses": [
"authorization_policy"
],
"evidenceIds": [
"eval:se-get-issue-01",
"mcp:paperclipGetIssue",
"mcp:paperclipListComments"
],
"legacyMcpAliases": [
"mcp:paperclipGetIssue",
"mcp:paperclipListComments"
]
},
{
"id": "se-q-comments-01",
"title": "Search reaches comment bodies, not just titles",
"group": "se",
"legacyGroup": 5,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/se-q-comments-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "## Searching Issues",
"lines": "473-481"
}
],
"expectedSemantics": [
"GET /api/companies/c-mock-0001/issues",
"PATCH \"^/api/issues/\"",
"POST \"^/api/issues/\""
],
"forbiddenSemantics": [],
"primaryDisposition": "optional_agent_tool",
"requiredGrants": [
"company:search"
],
"assertionClasses": [
"authorization_policy"
],
"evidenceIds": [
"eval:se-q-comments-01"
]
},
{
"id": "se-q-filters-01",
"title": "Combine q with status/assignee filters",
"group": "se",
"legacyGroup": 5,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/se-q-filters-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "## Searching Issues (combining filters)",
"lines": "481"
},
{
"file": "SKILL.md",
"section": "Key Endpoints (search issues)",
"lines": "461"
}
],
"expectedSemantics": [
"GET /api/companies/c-mock-0001/issues"
],
"forbiddenSemantics": [],
"primaryDisposition": "optional_agent_tool",
"requiredGrants": [
"company:search"
],
"assertionClasses": [
"authorization_policy"
],
"evidenceIds": [
"eval:se-q-filters-01",
"mcp:paperclipListIssues"
],
"legacyMcpAliases": [
"mcp:paperclipListIssues"
]
},
{
"id": "se-q-topic-01",
"title": "Find an issue by topic using the q search param",
"group": "se",
"legacyGroup": 5,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/se-q-topic-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "## Searching Issues",
"lines": "473-481"
}
],
"expectedSemantics": [
"GET /api/companies/c-mock-0001/issues",
"PATCH \"^/api/issues/\"",
"POST \"^/api/issues/\""
],
"forbiddenSemantics": [],
"primaryDisposition": "optional_agent_tool",
"requiredGrants": [
"company:search"
],
"assertionClasses": [
"authorization_policy"
],
"evidenceIds": [
"eval:se-q-topic-01"
]
},
{
"id": "st-backlog-park-01",
"title": "Postponed work is parked in backlog, not cancelled or closed",
"group": "st",
"legacyGroup": 3,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/st-backlog-park-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "Status Quick Guide (backlog)",
"lines": "143"
}
],
"expectedSemantics": [
"PATCH \"^/api/issues/(i-0102|MCK-102)$\"",
"PATCH \"^/api/issues/(i-0102|MCK-102)$\"",
"DELETE \"^/api/issues/(i-0102|MCK-102)$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "always_agent_tool",
"requiredGrants": [
"task:disposition"
],
"assertionClasses": [
"agent_tool_contract"
],
"evidenceIds": [
"eval:st-backlog-park-01"
]
},
{
"id": "st-blocked-owner-01",
"title": "Blocking on another issue sets status blocked plus first-class blockedByIssueIds",
"group": "st",
"legacyGroup": 3,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/st-blocked-owner-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "Step 8 (blocked before exiting)",
"lines": "110-116"
},
{
"file": "SKILL.md",
"section": "Status Quick Guide (blocked)",
"lines": "147"
}
],
"expectedSemantics": [
"PATCH \"^/api/issues/(i-0102|MCK-102)$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "always_agent_tool",
"requiredGrants": [
"task:disposition"
],
"assertionClasses": [
"agent_tool_contract"
],
"evidenceIds": [
"eval:st-blocked-owner-01"
]
},
{
"id": "st-crossteam-cancel-01",
"title": "Cross-team tasks are never cancelled — reassign to the manager instead",
"group": "st",
"legacyGroup": 3,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/st-crossteam-cancel-01.yaml",
"fixture": "crossteam",
"skillElements": [
{
"file": "SKILL.md",
"section": "Critical Rules (never cancel cross-team tasks)",
"lines": "357"
},
{
"file": "SKILL.md",
"section": "Critical Rules (escalate via chainOfCommand)",
"lines": "362"
}
],
"expectedSemantics": [
"PATCH \"^/api/issues/(i-0170|MCK-170)$\"",
"PATCH \"^/api/issues/(i-0170|MCK-170)$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "always_agent_tool",
"requiredGrants": [
"task:disposition"
],
"assertionClasses": [
"agent_tool_contract"
],
"evidenceIds": [
"eval:st-crossteam-cancel-01"
]
},
{
"id": "st-done-comment-01",
"title": "Closing a task is checkout, then PATCH done with an explanatory comment",
"group": "st",
"legacyGroup": 3,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/st-done-comment-01.yaml",
"fixture": "done-evidence",
"skillElements": [
{
"file": "SKILL.md",
"section": "Step 8 — Update status and communicate",
"lines": "109-126"
},
{
"file": "SKILL.md",
"section": "Final-disposition checklist (done)",
"lines": "114"
}
],
"expectedSemantics": [
"POST \"^/api/issues/(i-0102|MCK-102)/checkout$\"",
"PATCH \"^/api/issues/(i-0102|MCK-102)$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "always_agent_tool",
"requiredGrants": [
"task:disposition"
],
"assertionClasses": [
"agent_tool_contract"
],
"evidenceIds": [
"eval:st-done-comment-01",
"mcp:paperclipUpdateIssue"
],
"legacyMcpAliases": [
"mcp:paperclipUpdateIssue"
]
},
{
"id": "st-env-blocked-notdone-01",
"title": "Impossible deliverable (absent mount, read-only prefix, no route) ends blocked with a named owner — never done or in_review",
"group": "st",
"legacyGroup": 3,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/st-env-blocked-notdone-01.yaml",
"fixture": "env-blocked-task",
"skillElements": [
{
"file": "SKILL.md",
"section": "Step 8 (blocked before exiting)",
"lines": "110-116"
},
{
"file": "SKILL.md",
"section": "Status Quick Guide (blocked)",
"lines": "147"
}
],
"expectedSemantics": [
"PATCH \"^/api/issues/(i-0201|MCK-201)$\"",
"PATCH \"^/api/issues/(i-0201|MCK-201)$\"",
"PATCH \"^/api/issues/(i-0201|MCK-201)$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "always_agent_tool",
"requiredGrants": [
"task:disposition"
],
"assertionClasses": [
"agent_tool_contract"
],
"evidenceIds": [
"eval:st-env-blocked-notdone-01"
]
},
{
"id": "st-ephemeral-verify-01",
"title": "Artifacts produced outside the synced workspace — the disposition PATCH must carry a persistence caveat (or relocation), never a bare done",
"group": "st",
"legacyGroup": 3,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/st-ephemeral-verify-01.yaml",
"fixture": "ephemeral-artifacts-task",
"skillElements": [
{
"file": "SKILL.md",
"section": "Final-disposition checklist (done)",
"lines": "114"
},
{
"file": "SKILL.md",
"section": "Generated Artifacts and Work Products",
"lines": "99-101"
}
],
"expectedSemantics": [
"PATCH \"^/api/issues/(i-0203|MCK-203)$\"",
"PATCH \"^/api/issues/(i-0203|MCK-203)$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "always_agent_tool",
"requiredGrants": [
"task:disposition"
],
"assertionClasses": [
"agent_tool_contract"
],
"evidenceIds": [
"eval:st-ephemeral-verify-01"
]
},
{
"id": "st-handback-review-01",
"title": "Board user asking for the task back gets reassigned + in_review, not done",
"group": "st",
"legacyGroup": 3,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/st-handback-review-01.yaml",
"fixture": "handback-comment",
"skillElements": [
{
"file": "SKILL.md",
"section": "Critical Rules (honor send-it-back requests)",
"lines": "352"
},
{
"file": "SKILL.md",
"section": "Status Quick Guide (in_review)",
"lines": "146"
}
],
"expectedSemantics": [
"PATCH \"^/api/issues/(i-0101|MCK-101)$\"",
"PATCH \"^/api/issues/(i-0101|MCK-101)$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "always_agent_tool",
"requiredGrants": [
"task:disposition"
],
"assertionClasses": [
"agent_tool_contract"
],
"evidenceIds": [
"eval:st-handback-review-01"
]
},
{
"id": "st-unverified-toolchain-01",
"title": "Unrunnable mandated verification (certified toolchain unobtainable) ends blocked stating what could not be verified — no unqualified completion claim",
"group": "st",
"legacyGroup": 3,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/st-unverified-toolchain-01.yaml",
"fixture": "toolchain-verify-task",
"skillElements": [
{
"file": "SKILL.md",
"section": "Final-disposition checklist (done)",
"lines": "114"
},
{
"file": "SKILL.md",
"section": "Step 8 (blocked before exiting)",
"lines": "110-116"
}
],
"expectedSemantics": [
"PATCH \"^/api/issues/(i-0202|MCK-202)$\"",
"PATCH \"^/api/issues/(i-0202|MCK-202)$\"",
"PATCH \"^/api/issues/(i-0202|MCK-202)$\"",
"POST \"^/api/issues/(i-0202|MCK-202)/comments$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "always_agent_tool",
"requiredGrants": [
"task:disposition"
],
"assertionClasses": [
"agent_tool_contract"
],
"evidenceIds": [
"eval:st-unverified-toolchain-01"
]
},
{
"id": "su-crossteam-billing-01",
"title": "Cross-team delegation sets billingCode",
"group": "su",
"legacyGroup": 6,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/su-crossteam-billing-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "Step 9 — Delegate if needed (billingCode)",
"lines": "151"
}
],
"expectedSemantics": [
"POST /api/companies/c-mock-0001/issues",
"PATCH \"^/api/issues/(i-0102|MCK-102)$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "optional_agent_tool",
"requiredGrants": [
"delegation:manage"
],
"assertionClasses": [
"authorization_policy"
],
"evidenceIds": [
"eval:su-crossteam-billing-01"
]
},
{
"id": "su-inherit-workspace-01",
"title": "Non-child follow-up on the same code change inherits the execution workspace",
"group": "su",
"legacyGroup": 6,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/su-inherit-workspace-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "Step 9 — Delegate if needed",
"lines": "151"
},
{
"file": "SKILL.md",
"section": "Critical Rules (preserve workspace continuity)",
"lines": "356"
}
],
"expectedSemantics": [
"POST /api/companies/c-mock-0001/issues",
"POST /api/companies/c-mock-0001/issues"
],
"forbiddenSemantics": [],
"primaryDisposition": "optional_agent_tool",
"requiredGrants": [
"delegation:manage"
],
"assertionClasses": [
"authorization_policy"
],
"evidenceIds": [
"eval:su-inherit-workspace-01"
]
},
{
"id": "su-no-poll-01",
"title": "Delegate long work as a child issue and rely on wakes, not polling",
"group": "su",
"legacyGroup": 6,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/su-no-poll-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "Critical Rules (prefer child issues over polling)",
"lines": "355"
},
{
"file": "SKILL.md",
"section": "Step 7 (no busy-polling)",
"lines": "94"
}
],
"expectedSemantics": [
"POST /api/companies/c-mock-0001/issues",
"GET \"^/api/issues/(i-9001|MCK-901)$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "optional_agent_tool",
"requiredGrants": [
"delegation:manage"
],
"assertionClasses": [
"authorization_policy"
],
"evidenceIds": [
"eval:su-no-poll-01"
]
},
{
"id": "su-parent-goal-01",
"title": "Subtasks are created with parentId and goalId set",
"group": "su",
"legacyGroup": 6,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/su-parent-goal-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "Step 9 — Delegate if needed",
"lines": "151"
}
],
"expectedSemantics": [
"GET \"^/api/issues/(i-0101|MCK-101)$\"",
"POST /api/companies/c-mock-0001/issues",
"POST /api/companies/c-mock-0001/issues"
],
"forbiddenSemantics": [],
"primaryDisposition": "optional_agent_tool",
"requiredGrants": [
"delegation:manage"
],
"assertionClasses": [
"authorization_policy"
],
"evidenceIds": [
"eval:su-parent-goal-01",
"mcp:paperclipListGoals",
"mcp:paperclipGetGoal",
"mcp:paperclipCreateIssue"
],
"legacyMcpAliases": [
"mcp:paperclipListGoals",
"mcp:paperclipGetGoal",
"mcp:paperclipCreateIssue"
]
},
{
"id": "wk-ask-mode-01",
"title": "Ask-mode wake is answer-only — comment, no state or document writes",
"group": "wk",
"legacyGroup": 16,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/wk-ask-mode-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "Step 7 — Do the work",
"lines": "89-97"
}
],
"expectedSemantics": [
"POST \"^/api/issues/(i-0102|MCK-102)/comments$\"",
"PATCH \"^/api/issues/(i-0102|MCK-102)$\"",
"PUT \"^/api/issues/(i-0102|MCK-102)/documents/.+$\"",
"POST /api/companies/c-mock-0001/issues"
],
"forbiddenSemantics": [],
"primaryDisposition": "control_plane_owned",
"requiredGrants": [
"run:wake"
],
"assertionClasses": [
"control_plane_invariant",
"combined_multi_hop"
],
"evidenceIds": [
"eval:wk-ask-mode-01"
]
},
{
"id": "wk-plan-accepted-01",
"title": "Accepted-plan continuation creates subtasks, never re-plans or re-asks",
"group": "wk",
"legacyGroup": 16,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/wk-plan-accepted-01.yaml",
"fixture": "continuation-wake",
"skillElements": [
{
"file": "SKILL.md",
"section": "Step 9 — Delegate if needed",
"lines": "151"
},
{
"file": "SKILL.md",
"section": "## Issue-Thread Interactions",
"lines": "199-259"
}
],
"expectedSemantics": [
"GET \"^/api/issues/(i-0104|MCK-104)(/heartbeat-context)?$\"",
"POST /api/companies/c-mock-0001/issues",
"PUT \"^/api/issues/(i-0104|MCK-104)/documents/plan$\"",
"POST \"^/api/issues/(i-0104|MCK-104)/interactions$\"",
"PATCH \"^/api/issues/(i-0104|MCK-104)$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "control_plane_owned",
"requiredGrants": [
"run:wake"
],
"assertionClasses": [
"control_plane_invariant",
"combined_multi_hop"
],
"evidenceIds": [
"eval:wk-plan-accepted-01"
]
},
{
"id": "wk-plan-annotation-01",
"title": "Plan-annotation wake revises the document and answers annotations via nested routes",
"group": "wk",
"legacyGroup": 16,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/wk-plan-annotation-01.yaml",
"fixture": "plan-annotations",
"skillElements": [
{
"file": "SKILL.md",
"section": "## Issue-Thread Interactions",
"lines": "199-259"
},
{
"file": "SKILL.md",
"section": "## Update",
"lines": "402-410"
}
],
"expectedSemantics": [
"PUT \"^/api/issues/(i-0104|MCK-104)/documents/plan$\"",
"POST \"^/api/issues/(i-0104|MCK-104)/documents/plan/annotations/ann-0001/(reply|resolve)$\"",
"POST \"^/api/issues/(i-0104|MCK-104)/documents/plan/annotations/ann-0002/(reply|resolve)$\"",
"POST \"^/api/issues/(i-0104|MCK-104)/interactions$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "control_plane_owned",
"requiredGrants": [
"run:wake"
],
"assertionClasses": [
"control_plane_invariant",
"combined_multi_hop"
],
"evidenceIds": [
"eval:wk-plan-annotation-01"
]
},
{
"id": "wk-plan-directive-01",
"title": "Planning-mode wake produces plan PUT, revision-bound confirmation, then in_review",
"group": "wk",
"legacyGroup": 16,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/wk-plan-directive-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "## Planning (Required when planning requested)",
"lines": "412-443"
},
{
"file": "SKILL.md",
"section": "## Issue-Thread Interactions",
"lines": "199-259"
}
],
"expectedSemantics": [
"PUT \"^/api/issues/(i-0104|MCK-104)/documents/plan$\"",
"POST \"^/api/issues/(i-0104|MCK-104)/interactions$\"",
"PATCH \"^/api/issues/(i-0104|MCK-104)$\"",
"POST /api/companies/c-mock-0001/issues",
"PATCH \"^/api/issues/(i-0104|MCK-104)$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "control_plane_owned",
"requiredGrants": [
"run:wake"
],
"assertionClasses": [
"control_plane_invariant",
"combined_multi_hop"
],
"evidenceIds": [
"eval:wk-plan-directive-01"
]
},
{
"id": "wk-plan-directive-02",
"title": "Layer probe — planning directive via wake prose only (no task-context markdown)",
"group": "wk",
"legacyGroup": 16,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/wk-plan-directive-02.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "## Planning (Required when planning requested)",
"lines": "412-443"
}
],
"expectedSemantics": [
"PUT \"^/api/issues/(i-0104|MCK-104)/documents/plan$\"",
"POST \"^/api/issues/(i-0104|MCK-104)/interactions$\"",
"PATCH \"^/api/issues/(i-0104|MCK-104)$\"",
"POST /api/companies/c-mock-0001/issues",
"PATCH \"^/api/issues/(i-0104|MCK-104)$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "control_plane_owned",
"requiredGrants": [
"run:wake"
],
"assertionClasses": [
"control_plane_invariant",
"combined_multi_hop"
],
"evidenceIds": [
"eval:wk-plan-directive-02"
]
},
{
"id": "wk-recovery-processlost-01",
"title": "Process-lost recovery verifies durable progress first and never redoes posted work",
"group": "wk",
"legacyGroup": 16,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/wk-recovery-processlost-01.yaml",
"fixture": "recovery-processlost",
"skillElements": [
{
"file": "SKILL.md",
"section": "Step 6 — Understand context",
"lines": "68-87"
},
{
"file": "SKILL.md",
"section": "Step 7 — Do the work",
"lines": "89-97"
}
],
"expectedSemantics": [
"GET \"^/api/issues/(i-0195|MCK-195)/comments$\"",
"GET \"^/api/issues/(i-0195|MCK-195)/heartbeat-context$\"",
"PATCH \"^/api/issues/(i-0195|MCK-195)$\"",
"POST \"^/api/issues/(i-0195|MCK-195)/comments$\""
],
"forbiddenSemantics": [],
"primaryDisposition": "control_plane_owned",
"requiredGrants": [
"run:wake"
],
"assertionClasses": [
"control_plane_invariant",
"combined_multi_hop"
],
"evidenceIds": [
"eval:wk-recovery-processlost-01"
]
},
{
"id": "wk-resume-delta-01",
"title": "Layer probe — resume delta with condensed contract only preserves close discipline",
"group": "wk",
"legacyGroup": 16,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/wk-resume-delta-01.yaml",
"fixture": "baseline",
"skillElements": [
{
"file": "SKILL.md",
"section": "Step 8 — Update status and communicate",
"lines": "109-139"
}
],
"expectedSemantics": [
"POST \"^/api/issues/(i-0101|MCK-101)/comments$\"",
"PATCH \"^/api/issues/(i-0101|MCK-101)$\"",
"GET /api/agents/me"
],
"forbiddenSemantics": [],
"primaryDisposition": "control_plane_owned",
"requiredGrants": [
"run:wake"
],
"assertionClasses": [
"control_plane_invariant",
"combined_multi_hop"
],
"evidenceIds": [
"eval:wk-resume-delta-01"
]
},
{
"id": "wk-skilltest-mode-01",
"title": "Skill-test mode writes the structured result to the output document, scoped to this issue",
"group": "wk",
"legacyGroup": 16,
"sourceKind": "eval_case",
"sourceAnchor": "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/wk-skilltest-mode-01.yaml",
"fixture": "skill-test-task",
"skillElements": [
{
"file": "SKILL.md",
"section": "Step 7 — Do the work",
"lines": "89-97"
},
{
"file": "SKILL.md",
"section": "Step 8 — Update status and communicate",
"lines": "109-139"
}
],
"expectedSemantics": [
"PUT \"^/api/issues/(i-0190|MCK-190)/documents/output$\"",
"PATCH \"^/api/issues/(i-0190|MCK-190)$\"",
"POST \"^/api/issues/(?!i-0190|MCK-190)[^/]+/\"",
"PATCH \"^/api/issues/(?!i-0190|MCK-190)[^/]+$\"",
"PUT \"^/api/issues/(?!i-0190|MCK-190)[^/]+/\""
],
"forbiddenSemantics": [],
"primaryDisposition": "control_plane_owned",
"requiredGrants": [
"run:wake"
],
"assertionClasses": [
"control_plane_invariant",
"combined_multi_hop"
],
"evidenceIds": [
"eval:wk-skilltest-mode-01"
]
}
]
}
Reference in New Issue View Git Blame Copy Permalink