import { readFile } from "node:fs/promises"; import { relative, resolve } from "node:path"; import Ajv2020 from "ajv/dist/2020.js"; export const capabilityGroups = [ "hb", "co", "st", "cm", "se", "su", "bl", "dp", "ix", "ap", "ar", "er", "rf", "mh", "rs", "wk", ]; const groupPolicies = { hb: ["control_plane_owned", ["control_plane_invariant"], ["run:read"]], co: ["control_plane_owned", ["control_plane_invariant", "restraint_no_call"], ["task:checkout"]], st: ["always_agent_tool", ["agent_tool_contract"], ["task:disposition"]], cm: ["always_agent_tool", ["agent_tool_contract"], ["task:comment"]], se: ["optional_agent_tool", ["authorization_policy"], ["company:search"]], su: ["optional_agent_tool", ["authorization_policy"], ["delegation:manage"]], bl: ["always_agent_tool", ["agent_tool_contract", "combined_multi_hop"], ["task:block"]], dp: ["always_agent_tool", ["agent_tool_contract"], ["document:write"]], ix: ["always_agent_tool", ["agent_tool_contract", "combined_multi_hop"], ["interaction:request"]], ap: ["optional_agent_tool", ["authorization_policy", "combined_multi_hop"], ["approval:read"]], ar: ["always_agent_tool", ["agent_tool_contract"], ["artifact:register"]], er: ["control_plane_owned", ["control_plane_invariant", "restraint_no_call"], ["run:recover"]], rf: ["optional_agent_tool", ["authorization_policy", "combined_multi_hop"], ["admin:extended"]], mh: ["optional_agent_tool", ["authorization_policy", "combined_multi_hop"], ["delegation:manage"]], rs: ["control_plane_owned", ["control_plane_invariant", "restraint_no_call"], ["run:resume"]], wk: ["control_plane_owned", ["control_plane_invariant", "combined_multi_hop"], ["run:wake"]], }; // The skill and eval corpus are the normative behavior contract. These legacy // MCP names are retained only as traceability aliases into that contract; they // are not a second tool catalog and do not receive independent dispositions. const legacyMcpFoldTargets = { paperclipMe: "eval:hb-inbox-lite-01", paperclipInboxLite: "eval:hb-inbox-lite-01", paperclipListAgents: "eval:rf-api-mgr-heartbeat-01", paperclipListSkills: "eval:rf-cskill-audit-01", paperclipGetAgent: "eval:rf-api-mgr-heartbeat-01", paperclipListIssues: "eval:se-q-filters-01", paperclipGetIssue: "eval:se-get-issue-01", paperclipGetHeartbeatContext: "eval:hb-context-01", paperclipListComments: "eval:se-get-issue-01", paperclipGetComment: "eval:hb-wake-comment-01", paperclipListIssueApprovals: "eval:ap-board-approval-01", paperclipListDocuments: "eval:dp-base-revision-01", paperclipGetDocument: "eval:dp-base-revision-01", paperclipListDocumentRevisions: "eval:dp-base-revision-01", paperclipListProjects: "eval:rf-wf-project-setup-01", paperclipGetProject: "eval:rf-wf-project-setup-01", paperclipGetIssueWorkspaceRuntime: "eval:rf-iws-start-url-01", paperclipControlIssueWorkspaceServices: "eval:rf-iws-start-url-01", paperclipWaitForIssueWorkspaceService: "eval:rf-iws-target-restart-01", paperclipListGoals: "eval:su-parent-goal-01", paperclipGetGoal: "eval:su-parent-goal-01", paperclipListApprovals: "eval:ap-approval-wake-01", paperclipCreateApproval: "eval:ap-board-approval-01", paperclipGetApproval: "eval:ap-approval-wake-01", paperclipGetApprovalIssues: "eval:ap-approval-wake-01", paperclipListApprovalComments: "eval:ap-approval-deny-01", paperclipCreateIssue: "eval:su-parent-goal-01", paperclipUpdateIssue: "eval:st-done-comment-01", paperclipCheckoutIssue: "eval:co-body-contract-01", paperclipReleaseIssue: "eval:er-release-01", paperclipAddComment: "eval:cm-multiline-01", paperclipSuggestTasks: "eval:ix-suggest-tasks-01", paperclipAskUserQuestions: "eval:ix-questions-01", paperclipRequestConfirmation: "eval:ix-confirmation-plan-01", paperclipRequestCheckboxConfirmation: "eval:ix-checkbox-01", paperclipUpsertIssueDocument: "eval:dp-plan-doc-01", paperclipRestoreIssueDocumentRevision: "eval:dp-base-revision-01", paperclipLinkIssueApproval: "eval:ap-board-approval-01", paperclipUnlinkIssueApproval: "eval:ap-board-approval-01", paperclipApprovalDecision: "eval:ap-approval-wake-01", paperclipAddApprovalComment: "eval:ap-approval-deny-01", paperclipApiRequest: "eval:rf-api-404-report-01", }; const sourceHeader = "# GENERATED FILE — DO NOT EDIT. Run pnpm generate:capability-inventory.\n"; export function encodeInventory(value) { return `${sourceHeader}${JSON.stringify(value, null, 2)}\n`; } export function decodeInventory(source) { const json = source.startsWith(sourceHeader) ? source.slice(sourceHeader.length) : source; return JSON.parse(json); } function slug(value) { return value .toLowerCase() .replace(/[`*_]/g, "") .replace(/[^a-z0-9]+/g, "-") .replace(/(^-|-$)/g, ""); } function classifySkillHeading(heading) { const value = heading.toLowerCase(); if (/(authentication|identity|scope|checkout|lock|retry|dedupe|budget|wake|inbox|pick work|status quick guide|error handling|server-verified external chat)/.test(value)) { return "control_plane_owned"; } if (/(artifact|work product|comment|document|confirmation|question|approval follow-up|block|review|final disposition|issue lifecycle)/.test(value)) { return "always_agent_tool"; } return "optional_agent_tool"; } function parseSkillHeadings(text, file) { return text.split("\n").flatMap((line, index) => { const match = /^(#{1,4})\s+(.+?)\s*$/.exec(line); if (!match) return []; const title = match[2]; return [{ id: `skill:${file}:${slug(title) || "heading"}:${index + 1}`, sourceKind: "skill_heading", sourceAnchor: `${file}:${index + 1}`, title, expectedSemantics: `Skill guidance headed “${title}”.`, primaryDisposition: classifySkillHeading(title), requiredGrants: [], assertionClasses: ["control_plane_invariant"], evidenceIds: [`skill:${file}:${index + 1}`], }]; }); } function parseCase(source, sourceAnchor) { const id = /^id:\s*(.+)$/m.exec(source)?.[1]?.trim(); const title = /^title:\s*(.+)$/m.exec(source)?.[1]?.trim(); const group = /^group:\s*(\d+)$/m.exec(source)?.[1]; const fixture = /^fixture:\s*(.+)$/m.exec(source)?.[1]?.trim() ?? null; if (!id || !title || !group) throw new Error(`Invalid eval case at ${sourceAnchor}.`); const prefix = id.split("-")[0]; const policy = groupPolicies[prefix]; if (!policy) throw new Error(`Unknown eval group ${prefix} in ${sourceAnchor}.`); const expected = [...source.matchAll(/^\s+- method:\s*(\S+)\n\s+path(?:_pattern)?:\s*(.+)$/gm)] .map((match) => `${match[1]} ${match[2].trim()}`); const forbidden = [...source.matchAll(/^forbid:\s*$([\s\S]*?)(?=^[a-z_]+:|\n*$)/gm)] .flatMap((match) => [...match[1].matchAll(/method:\s*(\S+)/g)].map((method) => method[1])); const skillElements = [...source.matchAll(/- file:\s*(.+)\n\s+section:\s*"?(.+?)"?\n\s+lines:\s*"?(.+?)"?$/gm)] .map((match) => ({ file: match[1].trim(), section: match[2].trim(), lines: match[3].trim() })); return { id, title, group: prefix, legacyGroup: Number(group), sourceKind: "eval_case", sourceAnchor, fixture, skillElements, expectedSemantics: expected.length > 0 ? expected : ["No required HTTP call; evaluate the stated restraint or final state."], forbiddenSemantics: forbidden, primaryDisposition: policy[0], requiredGrants: policy[2], assertionClasses: policy[1], evidenceIds: [`eval:${id}`], }; } function parseMcpTools(source) { return [...source.matchAll(/makeTool\(\s*"([^"]+)"\s*,\s*"([^"]+)"/g)].map((match) => { const [name, description] = [match[1], match[2]]; const foldedInto = legacyMcpFoldTargets[name]; return { id: `mcp:${name}`, name, description, sourceKind: "legacy_mcp_alias", sourceAnchor: `packages/mcp-server/src/tools.ts:${source.slice(0, match.index).split("\n").length}`, expectedSemantics: description, foldedInto, evidenceId: `mcp:${name}`, }; }); } function foldLegacyMcpAliases(evaluations, legacyMcpAliases) { const aliasesByTarget = new Map(); for (const alias of legacyMcpAliases.rows) { const aliases = aliasesByTarget.get(alias.foldedInto) ?? []; aliases.push(alias); aliasesByTarget.set(alias.foldedInto, aliases); } return { ...evaluations, rows: evaluations.rows.map((row) => { const aliases = aliasesByTarget.get(`eval:${row.id}`) ?? []; if (aliases.length === 0) return row; return { ...row, legacyMcpAliases: aliases.map((alias) => alias.id), evidenceIds: [...row.evidenceIds, ...aliases.map((alias) => alias.evidenceId)], }; }), }; } export async function buildInventories({ repoRoot, evalRoot }) { const capabilities = await buildSkillInventory(repoRoot); const evalDirectory = resolve(evalRoot, "skills/paperclip/tests/cases"); const { readdir } = await import("node:fs/promises"); const caseNames = (await readdir(evalDirectory)).filter((name) => name.endsWith(".yaml")).sort(); const evalRows = await Promise.all(caseNames.map(async (name) => parseCase( await readFile(resolve(evalDirectory, name), "utf8"), `paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases/${name}`, ))); const legacyMcpAliases = await buildMcpInventory(repoRoot); const evaluations = foldLegacyMcpAliases({ schemaVersion: 2, inventoryRole: "normative", generatedFrom: "paperclip-evals/paperclip-skill-optimization/skills/paperclip/tests/cases", rows: evalRows, }, legacyMcpAliases); return { capabilities, evaluations, legacyMcpAliases, }; } export async function buildSkillInventory(repoRoot) { const skillRoot = resolve(repoRoot, "skills/paperclip"); const skillFiles = ["SKILL.md", "references/artifacts.md", "references/cases.md", "references/company-skills.md", "references/issue-workspaces.md", "references/routines.md", "references/workflows.md", "references/api-reference.md"]; const rows = (await Promise.all(skillFiles.map(async (file) => parseSkillHeadings( await readFile(resolve(skillRoot, file), "utf8"), `skills/paperclip/${file}`, )))).flat(); return { schemaVersion: 2, inventoryRole: "normative", generatedFrom: skillFiles.map((file) => `skills/paperclip/${file}`), rows, }; } export async function buildMcpInventory(repoRoot) { return { schemaVersion: 2, inventoryRole: "legacy_alias_index", generatedFrom: "packages/mcp-server/src/tools.ts", normativeSources: ["capabilities", "evaluations"], rows: parseMcpTools(await readFile(resolve(repoRoot, "packages/mcp-server/src/tools.ts"), "utf8")), }; } export function validateInventories(inventories) { const errors = []; const expectedCounts = { capabilities: 155, evaluations: 106, legacyMcpAliases: 42 }; const normativeNames = ["capabilities", "evaluations"]; const normativeRows = new Map(); const globalNormativeIds = new Set(); for (const name of normativeNames) { const inventory = inventories[name]; if (!inventory) { errors.push(`Missing normative inventory ${name}.`); continue; } if (inventory.schemaVersion !== 2 || inventory.inventoryRole !== "normative") { errors.push(`${name} must be a schema version 2 normative inventory.`); } const seen = new Set(); for (const row of inventory.rows) { if (!row.id || seen.has(row.id)) errors.push(`${name} contains a missing or duplicate id: ${row.id ?? ""}.`); seen.add(row.id); const normativeId = name === "evaluations" ? `eval:${row.id}` : row.id; if (globalNormativeIds.has(normativeId)) errors.push(`Normative row ${normativeId} is multiply classified.`); globalNormativeIds.add(normativeId); normativeRows.set(normativeId, row); if (!["control_plane_owned", "always_agent_tool", "optional_agent_tool"].includes(row.primaryDisposition)) { errors.push(`${name}:${row.id} has invalid primaryDisposition.`); } if (!row.sourceAnchor || !row.expectedSemantics || !Array.isArray(row.requiredGrants) || !Array.isArray(row.assertionClasses) || !Array.isArray(row.evidenceIds)) { errors.push(`${name}:${row.id} is missing normalized traceability fields.`); } } if (expectedCounts[name] && inventory.rows.length !== expectedCounts[name]) { errors.push(`${name} expected ${expectedCounts[name]} rows but found ${inventory.rows.length}.`); } } const groups = new Set((inventories.evaluations?.rows ?? []).map((row) => row.group)); for (const group of capabilityGroups) if (!groups.has(group)) errors.push(`evaluations is missing group ${group}.`); if (groups.size !== capabilityGroups.length) errors.push(`evaluations has unexpected groups: ${[...groups].sort().join(", ")}.`); const legacyMcpAliases = inventories.legacyMcpAliases; if (!legacyMcpAliases) { errors.push("Missing legacy MCP alias index."); return errors; } if (legacyMcpAliases.schemaVersion !== 2 || legacyMcpAliases.inventoryRole !== "legacy_alias_index") { errors.push("legacyMcpAliases must be a schema version 2 legacy alias index."); } const seenAliases = new Set(); for (const alias of legacyMcpAliases.rows) { if (!alias.id || seenAliases.has(alias.id)) { errors.push(`legacyMcpAliases contains a missing or duplicate id: ${alias.id ?? ""}.`); } seenAliases.add(alias.id); if (Object.hasOwn(alias, "primaryDisposition")) { errors.push(`legacyMcpAliases:${alias.id} must not define an independent primaryDisposition.`); } if (!alias.sourceAnchor || !alias.expectedSemantics || !alias.foldedInto || !alias.evidenceId) { errors.push(`legacyMcpAliases:${alias.id} is missing folded traceability fields.`); continue; } const target = normativeRows.get(alias.foldedInto); if (!target) { errors.push(`legacyMcpAliases:${alias.id} folds into unknown normative row ${alias.foldedInto}.`); continue; } const targetAliases = target.legacyMcpAliases ?? []; if (targetAliases.filter((id) => id === alias.id).length !== 1) { errors.push(`legacyMcpAliases:${alias.id} must be folded exactly once into ${alias.foldedInto}.`); } if (target.evidenceIds.filter((id) => id === alias.evidenceId).length !== 1) { errors.push(`legacyMcpAliases:${alias.id} evidence must be folded exactly once into ${alias.foldedInto}.`); } } if (legacyMcpAliases.rows.length !== expectedCounts.legacyMcpAliases) { errors.push(`legacyMcpAliases expected ${expectedCounts.legacyMcpAliases} rows but found ${legacyMcpAliases.rows.length}.`); } for (const [normativeId, row] of normativeRows) { for (const aliasId of row.legacyMcpAliases ?? []) { if (!seenAliases.has(aliasId)) errors.push(`${normativeId} references unknown legacy MCP alias ${aliasId}.`); } } return errors; } export function validateInventorySchema(inventories, schema) { const ajv = new Ajv2020({ allErrors: true, allowUnionTypes: true, strict: true, strictRequired: false, }); const validate = ajv.compile(schema); const errors = []; for (const [name, inventory] of Object.entries(inventories)) { if (validate(inventory)) continue; for (const error of validate.errors ?? []) { errors.push(`${name}${error.instancePath || "/"} ${error.message ?? "failed schema validation"}.`); } } return errors; } export function renderContractModule(inventories) { const counts = { skillReferenceCapabilities: inventories.capabilities.rows.length, evalCases: inventories.evaluations.rows.length, normativeRows: inventories.capabilities.rows.length + inventories.evaluations.rows.length, legacyMcpAliases: inventories.legacyMcpAliases.rows.length, }; return `// Generated by scripts/generate-capability-inventory.mjs. Do not edit.\n\nexport type CapabilityPrimaryDisposition =\n | "control_plane_owned"\n | "always_agent_tool"\n | "optional_agent_tool";\n\nexport const capabilityInventoryCounts = ${JSON.stringify(counts, null, 2)} as const;\n\nexport const capabilityEvalGroups = ${JSON.stringify(capabilityGroups, null, 2)} as const;\n\nexport const capabilityInventoryContract = {\n schemaVersion: 2,\n normativeInventories: [\n "spec/capability/capabilities.yaml",\n "spec/capability/eval-traceability.yaml",\n ],\n legacyAliasIndexes: ["spec/capability/mcp-tool-map.yaml"],\n primaryDispositions: [\n "control_plane_owned",\n "always_agent_tool",\n "optional_agent_tool",\n ] as const,\n} as const;\n`; } export function renderDocumentation(inventories) { const groupCounts = Object.fromEntries(capabilityGroups.map((group) => [group, inventories.evaluations.rows.filter((row) => row.group === group).length])); const capabilityLines = inventories.capabilities.rows.map((row) => `| ${row.id} | ${row.primaryDisposition} | ${row.sourceAnchor} |`).join("\n"); const mcpLines = inventories.legacyMcpAliases.rows.map((alias) => { const target = inventories.evaluations.rows.find((row) => `eval:${row.id}` === alias.foldedInto); return `| ${alias.name} | ${alias.foldedInto} | ${target?.primaryDisposition ?? "unknown"} | ${alias.sourceAnchor} |`; }).join("\n"); return `\n\n# Capability Capability Contract\n\nThis generated contract is a self-contained derivative of the Paperclip skill, its seven references, the Paperclip Evals corpus, and the legacy MCP tool surface. It does not import or contact the Paperclip control plane.\n\nThe skill/reference inventory and eval cases are the only normative behavior sources. Paperclip does not use the legacy MCP calls as a production capability surface; all MCP names below are traceability aliases folded into normative eval rows. Their disposition, grants, assertions, and evidence contract are inherited from the target row rather than classified independently.\n\n## Baseline Counts\n\n- Skill/reference headings: ${inventories.capabilities.rows.length}\n- Eval cases: ${inventories.evaluations.rows.length} across ${capabilityGroups.length} groups\n- Total normative rows: ${inventories.capabilities.rows.length + inventories.evaluations.rows.length}\n- Legacy MCP aliases folded into normative rows: ${inventories.legacyMcpAliases.rows.length}\n\n| Eval group | Cases |\n| --- | ---: |\n${Object.entries(groupCounts).map(([group, count]) => `| ${group} | ${count} |`).join("\n")}\n\n## Regeneration\n\n- \`pnpm --dir packages/paperclip-runner generate:capability-inventory\` imports the canonical baselines and rewrites every generated file.\n- \`pnpm --dir packages/paperclip-runner check:capability-inventory\` validates counts, uniqueness, normative dispositions, one-to-one MCP folds, required fields, and generated-file drift without requiring the external eval repository.\n\n## Skill / Reference Rows\n\n| Capability | Primary disposition | Source anchor |\n| --- | --- | --- |\n${capabilityLines}\n\n## Legacy MCP Alias Index\n\nThis is a compatibility/traceability index, not a tool catalog. “Inherited disposition” is shown only to make the normative target easy to audit.\n\n| Legacy MCP name | Folded into normative row | Inherited disposition | Source anchor |\n| --- | --- | --- | --- |\n${mcpLines}\n`; } export function packageRelative(repoRoot, path) { return relative(repoRoot, path).replaceAll("\\", "/"); }