import { randomUUID } from "node:crypto"; import os from "node:os"; import path from "node:path"; import { promises as fs } from "node:fs"; import { eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; import { agents, companies, companySkills, createDb, toolApplications, toolConnectionInstalls, toolConnections, toolProfileBindings, toolProfileEntries, toolProfiles, } from "@paperclipai/db"; import type { AdapterRuntimeMcpServer } from "@paperclipai/adapter-utils"; import type { PaperclipSkillEntry } from "@paperclipai/adapter-utils/server-utils"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase, } from "./helpers/embedded-postgres.js"; import { companySkillService } from "../services/company-skills.ts"; import { heartbeatService } from "../services/heartbeat.ts"; import { instanceSettingsService } from "../services/instance-settings.ts"; import { registerServerAdapter, unregisterServerAdapter } from "../adapters/index.ts"; const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; const TEST_ADAPTER_TYPE = "runtime_skill_capture"; if (!embeddedPostgresSupport.supported) { console.warn( `Skipping embedded Postgres heartbeat runtime skill tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`, ); } async function waitForRunToFinish( heartbeat: ReturnType, runId: string, timeoutMs = 5_000, ) { const deadline = Date.now() + timeoutMs; while (Date.now() < deadline) { const run = await heartbeat.getRun(runId); if (run && !["queued", "running"].includes(run.status)) return run; await new Promise((resolve) => setTimeout(resolve, 50)); } return await heartbeat.getRun(runId); } describeEmbeddedPostgres("heartbeat runtime skill version pins", () => { let db!: ReturnType; let tempDb: Awaited> | null = null; let oldPaperclipHome: string | undefined; let oldPaperclipApiUrl: string | undefined; let paperclipHome: string | null = null; const capturedRuns: Array<{ agentId: string; skills: PaperclipSkillEntry[]; mcpServers: AdapterRuntimeMcpServer[]; config: Record; serializedRuntimeInput: string; }> = []; const cleanupDirs = new Set(); beforeAll(async () => { tempDb = await startEmbeddedPostgresTestDatabase("heartbeat-runtime-skills-"); db = createDb(tempDb.connectionString); oldPaperclipHome = process.env.PAPERCLIP_HOME; paperclipHome = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-runtime-skills-home-")); process.env.PAPERCLIP_HOME = paperclipHome; // The server normalizes PAPERCLIP_API_URL into its own env at boot // (server/src/index.ts); heartbeat gateway delivery requires it, so pin // a deterministic value for tests that never boot the full server. oldPaperclipApiUrl = process.env.PAPERCLIP_API_URL; process.env.PAPERCLIP_API_URL = "http://127.0.0.1:3100/api"; registerServerAdapter({ type: TEST_ADAPTER_TYPE, execute: async (ctx) => { const serializedRuntimeInput = JSON.stringify({ config: ctx.config, context: ctx.context, runtimeMcp: ctx.runtimeMcp, }); await ctx.onLog("stdout", `${serializedRuntimeInput}\n`); capturedRuns.push({ agentId: ctx.agent.id, skills: (ctx.config.paperclipRuntimeSkills ?? []) as PaperclipSkillEntry[], mcpServers: ctx.runtimeMcp?.getServers() ?? [], config: ctx.config, serializedRuntimeInput, }); return { exitCode: 0, signal: null, timedOut: false, label: "Captured runtime skills", }; }, testEnvironment: async () => ({ adapterType: TEST_ADAPTER_TYPE, status: "pass", checks: [], testedAt: new Date().toISOString(), }), }); }, 20_000); afterEach(async () => { capturedRuns.length = 0; await instanceSettingsService(db).updateExperimental({ enableBetaSkills: false }); await new Promise((resolve) => setTimeout(resolve, 100)); await db.execute(sql.raw(` TRUNCATE TABLE "activity_log", "environment_leases", "environments", "heartbeat_run_events", "heartbeat_runs", "agent_wakeup_requests", "agent_runtime_state", "company_skill_versions", "company_skills", "agents", "companies" RESTART IDENTITY CASCADE `)); await Promise.all(Array.from(cleanupDirs, (dir) => fs.rm(dir, { recursive: true, force: true }))); cleanupDirs.clear(); }); afterAll(async () => { unregisterServerAdapter(TEST_ADAPTER_TYPE); if (oldPaperclipHome === undefined) delete process.env.PAPERCLIP_HOME; else process.env.PAPERCLIP_HOME = oldPaperclipHome; if (oldPaperclipApiUrl === undefined) delete process.env.PAPERCLIP_API_URL; else process.env.PAPERCLIP_API_URL = oldPaperclipApiUrl; if (paperclipHome) { await fs.rm(paperclipHome, { recursive: true, force: true }); } await tempDb?.cleanup(); }); it("materializes different pinned skill versions for different agents at runtime", async () => { const companyId = randomUUID(); const skillId = randomUUID(); const firstAgentId = randomUUID(); const secondAgentId = randomUUID(); const issuePrefix = `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`; const skillKey = `company/${companyId}/runtime-coach`; const skillDir = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-versioned-runtime-skill-")); cleanupDirs.add(skillDir); await db.insert(companies).values({ id: companyId, name: "Paperclip", issuePrefix, requireBoardApprovalForNewAgents: false, defaultResponsibleUserId: "responsible-user", }); await fs.writeFile(path.join(skillDir, "SKILL.md"), "# Runtime Coach\n\nVersion one.\n", "utf8"); await db.insert(companySkills).values({ id: skillId, companyId, key: skillKey, slug: "runtime-coach", name: "Runtime Coach", description: null, markdown: "# Runtime Coach\n\nVersion one.\n", sourceType: "local_path", sourceLocator: skillDir, trustLevel: "markdown_only", compatibility: "compatible", fileInventory: [{ path: "SKILL.md", kind: "skill" }], metadata: { sourceKind: "local_path" }, }); const skills = companySkillService(db); const versionOne = await skills.createVersion( companyId, skillId, { label: "v1" }, { type: "user", userId: "board" }, ); await fs.writeFile(path.join(skillDir, "SKILL.md"), "# Runtime Coach\n\nVersion two.\n", "utf8"); await db .update(companySkills) .set({ markdown: "# Runtime Coach\n\nVersion two.\n", updatedAt: new Date() }) .where(eq(companySkills.id, skillId)); const versionTwo = await skills.createVersion( companyId, skillId, { label: "v2" }, { type: "user", userId: "board" }, ); await db.insert(agents).values([ { id: firstAgentId, companyId, name: "Pinned V1", role: "engineer", status: "idle", adapterType: TEST_ADAPTER_TYPE, adapterConfig: { paperclipSkillSync: { desiredSkills: [{ key: skillKey, versionId: versionOne.id }], }, }, runtimeConfig: {}, permissions: {}, }, { id: secondAgentId, companyId, name: "Pinned V2", role: "engineer", status: "idle", adapterType: TEST_ADAPTER_TYPE, adapterConfig: { paperclipSkillSync: { desiredSkills: [{ key: skillKey, versionId: versionTwo.id }], }, }, runtimeConfig: {}, permissions: {}, }, ]); const settings = instanceSettingsService(db); await settings.updateExperimental({ enableBetaSkills: true }); const heartbeat = heartbeatService(db); const firstRun = await heartbeat.invoke(firstAgentId, "on_demand", {}, "manual"); expect(firstRun).not.toBeNull(); expect((await waitForRunToFinish(heartbeat, firstRun!.id))?.status).toBe("succeeded"); const secondRun = await heartbeat.invoke(secondAgentId, "on_demand", {}, "manual"); expect(secondRun).not.toBeNull(); expect((await waitForRunToFinish(heartbeat, secondRun!.id))?.status).toBe("succeeded"); const firstSkill = capturedRuns.find((run) => run.agentId === firstAgentId)?.skills .find((entry) => entry.key === skillKey); const secondSkill = capturedRuns.find((run) => run.agentId === secondAgentId)?.skills .find((entry) => entry.key === skillKey); expect(firstSkill).toMatchObject({ key: skillKey, versionId: versionOne.id, currentVersionId: versionTwo.id, sourceStatus: "available", }); expect(secondSkill).toMatchObject({ key: skillKey, versionId: versionTwo.id, currentVersionId: versionTwo.id, sourceStatus: "available", }); await expect(fs.readFile(path.join(firstSkill!.source, "SKILL.md"), "utf8")) .resolves.toContain("Version one."); await expect(fs.readFile(path.join(secondSkill!.source, "SKILL.md"), "utf8")) .resolves.toContain("Version two."); const firstSkillFile = path.join(firstSkill!.source, "SKILL.md"); const oldMtime = new Date("2024-01-01T00:00:00.000Z"); await fs.utimes(firstSkillFile, oldMtime, oldMtime); const repeatRun = await heartbeat.invoke(firstAgentId, "on_demand", {}, "manual"); expect(repeatRun).not.toBeNull(); expect((await waitForRunToFinish(heartbeat, repeatRun!.id))?.status).toBe("succeeded"); const repeatedSkill = capturedRuns .filter((run) => run.agentId === firstAgentId) .at(-1) ?.skills.find((entry) => entry.key === skillKey); expect(repeatedSkill).toMatchObject({ source: firstSkill!.source, versionId: versionOne.id, sourceStatus: "available", }); expect((await fs.stat(firstSkillFile)).mtime.toISOString()).toBe(oldMtime.toISOString()); await settings.updateExperimental({ enableBetaSkills: false }); const defaultRun = await heartbeat.invoke(firstAgentId, "on_demand", {}, "manual"); expect(defaultRun).not.toBeNull(); expect((await waitForRunToFinish(heartbeat, defaultRun!.id))?.status).toBe("succeeded"); const defaultSkill = capturedRuns .filter((run) => run.agentId === firstAgentId) .at(-1) ?.skills.find((entry) => entry.key === skillKey); expect(defaultSkill).toMatchObject({ key: skillKey, versionId: null, currentVersionId: versionTwo.id, sourceStatus: "available", }); await expect(fs.readFile(path.join(defaultSkill!.source, "SKILL.md"), "utf8")) .resolves.toContain("Version two."); const storedPreference = await db .select({ adapterConfig: agents.adapterConfig }) .from(agents) .where(eq(agents.id, firstAgentId)) .then((rows) => rows[0]?.adapterConfig); expect(storedPreference).toMatchObject({ paperclipSkillSync: { desiredSkills: [{ key: skillKey, versionId: versionOne.id }], }, }); await settings.updateExperimental({ enableBetaSkills: true }); const restoredRun = await heartbeat.invoke(firstAgentId, "on_demand", {}, "manual"); expect(restoredRun).not.toBeNull(); expect((await waitForRunToFinish(heartbeat, restoredRun!.id))?.status).toBe("succeeded"); const restoredSkill = capturedRuns .filter((run) => run.agentId === firstAgentId) .at(-1) ?.skills.find((entry) => entry.key === skillKey); expect(restoredSkill).toMatchObject({ versionId: versionOne.id, currentVersionId: versionTwo.id, sourceStatus: "available", }); await expect(fs.readFile(path.join(restoredSkill!.source, "SKILL.md"), "utf8")) .resolves.toContain("Version one."); }); it("delivers installed connections without exposing gateway bearers in adapter config or logs", async () => { const companyId = randomUUID(); const agentId = randomUUID(); await db.insert(companies).values({ id: companyId, name: "Runtime MCP Delivery", issuePrefix: `M${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`, requireBoardApprovalForNewAgents: false, defaultResponsibleUserId: "responsible-user", }); await db.insert(agents).values({ id: agentId, companyId, name: "Runtime MCP Capture", role: "engineer", status: "idle", adapterType: TEST_ADAPTER_TYPE, adapterConfig: {}, runtimeConfig: {}, permissions: {}, }); const [application] = await db.insert(toolApplications).values({ companyId, applicationKey: `runtime-${randomUUID().slice(0, 8)}`, name: "Runtime MCP", type: "mcp_http", status: "active", }).returning(); const [installed, uninstalled] = await db.insert(toolConnections).values([ { companyId, applicationId: application!.id, name: "Installed Runtime MCP", uid: `test/${randomUUID()}`, transport: "mcp_remote", status: "active", enabled: true, config: { url: "https://installed.example.test/mcp" }, }, { companyId, applicationId: application!.id, name: "Uninstalled Runtime MCP", uid: `test/${randomUUID()}`, transport: "mcp_remote", status: "active", enabled: true, config: { url: "https://uninstalled.example.test/mcp" }, }, ]).returning(); const [profile] = await db.insert(toolProfiles).values({ companyId, profileKey: `app:${installed!.id}`, name: installed!.name, defaultAction: "deny", }).returning(); await db.insert(toolProfileEntries).values({ companyId, profileId: profile!.id, selectorType: "connection", effect: "include", applicationId: application!.id, connectionId: installed!.id, }); await db.insert(toolProfileBindings).values({ companyId, profileId: profile!.id, targetType: "agent", targetId: agentId, }); await db.insert(toolConnectionInstalls).values({ companyId, connectionId: installed!.id, targetType: "agent", targetId: agentId, }); const heartbeat = heartbeatService(db); const run = await heartbeat.invoke(agentId, "on_demand", {}, "manual"); expect(run).not.toBeNull(); expect((await waitForRunToFinish(heartbeat, run!.id))?.status).toBe("succeeded"); const captured = capturedRuns.find((entry) => entry.agentId === agentId); expect(captured?.mcpServers).toHaveLength(1); expect(captured?.mcpServers[0]).toMatchObject({ connectionId: expect.stringMatching(/^assignment:[a-f0-9]{64}$/), name: "paperclip-assigned", token: expect.stringMatching(/^pcgw_/), url: expect.stringMatching(/\/mcp\/gateways\/gw_[a-f0-9]{32}$/), }); const runtimeProfiles = await db.select().from(toolProfiles); const runtimeProfile = runtimeProfiles.find((entry) => entry.profileKey.startsWith(`native:${agentId}:`) ); expect(runtimeProfile).toBeDefined(); const runtimeEntries = await db.select().from(toolProfileEntries) .where(eq(toolProfileEntries.profileId, runtimeProfile!.id)); expect(runtimeEntries.map((entry) => entry.connectionId)).toEqual([installed!.id]); expect(JSON.stringify(captured?.mcpServers)).not.toContain(uninstalled!.id); const bearer = captured?.mcpServers[0]?.token; expect(bearer).toMatch(/^pcgw_/); if (!bearer) throw new Error("Expected runtime MCP bearer"); expect(captured?.config).not.toHaveProperty("paperclipRuntimeMcpServers"); expect(JSON.stringify(captured?.config)).not.toContain(bearer); expect(captured?.serializedRuntimeInput).not.toContain(bearer); const log = await heartbeat.readLog(run!.id); expect(log.content).not.toContain(bearer); expect(log.content).not.toContain("pcgw_"); }); });