diff --git a/server/src/__tests__/heartbeat-runtime-mcp-servers.test.ts b/server/src/__tests__/heartbeat-runtime-mcp-servers.test.ts index d0dc7082dc..c93d328d5f 100644 --- a/server/src/__tests__/heartbeat-runtime-mcp-servers.test.ts +++ b/server/src/__tests__/heartbeat-runtime-mcp-servers.test.ts @@ -58,7 +58,7 @@ describeEmbeddedPostgres("heartbeat runtime MCP servers", () => { await tempDb?.cleanup(); }); - it("provisions one aggregate gateway and filters degraded access without blocking direct adapters", async () => { + it("provisions one aggregate gateway and omits unavailable access without blocking any runtime", async () => { process.env.PAPERCLIP_API_URL = "https://paperclip.example.test"; const [company] = await db.insert(companies).values({ name: `Runtime MCP ${randomUUID()}`, @@ -158,22 +158,35 @@ describeEmbeddedPostgres("heartbeat runtime MCP servers", () => { } expect(JSON.stringify(tokens)).not.toContain(first[0]!.token); - await db.update(toolConnections) - .set({ healthStatus: "degraded", healthMessage: "fixture unavailable" }) - .where(eq(toolConnections.id, installedConnection!.id)); - await expect( - buildPaperclipRuntimeMcpServers({ db, agent: agent!, runId: randomUUID() }), - ).resolves.toEqual([]); await expect( buildPaperclipRuntimeMcpServers({ db, agent: agent!, runId: randomUUID(), - failOnUnavailableAssignedConnection: true, + expectedAssignmentDigest: "0".repeat(64), }), - ).rejects.toThrow( - `assigned native MCP connection is unavailable: ${installedConnection!.id}`, - ); + ).resolves.toEqual([]); + expect(await db.select().from(toolMcpGatewayTokens)).toHaveLength(2); + + await db.update(toolConnections) + .set({ healthStatus: "degraded", healthMessage: "fixture unavailable" }) + .where(eq(toolConnections.id, installedConnection!.id)); + const unavailableReports: Array> = []; + await expect( + buildPaperclipRuntimeMcpServers({ + db, + agent: agent!, + runId: randomUUID(), + expectedAssignmentDigest: first[0]!.connectionId.slice("assignment:".length), + onUnavailableAssignedConnections: (connections) => { + unavailableReports.push(connections); + }, + }), + ).resolves.toEqual([]); + expect(unavailableReports).toEqual([[ + { id: installedConnection!.id, name: installedConnection!.name }, + ]]); + expect(await db.select().from(toolMcpGatewayTokens)).toHaveLength(2); await expect( createManagedMcpRunConfig({ db, diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 503c350d08..20a0e3a0d6 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -31,6 +31,7 @@ import { PROVIDER_QUOTA_MONITOR_SERVICE_NAME, envBindingSchema, isEnvironmentDriverSupportedForAdapter, + isToolConnectionAttentionHealth, type BillingType, type CostStatus, type EnvironmentLeaseStatus, @@ -3989,7 +3990,10 @@ export async function buildPaperclipRuntimeMcpServers(input: { db: Db; agent: Pick; runId: string; - failOnUnavailableAssignedConnection?: boolean; + expectedAssignmentDigest?: string | null; + onUnavailableAssignedConnections?: ( + connections: Array<{ id: string; name: string }>, + ) => void | Promise; }): Promise { const access = toolAccessService(input.db); const effective = await access.getEffectiveProfilesForAgent( @@ -4025,19 +4029,39 @@ export async function buildPaperclipRuntimeMcpServers(input: { permittedConnectionIds.has(connection.id) && connection.status === "active" && connection.enabled - && !["degraded", "failed", "error", "missing_secret"].includes(connection.healthStatus) + && !isToolConnectionAttentionHealth(connection.healthStatus) && (connection.transport === "mcp_remote" || connection.transport === "local_stdio") ); const unhealthyConnections = effective.installedConnections.filter((connection) => permittedConnectionIds.has(connection.id) && (connection.transport === "mcp_remote" || connection.transport === "local_stdio") - && (!connection.enabled || connection.status !== "active" || ["degraded", "failed", "error", "missing_secret"].includes(connection.healthStatus)), + && (!connection.enabled || connection.status !== "active" || isToolConnectionAttentionHealth(connection.healthStatus)), ); - if (input.failOnUnavailableAssignedConnection && unhealthyConnections.length) { - throw new Error( - `assigned native MCP connection is unavailable: ${unhealthyConnections.map((connection) => connection.id).join(", ")}`, - ); + if (unhealthyConnections.length && input.onUnavailableAssignedConnections) { + try { + await input.onUnavailableAssignedConnections( + unhealthyConnections + .map(({ id, name }) => ({ id, name })) + .sort((a, b) => a.name.localeCompare(b.name)), + ); + } catch (error) { + logger.warn( + { + companyId: input.agent.companyId, + agentId: input.agent.id, + runId: input.runId, + err: error, + }, + "failed to report unavailable runtime MCP connections", + ); + } } + const assignedConnectionIds = new Set( + assignedConnections.map((connection) => connection.id), + ); + const assignedTools = effective.allowedTools.filter((tool) => + assignedConnectionIds.has(tool.connectionId) + ); const service = createToolGatewayService(input.db); if (assignedConnections.length === 0) { await service.recordRuntimeMcpDeliveryDiagnostic({ @@ -4052,11 +4076,19 @@ export async function buildPaperclipRuntimeMcpServers(input: { version: 1, agentId: input.agent.id, connections: assignedConnections.map((connection) => connection.id).sort(), - tools: effective.allowedTools.map((tool) => tool.id).sort(), + tools: assignedTools.map((tool) => tool.id).sort(), }; const assignmentDigest = createHash("sha256") .update(JSON.stringify(assignment)) .digest("hex"); + // Native runs may lose access after their immutable context is captured, but + // they must never gain a new or changed assignment during dispatch. + if ( + input.expectedAssignmentDigest !== undefined + && input.expectedAssignmentDigest !== assignmentDigest + ) { + return []; + } const profileKey = `native:${input.agent.id}:${assignmentDigest}`; let [profile] = await input.db .select() @@ -4082,7 +4114,7 @@ export async function buildPaperclipRuntimeMcpServers(input: { applicationId: connection.applicationId, connectionId: connection.id, })), - ...effective.allowedTools + ...assignedTools .filter((tool) => !fullConnectionIds.has(tool.connectionId)) .map((tool) => ({ selectorType: "catalog_entry" as const, @@ -20522,20 +20554,29 @@ export function heartbeatService( if (nativeRuntimeResolution.kind === "native") { if (!nativeExecution || !nativeRunnerInstanceId) throw new Error("native_runtime_selection_not_persisted"); + const expectedNativeMcpDigest = + "runtimeContext" in nativeExecution + && nativeExecution.runtimeContext.mcp.bindingId + ? nativeExecution.runtimeContext.mcp.digest + : null; const nativeMcpServers = await buildPaperclipRuntimeMcpServers({ db, agent, runId: run.id, - failOnUnavailableAssignedConnection: true, + expectedAssignmentDigest: expectedNativeMcpDigest, + onUnavailableAssignedConnections: async (connections) => { + const names = connections.map((connection) => connection.name).join(", "); + await onLog( + "stderr", + `[paperclip] App connection${connections.length === 1 ? "" : "s"} unavailable: ${names}. Continuing this run without ${connections.length === 1 ? "it" : "them"}; reconnect from Apps to restore access.\n`, + ); + }, }); - if (!("runtimeContext" in nativeExecution) && nativeMcpServers.length) { - throw new Error("historical native runs cannot acquire newly assigned MCP access"); - } if ("runtimeContext" in nativeExecution) { if (nativeMcpServers.length > 1) throw new Error("native MCP realization must produce one aggregate gateway"); const server = nativeMcpServers[0] ?? null; const digest = server?.connectionId.startsWith("assignment:") ? server.connectionId.slice("assignment:".length) : null; - if (digest !== (nativeExecution.runtimeContext.mcp.bindingId ? nativeExecution.runtimeContext.mcp.digest : null)) { + if (digest && digest !== expectedNativeMcpDigest) { throw new Error("native MCP assignment digest mismatch"); } } diff --git a/server/src/services/native-runtime/runtime-context.test.ts b/server/src/services/native-runtime/runtime-context.test.ts index 475b249ecc..0f4db322dd 100644 --- a/server/src/services/native-runtime/runtime-context.test.ts +++ b/server/src/services/native-runtime/runtime-context.test.ts @@ -83,6 +83,102 @@ afterEach(async () => { }); describe("buildNativeRuntimeContext", () => { + it.each(["disabled", "degraded"] as const)( + "omits an unavailable native MCP connection when it is %s without aborting runtime context creation", + async (unavailableState) => { + serviceMocks.exportFiles.mockResolvedValue({ + entryFile: "AGENTS.md", + files: { "AGENTS.md": "Continue work without unavailable apps.\n" }, + }); + serviceMocks.getEffectiveProfilesForAgent.mockResolvedValue({ + agentId: "agent-1", + profiles: [], + entries: [{ effect: "include", connectionId: "connection-1" }], + bindings: [], + allowedTools: [{ id: "tool-1", connectionId: "connection-1" }], + allowedToolNames: ["issues.read"], + installedConnections: [{ + id: "connection-1", + transport: "mcp_remote", + enabled: unavailableState !== "disabled", + status: unavailableState === "disabled" ? "disabled" : "active", + healthStatus: unavailableState === "degraded" ? "degraded" : "healthy", + }], + }); + + const context = await buildNativeRuntimeContext({ + db: {} as Db, + agent: { + id: "agent-1", + companyId: "company-1", + name: "Reviewer", + adapterType: "paperclip_runner", + adapterConfig: {}, + }, + runId: "run-1", + runtimeConfig: {}, + runtimeSkillEntries: [], + }); + + expect(context.mcp.bindingId).toBeNull(); + expect(context.mcp.assignmentSetId).toMatch(/^sha256:[a-f0-9]{64}$/); + }, + ); + + it("keeps healthy native MCP connections when another assigned connection is unavailable", async () => { + serviceMocks.exportFiles.mockResolvedValue({ + entryFile: "AGENTS.md", + files: { "AGENTS.md": "Continue work with the apps that are available.\n" }, + }); + serviceMocks.getEffectiveProfilesForAgent.mockResolvedValue({ + agentId: "agent-1", + profiles: [], + entries: [ + { effect: "include", connectionId: "connection-expired" }, + { effect: "include", connectionId: "connection-healthy" }, + ], + bindings: [], + allowedTools: [ + { id: "tool-expired", connectionId: "connection-expired" }, + { id: "tool-healthy", connectionId: "connection-healthy" }, + ], + allowedToolNames: ["expired.read", "healthy.read"], + installedConnections: [ + { + id: "connection-expired", + transport: "mcp_remote", + enabled: true, + status: "active", + healthStatus: "degraded", + }, + { + id: "connection-healthy", + transport: "mcp_remote", + enabled: true, + status: "active", + healthStatus: "healthy", + }, + ], + }); + + const context = await buildNativeRuntimeContext({ + db: {} as Db, + agent: { + id: "agent-1", + companyId: "company-1", + name: "Reviewer", + adapterType: "paperclip_runner", + adapterConfig: {}, + }, + runId: "run-1", + runtimeConfig: {}, + runtimeSkillEntries: [], + }); + + expect(context.mcp.bindingId).toBe("native-mcp:run-1"); + expect(context.mcp.assignmentSetId).toMatch(/^sha256:[a-f0-9]{64}$/); + }); + it("materializes every instruction and selected-skill file as immutable, content-addressed context", async () => { serviceMocks.exportFiles.mockResolvedValue({ entryFile: "AGENTS.md", diff --git a/server/src/services/native-runtime/runtime-context.ts b/server/src/services/native-runtime/runtime-context.ts index ea84917a4e..7bc3abbe63 100644 --- a/server/src/services/native-runtime/runtime-context.ts +++ b/server/src/services/native-runtime/runtime-context.ts @@ -3,6 +3,7 @@ import fs from "node:fs/promises"; import path from "node:path"; import type { Db } from "@paperclipai/db"; import type { PaperclipSkillEntry } from "@paperclipai/adapter-utils/server-utils"; +import { isToolConnectionAttentionHealth } from "@paperclipai/shared"; import { PAPERCLIP_OPERATIONAL_SKILL_KEY, resolvePaperclipDesiredSkillNames, @@ -165,17 +166,23 @@ async function materializeSelectedSkills(runtimeConfig: Record, export async function resolveNativeRuntimeMcpSnapshot(input: { db: Db; agent: Pick; runId: string }) { const effective = await toolAccessService(input.db).getEffectiveProfilesForAgent(input.agent.companyId, input.agent.id); const permitted = new Set([...effective.entries.filter((entry) => entry.effect === "include" && entry.connectionId).map((entry) => entry.connectionId!), ...effective.allowedTools.map((tool) => tool.connectionId)]); - const unhealthy = effective.installedConnections.filter((connection) => + // App access is optional runtime context. Keep usable assignments pinned, but + // do not stop unrelated work because an assigned app needs attention. + const availableConnectionIds = new Set(effective.installedConnections.filter((connection) => permitted.has(connection.id) + && connection.status === "active" + && connection.enabled + && !isToolConnectionAttentionHealth(connection.healthStatus) && ["mcp_remote", "local_stdio"].includes(connection.transport) - && (!connection.enabled || connection.status !== "active" || ["degraded", "failed", "error", "missing_secret"].includes(connection.healthStatus)), - ); - if (unhealthy.length) throw new Error(`assigned native MCP connection is unavailable: ${unhealthy.map((connection) => connection.id).join(", ")}`); + ).map((connection) => connection.id)); const assignment = { version: 1, agentId: input.agent.id, - connections: effective.installedConnections.filter((connection) => permitted.has(connection.id) && connection.status === "active" && connection.enabled && ["mcp_remote", "local_stdio"].includes(connection.transport)).map((connection) => connection.id).sort(), - tools: effective.allowedTools.map((tool) => tool.id).sort(), + connections: [...availableConnectionIds].sort(), + tools: effective.allowedTools + .filter((tool) => availableConnectionIds.has(tool.connectionId)) + .map((tool) => tool.id) + .sort(), }; const assignmentDigest = sha256(JSON.stringify(assignment)); return { assignmentSetId: `sha256:${assignmentDigest}`, digest: assignmentDigest, bindingId: assignment.connections.length ? `native-mcp:${input.runId}` : null };