From 9c8adee48b15aa7d12becd767e461a83a306f552 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Tue, 14 Jul 2026 15:15:43 -0500 Subject: [PATCH] feat(mcp) [split 5/8]: integrate adapters and deployment runtime (#9560) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - Governed MCP access spans contracts, runtime enforcement, adapters, UI surfaces, and operator verification > - The parity reference PR #9534 is too large for effective automated or human review > - The feature therefore needs a linear stack whose individual diffs stay below the 100-file review limit > - This pull request is split 5/8 and focuses on remaining adapters, CLI, plugin examples, and deployment packaging > - The benefit is a standalone, testable review boundary while preserving byte-for-byte parity at the top of the stack ## Linked Issues or Issue Description - Related parity reference: #9534 - Problem: The backend runtime needs packaging, CLI propagation, worktree provisioning, release manifests, and remaining adapter/plugin consumers. - Proposed solution: Adds the remaining runtime/deployment integration after compile-required contracts and concrete MCP injection moved into lower server levels. - Alternatives considered: keeping #9534 as one 403-file review, or rewriting the feature to manufacture seams; both were rejected in favor of path extraction plus compile-driven boundary moves. - Roadmap alignment: this advances the existing governed MCP/tool-access work already represented by #9534; it does not introduce a separate roadmap initiative. - Stack position: base branch is `pap10341-split/04-server-runtime-wiring`. - Merge policy: merge bottom-up, in order, only after the complete eight-PR stack has been reviewed and the top-of-stack parity gate remains empty. - Requested review: QA for CLI, packaging, and worktree behavior; Greptile on every PR. ## What Changed - Adds the remaining runtime/deployment integration after compile-required contracts and concrete MCP injection moved into lower server levels. - Keeps this PR below 100 changed files and independently typecheckable. - Preserves the final tree from #9534 when combined with the other seven stack levels. ## Verification - `pnpm typecheck` - Focused CLI Vitest run — 4 files, 49 tests passed ## Risks - Packaging omissions could make the feature work in source but fail in Docker, worktrees, or release assembly. - Stack risk: merging out of order can expose incomplete layers; mitigate by following the documented bottom-up merge policy. - Parity risk: later edits to an intermediate branch can drift from #9534; mitigate by re-running the empty top-of-stack diff before merge. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - OpenAI Codex, exact model ID `gpt-5.4`; runtime-managed context window; medium reasoning with repository, shell, Git, GitHub CLI, and code-execution tools enabled. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] Internal references are omitted except the execution-plan link explicitly required for this coordinated split stack - [x] My branch name describes the change and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge ## Stack Coordination - Internal execution plan: [PAP-13874](/PAP/issues/PAP-13874#document-plan) - Parity reference: #9534 - Stack: #9556 → #9557 → #9558 → #9559 → #9560 → #9561 → #9562 → #9563 - Merge bottom-up only after full-stack review and an empty parity diff at #9563. --------- Co-authored-by: Paperclip --- .env.example | 1 + cli/src/__tests__/onboard.test.ts | 8 +++ cli/src/__tests__/project-goal.test.ts | 5 ++ .../__tests__/routine-plugin-parity.test.ts | 21 ++++-- cli/src/__tests__/worktree.test.ts | 51 ++++++++++++++- cli/src/commands/client/plugin.ts | 65 ++++++++++++++++++- cli/src/commands/worktree.ts | 12 +++- .../openclaw-gateway/src/ui/build-config.ts | 16 ++--- scripts/provision-worktree.sh | 18 ++++- 9 files changed, 174 insertions(+), 23 deletions(-) diff --git a/.env.example b/.env.example index 84e20d9605..e747df0914 100644 --- a/.env.example +++ b/.env.example @@ -2,6 +2,7 @@ DATABASE_URL=postgres://paperclip:paperclip@localhost:5432/paperclip PORT=3100 SERVE_UI=false BETTER_AUTH_SECRET=paperclip-dev-secret +PAPERCLIP_TOOL_ACTION_SIGNING_SECRET=paperclip-dev-tool-action-signing-secret-change-me # Discord webhook for daily merge digest (scripts/discord-daily-digest.sh) # DISCORD_WEBHOOK_URL=https://discord.com/api/webhooks/... diff --git a/cli/src/__tests__/onboard.test.ts b/cli/src/__tests__/onboard.test.ts index b7c01eb5cd..33890e703d 100644 --- a/cli/src/__tests__/onboard.test.ts +++ b/cli/src/__tests__/onboard.test.ts @@ -87,6 +87,14 @@ describe("onboard", () => { delete process.env.PAPERCLIP_AGENT_JWT_SECRET; delete process.env.PAPERCLIP_SECRETS_MASTER_KEY; delete process.env.PAPERCLIP_SECRETS_MASTER_KEY_FILE; + delete process.env.PAPERCLIP_DB_BACKUP_DIR; + delete process.env.PAPERCLIP_DB_BACKUP_ENABLED; + delete process.env.PAPERCLIP_DB_BACKUP_INTERVAL_MINUTES; + delete process.env.PAPERCLIP_DB_BACKUP_RETENTION_DAYS; + delete process.env.PAPERCLIP_STORAGE_PROVIDER; + delete process.env.PAPERCLIP_STORAGE_LOCAL_DIR; + delete process.env.PAPERCLIP_SECRETS_PROVIDER; + delete process.env.PAPERCLIP_SECRETS_STRICT_MODE; delete process.env.PAPERCLIP_HOME; delete process.env.PAPERCLIP_CONFIG; delete process.env.PAPERCLIP_INSTANCE_ID; diff --git a/cli/src/__tests__/project-goal.test.ts b/cli/src/__tests__/project-goal.test.ts index 2303a855d7..ba7bc235ee 100644 --- a/cli/src/__tests__/project-goal.test.ts +++ b/cli/src/__tests__/project-goal.test.ts @@ -1,3 +1,6 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; import { Command } from "commander"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { registerGoalCommands } from "../commands/client/goal.js"; @@ -25,10 +28,12 @@ describe("project and goal commands", () => { delete process.env.PAPERCLIP_API_KEY; delete process.env.PAPERCLIP_API_URL; delete process.env.PAPERCLIP_COMPANY_ID; + process.env.PAPERCLIP_CONTEXT = path.join(fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-project-goal-")), "context.json"); }); afterEach(() => { vi.restoreAllMocks(); + delete process.env.PAPERCLIP_CONTEXT; }); it("creates and updates projects with shared schemas", async () => { diff --git a/cli/src/__tests__/routine-plugin-parity.test.ts b/cli/src/__tests__/routine-plugin-parity.test.ts index e3086b0cbd..25192a9a9c 100644 --- a/cli/src/__tests__/routine-plugin-parity.test.ts +++ b/cli/src/__tests__/routine-plugin-parity.test.ts @@ -79,9 +79,9 @@ describe("routine and plugin parity commands", () => { await run(["plugin", "health", "plug"]); await run(["plugin", "logs", "plug"]); await run(["plugin", "upgrade", "plug"]); - await run(["plugin", "config", "plug"]); - await run(["plugin", "config:set", "plug", "--payload-json", "{}"]); - await run(["plugin", "config:test", "plug", "--payload-json", "{}"]); + await run(["plugin", "config", "plug", "--company-id", COMPANY_ID]); + await run(["plugin", "config:set", "plug", "--company-id", COMPANY_ID, "--payload-json", "{}"]); + await run(["plugin", "config:test", "plug", "--company-id", COMPANY_ID, "--payload-json", "{}"]); await run(["plugin", "jobs", "plug"]); await run(["plugin", "job:runs", "plug", "job1"]); await run(["plugin", "job:trigger", "plug", "job1"]); @@ -104,7 +104,7 @@ describe("routine and plugin parity commands", () => { ["GET", "http://localhost:3100/api/plugins/plug/health"], ["GET", "http://localhost:3100/api/plugins/plug/logs"], ["POST", "http://localhost:3100/api/plugins/plug/upgrade"], - ["GET", "http://localhost:3100/api/plugins/plug/config"], + ["GET", `http://localhost:3100/api/plugins/plug/config?companyId=${COMPANY_ID}`], ["POST", "http://localhost:3100/api/plugins/plug/config"], ["POST", "http://localhost:3100/api/plugins/plug/config/test"], ["GET", "http://localhost:3100/api/plugins/plug/jobs"], @@ -123,6 +123,19 @@ describe("routine and plugin parity commands", () => { ["PUT", `http://localhost:3100/api/plugins/plug/companies/${COMPANY_ID}/local-folders/source`], ]); }); + + it("resolves plugin config company context from the environment", async () => { + const fetchMock = vi.fn().mockImplementation(() => Promise.resolve(jsonResponse())); + vi.stubGlobal("fetch", fetchMock); + process.env.PAPERCLIP_COMPANY_ID = COMPANY_ID; + + await run(["plugin", "config", "plug"]); + + expect(fetchMock).toHaveBeenCalledWith( + `http://localhost:3100/api/plugins/plug/config?companyId=${COMPANY_ID}`, + expect.objectContaining({ method: "GET" }), + ); + }); }); function jsonResponse(body: unknown = { ok: true }, init: ResponseInit = { status: 200 }): Response { diff --git a/cli/src/__tests__/worktree.test.ts b/cli/src/__tests__/worktree.test.ts index bc23cf4de2..b75699caef 100644 --- a/cli/src/__tests__/worktree.test.ts +++ b/cli/src/__tests__/worktree.test.ts @@ -3,6 +3,7 @@ import os from "node:os"; import path from "node:path"; import { execFileSync } from "node:child_process"; import { randomUUID } from "node:crypto"; +import { createServer } from "node:net"; import { eq } from "drizzle-orm"; import { afterEach, describe, expect, it, vi } from "vitest"; import { @@ -28,6 +29,7 @@ import { resolveWorktreeReseedTargetPaths, resolveGitWorktreeAddArgs, resolvePnpmInstallInvocation, + resolveWorktreeSeedBackupEngine, resolveWorktreeMakeTargetPath, worktreeRepairCommand, worktreeInitCommand, @@ -62,6 +64,35 @@ if (!embeddedPostgresSupport.supported) { ); } +async function reserveTestPort(): Promise<{ port: number; release: () => Promise }> { + const server = createServer(); + server.unref(); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", () => { + server.off("error", reject); + resolve(); + }); + }); + const address = server.address(); + if (!address || typeof address === "string") { + server.close(); + throw new Error("Failed to reserve test port"); + } + let released = false; + return { + port: address.port, + release: () => new Promise((resolve, reject) => { + if (released) { + resolve(); + return; + } + released = true; + server.close((error) => (error ? reject(error) : resolve())); + }), + }; +} + afterEach(() => { process.chdir(ORIGINAL_CWD); for (const key of Object.keys(process.env)) { @@ -874,6 +905,11 @@ describe("worktree helpers", () => { } }); + it("uses streaming backup selection for full seeds and transformed backup selection for minimal seeds", () => { + expect(resolveWorktreeSeedBackupEngine(resolveWorktreeSeedPlan("full"))).toBe("auto"); + expect(resolveWorktreeSeedBackupEngine(resolveWorktreeSeedPlan("minimal"))).toBe("javascript"); + }); + itEmbeddedPostgres("reseed preserves the current worktree ports, instance id, and branding", async () => { const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-worktree-reseed-")); const repoRoot = path.join(tempRoot, "repo"); @@ -892,6 +928,10 @@ describe("worktree helpers", () => { }); const originalCwd = process.cwd(); const originalPaperclipConfig = process.env.PAPERCLIP_CONFIG; + const currentDatabaseReservation = await reserveTestPort(); + const sourceDatabaseReservation = await reserveTestPort(); + const currentDatabasePort = currentDatabaseReservation.port; + const sourceDatabasePort = sourceDatabaseReservation.port; try { fs.mkdirSync(path.dirname(currentPaths.configPath), { recursive: true }); @@ -904,13 +944,13 @@ describe("worktree helpers", () => { sourceConfig: buildSourceConfig(), paths: currentPaths, serverPort: 3114, - databasePort: 54341, + databasePort: currentDatabasePort, }); const sourceConfig = buildWorktreeConfig({ sourceConfig: buildSourceConfig(), paths: sourcePaths, serverPort: 3200, - databasePort: 54400, + databasePort: sourceDatabasePort, }); fs.writeFileSync(currentPaths.configPath, JSON.stringify(currentConfig, null, 2), "utf8"); fs.writeFileSync(sourcePaths.configPath, JSON.stringify(sourceConfig, null, 2), "utf8"); @@ -929,6 +969,9 @@ describe("worktree helpers", () => { delete process.env.PAPERCLIP_CONFIG; process.chdir(repoRoot); + await currentDatabaseReservation.release(); + await sourceDatabaseReservation.release(); + await worktreeReseedCommand({ fromConfig: sourcePaths.configPath, yes: true, @@ -938,12 +981,14 @@ describe("worktree helpers", () => { const rewrittenEnv = fs.readFileSync(currentPaths.envPath, "utf8"); expect(rewrittenConfig.server.port).toBe(3114); - expect(rewrittenConfig.database.embeddedPostgresPort).toBe(54341); + expect(rewrittenConfig.database.embeddedPostgresPort).toBe(currentDatabasePort); expect(rewrittenConfig.database.embeddedPostgresDataDir).toBe(currentPaths.embeddedPostgresDataDir); expect(rewrittenEnv).toContain(`PAPERCLIP_INSTANCE_ID=${currentInstanceId}`); expect(rewrittenEnv).toContain("PAPERCLIP_WORKTREE_NAME=existing-name"); expect(rewrittenEnv).toContain("PAPERCLIP_WORKTREE_COLOR=\"#112233\""); } finally { + await currentDatabaseReservation.release(); + await sourceDatabaseReservation.release(); process.chdir(originalCwd); if (originalPaperclipConfig === undefined) { delete process.env.PAPERCLIP_CONFIG; diff --git a/cli/src/commands/client/plugin.ts b/cli/src/commands/client/plugin.ts index e6e8bff04d..46b5da24ce 100644 --- a/cli/src/commands/client/plugin.ts +++ b/cli/src/commands/client/plugin.ts @@ -95,6 +95,15 @@ interface PluginCompanyOptions extends PluginJsonOptions { companyId?: string; } +function requireCompanyId(ctx: { companyId?: string }): string { + if (!ctx.companyId) { + throw new Error( + "Company ID is required. Pass --company-id, set PAPERCLIP_COMPANY_ID, or set context profile companyId via `paperclipai context set`.", + ); + } + return ctx.companyId; +} + interface PluginInitResult { outputDir: string; nextCommands: string[]; @@ -673,9 +682,9 @@ export function registerPluginCommands(program: Command): void { addPluginSubGet(plugin, "health", "Get plugin health", "health"); addPluginSubGet(plugin, "logs", "Get plugin logs", "logs"); addPluginSubPost(plugin, "upgrade", "Upgrade a plugin", "upgrade"); - addPluginSubGet(plugin, "config", "Get plugin config", "config"); - addPluginSubPost(plugin, "config:set", "Set plugin config", "config"); - addPluginSubPost(plugin, "config:test", "Test plugin config", "config/test"); + addPluginConfigGet(plugin, "config", "Get company-scoped plugin config"); + addPluginConfigPost(plugin, "config:set", "Set company-scoped plugin config", "config"); + addPluginConfigPost(plugin, "config:test", "Test company-scoped plugin config", "config/test"); addPluginSubGet(plugin, "jobs", "List plugin jobs", "jobs"); addPluginJobGet(plugin, "job:runs", "List plugin job runs", "runs"); addPluginJobPost(plugin, "job:trigger", "Trigger a plugin job", "trigger"); @@ -751,6 +760,56 @@ function addPluginSubPost(parent: Command, name: string, description: string, su })); } +function addPluginConfigGet(parent: Command, name: string, description: string): void { + addCommonClientOptions( + parent + .command(name) + .description(description) + .argument("", "Plugin ID or key") + .option("-C, --company-id ", "Company ID") + .action(async (pluginId: string, opts: PluginCompanyOptions) => { + try { + const ctx = resolveCommandContext(opts, { requireCompany: true }); + const companyId = requireCompanyId(ctx); + printOutput( + await ctx.api.get(`/api/plugins/${encodeURIComponent(pluginId)}/config?companyId=${encodeURIComponent(companyId)}`), + { json: ctx.json }, + ); + } catch (err) { + handleCommandError(err); + } + }), + { includeCompany: false }, + ); +} + +function addPluginConfigPost(parent: Command, name: string, description: string, suffix: string): void { + addCommonClientOptions( + parent + .command(name) + .description(description) + .argument("", "Plugin ID or key") + .option("-C, --company-id ", "Company ID") + .option("--payload-json ", "JSON payload", "{}") + .action(async (pluginId: string, opts: PluginCompanyOptions) => { + try { + const ctx = resolveCommandContext(opts, { requireCompany: true }); + const payload = parseJson(opts.payloadJson ?? "{}") as Record; + printOutput( + await ctx.api.post(`/api/plugins/${encodeURIComponent(pluginId)}/${suffix}`, { + ...payload, + companyId: ctx.companyId, + }), + { json: ctx.json }, + ); + } catch (err) { + handleCommandError(err); + } + }), + { includeCompany: false }, + ); +} + function addPluginJobGet(parent: Command, name: string, description: string, suffix: string): void { addCommonClientOptions(parent.command(name).description(description).argument("", "Plugin ID or key").argument("", "Job ID").action(async (pluginId: string, jobId: string, opts: BaseClientOptions) => { try { diff --git a/cli/src/commands/worktree.ts b/cli/src/commands/worktree.ts index a4c78cc022..ff4c6498f3 100644 --- a/cli/src/commands/worktree.ts +++ b/cli/src/commands/worktree.ts @@ -43,6 +43,7 @@ import { routineTriggers, runDatabaseBackup, runDatabaseRestore, + resetPostgresDatabase, createEmbeddedPostgresLogBuffer, formatEmbeddedPostgresError, prepareEmbeddedPostgresNativeRuntime, @@ -65,6 +66,7 @@ import { resolveWorktreeSeedPlan, resolveWorktreeLocalPaths, sanitizeWorktreeInstanceId, + type WorktreeSeedPlan, type WorktreeSeedMode, type WorktreeLocalPaths, } from "./worktree-lib.js"; @@ -1313,7 +1315,7 @@ async function seedWorktreeDatabase(input: { backupDir: path.resolve(input.targetPaths.backupDir, "seed"), retention: { dailyDays: 7, weeklyWeeks: 4, monthlyMonths: 1 }, filenamePrefix: `${input.instanceId}-seed`, - backupEngine: "javascript", + backupEngine: resolveWorktreeSeedBackupEngine(seedPlan), includeMigrationJournal: true, excludeTables: seedPlan.excludedTables, nullifyColumns: seedPlan.nullifyColumns, @@ -1325,7 +1327,7 @@ async function seedWorktreeDatabase(input: { ); const adminConnectionString = `postgres://paperclip:paperclip@127.0.0.1:${targetHandle.port}/postgres`; - await ensurePostgresDatabase(adminConnectionString, "paperclip"); + await resetPostgresDatabase(adminConnectionString, "paperclip"); const targetConnectionString = `postgres://paperclip:paperclip@127.0.0.1:${targetHandle.port}/paperclip`; await runDatabaseRestore({ connectionString: targetConnectionString, @@ -1357,6 +1359,12 @@ async function seedWorktreeDatabase(input: { } } +export function resolveWorktreeSeedBackupEngine(seedPlan: WorktreeSeedPlan): "auto" | "javascript" { + return seedPlan.excludedTables.length === 0 && Object.keys(seedPlan.nullifyColumns).length === 0 + ? "auto" + : "javascript"; +} + async function runWorktreeInit(opts: WorktreeInitOptions): Promise { const cwd = process.cwd(); const worktreeName = resolveSuggestedWorktreeName( diff --git a/packages/adapters/openclaw-gateway/src/ui/build-config.ts b/packages/adapters/openclaw-gateway/src/ui/build-config.ts index b13906d021..8694ded566 100644 --- a/packages/adapters/openclaw-gateway/src/ui/build-config.ts +++ b/packages/adapters/openclaw-gateway/src/ui/build-config.ts @@ -14,36 +14,36 @@ function parseJsonObject(text: string): Record | null { export function buildOpenClawGatewayConfig(v: CreateConfigValues): Record { const ac: Record = {}; - + // Required / Primary fields if (v.url) ac.url = v.url; if (v.authToken) ac.authToken = v.authToken; if (v.password) ac.password = v.password; if (v.agentId) ac.agentId = v.agentId; - + // Session routing fields if (v.sessionKeyStrategy) ac.sessionKeyStrategy = v.sessionKeyStrategy; if (v.sessionKey) ac.sessionKey = v.sessionKey; - + // Timeout fields if (typeof v.timeoutSec === "number") ac.timeoutSec = v.timeoutSec; if (typeof v.waitTimeoutMs === "number") ac.waitTimeoutMs = v.waitTimeoutMs; - + // Device auth fields if (typeof v.disableDeviceAuth === "boolean") ac.disableDeviceAuth = v.disableDeviceAuth; if (typeof v.autoPairOnFirstConnect === "boolean") ac.autoPairOnFirstConnect = v.autoPairOnFirstConnect; if (v.devicePrivateKeyPem) ac.devicePrivateKeyPem = v.devicePrivateKeyPem; - + // Gateway identity fields if (v.role) ac.role = v.role; if (v.scopes) { const parsed = v.scopes.split(",").map((s) => s.trim()).filter(Boolean); if (parsed.length > 0) ac.scopes = parsed; } - + // Paperclip API override if (v.paperclipApiUrl) ac.paperclipApiUrl = v.paperclipApiUrl; - + // Headers — parse headersJson first, then inject authToken on top const headers = parseJsonObject(v.headersJson ?? ""); if (headers) ac.headers = headers; @@ -56,7 +56,7 @@ export function buildOpenClawGatewayConfig(v: CreateConfigValues): Record