diff --git a/cli/src/__tests__/config-store.test.ts b/cli/src/__tests__/config-store.test.ts new file mode 100644 index 0000000000..21abb4bf60 --- /dev/null +++ b/cli/src/__tests__/config-store.test.ts @@ -0,0 +1,139 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + backupInvalidConfig, + readConfig, + writeConfig, +} from "../config/store.js"; +import { paperclipConfigSchema, type PaperclipConfig } from "../config/schema.js"; + +const roots: string[] = []; + +afterEach(() => { + vi.restoreAllMocks(); + for (const root of roots.splice(0)) { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +function createConfigPath(): string { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-config-store-")); + roots.push(root); + return path.join(root, "config.json"); +} + +function defaultConfig(): PaperclipConfig { + return paperclipConfigSchema.parse({ + $meta: { + version: 1, + updatedAt: "2026-08-06T00:00:00.000Z", + source: "configure", + }, + database: { mode: "embedded-postgres" }, + logging: { mode: "file" }, + server: {}, + }); +} + +describe("config store", () => { + it("preserves top-level and nested extension keys during a known-field update", () => { + const configPath = createConfigPath(); + fs.writeFileSync(configPath, JSON.stringify({ + ...defaultConfig(), + topLevelExtension: { enabled: true }, + server: { + ...defaultConfig().server, + serverExtension: "keep", + }, + storage: { + ...defaultConfig().storage, + localDisk: { + ...defaultConfig().storage.localDisk, + driverExtension: "keep", + }, + }, + }, null, 2)); + + const source = readConfig(configPath)!; + const { topLevelExtension: _topLevelExtension, ...knownConfig } = source; + const { serverExtension: _serverExtension, ...knownServer } = source.server; + const { driverExtension: _driverExtension, ...knownLocalDisk } = source.storage.localDisk; + const update: PaperclipConfig = { + ...knownConfig, + server: { + ...knownServer, + port: 3200, + }, + storage: { + ...source.storage, + localDisk: knownLocalDisk, + }, + }; + + expect(writeConfig(update, configPath)).toBe(true); + expect(JSON.parse(fs.readFileSync(configPath, "utf8"))).toMatchObject({ + topLevelExtension: { enabled: true }, + server: { + port: 3200, + serverExtension: "keep", + }, + storage: { + localDisk: { + driverExtension: "keep", + }, + }, + }); + }); + + it("skips semantic no-op writes and keeps the config mtime stable", () => { + const configPath = createConfigPath(); + const source = defaultConfig(); + fs.writeFileSync(configPath, `${JSON.stringify(source, null, 2)}\n`); + const stableTime = new Date("2020-01-01T00:00:00.000Z"); + fs.utimesSync(configPath, stableTime, stableTime); + + const update = { + ...source, + $meta: { + ...source.$meta, + source: "doctor" as const, + updatedAt: "2026-08-06T01:00:00.000Z", + }, + }; + + expect(writeConfig(update, configPath)).toBe(false); + expect(fs.statSync(configPath).mtimeMs).toBe(stableTime.getTime()); + expect(fs.existsSync(`${configPath}.backup`)).toBe(false); + }); + + it("backs up invalid bytes collision-safely and only replaces them through an atomic repair", () => { + const configPath = createConfigPath(); + const invalidBytes = Buffer.from('{"server": invalid}\n', "utf8"); + fs.writeFileSync(configPath, invalidBytes); + fs.writeFileSync(`${configPath}.invalid-1`, "existing backup"); + + const open = vi.spyOn(fs, "openSync"); + const sync = vi.spyOn(fs, "fsyncSync"); + const backupPath = backupInvalidConfig(configPath); + expect(backupPath).toBe(`${configPath}.invalid-2`); + expect(fs.readFileSync(backupPath)).toEqual(invalidBytes); + expect(open).toHaveBeenCalledWith(backupPath, "r"); + expect(open).toHaveBeenCalledWith(path.dirname(configPath), "r"); + expect(sync).toHaveBeenCalled(); + expect(() => writeConfig(defaultConfig(), configPath)).toThrow(/Refusing to overwrite invalid config/); + expect(fs.readFileSync(configPath)).toEqual(invalidBytes); + + open.mockClear(); + sync.mockClear(); + const rename = vi.spyOn(fs, "renameSync"); + expect(writeConfig(defaultConfig(), configPath, { invalidBackupPath: backupPath })).toBe(true); + expect(rename).toHaveBeenCalledWith(expect.stringMatching(/config\.json\.tmp-\d+-\d+$/), configPath); + expect(open).toHaveBeenCalledWith(path.dirname(configPath), "r"); + expect(open.mock.invocationCallOrder.at(-1)!).toBeGreaterThan(rename.mock.invocationCallOrder.at(-1)!); + expect(sync.mock.invocationCallOrder.at(-1)!).toBeGreaterThan(open.mock.invocationCallOrder.at(-1)!); + expect(readConfig(configPath)).not.toBeNull(); + expect(fs.readdirSync(path.dirname(configPath)).some((entry) => entry.includes(".tmp-"))).toBe(false); + }); +}); diff --git a/cli/src/__tests__/configure-repair.test.ts b/cli/src/__tests__/configure-repair.test.ts new file mode 100644 index 0000000000..8ffe383469 --- /dev/null +++ b/cli/src/__tests__/configure-repair.test.ts @@ -0,0 +1,83 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import * as prompts from "@clack/prompts"; +import { configure } from "../commands/configure.js"; +import { readConfig } from "../config/store.js"; + +vi.mock("@clack/prompts", () => ({ + intro: vi.fn(), + outro: vi.fn(), + cancel: vi.fn(), + confirm: vi.fn(), + select: vi.fn(), + isCancel: vi.fn(() => false), + log: { + error: vi.fn(), + message: vi.fn(), + step: vi.fn(), + success: vi.fn(), + warn: vi.fn(), + }, +})); + +vi.mock("../prompts/server.js", () => ({ + promptServer: vi.fn(async ({ currentServer, currentAuth }) => ({ + server: currentServer, + auth: currentAuth, + })), +})); + +const ORIGINAL_EXIT_CODE = process.exitCode; +let originalStdinIsTTY: boolean | undefined; +let originalStdoutIsTTY: boolean | undefined; + +beforeEach(() => { + originalStdinIsTTY = process.stdin.isTTY; + originalStdoutIsTTY = process.stdout.isTTY; + Object.defineProperty(process.stdin, "isTTY", { configurable: true, value: true }); + Object.defineProperty(process.stdout, "isTTY", { configurable: true, value: true }); + vi.mocked(prompts.confirm).mockResolvedValue(true); + vi.spyOn(console, "log").mockImplementation(() => undefined); +}); + +afterEach(() => { + Object.defineProperty(process.stdin, "isTTY", { + configurable: true, + value: originalStdinIsTTY, + }); + Object.defineProperty(process.stdout, "isTTY", { + configurable: true, + value: originalStdoutIsTTY, + }); + process.exitCode = ORIGINAL_EXIT_CODE; + vi.restoreAllMocks(); +}); + +describe("configure invalid-config repair", () => { + it("repairs only after confirmation and commits the staged config atomically", async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-configure-repair-")); + const configPath = path.join(root, "config.json"); + const invalidBytes = Buffer.from('{"server": invalid}\n', "utf8"); + fs.writeFileSync(configPath, invalidBytes); + const rename = vi.spyOn(fs, "renameSync"); + + try { + await configure({ config: configPath, section: "server" }); + + expect(prompts.confirm).toHaveBeenCalledWith({ + message: `Repair from defaults? The invalid original is backed up at ${configPath}.invalid-1.`, + initialValue: false, + }); + expect(fs.readFileSync(`${configPath}.invalid-1`)).toEqual(invalidBytes); + expect(readConfig(configPath)).not.toBeNull(); + expect(rename).toHaveBeenCalledWith( + expect.stringMatching(/config\.json\.tmp-\d+-\d+$/), + configPath, + ); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/cli/src/__tests__/configure.test.ts b/cli/src/__tests__/configure.test.ts index 74a37fc8fc..cbd7016879 100644 --- a/cli/src/__tests__/configure.test.ts +++ b/cli/src/__tests__/configure.test.ts @@ -96,4 +96,29 @@ describe("configure command", () => { fs.rmSync(root, { recursive: true, force: true }); } }); + + it("backs up invalid config bytes and refuses non-interactive replacement", async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-configure-invalid-")); + const configPath = path.join(root, "config.json"); + const invalidBytes = Buffer.from('{"server": invalid}\n', "utf8"); + const stdinDescriptor = Object.getOwnPropertyDescriptor(process.stdin, "isTTY"); + fs.writeFileSync(configPath, invalidBytes); + Object.defineProperty(process.stdin, "isTTY", { configurable: true, value: false }); + + try { + await configure({ config: configPath, section: "server" }); + + expect(process.exitCode).toBe(1); + expect(fs.readFileSync(configPath)).toEqual(invalidBytes); + expect(fs.readFileSync(`${configPath}.invalid-1`)).toEqual(invalidBytes); + expect(fs.existsSync(`${configPath}.backup`)).toBe(false); + } finally { + if (stdinDescriptor) { + Object.defineProperty(process.stdin, "isTTY", stdinDescriptor); + } else { + delete (process.stdin as { isTTY?: boolean }).isTTY; + } + fs.rmSync(root, { recursive: true, force: true }); + } + }); }); diff --git a/cli/src/__tests__/onboard.test.ts b/cli/src/__tests__/onboard.test.ts index 33890e703d..59a578b1f8 100644 --- a/cli/src/__tests__/onboard.test.ts +++ b/cli/src/__tests__/onboard.test.ts @@ -8,6 +8,7 @@ import type { PaperclipConfig } from "../config/schema.js"; const ORIGINAL_ENV = { ...process.env }; const ORIGINAL_CWD = process.cwd(); const ORIGINAL_PATH = process.env.PATH; +const ORIGINAL_EXIT_CODE = process.exitCode; function createExistingConfigFixture() { const root = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-onboard-")); @@ -107,6 +108,7 @@ describe("onboard", () => { afterEach(() => { process.env = { ...ORIGINAL_ENV }; process.chdir(ORIGINAL_CWD); + process.exitCode = ORIGINAL_EXIT_CODE; }); it("preserves an existing config when rerun without flags", async () => { @@ -129,6 +131,20 @@ describe("onboard", () => { expect(fs.existsSync(path.join(path.dirname(fixture.configPath), ".env"))).toBe(true); }); + it("backs up invalid config bytes and refuses --yes replacement", async () => { + const configPath = createFreshConfigPath(); + const invalidBytes = Buffer.from('{"database": invalid}\n', "utf8"); + fs.mkdirSync(path.dirname(configPath), { recursive: true }); + fs.writeFileSync(configPath, invalidBytes); + + await onboard({ config: configPath, yes: true, invokedByRun: true }); + + expect(process.exitCode).toBe(1); + expect(fs.readFileSync(configPath)).toEqual(invalidBytes); + expect(fs.readFileSync(`${configPath}.invalid-1`)).toEqual(invalidBytes); + expect(fs.existsSync(`${configPath}.backup`)).toBe(false); + }); + it("keeps --yes onboarding on local trusted loopback defaults", async () => { const configPath = createFreshConfigPath(); process.env.HOST = "0.0.0.0"; diff --git a/cli/src/commands/configure.ts b/cli/src/commands/configure.ts index 7c9b391bf0..126fcc414b 100644 --- a/cli/src/commands/configure.ts +++ b/cli/src/commands/configure.ts @@ -1,7 +1,16 @@ import * as p from "@clack/prompts"; import pc from "picocolors"; -import { readConfig, writeConfig, configExists, resolveConfigPath } from "../config/store.js"; -import type { PaperclipConfig } from "../config/schema.js"; +import { + backupInvalidConfig, + readConfig, + writeConfig, + configExists, + resolveConfigPath, +} from "../config/store.js"; +import { + findPaperclipConfigKeyWarnings, + type PaperclipConfig, +} from "../config/schema.js"; import { ensureLocalSecretsKeyFile } from "../config/secrets-key.js"; import { promptDatabase } from "../prompts/database.js"; import { promptLlm } from "../prompts/llm.js"; @@ -88,15 +97,39 @@ export async function configure(opts: { } let config: PaperclipConfig; + let invalidBackupPath: string | undefined; try { config = readConfig(opts.config) ?? defaultConfig(); + for (const warning of findPaperclipConfigKeyWarnings(config)) { + p.log.warn(`Unknown config key ${warning.path}; did you mean ${warning.suggestion}? It will be preserved.`); + } } catch (err) { - p.log.message( - pc.yellow( - `Existing config is invalid. Loading defaults so you can repair it now.\n${err instanceof Error ? err.message : String(err)}`, - ), + const backupPath = backupInvalidConfig(opts.config); + p.log.warn( + `Existing config is invalid. Preserved the original bytes at ${backupPath}.\n${err instanceof Error ? err.message : String(err)}`, ); + + if (!process.stdin.isTTY || !process.stdout.isTTY) { + p.log.error( + `Refusing to replace ${configPath} without confirmation. Rerun interactively to repair from defaults; the original and ${backupPath} are unchanged.`, + ); + p.outro(""); + process.exitCode = 1; + return; + } + + const repair = await p.confirm({ + message: `Repair from defaults? The invalid original is backed up at ${backupPath}.`, + initialValue: false, + }); + if (p.isCancel(repair) || !repair) { + p.cancel(`Configuration left unchanged. Invalid backup: ${backupPath}`); + process.exitCode = 1; + return; + } + config = defaultConfig(); + invalidBackupPath = backupPath; } let section: Section | undefined = opts.section as Section | undefined; @@ -179,8 +212,15 @@ export async function configure(opts: { config.$meta.updatedAt = new Date().toISOString(); config.$meta.source = "configure"; - writeConfig(config, opts.config); - p.log.success(`${SECTION_LABELS[section]} configuration updated.`); + const written = writeConfig(config, opts.config, { + invalidBackupPath, + }); + invalidBackupPath = undefined; + if (written) { + p.log.success(`${SECTION_LABELS[section]} configuration updated.`); + } else { + p.log.message(pc.dim(`${SECTION_LABELS[section]} configuration unchanged.`)); + } // If section was provided via CLI flag, don't loop if (opts.section) { diff --git a/cli/src/commands/onboard.ts b/cli/src/commands/onboard.ts index 3024360dda..31850714d7 100644 --- a/cli/src/commands/onboard.ts +++ b/cli/src/commands/onboard.ts @@ -17,8 +17,17 @@ import { type SecretProvider, type StorageProvider, } from "@paperclipai/shared"; -import { configExists, readConfig, resolveConfigPath, writeConfig } from "../config/store.js"; -import type { PaperclipConfig } from "../config/schema.js"; +import { + backupInvalidConfig, + configExists, + readConfig, + resolveConfigPath, + writeConfig, +} from "../config/store.js"; +import { + findPaperclipConfigKeyWarnings, + type PaperclipConfig, +} from "../config/schema.js"; import { ensureAgentJwtSecret, resolveAgentJwtEnvFile } from "../config/env.js"; import { ensureLocalSecretsKeyFile } from "../config/secrets-key.js"; import { promptDatabase } from "../prompts/database.js"; @@ -356,17 +365,45 @@ export async function onboard(opts: OnboardOptions): Promise { ); let existingConfig: PaperclipConfig | null = null; + let invalidBackupPath: string | undefined; if (configExists(opts.config)) { p.log.message(pc.dim(`${configPath} exists`)); try { existingConfig = readConfig(opts.config); + for (const warning of findPaperclipConfigKeyWarnings(existingConfig)) { + p.log.warn(`Unknown config key ${warning.path}; did you mean ${warning.suggestion}? It will be preserved.`); + } } catch (err) { - p.log.message( - pc.yellow( - `Existing config appears invalid and will be updated.\n${err instanceof Error ? err.message : String(err)}`, - ), + const backupPath = backupInvalidConfig(opts.config); + p.log.warn( + `Existing config is invalid. Preserved the original bytes at ${backupPath}.\n${err instanceof Error ? err.message : String(err)}`, ); + + const canConfirmRepair = + opts.yes !== true && + opts.invokedByRun !== true && + process.stdin.isTTY === true && + process.stdout.isTTY === true; + if (!canConfirmRepair) { + p.log.error( + `Refusing to replace ${configPath} without confirmation. Rerun interactively to repair from defaults; the original and ${backupPath} are unchanged.`, + ); + p.outro(""); + process.exitCode = 1; + return; + } + + const repair = await p.confirm({ + message: `Repair from defaults? The invalid original is backed up at ${backupPath}.`, + initialValue: false, + }); + if (p.isCancel(repair) || !repair) { + p.cancel(`Configuration left unchanged. Invalid backup: ${backupPath}`); + process.exitCode = 1; + return; + } + invalidBackupPath = backupPath; } } @@ -646,7 +683,9 @@ export async function onboard(opts: OnboardOptions): Promise { p.log.message(pc.dim(`Using existing local secrets key file at ${keyResult.path}`)); } - writeConfig(config, opts.config); + writeConfig(config, opts.config, { + invalidBackupPath, + }); if (tc) trackInstallCompleted(tc, { adapterType: server.deploymentMode, diff --git a/cli/src/config/schema.ts b/cli/src/config/schema.ts index 65ddeab733..799d8ba0d7 100644 --- a/cli/src/config/schema.ts +++ b/cli/src/config/schema.ts @@ -8,11 +8,15 @@ export { serverConfigSchema, authConfigSchema, telemetryConfigSchema, + updatesConfigSchema, storageConfigSchema, storageLocalDiskConfigSchema, storageS3ConfigSchema, secretsConfigSchema, secretsLocalEncryptedConfigSchema, + mergePaperclipConfig, + findPaperclipConfigKeyWarnings, + type ConfigKeyWarning, type PaperclipConfig, type LlmConfig, type DatabaseBackupConfig, @@ -27,4 +31,5 @@ export { type SecretsConfig, type SecretsLocalEncryptedConfig, type ConfigMeta, + type UpdatesConfig, } from "../../../packages/shared/src/config-schema.js"; diff --git a/cli/src/config/store.ts b/cli/src/config/store.ts index 8dddc77706..b1ab0229d1 100644 --- a/cli/src/config/store.ts +++ b/cli/src/config/store.ts @@ -1,6 +1,11 @@ import fs from "node:fs"; import path from "node:path"; -import { paperclipConfigSchema, type PaperclipConfig } from "./schema.js"; +import { isDeepStrictEqual } from "node:util"; +import { + mergePaperclipConfig, + paperclipConfigSchema, + type PaperclipConfig, +} from "./schema.js"; import { resolveDefaultConfigPath, resolvePaperclipInstanceId, @@ -95,24 +100,132 @@ export function readConfig(configPath?: string): PaperclipConfig | null { return parsed.data; } +function effectiveConfig(config: PaperclipConfig): Record { + const meta = { ...config.$meta } as Record; + delete meta.updatedAt; + delete meta.source; + return { + ...config, + $meta: meta, + }; +} + +function syncDirectory(directoryPath: string): void { + let directoryDescriptor: number | null = null; + try { + directoryDescriptor = fs.openSync(directoryPath, "r"); + fs.fsyncSync(directoryDescriptor); + } catch (error) { + const code = error instanceof Error && "code" in error ? error.code : null; + if (process.platform !== "win32" || !["EACCES", "EINVAL", "EISDIR", "ENOTSUP", "EPERM"].includes(String(code))) { + throw error; + } + } finally { + if (directoryDescriptor !== null) fs.closeSync(directoryDescriptor); + } +} + +function durableCopyFile(sourcePath: string, destinationPath: string, flags = 0): void { + fs.copyFileSync(sourcePath, destinationPath, flags); + fs.chmodSync(destinationPath, 0o600); + + const backupDescriptor = fs.openSync(destinationPath, "r"); + try { + fs.fsyncSync(backupDescriptor); + } finally { + fs.closeSync(backupDescriptor); + } + syncDirectory(path.dirname(destinationPath)); +} + +function atomicWriteFile(filePath: string, contents: string): void { + let attempt = 0; + + while (true) { + const temporaryPath = `${filePath}.tmp-${process.pid}-${attempt}`; + attempt += 1; + let fileDescriptor: number | null = null; + try { + fileDescriptor = fs.openSync(temporaryPath, "wx", 0o600); + fs.writeFileSync(fileDescriptor, contents, "utf8"); + fs.fsyncSync(fileDescriptor); + fs.closeSync(fileDescriptor); + fileDescriptor = null; + fs.renameSync(temporaryPath, filePath); + syncDirectory(path.dirname(filePath)); + return; + } catch (error) { + if (fileDescriptor !== null) fs.closeSync(fileDescriptor); + fs.rmSync(temporaryPath, { force: true }); + const code = error instanceof Error && "code" in error ? error.code : null; + if (code === "EEXIST") continue; + throw error; + } + } +} + +export function backupInvalidConfig(configPath?: string): string { + const filePath = resolveConfigPath(configPath); + if (!fs.existsSync(filePath)) { + throw new Error(`Cannot back up missing config at ${filePath}`); + } + + for (let suffix = 1; ; suffix += 1) { + const backupPath = `${filePath}.invalid-${suffix}`; + try { + durableCopyFile(filePath, backupPath, fs.constants.COPYFILE_EXCL); + return backupPath; + } catch (error) { + const code = error instanceof Error && "code" in error ? error.code : null; + if (code === "EEXIST") continue; + throw error; + } + } +} + export function writeConfig( config: PaperclipConfig, configPath?: string, -): void { + options: { invalidBackupPath?: string } = {}, +): boolean { const filePath = resolveConfigPath(configPath); const dir = path.dirname(filePath); fs.mkdirSync(dir, { recursive: true }); + let nextConfig = paperclipConfigSchema.parse(config); + if (fs.existsSync(filePath)) { + try { + const source = paperclipConfigSchema.parse(migrateLegacyConfig(parseJson(filePath))); + nextConfig = paperclipConfigSchema.parse(mergePaperclipConfig(source, nextConfig)); + if (isDeepStrictEqual(effectiveConfig(source), effectiveConfig(nextConfig))) { + return false; + } + } catch (error) { + const invalidBackupPath = options.invalidBackupPath; + if (!invalidBackupPath) { + throw new Error( + `Refusing to overwrite invalid config at ${filePath}: ${error instanceof Error ? error.message : String(error)}`, + ); + } + if ( + !fs.existsSync(invalidBackupPath) || + !fs.readFileSync(filePath).equals(fs.readFileSync(invalidBackupPath)) + ) { + throw new Error( + `Refusing to overwrite ${filePath} because it changed after the invalid backup was created`, + ); + } + } + } + // Backup existing config before overwriting if (fs.existsSync(filePath)) { const backupPath = filePath + ".backup"; - fs.copyFileSync(filePath, backupPath); - fs.chmodSync(backupPath, 0o600); + durableCopyFile(filePath, backupPath); } - fs.writeFileSync(filePath, JSON.stringify(config, null, 2) + "\n", { - mode: 0o600, - }); + atomicWriteFile(filePath, JSON.stringify(nextConfig, null, 2) + "\n"); + return true; } export function configExists(configPath?: string): boolean { diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 436c7bc85c..79af9ef36a 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -336,6 +336,14 @@ Every local install keeps runtime state directly under the selected instance roo `PAPERCLIP_HOME` and `PAPERCLIP_INSTANCE_ID` override the home root and instance id respectively. `paperclipai onboard` echoes the resolved values in its banner (`Local home: | instance: | config: `) so you can confirm where state will land before continuing. +Config updates preserve unrecognized top-level and nested keys so provider or +plugin extensions survive `configure` and worktree port repair. Likely +misspellings of known keys produce a warning but are not removed. If an +existing `config.json` is malformed, `onboard` and `configure` first create a +byte-for-byte sibling backup named `config.json.invalid-1` (then `-2`, and so +on). Repair from defaults requires an interactive confirmation; non-interactive +runs stop without replacing the original. + ## Database in Dev (Auto-Handled) For local development, leave `DATABASE_URL` unset. diff --git a/packages/shared/src/config-schema.test.ts b/packages/shared/src/config-schema.test.ts index 9e2a75ccf7..5810f2c152 100644 --- a/packages/shared/src/config-schema.test.ts +++ b/packages/shared/src/config-schema.test.ts @@ -1,5 +1,9 @@ import { describe, expect, it } from "vitest"; -import { paperclipConfigSchema } from "./config-schema.js"; +import { + findPaperclipConfigKeyWarnings, + mergePaperclipConfig, + paperclipConfigSchema, +} from "./config-schema.js"; describe("paperclip config schema", () => { it("defaults omitted runtime paths to legacy instance-root locations", () => { @@ -24,4 +28,110 @@ describe("paperclip config schema", () => { expect(parsed.storage.localDisk.baseDir).toBe("~/.paperclip/instances/default/data/storage"); expect(parsed.secrets.localEncrypted.keyFilePath).toBe("~/.paperclip/instances/default/secrets/master.key"); }); + + it("retains extension keys at the top level and every nested config boundary", () => { + const parsed = paperclipConfigSchema.parse({ + $meta: { + version: 1, + updatedAt: "2026-05-10T00:00:00.000Z", + source: "configure", + extensionMeta: "keep", + }, + database: { + mode: "embedded-postgres", + backup: { + backupExtension: "keep", + }, + }, + logging: { + mode: "file", + }, + server: { + serverExtension: "keep", + }, + storage: { + localDisk: { + driverExtension: "keep", + }, + s3: { + s3Extension: "keep", + }, + }, + secrets: { + localEncrypted: { + providerExtension: "keep", + }, + }, + topLevelExtension: { + enabled: true, + }, + }); + + expect(parsed).toMatchObject({ + $meta: { extensionMeta: "keep" }, + database: { backup: { backupExtension: "keep" } }, + server: { serverExtension: "keep" }, + storage: { + localDisk: { driverExtension: "keep" }, + s3: { s3Extension: "keep" }, + }, + secrets: { localEncrypted: { providerExtension: "keep" } }, + topLevelExtension: { enabled: true }, + }); + }); + + it("merges retained extensions without resurrecting removed known fields", () => { + const source = paperclipConfigSchema.parse({ + $meta: { + version: 1, + updatedAt: "2026-05-10T00:00:00.000Z", + source: "configure", + }, + llm: { + provider: "openai", + apiKey: "remove-me", + providerExtension: "keep", + }, + database: { mode: "embedded-postgres" }, + logging: { mode: "file" }, + server: { port: 3100, serverExtension: "keep" }, + topLevelExtension: "keep", + }); + const update = paperclipConfigSchema.parse({ + ...source, + llm: { + provider: "openai", + }, + server: { + ...source.server, + port: 3200, + }, + }); + + const merged = mergePaperclipConfig(source, update); + + expect(merged.server.port).toBe(3200); + expect(merged.server.serverExtension).toBe("keep"); + expect(merged.topLevelExtension).toBe("keep"); + expect(merged.llm?.providerExtension).toBe("keep"); + expect(merged.llm).not.toHaveProperty("apiKey"); + }); + + it("warns about likely misspellings while leaving arbitrary extensions alone", () => { + expect(findPaperclipConfigKeyWarnings({ + servr: {}, + server: { + ports: 3100, + pluginOptions: { + arbitrary: true, + }, + }, + extensionNamespace: { + port: 9999, + }, + })).toEqual([ + { path: "servr", suggestion: "server" }, + { path: "server.ports", suggestion: "server.port" }, + ]); + }); }); diff --git a/packages/shared/src/config-schema.ts b/packages/shared/src/config-schema.ts index 229abd1a02..190645062e 100644 --- a/packages/shared/src/config-schema.ts +++ b/packages/shared/src/config-schema.ts @@ -13,19 +13,19 @@ export const configMetaSchema = z.object({ version: z.literal(1), updatedAt: z.string(), source: z.enum(["onboard", "configure", "doctor"]), -}); +}).passthrough(); export const llmConfigSchema = z.object({ provider: z.enum(["claude", "openai"]), apiKey: z.string().optional(), -}); +}).passthrough(); export const databaseBackupConfigSchema = z.object({ enabled: z.boolean().default(true), intervalMinutes: z.number().int().min(1).max(7 * 24 * 60).default(60), retentionDays: z.number().int().min(1).max(3650).default(7), dir: z.string().default("~/.paperclip/instances/default/data/backups"), -}); +}).passthrough(); export const databaseConfigSchema = z.object({ mode: z.enum(["embedded-postgres", "postgres"]).default("embedded-postgres"), @@ -38,12 +38,12 @@ export const databaseConfigSchema = z.object({ retentionDays: 7, dir: "~/.paperclip/instances/default/data/backups", }), -}); +}).passthrough(); export const loggingConfigSchema = z.object({ mode: z.enum(["file", "cloud"]), logDir: z.string().default("~/.paperclip/instances/default/logs"), -}); +}).passthrough(); export const serverConfigSchema = z.object({ deploymentMode: z.enum(DEPLOYMENT_MODES).default("local_trusted"), @@ -54,17 +54,17 @@ export const serverConfigSchema = z.object({ port: z.number().int().min(1).max(65535).default(3100), allowedHostnames: z.array(z.string().min(1)).default([]), serveUi: z.boolean().default(true), -}); +}).passthrough(); export const authConfigSchema = z.object({ baseUrlMode: z.enum(AUTH_BASE_URL_MODES).default("auto"), publicBaseUrl: z.string().url().optional(), disableSignUp: z.boolean().default(false), -}); +}).passthrough(); export const storageLocalDiskConfigSchema = z.object({ baseDir: z.string().default("~/.paperclip/instances/default/data/storage"), -}); +}).passthrough(); export const storageS3ConfigSchema = z.object({ bucket: z.string().min(1).default("paperclip"), @@ -72,7 +72,7 @@ export const storageS3ConfigSchema = z.object({ endpoint: z.string().optional(), prefix: z.string().default(""), forcePathStyle: z.boolean().default(false), -}); +}).passthrough(); export const storageConfigSchema = z.object({ provider: z.enum(STORAGE_PROVIDERS).default("local_disk"), @@ -85,11 +85,11 @@ export const storageConfigSchema = z.object({ prefix: "", forcePathStyle: false, }), -}); +}).passthrough(); export const secretsLocalEncryptedConfigSchema = z.object({ keyFilePath: z.string().default("~/.paperclip/instances/default/secrets/master.key"), -}); +}).passthrough(); export const secretsConfigSchema = z.object({ provider: z.enum(SECRET_PROVIDERS).default("local_encrypted"), @@ -97,15 +97,15 @@ export const secretsConfigSchema = z.object({ localEncrypted: secretsLocalEncryptedConfigSchema.default({ keyFilePath: "~/.paperclip/instances/default/secrets/master.key", }), -}); +}).passthrough(); export const telemetryConfigSchema = z.object({ enabled: z.boolean().default(true), -}).default({}); +}).passthrough().default({}); export const updatesConfigSchema = z.object({ checkEnabled: z.boolean().default(true), -}).default({}); +}).passthrough().default({}); export const paperclipConfigSchema = z .object({ @@ -140,6 +140,7 @@ export const paperclipConfigSchema = z }, }), }) + .passthrough() .superRefine((value, ctx) => { if (value.server.deploymentMode === "local_trusted" && value.server.exposure !== "private") { ctx.addIssue({ @@ -203,3 +204,150 @@ export type TelemetryConfig = z.infer; export type UpdatesConfig = z.infer; export type ConfigMeta = z.infer; export type DatabaseBackupConfig = z.infer; + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function unwrapConfigSchema(schema: z.ZodTypeAny): z.ZodTypeAny { + let current = schema; + while (true) { + if (current instanceof z.ZodEffects) { + current = current.innerType(); + continue; + } + if (current instanceof z.ZodOptional) { + current = current.unwrap(); + continue; + } + if (current instanceof z.ZodDefault) { + current = current.removeDefault(); + continue; + } + return current; + } +} + +function mergeUnknownConfigKeys( + source: Record, + update: Record, + schema: z.ZodTypeAny, +): Record { + const objectSchema = unwrapConfigSchema(schema); + if (!(objectSchema instanceof z.ZodObject)) return { ...update }; + + const shape = objectSchema.shape as Record; + const merged = { ...update }; + + for (const [key, sourceValue] of Object.entries(source)) { + const childSchema = shape[key]; + if (childSchema === undefined) { + merged[key] = sourceValue; + continue; + } + + const updateValue = update[key]; + if (isRecord(sourceValue) && isRecord(updateValue)) { + merged[key] = mergeUnknownConfigKeys(sourceValue, updateValue, childSchema); + } + } + + return merged; +} + +/** + * Applies a config update while retaining extension keys from the parsed source. + * Known optional keys that are absent from the update stay absent, so callers can + * intentionally clear values such as llm.apiKey or auth.publicBaseUrl. + */ +export function mergePaperclipConfig( + source: PaperclipConfig, + update: PaperclipConfig, +): PaperclipConfig { + return mergeUnknownConfigKeys( + source as Record, + update as Record, + paperclipConfigSchema, + ) as PaperclipConfig; +} + +export type ConfigKeyWarning = { + path: string; + suggestion: string; +}; + +function editDistance(left: string, right: string): number { + const previous = Array.from({ length: right.length + 1 }, (_, index) => index); + + for (let leftIndex = 1; leftIndex <= left.length; leftIndex += 1) { + const current = [leftIndex]; + for (let rightIndex = 1; rightIndex <= right.length; rightIndex += 1) { + const substitutionCost = left[leftIndex - 1] === right[rightIndex - 1] ? 0 : 1; + current[rightIndex] = Math.min( + current[rightIndex - 1] + 1, + previous[rightIndex] + 1, + previous[rightIndex - 1] + substitutionCost, + ); + } + previous.splice(0, previous.length, ...current); + } + + return previous[right.length]; +} + +function nearMatch(key: string, candidates: string[]): string | null { + const normalizedKey = key.toLowerCase(); + let best: { candidate: string; distance: number } | null = null; + + for (const candidate of candidates) { + const normalizedCandidate = candidate.toLowerCase(); + if (normalizedKey === normalizedCandidate && key !== candidate) return candidate; + + const distance = editDistance(normalizedKey, normalizedCandidate); + const threshold = Math.max(normalizedKey.length, normalizedCandidate.length) >= 8 ? 2 : 1; + if (distance > threshold || (best && distance >= best.distance)) continue; + best = { candidate, distance }; + } + + return best?.candidate ?? null; +} + +function collectConfigKeyWarnings( + value: Record, + schema: z.ZodTypeAny, + prefix: string, + warnings: ConfigKeyWarning[], +): void { + const objectSchema = unwrapConfigSchema(schema); + if (!(objectSchema instanceof z.ZodObject)) return; + + const shape = objectSchema.shape as Record; + const knownKeys = Object.keys(shape); + + for (const [key, childValue] of Object.entries(value)) { + const childSchema = shape[key]; + const childPath = prefix ? `${prefix}.${key}` : key; + if (childSchema === undefined) { + const suggestion = nearMatch(key, knownKeys); + if (suggestion) { + warnings.push({ + path: childPath, + suggestion: prefix ? `${prefix}.${suggestion}` : suggestion, + }); + } + continue; + } + + if (isRecord(childValue)) { + collectConfigKeyWarnings(childValue, childSchema, childPath, warnings); + } + } +} + +/** Returns likely misspellings among retained extension keys without modifying them. */ +export function findPaperclipConfigKeyWarnings(config: unknown): ConfigKeyWarning[] { + if (!isRecord(config)) return []; + const warnings: ConfigKeyWarning[] = []; + collectConfigKeyWarnings(config, paperclipConfigSchema, "", warnings); + return warnings; +} diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index 512f54aeeb..2b3d6b918b 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -2262,7 +2262,12 @@ export { storageS3ConfigSchema, secretsLocalEncryptedConfigSchema, telemetryConfigSchema, + updatesConfigSchema, + mergePaperclipConfig, + findPaperclipConfigKeyWarnings, + type ConfigKeyWarning, type TelemetryConfig, + type UpdatesConfig, type PaperclipConfig, type LlmConfig, type DatabaseBackupConfig, diff --git a/server/src/__tests__/config-file.test.ts b/server/src/__tests__/config-file.test.ts index 9d5b2701ca..9cc9451138 100644 --- a/server/src/__tests__/config-file.test.ts +++ b/server/src/__tests__/config-file.test.ts @@ -1,7 +1,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { readConfigFile } from "../config-file.js"; const ORIGINAL_PAPERCLIP_CONFIG = process.env.PAPERCLIP_CONFIG; @@ -42,6 +42,7 @@ describe("readConfigFile", () => { }); afterEach(() => { + vi.restoreAllMocks(); if (ORIGINAL_PAPERCLIP_CONFIG === undefined) { delete process.env.PAPERCLIP_CONFIG; } else { @@ -89,4 +90,25 @@ describe("readConfigFile", () => { }, }); }); + + it("warns about likely misspellings without stripping them", () => { + const config = { + ...(minimalConfig() as Record), + server: { + ports: 3200, + }, + }; + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + writeConfig(configPath, config); + + expect(readConfigFile()).toMatchObject({ + server: { + port: 3100, + ports: 3200, + }, + }); + expect(warn).toHaveBeenCalledWith( + "Unknown config key server.ports; did you mean server.port? It will be preserved.", + ); + }); }); diff --git a/server/src/__tests__/worktree-config.test.ts b/server/src/__tests__/worktree-config.test.ts index 1ea6859f99..36a859d12b 100644 --- a/server/src/__tests__/worktree-config.test.ts +++ b/server/src/__tests__/worktree-config.test.ts @@ -1,3 +1,4 @@ +import fsSync from "node:fs"; import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; @@ -981,6 +982,83 @@ describe("worktree config repair", () => { expect(writtenConfig.auth.publicBaseUrl).toBe("https://paperclip.example"); }); + it("preserves top-level and nested config extensions while persisting runtime ports", async () => { + const tempRoot = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-worktree-config-extensions-")); + const worktreeRoot = path.join(tempRoot, "config-extensions"); + const paperclipDir = path.join(worktreeRoot, ".paperclip"); + const configPath = path.join(paperclipDir, "config.json"); + const isolatedHome = path.join(tempRoot, ".paperclip-worktrees"); + const instanceRoot = path.join(isolatedHome, "instances", "config-extensions"); + const base = buildIsolatedConfig(instanceRoot, 3101, 54331); + const config = { + ...base, + topLevelExtension: { enabled: true }, + database: { + ...base.database, + backup: { + ...base.database.backup, + backupExtension: "keep", + }, + }, + server: { + ...base.server, + serverExtension: "keep", + }, + storage: { + ...base.storage, + localDisk: { + ...base.storage.localDisk, + driverExtension: "keep", + }, + }, + }; + + await fs.mkdir(paperclipDir, { recursive: true }); + await fs.writeFile(configPath, `${JSON.stringify(config, null, 2)}\n`, "utf8"); + await fs.writeFile( + path.join(paperclipDir, ".env"), + ["# Paperclip environment variables", "PAPERCLIP_IN_WORKTREE=true", ""].join("\n"), + "utf8", + ); + + process.chdir(worktreeRoot); + process.env.PAPERCLIP_IN_WORKTREE = "true"; + process.env.PAPERCLIP_WORKTREE_NAME = "config-extensions"; + process.env.PAPERCLIP_HOME = isolatedHome; + process.env.PAPERCLIP_INSTANCE_ID = "config-extensions"; + process.env.PAPERCLIP_CONFIG = configPath; + delete process.env.PORT; + delete process.env.DATABASE_URL; + + const open = vi.spyOn(fsSync, "openSync"); + const sync = vi.spyOn(fsSync, "fsyncSync"); + maybePersistWorktreeRuntimePorts({ serverPort: 3103, databasePort: 54335 }); + + expect(open).toHaveBeenCalledWith(paperclipDir, "r"); + expect(sync).toHaveBeenCalled(); + + const writtenConfig = JSON.parse(await fs.readFile(configPath, "utf8")); + expect(writtenConfig).toMatchObject({ + topLevelExtension: { enabled: true }, + database: { + embeddedPostgresPort: 54335, + backup: { backupExtension: "keep" }, + }, + server: { + port: 3103, + serverExtension: "keep", + }, + storage: { + localDisk: { driverExtension: "keep" }, + }, + }); + + const stableTime = new Date("2020-01-01T00:00:00.000Z"); + await fs.utimes(configPath, stableTime, stableTime); + maybePersistWorktreeRuntimePorts({ serverPort: 3103, databasePort: 54335 }); + expect((await fs.stat(configPath)).mtimeMs).toBe(stableTime.getTime()); + }); + it("can update the in-memory config when auth URL already includes a port", () => { const { config, changed } = applyRuntimePortSelectionToConfig( buildLegacyConfig("/tmp/shared", "http://my-host.ts.net:3100"), diff --git a/server/src/config-file.ts b/server/src/config-file.ts index 9c150f4a78..c66685599f 100644 --- a/server/src/config-file.ts +++ b/server/src/config-file.ts @@ -1,5 +1,9 @@ import fs from "node:fs"; -import { paperclipConfigSchema, type PaperclipConfig } from "@paperclipai/shared"; +import { + findPaperclipConfigKeyWarnings, + paperclipConfigSchema, + type PaperclipConfig, +} from "@paperclipai/shared"; import { ZodError } from "zod"; import { resolvePaperclipConfigPath } from "./paths.js"; @@ -26,7 +30,13 @@ export function readConfigFile(): PaperclipConfig | null { } try { - return paperclipConfigSchema.parse(raw); + const config = paperclipConfigSchema.parse(raw); + for (const warning of findPaperclipConfigKeyWarnings(config)) { + console.warn( + `Unknown config key ${warning.path}; did you mean ${warning.suggestion}? It will be preserved.`, + ); + } + return config; } catch (error) { if (error instanceof ZodError) { throw new Error(`Invalid Paperclip config at ${configPath}: ${formatConfigValidationError(error)}`); diff --git a/server/src/worktree-config.ts b/server/src/worktree-config.ts index de8d824c18..de999134bd 100644 --- a/server/src/worktree-config.ts +++ b/server/src/worktree-config.ts @@ -1,7 +1,12 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import type { PaperclipConfig } from "@paperclipai/shared"; +import { isDeepStrictEqual } from "node:util"; +import { + mergePaperclipConfig, + paperclipConfigSchema, + type PaperclipConfig, +} from "@paperclipai/shared"; import { resolvePaperclipConfigPath, resolvePaperclipEnvPath } from "./paths.js"; function nonEmpty(value: string | null | undefined): string | null { @@ -305,9 +310,60 @@ function resolveWorktreeRuntimeContext( }; } -function writeConfigFile(configPath: string, config: PaperclipConfig): void { +function atomicWriteFile(filePath: string, contents: string): void { + let attempt = 0; + + while (true) { + const temporaryPath = `${filePath}.tmp-${process.pid}-${attempt}`; + attempt += 1; + let fileDescriptor: number | null = null; + try { + fileDescriptor = fs.openSync(temporaryPath, "wx", 0o600); + fs.writeFileSync(fileDescriptor, contents, "utf8"); + fs.fsyncSync(fileDescriptor); + fs.closeSync(fileDescriptor); + fileDescriptor = null; + fs.renameSync(temporaryPath, filePath); + let directoryDescriptor: number | null = null; + try { + directoryDescriptor = fs.openSync(path.dirname(filePath), "r"); + fs.fsyncSync(directoryDescriptor); + } catch (error) { + const code = error instanceof Error && "code" in error ? error.code : null; + if ( + process.platform !== "win32" || + !["EACCES", "EINVAL", "EISDIR", "ENOTSUP", "EPERM"].includes(String(code)) + ) { + throw error; + } + } finally { + if (directoryDescriptor !== null) fs.closeSync(directoryDescriptor); + } + return; + } catch (error) { + if (fileDescriptor !== null) fs.closeSync(fileDescriptor); + fs.rmSync(temporaryPath, { force: true }); + const code = error instanceof Error && "code" in error ? error.code : null; + if (code === "EEXIST") continue; + throw error; + } + } +} + +function writeConfigFile(configPath: string, config: PaperclipConfig): boolean { fs.mkdirSync(path.dirname(configPath), { recursive: true }); - fs.writeFileSync(configPath, JSON.stringify(config, null, 2) + "\n", { mode: 0o600 }); + const update = paperclipConfigSchema.parse(config); + const source = fs.existsSync(configPath) + ? paperclipConfigSchema.parse(JSON.parse(fs.readFileSync(configPath, "utf8"))) + : null; + const nextConfig = source + ? paperclipConfigSchema.parse(mergePaperclipConfig(source, update)) + : update; + + if (source && isDeepStrictEqual(source, nextConfig)) return false; + + atomicWriteFile(configPath, JSON.stringify(nextConfig, null, 2) + "\n"); + return true; } function resolveRepoManagedWorktreesRoot(worktreeRoot: string): string | null {