diff --git a/.env.example b/.env.example index e747df0914..ddd69484ba 100644 --- a/.env.example +++ b/.env.example @@ -4,5 +4,11 @@ SERVE_UI=false BETTER_AUTH_SECRET=paperclip-dev-secret PAPERCLIP_TOOL_ACTION_SIGNING_SECRET=paperclip-dev-tool-action-signing-secret-change-me +# Process-wide protection for expensive full-tree workspace Git scans. +# PAPERCLIP_WORKSPACE_GIT_SCAN_CONCURRENCY=2 +# PAPERCLIP_WORKSPACE_GIT_SCAN_QUEUE_CAPACITY=32 +# PAPERCLIP_WORKSPACE_GIT_SCAN_TIMEOUT_MS=8000 +# PAPERCLIP_WORKSPACE_GIT_SCAN_CACHE_TTL_MS=10000 + # Discord webhook for daily merge digest (scripts/discord-daily-digest.sh) # DISCORD_WEBHOOK_URL=https://discord.com/api/webhooks/... diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 5887970f06..dd287b1ee5 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -425,6 +425,21 @@ Agent, project, environment, secret, skill, and workspace config edits are sampl When effective run config changes, Paperclip may intentionally skip a saved adapter session, refresh persisted workspace runtime config, replace a reused execution workspace, or avoid reusing a sandbox/environment lease. Fresh execution can lose adapter-specific session, workspace, or sandbox state; correctness of the next run's config takes priority over continuity. Plain environment values affect freshness through value hashes; run result JSON and workspace operation logs expose only the non-sensitive freshness decision categories, without storing secret values, full env maps, provider credentials, or private path details. +## Workspace Git Scan Protection + +Paperclip applies one process-wide scheduler to expensive host-side workspace Git enumeration, including changed-file browsing, runtime/finalization cleanliness guards, and adapter sandbox-sync snapshots. The scheduler defaults to two active scans and a bounded queue of 32. Identical scans of the same canonical worktree share one subprocess, while successful changed-file listings are cached for 10 seconds. Correctness-sensitive runtime guards bypass the result cache. + +The cache intentionally trades up to a few seconds of changed-file freshness for stable server latency. The file browser retains an explicit refresh action, does not start its query while the panel or browser tab is hidden, and presents overloads as retryable failures rather than an empty workspace. A full queue returns `503` with code `workspace_git_scan_saturated`; a scan exceeding its wall-clock limit returns `504` with code `workspace_git_scan_timeout`. Both responses include `Retry-After: 1`. + +Environment overrides: + +- `PAPERCLIP_WORKSPACE_GIT_SCAN_CONCURRENCY` (default `2`, range `1`–`16`) +- `PAPERCLIP_WORKSPACE_GIT_SCAN_QUEUE_CAPACITY` (default `32`, range `0`–`1024`) +- `PAPERCLIP_WORKSPACE_GIT_SCAN_TIMEOUT_MS` (default `8000`, range `100`–`120000`) +- `PAPERCLIP_WORKSPACE_GIT_SCAN_CACHE_TTL_MS` (default `10000`, range `0`–`60000`) + +Structured `workspace_git_scan` logs expose the operation name, a non-reversible workspace-path hash, queue and execution durations, active/queued counts, cache and single-flight use, and terminal outcome. Saturation and timeout warnings are rate-limited so an overload does not create a second logging storm. + ## Worktree-local Instances When developing from multiple git worktrees, do not point two Paperclip servers at the same embedded PostgreSQL data directory. diff --git a/packages/adapter-utils/CHANGELOG.md b/packages/adapter-utils/CHANGELOG.md index 76cabbd73f..a59d037528 100644 --- a/packages/adapter-utils/CHANGELOG.md +++ b/packages/adapter-utils/CHANGELOG.md @@ -1,5 +1,11 @@ # @paperclipai/adapter-utils +## Unreleased + +### Patch Changes + +- Allow the Paperclip host to route adapter sandbox-sync full-tree Git enumeration through its process-wide bounded scheduler. + ## 0.3.1 ### Patch Changes diff --git a/packages/adapter-utils/src/git-workspace-sync.test.ts b/packages/adapter-utils/src/git-workspace-sync.test.ts index 3f5b70ca8c..1a233e4c5a 100644 --- a/packages/adapter-utils/src/git-workspace-sync.test.ts +++ b/packages/adapter-utils/src/git-workspace-sync.test.ts @@ -15,6 +15,7 @@ import { readGitWorkspaceSnapshot, runLocalGit, sanitizeGitRemoteUrl, + setExpensiveWorkspaceGitExecutor, withShallowGitWorkspaceClone, } from "./git-workspace-sync.js"; @@ -28,6 +29,7 @@ describe("git workspace sync", () => { const cleanupDirs: string[] = []; afterEach(async () => { + setExpensiveWorkspaceGitExecutor(null); while (cleanupDirs.length > 0) { const dir = cleanupDirs.pop(); if (!dir) continue; @@ -35,6 +37,31 @@ describe("git workspace sync", () => { } }); + it("delegates every host-side full-tree enumeration to the registered scheduler", async () => { + const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-git-scheduler-hook-")); + cleanupDirs.push(rootDir); + const repo = await createRepo(rootDir); + await writeFile(path.join(repo, "untracked.txt"), "untracked\n", "utf8"); + const operations: string[] = []; + setExpensiveWorkspaceGitExecutor(async (input) => { + operations.push(input.operation); + return await runLocalGit(input.localDir, [...input.args], { + timeout: input.timeout, + maxBuffer: input.maxBuffer, + }); + }); + + const snapshot = await readGitWorkspaceSnapshot(repo); + + expect(snapshot?.overlayPaths).toContain("untracked.txt"); + expect(operations.sort()).toEqual([ + "adapter_sync.deleted_files", + "adapter_sync.ignored_files", + "adapter_sync.overlay_diff", + "adapter_sync.untracked_files", + ]); + }); + async function createRepo(rootDir: string): Promise { const repo = path.join(rootDir, "repo"); await mkdir(repo, { recursive: true }); diff --git a/packages/adapter-utils/src/git-workspace-sync.ts b/packages/adapter-utils/src/git-workspace-sync.ts index dd0517ec41..fd3525ecc0 100644 --- a/packages/adapter-utils/src/git-workspace-sync.ts +++ b/packages/adapter-utils/src/git-workspace-sync.ts @@ -17,6 +17,29 @@ export interface GitWorkspaceSnapshot { ignoredPaths: string[]; } +export interface ExpensiveWorkspaceGitInput { + localDir: string; + args: readonly string[]; + operation: string; + timeout: number; + maxBuffer: number; +} + +export type ExpensiveWorkspaceGitExecutor = ( + input: ExpensiveWorkspaceGitInput, +) => Promise; + +let expensiveWorkspaceGitExecutor: ExpensiveWorkspaceGitExecutor | null = null; + +/** + * Lets a host process apply its process-wide admission policy to the adapter + * package's full-tree Git walks. Standalone adapter-utils consumers retain the + * existing timeout/buffer-bounded fallback. + */ +export function setExpensiveWorkspaceGitExecutor(executor: ExpensiveWorkspaceGitExecutor | null): void { + expensiveWorkspaceGitExecutor = executor; +} + export const GIT_ARCHIVE_EXCLUDES = [".git", ".git/*"] as const; function shellQuote(value: string) { @@ -53,6 +76,24 @@ export async function runLocalGit( }); } +async function runExpensiveWorkspaceGit( + localDir: string, + args: string[], + operation: string, + options: { timeout: number; maxBuffer: number }, +): Promise { + if (expensiveWorkspaceGitExecutor) { + return await expensiveWorkspaceGitExecutor({ + localDir, + args, + operation, + timeout: options.timeout, + maxBuffer: options.maxBuffer, + }); + } + return await runLocalGit(localDir, args, options); +} + export async function readGitWorkspaceSnapshot(localDir: string): Promise { try { const insideWorkTree = await runLocalGit(localDir, ["rev-parse", "--is-inside-work-tree"], { @@ -72,19 +113,19 @@ export async function readGitWorkspaceSnapshot(localDir: string): Promise