diff --git a/Dockerfile b/Dockerfile index e9e4c33f91..034ac98683 100644 --- a/Dockerfile +++ b/Dockerfile @@ -202,6 +202,9 @@ RUN set -eu; \ # doc/observability.md). # # CLOUD_BUNDLED_SERVER_DEPS names the optional peer packages to install. +# CLOUD_BUNDLED_OTEL_DEPS adds the tracing peers independently because release +# workflows explicitly override the former with the Sentry package. Installing +# tracing support does not enable it: an operator must still set an OTLP endpoint. # The value is a space-separated list, the same shape as # CLOUD_BUNDLED_PLUGINS above. The stage reads each package's version # from the `peerDependencies` block of `server/package.json` at build @@ -238,17 +241,19 @@ RUN set -eu; \ FROM build AS cloud-server-deps WORKDIR /app/.cloud-server-deps ARG CLOUD_BUNDLED_SERVER_DEPS="@sentry/node" +ARG CLOUD_BUNDLED_OTEL_DEPS="@opentelemetry/sdk-node @opentelemetry/auto-instrumentations-node @opentelemetry/resources @opentelemetry/semantic-conventions @opentelemetry/exporter-trace-otlp-grpc @opentelemetry/exporter-trace-otlp-proto @opentelemetry/exporter-trace-otlp-http" RUN set -eu; \ test -n "$CLOUD_BUNDLED_SERVER_DEPS" || { echo "ERROR: CLOUD_BUNDLED_SERVER_DEPS is empty; name at least one optional peer package to install" >&2; exit 1; }; \ echo '{"name":"paperclip-cloud-server-deps","private":true}' > package.json; \ specifiers=""; \ - for name in $CLOUD_BUNDLED_SERVER_DEPS; do \ + for name in $CLOUD_BUNDLED_SERVER_DEPS $CLOUD_BUNDLED_OTEL_DEPS; do \ version="$(node -e "const pkg=require('/app/server/package.json'); const name=process.argv[1]; const version=(pkg.peerDependencies||{})[name]; if(!version){console.error('ERROR: server/package.json declares no peerDependencies version for '+JSON.stringify(name));process.exit(1);} const meta=(pkg.peerDependenciesMeta||{})[name]; if(!meta||meta.optional!==true){console.error('ERROR: '+JSON.stringify(name)+' is not declared as an optional peer dependency in server/package.json; CLOUD_BUNDLED_SERVER_DEPS may name only optional peer packages');process.exit(1);} process.stdout.write(version);" "$name")"; \ test -n "$version" || { echo "ERROR: could not resolve a version for '$name'" >&2; exit 1; }; \ specifiers="$specifiers ${name}@${version}"; \ done; \ test -n "$specifiers" || { echo "ERROR: CLOUD_BUNDLED_SERVER_DEPS names no package" >&2; exit 1; }; \ - pnpm add --ignore-workspace --no-lockfile $specifiers + pnpm add --ignore-workspace --no-lockfile $specifiers \ + && node -e "const fs=require('node:fs'),path=require('node:path'),assert=require('node:assert/strict');const peers=require('/app/server/package.json').peerDependencies;for(const name of process.argv.slice(1)){let dir=path.dirname(require.resolve(name)),found=false;for(let depth=0;depth<12;depth++,dir=path.dirname(dir)){const file=path.join(dir,'package.json');if(!fs.existsSync(file))continue;const pkg=JSON.parse(fs.readFileSync(file,'utf8'));if(pkg.name!==name)continue;assert.equal(pkg.version,peers[name],name+' must match the server optional peer version');found=true;break;}assert(found,'Cannot verify installed optional peer '+name);}" $CLOUD_BUNDLED_SERVER_DEPS $CLOUD_BUNDLED_OTEL_DEPS # Use the same qualified interpreter as the Daytona provider-pack build. # The controller owns this pack and its manifest; remote OpenCode/ACPX launches @@ -277,7 +282,7 @@ ARG PAPERCLIP_BUILD_COMMIT RUN test -n "${PAPERCLIP_BUILD_COMMIT}" \ && pnpm --filter @paperclipai/paperclip-runner build:typescript \ && PAPERCLIP_RUNNER_SOURCE_REVISION="${PAPERCLIP_BUILD_COMMIT}" \ - node packages/paperclip-runner/scripts/build-provider-pack.mjs /provider-pack \ + node packages/paperclip-runner/scripts/assemble-provider-pack.mjs /provider-pack \ && node packages/paperclip-runner/scripts/verify-pi-provider-launch.mjs /provider-pack \ && chmod -R a+rX /provider-pack diff --git a/doc/observability.md b/doc/observability.md index e730ad2120..c31c2ba357 100644 --- a/doc/observability.md +++ b/doc/observability.md @@ -132,10 +132,76 @@ This document also holds three local instrumentation contracts: native runner traces, sandbox startup traces, and sandbox duplex transport instrumentation. Those sections follow below. +## Full sandbox lifecycle diagnostics + +With `OTEL_EXPORTER_OTLP_ENDPOINT` configured, `sandbox.run` measures the host +execution path for both runner generations. Its children cover authorization +and database work, workspace/environment acquisition, scoped work-folder +preparation, runtime context and launchers, dispatch, final saving and cleanup. +The Cloud image includes the exact optional OTel peers declared by the server; +self-hosted images retain the optional installation procedure above. The +endpoint gate remains required in both cases. + +`heartbeat.*` operations surround the actual awaited calls. Repeated database +calls share a name and have a numeric `operationIndex` for source correlation. +`work_folder.*` operations split binding checks, scope and history lookups, +attachment seeding/import, manifests, incoming/outgoing scans, metadata queries, +object GET response acquisition, body consumption, transport batches, repository +credentials/clone/checkout/setup, and checkpoint publication. Checkpoints also +measure object journal queries, existence checks, uploads, both repository +scans, manifest storage, and the final pointer transaction. A completed request +is not a completed body transfer: those are separate spans. + +The live host dispatch context parents native `task.run` when these diagnostics +are active. Native startup and agent spans remain beneath it. Previously +collected aggregate preparation durations are not backdated into a second +overlapping native preparation tree in this mode. Without the endpoint, native +local run-log behavior and its historical preparation tree remain unchanged. +Active contexts also propagate through the OpenTelemetry context manager, so +HTTP and database auto-instrumentation inherits the closest operation. + +Names and attributes are fixed operation labels, numeric indices/counts, finite +durations, byte counts, retry attempts, and boolean outcomes. Raw run identifiers +are hashed. New diagnostic attributes exclude paths, file contents, repository +URLs, credentials, commands, and exception messages. An operation's `failed` +outcome means that operation threw; the authoritative task outcome still comes +from its run row, since execution can catch an error and return normally. + +Filesystem helpers report their own monotonic phase offsets. These become +events on the measured host command span, with `clock=remote_relative`, +`startOffsetMs` and `durationMs`. They are **not** host-clock child spans: there +is no measured clock alignment. Compare host round-trip time and remote +execution time without labeling the difference pure network latency; it also +contains queueing and transport overhead. Body `readWaitMs` measures waits for +the next iterator result; `bodyNonReadMs` includes consumer processing and +backpressure. Neither is a direct network measurement. + +Startup, agent execution, checkpointing, and total task duration must be +reported separately. For a waterfall, subtract the **union** of child intervals +to find unattributed time; summing nested or parallel spans double counts work. +Keep remote-relative phases in their own lane and identify missing intervals +explicitly. A warm scoped folder means a prior manifest exists for the same +physical sandbox, not that the provider was already running. Repository reuse +means a usable checkout was found; restoring a checkpoint in a replacement +sandbox is distinct from finding an existing checkout. + +The SDK batches exports. The helper also keeps at most 20,000 local records +per run, writes them in batches of at most 250 after measured execution, and +reports discarded records in `dropped` (also on the exported root span). +Remote helpers cap detailed phases at 256 per command and report dropped +phases separately. No per-file synchronous logging or exporter call is added. +The local batch-write tail is outside `sandbox.run`; include it separately +when comparing whole-process completion. Exporter queue drops and collector +failures can still make exported traces incomplete: verify actual collector +records and limits before using a trace as performance evidence. Diagnostic +sink failures do not fail the task. See the local +[`sandbox.performance.batch` contract](run-log-events.md#sandbox-performance-batches). + ## Native Runner Trace Spans Paperclip Runner task runs emit a single foldable OpenTelemetry trace. This is -the native-run trace schema version `2`. `task.run` is the only full-run root; +the native-run trace schema version `2`. `task.run` is the native subtree root +(parented to live host dispatch in full lifecycle diagnostics); every other native span carries a real OpenTelemetry parent context rather than only a descriptive `parentName` field. diff --git a/doc/run-log-events.md b/doc/run-log-events.md index fa98161673..1b094fd61f 100644 --- a/doc/run-log-events.md +++ b/doc/run-log-events.md @@ -107,6 +107,30 @@ The payload never carries a command, an argument, a path, an environment value, or a raw identifier. The event rides the `ctx.onEvent` run-event bridge and is run-log-only. It needs no OTLP endpoint. +## Sandbox performance batches + +When full sandbox diagnostics are enabled by `OTEL_EXPORTER_OTLP_ENDPOINT`, the +host writes `sandbox.performance.batch` system events after measured execution. +These rows stay in the instance database; they are neither first-party Telemetry +events nor a replacement for inspecting actual OpenTelemetry exports. No batch +is produced with the endpoint unset. + +The payload schema is `paperclip.sandbox-performance.v1`, with a hashed +`runHash`, `records` (at most 250 per event), and the run's `dropped` count. +Each record contains a fixed operation name, span id, optional parent/trace id, +start time, duration, operation outcome, and closed safe attributes. Host times +use epoch milliseconds and monotonic durations. Records with +`clock: "remote_relative"` instead contain offsets from a remote command's +start and must not be placed on the host timeline as absolute timestamps. +The root has numeric retained-record and dropped-record counts. + +The default buffer holds 20,000 records per run; additional records increment +the drop count. A missing root or a nonzero drop count means the local record +set is incomplete. Event persistence happens in bounded batches after the +operation timings end, not synchronously for each file. A sink failure stops +batch persistence without changing the original task outcome. A successful +task alone therefore does not prove that all timing records were saved. + ## Related instrumentation The sandbox duplex transport also writes one run-log event as one of its three diff --git a/doc/sandbox-work-folders.md b/doc/sandbox-work-folders.md index 6453513051..c34565d34b 100644 --- a/doc/sandbox-work-folders.md +++ b/doc/sandbox-work-folders.md @@ -377,6 +377,17 @@ or patches change, update both expected hashes, and qualify a new sandbox image. Matching source files alone does not prove matching dependencies: acceptance also compares the actual app and sandbox production-lock hashes. +The qualification entry `build-provider-pack.mjs` (also exposed as +`pnpm --filter @paperclipai/paperclip-runner build:provider-pack`) requires +Docker with BuildKit and builds the canonical `linux/amd64` provider stage. +It uses the same digest-pinned Node interpreter, dedicated dependency graph, +and fresh TypeScript compilation as the sandbox image; host `node_modules`, +CI's root lock and previously compiled outputs are not assembly inputs. +Both Docker stages call the low-level assembler directly, avoiding recursion. +The entry verifies exported manifest and artifact bytes before atomically +replacing its output. Failed builds or validation preserve the previous pack. +This produces a local artifact and does not publish an image or release. + Native and legacy Git credential callbacks honor the same experimental duplex setting and provider capability gates. When streaming is disabled or unavailable, the file bridge remains supported. Credential acquisition allows 35 seconds per diff --git a/docker/daytona-runner/Dockerfile b/docker/daytona-runner/Dockerfile index 60b6398d4a..1d938956be 100644 --- a/docker/daytona-runner/Dockerfile +++ b/docker/daytona-runner/Dockerfile @@ -37,10 +37,14 @@ ARG PAPERCLIP_RUNNER_SOURCE_REVISION RUN test -n "${PAPERCLIP_RUNNER_SOURCE_REVISION}" RUN pnpm --filter @paperclipai/paperclip-runner build:typescript \ && PAPERCLIP_RUNNER_SOURCE_REVISION="${PAPERCLIP_RUNNER_SOURCE_REVISION}" \ - node packages/paperclip-runner/scripts/build-provider-pack.mjs /provider-pack \ + node packages/paperclip-runner/scripts/assemble-provider-pack.mjs /provider-pack \ && node packages/paperclip-runner/scripts/verify-pi-provider-launch.mjs /provider-pack \ && chmod -R a+rX /provider-pack +# The trusted qualification entry exports the identical canonical provider stage. +FROM scratch AS provider-pack-export +COPY --from=provider-pack-build /provider-pack / + # Fleet sandbox base image. Keep this section aligned with # paperclipai/paperclip-cloud/fleet-sandbox-image/Dockerfile. The only Paperclip # runner-specific addition is /usr/local/bin/paperclip-runnerd below. diff --git a/packages/paperclip-runner/package.json b/packages/paperclip-runner/package.json index 3a39ec27bb..9a7d0c691a 100644 --- a/packages/paperclip-runner/package.json +++ b/packages/paperclip-runner/package.json @@ -61,7 +61,7 @@ "build:typescript": "pnpm run ensure:eval-build-deps && pnpm run check:protocol-types && node ./node_modules/typescript/bin/tsc --version && node ./node_modules/typescript/bin/tsc -p tsconfig.json && node ./node_modules/typescript/bin/tsc -p tsconfig.surfaces.json && node scripts/build-verified-provider-entrypoints.mjs", "build:rust": "cargo build --manifest-path runner/Cargo.toml --locked --workspace --bins", "build:binary": "cargo build --release --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core --bin paperclip-runnerd && node scripts/stage-runner-binary.mjs", - "build:provider-pack": "pnpm run build:typescript && node scripts/build-provider-pack.mjs", + "build:provider-pack": "node scripts/build-provider-pack.mjs", "build:runner-binaries": "cargo build --manifest-path runner/Cargo.toml --locked --workspace --bins", "build:browser": "vite build --config vite.config.ts", "build:sdk": "vite build --config vite.sdk.config.ts", @@ -73,7 +73,7 @@ "typecheck:rust": "cargo fmt --manifest-path runner/Cargo.toml --all -- --check && cargo check --manifest-path runner/Cargo.toml --locked --workspace", "typecheck:browser": "tsc -p tsconfig.browser.json --noEmit", "test": "pnpm run test:typescript && pnpm run test:rust", - "test:typescript": "pnpm run ensure:eval-build-deps && pnpm run build:rust && node --test test/protocol-contract.test.mjs test/acpx-sidecar-contract.test.mjs test/acpx-codex-package-contract.test.mjs scripts/aws-agentcore-provisioning.test.mjs scripts/build-verified-provider-entrypoints.test.mjs scripts/local-provider-smoke-environment.test.mjs scripts/materialize-opencode-binary.test.mjs scripts/materialize-pi-binary.test.mjs scripts/portable-provider-shim.test.mjs scripts/provider-pack-layout.test.mjs && vitest run", + "test:typescript": "pnpm run ensure:eval-build-deps && pnpm run build:rust && node --test test/protocol-contract.test.mjs test/acpx-sidecar-contract.test.mjs test/acpx-codex-package-contract.test.mjs scripts/aws-agentcore-provisioning.test.mjs scripts/build-verified-provider-entrypoints.test.mjs scripts/local-provider-smoke-environment.test.mjs scripts/materialize-opencode-binary.test.mjs scripts/materialize-pi-binary.test.mjs scripts/portable-provider-shim.test.mjs scripts/provider-pack-layout.test.mjs scripts/build-provider-pack.test.mjs && vitest run", "test:rust": "cargo test --release --manifest-path runner/Cargo.toml --locked --workspace", "test:codex": "cargo test --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core --test codex_provider", "test:durable": "cargo test --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core durable::", diff --git a/packages/paperclip-runner/scripts/assemble-provider-pack.mjs b/packages/paperclip-runner/scripts/assemble-provider-pack.mjs new file mode 100644 index 0000000000..9168ba6ce4 --- /dev/null +++ b/packages/paperclip-runner/scripts/assemble-provider-pack.mjs @@ -0,0 +1,305 @@ +import { canonicalJson, sha256File, sha256Tree } from "./provider-pack-integrity.mjs"; +import { normalizeProviderPackLayout } from "./provider-pack-layout.mjs"; +import { portableProviderShim } from "./portable-provider-shim.mjs"; +import { createHash } from "node:crypto"; +import { execFileSync, spawnSync } from "node:child_process"; +import { + chmodSync, + copyFileSync, + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + realpathSync, + renameSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, relative, resolve } from "node:path"; +import { createRequire } from "node:module"; +import { fileURLToPath } from "node:url"; +import { materializePiBinary } from "./materialize-pi-binary.mjs"; + +const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const workspaceRoot = resolve(packageRoot, "../.."); +const outputArgument = process.argv.slice(2).find((value) => value !== "--"); +const outputRoot = resolve( + process.cwd(), + outputArgument ?? join(packageRoot, "provider-pack"), +); +if ( + outputRoot === workspaceRoot || + outputRoot === packageRoot || + outputRoot === "/" +) { + throw new Error(`Refusing unsafe provider-pack output path: ${outputRoot}`); +} + +const temporaryParent = mkdtempSync(join(tmpdir(), "paperclip-provider-pack-")); +const temporaryRoot = join(temporaryParent, "pack"); + +function writePortableNodeShim(name, entrypoint) { + const shimPath = join(temporaryRoot, "node_modules", ".bin", name); + writeFileSync(shimPath, portableProviderShim(entrypoint, { node: true })); + chmodSync(shimPath, 0o755); +} + +function writePortableExecutableShim(name, executable) { + const shimPath = join(temporaryRoot, "node_modules", ".bin", name); + writeFileSync(shimPath, portableProviderShim(executable)); + chmodSync(shimPath, 0o755); +} + +try { + const deployed = spawnSync( + "pnpm", + [ + "--filter", + "@paperclipai/paperclip-runner", + "deploy", + "--prod", + temporaryRoot, + ], + { cwd: workspaceRoot, encoding: "utf8", stdio: "inherit" }, + ); + if (deployed.status !== 0) { + throw new Error(`pnpm deploy failed with exit code ${deployed.status}`); + } + + normalizeProviderPackLayout(temporaryRoot); + + // Fail the image build if a bridge silently brings back an older/private + // provider CLI. A direct dependency alone does not deduplicate pnpm's graph. + const packRequire = createRequire(join(temporaryRoot, "package.json")); + const codexAcpRequire = createRequire(packRequire.resolve("@agentclientprotocol/codex-acp/package.json")); + if (realpathSync(codexAcpRequire.resolve("@openai/codex/package.json")) !== + realpathSync(packRequire.resolve("@openai/codex/package.json"))) { + throw new Error("Codex ACP must share the image's Codex installation"); + } + + // Reuse the already-qualified build interpreter instead of introducing a + // package-manager lifecycle hook or a second binary supply chain. The pack + // manifest binds the copied bytes, platform, architecture, and minimum + // version before any provider is launched. + const minimumNodeVersion = [24, 11, 0]; + const actualNodeVersion = process.versions.node.split(".").map(Number); + if ( + actualNodeVersion[0] < minimumNodeVersion[0] || + (actualNodeVersion[0] === minimumNodeVersion[0] && + (actualNodeVersion[1] < minimumNodeVersion[1] || + (actualNodeVersion[1] === minimumNodeVersion[1] && + actualNodeVersion[2] < minimumNodeVersion[2]))) + ) { + throw new Error("Provider pack build Node is older than 24.11.0"); + } + const stableNodeRoot = join(temporaryRoot, "node_modules", "node"); + if (existsSync(stableNodeRoot)) { + throw new Error( + "Provider pack deployment unexpectedly claimed the stable Node path", + ); + } + const stableNodeCommand = join(stableNodeRoot, "bin", "node"); + mkdirSync(dirname(stableNodeCommand), { recursive: true, mode: 0o755 }); + copyFileSync(process.execPath, stableNodeCommand); + chmodSync(stableNodeCommand, 0o755); + + // pnpm's generated .bin shims embed the temporary deployment directory in + // NODE_PATH. That makes an otherwise identical provider pack hash differ on + // every build and leaks a nonexistent host path after relocation. Replace + // every provider-facing shim with a pack-relative launcher that always uses + // the pinned Node executable owned by this pack. + // The image exposes these same installations to every adapter. Never add a + // separate global/runner-only CLI version; refresh these packages and their + // qualification digests together to the latest stable releases. + writePortableNodeShim("codex", "@openai/codex/bin/codex.js"); + const claudeAcpRequire = createRequire( + packRequire.resolve("@agentclientprotocol/claude-agent-acp/package.json"), + ); + // Use the ACP bridge's SDK dependency directly, avoiding a second peer- + // resolved SDK installation just to expose its CLI on the global PATH. + const sdkRequire = createRequire(claudeAcpRequire.resolve("@anthropic-ai/claude-agent-sdk")); + const claudeExecutable = sdkRequire.resolve( + `@anthropic-ai/claude-agent-sdk-${process.platform}-${process.arch}/claude`, + ); + writePortableExecutableShim( + "claude", + relative(realpathSync(join(temporaryRoot, "node_modules")), realpathSync(claudeExecutable)), + ); + writePortableExecutableShim("node", "node/bin/node"); + writePortableExecutableShim("opencode", "opencode-ai/bin/opencode.exe"); + writePortableNodeShim("acpx", "acpx/dist/cli.js"); + const piPackageRoot = realpathSync(join(temporaryRoot, "node_modules", "@earendil-works", "pi-coding-agent")); + await materializePiBinary(piPackageRoot); + writePortableExecutableShim("pi", "@earendil-works/pi-coding-agent/vendor/standalone/pi"); + writePortableNodeShim("pi-acp", "pi-acp/dist/index.js"); + writePortableNodeShim( + "claude-agent-acp", + "@agentclientprotocol/claude-agent-acp/dist/index.js", + ); + writePortableNodeShim( + "codex-acp", + "@agentclientprotocol/codex-acp/dist/index.js", + ); + + // pnpm deploy may retain a workspace self-link under its virtual store. It + // points outside the immutable pack and is not needed by the deployed + // package, so it must not be staged or captured. + rmSync( + join( + temporaryRoot, + "node_modules", + ".pnpm", + "node_modules", + "@paperclipai", + "paperclip-runner", + ), + { recursive: true, force: true }, + ); + + for (const shimName of readdirSync( + join(temporaryRoot, "node_modules", ".bin"), + )) { + const shimPath = join(temporaryRoot, "node_modules", ".bin", shimName); + const contents = readFileSync(shimPath, "utf8"); + if (contents.includes(temporaryParent)) { + throw new Error( + `Provider pack shim ${shimName} retains its temporary build path`, + ); + } + } + + // Exercise a clean import from the deployed dependency graph. This catches + // malformed patch hunks that can look correct in an existing pnpm store but + // land inside the wrong function when applied to a fresh acpx tarball. + const acpxImport = spawnSync( + join(temporaryRoot, "node_modules", "node", "bin", "node"), + ["--input-type=module", "-e", 'await import("acpx/runtime")'], + { + cwd: temporaryRoot, + encoding: "utf8", + timeout: 30_000, + }, + ); + if (acpxImport.status !== 0) { + throw new Error( + `Provider pack ACPX import failed: ${String(acpxImport.stderr ?? "") + .trim() + .slice(-2_000)}`, + ); + } + + const opencodeProxyPath = "dist/cli/opencode-app-server-proxy.cjs"; + const acpxSidecarPath = "dist/cli/acpx-runtime-sidecar.cjs"; + const opencodeCommand = "node_modules/.bin/opencode"; + const opencodeExecutable = "node_modules/opencode-ai/bin/opencode.exe"; + const nodeCommand = "node_modules/node/bin/node"; + const productionLock = "pnpm-lock.yaml"; + copyFileSync( + join(workspaceRoot, "pnpm-lock.yaml"), + join(temporaryRoot, productionLock), + ); + for (const relativePath of [ + nodeCommand, + productionLock, + opencodeProxyPath, + acpxSidecarPath, + opencodeCommand, + opencodeExecutable, + ]) { + if (!existsSync(join(temporaryRoot, relativePath))) { + throw new Error(`Provider pack is missing ${relativePath}`); + } + } + + const opencodeProxySha = sha256File(join(temporaryRoot, opencodeProxyPath)); + const acpxSidecarSha = sha256File(join(temporaryRoot, acpxSidecarPath)); + const distDigest = sha256Tree(join(temporaryRoot, "dist")); + const configuredRevision = + process.env.PAPERCLIP_RUNNER_SOURCE_REVISION?.trim(); + const revision = + configuredRevision ?? + execFileSync("git", ["rev-parse", "HEAD"], { + cwd: workspaceRoot, + encoding: "utf8", + }).trim(); + if (!/^[0-9a-f]{40}$/.test(revision)) { + throw new Error("PAPERCLIP_RUNNER_SOURCE_REVISION must be a full Git SHA"); + } + const dirty = configuredRevision + ? false + : spawnSync("git", ["diff", "--quiet", "--", "packages/paperclip-runner"], { + cwd: workspaceRoot, + }).status !== 0; + const payload = { + pins: { + nodeMinimum: minimumNodeVersion.join("."), + codex: "0.153.4", + opencode: "1.18.29", + acpx: "0.13.1", + claudeAcp: "0.70.0", + codexAcp: "1.6.2", + pi: "0.84.2", + piAcp: "0.0.33", + }, + target: { platform: process.platform, architecture: process.arch }, + runnerSourceRevision: `${revision}${dirty ? "-dirty" : ""}`, + distDigest, + bridgeDigest: `sha256:${createHash("sha256") + .update(opencodeProxySha) + .update("\n") + .update(acpxSidecarSha) + .update("\n") + .update(distDigest) + .digest("hex")}`, + acpxProfileDigests: { + pi: "sha256:24ff73fda6e3c76ddce2d359a79f5c4b8f292eb290e4d2ab85aac94676b2c2dc", + claude: + "sha256:9d73d1f0f121fb96cc8badb28c22d5bff02d8582eb2e40360a81c189e1b9422a", + codex: + "sha256:7a923b3829884d3cabcc9659d22cace3f86813e7bfffc90974b10140a45bc400", + }, + artifacts: { + nodeCommand: { + path: nodeCommand, + sha256: sha256File(join(temporaryRoot, nodeCommand)), + }, + productionLock: { + path: productionLock, + sha256: sha256File(join(temporaryRoot, productionLock)), + }, + opencodeCommand: { + path: opencodeCommand, + sha256: sha256File(join(temporaryRoot, opencodeCommand)), + }, + opencodeExecutable: { + path: opencodeExecutable, + sha256: sha256File(join(temporaryRoot, opencodeExecutable)), + }, + opencodeProxy: { + path: opencodeProxyPath, + sha256: opencodeProxySha, + }, + acpxSidecar: { path: acpxSidecarPath, sha256: acpxSidecarSha }, + }, + }; + const manifest = { + schema: "paperclip-runner/remote-provider-pack/v1", + digest: `sha256:${createHash("sha256") + .update(canonicalJson(payload)) + .digest("hex")}`, + payload, + }; + writeFileSync( + join(temporaryRoot, "provider-pack.json"), + `${JSON.stringify(manifest, null, 2)}\n`, + { mode: 0o600 }, + ); + + rmSync(outputRoot, { recursive: true, force: true }); + renameSync(temporaryRoot, outputRoot); + process.stdout.write(`${outputRoot}\n`); +} finally { + rmSync(temporaryParent, { recursive: true, force: true }); +} diff --git a/packages/paperclip-runner/scripts/build-provider-pack.mjs b/packages/paperclip-runner/scripts/build-provider-pack.mjs index 797595c8f9..f523ecf86a 100644 --- a/packages/paperclip-runner/scripts/build-provider-pack.mjs +++ b/packages/paperclip-runner/scripts/build-provider-pack.mjs @@ -1,351 +1,74 @@ -import { normalizeProviderPackLayout } from "./provider-pack-layout.mjs"; -import { portableProviderShim } from "./portable-provider-shim.mjs"; -import { createHash } from "node:crypto"; +// The trusted qualification workflow invokes this target-owned entry point. +// Never assemble from that job's CI-resolved lock, node_modules, shared dist, +// or setup-node interpreter: the image's isolated stage is the source of truth. import { execFileSync, spawnSync } from "node:child_process"; -import { - chmodSync, - copyFileSync, - existsSync, - mkdirSync, - mkdtempSync, - readdirSync, - readFileSync, - readlinkSync, - realpathSync, - renameSync, - rmSync, - writeFileSync, -} from "node:fs"; -import { tmpdir } from "node:os"; -import { dirname, join, relative, resolve } from "node:path"; -import { createRequire } from "node:module"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, realpathSync, renameSync, rmSync } from "node:fs"; +import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; import { fileURLToPath } from "node:url"; -import { materializePiBinary } from "./materialize-pi-binary.mjs"; +import { sha256File, verifyProviderPack } from "./provider-pack-integrity.mjs"; const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); -const workspaceRoot = resolve(packageRoot, "../.."); -const outputArgument = process.argv.slice(2).find((value) => value !== "--"); -const outputRoot = resolve( - process.cwd(), - outputArgument ?? join(packageRoot, "provider-pack"), -); -if ( - outputRoot === workspaceRoot || - outputRoot === packageRoot || - outputRoot === "/" -) { - throw new Error(`Refusing unsafe provider-pack output path: ${outputRoot}`); +const defaultWorkspaceRoot = resolve(packageRoot, "../.."); + +export function buildProviderPack({ workspaceRoot = defaultWorkspaceRoot, outputRoot = join(packageRoot, "provider-pack"), + revision = process.env.PAPERCLIP_RUNNER_SOURCE_REVISION?.trim(), run = spawnSync } = {}) { + workspaceRoot = realpathSync(workspaceRoot); + outputRoot = resolve(outputRoot); + const missingSegments = []; + let existingParent = outputRoot; + while (!existsSync(existingParent)) { missingSegments.unshift(basename(existingParent)); existingParent = dirname(existingParent); } + outputRoot = join(realpathSync(existingParent), ...missingSegments); + const sourcePackage = join(workspaceRoot, "packages/paperclip-runner"); + function contains(base, target) { + const child = relative(base, target); + return !isAbsolute(child) && child !== ".." && !child.startsWith(`..${sep}`); + } + if (outputRoot === dirname(outputRoot) || contains(outputRoot, workspaceRoot) + || (contains(workspaceRoot, outputRoot) && outputRoot !== join(sourcePackage, "provider-pack"))) { + throw new Error("Refusing unsafe provider-pack output path"); + } + if (revision === undefined) { + // CI supplies an authorized revision explicitly and may refresh only its + // root lock. Local implicit HEAD must not mislabel changed image inputs. + const dirty = execFileSync("git", ["status", "--porcelain", "--untracked-files=all", "--", + ".dockerignore", "docker/daytona-runner", "package.json", "pnpm-workspace.yaml", ".npmrc", "tsconfig.base.json", + "patches", "scripts/link-plugin-dev-sdk.mjs", "packages", "server/package.json", "ui/package.json", "cli/package.json", + ":(exclude)packages/paperclip-runner/provider-pack", ":(glob,exclude)packages/paperclip-runner/.paperclip-provider-pack-*/**"], + { cwd: workspaceRoot, encoding: "utf8" }); + if (dirty.trim()) throw new Error("Canonical provider inputs are dirty; an implicit HEAD revision would be misleading"); + revision = execFileSync("git", ["rev-parse", "HEAD"], { cwd: workspaceRoot, encoding: "utf8" }).trim(); + } + if (!/^[0-9a-f]{40}$/.test(revision)) throw new Error("PAPERCLIP_RUNNER_SOURCE_REVISION must be a full Git SHA"); + const dockerfile = join(workspaceRoot, "docker/daytona-runner/Dockerfile"); + const expectedLock = readFileSync(dockerfile, "utf8").match(/^ARG PAPERCLIP_RUNNER_LOCK_SHA256=([a-f0-9]{64})$/m)?.[1]; + if (!expectedLock) throw new Error("Canonical provider lock pin is missing"); + const actualLock = sha256File(join(workspaceRoot, "docker/daytona-runner/provider-dependencies.lock.yaml")); + if (actualLock !== `sha256:${expectedLock}`) throw new Error("Canonical provider lock integrity mismatch"); + mkdirSync(dirname(outputRoot), { recursive: true }); + // Same filesystem as OUTPUT, so successful publication is an atomic rename. + // Docker reads its inputs before this empty directory receives exported files. + const temporaryParent = mkdtempSync(join(dirname(outputRoot), ".paperclip-provider-pack-")); + const exported = join(temporaryParent, "exported"), backup = join(temporaryParent, "previous"); + let previousMoved = false, published = false; + try { + const build = run("docker", ["build", "--platform", "linux/amd64", "--target", "provider-pack-export", + "--file", dockerfile, "--build-arg", `PAPERCLIP_RUNNER_SOURCE_REVISION=${revision}`, + "--output", `type=local,dest=${exported}`, workspaceRoot], + { cwd: workspaceRoot, stdio: "inherit", timeout: 12 * 60 * 1000, killSignal: "SIGTERM" }); + if (build.error || build.status !== 0) throw new Error("Canonical provider-pack build failed"); + const manifest = verifyProviderPack(exported, { revision, lockSha256: expectedLock }); + if (existsSync(outputRoot)) { renameSync(outputRoot, backup); previousMoved = true; } + try { renameSync(exported, outputRoot); published = true; } + catch (error) { if (previousMoved) { renameSync(backup, outputRoot); previousMoved = false; } throw error; } + return { outputRoot, manifest }; + } finally { + // If even rollback fails, leave the prior pack in its recoverable backup. + if (!previousMoved || published) rmSync(temporaryParent, { recursive: true, force: true }); + } } -const temporaryParent = mkdtempSync(join(tmpdir(), "paperclip-provider-pack-")); -const temporaryRoot = join(temporaryParent, "pack"); - -function canonicalJson(value) { - if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; - if (value && typeof value === "object") { - return `{${Object.keys(value) - .sort() - .map((key) => `${JSON.stringify(key)}:${canonicalJson(value[key])}`) - .join(",")}}`; - } - return JSON.stringify(value); -} - -function sha256File(path) { - return `sha256:${createHash("sha256") - .update(readFileSync(path)) - .digest("hex")}`; -} - -function sha256Tree(root) { - const hash = createHash("sha256"); - const visit = (directory, prefix = "") => { - const entries = readdirSync(directory, { withFileTypes: true }).sort( - (left, right) => left.name.localeCompare(right.name), - ); - for (const entry of entries) { - const relativePath = prefix ? `${prefix}/${entry.name}` : entry.name; - const absolutePath = join(directory, entry.name); - if (entry.isDirectory()) { - hash.update(`directory\0${relativePath}\n`); - visit(absolutePath, relativePath); - } else if (entry.isFile()) { - hash.update(`file\0${relativePath}\0${sha256File(absolutePath)}\n`); - } else if (entry.isSymbolicLink()) { - hash.update( - `symlink\0${relativePath}\0${readlinkSync(absolutePath)}\n`, - ); - } else { - throw new Error( - `Provider pack tree contains unsupported entry ${relativePath}`, - ); - } - } - }; - visit(root); - return `sha256:${hash.digest("hex")}`; -} - -function writePortableNodeShim(name, entrypoint) { - const shimPath = join(temporaryRoot, "node_modules", ".bin", name); - writeFileSync(shimPath, portableProviderShim(entrypoint, { node: true })); - chmodSync(shimPath, 0o755); -} - -function writePortableExecutableShim(name, executable) { - const shimPath = join(temporaryRoot, "node_modules", ".bin", name); - writeFileSync(shimPath, portableProviderShim(executable)); - chmodSync(shimPath, 0o755); -} - -try { - const deployed = spawnSync( - "pnpm", - [ - "--filter", - "@paperclipai/paperclip-runner", - "deploy", - "--prod", - temporaryRoot, - ], - { cwd: workspaceRoot, encoding: "utf8", stdio: "inherit" }, - ); - if (deployed.status !== 0) { - throw new Error(`pnpm deploy failed with exit code ${deployed.status}`); - } - - normalizeProviderPackLayout(temporaryRoot); - - // Fail the image build if a bridge silently brings back an older/private - // provider CLI. A direct dependency alone does not deduplicate pnpm's graph. - const packRequire = createRequire(join(temporaryRoot, "package.json")); - const codexAcpRequire = createRequire(packRequire.resolve("@agentclientprotocol/codex-acp/package.json")); - if (realpathSync(codexAcpRequire.resolve("@openai/codex/package.json")) !== - realpathSync(packRequire.resolve("@openai/codex/package.json"))) { - throw new Error("Codex ACP must share the image's Codex installation"); - } - - // Reuse the already-qualified build interpreter instead of introducing a - // package-manager lifecycle hook or a second binary supply chain. The pack - // manifest binds the copied bytes, platform, architecture, and minimum - // version before any provider is launched. - const minimumNodeVersion = [24, 11, 0]; - const actualNodeVersion = process.versions.node.split(".").map(Number); - if ( - actualNodeVersion[0] < minimumNodeVersion[0] || - (actualNodeVersion[0] === minimumNodeVersion[0] && - (actualNodeVersion[1] < minimumNodeVersion[1] || - (actualNodeVersion[1] === minimumNodeVersion[1] && - actualNodeVersion[2] < minimumNodeVersion[2]))) - ) { - throw new Error("Provider pack build Node is older than 24.11.0"); - } - const stableNodeRoot = join(temporaryRoot, "node_modules", "node"); - if (existsSync(stableNodeRoot)) { - throw new Error( - "Provider pack deployment unexpectedly claimed the stable Node path", - ); - } - const stableNodeCommand = join(stableNodeRoot, "bin", "node"); - mkdirSync(dirname(stableNodeCommand), { recursive: true, mode: 0o755 }); - copyFileSync(process.execPath, stableNodeCommand); - chmodSync(stableNodeCommand, 0o755); - - // pnpm's generated .bin shims embed the temporary deployment directory in - // NODE_PATH. That makes an otherwise identical provider pack hash differ on - // every build and leaks a nonexistent host path after relocation. Replace - // every provider-facing shim with a pack-relative launcher that always uses - // the pinned Node executable owned by this pack. - // The image exposes these same installations to every adapter. Never add a - // separate global/runner-only CLI version; refresh these packages and their - // qualification digests together to the latest stable releases. - writePortableNodeShim("codex", "@openai/codex/bin/codex.js"); - const claudeAcpRequire = createRequire( - packRequire.resolve("@agentclientprotocol/claude-agent-acp/package.json"), - ); - // Use the ACP bridge's SDK dependency directly, avoiding a second peer- - // resolved SDK installation just to expose its CLI on the global PATH. - const sdkRequire = createRequire(claudeAcpRequire.resolve("@anthropic-ai/claude-agent-sdk")); - const claudeExecutable = sdkRequire.resolve( - `@anthropic-ai/claude-agent-sdk-${process.platform}-${process.arch}/claude`, - ); - writePortableExecutableShim( - "claude", - relative(realpathSync(join(temporaryRoot, "node_modules")), realpathSync(claudeExecutable)), - ); - writePortableExecutableShim("node", "node/bin/node"); - writePortableExecutableShim("opencode", "opencode-ai/bin/opencode.exe"); - writePortableNodeShim("acpx", "acpx/dist/cli.js"); - const piPackageRoot = realpathSync(join(temporaryRoot, "node_modules", "@earendil-works", "pi-coding-agent")); - await materializePiBinary(piPackageRoot); - writePortableExecutableShim("pi", "@earendil-works/pi-coding-agent/vendor/standalone/pi"); - writePortableNodeShim("pi-acp", "pi-acp/dist/index.js"); - writePortableNodeShim( - "claude-agent-acp", - "@agentclientprotocol/claude-agent-acp/dist/index.js", - ); - writePortableNodeShim( - "codex-acp", - "@agentclientprotocol/codex-acp/dist/index.js", - ); - - // pnpm deploy may retain a workspace self-link under its virtual store. It - // points outside the immutable pack and is not needed by the deployed - // package, so it must not be staged or captured. - rmSync( - join( - temporaryRoot, - "node_modules", - ".pnpm", - "node_modules", - "@paperclipai", - "paperclip-runner", - ), - { recursive: true, force: true }, - ); - - for (const shimName of readdirSync( - join(temporaryRoot, "node_modules", ".bin"), - )) { - const shimPath = join(temporaryRoot, "node_modules", ".bin", shimName); - const contents = readFileSync(shimPath, "utf8"); - if (contents.includes(temporaryParent)) { - throw new Error( - `Provider pack shim ${shimName} retains its temporary build path`, - ); - } - } - - // Exercise a clean import from the deployed dependency graph. This catches - // malformed patch hunks that can look correct in an existing pnpm store but - // land inside the wrong function when applied to a fresh acpx tarball. - const acpxImport = spawnSync( - join(temporaryRoot, "node_modules", "node", "bin", "node"), - ["--input-type=module", "-e", 'await import("acpx/runtime")'], - { - cwd: temporaryRoot, - encoding: "utf8", - timeout: 30_000, - }, - ); - if (acpxImport.status !== 0) { - throw new Error( - `Provider pack ACPX import failed: ${String(acpxImport.stderr ?? "") - .trim() - .slice(-2_000)}`, - ); - } - - const opencodeProxyPath = "dist/cli/opencode-app-server-proxy.cjs"; - const acpxSidecarPath = "dist/cli/acpx-runtime-sidecar.cjs"; - const opencodeCommand = "node_modules/.bin/opencode"; - const opencodeExecutable = "node_modules/opencode-ai/bin/opencode.exe"; - const nodeCommand = "node_modules/node/bin/node"; - const productionLock = "pnpm-lock.yaml"; - copyFileSync( - join(workspaceRoot, "pnpm-lock.yaml"), - join(temporaryRoot, productionLock), - ); - for (const relativePath of [ - nodeCommand, - productionLock, - opencodeProxyPath, - acpxSidecarPath, - opencodeCommand, - opencodeExecutable, - ]) { - if (!existsSync(join(temporaryRoot, relativePath))) { - throw new Error(`Provider pack is missing ${relativePath}`); - } - } - - const opencodeProxySha = sha256File(join(temporaryRoot, opencodeProxyPath)); - const acpxSidecarSha = sha256File(join(temporaryRoot, acpxSidecarPath)); - const distDigest = sha256Tree(join(temporaryRoot, "dist")); - const configuredRevision = - process.env.PAPERCLIP_RUNNER_SOURCE_REVISION?.trim(); - const revision = - configuredRevision ?? - execFileSync("git", ["rev-parse", "HEAD"], { - cwd: workspaceRoot, - encoding: "utf8", - }).trim(); - if (!/^[0-9a-f]{40}$/.test(revision)) { - throw new Error("PAPERCLIP_RUNNER_SOURCE_REVISION must be a full Git SHA"); - } - const dirty = configuredRevision - ? false - : spawnSync("git", ["diff", "--quiet", "--", "packages/paperclip-runner"], { - cwd: workspaceRoot, - }).status !== 0; - const payload = { - pins: { - nodeMinimum: minimumNodeVersion.join("."), - codex: "0.153.4", - opencode: "1.18.29", - acpx: "0.13.1", - claudeAcp: "0.70.0", - codexAcp: "1.6.2", - pi: "0.84.2", - piAcp: "0.0.33", - }, - target: { platform: process.platform, architecture: process.arch }, - runnerSourceRevision: `${revision}${dirty ? "-dirty" : ""}`, - distDigest, - bridgeDigest: `sha256:${createHash("sha256") - .update(opencodeProxySha) - .update("\n") - .update(acpxSidecarSha) - .update("\n") - .update(distDigest) - .digest("hex")}`, - acpxProfileDigests: { - pi: "sha256:24ff73fda6e3c76ddce2d359a79f5c4b8f292eb290e4d2ab85aac94676b2c2dc", - claude: - "sha256:9d73d1f0f121fb96cc8badb28c22d5bff02d8582eb2e40360a81c189e1b9422a", - codex: - "sha256:7a923b3829884d3cabcc9659d22cace3f86813e7bfffc90974b10140a45bc400", - }, - artifacts: { - nodeCommand: { - path: nodeCommand, - sha256: sha256File(join(temporaryRoot, nodeCommand)), - }, - productionLock: { - path: productionLock, - sha256: sha256File(join(temporaryRoot, productionLock)), - }, - opencodeCommand: { - path: opencodeCommand, - sha256: sha256File(join(temporaryRoot, opencodeCommand)), - }, - opencodeExecutable: { - path: opencodeExecutable, - sha256: sha256File(join(temporaryRoot, opencodeExecutable)), - }, - opencodeProxy: { - path: opencodeProxyPath, - sha256: opencodeProxySha, - }, - acpxSidecar: { path: acpxSidecarPath, sha256: acpxSidecarSha }, - }, - }; - const manifest = { - schema: "paperclip-runner/remote-provider-pack/v1", - digest: `sha256:${createHash("sha256") - .update(canonicalJson(payload)) - .digest("hex")}`, - payload, - }; - writeFileSync( - join(temporaryRoot, "provider-pack.json"), - `${JSON.stringify(manifest, null, 2)}\n`, - { mode: 0o600 }, - ); - - rmSync(outputRoot, { recursive: true, force: true }); - renameSync(temporaryRoot, outputRoot); - process.stdout.write(`${outputRoot}\n`); -} finally { - rmSync(temporaryParent, { recursive: true, force: true }); +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + const outputArgument = process.argv.slice(2).find((value) => value !== "--"); + const result = buildProviderPack({ outputRoot: resolve(process.cwd(), outputArgument ?? join(packageRoot, "provider-pack")) }); + process.stdout.write(`${result.outputRoot}\n`); } diff --git a/packages/paperclip-runner/scripts/build-provider-pack.test.mjs b/packages/paperclip-runner/scripts/build-provider-pack.test.mjs new file mode 100644 index 0000000000..5468bf9df5 --- /dev/null +++ b/packages/paperclip-runner/scripts/build-provider-pack.test.mjs @@ -0,0 +1,111 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { test } from "node:test"; +import { createHash } from "node:crypto"; +import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, readdirSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { buildProviderPack } from "./build-provider-pack.mjs"; +import { canonicalJson, sha256File, sha256Tree } from "./provider-pack-integrity.mjs"; +const revision = "a".repeat(40); +function fixture(runTest) { + const parent = mkdtempSync(join(tmpdir(), "canonical-provider-test-")); + const workspaceRoot = join(parent, "source"), outputRoot = join(parent, "output"); + const lock = "the dedicated immutable resolution\n"; + const hash = createHash("sha256").update(lock).digest("hex"); + mkdirSync(join(workspaceRoot, "docker/daytona-runner"), { recursive: true }); + writeFileSync(join(workspaceRoot, "docker/daytona-runner/provider-dependencies.lock.yaml"), lock); + writeFileSync(join(workspaceRoot, "docker/daytona-runner/Dockerfile"), `ARG PAPERCLIP_RUNNER_LOCK_SHA256=${hash}\n`); + writeFileSync(join(workspaceRoot, "pnpm-lock.yaml"), "different CI resolution"); + const calls = []; + function exported(args, tamper) { + const destination = args[args.indexOf("--output") + 1].slice("type=local,dest=".length); + const paths = { + nodeCommand: "node_modules/node/bin/node", productionLock: "pnpm-lock.yaml", + opencodeCommand: "node_modules/.bin/opencode", opencodeExecutable: "node_modules/opencode-ai/bin/opencode.exe", + opencodeProxy: "dist/cli/opencode-app-server-proxy.cjs", acpxSidecar: "dist/cli/acpx-runtime-sidecar.cjs", + }; + const artifacts = {}; + for (const [name, path] of Object.entries(paths)) { + const file = join(destination, path); + mkdirSync(dirname(file), { recursive: true }); + writeFileSync(file, name === "productionLock" ? lock : name); + artifacts[name] = { path, sha256: sha256File(file) }; + } + const distDigest = sha256Tree(join(destination, "dist")); + const payload = { target: { platform: "linux", architecture: "x64" }, + runnerSourceRevision: args.find((value) => value.startsWith("PAPERCLIP_RUNNER_SOURCE_REVISION=")).split("=")[1], + artifacts, distDigest, bridgeDigest: `sha256:${createHash("sha256").update(artifacts.opencodeProxy.sha256).update("\n") + .update(artifacts.acpxSidecar.sha256).update("\n").update(distDigest).digest("hex")}` }; + const manifest = { schema: "paperclip-runner/remote-provider-pack/v1", payload, + digest: `sha256:${createHash("sha256").update(canonicalJson(payload)).digest("hex")}` }; + writeFileSync(join(destination, "provider-pack.json"), JSON.stringify(manifest)); + tamper?.(destination); + } + const run = (command, args, options) => { calls.push({ command, args, options }); exported(args); return { status: 0 }; }; + try { runTest({ parent, workspaceRoot, outputRoot, hash, calls, run, exported }); } + finally { rmSync(parent, { recursive: true, force: true }); } +} +test("workflow entry isolates the canonical linux stage from root dependency graph and stale outputs", () => fixture((f) => { + mkdirSync(join(f.workspaceRoot, "node_modules")); + writeFileSync(join(f.workspaceRoot, "node_modules/stale"), "must not be assembled"); + const first = buildProviderPack({ ...f, revision }); + writeFileSync(join(f.workspaceRoot, "pnpm-lock.yaml"), "another CI lock"); + const second = buildProviderPack({ ...f, revision }); + assert.equal(first.manifest.digest, second.manifest.digest); + for (const call of f.calls) { + assert.equal(call.command, "docker"); + assert.equal(call.args[call.args.indexOf("--platform") + 1], "linux/amd64"); + assert.equal(call.args[call.args.indexOf("--target") + 1], "provider-pack-export"); + assert.ok(call.args.includes(`PAPERCLIP_RUNNER_SOURCE_REVISION=${revision}`)); + assert.equal(call.options.timeout, 720_000); + } + assert.equal(readFileSync(join(f.outputRoot, "pnpm-lock.yaml"), "utf8"), "the dedicated immutable resolution\n"); + assert.deepEqual(readdirSync(f.parent).sort(), ["output", "source"]); +})); +test("tampered dedicated lock fails before invoking Docker", () => fixture((f) => { + writeFileSync(join(f.workspaceRoot, "docker/daytona-runner/provider-dependencies.lock.yaml"), "changed"); + assert.throws(() => buildProviderPack({ ...f, revision }), /lock integrity mismatch/); + assert.equal(f.calls.length, 0); +})); +test("failed canonical build preserves prior pack and cleans only temporary output", () => fixture((f) => { + mkdirSync(f.outputRoot); writeFileSync(join(f.outputRoot, "old"), "retain"); + assert.throws(() => buildProviderPack({ ...f, revision, run: () => ({ status: 1 }) }), /build failed/); + assert.equal(readFileSync(join(f.outputRoot, "old"), "utf8"), "retain"); + assert.deepEqual(readdirSync(f.parent).sort(), ["output", "source"]); +})); +test("exported executable tampering is rejected without replacing the prior pack", () => fixture((f) => { + mkdirSync(f.outputRoot); writeFileSync(join(f.outputRoot, "old"), "retain"); + const run = (_command, args) => { f.exported(args, (root) => writeFileSync(join(root, "node_modules/node/bin/node"), "tampered")); return { status: 0 }; }; + assert.throws(() => buildProviderPack({ ...f, revision, run }), /artifact integrity mismatch/); + assert.equal(readFileSync(join(f.outputRoot, "old"), "utf8"), "retain"); +})); +test("arbitrary source-tree output and invalid revision are rejected before building", () => fixture((f) => { + for (const outputRoot of [f.workspaceRoot, join(f.workspaceRoot, "packages"), join(f.workspaceRoot, "..hidden"), dirname(f.workspaceRoot)]) { + assert.throws(() => buildProviderPack({ ...f, outputRoot, revision }), /unsafe/); + } + assert.throws(() => buildProviderPack({ ...f, revision: "not-a-sha" }), /full Git SHA/); + assert.equal(f.calls.length, 0); +})); + +test("implicit HEAD rejects dirty canonical inputs while explicit trusted revisions tolerate CI root-lock drift", () => fixture((f) => { + const previous = process.env.PAPERCLIP_RUNNER_SOURCE_REVISION; + delete process.env.PAPERCLIP_RUNNER_SOURCE_REVISION; + try { + execFileSync("git", ["init", f.workspaceRoot], { stdio: "ignore" }); + execFileSync("git", ["-C", f.workspaceRoot, "add", "."]); + execFileSync("git", ["-C", f.workspaceRoot, "-c", "user.name=Test", "-c", "user.email=test@example.com", "commit", "-m", "fixture"], { stdio: "ignore" }); + writeFileSync(join(f.workspaceRoot, "pnpm-lock.yaml"), "CI-owned changed root lock"); + buildProviderPack({ ...f }); + assert.equal(f.calls.length, 1); + mkdirSync(join(f.workspaceRoot, "packages/paperclip-runner/scripts"), { recursive: true }); + writeFileSync(join(f.workspaceRoot, "packages/paperclip-runner/scripts/new-source.mjs"), "changed source"); + assert.throws(() => buildProviderPack({ ...f }), /inputs are dirty/); + assert.equal(f.calls.length, 1); + buildProviderPack({ ...f, revision }); + assert.equal(f.calls.length, 2); + } finally { + if (previous === undefined) delete process.env.PAPERCLIP_RUNNER_SOURCE_REVISION; + else process.env.PAPERCLIP_RUNNER_SOURCE_REVISION = previous; + } +})); diff --git a/packages/paperclip-runner/scripts/provider-pack-integrity.mjs b/packages/paperclip-runner/scripts/provider-pack-integrity.mjs new file mode 100644 index 0000000000..99fc745263 --- /dev/null +++ b/packages/paperclip-runner/scripts/provider-pack-integrity.mjs @@ -0,0 +1,87 @@ +import { createHash } from "node:crypto"; +import { readdirSync, readFileSync, readlinkSync, realpathSync } from "node:fs"; +import { join, resolve, sep } from "node:path"; + +export function canonicalJson(value) { + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + if (value && typeof value === "object") { + return `{${Object.keys(value) + .sort() + .map((key) => `${JSON.stringify(key)}:${canonicalJson(value[key])}`) + .join(",")}}`; + } + return JSON.stringify(value); +} + +export function sha256File(path) { + return `sha256:${createHash("sha256") + .update(readFileSync(path)) + .digest("hex")}`; +} + +export function sha256Tree(root) { + const hash = createHash("sha256"); + const visit = (directory, prefix = "") => { + const entries = readdirSync(directory, { withFileTypes: true }).sort( + (left, right) => left.name.localeCompare(right.name), + ); + for (const entry of entries) { + const relativePath = prefix ? `${prefix}/${entry.name}` : entry.name; + const absolutePath = join(directory, entry.name); + if (entry.isDirectory()) { + hash.update(`directory\0${relativePath}\n`); + visit(absolutePath, relativePath); + } else if (entry.isFile()) { + hash.update(`file\0${relativePath}\0${sha256File(absolutePath)}\n`); + } else if (entry.isSymbolicLink()) { + hash.update( + `symlink\0${relativePath}\0${readlinkSync(absolutePath)}\n`, + ); + } else { + throw new Error( + `Provider pack tree contains unsupported entry ${relativePath}`, + ); + } + } + }; + visit(root); + return `sha256:${hash.digest("hex")}`; +} + + +/** Verify exported bytes before replacing any prior working provider pack. */ +export function verifyProviderPack(root, { revision, lockSha256 }) { + const manifest = JSON.parse(readFileSync(join(root, "provider-pack.json"), "utf8")); + const payload = manifest.payload; + if (manifest.schema !== "paperclip-runner/remote-provider-pack/v1" + || payload?.runnerSourceRevision !== revision + || payload.target?.platform !== "linux" || payload.target?.architecture !== "x64" + || manifest.digest !== `sha256:${createHash("sha256").update(canonicalJson(payload)).digest("hex")}`) { + throw new Error("Canonical provider-pack manifest mismatch"); + } + const expectedPaths = { + nodeCommand: "node_modules/node/bin/node", productionLock: "pnpm-lock.yaml", + opencodeCommand: "node_modules/.bin/opencode", opencodeExecutable: "node_modules/opencode-ai/bin/opencode.exe", + opencodeProxy: "dist/cli/opencode-app-server-proxy.cjs", acpxSidecar: "dist/cli/acpx-runtime-sidecar.cjs", + }; + const canonicalRoot = realpathSync(root); + for (const [name, expectedPath] of Object.entries(expectedPaths)) { + const artifact = payload.artifacts?.[name]; + if (artifact?.path !== expectedPath || !/^sha256:[a-f0-9]{64}$/.test(artifact.sha256 ?? "")) { + throw new Error("Canonical provider-pack artifact metadata mismatch"); + } + const file = realpathSync(resolve(root, expectedPath)); + if (!file.startsWith(`${canonicalRoot}${sep}`) || sha256File(file) !== artifact.sha256) { + throw new Error("Canonical provider-pack artifact integrity mismatch"); + } + } + if (payload.artifacts.productionLock.sha256 !== `sha256:${lockSha256}` || sha256Tree(join(root, "dist")) !== payload.distDigest) { + throw new Error("Canonical provider-pack lock or compiled output mismatch"); + } + const bridgeDigest = `sha256:${createHash("sha256") + .update(payload.artifacts.opencodeProxy.sha256).update("\n") + .update(payload.artifacts.acpxSidecar.sha256).update("\n") + .update(payload.distDigest).digest("hex")}`; + if (payload.bridgeDigest !== bridgeDigest) throw new Error("Canonical provider-pack bridge integrity mismatch"); + return manifest; +} diff --git a/packages/paperclip-runner/test/acpx-codex-package-contract.test.mjs b/packages/paperclip-runner/test/acpx-codex-package-contract.test.mjs index 9ea9df7dfb..4040956269 100644 --- a/packages/paperclip-runner/test/acpx-codex-package-contract.test.mjs +++ b/packages/paperclip-runner/test/acpx-codex-package-contract.test.mjs @@ -42,7 +42,7 @@ const runnerdAcpxBackend = await readFile( "utf8", ); const providerPackBuilder = await readFile( - new URL("../scripts/build-provider-pack.mjs", import.meta.url), + new URL("../scripts/assemble-provider-pack.mjs", import.meta.url), "utf8", ); const nativeSessionExecutor = await readFile( diff --git a/server/src/__tests__/sandbox-performance.test.ts b/server/src/__tests__/sandbox-performance.test.ts new file mode 100644 index 0000000000..0fa7d045a0 --- /dev/null +++ b/server/src/__tests__/sandbox-performance.test.ts @@ -0,0 +1,147 @@ +import { AsyncLocalStorage } from "node:async_hooks"; +import { ROOT_CONTEXT, trace, type Context } from "@opentelemetry/api"; +import { describe, expect, it, vi } from "vitest"; +import { getActiveStepContext } from "@paperclipai/adapter-utils/acpx-engine/startup-timing"; +import type { StartupTraceContextHandle } from "../instrumentation.js"; +import { + captureSandboxPerformanceContext, + hasSandboxPerformanceTrace, + measureSandboxOperation, + runWithSandboxPerformanceTrace, + setSandboxPerformanceRunAttributes, + type SandboxPerformanceRecord, +} from "../services/sandbox-performance.js"; + +function recordingContext() { + const active = new AsyncLocalStorage(); + const spans: Array<{ name: string; id: string; parentId?: string; attributes: Record; status?: unknown; ended: boolean; events: unknown[] }> = []; + let next = 1; + const tracing: StartupTraceContextHandle = { + tracer: { startSpan(name, options, parent) { + const id = (next++).toString(16).padStart(16, "0"); + const record = { name, id, parentId: trace.getSpanContext(parent as Context ?? active.getStore() ?? ROOT_CONTEXT)?.spanId, + attributes: { ...(options as { attributes?: Record })?.attributes }, + ended: false, status: undefined as unknown, events: [] as unknown[] }; + spans.push(record); + return { + ...trace.wrapSpanContext({ spanId: id, traceId: "1234567890abcdef1234567890abcdef", traceFlags: 1 }), + spanContext: () => ({ spanId: id, traceId: "1234567890abcdef1234567890abcdef", traceFlags: 1 }), + setAttribute(key: string, value: unknown) { record.attributes[key] = value; }, + setStatus(status: unknown) { record.status = status; }, + addEvent(name: string, attributes: unknown) { record.events.push({ name, attributes }); }, + end() { record.ended = true; }, + }; + } }, + contextWithSpan(span) { return trace.setSpan(active.getStore() ?? ROOT_CONTEXT, span as ReturnType); }, + withContext(context, work) { return active.run(context as Context, work); }, + }; + return { tracing, spans, active }; +} + +describe("sandbox performance trace", () => { + it("uses real contexts and keeps parallel branches separate across awaits", async () => { + const { tracing, spans, active } = recordingContext(); + const records: SandboxPerformanceRecord[] = []; + await runWithSandboxPerformanceTrace({ runId: "private-run-id", enabled: true, traceContext: tracing, + onBatch: async (batch) => { records.push(...batch.records); } }, async () => { + setSandboxPerformanceRunAttributes({ runtime: "legacy" }); + await Promise.all(["left", "right"].map((side) => measureSandboxOperation(`sandbox.${side}`, {}, async () => { + const parent = trace.getSpanContext(active.getStore()!)!.spanId; + await new Promise((resolve) => setImmediate(resolve)); + expect(trace.getSpanContext(active.getStore()!)?.spanId).toBe(parent); + expect(trace.getSpanContext(getActiveStepContext()?.parentContext as Context)?.spanId).toBe(parent); + await measureSandboxOperation(`sandbox.${side}.child`, {}, async () => undefined); + }))); + }); + const root = spans.find((span) => span.name === "sandbox.run")!; + for (const side of ["left", "right"]) { + const parent = spans.find((span) => span.name === `sandbox.${side}`)!; + expect(parent.parentId).toBe(root.id); + expect(spans.find((span) => span.name === `sandbox.${side}.child`)?.parentId).toBe(parent.id); + } + expect(root.attributes["paperclip.sandbox.runtime"]).toBe("legacy"); + expect(root.attributes["paperclip.sandbox.recordCount"]).toBe(5); + expect(root.attributes["paperclip.sandbox.dropped"]).toBe(0); + expect(records.every((record) => record.traceId === "1234567890abcdef1234567890abcdef")).toBe(true); + expect(new Set(records.map((record) => record.id)).size).toBe(5); + expect(spans.every((span) => span.ended)).toBe(true); + expect(hasSandboxPerformanceTrace()).toBe(false); + expect(active.getStore()).toBeUndefined(); + }); + + it("drops private attributes and error messages while preserving the original failure", async () => { + const { tracing, spans } = recordingContext(); + const records: SandboxPerformanceRecord[] = []; + const failure = new Error("private-secret-error"); + await expect(runWithSandboxPerformanceTrace({ runId: "private-run-id", enabled: true, traceContext: tracing, + onBatch: async (batch) => { records.push(...batch.records); } }, async () => { + await measureSandboxOperation("sandbox.read", { path: "/secret", credential: "private-token", scope: "task", bytes: 12, files: NaN, + operation: "https://private.example", outcome: "bad\nvalue" }, async () => { throw failure; }); + })).rejects.toBe(failure); + expect(records.map((record) => record.outcome)).toEqual(["failed", "failed"]); + expect(records[0]?.attributes).toEqual({ scope: "task", bytes: 12 }); + expect(spans.every((span) => (span.status as { code: number }).code === 2)).toBe(true); + expect(JSON.stringify({ spans, records })).not.toMatch(/private|secret|credential/); + }); + + it("bounds persistence, exposes lost records, and never writes batches during measured work", async () => { + const { tracing, spans } = recordingContext(); + let finished = false; + const batches: Array<{ records: SandboxPerformanceRecord[]; dropped: number }> = []; + await runWithSandboxPerformanceTrace({ runId: "run", enabled: true, traceContext: tracing, maxRecords: 300, + onBatch: async (batch) => { expect(finished).toBe(true); batches.push(batch); } }, async () => { + for (let i = 0; i < 350; i++) await measureSandboxOperation("sandbox.read", { fileIndex: i }, async () => undefined); + finished = true; + }); + expect(batches.map((batch) => batch.records.length)).toEqual([250, 50]); + expect(batches.every((batch) => batch.dropped === 51)).toBe(true); + const root = spans.find((span) => span.name === "sandbox.run")!; + expect(root.attributes["paperclip.sandbox.recordCount"]).toBe(300); + expect(root.attributes["paperclip.sandbox.dropped"]).toBe(51); + expect(spans).toHaveLength(351); + }); + + it("keeps remote timing relative instead of inventing a host timestamp", async () => { + const { tracing, spans } = recordingContext(); + const records: SandboxPerformanceRecord[] = []; + await runWithSandboxPerformanceTrace({ runId: "run", enabled: true, traceContext: tracing, + onBatch: async (batch) => { records.push(...batch.records); } }, async () => { + await measureSandboxOperation("sandbox.command", {}, async (span) => { + span.recordRemotePhase("sandbox.remote.hash", 7, 11, { bytes: 20 }); + span.recordRemotePhase("sandbox.invalid", -1, 5); + }); + }); + const remote = records.find((record) => record.clock === "remote_relative")!; + expect(remote).toMatchObject({ startedAtMs: 7, durationMs: 11, attributes: { clock: "remote_relative", bytes: 20 } }); + expect(remote.parentId).toBe(records.find((record) => record.name === "sandbox.command")!.id); + expect(spans.find((span) => span.name === "sandbox.command")!.events).toHaveLength(1); + expect(spans.some((span) => span.name === "sandbox.remote.hash")).toBe(false); + }); + + it("reparents lazy work to an open ancestor and ignores callbacks after trace closure", async () => { + const { tracing, spans } = recordingContext(); + let captured: ReturnType = (work) => work(); + await runWithSandboxPerformanceTrace({ runId: "run", enabled: true, traceContext: tracing }, async () => { + await measureSandboxOperation("sandbox.response", {}, async () => { captured = captureSandboxPerformanceContext(); }); + await captured(() => measureSandboxOperation("sandbox.body", {}, async () => undefined)); + }); + const root = spans.find((span) => span.name === "sandbox.run")!; + expect(spans.find((span) => span.name === "sandbox.body")?.parentId).toBe(root.id); + await captured(() => measureSandboxOperation("sandbox.too_late", {}, async () => { expect(hasSandboxPerformanceTrace()).toBe(false); })); + expect(spans).toHaveLength(3); + }); + + it("preserves disabled execution and fails open when tracing or persistence fails", async () => { + const tracer = vi.fn(() => { throw new Error("sink failure"); }); + const onBatch = vi.fn(async () => { throw new Error("database failure"); }); + const tracing: StartupTraceContextHandle = { tracer: { startSpan: tracer }, contextWithSpan: () => undefined }; + await expect(runWithSandboxPerformanceTrace({ runId: "run", enabled: false, traceContext: tracing, onBatch }, async () => { + expect(hasSandboxPerformanceTrace()).toBe(false); + return measureSandboxOperation("sandbox.noop", {}, async () => 42); + })).resolves.toBe(42); + expect(tracer).not.toHaveBeenCalled(); + expect(onBatch).not.toHaveBeenCalled(); + await expect(runWithSandboxPerformanceTrace({ runId: "run", enabled: true, traceContext: tracing, onBatch }, async () => 42)).resolves.toBe(42); + expect(onBatch).toHaveBeenCalledOnce(); + }); +}); diff --git a/server/src/__tests__/sandbox-work-folders.test.ts b/server/src/__tests__/sandbox-work-folders.test.ts index 52b4183eb4..7641e7c0d0 100644 --- a/server/src/__tests__/sandbox-work-folders.test.ts +++ b/server/src/__tests__/sandbox-work-folders.test.ts @@ -1,3 +1,4 @@ +import { runWithSandboxPerformanceTrace, type SandboxPerformanceRecord } from "../services/sandbox-performance.js"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { execFile } from "node:child_process"; import { promisify } from "node:util"; @@ -161,6 +162,63 @@ describe("shared sandbox work-folder lifecycle", () => { runner: { supportsSingleStreamStdinProgress: options.bulkStdin, execute: (input) => localTestWorkFolderRunner.execute({ ...input, env: { ...input.env, HOME: home } }) } } }); active.push(run); return run; } + it("records scoped startup and final checkpoint stages without private identities", async () => { + const task = randomUUID(); + await db.insert(issues).values({ id: task, companyId, projectId, title: "Instrumented task", assigneeAgentId: agentId }); + const home = path.join(root, "private-instrumented-home"); + const physicalId = randomUUID(); + const records: SandboxPerformanceRecord[] = []; + const intervals = vi.spyOn(globalThis, "setInterval"); + try { await runWithSandboxPerformanceTrace({ runId: randomUUID(), enabled: true, + onBatch: async (batch) => { records.push(...batch.records); } }, async () => { + const run = await prepare(home, randomUUID(), physicalId, null, { taskId: task }); + await fs.writeFile(path.join(home, "task", "private-file-name"), "private-file-content"); + const tick = intervals.mock.calls.find((call) => call[1] === 180_000)?.[0]; + expect(typeof tick).toBe("function"); + if (typeof tick !== "function") throw new Error("Missing periodic checkpoint callback"); + tick(); + await run.flush(); + await run.stop(); + }); } finally { intervals.mockRestore(); } + const prepared = records.find((record) => record.name === "work_folder.prepare")!; + expect(prepared.outcome).toBe("ok"); + expect(prepared.attributes).toMatchObject({ cold: true, warm: false, reused: false }); + for (const repo of records.filter((record) => record.name === "work_folder.repository.prepare")) { + expect(repo.attributes).toMatchObject({ exists: false, reused: false, cold: true, warm: false, cacheHit: false }); + } + const periodic = records.find((record) => record.name === "work_folder.checkpoint" && record.attributes.phase === "periodic")!; + expect(periodic.parentId).toBe(prepared.parentId); + expect(periodic.outcome).toBe("ok"); + expect(records.filter((record) => record.name === "work_folder.scope.incoming").map((record) => record.attributes.scope).sort()).toEqual(["agent", "project", "task", "user"]); + expect(records.filter((record) => record.name === "work_folder.repository.clone")).toHaveLength(2); + expect(records.filter((record) => record.name === "work_folder.repository.setup")).toHaveLength(2); + const finalized = records.find((record) => record.name === "work_folder.finalize")!; + const finalCheckpoint = records.find((record) => record.name === "work_folder.checkpoint" && record.attributes.phase === "final")!; + expect(finalCheckpoint.parentId).toBe(finalized.id); + expect(finalCheckpoint.outcome).toBe("ok"); + expect(records.some((record) => record.name === "work_folder.db.query" && record.attributes.requestCount === 1)).toBe(true); + expect(records.some((record) => record.name === "work_folder.progress.save")).toBe(true); + for (const secret of [task, companyId, agentId, home, "private-file-name", "private-file-content"]) expect(JSON.stringify(records)).not.toContain(secret); + // A new lease of the same physical sandbox is warm regardless of provider + // power state. A fresh physical sandbox restores saved repositories cold. + for (const samePhysicalSandbox of [true, false]) { + const observed: SandboxPerformanceRecord[] = []; + await runWithSandboxPerformanceTrace({ runId: randomUUID(), enabled: true, + onBatch: async (batch) => { observed.push(...batch.records); } }, async () => { + const resumed = await prepare(samePhysicalSandbox ? home : path.join(root, "instrumented-replacement"), + randomUUID(), samePhysicalSandbox ? physicalId : randomUUID(), null, { taskId: task }); + await resumed.stop(); + }); + expect(observed.find((record) => record.name === "work_folder.prepare")!.attributes) + .toMatchObject({ cold: !samePhysicalSandbox, warm: samePhysicalSandbox, reused: samePhysicalSandbox }); + const repositories = observed.filter((record) => record.name === "work_folder.repository.prepare"); + expect(repositories).toHaveLength(2); + for (const repo of repositories) expect(repo.attributes).toMatchObject({ exists: samePhysicalSandbox, + reused: samePhysicalSandbox, cold: !samePhysicalSandbox, warm: samePhysicalSandbox, cacheHit: !samePhysicalSandbox }); + expect(observed.filter((record) => record.name === "work_folder.repository.clone")).toHaveLength(0); + } + + }, 60_000); it("uses downloaded metadata when a shared file changes after the startup listing", async () => { const svc = workFolderService(db, storage); const folder = await svc.ensure({ companyId, scope: "project", ownerId: projectId }); diff --git a/server/src/__tests__/work-folder-performance.test.ts b/server/src/__tests__/work-folder-performance.test.ts new file mode 100644 index 0000000000..88c9572fb1 --- /dev/null +++ b/server/src/__tests__/work-folder-performance.test.ts @@ -0,0 +1,68 @@ +import { createHash } from "node:crypto"; +import { Readable } from "node:stream"; +import { describe, expect, it } from "vitest"; +import { prefetchWorkFiles } from "../services/work-folder-transfer.js"; +import { uploadWorkFolderObject } from "../services/work-folder-upload.js"; +import { measureSandboxOperation, measureSandboxStream, runWithSandboxPerformanceTrace, type SandboxPerformanceRecord } from "../services/sandbox-performance.js"; + +const entry = { path: "private-file-secret", kind: "file" as const, byteSize: 4, sha256: null, executable: false }; +async function traced(work: () => Promise) { + const records: SandboxPerformanceRecord[] = []; + await runWithSandboxPerformanceTrace({ runId: "private-run-secret", enabled: true, onBatch: async (batch) => { records.push(...batch.records); } }, work); + return records; +} +describe("work-folder performance instrumentation", () => { + it("separates ordered prefetch response waits from body consumption with safe attributes", async () => { + const records = await traced(async () => measureSandboxOperation("work_folder.scope.hydrate", { scope: "task" }, async () => { + for await (const transfer of prefetchWorkFiles([0, 1], async (fileIndex) => ({ + entry, body: measureSandboxStream("work_folder.object.body", { fileIndex }, Readable.from([Buffer.from("data")])), + }))) { + let body = ""; for await (const chunk of transfer.body!) body += String(chunk); + expect(body).toBe("data"); + } + })); + const parent = records.find((record) => record.name === "work_folder.scope.hydrate")!; + expect(records.filter((record) => record.name === "work_folder.prefetch.open")).toHaveLength(2); + expect(records.filter((record) => record.name === "work_folder.prefetch.wait")).toHaveLength(2); + const bodies = records.filter((record) => record.name === "work_folder.object.body"); + expect(bodies).toHaveLength(2); + expect(bodies.every((record) => record.attributes.bytes === 4 && record.parentId === parent.id)).toBe(true); + expect(JSON.stringify(records)).not.toContain("private-file-secret"); + expect(JSON.stringify(records)).not.toContain("private-run-secret"); + }); + it("records real PUT attempts and verified consumption without object keys or contents", async () => { + const bytes = Buffer.from("private-body-secret"); let attempts = 0; + const records = await traced(async () => { + await uploadWorkFolderObject({ async putObject(input) { + attempts++; for await (const _chunk of input.body) { /* actual consumption */ } + if (attempts === 1) throw Object.assign(new Error("transient"), { code: "ECONNRESET" }); + return { objectKey: input.objectKey, contentLength: bytes.length } as never; + } }, { objectKey: "private-object-secret", contentType: "text/plain", contentLength: bytes.length, + sha256: createHash("sha256").update(bytes).digest("hex"), createSource: () => Readable.from([bytes]) }); + }); + const puts = records.filter((record) => record.name === "work_folder.object.put"); + expect(puts.map((record) => [record.attributes.attempt, record.outcome])).toEqual([[1, "failed"], [2, "ok"]]); + expect(records.filter((record) => record.name === "work_folder.upload.consume").every((record) => record.attributes.bytes === bytes.length)).toBe(true); + expect(JSON.stringify(records)).not.toContain("private-object-secret"); + expect(JSON.stringify(records)).not.toContain("private-body-secret"); + }); + it("preserves queued response errors and closes unconsumed prefetched sources with tracing enabled", async () => { + const source = new Readable({ read() {} }); + await expect(traced(async () => { + for await (const transfer of prefetchWorkFiles([0, 1], async (index) => ({ entry, + body: measureSandboxStream("work_folder.object.body", { fileIndex: index }, index === 0 ? Readable.from([]) : source), + }))) { + if (transfer.entry === entry && !source.destroyed) { + source.destroy(new Error("queued response failed")); await new Promise((resolve) => setImmediate(resolve)); + } + for await (const _chunk of transfer.body!) { /* consume */ } + } + })).rejects.toThrow("queued response failed"); + expect(source.destroyed).toBe(true); + const abandoned = new Readable({ read() {} }); + await traced(async () => { + for await (const _transfer of prefetchWorkFiles([0], async () => ({ entry, body: measureSandboxStream("work_folder.object.body", {}, abandoned) }))) break; + }); + expect(abandoned.destroyed).toBe(true); + }); +}); diff --git a/server/src/__tests__/work-folder-repositories.test.ts b/server/src/__tests__/work-folder-repositories.test.ts index 2237c5b3ec..34d853397c 100644 --- a/server/src/__tests__/work-folder-repositories.test.ts +++ b/server/src/__tests__/work-folder-repositories.test.ts @@ -1,3 +1,11 @@ +import { runWithSandboxPerformanceTrace, type SandboxPerformanceRecord } from "../services/sandbox-performance.js"; +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { mkdtemp, mkdir, readFile, realpath, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { localTestWorkFolderRunner } from "./helpers/work-folder-runner.js"; +import { workFolderTransport } from "../services/work-folder-transport.js"; import { createHash, randomUUID } from "node:crypto"; import { Readable } from "node:stream"; import { setImmediate } from "node:timers/promises"; @@ -81,6 +89,57 @@ describe("bounded repository checkpoint transfers", () => { }; } + it("checkpoints Git ignore policy without uploading or removing local ignored dependencies", async () => { + const f = await fixture(); + const base = await realpath(await mkdtemp(path.join(os.tmpdir(), "repository-ignore-"))); + const root = path.join(base, "source"), restored = path.join(base, "restored"), staging = path.join(base, "staging"); + const git = promisify(execFile); + try { + await git("git", ["init", root]); + await writeFile(path.join(root, "tracked"), "tracked original"); + await git("git", ["-C", root, "add", "tracked"]); + await git("git", ["-C", root, "-c", "user.name=Test", "-c", "user.email=test@example.com", "commit", "-m", "initial"]); + await writeFile(path.join(root, ".gitignore"), "node_modules/\ntracked\n"); + await mkdir(path.join(root, "nested/cache"), { recursive: true }); + await mkdir(path.join(root, "node_modules")); + await writeFile(path.join(root, "nested/.gitignore"), "cache/\n"); + await writeFile(path.join(root, ".git/info/exclude"), "local-only\n"); + const ignored = ["node_modules/dependency", "nested/cache/dependency", "local-only"]; + for (const file of ignored) await writeFile(path.join(root, file), `ignored:${file}`); + await writeFile(path.join(root, "tracked"), "tracked changed"); + await writeFile(path.join(root, "nested/keep"), "untracked work"); + const actual = workFolderTransport({ ...localTestWorkFolderRunner, supportsSingleStreamStdinProgress: true }); + const service = workFolderRepositoryService(db, f.storage, actual); + const records: SandboxPerformanceRecord[] = []; + const trace = { runId: "repository-ignore", enabled: true, onBatch: async (batch: { records: SandboxPerformanceRecord[] }) => { records.push(...batch.records); } }; + await runWithSandboxPerformanceTrace(trace, () => service.checkpoint(f.binding, root)); + const current = await f.current(); + const manifest = JSON.parse(f.objects.get(current.checkpointKey!)!.toString()); + const durablePaths = manifest.files.map((entry: { path: string }) => entry.path); + expect(durablePaths).toContain("tracked"); + expect(durablePaths).toContain("nested/keep"); + expect(durablePaths).toContain(".git/info/exclude"); + for (const file of ignored) { + expect(durablePaths).not.toContain(file); + expect(await readFile(path.join(root, file), "utf8")).toBe(`ignored:${file}`); + expect([...f.objects.values()].some((body) => body.toString() === `ignored:${file}`)).toBe(false); + } + await mkdir(staging); + await runWithSandboxPerformanceTrace(trace, () => service.restore(current, restored, staging)); + for (const phase of ["lookup", "object_intent", "object_head", "object_upload", "manifest_upload", "publish_lock", "protect_objects", "publish_pointer", "manifest_download", "manifest_body", "manifest_decode", "object_download"]) { + expect(records.some((record) => record.name === `work_folder.repository.${phase}`)).toBe(true); + } + for (const secret of [root, restored, f.binding.id, companyId, "tracked changed"]) expect(JSON.stringify(records)).not.toContain(secret); + expect(await readFile(path.join(restored, "tracked"), "utf8")).toBe("tracked changed"); + expect(await readFile(path.join(restored, "nested/keep"), "utf8")).toBe("untracked work"); + for (const file of ignored) await expect(readFile(path.join(restored, file))).rejects.toMatchObject({ code: "ENOENT" }); + // Restoring into the existing workspace must preserve reusable ignored caches. + await service.restore(current, root, staging); + for (const file of ignored) expect(await readFile(path.join(root, file), "utf8")).toBe(`ignored:${file}`); + const listed = await git("git", ["-C", restored, "ls-files", "--cached", "--others", "--exclude-standard"]); + expect(listed.stdout).toContain("tracked"); + } finally { await rm(base, { recursive: true, force: true }); } + }, 60_000); it("bounds HEADs and streaming PUTs to four, deduplicates content, and preserves manifest order", async () => { const f = await fixture(); const entries = [f.file("z"), f.file("b"), f.file("same-z", "z"), diff --git a/server/src/__tests__/work-folder-transport.test.ts b/server/src/__tests__/work-folder-transport.test.ts index 48fb4fef6a..5960cdfef0 100644 --- a/server/src/__tests__/work-folder-transport.test.ts +++ b/server/src/__tests__/work-folder-transport.test.ts @@ -6,6 +6,7 @@ import { Readable } from "node:stream"; import { createHash } from "node:crypto"; import os from "node:os"; import path from "node:path"; +import { runWithSandboxPerformanceTrace, type SandboxPerformanceRecord } from "../services/sandbox-performance.js"; import { workFolderTransport } from "../services/work-folder-transport.js"; import { localTestWorkFolderRunner } from "./helpers/work-folder-runner.js"; @@ -18,6 +19,70 @@ describe("sandbox work folder transport with real Node and Git", () => { roots.push(dir); return dir; } afterEach(async () => { for (const dir of roots.splice(0)) await rm(dir, { recursive: true, force: true }); }); + it("records host and remote timing without changing bytes or exporting paths", async () => { + const dir = await root(), staging = await root(); + const payload = Buffer.from("private-file-content"); + const records: SandboxPerformanceRecord[] = []; + const fast = workFolderTransport({ ...localTestWorkFolderRunner, supportsSingleStreamStdinProgress: true }); + await runWithSandboxPerformanceTrace({ runId: "test-run", enabled: true, + onBatch: async (batch) => { records.push(...batch.records); } }, async () => { + await fast.write(dir, staging, { path: "sensitive-file-name", kind: "file", byteSize: payload.length, + sha256: createHash("sha256").update(payload).digest("hex"), executable: false }, Readable.from([payload])); + const files = await fast.scan(dir); + expect(await fast.readBatch!(dir, files.filter((entry) => entry.kind === "file"))).toEqual([payload]); + }); + const roundtrip = records.find((record) => record.name === "work_folder.transport.roundtrip" && record.attributes.operation === "batch"); + expect(roundtrip?.attributes.executionMs).toBeGreaterThanOrEqual(0); + expect(roundtrip?.attributes.transportOverheadMs).toBeGreaterThanOrEqual(0); + expect(records.some((record) => record.name === "work_folder.transport.incoming_body")).toBe(true); + expect(records.some((record) => record.name === "work_folder.transport.encode_bytes")).toBe(true); + const remote = records.filter((record) => record.clock === "remote_relative"); + expect(remote.length).toBeGreaterThan(0); + expect(remote.some((record) => record.name === "work_folder.remote.hash")).toBe(true); + expect(remote.some((record) => record.name === "work_folder.remote.scan")).toBe(true); + expect(remote.some((record) => record.name === "sandbox.invalid_operation")).toBe(false); + expect(remote.every((record) => records.some((parent) => parent.id === record.parentId && parent.name === "work_folder.transport.roundtrip"))).toBe(true); + for (const secret of [dir, staging, "sensitive-file-name", payload.toString()]) expect(JSON.stringify(records)).not.toContain(secret); + expect(await readFile(path.join(dir, "sensitive-file-name"))).toEqual(payload); + }); + it("keeps the helper's original raw result when performance is not requested", async () => { + const script = await readFile(new URL("../services/scripts/work-folder-io.mjs", import.meta.url), "utf8"); + const result = await exec(process.execPath, ["--input-type=module", "-e", script, + Buffer.from(JSON.stringify({ operation: "home" })).toString("base64")]); + expect(JSON.parse(result.stdout)).toEqual({ home: os.homedir() }); + expect(result.stdout).not.toContain("workFolderPerformanceVersion"); + }); + it("does not discard a valid result or retry a command because diagnostics are malformed", async () => { + const execute = vi.fn().mockResolvedValue({ exitCode: 0, timedOut: false, stderr: "", stdout: JSON.stringify({ + workFolderPerformanceVersion: 1, result: { home: "/home/daytona" }, performance: { executionMs: "invalid" }, + }) }); + const records: SandboxPerformanceRecord[] = []; + await runWithSandboxPerformanceTrace({ runId: "run", enabled: true, + onBatch: async (batch) => { records.push(...batch.records); } }, async () => { + expect(await workFolderTransport({ execute }).home()).toBe("/home/daytona"); + }); + expect(execute).toHaveBeenCalledOnce(); + expect(records.find((record) => record.name === "work_folder.transport.roundtrip")?.attributes.dropped).toBe(1); + execute.mockClear().mockResolvedValue({ exitCode: 1, timedOut: false, stderr: "socket hang up", stdout: "" }); + await expect(workFolderTransport({ execute }).home()).rejects.toThrow("socket hang up"); + expect(execute).toHaveBeenCalledOnce(); + }); + it("bounds remote phase records while retaining complete aggregate hash counts", async () => { + const dir = await root(); + await Promise.all(Array.from({ length: 300 }, (_, index) => writeFile(path.join(dir, `file-${index}`), "x"))); + const script = await readFile(new URL("../services/scripts/work-folder-io.mjs", import.meta.url), "utf8"); + const response = await exec(process.execPath, ["--input-type=module", "-e", script, + Buffer.from(JSON.stringify({ operation: "scan", root: dir, performance: true })).toString("base64")]); + const decoded = JSON.parse(response.stdout); + expect(decoded.result).toHaveLength(300); + expect(decoded.remotePhases).toHaveLength(256); + expect(decoded.performance.hashFiles).toBe(300); + expect(decoded.performance.hashBytes).toBe(300); + expect(decoded.performance.droppedPhases).toBeGreaterThan(0); + expect(decoded.performance.executionMs).toBeGreaterThanOrEqual(decoded.performance.scanMs); + expect(JSON.stringify(decoded.remotePhases)).not.toContain(dir); + expect(JSON.stringify(decoded.remotePhases)).not.toContain("file-"); + }); it("retries a lost read response without duplicating streamed bytes", async () => { const dir = await root(); const body = Buffer.alloc(700_000, "x"); @@ -188,12 +253,28 @@ describe("sandbox work folder transport with real Node and Git", () => { await exec("git", ["-C", dir, "add", "."]); await exec("git", ["-C", dir, "-c", "user.name=Test", "-c", "user.email=test@example.com", "commit", "-m", "initial"]); await writeFile(path.join(dir, "tracked"), "unstaged"); + // A tracked path remains durable even when a later ignore rule matches it. + await writeFile(path.join(dir, ".gitignore"), "node_modules/\ntracked\n"); + await mkdir(path.join(dir, "nested")); + await writeFile(path.join(dir, "nested/.gitignore"), "cache/\n*.generated\n"); + await mkdir(path.join(dir, "nested/cache")); + await writeFile(path.join(dir, "nested/cache/dependency"), "local cache"); + await writeFile(path.join(dir, "nested/result.generated"), "generated"); + await writeFile(path.join(dir, "nested/keep"), "new durable work"); + await writeFile(path.join(dir, ".git/info/exclude"), "local-only\n"); + await writeFile(path.join(dir, "local-only"), "local secret cache"); await writeFile(path.join(dir, "untracked"), "new"); await mkdir(path.join(dir, "node_modules")); await writeFile(path.join(dir, "node_modules/cache"), "ignored"); const paths = (await transport.scan(dir, true)).map((entry) => entry.path); expect(paths).toContain("tracked"); expect(paths).toContain("untracked"); + expect(paths).toContain("nested/keep"); + expect(paths).toContain("nested/.gitignore"); + expect(paths).toContain(".git/info/exclude"); + expect(paths).not.toContain("local-only"); + expect(paths).not.toContain("nested/result.generated"); + expect(paths.some((entry) => entry.startsWith("nested/cache/"))).toBe(false); expect(paths).toContain(".git/index"); expect(paths).toContain(".git/HEAD"); expect(paths).not.toContain(".git/config"); diff --git a/server/src/__tests__/work-folders.test.ts b/server/src/__tests__/work-folders.test.ts index 8f94ce2c69..e7549fb875 100644 --- a/server/src/__tests__/work-folders.test.ts +++ b/server/src/__tests__/work-folders.test.ts @@ -1,3 +1,4 @@ +import { runWithSandboxPerformanceTrace, type SandboxPerformanceRecord } from "../services/sandbox-performance.js"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { randomUUID } from "node:crypto"; import { mkdtemp, rm } from "node:fs/promises"; @@ -39,6 +40,29 @@ describe("durable work folders", () => { for await (const chunk of stream) buffers.push(Buffer.from(chunk)); return Buffer.concat(buffers).toString(); } + it("measures metadata, response wait, body bytes and progress without private paths", async () => { + const records: SandboxPerformanceRecord[] = []; + await runWithSandboxPerformanceTrace({ runId: randomUUID(), enabled: true, + onBatch: async (batch) => { records.push(...batch.records); } }, async () => { + const f = await folder(); + await svc.write(f, { path: "private-observed-file", body: Buffer.from("private-observed-content"), operationId: "private-operation-id" }); + const opened = await svc.content(f, "private-observed-file", 7); + let text = ""; for await (const chunk of opened.stream) text += String(chunk); + expect(text).toBe("private-observed-content"); + await svc.list(f); + }); + const names = records.map((record) => record.name); + for (const name of ["work_folder.scope.ensure", "work_folder.metadata.get", "work_folder.object.get_response", "work_folder.object.body", "work_folder.spool.consume", "work_folder.metadata.mutate", "work_folder.db.query"]) expect(names).toContain(name); + const response = records.find((record) => record.name === "work_folder.object.get_response")!; + const body = records.find((record) => record.name === "work_folder.object.body")!; + expect(response.attributes.requestCount).toBe(1); + expect(body.attributes.bytes).toBe(Buffer.byteLength("private-observed-content")); + expect(body.attributes.fileIndex).toBe(7); + expect(response.attributes.fileIndex).toBe(7); + expect(body.startedAtMs).toBeGreaterThanOrEqual(response.startedAtMs); + expect(records.filter((record) => record.name === "work_folder.db.query").every((record) => typeof record.attributes.operation === "string")).toBe(true); + for (const secret of [companyId, root, "private-observed-file", "private-observed-content", "private-operation-id"]) expect(JSON.stringify(records)).not.toContain(secret); + }); it("streams nested executable and empty files into durable storage", async () => { const f = await folder(); await svc.write(f, { path: "bin/run", body: Readable.from(["#!/bin/sh\n", "true\n"]), executable: true, operationId: "first" }); diff --git a/server/src/instrumentation.ts b/server/src/instrumentation.ts index 34275e1dbd..d1cb3910e3 100644 --- a/server/src/instrumentation.ts +++ b/server/src/instrumentation.ts @@ -126,6 +126,7 @@ export function getStartupTracer(name = "paperclip.startup"): StartupTracerHandl export interface StartupTraceContextHandle { readonly tracer: StartupTracerHandle; contextWithSpan(span: unknown): unknown; + withContext?(context: unknown, work: () => T): T; } /** @@ -158,7 +159,7 @@ export function getStartupTraceContext(name = "paperclip.startup"): StartupTrace getTracer(n: string): StartupTracerHandle; setSpan(context: unknown, span: unknown): unknown; }; - context?: { active(): unknown }; + context?: { active(): unknown; with?(context: unknown, work: () => T): T }; }; const trace = api.trace; const context = api.context; @@ -171,6 +172,7 @@ export function getStartupTraceContext(name = "paperclip.startup"): StartupTrace // Keep the method calls on `trace` / `context` so the api singletons stay // their own receiver. contextWithSpan: (span: unknown) => trace.setSpan(context.active(), span), + withContext: (token: unknown, work: () => T): T => context.with ? context.with(token, work) : work(), }; } catch (err) { if (!traceContextApiLoadFailed) { diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index da5af5b259..5f003028c4 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -1,3 +1,4 @@ +import { measureSandboxOperation, runWithSandboxPerformanceTrace, setSandboxPerformanceRunAttributes } from "./sandbox-performance.js"; import { initializeRunIdentity } from "./run-identity.js"; import { startNativeGitHubCallbackBridge } from "./native-github-bridge.js"; import { githubBrokerEnvironment } from "@paperclipai/adapter-utils/github-launcher"; @@ -17963,16 +17964,24 @@ export function heartbeatService( return promise; } - async function executeRun( + // The diagnostic path is opt-in and persists bounded batches after execution. + async function executeRun(runId: string, runOptions: Parameters[1] = {}) { + return runWithSandboxPerformanceTrace({ runId, onBatch: async (batch) => { + const observed = await getRun(runId); + if (observed) await appendRunEvent(observed, { eventType: "sandbox.performance.batch", stream: "system", level: "info", payload: batch }); + } }, () => executeRunMeasured(runId, runOptions)); + } + + async function executeRunMeasured( runId: string, runOptions: { nativeLeaseOwner?: string; nativeRestartRecovery?: NativeRestartRecoveryClaim; } = {}, ) { - if ((await getSchedulingSuppression()).suppressed) { + if ((await measureSandboxOperation("heartbeat.get_scheduling_suppression", { operationIndex: 0 }, async () => (getSchedulingSuppression()))).suppressed) { try { - await releaseRunClaimedJustBeforeSuppression(runId); + await measureSandboxOperation("heartbeat.release_run_claimed_just_before_suppression", { operationIndex: 1 }, async () => (releaseRunClaimedJustBeforeSuppression(runId))); } catch (err) { logger.error( { err, runId }, @@ -17982,12 +17991,13 @@ export function heartbeatService( return; } - let run = await getRun(runId); + let run = await measureSandboxOperation("heartbeat.get_run", { operationIndex: 2 }, async () => (getRun(runId))); if (!run) return; + setSandboxPerformanceRunAttributes({ runtime: run.runtimeMode === "native" ? "native" : "legacy" }); if (run.status !== "queued" && run.status !== "running") return; if (run.status === "queued") { - const claimed = await claimQueuedRun(run); + const claimed = await measureSandboxOperation("heartbeat.claim_queued_run", { operationIndex: 3 }, async () => (claimQueuedRun(run))); if (!claimed) { // claimQueuedRun can also leave the run queued when dependencies are unresolved. return; @@ -18027,17 +18037,17 @@ export function heartbeatService( persistedPidAlive || persistedProcessGroupAlive ) { - await markNativeOwnershipUnverified(run, { + await measureSandboxOperation("heartbeat.mark_native_ownership_unverified", { operationIndex: 4 }, async () => (markNativeOwnershipUnverified(run, { reason: "live_process_identifier", processPidAlive: trackedPidAlive || persistedPidAlive, processGroupAlive: trackedProcessGroupAlive || persistedProcessGroupAlive, - }); + }))); throw new Error(NATIVE_OWNERSHIP_UNVERIFIED_ERROR_CODE); } runningProcesses.delete(run.id); if (run.processPid || run.processGroupId || run.processStartedAt) { - const cleared = await db + const cleared = await measureSandboxOperation("heartbeat.db.update.set.where.returning.then", { operationIndex: 5 }, async () => (db .update(heartbeatRuns) .set({ processPid: null, @@ -18061,13 +18071,13 @@ export function heartbeatService( ), ) .returning() - .then((rows) => rows[0] ?? null); + .then((rows) => rows[0] ?? null))); if (!cleared) { - const current = await getRun(run.id); + const current = await measureSandboxOperation("heartbeat.get_run", { operationIndex: 6 }, async () => (getRun(run.id))); if (current) { - await markNativeOwnershipUnverified(current, { + await measureSandboxOperation("heartbeat.mark_native_ownership_unverified", { operationIndex: 7 }, async () => (markNativeOwnershipUnverified(current, { reason: "live_process_identifier", - }); + }))); } throw new Error(NATIVE_OWNERSHIP_UNVERIFIED_ERROR_CODE); } @@ -18105,30 +18115,30 @@ export function heartbeatService( let providerTraceFinalized = false; try { - const agent = await getAgent(run.agentId); + const agent = await measureSandboxOperation("heartbeat.get_agent", { operationIndex: 8 }, async () => (getAgent(run.agentId))); if (!agent) { - await setRunStatus(runId, "failed", { + await measureSandboxOperation("heartbeat.set_run_status", { operationIndex: 9 }, async () => (setRunStatus(runId, "failed", { error: "Agent not found", errorCode: "agent_not_found", finishedAt: new Date(), - }); - await setWakeupStatus(run.wakeupRequestId, "failed", { + }))); + await measureSandboxOperation("heartbeat.set_wakeup_status", { operationIndex: 10 }, async () => (setWakeupStatus(run.wakeupRequestId, "failed", { finishedAt: new Date(), error: "Agent not found", - }); - const failedRun = await getRun(runId); - if (failedRun) await releaseIssueExecutionAndPromote(failedRun); + }))); + const failedRun = await measureSandboxOperation("heartbeat.get_run", { operationIndex: 11 }, async () => (getRun(runId))); + if (failedRun) await measureSandboxOperation("heartbeat.release_issue_execution_and_promote", { operationIndex: 12 }, async () => (releaseIssueExecutionAndPromote(failedRun))); return; } - const runtime = await ensureRuntimeState(agent); + const runtime = await measureSandboxOperation("heartbeat.ensure_runtime_state", { operationIndex: 13 }, async () => (ensureRuntimeState(agent))); const context = parseObject(run.contextSnapshot); const providerTraceRequested = parseObject(context.debug).providerTrace === "raw"; if (providerTraceRequested) { if (context.providerTraceRequestSource === "agent_debug_setting") { try { - await logActivity(db, { + await measureSandboxOperation("heartbeat.log_activity", { operationIndex: 14 }, async () => (logActivity(db, { companyId: run.companyId, actorType: "system", actorId: "system", @@ -18143,7 +18153,7 @@ export function heartbeatService( retentionHours: 24, maxBytes: 64 * 1024 * 1024, }, - }); + }))); } catch (error) { logger.warn( { error, runId: run.id }, @@ -18152,7 +18162,7 @@ export function heartbeatService( } } try { - providerTraceCapture = await traceStore.prepare({ + providerTraceCapture = await measureSandboxOperation("heartbeat.trace_store.prepare", { operationIndex: 15 }, async () => (traceStore.prepare({ runId: run.id, companyId: run.companyId, provider: @@ -18161,7 +18171,7 @@ export function heartbeatService( requestedBy: readNonEmptyString(context.providerTraceRequestedBy) ?? "local-admin", - }); + }))); } catch (error) { logger.warn( { error, runId: run.id }, @@ -18173,12 +18183,12 @@ export function heartbeatService( const sessionCodec = getAdapterSessionCodec(agent.adapterType); const issueId = readNonEmptyString(context.issueId); let issueContext = issueId - ? await getIssueExecutionContext(agent.companyId, issueId) + ? await measureSandboxOperation("heartbeat.get_issue_execution_context", { operationIndex: 16 }, async () => (getIssueExecutionContext(agent.companyId, issueId))) : null; const issueDependencyReadiness = issueId - ? await issuesSvc + ? await measureSandboxOperation("heartbeat.issues_svc.list_dependency_readiness.then", { operationIndex: 17 }, async () => (issuesSvc .listDependencyReadiness(agent.companyId, [issueId]) - .then((rows) => rows.get(issueId) ?? null) + .then((rows) => rows.get(issueId) ?? null))) : null; if ( issueId && @@ -18190,22 +18200,22 @@ export function heartbeatService( // queued-run staleness gate. This is the final atomic guard before // dispatch: an operator parking the issue after claim but before this // checkout must not be overwritten by the continuation. - await issuesSvc.checkout(issueId, agent.id, ["in_progress"], run.id); + await measureSandboxOperation("heartbeat.issues_svc.checkout", { operationIndex: 18 }, async () => (issuesSvc.checkout(issueId, agent.id, ["in_progress"], run.id))); context[PAPERCLIP_HARNESS_CHECKOUT_KEY] = true; } catch (error) { if (!isCheckoutConflictError(error)) throw error; - const staleness = await evaluateQueuedRunStaleness( + const staleness = await measureSandboxOperation("heartbeat.evaluate_queued_run_staleness", { operationIndex: 19 }, async () => (evaluateQueuedRunStaleness( run, issueId, context, - ); + ))); if (staleness.stale) { - await cancelRunForStaleIssue(run, issueId, staleness); + await measureSandboxOperation("heartbeat.cancel_run_for_stale_issue", { operationIndex: 20 }, async () => (cancelRunForStaleIssue(run, issueId, staleness))); return; } throw error; } - issueContext = await getIssueExecutionContext(agent.companyId, issueId); + issueContext = await measureSandboxOperation("heartbeat.get_issue_execution_context", { operationIndex: 21 }, async () => (getIssueExecutionContext(agent.companyId, issueId))); } if ( issueId && @@ -18222,23 +18232,24 @@ export function heartbeatService( }) ) { try { - await issuesSvc.checkout( + await measureSandboxOperation("heartbeat.issues_svc.checkout", { operationIndex: 22 }, async () => (issuesSvc.checkout( issueId, agent.id, ["todo", "backlog", "blocked"], run.id, - ); + ))); context[PAPERCLIP_HARNESS_CHECKOUT_KEY] = true; } catch (error) { if (!isCheckoutConflictError(error)) throw error; context[PAPERCLIP_HARNESS_CHECKOUT_KEY] = false; } - issueContext = await getIssueExecutionContext(agent.companyId, issueId); + issueContext = await measureSandboxOperation("heartbeat.get_issue_execution_context", { operationIndex: 23 }, async () => (getIssueExecutionContext(agent.companyId, issueId))); } const wakeCommentId = deriveCommentId(context, null); + const wakeIssueContext = issueContext; const wakeCommentContext = - issueContext && wakeCommentId - ? await db + wakeIssueContext && wakeCommentId + ? await measureSandboxOperation("heartbeat.db.select.from.where.then", { operationIndex: 24 }, async () => (db .select({ id: issueComments.id, body: issueComments.body, @@ -18258,7 +18269,7 @@ export function heartbeatService( .where( and( eq(issueComments.id, wakeCommentId), - eq(issueComments.issueId, issueContext.id), + eq(issueComments.issueId, wakeIssueContext.id), eq(issueComments.companyId, agent.companyId), ), ) @@ -18272,7 +18283,7 @@ export function heartbeatService( metadata: null, } : row; - }) + }))) : null; const issueAssigneeOverrides = issueContext && issueContext.assigneeAgentId === agent.id @@ -18281,7 +18292,7 @@ export function heartbeatService( ) : null; const experimentalInstanceSettings = - await instanceSettings.getExperimental(); + await measureSandboxOperation("heartbeat.instance_settings.get_experimental", { operationIndex: 25 }, async () => (instanceSettings.getExperimental())); const isolatedWorkspacesEnabled = experimentalInstanceSettings.enableIsolatedWorkspaces; const parsedIssueExecutionWorkspaceSettings = @@ -18299,7 +18310,7 @@ export function heartbeatService( const contextProjectId = readNonEmptyString(context.projectId); const executionProjectId = issueContext?.projectId ?? contextProjectId; const projectContext = executionProjectId - ? await db + ? await measureSandboxOperation("heartbeat.db.select.from.where.then", { operationIndex: 26 }, async () => (db .select({ id: projects.id, executionWorkspacePolicy: projects.executionWorkspacePolicy, @@ -18313,7 +18324,7 @@ export function heartbeatService( eq(projects.companyId, agent.companyId), ), ) - .then((rows) => rows[0] ?? null) + .then((rows) => rows[0] ?? null))) : null; const acceptedPlanContinuationWake = issueContext ? readNonEmptyString(context.workspaceRefreshReason) === @@ -18324,14 +18335,14 @@ export function heartbeatService( readNonEmptyString(context.interactionStatus) === "accepted") : false; const acceptedPlanWakeRoutingDecision = issueContext - ? await resolveAcceptedPlanWakeRoutingDecision({ + ? await measureSandboxOperation("heartbeat.resolve_accepted_plan_wake_routing_decision", { operationIndex: 27 }, async () => (resolveAcceptedPlanWakeRoutingDecision({ db, companyId: agent.companyId, agentId: agent.id, issueId, acceptedPlanContinuationWake, contextSnapshot: context, - }) + }))) : null; if (acceptedPlanWakeRoutingDecision) { context.forceFreshSession = true; @@ -18351,17 +18362,17 @@ export function heartbeatService( } else { delete context.acceptedPlanWakeRouting; } - const routineEnvContext = await getRoutineEnvForExecutionIssue( + const routineEnvContext = await measureSandboxOperation("heartbeat.get_routine_env_for_execution_issue", { operationIndex: 28 }, async () => (getRoutineEnvForExecutionIssue( agent.companyId, issueContext, - ); - let responsibleUserId: string | null = await resolveResponsibleUserIdForRun({ + ))); + let responsibleUserId: string | null = await measureSandboxOperation("heartbeat.resolve_responsible_user_id_for_run", { operationIndex: 29 }, async () => (resolveResponsibleUserIdForRun({ run, contextSnapshot: context, issueContext, routineEnvContext, - }); - const identityContext = await initializeRunIdentity(db, { + }))); + const identityContext = await measureSandboxOperation("heartbeat.initialize_run_identity", { operationIndex: 30 }, async () => (initializeRunIdentity(db, { companyId: agent.companyId, runId: run.id, responsibleUserId, interactionId: readNonEmptyString(context.interactionId), issueId, messageIds: run.retryOfRunId || context.retryOfRunId ? [] : queuedCommentIdsFromRunContext(context).length @@ -18373,7 +18384,7 @@ export function heartbeatService( ?? (run.triggerDetail === "manual" || context.parentRunId ? null : issueContext?.continuationIdentityContextId ?? issueContext?.originIdentityContextId), parentRunId: run.retryOfRunId ?? readNonEmptyString(context.retryOfRunId) ?? readNonEmptyString(context.parentRunId), cause: readNonEmptyString(context.executionIdentityCause) ?? readNonEmptyString(context.wakeReason) ?? "dispatch", - }); + }))); // Initialization has persisted the active context, including an explicit // absence of identity inherited from an automatic continuation. responsibleUserId = identityContext.responsibleUserId; @@ -18384,16 +18395,17 @@ export function heartbeatService( issueContext && !issueContext.responsibleUserId ) { - await db + const responsibleIssueId = issueContext.id; + await measureSandboxOperation("heartbeat.db.update.set.where", { operationIndex: 31 }, async () => (db .update(issues) .set({ responsibleUserId, updatedAt: new Date() }) .where( and( eq(issues.companyId, agent.companyId), - eq(issues.id, issueContext.id), + eq(issues.id, responsibleIssueId), isNull(issues.responsibleUserId), ), - ); + ))); issueContext = { ...issueContext, responsibleUserId }; } const parsedProjectExecutionWorkspacePolicy = @@ -18426,12 +18438,12 @@ export function heartbeatService( }); const config = parseObject(agent.adapterConfig); const taskSession = taskKey - ? await getTaskSession( + ? await measureSandboxOperation("heartbeat.get_task_session", { operationIndex: 32 }, async () => (getTaskSession( agent.companyId, agent.id, agent.adapterType, taskKey, - ) + ))) : null; const taskSessionDecodedParams = normalizeSessionParams( sessionCodec.deserialize(taskSession?.sessionParamsJson ?? null), @@ -18476,7 +18488,7 @@ export function heartbeatService( } : null; const continuationSummary = issueRef - ? await getIssueContinuationSummaryDocument(db, issueRef.id) + ? await measureSandboxOperation("heartbeat.get_issue_continuation_summary_document", { operationIndex: 33 }, async () => (getIssueContinuationSummaryDocument(db, issueRef.id))) : null; const exposeLowTrustRaw = trustPreset.kind === "low_trust_review"; const safeContinuationSummary = @@ -18488,7 +18500,7 @@ export function heartbeatService( ? sanitizeQuarantinedCommentForHigherTrust(wakeCommentContext) : wakeCommentContext; const issueAncestors = issueRef - ? await issuesSvc.getAncestors(issueRef.id) + ? await measureSandboxOperation("heartbeat.issues_svc.get_ancestors", { operationIndex: 34 }, async () => (issuesSvc.getAncestors(issueRef.id))) : []; if (continuationSummary) { context.paperclipContinuationSummary = { @@ -18503,7 +18515,7 @@ export function heartbeatService( } const pinnedSkillTestContext = issueRef?.workMode === "skill_test" - ? await getPinnedSkillTestContext(agent.companyId, issueRef.id) + ? await measureSandboxOperation("heartbeat.get_pinned_skill_test_context", { operationIndex: 35 }, async () => (getPinnedSkillTestContext(agent.companyId, issueRef.id))) : null; if (pinnedSkillTestContext) { context.paperclipSkillTest = { @@ -18514,7 +18526,7 @@ export function heartbeatService( } else { delete context.paperclipSkillTest; } - const paperclipWakePayload = await buildPaperclipWakePayload({ + const paperclipWakePayload = await measureSandboxOperation("heartbeat.build_paperclip_wake_payload", { operationIndex: 36 }, async () => (buildPaperclipWakePayload({ db, companyId: agent.companyId, contextSnapshot: context, @@ -18536,7 +18548,7 @@ export function heartbeatService( simplifiedEnglishInteractions: experimentalInstanceSettings.enableSimplifiedEnglishInteractions === true, - }); + }))); if (paperclipWakePayload) { context[PAPERCLIP_WAKE_PAYLOAD_KEY] = paperclipWakePayload; } else { @@ -18611,14 +18623,14 @@ export function heartbeatService( delete context.paperclipTaskMarkdownCompact; } if (issueRef) { - const redactedWakeContext = await createRunSecretRedactionRegistry( + const redactedWakeContext = await measureSandboxOperation("heartbeat.create_run_secret_redaction_registry.redact_for_issue", { operationIndex: 37 }, async () => (createRunSecretRedactionRegistry( db, ).redactForIssue(agent.companyId, issueRef.id, { paperclipIssue: context.paperclipIssue, paperclipWakeComment: context.paperclipWakeComment, paperclipTaskMarkdown: context.paperclipTaskMarkdown, paperclipTaskMarkdownCompact: context.paperclipTaskMarkdownCompact, - }); + }))); context.paperclipIssue = redactedWakeContext.paperclipIssue; if (redactedWakeContext.paperclipWakeComment) { context.paperclipWakeComment = @@ -18647,10 +18659,10 @@ export function heartbeatService( : null; const persistedNativeExecutionWorkspaceId = persistedNativeExecutionInput?.binding.executionWorkspaceId ?? null; - const localEnvironment = await environmentsSvc.ensureLocalEnvironment( + const localEnvironment = await measureSandboxOperation("heartbeat.environments_svc.ensure_local_environment", { operationIndex: 38 }, async () => (environmentsSvc.ensureLocalEnvironment( agent.companyId, - ); - const resolvedInstanceSettings = await instanceSettings.get(); + ))); + const resolvedInstanceSettings = await measureSandboxOperation("heartbeat.instance_settings.get", { operationIndex: 39 }, async () => (instanceSettings.get())); // Managed-sandbox-only policy: a run that would land on the local // environment is redirected onto the platform-managed sandbox row, and // with no active managed row the resolution fails closed @@ -18658,10 +18670,10 @@ export function heartbeatService( // kubernetes execution mode below, which takes precedence when both // regimes are active. const managedSandboxOnly = - (await instanceSettings.getExperimental()).enableManagedSandboxOnly === + (await measureSandboxOperation("heartbeat.instance_settings.get_experimental", { operationIndex: 40 }, async () => (instanceSettings.getExperimental()))).enableManagedSandboxOnly === true; const managedSandboxEnvironment = managedSandboxOnly - ? await environmentsSvc.findManagedSandboxEnvironment(agent.companyId) + ? await measureSandboxOperation("heartbeat.environments_svc.find_managed_sandbox_environment", { operationIndex: 41 }, async () => (environmentsSvc.findManagedSandboxEnvironment(agent.companyId))) : null; const environmentResolution = resolveExecutionWorkspaceEnvironmentId({ agentDefaultEnvironmentId: agent.defaultEnvironmentId, @@ -18688,7 +18700,7 @@ export function heartbeatService( let selectedEnvironmentId = environmentResolution.environmentId; if (executionForcedToKubernetes) { let kubernetesEnvironment = - await environmentsSvc.findKubernetesEnvironment(agent.companyId); + await measureSandboxOperation("heartbeat.environments_svc.find_kubernetes_environment", { operationIndex: 42 }, async () => (environmentsSvc.findKubernetesEnvironment(agent.companyId))); if (!kubernetesEnvironment) { // Lazy recovery for companies created after the startup bootstrap ran // (the boot hook only provisions environments for companies that exist @@ -18713,12 +18725,12 @@ export function heartbeatService( }`; } if (bootstrap) { - await environmentsSvc.ensureKubernetesEnvironment( + await measureSandboxOperation("heartbeat.environments_svc.ensure_kubernetes_environment", { operationIndex: 43 }, async () => (environmentsSvc.ensureKubernetesEnvironment( agent.companyId, bootstrap.kubernetesConfig, - ); + ))); kubernetesEnvironment = - await environmentsSvc.findKubernetesEnvironment(agent.companyId); + await measureSandboxOperation("heartbeat.environments_svc.find_kubernetes_environment", { operationIndex: 44 }, async () => (environmentsSvc.findKubernetesEnvironment(agent.companyId))); } else { logger.warn( { @@ -18757,21 +18769,21 @@ export function heartbeatService( selectedEnvironmentId === localEnvironment.id ? localEnvironment : selectedEnvironmentId - ? await environmentsSvc.getById(selectedEnvironmentId) + ? await measureSandboxOperation("heartbeat.environments_svc.get_by_id", { operationIndex: 45 }, async () => (environmentsSvc.getById(selectedEnvironmentId))) : null; const unboundLegacyWorkspaceId = persistedNativeExecutionWorkspaceId ? null - : await findUnboundLegacyTaskWorkspace(db, { + : await measureSandboxOperation("heartbeat.find_unbound_legacy_task_workspace", { operationIndex: 46 }, async () => (findUnboundLegacyTaskWorkspace(db, { companyId: agent.companyId, issueId, projectId: issueRef?.projectId ?? null, agentId: agent.id, responsibleUserId: run.responsibleUserId, adapterType: agent.adapterType, environment: selectedEnvironmentForConfig, executionWorkspaceId: readNonEmptyString(issueRef?.executionWorkspaceId), executionWorkspacePreference: issueRef?.executionWorkspacePreference ?? null, - }); + }))); const requestedExecutionWorkspaceId = persistedNativeExecutionWorkspaceId ?? readNonEmptyString(issueRef?.executionWorkspaceId) ?? unboundLegacyWorkspaceId; const existingExecutionWorkspace = requestedExecutionWorkspaceId - ? await executionWorkspacesSvc.getById(requestedExecutionWorkspaceId) + ? await measureSandboxOperation("heartbeat.execution_workspaces_svc.get_by_id", { operationIndex: 47 }, async () => (executionWorkspacesSvc.getById(requestedExecutionWorkspaceId))) : null; const nativeRecoveryExecutionWorkspaceId = resolveNativeRecoveryExecutionWorkspaceBinding({ @@ -18807,13 +18819,14 @@ export function heartbeatService( issueRef?.projectWorkspaceId && effectiveExecutionWorkspaceMode === "shared_workspace" ) { - const workspaceHolder = await findSharedWorkspaceHolder({ + const projectWorkspaceId = issueRef.projectWorkspaceId; + const workspaceHolder = await measureSandboxOperation("heartbeat.find_shared_workspace_holder", { operationIndex: 48 }, async () => (findSharedWorkspaceHolder({ companyId: agent.companyId, - projectWorkspaceId: issueRef.projectWorkspaceId, + projectWorkspaceId, excludeIssueId: issueRef.id, excludeRunId: run.id, honorIsolatedWorkspaceModes: isolatedWorkspacesEnabled, - }); + }))); if (workspaceHolder) { const environmentDriver = selectedEnvironmentForConfig?.driver ?? null; @@ -18890,18 +18903,18 @@ export function heartbeatService( const executionRunConfig = stripWorkspaceRuntimeFromExecutionRunConfig(mergedConfig); const runScopedMentionedSkillKeys = - await resolveRunScopedMentionedSkillKeys({ + await measureSandboxOperation("heartbeat.resolve_run_scoped_mentioned_skill_keys", { operationIndex: 49 }, async () => (resolveRunScopedMentionedSkillKeys({ db, companyId: agent.companyId, issueId, - }); + }))); const runScopedSkillKeys = acceptedPlanContinuationWake && !acceptedPlanWakeRoutingDecision?.suppressAcceptedContinuation ? [...runScopedMentionedSkillKeys, ACCEPTED_PLAN_CONVERSION_SKILL_KEY] : runScopedMentionedSkillKeys; const { resolvedConfig, secretKeys, secretManifest } = - await resolveExecutionRunAdapterConfig({ + await measureSandboxOperation("heartbeat.resolve_execution_run_adapter_config", { operationIndex: 50 }, async () => (resolveExecutionRunAdapterConfig({ managedGitHubCredentials: true, companyId: agent.companyId, agentId: agent.id, @@ -18919,7 +18932,7 @@ export function heartbeatService( routineEnv: routineEnvContext.env, secretsSvc, trustPreset, - }); + }))); if (secretManifest.length > 0) { context.paperclipSecrets = { manifest: secretManifest, @@ -18934,7 +18947,7 @@ export function heartbeatService( const runtimeSkillPreference = readPaperclipSkillSyncPreference( effectiveResolvedConfig, ); - const runtimeSkillEntries = await companySkills.listRuntimeSkillEntries( + const runtimeSkillEntries = await measureSandboxOperation("heartbeat.company_skills.list_runtime_skill_entries", { operationIndex: 51 }, async () => (companySkills.listRuntimeSkillEntries( agent.companyId, { versionSelections: skillVersionSelectionMap( @@ -18945,17 +18958,17 @@ export function heartbeatService( }, ), }, - ); + ))); let runtimeConfig: Record = { ...effectiveResolvedConfig, paperclipRuntimeSkills: runtimeSkillEntries, }; - const latestAgentConfigRevision = await getLatestAgentConfigRevision( + const latestAgentConfigRevision = await measureSandboxOperation("heartbeat.get_latest_agent_config_revision", { operationIndex: 52 }, async () => (getLatestAgentConfigRevision( agent.companyId, agent.id, - ); + ))); const sessionConfigMetadata = - await buildEffectiveRunSessionConfigMetadata({ + await measureSandboxOperation("heartbeat.build_effective_run_session_config_metadata", { operationIndex: 53 }, async () => (buildEffectiveRunSessionConfigMetadata({ adapterType: agent.adapterType, effectiveAdapterConfig: runtimeConfig, agentRuntimeConfig: agent.runtimeConfig, @@ -19018,7 +19031,7 @@ export function heartbeatService( latestAgentConfigRevision.createdAt.toISOString(), } : null, - }); + }))); const configuredModel = readConfiguredModelFromAdapterConfig(runtimeConfig); const wakeSessionResetReason = describeSessionResetReason(context); @@ -19056,7 +19069,7 @@ export function heartbeatService( const { selectedEnvironmentDriver: lowTrustPreflightEnvironmentDriver, workspace: resolvedWorkspace, - } = await resolveWorkspaceAfterLowTrustPreflight({ + } = await measureSandboxOperation("heartbeat.resolve_workspace_after_low_trust_preflight", { operationIndex: 54 }, async () => (resolveWorkspaceAfterLowTrustPreflight({ db, trustPreset, isolatedWorkspacesEnabled, @@ -19069,13 +19082,13 @@ export function heartbeatService( } : null, resolveSelectedEnvironmentDriver: async () => { - const preflightEnvironment = await envOrchestrator.resolveEnvironment( + const preflightEnvironment = await measureSandboxOperation("heartbeat.env_orchestrator.resolve_environment", { operationIndex: 55 }, async () => (envOrchestrator.resolveEnvironment( { companyId: agent.companyId, selectedEnvironmentId, localEnvironmentId: localEnvironment.id, }, - ); + ))); return preflightEnvironment.driver; }, resolveWorkspace: () => @@ -19089,7 +19102,7 @@ export function heartbeatService( executionEnvironmentDriver: selectedEnvironmentForConfig?.driver ?? null, }), - }); + }))); const hostExecutionWorkspaceConfig = stripHostWorkspaceProvisionForLowTrustSandbox({ config: mergedConfig, @@ -19105,7 +19118,7 @@ export function heartbeatService( repoRef: resolvedWorkspace.repoRef, additionalWorkspaces: resolvedWorkspace.additionalWorkspaces, } satisfies ExecutionWorkspaceInput; - await assertGitWorktreeBaseWorkspaceReady({ + await measureSandboxOperation("heartbeat.assert_git_worktree_base_workspace_ready", { operationIndex: 56 }, async () => (assertGitWorktreeBaseWorkspaceReady({ requestedExecutionWorkspaceMode, config: hostExecutionWorkspaceConfig, issue: issueRef, @@ -19114,7 +19127,7 @@ export function heartbeatService( baseCwdFallback: resolvedWorkspace.baseCwdFallback, materializationFailures: resolvedWorkspace.materializationFailures, }, - }); + }))); const workspaceStrategyForFingerprint = parseObject( hostExecutionWorkspaceConfig.workspaceStrategy, ); @@ -19238,7 +19251,7 @@ export function heartbeatService( executionWorkspace, reusedExecutionWorkspace, policy: resolvedWorkspaceReusePolicy, - } = await provisionExecutionWorkspaceForFreshnessDecision( + } = await measureSandboxOperation("heartbeat.provision_execution_workspace_for_freshness_decision", { operationIndex: 57 }, async () => (provisionExecutionWorkspaceForFreshnessDecision( { requestedShouldReuseExisting, existingExecutionWorkspaceId: @@ -19335,7 +19348,7 @@ export function heartbeatService( resolveGitAuth: workspaceGitAuthProvider, }), }, - ); + ))); const resolvedProjectId = executionWorkspace.projectId ?? issueRef?.projectId ?? @@ -19392,11 +19405,11 @@ export function heartbeatService( } if (Object.keys(nextIssuePatch).length > 0) { if (warmReusableExecutionWorkspace && !isolatedWorkspacesEnabled) { - await bindWarmSandboxWorkspace(db, { + await measureSandboxOperation("heartbeat.bind_warm_sandbox_workspace", { operationIndex: 58 }, async () => (bindWarmSandboxWorkspace(db, { companyId: agent.companyId, issueId, runId: run.id, agentId: agent.id, workspaceId: workspace.id, - }); + }))); } else { - await issuesSvc.update(issueId, nextIssuePatch); + await measureSandboxOperation("heartbeat.issues_svc.update", { operationIndex: 59 }, async () => (issuesSvc.update(issueId, nextIssuePatch))); } issueExecutionWorkspaceIdForRun = workspace.id; issueProjectWorkspaceIdForRun = @@ -19446,11 +19459,12 @@ export function heartbeatService( executionWorkspace.worktreePath ) { try { + const ownershipWorktreePath = executionWorkspace.worktreePath; persistedWorktreeInstanceRoot = ( - await readManagedWorktreeInstanceOwnership( - executionWorkspace.worktreePath, - ) + await measureSandboxOperation("heartbeat.read_managed_worktree_instance_ownership", { operationIndex: 60 }, async () => (readManagedWorktreeInstanceOwnership( + ownershipWorktreePath, + ))) )?.instanceRoot ?? null; } catch (error) { logger.warn( @@ -19482,7 +19496,7 @@ export function heartbeatService( persistedExecutionWorkspace = resolvedWorkspaceReusePolicy.shouldRestoreExistingWorkspace && reusableExistingExecutionWorkspace - ? await executionWorkspacesSvc.update( + ? await measureSandboxOperation("heartbeat.execution_workspaces_svc.update", { operationIndex: 61 }, async () => (executionWorkspacesSvc.update( reusableExistingExecutionWorkspace.id, { cwd: executionWorkspace.cwd, @@ -19503,9 +19517,9 @@ export function heartbeatService( resolvedProjectWorkspaceId, ), }, - ) + ))) : resolvedProjectId - ? await executionWorkspacesSvc.create({ + ? await measureSandboxOperation("heartbeat.execution_workspaces_svc.create", { operationIndex: 62 }, async () => (executionWorkspacesSvc.create({ companyId: agent.companyId, projectId: resolvedProjectId, projectWorkspaceId: resolvedProjectWorkspaceId, @@ -19539,12 +19553,12 @@ export function heartbeatService( lastUsedAt: new Date(), openedAt: new Date(), metadata: nextExecutionWorkspaceMetadata, - }) + }))) : null; } catch (error) { if (executionWorkspace.created) { try { - await cleanupExecutionWorkspaceArtifacts({ + await measureSandboxOperation("heartbeat.cleanup_execution_workspace_artifacts", { operationIndex: 63 }, async () => (cleanupExecutionWorkspaceArtifacts({ workspace: { id: reusableExistingExecutionWorkspace?.id ?? @@ -19575,7 +19589,7 @@ export function heartbeatService( ?.teardownCommand ?? null, recorder: workspaceOperationRecorder, - }); + }))); } catch (cleanupError) { logger.warn( { @@ -19593,10 +19607,10 @@ export function heartbeatService( } throw error; } - await workspaceOperationRecorder.attachExecutionWorkspaceId( + await measureSandboxOperation("heartbeat.workspace_operation_recorder.attach_execution_workspace_id", { operationIndex: 64 }, async () => (workspaceOperationRecorder.attachExecutionWorkspaceId( persistedExecutionWorkspace?.id ?? null, - ); - await recordWorkspaceConfigFreshnessOperation({ + ))); + await measureSandboxOperation("heartbeat.record_workspace_config_freshness_operation", { operationIndex: 65 }, async () => (recordWorkspaceConfigFreshnessOperation({ recorder: workspaceOperationRecorder, runId: run.id, decision: workspaceConfigFreshness, @@ -19608,7 +19622,7 @@ export function heartbeatService( previousWorkspaceId: workspaceReuseRequest.requestedExecutionWorkspaceId, activeWorkspaceId: persistedExecutionWorkspace?.id ?? null, - }); + }))); if ( reusableExistingExecutionWorkspace && persistedExecutionWorkspace && @@ -19616,24 +19630,24 @@ export function heartbeatService( persistedExecutionWorkspace.id && reusableExistingExecutionWorkspace.status === "active" ) { - await executionWorkspacesSvc.update( + await measureSandboxOperation("heartbeat.execution_workspaces_svc.update", { operationIndex: 66 }, async () => (executionWorkspacesSvc.update( reusableExistingExecutionWorkspace.id, { status: "idle", cleanupReason: null, }, - ); + ))); } - await bindIssueToPersistedExecutionWorkspace(persistedExecutionWorkspace); + await measureSandboxOperation("heartbeat.bind_issue_to_persisted_execution_workspace", { operationIndex: 67 }, async () => (bindIssueToPersistedExecutionWorkspace(persistedExecutionWorkspace))); if (persistedExecutionWorkspace) { context.executionWorkspaceId = persistedExecutionWorkspace.id; - await db + await measureSandboxOperation("heartbeat.db.update.set.where", { operationIndex: 68 }, async () => (db .update(heartbeatRuns) .set({ contextSnapshot: context, updatedAt: new Date(), }) - .where(eq(heartbeatRuns.id, run.id)); + .where(eq(heartbeatRuns.id, run.id)))); } const nativeRunnerPreparationSpans: NativeRunHistoricalSpan[] = []; const environmentAcquireStartedAtMs = Date.now(); @@ -19641,7 +19655,7 @@ export function heartbeatService( ReturnType >; try { - acquiredEnvironment = await envOrchestrator.acquireForRun({ + acquiredEnvironment = await measureSandboxOperation("heartbeat.env_orchestrator.acquire_for_run", { operationIndex: 69 }, async () => (envOrchestrator.acquireForRun({ companyId: agent.companyId, selectedEnvironmentId, localEnvironmentId: localEnvironment.id, @@ -19651,7 +19665,7 @@ export function heartbeatService( agentId: agent.id, persistedExecutionWorkspace, executionWorkspaceSettings: environmentExecutionWorkspaceSettings, - }); + }))); nativeRunnerPreparationSpans.push({ name: "environment.acquire", parentName: "task.run", @@ -19711,12 +19725,12 @@ export function heartbeatService( }, emitTransportEvent: (event) => { void (async () => { - await appendRunEvent(run, { + await measureSandboxOperation("heartbeat.append_run_event", { operationIndex: 70 }, async () => (appendRunEvent(run, { eventType: event.name, stream: "system", level: event.dimensions.outcome === "error" ? "warn" : "info", payload: { ...event.dimensions }, - }); + }))); })().catch(() => {}); }, }, @@ -19726,7 +19740,7 @@ export function heartbeatService( ReturnType >; try { - realizationResult = await envOrchestrator.realizeForRun({ + realizationResult = await measureSandboxOperation("heartbeat.env_orchestrator.realize_for_run", { operationIndex: 71 }, async () => (envOrchestrator.realizeForRun({ environment: selectedEnvironment, lease: activeEnvironmentLease.lease, adapterType: agent.adapterType, @@ -19737,7 +19751,7 @@ export function heartbeatService( effectiveExecutionWorkspaceMode, persistedExecutionWorkspace, duplexObservabilityRecorder, - }); + }))); nativeRunnerPreparationSpans.push({ name: "environment.workspace.realize", parentName: "task.run", @@ -19767,7 +19781,7 @@ export function heartbeatService( // after the host-side provisioning boundary above. Bind that final ID to // the issue before dispatch so warm turns reuse the exact same workspace // and lease scope instead of silently creating a per-run replacement. - await bindIssueToPersistedExecutionWorkspace(persistedExecutionWorkspace); + await measureSandboxOperation("heartbeat.bind_issue_to_persisted_execution_workspace", { operationIndex: 72 }, async () => (bindIssueToPersistedExecutionWorkspace(persistedExecutionWorkspace))); const workspaceRealization = realizationResult.workspaceRealization; const executionTarget = realizationResult.executionTarget; if (executionTarget?.kind === "remote" && executionTarget.transport === "sandbox" @@ -19778,16 +19792,16 @@ export function heartbeatService( // both legacy and native dispatch. Local execution never enters here. workFolderSaveFailed = true; workFolderLeaseId = activeEnvironmentLease.lease.id; - sandboxWorkFolders = await prepareSandboxWorkFolders({ db, companyId: run.companyId, runId: run.id, + sandboxWorkFolders = await measureSandboxOperation("heartbeat.prepare_sandbox_work_folders", { operationIndex: 73 }, async () => (prepareSandboxWorkFolders({ db, companyId: run.companyId, runId: run.id, agentId: agent.id, responsibleUserId: run.responsibleUserId ?? null, taskId: issueRef?.id ?? null, projectId: issueRef?.projectId ?? null, target: executionTarget, primaryWorkspaceId: executionWorkspace.workspaceId, primaryBranchName: executionWorkspace.branchName, - sandboxKey: workFolderSandboxKey(activeEnvironmentLease.lease) }); + sandboxKey: workFolderSandboxKey(activeEnvironmentLease.lease) }))); if (sandboxWorkFolders.identityChanged) { taskSessionForRun = null; previousSessionParams = null; } executionTarget.workFolderHome = sandboxWorkFolders.home; executionTarget.remoteCwd = sandboxWorkFolders.primaryRepo; const nextLeaseMetadata = { ...activeEnvironmentLease.lease.metadata, remoteCwd: sandboxWorkFolders.primaryRepo, workFolderHome: sandboxWorkFolders.home }; - await db.update(environmentLeases).set({ metadata: nextLeaseMetadata, updatedAt: new Date() }).where(eq(environmentLeases.id, activeEnvironmentLease.lease.id)); + await measureSandboxOperation("heartbeat.db.update.set.where", { operationIndex: 74 }, async () => (db.update(environmentLeases).set({ metadata: nextLeaseMetadata, updatedAt: new Date() }).where(eq(environmentLeases.id, activeEnvironmentLease.lease.id)))); activeEnvironmentLease = { ...activeEnvironmentLease, lease: { ...activeEnvironmentLease.lease, metadata: nextLeaseMetadata } }; runtimeConfig = { ...runtimeConfig, env: { ...parseObject(runtimeConfig.env), ...sandboxWorkFolders.env } }; context.paperclipWorkFolders = { ...sandboxWorkFolders.manifest, primaryRepo: sandboxWorkFolders.primaryRepo }; @@ -19811,26 +19825,26 @@ export function heartbeatService( ) { return { dispatched: true, resultPromise: dispatch(() => {}) }; } - await options.beforeResolvedInteractionContinuationDispatchCheck?.({ + await measureSandboxOperation("heartbeat.options.before_resolved_interaction_continuation_dispatch_check", { operationIndex: 75 }, async () => (options.beforeResolvedInteractionContinuationDispatchCheck?.({ runId: run.id, issueId, - }); + }))); - const gate = await db.transaction(async (tx) => { - const lockedIssue = await tx + const gate = await measureSandboxOperation("heartbeat.db.transaction", { operationIndex: 76 }, async () => (db.transaction(async (tx) => { + const lockedIssue = await measureSandboxOperation("heartbeat.tx.select.from.where.for.then", { operationIndex: 77 }, async () => (tx .select({ executionRunId: issues.executionRunId }) .from(issues) .where( and(eq(issues.id, issueId), eq(issues.companyId, run.companyId)), ) .for("update") - .then((rows) => rows[0] ?? null); - const staleness = await evaluateQueuedRunStaleness( + .then((rows) => rows[0] ?? null))); + const staleness = await measureSandboxOperation("heartbeat.evaluate_queued_run_staleness", { operationIndex: 78 }, async () => (evaluateQueuedRunStaleness( run, issueId, context, tx as unknown as Db, - ); + ))); if (staleness.stale) { return { dispatched: false as const, staleness }; } @@ -19851,10 +19865,10 @@ export function heartbeatService( }; } - await options.afterResolvedInteractionContinuationDispatchCheck?.({ + await measureSandboxOperation("heartbeat.options.after_resolved_interaction_continuation_dispatch_check", { operationIndex: 79 }, async () => (options.afterResolvedInteractionContinuationDispatchCheck?.({ runId: run.id, issueId, - }); + }))); let dispatchStarted = false; let resolveDispatchStarted!: () => void; const dispatchStartedPromise = new Promise((resolve) => { @@ -19872,23 +19886,23 @@ export function heartbeatService( // spawn, settling the adapter promise also releases the gate. const resultPromise = dispatch(markDispatchStarted); void resultPromise.then(markDispatchStarted, markDispatchStarted); - await dispatchStartedPromise; + await measureSandboxOperation("heartbeat.dispatch_started_promise", { operationIndex: 80 }, async () => (dispatchStartedPromise)); return { dispatched: true as const, resultPromise }; - }); + }))); if (gate.dispatched) return gate; - await cancelRunForStaleIssue(run, issueId, gate.staleness); + await measureSandboxOperation("heartbeat.cancel_run_for_stale_issue", { operationIndex: 81 }, async () => (cancelRunForStaleIssue(run, issueId, gate.staleness))); return { dispatched: false }; }; if (!executionTarget || executionTarget.kind === "local") { try { - runScratch = await prepareHeartbeatRunScratch({ + runScratch = await measureSandboxOperation("heartbeat.prepare_heartbeat_run_scratch", { operationIndex: 82 }, async () => (prepareHeartbeatRunScratch({ companyId: agent.companyId, agentId: agent.id, runId: run.id, issueId: issueRef?.id ?? null, issueIdentifier: issueRef?.identifier ?? null, - }); + }))); const existingRuntimeEnv = parseObject(runtimeConfig.env); const scratchEnv = buildHeartbeatRunScratchEnv( existingRuntimeEnv, @@ -19932,10 +19946,10 @@ export function heartbeatService( url: configuredPaperclipApiBaseUrl() ?? "", token: githubBrokerToken?.token ?? "", }); githubLauncherLocation = { runId: run.id, target: executionTarget }; - runtimeConfig = { ...runtimeConfig, env: await prepareGitHubOperationLaunchers({ + runtimeConfig = { ...runtimeConfig, env: await measureSandboxOperation("heartbeat.prepare_git_hub_operation_launchers", { operationIndex: 83 }, async () => (prepareGitHubOperationLaunchers({ runId: run.id, target: executionTarget, cwd: executionWorkspace.cwd, env: githubBrokerEnv, - }) }; + }))) }; secretKeys.add("PAPERCLIP_GITHUB_BROKER_TOKEN"); context.paperclipEnvironment = { id: selectedEnvironment.id, @@ -19965,13 +19979,13 @@ export function heartbeatService( } : {}), }; - await db + await measureSandboxOperation("heartbeat.db.update.set.where", { operationIndex: 84 }, async () => (db .update(heartbeatRuns) .set({ contextSnapshot: context, updatedAt: new Date(), }) - .where(eq(heartbeatRuns.id, run.id)); + .where(eq(heartbeatRuns.id, run.id)))); const runtimeSessionResolution = resolveRuntimeSessionParamsForWorkspace({ agentId: agent.id, previousSessionParams, @@ -20013,12 +20027,12 @@ export function heartbeatService( branchName: executionWorkspace.branchName, worktreePath: executionWorkspace.worktreePath, realization: workspaceRealization, - agentHome: await (async () => { + agentHome: await measureSandboxOperation("heartbeat.inline_operation", { operationIndex: 85 }, async () => ((async () => { if (sandboxWorkFolders) return sandboxWorkFolders.env.AGENT_HOME; const home = resolveDefaultAgentWorkspaceDir(agent.id); - await fs.mkdir(home, { recursive: true }); + await measureSandboxOperation("heartbeat.fs.mkdir", { operationIndex: 86 }, async () => (fs.mkdir(home, { recursive: true }))); return home; - })(), + })())), }; context.paperclipWorkspaces = sandboxWorkFolders ? sandboxWorkFolders.manifest.repositories.map((repo) => ({ workspaceId: repo.workspaceId, projectId: sandboxWorkFolders!.manifest.projectId, @@ -20125,12 +20139,12 @@ export function heartbeatService( runtimeSessionParamsForAdapter = null; previousSessionDisplayId = null; } - const sessionCompaction = await evaluateSessionCompaction({ + const sessionCompaction = await measureSandboxOperation("heartbeat.evaluate_session_compaction", { operationIndex: 87 }, async () => (evaluateSessionCompaction({ agent, sessionId: previousSessionDisplayId ?? runtimeSessionIdForAdapter, issueId, continuationSummaryBody: continuationSummary?.body ?? null, - }); + }))); if (sessionCompaction.rotate) { context.paperclipSessionHandoffMarkdown = sessionCompaction.handoffMarkdown; @@ -20217,7 +20231,7 @@ export function heartbeatService( pendingOutputProgress.at.getTime() - lastOutputFlushAt.getTime() >= ACTIVE_RUN_OUTPUT_PROGRESS_FLUSH_INTERVAL_MS; if (!shouldFlush) return; - await db + await measureSandboxOperation("heartbeat.db.update.set.where", { operationIndex: 88 }, async () => (db .update(heartbeatRuns) .set({ lastOutputAt: pendingOutputProgress.at, @@ -20226,13 +20240,13 @@ export function heartbeatService( lastOutputBytes: pendingOutputProgress.bytes, updatedAt: new Date(), }) - .where(eq(heartbeatRuns.id, run.id)); + .where(eq(heartbeatRuns.id, run.id)))); lastOutputFlushAt = pendingOutputProgress.at; outputProgressState.pending = null; }; try { const startedAt = run.startedAt ?? new Date(); - const runningWithSession = await db + const runningWithSession = await measureSandboxOperation("heartbeat.db.update.set.where.returning.then", { operationIndex: 89 }, async () => (db .update(heartbeatRuns) .set({ startedAt, @@ -20243,12 +20257,12 @@ export function heartbeatService( }) .where(eq(heartbeatRuns.id, run.id)) .returning() - .then((rows) => rows[0] ?? null); + .then((rows) => rows[0] ?? null))); if (runningWithSession) run = runningWithSession; // Pause Durability: flip to "running" ONLY if the agent is still invokable. // Atomic conditional UPDATE is the sole gate (no read-then-write); 0 rows => abort. - const runningAgent = await db + const runningAgent = await measureSandboxOperation("heartbeat.db.update.set.where.returning.then", { operationIndex: 90 }, async () => (db .update(agents) .set({ status: "running", updatedAt: new Date() }) .where( @@ -20258,7 +20272,7 @@ export function heartbeatService( ), ) .returning() - .then((rows) => rows[0] ?? null); + .then((rows) => rows[0] ?? null))); if (!runningAgent) { logger.warn( @@ -20267,7 +20281,7 @@ export function heartbeatService( ); const abortReason = "Cancelled: agent not invokable at execution-start"; - await setRunStatus(run.id, "cancelled", { + await measureSandboxOperation("heartbeat.set_run_status", { operationIndex: 91 }, async () => (setRunStatus(run.id, "cancelled", { finishedAt: new Date(), error: abortReason, errorCode: "agent_not_invokable", @@ -20280,12 +20294,12 @@ export function heartbeatService( }), } : {}), - }); - await setWakeupStatus(run.wakeupRequestId, "cancelled", { + }))); + await measureSandboxOperation("heartbeat.set_wakeup_status", { operationIndex: 92 }, async () => (setWakeupStatus(run.wakeupRequestId, "cancelled", { finishedAt: new Date(), error: abortReason, - }); - await releaseIssueExecutionAndPromote(run); + }))); + await measureSandboxOperation("heartbeat.release_issue_execution_and_promote", { operationIndex: 93 }, async () => (releaseIssueExecutionAndPromote(run))); return; } @@ -20300,30 +20314,31 @@ export function heartbeatService( }); const currentRun = run; - await appendRunEvent(currentRun, { + await measureSandboxOperation("heartbeat.append_run_event", { operationIndex: 94 }, async () => (appendRunEvent(currentRun, { eventType: "lifecycle", stream: "system", level: "info", message: "run started", - }); + }))); - handle = await runLogStore.begin({ + handle = await measureSandboxOperation("heartbeat.run_log_store.begin", { operationIndex: 95 }, async () => (runLogStore.begin({ companyId: run.companyId, agentId: run.agentId, runId, - }); + }))); - await db + const openedLogHandle = handle; + await measureSandboxOperation("heartbeat.db.update.set.where", { operationIndex: 96 }, async () => (db .update(heartbeatRuns) .set({ - logStore: handle.store, - logRef: handle.logRef, + logStore: openedLogHandle.store, + logRef: openedLogHandle.logRef, updatedAt: new Date(), }) - .where(eq(heartbeatRuns.id, runId)); + .where(eq(heartbeatRuns.id, runId)))); const currentUserRedactionOptions = - await getCurrentUserRedactionOptions(); + await measureSandboxOperation("heartbeat.get_current_user_redaction_options", { operationIndex: 97 }, async () => (getCurrentUserRedactionOptions())); const onLog = async (stream: "stdout" | "stderr", chunk: string) => { const sanitizedChunk = compactRunLogChunk( redactCurrentUserText(chunk, currentUserRedactionOptions), @@ -20338,12 +20353,13 @@ export function heartbeatService( const chunkSeq = outputSeq; let appendedBytes = 0; if (handle) { - appendedBytes = await runLogStore.append(handle, { + const appendHandle = handle; + appendedBytes = await measureSandboxOperation("heartbeat.run_log_store.append", { operationIndex: 98 }, async () => (runLogStore.append(appendHandle, { stream, chunk: sanitizedChunk, ts, seq: chunkSeq, - }); + }))); persistedLogBytes += appendedBytes; } outputProgressState.pending = { @@ -20352,7 +20368,7 @@ export function heartbeatService( stream, bytes: persistedLogBytes, }; - await flushOutputProgress(); + await measureSandboxOperation("heartbeat.flush_output_progress", { operationIndex: 99 }, async () => (flushOutputProgress())); // Streamed CLI output is real run activity: keep the in-memory // runtime status ("Working... / X ago") fresh between structured @@ -20400,16 +20416,16 @@ export function heartbeatService( }); }; if (runScopedMentionedSkillKeys.length > 0) { - await onLog( + await measureSandboxOperation("heartbeat.on_log", { operationIndex: 100 }, async () => (onLog( "stdout", `[paperclip] Enabled run-scoped skills from issue mentions: ${runScopedMentionedSkillKeys.join(", ")}\n`, - ); + ))); } for (const warning of runtimeWorkspaceWarnings) { const logEntry = formatRuntimeWorkspaceWarningLog(warning); - await onLog(logEntry.stream, logEntry.chunk); + await measureSandboxOperation("heartbeat.on_log", { operationIndex: 101 }, async () => (onLog(logEntry.stream, logEntry.chunk))); } - await assertGitSensitiveAdapterWorkspaceValid({ + await measureSandboxOperation("heartbeat.assert_git_sensitive_adapter_workspace_valid", { operationIndex: 102 }, async () => (assertGitSensitiveAdapterWorkspaceValid({ adapterType: agent.adapterType, agentId: agent.id, issue: issueRef @@ -20426,14 +20442,14 @@ export function heartbeatService( executionTarget, environmentDriver: selectedEnvironment.driver, leaseMetadata: activeEnvironmentLease.lease.metadata, - }); + }))); const adapterEnv = Object.fromEntries( Object.entries({ ...parseObject(runtimeConfig.env), ...sandboxWorkFolders?.env }).filter( (entry): entry is [string, string] => typeof entry[0] === "string" && typeof entry[1] === "string", ), ); - const runtimeServices = await ensureRuntimeServicesForRun({ + const runtimeServices = await measureSandboxOperation("heartbeat.ensure_runtime_services_for_run", { operationIndex: 103 }, async () => (ensureRuntimeServicesForRun({ db, runId: run.id, agent: { @@ -20451,19 +20467,19 @@ export function heartbeatService( adapterEnv, onLog, recorder: workspaceOperationRecorder, - }); + }))); if (runtimeServices.length > 0) { context.paperclipRuntimeServices = runtimeServices; context.paperclipRuntimePrimaryUrl = runtimeServices.find((service) => readNonEmptyString(service.url)) ?.url ?? null; - await db + await measureSandboxOperation("heartbeat.db.update.set.where", { operationIndex: 104 }, async () => (db .update(heartbeatRuns) .set({ contextSnapshot: context, updatedAt: new Date(), }) - .where(eq(heartbeatRuns.id, run.id)); + .where(eq(heartbeatRuns.id, run.id)))); } if ( issueId && @@ -20471,19 +20487,19 @@ export function heartbeatService( runtimeServices.some((service) => !service.reused)) ) { try { - await postWorkspaceReadyComment({ + await measureSandboxOperation("heartbeat.post_workspace_ready_comment", { operationIndex: 105 }, async () => (postWorkspaceReadyComment({ issuesSvc, issueId, agentId: agent.id, runId: run.id, workspace: executionWorkspace, runtimeServices, - }); + }))); } catch (err) { - await onLog( + await measureSandboxOperation("heartbeat.on_log", { operationIndex: 106 }, async () => (onLog( "stderr", `[paperclip] Failed to post workspace-ready comment: ${err instanceof Error ? err.message : String(err)}\n`, - ); + ))); } } const onAdapterMeta = async (meta: AdapterInvocationMeta) => { @@ -20492,26 +20508,26 @@ export function heartbeatService( if (key in meta.env) meta.env[key] = "***REDACTED***"; } } - await appendRunEvent(currentRun, { + await measureSandboxOperation("heartbeat.append_run_event", { operationIndex: 107 }, async () => (appendRunEvent(currentRun, { eventType: "adapter.invoke", stream: "system", level: "info", message: "adapter invocation", payload: meta as unknown as Record, - }); + }))); }; const onAdapterEvent = async (event: AdapterRuntimeEvent) => { const eventType = event.eventType.trim(); if (!eventType) return; - await appendRunEvent(currentRun, { + await measureSandboxOperation("heartbeat.append_run_event", { operationIndex: 108 }, async () => (appendRunEvent(currentRun, { eventType: eventType.slice(0, 120), stream: event.stream, level: event.level, color: event.color, message: event.message, payload: event.payload, - }); + }))); }; const adapter = getServerAdapter(agent.adapterType); @@ -20540,32 +20556,33 @@ export function heartbeatService( } const nativeExecutionWorkspaceId = persistedExecutionWorkspace?.id ?? run.id; - const persistedContract = run.completionContractId - ? await db + const completionContractId = run.completionContractId; + const persistedContract = completionContractId + ? await measureSandboxOperation("heartbeat.db.select.from.where.limit.then", { operationIndex: 109 }, async () => (db .select() .from(completionContracts) .where( and( - eq(completionContracts.id, run.completionContractId), + eq(completionContracts.id, completionContractId), eq(completionContracts.companyId, agent.companyId), eq(completionContracts.issueId, issueRef.id), ), ) .limit(1) - .then((rows) => rows[0] ?? null) + .then((rows) => rows[0] ?? null))) : null; const completionContract = persistedContract ? { row: persistedContract, contract: persistedContract.contractJson as never, } - : await ensureNativeCompletionContract({ + : await measureSandboxOperation("heartbeat.ensure_native_completion_contract", { operationIndex: 110 }, async () => (ensureNativeCompletionContract({ db, companyId: agent.companyId, issue: issueRef, actorId: agent.id, immediateRequest: safeWakeCommentContext?.body ?? null, - }); + }))); const taskNativeSessionId = sandboxWorkFolders?.identityChanged ? null : readNonEmptyString( taskSessionDecodedParams?.sessionId, ); @@ -20573,8 +20590,9 @@ export function heartbeatService( // recovery existed. Only an entirely unused replacement row may // inherit its source checkpoint; any process/provider evidence on the // replacement makes the ownership ambiguous and therefore ineligible. - const legacyRetrySource = !sandboxWorkFolders?.identityChanged && run.retryOfRunId - ? await db + const retryOfRunId = run.retryOfRunId; + const legacyRetrySource = !sandboxWorkFolders?.identityChanged && retryOfRunId + ? await measureSandboxOperation("heartbeat.db.select.from.where.limit.then", { operationIndex: 111 }, async () => (db .select({ id: heartbeatRuns.id, companyId: heartbeatRuns.companyId, @@ -20588,16 +20606,16 @@ export function heartbeatService( .from(heartbeatRuns) .where( and( - eq(heartbeatRuns.id, run.retryOfRunId), + eq(heartbeatRuns.id, retryOfRunId), eq(heartbeatRuns.companyId, agent.companyId), eq(heartbeatRuns.agentId, agent.id), ), ) .limit(1) - .then((rows) => rows[0] ?? null) + .then((rows) => rows[0] ?? null))) : null; const legacyRetryHasProviderEvidence = legacyRetrySource - ? await db + ? await measureSandboxOperation("heartbeat.db.select.from.where.limit.then", { operationIndex: 112 }, async () => (db .select({ id: heartbeatRunEvents.id }) .from(heartbeatRunEvents) .where( @@ -20615,7 +20633,7 @@ export function heartbeatService( ), ) .limit(1) - .then((rows) => rows.length > 0) + .then((rows) => rows.length > 0))) : false; const compatibleLegacyRetrySource = isUnusedLegacyNativeRetryReplacement({ @@ -20640,7 +20658,7 @@ export function heartbeatService( taskResumeRunId ?? compatibleLegacyRetrySource?.id ?? null; const previousNativeRun = resumableTaskSessionId && priorNativeRunId - ? await db + ? await measureSandboxOperation("heartbeat.db.select.from.where.limit.then", { operationIndex: 113 }, async () => (db .select({ id: heartbeatRuns.id, companyId: heartbeatRuns.companyId, @@ -20662,7 +20680,7 @@ export function heartbeatService( ), ) .limit(1) - .then((rows) => rows[0] ?? null) + .then((rows) => rows[0] ?? null))) : null; nativeRunnerInstanceId = previousNativeRun?.runnerInstanceId && @@ -20719,12 +20737,13 @@ export function heartbeatService( "native_execution_input_persisted_binding_mismatch", ); if (nativeExecution.provider.kind === "claude_managed") { - const recoveryProfile = await managedAgentProfileService( + const managedProfileId = nativeExecution.provider.managedProfile.profileId; + const recoveryProfile = await measureSandboxOperation("heartbeat.managed_agent_profile_service.require_qualified", { operationIndex: 114 }, async () => (managedAgentProfileService( db, ).requireQualified( agent.companyId, - nativeExecution.provider.managedProfile.profileId, - ); + managedProfileId, + ))); assertManagedProfileRecoveryBinding({ adapterConfig: agent.adapterConfig, snapshot: nativeExecution.provider.managedProfile, @@ -20732,13 +20751,14 @@ export function heartbeatService( }); } if (nativeExecution.provider.kind === "aws_agentcore") { - const recoveryProfile = await remoteAgentProfileService( + const agentCoreProfileId = nativeExecution.provider.agentCoreProfile.profileId; + const recoveryProfile = await measureSandboxOperation("heartbeat.remote_agent_profile_service.require_qualified", { operationIndex: 115 }, async () => (remoteAgentProfileService( db, ).requireQualified( agent.companyId, - nativeExecution.provider.agentCoreProfile.profileId, + agentCoreProfileId, "aws_bedrock_agentcore_harness", - ); + ))); assertAgentCoreProfileRecoveryBinding({ snapshot: nativeExecution.provider.agentCoreProfile, stored: recoveryProfile, @@ -20747,34 +20767,34 @@ export function heartbeatService( } else { const interactionId = readNonEmptyString(context.interactionId); const interactionResponses = - await materializeNativeInteractionResponses({ + await measureSandboxOperation("heartbeat.materialize_native_interaction_responses", { operationIndex: 116 }, async () => (materializeNativeInteractionResponses({ db, companyId: agent.companyId, issueId: issueRef.id, runId: run.id, agentId: agent.id, interactionIds: interactionId ? [interactionId] : [], - }); + }))); const runnerAdapterConfig = parseObject(agent.adapterConfig); const managedProfile = nativeRuntimeResolution.profile.backend === "claude_managed_agents_api" - ? await managedAgentProfileService(db).requireQualified( + ? await measureSandboxOperation("heartbeat.managed_agent_profile_service.require_qualified", { operationIndex: 117 }, async () => (managedAgentProfileService(db).requireQualified( agent.companyId, readNonEmptyString(runnerAdapterConfig.managedProfileId) ?? "", - ) + ))) : null; const agentCoreProfile = nativeRuntimeResolution.profile.backend === "aws_agentcore_harness_api" - ? await remoteAgentProfileService(db).requireQualified( + ? await measureSandboxOperation("heartbeat.remote_agent_profile_service.require_qualified", { operationIndex: 118 }, async () => (remoteAgentProfileService(db).requireQualified( agent.companyId, readNonEmptyString( runnerAdapterConfig.agentCoreProfileId, ) ?? "", "aws_bedrock_agentcore_harness", - ) + ))) : null; if (managedProfile) { const rawApiKeyBinding = parseObject( @@ -20808,29 +20828,29 @@ export function heartbeatService( : ("default" as const); const pinnedPlan = executionMode === "plan" - ? await documentService(db).getIssueDocumentByKey( + ? await measureSandboxOperation("heartbeat.document_service.get_issue_document_by_key", { operationIndex: 119 }, async () => (documentService(db).getIssueDocumentByKey( issueRef.id, "plan", - ) + ))) : null; const pinnedReviewContext = executionMode === "plan" - ? await buildPlanReviewContext({ + ? await measureSandboxOperation("heartbeat.build_plan_review_context", { operationIndex: 120 }, async () => (buildPlanReviewContext({ db, companyId: agent.companyId, issueId: issueRef.id, issueWorkMode: issueRef.workMode, interactionId: readNonEmptyString(context.interactionId), - }) + }))) : null; const pinnedPlanMarkdown = pinnedPlan?.body ?? ""; - const nativeRuntimeContext = await buildNativeRuntimeContext({ + const nativeRuntimeContext = await measureSandboxOperation("heartbeat.build_native_runtime_context", { operationIndex: 121 }, async () => (buildNativeRuntimeContext({ db, agent, runId: run.id, runtimeConfig, runtimeSkillEntries, - }); + }))); nativeExecution = buildNativeExecutionInput({ companyId: agent.companyId, runId: run.id, @@ -20944,14 +20964,14 @@ export function heartbeatService( "destroy_after_turn" ? "destroy" : undefined; - await db.transaction(async (tx) => { - const lockedRun = await tx + await measureSandboxOperation("heartbeat.db.transaction", { operationIndex: 122 }, async () => (db.transaction(async (tx) => { + const lockedRun = await measureSandboxOperation("heartbeat.tx.select.from.where.for.limit.then", { operationIndex: 123 }, async () => (tx .select() .from(heartbeatRuns) .where(eq(heartbeatRuns.id, run.id)) .for("update") .limit(1) - .then((rows) => rows[0] ?? null); + .then((rows) => rows[0] ?? null))); if (!lockedRun) throw new Error("native_runtime_run_missing"); if ( lockedRun.runtimeModeResolvedAt && @@ -20960,7 +20980,7 @@ export function heartbeatService( throw new Error("native_runtime_mode_conflict"); } const lockedProfile = parseObject(lockedRun.runnerProfileJson); - await tx + await measureSandboxOperation("heartbeat.tx.update.set.where", { operationIndex: 124 }, async () => (tx .update(heartbeatRuns) .set({ runtimeMode: "native", @@ -21036,8 +21056,8 @@ export function heartbeatService( lockedRun.nativePhaseUpdatedAt ?? new Date(), updatedAt: new Date(), }) - .where(eq(heartbeatRuns.id, run.id)); - await tx + .where(eq(heartbeatRuns.id, run.id)))); + await measureSandboxOperation("heartbeat.tx.insert.values.on_conflict_do_nothing", { operationIndex: 125 }, async () => (tx .insert(nativeRunFinalizations) .values({ runId: run.id, @@ -21045,9 +21065,9 @@ export function heartbeatService( issueId: issueRef.id, phase: "observed", }) - .onConflictDoNothing(); - }); - nativeWorkspaceSync = sandboxWorkFolders ? null : await prepareNativeWorkspaceSync({ + .onConflictDoNothing())); + }))); + nativeWorkspaceSync = sandboxWorkFolders ? null : await measureSandboxOperation("heartbeat.prepare_native_workspace_sync", { operationIndex: 126 }, async () => (prepareNativeWorkspaceSync({ db, runId: run.id, companyId: agent.companyId, @@ -21058,7 +21078,7 @@ export function heartbeatService( restartRecovery: runOptions.nativeRestartRecovery, sameRunRecovery: Boolean(runOptions.nativeLeaseOwner), resourceDisposition: providerResourceDispositionForRun, - }); + }))); } else { const legacyWarmLifecycle = executionTarget?.kind === "remote" && @@ -21072,7 +21092,7 @@ export function heartbeatService( if (legacyWarmLifecycle?.sandboxResource === "keep_running") { providerResourceDispositionForRun = "keep_running"; } - await db + await measureSandboxOperation("heartbeat.db.update.set.where", { operationIndex: 127 }, async () => (db .update(heartbeatRuns) .set({ runtimeMode: "legacy", @@ -21091,7 +21111,7 @@ export function heartbeatService( : null, updatedAt: new Date(), }) - .where(eq(heartbeatRuns.id, run.id)); + .where(eq(heartbeatRuns.id, run.id)))); } const localAgentJwtScope = issueRef?.workMode === "skill_test" @@ -21127,9 +21147,9 @@ export function heartbeatService( let adapterFinalizeOutcome: "succeeded" | "failed" | null = null; const inspectFinalizeWorkspaceBranch = async () => { const workspaceRecord = persistedExecutionWorkspace?.id - ? await executionWorkspacesSvc.getById( + ? await measureSandboxOperation("heartbeat.execution_workspaces_svc.get_by_id", { operationIndex: 128 }, async () => (executionWorkspacesSvc.getById( persistedExecutionWorkspace.id, - ) + ))) : persistedExecutionWorkspace; if (workspaceRecord?.strategyType !== "git_worktree") return null; @@ -21143,10 +21163,10 @@ export function heartbeatService( readNonEmptyString(executionWorkspace.branchName); if (!worktreePath || !expectedBranchName) return null; - const inspection = await inspectManagedGitWorktreeBranch({ + const inspection = await measureSandboxOperation("heartbeat.inspect_managed_git_worktree_branch", { operationIndex: 129 }, async () => (inspectManagedGitWorktreeBranch({ worktreePath, expectedBranchName, - }); + }))); return { workspaceRecord, inspection }; }; const recordWorkspaceFinalize = async ( @@ -21158,7 +21178,7 @@ export function heartbeatService( let finalizeBranchRepairMetadata: Record | null = null; if (status === "succeeded") { - const branchInspection = await inspectFinalizeWorkspaceBranch(); + const branchInspection = await measureSandboxOperation("heartbeat.inspect_finalize_workspace_branch", { operationIndex: 130 }, async () => (inspectFinalizeWorkspaceBranch())); if (branchInspection) { let inspection = branchInspection.inspection; const initialManagedGitWorktreeBranch = @@ -21168,11 +21188,12 @@ export function heartbeatService( inspection.reasonCode === "branch_mismatch" && inspection.repoRoot ) { + const repoRoot = inspection.repoRoot; let repairedExpectedBranchName = inspection.expectedBranchName; try { - const coherence = await ensureGitWorktreeBranchCoherent({ + const coherence = await measureSandboxOperation("heartbeat.ensure_git_worktree_branch_coherent", { operationIndex: 131 }, async () => (ensureGitWorktreeBranchCoherent({ db, - repoRoot: inspection.repoRoot, + repoRoot, worktreePath: inspection.worktreePath, expectedBranchName: inspection.expectedBranchName, actualBranchName: inspection.actualBranchName, @@ -21195,7 +21216,7 @@ export function heartbeatService( persistForwardReconcile: false, reconcileOperationPhase: "workspace_finalize", recorder: workspaceOperationRecorder, - }); + }))); if ( coherence.branchName && coherence.branchName !== @@ -21221,7 +21242,7 @@ export function heartbeatService( ? repairErr.message : String(repairErr), }; - await workspaceOperationRecorder.recordOperation({ + await measureSandboxOperation("heartbeat.workspace_operation_recorder.record_operation", { operationIndex: 132 }, async () => (workspaceOperationRecorder.recordOperation({ phase: "workspace_finalize", cwd: executionWorkspace.cwd, metadata: { @@ -21244,17 +21265,17 @@ export function heartbeatService( status: "failed", stderr: `Managed git worktree branch check failed: ${repairErr instanceof Error ? repairErr.message : String(repairErr)}\n`, }), - }); + }))); adapterFinalizeOutcome = "failed"; throw repairErr; } const repairedInspection = - await inspectManagedGitWorktreeBranch({ + await measureSandboxOperation("heartbeat.inspect_managed_git_worktree_branch", { operationIndex: 133 }, async () => (inspectManagedGitWorktreeBranch({ worktreePath: inspection.worktreePath, expectedBranchName: repairedExpectedBranchName, repoRoot: inspection.repoRoot, - }); + }))); finalizeBranchRepairMetadata = { attempted: true, succeeded: repairedInspection.valid, @@ -21280,7 +21301,7 @@ export function heartbeatService( executionWorkspaceId: branchInspection.workspaceRecord.id, inspection: managedGitWorktreeBranch, }); - await workspaceOperationRecorder.recordOperation({ + await measureSandboxOperation("heartbeat.workspace_operation_recorder.record_operation", { operationIndex: 134 }, async () => (workspaceOperationRecorder.recordOperation({ phase: "workspace_finalize", cwd: executionWorkspace.cwd, metadata: { @@ -21299,7 +21320,7 @@ export function heartbeatService( status: "failed", stderr: `Managed git worktree branch check failed: ${inspection.reason ?? "unknown branch mismatch"}\n`, }), - }); + }))); adapterFinalizeOutcome = "failed"; throw new WorkspaceValidationFailure( `Execution workspace ${branchInspection.workspaceRecord.id} expected git worktree branch "${inspection.expectedBranchName}" at "${inspection.worktreePath}", but ${inspection.reason ?? "the checked-out branch could not be verified"}. Record a sanctioned execution-workspace branch transition or restore the workspace branch before completing the run.`, @@ -21320,7 +21341,7 @@ export function heartbeatService( } } } - await workspaceOperationRecorder.recordOperation({ + await measureSandboxOperation("heartbeat.workspace_operation_recorder.record_operation", { operationIndex: 135 }, async () => (workspaceOperationRecorder.recordOperation({ phase: "workspace_finalize", cwd: executionWorkspace.cwd, metadata: { @@ -21338,7 +21359,7 @@ export function heartbeatService( : {}), }, run: async () => ({ status }), - }); + }))); // Only mark the outcome after the row landed, so a transient write // failure on the succeeded path can still be recovered by recording // finalize=failed from the catch path below. @@ -21355,7 +21376,7 @@ export function heartbeatService( nativeExecution.runtimeContext.mcp.bindingId ? nativeExecution.runtimeContext.mcp.digest : null; - const nativeMcpServers = await buildPaperclipRuntimeMcpServers({ + const nativeMcpServers = await measureSandboxOperation("heartbeat.build_paperclip_runtime_mcp_servers", { operationIndex: 136 }, async () => (buildPaperclipRuntimeMcpServers({ db, agent, runId: run.id, @@ -21364,12 +21385,12 @@ export function heartbeatService( const names = connections .map((connection) => connection.name) .join(", "); - await onLog( + await measureSandboxOperation("heartbeat.on_log", { operationIndex: 137 }, async () => (onLog( "stderr", `[paperclip] App connection${connections.length === 1 ? "" : "s"} unavailable: ${names}. Continuing this run without ${connections.length === 1 ? "it" : "them"}; reconnect from Apps to restore access.\n`, - ); + ))); }, - }); + }))); if ("runtimeContext" in nativeExecution) { if (nativeMcpServers.length > 1) throw new Error( @@ -21440,21 +21461,21 @@ export function heartbeatService( let nativeGitHubBridge: Awaited> = null; if (executionTarget?.kind === "remote" && adapterEnv.PAPERCLIP_GITHUB_BROKER_TOKEN) { try { - nativeGitHubBridge = await startNativeGitHubCallbackBridge({ + nativeGitHubBridge = await measureSandboxOperation("heartbeat.start_native_git_hub_callback_bridge", { operationIndex: 138 }, async () => (startNativeGitHubCallbackBridge({ runId: run.id, target: executionTarget, hostApiToken: adapterEnv.PAPERCLIP_GITHUB_BROKER_TOKEN, // Forward inside this API process. The public tenant origin // requires a browser session and rejects runtime capabilities. onLog, - }); + }))); } catch { - await onLog("stderr", "[paperclip] GitHub runtime transport unavailable; continuing without managed GitHub access.\n"); + await measureSandboxOperation("heartbeat.on_log", { operationIndex: 139 }, async () => (onLog("stderr", "[paperclip] GitHub runtime transport unavailable; continuing without managed GitHub access.\n"))); } } try { const guardedDispatch = - await dispatchResolvedInteractionContinuationWithAtomicGate( + await measureSandboxOperation("heartbeat.dispatch_resolved_interaction_continuation_with_atomic_gate", { operationIndex: 140 }, async () => (dispatchResolvedInteractionContinuationWithAtomicGate( (markDispatchStarted) => executePaperclipNativeSession({ db, @@ -21522,14 +21543,14 @@ export function heartbeatService( enqueueWakeup, onSpawn: async (meta) => { markDispatchStarted(); - await persistRunProcessMetadata(run.id, meta); + await measureSandboxOperation("heartbeat.persist_run_process_metadata", { operationIndex: 141 }, async () => (persistRunProcessMetadata(run.id, meta))); }, }), - ); + ))); if (!guardedDispatch.dispatched) return; - adapterResult = await guardedDispatch.resultPromise; + adapterResult = await measureSandboxOperation("heartbeat.guarded_dispatch.result_promise", { operationIndex: 142 }, async () => (guardedDispatch.resultPromise)); } finally { - await nativeGitHubBridge?.stop(); + await measureSandboxOperation("heartbeat.native_git_hub_bridge.stop", { operationIndex: 143 }, async () => (nativeGitHubBridge?.stop())); } } else { const interactionId = readNonEmptyString(context.interactionId); @@ -21539,14 +21560,14 @@ export function heartbeatService( readNonEmptyString(context.interactionKind) === "ask_user_questions" && readNonEmptyString(context.interactionStatus) === "answered" - ? await materializeLegacyQuestionResponseWakeProjection({ + ? await measureSandboxOperation("heartbeat.materialize_legacy_question_response_wake_projection", { operationIndex: 144 }, async () => (materializeLegacyQuestionResponseWakeProjection({ db, companyId: agent.companyId, issueId: issueRef.id, runId: run.id, agentId: agent.id, interactionId, - }) + }))) : null; // Do not write the answer projection back to `context`: legacy // adapters need it in their prompt, but the authoritative answers @@ -21579,11 +21600,11 @@ export function heartbeatService( "runtime connection tools could not be delivered", ); } - const runtimeMcpServers = await buildPaperclipRuntimeMcpServers({ + const runtimeMcpServers = await measureSandboxOperation("heartbeat.build_paperclip_runtime_mcp_servers", { operationIndex: 145 }, async () => (buildPaperclipRuntimeMcpServers({ db, agent, runId: run.id, - }); + }))); const runtimeToolDelivery = adapter.runtimeToolDelivery ?? "invocation_context"; if (runtimeTools && runtimeToolDelivery === "native_mcp") { @@ -21598,19 +21619,19 @@ export function heartbeatService( if (runtimeTools && runtimeToolDelivery === "invocation_context") { adapterContext.paperclipRuntimeTools = runtimeTools; } - const managedMcpConfig = await createManagedMcpRunConfig({ + const managedMcpConfig = await measureSandboxOperation("heartbeat.create_managed_mcp_run_config", { operationIndex: 146 }, async () => (createManagedMcpRunConfig({ db, agent, runId: run.id, config: runtimeConfig, projectId: issueRef?.projectId ?? null, issueId: issueRef?.id ?? null, - }); + }))); if (managedMcpConfig) { adapterContext.paperclipManagedMcp = managedMcpConfig; } const guardedDispatch = - await dispatchResolvedInteractionContinuationWithAtomicGate( + await measureSandboxOperation("heartbeat.dispatch_resolved_interaction_continuation_with_atomic_gate", { operationIndex: 147 }, async () => (dispatchResolvedInteractionContinuationWithAtomicGate( (markDispatchStarted) => adapter.execute({ runId: run.id, @@ -21636,16 +21657,16 @@ export function heartbeatService( onEvent: onAdapterEvent, startupTraceContext: getStartupTraceContext(), onRuntimeProgress: async (progress) => { - await recordCurrentHeartbeatRunRuntimeProgress( + await measureSandboxOperation("heartbeat.record_current_heartbeat_run_runtime_progress", { operationIndex: 148 }, async () => (recordCurrentHeartbeatRunRuntimeProgress( run, progress, issueId, - ); + ))); }, onDispatch: markDispatchStarted, onSpawn: async (meta) => { markDispatchStarted(); - await persistRunProcessMetadata(run.id, { + await measureSandboxOperation("heartbeat.persist_run_process_metadata", { operationIndex: 149 }, async () => (persistRunProcessMetadata(run.id, { pid: meta.pid, processGroupId: "processGroupId" in meta && @@ -21653,13 +21674,13 @@ export function heartbeatService( ? meta.processGroupId : null, startedAt: meta.startedAt, - }); + }))); }, authToken: authToken ?? undefined, }), - ); + ))); if (!guardedDispatch.dispatched) return; - adapterResult = await guardedDispatch.resultPromise; + adapterResult = await measureSandboxOperation("heartbeat.guarded_dispatch.result_promise", { operationIndex: 150 }, async () => (guardedDispatch.resultPromise)); } // Adapter returned cleanly, which means its workspace-restore finally // block also ran without throwing. Record the workspace_finalize @@ -21684,7 +21705,7 @@ export function heartbeatService( previousDisplayId: runtimeForAdapter.sessionDisplayId, previousLegacySessionId: runtimeForAdapter.sessionId, }); - await upsertTaskSession({ + await measureSandboxOperation("heartbeat.upsert_task_session", { operationIndex: 151 }, async () => (upsertTaskSession({ companyId: agent.companyId, agentId: agent.id, adapterType: agent.adapterType, @@ -21695,34 +21716,36 @@ export function heartbeatService( sessionDisplayId: sessionState.displayId, lastRunId: run.id, lastError: adapterResult.errorMessage ?? null, - }); + }))); nativeTaskSessionPersisted = true; }; } workFolderSaveFailed = true; - await sandboxWorkFolders.stop(beforeWorkFolderCompletion); + const completedWorkFolders = sandboxWorkFolders; + await measureSandboxOperation("heartbeat.sandbox_work_folders.stop", { operationIndex: 152 }, async () => (completedWorkFolders.stop(beforeWorkFolderCompletion))); sandboxWorkFolders = null; workFolderSaveFailed = false; } if (nativeWorkspaceSync) { - await nativeWorkspaceSync.restoreWorkspace(); + const workspaceSync = nativeWorkspaceSync; + await measureSandboxOperation("heartbeat.native_workspace_sync.restore_workspace", { operationIndex: 153 }, async () => (workspaceSync.restoreWorkspace())); } - await recordWorkspaceFinalize("succeeded"); + await measureSandboxOperation("heartbeat.record_workspace_finalize", { operationIndex: 154 }, async () => (recordWorkspaceFinalize("succeeded"))); if (adapterResult.nativeFinalization) { adapterResult.nativeFinalization.workspaceFinalizeStatus = "succeeded"; try { - const finalized = await finalizeNativeRun({ + const finalized = await measureSandboxOperation("heartbeat.finalize_native_run", { operationIndex: 155 }, async () => (finalizeNativeRun({ db, runId: run.id, workspaceFinalizeStatus: "succeeded", preserveProviderAttempt: Boolean(nativeWorkspaceSync), - }); - await dispatchPendingNativeStatusWakeups({ + }))); + await measureSandboxOperation("heartbeat.dispatch_pending_native_status_wakeups", { operationIndex: 156 }, async () => (dispatchPendingNativeStatusWakeups({ companyId: run.companyId, - }); + }))); if (finalized.phase === "committed") { - await nativeWorkspaceSync?.cleanup(); + await measureSandboxOperation("heartbeat.native_workspace_sync.cleanup", { operationIndex: 157 }, async () => (nativeWorkspaceSync?.cleanup())); } } catch (finalizeErr) { logger.warn( @@ -21734,7 +21757,7 @@ export function heartbeatService( } catch (adapterErr) { const nativeResumeScheduled = nativeRuntimeResolution.kind === "native" - ? await db + ? await measureSandboxOperation("heartbeat.db.select.from.where.limit.then", { operationIndex: 158 }, async () => (db .select({ phase: nativeRunFinalizations.phase, resultId: nativeRunFinalizations.resultId, @@ -21746,7 +21769,7 @@ export function heartbeatService( (rows) => rows[0]?.phase === "retryable_failure" && rows[0]?.resultId === null, - ) + ))) : false; if (nativeResumeScheduled) { nativeSessionResumeScheduled = true; @@ -21758,12 +21781,12 @@ export function heartbeatService( // check keeps the gate closed instead of waking on stale local state, // and surface the original error to the caller. try { - await recordWorkspaceFinalize("failed", { + await measureSandboxOperation("heartbeat.record_workspace_finalize", { operationIndex: 159 }, async () => (recordWorkspaceFinalize("failed", { errorMessage: adapterErr instanceof Error ? adapterErr.message : String(adapterErr), - }); + }))); } catch (recordErr) { logger.warn( { @@ -21775,12 +21798,12 @@ export function heartbeatService( ); } if (nativeRuntimeResolution.kind === "native") { - const proposedResult = await db + const proposedResult = await measureSandboxOperation("heartbeat.db.select.from.where.limit.then", { operationIndex: 160 }, async () => (db .select({ resultId: nativeRunFinalizations.resultId }) .from(nativeRunFinalizations) .where(eq(nativeRunFinalizations.runId, run.id)) .limit(1) - .then((rows) => rows[0]?.resultId ?? null); + .then((rows) => rows[0]?.resultId ?? null))); if (proposedResult && nativeWorkspaceSync) { const workspaceFailureMessage = adapterErr instanceof Error ? adapterErr.message : ""; @@ -21788,7 +21811,7 @@ export function heartbeatService( workspaceFailureMessage === "workspace_sync_out_unrecoverable" || workspaceFailureMessage.includes("daytona_sandbox_not_found"); - const failure = await recordNativeFinalizationFailure({ + const failure = await measureSandboxOperation("heartbeat.record_native_finalization_failure", { operationIndex: 161 }, async () => (recordNativeFinalizationFailure({ db, runId: run.id, error: new Error( @@ -21799,7 +21822,7 @@ export function heartbeatService( projectRunStatus: true, failureScope: "workspace", permanent: unrecoverable, - }); + }))); nativeWorkspaceFinalizeScheduled = true; throw new NativeWorkspaceFinalizeScheduledError( adapterErr, @@ -21810,14 +21833,14 @@ export function heartbeatService( ); } try { - await finalizeNativeRun({ + await measureSandboxOperation("heartbeat.finalize_native_run", { operationIndex: 162 }, async () => (finalizeNativeRun({ db, runId: run.id, workspaceFinalizeStatus: "failed", - }); - await dispatchPendingNativeStatusWakeups({ + }))); + await measureSandboxOperation("heartbeat.dispatch_pending_native_status_wakeups", { operationIndex: 163 }, async () => (dispatchPendingNativeStatusWakeups({ companyId: run.companyId, - }); + }))); } catch (finalizeErr) { logger.warn( { err: finalizeErr, runId: run.id }, @@ -21828,11 +21851,11 @@ export function heartbeatService( throw adapterErr; } finally { try { - await revokeHeartbeatRunGatewayTokens({ + await measureSandboxOperation("heartbeat.revoke_heartbeat_run_gateway_tokens", { operationIndex: 164 }, async () => (revokeHeartbeatRunGatewayTokens({ db, companyId: agent.companyId, runId: run.id, - }); + }))); } catch (revokeErr) { logger.warn( { err: revokeErr, runId: run.id, companyId: agent.companyId }, @@ -21876,8 +21899,9 @@ export function heartbeatService( "run referenced-project remote staging", ); } - const adapterManagedRuntimeServices = adapterResult.runtimeServices - ? await persistAdapterManagedRuntimeServices({ + const reportedRuntimeServices = adapterResult.runtimeServices; + const adapterManagedRuntimeServices = reportedRuntimeServices + ? await measureSandboxOperation("heartbeat.persist_adapter_managed_runtime_services", { operationIndex: 165 }, async () => (persistAdapterManagedRuntimeServices({ db, adapterType: agent.adapterType, runId: run.id, @@ -21888,8 +21912,8 @@ export function heartbeatService( }, issue: issueRef, workspace: executionWorkspace, - reports: adapterResult.runtimeServices, - }) + reports: reportedRuntimeServices, + }))) : []; if (adapterManagedRuntimeServices.length > 0) { const combinedRuntimeServices = [ @@ -21901,33 +21925,33 @@ export function heartbeatService( combinedRuntimeServices.find((service) => readNonEmptyString(service.url), )?.url ?? null; - await db + await measureSandboxOperation("heartbeat.db.update.set.where", { operationIndex: 166 }, async () => (db .update(heartbeatRuns) .set({ contextSnapshot: context, updatedAt: new Date(), }) - .where(eq(heartbeatRuns.id, run.id)); + .where(eq(heartbeatRuns.id, run.id)))); if (issueId) { try { - await postWorkspaceReadyComment({ + await measureSandboxOperation("heartbeat.post_workspace_ready_comment", { operationIndex: 167 }, async () => (postWorkspaceReadyComment({ issuesSvc, issueId, agentId: agent.id, runId: run.id, workspace: executionWorkspace, runtimeServices: adapterManagedRuntimeServices, - }); + }))); } catch (err) { - await onLog( + await measureSandboxOperation("heartbeat.on_log", { operationIndex: 168 }, async () => (onLog( "stderr", `[paperclip] Failed to post adapter-managed runtime comment: ${err instanceof Error ? err.message : String(err)}\n`, - ); + ))); } } } let outcome: RunSessionOutcome; - const latestRun = await getRun(run.id); + const latestRun = await measureSandboxOperation("heartbeat.get_run", { operationIndex: 169 }, async () => (getRun(run.id))); if (isHeartbeatRunTerminalStatus(latestRun?.status)) { outcome = latestRun.status; } else if (adapterResult.nativeFinalization) { @@ -21960,14 +21984,14 @@ export function heartbeatService( previousLegacySessionId: runtimeForAdapter.sessionId, }); const rawUsage = normalizeUsageTotals(adapterResult.usage); - const sessionUsageResolution = await resolveNormalizedUsageForSession({ + const sessionUsageResolution = await measureSandboxOperation("heartbeat.resolve_normalized_usage_for_session", { operationIndex: 170 }, async () => (resolveNormalizedUsageForSession({ agentId: agent.id, runId: run.id, sessionId: nextSessionState.displayId ?? nextSessionState.legacySessionId, rawUsage, usageBasis: adapterResult.usageBasis ?? null, - }); + }))); const normalizedUsage = sessionUsageResolution.normalizedUsage; const runErrorMessage = outcome === "cancelled" @@ -21998,17 +22022,18 @@ export function heartbeatService( compressed: boolean; } | null = null; if (handle) { - logSummary = await runLogStore.finalize(handle); + const finalizeHandle = handle; + logSummary = await measureSandboxOperation("heartbeat.run_log_store.finalize", { operationIndex: 171 }, async () => (runLogStore.finalize(finalizeHandle))); } const finalLogBytes = logSummary?.bytes; if (outputProgressState.pending && typeof finalLogBytes === "number") { outputProgressState.pending.bytes = finalLogBytes; } - await flushOutputProgress({ force: true }); + await measureSandboxOperation("heartbeat.flush_output_progress", { operationIndex: 172 }, async () => (flushOutputProgress({ force: true }))); if (providerTraceCapture) { try { - await traceStore.finalize(run.id, run.companyId); + await measureSandboxOperation("heartbeat.trace_store.finalize", { operationIndex: 173 }, async () => (traceStore.finalize(run.id, run.companyId))); providerTraceFinalized = true; } catch (error) { logger.warn( @@ -22121,11 +22146,11 @@ export function heartbeatService( logSha256: logSummary?.sha256, logCompressed: logSummary?.compressed ?? false, }; - const persistedRunWrite = await setRunStatusIfRunning( + const persistedRunWrite = await measureSandboxOperation("heartbeat.set_run_status_if_running", { operationIndex: 174 }, async () => (setRunStatusIfRunning( run.id, status, finalRunPatch, - ); + ))); let persistedRun: typeof heartbeatRuns.$inferSelect | null = persistedRunWrite.run; if (!persistedRunWrite.updated) { @@ -22141,7 +22166,7 @@ export function heartbeatService( adapterResult.nativeFinalization && persistedRunWrite.run?.status === status ) { - persistedRun = await db + persistedRun = await measureSandboxOperation("heartbeat.db.update.set.where.returning.then", { operationIndex: 175 }, async () => (db .update(heartbeatRuns) .set({ ...finalRunPatch, @@ -22156,7 +22181,7 @@ export function heartbeatService( ), ) .returning() - .then((rows) => rows[0] ?? null); + .then((rows) => rows[0] ?? null))); } if (!persistedRun) { logger.info( @@ -22171,25 +22196,26 @@ export function heartbeatService( } } if (persistedRun) { + const runToClassify = persistedRun; persistedRun = - (await classifyAndPersistRunLiveness( - persistedRun, + (await measureSandboxOperation("heartbeat.classify_and_persist_run_liveness", { operationIndex: 176 }, async () => (classifyAndPersistRunLiveness( + runToClassify, persistedResultJson, - )) ?? persistedRun; + )))) ?? persistedRun; } - await setWakeupStatus( + await measureSandboxOperation("heartbeat.set_wakeup_status", { operationIndex: 177 }, async () => (setWakeupStatus( run.wakeupRequestId, outcome === "succeeded" ? "completed" : status, { finishedAt: new Date(), error: runErrorMessage, }, - ); + ))); - const finalizedRun = persistedRun ?? (await getRun(run.id)); + const finalizedRun = persistedRun ?? (await measureSandboxOperation("heartbeat.get_run", { operationIndex: 178 }, async () => (getRun(run.id)))); if (finalizedRun) { - await appendRunEvent(finalizedRun, { + await measureSandboxOperation("heartbeat.append_run_event", { operationIndex: 179 }, async () => (appendRunEvent(finalizedRun, { eventType: "lifecycle", stream: "system", level: outcome === "succeeded" ? "info" : "error", @@ -22198,45 +22224,45 @@ export function heartbeatService( status, exitCode: adapterResult.exitCode, }, - }); + }))); try { - await completeSkillTestRunForHeartbeatOutcome({ + await measureSandboxOperation("heartbeat.complete_skill_test_run_for_heartbeat_outcome", { operationIndex: 180 }, async () => (completeSkillTestRunForHeartbeatOutcome({ run: finalizedRun, issueId, issueWorkMode: issueRef?.workMode ?? null, outcome, error: runErrorMessage, - }); + }))); } catch (err) { logger.warn( { err, runId: finalizedRun.id, issueId }, "failed to complete skill test run after heartbeat finalization", ); - await onLog( + await measureSandboxOperation("heartbeat.on_log", { operationIndex: 181 }, async () => (onLog( "stderr", `[paperclip] Failed to complete skill test run: ${err instanceof Error ? err.message : String(err)}\n`, - ); + ))); } const livenessRun = finalizedRun; - await refreshContinuationSummaryForRun(livenessRun, agent); + await measureSandboxOperation("heartbeat.refresh_continuation_summary_for_run", { operationIndex: 182 }, async () => (refreshContinuationSummaryForRun(livenessRun, agent))); const skipRunIssueComment = parseObject(livenessRun.contextSnapshot).skipIssueComment === true; let resolvedPresentationDecision: RunPresentationDecision | null = null; try { const existingRunComment = issueId - ? await findRunIssueComment( + ? await measureSandboxOperation("heartbeat.find_run_issue_comment", { operationIndex: 183 }, async () => (findRunIssueComment( livenessRun.id, livenessRun.companyId, issueId, persistedResultJson, - ) + ))) : null; const finalAgentMessage = - await findLatestCompletedFinalAgentMessage( + await measureSandboxOperation("heartbeat.find_latest_completed_final_agent_message", { operationIndex: 184 }, async () => (findLatestCompletedFinalAgentMessage( livenessRun.id, livenessRun.companyId, - ); + ))); const resolved = resolveHeartbeatRunResponse({ resultJson: persistedResultJson, existingComment: existingRunComment, @@ -22251,11 +22277,12 @@ export function heartbeatService( presentationDecision.commentAction === "create" && resolved.text ) { - const comment = await issuesSvc.addComment( + const resolvedText = resolved.text; + const comment = await measureSandboxOperation("heartbeat.issues_svc.add_comment", { operationIndex: 185 }, async () => (issuesSvc.addComment( issueId, - resolved.text, + resolvedText, { agentId: agent.id, runId: livenessRun.id }, - ); + ))); presentationDecision = { ...presentationDecision, commentId: comment.id, @@ -22264,7 +22291,7 @@ export function heartbeatService( "resolved_response_materialized", ], }; - await logActivity(db, { + await measureSandboxOperation("heartbeat.log_activity", { operationIndex: 186 }, async () => (logActivity(db, { companyId: livenessRun.companyId, actorType: "agent", actorId: agent.id, @@ -22283,7 +22310,7 @@ export function heartbeatService( source: "run_presentation_resolver", presentationSource: presentationDecision.chosenSource, }, - }); + }))); } else if (presentationDecision.commentAction === "create") { presentationDecision = { ...presentationDecision, @@ -22297,7 +22324,7 @@ export function heartbeatService( }; } - await db + await measureSandboxOperation("heartbeat.db.update.set.where", { operationIndex: 187 }, async () => (db .update(heartbeatRuns) .set({ resultJson: { @@ -22306,32 +22333,32 @@ export function heartbeatService( }, updatedAt: new Date(), }) - .where(eq(heartbeatRuns.id, livenessRun.id)); - await appendRunEvent(livenessRun, { + .where(eq(heartbeatRuns.id, livenessRun.id)))); + await measureSandboxOperation("heartbeat.append_run_event", { operationIndex: 188 }, async () => (appendRunEvent(livenessRun, { eventType: "run.presentation.resolved", stream: "system", level: "info", message: "run presentation resolved", payload: { presentationDecision }, - }); + }))); resolvedPresentationDecision = presentationDecision; } catch (err) { - await onLog( + await measureSandboxOperation("heartbeat.on_log", { operationIndex: 189 }, async () => (onLog( "stderr", `[paperclip] Failed to resolve run presentation: ${err instanceof Error ? err.message : String(err)}\n`, - ); + ))); } if (outcome === "failed" && isMaxTurnExhaustionRun(livenessRun)) { const policy = parseMaxTurnContinuationPolicy(agent); if (policy.enabled && policy.maxAttempts > 0) { - await scheduleBoundedRetryForRun(livenessRun, agent, { + await measureSandboxOperation("heartbeat.schedule_bounded_retry_for_run", { operationIndex: 190 }, async () => (scheduleBoundedRetryForRun(livenessRun, agent, { retryReason: MAX_TURN_CONTINUATION_RETRY_REASON, wakeReason: MAX_TURN_CONTINUATION_WAKE_REASON, maxAttempts: policy.maxAttempts, delayMs: policy.delayMs, - }); + }))); } else { - await appendRunEvent(livenessRun, { + await measureSandboxOperation("heartbeat.append_run_event", { operationIndex: 191 }, async () => (appendRunEvent(livenessRun, { eventType: "lifecycle", stream: "system", level: "warn", @@ -22341,23 +22368,23 @@ export function heartbeatService( retryReason: MAX_TURN_CONTINUATION_RETRY_REASON, policy, }, - }); + }))); } } else if ( outcome === "failed" && readTransientRecoveryContractFromRun(livenessRun) ) { - await scheduleBoundedRetryForRun(livenessRun, agent); + await measureSandboxOperation("heartbeat.schedule_bounded_retry_for_run", { operationIndex: 192 }, async () => (scheduleBoundedRetryForRun(livenessRun, agent))); } - const issueCommentPolicyResult = await finalizeIssueCommentPolicy( + const issueCommentPolicyResult = await measureSandboxOperation("heartbeat.finalize_issue_comment_policy", { operationIndex: 193 }, async () => (finalizeIssueCommentPolicy( livenessRun, agent, resolvedPresentationDecision, - ); - await releaseIssueExecutionAndPromote(livenessRun); - await handleRunLivenessContinuation(livenessRun); - await handleIssueReviewPathDisposition(livenessRun); - await handleSuccessfulRunHandoff( + ))); + await measureSandboxOperation("heartbeat.release_issue_execution_and_promote", { operationIndex: 194 }, async () => (releaseIssueExecutionAndPromote(livenessRun))); + await measureSandboxOperation("heartbeat.handle_run_liveness_continuation", { operationIndex: 195 }, async () => (handleRunLivenessContinuation(livenessRun))); + await measureSandboxOperation("heartbeat.handle_issue_review_path_disposition", { operationIndex: 196 }, async () => (handleIssueReviewPathDisposition(livenessRun))); + await measureSandboxOperation("heartbeat.handle_successful_run_handoff", { operationIndex: 197 }, async () => (handleSuccessfulRunHandoff( issueCommentPolicyResult.outcome === "retry_queued" || issueCommentPolicyResult.outcome === "retry_exhausted" ? { @@ -22366,7 +22393,7 @@ export function heartbeatService( } : livenessRun, agent, - ); + ))); // Dependency wake re-check: if this run's issue was marked done mid-run, // the route-time `issue_blockers_resolved` wake may have been gated by @@ -22375,18 +22402,18 @@ export function heartbeatService( // readiness, active-path, and observability rules. if (issueId && finalizedRun) { try { - const blockerIssueStatus = await db + const blockerIssueStatus = await measureSandboxOperation("heartbeat.db.select.from.where.then", { operationIndex: 198 }, async () => (db .select({ status: issues.status }) .from(issues) .where(eq(issues.id, issueId)) - .then((rows) => rows[0]?.status ?? null); + .then((rows) => rows[0]?.status ?? null))); if (blockerIssueStatus === "done") { - await recovery.reconcileResolvedDependencyWakeBackstop({ + await measureSandboxOperation("heartbeat.recovery.reconcile_resolved_dependency_wake_backstop", { operationIndex: 199 }, async () => (recovery.reconcileResolvedDependencyWakeBackstop({ runId: finalizedRun.id, companyId: finalizedRun.companyId, blockerIssueId: issueId, source: "workspace.finalize", - }); + }))); } } catch (finalizeWakeErr) { logger.warn( @@ -22398,7 +22425,7 @@ export function heartbeatService( } if (finalizedRun) { - await updateRuntimeState( + await measureSandboxOperation("heartbeat.update_runtime_state", { operationIndex: 200 }, async () => (updateRuntimeState( agent, finalizedRun, adapterResult, @@ -22406,18 +22433,18 @@ export function heartbeatService( legacySessionId: nextSessionState.legacySessionId, }, normalizedUsage, - ); + ))); if (taskKey && !nativeTaskSessionPersisted) { if ( adapterResult.clearSession || (!nextSessionState.params && !nextSessionState.displayId) ) { - await clearTaskSessions(agent.companyId, agent.id, { + await measureSandboxOperation("heartbeat.clear_task_sessions", { operationIndex: 201 }, async () => (clearTaskSessions(agent.companyId, agent.id, { taskKey, adapterType: agent.adapterType, - }); + }))); } else { - await upsertTaskSession({ + await measureSandboxOperation("heartbeat.upsert_task_session", { operationIndex: 202 }, async () => (upsertTaskSession({ companyId: agent.companyId, agentId: agent.id, adapterType: agent.adapterType, @@ -22431,11 +22458,11 @@ export function heartbeatService( sessionDisplayId: nextSessionState.displayId, lastRunId: finalizedRun.id, lastError: runErrorMessage, - }); + }))); } } } - await finalizeAgentStatus(agent.id, outcome, runErrorMessage, { + await measureSandboxOperation("heartbeat.finalize_agent_status", { operationIndex: 203 }, async () => (finalizeAgentStatus(agent.id, outcome, runErrorMessage, { keepIdleOnFailure: outcome === "failed" && ((finalizedRun @@ -22443,13 +22470,13 @@ export function heartbeatService( : runErrorCode === "provider_quota") || isWorkspaceSyncConflictFailure(adapterResult.errorMessage)), wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run), - }); + }))); } catch (err) { if (err instanceof NativeCancellationPendingRecoveryError) { - await cancelRunInternal( + await measureSandboxOperation("heartbeat.cancel_run_internal", { operationIndex: 204 }, async () => (cancelRunInternal( run.id, "Recovered durable native run cancellation", - ); + ))); return; } if (err instanceof NativeSessionResumeScheduledError) { @@ -22462,7 +22489,7 @@ export function heartbeatService( ) ? "semantic_result_missing" : "native_session_interrupted"; - const coordinator = await db + const coordinator = await measureSandboxOperation("heartbeat.db.select.from.where.limit.then", { operationIndex: 205 }, async () => (db .select({ nextAttemptAt: nativeRunFinalizations.nextAttemptAt, attempt: nativeRunFinalizations.attempt, @@ -22471,8 +22498,8 @@ export function heartbeatService( .from(nativeRunFinalizations) .where(eq(nativeRunFinalizations.runId, run.id)) .limit(1) - .then((rows) => rows[0] ?? null); - await appendRunEvent(run, { + .then((rows) => rows[0] ?? null))); + await measureSandboxOperation("heartbeat.append_run_event.catch", { operationIndex: 206 }, async () => (appendRunEvent(run, { eventType: "lifecycle", stream: "system", level: "warn", @@ -22492,7 +22519,7 @@ export function heartbeatService( // why a live warm runner was replaced. failureDetail: coordinator?.failureDetail ?? null, }, - }).catch(() => undefined); + }).catch(() => undefined))); if (coordinator?.nextAttemptAt) { scheduleNativeSessionResumeDispatch( run.id, @@ -22502,7 +22529,7 @@ export function heartbeatService( return; } if (err instanceof NativeWorkspaceFinalizeScheduledError) { - const coordinator = await db + const coordinator = await measureSandboxOperation("heartbeat.db.select.from.where.limit.then", { operationIndex: 207 }, async () => (db .select({ nextAttemptAt: nativeRunFinalizations.nextAttemptAt, attempt: nativeRunFinalizations.attempt, @@ -22510,8 +22537,8 @@ export function heartbeatService( .from(nativeRunFinalizations) .where(eq(nativeRunFinalizations.runId, run.id)) .limit(1) - .then((rows) => rows[0] ?? null); - await appendRunEvent(run, { + .then((rows) => rows[0] ?? null))); + await measureSandboxOperation("heartbeat.append_run_event.catch", { operationIndex: 208 }, async () => (appendRunEvent(run, { eventType: "lifecycle", stream: "system", level: err.terminalFailure ? "error" : "warn", @@ -22524,25 +22551,25 @@ export function heartbeatService( fallbackSuppressed: true, retryReasonCode: err.reasonCode, }, - }).catch(() => undefined); + }).catch(() => undefined))); if (err.terminalFailure) { // The durable coordinator already failed the run, blocked the // issue, and cleared its execution lock. Let ordinary teardown // release the now-useless lease and return the agent to service. nativeWorkspaceFinalizeScheduled = false; providerResourceDispositionForRun = "stop_and_retain"; - await finalizeAgentStatus( + await measureSandboxOperation("heartbeat.finalize_agent_status.catch", { operationIndex: 209 }, async () => (finalizeAgentStatus( run.agentId, "failed", "native_workspace_sync_out_unrecoverable", { wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run) }, - ).catch(() => undefined); + ).catch(() => undefined))); } return; } const message = redactCurrentUserText( err instanceof Error ? err.message : "Unknown adapter failure", - await getCurrentUserRedactionOptions(), + await measureSandboxOperation("heartbeat.get_current_user_redaction_options", { operationIndex: 210 }, async () => (getCurrentUserRedactionOptions())), ); const workspaceValidationFailure = isWorkspaceValidationFailure(err) ? err @@ -22554,13 +22581,13 @@ export function heartbeatService( : null; const recordedResponsibleUserDenialCode = normalizeResponsibleUserDenialCode( - (await getRun(run.id).catch(() => null))?.errorCode, + (await measureSandboxOperation("heartbeat.get_run.catch", { operationIndex: 211 }, async () => (getRun(run.id).catch(() => null))))?.errorCode, ); // The runtime resolution is scoped to the adapter try block. The // durable coordinator is also the stronger authority here: legacy // runs simply have no row, while native result-less exhaustion keeps // its named failure instead of being flattened to `adapter_failed`. - const nativeTerminalFailureCode = await db + const nativeTerminalFailureCode = await measureSandboxOperation("heartbeat.db.select.from.where.limit.then.catch", { operationIndex: 212 }, async () => (db .select({ phase: nativeRunFinalizations.phase, resultId: nativeRunFinalizations.resultId, @@ -22576,7 +22603,7 @@ export function heartbeatService( ? coordinator.failureCode : null; }) - .catch(() => null); + .catch(() => null))); const failureErrorCode = workspaceValidationFailure?.code ?? configurationIncompleteFailure?.code ?? @@ -22592,7 +22619,8 @@ export function heartbeatService( } | null = null; if (handle) { try { - logSummary = await runLogStore.finalize(handle); + const finalizeHandle = handle; + logSummary = await measureSandboxOperation("heartbeat.run_log_store.finalize", { operationIndex: 213 }, async () => (runLogStore.finalize(finalizeHandle))); } catch (finalizeErr) { logger.warn( { err: finalizeErr, runId }, @@ -22604,14 +22632,14 @@ export function heartbeatService( if (outputProgressState.pending && typeof finalLogBytes === "number") { outputProgressState.pending.bytes = finalLogBytes; } - await flushOutputProgress({ force: true }).catch((flushErr) => { + await measureSandboxOperation("heartbeat.flush_output_progress.catch", { operationIndex: 214 }, async () => (flushOutputProgress({ force: true }).catch((flushErr) => { logger.warn( { err: flushErr, runId }, "failed to flush run output progress after error", ); - }); + }))); - const failedRunWrite = await setRunStatusIfRunning(run.id, "failed", { + const failedRunWrite = await measureSandboxOperation("heartbeat.set_run_status_if_running", { operationIndex: 215 }, async () => (setRunStatusIfRunning(run.id, "failed", { error: message, errorCode: failureErrorCode, finishedAt: new Date(), @@ -22628,7 +22656,7 @@ export function heartbeatService( logBytes: logSummary?.bytes, logSha256: logSummary?.sha256, logCompressed: logSummary?.compressed ?? false, - }); + }))); if (!failedRunWrite.updated) { logger.info( { @@ -22642,55 +22670,55 @@ export function heartbeatService( } const failedRun = failedRunWrite.run; - await setWakeupStatus(run.wakeupRequestId, "failed", { + await measureSandboxOperation("heartbeat.set_wakeup_status", { operationIndex: 216 }, async () => (setWakeupStatus(run.wakeupRequestId, "failed", { finishedAt: new Date(), error: message, - }); + }))); if (failedRun) { - await appendRunEvent(failedRun, { + await measureSandboxOperation("heartbeat.append_run_event", { operationIndex: 217 }, async () => (appendRunEvent(failedRun, { eventType: "error", stream: "system", level: "error", message, - }); + }))); const livenessRun = - (await classifyAndPersistRunLiveness(failedRun)) ?? failedRun; + (await measureSandboxOperation("heartbeat.classify_and_persist_run_liveness", { operationIndex: 218 }, async () => (classifyAndPersistRunLiveness(failedRun)))) ?? failedRun; try { - await completeSkillTestRunForHeartbeatOutcome({ + await measureSandboxOperation("heartbeat.complete_skill_test_run_for_heartbeat_outcome", { operationIndex: 219 }, async () => (completeSkillTestRunForHeartbeatOutcome({ run: livenessRun, issueId, issueWorkMode: issueRef?.workMode ?? null, outcome: "failed", error: message, - }); + }))); } catch (err) { logger.warn( { err, runId: livenessRun.id, issueId }, "failed to complete skill test run after heartbeat adapter failure", ); } - await refreshContinuationSummaryForRun(livenessRun, agent); + await measureSandboxOperation("heartbeat.refresh_continuation_summary_for_run", { operationIndex: 220 }, async () => (refreshContinuationSummaryForRun(livenessRun, agent))); if ( !isWorkspaceValidationFailedRun(livenessRun) && !isConfigurationIncompleteFailedRun(livenessRun) ) { - await finalizeIssueCommentPolicy(livenessRun, agent); + await measureSandboxOperation("heartbeat.finalize_issue_comment_policy", { operationIndex: 221 }, async () => (finalizeIssueCommentPolicy(livenessRun, agent))); } - await scheduleInteractionContinuationInfrastructureRetryIfEligible( + await measureSandboxOperation("heartbeat.schedule_interaction_continuation_infrastructure_retry_if_eligible", { operationIndex: 222 }, async () => (scheduleInteractionContinuationInfrastructureRetryIfEligible( livenessRun, agent, - ); - await releaseIssueExecutionAndPromote(livenessRun, { + ))); + await measureSandboxOperation("heartbeat.release_issue_execution_and_promote", { operationIndex: 223 }, async () => (releaseIssueExecutionAndPromote(livenessRun, { // Native recovery owns the original heartbeat run through // exhaustion. Once its durable coordinator has classified a // terminal failure, generic issue recovery must not create a // replacement retryOfRunId chain for the same provider work. suppressImmediateRecovery: nativeTerminalFailureCode !== null, - }); - await handleIssueReviewPathDisposition(livenessRun); + }))); + await measureSandboxOperation("heartbeat.handle_issue_review_path_disposition", { operationIndex: 224 }, async () => (handleIssueReviewPathDisposition(livenessRun))); - await updateRuntimeState( + await measureSandboxOperation("heartbeat.update_runtime_state", { operationIndex: 225 }, async () => (updateRuntimeState( agent, livenessRun, { @@ -22702,14 +22730,14 @@ export function heartbeatService( { legacySessionId: runtimeForAdapter.sessionId, }, - ); + ))); if ( taskKey && !nativeTaskSessionPersisted && (previousSessionParams || previousSessionDisplayId || taskSession) ) { - await upsertTaskSession({ + await measureSandboxOperation("heartbeat.upsert_task_session", { operationIndex: 226 }, async () => (upsertTaskSession({ companyId: agent.companyId, agentId: agent.id, adapterType: agent.adapterType, @@ -22722,14 +22750,14 @@ export function heartbeatService( sessionDisplayId: previousSessionDisplayId, lastRunId: failedRun.id, lastError: message, - }); + }))); } } - await finalizeAgentStatus(agent.id, "failed", message, { + await measureSandboxOperation("heartbeat.finalize_agent_status", { operationIndex: 227 }, async () => (finalizeAgentStatus(agent.id, "failed", message, { wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run), keepIdleOnFailure: isWorkspaceSyncConflictFailure(message), - }); + }))); } } catch (outerErr) { if (isWorkspaceBusyDeferral(outerErr)) { @@ -22737,14 +22765,14 @@ export function heartbeatService( // failure: park the run as a bounded scheduled retry and leave the // holder undisturbed. The finally block below still releases // leases, runtime services, and scratch for this run. - await finalizeWorkspaceBusyDeferral(run, outerErr).catch( + await measureSandboxOperation("heartbeat.finalize_workspace_busy_deferral.catch", { operationIndex: 228 }, async () => (finalizeWorkspaceBusyDeferral(run, outerErr).catch( (deferralErr) => { logger.error( { err: deferralErr, runId }, "failed to finalize workspace-busy deferral", ); }, - ); + ))); } else { // Setup code before adapter.execute threw (e.g. ensureRuntimeState, resolveWorkspaceForRun). // The inner catch did not fire, so we must record the failure here. @@ -22752,7 +22780,7 @@ export function heartbeatService( outerErr instanceof Error ? outerErr.message : "Unknown setup failure", - await getCurrentUserRedactionOptions(), + await measureSandboxOperation("heartbeat.get_current_user_redaction_options", { operationIndex: 229 }, async () => (getCurrentUserRedactionOptions())), ); // A missing secret/env binding is a known pre-dispatch configuration gap, // not an opaque setup crash. Surface it with its own errorCode so the @@ -22771,7 +22799,7 @@ export function heartbeatService( : null; const recordedResponsibleUserDenialCode = normalizeResponsibleUserDenialCode( - (await getRun(runId).catch(() => null))?.errorCode, + (await measureSandboxOperation("heartbeat.get_run.catch", { operationIndex: 230 }, async () => (getRun(runId).catch(() => null))))?.errorCode, ); const setupFailureErrorCode = workspaceValidationSetupFailure?.code ?? @@ -22785,8 +22813,8 @@ export function heartbeatService( { err: outerErr, runId }, "heartbeat execution setup failed", ); - const setupFailureAgent = await getAgent(run.agentId).catch(() => null); - const setupFailureWrite = await setRunStatusIfRunning(runId, "failed", { + const setupFailureAgent = await measureSandboxOperation("heartbeat.get_agent.catch", { operationIndex: 231 }, async () => (getAgent(run.agentId).catch(() => null))); + const setupFailureWrite = await measureSandboxOperation("heartbeat.set_run_status_if_running.catch", { operationIndex: 232 }, async () => (setRunStatusIfRunning(runId, "failed", { error: message, errorCode: setupFailureErrorCode, finishedAt: new Date(), @@ -22811,7 +22839,7 @@ export function heartbeatService( ), } : {}), - }).catch(() => ({ run: null, updated: false as const })); + }).catch(() => ({ run: null, updated: false as const })))); if (!setupFailureWrite.updated) { logger.info( { @@ -22822,29 +22850,29 @@ export function heartbeatService( "skipping late setup failure finalization because the run already left running state", ); } else { - await setWakeupStatus(run.wakeupRequestId, "failed", { + await measureSandboxOperation("heartbeat.set_wakeup_status.catch", { operationIndex: 233 }, async () => (setWakeupStatus(run.wakeupRequestId, "failed", { finishedAt: new Date(), error: message, - }).catch(() => undefined); + }).catch(() => undefined))); } - const failedRun = await getRun(runId).catch(() => null); + const failedRun = await measureSandboxOperation("heartbeat.get_run.catch", { operationIndex: 234 }, async () => (getRun(runId).catch(() => null))); if (setupFailureWrite.updated && failedRun) { // Emit a run-log event so the failure is visible in the run timeline, // consistent with what the inner catch block does for adapter failures. - await appendRunEvent(failedRun, { + await measureSandboxOperation("heartbeat.append_run_event.catch", { operationIndex: 235 }, async () => (appendRunEvent(failedRun, { eventType: "error", stream: "system", level: "error", message, - }).catch(() => undefined); - const livenessRun = await classifyAndPersistRunLiveness( + }).catch(() => undefined))); + const livenessRun = await measureSandboxOperation("heartbeat.classify_and_persist_run_liveness.catch", { operationIndex: 236 }, async () => (classifyAndPersistRunLiveness( failedRun, - ).catch(() => failedRun); + ).catch(() => failedRun))); const setupFailureIssueId = readNonEmptyString( parseObject(livenessRun.contextSnapshot).issueId, ); if (setupFailureIssueId) { - await completeSkillTestRunForHeartbeatOutcome({ + await measureSandboxOperation("heartbeat.complete_skill_test_run_for_heartbeat_outcome.catch", { operationIndex: 237 }, async () => (completeSkillTestRunForHeartbeatOutcome({ run: livenessRun, issueId: setupFailureIssueId, outcome: "failed", @@ -22858,25 +22886,25 @@ export function heartbeatService( }, "failed to complete skill test run after heartbeat setup failure", ); - }); + }))); } const failedAgent = setupFailureAgent ?? - (await getAgent(run.agentId).catch(() => null)); + (await measureSandboxOperation("heartbeat.get_agent.catch", { operationIndex: 238 }, async () => (getAgent(run.agentId).catch(() => null)))); if (failedAgent) { - await refreshContinuationSummaryForRun( + await measureSandboxOperation("heartbeat.refresh_continuation_summary_for_run.catch", { operationIndex: 239 }, async () => (refreshContinuationSummaryForRun( livenessRun, failedAgent, - ).catch(() => undefined); + ).catch(() => undefined))); if ( !isWorkspaceValidationFailedRun(livenessRun) && !isConfigurationIncompleteFailedRun(livenessRun) ) { - await finalizeIssueCommentPolicy(livenessRun, failedAgent).catch( + await measureSandboxOperation("heartbeat.finalize_issue_comment_policy.catch", { operationIndex: 240 }, async () => (finalizeIssueCommentPolicy(livenessRun, failedAgent).catch( () => undefined, - ); + ))); } - await scheduleInteractionContinuationInfrastructureRetryIfEligible( + await measureSandboxOperation("heartbeat.schedule_interaction_continuation_infrastructure_retry_if_eligible.catch", { operationIndex: 241 }, async () => (scheduleInteractionContinuationInfrastructureRetryIfEligible( livenessRun, failedAgent, ).catch((retryError) => { @@ -22884,40 +22912,41 @@ export function heartbeatService( { err: retryError, runId: livenessRun.id }, "failed to schedule interaction continuation retry after setup failure", ); - }); + }))); } - await releaseIssueExecutionAndPromote(livenessRun).catch( + await measureSandboxOperation("heartbeat.release_issue_execution_and_promote.catch", { operationIndex: 242 }, async () => (releaseIssueExecutionAndPromote(livenessRun).catch( (releaseError) => { logger.error( { err: releaseError, runId }, "failed to release issue execution after heartbeat setup failure", ); }, - ); - await handleIssueReviewPathDisposition(livenessRun).catch( + ))); + await measureSandboxOperation("heartbeat.handle_issue_review_path_disposition.catch", { operationIndex: 243 }, async () => (handleIssueReviewPathDisposition(livenessRun).catch( (reviewPathError) => { logger.error( { err: reviewPathError, runId }, "failed to evaluate review-path disposition after heartbeat setup failure", ); }, - ); + ))); } // Ensure the agent is not left stuck in "running" if the setup-failure // path owned the terminal transition. If another path already finalized // the run, keep that terminal outcome authoritative. if (setupFailureWrite.updated) { - await finalizeAgentStatus(run.agentId, "failed", message, { + await measureSandboxOperation("heartbeat.finalize_agent_status.catch", { operationIndex: 244 }, async () => (finalizeAgentStatus(run.agentId, "failed", message, { wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run), - }).catch(() => undefined); + }).catch(() => undefined))); } } } finally { if (sandboxWorkFolders) { - try { await sandboxWorkFolders.stop(beforeWorkFolderCompletion); workFolderSaveFailed = false; } + const remainingWorkFolders = sandboxWorkFolders; + try { await measureSandboxOperation("heartbeat.sandbox_work_folders.stop", { operationIndex: 245 }, async () => (remainingWorkFolders.stop(beforeWorkFolderCompletion))); workFolderSaveFailed = false; } catch (error) { workFolderSaveFailed = true; logger.error({ err: error, runId: run.id }, "Work folder save failed; retaining sandbox for recovery"); } } - let latestRun = await getRun(run.id).catch(() => null); + let latestRun = await measureSandboxOperation("heartbeat.get_run.catch", { operationIndex: 246 }, async () => (getRun(run.id).catch(() => null))); // Trace capture is debug-only and must settle independently of every // provider outcome. Adapter/setup failures used to skip the success-path // finalizer, leaving metadata permanently stuck at `capturing` even when @@ -22930,7 +22959,7 @@ export function heartbeatService( !nativeSessionResumeScheduled ) { try { - await traceStore.finalize(run.id, run.companyId); + await measureSandboxOperation("heartbeat.trace_store.finalize", { operationIndex: 247 }, async () => (traceStore.finalize(run.id, run.companyId))); providerTraceFinalized = true; } catch (traceFinalizeError) { logger.warn( @@ -22948,7 +22977,8 @@ export function heartbeatService( !nativeSessionResumeScheduled && !nativeWorkspaceFinalizeScheduled ) { - latestRun = await terminalizeRunOnLeaseRelease(latestRun).catch( + const runToTerminalize = latestRun; + latestRun = await measureSandboxOperation("heartbeat.terminalize_run_on_lease_release.catch", { operationIndex: 248 }, async () => (terminalizeRunOnLeaseRelease(runToTerminalize).catch( (terminalizeErr) => { logger.error( { err: terminalizeErr, runId: run.id }, @@ -22956,7 +22986,7 @@ export function heartbeatService( ); return latestRun; }, - ); + ))); } // Warm retention is earned only by a fully successful turn. A failed, // cancelled, or timed-out run stops the reusable sandbox so the next @@ -22970,16 +23000,18 @@ export function heartbeatService( // Keep launchers during same-run recovery. At a terminal boundary all // operations have settled; clean before the remote lease can be stopped. if (githubLauncherLocation && latestRun && isHeartbeatRunTerminalStatus(latestRun.status)) { - await cleanupGitHubOperationLaunchers(githubLauncherLocation).catch((err) => { + const launchersToClean = githubLauncherLocation; + await measureSandboxOperation("heartbeat.cleanup_git_hub_operation_launchers.catch", { operationIndex: 249 }, async () => (cleanupGitHubOperationLaunchers(launchersToClean).catch((err) => { logger.warn({ err, runId: run.id }, "failed to clean managed GitHub launchers"); - }); + }))); } if (workFolderSaveFailed && workFolderLeaseId) { - await retainUnsavedWorkFolderLease(db, { id: workFolderLeaseId, companyId: run.companyId }).catch((error) => { + const unsavedLeaseId = workFolderLeaseId; + await measureSandboxOperation("heartbeat.retain_unsaved_work_folder_lease.catch", { operationIndex: 250 }, async () => (retainUnsavedWorkFolderLease(db, { id: unsavedLeaseId, companyId: run.companyId }).catch((error) => { logger.error({ err: error, runId: run.id }, "Could not record work folder retention; lease remains active"); - }); + }))); } - if (!workFolderSaveFailed) await releaseEnvironmentLeasesForRun({ + if (!workFolderSaveFailed) await measureSandboxOperation("heartbeat.release_environment_leases_for_run", { operationIndex: 251 }, async () => (releaseEnvironmentLeasesForRun({ runId: run.id, companyId: run.companyId, agentId: run.agentId, @@ -22987,8 +23019,8 @@ export function heartbeatService( failureReason: latestRun?.error ?? undefined, providerResourceDisposition: providerResourceDispositionForRun, nativeLifecycleTelemetry: nativeLifecycleTelemetryForRun, - }); - await releaseRuntimeServicesForRun(run.id).catch(() => undefined); + }))); + await measureSandboxOperation("heartbeat.release_runtime_services_for_run.catch", { operationIndex: 252 }, async () => (releaseRuntimeServicesForRun(run.id).catch(() => undefined))); } if ( runScratch && @@ -23000,11 +23032,11 @@ export function heartbeatService( ReturnType > | null = null; try { - scratchCleanup = await cleanupHeartbeatRunScratch({ + scratchCleanup = await measureSandboxOperation("heartbeat.cleanup_heartbeat_run_scratch", { operationIndex: 253 }, async () => (cleanupHeartbeatRunScratch({ scratch: scratchForCleanup, processGroupId: latestRun.processGroupId, isProcessGroupAlive, - }); + }))); } catch (scratchCleanupError) { logger.warn( { @@ -23014,7 +23046,7 @@ export function heartbeatService( }, "failed to clean heartbeat run scratch directory", ); - await appendRunEvent(latestRun, { + await measureSandboxOperation("heartbeat.append_run_event.catch", { operationIndex: 254 }, async () => (appendRunEvent(latestRun, { eventType: "error", stream: "system", level: "warn", @@ -23026,10 +23058,10 @@ export function heartbeatService( ? scratchCleanupError.message : String(scratchCleanupError), }, - }).catch(() => undefined); + }).catch(() => undefined))); } if (scratchCleanup) { - await appendRunEvent(latestRun, { + await measureSandboxOperation("heartbeat.append_run_event.catch", { operationIndex: 255 }, async () => (appendRunEvent(latestRun, { eventType: "lifecycle", stream: "system", level: scratchCleanup.removed ? "info" : "warn", @@ -23046,7 +23078,7 @@ export function heartbeatService( }, "failed to record heartbeat run scratch cleanup event", ); - }); + }))); } } activeRunExecutions.delete(run.id); @@ -23055,7 +23087,7 @@ export function heartbeatService( !nativeWorkspaceFinalizeScheduled && !shutdownInProgress ) { - await startNextQueuedRunForAgent(run.agentId); + await measureSandboxOperation("heartbeat.start_next_queued_run_for_agent", { operationIndex: 256 }, async () => (startNextQueuedRunForAgent(run.agentId))); } } } diff --git a/server/src/services/native-runtime/native-run-trace.test.ts b/server/src/services/native-runtime/native-run-trace.test.ts index 88c96b58be..c70c290b33 100644 --- a/server/src/services/native-runtime/native-run-trace.test.ts +++ b/server/src/services/native-runtime/native-run-trace.test.ts @@ -62,6 +62,30 @@ function createRecordingTraceContext(): { } describe("native runner performance trace", () => { + it("parents to live host dispatch and activates its context for instrumented clients", async () => { + const { traceContext, spans } = createRecordingTraceContext(); + let activeContext: unknown; + traceContext.withContext = (context, work) => { + const previous = activeContext; + activeContext = context; + try { return work(); } finally { activeContext = previous; } + }; + const host = traceContext.tracer.startSpan("heartbeat.native_dispatch"); + const trace = createNativeRunTrace({ runId: "run", traceContext, + parentContext: traceContext.contextWithSpan(host) }); + const startup = trace.start("runner.session.startup"); + trace.run(startup, () => { + const context = activeContext as { span?: RecordedSpan }; + expect(context.span?.name).toBe("runner.session.startup"); + traceContext.tracer.startSpan("http.request", {}, activeContext).end(); + }); + await trace.end(startup); + await trace.finish("ok"); + expect(spans.find((span) => span.name === "task.run")?.parentName).toBe("heartbeat.native_dispatch"); + expect(spans.find((span) => span.name === "http.request")?.parentName).toBe("runner.session.startup"); + expect(activeContext).toBeUndefined(); + }); + it("persists measured spans with bounded run-relative timing", async () => { const events: AdapterRuntimeEvent[] = []; const trace = createNativeRunTrace({ diff --git a/server/src/services/native-runtime/native-run-trace.ts b/server/src/services/native-runtime/native-run-trace.ts index 796533ee7e..2bb4be1063 100644 --- a/server/src/services/native-runtime/native-run-trace.ts +++ b/server/src/services/native-runtime/native-run-trace.ts @@ -103,6 +103,8 @@ export function createNativeRunTrace(input: { startedAtMs?: number; onEvent?: NativeRunTraceSink; traceContext?: StartupTraceContextHandle; + /** The live host dispatch span when full sandbox diagnostics are enabled. */ + parentContext?: unknown; }) { const tracing = input.traceContext ?? getStartupTraceContext("paperclip.native-runner"); @@ -118,7 +120,7 @@ export function createNativeRunTrace(input: { "paperclip.task.run.trace_schema_version": NATIVE_RUN_TRACE_SCHEMA_VERSION, }, - }); + }, input.parentContext); } catch { rootSpan = NOOP_SPAN; } @@ -350,9 +352,10 @@ export function createNativeRunTrace(input: { }, }, ); - return scopeStorage.run(state, () => + const work = () => scopeStorage.run(state, () => runWithRuntimeParent(dynamicContext, fn), ); + return tracing.withContext ? tracing.withContext(dynamicContext, work) : work(); }; const record = async (span: NativeRunHistoricalSpan): Promise => { diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 056bc7bad9..c2be96961a 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -1,5 +1,7 @@ import { createHash, randomUUID } from "node:crypto"; import { execFileSync } from "node:child_process"; +import { hasSandboxPerformanceTrace } from "../sandbox-performance.js"; +import { getActiveStepContext } from "@paperclipai/adapter-utils/acpx-engine/startup-timing"; import { chmodSync, closeSync, @@ -3759,7 +3761,11 @@ async function executePaperclipNativeSessionWithinScope( "paperclip_runner_provider_unsupported: ACPX Pi requires the native runner's descriptor-confined verified launch", ); } - const earliestPreparationStart = input.preparationSpans?.reduce( + // Full host diagnostics already measured preparation in its real scopes. + // Do not backdate a second aggregate over those operations or detach this + // native subtree from the live dispatch parent. + const hostMeasured = hasSandboxPerformanceTrace(); + const earliestPreparationStart = hostMeasured ? Date.now() : input.preparationSpans?.reduce( (earliest, span) => Math.min(earliest, span.startedAtMs), Date.now(), ); @@ -3767,8 +3773,9 @@ async function executePaperclipNativeSessionWithinScope( runId: input.execution.binding.runId, startedAtMs: earliestPreparationStart, onEvent: input.onEvent, + parentContext: hostMeasured ? getActiveStepContext()?.parentContext : undefined, }); - const preparationSpans = input.preparationSpans ?? []; + const preparationSpans = hostMeasured ? [] : input.preparationSpans ?? []; const taskPrepareScope = trace.start("task.prepare", { parentName: "task.run", startedAtMs: earliestPreparationStart, diff --git a/server/src/services/sandbox-performance.ts b/server/src/services/sandbox-performance.ts new file mode 100644 index 0000000000..3be762b871 --- /dev/null +++ b/server/src/services/sandbox-performance.ts @@ -0,0 +1,216 @@ +import { AsyncLocalStorage } from "node:async_hooks"; +import { createHash, randomBytes } from "node:crypto"; +import { Readable } from "node:stream"; +import { performance } from "node:perf_hooks"; +import { getActiveStepContext, runWithRuntimeParent } from "@paperclipai/adapter-utils/acpx-engine/startup-timing"; +import { getStartupTraceContext, traceparentFromContextToken, type StartupTraceContextHandle } from "../instrumentation.js"; + +type Attributes = Record; +type Outcome = "ok" | "failed" | "cancelled"; +export interface SandboxPerformanceRecord { + name: string; id: string; parentId?: string; traceId?: string; + startedAtMs: number; durationMs: number; outcome: Outcome; attributes: Attributes; + clock?: "remote_relative"; +} +interface TraceState { + tracing: StartupTraceContextHandle; records: SandboxPerformanceRecord[]; + limit: number; dropped: number; runHash: string; startedAtMs: number; closed?: boolean; + root?: SandboxOperation; +} +interface ScopeState { trace: TraceState; id: string; context: unknown; parent?: ScopeState; ended?: boolean; } +const scopes = new AsyncLocalStorage(); +function openScope(scope: ScopeState | undefined) { + while (scope?.ended) scope = scope.parent; + return scope; +} +const numericKeys = new Set([ + "bytes", "files", "requestCount", "attempt", "retries", "parallelism", "repositoryIndex", "fileIndex", "scopeIndex", "chunkIndex", + "objects", "entries", "batchIndex", "batches", "operations", "completed", "offset", "limit", "queueDepth", "dropped", + "executionMs", "transportOverheadMs", "durationMs", "startOffsetMs", "endOffsetMs", "waitMs", "gitListMs", "hashMs", "readMs", "writeMs", "publishMs", "decodeMs", "encodeMs", + "inputBytes", "outputBytes", "roundtripMs", "hashFiles", "hashBytes", "droppedPhases", "scanMs", "listMs", "recordCount", "operationIndex", "readWaitMs", "bodyNonReadMs", "chunks", +]); +const booleanKeys = new Set(["cold", "warm", "reused", "cacheHit", "exists", "changed", "repository", "bound", "tracked", "ignored", "cancelled", "final", "enabled", "repeated"]); +// String attributes describe operations, never user input. Names and string +// values must also pass a token grammar; callers must use fixed literals. +const stringKeys = new Set(["scope", "phase", "operation", "runtime", "outcome", "mode", "source", "trigger", "clock", "strategy"]); +function safe(attributes: Attributes): Attributes { + const result: Attributes = {}; + for (const [key, value] of Object.entries(attributes)) { + if (numericKeys.has(key) && typeof value === "number" && Number.isFinite(value) && value >= 0) result[key] = value; + else if (booleanKeys.has(key) && typeof value === "boolean") result[key] = value; + else if (stringKeys.has(key) && typeof value === "string" && /^[a-z][a-z0-9_-]{0,47}$/.test(value)) result[key] = value; + } + return result; +} +function safeName(name: string) { return /^[a-z][a-z0-9_.]{0,95}$/.test(name) ? name : "sandbox.invalid_operation"; } +function identity(context: unknown) { + const value = traceparentFromContextToken(context)?.split("-"); + return value && !/^0+$/.test(value[1]!) && !/^0+$/.test(value[2]!) ? { traceId: value[1], spanId: value[2] } : undefined; +} +function record(trace: TraceState, entry: SandboxPerformanceRecord) { + if (trace.closed) return; + if (trace.records.length < trace.limit) trace.records.push(entry); + else trace.dropped++; +} +export interface SandboxOperation { + set(attributes: Attributes): void; + end(outcome?: Outcome): void; + run(work: () => T): T; + recordRemotePhase(name: string, startOffsetMs: number, durationMs: number, attributes?: Attributes): void; +} +const noop: SandboxOperation = { set() {}, end() {}, run: (work) => work(), recordRemotePhase() {} }; + +/** Capture at stream creation, not when an unrelated consumer later reads it. */ +export function captureSandboxPerformanceContext(): (work: () => T) => T { + const scope = scopes.getStore(); + const parent = getActiveStepContext()?.parentContext ?? scope?.context; + return (work) => { + const current = openScope(scope); + const token = current === scope ? parent : current?.context; + const within = () => scopes.run(current, () => runWithRuntimeParent(token, work)); + return current?.trace.tracing.withContext ? current.trace.tracing.withContext(token, within) : within(); + }; +} +export function hasSandboxPerformanceTrace() { return Boolean(openScope(scopes.getStore()) && !scopes.getStore()?.trace.closed); } +export function setSandboxPerformanceRunAttributes(attributes: Attributes): void { + scopes.getStore()?.trace.root?.set(attributes); +} + +export function startSandboxOperation(name: string, attributes: Attributes = {}): SandboxOperation { + const current = openScope(scopes.getStore()); + if (!current || current.trace.closed) return noop; + const trace = current.trace; + const parentContext = getActiveStepContext()?.parentContext ?? current.context; + const parentId = identity(parentContext)?.spanId ?? current.id; + const startedAtMs = Date.now(), started = performance.now(); + const values = safe(attributes); + let span: ReturnType | undefined; + let context = parentContext; + name = safeName(name); + try { + span = trace.tracing.tracer.startSpan(name, { startTime: startedAtMs, attributes: { "paperclip.sandbox.run_hash": trace.runHash } }, parentContext); + context = trace.tracing.contextWithSpan(span); + } catch { /* Diagnostics must never change task behavior. */ } + const ids = span ? identity(context) : undefined, id = ids?.spanId ?? randomBytes(8).toString("hex"); + const child: ScopeState = { trace, id, context, parent: current }; + let ended = false; + return { + set(next) { if (!ended) Object.assign(values, safe(next)); }, + run(work) { + const within = () => scopes.run(child, () => runWithRuntimeParent(context, work)); + return trace.tracing.withContext ? trace.tracing.withContext(context, within) : within(); + }, + recordRemotePhase(remoteName, startOffsetMs, durationMs, remoteAttributes = {}) { + if (ended || !Number.isFinite(startOffsetMs) || startOffsetMs < 0 || !Number.isFinite(durationMs) || durationMs < 0) return; + const attrs = safe({ ...remoteAttributes, clock: "remote_relative", startOffsetMs, durationMs }); + // Remote offsets have no measured host-clock alignment. Preserve them as + // events on the real command span; never fabricate host timestamps. + try { (span as typeof span & { addEvent?: (n: string, a: Attributes) => void })?.addEvent?.(safeName(remoteName), attrs); } catch { /* fail open */ } + record(trace, { name: safeName(remoteName), id: randomBytes(8).toString("hex"), parentId: id, traceId: ids?.traceId, + startedAtMs: startOffsetMs, durationMs, outcome: "ok", attributes: attrs, clock: "remote_relative" }); + }, + end(outcome = "ok") { + if (ended) return; + ended = true; + child.ended = true; + const durationMs = Math.max(0, performance.now() - started); + try { + for (const [key, value] of Object.entries(values)) span?.setAttribute(`paperclip.sandbox.${key}`, value); + span?.setAttribute("paperclip.sandbox.outcome", outcome); + span?.setAttribute("paperclip.sandbox.duration_ms", durationMs); + if (outcome !== "ok") span?.setStatus({ code: 2 }); // Never export the exception message. + span?.end(startedAtMs + durationMs); + } catch { /* fail open */ } + record(trace, { name, id, parentId, traceId: ids?.traceId, startedAtMs, durationMs, outcome, attributes: values }); + }, + }; +} + +export async function measureSandboxOperation(name: string, attributes: Attributes, work: (span: SandboxOperation) => Promise): Promise { + if (!scopes.getStore()) return work(noop); + const span = startSandboxOperation(name, attributes); + let outcome: Outcome = "ok"; + try { return await span.run(() => work(span)); } + catch (error) { outcome = "failed"; throw error; } + finally { span.end(outcome); } +} + +/** Lazy, bounded body consumption. GET response wait is a separate operation. */ +export function measureSandboxStream(name: string, attributes: Attributes, source: Readable): Readable { + if (!hasSandboxPerformanceTrace()) return source; + const within = captureSandboxPerformanceContext(); + let earlyError: Error | undefined; + // A prefetched response can fail before anybody starts consuming the wrapper. + // Keep its error observable without starting the stream or buffering bytes. + const onSourceError = (error: Error) => { earlyError = error; }; + source.on("error", onSourceError); + const wrapped = Readable.from((async function* () { + const span = within(() => startSandboxOperation(name, attributes)); + let bytes = 0, chunks = 0, readWaitMs = 0, complete = false, outcome: Outcome = "cancelled"; + const bodyStarted = performance.now(); + const iterator = source[Symbol.asyncIterator](); + try { + if (earlyError) throw earlyError; + for (;;) { + const readStarted = performance.now(); + let next: IteratorResult; + try { next = await span.run(() => iterator.next()); } + finally { readWaitMs += performance.now() - readStarted; } + if (next.done) { complete = true; outcome = "ok"; break; } + bytes += Buffer.isBuffer(next.value) ? next.value.length : Buffer.byteLength(String(next.value)); + chunks++; + yield next.value; + } + } catch (error) { outcome = "failed"; throw error; } + finally { + try { + if (!complete) { try { await iterator.return?.(); } finally { source.destroy(); } } + } finally { + span.set({ bytes, chunks, readWaitMs, bodyNonReadMs: Math.max(0, performance.now() - bodyStarted - readWaitMs) }); span.end(outcome); + } + } + })()); + wrapped.once("close", () => { + // Also closes a response whose lazy generator was never entered. + source.destroy(); + if (source.closed) source.removeListener("error", onSourceError); + else source.once("close", () => source.removeListener("error", onSourceError)); + }); + const destroy = wrapped.destroy.bind(wrapped); + wrapped.destroy = (error?: Error) => { + source.destroy(); + return destroy(error); + }; + return wrapped; +} + +export async function runWithSandboxPerformanceTrace(input: { + runId: string; enabled?: boolean; traceContext?: StartupTraceContextHandle; maxRecords?: number; + onBatch?: (batch: { schema: "paperclip.sandbox-performance.v1"; runHash: string; records: SandboxPerformanceRecord[]; dropped: number }) => Promise; +}, work: () => Promise): Promise { + if (!(input.enabled ?? Boolean(process.env.OTEL_EXPORTER_OTLP_ENDPOINT?.trim()))) return work(); + const trace: TraceState = { tracing: input.traceContext ?? getStartupTraceContext("paperclip.sandbox"), records: [], + limit: Number.isFinite(input.maxRecords) ? Math.max(1, Math.min(50_000, Math.floor(input.maxRecords!))) : 20_000, dropped: 0, + runHash: createHash("sha256").update(input.runId).digest("hex").slice(0, 12), startedAtMs: Date.now() }; + const initial = { trace, id: "", context: getActiveStepContext()?.parentContext }; + try { + return await scopes.run(initial, () => measureSandboxOperation("sandbox.run", {}, async (span) => { + trace.root = span; + try { return await work(); } + finally { + span.set({ recordCount: trace.records.length + (trace.records.length < trace.limit ? 1 : 0), + dropped: trace.dropped + (trace.records.length >= trace.limit ? 1 : 0) }); + } + })); + } + finally { + trace.closed = true; + // No per-file DB writes, no synchronous exporter call and no unbounded + // promise queue. Persist at most 250 records per batch after measured work. + // The SDK independently exports ended spans through its batch processor. + for (let offset = 0; offset < trace.records.length; offset += 250) { + try { await input.onBatch?.({ schema: "paperclip.sandbox-performance.v1", runHash: trace.runHash, + records: trace.records.slice(offset, offset + 250), dropped: trace.dropped }); } catch { break; } + } + } +} diff --git a/server/src/services/sandbox-work-folders.ts b/server/src/services/sandbox-work-folders.ts index ab2ca0b34f..1a1fb0522f 100644 --- a/server/src/services/sandbox-work-folders.ts +++ b/server/src/services/sandbox-work-folders.ts @@ -1,3 +1,4 @@ +import { measureSandboxOperation, measureSandboxStream, captureSandboxPerformanceContext } from "./sandbox-performance.js"; import { prefetchWorkFiles } from "./work-folder-transfer.js"; import { createHash, randomUUID } from "node:crypto"; import path from "node:path"; @@ -34,376 +35,416 @@ export async function prepareSandboxWorkFolders(input: { primaryWorkspaceId?: string | null; primaryBranchName?: string | null; storage?: StorageProvider; sandboxKey?: string; }) { - const { db, target } = input; - if (!target.runner || !target.leaseId) throw new Error("Sandbox file transport is unavailable"); - async function assertBindings() { - const memberships: Array<{ companyId: string; membershipRole: string | null; status: string }> = []; - const deny = () => { throw new Error("Sandbox work-folder access is no longer authorized; working files were retained"); }; - const [run] = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.id, input.runId), eq(heartbeatRuns.companyId, input.companyId))); - if (!run || run.agentId !== input.agentId || run.responsibleUserId !== input.responsibleUserId) deny(); - const [agent] = await db.select({ id: agents.id }).from(agents).where(and(eq(agents.id, input.agentId), eq(agents.companyId, input.companyId))); - if (!agent) deny(); - if (input.taskId) { - const [task] = await db.select({ id: issues.id }).from(issues).where(and(eq(issues.id, input.taskId), eq(issues.companyId, input.companyId))); - if (!task) deny(); + const runInRunContext = captureSandboxPerformanceContext(); + return measureSandboxOperation("work_folder.prepare", { phase: "prepare" }, async (prepareSpan) => { + const { db, target, taskId, projectId, responsibleUserId } = input; + if (!target.runner || !target.leaseId) throw new Error("Sandbox file transport is unavailable"); + async function assertBindings() { + return measureSandboxOperation("work_folder.authorization", { operation: "validate_bindings" }, async () => { + const memberships: Array<{ companyId: string; membershipRole: string | null; status: string }> = []; + const deny = () => { throw new Error("Sandbox work-folder access is no longer authorized; working files were retained"); }; + const [run] = await measureSandboxOperation("work_folder.db.query", { operation: "select_heartbeat_runs", requestCount: 1 }, async () => (db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.id, input.runId), eq(heartbeatRuns.companyId, input.companyId))))); + if (!run || run.agentId !== input.agentId || run.responsibleUserId !== responsibleUserId) deny(); + const [agent] = await measureSandboxOperation("work_folder.db.query", { operation: "select_agents", requestCount: 1 }, async () => (db.select({ id: agents.id }).from(agents).where(and(eq(agents.id, input.agentId), eq(agents.companyId, input.companyId))))); + if (!agent) deny(); + if (taskId) { + const [task] = await measureSandboxOperation("work_folder.db.query", { operation: "select_issues", requestCount: 1 }, async () => (db.select({ id: issues.id }).from(issues).where(and(eq(issues.id, taskId), eq(issues.companyId, input.companyId))))); + if (!task) deny(); + } + if (projectId) { + const [project] = await measureSandboxOperation("work_folder.db.query", { operation: "select_projects", requestCount: 1 }, async () => (db.select({ id: projects.id }).from(projects).where(and(eq(projects.id, projectId), eq(projects.companyId, input.companyId))))); + if (!project) deny(); + } + if (responsibleUserId) { + const [membership] = await measureSandboxOperation("work_folder.db.query", { operation: "select_company_memberships", requestCount: 1 }, async () => (db.select().from(companyMemberships).where(and(eq(companyMemberships.companyId, input.companyId), + eq(companyMemberships.principalType, "user"), eq(companyMemberships.principalId, responsibleUserId), eq(companyMemberships.status, "active"))))); + if (!membership || membership.membershipRole === "viewer") deny(); + if (membership) memberships.push({ companyId: membership.companyId, membershipRole: membership.membershipRole, status: membership.status }); + } + for (const [scope, ownerId] of [["task", taskId], ["agent", input.agentId], ["project", projectId]] as const) { + if (!ownerId) continue; + await assertWorkFolderAccess(db, { type: "agent", source: "agent_jwt", companyId: input.companyId, + agentId: input.agentId, runId: input.runId, onBehalfOfUserId: responsibleUserId, onBehalfOfMemberships: memberships }, + { companyId: input.companyId, scope, ownerId }, true); + } + }); } - if (input.projectId) { - const [project] = await db.select({ id: projects.id }).from(projects).where(and(eq(projects.id, input.projectId), eq(projects.companyId, input.companyId))); - if (!project) deny(); + // Host-side transfers do not go through HTTP authorization middleware. Check + // the authoritative bindings here too, including after membership revocation. + // An ended heartbeat may still flush; its immutable identity must still match. + await assertBindings(); + const storage = input.storage ?? createStorageProviderFromConfig(loadConfig()); + const svc = workFolderService(db, storage); + const transport = workFolderTransport(target.runner); + const repositories = workFolderRepositoryService(db, storage, transport); + const home = await transport.home(); + const paths = workFolderPaths(home); + for (const value of Object.values(paths)) await transport.mkdirRoot(value); + const staging = paths[".paperclip-work-folders"]!; + const owners = { task: taskId, agent: input.agentId, user: responsibleUserId, project: projectId }; + const folders: Partial> = {}; + for (const scope of WORK_FOLDER_SCOPES) { + const ownerId = owners[scope]; + if (ownerId) folders[scope] = await svc.ensure({ companyId: input.companyId, scope, ownerId }); } - if (input.responsibleUserId) { - const [membership] = await db.select().from(companyMemberships).where(and(eq(companyMemberships.companyId, input.companyId), - eq(companyMemberships.principalType, "user"), eq(companyMemberships.principalId, input.responsibleUserId), eq(companyMemberships.status, "active"))); - if (!membership || membership.membershipRole === "viewer") deny(); - if (membership) memberships.push({ companyId: membership.companyId, membershipRole: membership.membershipRole, status: membership.status }); + const manifest: SandboxWorkFolderManifest = { version: 1, companyId: input.companyId, runId: input.runId, + taskId: taskId, agentId: input.agentId, responsibleUserId: responsibleUserId, + projectId: projectId, leaseId: target.leaseId, sandboxKey: input.sandboxKey ?? target.leaseId, home, + folders: { task: folders.task?.id ?? null, agent: folders.agent!.id, user: folders.user?.id ?? null, project: folders.project?.id ?? null }, repositories: [] }; + const [previous] = await measureSandboxOperation("work_folder.history.sandbox", { operation: "select_work_folder_runs", requestCount: 1 }, async () => (db.select().from(workFolderRuns).where(and(eq(workFolderRuns.companyId, input.companyId), + sql`coalesce(${workFolderRuns.manifest}->>'sandboxKey', ${workFolderRuns.manifest}->>'leaseId') = ${manifest.sandboxKey}`)).orderBy(desc(workFolderRuns.updatedAt)).limit(1))); + if (previous && (previous.manifest.taskId !== taskId || previous.manifest.agentId !== input.agentId + || previous.manifest.responsibleUserId !== responsibleUserId || previous.manifest.projectId !== projectId)) { + throw new Error("Sandbox file identity changed; acquire a fresh sandbox before continuing"); } - for (const [scope, ownerId] of [["task", input.taskId], ["agent", input.agentId], ["project", input.projectId]] as const) { - if (!ownerId) continue; - await assertWorkFolderAccess(db, { type: "agent", source: "agent_jwt", companyId: input.companyId, - agentId: input.agentId, runId: input.runId, onBehalfOfUserId: input.responsibleUserId, onBehalfOfMemberships: memberships }, - { companyId: input.companyId, scope, ownerId }, true); - } - } - // Host-side transfers do not go through HTTP authorization middleware. Check - // the authoritative bindings here too, including after membership revocation. - // An ended heartbeat may still flush; its immutable identity must still match. - await assertBindings(); - const storage = input.storage ?? createStorageProviderFromConfig(loadConfig()); - const svc = workFolderService(db, storage); - const transport = workFolderTransport(target.runner); - const repositories = workFolderRepositoryService(db, storage, transport); - const home = await transport.home(); - const paths = workFolderPaths(home); - for (const value of Object.values(paths)) await transport.mkdirRoot(value); - const staging = paths[".paperclip-work-folders"]!; - const owners = { task: input.taskId, agent: input.agentId, user: input.responsibleUserId, project: input.projectId }; - const folders: Partial> = {}; - for (const scope of WORK_FOLDER_SCOPES) { - const ownerId = owners[scope]; - if (ownerId) folders[scope] = await svc.ensure({ companyId: input.companyId, scope, ownerId }); - } - const manifest: SandboxWorkFolderManifest = { version: 1, companyId: input.companyId, runId: input.runId, - taskId: input.taskId, agentId: input.agentId, responsibleUserId: input.responsibleUserId, - projectId: input.projectId, leaseId: target.leaseId, sandboxKey: input.sandboxKey ?? target.leaseId, home, - folders: { task: folders.task?.id ?? null, agent: folders.agent!.id, user: folders.user?.id ?? null, project: folders.project?.id ?? null }, repositories: [] }; - const [previous] = await db.select().from(workFolderRuns).where(and(eq(workFolderRuns.companyId, input.companyId), - sql`coalesce(${workFolderRuns.manifest}->>'sandboxKey', ${workFolderRuns.manifest}->>'leaseId') = ${manifest.sandboxKey}`)).orderBy(desc(workFolderRuns.updatedAt)).limit(1); - if (previous && (previous.manifest.taskId !== input.taskId || previous.manifest.agentId !== input.agentId - || previous.manifest.responsibleUserId !== input.responsibleUserId || previous.manifest.projectId !== input.projectId)) { - throw new Error("Sandbox file identity changed; acquire a fresh sandbox before continuing"); - } - const [previousTaskRun] = input.taskId ? await db.select({ manifest: workFolderRuns.manifest }).from(workFolderRuns) - .where(and(eq(workFolderRuns.companyId, input.companyId), sql`${workFolderRuns.manifest}->>'taskId' = ${input.taskId}`)) - .orderBy(desc(workFolderRuns.updatedAt)).limit(1) : []; - const identityChanged = Boolean(previousTaskRun && (previousTaskRun.manifest.agentId !== input.agentId - || previousTaskRun.manifest.responsibleUserId !== input.responsibleUserId)); - const baselines: Record = previous?.baselines ?? {}; - const pendingOperations = previous?.pendingOperations ?? {}; - await db.insert(workFolderRuns).values({ runId: input.runId, companyId: input.companyId, manifest, baselines, pendingOperations }) - .onConflictDoUpdate({ target: workFolderRuns.runId, set: { manifest, state: "starting", updatedAt: new Date() } }); + // These dimensions describe validated work-folder history for this physical + // sandbox key, not whether the provider was powered on or restarted. + prepareSpan.set({ cold: !previous, warm: Boolean(previous), reused: Boolean(previous) }); + const [previousTaskRun] = taskId ? await measureSandboxOperation("work_folder.history.task", { operation: "select_work_folder_runs", requestCount: 1 }, async () => (db.select({ manifest: workFolderRuns.manifest }).from(workFolderRuns) + .where(and(eq(workFolderRuns.companyId, input.companyId), sql`${workFolderRuns.manifest}->>'taskId' = ${taskId}`)) + .orderBy(desc(workFolderRuns.updatedAt)).limit(1))) : []; + const identityChanged = Boolean(previousTaskRun && (previousTaskRun.manifest.agentId !== input.agentId + || previousTaskRun.manifest.responsibleUserId !== responsibleUserId)); + const baselines: Record = previous?.baselines ?? {}; + const pendingOperations = previous?.pendingOperations ?? {}; + await measureSandboxOperation("work_folder.manifest.initialize", { operation: "insert_work_folder_runs", requestCount: 1 }, async () => (db.insert(workFolderRuns).values({ runId: input.runId, companyId: input.companyId, manifest, baselines, pendingOperations }) + .onConflictDoUpdate({ target: workFolderRuns.runId, set: { manifest, state: "starting", updatedAt: new Date() } }))); - async function seedAttachments() { - if (!folders.task || !input.taskId) return; - const attached = await db.select({ attachment: issueAttachments, asset: assets }).from(issueAttachments) - .innerJoin(assets, and(eq(assets.id, issueAttachments.assetId), eq(assets.companyId, issueAttachments.companyId))) - .where(and(eq(issueAttachments.issueId, input.taskId), eq(issueAttachments.companyId, input.companyId))) - .orderBy(asc(issueAttachments.createdAt), asc(issueAttachments.id)); - for (const { attachment, asset } of attached) { - const operationId = `attachment:${attachment.id}`; - const [seeded] = await db.select().from(workFileOperations).where(and(eq(workFileOperations.folderId, folders.task.id), - eq(workFileOperations.operationId, operationId))); - if (seeded) continue; - const original = (asset.originalFilename ?? "attachment").split(/[\\/]/).at(-1)!.replace(/[\x00-\x1f\x7f]/g, "_").slice(0, 180) || "attachment"; - // The ID makes the destination independent of concurrent uploads and - // earlier seeding attempts. Even dot/reserved filenames become safe. - const extension = path.posix.extname(original); - const filename = `${original.slice(0, original.length - extension.length)}-${attachment.id}${extension}`; - const result = await storage.getObject({ objectKey: asset.objectKey }); - try { await svc.write(folders.task, { path: filename, body: result.stream, contentType: asset.contentType, operationId, onlyIfMissing: true }); } - finally { result.stream.destroy(); } + async function seedAttachments() { + return measureSandboxOperation("work_folder.attachments", { phase: "prepare" }, async (span) => { + const taskFolder = folders.task; + if (!taskFolder || !taskId) return; + const attached = await measureSandboxOperation("work_folder.db.query", { operation: "select_issue_attachments", requestCount: 1 }, async () => (db.select({ attachment: issueAttachments, asset: assets }).from(issueAttachments) + .innerJoin(assets, and(eq(assets.id, issueAttachments.assetId), eq(assets.companyId, issueAttachments.companyId))) + .where(and(eq(issueAttachments.issueId, taskId), eq(issueAttachments.companyId, input.companyId))) + .orderBy(asc(issueAttachments.createdAt), asc(issueAttachments.id)))); + span.set({ files: attached.length }); + for (const [fileIndex, { attachment, asset }] of attached.entries()) { + const operationId = `attachment:${attachment.id}`; + const [seeded] = await measureSandboxOperation("work_folder.db.query", { operation: "select_work_file_operations", requestCount: 1 }, async () => (db.select().from(workFileOperations).where(and(eq(workFileOperations.folderId, taskFolder.id), + eq(workFileOperations.operationId, operationId))))); + if (seeded) continue; + const original = (asset.originalFilename ?? "attachment").split(/[\\/]/).at(-1)!.replace(/[\x00-\x1f\x7f]/g, "_").slice(0, 180) || "attachment"; + // The ID makes the destination independent of concurrent uploads and + // earlier seeding attempts. Even dot/reserved filenames become safe. + const extension = path.posix.extname(original); + const filename = `${original.slice(0, original.length - extension.length)}-${attachment.id}${extension}`; + const result = await measureSandboxOperation("work_folder.attachment.get_response", { requestCount: 1, bytes: asset.byteSize, fileIndex }, async () => (storage.getObject({ objectKey: asset.objectKey }))); + try { await svc.write(taskFolder, { path: filename, body: measureSandboxStream("work_folder.attachment.body", { bytes: asset.byteSize, scope: "task", fileIndex }, result.stream), contentType: asset.contentType, operationId, onlyIfMissing: true }); } + finally { result.stream.destroy(); } + } + }); } - } - async function importAgentFiles() { - const folder = folders.agent!; - if (folder.importedAt) return; - const root = resolveDefaultAgentWorkspaceDir(input.agentId); - for await (const file of managedAgentFiles(root)) { - const [receipt] = await db.select({ id: workFileOperations.id }).from(workFileOperations).where(and( - eq(workFileOperations.folderId, folder.id), eq(workFileOperations.operationId, `import:${file.path}`))); - if (receipt) continue; - await svc.write(folder, { ...file, operationId: `import:${file.path}`, onlyIfMissing: true }); + async function importAgentFiles() { + return measureSandboxOperation("work_folder.agent_import", { phase: "prepare" }, async () => { + const folder = folders.agent!; + if (folder.importedAt) return; + const root = resolveDefaultAgentWorkspaceDir(input.agentId); + for await (const file of managedAgentFiles(root)) { + const [receipt] = await measureSandboxOperation("work_folder.db.query", { operation: "select_work_file_operations", requestCount: 1 }, async () => (db.select({ id: workFileOperations.id }).from(workFileOperations).where(and( + eq(workFileOperations.folderId, folder.id), eq(workFileOperations.operationId, `import:${file.path}`))))); + if (receipt) continue; + await svc.write(folder, { ...file, operationId: `import:${file.path}`, onlyIfMissing: true }); + } + await measureSandboxOperation("work_folder.db.query", { operation: "update_work_folders", requestCount: 1 }, async () => (db.update(workFolders).set({ importedAt: new Date() }).where(eq(workFolders.id, folder.id)))); + }); } - await db.update(workFolders).set({ importedAt: new Date() }).where(eq(workFolders.id, folder.id)); - } - async function reconcileIncoming(scope: WorkFolderScope, current: WorkTreeEntry[]) { - const targets = baselines[`incoming:${scope}`]; - const removed = baselines[`incomingRemoved:${scope}`]; - if (!targets && !removed) return; - const before = new Map((baselines[scope] ?? []).map((entry) => [entry.path, entry])); - const observed = new Map(current.map((entry) => [entry.path, entry])); - // A failed command may have published some imports or removed some files. - // Only adopt effects actually observed on disk; different bytes remain - // genuine local edits and must still pass through outgoing synchronization. - for (const entry of removed ?? []) if (!observed.has(entry.path)) before.delete(entry.path); - for (const entry of targets ?? []) { - if (signature(observed.get(entry.path)) === signature(entry)) before.set(entry.path, entry); - } - baselines[scope] = [...before.values()]; - delete baselines[`incoming:${scope}`]; - delete baselines[`incomingRemoved:${scope}`]; - // Persist the reconciled baseline and remove its provenance atomically, - // before any outgoing write can be accepted by the shared collection. - await saveState("saving"); - } - async function outgoing(scope: WorkFolderScope) { - const folder = folders[scope]; - if (!folder) return; - const current = await transport.scan(paths[scope]!); - await reconcileIncoming(scope, current); - const before = new Map((baselines[scope] ?? []).map((entry) => [entry.path, entry])); - const after = new Map(current.map((entry) => [entry.path, entry])); - async function operation(filePath: string, nextSignature: string, apply: (id: string) => Promise, accept: () => void) { - const key = `${scope}/${filePath}`; - if (pendingOperations[key]?.signature !== nextSignature) pendingOperations[key] = { id: randomUUID(), signature: nextSignature }; - // Persist the receipt ID BEFORE sending bytes. A process restart or lost - // COMMIT response must retry this ID, not overwrite another run's edit. - await saveState("saving"); - await apply(pendingOperations[key]!.id); - accept(); + async function reconcileIncoming(scope: WorkFolderScope, current: WorkTreeEntry[]) { + const targets = baselines[`incoming:${scope}`]; + const removed = baselines[`incomingRemoved:${scope}`]; + if (!targets && !removed) return; + const before = new Map((baselines[scope] ?? []).map((entry) => [entry.path, entry])); + const observed = new Map(current.map((entry) => [entry.path, entry])); + // A failed command may have published some imports or removed some files. + // Only adopt effects actually observed on disk; different bytes remain + // genuine local edits and must still pass through outgoing synchronization. + for (const entry of removed ?? []) if (!observed.has(entry.path)) before.delete(entry.path); + for (const entry of targets ?? []) { + if (signature(observed.get(entry.path)) === signature(entry)) before.set(entry.path, entry); + } baselines[scope] = [...before.values()]; - delete pendingOperations[key]; + delete baselines[`incoming:${scope}`]; + delete baselines[`incomingRemoved:${scope}`]; + // Persist the reconciled baseline and remove its provenance atomically, + // before any outgoing write can be accepted by the shared collection. await saveState("saving"); } - for (const entry of current) { - if (signature(before.get(entry.path)) === signature(entry)) continue; - const body = entry.kind === "file" ? transport.read(paths[scope]!, entry.path, entry.byteSize) : undefined; - try { - await operation(entry.path, signature(entry), (operationId) => svc.write(folder, { path: entry.path, body, - kind: entry.kind, replaceKind: true, executable: entry.executable, expectedSha256: entry.sha256, operationId }), () => { - if (before.get(entry.path)?.kind !== entry.kind) for (const key of before.keys()) { - if (key.startsWith(`${entry.path}/`)) before.delete(key); - } - before.set(entry.path, entry); - }); - } finally { body?.destroy(); } - } - for (const old of [...before.values()].sort((a, b) => a.path.length - b.path.length)) { - if (!after.has(old.path) && before.has(old.path)) { - await operation(old.path, "missing", (operationId) => svc.remove(folder, old.path, operationId), () => { - for (const key of before.keys()) if (key === old.path || key.startsWith(`${old.path}/`)) before.delete(key); - }); - } - } - baselines[scope] = [...before.values()]; - } - async function incoming(scope: WorkFolderScope) { - const folder = folders[scope]; - if (!folder) return; - const current = new Map((await transport.scan(paths[scope]!)).map((entry) => [entry.path, entry])); - const saved: WorkTreeEntry[] = []; - let cursor: string | undefined; - do { - const page = await svc.list(folder, { cursor, limit: 1000 }); - for (const file of page.files) saved.push({ path: file.path, kind: file.kind, byteSize: file.byteSize, sha256: file.sha256, executable: file.executable }); - cursor = page.nextCursor ?? undefined; - } while (cursor); - const desired = new Map(saved.map((entry) => [entry.path, entry])); - // Persist all deletion intents before the first removal, including children - // that can disappear when a directory is replaced. A lost response must not - // turn an imported deletion into a new delete against a newer shared file. - const removed = [...current.values()].filter((entry) => !desired.has(entry.path) || desired.get(entry.path)!.kind !== entry.kind) - .sort((a, b) => b.path.length - a.path.length); - if (removed.length) { - baselines[`incomingRemoved:${scope}`] = removed; - await saveState("starting"); - } - for (const entry of removed) { - await transport.remove(paths[scope]!, entry.path); - current.delete(entry.path); - } - const changed = saved.sort((a, b) => a.path.length - b.path.length) - .filter((entry) => signature(current.get(entry.path)) !== signature(entry)); - await transport.writeMany(paths[scope]!, staging, prefetchWorkFiles(changed, async (entry) => { - if (entry.kind === "directory") return { entry }; - const result = await svc.content(folder, entry.path); - // A shared file can change after listing. Validate and baseline the - // version opened by content(), whose metadata and stream belong together. - Object.assign(entry, { byteSize: result.file.byteSize, sha256: result.file.sha256, executable: result.file.executable }); - return { entry, body: result.stream }; - }), async (entries) => { - const targets = new Map((baselines[`incoming:${scope}`] ?? []).map((entry) => [entry.path, entry])); - for (const entry of entries) { - targets.set(entry.path, entry); - // Publishing nested files can create parents absent from the listing. - // Record those directory imports too, so they cannot be mistaken for - // agent-created directories after an interrupted batch. - let parent = path.posix.dirname(entry.path); - while (parent !== ".") { - if (!targets.has(parent)) targets.set(parent, { path: parent, kind: "directory", byteSize: 0, sha256: null, executable: false }); - parent = path.posix.dirname(parent); + async function outgoing(scope: WorkFolderScope) { + return measureSandboxOperation("work_folder.scope.outgoing", { scope }, async (span) => { + const folder = folders[scope]; + if (!folder) return; + const current = await measureSandboxOperation("work_folder.scope.disk_scan", { scope }, async () => (transport.scan(paths[scope]!))); + span.set({ files: current.length }); + await reconcileIncoming(scope, current); + const before = new Map((baselines[scope] ?? []).map((entry) => [entry.path, entry])); + const after = new Map(current.map((entry) => [entry.path, entry])); + async function operation(filePath: string, nextSignature: string, apply: (id: string) => Promise, accept: () => void) { + const key = `${scope}/${filePath}`; + if (pendingOperations[key]?.signature !== nextSignature) pendingOperations[key] = { id: randomUUID(), signature: nextSignature }; + // Persist the receipt ID BEFORE sending bytes. A process restart or lost + // COMMIT response must retry this ID, not overwrite another run's edit. + await saveState("saving"); + await apply(pendingOperations[key]!.id); + accept(); + baselines[scope] = [...before.values()]; + delete pendingOperations[key]; + await saveState("saving"); } - } - baselines[`incoming:${scope}`] = [...targets.values()]; - await saveState("starting"); - }); - baselines[scope] = saved; - delete baselines[`incoming:${scope}`]; - delete baselines[`incomingRemoved:${scope}`]; - await saveState("starting"); - } + for (const entry of current) { + if (signature(before.get(entry.path)) === signature(entry)) continue; + const body = entry.kind === "file" ? transport.read(paths[scope]!, entry.path, entry.byteSize) : undefined; + try { + await operation(entry.path, signature(entry), (operationId) => svc.write(folder, { path: entry.path, body, + kind: entry.kind, replaceKind: true, executable: entry.executable, expectedSha256: entry.sha256, operationId }), () => { + if (before.get(entry.path)?.kind !== entry.kind) for (const key of before.keys()) { + if (key.startsWith(`${entry.path}/`)) before.delete(key); + } + before.set(entry.path, entry); + }); + } finally { body?.destroy(); } + } + for (const old of [...before.values()].sort((a, b) => a.path.length - b.path.length)) { + if (!after.has(old.path) && before.has(old.path)) { + await operation(old.path, "missing", (operationId) => svc.remove(folder, old.path, operationId), () => { + for (const key of before.keys()) if (key === old.path || key.startsWith(`${old.path}/`)) before.delete(key); + }); + } + } + baselines[scope] = [...before.values()]; + }); + } + async function incoming(scope: WorkFolderScope) { + return measureSandboxOperation("work_folder.scope.incoming", { scope }, async (span) => { + const folder = folders[scope]; + if (!folder) return; + const current = new Map((await measureSandboxOperation("work_folder.scope.disk_scan", { scope }, async () => (transport.scan(paths[scope]!)))).map((entry) => [entry.path, entry])); + const saved: WorkTreeEntry[] = []; + let cursor: string | undefined; + do { + const page = await svc.list(folder, { cursor, limit: 1000 }); + for (const file of page.files) saved.push({ path: file.path, kind: file.kind, byteSize: file.byteSize, sha256: file.sha256, executable: file.executable }); + cursor = page.nextCursor ?? undefined; + } while (cursor); + span.set({ files: saved.length }); + const desired = new Map(saved.map((entry) => [entry.path, entry])); + // Persist all deletion intents before the first removal, including children + // that can disappear when a directory is replaced. A lost response must not + // turn an imported deletion into a new delete against a newer shared file. + const removed = [...current.values()].filter((entry) => !desired.has(entry.path) || desired.get(entry.path)!.kind !== entry.kind) + .sort((a, b) => b.path.length - a.path.length); + if (removed.length) { + baselines[`incomingRemoved:${scope}`] = removed; + await saveState("starting"); + } + for (const entry of removed) { + await transport.remove(paths[scope]!, entry.path); + current.delete(entry.path); + } + const changed = saved.sort((a, b) => a.path.length - b.path.length) + .filter((entry) => signature(current.get(entry.path)) !== signature(entry)); + await measureSandboxOperation("work_folder.scope.hydrate", { scope, files: changed.length, parallelism: 4 }, async () => (transport.writeMany(paths[scope]!, staging, prefetchWorkFiles(changed, async (entry, fileIndex) => { + if (entry.kind === "directory") return { entry }; + const result = await svc.content(folder, entry.path, fileIndex); + // A shared file can change after listing. Validate and baseline the + // version opened by content(), whose metadata and stream belong together. + Object.assign(entry, { byteSize: result.file.byteSize, sha256: result.file.sha256, executable: result.file.executable }); + return { entry, body: result.stream }; + }), async (entries) => { + const targets = new Map((baselines[`incoming:${scope}`] ?? []).map((entry) => [entry.path, entry])); + for (const entry of entries) { + targets.set(entry.path, entry); + // Publishing nested files can create parents absent from the listing. + // Record those directory imports too, so they cannot be mistaken for + // agent-created directories after an interrupted batch. + let parent = path.posix.dirname(entry.path); + while (parent !== ".") { + if (!targets.has(parent)) targets.set(parent, { path: parent, kind: "directory", byteSize: 0, sha256: null, executable: false }); + parent = path.posix.dirname(parent); + } + } + baselines[`incoming:${scope}`] = [...targets.values()]; + await saveState("starting"); + }))); + baselines[scope] = saved; + delete baselines[`incoming:${scope}`]; + delete baselines[`incomingRemoved:${scope}`]; + await saveState("starting"); + }); + } - const bindings: Array<{ binding: typeof taskRepositoryBindings.$inferSelect; root: string }> = []; - async function prepareRepositories() { - if (!input.taskId || !input.projectId) return; - const workspaces = await db.select().from(projectWorkspaces).where(and(eq(projectWorkspaces.companyId, input.companyId), - eq(projectWorkspaces.projectId, input.projectId))).orderBy(desc(projectWorkspaces.isPrimary), asc(projectWorkspaces.createdAt)); - const existing = await db.select().from(taskRepositoryBindings).where(and(eq(taskRepositoryBindings.companyId, input.companyId), eq(taskRepositoryBindings.taskId, input.taskId))); - const names = new Set(existing.map((binding) => binding.name)); - const resolveGitAuth = createGitRemoteAuthProvider(db, input.companyId, { responsibleUserId: input.responsibleUserId, agentId: input.agentId, issueId: input.taskId, heartbeatRunId: input.runId }); - for (const workspace of workspaces.filter((entry) => entry.repoUrl)) { - const primary = input.primaryWorkspaceId ? workspace.id === input.primaryWorkspaceId : workspace.isPrimary; - let binding = existing.find((entry) => entry.workspaceId === workspace.id); - if (!binding) { - const baseName = repoName(workspace.repoUrl!.split(/[/:]/).at(-1) ?? workspace.name, workspace.id); - const name = names.has(baseName) ? `${baseName}-${workspace.id.slice(0, 8)}` : baseName; - names.add(name); - [binding] = await db.insert(taskRepositoryBindings).values({ companyId: input.companyId, taskId: input.taskId, - workspaceId: workspace.id, name, repoUrl: workspace.repoUrl, repoRef: workspace.repoRef ?? workspace.defaultRef }).returning(); - } - if (!binding) throw new Error("Repository binding could not be created"); - if (binding.repoUrl !== workspace.repoUrl) throw new Error(`Repository ${binding.name} configuration changed; saved work was retained`); - if (binding.repoRef !== (workspace.repoRef ?? workspace.defaultRef)) throw new Error(`Repository ${binding.name} starting ref changed; saved work was retained`); - const root = path.posix.join(paths.repos!, binding.name); - const probe = await target.runner!.execute({ command: "git", args: ["-C", root, "rev-parse", "--git-dir"], bypassSession: true, timeoutMs: 10_000 }); - const freshCheckout = probe.exitCode !== 0; - if (freshCheckout) { - // Publish the checkout directory only after every restore object or - // clone step completes. An interrupted attempt cannot masquerade as a - // reusable checkout merely because it contains a .git directory. - const temporary = path.posix.join(staging, `repo-${binding.id}-${randomUUID()}`); - const restored = await repositories.restore(binding, temporary, staging); - if (!restored) { - const auth = await resolveGitAuth(workspace.repoUrl!); - const result = await target.runner!.execute({ command: "git", args: [...(auth?.configArgs ?? []), "clone", "--no-hardlinks", - "--", workspace.repoUrl!, temporary], - env: { GIT_TERMINAL_PROMPT: "0", ...(auth?.env ?? {}) }, bypassSession: true, timeoutMs: 300_000 }); - if (result.exitCode !== 0 || result.timedOut) throw new Error(`Required repository ${binding.name} could not be cloned`); - if (binding.repoRef) { - const checkout = await target.runner!.execute({ command: "git", args: ["-C", temporary, "checkout", binding.repoRef, "--"], bypassSession: true, timeoutMs: 60_000 }); - if (checkout.exitCode !== 0 || checkout.timedOut) throw new Error(`Required repository ${binding.name} ref could not be checked out`); - } - if (primary && input.primaryBranchName) { - const branch = input.primaryBranchName; - const valid = await target.runner!.execute({ command: "git", args: ["check-ref-format", "--branch", branch], bypassSession: true, timeoutMs: 10_000 }); - if (valid.exitCode !== 0 || valid.stdout.trim() !== branch) throw new Error(`Required repository ${binding.name} branch is invalid`); - // Honor the task's existing branch policy on the initial clone. - // Restores and warm starts keep the saved HEAD and index untouched. - const checkout = await target.runner!.execute({ command: "git", args: ["-C", temporary, "checkout", branch, "--"], bypassSession: true, timeoutMs: 60_000 }); - if (checkout.exitCode !== 0) { - const create = await target.runner!.execute({ command: "git", args: ["-C", temporary, "checkout", "-b", branch], bypassSession: true, timeoutMs: 60_000 }); - if (create.exitCode !== 0 || create.timedOut) throw new Error(`Required repository ${binding.name} task branch could not be created`); + const bindings: Array<{ binding: typeof taskRepositoryBindings.$inferSelect; root: string }> = []; + async function prepareRepositories() { + return measureSandboxOperation("work_folder.repositories.prepare", { phase: "prepare" }, async () => { + if (!taskId || !projectId) return; + const workspaces = await measureSandboxOperation("work_folder.db.query", { operation: "select_project_workspaces", requestCount: 1 }, async () => (db.select().from(projectWorkspaces).where(and(eq(projectWorkspaces.companyId, input.companyId), + eq(projectWorkspaces.projectId, projectId))).orderBy(desc(projectWorkspaces.isPrimary), asc(projectWorkspaces.createdAt)))); + const existing = await measureSandboxOperation("work_folder.db.query", { operation: "select_task_repository_bindings", requestCount: 1 }, async () => (db.select().from(taskRepositoryBindings).where(and(eq(taskRepositoryBindings.companyId, input.companyId), eq(taskRepositoryBindings.taskId, taskId))))); + const names = new Set(existing.map((binding) => binding.name)); + const resolveGitAuth = createGitRemoteAuthProvider(db, input.companyId, { responsibleUserId: responsibleUserId, agentId: input.agentId, issueId: taskId, heartbeatRunId: input.runId }); + for (const [repositoryIndex, workspace] of workspaces.filter((entry) => entry.repoUrl).entries()) { + await measureSandboxOperation("work_folder.repository.prepare", { repositoryIndex }, async (repositorySpan) => { + const primary = input.primaryWorkspaceId ? workspace.id === input.primaryWorkspaceId : workspace.isPrimary; + let binding = existing.find((entry) => entry.workspaceId === workspace.id); + if (!binding) { + const baseName = repoName(workspace.repoUrl!.split(/[/:]/).at(-1) ?? workspace.name, workspace.id); + const name = names.has(baseName) ? `${baseName}-${workspace.id.slice(0, 8)}` : baseName; + names.add(name); + [binding] = await measureSandboxOperation("work_folder.db.query", { operation: "insert_task_repository_bindings", requestCount: 1 }, async () => (db.insert(taskRepositoryBindings).values({ companyId: input.companyId, taskId: taskId, + workspaceId: workspace.id, name, repoUrl: workspace.repoUrl, repoRef: workspace.repoRef ?? workspace.defaultRef }).returning())); } - } - } else { - const init = await target.runner!.execute({ command: "git", args: ["-C", temporary, "init"], bypassSession: true, timeoutMs: 10_000 }); - if (init.exitCode !== 0) throw new Error(`Repository ${binding.name} could not be restored`); - const remote = await target.runner!.execute({ command: "git", args: ["-C", temporary, "remote", "add", "origin", binding.repoUrl!], bypassSession: true, timeoutMs: 10_000 }); - if (remote.exitCode !== 0) throw new Error(`Repository ${binding.name} remote could not be restored`); + if (!binding) throw new Error("Repository binding could not be created"); + if (binding.repoUrl !== workspace.repoUrl) throw new Error(`Repository ${binding.name} configuration changed; saved work was retained`); + if (binding.repoRef !== (workspace.repoRef ?? workspace.defaultRef)) throw new Error(`Repository ${binding.name} starting ref changed; saved work was retained`); + const root = path.posix.join(paths.repos!, binding.name); + const probe = await measureSandboxOperation("work_folder.repository.probe", { repositoryIndex, requestCount: 1 }, async () => (target.runner!.execute({ command: "git", args: ["-C", root, "rev-parse", "--git-dir"], bypassSession: true, timeoutMs: 10_000 }))); + const freshCheckout = probe.exitCode !== 0; + // Checkout reuse is observed on disk; a durable restore remains cold. + repositorySpan.set({ exists: !freshCheckout, reused: !freshCheckout, cold: freshCheckout, warm: !freshCheckout, cacheHit: false }); + if (freshCheckout) { + // Publish the checkout directory only after every restore object or + // clone step completes. An interrupted attempt cannot masquerade as a + // reusable checkout merely because it contains a .git directory. + const temporary = path.posix.join(staging, `repo-${binding.id}-${randomUUID()}`); + const restored = await repositories.restore(binding, temporary, staging); + repositorySpan.set({ cacheHit: restored }); + if (!restored) { + const auth = await measureSandboxOperation("work_folder.repository.credentials", { repositoryIndex }, async () => (resolveGitAuth(workspace.repoUrl!))); + const result = await measureSandboxOperation("work_folder.repository.clone", { repositoryIndex, requestCount: 1 }, async () => (target.runner!.execute({ command: "git", args: [...(auth?.configArgs ?? []), "clone", "--no-hardlinks", + "--", workspace.repoUrl!, temporary], + env: { GIT_TERMINAL_PROMPT: "0", ...(auth?.env ?? {}) }, bypassSession: true, timeoutMs: 300_000 }))); + if (result.exitCode !== 0 || result.timedOut) throw new Error(`Required repository ${binding.name} could not be cloned`); + const repoRef = binding.repoRef; + if (repoRef) { + const checkout = await measureSandboxOperation("work_folder.repository.checkout", { repositoryIndex, requestCount: 1 }, async () => (target.runner!.execute({ command: "git", args: ["-C", temporary, "checkout", repoRef, "--"], bypassSession: true, timeoutMs: 60_000 }))); + if (checkout.exitCode !== 0 || checkout.timedOut) throw new Error(`Required repository ${binding.name} ref could not be checked out`); + } + if (primary && input.primaryBranchName) { + const branch = input.primaryBranchName; + const valid = await measureSandboxOperation("work_folder.repository.validate_branch", { repositoryIndex, requestCount: 1 }, async () => (target.runner!.execute({ command: "git", args: ["check-ref-format", "--branch", branch], bypassSession: true, timeoutMs: 10_000 }))); + if (valid.exitCode !== 0 || valid.stdout.trim() !== branch) throw new Error(`Required repository ${binding.name} branch is invalid`); + // Honor the task's existing branch policy on the initial clone. + // Restores and warm starts keep the saved HEAD and index untouched. + const checkout = await measureSandboxOperation("work_folder.repository.checkout", { repositoryIndex, requestCount: 1 }, async () => (target.runner!.execute({ command: "git", args: ["-C", temporary, "checkout", branch, "--"], bypassSession: true, timeoutMs: 60_000 }))); + if (checkout.exitCode !== 0) { + const create = await measureSandboxOperation("work_folder.repository.create_branch", { repositoryIndex, requestCount: 1 }, async () => (target.runner!.execute({ command: "git", args: ["-C", temporary, "checkout", "-b", branch], bypassSession: true, timeoutMs: 60_000 }))); + if (create.exitCode !== 0 || create.timedOut) throw new Error(`Required repository ${binding.name} task branch could not be created`); + } + } + } else { + const init = await measureSandboxOperation("work_folder.repository.restore_init", { repositoryIndex, requestCount: 1 }, async () => (target.runner!.execute({ command: "git", args: ["-C", temporary, "init"], bypassSession: true, timeoutMs: 10_000 }))); + if (init.exitCode !== 0) throw new Error(`Repository ${binding.name} could not be restored`); + const remote = await measureSandboxOperation("work_folder.repository.restore_remote", { repositoryIndex, requestCount: 1 }, async () => (target.runner!.execute({ command: "git", args: ["-C", temporary, "remote", "add", "origin", binding.repoUrl!], bypassSession: true, timeoutMs: 10_000 }))); + if (remote.exitCode !== 0) throw new Error(`Repository ${binding.name} remote could not be restored`); + } + await measureSandboxOperation("work_folder.repository.publish", { repositoryIndex }, async () => (transport.moveRoot(temporary, root))); + } + // Warm checkouts retain completed setup. A replacement only restores + // durable repository files, so setup must recreate ignored dependencies + // and caches that are deliberately outside the checkpoint guarantee. + const setupCommand = workspace.setupCommand; + if ((!binding.setupComplete || freshCheckout) && setupCommand) { + const setup = await measureSandboxOperation("work_folder.repository.setup", { repositoryIndex, requestCount: 1 }, async () => (target.runner!.execute({ command: "sh", args: ["-c", setupCommand], cwd: root, bypassSession: true, timeoutMs: 300_000 }))); + if (setup.exitCode !== 0 || setup.timedOut) throw new Error(`Repository ${binding.name} setup failed`); + } + await measureSandboxOperation("work_folder.db.query", { operation: "update_task_repository_bindings", requestCount: 1 }, async () => (db.update(taskRepositoryBindings).set({ setupComplete: true, retiredAt: null }).where(eq(taskRepositoryBindings.id, binding.id)))); + bindings.push({ binding, root }); + manifest.repositories.push({ bindingId: binding.id, workspaceId: workspace.id, name: binding.name, primary }); + await saveState("starting"); + }); } - await transport.moveRoot(temporary, root); - } - // Warm checkouts retain completed setup. A replacement only restores - // durable repository files, so setup must recreate ignored dependencies - // and caches that are deliberately outside the checkpoint guarantee. - if ((!binding.setupComplete || freshCheckout) && workspace.setupCommand) { - const setup = await target.runner!.execute({ command: "sh", args: ["-c", workspace.setupCommand], cwd: root, bypassSession: true, timeoutMs: 300_000 }); - if (setup.exitCode !== 0 || setup.timedOut) throw new Error(`Repository ${binding.name} setup failed`); - } - await db.update(taskRepositoryBindings).set({ setupComplete: true, retiredAt: null }).where(eq(taskRepositoryBindings.id, binding.id)); - bindings.push({ binding, root }); - manifest.repositories.push({ bindingId: binding.id, workspaceId: workspace.id, name: binding.name, primary }); + for (const old of existing) if (!workspaces.some((workspace) => workspace.id === old.workspaceId)) { + await measureSandboxOperation("work_folder.db.query", { operation: "update_task_repository_bindings", requestCount: 1 }, async () => (db.update(taskRepositoryBindings).set({ retiredAt: new Date() }).where(eq(taskRepositoryBindings.id, old.id)))); + } + }); + } + async function saveState(state: "starting" | "saving" | "saved" | "failed", error: string | null = null) { + return measureSandboxOperation("work_folder.progress.save", { phase: state, requestCount: 1 }, async () => { + await measureSandboxOperation("work_folder.db.query", { operation: "update_work_folder_runs", requestCount: 1 }, async () => (db.update(workFolderRuns).set({ state, baselines, pendingOperations, manifest, error, updatedAt: new Date(), + ...(state === "saved" ? { lastSavedAt: new Date() } : {}) }).where(eq(workFolderRuns.runId, input.runId)))); + }); + } + async function recordCheckpoint(action: string) { + return measureSandboxOperation("work_folder.activity.record", { requestCount: 1 }, async () => { + await logActivity(db, { companyId: input.companyId, actorType: "agent", actorId: input.agentId, + agentId: input.agentId, runId: input.runId, issueId: taskId, + responsibleUserIdOverride: responsibleUserId, action, entityType: "heartbeat_run", entityId: input.runId, + details: { phase: "started", scopes: WORK_FOLDER_SCOPES.filter((scope) => Boolean(folders[scope])), repositories: bindings.length } }); + }); + } + try { + // Record intent before mutations. An unavailable audit store blocks new + // work instead of turning an already completed save into a false failure. + // The run's persisted state/lastSavedAt records checkpoint completion. + await recordCheckpoint("work_folder.prepared"); + await seedAttachments(); + await importAgentFiles(); + // A resumed sandbox can hold edits newer than its last completed checkpoint. + if (previous) for (const scope of WORK_FOLDER_SCOPES) await outgoing(scope); + for (const scope of WORK_FOLDER_SCOPES) await incoming(scope); + await prepareRepositories(); await saveState("starting"); + if (previous?.refreshRequested) await measureSandboxOperation("work_folder.db.query", { operation: "update_work_folder_runs", requestCount: 1 }, async () => (db.update(workFolderRuns).set({ refreshRequested: false }) + .where(eq(workFolderRuns.runId, previous.runId)))); + } catch (error) { + await saveState("failed", "Work folder preparation failed; existing files were retained"); + throw error; } - for (const old of existing) if (!workspaces.some((workspace) => workspace.id === old.workspaceId)) { - await db.update(taskRepositoryBindings).set({ retiredAt: new Date() }).where(eq(taskRepositoryBindings.id, old.id)); - } - } - async function saveState(state: "starting" | "saving" | "saved" | "failed", error: string | null = null) { - await db.update(workFolderRuns).set({ state, baselines, pendingOperations, manifest, error, updatedAt: new Date(), - ...(state === "saved" ? { lastSavedAt: new Date() } : {}) }).where(eq(workFolderRuns.runId, input.runId)); - } - async function recordCheckpoint(action: string) { - await logActivity(db, { companyId: input.companyId, actorType: "agent", actorId: input.agentId, - agentId: input.agentId, runId: input.runId, issueId: input.taskId, - responsibleUserIdOverride: input.responsibleUserId, action, entityType: "heartbeat_run", entityId: input.runId, - details: { phase: "started", scopes: WORK_FOLDER_SCOPES.filter((scope) => Boolean(folders[scope])), repositories: bindings.length } }); - } - try { - // Record intent before mutations. An unavailable audit store blocks new - // work instead of turning an already completed save into a false failure. - // The run's persisted state/lastSavedAt records checkpoint completion. - await recordCheckpoint("work_folder.prepared"); - await seedAttachments(); - await importAgentFiles(); - // A resumed sandbox can hold edits newer than its last completed checkpoint. - if (previous) for (const scope of WORK_FOLDER_SCOPES) await outgoing(scope); - for (const scope of WORK_FOLDER_SCOPES) await incoming(scope); - await prepareRepositories(); - await saveState("starting"); - if (previous?.refreshRequested) await db.update(workFolderRuns).set({ refreshRequested: false }) - .where(eq(workFolderRuns.runId, previous.runId)); - } catch (error) { - await saveState("failed", "Work folder preparation failed; existing files were retained"); - throw error; - } - const checkpointer = startWorkFolderCheckpointer({ - async checkpoint() { - await assertBindings(); - await recordCheckpoint("work_folder.checkpoint"); - await saveState("saving"); - for (const scope of WORK_FOLDER_SCOPES) await outgoing(scope); - for (const { binding, root } of bindings) await repositories.checkpoint(binding, root); - await saveState("saved"); - }, - async onError(error) { - const failure = error as { name?: unknown; code?: unknown; $metadata?: { httpStatusCode?: unknown } } | null; - // Do not log SDK request objects, headers, file contents or credentials. - const label = (value: unknown) => typeof value === "string" && /^[A-Za-z0-9_]{1,80}$/.test(value) ? value : null; - logger.warn({ runId: input.runId, errorName: label(failure?.name), errorCode: label(failure?.code), - httpStatus: typeof failure?.$metadata?.httpStatusCode === "number" ? failure.$metadata.httpStatusCode : null }, - "Work folder checkpoint failed; retaining sandbox for recovery"); - await saveState("failed", "Files could not be saved; the sandbox must be retained for recovery"); - }, - }); - let completion: Promise | null = null; - function stop(beforeCompletion?: () => Promise) { - // Error teardown must observe the original outcome, including failures - // after the data save. A later run owns any recovery of this working copy. - completion ??= (async () => { - await checkpointer.stop(); - const [run] = await db.select({ refreshRequested: workFolderRuns.refreshRequested }).from(workFolderRuns) - .where(eq(workFolderRuns.runId, input.runId)); - if (run?.refreshRequested) { - // The successful final flush protects edits before incoming refresh. - try { + const checkpointer = startWorkFolderCheckpointer({ + async checkpoint() { + const checkpoint = () => measureSandboxOperation("work_folder.checkpoint", { phase: completion ? "final" : explicitFlushes ? "explicit" : "periodic" }, async () => { await assertBindings(); - for (const scope of WORK_FOLDER_SCOPES) await incoming(scope); - await db.update(workFolderRuns).set({ refreshRequested: false, baselines, updatedAt: new Date() }) - .where(eq(workFolderRuns.runId, input.runId)); - } catch (error) { - await saveState("failed", "Work folder refresh failed; existing files were retained"); - throw error; + await recordCheckpoint("work_folder.checkpoint"); + await saveState("saving"); + for (const scope of WORK_FOLDER_SCOPES) await outgoing(scope); + for (const [repositoryIndex, { binding, root }] of bindings.entries()) await measureSandboxOperation("work_folder.repository.checkpoint", { repositoryIndex }, async () => repositories.checkpoint(binding, root)); + await saveState("saved"); + }); + return completion || explicitFlushes ? checkpoint() : runInRunContext(checkpoint); + }, + async onError(error) { + const failure = error as { name?: unknown; code?: unknown; $metadata?: { httpStatusCode?: unknown } } | null; + // Do not log SDK request objects, headers, file contents or credentials. + const label = (value: unknown) => typeof value === "string" && /^[A-Za-z0-9_]{1,80}$/.test(value) ? value : null; + logger.warn({ runId: input.runId, errorName: label(failure?.name), errorCode: label(failure?.code), + httpStatus: typeof failure?.$metadata?.httpStatusCode === "number" ? failure.$metadata.httpStatusCode : null }, + "Work folder checkpoint failed; retaining sandbox for recovery"); + await saveState("failed", "Files could not be saved; the sandbox must be retained for recovery"); + }, + }); + let completion: Promise | null = null; + let explicitFlushes = 0; + function stop(beforeCompletion?: () => Promise) { + // Error teardown must observe the original outcome, including failures + // after the data save. A later run owns any recovery of this working copy. + completion ??= measureSandboxOperation("work_folder.finalize", { phase: "final" }, async () => { + await checkpointer.stop(); + const [run] = await measureSandboxOperation("work_folder.db.query", { operation: "select_work_folder_runs", requestCount: 1 }, async () => (db.select({ refreshRequested: workFolderRuns.refreshRequested }).from(workFolderRuns) + .where(eq(workFolderRuns.runId, input.runId)))); + if (run?.refreshRequested) { + // The successful final flush protects edits before incoming refresh. + try { + await assertBindings(); + for (const scope of WORK_FOLDER_SCOPES) await incoming(scope); + await measureSandboxOperation("work_folder.db.query", { operation: "update_work_folder_runs", requestCount: 1 }, async () => (db.update(workFolderRuns).set({ refreshRequested: false, baselines, updatedAt: new Date() }) + .where(eq(workFolderRuns.runId, input.runId)))); + } catch (error) { + await saveState("failed", "Work folder refresh failed; existing files were retained"); + throw error; + } } - } - // Publish resume identity after data is durable, before releasing a turn. - await beforeCompletion?.(); - manifest.finalCheckpointAt = new Date().toISOString(); - await saveState("saved"); - })(); - return completion; - } - return { manifest, home, identityChanged, primaryRepo: bindings.find(({ binding }) => manifest.repositories.some((repo) => repo.bindingId === binding.id && repo.primary))?.root ?? bindings[0]?.root ?? paths.task!, - env: { HOME: home, AGENT_HOME: paths.agent!, PAPERCLIP_PRIMARY_REPO: bindings.find(({ binding }) => manifest.repositories.some((repo) => repo.bindingId === binding.id && repo.primary))?.root ?? bindings[0]?.root ?? paths.task!, PAPERCLIP_TASK_DIR: paths.task!, PAPERCLIP_AGENT_DIR: paths.agent!, - PAPERCLIP_USER_DIR: paths.user!, PAPERCLIP_PROJECT_DIR: paths.project!, PAPERCLIP_REPOS_DIR: paths.repos! }, - flush: checkpointer.flush, stop }; + // Publish resume identity after data is durable, before releasing a turn. + await beforeCompletion?.(); + manifest.finalCheckpointAt = new Date().toISOString(); + await saveState("saved"); + }); + return completion; + } + return { manifest, home, identityChanged, primaryRepo: bindings.find(({ binding }) => manifest.repositories.some((repo) => repo.bindingId === binding.id && repo.primary))?.root ?? bindings[0]?.root ?? paths.task!, + env: { HOME: home, AGENT_HOME: paths.agent!, PAPERCLIP_PRIMARY_REPO: bindings.find(({ binding }) => manifest.repositories.some((repo) => repo.bindingId === binding.id && repo.primary))?.root ?? bindings[0]?.root ?? paths.task!, PAPERCLIP_TASK_DIR: paths.task!, PAPERCLIP_AGENT_DIR: paths.agent!, + PAPERCLIP_USER_DIR: paths.user!, PAPERCLIP_PROJECT_DIR: paths.project!, PAPERCLIP_REPOS_DIR: paths.repos! }, + flush: () => measureSandboxOperation("work_folder.flush", { phase: "explicit" }, async () => { + explicitFlushes++; + try { await checkpointer.flush(); } finally { explicitFlushes--; } + }), stop }; + }); } diff --git a/server/src/services/scripts/work-folder-io.mjs b/server/src/services/scripts/work-folder-io.mjs index 72c4489891..28ac57bec2 100644 --- a/server/src/services/scripts/work-folder-io.mjs +++ b/server/src/services/scripts/work-folder-io.mjs @@ -6,7 +6,30 @@ import os from "node:os"; import { createHash, createHmac, randomUUID, timingSafeEqual } from "node:crypto"; import { execFileSync } from "node:child_process"; -let input = JSON.parse(Buffer.from(process.argv[1], "base64").toString("utf8")); +const commandStartedAt = performance.now(); +const remotePhases = []; +let performanceRequested = true; +const metrics = { droppedPhases: 0, executionMs: 0, scanMs: 0, listMs: 0, gitListMs: 0, hashMs: 0, readMs: 0, writeMs: 0, publishMs: 0, + decodeMs: 0, encodeMs: 0, files: 0, bytes: 0, hashFiles: 0, hashBytes: 0, requestCount: 0 }; +function measured(key, work) { + if (!performanceRequested) return work(); + const startedAt = performance.now(); + try { return work(); } finally { + const durationMs = performance.now() - startedAt; + metrics[key] += durationMs; + if (remotePhases.length < 256) remotePhases.push({ phase: key, startOffsetMs: startedAt - commandStartedAt, durationMs }); + else metrics.droppedPhases++; + } +} +let input = measured("decodeMs", () => JSON.parse(Buffer.from(process.argv[1], "base64").toString("utf8"))); +performanceRequested = input.performance === true; +function output(result) { + const encoded = measured("encodeMs", () => JSON.stringify(result)); + metrics.executionMs = performance.now() - commandStartedAt; + process.stdout.write(performanceRequested + ? `{"workFolderPerformanceVersion":1,"result":${encoded},"performance":${JSON.stringify(metrics)},"remotePhases":${JSON.stringify(remotePhases)}}` + : encoded); +} const MAX_CHUNK = 256 * 1024; const MAX_ENTRIES = 100_000; function safeRelative(value) { @@ -51,18 +74,22 @@ function checked(target, directory = false) { } function children(target) { const fd = checked(target, true); - try { return fs.readdirSync(process.platform === "linux" ? `/proc/self/fd/${fd}` : target).sort(); } + try { return measured("listMs", () => fs.readdirSync(process.platform === "linux" ? `/proc/self/fd/${fd}` : target).sort()); } finally { fs.closeSync(fd); } } function checksum(target) { - const fd = checked(target); - try { - const hash = createHash("sha256"); - const buffer = Buffer.alloc(MAX_CHUNK); - let count; - while ((count = fs.readSync(fd, buffer, 0, buffer.length, null)) > 0) hash.update(buffer.subarray(0, count)); - return hash.digest("hex"); - } finally { fs.closeSync(fd); } + return measured("hashMs", () => { + const fd = checked(target); + try { + const hash = createHash("sha256"); + const buffer = Buffer.alloc(MAX_CHUNK); + let count; + while ((count = fs.readSync(fd, buffer, 0, buffer.length, null)) > 0) { + metrics.hashBytes += count; hash.update(buffer.subarray(0, count)); + } + return hash.digest("hex"); + } finally { fs.closeSync(fd); metrics.hashFiles++; } + }); } function ensureDirectory(target) { withParent(target, false, (anchored) => { @@ -110,8 +137,8 @@ function scan() { const gitDir = path.join(input.root, ".git"); const fd = checked(gitDir, true); fs.closeSync(fd); if (fs.existsSync(path.join(gitDir, "objects/info/alternates"))) throw new Error("repository_is_not_independent"); - const files = execFileSync("git", ["-C", input.root, "ls-files", "--cached", "--others", "--exclude-standard", "-z"], - { encoding: "utf8", maxBuffer: 16 * 1024 * 1024 }).split("\0").filter(Boolean); + const files = measured("gitListMs", () => execFileSync("git", ["-C", input.root, "ls-files", "--cached", "--others", "--exclude-standard", "-z"], + { encoding: "utf8", maxBuffer: 16 * 1024 * 1024 })).split("\0").filter(Boolean); for (const relative of [...new Set(files)].sort()) { // Git reports nested repositories with a trailing slash. Private runner // caches can contain them and must be excluded before path validation. @@ -131,6 +158,7 @@ function scan() { function execute(request) { input = request; +metrics.requestCount++; let result; if (input.operation === "home") { result = { home: os.homedir() }; @@ -145,21 +173,22 @@ if (input.operation === "home") { })); result = {}; } else { const rootFd = checked(input.root, true); fs.closeSync(rootFd); - if (input.operation === "scan") result = scan(); + if (input.operation === "scan") result = measured("scanMs", () => scan()); else if (input.operation === "read") { const fd = checked(full(input.path)); try { const length = input.length ?? MAX_CHUNK; if (!Number.isSafeInteger(length) || length < 1 || length > 1024 * 1024) throw new Error("invalid_read_length"); const buffer = Buffer.alloc(length); - const count = fs.readSync(fd, buffer, 0, buffer.length, input.offset); - result = { data: buffer.subarray(0, count).toString("base64") }; + const count = measured("readMs", () => fs.readSync(fd, buffer, 0, buffer.length, input.offset)); + metrics.files++; metrics.bytes += count; + result = { data: measured("encodeMs", () => buffer.subarray(0, count).toString("base64")) }; } finally { fs.closeSync(fd); } } else if (input.operation === "mkdir") { parents(input.path); ensureDirectory(full(input.path)); result = {}; } else if (input.operation === "write") { parents(input.path); - const buffer = Buffer.from(input.data, "base64"); + const buffer = measured("decodeMs", () => Buffer.from(input.data, "base64")); if (buffer.length > MAX_CHUNK) throw new Error("chunk_too_large"); const target = full(input.path); // Temporary writes happen in a separate host-selected staging root. @@ -170,7 +199,8 @@ if (input.operation === "home") { const stat = fs.fstatSync(fd); if (!stat.isFile() || stat.nlink !== 1) throw new Error("unsupported_file"); if (stat.size !== input.offset) throw new Error("invalid_chunk_offset"); - fs.writeSync(fd, buffer, 0, buffer.length, input.offset); + measured("writeMs", () => fs.writeSync(fd, buffer, 0, buffer.length, input.offset)); + metrics.files++; metrics.bytes += buffer.length; } finally { fs.closeSync(fd); } }); result = {}; @@ -182,7 +212,7 @@ if (input.operation === "home") { try { fs.fchmodSync(sourceFd, input.executable ? 0o700 : 0o600); } finally { fs.closeSync(sourceFd); } const target = full(input.path); try { const fd = checked(target); fs.closeSync(fd); } catch (error) { if (error.code !== "ENOENT") throw error; } - withParent(source, false, (from) => withParent(target, false, (to) => fs.renameSync(from, to))); result = {}; + measured("publishMs", () => withParent(source, false, (from) => withParent(target, false, (to) => fs.renameSync(from, to)))); result = {}; } else if (input.operation === "symlink") { parents(input.path); const target = full(input.path); @@ -314,9 +344,9 @@ if (request.operation === "read-batch") { } const results = request.entries.map((entry) => execute({ operation: "read", root: request.root, path: entry.path, offset: 0, length: Math.max(1, entry.byteSize) })); - process.stdout.write(JSON.stringify(results)); + output(results); } else if (request.operation === "batch-status") { - process.stdout.write(JSON.stringify(batchReceipt(request).read())); + output(batchReceipt(request).read()); } else if (request.operation === "batch") { // Stdin is bounded and hashed exactly, before parsing or claiming execution. const chunks = []; @@ -331,7 +361,7 @@ if (request.operation === "read-batch") { const body = Buffer.concat(chunks); if (createHash("sha256").update(body).digest("hex") !== request.batchSha256) throw new Error("batch_body_hash_mismatch"); const receipt = batchReceipt(request); - const operations = JSON.parse(body.toString("utf8")); + const operations = measured("decodeMs", () => JSON.parse(body.toString("utf8"))); if (!Array.isArray(operations) || operations.length > 512) throw new Error("invalid_batch"); for (const operation of operations) { if (!operation || !["write", "publish", "mkdir"].includes(operation.operation)) throw new Error("invalid_batch_operation"); @@ -339,8 +369,8 @@ if (request.operation === "read-batch") { } if (!receipt.claim()) { const status = receipt.read(); - if (status.state === "completed") process.stdout.write(JSON.stringify({ completed: status.completed })); - else if (status.state === "running") process.stdout.write(JSON.stringify({ pending: true })); + if (status.state === "completed") output({ completed: status.completed }); + else if (status.state === "running") output({ pending: true }); else throw new Error(status.state === "failed" ? "batch_execution_failed" : "batch_receipt_disappeared"); } else { try { @@ -358,8 +388,8 @@ if (request.operation === "read-batch") { "hardlink_not_allowed", "chunk_too_large", "invalid_chunk_offset", "content_changed_during_transfer"]; throw new Error(safeErrors.includes(error.message) ? error.message : "batch_execution_failed"); } - process.stdout.write(JSON.stringify({ completed: operations.length })); + output({ completed: operations.length }); } } else { - process.stdout.write(JSON.stringify(execute(request))); + output(execute(request)); } diff --git a/server/src/services/work-folder-access.ts b/server/src/services/work-folder-access.ts index e1f6ec0266..05b9d3ced3 100644 --- a/server/src/services/work-folder-access.ts +++ b/server/src/services/work-folder-access.ts @@ -1,3 +1,4 @@ +import { measureSandboxOperation } from "./sandbox-performance.js"; import { and, eq } from "drizzle-orm"; import { agents, companyMemberships, heartbeatRuns, issues, projects, workFolderRuns, type Db } from "@paperclipai/db"; import type { WorkFolderOwner } from "@paperclipai/shared"; @@ -6,55 +7,57 @@ import { authorizationService, type AuthorizationActor, type AuthorizationResour /** Private-file access never inherits the responsible-user shadow-mode bypass. */ export async function assertWorkFolderAccess(db: Db, actor: AuthorizationActor, owner: WorkFolderOwner, write: boolean) { - const deny = () => { throw notFound("Work folder not found"); }; - if (actor.type === "none") deny(); - if (actor.type === "agent" && actor.companyId !== owner.companyId) deny(); - if (actor.type === "board" && actor.source !== "local_implicit" && !actor.companyIds?.includes(owner.companyId)) deny(); - const userId = actor.type === "board" ? actor.userId : actor.onBehalfOfUserId; - if (actor.source !== "local_implicit" && userId) { - const [membership] = await db.select().from(companyMemberships).where(and( - eq(companyMemberships.companyId, owner.companyId), eq(companyMemberships.principalType, "user"), - eq(companyMemberships.principalId, userId), eq(companyMemberships.status, "active"))); - if (!membership || (write && membership.membershipRole === "viewer")) deny(); - } - if (owner.scope === "user") { - if (!userId || userId !== owner.ownerId) deny(); - if (actor.type === "agent") { - if (!actor.runId || !actor.agentId || actor.source !== "agent_jwt") deny(); - const [run] = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.id, actor.runId!), - eq(heartbeatRuns.companyId, owner.companyId), eq(heartbeatRuns.agentId, actor.agentId!))); - const [binding] = await db.select().from(workFolderRuns).where(eq(workFolderRuns.runId, actor.runId!)); - if (!run || run.status !== "running" || run.responsibleUserId !== owner.ownerId - || binding?.manifest.responsibleUserId !== owner.ownerId) deny(); + return measureSandboxOperation("work_folder.access", { scope: owner.scope, operation: write ? "write" : "read" }, async () => { + const deny = () => { throw notFound("Work folder not found"); }; + if (actor.type === "none") deny(); + if (actor.type === "agent" && actor.companyId !== owner.companyId) deny(); + if (actor.type === "board" && actor.source !== "local_implicit" && !actor.companyIds?.includes(owner.companyId)) deny(); + const userId = actor.type === "board" ? actor.userId : actor.onBehalfOfUserId; + if (actor.source !== "local_implicit" && userId) { + const [membership] = await measureSandboxOperation("work_folder.db.query", { operation: "select_company_memberships", requestCount: 1 }, async () => (db.select().from(companyMemberships).where(and( + eq(companyMemberships.companyId, owner.companyId), eq(companyMemberships.principalType, "user"), + eq(companyMemberships.principalId, userId), eq(companyMemberships.status, "active"))))); + if (!membership || (write && membership.membershipRole === "viewer")) deny(); } - return; - } - let resource: AuthorizationResource; - let action: "issue:read" | "issue:mutate" | "agent:read" | "agent_config:update" | "project:read"; - if (owner.scope === "task") { - const [issue] = await db.select().from(issues).where(and(eq(issues.id, owner.ownerId), eq(issues.companyId, owner.companyId))); - if (!issue) return deny(); - resource = { type: "issue", companyId: owner.companyId, issueId: issue.id, projectId: issue.projectId, - parentIssueId: issue.parentId, assigneeAgentId: issue.assigneeAgentId, assigneeUserId: issue.assigneeUserId, - originKind: issue.originKind, originId: issue.originId, status: issue.status }; - action = write ? "issue:mutate" : "issue:read"; - } else if (owner.scope === "agent") { - const [agent] = await db.select().from(agents).where(and(eq(agents.id, owner.ownerId), eq(agents.companyId, owner.companyId))); - if (!agent) return deny(); - if (write && actor.type === "agent" && actor.agentId !== agent.id) deny(); - resource = { type: "agent", companyId: owner.companyId, agentId: agent.id }; - action = "agent:read"; - } else { - const [project] = await db.select().from(projects).where(and(eq(projects.id, owner.ownerId), eq(projects.companyId, owner.companyId))); - if (!project) return deny(); - resource = { type: "project", companyId: owner.companyId, projectId: project.id }; - action = "project:read"; - } - const authz = authorizationService(db); - if (!(await authz.decide({ actor, action, resource })).allowed) deny(); - if (actor.type === "agent" && userId) { - const responsibleActor: AuthorizationActor = { type: "board", source: "session", userId, - companyIds: [owner.companyId], memberships: actor.onBehalfOfMemberships, ignoreInstanceAdmin: true }; - if (!(await authz.decide({ actor: responsibleActor, action, resource })).allowed) deny(); - } + if (owner.scope === "user") { + if (!userId || userId !== owner.ownerId) deny(); + if (actor.type === "agent") { + if (!actor.runId || !actor.agentId || actor.source !== "agent_jwt") deny(); + const [run] = await measureSandboxOperation("work_folder.db.query", { operation: "select_heartbeat_runs", requestCount: 1 }, async () => (db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.id, actor.runId!), + eq(heartbeatRuns.companyId, owner.companyId), eq(heartbeatRuns.agentId, actor.agentId!))))); + const [binding] = await measureSandboxOperation("work_folder.db.query", { operation: "select_work_folder_runs", requestCount: 1 }, async () => (db.select().from(workFolderRuns).where(eq(workFolderRuns.runId, actor.runId!)))); + if (!run || run.status !== "running" || run.responsibleUserId !== owner.ownerId + || binding?.manifest.responsibleUserId !== owner.ownerId) deny(); + } + return; + } + let resource: AuthorizationResource; + let action: "issue:read" | "issue:mutate" | "agent:read" | "agent_config:update" | "project:read"; + if (owner.scope === "task") { + const [issue] = await measureSandboxOperation("work_folder.db.query", { operation: "select_issues", requestCount: 1 }, async () => (db.select().from(issues).where(and(eq(issues.id, owner.ownerId), eq(issues.companyId, owner.companyId))))); + if (!issue) return deny(); + resource = { type: "issue", companyId: owner.companyId, issueId: issue.id, projectId: issue.projectId, + parentIssueId: issue.parentId, assigneeAgentId: issue.assigneeAgentId, assigneeUserId: issue.assigneeUserId, + originKind: issue.originKind, originId: issue.originId, status: issue.status }; + action = write ? "issue:mutate" : "issue:read"; + } else if (owner.scope === "agent") { + const [agent] = await measureSandboxOperation("work_folder.db.query", { operation: "select_agents", requestCount: 1 }, async () => (db.select().from(agents).where(and(eq(agents.id, owner.ownerId), eq(agents.companyId, owner.companyId))))); + if (!agent) return deny(); + if (write && actor.type === "agent" && actor.agentId !== agent.id) deny(); + resource = { type: "agent", companyId: owner.companyId, agentId: agent.id }; + action = "agent:read"; + } else { + const [project] = await measureSandboxOperation("work_folder.db.query", { operation: "select_projects", requestCount: 1 }, async () => (db.select().from(projects).where(and(eq(projects.id, owner.ownerId), eq(projects.companyId, owner.companyId))))); + if (!project) return deny(); + resource = { type: "project", companyId: owner.companyId, projectId: project.id }; + action = "project:read"; + } + const authz = authorizationService(db); + if (!(await measureSandboxOperation("work_folder.authorization.decide", { scope: owner.scope, operation: write ? "write" : "read" }, async () => (authz.decide({ actor, action, resource })))).allowed) deny(); + if (actor.type === "agent" && userId) { + const responsibleActor: AuthorizationActor = { type: "board", source: "session", userId, + companyIds: [owner.companyId], memberships: actor.onBehalfOfMemberships, ignoreInstanceAdmin: true }; + if (!(await measureSandboxOperation("work_folder.authorization.decide", { scope: owner.scope, operation: write ? "write" : "read" }, async () => (authz.decide({ actor: responsibleActor, action, resource })))).allowed) deny(); + } + }); } diff --git a/server/src/services/work-folder-agent-import.ts b/server/src/services/work-folder-agent-import.ts index 3805de6318..660dd87cc6 100644 --- a/server/src/services/work-folder-agent-import.ts +++ b/server/src/services/work-folder-agent-import.ts @@ -1,3 +1,4 @@ +import { measureSandboxOperation, measureSandboxStream } from "./sandbox-performance.js"; import { constants } from "node:fs"; import fs from "node:fs/promises"; import path from "node:path"; @@ -10,39 +11,39 @@ export interface ManagedAgentFile { path: string; kind: "file" | "directory"; ex export async function* managedAgentFiles(root: string): AsyncGenerator { const absolute = path.resolve(root); let current = path.parse(absolute).root; - let parent = await fs.open(current, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW); + let parent = await measureSandboxOperation("work_folder.agent_import.disk", { operation: "open" }, async () => (fs.open(current, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW))); const anchored = (fd: number, fallback: string) => process.platform === "linux" ? `/proc/self/fd/${fd}` : fallback; try { for (const segment of absolute.slice(current.length).split(path.sep).filter(Boolean)) { const next = path.join(anchored(parent.fd, current), segment); - const child = await fs.open(next, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW).catch((error) => { + const child = await measureSandboxOperation("work_folder.agent_import.disk", { operation: "open" }, async () => (fs.open(next, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW).catch((error) => { if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; throw error; - }); + }))); if (!child) return; - await parent.close(); parent = child; current = path.join(current, segment); + await measureSandboxOperation("work_folder.agent_import.disk", { operation: "close" }, async () => (parent.close())); parent = child; current = path.join(current, segment); } let entries = 0; async function* visit(directory: typeof parent, fallback: string, relative: string): AsyncGenerator { - for (const name of (await fs.readdir(anchored(directory.fd, fallback))).sort()) { + for (const name of (await measureSandboxOperation("work_folder.agent_import.disk", { operation: "list" }, async () => (fs.readdir(anchored(directory.fd, fallback))))).sort()) { if (excluded.has(name)) continue; if (++entries > 100_000) throw new Error("Managed agent import exceeds its file limit"); const filename = relative ? `${relative}/${name}` : name; const source = path.join(anchored(directory.fd, fallback), name); - const handle = await fs.open(source, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + const handle = await measureSandboxOperation("work_folder.agent_import.disk", { operation: "open" }, async () => (fs.open(source, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK))); try { - const stat = await handle.stat(); + const stat = await measureSandboxOperation("work_folder.agent_import.disk", { operation: "metadata" }, async () => (handle.stat())); if (stat.isDirectory()) { yield { path: filename, kind: "directory", executable: false }; yield* visit(handle, path.join(fallback, name), filename); } else if (stat.isFile() && stat.nlink === 1 && stat.size <= 1024 ** 3) { - const body = handle.createReadStream({ autoClose: false, highWaterMark: 256 * 1024 }); + const body = measureSandboxStream("work_folder.agent_import.body", { fileIndex: entries - 1, bytes: stat.size }, handle.createReadStream({ autoClose: false, highWaterMark: 256 * 1024 })); try { yield { path: filename, kind: "file", executable: Boolean(stat.mode & 0o111), body }; } finally { body.destroy(); } } else throw new Error("Managed agent files cannot contain hard links or special files"); - } finally { await handle.close(); } + } finally { await measureSandboxOperation("work_folder.agent_import.disk", { operation: "close" }, async () => (handle.close())); } } } yield* visit(parent, absolute, ""); - } finally { await parent.close(); } + } finally { await measureSandboxOperation("work_folder.agent_import.disk", { operation: "close" }, async () => (parent.close())); } } diff --git a/server/src/services/work-folder-garbage.ts b/server/src/services/work-folder-garbage.ts index 9ffe9b29a7..2d15d1d816 100644 --- a/server/src/services/work-folder-garbage.ts +++ b/server/src/services/work-folder-garbage.ts @@ -1,3 +1,4 @@ +import { measureSandboxOperation } from "./sandbox-performance.js"; import { and, eq, sql } from "drizzle-orm"; import { workFolderObjects, workFiles, taskRepositoryBindings, type Db } from "@paperclipai/db"; import type { StorageProvider } from "../storage/types.js"; @@ -6,39 +7,43 @@ import type { StorageProvider } from "../storage/types.js"; export async function registerWorkFolderObject(db: Db, storage: StorageProvider, input: { objectKey: string; companyId: string; folderId?: string; repositoryBindingId?: string; }) { - await db.insert(workFolderObjects).values({ ...input, provider: storage.id, - deleteAfter: new Date(Date.now() + 24 * 60 * 60 * 1000) }).onConflictDoUpdate({ - target: workFolderObjects.objectKey, - set: { deleteAfter: sql`case when ${workFolderObjects.deleteAfter} is null then null else ${new Date(Date.now() + 24 * 60 * 60 * 1000).toISOString()}::timestamptz end` }, - }); + return measureSandboxOperation("work_folder.object.register", { requestCount: 1 }, async () => { + await measureSandboxOperation("work_folder.db.query", { operation: "insert_work_folder_objects", requestCount: 1 }, async () => (db.insert(workFolderObjects).values({ ...input, provider: storage.id, + deleteAfter: new Date(Date.now() + 24 * 60 * 60 * 1000) }).onConflictDoUpdate({ + target: workFolderObjects.objectKey, + set: { deleteAfter: sql`case when ${workFolderObjects.deleteAfter} is null then null else ${new Date(Date.now() + 24 * 60 * 60 * 1000).toISOString()}::timestamptz end` }, + }))); + }); } /** Bounded, retryable cleanup. Committed trash stays referenced until explicit purge. */ export async function collectWorkFolderGarbage(db: Db, storage: StorageProvider, now = new Date(), limit = 100) { - // Polymorphic owners cannot use a single foreign key. Permanent deletion is - // detected against the authoritative owner tables, including auth users. - await db.execute(sql`delete from work_folders f where - (f.scope = 'task' and not exists (select 1 from issues i where i.id::text = f.owner_id and i.company_id = f.company_id)) or - (f.scope = 'agent' and not exists (select 1 from agents a where a.id::text = f.owner_id and a.company_id = f.company_id)) or - (f.scope = 'project' and not exists (select 1 from projects p where p.id::text = f.owner_id and p.company_id = f.company_id)) or - (f.scope = 'user' and not exists (select 1 from "user" u where u.id = f.owner_id))`); - let deleted = 0; - await db.transaction(async (tx) => { - const candidates = await tx.select().from(workFolderObjects).where(and(eq(workFolderObjects.provider, storage.id), - sql`(${workFolderObjects.deleteAfter} <= ${now.toISOString()}::timestamptz or (${workFolderObjects.deleteAfter} is null and ( - (${workFolderObjects.folderId} is not null and not exists (select 1 from ${workFiles} where ${workFiles.objectKey} = ${workFolderObjects.objectKey})) or - (${workFolderObjects.repositoryBindingId} is not null and not exists (select 1 from ${taskRepositoryBindings} where ${taskRepositoryBindings.id} = ${workFolderObjects.repositoryBindingId})) - )))`, - sql`not exists (select 1 from ${workFiles} where ${workFiles.objectKey} = ${workFolderObjects.objectKey})`, - )).limit(Math.max(1, Math.min(limit, 1000))).for("update", { skipLocked: true }); - for (const object of candidates) { - const prefix = object.folderId ? `${object.companyId}/work-folders/${object.folderId}/` - : `${object.companyId}/task-repositories/${object.repositoryBindingId}/`; - if (!object.objectKey.startsWith(prefix)) throw new Error("Work folder garbage ownership mismatch"); - await storage.deleteObject({ objectKey: object.objectKey }); - await tx.delete(workFolderObjects).where(eq(workFolderObjects.objectKey, object.objectKey)); - deleted++; - } + return measureSandboxOperation("work_folder.garbage.collect", { files: limit }, async () => { + // Polymorphic owners cannot use a single foreign key. Permanent deletion is + // detected against the authoritative owner tables, including auth users. + await measureSandboxOperation("work_folder.db.query", { operation: "execute", requestCount: 1 }, async () => (db.execute(sql`delete from work_folders f where + (f.scope = 'task' and not exists (select 1 from issues i where i.id::text = f.owner_id and i.company_id = f.company_id)) or + (f.scope = 'agent' and not exists (select 1 from agents a where a.id::text = f.owner_id and a.company_id = f.company_id)) or + (f.scope = 'project' and not exists (select 1 from projects p where p.id::text = f.owner_id and p.company_id = f.company_id)) or + (f.scope = 'user' and not exists (select 1 from "user" u where u.id = f.owner_id))`))); + let deleted = 0; + await db.transaction(async (tx) => { + const candidates = await measureSandboxOperation("work_folder.db.query", { operation: "select_work_folder_objects", requestCount: 1 }, async () => (tx.select().from(workFolderObjects).where(and(eq(workFolderObjects.provider, storage.id), + sql`(${workFolderObjects.deleteAfter} <= ${now.toISOString()}::timestamptz or (${workFolderObjects.deleteAfter} is null and ( + (${workFolderObjects.folderId} is not null and not exists (select 1 from ${workFiles} where ${workFiles.objectKey} = ${workFolderObjects.objectKey})) or + (${workFolderObjects.repositoryBindingId} is not null and not exists (select 1 from ${taskRepositoryBindings} where ${taskRepositoryBindings.id} = ${workFolderObjects.repositoryBindingId})) + )))`, + sql`not exists (select 1 from ${workFiles} where ${workFiles.objectKey} = ${workFolderObjects.objectKey})`, + )).limit(Math.max(1, Math.min(limit, 1000))).for("update", { skipLocked: true }))); + for (const object of candidates) { + const prefix = object.folderId ? `${object.companyId}/work-folders/${object.folderId}/` + : `${object.companyId}/task-repositories/${object.repositoryBindingId}/`; + if (!object.objectKey.startsWith(prefix)) throw new Error("Work folder garbage ownership mismatch"); + await measureSandboxOperation("work_folder.object.delete", { requestCount: 1 }, async () => (storage.deleteObject({ objectKey: object.objectKey }))); + await measureSandboxOperation("work_folder.db.query", { operation: "delete_work_folder_objects", requestCount: 1 }, async () => (tx.delete(workFolderObjects).where(eq(workFolderObjects.objectKey, object.objectKey)))); + deleted++; + } + }); + return { deleted }; }); - return { deleted }; } diff --git a/server/src/services/work-folder-read-cache.ts b/server/src/services/work-folder-read-cache.ts index 6b1ea222ab..ed0edfafa7 100644 --- a/server/src/services/work-folder-read-cache.ts +++ b/server/src/services/work-folder-read-cache.ts @@ -1,3 +1,4 @@ +import { captureSandboxPerformanceContext, measureSandboxOperation, measureSandboxStream } from "./sandbox-performance.js"; import { Readable } from "node:stream"; import type { WorkTreeEntry } from "./work-folder-transport.js"; @@ -37,7 +38,7 @@ export function createWorkFolderReadCache( cached.delete(group); cached.set(group, result); return result; } - result = Promise.resolve().then(() => load(group)).then((buffers) => { + result = measureSandboxOperation("work_folder.read_cache.load", { files: group.length, bytes: group.reduce((sum, entry) => sum + entry.byteSize, 0) }, () => load(group)).then((buffers) => { if (buffers.length !== group.length || buffers.some((buffer, index) => !Buffer.isBuffer(buffer) || buffer.length !== group[index]!.byteSize)) { throw new Error("Work folder batch content does not match its entries"); @@ -58,15 +59,17 @@ export function createWorkFolderReadCache( const location = locations.get(entry.path); // Mark each source request, even if its stream is never consumed. A retry // must reopen the physical file instead of replaying possibly stale bytes. - return Readable.from((async function* () { + const inContext = captureSandboxPerformanceContext(); + return measureSandboxStream("work_folder.read_cache.body", { bytes: entry.byteSize, repeated }, Readable.from((async function* () { if (cleared || repeated || !location) { - const source = fallback(entry); + const source = inContext(() => fallback(entry)); try { yield* source; } finally { source.destroy(); } return; } - const buffers = await getBatch(location.batch); + const buffers = await inContext(() => measureSandboxOperation("work_folder.read_cache.lookup", + { cacheHit: cached.has(location.batch), files: location.batch.length }, async () => getBatch(location.batch))); yield buffers[location.index]!; - })()); + })())); } function clear() { cleared = true; diff --git a/server/src/services/work-folder-repositories.ts b/server/src/services/work-folder-repositories.ts index 2122f02152..2157cbff8f 100644 --- a/server/src/services/work-folder-repositories.ts +++ b/server/src/services/work-folder-repositories.ts @@ -1,3 +1,4 @@ +import { measureSandboxOperation, measureSandboxStream } from "./sandbox-performance.js"; import { createWorkFolderReadCache } from "./work-folder-read-cache.js"; import { prefetchWorkFiles } from "./work-folder-transfer.js"; import { createHash, randomUUID } from "node:crypto"; @@ -21,18 +22,23 @@ function signature(entries: WorkTreeEntry[]) { } export function workFolderRepositoryService(db: Db, storage: StorageProvider, transport: WorkFolderTransport) { + const indexes = new Map(); + function repositoryIndex(binding: Binding) { + if (!indexes.has(binding.id)) indexes.set(binding.id, indexes.size); + return indexes.get(binding.id)!; + } const knownByBinding = new Map>(); async function checkpoint(binding: Binding, root: string) { // Another sandbox can have published since this coordinator loaded the // binding. Cache only the current complete checkpoint's protected objects. - const [current] = await db.select().from(taskRepositoryBindings).where(and( - eq(taskRepositoryBindings.id, binding.id), eq(taskRepositoryBindings.companyId, binding.companyId))); + const [current] = await measureSandboxOperation("work_folder.repository.lookup", { scope: "repos" }, async () => db.select().from(taskRepositoryBindings).where(and( + eq(taskRepositoryBindings.id, binding.id), eq(taskRepositoryBindings.companyId, binding.companyId)))); if (!current) throw new Error("Repository owner was deleted during checkpoint"); if (current.checkpointKey !== binding.checkpointKey) knownByBinding.delete(binding.id); Object.assign(binding, current); const startedAt = Date.now(); const entries = await transport.scan(root, true); - const digest = signature(entries); + const digest = await measureSandboxOperation("work_folder.repository.signature", { files: entries.length }, async () => signature(entries)); if (binding.checkpointSha256 === digest) return; const prefix = `${binding.companyId}/task-repositories/${binding.id}/`; if (!knownByBinding.has(binding.id) && binding.checkpointKey) await loadManifest(binding); @@ -57,17 +63,20 @@ export function workFolderRepositoryService(db: Db, storage: StorageProvider, tr try { await Promise.all(Array.from({ length: Math.min(4, objects.length) }, async () => { while (!failure) { - const object = objects[next++]; + const fileIndex = next++; + const object = objects[fileIndex]; if (!object) return; const [objectKey, entry] = object; try { - await registerWorkFolderObject(db, storage, { objectKey, companyId: binding.companyId, repositoryBindingId: binding.id }); + await measureSandboxOperation("work_folder.repository.object_intent", { fileIndex }, () => registerWorkFolderObject(db, storage, { objectKey, companyId: binding.companyId, repositoryBindingId: binding.id })); if (failure) return; - const { exists } = await storage.headObject({ objectKey }); + const { exists } = await measureSandboxOperation("work_folder.repository.object_head", { fileIndex, bytes: entry.byteSize, requestCount: 1 }, async (span) => { + const result = await storage.headObject({ objectKey }); span.set({ exists: result.exists }); return result; + }); if (!exists && !failure) { - await uploadWorkFolderObject(storage, { objectKey, contentType: "application/octet-stream", + await measureSandboxOperation("work_folder.repository.object_upload", { fileIndex, bytes: entry.byteSize, parallelism: 4 }, () => uploadWorkFolderObject(storage, { objectKey, contentType: "application/octet-stream", contentLength: entry.byteSize, sha256: entry.sha256!, - createSource: () => readCache ? readCache.read(entry) : transport.read(root, entry.path, entry.byteSize) }); + createSource: () => readCache ? readCache.read(entry) : transport.read(root, entry.path, entry.byteSize) })); } } catch (error) { // Stop scheduling after the first error, but drain the other workers @@ -79,35 +88,39 @@ export function workFolderRepositoryService(db: Db, storage: StorageProvider, tr } finally { readCache?.clear(); } if (failure) throw failure.error; // Never publish a torn Git index/worktree snapshot as a completed save. - if (signature(await transport.scan(root, true)) !== digest) throw new Error("Repository changed during checkpoint; retry required"); + await measureSandboxOperation("work_folder.repository.verify_snapshot", { files: entries.length }, async () => { + const verifiedEntries = await transport.scan(root, true); + const verifiedDigest = await measureSandboxOperation("work_folder.repository.signature", { files: verifiedEntries.length }, async () => signature(verifiedEntries)); + if (verifiedDigest !== digest) throw new Error("Repository changed during checkpoint; retry required"); + }); const checkpointKey = `${prefix}checkpoints/${randomUUID()}.json`; - const body = Buffer.from(JSON.stringify({ version: 1, bindingId: binding.id, files })); + const body = await measureSandboxOperation("work_folder.repository.manifest_encode", { files: files.length }, async () => Buffer.from(JSON.stringify({ version: 1, bindingId: binding.id, files }))); await registerWorkFolderObject(db, storage, { objectKey: checkpointKey, companyId: binding.companyId, repositoryBindingId: binding.id }); - await storage.putObject({ objectKey: checkpointKey, body, contentType: "application/json", contentLength: body.length }); + await measureSandboxOperation("work_folder.repository.manifest_upload", { bytes: body.length, requestCount: 1 }, () => storage.putObject({ objectKey: checkpointKey, body, contentType: "application/json", contentLength: body.length })); // Retired objects have a 24-hour grace period. A checkpoint must finish // within that window even when a competing run advances the pointer. if (Date.now() - startedAt > 60 * 60 * 1000) throw new Error("Repository checkpoint exceeded the one-hour save limit; retry required"); - await db.transaction(async (tx) => { - const [owner] = await tx.select({ id: taskRepositoryBindings.id }).from(taskRepositoryBindings) - .where(and(eq(taskRepositoryBindings.id, binding.id), eq(taskRepositoryBindings.companyId, binding.companyId))).for("update"); + await measureSandboxOperation("work_folder.repository.publish", { files: files.length }, () => db.transaction(async (tx) => { + const [owner] = await measureSandboxOperation("work_folder.repository.publish_lock", {}, async () => tx.select({ id: taskRepositoryBindings.id }).from(taskRepositoryBindings) + .where(and(eq(taskRepositoryBindings.id, binding.id), eq(taskRepositoryBindings.companyId, binding.companyId))).for("update")); if (!owner) throw new Error("Repository owner was deleted during checkpoint"); // Retire superseded manifests and blobs in the same transaction that // protects ALL current objects and advances the complete-checkpoint pointer. // The grace period also lets an already-started restore finish safely. - await tx.update(workFolderObjects).set({ deleteAfter: new Date(Date.now() + 24 * 60 * 60 * 1000) }) - .where(and(eq(workFolderObjects.repositoryBindingId, binding.id), eq(workFolderObjects.companyId, binding.companyId), isNull(workFolderObjects.deleteAfter))); + await measureSandboxOperation("work_folder.repository.retire_objects", {}, async () => tx.update(workFolderObjects).set({ deleteAfter: new Date(Date.now() + 24 * 60 * 60 * 1000) }) + .where(and(eq(workFolderObjects.repositoryBindingId, binding.id), eq(workFolderObjects.companyId, binding.companyId), isNull(workFolderObjects.deleteAfter)))); const published = [checkpointKey, ...new Set(files.flatMap((file) => file.objectKey ? [file.objectKey] : []))]; for (let offset = 0; offset < published.length; offset += 1000) { const batch = published.slice(offset, offset + 1000); - const protectedObjects = await tx.update(workFolderObjects).set({ deleteAfter: null }) + const protectedObjects = await measureSandboxOperation("work_folder.repository.protect_objects", { objects: batch.length, batchIndex: Math.floor(offset / 1000) }, async () => tx.update(workFolderObjects).set({ deleteAfter: null }) .where(and(eq(workFolderObjects.repositoryBindingId, binding.id), inArray(workFolderObjects.objectKey, batch))) - .returning({ key: workFolderObjects.objectKey }); + .returning({ key: workFolderObjects.objectKey })); if (protectedObjects.length !== batch.length) throw new Error("Repository objects expired during checkpoint; retry required"); } - const updated = await tx.update(taskRepositoryBindings).set({ checkpointKey, checkpointSha256: digest, checkpointAt: new Date() }) - .where(and(eq(taskRepositoryBindings.id, binding.id), eq(taskRepositoryBindings.companyId, binding.companyId))).returning({ id: taskRepositoryBindings.id }); + const updated = await measureSandboxOperation("work_folder.repository.publish_pointer", {}, async () => tx.update(taskRepositoryBindings).set({ checkpointKey, checkpointSha256: digest, checkpointAt: new Date() }) + .where(and(eq(taskRepositoryBindings.id, binding.id), eq(taskRepositoryBindings.companyId, binding.companyId))).returning({ id: taskRepositoryBindings.id })); if (!updated.length) throw new Error("Repository owner was deleted during checkpoint"); - }); + })); knownByBinding.set(binding.id, new Set(files.flatMap((file) => file.objectKey ? [file.objectKey] : []))); binding.checkpointKey = checkpointKey; binding.checkpointSha256 = digest; @@ -117,15 +130,15 @@ export function workFolderRepositoryService(db: Db, storage: StorageProvider, tr if (!binding.checkpointKey) throw new Error("Repository checkpoint is missing"); const prefix = `${binding.companyId}/task-repositories/${binding.id}/`; if (!binding.checkpointKey.startsWith(prefix)) throw new Error("Repository checkpoint ownership mismatch"); - const { stream } = await storage.getObject({ objectKey: binding.checkpointKey }); + const { stream } = await measureSandboxOperation("work_folder.repository.manifest_download", { requestCount: 1 }, () => storage.getObject({ objectKey: binding.checkpointKey! })); const chunks: Buffer[] = []; let length = 0; - for await (const chunk of stream) { + for await (const chunk of measureSandboxStream("work_folder.repository.manifest_body", {}, stream)) { length += chunk.length; if (length > 64 * 1024 * 1024) { stream.destroy(); throw new Error("Repository checkpoint manifest exceeds size limit"); } chunks.push(Buffer.from(chunk)); } - const manifest = checkpointSchema.parse(JSON.parse(Buffer.concat(chunks).toString("utf8"))); + const manifest = await measureSandboxOperation("work_folder.repository.manifest_decode", { bytes: length }, async () => checkpointSchema.parse(JSON.parse(Buffer.concat(chunks).toString("utf8")))); if (manifest.bindingId !== binding.id) throw new Error("Repository checkpoint ownership mismatch"); for (const entry of manifest.files) { validateWorkFilePath(entry.path); @@ -139,14 +152,14 @@ export function workFolderRepositoryService(db: Db, storage: StorageProvider, tr async function restore(binding: Binding, root: string, stagingRoot: string) { if (!binding.checkpointKey) return false; - const manifest = await loadManifest(binding); + const manifest = await measureSandboxOperation("work_folder.repository.manifest_load", {}, () => loadManifest(binding)); await transport.mkdirRoot(root); await transport.writeMany(root, stagingRoot, prefetchWorkFiles( - manifest.files.filter((entry) => !entry.linkTarget), async (entry) => { + manifest.files.filter((entry) => !entry.linkTarget), async (entry, fileIndex) => { if (entry.kind === "directory") return { entry }; if (!entry.objectKey) throw new Error("Repository checkpoint file is missing"); - const result = await storage.getObject({ objectKey: entry.objectKey }); - return { entry, body: result.stream }; + const result = await measureSandboxOperation("work_folder.repository.object_download", { fileIndex, bytes: entry.byteSize, requestCount: 1 }, () => storage.getObject({ objectKey: entry.objectKey! })); + return { entry, body: measureSandboxStream("work_folder.repository.object_body", { fileIndex, bytes: entry.byteSize }, result.stream) }; })); // Restore links only after ordinary files. No transfer follows a link as // a parent, and symlink() still confines its target to this repository. @@ -155,5 +168,10 @@ export function workFolderRepositoryService(db: Db, storage: StorageProvider, tr } return true; } - return { checkpoint, restore }; + return { + checkpoint: (binding: Binding, root: string) => measureSandboxOperation("work_folder.repository.checkpoint", + { scope: "repos", repositoryIndex: repositoryIndex(binding) }, () => checkpoint(binding, root)), + restore: (binding: Binding, root: string, stagingRoot: string) => measureSandboxOperation("work_folder.repository.restore", + { scope: "repos", repositoryIndex: repositoryIndex(binding) }, () => restore(binding, root, stagingRoot)), + }; } diff --git a/server/src/services/work-folder-transfer.ts b/server/src/services/work-folder-transfer.ts index 6312c8abf9..e9ce4a2693 100644 --- a/server/src/services/work-folder-transfer.ts +++ b/server/src/services/work-folder-transfer.ts @@ -1,34 +1,40 @@ +import { measureSandboxOperation } from "./sandbox-performance.js"; import type { WorkFileTransfer } from "./work-folder-transport.js"; // Open a few storage responses ahead without buffering their bodies. Drain all // pending opens on failure so abandoned HTTP response streams are closed too. export async function* prefetchWorkFiles( entries: Iterable, - open: (entry: T) => Promise, + open: (entry: T, fileIndex: number) => Promise, ): AsyncGenerator { type Result = { value: WorkFileTransfer } | { error: unknown }; const iterator = entries[Symbol.iterator](); const pending: Array> = []; + let opened = 0, consumed = 0; function enqueue() { const next = iterator.next(); - if (!next.done) pending.push(Promise.resolve().then(() => open(next.value)) + if (!next.done) { + const fileIndex = opened++; + pending.push(Promise.resolve().then(() => measureSandboxOperation("work_folder.prefetch.open", { fileIndex, parallelism: 4 }, async () => open(next.value, fileIndex))) .then((value): Result => { // A response can fail while queued, before its async iterator exists. // Keep that error handled; consuming the stream still throws it. value.body?.on("error", () => {}); return { value }; }, (error): Result => ({ error }))); + } } try { for (let i = 0; i < 4; i++) enqueue(); while (pending.length) { - const result = await pending.shift()!; + const next = pending.shift()!; + const result = await measureSandboxOperation("work_folder.prefetch.wait", { fileIndex: consumed++, parallelism: 4 }, async () => next); if ("error" in result) throw result.error; try { yield result.value; } finally { result.value.body?.destroy(); } enqueue(); } } finally { - for (const result of await Promise.all(pending)) { + for (const result of await measureSandboxOperation("work_folder.prefetch.drain", { files: pending.length, parallelism: 4 }, async () => Promise.all(pending))) { if ("value" in result) result.value.body?.destroy(); } iterator.return?.(); diff --git a/server/src/services/work-folder-transport.ts b/server/src/services/work-folder-transport.ts index aa79e6a1f2..639383c93a 100644 --- a/server/src/services/work-folder-transport.ts +++ b/server/src/services/work-folder-transport.ts @@ -1,3 +1,4 @@ +import { captureSandboxPerformanceContext, hasSandboxPerformanceTrace, measureSandboxOperation, measureSandboxStream } from "./sandbox-performance.js"; import { readFile } from "node:fs/promises"; import { Readable } from "node:stream"; import { createHash, randomBytes, randomUUID } from "node:crypto"; @@ -7,6 +8,18 @@ import { z } from "zod"; import type { CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime"; import { validateWorkFilePath } from "@paperclipai/shared"; +const remoteNumber = z.number().finite().nonnegative(); +const remotePerformanceSchema = z.object({ result: z.unknown(), performance: z.object({ + executionMs: remoteNumber, scanMs: remoteNumber, listMs: remoteNumber, gitListMs: remoteNumber, hashMs: remoteNumber, readMs: remoteNumber, + writeMs: remoteNumber, publishMs: remoteNumber, decodeMs: remoteNumber, encodeMs: remoteNumber, + files: remoteNumber, bytes: remoteNumber, hashFiles: remoteNumber, hashBytes: remoteNumber, + requestCount: remoteNumber, droppedPhases: remoteNumber, +}), remotePhases: z.array(z.object({ phase: z.enum(["scanMs", "listMs", "gitListMs", "hashMs", "readMs", "writeMs", "publishMs", "decodeMs", "encodeMs"]), + startOffsetMs: remoteNumber, durationMs: remoteNumber })).max(256) }); + +const remotePhaseNames = { scanMs: "scan", listMs: "list", gitListMs: "git_list", hashMs: "hash", readMs: "read", + writeMs: "write", publishMs: "publish", decodeMs: "decode", encodeMs: "encode" } as const; + const entrySchema = z.object({ path: z.string().refine((value) => { try { validateWorkFilePath(value); return true; } catch { return false; } }), kind: z.enum(["file", "directory"]), byteSize: z.number().int().nonnegative().max(1024 ** 3), sha256: z.string().regex(/^[a-f0-9]{64}$/).nullable(), executable: z.boolean(), linkTarget: z.string().max(1024).optional() }); @@ -38,27 +51,53 @@ export function workFolderTransport(runner: CommandManagedRuntimeRunner) { // small-argv transport without assuming additional capabilities. const bulkStdin = Boolean(runner.syncIn && runner.syncOut) || runner.supportsSingleStreamStdinProgress === true; async function command(input: Record, stdin?: string, deadline = Date.now() + 120_000): Promise { - source ??= readFile(new URL("./scripts/work-folder-io.mjs", import.meta.url), "utf8"); - const args = ["--input-type=module", "-e", await source, Buffer.from(JSON.stringify(input)).toString("base64")]; - const readOnly = ["home", "scan", "read", "read-batch", "batch-status"].includes(String(input.operation)); - let result; - for (let attempt = 0; ; attempt++) { - if (Date.now() >= deadline) throw new Error("Work folder transfer deadline exceeded"); - try { - result = await runner.execute({ command: "node", args, ...(stdin === undefined ? {} : { stdin }), bypassSession: true, - timeoutMs: Math.max(1, deadline - Date.now()) }); - break; - } catch (error) { - // A lost read response is safe to repeat. Staging writes, publishes and - // moves may already have happened, so never replay them here. - const waitMs = 250 * (attempt + 1); - if (!readOnly || attempt >= 2 || !transientTransportFailure(error) || Date.now() + waitMs >= deadline) throw error; - await delay(waitMs); + const operation = String(input.operation); + return measureSandboxOperation("work_folder.transport.command", { operation }, async () => { + const args = await measureSandboxOperation("work_folder.transport.encode", { operation }, async (span) => { + source ??= readFile(new URL("./scripts/work-folder-io.mjs", import.meta.url), "utf8"); + const encoded = Buffer.from(JSON.stringify({ ...input, performance: hasSandboxPerformanceTrace() })).toString("base64"); + span.set({ bytes: Buffer.byteLength(encoded), inputBytes: stdin === undefined ? 0 : Buffer.byteLength(stdin) }); + return ["--input-type=module", "-e", await source, encoded]; + }); + const readOnly = ["home", "scan", "read", "read-batch", "batch-status"].includes(operation); + for (let attempt = 0; ; attempt++) { + if (Date.now() >= deadline) throw new Error("Work folder transfer deadline exceeded"); + let executionReturned = false; + try { + return await measureSandboxOperation("work_folder.transport.roundtrip", { operation, attempt: attempt + 1, requestCount: 1 }, async (span) => { + const startedAt = performance.now(); + const result = await runner.execute({ command: "node", args, ...(stdin === undefined ? {} : { stdin }), bypassSession: true, + timeoutMs: Math.max(1, deadline - Date.now()) }); + executionReturned = true; + const roundtripMs = performance.now() - startedAt; + span.set({ roundtripMs, outputBytes: Buffer.byteLength(result.stdout) }); + if (result.exitCode !== 0 || result.timedOut) throw new Error(`Work folder ${operation} failed: ${result.stderr.slice(0, 1500)}`); + return measureSandboxOperation("work_folder.transport.decode", { operation }, async () => { + const decoded = JSON.parse(result.stdout); + // Older helpers and runners retain the unwrapped result contract. + if (decoded?.workFolderPerformanceVersion !== 1) return decoded; + const validated = remotePerformanceSchema.safeParse(decoded); + // Timing is diagnostic. A malformed timing envelope must not + // discard a valid command result or change its retry semantics. + if (!validated.success) { span.set({ dropped: 1 }); return decoded.result; } + const details = validated.data; + span.set({ ...details.performance, transportOverheadMs: Math.max(0, roundtripMs - details.performance.executionMs) }); + for (const phase of details.remotePhases) { + span.recordRemotePhase(`work_folder.remote.${remotePhaseNames[phase.phase]}`, phase.startOffsetMs, phase.durationMs, { operation }); + } + return details.result; + }); + }); + } catch (error) { + // Mutations may already have happened. Only reads repeat here. + const waitMs = 250 * (attempt + 1); + if (executionReturned || !readOnly || attempt >= 2 || !transientTransportFailure(error) || Date.now() + waitMs >= deadline) throw error; + await measureSandboxOperation("work_folder.transport.backoff", { operation, attempt: attempt + 1, waitMs }, () => delay(waitMs)); + } } - } - if (result.exitCode !== 0 || result.timedOut) throw new Error(`Work folder ${String(input.operation)} failed: ${result.stderr.slice(0, 1500)}`); - return JSON.parse(result.stdout); + }); } + async function writeBatch(root: string, stagingRoot: string, body: string) { const deadline = Date.now() + 120_000; const identity = { root, stagingRoot, batchId: randomUUID(), @@ -88,14 +127,18 @@ export function workFolderTransport(runner: CommandManagedRuntimeRunner) { // An upstream error may have lost only the response. First inspect the // sandbox receipt. A claimed batch is never replayed; it must complete // or remain visibly recoverable when this bounded wait expires. - const status = statusSchema.parse(await command({ operation: "batch-status", ...identity }, undefined, deadline)); + const status = await measureSandboxOperation("work_folder.transport.receipt_reconcile", { attempt: attempts }, async (span) => { + const result = statusSchema.parse(await command({ operation: "batch-status", ...identity }, undefined, deadline)); + span.set({ operation: result.state }); + return result; + }); if (status.state === "completed") return { completed: status.completed }; if (status.state === "failed") throw new Error(`Work folder batch failed: ${status.error}`); pending = status.state === "running"; if (!pending && attempts >= 3) throw lastError ?? new Error("Work folder batch receipt is missing"); const waitMs = pending ? 500 : 250 * attempts; if (Date.now() + waitMs >= deadline) break; - await delay(waitMs); + await measureSandboxOperation("work_folder.transport.receipt_wait", { waitMs, attempt: attempts }, () => delay(waitMs)); } throw new Error("Work folder batch outcome is uncertain; retaining sandbox for recovery"); } @@ -104,20 +147,25 @@ export function workFolderTransport(runner: CommandManagedRuntimeRunner) { return result.home; } async function scan(root: string, repository = false) { - return z.array(entrySchema).max(100_000).parse(await command({ operation: "scan", root, repository })); + return measureSandboxOperation("work_folder.transport.scan", { repository }, async (span) => { + const entries = z.array(entrySchema).max(100_000).parse(await command({ operation: "scan", root, repository })); + span.set({ files: entries.length, bytes: entries.reduce((sum, entry) => sum + entry.byteSize, 0) }); + return entries; + }); } function read(root: string, filePath: string, byteSize: number) { validateWorkFilePath(filePath); - return Readable.from((async function* () { + const inContext = captureSandboxPerformanceContext(); + return measureSandboxStream("work_folder.transport.read_body", { bytes: byteSize }, Readable.from((async function* () { for (let offset = 0; offset < byteSize;) { const length = bulkStdin ? 1024 * 1024 : 256 * 1024; - const result = z.object({ data: z.string().max(Math.ceil(length / 3) * 4) }).parse(await command({ operation: "read", root, path: filePath, offset, length })); - const bytes = Buffer.from(result.data, "base64"); + const result = z.object({ data: z.string().max(Math.ceil(length / 3) * 4) }).parse(await inContext(() => command({ operation: "read", root, path: filePath, offset, length }))); + const bytes = await inContext(() => measureSandboxOperation("work_folder.transport.decode_bytes", { chunkIndex: Math.floor(offset / length) }, async () => Buffer.from(result.data, "base64"))); if (bytes.length === 0 || offset + bytes.length > byteSize) throw new Error("Work file changed during transfer"); offset += bytes.length; yield bytes; } - })()); + })())); } async function readBatch(root: string, entries: WorkTreeEntry[]) { if (entries.length > 64) throw new Error("Work folder read batch exceeds entry limit"); @@ -132,20 +180,20 @@ export function workFolderTransport(runner: CommandManagedRuntimeRunner) { const result = z.array(z.object({ data: z.string().max(Math.ceil(1024 * 1024 / 3) * 4) })).max(64) .parse(await command({ operation: "read-batch", root, entries: entries.map(({ path, byteSize }) => ({ path, byteSize })) })); if (result.length !== entries.length) throw new Error("Work folder read batch did not complete"); - return result.map(({ data }, index) => { + return measureSandboxOperation("work_folder.transport.decode_batch", { files: entries.length, bytes }, async () => result.map(({ data }, index) => { const buffer = Buffer.from(data, "base64"); if (buffer.length !== entries[index]!.byteSize) throw new Error("Work file changed during transfer"); return buffer; - }); + })); } async function writeChunks(root: string, stagingRoot: string, entry: WorkTreeEntry, body: Readable) { const stagingPath = randomUUID(); let offset = 0; - for await (const value of body) { + for await (const value of measureSandboxStream("work_folder.transport.incoming_body", { bytes: entry.byteSize }, body)) { const chunk = Buffer.isBuffer(value) ? value : Buffer.from(value); for (let start = 0; start < chunk.length; start += WRITE_CHUNK_BYTES) { const bytes = chunk.subarray(start, start + WRITE_CHUNK_BYTES); - await command({ operation: "write", root: stagingRoot, path: stagingPath, offset, data: bytes.toString("base64") }); + await command({ operation: "write", root: stagingRoot, path: stagingPath, offset, data: await measureSandboxOperation("work_folder.transport.encode_bytes", { bytes: bytes.length }, async () => bytes.toString("base64")) }); offset += bytes.length; } } @@ -161,7 +209,7 @@ export function workFolderTransport(runner: CommandManagedRuntimeRunner) { let bufferedBytes = 0; async function flush() { if (!operations.length) return; - const body = JSON.stringify(operations); + const body = await measureSandboxOperation("work_folder.transport.encode_batch", { files: publishedEntries.length, bytes: bufferedBytes, requestCount: operations.length }, async () => JSON.stringify(operations)); if (Buffer.byteLength(body) > 8 * 1024 * 1024) throw new Error("Work folder batch exceeds transfer limit"); if (publishedEntries.length) await beforePublish?.(publishedEntries); const result = await writeBatch(root, stagingRoot, body); @@ -194,12 +242,12 @@ export function workFolderTransport(runner: CommandManagedRuntimeRunner) { } const stagingPath = randomUUID(); let offset = 0; - for await (const value of body) { + for await (const value of measureSandboxStream("work_folder.transport.incoming_body", { bytes: entry.byteSize }, body)) { const chunk = Buffer.isBuffer(value) ? value : Buffer.from(value); for (let start = 0; start < chunk.length; start += 256 * 1024) { const bytes = chunk.subarray(start, start + 256 * 1024); if (offset + bytes.length > entry.byteSize) throw new Error("Work file size changed during transfer"); - await append({ operation: "write", path: stagingPath, offset, data: bytes.toString("base64") }, bytes.length); + await append({ operation: "write", path: stagingPath, offset, data: await measureSandboxOperation("work_folder.transport.encode_bytes", { bytes: bytes.length }, async () => bytes.toString("base64")) }, bytes.length); offset += bytes.length; } } diff --git a/server/src/services/work-folder-upload.ts b/server/src/services/work-folder-upload.ts index 11e4cffd6a..27e879528a 100644 --- a/server/src/services/work-folder-upload.ts +++ b/server/src/services/work-folder-upload.ts @@ -1,3 +1,4 @@ +import { measureSandboxOperation } from "./sandbox-performance.js"; import { createHash } from "node:crypto"; import { Readable, Transform } from "node:stream"; import { pipeline } from "node:stream/promises"; @@ -21,42 +22,46 @@ export async function uploadWorkFolderObject(storage: Pick Readable; }) { - for (let attempt = 0; attempt < 3; attempt++) { - const source = input.createSource(); - const hash = createHash("sha256"); - let bytes = 0; - let validationError: Error | undefined; - const changed = () => validationError ??= new Error("Work file changed during upload; retry the checkpoint"); - const verify = new Transform({ - transform(chunk: Buffer, _encoding, callback) { - bytes += chunk.length; - if (bytes > input.contentLength) return callback(changed()); - hash.update(chunk); - callback(null, chunk); - }, - flush(callback) { - callback(bytes === input.contentLength && hash.digest("hex") === input.sha256 ? undefined : changed()); - }, - }); - const transferred = pipeline(source, verify); - const uploaded = Promise.resolve().then(() => storage.putObject({ - objectKey: input.objectKey, contentType: input.contentType, - contentLength: input.contentLength, body: verify, - })); - let failure: unknown; - try { - // Observe both promises immediately. An early HTTP success cannot publish - // unvalidated content, and a failed request must release its source reader. - await Promise.all([transferred, uploaded]); - return; - } catch (error) { - failure = validationError ?? error; - } finally { - source.destroy(); - verify.destroy(); - await Promise.allSettled([transferred, uploaded]); + return measureSandboxOperation("work_folder.object.upload", { bytes: input.contentLength }, async () => { + for (let attempt = 0; attempt < 3; attempt++) { + const source = input.createSource(); + const hash = createHash("sha256"); + let bytes = 0; + let validationError: Error | undefined; + const changed = () => validationError ??= new Error("Work file changed during upload; retry the checkpoint"); + const verify = new Transform({ + transform(chunk: Buffer, _encoding, callback) { + bytes += chunk.length; + if (bytes > input.contentLength) return callback(changed()); + hash.update(chunk); + callback(null, chunk); + }, + flush(callback) { + callback(bytes === input.contentLength && hash.digest("hex") === input.sha256 ? undefined : changed()); + }, + }); + const transferred = measureSandboxOperation("work_folder.upload.consume", { attempt: attempt + 1, bytes: input.contentLength }, async (span) => { + try { await pipeline(source, verify); } finally { span.set({ bytes }); } + }); + const uploaded = Promise.resolve().then(() => measureSandboxOperation("work_folder.object.put", { attempt: attempt + 1, bytes: input.contentLength, requestCount: 1 }, async () => storage.putObject({ + objectKey: input.objectKey, contentType: input.contentType, + contentLength: input.contentLength, body: verify, + }))); + let failure: unknown; + try { + // Observe both promises immediately. An early HTTP success cannot publish + // unvalidated content, and a failed request must release its source reader. + await Promise.all([transferred, uploaded]); + return; + } catch (error) { + failure = validationError ?? error; + } finally { + source.destroy(); + verify.destroy(); + await Promise.allSettled([transferred, uploaded]); + } + if (validationError || attempt === 2 || !transientUploadFailure(failure)) throw failure; + await measureSandboxOperation("work_folder.upload.retry_wait", { attempt: attempt + 1 }, async () => delay(250 * (attempt + 1))); } - if (validationError || attempt === 2 || !transientUploadFailure(failure)) throw failure; - await delay(250 * (attempt + 1)); - } + }); } diff --git a/server/src/services/work-folders.ts b/server/src/services/work-folders.ts index c8f015cfe4..67ea28c88a 100644 --- a/server/src/services/work-folders.ts +++ b/server/src/services/work-folders.ts @@ -1,3 +1,4 @@ +import { measureSandboxOperation, measureSandboxStream } from "./sandbox-performance.js"; import { createHash, randomUUID } from "node:crypto"; import { createReadStream, createWriteStream } from "node:fs"; import { mkdtemp, rm } from "node:fs/promises"; @@ -30,55 +31,67 @@ function validPath(value: string) { /** The caller must authorize the folder owner before using this host-only service. */ export function workFolderService(db: Db, storage: StorageProvider) { async function ensure(owner: WorkFolderOwner): Promise { - await db.insert(workFolders).values(owner).onConflictDoNothing(); - const [folder] = await db.select().from(workFolders).where(and(eq(workFolders.companyId, owner.companyId), - eq(workFolders.scope, owner.scope), eq(workFolders.ownerId, owner.ownerId))); - if (!folder) throw notFound("Work folder not found"); - return folder; + return measureSandboxOperation("work_folder.scope.ensure", { scope: owner.scope }, async () => { + await measureSandboxOperation("work_folder.db.query", { operation: "insert_work_folders", requestCount: 1 }, async () => (db.insert(workFolders).values(owner).onConflictDoNothing())); + const [folder] = await measureSandboxOperation("work_folder.db.query", { operation: "select_work_folders", requestCount: 1 }, async () => (db.select().from(workFolders).where(and(eq(workFolders.companyId, owner.companyId), + eq(workFolders.scope, owner.scope), eq(workFolders.ownerId, owner.ownerId))))); + if (!folder) throw notFound("Work folder not found"); + return folder; + }); } async function list(folder: Folder, options: { trash?: boolean; cursor?: string; limit?: number } = {}) { - const limit = Math.max(1, Math.min(1000, options.limit ?? 200)); - const rows = await db.select().from(workFiles).where(and(eq(workFiles.folderId, folder.id), - eq(workFiles.companyId, folder.companyId), options.trash ? isNotNull(workFiles.deletedAt) : isNull(workFiles.deletedAt), - options.cursor ? gt(workFiles.id, options.cursor) : undefined)).orderBy(asc(workFiles.id)).limit(limit + 1); - const [saved] = await db.select({ at: max(workFileOperations.createdAt) }).from(workFileOperations) - .where(and(eq(workFileOperations.companyId, folder.companyId), eq(workFileOperations.folderId, folder.id))); - return { id: folder.id, owner: { companyId: folder.companyId, scope: folder.scope, ownerId: folder.ownerId }, - lastOperationAt: saved?.at?.toISOString() ?? null, - files: rows.slice(0, limit).map(workFileDto), nextCursor: rows.length > limit ? rows[limit - 1]!.id : null }; + return measureSandboxOperation("work_folder.metadata.list", { scope: folder.scope }, async (span) => { + const limit = Math.max(1, Math.min(1000, options.limit ?? 200)); + const rows = await measureSandboxOperation("work_folder.db.query", { operation: "select_work_files", requestCount: 1 }, async () => (db.select().from(workFiles).where(and(eq(workFiles.folderId, folder.id), + eq(workFiles.companyId, folder.companyId), options.trash ? isNotNull(workFiles.deletedAt) : isNull(workFiles.deletedAt), + options.cursor ? gt(workFiles.id, options.cursor) : undefined)).orderBy(asc(workFiles.id)).limit(limit + 1))); + const [saved] = await measureSandboxOperation("work_folder.db.query", { operation: "select_work_file_operations", requestCount: 1 }, async () => (db.select({ at: max(workFileOperations.createdAt) }).from(workFileOperations) + .where(and(eq(workFileOperations.companyId, folder.companyId), eq(workFileOperations.folderId, folder.id))))); + span.set({ files: Math.min(rows.length, limit) }); + return { id: folder.id, owner: { companyId: folder.companyId, scope: folder.scope, ownerId: folder.ownerId }, + lastOperationAt: saved?.at?.toISOString() ?? null, + files: rows.slice(0, limit).map(workFileDto), nextCursor: rows.length > limit ? rows[limit - 1]!.id : null }; + }); } async function get(folder: Folder, filePath: string) { - const [row] = await db.select().from(workFiles).where(and(eq(workFiles.companyId, folder.companyId), - eq(workFiles.folderId, folder.id), eq(workFiles.path, validPath(filePath)), isNull(workFiles.deletedAt))); - if (!row) throw notFound("Work file not found"); - return row; + return measureSandboxOperation("work_folder.metadata.get", { scope: folder.scope }, async () => { + const [row] = await measureSandboxOperation("work_folder.db.query", { operation: "select_work_files", requestCount: 1 }, async () => (db.select().from(workFiles).where(and(eq(workFiles.companyId, folder.companyId), + eq(workFiles.folderId, folder.id), eq(workFiles.path, validPath(filePath)), isNull(workFiles.deletedAt))))); + if (!row) throw notFound("Work file not found"); + return row; + }); } - async function content(folder: Folder, filePath: string) { - const row = await get(folder, filePath); - if (row.kind !== "file" || !row.objectKey) throw badRequest("Path is a directory"); - return { file: workFileDto(row), ...(await storage.getObject({ objectKey: row.objectKey })) }; + async function content(folder: Folder, filePath: string, fileIndex?: number) { + return measureSandboxOperation("work_folder.content.open", { scope: folder.scope, ...(fileIndex === undefined ? {} : { fileIndex }) }, async () => { + const row = await get(folder, filePath); + if (row.kind !== "file" || !row.objectKey) throw badRequest("Path is a directory"); + const object = await measureSandboxOperation("work_folder.object.get_response", { scope: folder.scope, requestCount: 1, bytes: row.byteSize, ...(fileIndex === undefined ? {} : { fileIndex }) }, async () => storage.getObject({ objectKey: row.objectKey! })); + return { file: workFileDto(row), ...object, stream: measureSandboxStream("work_folder.object.body", { scope: folder.scope, bytes: row.byteSize, ...(fileIndex === undefined ? {} : { fileIndex }) }, object.stream) }; + }); } async function mutate(folder: Folder, operationId: string, fingerprint: string, apply: (tx: Parameters[0]>[0]) => Promise) { - if (!operationId || operationId.length > 256) throw badRequest("An operation ID is required"); - return db.transaction(async (tx) => { - // Serialize acceptance order across all app processes, including mkdir/delete races. - const [locked] = await tx.select().from(workFolders).where(and(eq(workFolders.id, folder.id), - eq(workFolders.companyId, folder.companyId))).for("update"); - if (!locked) throw notFound("Work folder not found"); - const [receipt] = await tx.select().from(workFileOperations).where(and( - eq(workFileOperations.folderId, folder.id), eq(workFileOperations.operationId, operationId))); - if (receipt) { - if (receipt.fingerprint !== fingerprint) throw conflict("Operation ID was already used for different content"); - return { applied: false as const }; - } - const result = await apply(tx); - await tx.insert(workFileOperations).values({ companyId: folder.companyId, folderId: folder.id, operationId, fingerprint }); - return { applied: true as const, result }; + return measureSandboxOperation("work_folder.metadata.mutate", { scope: folder.scope }, async () => { + if (!operationId || operationId.length > 256) throw badRequest("An operation ID is required"); + return db.transaction(async (tx) => { + // Serialize acceptance order across all app processes, including mkdir/delete races. + const [locked] = await measureSandboxOperation("work_folder.db.query", { operation: "select_work_folders", requestCount: 1 }, async () => (tx.select().from(workFolders).where(and(eq(workFolders.id, folder.id), + eq(workFolders.companyId, folder.companyId))).for("update"))); + if (!locked) throw notFound("Work folder not found"); + const [receipt] = await measureSandboxOperation("work_folder.db.query", { operation: "select_work_file_operations", requestCount: 1 }, async () => (tx.select().from(workFileOperations).where(and( + eq(workFileOperations.folderId, folder.id), eq(workFileOperations.operationId, operationId))))); + if (receipt) { + if (receipt.fingerprint !== fingerprint) throw conflict("Operation ID was already used for different content"); + return { applied: false as const }; + } + const result = await apply(tx); + await measureSandboxOperation("work_folder.db.query", { operation: "insert_work_file_operations", requestCount: 1 }, async () => (tx.insert(workFileOperations).values({ companyId: folder.companyId, folderId: folder.id, operationId, fingerprint }))); + return { applied: true as const, result }; + }); }); } @@ -87,127 +100,132 @@ export function workFolderService(db: Db, storage: StorageProvider) { kind?: "file" | "directory"; operationId: string; maxBytes?: number; expectedSha256?: string | null; replaceKind?: boolean; onlyIfMissing?: boolean; }) { - const filePath = validPath(input.path); - const kind = input.kind ?? "file"; - const directory = await mkdtemp(path.join(os.tmpdir(), "paperclip-work-file-")); - const spool = path.join(directory, "content"); - const hash = createHash("sha256"); - let byteSize = 0; - let sha256: string | null = null; - let objectKey: string | null = null; - let discardUpload = false; - try { - if (kind === "file") { - const source = Buffer.isBuffer(input.body) ? Readable.from([input.body]) : input.body ?? Readable.from([]); - await pipeline(source, new Transform({ transform(chunk: Buffer, _encoding, callback) { - byteSize += chunk.length; - if (byteSize > (input.maxBytes ?? MAX_WORK_FILE_BYTES)) return callback(payloadTooLarge("Work file exceeds the size limit")); - hash.update(chunk); - callback(null, chunk); - } }), createWriteStream(spool, { mode: 0o600 })); - sha256 = hash.digest("hex"); + return measureSandboxOperation("work_folder.content.write", { scope: folder.scope }, async (span) => { + const filePath = validPath(input.path); + const kind = input.kind ?? "file"; + const directory = await measureSandboxOperation("work_folder.spool.create", { scope: folder.scope }, async () => mkdtemp(path.join(os.tmpdir(), "paperclip-work-file-"))); + const spool = path.join(directory, "content"); + const hash = createHash("sha256"); + let byteSize = 0; + let sha256: string | null = null; + let objectKey: string | null = null; + let discardUpload = false; + try { + if (kind === "file") { + const source = Buffer.isBuffer(input.body) ? Readable.from([input.body]) : input.body ?? Readable.from([]); + await measureSandboxOperation("work_folder.spool.consume", { scope: folder.scope }, async (span) => { + try { await pipeline(source, new Transform({ transform(chunk: Buffer, _encoding, callback) { + byteSize += chunk.length; + if (byteSize > (input.maxBytes ?? MAX_WORK_FILE_BYTES)) return callback(payloadTooLarge("Work file exceeds the size limit")); + hash.update(chunk); + callback(null, chunk); + } }), createWriteStream(spool, { mode: 0o600 })); } finally { span.set({ bytes: byteSize }); } + }); + sha256 = hash.digest("hex"); + if (input.expectedSha256 && input.expectedSha256 !== sha256) throw conflict("File changed during transfer; retry the checkpoint"); + objectKey = `${folder.companyId}/work-folders/${folder.id}/${randomUUID()}`; + await registerWorkFolderObject(db, storage, { objectKey, companyId: folder.companyId, folderId: folder.id }); + await uploadWorkFolderObject(storage, { objectKey, createSource: () => createReadStream(spool), + contentLength: byteSize, sha256, contentType: input.contentType ?? "application/octet-stream" }); + } if (input.expectedSha256 && input.expectedSha256 !== sha256) throw conflict("File changed during transfer; retry the checkpoint"); - objectKey = `${folder.companyId}/work-folders/${folder.id}/${randomUUID()}`; - await registerWorkFolderObject(db, storage, { objectKey, companyId: folder.companyId, folderId: folder.id }); - await uploadWorkFolderObject(storage, { objectKey, createSource: () => createReadStream(spool), - contentLength: byteSize, sha256, contentType: input.contentType ?? "application/octet-stream" }); + const value = { kind, objectKey, byteSize, sha256, executable: input.executable ?? false, + contentType: input.contentType ?? "application/octet-stream", updatedAt: new Date() }; + const fingerprint = JSON.stringify(["write", filePath, kind, sha256, value.executable, value.contentType, Boolean(input.replaceKind), Boolean(input.onlyIfMissing)]); + const result = await mutate(folder, input.operationId, fingerprint, async (tx) => { + const parts = filePath.split("/"); + for (let i = 1; i < parts.length; i++) { + const parent = parts.slice(0, i).join("/"); + const [existing] = await measureSandboxOperation("work_folder.db.query", { operation: "select_work_files", requestCount: 1 }, async () => (tx.select().from(workFiles).where(and(eq(workFiles.folderId, folder.id), + eq(workFiles.path, parent), isNull(workFiles.deletedAt))))); + if (existing && existing.kind !== "directory") throw conflict("A parent path is a file"); + if (!existing) await measureSandboxOperation("work_folder.db.query", { operation: "insert_work_files", requestCount: 1 }, async () => (tx.insert(workFiles).values({ companyId: folder.companyId, folderId: folder.id, + path: parent, kind: "directory" }))); + } + const previousRows = await measureSandboxOperation("work_folder.db.query", { operation: "select_work_files", requestCount: 1 }, async () => (tx.select().from(workFiles).where(and(eq(workFiles.folderId, folder.id), + eq(workFiles.path, filePath), isNull(workFiles.deletedAt))))); + let previous: FileRow | undefined = previousRows[0]; + if (previous && input.onlyIfMissing) return { oldKey: null, unusedUpload: true }; + if (previous && previous.kind !== kind) { + if (!input.replaceKind) throw conflict("Delete the existing path before changing its kind"); + const prefix = `${filePath}/`; + await measureSandboxOperation("work_folder.db.query", { operation: "update_work_files", requestCount: 1 }, async () => (tx.update(workFiles).set({ deletedAt: new Date(), updatedAt: new Date() }).where(and( + eq(workFiles.folderId, folder.id), isNull(workFiles.deletedAt), + sql`(${workFiles.path} = ${filePath} or left(${workFiles.path}, ${prefix.length}) = ${prefix})`)))); + previous = undefined; + } + if (previous) { + await measureSandboxOperation("work_folder.db.query", { operation: "update_work_files", requestCount: 1 }, async () => (tx.update(workFiles).set(value).where(eq(workFiles.id, previous.id)))); + } else { + await measureSandboxOperation("work_folder.db.query", { operation: "insert_work_files", requestCount: 1 }, async () => (tx.insert(workFiles).values({ ...value, companyId: folder.companyId, folderId: folder.id, path: filePath }))); + } + if (objectKey) await measureSandboxOperation("work_folder.db.query", { operation: "update_work_folder_objects", requestCount: 1 }, async () => (tx.update(workFolderObjects).set({ deleteAfter: null }).where(eq(workFolderObjects.objectKey, objectKey!)))); + if (previous?.objectKey) await measureSandboxOperation("work_folder.db.query", { operation: "update_work_folder_objects", requestCount: 1 }, async () => (tx.update(workFolderObjects).set({ deleteAfter: new Date() }).where(eq(workFolderObjects.objectKey, previous!.objectKey!)))); + return { oldKey: previous?.objectKey ?? null, unusedUpload: false }; + }); + discardUpload = !result.applied || result.result.unusedUpload; + // Object keys are private to this service; receipts contain no overwritten content. + // Cleanup is journaled in the same transaction as replacement. Storage + // outages and lost commit replies cannot orphan the only current object. + return { applied: result.applied }; + } finally { + // A lost database response can mean COMMIT succeeded. Retain an uncertain + // upload for reconciliation; deleting it here could destroy saved content. + if (objectKey && discardUpload) await measureSandboxOperation("work_folder.db.query", { operation: "update_work_folder_objects", requestCount: 1 }, async () => (db.update(workFolderObjects).set({ deleteAfter: new Date() }).where(eq(workFolderObjects.objectKey, objectKey!)))); + span.set({ bytes: byteSize }); + await measureSandboxOperation("work_folder.spool.cleanup", { scope: folder.scope }, async () => rm(directory, { recursive: true, force: true })); } - if (input.expectedSha256 && input.expectedSha256 !== sha256) throw conflict("File changed during transfer; retry the checkpoint"); - const value = { kind, objectKey, byteSize, sha256, executable: input.executable ?? false, - contentType: input.contentType ?? "application/octet-stream", updatedAt: new Date() }; - const fingerprint = JSON.stringify(["write", filePath, kind, sha256, value.executable, value.contentType, Boolean(input.replaceKind), Boolean(input.onlyIfMissing)]); - const result = await mutate(folder, input.operationId, fingerprint, async (tx) => { - const parts = filePath.split("/"); - for (let i = 1; i < parts.length; i++) { - const parent = parts.slice(0, i).join("/"); - const [existing] = await tx.select().from(workFiles).where(and(eq(workFiles.folderId, folder.id), - eq(workFiles.path, parent), isNull(workFiles.deletedAt))); - if (existing && existing.kind !== "directory") throw conflict("A parent path is a file"); - if (!existing) await tx.insert(workFiles).values({ companyId: folder.companyId, folderId: folder.id, - path: parent, kind: "directory" }); - } - const previousRows = await tx.select().from(workFiles).where(and(eq(workFiles.folderId, folder.id), - eq(workFiles.path, filePath), isNull(workFiles.deletedAt))); - let previous: FileRow | undefined = previousRows[0]; - if (previous && input.onlyIfMissing) return { oldKey: null, unusedUpload: true }; - if (previous && previous.kind !== kind) { - if (!input.replaceKind) throw conflict("Delete the existing path before changing its kind"); - const prefix = `${filePath}/`; - await tx.update(workFiles).set({ deletedAt: new Date(), updatedAt: new Date() }).where(and( - eq(workFiles.folderId, folder.id), isNull(workFiles.deletedAt), - sql`(${workFiles.path} = ${filePath} or left(${workFiles.path}, ${prefix.length}) = ${prefix})`)); - previous = undefined; - } - if (previous) { - await tx.update(workFiles).set(value).where(eq(workFiles.id, previous.id)); - } else { - await tx.insert(workFiles).values({ ...value, companyId: folder.companyId, folderId: folder.id, path: filePath }); - } - if (objectKey) await tx.update(workFolderObjects).set({ deleteAfter: null }).where(eq(workFolderObjects.objectKey, objectKey)); - if (previous?.objectKey) await tx.update(workFolderObjects).set({ deleteAfter: new Date() }).where(eq(workFolderObjects.objectKey, previous.objectKey)); - return { oldKey: previous?.objectKey ?? null, unusedUpload: false }; - }); - discardUpload = !result.applied || result.result.unusedUpload; - // Object keys are private to this service; receipts contain no overwritten content. - // Cleanup is journaled in the same transaction as replacement. Storage - // outages and lost commit replies cannot orphan the only current object. - return { applied: result.applied }; - } finally { - // A lost database response can mean COMMIT succeeded. Retain an uncertain - // upload for reconciliation; deleting it here could destroy saved content. - if (objectKey && discardUpload) await db.update(workFolderObjects).set({ deleteAfter: new Date() }).where(eq(workFolderObjects.objectKey, objectKey)); - await rm(directory, { recursive: true, force: true }); - } + }); } async function remove(folder: Folder, filePath: string, operationId: string) { const normalized = validPath(filePath); return mutate(folder, operationId, JSON.stringify(["delete", normalized]), async (tx) => { const prefix = `${normalized}/`; - await tx.update(workFiles).set({ deletedAt: new Date(), updatedAt: new Date() }).where(and( + await measureSandboxOperation("work_folder.db.query", { operation: "update_work_files", requestCount: 1 }, async () => (tx.update(workFiles).set({ deletedAt: new Date(), updatedAt: new Date() }).where(and( eq(workFiles.folderId, folder.id), isNull(workFiles.deletedAt), - sql`(${workFiles.path} = ${normalized} or left(${workFiles.path}, ${prefix.length}) = ${prefix})`)); + sql`(${workFiles.path} = ${normalized} or left(${workFiles.path}, ${prefix.length}) = ${prefix})`)))); }); } async function restore(folder: Folder, fileId: string, operationId: string) { return mutate(folder, operationId, JSON.stringify(["restore", fileId]), async (tx) => { - const [deleted] = await tx.select().from(workFiles).where(and(eq(workFiles.folderId, folder.id), - eq(workFiles.id, fileId), isNotNull(workFiles.deletedAt))); + const [deleted] = await measureSandboxOperation("work_folder.db.query", { operation: "select_work_files", requestCount: 1 }, async () => (tx.select().from(workFiles).where(and(eq(workFiles.folderId, folder.id), + eq(workFiles.id, fileId), isNotNull(workFiles.deletedAt))))); if (!deleted) throw notFound("Deleted file not found"); const prefix = `${deleted.path}/`; - const restoreRows = deleted.kind === "directory" ? await tx.select().from(workFiles).where(and( + const restoreRows = deleted.kind === "directory" ? await measureSandboxOperation("work_folder.db.query", { operation: "select_work_files", requestCount: 1 }, async () => (tx.select().from(workFiles).where(and( eq(workFiles.folderId, folder.id), eq(workFiles.deletedAt, deleted.deletedAt!), - sql`(${workFiles.path} = ${deleted.path} or left(${workFiles.path}, ${prefix.length}) = ${prefix})`)) : [deleted]; + sql`(${workFiles.path} = ${deleted.path} or left(${workFiles.path}, ${prefix.length}) = ${prefix})`)))) : [deleted]; for (const row of restoreRows) { - const [occupied] = await tx.select().from(workFiles).where(and(eq(workFiles.folderId, folder.id), - eq(workFiles.path, row.path), isNull(workFiles.deletedAt))); + const [occupied] = await measureSandboxOperation("work_folder.db.query", { operation: "select_work_files", requestCount: 1 }, async () => (tx.select().from(workFiles).where(and(eq(workFiles.folderId, folder.id), + eq(workFiles.path, row.path), isNull(workFiles.deletedAt))))); if (occupied) throw conflict("Delete the current file before restoring this deleted copy"); } const parts = deleted.path.split("/"); for (let i = 1; i < parts.length; i++) { const parent = parts.slice(0, i).join("/"); - const [existing] = await tx.select().from(workFiles).where(and(eq(workFiles.folderId, folder.id), - eq(workFiles.path, parent), isNull(workFiles.deletedAt))); + const [existing] = await measureSandboxOperation("work_folder.db.query", { operation: "select_work_files", requestCount: 1 }, async () => (tx.select().from(workFiles).where(and(eq(workFiles.folderId, folder.id), + eq(workFiles.path, parent), isNull(workFiles.deletedAt))))); if (existing && existing.kind !== "directory") throw conflict("A parent path is a file"); - if (!existing) await tx.insert(workFiles).values({ companyId: folder.companyId, folderId: folder.id, - path: parent, kind: "directory" }); + if (!existing) await measureSandboxOperation("work_folder.db.query", { operation: "insert_work_files", requestCount: 1 }, async () => (tx.insert(workFiles).values({ companyId: folder.companyId, folderId: folder.id, + path: parent, kind: "directory" }))); } - for (const row of restoreRows) await tx.update(workFiles).set({ deletedAt: null, updatedAt: new Date() }).where(eq(workFiles.id, row.id)); + for (const row of restoreRows) await measureSandboxOperation("work_folder.db.query", { operation: "update_work_files", requestCount: 1 }, async () => (tx.update(workFiles).set({ deletedAt: null, updatedAt: new Date() }).where(eq(workFiles.id, row.id)))); }); } async function purge(folder: Folder, fileId: string, operationId: string) { return mutate(folder, operationId, JSON.stringify(["purge", fileId]), async (tx) => { - const [deleted] = await tx.select().from(workFiles).where(and(eq(workFiles.folderId, folder.id), - eq(workFiles.id, fileId), isNotNull(workFiles.deletedAt))); + const [deleted] = await measureSandboxOperation("work_folder.db.query", { operation: "select_work_files", requestCount: 1 }, async () => (tx.select().from(workFiles).where(and(eq(workFiles.folderId, folder.id), + eq(workFiles.id, fileId), isNotNull(workFiles.deletedAt))))); if (!deleted) throw notFound("Deleted file not found"); const prefix = `${deleted.path}/`; - const rows = await tx.delete(workFiles).where(and(eq(workFiles.folderId, folder.id), + const rows = await measureSandboxOperation("work_folder.db.query", { operation: "delete_work_files", requestCount: 1 }, async () => (tx.delete(workFiles).where(and(eq(workFiles.folderId, folder.id), eq(workFiles.deletedAt, deleted.deletedAt!), deleted.kind === "directory" ? sql`(${workFiles.path} = ${deleted.path} or left(${workFiles.path}, ${prefix.length}) = ${prefix})` - : eq(workFiles.id, fileId))).returning({ objectKey: workFiles.objectKey }); - for (const row of rows) if (row.objectKey) await tx.update(workFolderObjects).set({ deleteAfter: new Date() }).where(eq(workFolderObjects.objectKey, row.objectKey)); + : eq(workFiles.id, fileId))).returning({ objectKey: workFiles.objectKey }))); + for (const row of rows) if (row.objectKey) await measureSandboxOperation("work_folder.db.query", { operation: "update_work_folder_objects", requestCount: 1 }, async () => (tx.update(workFolderObjects).set({ deleteAfter: new Date() }).where(eq(workFolderObjects.objectKey, row.objectKey!)))); }); } return { ensure, list, get, content, write, remove, restore, purge }; diff --git a/tests/runner-e2e/daytona-image-content.ts b/tests/runner-e2e/daytona-image-content.ts index aa8642ed05..0472de0351 100644 --- a/tests/runner-e2e/daytona-image-content.ts +++ b/tests/runner-e2e/daytona-image-content.ts @@ -76,6 +76,8 @@ export const DAYTONA_IMAGE_INPUT_PATHS = [ "packages/paperclip-runner/runner/crates", "packages/paperclip-runner/scripts/acpx-sidecar-contract.mjs", "packages/paperclip-runner/scripts/build-provider-pack.mjs", + "packages/paperclip-runner/scripts/assemble-provider-pack.mjs", + "packages/paperclip-runner/scripts/provider-pack-integrity.mjs", "packages/paperclip-runner/scripts/provider-pack-layout.mjs", "packages/paperclip-runner/scripts/materialize-pi-binary.mjs", "packages/paperclip-runner/scripts/portable-provider-shim.mjs", @@ -195,7 +197,9 @@ export function extractDaytonaBaseImages(dockerfile: string): string[] { } const reference = match[1]!; - if (!stageAliases.has(reference)) { + // Docker's scratch is the built-in empty filesystem, not a registry image. + // Its declaration remains covered by the Dockerfile content hash. + if (reference !== "scratch" && !stageAliases.has(reference)) { assertPinnedBaseImage(reference); baseImages.push(reference); } diff --git a/tests/runner-e2e/daytona-image.test.ts b/tests/runner-e2e/daytona-image.test.ts index 953d16dd5e..ff629ade20 100644 --- a/tests/runner-e2e/daytona-image.test.ts +++ b/tests/runner-e2e/daytona-image.test.ts @@ -33,7 +33,7 @@ describe("runner E2E Daytona image contract", () => { ]); const normalizedDockerfile = dockerfile.replace(/\\\r?\n\s*/g, " "); expect(dockerfile).toContain("--bin paperclip-runnerd"); - expect(dockerfile).toContain("build-provider-pack.mjs /provider-pack"); + expect(dockerfile).toContain("assemble-provider-pack.mjs /provider-pack"); expect(normalizedDockerfile).not.toContain( "COPY packages/paperclip-eval-kernel ./packages/paperclip-eval-kernel", ); @@ -166,6 +166,21 @@ describe("runner E2E Daytona image contract", () => { expect(cliInstall).toBeLessThan(finalMetadataArgs); }); + it("exports the canonical provider stage without a mutable base or recursive builder", async () => { + const dockerfile = await readFile(path.join(repositoryRoot, "docker/daytona-runner/Dockerfile"), "utf8"); + expect(dockerfile).toContain("FROM scratch AS provider-pack-export\nCOPY --from=provider-pack-build /provider-pack /"); + expect(extractDaytonaBaseImages(dockerfile)).not.toContain("scratch"); + expect(() => extractDaytonaBaseImages("FROM node:latest\nFROM scratch AS exported")).toThrow("immutable"); + const builder = await readFile(path.join(repositoryRoot, "packages/paperclip-runner/scripts/build-provider-pack.mjs"), "utf8"); + expect(builder).toContain('"--target", "provider-pack-export"'); + expect(builder).toContain('"--platform", "linux/amd64"'); + expect(builder).toContain("verifyProviderPack(exported"); + expect(dockerfile).not.toContain("build-provider-pack.mjs /provider-pack"); + for (const file of ["assemble-provider-pack.mjs", "provider-pack-integrity.mjs"]) { + expect(DAYTONA_IMAGE_INPUT_PATHS).toContain(`packages/paperclip-runner/scripts/${file}`); + } + }); + it("builds both provider packs from the same verified dedicated lock before compiling", async () => { const lockPath = "docker/daytona-runner/provider-dependencies.lock.yaml"; const lock = await readFile(path.join(repositoryRoot, lockPath)); @@ -189,7 +204,7 @@ describe("runner E2E Daytona image contract", () => { "pnpm --filter @paperclipai/paperclip-runner build:typescript", ); const pack = normalized.indexOf( - "node packages/paperclip-runner/scripts/build-provider-pack.mjs /provider-pack", + "node packages/paperclip-runner/scripts/assemble-provider-pack.mjs /provider-pack", ); expect(normalized).toContain(`ARG PAPERCLIP_RUNNER_LOCK_SHA256=${lockDigest}`); expect(copy).toBeGreaterThan(0);