Measure sandbox lifecycle and align qualification provider packs
Add opt-in bounded OpenTelemetry diagnostics across host preparation, scoped files, repository checkpoints, transport phases, and native execution. Preserve context parentage and stream cleanup, and retain diagnostic batches without per-file log writes. Build qualification packs through the canonical frozen provider stage so lockfile, interpreter, and compiled bytes match the sandbox image. Keep ignored untracked repository files out of durable checkpoints. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Dotta
parent
310d5e3627
commit
4cd6a81e4a
11
Dockerfile
11
Dockerfile
|
|
@ -202,6 +202,9 @@ RUN set -eu; \
|
|||
# doc/observability.md).
|
||||
#
|
||||
# CLOUD_BUNDLED_SERVER_DEPS names the optional peer packages to install.
|
||||
# CLOUD_BUNDLED_OTEL_DEPS adds the tracing peers independently because release
|
||||
# workflows explicitly override the former with the Sentry package. Installing
|
||||
# tracing support does not enable it: an operator must still set an OTLP endpoint.
|
||||
# The value is a space-separated list, the same shape as
|
||||
# CLOUD_BUNDLED_PLUGINS above. The stage reads each package's version
|
||||
# from the `peerDependencies` block of `server/package.json` at build
|
||||
|
|
@ -238,17 +241,19 @@ RUN set -eu; \
|
|||
FROM build AS cloud-server-deps
|
||||
WORKDIR /app/.cloud-server-deps
|
||||
ARG CLOUD_BUNDLED_SERVER_DEPS="@sentry/node"
|
||||
ARG CLOUD_BUNDLED_OTEL_DEPS="@opentelemetry/sdk-node @opentelemetry/auto-instrumentations-node @opentelemetry/resources @opentelemetry/semantic-conventions @opentelemetry/exporter-trace-otlp-grpc @opentelemetry/exporter-trace-otlp-proto @opentelemetry/exporter-trace-otlp-http"
|
||||
RUN set -eu; \
|
||||
test -n "$CLOUD_BUNDLED_SERVER_DEPS" || { echo "ERROR: CLOUD_BUNDLED_SERVER_DEPS is empty; name at least one optional peer package to install" >&2; exit 1; }; \
|
||||
echo '{"name":"paperclip-cloud-server-deps","private":true}' > package.json; \
|
||||
specifiers=""; \
|
||||
for name in $CLOUD_BUNDLED_SERVER_DEPS; do \
|
||||
for name in $CLOUD_BUNDLED_SERVER_DEPS $CLOUD_BUNDLED_OTEL_DEPS; do \
|
||||
version="$(node -e "const pkg=require('/app/server/package.json'); const name=process.argv[1]; const version=(pkg.peerDependencies||{})[name]; if(!version){console.error('ERROR: server/package.json declares no peerDependencies version for '+JSON.stringify(name));process.exit(1);} const meta=(pkg.peerDependenciesMeta||{})[name]; if(!meta||meta.optional!==true){console.error('ERROR: '+JSON.stringify(name)+' is not declared as an optional peer dependency in server/package.json; CLOUD_BUNDLED_SERVER_DEPS may name only optional peer packages');process.exit(1);} process.stdout.write(version);" "$name")"; \
|
||||
test -n "$version" || { echo "ERROR: could not resolve a version for '$name'" >&2; exit 1; }; \
|
||||
specifiers="$specifiers ${name}@${version}"; \
|
||||
done; \
|
||||
test -n "$specifiers" || { echo "ERROR: CLOUD_BUNDLED_SERVER_DEPS names no package" >&2; exit 1; }; \
|
||||
pnpm add --ignore-workspace --no-lockfile $specifiers
|
||||
pnpm add --ignore-workspace --no-lockfile $specifiers \
|
||||
&& node -e "const fs=require('node:fs'),path=require('node:path'),assert=require('node:assert/strict');const peers=require('/app/server/package.json').peerDependencies;for(const name of process.argv.slice(1)){let dir=path.dirname(require.resolve(name)),found=false;for(let depth=0;depth<12;depth++,dir=path.dirname(dir)){const file=path.join(dir,'package.json');if(!fs.existsSync(file))continue;const pkg=JSON.parse(fs.readFileSync(file,'utf8'));if(pkg.name!==name)continue;assert.equal(pkg.version,peers[name],name+' must match the server optional peer version');found=true;break;}assert(found,'Cannot verify installed optional peer '+name);}" $CLOUD_BUNDLED_SERVER_DEPS $CLOUD_BUNDLED_OTEL_DEPS
|
||||
|
||||
# Use the same qualified interpreter as the Daytona provider-pack build.
|
||||
# The controller owns this pack and its manifest; remote OpenCode/ACPX launches
|
||||
|
|
@ -277,7 +282,7 @@ ARG PAPERCLIP_BUILD_COMMIT
|
|||
RUN test -n "${PAPERCLIP_BUILD_COMMIT}" \
|
||||
&& pnpm --filter @paperclipai/paperclip-runner build:typescript \
|
||||
&& PAPERCLIP_RUNNER_SOURCE_REVISION="${PAPERCLIP_BUILD_COMMIT}" \
|
||||
node packages/paperclip-runner/scripts/build-provider-pack.mjs /provider-pack \
|
||||
node packages/paperclip-runner/scripts/assemble-provider-pack.mjs /provider-pack \
|
||||
&& node packages/paperclip-runner/scripts/verify-pi-provider-launch.mjs /provider-pack \
|
||||
&& chmod -R a+rX /provider-pack
|
||||
|
||||
|
|
|
|||
|
|
@ -132,10 +132,76 @@ This document also holds three local instrumentation contracts: native runner
|
|||
traces, sandbox startup traces, and sandbox duplex transport instrumentation.
|
||||
Those sections follow below.
|
||||
|
||||
## Full sandbox lifecycle diagnostics
|
||||
|
||||
With `OTEL_EXPORTER_OTLP_ENDPOINT` configured, `sandbox.run` measures the host
|
||||
execution path for both runner generations. Its children cover authorization
|
||||
and database work, workspace/environment acquisition, scoped work-folder
|
||||
preparation, runtime context and launchers, dispatch, final saving and cleanup.
|
||||
The Cloud image includes the exact optional OTel peers declared by the server;
|
||||
self-hosted images retain the optional installation procedure above. The
|
||||
endpoint gate remains required in both cases.
|
||||
|
||||
`heartbeat.*` operations surround the actual awaited calls. Repeated database
|
||||
calls share a name and have a numeric `operationIndex` for source correlation.
|
||||
`work_folder.*` operations split binding checks, scope and history lookups,
|
||||
attachment seeding/import, manifests, incoming/outgoing scans, metadata queries,
|
||||
object GET response acquisition, body consumption, transport batches, repository
|
||||
credentials/clone/checkout/setup, and checkpoint publication. Checkpoints also
|
||||
measure object journal queries, existence checks, uploads, both repository
|
||||
scans, manifest storage, and the final pointer transaction. A completed request
|
||||
is not a completed body transfer: those are separate spans.
|
||||
|
||||
The live host dispatch context parents native `task.run` when these diagnostics
|
||||
are active. Native startup and agent spans remain beneath it. Previously
|
||||
collected aggregate preparation durations are not backdated into a second
|
||||
overlapping native preparation tree in this mode. Without the endpoint, native
|
||||
local run-log behavior and its historical preparation tree remain unchanged.
|
||||
Active contexts also propagate through the OpenTelemetry context manager, so
|
||||
HTTP and database auto-instrumentation inherits the closest operation.
|
||||
|
||||
Names and attributes are fixed operation labels, numeric indices/counts, finite
|
||||
durations, byte counts, retry attempts, and boolean outcomes. Raw run identifiers
|
||||
are hashed. New diagnostic attributes exclude paths, file contents, repository
|
||||
URLs, credentials, commands, and exception messages. An operation's `failed`
|
||||
outcome means that operation threw; the authoritative task outcome still comes
|
||||
from its run row, since execution can catch an error and return normally.
|
||||
|
||||
Filesystem helpers report their own monotonic phase offsets. These become
|
||||
events on the measured host command span, with `clock=remote_relative`,
|
||||
`startOffsetMs` and `durationMs`. They are **not** host-clock child spans: there
|
||||
is no measured clock alignment. Compare host round-trip time and remote
|
||||
execution time without labeling the difference pure network latency; it also
|
||||
contains queueing and transport overhead. Body `readWaitMs` measures waits for
|
||||
the next iterator result; `bodyNonReadMs` includes consumer processing and
|
||||
backpressure. Neither is a direct network measurement.
|
||||
|
||||
Startup, agent execution, checkpointing, and total task duration must be
|
||||
reported separately. For a waterfall, subtract the **union** of child intervals
|
||||
to find unattributed time; summing nested or parallel spans double counts work.
|
||||
Keep remote-relative phases in their own lane and identify missing intervals
|
||||
explicitly. A warm scoped folder means a prior manifest exists for the same
|
||||
physical sandbox, not that the provider was already running. Repository reuse
|
||||
means a usable checkout was found; restoring a checkpoint in a replacement
|
||||
sandbox is distinct from finding an existing checkout.
|
||||
|
||||
The SDK batches exports. The helper also keeps at most 20,000 local records
|
||||
per run, writes them in batches of at most 250 after measured execution, and
|
||||
reports discarded records in `dropped` (also on the exported root span).
|
||||
Remote helpers cap detailed phases at 256 per command and report dropped
|
||||
phases separately. No per-file synchronous logging or exporter call is added.
|
||||
The local batch-write tail is outside `sandbox.run`; include it separately
|
||||
when comparing whole-process completion. Exporter queue drops and collector
|
||||
failures can still make exported traces incomplete: verify actual collector
|
||||
records and limits before using a trace as performance evidence. Diagnostic
|
||||
sink failures do not fail the task. See the local
|
||||
[`sandbox.performance.batch` contract](run-log-events.md#sandbox-performance-batches).
|
||||
|
||||
## Native Runner Trace Spans
|
||||
|
||||
Paperclip Runner task runs emit a single foldable OpenTelemetry trace. This is
|
||||
the native-run trace schema version `2`. `task.run` is the only full-run root;
|
||||
the native-run trace schema version `2`. `task.run` is the native subtree root
|
||||
(parented to live host dispatch in full lifecycle diagnostics);
|
||||
every other native span carries a real OpenTelemetry parent context rather than
|
||||
only a descriptive `parentName` field.
|
||||
|
||||
|
|
|
|||
|
|
@ -107,6 +107,30 @@ The payload never carries a command, an argument, a path, an environment value,
|
|||
or a raw identifier. The event rides the `ctx.onEvent` run-event bridge and is
|
||||
run-log-only. It needs no OTLP endpoint.
|
||||
|
||||
## Sandbox performance batches
|
||||
|
||||
When full sandbox diagnostics are enabled by `OTEL_EXPORTER_OTLP_ENDPOINT`, the
|
||||
host writes `sandbox.performance.batch` system events after measured execution.
|
||||
These rows stay in the instance database; they are neither first-party Telemetry
|
||||
events nor a replacement for inspecting actual OpenTelemetry exports. No batch
|
||||
is produced with the endpoint unset.
|
||||
|
||||
The payload schema is `paperclip.sandbox-performance.v1`, with a hashed
|
||||
`runHash`, `records` (at most 250 per event), and the run's `dropped` count.
|
||||
Each record contains a fixed operation name, span id, optional parent/trace id,
|
||||
start time, duration, operation outcome, and closed safe attributes. Host times
|
||||
use epoch milliseconds and monotonic durations. Records with
|
||||
`clock: "remote_relative"` instead contain offsets from a remote command's
|
||||
start and must not be placed on the host timeline as absolute timestamps.
|
||||
The root has numeric retained-record and dropped-record counts.
|
||||
|
||||
The default buffer holds 20,000 records per run; additional records increment
|
||||
the drop count. A missing root or a nonzero drop count means the local record
|
||||
set is incomplete. Event persistence happens in bounded batches after the
|
||||
operation timings end, not synchronously for each file. A sink failure stops
|
||||
batch persistence without changing the original task outcome. A successful
|
||||
task alone therefore does not prove that all timing records were saved.
|
||||
|
||||
## Related instrumentation
|
||||
|
||||
The sandbox duplex transport also writes one run-log event as one of its three
|
||||
|
|
|
|||
|
|
@ -377,6 +377,17 @@ or patches change, update both expected hashes, and qualify a new sandbox image.
|
|||
Matching source files alone does not prove matching dependencies: acceptance
|
||||
also compares the actual app and sandbox production-lock hashes.
|
||||
|
||||
The qualification entry `build-provider-pack.mjs` (also exposed as
|
||||
`pnpm --filter @paperclipai/paperclip-runner build:provider-pack`) requires
|
||||
Docker with BuildKit and builds the canonical `linux/amd64` provider stage.
|
||||
It uses the same digest-pinned Node interpreter, dedicated dependency graph,
|
||||
and fresh TypeScript compilation as the sandbox image; host `node_modules`,
|
||||
CI's root lock and previously compiled outputs are not assembly inputs.
|
||||
Both Docker stages call the low-level assembler directly, avoiding recursion.
|
||||
The entry verifies exported manifest and artifact bytes before atomically
|
||||
replacing its output. Failed builds or validation preserve the previous pack.
|
||||
This produces a local artifact and does not publish an image or release.
|
||||
|
||||
Native and legacy Git credential callbacks honor the same experimental duplex
|
||||
setting and provider capability gates. When streaming is disabled or unavailable,
|
||||
the file bridge remains supported. Credential acquisition allows 35 seconds per
|
||||
|
|
|
|||
|
|
@ -37,10 +37,14 @@ ARG PAPERCLIP_RUNNER_SOURCE_REVISION
|
|||
RUN test -n "${PAPERCLIP_RUNNER_SOURCE_REVISION}"
|
||||
RUN pnpm --filter @paperclipai/paperclip-runner build:typescript \
|
||||
&& PAPERCLIP_RUNNER_SOURCE_REVISION="${PAPERCLIP_RUNNER_SOURCE_REVISION}" \
|
||||
node packages/paperclip-runner/scripts/build-provider-pack.mjs /provider-pack \
|
||||
node packages/paperclip-runner/scripts/assemble-provider-pack.mjs /provider-pack \
|
||||
&& node packages/paperclip-runner/scripts/verify-pi-provider-launch.mjs /provider-pack \
|
||||
&& chmod -R a+rX /provider-pack
|
||||
|
||||
# The trusted qualification entry exports the identical canonical provider stage.
|
||||
FROM scratch AS provider-pack-export
|
||||
COPY --from=provider-pack-build /provider-pack /
|
||||
|
||||
# Fleet sandbox base image. Keep this section aligned with
|
||||
# paperclipai/paperclip-cloud/fleet-sandbox-image/Dockerfile. The only Paperclip
|
||||
# runner-specific addition is /usr/local/bin/paperclip-runnerd below.
|
||||
|
|
|
|||
|
|
@ -61,7 +61,7 @@
|
|||
"build:typescript": "pnpm run ensure:eval-build-deps && pnpm run check:protocol-types && node ./node_modules/typescript/bin/tsc --version && node ./node_modules/typescript/bin/tsc -p tsconfig.json && node ./node_modules/typescript/bin/tsc -p tsconfig.surfaces.json && node scripts/build-verified-provider-entrypoints.mjs",
|
||||
"build:rust": "cargo build --manifest-path runner/Cargo.toml --locked --workspace --bins",
|
||||
"build:binary": "cargo build --release --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core --bin paperclip-runnerd && node scripts/stage-runner-binary.mjs",
|
||||
"build:provider-pack": "pnpm run build:typescript && node scripts/build-provider-pack.mjs",
|
||||
"build:provider-pack": "node scripts/build-provider-pack.mjs",
|
||||
"build:runner-binaries": "cargo build --manifest-path runner/Cargo.toml --locked --workspace --bins",
|
||||
"build:browser": "vite build --config vite.config.ts",
|
||||
"build:sdk": "vite build --config vite.sdk.config.ts",
|
||||
|
|
@ -73,7 +73,7 @@
|
|||
"typecheck:rust": "cargo fmt --manifest-path runner/Cargo.toml --all -- --check && cargo check --manifest-path runner/Cargo.toml --locked --workspace",
|
||||
"typecheck:browser": "tsc -p tsconfig.browser.json --noEmit",
|
||||
"test": "pnpm run test:typescript && pnpm run test:rust",
|
||||
"test:typescript": "pnpm run ensure:eval-build-deps && pnpm run build:rust && node --test test/protocol-contract.test.mjs test/acpx-sidecar-contract.test.mjs test/acpx-codex-package-contract.test.mjs scripts/aws-agentcore-provisioning.test.mjs scripts/build-verified-provider-entrypoints.test.mjs scripts/local-provider-smoke-environment.test.mjs scripts/materialize-opencode-binary.test.mjs scripts/materialize-pi-binary.test.mjs scripts/portable-provider-shim.test.mjs scripts/provider-pack-layout.test.mjs && vitest run",
|
||||
"test:typescript": "pnpm run ensure:eval-build-deps && pnpm run build:rust && node --test test/protocol-contract.test.mjs test/acpx-sidecar-contract.test.mjs test/acpx-codex-package-contract.test.mjs scripts/aws-agentcore-provisioning.test.mjs scripts/build-verified-provider-entrypoints.test.mjs scripts/local-provider-smoke-environment.test.mjs scripts/materialize-opencode-binary.test.mjs scripts/materialize-pi-binary.test.mjs scripts/portable-provider-shim.test.mjs scripts/provider-pack-layout.test.mjs scripts/build-provider-pack.test.mjs && vitest run",
|
||||
"test:rust": "cargo test --release --manifest-path runner/Cargo.toml --locked --workspace",
|
||||
"test:codex": "cargo test --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core --test codex_provider",
|
||||
"test:durable": "cargo test --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core durable::",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,305 @@
|
|||
import { canonicalJson, sha256File, sha256Tree } from "./provider-pack-integrity.mjs";
|
||||
import { normalizeProviderPackLayout } from "./provider-pack-layout.mjs";
|
||||
import { portableProviderShim } from "./portable-provider-shim.mjs";
|
||||
import { createHash } from "node:crypto";
|
||||
import { execFileSync, spawnSync } from "node:child_process";
|
||||
import {
|
||||
chmodSync,
|
||||
copyFileSync,
|
||||
existsSync,
|
||||
mkdirSync,
|
||||
mkdtempSync,
|
||||
readdirSync,
|
||||
readFileSync,
|
||||
realpathSync,
|
||||
renameSync,
|
||||
rmSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join, relative, resolve } from "node:path";
|
||||
import { createRequire } from "node:module";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { materializePiBinary } from "./materialize-pi-binary.mjs";
|
||||
|
||||
const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const workspaceRoot = resolve(packageRoot, "../..");
|
||||
const outputArgument = process.argv.slice(2).find((value) => value !== "--");
|
||||
const outputRoot = resolve(
|
||||
process.cwd(),
|
||||
outputArgument ?? join(packageRoot, "provider-pack"),
|
||||
);
|
||||
if (
|
||||
outputRoot === workspaceRoot ||
|
||||
outputRoot === packageRoot ||
|
||||
outputRoot === "/"
|
||||
) {
|
||||
throw new Error(`Refusing unsafe provider-pack output path: ${outputRoot}`);
|
||||
}
|
||||
|
||||
const temporaryParent = mkdtempSync(join(tmpdir(), "paperclip-provider-pack-"));
|
||||
const temporaryRoot = join(temporaryParent, "pack");
|
||||
|
||||
function writePortableNodeShim(name, entrypoint) {
|
||||
const shimPath = join(temporaryRoot, "node_modules", ".bin", name);
|
||||
writeFileSync(shimPath, portableProviderShim(entrypoint, { node: true }));
|
||||
chmodSync(shimPath, 0o755);
|
||||
}
|
||||
|
||||
function writePortableExecutableShim(name, executable) {
|
||||
const shimPath = join(temporaryRoot, "node_modules", ".bin", name);
|
||||
writeFileSync(shimPath, portableProviderShim(executable));
|
||||
chmodSync(shimPath, 0o755);
|
||||
}
|
||||
|
||||
try {
|
||||
const deployed = spawnSync(
|
||||
"pnpm",
|
||||
[
|
||||
"--filter",
|
||||
"@paperclipai/paperclip-runner",
|
||||
"deploy",
|
||||
"--prod",
|
||||
temporaryRoot,
|
||||
],
|
||||
{ cwd: workspaceRoot, encoding: "utf8", stdio: "inherit" },
|
||||
);
|
||||
if (deployed.status !== 0) {
|
||||
throw new Error(`pnpm deploy failed with exit code ${deployed.status}`);
|
||||
}
|
||||
|
||||
normalizeProviderPackLayout(temporaryRoot);
|
||||
|
||||
// Fail the image build if a bridge silently brings back an older/private
|
||||
// provider CLI. A direct dependency alone does not deduplicate pnpm's graph.
|
||||
const packRequire = createRequire(join(temporaryRoot, "package.json"));
|
||||
const codexAcpRequire = createRequire(packRequire.resolve("@agentclientprotocol/codex-acp/package.json"));
|
||||
if (realpathSync(codexAcpRequire.resolve("@openai/codex/package.json")) !==
|
||||
realpathSync(packRequire.resolve("@openai/codex/package.json"))) {
|
||||
throw new Error("Codex ACP must share the image's Codex installation");
|
||||
}
|
||||
|
||||
// Reuse the already-qualified build interpreter instead of introducing a
|
||||
// package-manager lifecycle hook or a second binary supply chain. The pack
|
||||
// manifest binds the copied bytes, platform, architecture, and minimum
|
||||
// version before any provider is launched.
|
||||
const minimumNodeVersion = [24, 11, 0];
|
||||
const actualNodeVersion = process.versions.node.split(".").map(Number);
|
||||
if (
|
||||
actualNodeVersion[0] < minimumNodeVersion[0] ||
|
||||
(actualNodeVersion[0] === minimumNodeVersion[0] &&
|
||||
(actualNodeVersion[1] < minimumNodeVersion[1] ||
|
||||
(actualNodeVersion[1] === minimumNodeVersion[1] &&
|
||||
actualNodeVersion[2] < minimumNodeVersion[2])))
|
||||
) {
|
||||
throw new Error("Provider pack build Node is older than 24.11.0");
|
||||
}
|
||||
const stableNodeRoot = join(temporaryRoot, "node_modules", "node");
|
||||
if (existsSync(stableNodeRoot)) {
|
||||
throw new Error(
|
||||
"Provider pack deployment unexpectedly claimed the stable Node path",
|
||||
);
|
||||
}
|
||||
const stableNodeCommand = join(stableNodeRoot, "bin", "node");
|
||||
mkdirSync(dirname(stableNodeCommand), { recursive: true, mode: 0o755 });
|
||||
copyFileSync(process.execPath, stableNodeCommand);
|
||||
chmodSync(stableNodeCommand, 0o755);
|
||||
|
||||
// pnpm's generated .bin shims embed the temporary deployment directory in
|
||||
// NODE_PATH. That makes an otherwise identical provider pack hash differ on
|
||||
// every build and leaks a nonexistent host path after relocation. Replace
|
||||
// every provider-facing shim with a pack-relative launcher that always uses
|
||||
// the pinned Node executable owned by this pack.
|
||||
// The image exposes these same installations to every adapter. Never add a
|
||||
// separate global/runner-only CLI version; refresh these packages and their
|
||||
// qualification digests together to the latest stable releases.
|
||||
writePortableNodeShim("codex", "@openai/codex/bin/codex.js");
|
||||
const claudeAcpRequire = createRequire(
|
||||
packRequire.resolve("@agentclientprotocol/claude-agent-acp/package.json"),
|
||||
);
|
||||
// Use the ACP bridge's SDK dependency directly, avoiding a second peer-
|
||||
// resolved SDK installation just to expose its CLI on the global PATH.
|
||||
const sdkRequire = createRequire(claudeAcpRequire.resolve("@anthropic-ai/claude-agent-sdk"));
|
||||
const claudeExecutable = sdkRequire.resolve(
|
||||
`@anthropic-ai/claude-agent-sdk-${process.platform}-${process.arch}/claude`,
|
||||
);
|
||||
writePortableExecutableShim(
|
||||
"claude",
|
||||
relative(realpathSync(join(temporaryRoot, "node_modules")), realpathSync(claudeExecutable)),
|
||||
);
|
||||
writePortableExecutableShim("node", "node/bin/node");
|
||||
writePortableExecutableShim("opencode", "opencode-ai/bin/opencode.exe");
|
||||
writePortableNodeShim("acpx", "acpx/dist/cli.js");
|
||||
const piPackageRoot = realpathSync(join(temporaryRoot, "node_modules", "@earendil-works", "pi-coding-agent"));
|
||||
await materializePiBinary(piPackageRoot);
|
||||
writePortableExecutableShim("pi", "@earendil-works/pi-coding-agent/vendor/standalone/pi");
|
||||
writePortableNodeShim("pi-acp", "pi-acp/dist/index.js");
|
||||
writePortableNodeShim(
|
||||
"claude-agent-acp",
|
||||
"@agentclientprotocol/claude-agent-acp/dist/index.js",
|
||||
);
|
||||
writePortableNodeShim(
|
||||
"codex-acp",
|
||||
"@agentclientprotocol/codex-acp/dist/index.js",
|
||||
);
|
||||
|
||||
// pnpm deploy may retain a workspace self-link under its virtual store. It
|
||||
// points outside the immutable pack and is not needed by the deployed
|
||||
// package, so it must not be staged or captured.
|
||||
rmSync(
|
||||
join(
|
||||
temporaryRoot,
|
||||
"node_modules",
|
||||
".pnpm",
|
||||
"node_modules",
|
||||
"@paperclipai",
|
||||
"paperclip-runner",
|
||||
),
|
||||
{ recursive: true, force: true },
|
||||
);
|
||||
|
||||
for (const shimName of readdirSync(
|
||||
join(temporaryRoot, "node_modules", ".bin"),
|
||||
)) {
|
||||
const shimPath = join(temporaryRoot, "node_modules", ".bin", shimName);
|
||||
const contents = readFileSync(shimPath, "utf8");
|
||||
if (contents.includes(temporaryParent)) {
|
||||
throw new Error(
|
||||
`Provider pack shim ${shimName} retains its temporary build path`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Exercise a clean import from the deployed dependency graph. This catches
|
||||
// malformed patch hunks that can look correct in an existing pnpm store but
|
||||
// land inside the wrong function when applied to a fresh acpx tarball.
|
||||
const acpxImport = spawnSync(
|
||||
join(temporaryRoot, "node_modules", "node", "bin", "node"),
|
||||
["--input-type=module", "-e", 'await import("acpx/runtime")'],
|
||||
{
|
||||
cwd: temporaryRoot,
|
||||
encoding: "utf8",
|
||||
timeout: 30_000,
|
||||
},
|
||||
);
|
||||
if (acpxImport.status !== 0) {
|
||||
throw new Error(
|
||||
`Provider pack ACPX import failed: ${String(acpxImport.stderr ?? "")
|
||||
.trim()
|
||||
.slice(-2_000)}`,
|
||||
);
|
||||
}
|
||||
|
||||
const opencodeProxyPath = "dist/cli/opencode-app-server-proxy.cjs";
|
||||
const acpxSidecarPath = "dist/cli/acpx-runtime-sidecar.cjs";
|
||||
const opencodeCommand = "node_modules/.bin/opencode";
|
||||
const opencodeExecutable = "node_modules/opencode-ai/bin/opencode.exe";
|
||||
const nodeCommand = "node_modules/node/bin/node";
|
||||
const productionLock = "pnpm-lock.yaml";
|
||||
copyFileSync(
|
||||
join(workspaceRoot, "pnpm-lock.yaml"),
|
||||
join(temporaryRoot, productionLock),
|
||||
);
|
||||
for (const relativePath of [
|
||||
nodeCommand,
|
||||
productionLock,
|
||||
opencodeProxyPath,
|
||||
acpxSidecarPath,
|
||||
opencodeCommand,
|
||||
opencodeExecutable,
|
||||
]) {
|
||||
if (!existsSync(join(temporaryRoot, relativePath))) {
|
||||
throw new Error(`Provider pack is missing ${relativePath}`);
|
||||
}
|
||||
}
|
||||
|
||||
const opencodeProxySha = sha256File(join(temporaryRoot, opencodeProxyPath));
|
||||
const acpxSidecarSha = sha256File(join(temporaryRoot, acpxSidecarPath));
|
||||
const distDigest = sha256Tree(join(temporaryRoot, "dist"));
|
||||
const configuredRevision =
|
||||
process.env.PAPERCLIP_RUNNER_SOURCE_REVISION?.trim();
|
||||
const revision =
|
||||
configuredRevision ??
|
||||
execFileSync("git", ["rev-parse", "HEAD"], {
|
||||
cwd: workspaceRoot,
|
||||
encoding: "utf8",
|
||||
}).trim();
|
||||
if (!/^[0-9a-f]{40}$/.test(revision)) {
|
||||
throw new Error("PAPERCLIP_RUNNER_SOURCE_REVISION must be a full Git SHA");
|
||||
}
|
||||
const dirty = configuredRevision
|
||||
? false
|
||||
: spawnSync("git", ["diff", "--quiet", "--", "packages/paperclip-runner"], {
|
||||
cwd: workspaceRoot,
|
||||
}).status !== 0;
|
||||
const payload = {
|
||||
pins: {
|
||||
nodeMinimum: minimumNodeVersion.join("."),
|
||||
codex: "0.153.4",
|
||||
opencode: "1.18.29",
|
||||
acpx: "0.13.1",
|
||||
claudeAcp: "0.70.0",
|
||||
codexAcp: "1.6.2",
|
||||
pi: "0.84.2",
|
||||
piAcp: "0.0.33",
|
||||
},
|
||||
target: { platform: process.platform, architecture: process.arch },
|
||||
runnerSourceRevision: `${revision}${dirty ? "-dirty" : ""}`,
|
||||
distDigest,
|
||||
bridgeDigest: `sha256:${createHash("sha256")
|
||||
.update(opencodeProxySha)
|
||||
.update("\n")
|
||||
.update(acpxSidecarSha)
|
||||
.update("\n")
|
||||
.update(distDigest)
|
||||
.digest("hex")}`,
|
||||
acpxProfileDigests: {
|
||||
pi: "sha256:24ff73fda6e3c76ddce2d359a79f5c4b8f292eb290e4d2ab85aac94676b2c2dc",
|
||||
claude:
|
||||
"sha256:9d73d1f0f121fb96cc8badb28c22d5bff02d8582eb2e40360a81c189e1b9422a",
|
||||
codex:
|
||||
"sha256:7a923b3829884d3cabcc9659d22cace3f86813e7bfffc90974b10140a45bc400",
|
||||
},
|
||||
artifacts: {
|
||||
nodeCommand: {
|
||||
path: nodeCommand,
|
||||
sha256: sha256File(join(temporaryRoot, nodeCommand)),
|
||||
},
|
||||
productionLock: {
|
||||
path: productionLock,
|
||||
sha256: sha256File(join(temporaryRoot, productionLock)),
|
||||
},
|
||||
opencodeCommand: {
|
||||
path: opencodeCommand,
|
||||
sha256: sha256File(join(temporaryRoot, opencodeCommand)),
|
||||
},
|
||||
opencodeExecutable: {
|
||||
path: opencodeExecutable,
|
||||
sha256: sha256File(join(temporaryRoot, opencodeExecutable)),
|
||||
},
|
||||
opencodeProxy: {
|
||||
path: opencodeProxyPath,
|
||||
sha256: opencodeProxySha,
|
||||
},
|
||||
acpxSidecar: { path: acpxSidecarPath, sha256: acpxSidecarSha },
|
||||
},
|
||||
};
|
||||
const manifest = {
|
||||
schema: "paperclip-runner/remote-provider-pack/v1",
|
||||
digest: `sha256:${createHash("sha256")
|
||||
.update(canonicalJson(payload))
|
||||
.digest("hex")}`,
|
||||
payload,
|
||||
};
|
||||
writeFileSync(
|
||||
join(temporaryRoot, "provider-pack.json"),
|
||||
`${JSON.stringify(manifest, null, 2)}\n`,
|
||||
{ mode: 0o600 },
|
||||
);
|
||||
|
||||
rmSync(outputRoot, { recursive: true, force: true });
|
||||
renameSync(temporaryRoot, outputRoot);
|
||||
process.stdout.write(`${outputRoot}\n`);
|
||||
} finally {
|
||||
rmSync(temporaryParent, { recursive: true, force: true });
|
||||
}
|
||||
|
|
@ -1,351 +1,74 @@
|
|||
import { normalizeProviderPackLayout } from "./provider-pack-layout.mjs";
|
||||
import { portableProviderShim } from "./portable-provider-shim.mjs";
|
||||
import { createHash } from "node:crypto";
|
||||
// The trusted qualification workflow invokes this target-owned entry point.
|
||||
// Never assemble from that job's CI-resolved lock, node_modules, shared dist,
|
||||
// or setup-node interpreter: the image's isolated stage is the source of truth.
|
||||
import { execFileSync, spawnSync } from "node:child_process";
|
||||
import {
|
||||
chmodSync,
|
||||
copyFileSync,
|
||||
existsSync,
|
||||
mkdirSync,
|
||||
mkdtempSync,
|
||||
readdirSync,
|
||||
readFileSync,
|
||||
readlinkSync,
|
||||
realpathSync,
|
||||
renameSync,
|
||||
rmSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join, relative, resolve } from "node:path";
|
||||
import { createRequire } from "node:module";
|
||||
import { existsSync, mkdirSync, mkdtempSync, readFileSync, realpathSync, renameSync, rmSync } from "node:fs";
|
||||
import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { materializePiBinary } from "./materialize-pi-binary.mjs";
|
||||
import { sha256File, verifyProviderPack } from "./provider-pack-integrity.mjs";
|
||||
|
||||
const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const workspaceRoot = resolve(packageRoot, "../..");
|
||||
const outputArgument = process.argv.slice(2).find((value) => value !== "--");
|
||||
const outputRoot = resolve(
|
||||
process.cwd(),
|
||||
outputArgument ?? join(packageRoot, "provider-pack"),
|
||||
);
|
||||
if (
|
||||
outputRoot === workspaceRoot ||
|
||||
outputRoot === packageRoot ||
|
||||
outputRoot === "/"
|
||||
) {
|
||||
throw new Error(`Refusing unsafe provider-pack output path: ${outputRoot}`);
|
||||
const defaultWorkspaceRoot = resolve(packageRoot, "../..");
|
||||
|
||||
export function buildProviderPack({ workspaceRoot = defaultWorkspaceRoot, outputRoot = join(packageRoot, "provider-pack"),
|
||||
revision = process.env.PAPERCLIP_RUNNER_SOURCE_REVISION?.trim(), run = spawnSync } = {}) {
|
||||
workspaceRoot = realpathSync(workspaceRoot);
|
||||
outputRoot = resolve(outputRoot);
|
||||
const missingSegments = [];
|
||||
let existingParent = outputRoot;
|
||||
while (!existsSync(existingParent)) { missingSegments.unshift(basename(existingParent)); existingParent = dirname(existingParent); }
|
||||
outputRoot = join(realpathSync(existingParent), ...missingSegments);
|
||||
const sourcePackage = join(workspaceRoot, "packages/paperclip-runner");
|
||||
function contains(base, target) {
|
||||
const child = relative(base, target);
|
||||
return !isAbsolute(child) && child !== ".." && !child.startsWith(`..${sep}`);
|
||||
}
|
||||
if (outputRoot === dirname(outputRoot) || contains(outputRoot, workspaceRoot)
|
||||
|| (contains(workspaceRoot, outputRoot) && outputRoot !== join(sourcePackage, "provider-pack"))) {
|
||||
throw new Error("Refusing unsafe provider-pack output path");
|
||||
}
|
||||
if (revision === undefined) {
|
||||
// CI supplies an authorized revision explicitly and may refresh only its
|
||||
// root lock. Local implicit HEAD must not mislabel changed image inputs.
|
||||
const dirty = execFileSync("git", ["status", "--porcelain", "--untracked-files=all", "--",
|
||||
".dockerignore", "docker/daytona-runner", "package.json", "pnpm-workspace.yaml", ".npmrc", "tsconfig.base.json",
|
||||
"patches", "scripts/link-plugin-dev-sdk.mjs", "packages", "server/package.json", "ui/package.json", "cli/package.json",
|
||||
":(exclude)packages/paperclip-runner/provider-pack", ":(glob,exclude)packages/paperclip-runner/.paperclip-provider-pack-*/**"],
|
||||
{ cwd: workspaceRoot, encoding: "utf8" });
|
||||
if (dirty.trim()) throw new Error("Canonical provider inputs are dirty; an implicit HEAD revision would be misleading");
|
||||
revision = execFileSync("git", ["rev-parse", "HEAD"], { cwd: workspaceRoot, encoding: "utf8" }).trim();
|
||||
}
|
||||
if (!/^[0-9a-f]{40}$/.test(revision)) throw new Error("PAPERCLIP_RUNNER_SOURCE_REVISION must be a full Git SHA");
|
||||
const dockerfile = join(workspaceRoot, "docker/daytona-runner/Dockerfile");
|
||||
const expectedLock = readFileSync(dockerfile, "utf8").match(/^ARG PAPERCLIP_RUNNER_LOCK_SHA256=([a-f0-9]{64})$/m)?.[1];
|
||||
if (!expectedLock) throw new Error("Canonical provider lock pin is missing");
|
||||
const actualLock = sha256File(join(workspaceRoot, "docker/daytona-runner/provider-dependencies.lock.yaml"));
|
||||
if (actualLock !== `sha256:${expectedLock}`) throw new Error("Canonical provider lock integrity mismatch");
|
||||
mkdirSync(dirname(outputRoot), { recursive: true });
|
||||
// Same filesystem as OUTPUT, so successful publication is an atomic rename.
|
||||
// Docker reads its inputs before this empty directory receives exported files.
|
||||
const temporaryParent = mkdtempSync(join(dirname(outputRoot), ".paperclip-provider-pack-"));
|
||||
const exported = join(temporaryParent, "exported"), backup = join(temporaryParent, "previous");
|
||||
let previousMoved = false, published = false;
|
||||
try {
|
||||
const build = run("docker", ["build", "--platform", "linux/amd64", "--target", "provider-pack-export",
|
||||
"--file", dockerfile, "--build-arg", `PAPERCLIP_RUNNER_SOURCE_REVISION=${revision}`,
|
||||
"--output", `type=local,dest=${exported}`, workspaceRoot],
|
||||
{ cwd: workspaceRoot, stdio: "inherit", timeout: 12 * 60 * 1000, killSignal: "SIGTERM" });
|
||||
if (build.error || build.status !== 0) throw new Error("Canonical provider-pack build failed");
|
||||
const manifest = verifyProviderPack(exported, { revision, lockSha256: expectedLock });
|
||||
if (existsSync(outputRoot)) { renameSync(outputRoot, backup); previousMoved = true; }
|
||||
try { renameSync(exported, outputRoot); published = true; }
|
||||
catch (error) { if (previousMoved) { renameSync(backup, outputRoot); previousMoved = false; } throw error; }
|
||||
return { outputRoot, manifest };
|
||||
} finally {
|
||||
// If even rollback fails, leave the prior pack in its recoverable backup.
|
||||
if (!previousMoved || published) rmSync(temporaryParent, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
const temporaryParent = mkdtempSync(join(tmpdir(), "paperclip-provider-pack-"));
|
||||
const temporaryRoot = join(temporaryParent, "pack");
|
||||
|
||||
function canonicalJson(value) {
|
||||
if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`;
|
||||
if (value && typeof value === "object") {
|
||||
return `{${Object.keys(value)
|
||||
.sort()
|
||||
.map((key) => `${JSON.stringify(key)}:${canonicalJson(value[key])}`)
|
||||
.join(",")}}`;
|
||||
}
|
||||
return JSON.stringify(value);
|
||||
}
|
||||
|
||||
function sha256File(path) {
|
||||
return `sha256:${createHash("sha256")
|
||||
.update(readFileSync(path))
|
||||
.digest("hex")}`;
|
||||
}
|
||||
|
||||
function sha256Tree(root) {
|
||||
const hash = createHash("sha256");
|
||||
const visit = (directory, prefix = "") => {
|
||||
const entries = readdirSync(directory, { withFileTypes: true }).sort(
|
||||
(left, right) => left.name.localeCompare(right.name),
|
||||
);
|
||||
for (const entry of entries) {
|
||||
const relativePath = prefix ? `${prefix}/${entry.name}` : entry.name;
|
||||
const absolutePath = join(directory, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
hash.update(`directory\0${relativePath}\n`);
|
||||
visit(absolutePath, relativePath);
|
||||
} else if (entry.isFile()) {
|
||||
hash.update(`file\0${relativePath}\0${sha256File(absolutePath)}\n`);
|
||||
} else if (entry.isSymbolicLink()) {
|
||||
hash.update(
|
||||
`symlink\0${relativePath}\0${readlinkSync(absolutePath)}\n`,
|
||||
);
|
||||
} else {
|
||||
throw new Error(
|
||||
`Provider pack tree contains unsupported entry ${relativePath}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
};
|
||||
visit(root);
|
||||
return `sha256:${hash.digest("hex")}`;
|
||||
}
|
||||
|
||||
function writePortableNodeShim(name, entrypoint) {
|
||||
const shimPath = join(temporaryRoot, "node_modules", ".bin", name);
|
||||
writeFileSync(shimPath, portableProviderShim(entrypoint, { node: true }));
|
||||
chmodSync(shimPath, 0o755);
|
||||
}
|
||||
|
||||
function writePortableExecutableShim(name, executable) {
|
||||
const shimPath = join(temporaryRoot, "node_modules", ".bin", name);
|
||||
writeFileSync(shimPath, portableProviderShim(executable));
|
||||
chmodSync(shimPath, 0o755);
|
||||
}
|
||||
|
||||
try {
|
||||
const deployed = spawnSync(
|
||||
"pnpm",
|
||||
[
|
||||
"--filter",
|
||||
"@paperclipai/paperclip-runner",
|
||||
"deploy",
|
||||
"--prod",
|
||||
temporaryRoot,
|
||||
],
|
||||
{ cwd: workspaceRoot, encoding: "utf8", stdio: "inherit" },
|
||||
);
|
||||
if (deployed.status !== 0) {
|
||||
throw new Error(`pnpm deploy failed with exit code ${deployed.status}`);
|
||||
}
|
||||
|
||||
normalizeProviderPackLayout(temporaryRoot);
|
||||
|
||||
// Fail the image build if a bridge silently brings back an older/private
|
||||
// provider CLI. A direct dependency alone does not deduplicate pnpm's graph.
|
||||
const packRequire = createRequire(join(temporaryRoot, "package.json"));
|
||||
const codexAcpRequire = createRequire(packRequire.resolve("@agentclientprotocol/codex-acp/package.json"));
|
||||
if (realpathSync(codexAcpRequire.resolve("@openai/codex/package.json")) !==
|
||||
realpathSync(packRequire.resolve("@openai/codex/package.json"))) {
|
||||
throw new Error("Codex ACP must share the image's Codex installation");
|
||||
}
|
||||
|
||||
// Reuse the already-qualified build interpreter instead of introducing a
|
||||
// package-manager lifecycle hook or a second binary supply chain. The pack
|
||||
// manifest binds the copied bytes, platform, architecture, and minimum
|
||||
// version before any provider is launched.
|
||||
const minimumNodeVersion = [24, 11, 0];
|
||||
const actualNodeVersion = process.versions.node.split(".").map(Number);
|
||||
if (
|
||||
actualNodeVersion[0] < minimumNodeVersion[0] ||
|
||||
(actualNodeVersion[0] === minimumNodeVersion[0] &&
|
||||
(actualNodeVersion[1] < minimumNodeVersion[1] ||
|
||||
(actualNodeVersion[1] === minimumNodeVersion[1] &&
|
||||
actualNodeVersion[2] < minimumNodeVersion[2])))
|
||||
) {
|
||||
throw new Error("Provider pack build Node is older than 24.11.0");
|
||||
}
|
||||
const stableNodeRoot = join(temporaryRoot, "node_modules", "node");
|
||||
if (existsSync(stableNodeRoot)) {
|
||||
throw new Error(
|
||||
"Provider pack deployment unexpectedly claimed the stable Node path",
|
||||
);
|
||||
}
|
||||
const stableNodeCommand = join(stableNodeRoot, "bin", "node");
|
||||
mkdirSync(dirname(stableNodeCommand), { recursive: true, mode: 0o755 });
|
||||
copyFileSync(process.execPath, stableNodeCommand);
|
||||
chmodSync(stableNodeCommand, 0o755);
|
||||
|
||||
// pnpm's generated .bin shims embed the temporary deployment directory in
|
||||
// NODE_PATH. That makes an otherwise identical provider pack hash differ on
|
||||
// every build and leaks a nonexistent host path after relocation. Replace
|
||||
// every provider-facing shim with a pack-relative launcher that always uses
|
||||
// the pinned Node executable owned by this pack.
|
||||
// The image exposes these same installations to every adapter. Never add a
|
||||
// separate global/runner-only CLI version; refresh these packages and their
|
||||
// qualification digests together to the latest stable releases.
|
||||
writePortableNodeShim("codex", "@openai/codex/bin/codex.js");
|
||||
const claudeAcpRequire = createRequire(
|
||||
packRequire.resolve("@agentclientprotocol/claude-agent-acp/package.json"),
|
||||
);
|
||||
// Use the ACP bridge's SDK dependency directly, avoiding a second peer-
|
||||
// resolved SDK installation just to expose its CLI on the global PATH.
|
||||
const sdkRequire = createRequire(claudeAcpRequire.resolve("@anthropic-ai/claude-agent-sdk"));
|
||||
const claudeExecutable = sdkRequire.resolve(
|
||||
`@anthropic-ai/claude-agent-sdk-${process.platform}-${process.arch}/claude`,
|
||||
);
|
||||
writePortableExecutableShim(
|
||||
"claude",
|
||||
relative(realpathSync(join(temporaryRoot, "node_modules")), realpathSync(claudeExecutable)),
|
||||
);
|
||||
writePortableExecutableShim("node", "node/bin/node");
|
||||
writePortableExecutableShim("opencode", "opencode-ai/bin/opencode.exe");
|
||||
writePortableNodeShim("acpx", "acpx/dist/cli.js");
|
||||
const piPackageRoot = realpathSync(join(temporaryRoot, "node_modules", "@earendil-works", "pi-coding-agent"));
|
||||
await materializePiBinary(piPackageRoot);
|
||||
writePortableExecutableShim("pi", "@earendil-works/pi-coding-agent/vendor/standalone/pi");
|
||||
writePortableNodeShim("pi-acp", "pi-acp/dist/index.js");
|
||||
writePortableNodeShim(
|
||||
"claude-agent-acp",
|
||||
"@agentclientprotocol/claude-agent-acp/dist/index.js",
|
||||
);
|
||||
writePortableNodeShim(
|
||||
"codex-acp",
|
||||
"@agentclientprotocol/codex-acp/dist/index.js",
|
||||
);
|
||||
|
||||
// pnpm deploy may retain a workspace self-link under its virtual store. It
|
||||
// points outside the immutable pack and is not needed by the deployed
|
||||
// package, so it must not be staged or captured.
|
||||
rmSync(
|
||||
join(
|
||||
temporaryRoot,
|
||||
"node_modules",
|
||||
".pnpm",
|
||||
"node_modules",
|
||||
"@paperclipai",
|
||||
"paperclip-runner",
|
||||
),
|
||||
{ recursive: true, force: true },
|
||||
);
|
||||
|
||||
for (const shimName of readdirSync(
|
||||
join(temporaryRoot, "node_modules", ".bin"),
|
||||
)) {
|
||||
const shimPath = join(temporaryRoot, "node_modules", ".bin", shimName);
|
||||
const contents = readFileSync(shimPath, "utf8");
|
||||
if (contents.includes(temporaryParent)) {
|
||||
throw new Error(
|
||||
`Provider pack shim ${shimName} retains its temporary build path`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Exercise a clean import from the deployed dependency graph. This catches
|
||||
// malformed patch hunks that can look correct in an existing pnpm store but
|
||||
// land inside the wrong function when applied to a fresh acpx tarball.
|
||||
const acpxImport = spawnSync(
|
||||
join(temporaryRoot, "node_modules", "node", "bin", "node"),
|
||||
["--input-type=module", "-e", 'await import("acpx/runtime")'],
|
||||
{
|
||||
cwd: temporaryRoot,
|
||||
encoding: "utf8",
|
||||
timeout: 30_000,
|
||||
},
|
||||
);
|
||||
if (acpxImport.status !== 0) {
|
||||
throw new Error(
|
||||
`Provider pack ACPX import failed: ${String(acpxImport.stderr ?? "")
|
||||
.trim()
|
||||
.slice(-2_000)}`,
|
||||
);
|
||||
}
|
||||
|
||||
const opencodeProxyPath = "dist/cli/opencode-app-server-proxy.cjs";
|
||||
const acpxSidecarPath = "dist/cli/acpx-runtime-sidecar.cjs";
|
||||
const opencodeCommand = "node_modules/.bin/opencode";
|
||||
const opencodeExecutable = "node_modules/opencode-ai/bin/opencode.exe";
|
||||
const nodeCommand = "node_modules/node/bin/node";
|
||||
const productionLock = "pnpm-lock.yaml";
|
||||
copyFileSync(
|
||||
join(workspaceRoot, "pnpm-lock.yaml"),
|
||||
join(temporaryRoot, productionLock),
|
||||
);
|
||||
for (const relativePath of [
|
||||
nodeCommand,
|
||||
productionLock,
|
||||
opencodeProxyPath,
|
||||
acpxSidecarPath,
|
||||
opencodeCommand,
|
||||
opencodeExecutable,
|
||||
]) {
|
||||
if (!existsSync(join(temporaryRoot, relativePath))) {
|
||||
throw new Error(`Provider pack is missing ${relativePath}`);
|
||||
}
|
||||
}
|
||||
|
||||
const opencodeProxySha = sha256File(join(temporaryRoot, opencodeProxyPath));
|
||||
const acpxSidecarSha = sha256File(join(temporaryRoot, acpxSidecarPath));
|
||||
const distDigest = sha256Tree(join(temporaryRoot, "dist"));
|
||||
const configuredRevision =
|
||||
process.env.PAPERCLIP_RUNNER_SOURCE_REVISION?.trim();
|
||||
const revision =
|
||||
configuredRevision ??
|
||||
execFileSync("git", ["rev-parse", "HEAD"], {
|
||||
cwd: workspaceRoot,
|
||||
encoding: "utf8",
|
||||
}).trim();
|
||||
if (!/^[0-9a-f]{40}$/.test(revision)) {
|
||||
throw new Error("PAPERCLIP_RUNNER_SOURCE_REVISION must be a full Git SHA");
|
||||
}
|
||||
const dirty = configuredRevision
|
||||
? false
|
||||
: spawnSync("git", ["diff", "--quiet", "--", "packages/paperclip-runner"], {
|
||||
cwd: workspaceRoot,
|
||||
}).status !== 0;
|
||||
const payload = {
|
||||
pins: {
|
||||
nodeMinimum: minimumNodeVersion.join("."),
|
||||
codex: "0.153.4",
|
||||
opencode: "1.18.29",
|
||||
acpx: "0.13.1",
|
||||
claudeAcp: "0.70.0",
|
||||
codexAcp: "1.6.2",
|
||||
pi: "0.84.2",
|
||||
piAcp: "0.0.33",
|
||||
},
|
||||
target: { platform: process.platform, architecture: process.arch },
|
||||
runnerSourceRevision: `${revision}${dirty ? "-dirty" : ""}`,
|
||||
distDigest,
|
||||
bridgeDigest: `sha256:${createHash("sha256")
|
||||
.update(opencodeProxySha)
|
||||
.update("\n")
|
||||
.update(acpxSidecarSha)
|
||||
.update("\n")
|
||||
.update(distDigest)
|
||||
.digest("hex")}`,
|
||||
acpxProfileDigests: {
|
||||
pi: "sha256:24ff73fda6e3c76ddce2d359a79f5c4b8f292eb290e4d2ab85aac94676b2c2dc",
|
||||
claude:
|
||||
"sha256:9d73d1f0f121fb96cc8badb28c22d5bff02d8582eb2e40360a81c189e1b9422a",
|
||||
codex:
|
||||
"sha256:7a923b3829884d3cabcc9659d22cace3f86813e7bfffc90974b10140a45bc400",
|
||||
},
|
||||
artifacts: {
|
||||
nodeCommand: {
|
||||
path: nodeCommand,
|
||||
sha256: sha256File(join(temporaryRoot, nodeCommand)),
|
||||
},
|
||||
productionLock: {
|
||||
path: productionLock,
|
||||
sha256: sha256File(join(temporaryRoot, productionLock)),
|
||||
},
|
||||
opencodeCommand: {
|
||||
path: opencodeCommand,
|
||||
sha256: sha256File(join(temporaryRoot, opencodeCommand)),
|
||||
},
|
||||
opencodeExecutable: {
|
||||
path: opencodeExecutable,
|
||||
sha256: sha256File(join(temporaryRoot, opencodeExecutable)),
|
||||
},
|
||||
opencodeProxy: {
|
||||
path: opencodeProxyPath,
|
||||
sha256: opencodeProxySha,
|
||||
},
|
||||
acpxSidecar: { path: acpxSidecarPath, sha256: acpxSidecarSha },
|
||||
},
|
||||
};
|
||||
const manifest = {
|
||||
schema: "paperclip-runner/remote-provider-pack/v1",
|
||||
digest: `sha256:${createHash("sha256")
|
||||
.update(canonicalJson(payload))
|
||||
.digest("hex")}`,
|
||||
payload,
|
||||
};
|
||||
writeFileSync(
|
||||
join(temporaryRoot, "provider-pack.json"),
|
||||
`${JSON.stringify(manifest, null, 2)}\n`,
|
||||
{ mode: 0o600 },
|
||||
);
|
||||
|
||||
rmSync(outputRoot, { recursive: true, force: true });
|
||||
renameSync(temporaryRoot, outputRoot);
|
||||
process.stdout.write(`${outputRoot}\n`);
|
||||
} finally {
|
||||
rmSync(temporaryParent, { recursive: true, force: true });
|
||||
if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
||||
const outputArgument = process.argv.slice(2).find((value) => value !== "--");
|
||||
const result = buildProviderPack({ outputRoot: resolve(process.cwd(), outputArgument ?? join(packageRoot, "provider-pack")) });
|
||||
process.stdout.write(`${result.outputRoot}\n`);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,111 @@
|
|||
import assert from "node:assert/strict";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { test } from "node:test";
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, readdirSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
import { buildProviderPack } from "./build-provider-pack.mjs";
|
||||
import { canonicalJson, sha256File, sha256Tree } from "./provider-pack-integrity.mjs";
|
||||
const revision = "a".repeat(40);
|
||||
function fixture(runTest) {
|
||||
const parent = mkdtempSync(join(tmpdir(), "canonical-provider-test-"));
|
||||
const workspaceRoot = join(parent, "source"), outputRoot = join(parent, "output");
|
||||
const lock = "the dedicated immutable resolution\n";
|
||||
const hash = createHash("sha256").update(lock).digest("hex");
|
||||
mkdirSync(join(workspaceRoot, "docker/daytona-runner"), { recursive: true });
|
||||
writeFileSync(join(workspaceRoot, "docker/daytona-runner/provider-dependencies.lock.yaml"), lock);
|
||||
writeFileSync(join(workspaceRoot, "docker/daytona-runner/Dockerfile"), `ARG PAPERCLIP_RUNNER_LOCK_SHA256=${hash}\n`);
|
||||
writeFileSync(join(workspaceRoot, "pnpm-lock.yaml"), "different CI resolution");
|
||||
const calls = [];
|
||||
function exported(args, tamper) {
|
||||
const destination = args[args.indexOf("--output") + 1].slice("type=local,dest=".length);
|
||||
const paths = {
|
||||
nodeCommand: "node_modules/node/bin/node", productionLock: "pnpm-lock.yaml",
|
||||
opencodeCommand: "node_modules/.bin/opencode", opencodeExecutable: "node_modules/opencode-ai/bin/opencode.exe",
|
||||
opencodeProxy: "dist/cli/opencode-app-server-proxy.cjs", acpxSidecar: "dist/cli/acpx-runtime-sidecar.cjs",
|
||||
};
|
||||
const artifacts = {};
|
||||
for (const [name, path] of Object.entries(paths)) {
|
||||
const file = join(destination, path);
|
||||
mkdirSync(dirname(file), { recursive: true });
|
||||
writeFileSync(file, name === "productionLock" ? lock : name);
|
||||
artifacts[name] = { path, sha256: sha256File(file) };
|
||||
}
|
||||
const distDigest = sha256Tree(join(destination, "dist"));
|
||||
const payload = { target: { platform: "linux", architecture: "x64" },
|
||||
runnerSourceRevision: args.find((value) => value.startsWith("PAPERCLIP_RUNNER_SOURCE_REVISION=")).split("=")[1],
|
||||
artifacts, distDigest, bridgeDigest: `sha256:${createHash("sha256").update(artifacts.opencodeProxy.sha256).update("\n")
|
||||
.update(artifacts.acpxSidecar.sha256).update("\n").update(distDigest).digest("hex")}` };
|
||||
const manifest = { schema: "paperclip-runner/remote-provider-pack/v1", payload,
|
||||
digest: `sha256:${createHash("sha256").update(canonicalJson(payload)).digest("hex")}` };
|
||||
writeFileSync(join(destination, "provider-pack.json"), JSON.stringify(manifest));
|
||||
tamper?.(destination);
|
||||
}
|
||||
const run = (command, args, options) => { calls.push({ command, args, options }); exported(args); return { status: 0 }; };
|
||||
try { runTest({ parent, workspaceRoot, outputRoot, hash, calls, run, exported }); }
|
||||
finally { rmSync(parent, { recursive: true, force: true }); }
|
||||
}
|
||||
test("workflow entry isolates the canonical linux stage from root dependency graph and stale outputs", () => fixture((f) => {
|
||||
mkdirSync(join(f.workspaceRoot, "node_modules"));
|
||||
writeFileSync(join(f.workspaceRoot, "node_modules/stale"), "must not be assembled");
|
||||
const first = buildProviderPack({ ...f, revision });
|
||||
writeFileSync(join(f.workspaceRoot, "pnpm-lock.yaml"), "another CI lock");
|
||||
const second = buildProviderPack({ ...f, revision });
|
||||
assert.equal(first.manifest.digest, second.manifest.digest);
|
||||
for (const call of f.calls) {
|
||||
assert.equal(call.command, "docker");
|
||||
assert.equal(call.args[call.args.indexOf("--platform") + 1], "linux/amd64");
|
||||
assert.equal(call.args[call.args.indexOf("--target") + 1], "provider-pack-export");
|
||||
assert.ok(call.args.includes(`PAPERCLIP_RUNNER_SOURCE_REVISION=${revision}`));
|
||||
assert.equal(call.options.timeout, 720_000);
|
||||
}
|
||||
assert.equal(readFileSync(join(f.outputRoot, "pnpm-lock.yaml"), "utf8"), "the dedicated immutable resolution\n");
|
||||
assert.deepEqual(readdirSync(f.parent).sort(), ["output", "source"]);
|
||||
}));
|
||||
test("tampered dedicated lock fails before invoking Docker", () => fixture((f) => {
|
||||
writeFileSync(join(f.workspaceRoot, "docker/daytona-runner/provider-dependencies.lock.yaml"), "changed");
|
||||
assert.throws(() => buildProviderPack({ ...f, revision }), /lock integrity mismatch/);
|
||||
assert.equal(f.calls.length, 0);
|
||||
}));
|
||||
test("failed canonical build preserves prior pack and cleans only temporary output", () => fixture((f) => {
|
||||
mkdirSync(f.outputRoot); writeFileSync(join(f.outputRoot, "old"), "retain");
|
||||
assert.throws(() => buildProviderPack({ ...f, revision, run: () => ({ status: 1 }) }), /build failed/);
|
||||
assert.equal(readFileSync(join(f.outputRoot, "old"), "utf8"), "retain");
|
||||
assert.deepEqual(readdirSync(f.parent).sort(), ["output", "source"]);
|
||||
}));
|
||||
test("exported executable tampering is rejected without replacing the prior pack", () => fixture((f) => {
|
||||
mkdirSync(f.outputRoot); writeFileSync(join(f.outputRoot, "old"), "retain");
|
||||
const run = (_command, args) => { f.exported(args, (root) => writeFileSync(join(root, "node_modules/node/bin/node"), "tampered")); return { status: 0 }; };
|
||||
assert.throws(() => buildProviderPack({ ...f, revision, run }), /artifact integrity mismatch/);
|
||||
assert.equal(readFileSync(join(f.outputRoot, "old"), "utf8"), "retain");
|
||||
}));
|
||||
test("arbitrary source-tree output and invalid revision are rejected before building", () => fixture((f) => {
|
||||
for (const outputRoot of [f.workspaceRoot, join(f.workspaceRoot, "packages"), join(f.workspaceRoot, "..hidden"), dirname(f.workspaceRoot)]) {
|
||||
assert.throws(() => buildProviderPack({ ...f, outputRoot, revision }), /unsafe/);
|
||||
}
|
||||
assert.throws(() => buildProviderPack({ ...f, revision: "not-a-sha" }), /full Git SHA/);
|
||||
assert.equal(f.calls.length, 0);
|
||||
}));
|
||||
|
||||
test("implicit HEAD rejects dirty canonical inputs while explicit trusted revisions tolerate CI root-lock drift", () => fixture((f) => {
|
||||
const previous = process.env.PAPERCLIP_RUNNER_SOURCE_REVISION;
|
||||
delete process.env.PAPERCLIP_RUNNER_SOURCE_REVISION;
|
||||
try {
|
||||
execFileSync("git", ["init", f.workspaceRoot], { stdio: "ignore" });
|
||||
execFileSync("git", ["-C", f.workspaceRoot, "add", "."]);
|
||||
execFileSync("git", ["-C", f.workspaceRoot, "-c", "user.name=Test", "-c", "user.email=test@example.com", "commit", "-m", "fixture"], { stdio: "ignore" });
|
||||
writeFileSync(join(f.workspaceRoot, "pnpm-lock.yaml"), "CI-owned changed root lock");
|
||||
buildProviderPack({ ...f });
|
||||
assert.equal(f.calls.length, 1);
|
||||
mkdirSync(join(f.workspaceRoot, "packages/paperclip-runner/scripts"), { recursive: true });
|
||||
writeFileSync(join(f.workspaceRoot, "packages/paperclip-runner/scripts/new-source.mjs"), "changed source");
|
||||
assert.throws(() => buildProviderPack({ ...f }), /inputs are dirty/);
|
||||
assert.equal(f.calls.length, 1);
|
||||
buildProviderPack({ ...f, revision });
|
||||
assert.equal(f.calls.length, 2);
|
||||
} finally {
|
||||
if (previous === undefined) delete process.env.PAPERCLIP_RUNNER_SOURCE_REVISION;
|
||||
else process.env.PAPERCLIP_RUNNER_SOURCE_REVISION = previous;
|
||||
}
|
||||
}));
|
||||
|
|
@ -0,0 +1,87 @@
|
|||
import { createHash } from "node:crypto";
|
||||
import { readdirSync, readFileSync, readlinkSync, realpathSync } from "node:fs";
|
||||
import { join, resolve, sep } from "node:path";
|
||||
|
||||
export function canonicalJson(value) {
|
||||
if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`;
|
||||
if (value && typeof value === "object") {
|
||||
return `{${Object.keys(value)
|
||||
.sort()
|
||||
.map((key) => `${JSON.stringify(key)}:${canonicalJson(value[key])}`)
|
||||
.join(",")}}`;
|
||||
}
|
||||
return JSON.stringify(value);
|
||||
}
|
||||
|
||||
export function sha256File(path) {
|
||||
return `sha256:${createHash("sha256")
|
||||
.update(readFileSync(path))
|
||||
.digest("hex")}`;
|
||||
}
|
||||
|
||||
export function sha256Tree(root) {
|
||||
const hash = createHash("sha256");
|
||||
const visit = (directory, prefix = "") => {
|
||||
const entries = readdirSync(directory, { withFileTypes: true }).sort(
|
||||
(left, right) => left.name.localeCompare(right.name),
|
||||
);
|
||||
for (const entry of entries) {
|
||||
const relativePath = prefix ? `${prefix}/${entry.name}` : entry.name;
|
||||
const absolutePath = join(directory, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
hash.update(`directory\0${relativePath}\n`);
|
||||
visit(absolutePath, relativePath);
|
||||
} else if (entry.isFile()) {
|
||||
hash.update(`file\0${relativePath}\0${sha256File(absolutePath)}\n`);
|
||||
} else if (entry.isSymbolicLink()) {
|
||||
hash.update(
|
||||
`symlink\0${relativePath}\0${readlinkSync(absolutePath)}\n`,
|
||||
);
|
||||
} else {
|
||||
throw new Error(
|
||||
`Provider pack tree contains unsupported entry ${relativePath}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
};
|
||||
visit(root);
|
||||
return `sha256:${hash.digest("hex")}`;
|
||||
}
|
||||
|
||||
|
||||
/** Verify exported bytes before replacing any prior working provider pack. */
|
||||
export function verifyProviderPack(root, { revision, lockSha256 }) {
|
||||
const manifest = JSON.parse(readFileSync(join(root, "provider-pack.json"), "utf8"));
|
||||
const payload = manifest.payload;
|
||||
if (manifest.schema !== "paperclip-runner/remote-provider-pack/v1"
|
||||
|| payload?.runnerSourceRevision !== revision
|
||||
|| payload.target?.platform !== "linux" || payload.target?.architecture !== "x64"
|
||||
|| manifest.digest !== `sha256:${createHash("sha256").update(canonicalJson(payload)).digest("hex")}`) {
|
||||
throw new Error("Canonical provider-pack manifest mismatch");
|
||||
}
|
||||
const expectedPaths = {
|
||||
nodeCommand: "node_modules/node/bin/node", productionLock: "pnpm-lock.yaml",
|
||||
opencodeCommand: "node_modules/.bin/opencode", opencodeExecutable: "node_modules/opencode-ai/bin/opencode.exe",
|
||||
opencodeProxy: "dist/cli/opencode-app-server-proxy.cjs", acpxSidecar: "dist/cli/acpx-runtime-sidecar.cjs",
|
||||
};
|
||||
const canonicalRoot = realpathSync(root);
|
||||
for (const [name, expectedPath] of Object.entries(expectedPaths)) {
|
||||
const artifact = payload.artifacts?.[name];
|
||||
if (artifact?.path !== expectedPath || !/^sha256:[a-f0-9]{64}$/.test(artifact.sha256 ?? "")) {
|
||||
throw new Error("Canonical provider-pack artifact metadata mismatch");
|
||||
}
|
||||
const file = realpathSync(resolve(root, expectedPath));
|
||||
if (!file.startsWith(`${canonicalRoot}${sep}`) || sha256File(file) !== artifact.sha256) {
|
||||
throw new Error("Canonical provider-pack artifact integrity mismatch");
|
||||
}
|
||||
}
|
||||
if (payload.artifacts.productionLock.sha256 !== `sha256:${lockSha256}` || sha256Tree(join(root, "dist")) !== payload.distDigest) {
|
||||
throw new Error("Canonical provider-pack lock or compiled output mismatch");
|
||||
}
|
||||
const bridgeDigest = `sha256:${createHash("sha256")
|
||||
.update(payload.artifacts.opencodeProxy.sha256).update("\n")
|
||||
.update(payload.artifacts.acpxSidecar.sha256).update("\n")
|
||||
.update(payload.distDigest).digest("hex")}`;
|
||||
if (payload.bridgeDigest !== bridgeDigest) throw new Error("Canonical provider-pack bridge integrity mismatch");
|
||||
return manifest;
|
||||
}
|
||||
|
|
@ -42,7 +42,7 @@ const runnerdAcpxBackend = await readFile(
|
|||
"utf8",
|
||||
);
|
||||
const providerPackBuilder = await readFile(
|
||||
new URL("../scripts/build-provider-pack.mjs", import.meta.url),
|
||||
new URL("../scripts/assemble-provider-pack.mjs", import.meta.url),
|
||||
"utf8",
|
||||
);
|
||||
const nativeSessionExecutor = await readFile(
|
||||
|
|
|
|||
|
|
@ -0,0 +1,147 @@
|
|||
import { AsyncLocalStorage } from "node:async_hooks";
|
||||
import { ROOT_CONTEXT, trace, type Context } from "@opentelemetry/api";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { getActiveStepContext } from "@paperclipai/adapter-utils/acpx-engine/startup-timing";
|
||||
import type { StartupTraceContextHandle } from "../instrumentation.js";
|
||||
import {
|
||||
captureSandboxPerformanceContext,
|
||||
hasSandboxPerformanceTrace,
|
||||
measureSandboxOperation,
|
||||
runWithSandboxPerformanceTrace,
|
||||
setSandboxPerformanceRunAttributes,
|
||||
type SandboxPerformanceRecord,
|
||||
} from "../services/sandbox-performance.js";
|
||||
|
||||
function recordingContext() {
|
||||
const active = new AsyncLocalStorage<Context>();
|
||||
const spans: Array<{ name: string; id: string; parentId?: string; attributes: Record<string, unknown>; status?: unknown; ended: boolean; events: unknown[] }> = [];
|
||||
let next = 1;
|
||||
const tracing: StartupTraceContextHandle = {
|
||||
tracer: { startSpan(name, options, parent) {
|
||||
const id = (next++).toString(16).padStart(16, "0");
|
||||
const record = { name, id, parentId: trace.getSpanContext(parent as Context ?? active.getStore() ?? ROOT_CONTEXT)?.spanId,
|
||||
attributes: { ...(options as { attributes?: Record<string, unknown> })?.attributes },
|
||||
ended: false, status: undefined as unknown, events: [] as unknown[] };
|
||||
spans.push(record);
|
||||
return {
|
||||
...trace.wrapSpanContext({ spanId: id, traceId: "1234567890abcdef1234567890abcdef", traceFlags: 1 }),
|
||||
spanContext: () => ({ spanId: id, traceId: "1234567890abcdef1234567890abcdef", traceFlags: 1 }),
|
||||
setAttribute(key: string, value: unknown) { record.attributes[key] = value; },
|
||||
setStatus(status: unknown) { record.status = status; },
|
||||
addEvent(name: string, attributes: unknown) { record.events.push({ name, attributes }); },
|
||||
end() { record.ended = true; },
|
||||
};
|
||||
} },
|
||||
contextWithSpan(span) { return trace.setSpan(active.getStore() ?? ROOT_CONTEXT, span as ReturnType<typeof trace.wrapSpanContext>); },
|
||||
withContext(context, work) { return active.run(context as Context, work); },
|
||||
};
|
||||
return { tracing, spans, active };
|
||||
}
|
||||
|
||||
describe("sandbox performance trace", () => {
|
||||
it("uses real contexts and keeps parallel branches separate across awaits", async () => {
|
||||
const { tracing, spans, active } = recordingContext();
|
||||
const records: SandboxPerformanceRecord[] = [];
|
||||
await runWithSandboxPerformanceTrace({ runId: "private-run-id", enabled: true, traceContext: tracing,
|
||||
onBatch: async (batch) => { records.push(...batch.records); } }, async () => {
|
||||
setSandboxPerformanceRunAttributes({ runtime: "legacy" });
|
||||
await Promise.all(["left", "right"].map((side) => measureSandboxOperation(`sandbox.${side}`, {}, async () => {
|
||||
const parent = trace.getSpanContext(active.getStore()!)!.spanId;
|
||||
await new Promise<void>((resolve) => setImmediate(resolve));
|
||||
expect(trace.getSpanContext(active.getStore()!)?.spanId).toBe(parent);
|
||||
expect(trace.getSpanContext(getActiveStepContext()?.parentContext as Context)?.spanId).toBe(parent);
|
||||
await measureSandboxOperation(`sandbox.${side}.child`, {}, async () => undefined);
|
||||
})));
|
||||
});
|
||||
const root = spans.find((span) => span.name === "sandbox.run")!;
|
||||
for (const side of ["left", "right"]) {
|
||||
const parent = spans.find((span) => span.name === `sandbox.${side}`)!;
|
||||
expect(parent.parentId).toBe(root.id);
|
||||
expect(spans.find((span) => span.name === `sandbox.${side}.child`)?.parentId).toBe(parent.id);
|
||||
}
|
||||
expect(root.attributes["paperclip.sandbox.runtime"]).toBe("legacy");
|
||||
expect(root.attributes["paperclip.sandbox.recordCount"]).toBe(5);
|
||||
expect(root.attributes["paperclip.sandbox.dropped"]).toBe(0);
|
||||
expect(records.every((record) => record.traceId === "1234567890abcdef1234567890abcdef")).toBe(true);
|
||||
expect(new Set(records.map((record) => record.id)).size).toBe(5);
|
||||
expect(spans.every((span) => span.ended)).toBe(true);
|
||||
expect(hasSandboxPerformanceTrace()).toBe(false);
|
||||
expect(active.getStore()).toBeUndefined();
|
||||
});
|
||||
|
||||
it("drops private attributes and error messages while preserving the original failure", async () => {
|
||||
const { tracing, spans } = recordingContext();
|
||||
const records: SandboxPerformanceRecord[] = [];
|
||||
const failure = new Error("private-secret-error");
|
||||
await expect(runWithSandboxPerformanceTrace({ runId: "private-run-id", enabled: true, traceContext: tracing,
|
||||
onBatch: async (batch) => { records.push(...batch.records); } }, async () => {
|
||||
await measureSandboxOperation("sandbox.read", { path: "/secret", credential: "private-token", scope: "task", bytes: 12, files: NaN,
|
||||
operation: "https://private.example", outcome: "bad\nvalue" }, async () => { throw failure; });
|
||||
})).rejects.toBe(failure);
|
||||
expect(records.map((record) => record.outcome)).toEqual(["failed", "failed"]);
|
||||
expect(records[0]?.attributes).toEqual({ scope: "task", bytes: 12 });
|
||||
expect(spans.every((span) => (span.status as { code: number }).code === 2)).toBe(true);
|
||||
expect(JSON.stringify({ spans, records })).not.toMatch(/private|secret|credential/);
|
||||
});
|
||||
|
||||
it("bounds persistence, exposes lost records, and never writes batches during measured work", async () => {
|
||||
const { tracing, spans } = recordingContext();
|
||||
let finished = false;
|
||||
const batches: Array<{ records: SandboxPerformanceRecord[]; dropped: number }> = [];
|
||||
await runWithSandboxPerformanceTrace({ runId: "run", enabled: true, traceContext: tracing, maxRecords: 300,
|
||||
onBatch: async (batch) => { expect(finished).toBe(true); batches.push(batch); } }, async () => {
|
||||
for (let i = 0; i < 350; i++) await measureSandboxOperation("sandbox.read", { fileIndex: i }, async () => undefined);
|
||||
finished = true;
|
||||
});
|
||||
expect(batches.map((batch) => batch.records.length)).toEqual([250, 50]);
|
||||
expect(batches.every((batch) => batch.dropped === 51)).toBe(true);
|
||||
const root = spans.find((span) => span.name === "sandbox.run")!;
|
||||
expect(root.attributes["paperclip.sandbox.recordCount"]).toBe(300);
|
||||
expect(root.attributes["paperclip.sandbox.dropped"]).toBe(51);
|
||||
expect(spans).toHaveLength(351);
|
||||
});
|
||||
|
||||
it("keeps remote timing relative instead of inventing a host timestamp", async () => {
|
||||
const { tracing, spans } = recordingContext();
|
||||
const records: SandboxPerformanceRecord[] = [];
|
||||
await runWithSandboxPerformanceTrace({ runId: "run", enabled: true, traceContext: tracing,
|
||||
onBatch: async (batch) => { records.push(...batch.records); } }, async () => {
|
||||
await measureSandboxOperation("sandbox.command", {}, async (span) => {
|
||||
span.recordRemotePhase("sandbox.remote.hash", 7, 11, { bytes: 20 });
|
||||
span.recordRemotePhase("sandbox.invalid", -1, 5);
|
||||
});
|
||||
});
|
||||
const remote = records.find((record) => record.clock === "remote_relative")!;
|
||||
expect(remote).toMatchObject({ startedAtMs: 7, durationMs: 11, attributes: { clock: "remote_relative", bytes: 20 } });
|
||||
expect(remote.parentId).toBe(records.find((record) => record.name === "sandbox.command")!.id);
|
||||
expect(spans.find((span) => span.name === "sandbox.command")!.events).toHaveLength(1);
|
||||
expect(spans.some((span) => span.name === "sandbox.remote.hash")).toBe(false);
|
||||
});
|
||||
|
||||
it("reparents lazy work to an open ancestor and ignores callbacks after trace closure", async () => {
|
||||
const { tracing, spans } = recordingContext();
|
||||
let captured: ReturnType<typeof captureSandboxPerformanceContext> = (work) => work();
|
||||
await runWithSandboxPerformanceTrace({ runId: "run", enabled: true, traceContext: tracing }, async () => {
|
||||
await measureSandboxOperation("sandbox.response", {}, async () => { captured = captureSandboxPerformanceContext(); });
|
||||
await captured(() => measureSandboxOperation("sandbox.body", {}, async () => undefined));
|
||||
});
|
||||
const root = spans.find((span) => span.name === "sandbox.run")!;
|
||||
expect(spans.find((span) => span.name === "sandbox.body")?.parentId).toBe(root.id);
|
||||
await captured(() => measureSandboxOperation("sandbox.too_late", {}, async () => { expect(hasSandboxPerformanceTrace()).toBe(false); }));
|
||||
expect(spans).toHaveLength(3);
|
||||
});
|
||||
|
||||
it("preserves disabled execution and fails open when tracing or persistence fails", async () => {
|
||||
const tracer = vi.fn(() => { throw new Error("sink failure"); });
|
||||
const onBatch = vi.fn(async () => { throw new Error("database failure"); });
|
||||
const tracing: StartupTraceContextHandle = { tracer: { startSpan: tracer }, contextWithSpan: () => undefined };
|
||||
await expect(runWithSandboxPerformanceTrace({ runId: "run", enabled: false, traceContext: tracing, onBatch }, async () => {
|
||||
expect(hasSandboxPerformanceTrace()).toBe(false);
|
||||
return measureSandboxOperation("sandbox.noop", {}, async () => 42);
|
||||
})).resolves.toBe(42);
|
||||
expect(tracer).not.toHaveBeenCalled();
|
||||
expect(onBatch).not.toHaveBeenCalled();
|
||||
await expect(runWithSandboxPerformanceTrace({ runId: "run", enabled: true, traceContext: tracing, onBatch }, async () => 42)).resolves.toBe(42);
|
||||
expect(onBatch).toHaveBeenCalledOnce();
|
||||
});
|
||||
});
|
||||
|
|
@ -1,3 +1,4 @@
|
|||
import { runWithSandboxPerformanceTrace, type SandboxPerformanceRecord } from "../services/sandbox-performance.js";
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { execFile } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
|
|
@ -161,6 +162,63 @@ describe("shared sandbox work-folder lifecycle", () => {
|
|||
runner: { supportsSingleStreamStdinProgress: options.bulkStdin, execute: (input) => localTestWorkFolderRunner.execute({ ...input, env: { ...input.env, HOME: home } }) } } });
|
||||
active.push(run); return run;
|
||||
}
|
||||
it("records scoped startup and final checkpoint stages without private identities", async () => {
|
||||
const task = randomUUID();
|
||||
await db.insert(issues).values({ id: task, companyId, projectId, title: "Instrumented task", assigneeAgentId: agentId });
|
||||
const home = path.join(root, "private-instrumented-home");
|
||||
const physicalId = randomUUID();
|
||||
const records: SandboxPerformanceRecord[] = [];
|
||||
const intervals = vi.spyOn(globalThis, "setInterval");
|
||||
try { await runWithSandboxPerformanceTrace({ runId: randomUUID(), enabled: true,
|
||||
onBatch: async (batch) => { records.push(...batch.records); } }, async () => {
|
||||
const run = await prepare(home, randomUUID(), physicalId, null, { taskId: task });
|
||||
await fs.writeFile(path.join(home, "task", "private-file-name"), "private-file-content");
|
||||
const tick = intervals.mock.calls.find((call) => call[1] === 180_000)?.[0];
|
||||
expect(typeof tick).toBe("function");
|
||||
if (typeof tick !== "function") throw new Error("Missing periodic checkpoint callback");
|
||||
tick();
|
||||
await run.flush();
|
||||
await run.stop();
|
||||
}); } finally { intervals.mockRestore(); }
|
||||
const prepared = records.find((record) => record.name === "work_folder.prepare")!;
|
||||
expect(prepared.outcome).toBe("ok");
|
||||
expect(prepared.attributes).toMatchObject({ cold: true, warm: false, reused: false });
|
||||
for (const repo of records.filter((record) => record.name === "work_folder.repository.prepare")) {
|
||||
expect(repo.attributes).toMatchObject({ exists: false, reused: false, cold: true, warm: false, cacheHit: false });
|
||||
}
|
||||
const periodic = records.find((record) => record.name === "work_folder.checkpoint" && record.attributes.phase === "periodic")!;
|
||||
expect(periodic.parentId).toBe(prepared.parentId);
|
||||
expect(periodic.outcome).toBe("ok");
|
||||
expect(records.filter((record) => record.name === "work_folder.scope.incoming").map((record) => record.attributes.scope).sort()).toEqual(["agent", "project", "task", "user"]);
|
||||
expect(records.filter((record) => record.name === "work_folder.repository.clone")).toHaveLength(2);
|
||||
expect(records.filter((record) => record.name === "work_folder.repository.setup")).toHaveLength(2);
|
||||
const finalized = records.find((record) => record.name === "work_folder.finalize")!;
|
||||
const finalCheckpoint = records.find((record) => record.name === "work_folder.checkpoint" && record.attributes.phase === "final")!;
|
||||
expect(finalCheckpoint.parentId).toBe(finalized.id);
|
||||
expect(finalCheckpoint.outcome).toBe("ok");
|
||||
expect(records.some((record) => record.name === "work_folder.db.query" && record.attributes.requestCount === 1)).toBe(true);
|
||||
expect(records.some((record) => record.name === "work_folder.progress.save")).toBe(true);
|
||||
for (const secret of [task, companyId, agentId, home, "private-file-name", "private-file-content"]) expect(JSON.stringify(records)).not.toContain(secret);
|
||||
// A new lease of the same physical sandbox is warm regardless of provider
|
||||
// power state. A fresh physical sandbox restores saved repositories cold.
|
||||
for (const samePhysicalSandbox of [true, false]) {
|
||||
const observed: SandboxPerformanceRecord[] = [];
|
||||
await runWithSandboxPerformanceTrace({ runId: randomUUID(), enabled: true,
|
||||
onBatch: async (batch) => { observed.push(...batch.records); } }, async () => {
|
||||
const resumed = await prepare(samePhysicalSandbox ? home : path.join(root, "instrumented-replacement"),
|
||||
randomUUID(), samePhysicalSandbox ? physicalId : randomUUID(), null, { taskId: task });
|
||||
await resumed.stop();
|
||||
});
|
||||
expect(observed.find((record) => record.name === "work_folder.prepare")!.attributes)
|
||||
.toMatchObject({ cold: !samePhysicalSandbox, warm: samePhysicalSandbox, reused: samePhysicalSandbox });
|
||||
const repositories = observed.filter((record) => record.name === "work_folder.repository.prepare");
|
||||
expect(repositories).toHaveLength(2);
|
||||
for (const repo of repositories) expect(repo.attributes).toMatchObject({ exists: samePhysicalSandbox,
|
||||
reused: samePhysicalSandbox, cold: !samePhysicalSandbox, warm: samePhysicalSandbox, cacheHit: !samePhysicalSandbox });
|
||||
expect(observed.filter((record) => record.name === "work_folder.repository.clone")).toHaveLength(0);
|
||||
}
|
||||
|
||||
}, 60_000);
|
||||
it("uses downloaded metadata when a shared file changes after the startup listing", async () => {
|
||||
const svc = workFolderService(db, storage);
|
||||
const folder = await svc.ensure({ companyId, scope: "project", ownerId: projectId });
|
||||
|
|
|
|||
|
|
@ -0,0 +1,68 @@
|
|||
import { createHash } from "node:crypto";
|
||||
import { Readable } from "node:stream";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { prefetchWorkFiles } from "../services/work-folder-transfer.js";
|
||||
import { uploadWorkFolderObject } from "../services/work-folder-upload.js";
|
||||
import { measureSandboxOperation, measureSandboxStream, runWithSandboxPerformanceTrace, type SandboxPerformanceRecord } from "../services/sandbox-performance.js";
|
||||
|
||||
const entry = { path: "private-file-secret", kind: "file" as const, byteSize: 4, sha256: null, executable: false };
|
||||
async function traced(work: () => Promise<void>) {
|
||||
const records: SandboxPerformanceRecord[] = [];
|
||||
await runWithSandboxPerformanceTrace({ runId: "private-run-secret", enabled: true, onBatch: async (batch) => { records.push(...batch.records); } }, work);
|
||||
return records;
|
||||
}
|
||||
describe("work-folder performance instrumentation", () => {
|
||||
it("separates ordered prefetch response waits from body consumption with safe attributes", async () => {
|
||||
const records = await traced(async () => measureSandboxOperation("work_folder.scope.hydrate", { scope: "task" }, async () => {
|
||||
for await (const transfer of prefetchWorkFiles([0, 1], async (fileIndex) => ({
|
||||
entry, body: measureSandboxStream("work_folder.object.body", { fileIndex }, Readable.from([Buffer.from("data")])),
|
||||
}))) {
|
||||
let body = ""; for await (const chunk of transfer.body!) body += String(chunk);
|
||||
expect(body).toBe("data");
|
||||
}
|
||||
}));
|
||||
const parent = records.find((record) => record.name === "work_folder.scope.hydrate")!;
|
||||
expect(records.filter((record) => record.name === "work_folder.prefetch.open")).toHaveLength(2);
|
||||
expect(records.filter((record) => record.name === "work_folder.prefetch.wait")).toHaveLength(2);
|
||||
const bodies = records.filter((record) => record.name === "work_folder.object.body");
|
||||
expect(bodies).toHaveLength(2);
|
||||
expect(bodies.every((record) => record.attributes.bytes === 4 && record.parentId === parent.id)).toBe(true);
|
||||
expect(JSON.stringify(records)).not.toContain("private-file-secret");
|
||||
expect(JSON.stringify(records)).not.toContain("private-run-secret");
|
||||
});
|
||||
it("records real PUT attempts and verified consumption without object keys or contents", async () => {
|
||||
const bytes = Buffer.from("private-body-secret"); let attempts = 0;
|
||||
const records = await traced(async () => {
|
||||
await uploadWorkFolderObject({ async putObject(input) {
|
||||
attempts++; for await (const _chunk of input.body) { /* actual consumption */ }
|
||||
if (attempts === 1) throw Object.assign(new Error("transient"), { code: "ECONNRESET" });
|
||||
return { objectKey: input.objectKey, contentLength: bytes.length } as never;
|
||||
} }, { objectKey: "private-object-secret", contentType: "text/plain", contentLength: bytes.length,
|
||||
sha256: createHash("sha256").update(bytes).digest("hex"), createSource: () => Readable.from([bytes]) });
|
||||
});
|
||||
const puts = records.filter((record) => record.name === "work_folder.object.put");
|
||||
expect(puts.map((record) => [record.attributes.attempt, record.outcome])).toEqual([[1, "failed"], [2, "ok"]]);
|
||||
expect(records.filter((record) => record.name === "work_folder.upload.consume").every((record) => record.attributes.bytes === bytes.length)).toBe(true);
|
||||
expect(JSON.stringify(records)).not.toContain("private-object-secret");
|
||||
expect(JSON.stringify(records)).not.toContain("private-body-secret");
|
||||
});
|
||||
it("preserves queued response errors and closes unconsumed prefetched sources with tracing enabled", async () => {
|
||||
const source = new Readable({ read() {} });
|
||||
await expect(traced(async () => {
|
||||
for await (const transfer of prefetchWorkFiles([0, 1], async (index) => ({ entry,
|
||||
body: measureSandboxStream("work_folder.object.body", { fileIndex: index }, index === 0 ? Readable.from([]) : source),
|
||||
}))) {
|
||||
if (transfer.entry === entry && !source.destroyed) {
|
||||
source.destroy(new Error("queued response failed")); await new Promise((resolve) => setImmediate(resolve));
|
||||
}
|
||||
for await (const _chunk of transfer.body!) { /* consume */ }
|
||||
}
|
||||
})).rejects.toThrow("queued response failed");
|
||||
expect(source.destroyed).toBe(true);
|
||||
const abandoned = new Readable({ read() {} });
|
||||
await traced(async () => {
|
||||
for await (const _transfer of prefetchWorkFiles([0], async () => ({ entry, body: measureSandboxStream("work_folder.object.body", {}, abandoned) }))) break;
|
||||
});
|
||||
expect(abandoned.destroyed).toBe(true);
|
||||
});
|
||||
});
|
||||
|
|
@ -1,3 +1,11 @@
|
|||
import { runWithSandboxPerformanceTrace, type SandboxPerformanceRecord } from "../services/sandbox-performance.js";
|
||||
import { execFile } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
import { mkdtemp, mkdir, readFile, realpath, rm, writeFile } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { localTestWorkFolderRunner } from "./helpers/work-folder-runner.js";
|
||||
import { workFolderTransport } from "../services/work-folder-transport.js";
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { Readable } from "node:stream";
|
||||
import { setImmediate } from "node:timers/promises";
|
||||
|
|
@ -81,6 +89,57 @@ describe("bounded repository checkpoint transfers", () => {
|
|||
};
|
||||
}
|
||||
|
||||
it("checkpoints Git ignore policy without uploading or removing local ignored dependencies", async () => {
|
||||
const f = await fixture();
|
||||
const base = await realpath(await mkdtemp(path.join(os.tmpdir(), "repository-ignore-")));
|
||||
const root = path.join(base, "source"), restored = path.join(base, "restored"), staging = path.join(base, "staging");
|
||||
const git = promisify(execFile);
|
||||
try {
|
||||
await git("git", ["init", root]);
|
||||
await writeFile(path.join(root, "tracked"), "tracked original");
|
||||
await git("git", ["-C", root, "add", "tracked"]);
|
||||
await git("git", ["-C", root, "-c", "user.name=Test", "-c", "user.email=test@example.com", "commit", "-m", "initial"]);
|
||||
await writeFile(path.join(root, ".gitignore"), "node_modules/\ntracked\n");
|
||||
await mkdir(path.join(root, "nested/cache"), { recursive: true });
|
||||
await mkdir(path.join(root, "node_modules"));
|
||||
await writeFile(path.join(root, "nested/.gitignore"), "cache/\n");
|
||||
await writeFile(path.join(root, ".git/info/exclude"), "local-only\n");
|
||||
const ignored = ["node_modules/dependency", "nested/cache/dependency", "local-only"];
|
||||
for (const file of ignored) await writeFile(path.join(root, file), `ignored:${file}`);
|
||||
await writeFile(path.join(root, "tracked"), "tracked changed");
|
||||
await writeFile(path.join(root, "nested/keep"), "untracked work");
|
||||
const actual = workFolderTransport({ ...localTestWorkFolderRunner, supportsSingleStreamStdinProgress: true });
|
||||
const service = workFolderRepositoryService(db, f.storage, actual);
|
||||
const records: SandboxPerformanceRecord[] = [];
|
||||
const trace = { runId: "repository-ignore", enabled: true, onBatch: async (batch: { records: SandboxPerformanceRecord[] }) => { records.push(...batch.records); } };
|
||||
await runWithSandboxPerformanceTrace(trace, () => service.checkpoint(f.binding, root));
|
||||
const current = await f.current();
|
||||
const manifest = JSON.parse(f.objects.get(current.checkpointKey!)!.toString());
|
||||
const durablePaths = manifest.files.map((entry: { path: string }) => entry.path);
|
||||
expect(durablePaths).toContain("tracked");
|
||||
expect(durablePaths).toContain("nested/keep");
|
||||
expect(durablePaths).toContain(".git/info/exclude");
|
||||
for (const file of ignored) {
|
||||
expect(durablePaths).not.toContain(file);
|
||||
expect(await readFile(path.join(root, file), "utf8")).toBe(`ignored:${file}`);
|
||||
expect([...f.objects.values()].some((body) => body.toString() === `ignored:${file}`)).toBe(false);
|
||||
}
|
||||
await mkdir(staging);
|
||||
await runWithSandboxPerformanceTrace(trace, () => service.restore(current, restored, staging));
|
||||
for (const phase of ["lookup", "object_intent", "object_head", "object_upload", "manifest_upload", "publish_lock", "protect_objects", "publish_pointer", "manifest_download", "manifest_body", "manifest_decode", "object_download"]) {
|
||||
expect(records.some((record) => record.name === `work_folder.repository.${phase}`)).toBe(true);
|
||||
}
|
||||
for (const secret of [root, restored, f.binding.id, companyId, "tracked changed"]) expect(JSON.stringify(records)).not.toContain(secret);
|
||||
expect(await readFile(path.join(restored, "tracked"), "utf8")).toBe("tracked changed");
|
||||
expect(await readFile(path.join(restored, "nested/keep"), "utf8")).toBe("untracked work");
|
||||
for (const file of ignored) await expect(readFile(path.join(restored, file))).rejects.toMatchObject({ code: "ENOENT" });
|
||||
// Restoring into the existing workspace must preserve reusable ignored caches.
|
||||
await service.restore(current, root, staging);
|
||||
for (const file of ignored) expect(await readFile(path.join(root, file), "utf8")).toBe(`ignored:${file}`);
|
||||
const listed = await git("git", ["-C", restored, "ls-files", "--cached", "--others", "--exclude-standard"]);
|
||||
expect(listed.stdout).toContain("tracked");
|
||||
} finally { await rm(base, { recursive: true, force: true }); }
|
||||
}, 60_000);
|
||||
it("bounds HEADs and streaming PUTs to four, deduplicates content, and preserves manifest order", async () => {
|
||||
const f = await fixture();
|
||||
const entries = [f.file("z"), f.file("b"), f.file("same-z", "z"),
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import { Readable } from "node:stream";
|
|||
import { createHash } from "node:crypto";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { runWithSandboxPerformanceTrace, type SandboxPerformanceRecord } from "../services/sandbox-performance.js";
|
||||
import { workFolderTransport } from "../services/work-folder-transport.js";
|
||||
import { localTestWorkFolderRunner } from "./helpers/work-folder-runner.js";
|
||||
|
||||
|
|
@ -18,6 +19,70 @@ describe("sandbox work folder transport with real Node and Git", () => {
|
|||
roots.push(dir); return dir;
|
||||
}
|
||||
afterEach(async () => { for (const dir of roots.splice(0)) await rm(dir, { recursive: true, force: true }); });
|
||||
it("records host and remote timing without changing bytes or exporting paths", async () => {
|
||||
const dir = await root(), staging = await root();
|
||||
const payload = Buffer.from("private-file-content");
|
||||
const records: SandboxPerformanceRecord[] = [];
|
||||
const fast = workFolderTransport({ ...localTestWorkFolderRunner, supportsSingleStreamStdinProgress: true });
|
||||
await runWithSandboxPerformanceTrace({ runId: "test-run", enabled: true,
|
||||
onBatch: async (batch) => { records.push(...batch.records); } }, async () => {
|
||||
await fast.write(dir, staging, { path: "sensitive-file-name", kind: "file", byteSize: payload.length,
|
||||
sha256: createHash("sha256").update(payload).digest("hex"), executable: false }, Readable.from([payload]));
|
||||
const files = await fast.scan(dir);
|
||||
expect(await fast.readBatch!(dir, files.filter((entry) => entry.kind === "file"))).toEqual([payload]);
|
||||
});
|
||||
const roundtrip = records.find((record) => record.name === "work_folder.transport.roundtrip" && record.attributes.operation === "batch");
|
||||
expect(roundtrip?.attributes.executionMs).toBeGreaterThanOrEqual(0);
|
||||
expect(roundtrip?.attributes.transportOverheadMs).toBeGreaterThanOrEqual(0);
|
||||
expect(records.some((record) => record.name === "work_folder.transport.incoming_body")).toBe(true);
|
||||
expect(records.some((record) => record.name === "work_folder.transport.encode_bytes")).toBe(true);
|
||||
const remote = records.filter((record) => record.clock === "remote_relative");
|
||||
expect(remote.length).toBeGreaterThan(0);
|
||||
expect(remote.some((record) => record.name === "work_folder.remote.hash")).toBe(true);
|
||||
expect(remote.some((record) => record.name === "work_folder.remote.scan")).toBe(true);
|
||||
expect(remote.some((record) => record.name === "sandbox.invalid_operation")).toBe(false);
|
||||
expect(remote.every((record) => records.some((parent) => parent.id === record.parentId && parent.name === "work_folder.transport.roundtrip"))).toBe(true);
|
||||
for (const secret of [dir, staging, "sensitive-file-name", payload.toString()]) expect(JSON.stringify(records)).not.toContain(secret);
|
||||
expect(await readFile(path.join(dir, "sensitive-file-name"))).toEqual(payload);
|
||||
});
|
||||
it("keeps the helper's original raw result when performance is not requested", async () => {
|
||||
const script = await readFile(new URL("../services/scripts/work-folder-io.mjs", import.meta.url), "utf8");
|
||||
const result = await exec(process.execPath, ["--input-type=module", "-e", script,
|
||||
Buffer.from(JSON.stringify({ operation: "home" })).toString("base64")]);
|
||||
expect(JSON.parse(result.stdout)).toEqual({ home: os.homedir() });
|
||||
expect(result.stdout).not.toContain("workFolderPerformanceVersion");
|
||||
});
|
||||
it("does not discard a valid result or retry a command because diagnostics are malformed", async () => {
|
||||
const execute = vi.fn().mockResolvedValue({ exitCode: 0, timedOut: false, stderr: "", stdout: JSON.stringify({
|
||||
workFolderPerformanceVersion: 1, result: { home: "/home/daytona" }, performance: { executionMs: "invalid" },
|
||||
}) });
|
||||
const records: SandboxPerformanceRecord[] = [];
|
||||
await runWithSandboxPerformanceTrace({ runId: "run", enabled: true,
|
||||
onBatch: async (batch) => { records.push(...batch.records); } }, async () => {
|
||||
expect(await workFolderTransport({ execute }).home()).toBe("/home/daytona");
|
||||
});
|
||||
expect(execute).toHaveBeenCalledOnce();
|
||||
expect(records.find((record) => record.name === "work_folder.transport.roundtrip")?.attributes.dropped).toBe(1);
|
||||
execute.mockClear().mockResolvedValue({ exitCode: 1, timedOut: false, stderr: "socket hang up", stdout: "" });
|
||||
await expect(workFolderTransport({ execute }).home()).rejects.toThrow("socket hang up");
|
||||
expect(execute).toHaveBeenCalledOnce();
|
||||
});
|
||||
it("bounds remote phase records while retaining complete aggregate hash counts", async () => {
|
||||
const dir = await root();
|
||||
await Promise.all(Array.from({ length: 300 }, (_, index) => writeFile(path.join(dir, `file-${index}`), "x")));
|
||||
const script = await readFile(new URL("../services/scripts/work-folder-io.mjs", import.meta.url), "utf8");
|
||||
const response = await exec(process.execPath, ["--input-type=module", "-e", script,
|
||||
Buffer.from(JSON.stringify({ operation: "scan", root: dir, performance: true })).toString("base64")]);
|
||||
const decoded = JSON.parse(response.stdout);
|
||||
expect(decoded.result).toHaveLength(300);
|
||||
expect(decoded.remotePhases).toHaveLength(256);
|
||||
expect(decoded.performance.hashFiles).toBe(300);
|
||||
expect(decoded.performance.hashBytes).toBe(300);
|
||||
expect(decoded.performance.droppedPhases).toBeGreaterThan(0);
|
||||
expect(decoded.performance.executionMs).toBeGreaterThanOrEqual(decoded.performance.scanMs);
|
||||
expect(JSON.stringify(decoded.remotePhases)).not.toContain(dir);
|
||||
expect(JSON.stringify(decoded.remotePhases)).not.toContain("file-");
|
||||
});
|
||||
it("retries a lost read response without duplicating streamed bytes", async () => {
|
||||
const dir = await root();
|
||||
const body = Buffer.alloc(700_000, "x");
|
||||
|
|
@ -188,12 +253,28 @@ describe("sandbox work folder transport with real Node and Git", () => {
|
|||
await exec("git", ["-C", dir, "add", "."]);
|
||||
await exec("git", ["-C", dir, "-c", "user.name=Test", "-c", "user.email=test@example.com", "commit", "-m", "initial"]);
|
||||
await writeFile(path.join(dir, "tracked"), "unstaged");
|
||||
// A tracked path remains durable even when a later ignore rule matches it.
|
||||
await writeFile(path.join(dir, ".gitignore"), "node_modules/\ntracked\n");
|
||||
await mkdir(path.join(dir, "nested"));
|
||||
await writeFile(path.join(dir, "nested/.gitignore"), "cache/\n*.generated\n");
|
||||
await mkdir(path.join(dir, "nested/cache"));
|
||||
await writeFile(path.join(dir, "nested/cache/dependency"), "local cache");
|
||||
await writeFile(path.join(dir, "nested/result.generated"), "generated");
|
||||
await writeFile(path.join(dir, "nested/keep"), "new durable work");
|
||||
await writeFile(path.join(dir, ".git/info/exclude"), "local-only\n");
|
||||
await writeFile(path.join(dir, "local-only"), "local secret cache");
|
||||
await writeFile(path.join(dir, "untracked"), "new");
|
||||
await mkdir(path.join(dir, "node_modules"));
|
||||
await writeFile(path.join(dir, "node_modules/cache"), "ignored");
|
||||
const paths = (await transport.scan(dir, true)).map((entry) => entry.path);
|
||||
expect(paths).toContain("tracked");
|
||||
expect(paths).toContain("untracked");
|
||||
expect(paths).toContain("nested/keep");
|
||||
expect(paths).toContain("nested/.gitignore");
|
||||
expect(paths).toContain(".git/info/exclude");
|
||||
expect(paths).not.toContain("local-only");
|
||||
expect(paths).not.toContain("nested/result.generated");
|
||||
expect(paths.some((entry) => entry.startsWith("nested/cache/"))).toBe(false);
|
||||
expect(paths).toContain(".git/index");
|
||||
expect(paths).toContain(".git/HEAD");
|
||||
expect(paths).not.toContain(".git/config");
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import { runWithSandboxPerformanceTrace, type SandboxPerformanceRecord } from "../services/sandbox-performance.js";
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
|
|
@ -39,6 +40,29 @@ describe("durable work folders", () => {
|
|||
for await (const chunk of stream) buffers.push(Buffer.from(chunk));
|
||||
return Buffer.concat(buffers).toString();
|
||||
}
|
||||
it("measures metadata, response wait, body bytes and progress without private paths", async () => {
|
||||
const records: SandboxPerformanceRecord[] = [];
|
||||
await runWithSandboxPerformanceTrace({ runId: randomUUID(), enabled: true,
|
||||
onBatch: async (batch) => { records.push(...batch.records); } }, async () => {
|
||||
const f = await folder();
|
||||
await svc.write(f, { path: "private-observed-file", body: Buffer.from("private-observed-content"), operationId: "private-operation-id" });
|
||||
const opened = await svc.content(f, "private-observed-file", 7);
|
||||
let text = ""; for await (const chunk of opened.stream) text += String(chunk);
|
||||
expect(text).toBe("private-observed-content");
|
||||
await svc.list(f);
|
||||
});
|
||||
const names = records.map((record) => record.name);
|
||||
for (const name of ["work_folder.scope.ensure", "work_folder.metadata.get", "work_folder.object.get_response", "work_folder.object.body", "work_folder.spool.consume", "work_folder.metadata.mutate", "work_folder.db.query"]) expect(names).toContain(name);
|
||||
const response = records.find((record) => record.name === "work_folder.object.get_response")!;
|
||||
const body = records.find((record) => record.name === "work_folder.object.body")!;
|
||||
expect(response.attributes.requestCount).toBe(1);
|
||||
expect(body.attributes.bytes).toBe(Buffer.byteLength("private-observed-content"));
|
||||
expect(body.attributes.fileIndex).toBe(7);
|
||||
expect(response.attributes.fileIndex).toBe(7);
|
||||
expect(body.startedAtMs).toBeGreaterThanOrEqual(response.startedAtMs);
|
||||
expect(records.filter((record) => record.name === "work_folder.db.query").every((record) => typeof record.attributes.operation === "string")).toBe(true);
|
||||
for (const secret of [companyId, root, "private-observed-file", "private-observed-content", "private-operation-id"]) expect(JSON.stringify(records)).not.toContain(secret);
|
||||
});
|
||||
it("streams nested executable and empty files into durable storage", async () => {
|
||||
const f = await folder();
|
||||
await svc.write(f, { path: "bin/run", body: Readable.from(["#!/bin/sh\n", "true\n"]), executable: true, operationId: "first" });
|
||||
|
|
|
|||
|
|
@ -126,6 +126,7 @@ export function getStartupTracer(name = "paperclip.startup"): StartupTracerHandl
|
|||
export interface StartupTraceContextHandle {
|
||||
readonly tracer: StartupTracerHandle;
|
||||
contextWithSpan(span: unknown): unknown;
|
||||
withContext?<T>(context: unknown, work: () => T): T;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -158,7 +159,7 @@ export function getStartupTraceContext(name = "paperclip.startup"): StartupTrace
|
|||
getTracer(n: string): StartupTracerHandle;
|
||||
setSpan(context: unknown, span: unknown): unknown;
|
||||
};
|
||||
context?: { active(): unknown };
|
||||
context?: { active(): unknown; with?<T>(context: unknown, work: () => T): T };
|
||||
};
|
||||
const trace = api.trace;
|
||||
const context = api.context;
|
||||
|
|
@ -171,6 +172,7 @@ export function getStartupTraceContext(name = "paperclip.startup"): StartupTrace
|
|||
// Keep the method calls on `trace` / `context` so the api singletons stay
|
||||
// their own receiver.
|
||||
contextWithSpan: (span: unknown) => trace.setSpan(context.active(), span),
|
||||
withContext: <T>(token: unknown, work: () => T): T => context.with ? context.with(token, work) : work(),
|
||||
};
|
||||
} catch (err) {
|
||||
if (!traceContextApiLoadFailed) {
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -62,6 +62,30 @@ function createRecordingTraceContext(): {
|
|||
}
|
||||
|
||||
describe("native runner performance trace", () => {
|
||||
it("parents to live host dispatch and activates its context for instrumented clients", async () => {
|
||||
const { traceContext, spans } = createRecordingTraceContext();
|
||||
let activeContext: unknown;
|
||||
traceContext.withContext = (context, work) => {
|
||||
const previous = activeContext;
|
||||
activeContext = context;
|
||||
try { return work(); } finally { activeContext = previous; }
|
||||
};
|
||||
const host = traceContext.tracer.startSpan("heartbeat.native_dispatch");
|
||||
const trace = createNativeRunTrace({ runId: "run", traceContext,
|
||||
parentContext: traceContext.contextWithSpan(host) });
|
||||
const startup = trace.start("runner.session.startup");
|
||||
trace.run(startup, () => {
|
||||
const context = activeContext as { span?: RecordedSpan };
|
||||
expect(context.span?.name).toBe("runner.session.startup");
|
||||
traceContext.tracer.startSpan("http.request", {}, activeContext).end();
|
||||
});
|
||||
await trace.end(startup);
|
||||
await trace.finish("ok");
|
||||
expect(spans.find((span) => span.name === "task.run")?.parentName).toBe("heartbeat.native_dispatch");
|
||||
expect(spans.find((span) => span.name === "http.request")?.parentName).toBe("runner.session.startup");
|
||||
expect(activeContext).toBeUndefined();
|
||||
});
|
||||
|
||||
it("persists measured spans with bounded run-relative timing", async () => {
|
||||
const events: AdapterRuntimeEvent[] = [];
|
||||
const trace = createNativeRunTrace({
|
||||
|
|
|
|||
|
|
@ -103,6 +103,8 @@ export function createNativeRunTrace(input: {
|
|||
startedAtMs?: number;
|
||||
onEvent?: NativeRunTraceSink;
|
||||
traceContext?: StartupTraceContextHandle;
|
||||
/** The live host dispatch span when full sandbox diagnostics are enabled. */
|
||||
parentContext?: unknown;
|
||||
}) {
|
||||
const tracing =
|
||||
input.traceContext ?? getStartupTraceContext("paperclip.native-runner");
|
||||
|
|
@ -118,7 +120,7 @@ export function createNativeRunTrace(input: {
|
|||
"paperclip.task.run.trace_schema_version":
|
||||
NATIVE_RUN_TRACE_SCHEMA_VERSION,
|
||||
},
|
||||
});
|
||||
}, input.parentContext);
|
||||
} catch {
|
||||
rootSpan = NOOP_SPAN;
|
||||
}
|
||||
|
|
@ -350,9 +352,10 @@ export function createNativeRunTrace(input: {
|
|||
},
|
||||
},
|
||||
);
|
||||
return scopeStorage.run(state, () =>
|
||||
const work = () => scopeStorage.run(state, () =>
|
||||
runWithRuntimeParent(dynamicContext, fn),
|
||||
);
|
||||
return tracing.withContext ? tracing.withContext(dynamicContext, work) : work();
|
||||
};
|
||||
|
||||
const record = async (span: NativeRunHistoricalSpan): Promise<void> => {
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { hasSandboxPerformanceTrace } from "../sandbox-performance.js";
|
||||
import { getActiveStepContext } from "@paperclipai/adapter-utils/acpx-engine/startup-timing";
|
||||
import {
|
||||
chmodSync,
|
||||
closeSync,
|
||||
|
|
@ -3759,7 +3761,11 @@ async function executePaperclipNativeSessionWithinScope(
|
|||
"paperclip_runner_provider_unsupported: ACPX Pi requires the native runner's descriptor-confined verified launch",
|
||||
);
|
||||
}
|
||||
const earliestPreparationStart = input.preparationSpans?.reduce(
|
||||
// Full host diagnostics already measured preparation in its real scopes.
|
||||
// Do not backdate a second aggregate over those operations or detach this
|
||||
// native subtree from the live dispatch parent.
|
||||
const hostMeasured = hasSandboxPerformanceTrace();
|
||||
const earliestPreparationStart = hostMeasured ? Date.now() : input.preparationSpans?.reduce(
|
||||
(earliest, span) => Math.min(earliest, span.startedAtMs),
|
||||
Date.now(),
|
||||
);
|
||||
|
|
@ -3767,8 +3773,9 @@ async function executePaperclipNativeSessionWithinScope(
|
|||
runId: input.execution.binding.runId,
|
||||
startedAtMs: earliestPreparationStart,
|
||||
onEvent: input.onEvent,
|
||||
parentContext: hostMeasured ? getActiveStepContext()?.parentContext : undefined,
|
||||
});
|
||||
const preparationSpans = input.preparationSpans ?? [];
|
||||
const preparationSpans = hostMeasured ? [] : input.preparationSpans ?? [];
|
||||
const taskPrepareScope = trace.start("task.prepare", {
|
||||
parentName: "task.run",
|
||||
startedAtMs: earliestPreparationStart,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,216 @@
|
|||
import { AsyncLocalStorage } from "node:async_hooks";
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
import { Readable } from "node:stream";
|
||||
import { performance } from "node:perf_hooks";
|
||||
import { getActiveStepContext, runWithRuntimeParent } from "@paperclipai/adapter-utils/acpx-engine/startup-timing";
|
||||
import { getStartupTraceContext, traceparentFromContextToken, type StartupTraceContextHandle } from "../instrumentation.js";
|
||||
|
||||
type Attributes = Record<string, string | number | boolean>;
|
||||
type Outcome = "ok" | "failed" | "cancelled";
|
||||
export interface SandboxPerformanceRecord {
|
||||
name: string; id: string; parentId?: string; traceId?: string;
|
||||
startedAtMs: number; durationMs: number; outcome: Outcome; attributes: Attributes;
|
||||
clock?: "remote_relative";
|
||||
}
|
||||
interface TraceState {
|
||||
tracing: StartupTraceContextHandle; records: SandboxPerformanceRecord[];
|
||||
limit: number; dropped: number; runHash: string; startedAtMs: number; closed?: boolean;
|
||||
root?: SandboxOperation;
|
||||
}
|
||||
interface ScopeState { trace: TraceState; id: string; context: unknown; parent?: ScopeState; ended?: boolean; }
|
||||
const scopes = new AsyncLocalStorage<ScopeState | undefined>();
|
||||
function openScope(scope: ScopeState | undefined) {
|
||||
while (scope?.ended) scope = scope.parent;
|
||||
return scope;
|
||||
}
|
||||
const numericKeys = new Set([
|
||||
"bytes", "files", "requestCount", "attempt", "retries", "parallelism", "repositoryIndex", "fileIndex", "scopeIndex", "chunkIndex",
|
||||
"objects", "entries", "batchIndex", "batches", "operations", "completed", "offset", "limit", "queueDepth", "dropped",
|
||||
"executionMs", "transportOverheadMs", "durationMs", "startOffsetMs", "endOffsetMs", "waitMs", "gitListMs", "hashMs", "readMs", "writeMs", "publishMs", "decodeMs", "encodeMs",
|
||||
"inputBytes", "outputBytes", "roundtripMs", "hashFiles", "hashBytes", "droppedPhases", "scanMs", "listMs", "recordCount", "operationIndex", "readWaitMs", "bodyNonReadMs", "chunks",
|
||||
]);
|
||||
const booleanKeys = new Set(["cold", "warm", "reused", "cacheHit", "exists", "changed", "repository", "bound", "tracked", "ignored", "cancelled", "final", "enabled", "repeated"]);
|
||||
// String attributes describe operations, never user input. Names and string
|
||||
// values must also pass a token grammar; callers must use fixed literals.
|
||||
const stringKeys = new Set(["scope", "phase", "operation", "runtime", "outcome", "mode", "source", "trigger", "clock", "strategy"]);
|
||||
function safe(attributes: Attributes): Attributes {
|
||||
const result: Attributes = {};
|
||||
for (const [key, value] of Object.entries(attributes)) {
|
||||
if (numericKeys.has(key) && typeof value === "number" && Number.isFinite(value) && value >= 0) result[key] = value;
|
||||
else if (booleanKeys.has(key) && typeof value === "boolean") result[key] = value;
|
||||
else if (stringKeys.has(key) && typeof value === "string" && /^[a-z][a-z0-9_-]{0,47}$/.test(value)) result[key] = value;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
function safeName(name: string) { return /^[a-z][a-z0-9_.]{0,95}$/.test(name) ? name : "sandbox.invalid_operation"; }
|
||||
function identity(context: unknown) {
|
||||
const value = traceparentFromContextToken(context)?.split("-");
|
||||
return value && !/^0+$/.test(value[1]!) && !/^0+$/.test(value[2]!) ? { traceId: value[1], spanId: value[2] } : undefined;
|
||||
}
|
||||
function record(trace: TraceState, entry: SandboxPerformanceRecord) {
|
||||
if (trace.closed) return;
|
||||
if (trace.records.length < trace.limit) trace.records.push(entry);
|
||||
else trace.dropped++;
|
||||
}
|
||||
export interface SandboxOperation {
|
||||
set(attributes: Attributes): void;
|
||||
end(outcome?: Outcome): void;
|
||||
run<T>(work: () => T): T;
|
||||
recordRemotePhase(name: string, startOffsetMs: number, durationMs: number, attributes?: Attributes): void;
|
||||
}
|
||||
const noop: SandboxOperation = { set() {}, end() {}, run: (work) => work(), recordRemotePhase() {} };
|
||||
|
||||
/** Capture at stream creation, not when an unrelated consumer later reads it. */
|
||||
export function captureSandboxPerformanceContext(): <T>(work: () => T) => T {
|
||||
const scope = scopes.getStore();
|
||||
const parent = getActiveStepContext()?.parentContext ?? scope?.context;
|
||||
return (work) => {
|
||||
const current = openScope(scope);
|
||||
const token = current === scope ? parent : current?.context;
|
||||
const within = () => scopes.run(current, () => runWithRuntimeParent(token, work));
|
||||
return current?.trace.tracing.withContext ? current.trace.tracing.withContext(token, within) : within();
|
||||
};
|
||||
}
|
||||
export function hasSandboxPerformanceTrace() { return Boolean(openScope(scopes.getStore()) && !scopes.getStore()?.trace.closed); }
|
||||
export function setSandboxPerformanceRunAttributes(attributes: Attributes): void {
|
||||
scopes.getStore()?.trace.root?.set(attributes);
|
||||
}
|
||||
|
||||
export function startSandboxOperation(name: string, attributes: Attributes = {}): SandboxOperation {
|
||||
const current = openScope(scopes.getStore());
|
||||
if (!current || current.trace.closed) return noop;
|
||||
const trace = current.trace;
|
||||
const parentContext = getActiveStepContext()?.parentContext ?? current.context;
|
||||
const parentId = identity(parentContext)?.spanId ?? current.id;
|
||||
const startedAtMs = Date.now(), started = performance.now();
|
||||
const values = safe(attributes);
|
||||
let span: ReturnType<StartupTraceContextHandle["tracer"]["startSpan"]> | undefined;
|
||||
let context = parentContext;
|
||||
name = safeName(name);
|
||||
try {
|
||||
span = trace.tracing.tracer.startSpan(name, { startTime: startedAtMs, attributes: { "paperclip.sandbox.run_hash": trace.runHash } }, parentContext);
|
||||
context = trace.tracing.contextWithSpan(span);
|
||||
} catch { /* Diagnostics must never change task behavior. */ }
|
||||
const ids = span ? identity(context) : undefined, id = ids?.spanId ?? randomBytes(8).toString("hex");
|
||||
const child: ScopeState = { trace, id, context, parent: current };
|
||||
let ended = false;
|
||||
return {
|
||||
set(next) { if (!ended) Object.assign(values, safe(next)); },
|
||||
run(work) {
|
||||
const within = () => scopes.run(child, () => runWithRuntimeParent(context, work));
|
||||
return trace.tracing.withContext ? trace.tracing.withContext(context, within) : within();
|
||||
},
|
||||
recordRemotePhase(remoteName, startOffsetMs, durationMs, remoteAttributes = {}) {
|
||||
if (ended || !Number.isFinite(startOffsetMs) || startOffsetMs < 0 || !Number.isFinite(durationMs) || durationMs < 0) return;
|
||||
const attrs = safe({ ...remoteAttributes, clock: "remote_relative", startOffsetMs, durationMs });
|
||||
// Remote offsets have no measured host-clock alignment. Preserve them as
|
||||
// events on the real command span; never fabricate host timestamps.
|
||||
try { (span as typeof span & { addEvent?: (n: string, a: Attributes) => void })?.addEvent?.(safeName(remoteName), attrs); } catch { /* fail open */ }
|
||||
record(trace, { name: safeName(remoteName), id: randomBytes(8).toString("hex"), parentId: id, traceId: ids?.traceId,
|
||||
startedAtMs: startOffsetMs, durationMs, outcome: "ok", attributes: attrs, clock: "remote_relative" });
|
||||
},
|
||||
end(outcome = "ok") {
|
||||
if (ended) return;
|
||||
ended = true;
|
||||
child.ended = true;
|
||||
const durationMs = Math.max(0, performance.now() - started);
|
||||
try {
|
||||
for (const [key, value] of Object.entries(values)) span?.setAttribute(`paperclip.sandbox.${key}`, value);
|
||||
span?.setAttribute("paperclip.sandbox.outcome", outcome);
|
||||
span?.setAttribute("paperclip.sandbox.duration_ms", durationMs);
|
||||
if (outcome !== "ok") span?.setStatus({ code: 2 }); // Never export the exception message.
|
||||
span?.end(startedAtMs + durationMs);
|
||||
} catch { /* fail open */ }
|
||||
record(trace, { name, id, parentId, traceId: ids?.traceId, startedAtMs, durationMs, outcome, attributes: values });
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export async function measureSandboxOperation<T>(name: string, attributes: Attributes, work: (span: SandboxOperation) => Promise<T>): Promise<T> {
|
||||
if (!scopes.getStore()) return work(noop);
|
||||
const span = startSandboxOperation(name, attributes);
|
||||
let outcome: Outcome = "ok";
|
||||
try { return await span.run(() => work(span)); }
|
||||
catch (error) { outcome = "failed"; throw error; }
|
||||
finally { span.end(outcome); }
|
||||
}
|
||||
|
||||
/** Lazy, bounded body consumption. GET response wait is a separate operation. */
|
||||
export function measureSandboxStream(name: string, attributes: Attributes, source: Readable): Readable {
|
||||
if (!hasSandboxPerformanceTrace()) return source;
|
||||
const within = captureSandboxPerformanceContext();
|
||||
let earlyError: Error | undefined;
|
||||
// A prefetched response can fail before anybody starts consuming the wrapper.
|
||||
// Keep its error observable without starting the stream or buffering bytes.
|
||||
const onSourceError = (error: Error) => { earlyError = error; };
|
||||
source.on("error", onSourceError);
|
||||
const wrapped = Readable.from((async function* () {
|
||||
const span = within(() => startSandboxOperation(name, attributes));
|
||||
let bytes = 0, chunks = 0, readWaitMs = 0, complete = false, outcome: Outcome = "cancelled";
|
||||
const bodyStarted = performance.now();
|
||||
const iterator = source[Symbol.asyncIterator]();
|
||||
try {
|
||||
if (earlyError) throw earlyError;
|
||||
for (;;) {
|
||||
const readStarted = performance.now();
|
||||
let next: IteratorResult<unknown>;
|
||||
try { next = await span.run(() => iterator.next()); }
|
||||
finally { readWaitMs += performance.now() - readStarted; }
|
||||
if (next.done) { complete = true; outcome = "ok"; break; }
|
||||
bytes += Buffer.isBuffer(next.value) ? next.value.length : Buffer.byteLength(String(next.value));
|
||||
chunks++;
|
||||
yield next.value;
|
||||
}
|
||||
} catch (error) { outcome = "failed"; throw error; }
|
||||
finally {
|
||||
try {
|
||||
if (!complete) { try { await iterator.return?.(); } finally { source.destroy(); } }
|
||||
} finally {
|
||||
span.set({ bytes, chunks, readWaitMs, bodyNonReadMs: Math.max(0, performance.now() - bodyStarted - readWaitMs) }); span.end(outcome);
|
||||
}
|
||||
}
|
||||
})());
|
||||
wrapped.once("close", () => {
|
||||
// Also closes a response whose lazy generator was never entered.
|
||||
source.destroy();
|
||||
if (source.closed) source.removeListener("error", onSourceError);
|
||||
else source.once("close", () => source.removeListener("error", onSourceError));
|
||||
});
|
||||
const destroy = wrapped.destroy.bind(wrapped);
|
||||
wrapped.destroy = (error?: Error) => {
|
||||
source.destroy();
|
||||
return destroy(error);
|
||||
};
|
||||
return wrapped;
|
||||
}
|
||||
|
||||
export async function runWithSandboxPerformanceTrace<T>(input: {
|
||||
runId: string; enabled?: boolean; traceContext?: StartupTraceContextHandle; maxRecords?: number;
|
||||
onBatch?: (batch: { schema: "paperclip.sandbox-performance.v1"; runHash: string; records: SandboxPerformanceRecord[]; dropped: number }) => Promise<void>;
|
||||
}, work: () => Promise<T>): Promise<T> {
|
||||
if (!(input.enabled ?? Boolean(process.env.OTEL_EXPORTER_OTLP_ENDPOINT?.trim()))) return work();
|
||||
const trace: TraceState = { tracing: input.traceContext ?? getStartupTraceContext("paperclip.sandbox"), records: [],
|
||||
limit: Number.isFinite(input.maxRecords) ? Math.max(1, Math.min(50_000, Math.floor(input.maxRecords!))) : 20_000, dropped: 0,
|
||||
runHash: createHash("sha256").update(input.runId).digest("hex").slice(0, 12), startedAtMs: Date.now() };
|
||||
const initial = { trace, id: "", context: getActiveStepContext()?.parentContext };
|
||||
try {
|
||||
return await scopes.run(initial, () => measureSandboxOperation("sandbox.run", {}, async (span) => {
|
||||
trace.root = span;
|
||||
try { return await work(); }
|
||||
finally {
|
||||
span.set({ recordCount: trace.records.length + (trace.records.length < trace.limit ? 1 : 0),
|
||||
dropped: trace.dropped + (trace.records.length >= trace.limit ? 1 : 0) });
|
||||
}
|
||||
}));
|
||||
}
|
||||
finally {
|
||||
trace.closed = true;
|
||||
// No per-file DB writes, no synchronous exporter call and no unbounded
|
||||
// promise queue. Persist at most 250 records per batch after measured work.
|
||||
// The SDK independently exports ended spans through its batch processor.
|
||||
for (let offset = 0; offset < trace.records.length; offset += 250) {
|
||||
try { await input.onBatch?.({ schema: "paperclip.sandbox-performance.v1", runHash: trace.runHash,
|
||||
records: trace.records.slice(offset, offset + 250), dropped: trace.dropped }); } catch { break; }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,3 +1,4 @@
|
|||
import { measureSandboxOperation, measureSandboxStream, captureSandboxPerformanceContext } from "./sandbox-performance.js";
|
||||
import { prefetchWorkFiles } from "./work-folder-transfer.js";
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import path from "node:path";
|
||||
|
|
@ -34,376 +35,416 @@ export async function prepareSandboxWorkFolders(input: {
|
|||
primaryWorkspaceId?: string | null; primaryBranchName?: string | null;
|
||||
storage?: StorageProvider; sandboxKey?: string;
|
||||
}) {
|
||||
const { db, target } = input;
|
||||
if (!target.runner || !target.leaseId) throw new Error("Sandbox file transport is unavailable");
|
||||
async function assertBindings() {
|
||||
const memberships: Array<{ companyId: string; membershipRole: string | null; status: string }> = [];
|
||||
const deny = () => { throw new Error("Sandbox work-folder access is no longer authorized; working files were retained"); };
|
||||
const [run] = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.id, input.runId), eq(heartbeatRuns.companyId, input.companyId)));
|
||||
if (!run || run.agentId !== input.agentId || run.responsibleUserId !== input.responsibleUserId) deny();
|
||||
const [agent] = await db.select({ id: agents.id }).from(agents).where(and(eq(agents.id, input.agentId), eq(agents.companyId, input.companyId)));
|
||||
if (!agent) deny();
|
||||
if (input.taskId) {
|
||||
const [task] = await db.select({ id: issues.id }).from(issues).where(and(eq(issues.id, input.taskId), eq(issues.companyId, input.companyId)));
|
||||
if (!task) deny();
|
||||
const runInRunContext = captureSandboxPerformanceContext();
|
||||
return measureSandboxOperation("work_folder.prepare", { phase: "prepare" }, async (prepareSpan) => {
|
||||
const { db, target, taskId, projectId, responsibleUserId } = input;
|
||||
if (!target.runner || !target.leaseId) throw new Error("Sandbox file transport is unavailable");
|
||||
async function assertBindings() {
|
||||
return measureSandboxOperation("work_folder.authorization", { operation: "validate_bindings" }, async () => {
|
||||
const memberships: Array<{ companyId: string; membershipRole: string | null; status: string }> = [];
|
||||
const deny = () => { throw new Error("Sandbox work-folder access is no longer authorized; working files were retained"); };
|
||||
const [run] = await measureSandboxOperation("work_folder.db.query", { operation: "select_heartbeat_runs", requestCount: 1 }, async () => (db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.id, input.runId), eq(heartbeatRuns.companyId, input.companyId)))));
|
||||
if (!run || run.agentId !== input.agentId || run.responsibleUserId !== responsibleUserId) deny();
|
||||
const [agent] = await measureSandboxOperation("work_folder.db.query", { operation: "select_agents", requestCount: 1 }, async () => (db.select({ id: agents.id }).from(agents).where(and(eq(agents.id, input.agentId), eq(agents.companyId, input.companyId)))));
|
||||
if (!agent) deny();
|
||||
if (taskId) {
|
||||
const [task] = await measureSandboxOperation("work_folder.db.query", { operation: "select_issues", requestCount: 1 }, async () => (db.select({ id: issues.id }).from(issues).where(and(eq(issues.id, taskId), eq(issues.companyId, input.companyId)))));
|
||||
if (!task) deny();
|
||||
}
|
||||
if (projectId) {
|
||||
const [project] = await measureSandboxOperation("work_folder.db.query", { operation: "select_projects", requestCount: 1 }, async () => (db.select({ id: projects.id }).from(projects).where(and(eq(projects.id, projectId), eq(projects.companyId, input.companyId)))));
|
||||
if (!project) deny();
|
||||
}
|
||||
if (responsibleUserId) {
|
||||
const [membership] = await measureSandboxOperation("work_folder.db.query", { operation: "select_company_memberships", requestCount: 1 }, async () => (db.select().from(companyMemberships).where(and(eq(companyMemberships.companyId, input.companyId),
|
||||
eq(companyMemberships.principalType, "user"), eq(companyMemberships.principalId, responsibleUserId), eq(companyMemberships.status, "active")))));
|
||||
if (!membership || membership.membershipRole === "viewer") deny();
|
||||
if (membership) memberships.push({ companyId: membership.companyId, membershipRole: membership.membershipRole, status: membership.status });
|
||||
}
|
||||
for (const [scope, ownerId] of [["task", taskId], ["agent", input.agentId], ["project", projectId]] as const) {
|
||||
if (!ownerId) continue;
|
||||
await assertWorkFolderAccess(db, { type: "agent", source: "agent_jwt", companyId: input.companyId,
|
||||
agentId: input.agentId, runId: input.runId, onBehalfOfUserId: responsibleUserId, onBehalfOfMemberships: memberships },
|
||||
{ companyId: input.companyId, scope, ownerId }, true);
|
||||
}
|
||||
});
|
||||
}
|
||||
if (input.projectId) {
|
||||
const [project] = await db.select({ id: projects.id }).from(projects).where(and(eq(projects.id, input.projectId), eq(projects.companyId, input.companyId)));
|
||||
if (!project) deny();
|
||||
// Host-side transfers do not go through HTTP authorization middleware. Check
|
||||
// the authoritative bindings here too, including after membership revocation.
|
||||
// An ended heartbeat may still flush; its immutable identity must still match.
|
||||
await assertBindings();
|
||||
const storage = input.storage ?? createStorageProviderFromConfig(loadConfig());
|
||||
const svc = workFolderService(db, storage);
|
||||
const transport = workFolderTransport(target.runner);
|
||||
const repositories = workFolderRepositoryService(db, storage, transport);
|
||||
const home = await transport.home();
|
||||
const paths = workFolderPaths(home);
|
||||
for (const value of Object.values(paths)) await transport.mkdirRoot(value);
|
||||
const staging = paths[".paperclip-work-folders"]!;
|
||||
const owners = { task: taskId, agent: input.agentId, user: responsibleUserId, project: projectId };
|
||||
const folders: Partial<Record<WorkFolderScope, typeof workFolders.$inferSelect>> = {};
|
||||
for (const scope of WORK_FOLDER_SCOPES) {
|
||||
const ownerId = owners[scope];
|
||||
if (ownerId) folders[scope] = await svc.ensure({ companyId: input.companyId, scope, ownerId });
|
||||
}
|
||||
if (input.responsibleUserId) {
|
||||
const [membership] = await db.select().from(companyMemberships).where(and(eq(companyMemberships.companyId, input.companyId),
|
||||
eq(companyMemberships.principalType, "user"), eq(companyMemberships.principalId, input.responsibleUserId), eq(companyMemberships.status, "active")));
|
||||
if (!membership || membership.membershipRole === "viewer") deny();
|
||||
if (membership) memberships.push({ companyId: membership.companyId, membershipRole: membership.membershipRole, status: membership.status });
|
||||
const manifest: SandboxWorkFolderManifest = { version: 1, companyId: input.companyId, runId: input.runId,
|
||||
taskId: taskId, agentId: input.agentId, responsibleUserId: responsibleUserId,
|
||||
projectId: projectId, leaseId: target.leaseId, sandboxKey: input.sandboxKey ?? target.leaseId, home,
|
||||
folders: { task: folders.task?.id ?? null, agent: folders.agent!.id, user: folders.user?.id ?? null, project: folders.project?.id ?? null }, repositories: [] };
|
||||
const [previous] = await measureSandboxOperation("work_folder.history.sandbox", { operation: "select_work_folder_runs", requestCount: 1 }, async () => (db.select().from(workFolderRuns).where(and(eq(workFolderRuns.companyId, input.companyId),
|
||||
sql`coalesce(${workFolderRuns.manifest}->>'sandboxKey', ${workFolderRuns.manifest}->>'leaseId') = ${manifest.sandboxKey}`)).orderBy(desc(workFolderRuns.updatedAt)).limit(1)));
|
||||
if (previous && (previous.manifest.taskId !== taskId || previous.manifest.agentId !== input.agentId
|
||||
|| previous.manifest.responsibleUserId !== responsibleUserId || previous.manifest.projectId !== projectId)) {
|
||||
throw new Error("Sandbox file identity changed; acquire a fresh sandbox before continuing");
|
||||
}
|
||||
for (const [scope, ownerId] of [["task", input.taskId], ["agent", input.agentId], ["project", input.projectId]] as const) {
|
||||
if (!ownerId) continue;
|
||||
await assertWorkFolderAccess(db, { type: "agent", source: "agent_jwt", companyId: input.companyId,
|
||||
agentId: input.agentId, runId: input.runId, onBehalfOfUserId: input.responsibleUserId, onBehalfOfMemberships: memberships },
|
||||
{ companyId: input.companyId, scope, ownerId }, true);
|
||||
}
|
||||
}
|
||||
// Host-side transfers do not go through HTTP authorization middleware. Check
|
||||
// the authoritative bindings here too, including after membership revocation.
|
||||
// An ended heartbeat may still flush; its immutable identity must still match.
|
||||
await assertBindings();
|
||||
const storage = input.storage ?? createStorageProviderFromConfig(loadConfig());
|
||||
const svc = workFolderService(db, storage);
|
||||
const transport = workFolderTransport(target.runner);
|
||||
const repositories = workFolderRepositoryService(db, storage, transport);
|
||||
const home = await transport.home();
|
||||
const paths = workFolderPaths(home);
|
||||
for (const value of Object.values(paths)) await transport.mkdirRoot(value);
|
||||
const staging = paths[".paperclip-work-folders"]!;
|
||||
const owners = { task: input.taskId, agent: input.agentId, user: input.responsibleUserId, project: input.projectId };
|
||||
const folders: Partial<Record<WorkFolderScope, typeof workFolders.$inferSelect>> = {};
|
||||
for (const scope of WORK_FOLDER_SCOPES) {
|
||||
const ownerId = owners[scope];
|
||||
if (ownerId) folders[scope] = await svc.ensure({ companyId: input.companyId, scope, ownerId });
|
||||
}
|
||||
const manifest: SandboxWorkFolderManifest = { version: 1, companyId: input.companyId, runId: input.runId,
|
||||
taskId: input.taskId, agentId: input.agentId, responsibleUserId: input.responsibleUserId,
|
||||
projectId: input.projectId, leaseId: target.leaseId, sandboxKey: input.sandboxKey ?? target.leaseId, home,
|
||||
folders: { task: folders.task?.id ?? null, agent: folders.agent!.id, user: folders.user?.id ?? null, project: folders.project?.id ?? null }, repositories: [] };
|
||||
const [previous] = await db.select().from(workFolderRuns).where(and(eq(workFolderRuns.companyId, input.companyId),
|
||||
sql`coalesce(${workFolderRuns.manifest}->>'sandboxKey', ${workFolderRuns.manifest}->>'leaseId') = ${manifest.sandboxKey}`)).orderBy(desc(workFolderRuns.updatedAt)).limit(1);
|
||||
if (previous && (previous.manifest.taskId !== input.taskId || previous.manifest.agentId !== input.agentId
|
||||
|| previous.manifest.responsibleUserId !== input.responsibleUserId || previous.manifest.projectId !== input.projectId)) {
|
||||
throw new Error("Sandbox file identity changed; acquire a fresh sandbox before continuing");
|
||||
}
|
||||
const [previousTaskRun] = input.taskId ? await db.select({ manifest: workFolderRuns.manifest }).from(workFolderRuns)
|
||||
.where(and(eq(workFolderRuns.companyId, input.companyId), sql`${workFolderRuns.manifest}->>'taskId' = ${input.taskId}`))
|
||||
.orderBy(desc(workFolderRuns.updatedAt)).limit(1) : [];
|
||||
const identityChanged = Boolean(previousTaskRun && (previousTaskRun.manifest.agentId !== input.agentId
|
||||
|| previousTaskRun.manifest.responsibleUserId !== input.responsibleUserId));
|
||||
const baselines: Record<string, WorkTreeEntry[]> = previous?.baselines ?? {};
|
||||
const pendingOperations = previous?.pendingOperations ?? {};
|
||||
await db.insert(workFolderRuns).values({ runId: input.runId, companyId: input.companyId, manifest, baselines, pendingOperations })
|
||||
.onConflictDoUpdate({ target: workFolderRuns.runId, set: { manifest, state: "starting", updatedAt: new Date() } });
|
||||
// These dimensions describe validated work-folder history for this physical
|
||||
// sandbox key, not whether the provider was powered on or restarted.
|
||||
prepareSpan.set({ cold: !previous, warm: Boolean(previous), reused: Boolean(previous) });
|
||||
const [previousTaskRun] = taskId ? await measureSandboxOperation("work_folder.history.task", { operation: "select_work_folder_runs", requestCount: 1 }, async () => (db.select({ manifest: workFolderRuns.manifest }).from(workFolderRuns)
|
||||
.where(and(eq(workFolderRuns.companyId, input.companyId), sql`${workFolderRuns.manifest}->>'taskId' = ${taskId}`))
|
||||
.orderBy(desc(workFolderRuns.updatedAt)).limit(1))) : [];
|
||||
const identityChanged = Boolean(previousTaskRun && (previousTaskRun.manifest.agentId !== input.agentId
|
||||
|| previousTaskRun.manifest.responsibleUserId !== responsibleUserId));
|
||||
const baselines: Record<string, WorkTreeEntry[]> = previous?.baselines ?? {};
|
||||
const pendingOperations = previous?.pendingOperations ?? {};
|
||||
await measureSandboxOperation("work_folder.manifest.initialize", { operation: "insert_work_folder_runs", requestCount: 1 }, async () => (db.insert(workFolderRuns).values({ runId: input.runId, companyId: input.companyId, manifest, baselines, pendingOperations })
|
||||
.onConflictDoUpdate({ target: workFolderRuns.runId, set: { manifest, state: "starting", updatedAt: new Date() } })));
|
||||
|
||||
async function seedAttachments() {
|
||||
if (!folders.task || !input.taskId) return;
|
||||
const attached = await db.select({ attachment: issueAttachments, asset: assets }).from(issueAttachments)
|
||||
.innerJoin(assets, and(eq(assets.id, issueAttachments.assetId), eq(assets.companyId, issueAttachments.companyId)))
|
||||
.where(and(eq(issueAttachments.issueId, input.taskId), eq(issueAttachments.companyId, input.companyId)))
|
||||
.orderBy(asc(issueAttachments.createdAt), asc(issueAttachments.id));
|
||||
for (const { attachment, asset } of attached) {
|
||||
const operationId = `attachment:${attachment.id}`;
|
||||
const [seeded] = await db.select().from(workFileOperations).where(and(eq(workFileOperations.folderId, folders.task.id),
|
||||
eq(workFileOperations.operationId, operationId)));
|
||||
if (seeded) continue;
|
||||
const original = (asset.originalFilename ?? "attachment").split(/[\\/]/).at(-1)!.replace(/[\x00-\x1f\x7f]/g, "_").slice(0, 180) || "attachment";
|
||||
// The ID makes the destination independent of concurrent uploads and
|
||||
// earlier seeding attempts. Even dot/reserved filenames become safe.
|
||||
const extension = path.posix.extname(original);
|
||||
const filename = `${original.slice(0, original.length - extension.length)}-${attachment.id}${extension}`;
|
||||
const result = await storage.getObject({ objectKey: asset.objectKey });
|
||||
try { await svc.write(folders.task, { path: filename, body: result.stream, contentType: asset.contentType, operationId, onlyIfMissing: true }); }
|
||||
finally { result.stream.destroy(); }
|
||||
async function seedAttachments() {
|
||||
return measureSandboxOperation("work_folder.attachments", { phase: "prepare" }, async (span) => {
|
||||
const taskFolder = folders.task;
|
||||
if (!taskFolder || !taskId) return;
|
||||
const attached = await measureSandboxOperation("work_folder.db.query", { operation: "select_issue_attachments", requestCount: 1 }, async () => (db.select({ attachment: issueAttachments, asset: assets }).from(issueAttachments)
|
||||
.innerJoin(assets, and(eq(assets.id, issueAttachments.assetId), eq(assets.companyId, issueAttachments.companyId)))
|
||||
.where(and(eq(issueAttachments.issueId, taskId), eq(issueAttachments.companyId, input.companyId)))
|
||||
.orderBy(asc(issueAttachments.createdAt), asc(issueAttachments.id))));
|
||||
span.set({ files: attached.length });
|
||||
for (const [fileIndex, { attachment, asset }] of attached.entries()) {
|
||||
const operationId = `attachment:${attachment.id}`;
|
||||
const [seeded] = await measureSandboxOperation("work_folder.db.query", { operation: "select_work_file_operations", requestCount: 1 }, async () => (db.select().from(workFileOperations).where(and(eq(workFileOperations.folderId, taskFolder.id),
|
||||
eq(workFileOperations.operationId, operationId)))));
|
||||
if (seeded) continue;
|
||||
const original = (asset.originalFilename ?? "attachment").split(/[\\/]/).at(-1)!.replace(/[\x00-\x1f\x7f]/g, "_").slice(0, 180) || "attachment";
|
||||
// The ID makes the destination independent of concurrent uploads and
|
||||
// earlier seeding attempts. Even dot/reserved filenames become safe.
|
||||
const extension = path.posix.extname(original);
|
||||
const filename = `${original.slice(0, original.length - extension.length)}-${attachment.id}${extension}`;
|
||||
const result = await measureSandboxOperation("work_folder.attachment.get_response", { requestCount: 1, bytes: asset.byteSize, fileIndex }, async () => (storage.getObject({ objectKey: asset.objectKey })));
|
||||
try { await svc.write(taskFolder, { path: filename, body: measureSandboxStream("work_folder.attachment.body", { bytes: asset.byteSize, scope: "task", fileIndex }, result.stream), contentType: asset.contentType, operationId, onlyIfMissing: true }); }
|
||||
finally { result.stream.destroy(); }
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
async function importAgentFiles() {
|
||||
const folder = folders.agent!;
|
||||
if (folder.importedAt) return;
|
||||
const root = resolveDefaultAgentWorkspaceDir(input.agentId);
|
||||
for await (const file of managedAgentFiles(root)) {
|
||||
const [receipt] = await db.select({ id: workFileOperations.id }).from(workFileOperations).where(and(
|
||||
eq(workFileOperations.folderId, folder.id), eq(workFileOperations.operationId, `import:${file.path}`)));
|
||||
if (receipt) continue;
|
||||
await svc.write(folder, { ...file, operationId: `import:${file.path}`, onlyIfMissing: true });
|
||||
async function importAgentFiles() {
|
||||
return measureSandboxOperation("work_folder.agent_import", { phase: "prepare" }, async () => {
|
||||
const folder = folders.agent!;
|
||||
if (folder.importedAt) return;
|
||||
const root = resolveDefaultAgentWorkspaceDir(input.agentId);
|
||||
for await (const file of managedAgentFiles(root)) {
|
||||
const [receipt] = await measureSandboxOperation("work_folder.db.query", { operation: "select_work_file_operations", requestCount: 1 }, async () => (db.select({ id: workFileOperations.id }).from(workFileOperations).where(and(
|
||||
eq(workFileOperations.folderId, folder.id), eq(workFileOperations.operationId, `import:${file.path}`)))));
|
||||
if (receipt) continue;
|
||||
await svc.write(folder, { ...file, operationId: `import:${file.path}`, onlyIfMissing: true });
|
||||
}
|
||||
await measureSandboxOperation("work_folder.db.query", { operation: "update_work_folders", requestCount: 1 }, async () => (db.update(workFolders).set({ importedAt: new Date() }).where(eq(workFolders.id, folder.id))));
|
||||
});
|
||||
}
|
||||
await db.update(workFolders).set({ importedAt: new Date() }).where(eq(workFolders.id, folder.id));
|
||||
}
|
||||
async function reconcileIncoming(scope: WorkFolderScope, current: WorkTreeEntry[]) {
|
||||
const targets = baselines[`incoming:${scope}`];
|
||||
const removed = baselines[`incomingRemoved:${scope}`];
|
||||
if (!targets && !removed) return;
|
||||
const before = new Map((baselines[scope] ?? []).map((entry) => [entry.path, entry]));
|
||||
const observed = new Map(current.map((entry) => [entry.path, entry]));
|
||||
// A failed command may have published some imports or removed some files.
|
||||
// Only adopt effects actually observed on disk; different bytes remain
|
||||
// genuine local edits and must still pass through outgoing synchronization.
|
||||
for (const entry of removed ?? []) if (!observed.has(entry.path)) before.delete(entry.path);
|
||||
for (const entry of targets ?? []) {
|
||||
if (signature(observed.get(entry.path)) === signature(entry)) before.set(entry.path, entry);
|
||||
}
|
||||
baselines[scope] = [...before.values()];
|
||||
delete baselines[`incoming:${scope}`];
|
||||
delete baselines[`incomingRemoved:${scope}`];
|
||||
// Persist the reconciled baseline and remove its provenance atomically,
|
||||
// before any outgoing write can be accepted by the shared collection.
|
||||
await saveState("saving");
|
||||
}
|
||||
async function outgoing(scope: WorkFolderScope) {
|
||||
const folder = folders[scope];
|
||||
if (!folder) return;
|
||||
const current = await transport.scan(paths[scope]!);
|
||||
await reconcileIncoming(scope, current);
|
||||
const before = new Map((baselines[scope] ?? []).map((entry) => [entry.path, entry]));
|
||||
const after = new Map(current.map((entry) => [entry.path, entry]));
|
||||
async function operation(filePath: string, nextSignature: string, apply: (id: string) => Promise<unknown>, accept: () => void) {
|
||||
const key = `${scope}/${filePath}`;
|
||||
if (pendingOperations[key]?.signature !== nextSignature) pendingOperations[key] = { id: randomUUID(), signature: nextSignature };
|
||||
// Persist the receipt ID BEFORE sending bytes. A process restart or lost
|
||||
// COMMIT response must retry this ID, not overwrite another run's edit.
|
||||
await saveState("saving");
|
||||
await apply(pendingOperations[key]!.id);
|
||||
accept();
|
||||
async function reconcileIncoming(scope: WorkFolderScope, current: WorkTreeEntry[]) {
|
||||
const targets = baselines[`incoming:${scope}`];
|
||||
const removed = baselines[`incomingRemoved:${scope}`];
|
||||
if (!targets && !removed) return;
|
||||
const before = new Map((baselines[scope] ?? []).map((entry) => [entry.path, entry]));
|
||||
const observed = new Map(current.map((entry) => [entry.path, entry]));
|
||||
// A failed command may have published some imports or removed some files.
|
||||
// Only adopt effects actually observed on disk; different bytes remain
|
||||
// genuine local edits and must still pass through outgoing synchronization.
|
||||
for (const entry of removed ?? []) if (!observed.has(entry.path)) before.delete(entry.path);
|
||||
for (const entry of targets ?? []) {
|
||||
if (signature(observed.get(entry.path)) === signature(entry)) before.set(entry.path, entry);
|
||||
}
|
||||
baselines[scope] = [...before.values()];
|
||||
delete pendingOperations[key];
|
||||
delete baselines[`incoming:${scope}`];
|
||||
delete baselines[`incomingRemoved:${scope}`];
|
||||
// Persist the reconciled baseline and remove its provenance atomically,
|
||||
// before any outgoing write can be accepted by the shared collection.
|
||||
await saveState("saving");
|
||||
}
|
||||
for (const entry of current) {
|
||||
if (signature(before.get(entry.path)) === signature(entry)) continue;
|
||||
const body = entry.kind === "file" ? transport.read(paths[scope]!, entry.path, entry.byteSize) : undefined;
|
||||
try {
|
||||
await operation(entry.path, signature(entry), (operationId) => svc.write(folder, { path: entry.path, body,
|
||||
kind: entry.kind, replaceKind: true, executable: entry.executable, expectedSha256: entry.sha256, operationId }), () => {
|
||||
if (before.get(entry.path)?.kind !== entry.kind) for (const key of before.keys()) {
|
||||
if (key.startsWith(`${entry.path}/`)) before.delete(key);
|
||||
}
|
||||
before.set(entry.path, entry);
|
||||
});
|
||||
} finally { body?.destroy(); }
|
||||
}
|
||||
for (const old of [...before.values()].sort((a, b) => a.path.length - b.path.length)) {
|
||||
if (!after.has(old.path) && before.has(old.path)) {
|
||||
await operation(old.path, "missing", (operationId) => svc.remove(folder, old.path, operationId), () => {
|
||||
for (const key of before.keys()) if (key === old.path || key.startsWith(`${old.path}/`)) before.delete(key);
|
||||
});
|
||||
}
|
||||
}
|
||||
baselines[scope] = [...before.values()];
|
||||
}
|
||||
async function incoming(scope: WorkFolderScope) {
|
||||
const folder = folders[scope];
|
||||
if (!folder) return;
|
||||
const current = new Map((await transport.scan(paths[scope]!)).map((entry) => [entry.path, entry]));
|
||||
const saved: WorkTreeEntry[] = [];
|
||||
let cursor: string | undefined;
|
||||
do {
|
||||
const page = await svc.list(folder, { cursor, limit: 1000 });
|
||||
for (const file of page.files) saved.push({ path: file.path, kind: file.kind, byteSize: file.byteSize, sha256: file.sha256, executable: file.executable });
|
||||
cursor = page.nextCursor ?? undefined;
|
||||
} while (cursor);
|
||||
const desired = new Map(saved.map((entry) => [entry.path, entry]));
|
||||
// Persist all deletion intents before the first removal, including children
|
||||
// that can disappear when a directory is replaced. A lost response must not
|
||||
// turn an imported deletion into a new delete against a newer shared file.
|
||||
const removed = [...current.values()].filter((entry) => !desired.has(entry.path) || desired.get(entry.path)!.kind !== entry.kind)
|
||||
.sort((a, b) => b.path.length - a.path.length);
|
||||
if (removed.length) {
|
||||
baselines[`incomingRemoved:${scope}`] = removed;
|
||||
await saveState("starting");
|
||||
}
|
||||
for (const entry of removed) {
|
||||
await transport.remove(paths[scope]!, entry.path);
|
||||
current.delete(entry.path);
|
||||
}
|
||||
const changed = saved.sort((a, b) => a.path.length - b.path.length)
|
||||
.filter((entry) => signature(current.get(entry.path)) !== signature(entry));
|
||||
await transport.writeMany(paths[scope]!, staging, prefetchWorkFiles(changed, async (entry) => {
|
||||
if (entry.kind === "directory") return { entry };
|
||||
const result = await svc.content(folder, entry.path);
|
||||
// A shared file can change after listing. Validate and baseline the
|
||||
// version opened by content(), whose metadata and stream belong together.
|
||||
Object.assign(entry, { byteSize: result.file.byteSize, sha256: result.file.sha256, executable: result.file.executable });
|
||||
return { entry, body: result.stream };
|
||||
}), async (entries) => {
|
||||
const targets = new Map((baselines[`incoming:${scope}`] ?? []).map((entry) => [entry.path, entry]));
|
||||
for (const entry of entries) {
|
||||
targets.set(entry.path, entry);
|
||||
// Publishing nested files can create parents absent from the listing.
|
||||
// Record those directory imports too, so they cannot be mistaken for
|
||||
// agent-created directories after an interrupted batch.
|
||||
let parent = path.posix.dirname(entry.path);
|
||||
while (parent !== ".") {
|
||||
if (!targets.has(parent)) targets.set(parent, { path: parent, kind: "directory", byteSize: 0, sha256: null, executable: false });
|
||||
parent = path.posix.dirname(parent);
|
||||
async function outgoing(scope: WorkFolderScope) {
|
||||
return measureSandboxOperation("work_folder.scope.outgoing", { scope }, async (span) => {
|
||||
const folder = folders[scope];
|
||||
if (!folder) return;
|
||||
const current = await measureSandboxOperation("work_folder.scope.disk_scan", { scope }, async () => (transport.scan(paths[scope]!)));
|
||||
span.set({ files: current.length });
|
||||
await reconcileIncoming(scope, current);
|
||||
const before = new Map((baselines[scope] ?? []).map((entry) => [entry.path, entry]));
|
||||
const after = new Map(current.map((entry) => [entry.path, entry]));
|
||||
async function operation(filePath: string, nextSignature: string, apply: (id: string) => Promise<unknown>, accept: () => void) {
|
||||
const key = `${scope}/${filePath}`;
|
||||
if (pendingOperations[key]?.signature !== nextSignature) pendingOperations[key] = { id: randomUUID(), signature: nextSignature };
|
||||
// Persist the receipt ID BEFORE sending bytes. A process restart or lost
|
||||
// COMMIT response must retry this ID, not overwrite another run's edit.
|
||||
await saveState("saving");
|
||||
await apply(pendingOperations[key]!.id);
|
||||
accept();
|
||||
baselines[scope] = [...before.values()];
|
||||
delete pendingOperations[key];
|
||||
await saveState("saving");
|
||||
}
|
||||
}
|
||||
baselines[`incoming:${scope}`] = [...targets.values()];
|
||||
await saveState("starting");
|
||||
});
|
||||
baselines[scope] = saved;
|
||||
delete baselines[`incoming:${scope}`];
|
||||
delete baselines[`incomingRemoved:${scope}`];
|
||||
await saveState("starting");
|
||||
}
|
||||
for (const entry of current) {
|
||||
if (signature(before.get(entry.path)) === signature(entry)) continue;
|
||||
const body = entry.kind === "file" ? transport.read(paths[scope]!, entry.path, entry.byteSize) : undefined;
|
||||
try {
|
||||
await operation(entry.path, signature(entry), (operationId) => svc.write(folder, { path: entry.path, body,
|
||||
kind: entry.kind, replaceKind: true, executable: entry.executable, expectedSha256: entry.sha256, operationId }), () => {
|
||||
if (before.get(entry.path)?.kind !== entry.kind) for (const key of before.keys()) {
|
||||
if (key.startsWith(`${entry.path}/`)) before.delete(key);
|
||||
}
|
||||
before.set(entry.path, entry);
|
||||
});
|
||||
} finally { body?.destroy(); }
|
||||
}
|
||||
for (const old of [...before.values()].sort((a, b) => a.path.length - b.path.length)) {
|
||||
if (!after.has(old.path) && before.has(old.path)) {
|
||||
await operation(old.path, "missing", (operationId) => svc.remove(folder, old.path, operationId), () => {
|
||||
for (const key of before.keys()) if (key === old.path || key.startsWith(`${old.path}/`)) before.delete(key);
|
||||
});
|
||||
}
|
||||
}
|
||||
baselines[scope] = [...before.values()];
|
||||
});
|
||||
}
|
||||
async function incoming(scope: WorkFolderScope) {
|
||||
return measureSandboxOperation("work_folder.scope.incoming", { scope }, async (span) => {
|
||||
const folder = folders[scope];
|
||||
if (!folder) return;
|
||||
const current = new Map((await measureSandboxOperation("work_folder.scope.disk_scan", { scope }, async () => (transport.scan(paths[scope]!)))).map((entry) => [entry.path, entry]));
|
||||
const saved: WorkTreeEntry[] = [];
|
||||
let cursor: string | undefined;
|
||||
do {
|
||||
const page = await svc.list(folder, { cursor, limit: 1000 });
|
||||
for (const file of page.files) saved.push({ path: file.path, kind: file.kind, byteSize: file.byteSize, sha256: file.sha256, executable: file.executable });
|
||||
cursor = page.nextCursor ?? undefined;
|
||||
} while (cursor);
|
||||
span.set({ files: saved.length });
|
||||
const desired = new Map(saved.map((entry) => [entry.path, entry]));
|
||||
// Persist all deletion intents before the first removal, including children
|
||||
// that can disappear when a directory is replaced. A lost response must not
|
||||
// turn an imported deletion into a new delete against a newer shared file.
|
||||
const removed = [...current.values()].filter((entry) => !desired.has(entry.path) || desired.get(entry.path)!.kind !== entry.kind)
|
||||
.sort((a, b) => b.path.length - a.path.length);
|
||||
if (removed.length) {
|
||||
baselines[`incomingRemoved:${scope}`] = removed;
|
||||
await saveState("starting");
|
||||
}
|
||||
for (const entry of removed) {
|
||||
await transport.remove(paths[scope]!, entry.path);
|
||||
current.delete(entry.path);
|
||||
}
|
||||
const changed = saved.sort((a, b) => a.path.length - b.path.length)
|
||||
.filter((entry) => signature(current.get(entry.path)) !== signature(entry));
|
||||
await measureSandboxOperation("work_folder.scope.hydrate", { scope, files: changed.length, parallelism: 4 }, async () => (transport.writeMany(paths[scope]!, staging, prefetchWorkFiles(changed, async (entry, fileIndex) => {
|
||||
if (entry.kind === "directory") return { entry };
|
||||
const result = await svc.content(folder, entry.path, fileIndex);
|
||||
// A shared file can change after listing. Validate and baseline the
|
||||
// version opened by content(), whose metadata and stream belong together.
|
||||
Object.assign(entry, { byteSize: result.file.byteSize, sha256: result.file.sha256, executable: result.file.executable });
|
||||
return { entry, body: result.stream };
|
||||
}), async (entries) => {
|
||||
const targets = new Map((baselines[`incoming:${scope}`] ?? []).map((entry) => [entry.path, entry]));
|
||||
for (const entry of entries) {
|
||||
targets.set(entry.path, entry);
|
||||
// Publishing nested files can create parents absent from the listing.
|
||||
// Record those directory imports too, so they cannot be mistaken for
|
||||
// agent-created directories after an interrupted batch.
|
||||
let parent = path.posix.dirname(entry.path);
|
||||
while (parent !== ".") {
|
||||
if (!targets.has(parent)) targets.set(parent, { path: parent, kind: "directory", byteSize: 0, sha256: null, executable: false });
|
||||
parent = path.posix.dirname(parent);
|
||||
}
|
||||
}
|
||||
baselines[`incoming:${scope}`] = [...targets.values()];
|
||||
await saveState("starting");
|
||||
})));
|
||||
baselines[scope] = saved;
|
||||
delete baselines[`incoming:${scope}`];
|
||||
delete baselines[`incomingRemoved:${scope}`];
|
||||
await saveState("starting");
|
||||
});
|
||||
}
|
||||
|
||||
const bindings: Array<{ binding: typeof taskRepositoryBindings.$inferSelect; root: string }> = [];
|
||||
async function prepareRepositories() {
|
||||
if (!input.taskId || !input.projectId) return;
|
||||
const workspaces = await db.select().from(projectWorkspaces).where(and(eq(projectWorkspaces.companyId, input.companyId),
|
||||
eq(projectWorkspaces.projectId, input.projectId))).orderBy(desc(projectWorkspaces.isPrimary), asc(projectWorkspaces.createdAt));
|
||||
const existing = await db.select().from(taskRepositoryBindings).where(and(eq(taskRepositoryBindings.companyId, input.companyId), eq(taskRepositoryBindings.taskId, input.taskId)));
|
||||
const names = new Set(existing.map((binding) => binding.name));
|
||||
const resolveGitAuth = createGitRemoteAuthProvider(db, input.companyId, { responsibleUserId: input.responsibleUserId, agentId: input.agentId, issueId: input.taskId, heartbeatRunId: input.runId });
|
||||
for (const workspace of workspaces.filter((entry) => entry.repoUrl)) {
|
||||
const primary = input.primaryWorkspaceId ? workspace.id === input.primaryWorkspaceId : workspace.isPrimary;
|
||||
let binding = existing.find((entry) => entry.workspaceId === workspace.id);
|
||||
if (!binding) {
|
||||
const baseName = repoName(workspace.repoUrl!.split(/[/:]/).at(-1) ?? workspace.name, workspace.id);
|
||||
const name = names.has(baseName) ? `${baseName}-${workspace.id.slice(0, 8)}` : baseName;
|
||||
names.add(name);
|
||||
[binding] = await db.insert(taskRepositoryBindings).values({ companyId: input.companyId, taskId: input.taskId,
|
||||
workspaceId: workspace.id, name, repoUrl: workspace.repoUrl, repoRef: workspace.repoRef ?? workspace.defaultRef }).returning();
|
||||
}
|
||||
if (!binding) throw new Error("Repository binding could not be created");
|
||||
if (binding.repoUrl !== workspace.repoUrl) throw new Error(`Repository ${binding.name} configuration changed; saved work was retained`);
|
||||
if (binding.repoRef !== (workspace.repoRef ?? workspace.defaultRef)) throw new Error(`Repository ${binding.name} starting ref changed; saved work was retained`);
|
||||
const root = path.posix.join(paths.repos!, binding.name);
|
||||
const probe = await target.runner!.execute({ command: "git", args: ["-C", root, "rev-parse", "--git-dir"], bypassSession: true, timeoutMs: 10_000 });
|
||||
const freshCheckout = probe.exitCode !== 0;
|
||||
if (freshCheckout) {
|
||||
// Publish the checkout directory only after every restore object or
|
||||
// clone step completes. An interrupted attempt cannot masquerade as a
|
||||
// reusable checkout merely because it contains a .git directory.
|
||||
const temporary = path.posix.join(staging, `repo-${binding.id}-${randomUUID()}`);
|
||||
const restored = await repositories.restore(binding, temporary, staging);
|
||||
if (!restored) {
|
||||
const auth = await resolveGitAuth(workspace.repoUrl!);
|
||||
const result = await target.runner!.execute({ command: "git", args: [...(auth?.configArgs ?? []), "clone", "--no-hardlinks",
|
||||
"--", workspace.repoUrl!, temporary],
|
||||
env: { GIT_TERMINAL_PROMPT: "0", ...(auth?.env ?? {}) }, bypassSession: true, timeoutMs: 300_000 });
|
||||
if (result.exitCode !== 0 || result.timedOut) throw new Error(`Required repository ${binding.name} could not be cloned`);
|
||||
if (binding.repoRef) {
|
||||
const checkout = await target.runner!.execute({ command: "git", args: ["-C", temporary, "checkout", binding.repoRef, "--"], bypassSession: true, timeoutMs: 60_000 });
|
||||
if (checkout.exitCode !== 0 || checkout.timedOut) throw new Error(`Required repository ${binding.name} ref could not be checked out`);
|
||||
}
|
||||
if (primary && input.primaryBranchName) {
|
||||
const branch = input.primaryBranchName;
|
||||
const valid = await target.runner!.execute({ command: "git", args: ["check-ref-format", "--branch", branch], bypassSession: true, timeoutMs: 10_000 });
|
||||
if (valid.exitCode !== 0 || valid.stdout.trim() !== branch) throw new Error(`Required repository ${binding.name} branch is invalid`);
|
||||
// Honor the task's existing branch policy on the initial clone.
|
||||
// Restores and warm starts keep the saved HEAD and index untouched.
|
||||
const checkout = await target.runner!.execute({ command: "git", args: ["-C", temporary, "checkout", branch, "--"], bypassSession: true, timeoutMs: 60_000 });
|
||||
if (checkout.exitCode !== 0) {
|
||||
const create = await target.runner!.execute({ command: "git", args: ["-C", temporary, "checkout", "-b", branch], bypassSession: true, timeoutMs: 60_000 });
|
||||
if (create.exitCode !== 0 || create.timedOut) throw new Error(`Required repository ${binding.name} task branch could not be created`);
|
||||
const bindings: Array<{ binding: typeof taskRepositoryBindings.$inferSelect; root: string }> = [];
|
||||
async function prepareRepositories() {
|
||||
return measureSandboxOperation("work_folder.repositories.prepare", { phase: "prepare" }, async () => {
|
||||
if (!taskId || !projectId) return;
|
||||
const workspaces = await measureSandboxOperation("work_folder.db.query", { operation: "select_project_workspaces", requestCount: 1 }, async () => (db.select().from(projectWorkspaces).where(and(eq(projectWorkspaces.companyId, input.companyId),
|
||||
eq(projectWorkspaces.projectId, projectId))).orderBy(desc(projectWorkspaces.isPrimary), asc(projectWorkspaces.createdAt))));
|
||||
const existing = await measureSandboxOperation("work_folder.db.query", { operation: "select_task_repository_bindings", requestCount: 1 }, async () => (db.select().from(taskRepositoryBindings).where(and(eq(taskRepositoryBindings.companyId, input.companyId), eq(taskRepositoryBindings.taskId, taskId)))));
|
||||
const names = new Set(existing.map((binding) => binding.name));
|
||||
const resolveGitAuth = createGitRemoteAuthProvider(db, input.companyId, { responsibleUserId: responsibleUserId, agentId: input.agentId, issueId: taskId, heartbeatRunId: input.runId });
|
||||
for (const [repositoryIndex, workspace] of workspaces.filter((entry) => entry.repoUrl).entries()) {
|
||||
await measureSandboxOperation("work_folder.repository.prepare", { repositoryIndex }, async (repositorySpan) => {
|
||||
const primary = input.primaryWorkspaceId ? workspace.id === input.primaryWorkspaceId : workspace.isPrimary;
|
||||
let binding = existing.find((entry) => entry.workspaceId === workspace.id);
|
||||
if (!binding) {
|
||||
const baseName = repoName(workspace.repoUrl!.split(/[/:]/).at(-1) ?? workspace.name, workspace.id);
|
||||
const name = names.has(baseName) ? `${baseName}-${workspace.id.slice(0, 8)}` : baseName;
|
||||
names.add(name);
|
||||
[binding] = await measureSandboxOperation("work_folder.db.query", { operation: "insert_task_repository_bindings", requestCount: 1 }, async () => (db.insert(taskRepositoryBindings).values({ companyId: input.companyId, taskId: taskId,
|
||||
workspaceId: workspace.id, name, repoUrl: workspace.repoUrl, repoRef: workspace.repoRef ?? workspace.defaultRef }).returning()));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const init = await target.runner!.execute({ command: "git", args: ["-C", temporary, "init"], bypassSession: true, timeoutMs: 10_000 });
|
||||
if (init.exitCode !== 0) throw new Error(`Repository ${binding.name} could not be restored`);
|
||||
const remote = await target.runner!.execute({ command: "git", args: ["-C", temporary, "remote", "add", "origin", binding.repoUrl!], bypassSession: true, timeoutMs: 10_000 });
|
||||
if (remote.exitCode !== 0) throw new Error(`Repository ${binding.name} remote could not be restored`);
|
||||
if (!binding) throw new Error("Repository binding could not be created");
|
||||
if (binding.repoUrl !== workspace.repoUrl) throw new Error(`Repository ${binding.name} configuration changed; saved work was retained`);
|
||||
if (binding.repoRef !== (workspace.repoRef ?? workspace.defaultRef)) throw new Error(`Repository ${binding.name} starting ref changed; saved work was retained`);
|
||||
const root = path.posix.join(paths.repos!, binding.name);
|
||||
const probe = await measureSandboxOperation("work_folder.repository.probe", { repositoryIndex, requestCount: 1 }, async () => (target.runner!.execute({ command: "git", args: ["-C", root, "rev-parse", "--git-dir"], bypassSession: true, timeoutMs: 10_000 })));
|
||||
const freshCheckout = probe.exitCode !== 0;
|
||||
// Checkout reuse is observed on disk; a durable restore remains cold.
|
||||
repositorySpan.set({ exists: !freshCheckout, reused: !freshCheckout, cold: freshCheckout, warm: !freshCheckout, cacheHit: false });
|
||||
if (freshCheckout) {
|
||||
// Publish the checkout directory only after every restore object or
|
||||
// clone step completes. An interrupted attempt cannot masquerade as a
|
||||
// reusable checkout merely because it contains a .git directory.
|
||||
const temporary = path.posix.join(staging, `repo-${binding.id}-${randomUUID()}`);
|
||||
const restored = await repositories.restore(binding, temporary, staging);
|
||||
repositorySpan.set({ cacheHit: restored });
|
||||
if (!restored) {
|
||||
const auth = await measureSandboxOperation("work_folder.repository.credentials", { repositoryIndex }, async () => (resolveGitAuth(workspace.repoUrl!)));
|
||||
const result = await measureSandboxOperation("work_folder.repository.clone", { repositoryIndex, requestCount: 1 }, async () => (target.runner!.execute({ command: "git", args: [...(auth?.configArgs ?? []), "clone", "--no-hardlinks",
|
||||
"--", workspace.repoUrl!, temporary],
|
||||
env: { GIT_TERMINAL_PROMPT: "0", ...(auth?.env ?? {}) }, bypassSession: true, timeoutMs: 300_000 })));
|
||||
if (result.exitCode !== 0 || result.timedOut) throw new Error(`Required repository ${binding.name} could not be cloned`);
|
||||
const repoRef = binding.repoRef;
|
||||
if (repoRef) {
|
||||
const checkout = await measureSandboxOperation("work_folder.repository.checkout", { repositoryIndex, requestCount: 1 }, async () => (target.runner!.execute({ command: "git", args: ["-C", temporary, "checkout", repoRef, "--"], bypassSession: true, timeoutMs: 60_000 })));
|
||||
if (checkout.exitCode !== 0 || checkout.timedOut) throw new Error(`Required repository ${binding.name} ref could not be checked out`);
|
||||
}
|
||||
if (primary && input.primaryBranchName) {
|
||||
const branch = input.primaryBranchName;
|
||||
const valid = await measureSandboxOperation("work_folder.repository.validate_branch", { repositoryIndex, requestCount: 1 }, async () => (target.runner!.execute({ command: "git", args: ["check-ref-format", "--branch", branch], bypassSession: true, timeoutMs: 10_000 })));
|
||||
if (valid.exitCode !== 0 || valid.stdout.trim() !== branch) throw new Error(`Required repository ${binding.name} branch is invalid`);
|
||||
// Honor the task's existing branch policy on the initial clone.
|
||||
// Restores and warm starts keep the saved HEAD and index untouched.
|
||||
const checkout = await measureSandboxOperation("work_folder.repository.checkout", { repositoryIndex, requestCount: 1 }, async () => (target.runner!.execute({ command: "git", args: ["-C", temporary, "checkout", branch, "--"], bypassSession: true, timeoutMs: 60_000 })));
|
||||
if (checkout.exitCode !== 0) {
|
||||
const create = await measureSandboxOperation("work_folder.repository.create_branch", { repositoryIndex, requestCount: 1 }, async () => (target.runner!.execute({ command: "git", args: ["-C", temporary, "checkout", "-b", branch], bypassSession: true, timeoutMs: 60_000 })));
|
||||
if (create.exitCode !== 0 || create.timedOut) throw new Error(`Required repository ${binding.name} task branch could not be created`);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const init = await measureSandboxOperation("work_folder.repository.restore_init", { repositoryIndex, requestCount: 1 }, async () => (target.runner!.execute({ command: "git", args: ["-C", temporary, "init"], bypassSession: true, timeoutMs: 10_000 })));
|
||||
if (init.exitCode !== 0) throw new Error(`Repository ${binding.name} could not be restored`);
|
||||
const remote = await measureSandboxOperation("work_folder.repository.restore_remote", { repositoryIndex, requestCount: 1 }, async () => (target.runner!.execute({ command: "git", args: ["-C", temporary, "remote", "add", "origin", binding.repoUrl!], bypassSession: true, timeoutMs: 10_000 })));
|
||||
if (remote.exitCode !== 0) throw new Error(`Repository ${binding.name} remote could not be restored`);
|
||||
}
|
||||
await measureSandboxOperation("work_folder.repository.publish", { repositoryIndex }, async () => (transport.moveRoot(temporary, root)));
|
||||
}
|
||||
// Warm checkouts retain completed setup. A replacement only restores
|
||||
// durable repository files, so setup must recreate ignored dependencies
|
||||
// and caches that are deliberately outside the checkpoint guarantee.
|
||||
const setupCommand = workspace.setupCommand;
|
||||
if ((!binding.setupComplete || freshCheckout) && setupCommand) {
|
||||
const setup = await measureSandboxOperation("work_folder.repository.setup", { repositoryIndex, requestCount: 1 }, async () => (target.runner!.execute({ command: "sh", args: ["-c", setupCommand], cwd: root, bypassSession: true, timeoutMs: 300_000 })));
|
||||
if (setup.exitCode !== 0 || setup.timedOut) throw new Error(`Repository ${binding.name} setup failed`);
|
||||
}
|
||||
await measureSandboxOperation("work_folder.db.query", { operation: "update_task_repository_bindings", requestCount: 1 }, async () => (db.update(taskRepositoryBindings).set({ setupComplete: true, retiredAt: null }).where(eq(taskRepositoryBindings.id, binding.id))));
|
||||
bindings.push({ binding, root });
|
||||
manifest.repositories.push({ bindingId: binding.id, workspaceId: workspace.id, name: binding.name, primary });
|
||||
await saveState("starting");
|
||||
});
|
||||
}
|
||||
await transport.moveRoot(temporary, root);
|
||||
}
|
||||
// Warm checkouts retain completed setup. A replacement only restores
|
||||
// durable repository files, so setup must recreate ignored dependencies
|
||||
// and caches that are deliberately outside the checkpoint guarantee.
|
||||
if ((!binding.setupComplete || freshCheckout) && workspace.setupCommand) {
|
||||
const setup = await target.runner!.execute({ command: "sh", args: ["-c", workspace.setupCommand], cwd: root, bypassSession: true, timeoutMs: 300_000 });
|
||||
if (setup.exitCode !== 0 || setup.timedOut) throw new Error(`Repository ${binding.name} setup failed`);
|
||||
}
|
||||
await db.update(taskRepositoryBindings).set({ setupComplete: true, retiredAt: null }).where(eq(taskRepositoryBindings.id, binding.id));
|
||||
bindings.push({ binding, root });
|
||||
manifest.repositories.push({ bindingId: binding.id, workspaceId: workspace.id, name: binding.name, primary });
|
||||
for (const old of existing) if (!workspaces.some((workspace) => workspace.id === old.workspaceId)) {
|
||||
await measureSandboxOperation("work_folder.db.query", { operation: "update_task_repository_bindings", requestCount: 1 }, async () => (db.update(taskRepositoryBindings).set({ retiredAt: new Date() }).where(eq(taskRepositoryBindings.id, old.id))));
|
||||
}
|
||||
});
|
||||
}
|
||||
async function saveState(state: "starting" | "saving" | "saved" | "failed", error: string | null = null) {
|
||||
return measureSandboxOperation("work_folder.progress.save", { phase: state, requestCount: 1 }, async () => {
|
||||
await measureSandboxOperation("work_folder.db.query", { operation: "update_work_folder_runs", requestCount: 1 }, async () => (db.update(workFolderRuns).set({ state, baselines, pendingOperations, manifest, error, updatedAt: new Date(),
|
||||
...(state === "saved" ? { lastSavedAt: new Date() } : {}) }).where(eq(workFolderRuns.runId, input.runId))));
|
||||
});
|
||||
}
|
||||
async function recordCheckpoint(action: string) {
|
||||
return measureSandboxOperation("work_folder.activity.record", { requestCount: 1 }, async () => {
|
||||
await logActivity(db, { companyId: input.companyId, actorType: "agent", actorId: input.agentId,
|
||||
agentId: input.agentId, runId: input.runId, issueId: taskId,
|
||||
responsibleUserIdOverride: responsibleUserId, action, entityType: "heartbeat_run", entityId: input.runId,
|
||||
details: { phase: "started", scopes: WORK_FOLDER_SCOPES.filter((scope) => Boolean(folders[scope])), repositories: bindings.length } });
|
||||
});
|
||||
}
|
||||
try {
|
||||
// Record intent before mutations. An unavailable audit store blocks new
|
||||
// work instead of turning an already completed save into a false failure.
|
||||
// The run's persisted state/lastSavedAt records checkpoint completion.
|
||||
await recordCheckpoint("work_folder.prepared");
|
||||
await seedAttachments();
|
||||
await importAgentFiles();
|
||||
// A resumed sandbox can hold edits newer than its last completed checkpoint.
|
||||
if (previous) for (const scope of WORK_FOLDER_SCOPES) await outgoing(scope);
|
||||
for (const scope of WORK_FOLDER_SCOPES) await incoming(scope);
|
||||
await prepareRepositories();
|
||||
await saveState("starting");
|
||||
if (previous?.refreshRequested) await measureSandboxOperation("work_folder.db.query", { operation: "update_work_folder_runs", requestCount: 1 }, async () => (db.update(workFolderRuns).set({ refreshRequested: false })
|
||||
.where(eq(workFolderRuns.runId, previous.runId))));
|
||||
} catch (error) {
|
||||
await saveState("failed", "Work folder preparation failed; existing files were retained");
|
||||
throw error;
|
||||
}
|
||||
for (const old of existing) if (!workspaces.some((workspace) => workspace.id === old.workspaceId)) {
|
||||
await db.update(taskRepositoryBindings).set({ retiredAt: new Date() }).where(eq(taskRepositoryBindings.id, old.id));
|
||||
}
|
||||
}
|
||||
async function saveState(state: "starting" | "saving" | "saved" | "failed", error: string | null = null) {
|
||||
await db.update(workFolderRuns).set({ state, baselines, pendingOperations, manifest, error, updatedAt: new Date(),
|
||||
...(state === "saved" ? { lastSavedAt: new Date() } : {}) }).where(eq(workFolderRuns.runId, input.runId));
|
||||
}
|
||||
async function recordCheckpoint(action: string) {
|
||||
await logActivity(db, { companyId: input.companyId, actorType: "agent", actorId: input.agentId,
|
||||
agentId: input.agentId, runId: input.runId, issueId: input.taskId,
|
||||
responsibleUserIdOverride: input.responsibleUserId, action, entityType: "heartbeat_run", entityId: input.runId,
|
||||
details: { phase: "started", scopes: WORK_FOLDER_SCOPES.filter((scope) => Boolean(folders[scope])), repositories: bindings.length } });
|
||||
}
|
||||
try {
|
||||
// Record intent before mutations. An unavailable audit store blocks new
|
||||
// work instead of turning an already completed save into a false failure.
|
||||
// The run's persisted state/lastSavedAt records checkpoint completion.
|
||||
await recordCheckpoint("work_folder.prepared");
|
||||
await seedAttachments();
|
||||
await importAgentFiles();
|
||||
// A resumed sandbox can hold edits newer than its last completed checkpoint.
|
||||
if (previous) for (const scope of WORK_FOLDER_SCOPES) await outgoing(scope);
|
||||
for (const scope of WORK_FOLDER_SCOPES) await incoming(scope);
|
||||
await prepareRepositories();
|
||||
await saveState("starting");
|
||||
if (previous?.refreshRequested) await db.update(workFolderRuns).set({ refreshRequested: false })
|
||||
.where(eq(workFolderRuns.runId, previous.runId));
|
||||
} catch (error) {
|
||||
await saveState("failed", "Work folder preparation failed; existing files were retained");
|
||||
throw error;
|
||||
}
|
||||
const checkpointer = startWorkFolderCheckpointer({
|
||||
async checkpoint() {
|
||||
await assertBindings();
|
||||
await recordCheckpoint("work_folder.checkpoint");
|
||||
await saveState("saving");
|
||||
for (const scope of WORK_FOLDER_SCOPES) await outgoing(scope);
|
||||
for (const { binding, root } of bindings) await repositories.checkpoint(binding, root);
|
||||
await saveState("saved");
|
||||
},
|
||||
async onError(error) {
|
||||
const failure = error as { name?: unknown; code?: unknown; $metadata?: { httpStatusCode?: unknown } } | null;
|
||||
// Do not log SDK request objects, headers, file contents or credentials.
|
||||
const label = (value: unknown) => typeof value === "string" && /^[A-Za-z0-9_]{1,80}$/.test(value) ? value : null;
|
||||
logger.warn({ runId: input.runId, errorName: label(failure?.name), errorCode: label(failure?.code),
|
||||
httpStatus: typeof failure?.$metadata?.httpStatusCode === "number" ? failure.$metadata.httpStatusCode : null },
|
||||
"Work folder checkpoint failed; retaining sandbox for recovery");
|
||||
await saveState("failed", "Files could not be saved; the sandbox must be retained for recovery");
|
||||
},
|
||||
});
|
||||
let completion: Promise<void> | null = null;
|
||||
function stop(beforeCompletion?: () => Promise<void>) {
|
||||
// Error teardown must observe the original outcome, including failures
|
||||
// after the data save. A later run owns any recovery of this working copy.
|
||||
completion ??= (async () => {
|
||||
await checkpointer.stop();
|
||||
const [run] = await db.select({ refreshRequested: workFolderRuns.refreshRequested }).from(workFolderRuns)
|
||||
.where(eq(workFolderRuns.runId, input.runId));
|
||||
if (run?.refreshRequested) {
|
||||
// The successful final flush protects edits before incoming refresh.
|
||||
try {
|
||||
const checkpointer = startWorkFolderCheckpointer({
|
||||
async checkpoint() {
|
||||
const checkpoint = () => measureSandboxOperation("work_folder.checkpoint", { phase: completion ? "final" : explicitFlushes ? "explicit" : "periodic" }, async () => {
|
||||
await assertBindings();
|
||||
for (const scope of WORK_FOLDER_SCOPES) await incoming(scope);
|
||||
await db.update(workFolderRuns).set({ refreshRequested: false, baselines, updatedAt: new Date() })
|
||||
.where(eq(workFolderRuns.runId, input.runId));
|
||||
} catch (error) {
|
||||
await saveState("failed", "Work folder refresh failed; existing files were retained");
|
||||
throw error;
|
||||
await recordCheckpoint("work_folder.checkpoint");
|
||||
await saveState("saving");
|
||||
for (const scope of WORK_FOLDER_SCOPES) await outgoing(scope);
|
||||
for (const [repositoryIndex, { binding, root }] of bindings.entries()) await measureSandboxOperation("work_folder.repository.checkpoint", { repositoryIndex }, async () => repositories.checkpoint(binding, root));
|
||||
await saveState("saved");
|
||||
});
|
||||
return completion || explicitFlushes ? checkpoint() : runInRunContext(checkpoint);
|
||||
},
|
||||
async onError(error) {
|
||||
const failure = error as { name?: unknown; code?: unknown; $metadata?: { httpStatusCode?: unknown } } | null;
|
||||
// Do not log SDK request objects, headers, file contents or credentials.
|
||||
const label = (value: unknown) => typeof value === "string" && /^[A-Za-z0-9_]{1,80}$/.test(value) ? value : null;
|
||||
logger.warn({ runId: input.runId, errorName: label(failure?.name), errorCode: label(failure?.code),
|
||||
httpStatus: typeof failure?.$metadata?.httpStatusCode === "number" ? failure.$metadata.httpStatusCode : null },
|
||||
"Work folder checkpoint failed; retaining sandbox for recovery");
|
||||
await saveState("failed", "Files could not be saved; the sandbox must be retained for recovery");
|
||||
},
|
||||
});
|
||||
let completion: Promise<void> | null = null;
|
||||
let explicitFlushes = 0;
|
||||
function stop(beforeCompletion?: () => Promise<void>) {
|
||||
// Error teardown must observe the original outcome, including failures
|
||||
// after the data save. A later run owns any recovery of this working copy.
|
||||
completion ??= measureSandboxOperation("work_folder.finalize", { phase: "final" }, async () => {
|
||||
await checkpointer.stop();
|
||||
const [run] = await measureSandboxOperation("work_folder.db.query", { operation: "select_work_folder_runs", requestCount: 1 }, async () => (db.select({ refreshRequested: workFolderRuns.refreshRequested }).from(workFolderRuns)
|
||||
.where(eq(workFolderRuns.runId, input.runId))));
|
||||
if (run?.refreshRequested) {
|
||||
// The successful final flush protects edits before incoming refresh.
|
||||
try {
|
||||
await assertBindings();
|
||||
for (const scope of WORK_FOLDER_SCOPES) await incoming(scope);
|
||||
await measureSandboxOperation("work_folder.db.query", { operation: "update_work_folder_runs", requestCount: 1 }, async () => (db.update(workFolderRuns).set({ refreshRequested: false, baselines, updatedAt: new Date() })
|
||||
.where(eq(workFolderRuns.runId, input.runId))));
|
||||
} catch (error) {
|
||||
await saveState("failed", "Work folder refresh failed; existing files were retained");
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Publish resume identity after data is durable, before releasing a turn.
|
||||
await beforeCompletion?.();
|
||||
manifest.finalCheckpointAt = new Date().toISOString();
|
||||
await saveState("saved");
|
||||
})();
|
||||
return completion;
|
||||
}
|
||||
return { manifest, home, identityChanged, primaryRepo: bindings.find(({ binding }) => manifest.repositories.some((repo) => repo.bindingId === binding.id && repo.primary))?.root ?? bindings[0]?.root ?? paths.task!,
|
||||
env: { HOME: home, AGENT_HOME: paths.agent!, PAPERCLIP_PRIMARY_REPO: bindings.find(({ binding }) => manifest.repositories.some((repo) => repo.bindingId === binding.id && repo.primary))?.root ?? bindings[0]?.root ?? paths.task!, PAPERCLIP_TASK_DIR: paths.task!, PAPERCLIP_AGENT_DIR: paths.agent!,
|
||||
PAPERCLIP_USER_DIR: paths.user!, PAPERCLIP_PROJECT_DIR: paths.project!, PAPERCLIP_REPOS_DIR: paths.repos! },
|
||||
flush: checkpointer.flush, stop };
|
||||
// Publish resume identity after data is durable, before releasing a turn.
|
||||
await beforeCompletion?.();
|
||||
manifest.finalCheckpointAt = new Date().toISOString();
|
||||
await saveState("saved");
|
||||
});
|
||||
return completion;
|
||||
}
|
||||
return { manifest, home, identityChanged, primaryRepo: bindings.find(({ binding }) => manifest.repositories.some((repo) => repo.bindingId === binding.id && repo.primary))?.root ?? bindings[0]?.root ?? paths.task!,
|
||||
env: { HOME: home, AGENT_HOME: paths.agent!, PAPERCLIP_PRIMARY_REPO: bindings.find(({ binding }) => manifest.repositories.some((repo) => repo.bindingId === binding.id && repo.primary))?.root ?? bindings[0]?.root ?? paths.task!, PAPERCLIP_TASK_DIR: paths.task!, PAPERCLIP_AGENT_DIR: paths.agent!,
|
||||
PAPERCLIP_USER_DIR: paths.user!, PAPERCLIP_PROJECT_DIR: paths.project!, PAPERCLIP_REPOS_DIR: paths.repos! },
|
||||
flush: () => measureSandboxOperation("work_folder.flush", { phase: "explicit" }, async () => {
|
||||
explicitFlushes++;
|
||||
try { await checkpointer.flush(); } finally { explicitFlushes--; }
|
||||
}), stop };
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,7 +6,30 @@ import os from "node:os";
|
|||
import { createHash, createHmac, randomUUID, timingSafeEqual } from "node:crypto";
|
||||
import { execFileSync } from "node:child_process";
|
||||
|
||||
let input = JSON.parse(Buffer.from(process.argv[1], "base64").toString("utf8"));
|
||||
const commandStartedAt = performance.now();
|
||||
const remotePhases = [];
|
||||
let performanceRequested = true;
|
||||
const metrics = { droppedPhases: 0, executionMs: 0, scanMs: 0, listMs: 0, gitListMs: 0, hashMs: 0, readMs: 0, writeMs: 0, publishMs: 0,
|
||||
decodeMs: 0, encodeMs: 0, files: 0, bytes: 0, hashFiles: 0, hashBytes: 0, requestCount: 0 };
|
||||
function measured(key, work) {
|
||||
if (!performanceRequested) return work();
|
||||
const startedAt = performance.now();
|
||||
try { return work(); } finally {
|
||||
const durationMs = performance.now() - startedAt;
|
||||
metrics[key] += durationMs;
|
||||
if (remotePhases.length < 256) remotePhases.push({ phase: key, startOffsetMs: startedAt - commandStartedAt, durationMs });
|
||||
else metrics.droppedPhases++;
|
||||
}
|
||||
}
|
||||
let input = measured("decodeMs", () => JSON.parse(Buffer.from(process.argv[1], "base64").toString("utf8")));
|
||||
performanceRequested = input.performance === true;
|
||||
function output(result) {
|
||||
const encoded = measured("encodeMs", () => JSON.stringify(result));
|
||||
metrics.executionMs = performance.now() - commandStartedAt;
|
||||
process.stdout.write(performanceRequested
|
||||
? `{"workFolderPerformanceVersion":1,"result":${encoded},"performance":${JSON.stringify(metrics)},"remotePhases":${JSON.stringify(remotePhases)}}`
|
||||
: encoded);
|
||||
}
|
||||
const MAX_CHUNK = 256 * 1024;
|
||||
const MAX_ENTRIES = 100_000;
|
||||
function safeRelative(value) {
|
||||
|
|
@ -51,18 +74,22 @@ function checked(target, directory = false) {
|
|||
}
|
||||
function children(target) {
|
||||
const fd = checked(target, true);
|
||||
try { return fs.readdirSync(process.platform === "linux" ? `/proc/self/fd/${fd}` : target).sort(); }
|
||||
try { return measured("listMs", () => fs.readdirSync(process.platform === "linux" ? `/proc/self/fd/${fd}` : target).sort()); }
|
||||
finally { fs.closeSync(fd); }
|
||||
}
|
||||
function checksum(target) {
|
||||
const fd = checked(target);
|
||||
try {
|
||||
const hash = createHash("sha256");
|
||||
const buffer = Buffer.alloc(MAX_CHUNK);
|
||||
let count;
|
||||
while ((count = fs.readSync(fd, buffer, 0, buffer.length, null)) > 0) hash.update(buffer.subarray(0, count));
|
||||
return hash.digest("hex");
|
||||
} finally { fs.closeSync(fd); }
|
||||
return measured("hashMs", () => {
|
||||
const fd = checked(target);
|
||||
try {
|
||||
const hash = createHash("sha256");
|
||||
const buffer = Buffer.alloc(MAX_CHUNK);
|
||||
let count;
|
||||
while ((count = fs.readSync(fd, buffer, 0, buffer.length, null)) > 0) {
|
||||
metrics.hashBytes += count; hash.update(buffer.subarray(0, count));
|
||||
}
|
||||
return hash.digest("hex");
|
||||
} finally { fs.closeSync(fd); metrics.hashFiles++; }
|
||||
});
|
||||
}
|
||||
function ensureDirectory(target) {
|
||||
withParent(target, false, (anchored) => {
|
||||
|
|
@ -110,8 +137,8 @@ function scan() {
|
|||
const gitDir = path.join(input.root, ".git");
|
||||
const fd = checked(gitDir, true); fs.closeSync(fd);
|
||||
if (fs.existsSync(path.join(gitDir, "objects/info/alternates"))) throw new Error("repository_is_not_independent");
|
||||
const files = execFileSync("git", ["-C", input.root, "ls-files", "--cached", "--others", "--exclude-standard", "-z"],
|
||||
{ encoding: "utf8", maxBuffer: 16 * 1024 * 1024 }).split("\0").filter(Boolean);
|
||||
const files = measured("gitListMs", () => execFileSync("git", ["-C", input.root, "ls-files", "--cached", "--others", "--exclude-standard", "-z"],
|
||||
{ encoding: "utf8", maxBuffer: 16 * 1024 * 1024 })).split("\0").filter(Boolean);
|
||||
for (const relative of [...new Set(files)].sort()) {
|
||||
// Git reports nested repositories with a trailing slash. Private runner
|
||||
// caches can contain them and must be excluded before path validation.
|
||||
|
|
@ -131,6 +158,7 @@ function scan() {
|
|||
|
||||
function execute(request) {
|
||||
input = request;
|
||||
metrics.requestCount++;
|
||||
let result;
|
||||
if (input.operation === "home") {
|
||||
result = { home: os.homedir() };
|
||||
|
|
@ -145,21 +173,22 @@ if (input.operation === "home") {
|
|||
})); result = {};
|
||||
} else {
|
||||
const rootFd = checked(input.root, true); fs.closeSync(rootFd);
|
||||
if (input.operation === "scan") result = scan();
|
||||
if (input.operation === "scan") result = measured("scanMs", () => scan());
|
||||
else if (input.operation === "read") {
|
||||
const fd = checked(full(input.path));
|
||||
try {
|
||||
const length = input.length ?? MAX_CHUNK;
|
||||
if (!Number.isSafeInteger(length) || length < 1 || length > 1024 * 1024) throw new Error("invalid_read_length");
|
||||
const buffer = Buffer.alloc(length);
|
||||
const count = fs.readSync(fd, buffer, 0, buffer.length, input.offset);
|
||||
result = { data: buffer.subarray(0, count).toString("base64") };
|
||||
const count = measured("readMs", () => fs.readSync(fd, buffer, 0, buffer.length, input.offset));
|
||||
metrics.files++; metrics.bytes += count;
|
||||
result = { data: measured("encodeMs", () => buffer.subarray(0, count).toString("base64")) };
|
||||
} finally { fs.closeSync(fd); }
|
||||
} else if (input.operation === "mkdir") {
|
||||
parents(input.path); ensureDirectory(full(input.path)); result = {};
|
||||
} else if (input.operation === "write") {
|
||||
parents(input.path);
|
||||
const buffer = Buffer.from(input.data, "base64");
|
||||
const buffer = measured("decodeMs", () => Buffer.from(input.data, "base64"));
|
||||
if (buffer.length > MAX_CHUNK) throw new Error("chunk_too_large");
|
||||
const target = full(input.path);
|
||||
// Temporary writes happen in a separate host-selected staging root.
|
||||
|
|
@ -170,7 +199,8 @@ if (input.operation === "home") {
|
|||
const stat = fs.fstatSync(fd);
|
||||
if (!stat.isFile() || stat.nlink !== 1) throw new Error("unsupported_file");
|
||||
if (stat.size !== input.offset) throw new Error("invalid_chunk_offset");
|
||||
fs.writeSync(fd, buffer, 0, buffer.length, input.offset);
|
||||
measured("writeMs", () => fs.writeSync(fd, buffer, 0, buffer.length, input.offset));
|
||||
metrics.files++; metrics.bytes += buffer.length;
|
||||
} finally { fs.closeSync(fd); }
|
||||
});
|
||||
result = {};
|
||||
|
|
@ -182,7 +212,7 @@ if (input.operation === "home") {
|
|||
try { fs.fchmodSync(sourceFd, input.executable ? 0o700 : 0o600); } finally { fs.closeSync(sourceFd); }
|
||||
const target = full(input.path);
|
||||
try { const fd = checked(target); fs.closeSync(fd); } catch (error) { if (error.code !== "ENOENT") throw error; }
|
||||
withParent(source, false, (from) => withParent(target, false, (to) => fs.renameSync(from, to))); result = {};
|
||||
measured("publishMs", () => withParent(source, false, (from) => withParent(target, false, (to) => fs.renameSync(from, to)))); result = {};
|
||||
} else if (input.operation === "symlink") {
|
||||
parents(input.path);
|
||||
const target = full(input.path);
|
||||
|
|
@ -314,9 +344,9 @@ if (request.operation === "read-batch") {
|
|||
}
|
||||
const results = request.entries.map((entry) => execute({ operation: "read", root: request.root,
|
||||
path: entry.path, offset: 0, length: Math.max(1, entry.byteSize) }));
|
||||
process.stdout.write(JSON.stringify(results));
|
||||
output(results);
|
||||
} else if (request.operation === "batch-status") {
|
||||
process.stdout.write(JSON.stringify(batchReceipt(request).read()));
|
||||
output(batchReceipt(request).read());
|
||||
} else if (request.operation === "batch") {
|
||||
// Stdin is bounded and hashed exactly, before parsing or claiming execution.
|
||||
const chunks = [];
|
||||
|
|
@ -331,7 +361,7 @@ if (request.operation === "read-batch") {
|
|||
const body = Buffer.concat(chunks);
|
||||
if (createHash("sha256").update(body).digest("hex") !== request.batchSha256) throw new Error("batch_body_hash_mismatch");
|
||||
const receipt = batchReceipt(request);
|
||||
const operations = JSON.parse(body.toString("utf8"));
|
||||
const operations = measured("decodeMs", () => JSON.parse(body.toString("utf8")));
|
||||
if (!Array.isArray(operations) || operations.length > 512) throw new Error("invalid_batch");
|
||||
for (const operation of operations) {
|
||||
if (!operation || !["write", "publish", "mkdir"].includes(operation.operation)) throw new Error("invalid_batch_operation");
|
||||
|
|
@ -339,8 +369,8 @@ if (request.operation === "read-batch") {
|
|||
}
|
||||
if (!receipt.claim()) {
|
||||
const status = receipt.read();
|
||||
if (status.state === "completed") process.stdout.write(JSON.stringify({ completed: status.completed }));
|
||||
else if (status.state === "running") process.stdout.write(JSON.stringify({ pending: true }));
|
||||
if (status.state === "completed") output({ completed: status.completed });
|
||||
else if (status.state === "running") output({ pending: true });
|
||||
else throw new Error(status.state === "failed" ? "batch_execution_failed" : "batch_receipt_disappeared");
|
||||
} else {
|
||||
try {
|
||||
|
|
@ -358,8 +388,8 @@ if (request.operation === "read-batch") {
|
|||
"hardlink_not_allowed", "chunk_too_large", "invalid_chunk_offset", "content_changed_during_transfer"];
|
||||
throw new Error(safeErrors.includes(error.message) ? error.message : "batch_execution_failed");
|
||||
}
|
||||
process.stdout.write(JSON.stringify({ completed: operations.length }));
|
||||
output({ completed: operations.length });
|
||||
}
|
||||
} else {
|
||||
process.stdout.write(JSON.stringify(execute(request)));
|
||||
output(execute(request));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import { measureSandboxOperation } from "./sandbox-performance.js";
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { agents, companyMemberships, heartbeatRuns, issues, projects, workFolderRuns, type Db } from "@paperclipai/db";
|
||||
import type { WorkFolderOwner } from "@paperclipai/shared";
|
||||
|
|
@ -6,55 +7,57 @@ import { authorizationService, type AuthorizationActor, type AuthorizationResour
|
|||
|
||||
/** Private-file access never inherits the responsible-user shadow-mode bypass. */
|
||||
export async function assertWorkFolderAccess(db: Db, actor: AuthorizationActor, owner: WorkFolderOwner, write: boolean) {
|
||||
const deny = () => { throw notFound("Work folder not found"); };
|
||||
if (actor.type === "none") deny();
|
||||
if (actor.type === "agent" && actor.companyId !== owner.companyId) deny();
|
||||
if (actor.type === "board" && actor.source !== "local_implicit" && !actor.companyIds?.includes(owner.companyId)) deny();
|
||||
const userId = actor.type === "board" ? actor.userId : actor.onBehalfOfUserId;
|
||||
if (actor.source !== "local_implicit" && userId) {
|
||||
const [membership] = await db.select().from(companyMemberships).where(and(
|
||||
eq(companyMemberships.companyId, owner.companyId), eq(companyMemberships.principalType, "user"),
|
||||
eq(companyMemberships.principalId, userId), eq(companyMemberships.status, "active")));
|
||||
if (!membership || (write && membership.membershipRole === "viewer")) deny();
|
||||
}
|
||||
if (owner.scope === "user") {
|
||||
if (!userId || userId !== owner.ownerId) deny();
|
||||
if (actor.type === "agent") {
|
||||
if (!actor.runId || !actor.agentId || actor.source !== "agent_jwt") deny();
|
||||
const [run] = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.id, actor.runId!),
|
||||
eq(heartbeatRuns.companyId, owner.companyId), eq(heartbeatRuns.agentId, actor.agentId!)));
|
||||
const [binding] = await db.select().from(workFolderRuns).where(eq(workFolderRuns.runId, actor.runId!));
|
||||
if (!run || run.status !== "running" || run.responsibleUserId !== owner.ownerId
|
||||
|| binding?.manifest.responsibleUserId !== owner.ownerId) deny();
|
||||
return measureSandboxOperation("work_folder.access", { scope: owner.scope, operation: write ? "write" : "read" }, async () => {
|
||||
const deny = () => { throw notFound("Work folder not found"); };
|
||||
if (actor.type === "none") deny();
|
||||
if (actor.type === "agent" && actor.companyId !== owner.companyId) deny();
|
||||
if (actor.type === "board" && actor.source !== "local_implicit" && !actor.companyIds?.includes(owner.companyId)) deny();
|
||||
const userId = actor.type === "board" ? actor.userId : actor.onBehalfOfUserId;
|
||||
if (actor.source !== "local_implicit" && userId) {
|
||||
const [membership] = await measureSandboxOperation("work_folder.db.query", { operation: "select_company_memberships", requestCount: 1 }, async () => (db.select().from(companyMemberships).where(and(
|
||||
eq(companyMemberships.companyId, owner.companyId), eq(companyMemberships.principalType, "user"),
|
||||
eq(companyMemberships.principalId, userId), eq(companyMemberships.status, "active")))));
|
||||
if (!membership || (write && membership.membershipRole === "viewer")) deny();
|
||||
}
|
||||
return;
|
||||
}
|
||||
let resource: AuthorizationResource;
|
||||
let action: "issue:read" | "issue:mutate" | "agent:read" | "agent_config:update" | "project:read";
|
||||
if (owner.scope === "task") {
|
||||
const [issue] = await db.select().from(issues).where(and(eq(issues.id, owner.ownerId), eq(issues.companyId, owner.companyId)));
|
||||
if (!issue) return deny();
|
||||
resource = { type: "issue", companyId: owner.companyId, issueId: issue.id, projectId: issue.projectId,
|
||||
parentIssueId: issue.parentId, assigneeAgentId: issue.assigneeAgentId, assigneeUserId: issue.assigneeUserId,
|
||||
originKind: issue.originKind, originId: issue.originId, status: issue.status };
|
||||
action = write ? "issue:mutate" : "issue:read";
|
||||
} else if (owner.scope === "agent") {
|
||||
const [agent] = await db.select().from(agents).where(and(eq(agents.id, owner.ownerId), eq(agents.companyId, owner.companyId)));
|
||||
if (!agent) return deny();
|
||||
if (write && actor.type === "agent" && actor.agentId !== agent.id) deny();
|
||||
resource = { type: "agent", companyId: owner.companyId, agentId: agent.id };
|
||||
action = "agent:read";
|
||||
} else {
|
||||
const [project] = await db.select().from(projects).where(and(eq(projects.id, owner.ownerId), eq(projects.companyId, owner.companyId)));
|
||||
if (!project) return deny();
|
||||
resource = { type: "project", companyId: owner.companyId, projectId: project.id };
|
||||
action = "project:read";
|
||||
}
|
||||
const authz = authorizationService(db);
|
||||
if (!(await authz.decide({ actor, action, resource })).allowed) deny();
|
||||
if (actor.type === "agent" && userId) {
|
||||
const responsibleActor: AuthorizationActor = { type: "board", source: "session", userId,
|
||||
companyIds: [owner.companyId], memberships: actor.onBehalfOfMemberships, ignoreInstanceAdmin: true };
|
||||
if (!(await authz.decide({ actor: responsibleActor, action, resource })).allowed) deny();
|
||||
}
|
||||
if (owner.scope === "user") {
|
||||
if (!userId || userId !== owner.ownerId) deny();
|
||||
if (actor.type === "agent") {
|
||||
if (!actor.runId || !actor.agentId || actor.source !== "agent_jwt") deny();
|
||||
const [run] = await measureSandboxOperation("work_folder.db.query", { operation: "select_heartbeat_runs", requestCount: 1 }, async () => (db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.id, actor.runId!),
|
||||
eq(heartbeatRuns.companyId, owner.companyId), eq(heartbeatRuns.agentId, actor.agentId!)))));
|
||||
const [binding] = await measureSandboxOperation("work_folder.db.query", { operation: "select_work_folder_runs", requestCount: 1 }, async () => (db.select().from(workFolderRuns).where(eq(workFolderRuns.runId, actor.runId!))));
|
||||
if (!run || run.status !== "running" || run.responsibleUserId !== owner.ownerId
|
||||
|| binding?.manifest.responsibleUserId !== owner.ownerId) deny();
|
||||
}
|
||||
return;
|
||||
}
|
||||
let resource: AuthorizationResource;
|
||||
let action: "issue:read" | "issue:mutate" | "agent:read" | "agent_config:update" | "project:read";
|
||||
if (owner.scope === "task") {
|
||||
const [issue] = await measureSandboxOperation("work_folder.db.query", { operation: "select_issues", requestCount: 1 }, async () => (db.select().from(issues).where(and(eq(issues.id, owner.ownerId), eq(issues.companyId, owner.companyId)))));
|
||||
if (!issue) return deny();
|
||||
resource = { type: "issue", companyId: owner.companyId, issueId: issue.id, projectId: issue.projectId,
|
||||
parentIssueId: issue.parentId, assigneeAgentId: issue.assigneeAgentId, assigneeUserId: issue.assigneeUserId,
|
||||
originKind: issue.originKind, originId: issue.originId, status: issue.status };
|
||||
action = write ? "issue:mutate" : "issue:read";
|
||||
} else if (owner.scope === "agent") {
|
||||
const [agent] = await measureSandboxOperation("work_folder.db.query", { operation: "select_agents", requestCount: 1 }, async () => (db.select().from(agents).where(and(eq(agents.id, owner.ownerId), eq(agents.companyId, owner.companyId)))));
|
||||
if (!agent) return deny();
|
||||
if (write && actor.type === "agent" && actor.agentId !== agent.id) deny();
|
||||
resource = { type: "agent", companyId: owner.companyId, agentId: agent.id };
|
||||
action = "agent:read";
|
||||
} else {
|
||||
const [project] = await measureSandboxOperation("work_folder.db.query", { operation: "select_projects", requestCount: 1 }, async () => (db.select().from(projects).where(and(eq(projects.id, owner.ownerId), eq(projects.companyId, owner.companyId)))));
|
||||
if (!project) return deny();
|
||||
resource = { type: "project", companyId: owner.companyId, projectId: project.id };
|
||||
action = "project:read";
|
||||
}
|
||||
const authz = authorizationService(db);
|
||||
if (!(await measureSandboxOperation("work_folder.authorization.decide", { scope: owner.scope, operation: write ? "write" : "read" }, async () => (authz.decide({ actor, action, resource })))).allowed) deny();
|
||||
if (actor.type === "agent" && userId) {
|
||||
const responsibleActor: AuthorizationActor = { type: "board", source: "session", userId,
|
||||
companyIds: [owner.companyId], memberships: actor.onBehalfOfMemberships, ignoreInstanceAdmin: true };
|
||||
if (!(await measureSandboxOperation("work_folder.authorization.decide", { scope: owner.scope, operation: write ? "write" : "read" }, async () => (authz.decide({ actor: responsibleActor, action, resource })))).allowed) deny();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import { measureSandboxOperation, measureSandboxStream } from "./sandbox-performance.js";
|
||||
import { constants } from "node:fs";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
|
@ -10,39 +11,39 @@ export interface ManagedAgentFile { path: string; kind: "file" | "directory"; ex
|
|||
export async function* managedAgentFiles(root: string): AsyncGenerator<ManagedAgentFile> {
|
||||
const absolute = path.resolve(root);
|
||||
let current = path.parse(absolute).root;
|
||||
let parent = await fs.open(current, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW);
|
||||
let parent = await measureSandboxOperation("work_folder.agent_import.disk", { operation: "open" }, async () => (fs.open(current, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW)));
|
||||
const anchored = (fd: number, fallback: string) => process.platform === "linux" ? `/proc/self/fd/${fd}` : fallback;
|
||||
try {
|
||||
for (const segment of absolute.slice(current.length).split(path.sep).filter(Boolean)) {
|
||||
const next = path.join(anchored(parent.fd, current), segment);
|
||||
const child = await fs.open(next, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW).catch((error) => {
|
||||
const child = await measureSandboxOperation("work_folder.agent_import.disk", { operation: "open" }, async () => (fs.open(next, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW).catch((error) => {
|
||||
if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
|
||||
throw error;
|
||||
});
|
||||
})));
|
||||
if (!child) return;
|
||||
await parent.close(); parent = child; current = path.join(current, segment);
|
||||
await measureSandboxOperation("work_folder.agent_import.disk", { operation: "close" }, async () => (parent.close())); parent = child; current = path.join(current, segment);
|
||||
}
|
||||
let entries = 0;
|
||||
async function* visit(directory: typeof parent, fallback: string, relative: string): AsyncGenerator<ManagedAgentFile> {
|
||||
for (const name of (await fs.readdir(anchored(directory.fd, fallback))).sort()) {
|
||||
for (const name of (await measureSandboxOperation("work_folder.agent_import.disk", { operation: "list" }, async () => (fs.readdir(anchored(directory.fd, fallback))))).sort()) {
|
||||
if (excluded.has(name)) continue;
|
||||
if (++entries > 100_000) throw new Error("Managed agent import exceeds its file limit");
|
||||
const filename = relative ? `${relative}/${name}` : name;
|
||||
const source = path.join(anchored(directory.fd, fallback), name);
|
||||
const handle = await fs.open(source, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
|
||||
const handle = await measureSandboxOperation("work_folder.agent_import.disk", { operation: "open" }, async () => (fs.open(source, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK)));
|
||||
try {
|
||||
const stat = await handle.stat();
|
||||
const stat = await measureSandboxOperation("work_folder.agent_import.disk", { operation: "metadata" }, async () => (handle.stat()));
|
||||
if (stat.isDirectory()) {
|
||||
yield { path: filename, kind: "directory", executable: false };
|
||||
yield* visit(handle, path.join(fallback, name), filename);
|
||||
} else if (stat.isFile() && stat.nlink === 1 && stat.size <= 1024 ** 3) {
|
||||
const body = handle.createReadStream({ autoClose: false, highWaterMark: 256 * 1024 });
|
||||
const body = measureSandboxStream("work_folder.agent_import.body", { fileIndex: entries - 1, bytes: stat.size }, handle.createReadStream({ autoClose: false, highWaterMark: 256 * 1024 }));
|
||||
try { yield { path: filename, kind: "file", executable: Boolean(stat.mode & 0o111), body }; }
|
||||
finally { body.destroy(); }
|
||||
} else throw new Error("Managed agent files cannot contain hard links or special files");
|
||||
} finally { await handle.close(); }
|
||||
} finally { await measureSandboxOperation("work_folder.agent_import.disk", { operation: "close" }, async () => (handle.close())); }
|
||||
}
|
||||
}
|
||||
yield* visit(parent, absolute, "");
|
||||
} finally { await parent.close(); }
|
||||
} finally { await measureSandboxOperation("work_folder.agent_import.disk", { operation: "close" }, async () => (parent.close())); }
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import { measureSandboxOperation } from "./sandbox-performance.js";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { workFolderObjects, workFiles, taskRepositoryBindings, type Db } from "@paperclipai/db";
|
||||
import type { StorageProvider } from "../storage/types.js";
|
||||
|
|
@ -6,39 +7,43 @@ import type { StorageProvider } from "../storage/types.js";
|
|||
export async function registerWorkFolderObject(db: Db, storage: StorageProvider, input: {
|
||||
objectKey: string; companyId: string; folderId?: string; repositoryBindingId?: string;
|
||||
}) {
|
||||
await db.insert(workFolderObjects).values({ ...input, provider: storage.id,
|
||||
deleteAfter: new Date(Date.now() + 24 * 60 * 60 * 1000) }).onConflictDoUpdate({
|
||||
target: workFolderObjects.objectKey,
|
||||
set: { deleteAfter: sql`case when ${workFolderObjects.deleteAfter} is null then null else ${new Date(Date.now() + 24 * 60 * 60 * 1000).toISOString()}::timestamptz end` },
|
||||
});
|
||||
return measureSandboxOperation("work_folder.object.register", { requestCount: 1 }, async () => {
|
||||
await measureSandboxOperation("work_folder.db.query", { operation: "insert_work_folder_objects", requestCount: 1 }, async () => (db.insert(workFolderObjects).values({ ...input, provider: storage.id,
|
||||
deleteAfter: new Date(Date.now() + 24 * 60 * 60 * 1000) }).onConflictDoUpdate({
|
||||
target: workFolderObjects.objectKey,
|
||||
set: { deleteAfter: sql`case when ${workFolderObjects.deleteAfter} is null then null else ${new Date(Date.now() + 24 * 60 * 60 * 1000).toISOString()}::timestamptz end` },
|
||||
})));
|
||||
});
|
||||
}
|
||||
|
||||
/** Bounded, retryable cleanup. Committed trash stays referenced until explicit purge. */
|
||||
export async function collectWorkFolderGarbage(db: Db, storage: StorageProvider, now = new Date(), limit = 100) {
|
||||
// Polymorphic owners cannot use a single foreign key. Permanent deletion is
|
||||
// detected against the authoritative owner tables, including auth users.
|
||||
await db.execute(sql`delete from work_folders f where
|
||||
(f.scope = 'task' and not exists (select 1 from issues i where i.id::text = f.owner_id and i.company_id = f.company_id)) or
|
||||
(f.scope = 'agent' and not exists (select 1 from agents a where a.id::text = f.owner_id and a.company_id = f.company_id)) or
|
||||
(f.scope = 'project' and not exists (select 1 from projects p where p.id::text = f.owner_id and p.company_id = f.company_id)) or
|
||||
(f.scope = 'user' and not exists (select 1 from "user" u where u.id = f.owner_id))`);
|
||||
let deleted = 0;
|
||||
await db.transaction(async (tx) => {
|
||||
const candidates = await tx.select().from(workFolderObjects).where(and(eq(workFolderObjects.provider, storage.id),
|
||||
sql`(${workFolderObjects.deleteAfter} <= ${now.toISOString()}::timestamptz or (${workFolderObjects.deleteAfter} is null and (
|
||||
(${workFolderObjects.folderId} is not null and not exists (select 1 from ${workFiles} where ${workFiles.objectKey} = ${workFolderObjects.objectKey})) or
|
||||
(${workFolderObjects.repositoryBindingId} is not null and not exists (select 1 from ${taskRepositoryBindings} where ${taskRepositoryBindings.id} = ${workFolderObjects.repositoryBindingId}))
|
||||
)))`,
|
||||
sql`not exists (select 1 from ${workFiles} where ${workFiles.objectKey} = ${workFolderObjects.objectKey})`,
|
||||
)).limit(Math.max(1, Math.min(limit, 1000))).for("update", { skipLocked: true });
|
||||
for (const object of candidates) {
|
||||
const prefix = object.folderId ? `${object.companyId}/work-folders/${object.folderId}/`
|
||||
: `${object.companyId}/task-repositories/${object.repositoryBindingId}/`;
|
||||
if (!object.objectKey.startsWith(prefix)) throw new Error("Work folder garbage ownership mismatch");
|
||||
await storage.deleteObject({ objectKey: object.objectKey });
|
||||
await tx.delete(workFolderObjects).where(eq(workFolderObjects.objectKey, object.objectKey));
|
||||
deleted++;
|
||||
}
|
||||
return measureSandboxOperation("work_folder.garbage.collect", { files: limit }, async () => {
|
||||
// Polymorphic owners cannot use a single foreign key. Permanent deletion is
|
||||
// detected against the authoritative owner tables, including auth users.
|
||||
await measureSandboxOperation("work_folder.db.query", { operation: "execute", requestCount: 1 }, async () => (db.execute(sql`delete from work_folders f where
|
||||
(f.scope = 'task' and not exists (select 1 from issues i where i.id::text = f.owner_id and i.company_id = f.company_id)) or
|
||||
(f.scope = 'agent' and not exists (select 1 from agents a where a.id::text = f.owner_id and a.company_id = f.company_id)) or
|
||||
(f.scope = 'project' and not exists (select 1 from projects p where p.id::text = f.owner_id and p.company_id = f.company_id)) or
|
||||
(f.scope = 'user' and not exists (select 1 from "user" u where u.id = f.owner_id))`)));
|
||||
let deleted = 0;
|
||||
await db.transaction(async (tx) => {
|
||||
const candidates = await measureSandboxOperation("work_folder.db.query", { operation: "select_work_folder_objects", requestCount: 1 }, async () => (tx.select().from(workFolderObjects).where(and(eq(workFolderObjects.provider, storage.id),
|
||||
sql`(${workFolderObjects.deleteAfter} <= ${now.toISOString()}::timestamptz or (${workFolderObjects.deleteAfter} is null and (
|
||||
(${workFolderObjects.folderId} is not null and not exists (select 1 from ${workFiles} where ${workFiles.objectKey} = ${workFolderObjects.objectKey})) or
|
||||
(${workFolderObjects.repositoryBindingId} is not null and not exists (select 1 from ${taskRepositoryBindings} where ${taskRepositoryBindings.id} = ${workFolderObjects.repositoryBindingId}))
|
||||
)))`,
|
||||
sql`not exists (select 1 from ${workFiles} where ${workFiles.objectKey} = ${workFolderObjects.objectKey})`,
|
||||
)).limit(Math.max(1, Math.min(limit, 1000))).for("update", { skipLocked: true })));
|
||||
for (const object of candidates) {
|
||||
const prefix = object.folderId ? `${object.companyId}/work-folders/${object.folderId}/`
|
||||
: `${object.companyId}/task-repositories/${object.repositoryBindingId}/`;
|
||||
if (!object.objectKey.startsWith(prefix)) throw new Error("Work folder garbage ownership mismatch");
|
||||
await measureSandboxOperation("work_folder.object.delete", { requestCount: 1 }, async () => (storage.deleteObject({ objectKey: object.objectKey })));
|
||||
await measureSandboxOperation("work_folder.db.query", { operation: "delete_work_folder_objects", requestCount: 1 }, async () => (tx.delete(workFolderObjects).where(eq(workFolderObjects.objectKey, object.objectKey))));
|
||||
deleted++;
|
||||
}
|
||||
});
|
||||
return { deleted };
|
||||
});
|
||||
return { deleted };
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import { captureSandboxPerformanceContext, measureSandboxOperation, measureSandboxStream } from "./sandbox-performance.js";
|
||||
import { Readable } from "node:stream";
|
||||
import type { WorkTreeEntry } from "./work-folder-transport.js";
|
||||
|
||||
|
|
@ -37,7 +38,7 @@ export function createWorkFolderReadCache(
|
|||
cached.delete(group); cached.set(group, result);
|
||||
return result;
|
||||
}
|
||||
result = Promise.resolve().then(() => load(group)).then((buffers) => {
|
||||
result = measureSandboxOperation("work_folder.read_cache.load", { files: group.length, bytes: group.reduce((sum, entry) => sum + entry.byteSize, 0) }, () => load(group)).then((buffers) => {
|
||||
if (buffers.length !== group.length || buffers.some((buffer, index) =>
|
||||
!Buffer.isBuffer(buffer) || buffer.length !== group[index]!.byteSize)) {
|
||||
throw new Error("Work folder batch content does not match its entries");
|
||||
|
|
@ -58,15 +59,17 @@ export function createWorkFolderReadCache(
|
|||
const location = locations.get(entry.path);
|
||||
// Mark each source request, even if its stream is never consumed. A retry
|
||||
// must reopen the physical file instead of replaying possibly stale bytes.
|
||||
return Readable.from((async function* () {
|
||||
const inContext = captureSandboxPerformanceContext();
|
||||
return measureSandboxStream("work_folder.read_cache.body", { bytes: entry.byteSize, repeated }, Readable.from((async function* () {
|
||||
if (cleared || repeated || !location) {
|
||||
const source = fallback(entry);
|
||||
const source = inContext(() => fallback(entry));
|
||||
try { yield* source; } finally { source.destroy(); }
|
||||
return;
|
||||
}
|
||||
const buffers = await getBatch(location.batch);
|
||||
const buffers = await inContext(() => measureSandboxOperation("work_folder.read_cache.lookup",
|
||||
{ cacheHit: cached.has(location.batch), files: location.batch.length }, async () => getBatch(location.batch)));
|
||||
yield buffers[location.index]!;
|
||||
})());
|
||||
})()));
|
||||
}
|
||||
function clear() {
|
||||
cleared = true;
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import { measureSandboxOperation, measureSandboxStream } from "./sandbox-performance.js";
|
||||
import { createWorkFolderReadCache } from "./work-folder-read-cache.js";
|
||||
import { prefetchWorkFiles } from "./work-folder-transfer.js";
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
|
|
@ -21,18 +22,23 @@ function signature(entries: WorkTreeEntry[]) {
|
|||
}
|
||||
|
||||
export function workFolderRepositoryService(db: Db, storage: StorageProvider, transport: WorkFolderTransport) {
|
||||
const indexes = new Map<string, number>();
|
||||
function repositoryIndex(binding: Binding) {
|
||||
if (!indexes.has(binding.id)) indexes.set(binding.id, indexes.size);
|
||||
return indexes.get(binding.id)!;
|
||||
}
|
||||
const knownByBinding = new Map<string, Set<string>>();
|
||||
async function checkpoint(binding: Binding, root: string) {
|
||||
// Another sandbox can have published since this coordinator loaded the
|
||||
// binding. Cache only the current complete checkpoint's protected objects.
|
||||
const [current] = await db.select().from(taskRepositoryBindings).where(and(
|
||||
eq(taskRepositoryBindings.id, binding.id), eq(taskRepositoryBindings.companyId, binding.companyId)));
|
||||
const [current] = await measureSandboxOperation("work_folder.repository.lookup", { scope: "repos" }, async () => db.select().from(taskRepositoryBindings).where(and(
|
||||
eq(taskRepositoryBindings.id, binding.id), eq(taskRepositoryBindings.companyId, binding.companyId))));
|
||||
if (!current) throw new Error("Repository owner was deleted during checkpoint");
|
||||
if (current.checkpointKey !== binding.checkpointKey) knownByBinding.delete(binding.id);
|
||||
Object.assign(binding, current);
|
||||
const startedAt = Date.now();
|
||||
const entries = await transport.scan(root, true);
|
||||
const digest = signature(entries);
|
||||
const digest = await measureSandboxOperation("work_folder.repository.signature", { files: entries.length }, async () => signature(entries));
|
||||
if (binding.checkpointSha256 === digest) return;
|
||||
const prefix = `${binding.companyId}/task-repositories/${binding.id}/`;
|
||||
if (!knownByBinding.has(binding.id) && binding.checkpointKey) await loadManifest(binding);
|
||||
|
|
@ -57,17 +63,20 @@ export function workFolderRepositoryService(db: Db, storage: StorageProvider, tr
|
|||
try {
|
||||
await Promise.all(Array.from({ length: Math.min(4, objects.length) }, async () => {
|
||||
while (!failure) {
|
||||
const object = objects[next++];
|
||||
const fileIndex = next++;
|
||||
const object = objects[fileIndex];
|
||||
if (!object) return;
|
||||
const [objectKey, entry] = object;
|
||||
try {
|
||||
await registerWorkFolderObject(db, storage, { objectKey, companyId: binding.companyId, repositoryBindingId: binding.id });
|
||||
await measureSandboxOperation("work_folder.repository.object_intent", { fileIndex }, () => registerWorkFolderObject(db, storage, { objectKey, companyId: binding.companyId, repositoryBindingId: binding.id }));
|
||||
if (failure) return;
|
||||
const { exists } = await storage.headObject({ objectKey });
|
||||
const { exists } = await measureSandboxOperation("work_folder.repository.object_head", { fileIndex, bytes: entry.byteSize, requestCount: 1 }, async (span) => {
|
||||
const result = await storage.headObject({ objectKey }); span.set({ exists: result.exists }); return result;
|
||||
});
|
||||
if (!exists && !failure) {
|
||||
await uploadWorkFolderObject(storage, { objectKey, contentType: "application/octet-stream",
|
||||
await measureSandboxOperation("work_folder.repository.object_upload", { fileIndex, bytes: entry.byteSize, parallelism: 4 }, () => uploadWorkFolderObject(storage, { objectKey, contentType: "application/octet-stream",
|
||||
contentLength: entry.byteSize, sha256: entry.sha256!,
|
||||
createSource: () => readCache ? readCache.read(entry) : transport.read(root, entry.path, entry.byteSize) });
|
||||
createSource: () => readCache ? readCache.read(entry) : transport.read(root, entry.path, entry.byteSize) }));
|
||||
}
|
||||
} catch (error) {
|
||||
// Stop scheduling after the first error, but drain the other workers
|
||||
|
|
@ -79,35 +88,39 @@ export function workFolderRepositoryService(db: Db, storage: StorageProvider, tr
|
|||
} finally { readCache?.clear(); }
|
||||
if (failure) throw failure.error;
|
||||
// Never publish a torn Git index/worktree snapshot as a completed save.
|
||||
if (signature(await transport.scan(root, true)) !== digest) throw new Error("Repository changed during checkpoint; retry required");
|
||||
await measureSandboxOperation("work_folder.repository.verify_snapshot", { files: entries.length }, async () => {
|
||||
const verifiedEntries = await transport.scan(root, true);
|
||||
const verifiedDigest = await measureSandboxOperation("work_folder.repository.signature", { files: verifiedEntries.length }, async () => signature(verifiedEntries));
|
||||
if (verifiedDigest !== digest) throw new Error("Repository changed during checkpoint; retry required");
|
||||
});
|
||||
const checkpointKey = `${prefix}checkpoints/${randomUUID()}.json`;
|
||||
const body = Buffer.from(JSON.stringify({ version: 1, bindingId: binding.id, files }));
|
||||
const body = await measureSandboxOperation("work_folder.repository.manifest_encode", { files: files.length }, async () => Buffer.from(JSON.stringify({ version: 1, bindingId: binding.id, files })));
|
||||
await registerWorkFolderObject(db, storage, { objectKey: checkpointKey, companyId: binding.companyId, repositoryBindingId: binding.id });
|
||||
await storage.putObject({ objectKey: checkpointKey, body, contentType: "application/json", contentLength: body.length });
|
||||
await measureSandboxOperation("work_folder.repository.manifest_upload", { bytes: body.length, requestCount: 1 }, () => storage.putObject({ objectKey: checkpointKey, body, contentType: "application/json", contentLength: body.length }));
|
||||
// Retired objects have a 24-hour grace period. A checkpoint must finish
|
||||
// within that window even when a competing run advances the pointer.
|
||||
if (Date.now() - startedAt > 60 * 60 * 1000) throw new Error("Repository checkpoint exceeded the one-hour save limit; retry required");
|
||||
await db.transaction(async (tx) => {
|
||||
const [owner] = await tx.select({ id: taskRepositoryBindings.id }).from(taskRepositoryBindings)
|
||||
.where(and(eq(taskRepositoryBindings.id, binding.id), eq(taskRepositoryBindings.companyId, binding.companyId))).for("update");
|
||||
await measureSandboxOperation("work_folder.repository.publish", { files: files.length }, () => db.transaction(async (tx) => {
|
||||
const [owner] = await measureSandboxOperation("work_folder.repository.publish_lock", {}, async () => tx.select({ id: taskRepositoryBindings.id }).from(taskRepositoryBindings)
|
||||
.where(and(eq(taskRepositoryBindings.id, binding.id), eq(taskRepositoryBindings.companyId, binding.companyId))).for("update"));
|
||||
if (!owner) throw new Error("Repository owner was deleted during checkpoint");
|
||||
// Retire superseded manifests and blobs in the same transaction that
|
||||
// protects ALL current objects and advances the complete-checkpoint pointer.
|
||||
// The grace period also lets an already-started restore finish safely.
|
||||
await tx.update(workFolderObjects).set({ deleteAfter: new Date(Date.now() + 24 * 60 * 60 * 1000) })
|
||||
.where(and(eq(workFolderObjects.repositoryBindingId, binding.id), eq(workFolderObjects.companyId, binding.companyId), isNull(workFolderObjects.deleteAfter)));
|
||||
await measureSandboxOperation("work_folder.repository.retire_objects", {}, async () => tx.update(workFolderObjects).set({ deleteAfter: new Date(Date.now() + 24 * 60 * 60 * 1000) })
|
||||
.where(and(eq(workFolderObjects.repositoryBindingId, binding.id), eq(workFolderObjects.companyId, binding.companyId), isNull(workFolderObjects.deleteAfter))));
|
||||
const published = [checkpointKey, ...new Set(files.flatMap((file) => file.objectKey ? [file.objectKey] : []))];
|
||||
for (let offset = 0; offset < published.length; offset += 1000) {
|
||||
const batch = published.slice(offset, offset + 1000);
|
||||
const protectedObjects = await tx.update(workFolderObjects).set({ deleteAfter: null })
|
||||
const protectedObjects = await measureSandboxOperation("work_folder.repository.protect_objects", { objects: batch.length, batchIndex: Math.floor(offset / 1000) }, async () => tx.update(workFolderObjects).set({ deleteAfter: null })
|
||||
.where(and(eq(workFolderObjects.repositoryBindingId, binding.id), inArray(workFolderObjects.objectKey, batch)))
|
||||
.returning({ key: workFolderObjects.objectKey });
|
||||
.returning({ key: workFolderObjects.objectKey }));
|
||||
if (protectedObjects.length !== batch.length) throw new Error("Repository objects expired during checkpoint; retry required");
|
||||
}
|
||||
const updated = await tx.update(taskRepositoryBindings).set({ checkpointKey, checkpointSha256: digest, checkpointAt: new Date() })
|
||||
.where(and(eq(taskRepositoryBindings.id, binding.id), eq(taskRepositoryBindings.companyId, binding.companyId))).returning({ id: taskRepositoryBindings.id });
|
||||
const updated = await measureSandboxOperation("work_folder.repository.publish_pointer", {}, async () => tx.update(taskRepositoryBindings).set({ checkpointKey, checkpointSha256: digest, checkpointAt: new Date() })
|
||||
.where(and(eq(taskRepositoryBindings.id, binding.id), eq(taskRepositoryBindings.companyId, binding.companyId))).returning({ id: taskRepositoryBindings.id }));
|
||||
if (!updated.length) throw new Error("Repository owner was deleted during checkpoint");
|
||||
});
|
||||
}));
|
||||
knownByBinding.set(binding.id, new Set(files.flatMap((file) => file.objectKey ? [file.objectKey] : [])));
|
||||
binding.checkpointKey = checkpointKey;
|
||||
binding.checkpointSha256 = digest;
|
||||
|
|
@ -117,15 +130,15 @@ export function workFolderRepositoryService(db: Db, storage: StorageProvider, tr
|
|||
if (!binding.checkpointKey) throw new Error("Repository checkpoint is missing");
|
||||
const prefix = `${binding.companyId}/task-repositories/${binding.id}/`;
|
||||
if (!binding.checkpointKey.startsWith(prefix)) throw new Error("Repository checkpoint ownership mismatch");
|
||||
const { stream } = await storage.getObject({ objectKey: binding.checkpointKey });
|
||||
const { stream } = await measureSandboxOperation("work_folder.repository.manifest_download", { requestCount: 1 }, () => storage.getObject({ objectKey: binding.checkpointKey! }));
|
||||
const chunks: Buffer[] = [];
|
||||
let length = 0;
|
||||
for await (const chunk of stream) {
|
||||
for await (const chunk of measureSandboxStream("work_folder.repository.manifest_body", {}, stream)) {
|
||||
length += chunk.length;
|
||||
if (length > 64 * 1024 * 1024) { stream.destroy(); throw new Error("Repository checkpoint manifest exceeds size limit"); }
|
||||
chunks.push(Buffer.from(chunk));
|
||||
}
|
||||
const manifest = checkpointSchema.parse(JSON.parse(Buffer.concat(chunks).toString("utf8")));
|
||||
const manifest = await measureSandboxOperation("work_folder.repository.manifest_decode", { bytes: length }, async () => checkpointSchema.parse(JSON.parse(Buffer.concat(chunks).toString("utf8"))));
|
||||
if (manifest.bindingId !== binding.id) throw new Error("Repository checkpoint ownership mismatch");
|
||||
for (const entry of manifest.files) {
|
||||
validateWorkFilePath(entry.path);
|
||||
|
|
@ -139,14 +152,14 @@ export function workFolderRepositoryService(db: Db, storage: StorageProvider, tr
|
|||
|
||||
async function restore(binding: Binding, root: string, stagingRoot: string) {
|
||||
if (!binding.checkpointKey) return false;
|
||||
const manifest = await loadManifest(binding);
|
||||
const manifest = await measureSandboxOperation("work_folder.repository.manifest_load", {}, () => loadManifest(binding));
|
||||
await transport.mkdirRoot(root);
|
||||
await transport.writeMany(root, stagingRoot, prefetchWorkFiles(
|
||||
manifest.files.filter((entry) => !entry.linkTarget), async (entry) => {
|
||||
manifest.files.filter((entry) => !entry.linkTarget), async (entry, fileIndex) => {
|
||||
if (entry.kind === "directory") return { entry };
|
||||
if (!entry.objectKey) throw new Error("Repository checkpoint file is missing");
|
||||
const result = await storage.getObject({ objectKey: entry.objectKey });
|
||||
return { entry, body: result.stream };
|
||||
const result = await measureSandboxOperation("work_folder.repository.object_download", { fileIndex, bytes: entry.byteSize, requestCount: 1 }, () => storage.getObject({ objectKey: entry.objectKey! }));
|
||||
return { entry, body: measureSandboxStream("work_folder.repository.object_body", { fileIndex, bytes: entry.byteSize }, result.stream) };
|
||||
}));
|
||||
// Restore links only after ordinary files. No transfer follows a link as
|
||||
// a parent, and symlink() still confines its target to this repository.
|
||||
|
|
@ -155,5 +168,10 @@ export function workFolderRepositoryService(db: Db, storage: StorageProvider, tr
|
|||
}
|
||||
return true;
|
||||
}
|
||||
return { checkpoint, restore };
|
||||
return {
|
||||
checkpoint: (binding: Binding, root: string) => measureSandboxOperation("work_folder.repository.checkpoint",
|
||||
{ scope: "repos", repositoryIndex: repositoryIndex(binding) }, () => checkpoint(binding, root)),
|
||||
restore: (binding: Binding, root: string, stagingRoot: string) => measureSandboxOperation("work_folder.repository.restore",
|
||||
{ scope: "repos", repositoryIndex: repositoryIndex(binding) }, () => restore(binding, root, stagingRoot)),
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,34 +1,40 @@
|
|||
import { measureSandboxOperation } from "./sandbox-performance.js";
|
||||
import type { WorkFileTransfer } from "./work-folder-transport.js";
|
||||
|
||||
// Open a few storage responses ahead without buffering their bodies. Drain all
|
||||
// pending opens on failure so abandoned HTTP response streams are closed too.
|
||||
export async function* prefetchWorkFiles<T>(
|
||||
entries: Iterable<T>,
|
||||
open: (entry: T) => Promise<WorkFileTransfer>,
|
||||
open: (entry: T, fileIndex: number) => Promise<WorkFileTransfer>,
|
||||
): AsyncGenerator<WorkFileTransfer> {
|
||||
type Result = { value: WorkFileTransfer } | { error: unknown };
|
||||
const iterator = entries[Symbol.iterator]();
|
||||
const pending: Array<Promise<Result>> = [];
|
||||
let opened = 0, consumed = 0;
|
||||
function enqueue() {
|
||||
const next = iterator.next();
|
||||
if (!next.done) pending.push(Promise.resolve().then(() => open(next.value))
|
||||
if (!next.done) {
|
||||
const fileIndex = opened++;
|
||||
pending.push(Promise.resolve().then(() => measureSandboxOperation("work_folder.prefetch.open", { fileIndex, parallelism: 4 }, async () => open(next.value, fileIndex)))
|
||||
.then((value): Result => {
|
||||
// A response can fail while queued, before its async iterator exists.
|
||||
// Keep that error handled; consuming the stream still throws it.
|
||||
value.body?.on("error", () => {});
|
||||
return { value };
|
||||
}, (error): Result => ({ error })));
|
||||
}
|
||||
}
|
||||
try {
|
||||
for (let i = 0; i < 4; i++) enqueue();
|
||||
while (pending.length) {
|
||||
const result = await pending.shift()!;
|
||||
const next = pending.shift()!;
|
||||
const result = await measureSandboxOperation("work_folder.prefetch.wait", { fileIndex: consumed++, parallelism: 4 }, async () => next);
|
||||
if ("error" in result) throw result.error;
|
||||
try { yield result.value; } finally { result.value.body?.destroy(); }
|
||||
enqueue();
|
||||
}
|
||||
} finally {
|
||||
for (const result of await Promise.all(pending)) {
|
||||
for (const result of await measureSandboxOperation("work_folder.prefetch.drain", { files: pending.length, parallelism: 4 }, async () => Promise.all(pending))) {
|
||||
if ("value" in result) result.value.body?.destroy();
|
||||
}
|
||||
iterator.return?.();
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import { captureSandboxPerformanceContext, hasSandboxPerformanceTrace, measureSandboxOperation, measureSandboxStream } from "./sandbox-performance.js";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { Readable } from "node:stream";
|
||||
import { createHash, randomBytes, randomUUID } from "node:crypto";
|
||||
|
|
@ -7,6 +8,18 @@ import { z } from "zod";
|
|||
import type { CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime";
|
||||
import { validateWorkFilePath } from "@paperclipai/shared";
|
||||
|
||||
const remoteNumber = z.number().finite().nonnegative();
|
||||
const remotePerformanceSchema = z.object({ result: z.unknown(), performance: z.object({
|
||||
executionMs: remoteNumber, scanMs: remoteNumber, listMs: remoteNumber, gitListMs: remoteNumber, hashMs: remoteNumber, readMs: remoteNumber,
|
||||
writeMs: remoteNumber, publishMs: remoteNumber, decodeMs: remoteNumber, encodeMs: remoteNumber,
|
||||
files: remoteNumber, bytes: remoteNumber, hashFiles: remoteNumber, hashBytes: remoteNumber,
|
||||
requestCount: remoteNumber, droppedPhases: remoteNumber,
|
||||
}), remotePhases: z.array(z.object({ phase: z.enum(["scanMs", "listMs", "gitListMs", "hashMs", "readMs", "writeMs", "publishMs", "decodeMs", "encodeMs"]),
|
||||
startOffsetMs: remoteNumber, durationMs: remoteNumber })).max(256) });
|
||||
|
||||
const remotePhaseNames = { scanMs: "scan", listMs: "list", gitListMs: "git_list", hashMs: "hash", readMs: "read",
|
||||
writeMs: "write", publishMs: "publish", decodeMs: "decode", encodeMs: "encode" } as const;
|
||||
|
||||
const entrySchema = z.object({ path: z.string().refine((value) => { try { validateWorkFilePath(value); return true; } catch { return false; } }),
|
||||
kind: z.enum(["file", "directory"]), byteSize: z.number().int().nonnegative().max(1024 ** 3),
|
||||
sha256: z.string().regex(/^[a-f0-9]{64}$/).nullable(), executable: z.boolean(), linkTarget: z.string().max(1024).optional() });
|
||||
|
|
@ -38,27 +51,53 @@ export function workFolderTransport(runner: CommandManagedRuntimeRunner) {
|
|||
// small-argv transport without assuming additional capabilities.
|
||||
const bulkStdin = Boolean(runner.syncIn && runner.syncOut) || runner.supportsSingleStreamStdinProgress === true;
|
||||
async function command(input: Record<string, unknown>, stdin?: string, deadline = Date.now() + 120_000): Promise<unknown> {
|
||||
source ??= readFile(new URL("./scripts/work-folder-io.mjs", import.meta.url), "utf8");
|
||||
const args = ["--input-type=module", "-e", await source, Buffer.from(JSON.stringify(input)).toString("base64")];
|
||||
const readOnly = ["home", "scan", "read", "read-batch", "batch-status"].includes(String(input.operation));
|
||||
let result;
|
||||
for (let attempt = 0; ; attempt++) {
|
||||
if (Date.now() >= deadline) throw new Error("Work folder transfer deadline exceeded");
|
||||
try {
|
||||
result = await runner.execute({ command: "node", args, ...(stdin === undefined ? {} : { stdin }), bypassSession: true,
|
||||
timeoutMs: Math.max(1, deadline - Date.now()) });
|
||||
break;
|
||||
} catch (error) {
|
||||
// A lost read response is safe to repeat. Staging writes, publishes and
|
||||
// moves may already have happened, so never replay them here.
|
||||
const waitMs = 250 * (attempt + 1);
|
||||
if (!readOnly || attempt >= 2 || !transientTransportFailure(error) || Date.now() + waitMs >= deadline) throw error;
|
||||
await delay(waitMs);
|
||||
const operation = String(input.operation);
|
||||
return measureSandboxOperation("work_folder.transport.command", { operation }, async () => {
|
||||
const args = await measureSandboxOperation("work_folder.transport.encode", { operation }, async (span) => {
|
||||
source ??= readFile(new URL("./scripts/work-folder-io.mjs", import.meta.url), "utf8");
|
||||
const encoded = Buffer.from(JSON.stringify({ ...input, performance: hasSandboxPerformanceTrace() })).toString("base64");
|
||||
span.set({ bytes: Buffer.byteLength(encoded), inputBytes: stdin === undefined ? 0 : Buffer.byteLength(stdin) });
|
||||
return ["--input-type=module", "-e", await source, encoded];
|
||||
});
|
||||
const readOnly = ["home", "scan", "read", "read-batch", "batch-status"].includes(operation);
|
||||
for (let attempt = 0; ; attempt++) {
|
||||
if (Date.now() >= deadline) throw new Error("Work folder transfer deadline exceeded");
|
||||
let executionReturned = false;
|
||||
try {
|
||||
return await measureSandboxOperation("work_folder.transport.roundtrip", { operation, attempt: attempt + 1, requestCount: 1 }, async (span) => {
|
||||
const startedAt = performance.now();
|
||||
const result = await runner.execute({ command: "node", args, ...(stdin === undefined ? {} : { stdin }), bypassSession: true,
|
||||
timeoutMs: Math.max(1, deadline - Date.now()) });
|
||||
executionReturned = true;
|
||||
const roundtripMs = performance.now() - startedAt;
|
||||
span.set({ roundtripMs, outputBytes: Buffer.byteLength(result.stdout) });
|
||||
if (result.exitCode !== 0 || result.timedOut) throw new Error(`Work folder ${operation} failed: ${result.stderr.slice(0, 1500)}`);
|
||||
return measureSandboxOperation("work_folder.transport.decode", { operation }, async () => {
|
||||
const decoded = JSON.parse(result.stdout);
|
||||
// Older helpers and runners retain the unwrapped result contract.
|
||||
if (decoded?.workFolderPerformanceVersion !== 1) return decoded;
|
||||
const validated = remotePerformanceSchema.safeParse(decoded);
|
||||
// Timing is diagnostic. A malformed timing envelope must not
|
||||
// discard a valid command result or change its retry semantics.
|
||||
if (!validated.success) { span.set({ dropped: 1 }); return decoded.result; }
|
||||
const details = validated.data;
|
||||
span.set({ ...details.performance, transportOverheadMs: Math.max(0, roundtripMs - details.performance.executionMs) });
|
||||
for (const phase of details.remotePhases) {
|
||||
span.recordRemotePhase(`work_folder.remote.${remotePhaseNames[phase.phase]}`, phase.startOffsetMs, phase.durationMs, { operation });
|
||||
}
|
||||
return details.result;
|
||||
});
|
||||
});
|
||||
} catch (error) {
|
||||
// Mutations may already have happened. Only reads repeat here.
|
||||
const waitMs = 250 * (attempt + 1);
|
||||
if (executionReturned || !readOnly || attempt >= 2 || !transientTransportFailure(error) || Date.now() + waitMs >= deadline) throw error;
|
||||
await measureSandboxOperation("work_folder.transport.backoff", { operation, attempt: attempt + 1, waitMs }, () => delay(waitMs));
|
||||
}
|
||||
}
|
||||
}
|
||||
if (result.exitCode !== 0 || result.timedOut) throw new Error(`Work folder ${String(input.operation)} failed: ${result.stderr.slice(0, 1500)}`);
|
||||
return JSON.parse(result.stdout);
|
||||
});
|
||||
}
|
||||
|
||||
async function writeBatch(root: string, stagingRoot: string, body: string) {
|
||||
const deadline = Date.now() + 120_000;
|
||||
const identity = { root, stagingRoot, batchId: randomUUID(),
|
||||
|
|
@ -88,14 +127,18 @@ export function workFolderTransport(runner: CommandManagedRuntimeRunner) {
|
|||
// An upstream error may have lost only the response. First inspect the
|
||||
// sandbox receipt. A claimed batch is never replayed; it must complete
|
||||
// or remain visibly recoverable when this bounded wait expires.
|
||||
const status = statusSchema.parse(await command({ operation: "batch-status", ...identity }, undefined, deadline));
|
||||
const status = await measureSandboxOperation("work_folder.transport.receipt_reconcile", { attempt: attempts }, async (span) => {
|
||||
const result = statusSchema.parse(await command({ operation: "batch-status", ...identity }, undefined, deadline));
|
||||
span.set({ operation: result.state });
|
||||
return result;
|
||||
});
|
||||
if (status.state === "completed") return { completed: status.completed };
|
||||
if (status.state === "failed") throw new Error(`Work folder batch failed: ${status.error}`);
|
||||
pending = status.state === "running";
|
||||
if (!pending && attempts >= 3) throw lastError ?? new Error("Work folder batch receipt is missing");
|
||||
const waitMs = pending ? 500 : 250 * attempts;
|
||||
if (Date.now() + waitMs >= deadline) break;
|
||||
await delay(waitMs);
|
||||
await measureSandboxOperation("work_folder.transport.receipt_wait", { waitMs, attempt: attempts }, () => delay(waitMs));
|
||||
}
|
||||
throw new Error("Work folder batch outcome is uncertain; retaining sandbox for recovery");
|
||||
}
|
||||
|
|
@ -104,20 +147,25 @@ export function workFolderTransport(runner: CommandManagedRuntimeRunner) {
|
|||
return result.home;
|
||||
}
|
||||
async function scan(root: string, repository = false) {
|
||||
return z.array(entrySchema).max(100_000).parse(await command({ operation: "scan", root, repository }));
|
||||
return measureSandboxOperation("work_folder.transport.scan", { repository }, async (span) => {
|
||||
const entries = z.array(entrySchema).max(100_000).parse(await command({ operation: "scan", root, repository }));
|
||||
span.set({ files: entries.length, bytes: entries.reduce((sum, entry) => sum + entry.byteSize, 0) });
|
||||
return entries;
|
||||
});
|
||||
}
|
||||
function read(root: string, filePath: string, byteSize: number) {
|
||||
validateWorkFilePath(filePath);
|
||||
return Readable.from((async function* () {
|
||||
const inContext = captureSandboxPerformanceContext();
|
||||
return measureSandboxStream("work_folder.transport.read_body", { bytes: byteSize }, Readable.from((async function* () {
|
||||
for (let offset = 0; offset < byteSize;) {
|
||||
const length = bulkStdin ? 1024 * 1024 : 256 * 1024;
|
||||
const result = z.object({ data: z.string().max(Math.ceil(length / 3) * 4) }).parse(await command({ operation: "read", root, path: filePath, offset, length }));
|
||||
const bytes = Buffer.from(result.data, "base64");
|
||||
const result = z.object({ data: z.string().max(Math.ceil(length / 3) * 4) }).parse(await inContext(() => command({ operation: "read", root, path: filePath, offset, length })));
|
||||
const bytes = await inContext(() => measureSandboxOperation("work_folder.transport.decode_bytes", { chunkIndex: Math.floor(offset / length) }, async () => Buffer.from(result.data, "base64")));
|
||||
if (bytes.length === 0 || offset + bytes.length > byteSize) throw new Error("Work file changed during transfer");
|
||||
offset += bytes.length;
|
||||
yield bytes;
|
||||
}
|
||||
})());
|
||||
})()));
|
||||
}
|
||||
async function readBatch(root: string, entries: WorkTreeEntry[]) {
|
||||
if (entries.length > 64) throw new Error("Work folder read batch exceeds entry limit");
|
||||
|
|
@ -132,20 +180,20 @@ export function workFolderTransport(runner: CommandManagedRuntimeRunner) {
|
|||
const result = z.array(z.object({ data: z.string().max(Math.ceil(1024 * 1024 / 3) * 4) })).max(64)
|
||||
.parse(await command({ operation: "read-batch", root, entries: entries.map(({ path, byteSize }) => ({ path, byteSize })) }));
|
||||
if (result.length !== entries.length) throw new Error("Work folder read batch did not complete");
|
||||
return result.map(({ data }, index) => {
|
||||
return measureSandboxOperation("work_folder.transport.decode_batch", { files: entries.length, bytes }, async () => result.map(({ data }, index) => {
|
||||
const buffer = Buffer.from(data, "base64");
|
||||
if (buffer.length !== entries[index]!.byteSize) throw new Error("Work file changed during transfer");
|
||||
return buffer;
|
||||
});
|
||||
}));
|
||||
}
|
||||
async function writeChunks(root: string, stagingRoot: string, entry: WorkTreeEntry, body: Readable) {
|
||||
const stagingPath = randomUUID();
|
||||
let offset = 0;
|
||||
for await (const value of body) {
|
||||
for await (const value of measureSandboxStream("work_folder.transport.incoming_body", { bytes: entry.byteSize }, body)) {
|
||||
const chunk = Buffer.isBuffer(value) ? value : Buffer.from(value);
|
||||
for (let start = 0; start < chunk.length; start += WRITE_CHUNK_BYTES) {
|
||||
const bytes = chunk.subarray(start, start + WRITE_CHUNK_BYTES);
|
||||
await command({ operation: "write", root: stagingRoot, path: stagingPath, offset, data: bytes.toString("base64") });
|
||||
await command({ operation: "write", root: stagingRoot, path: stagingPath, offset, data: await measureSandboxOperation("work_folder.transport.encode_bytes", { bytes: bytes.length }, async () => bytes.toString("base64")) });
|
||||
offset += bytes.length;
|
||||
}
|
||||
}
|
||||
|
|
@ -161,7 +209,7 @@ export function workFolderTransport(runner: CommandManagedRuntimeRunner) {
|
|||
let bufferedBytes = 0;
|
||||
async function flush() {
|
||||
if (!operations.length) return;
|
||||
const body = JSON.stringify(operations);
|
||||
const body = await measureSandboxOperation("work_folder.transport.encode_batch", { files: publishedEntries.length, bytes: bufferedBytes, requestCount: operations.length }, async () => JSON.stringify(operations));
|
||||
if (Buffer.byteLength(body) > 8 * 1024 * 1024) throw new Error("Work folder batch exceeds transfer limit");
|
||||
if (publishedEntries.length) await beforePublish?.(publishedEntries);
|
||||
const result = await writeBatch(root, stagingRoot, body);
|
||||
|
|
@ -194,12 +242,12 @@ export function workFolderTransport(runner: CommandManagedRuntimeRunner) {
|
|||
}
|
||||
const stagingPath = randomUUID();
|
||||
let offset = 0;
|
||||
for await (const value of body) {
|
||||
for await (const value of measureSandboxStream("work_folder.transport.incoming_body", { bytes: entry.byteSize }, body)) {
|
||||
const chunk = Buffer.isBuffer(value) ? value : Buffer.from(value);
|
||||
for (let start = 0; start < chunk.length; start += 256 * 1024) {
|
||||
const bytes = chunk.subarray(start, start + 256 * 1024);
|
||||
if (offset + bytes.length > entry.byteSize) throw new Error("Work file size changed during transfer");
|
||||
await append({ operation: "write", path: stagingPath, offset, data: bytes.toString("base64") }, bytes.length);
|
||||
await append({ operation: "write", path: stagingPath, offset, data: await measureSandboxOperation("work_folder.transport.encode_bytes", { bytes: bytes.length }, async () => bytes.toString("base64")) }, bytes.length);
|
||||
offset += bytes.length;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import { measureSandboxOperation } from "./sandbox-performance.js";
|
||||
import { createHash } from "node:crypto";
|
||||
import { Readable, Transform } from "node:stream";
|
||||
import { pipeline } from "node:stream/promises";
|
||||
|
|
@ -21,42 +22,46 @@ export async function uploadWorkFolderObject(storage: Pick<StorageProvider, "put
|
|||
sha256: string;
|
||||
createSource: () => Readable;
|
||||
}) {
|
||||
for (let attempt = 0; attempt < 3; attempt++) {
|
||||
const source = input.createSource();
|
||||
const hash = createHash("sha256");
|
||||
let bytes = 0;
|
||||
let validationError: Error | undefined;
|
||||
const changed = () => validationError ??= new Error("Work file changed during upload; retry the checkpoint");
|
||||
const verify = new Transform({
|
||||
transform(chunk: Buffer, _encoding, callback) {
|
||||
bytes += chunk.length;
|
||||
if (bytes > input.contentLength) return callback(changed());
|
||||
hash.update(chunk);
|
||||
callback(null, chunk);
|
||||
},
|
||||
flush(callback) {
|
||||
callback(bytes === input.contentLength && hash.digest("hex") === input.sha256 ? undefined : changed());
|
||||
},
|
||||
});
|
||||
const transferred = pipeline(source, verify);
|
||||
const uploaded = Promise.resolve().then(() => storage.putObject({
|
||||
objectKey: input.objectKey, contentType: input.contentType,
|
||||
contentLength: input.contentLength, body: verify,
|
||||
}));
|
||||
let failure: unknown;
|
||||
try {
|
||||
// Observe both promises immediately. An early HTTP success cannot publish
|
||||
// unvalidated content, and a failed request must release its source reader.
|
||||
await Promise.all([transferred, uploaded]);
|
||||
return;
|
||||
} catch (error) {
|
||||
failure = validationError ?? error;
|
||||
} finally {
|
||||
source.destroy();
|
||||
verify.destroy();
|
||||
await Promise.allSettled([transferred, uploaded]);
|
||||
return measureSandboxOperation("work_folder.object.upload", { bytes: input.contentLength }, async () => {
|
||||
for (let attempt = 0; attempt < 3; attempt++) {
|
||||
const source = input.createSource();
|
||||
const hash = createHash("sha256");
|
||||
let bytes = 0;
|
||||
let validationError: Error | undefined;
|
||||
const changed = () => validationError ??= new Error("Work file changed during upload; retry the checkpoint");
|
||||
const verify = new Transform({
|
||||
transform(chunk: Buffer, _encoding, callback) {
|
||||
bytes += chunk.length;
|
||||
if (bytes > input.contentLength) return callback(changed());
|
||||
hash.update(chunk);
|
||||
callback(null, chunk);
|
||||
},
|
||||
flush(callback) {
|
||||
callback(bytes === input.contentLength && hash.digest("hex") === input.sha256 ? undefined : changed());
|
||||
},
|
||||
});
|
||||
const transferred = measureSandboxOperation("work_folder.upload.consume", { attempt: attempt + 1, bytes: input.contentLength }, async (span) => {
|
||||
try { await pipeline(source, verify); } finally { span.set({ bytes }); }
|
||||
});
|
||||
const uploaded = Promise.resolve().then(() => measureSandboxOperation("work_folder.object.put", { attempt: attempt + 1, bytes: input.contentLength, requestCount: 1 }, async () => storage.putObject({
|
||||
objectKey: input.objectKey, contentType: input.contentType,
|
||||
contentLength: input.contentLength, body: verify,
|
||||
})));
|
||||
let failure: unknown;
|
||||
try {
|
||||
// Observe both promises immediately. An early HTTP success cannot publish
|
||||
// unvalidated content, and a failed request must release its source reader.
|
||||
await Promise.all([transferred, uploaded]);
|
||||
return;
|
||||
} catch (error) {
|
||||
failure = validationError ?? error;
|
||||
} finally {
|
||||
source.destroy();
|
||||
verify.destroy();
|
||||
await Promise.allSettled([transferred, uploaded]);
|
||||
}
|
||||
if (validationError || attempt === 2 || !transientUploadFailure(failure)) throw failure;
|
||||
await measureSandboxOperation("work_folder.upload.retry_wait", { attempt: attempt + 1 }, async () => delay(250 * (attempt + 1)));
|
||||
}
|
||||
if (validationError || attempt === 2 || !transientUploadFailure(failure)) throw failure;
|
||||
await delay(250 * (attempt + 1));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import { measureSandboxOperation, measureSandboxStream } from "./sandbox-performance.js";
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { createReadStream, createWriteStream } from "node:fs";
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
|
|
@ -30,55 +31,67 @@ function validPath(value: string) {
|
|||
/** The caller must authorize the folder owner before using this host-only service. */
|
||||
export function workFolderService(db: Db, storage: StorageProvider) {
|
||||
async function ensure(owner: WorkFolderOwner): Promise<Folder> {
|
||||
await db.insert(workFolders).values(owner).onConflictDoNothing();
|
||||
const [folder] = await db.select().from(workFolders).where(and(eq(workFolders.companyId, owner.companyId),
|
||||
eq(workFolders.scope, owner.scope), eq(workFolders.ownerId, owner.ownerId)));
|
||||
if (!folder) throw notFound("Work folder not found");
|
||||
return folder;
|
||||
return measureSandboxOperation("work_folder.scope.ensure", { scope: owner.scope }, async () => {
|
||||
await measureSandboxOperation("work_folder.db.query", { operation: "insert_work_folders", requestCount: 1 }, async () => (db.insert(workFolders).values(owner).onConflictDoNothing()));
|
||||
const [folder] = await measureSandboxOperation("work_folder.db.query", { operation: "select_work_folders", requestCount: 1 }, async () => (db.select().from(workFolders).where(and(eq(workFolders.companyId, owner.companyId),
|
||||
eq(workFolders.scope, owner.scope), eq(workFolders.ownerId, owner.ownerId)))));
|
||||
if (!folder) throw notFound("Work folder not found");
|
||||
return folder;
|
||||
});
|
||||
}
|
||||
|
||||
async function list(folder: Folder, options: { trash?: boolean; cursor?: string; limit?: number } = {}) {
|
||||
const limit = Math.max(1, Math.min(1000, options.limit ?? 200));
|
||||
const rows = await db.select().from(workFiles).where(and(eq(workFiles.folderId, folder.id),
|
||||
eq(workFiles.companyId, folder.companyId), options.trash ? isNotNull(workFiles.deletedAt) : isNull(workFiles.deletedAt),
|
||||
options.cursor ? gt(workFiles.id, options.cursor) : undefined)).orderBy(asc(workFiles.id)).limit(limit + 1);
|
||||
const [saved] = await db.select({ at: max(workFileOperations.createdAt) }).from(workFileOperations)
|
||||
.where(and(eq(workFileOperations.companyId, folder.companyId), eq(workFileOperations.folderId, folder.id)));
|
||||
return { id: folder.id, owner: { companyId: folder.companyId, scope: folder.scope, ownerId: folder.ownerId },
|
||||
lastOperationAt: saved?.at?.toISOString() ?? null,
|
||||
files: rows.slice(0, limit).map(workFileDto), nextCursor: rows.length > limit ? rows[limit - 1]!.id : null };
|
||||
return measureSandboxOperation("work_folder.metadata.list", { scope: folder.scope }, async (span) => {
|
||||
const limit = Math.max(1, Math.min(1000, options.limit ?? 200));
|
||||
const rows = await measureSandboxOperation("work_folder.db.query", { operation: "select_work_files", requestCount: 1 }, async () => (db.select().from(workFiles).where(and(eq(workFiles.folderId, folder.id),
|
||||
eq(workFiles.companyId, folder.companyId), options.trash ? isNotNull(workFiles.deletedAt) : isNull(workFiles.deletedAt),
|
||||
options.cursor ? gt(workFiles.id, options.cursor) : undefined)).orderBy(asc(workFiles.id)).limit(limit + 1)));
|
||||
const [saved] = await measureSandboxOperation("work_folder.db.query", { operation: "select_work_file_operations", requestCount: 1 }, async () => (db.select({ at: max(workFileOperations.createdAt) }).from(workFileOperations)
|
||||
.where(and(eq(workFileOperations.companyId, folder.companyId), eq(workFileOperations.folderId, folder.id)))));
|
||||
span.set({ files: Math.min(rows.length, limit) });
|
||||
return { id: folder.id, owner: { companyId: folder.companyId, scope: folder.scope, ownerId: folder.ownerId },
|
||||
lastOperationAt: saved?.at?.toISOString() ?? null,
|
||||
files: rows.slice(0, limit).map(workFileDto), nextCursor: rows.length > limit ? rows[limit - 1]!.id : null };
|
||||
});
|
||||
}
|
||||
|
||||
async function get(folder: Folder, filePath: string) {
|
||||
const [row] = await db.select().from(workFiles).where(and(eq(workFiles.companyId, folder.companyId),
|
||||
eq(workFiles.folderId, folder.id), eq(workFiles.path, validPath(filePath)), isNull(workFiles.deletedAt)));
|
||||
if (!row) throw notFound("Work file not found");
|
||||
return row;
|
||||
return measureSandboxOperation("work_folder.metadata.get", { scope: folder.scope }, async () => {
|
||||
const [row] = await measureSandboxOperation("work_folder.db.query", { operation: "select_work_files", requestCount: 1 }, async () => (db.select().from(workFiles).where(and(eq(workFiles.companyId, folder.companyId),
|
||||
eq(workFiles.folderId, folder.id), eq(workFiles.path, validPath(filePath)), isNull(workFiles.deletedAt)))));
|
||||
if (!row) throw notFound("Work file not found");
|
||||
return row;
|
||||
});
|
||||
}
|
||||
|
||||
async function content(folder: Folder, filePath: string) {
|
||||
const row = await get(folder, filePath);
|
||||
if (row.kind !== "file" || !row.objectKey) throw badRequest("Path is a directory");
|
||||
return { file: workFileDto(row), ...(await storage.getObject({ objectKey: row.objectKey })) };
|
||||
async function content(folder: Folder, filePath: string, fileIndex?: number) {
|
||||
return measureSandboxOperation("work_folder.content.open", { scope: folder.scope, ...(fileIndex === undefined ? {} : { fileIndex }) }, async () => {
|
||||
const row = await get(folder, filePath);
|
||||
if (row.kind !== "file" || !row.objectKey) throw badRequest("Path is a directory");
|
||||
const object = await measureSandboxOperation("work_folder.object.get_response", { scope: folder.scope, requestCount: 1, bytes: row.byteSize, ...(fileIndex === undefined ? {} : { fileIndex }) }, async () => storage.getObject({ objectKey: row.objectKey! }));
|
||||
return { file: workFileDto(row), ...object, stream: measureSandboxStream("work_folder.object.body", { scope: folder.scope, bytes: row.byteSize, ...(fileIndex === undefined ? {} : { fileIndex }) }, object.stream) };
|
||||
});
|
||||
}
|
||||
|
||||
async function mutate<T>(folder: Folder, operationId: string, fingerprint: string,
|
||||
apply: (tx: Parameters<Parameters<Db["transaction"]>[0]>[0]) => Promise<T>) {
|
||||
if (!operationId || operationId.length > 256) throw badRequest("An operation ID is required");
|
||||
return db.transaction(async (tx) => {
|
||||
// Serialize acceptance order across all app processes, including mkdir/delete races.
|
||||
const [locked] = await tx.select().from(workFolders).where(and(eq(workFolders.id, folder.id),
|
||||
eq(workFolders.companyId, folder.companyId))).for("update");
|
||||
if (!locked) throw notFound("Work folder not found");
|
||||
const [receipt] = await tx.select().from(workFileOperations).where(and(
|
||||
eq(workFileOperations.folderId, folder.id), eq(workFileOperations.operationId, operationId)));
|
||||
if (receipt) {
|
||||
if (receipt.fingerprint !== fingerprint) throw conflict("Operation ID was already used for different content");
|
||||
return { applied: false as const };
|
||||
}
|
||||
const result = await apply(tx);
|
||||
await tx.insert(workFileOperations).values({ companyId: folder.companyId, folderId: folder.id, operationId, fingerprint });
|
||||
return { applied: true as const, result };
|
||||
return measureSandboxOperation("work_folder.metadata.mutate", { scope: folder.scope }, async () => {
|
||||
if (!operationId || operationId.length > 256) throw badRequest("An operation ID is required");
|
||||
return db.transaction(async (tx) => {
|
||||
// Serialize acceptance order across all app processes, including mkdir/delete races.
|
||||
const [locked] = await measureSandboxOperation("work_folder.db.query", { operation: "select_work_folders", requestCount: 1 }, async () => (tx.select().from(workFolders).where(and(eq(workFolders.id, folder.id),
|
||||
eq(workFolders.companyId, folder.companyId))).for("update")));
|
||||
if (!locked) throw notFound("Work folder not found");
|
||||
const [receipt] = await measureSandboxOperation("work_folder.db.query", { operation: "select_work_file_operations", requestCount: 1 }, async () => (tx.select().from(workFileOperations).where(and(
|
||||
eq(workFileOperations.folderId, folder.id), eq(workFileOperations.operationId, operationId)))));
|
||||
if (receipt) {
|
||||
if (receipt.fingerprint !== fingerprint) throw conflict("Operation ID was already used for different content");
|
||||
return { applied: false as const };
|
||||
}
|
||||
const result = await apply(tx);
|
||||
await measureSandboxOperation("work_folder.db.query", { operation: "insert_work_file_operations", requestCount: 1 }, async () => (tx.insert(workFileOperations).values({ companyId: folder.companyId, folderId: folder.id, operationId, fingerprint })));
|
||||
return { applied: true as const, result };
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -87,127 +100,132 @@ export function workFolderService(db: Db, storage: StorageProvider) {
|
|||
kind?: "file" | "directory"; operationId: string; maxBytes?: number; expectedSha256?: string | null;
|
||||
replaceKind?: boolean; onlyIfMissing?: boolean;
|
||||
}) {
|
||||
const filePath = validPath(input.path);
|
||||
const kind = input.kind ?? "file";
|
||||
const directory = await mkdtemp(path.join(os.tmpdir(), "paperclip-work-file-"));
|
||||
const spool = path.join(directory, "content");
|
||||
const hash = createHash("sha256");
|
||||
let byteSize = 0;
|
||||
let sha256: string | null = null;
|
||||
let objectKey: string | null = null;
|
||||
let discardUpload = false;
|
||||
try {
|
||||
if (kind === "file") {
|
||||
const source = Buffer.isBuffer(input.body) ? Readable.from([input.body]) : input.body ?? Readable.from([]);
|
||||
await pipeline(source, new Transform({ transform(chunk: Buffer, _encoding, callback) {
|
||||
byteSize += chunk.length;
|
||||
if (byteSize > (input.maxBytes ?? MAX_WORK_FILE_BYTES)) return callback(payloadTooLarge("Work file exceeds the size limit"));
|
||||
hash.update(chunk);
|
||||
callback(null, chunk);
|
||||
} }), createWriteStream(spool, { mode: 0o600 }));
|
||||
sha256 = hash.digest("hex");
|
||||
return measureSandboxOperation("work_folder.content.write", { scope: folder.scope }, async (span) => {
|
||||
const filePath = validPath(input.path);
|
||||
const kind = input.kind ?? "file";
|
||||
const directory = await measureSandboxOperation("work_folder.spool.create", { scope: folder.scope }, async () => mkdtemp(path.join(os.tmpdir(), "paperclip-work-file-")));
|
||||
const spool = path.join(directory, "content");
|
||||
const hash = createHash("sha256");
|
||||
let byteSize = 0;
|
||||
let sha256: string | null = null;
|
||||
let objectKey: string | null = null;
|
||||
let discardUpload = false;
|
||||
try {
|
||||
if (kind === "file") {
|
||||
const source = Buffer.isBuffer(input.body) ? Readable.from([input.body]) : input.body ?? Readable.from([]);
|
||||
await measureSandboxOperation("work_folder.spool.consume", { scope: folder.scope }, async (span) => {
|
||||
try { await pipeline(source, new Transform({ transform(chunk: Buffer, _encoding, callback) {
|
||||
byteSize += chunk.length;
|
||||
if (byteSize > (input.maxBytes ?? MAX_WORK_FILE_BYTES)) return callback(payloadTooLarge("Work file exceeds the size limit"));
|
||||
hash.update(chunk);
|
||||
callback(null, chunk);
|
||||
} }), createWriteStream(spool, { mode: 0o600 })); } finally { span.set({ bytes: byteSize }); }
|
||||
});
|
||||
sha256 = hash.digest("hex");
|
||||
if (input.expectedSha256 && input.expectedSha256 !== sha256) throw conflict("File changed during transfer; retry the checkpoint");
|
||||
objectKey = `${folder.companyId}/work-folders/${folder.id}/${randomUUID()}`;
|
||||
await registerWorkFolderObject(db, storage, { objectKey, companyId: folder.companyId, folderId: folder.id });
|
||||
await uploadWorkFolderObject(storage, { objectKey, createSource: () => createReadStream(spool),
|
||||
contentLength: byteSize, sha256, contentType: input.contentType ?? "application/octet-stream" });
|
||||
}
|
||||
if (input.expectedSha256 && input.expectedSha256 !== sha256) throw conflict("File changed during transfer; retry the checkpoint");
|
||||
objectKey = `${folder.companyId}/work-folders/${folder.id}/${randomUUID()}`;
|
||||
await registerWorkFolderObject(db, storage, { objectKey, companyId: folder.companyId, folderId: folder.id });
|
||||
await uploadWorkFolderObject(storage, { objectKey, createSource: () => createReadStream(spool),
|
||||
contentLength: byteSize, sha256, contentType: input.contentType ?? "application/octet-stream" });
|
||||
const value = { kind, objectKey, byteSize, sha256, executable: input.executable ?? false,
|
||||
contentType: input.contentType ?? "application/octet-stream", updatedAt: new Date() };
|
||||
const fingerprint = JSON.stringify(["write", filePath, kind, sha256, value.executable, value.contentType, Boolean(input.replaceKind), Boolean(input.onlyIfMissing)]);
|
||||
const result = await mutate(folder, input.operationId, fingerprint, async (tx) => {
|
||||
const parts = filePath.split("/");
|
||||
for (let i = 1; i < parts.length; i++) {
|
||||
const parent = parts.slice(0, i).join("/");
|
||||
const [existing] = await measureSandboxOperation("work_folder.db.query", { operation: "select_work_files", requestCount: 1 }, async () => (tx.select().from(workFiles).where(and(eq(workFiles.folderId, folder.id),
|
||||
eq(workFiles.path, parent), isNull(workFiles.deletedAt)))));
|
||||
if (existing && existing.kind !== "directory") throw conflict("A parent path is a file");
|
||||
if (!existing) await measureSandboxOperation("work_folder.db.query", { operation: "insert_work_files", requestCount: 1 }, async () => (tx.insert(workFiles).values({ companyId: folder.companyId, folderId: folder.id,
|
||||
path: parent, kind: "directory" })));
|
||||
}
|
||||
const previousRows = await measureSandboxOperation("work_folder.db.query", { operation: "select_work_files", requestCount: 1 }, async () => (tx.select().from(workFiles).where(and(eq(workFiles.folderId, folder.id),
|
||||
eq(workFiles.path, filePath), isNull(workFiles.deletedAt)))));
|
||||
let previous: FileRow | undefined = previousRows[0];
|
||||
if (previous && input.onlyIfMissing) return { oldKey: null, unusedUpload: true };
|
||||
if (previous && previous.kind !== kind) {
|
||||
if (!input.replaceKind) throw conflict("Delete the existing path before changing its kind");
|
||||
const prefix = `${filePath}/`;
|
||||
await measureSandboxOperation("work_folder.db.query", { operation: "update_work_files", requestCount: 1 }, async () => (tx.update(workFiles).set({ deletedAt: new Date(), updatedAt: new Date() }).where(and(
|
||||
eq(workFiles.folderId, folder.id), isNull(workFiles.deletedAt),
|
||||
sql`(${workFiles.path} = ${filePath} or left(${workFiles.path}, ${prefix.length}) = ${prefix})`))));
|
||||
previous = undefined;
|
||||
}
|
||||
if (previous) {
|
||||
await measureSandboxOperation("work_folder.db.query", { operation: "update_work_files", requestCount: 1 }, async () => (tx.update(workFiles).set(value).where(eq(workFiles.id, previous.id))));
|
||||
} else {
|
||||
await measureSandboxOperation("work_folder.db.query", { operation: "insert_work_files", requestCount: 1 }, async () => (tx.insert(workFiles).values({ ...value, companyId: folder.companyId, folderId: folder.id, path: filePath })));
|
||||
}
|
||||
if (objectKey) await measureSandboxOperation("work_folder.db.query", { operation: "update_work_folder_objects", requestCount: 1 }, async () => (tx.update(workFolderObjects).set({ deleteAfter: null }).where(eq(workFolderObjects.objectKey, objectKey!))));
|
||||
if (previous?.objectKey) await measureSandboxOperation("work_folder.db.query", { operation: "update_work_folder_objects", requestCount: 1 }, async () => (tx.update(workFolderObjects).set({ deleteAfter: new Date() }).where(eq(workFolderObjects.objectKey, previous!.objectKey!))));
|
||||
return { oldKey: previous?.objectKey ?? null, unusedUpload: false };
|
||||
});
|
||||
discardUpload = !result.applied || result.result.unusedUpload;
|
||||
// Object keys are private to this service; receipts contain no overwritten content.
|
||||
// Cleanup is journaled in the same transaction as replacement. Storage
|
||||
// outages and lost commit replies cannot orphan the only current object.
|
||||
return { applied: result.applied };
|
||||
} finally {
|
||||
// A lost database response can mean COMMIT succeeded. Retain an uncertain
|
||||
// upload for reconciliation; deleting it here could destroy saved content.
|
||||
if (objectKey && discardUpload) await measureSandboxOperation("work_folder.db.query", { operation: "update_work_folder_objects", requestCount: 1 }, async () => (db.update(workFolderObjects).set({ deleteAfter: new Date() }).where(eq(workFolderObjects.objectKey, objectKey!))));
|
||||
span.set({ bytes: byteSize });
|
||||
await measureSandboxOperation("work_folder.spool.cleanup", { scope: folder.scope }, async () => rm(directory, { recursive: true, force: true }));
|
||||
}
|
||||
if (input.expectedSha256 && input.expectedSha256 !== sha256) throw conflict("File changed during transfer; retry the checkpoint");
|
||||
const value = { kind, objectKey, byteSize, sha256, executable: input.executable ?? false,
|
||||
contentType: input.contentType ?? "application/octet-stream", updatedAt: new Date() };
|
||||
const fingerprint = JSON.stringify(["write", filePath, kind, sha256, value.executable, value.contentType, Boolean(input.replaceKind), Boolean(input.onlyIfMissing)]);
|
||||
const result = await mutate(folder, input.operationId, fingerprint, async (tx) => {
|
||||
const parts = filePath.split("/");
|
||||
for (let i = 1; i < parts.length; i++) {
|
||||
const parent = parts.slice(0, i).join("/");
|
||||
const [existing] = await tx.select().from(workFiles).where(and(eq(workFiles.folderId, folder.id),
|
||||
eq(workFiles.path, parent), isNull(workFiles.deletedAt)));
|
||||
if (existing && existing.kind !== "directory") throw conflict("A parent path is a file");
|
||||
if (!existing) await tx.insert(workFiles).values({ companyId: folder.companyId, folderId: folder.id,
|
||||
path: parent, kind: "directory" });
|
||||
}
|
||||
const previousRows = await tx.select().from(workFiles).where(and(eq(workFiles.folderId, folder.id),
|
||||
eq(workFiles.path, filePath), isNull(workFiles.deletedAt)));
|
||||
let previous: FileRow | undefined = previousRows[0];
|
||||
if (previous && input.onlyIfMissing) return { oldKey: null, unusedUpload: true };
|
||||
if (previous && previous.kind !== kind) {
|
||||
if (!input.replaceKind) throw conflict("Delete the existing path before changing its kind");
|
||||
const prefix = `${filePath}/`;
|
||||
await tx.update(workFiles).set({ deletedAt: new Date(), updatedAt: new Date() }).where(and(
|
||||
eq(workFiles.folderId, folder.id), isNull(workFiles.deletedAt),
|
||||
sql`(${workFiles.path} = ${filePath} or left(${workFiles.path}, ${prefix.length}) = ${prefix})`));
|
||||
previous = undefined;
|
||||
}
|
||||
if (previous) {
|
||||
await tx.update(workFiles).set(value).where(eq(workFiles.id, previous.id));
|
||||
} else {
|
||||
await tx.insert(workFiles).values({ ...value, companyId: folder.companyId, folderId: folder.id, path: filePath });
|
||||
}
|
||||
if (objectKey) await tx.update(workFolderObjects).set({ deleteAfter: null }).where(eq(workFolderObjects.objectKey, objectKey));
|
||||
if (previous?.objectKey) await tx.update(workFolderObjects).set({ deleteAfter: new Date() }).where(eq(workFolderObjects.objectKey, previous.objectKey));
|
||||
return { oldKey: previous?.objectKey ?? null, unusedUpload: false };
|
||||
});
|
||||
discardUpload = !result.applied || result.result.unusedUpload;
|
||||
// Object keys are private to this service; receipts contain no overwritten content.
|
||||
// Cleanup is journaled in the same transaction as replacement. Storage
|
||||
// outages and lost commit replies cannot orphan the only current object.
|
||||
return { applied: result.applied };
|
||||
} finally {
|
||||
// A lost database response can mean COMMIT succeeded. Retain an uncertain
|
||||
// upload for reconciliation; deleting it here could destroy saved content.
|
||||
if (objectKey && discardUpload) await db.update(workFolderObjects).set({ deleteAfter: new Date() }).where(eq(workFolderObjects.objectKey, objectKey));
|
||||
await rm(directory, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async function remove(folder: Folder, filePath: string, operationId: string) {
|
||||
const normalized = validPath(filePath);
|
||||
return mutate(folder, operationId, JSON.stringify(["delete", normalized]), async (tx) => {
|
||||
const prefix = `${normalized}/`;
|
||||
await tx.update(workFiles).set({ deletedAt: new Date(), updatedAt: new Date() }).where(and(
|
||||
await measureSandboxOperation("work_folder.db.query", { operation: "update_work_files", requestCount: 1 }, async () => (tx.update(workFiles).set({ deletedAt: new Date(), updatedAt: new Date() }).where(and(
|
||||
eq(workFiles.folderId, folder.id), isNull(workFiles.deletedAt),
|
||||
sql`(${workFiles.path} = ${normalized} or left(${workFiles.path}, ${prefix.length}) = ${prefix})`));
|
||||
sql`(${workFiles.path} = ${normalized} or left(${workFiles.path}, ${prefix.length}) = ${prefix})`))));
|
||||
});
|
||||
}
|
||||
|
||||
async function restore(folder: Folder, fileId: string, operationId: string) {
|
||||
return mutate(folder, operationId, JSON.stringify(["restore", fileId]), async (tx) => {
|
||||
const [deleted] = await tx.select().from(workFiles).where(and(eq(workFiles.folderId, folder.id),
|
||||
eq(workFiles.id, fileId), isNotNull(workFiles.deletedAt)));
|
||||
const [deleted] = await measureSandboxOperation("work_folder.db.query", { operation: "select_work_files", requestCount: 1 }, async () => (tx.select().from(workFiles).where(and(eq(workFiles.folderId, folder.id),
|
||||
eq(workFiles.id, fileId), isNotNull(workFiles.deletedAt)))));
|
||||
if (!deleted) throw notFound("Deleted file not found");
|
||||
const prefix = `${deleted.path}/`;
|
||||
const restoreRows = deleted.kind === "directory" ? await tx.select().from(workFiles).where(and(
|
||||
const restoreRows = deleted.kind === "directory" ? await measureSandboxOperation("work_folder.db.query", { operation: "select_work_files", requestCount: 1 }, async () => (tx.select().from(workFiles).where(and(
|
||||
eq(workFiles.folderId, folder.id), eq(workFiles.deletedAt, deleted.deletedAt!),
|
||||
sql`(${workFiles.path} = ${deleted.path} or left(${workFiles.path}, ${prefix.length}) = ${prefix})`)) : [deleted];
|
||||
sql`(${workFiles.path} = ${deleted.path} or left(${workFiles.path}, ${prefix.length}) = ${prefix})`)))) : [deleted];
|
||||
for (const row of restoreRows) {
|
||||
const [occupied] = await tx.select().from(workFiles).where(and(eq(workFiles.folderId, folder.id),
|
||||
eq(workFiles.path, row.path), isNull(workFiles.deletedAt)));
|
||||
const [occupied] = await measureSandboxOperation("work_folder.db.query", { operation: "select_work_files", requestCount: 1 }, async () => (tx.select().from(workFiles).where(and(eq(workFiles.folderId, folder.id),
|
||||
eq(workFiles.path, row.path), isNull(workFiles.deletedAt)))));
|
||||
if (occupied) throw conflict("Delete the current file before restoring this deleted copy");
|
||||
}
|
||||
const parts = deleted.path.split("/");
|
||||
for (let i = 1; i < parts.length; i++) {
|
||||
const parent = parts.slice(0, i).join("/");
|
||||
const [existing] = await tx.select().from(workFiles).where(and(eq(workFiles.folderId, folder.id),
|
||||
eq(workFiles.path, parent), isNull(workFiles.deletedAt)));
|
||||
const [existing] = await measureSandboxOperation("work_folder.db.query", { operation: "select_work_files", requestCount: 1 }, async () => (tx.select().from(workFiles).where(and(eq(workFiles.folderId, folder.id),
|
||||
eq(workFiles.path, parent), isNull(workFiles.deletedAt)))));
|
||||
if (existing && existing.kind !== "directory") throw conflict("A parent path is a file");
|
||||
if (!existing) await tx.insert(workFiles).values({ companyId: folder.companyId, folderId: folder.id,
|
||||
path: parent, kind: "directory" });
|
||||
if (!existing) await measureSandboxOperation("work_folder.db.query", { operation: "insert_work_files", requestCount: 1 }, async () => (tx.insert(workFiles).values({ companyId: folder.companyId, folderId: folder.id,
|
||||
path: parent, kind: "directory" })));
|
||||
}
|
||||
for (const row of restoreRows) await tx.update(workFiles).set({ deletedAt: null, updatedAt: new Date() }).where(eq(workFiles.id, row.id));
|
||||
for (const row of restoreRows) await measureSandboxOperation("work_folder.db.query", { operation: "update_work_files", requestCount: 1 }, async () => (tx.update(workFiles).set({ deletedAt: null, updatedAt: new Date() }).where(eq(workFiles.id, row.id))));
|
||||
});
|
||||
}
|
||||
|
||||
async function purge(folder: Folder, fileId: string, operationId: string) {
|
||||
return mutate(folder, operationId, JSON.stringify(["purge", fileId]), async (tx) => {
|
||||
const [deleted] = await tx.select().from(workFiles).where(and(eq(workFiles.folderId, folder.id),
|
||||
eq(workFiles.id, fileId), isNotNull(workFiles.deletedAt)));
|
||||
const [deleted] = await measureSandboxOperation("work_folder.db.query", { operation: "select_work_files", requestCount: 1 }, async () => (tx.select().from(workFiles).where(and(eq(workFiles.folderId, folder.id),
|
||||
eq(workFiles.id, fileId), isNotNull(workFiles.deletedAt)))));
|
||||
if (!deleted) throw notFound("Deleted file not found");
|
||||
const prefix = `${deleted.path}/`;
|
||||
const rows = await tx.delete(workFiles).where(and(eq(workFiles.folderId, folder.id),
|
||||
const rows = await measureSandboxOperation("work_folder.db.query", { operation: "delete_work_files", requestCount: 1 }, async () => (tx.delete(workFiles).where(and(eq(workFiles.folderId, folder.id),
|
||||
eq(workFiles.deletedAt, deleted.deletedAt!), deleted.kind === "directory"
|
||||
? sql`(${workFiles.path} = ${deleted.path} or left(${workFiles.path}, ${prefix.length}) = ${prefix})`
|
||||
: eq(workFiles.id, fileId))).returning({ objectKey: workFiles.objectKey });
|
||||
for (const row of rows) if (row.objectKey) await tx.update(workFolderObjects).set({ deleteAfter: new Date() }).where(eq(workFolderObjects.objectKey, row.objectKey));
|
||||
: eq(workFiles.id, fileId))).returning({ objectKey: workFiles.objectKey })));
|
||||
for (const row of rows) if (row.objectKey) await measureSandboxOperation("work_folder.db.query", { operation: "update_work_folder_objects", requestCount: 1 }, async () => (tx.update(workFolderObjects).set({ deleteAfter: new Date() }).where(eq(workFolderObjects.objectKey, row.objectKey!))));
|
||||
});
|
||||
}
|
||||
return { ensure, list, get, content, write, remove, restore, purge };
|
||||
|
|
|
|||
|
|
@ -76,6 +76,8 @@ export const DAYTONA_IMAGE_INPUT_PATHS = [
|
|||
"packages/paperclip-runner/runner/crates",
|
||||
"packages/paperclip-runner/scripts/acpx-sidecar-contract.mjs",
|
||||
"packages/paperclip-runner/scripts/build-provider-pack.mjs",
|
||||
"packages/paperclip-runner/scripts/assemble-provider-pack.mjs",
|
||||
"packages/paperclip-runner/scripts/provider-pack-integrity.mjs",
|
||||
"packages/paperclip-runner/scripts/provider-pack-layout.mjs",
|
||||
"packages/paperclip-runner/scripts/materialize-pi-binary.mjs",
|
||||
"packages/paperclip-runner/scripts/portable-provider-shim.mjs",
|
||||
|
|
@ -195,7 +197,9 @@ export function extractDaytonaBaseImages(dockerfile: string): string[] {
|
|||
}
|
||||
|
||||
const reference = match[1]!;
|
||||
if (!stageAliases.has(reference)) {
|
||||
// Docker's scratch is the built-in empty filesystem, not a registry image.
|
||||
// Its declaration remains covered by the Dockerfile content hash.
|
||||
if (reference !== "scratch" && !stageAliases.has(reference)) {
|
||||
assertPinnedBaseImage(reference);
|
||||
baseImages.push(reference);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ describe("runner E2E Daytona image contract", () => {
|
|||
]);
|
||||
const normalizedDockerfile = dockerfile.replace(/\\\r?\n\s*/g, " ");
|
||||
expect(dockerfile).toContain("--bin paperclip-runnerd");
|
||||
expect(dockerfile).toContain("build-provider-pack.mjs /provider-pack");
|
||||
expect(dockerfile).toContain("assemble-provider-pack.mjs /provider-pack");
|
||||
expect(normalizedDockerfile).not.toContain(
|
||||
"COPY packages/paperclip-eval-kernel ./packages/paperclip-eval-kernel",
|
||||
);
|
||||
|
|
@ -166,6 +166,21 @@ describe("runner E2E Daytona image contract", () => {
|
|||
expect(cliInstall).toBeLessThan(finalMetadataArgs);
|
||||
});
|
||||
|
||||
it("exports the canonical provider stage without a mutable base or recursive builder", async () => {
|
||||
const dockerfile = await readFile(path.join(repositoryRoot, "docker/daytona-runner/Dockerfile"), "utf8");
|
||||
expect(dockerfile).toContain("FROM scratch AS provider-pack-export\nCOPY --from=provider-pack-build /provider-pack /");
|
||||
expect(extractDaytonaBaseImages(dockerfile)).not.toContain("scratch");
|
||||
expect(() => extractDaytonaBaseImages("FROM node:latest\nFROM scratch AS exported")).toThrow("immutable");
|
||||
const builder = await readFile(path.join(repositoryRoot, "packages/paperclip-runner/scripts/build-provider-pack.mjs"), "utf8");
|
||||
expect(builder).toContain('"--target", "provider-pack-export"');
|
||||
expect(builder).toContain('"--platform", "linux/amd64"');
|
||||
expect(builder).toContain("verifyProviderPack(exported");
|
||||
expect(dockerfile).not.toContain("build-provider-pack.mjs /provider-pack");
|
||||
for (const file of ["assemble-provider-pack.mjs", "provider-pack-integrity.mjs"]) {
|
||||
expect(DAYTONA_IMAGE_INPUT_PATHS).toContain(`packages/paperclip-runner/scripts/${file}`);
|
||||
}
|
||||
});
|
||||
|
||||
it("builds both provider packs from the same verified dedicated lock before compiling", async () => {
|
||||
const lockPath = "docker/daytona-runner/provider-dependencies.lock.yaml";
|
||||
const lock = await readFile(path.join(repositoryRoot, lockPath));
|
||||
|
|
@ -189,7 +204,7 @@ describe("runner E2E Daytona image contract", () => {
|
|||
"pnpm --filter @paperclipai/paperclip-runner build:typescript",
|
||||
);
|
||||
const pack = normalized.indexOf(
|
||||
"node packages/paperclip-runner/scripts/build-provider-pack.mjs /provider-pack",
|
||||
"node packages/paperclip-runner/scripts/assemble-provider-pack.mjs /provider-pack",
|
||||
);
|
||||
expect(normalized).toContain(`ARG PAPERCLIP_RUNNER_LOCK_SHA256=${lockDigest}`);
|
||||
expect(copy).toBeGreaterThan(0);
|
||||
|
|
|
|||
Loading…
Reference in New Issue