fix(adapter-utils): honor .gitignore for referenced-project staging (#12184)
## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work. > - Sandbox adapters stage project files before an agent starts. > - Referenced projects ignored Git-ignored paths and copied large local directories. > - This behavior increased staging time and disk use, and it differed from anchor workspaces. > - This pull request resolves Git-ignored paths once and shares that result across all referenced-project consumers. > - The benefit is smaller, faster, and consistent project staging. ## Linked Issues or Issue Description No public GitHub issue exists for this bug. **What happened?** Referenced-project staging copied Git-ignored paths, except for a fixed list of heavy directory names. A large repository therefore used much more time and disk space than the same repository in an anchor workspace. **Expected behavior** Referenced-project staging should exclude the same Git-ignored paths that the workspace staging path excludes. **Steps to reproduce** 1. Create a referenced project with a large Git-ignored directory. 2. Start a sandbox or SSH run that stages the referenced project. 3. Observe that the ignored directory enters the staged content. **Paperclip version or commit** Commit `9964b034bbff24e700c8eccf5a8b1fc3daa44bf2`. **Deployment mode** Built from source. ## What Changed - Resolve each referenced project's Git-ignored paths once before staging. - Carry the resolved paths as a required field on `SandboxAdditionalSource`. - Reuse the resolved paths in sandbox staging, SSH staging, and content-signature code. - Harden the read-only Git helper with a bounded process, a reduced environment, and disabled system and global configuration. - Fail closed on Git errors, timeouts, and invalid path relations. - Escape tar glob metacharacters in ignore-derived exclude entries. - Add and update unit tests for the resolver and its three consumers. ## Verification - `pnpm vitest run --config packages/adapter-utils/vitest.config.ts` passes 266 tests locally. - `pnpm exec tsc --noEmit -p packages/adapter-utils/tsconfig.json` passes locally. - CI must pass on this pull request. - Greptile must report 5/5 with no unresolved comments before merge. ## Risks - A Git error or timeout now prevents staging for the affected referenced project. - The resolver uses a bounded read-only Git process and fails closed by design. - The change stays inside `packages/adapter-utils` and does not change the database schema. ## Model Used Claude Sonnet 5 (Anthropic) assisted the implementation with code execution and tool use. The exact context window and reasoning mode are not recorded. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes: #` / `Refs: #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Nicky Leach
GitHub
parent
9fc2f594ae
commit
6880213de5
|
|
@ -31,11 +31,13 @@ import {
|
|||
findAncestorBin,
|
||||
geminiVersionSupportsNativeAcpFlag,
|
||||
parseGeminiVersionParts,
|
||||
referencedSourceContentSignature,
|
||||
rewriteGeminiAcpFlagForVersion,
|
||||
summarizeAcpxTurnUsage,
|
||||
type AcpxEngineExecutorOptions,
|
||||
} from "./execute.js";
|
||||
import { runChildProcess } from "../server-utils.js";
|
||||
import { setExpensiveWorkspaceGitExecutor } from "../git-workspace-sync.js";
|
||||
import {
|
||||
getActiveStepContext,
|
||||
runWithRuntimeParent,
|
||||
|
|
@ -1417,6 +1419,50 @@ describe("shared ACPX engine runtime behavior", () => {
|
|||
expect(fp(repointed)).not.toBe(fp(two));
|
||||
});
|
||||
|
||||
it("routes every referenced project's Git-ignore scan through the registered scheduler, not an unbounded direct spawn", async () => {
|
||||
const root = await makeTempRoot();
|
||||
const stateDir = path.join(root, "state");
|
||||
const cwd = path.join(root, "workspace");
|
||||
const baseConfig = { agentCommand: "node ./fake-acp.js", stateDir };
|
||||
// The paths need not exist: the scheduler hook intercepts the scan before
|
||||
// any real `git` process (or even a directory-existence check) runs.
|
||||
const projectPaths = ["/host/project-a", "/host/project-b", "/host/project-c"];
|
||||
|
||||
const scannedPaths: string[] = [];
|
||||
setExpensiveWorkspaceGitExecutor(async (input) => {
|
||||
scannedPaths.push(input.localDir);
|
||||
const error = new Error("fatal: not a git repository (or any of the parent directories): .git");
|
||||
throw Object.assign(error, { stdout: "", stderr: error.message });
|
||||
});
|
||||
|
||||
try {
|
||||
await runExecutor(baseConfig, {
|
||||
context: {
|
||||
taskId: "issue-1",
|
||||
wakeReason: "issue_assigned",
|
||||
paperclipWorkspace: {
|
||||
cwd,
|
||||
realization: {
|
||||
additional: projectPaths.map((localPath, index) => ({
|
||||
path: localPath,
|
||||
projectId: String.fromCharCode(97 + index),
|
||||
})),
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
} finally {
|
||||
setExpensiveWorkspaceGitExecutor(null);
|
||||
}
|
||||
|
||||
// Every referenced project's scan reached the registered scheduler hook.
|
||||
// The `Promise.all` fan-out over projects can no longer bypass it by
|
||||
// spawning `git` directly, so a host process that bounds concurrent scans
|
||||
// there bounds referenced-project scans too, however many projects a run
|
||||
// configures.
|
||||
expect(scannedPaths.sort()).toEqual([...projectPaths].sort());
|
||||
});
|
||||
|
||||
it("busts the session fingerprint when referenced-project files change at the same host path", async () => {
|
||||
const root = await makeTempRoot();
|
||||
const stateDir = path.join(root, "state");
|
||||
|
|
@ -1513,6 +1559,53 @@ describe("shared ACPX engine runtime behavior", () => {
|
|||
expect(fp(after)).not.toBe(fp(before));
|
||||
});
|
||||
|
||||
describe("referencedSourceContentSignature", () => {
|
||||
it("returns a stable marker without walking the tree when the ignore resolution failed", async () => {
|
||||
const root = await makeTempRoot();
|
||||
// A directory that does not exist: a walk would throw ENOENT. The
|
||||
// `failed` resolution must short-circuit before any `fs.readdir` call.
|
||||
const localPath = path.join(root, "does-not-exist");
|
||||
|
||||
const signature = await referencedSourceContentSignature(localPath, { kind: "failed", reason: "git status timed out" });
|
||||
|
||||
expect(signature).toBe("unreadable:git status timed out");
|
||||
});
|
||||
|
||||
it("skips a Git-ignored file (exact match) so its content never affects the signature", async () => {
|
||||
const root = await makeTempRoot();
|
||||
const localPath = path.join(root, "project");
|
||||
await fs.mkdir(localPath, { recursive: true });
|
||||
await fs.writeFile(path.join(localPath, "kept.txt"), "kept\n", "utf8");
|
||||
await fs.writeFile(path.join(localPath, "secret.env"), "TOKEN=1\n", "utf8");
|
||||
|
||||
const resolution = { kind: "git" as const, ignoredPaths: ["secret.env"] };
|
||||
const before = await referencedSourceContentSignature(localPath, resolution);
|
||||
await fs.writeFile(path.join(localPath, "secret.env"), "TOKEN=2\n", "utf8");
|
||||
const afterIgnoredEdit = await referencedSourceContentSignature(localPath, resolution);
|
||||
await fs.writeFile(path.join(localPath, "kept.txt"), "kept, changed\n", "utf8");
|
||||
const afterKeptEdit = await referencedSourceContentSignature(localPath, resolution);
|
||||
|
||||
// Editing the ignored file never busts the signature; editing the kept file does.
|
||||
expect(afterIgnoredEdit).toBe(before);
|
||||
expect(afterKeptEdit).not.toBe(before);
|
||||
});
|
||||
|
||||
it("skips every file under a Git-ignored directory (prefix match)", async () => {
|
||||
const root = await makeTempRoot();
|
||||
const localPath = path.join(root, "project");
|
||||
await fs.mkdir(path.join(localPath, "build", "nested"), { recursive: true });
|
||||
await fs.writeFile(path.join(localPath, "kept.txt"), "kept\n", "utf8");
|
||||
await fs.writeFile(path.join(localPath, "build", "nested", "artifact.js"), "v1\n", "utf8");
|
||||
|
||||
const resolution = { kind: "git" as const, ignoredPaths: ["build"] };
|
||||
const before = await referencedSourceContentSignature(localPath, resolution);
|
||||
await fs.writeFile(path.join(localPath, "build", "nested", "artifact.js"), "v2\n", "utf8");
|
||||
const after = await referencedSourceContentSignature(localPath, resolution);
|
||||
|
||||
expect(after).toBe(before);
|
||||
});
|
||||
});
|
||||
|
||||
it("shapes ACPX session env for remote execution identities", async () => {
|
||||
const root = await makeTempRoot();
|
||||
const localCwd = path.join(root, "local");
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ import {
|
|||
prepareAdapterExecutionTargetRuntime,
|
||||
readAdapterExecutionTarget,
|
||||
resolveAdapterExecutionTargetTimeout,
|
||||
resolveReferencedSourceIgnore,
|
||||
runAdapterExecutionTargetShellCommand,
|
||||
startAdapterExecutionTargetPaperclipBridge,
|
||||
startAdapterExecutionTargetProcessSessionBridge,
|
||||
|
|
@ -31,6 +32,7 @@ import {
|
|||
type AdapterExecutionTargetTimeoutResolution,
|
||||
type AdapterManagedRuntimeAsset,
|
||||
type PreparedAdapterExecutionTargetRuntime,
|
||||
type ReferencedSourceIgnoreResolution,
|
||||
type SandboxAdditionalSource,
|
||||
} from "@paperclipai/adapter-utils/execution-target";
|
||||
import type { DuplexLossReason } from "../duplex-observability.js";
|
||||
|
|
@ -519,9 +521,14 @@ export function finalizeLaunchEnvironment(
|
|||
}
|
||||
|
||||
// Directory names the staging path never ships for a referenced project (heavy
|
||||
// build/cache output and git history). The content signature skips them so it
|
||||
// reflects only the staged tree and never reads their bytes. Keep this set equal
|
||||
// to the staging excludes in the sandbox and remote runtimes.
|
||||
// build/cache output and git history), applied regardless of the project's
|
||||
// ignore resolution. The content signature skips them so it reflects only the
|
||||
// staged tree and never reads their bytes. Keep this set equal to the fixed
|
||||
// excludes the sandbox and SSH runtimes always apply. A project's OWN resolved
|
||||
// Git-ignored paths (see `resolveReferencedSourceIgnore`) are matched
|
||||
// separately, by relative path, inside `referencedSourceContentSignature` — that
|
||||
// is the real invariant now: the signature and both staging lanes must consume
|
||||
// the SAME one resolution per project, not just this fixed name list.
|
||||
const REFERENCED_SOURCE_SIGNATURE_SKIP_DIRS = new Set([
|
||||
"node_modules",
|
||||
"vendor",
|
||||
|
|
@ -551,12 +558,30 @@ const REFERENCED_SOURCE_SIGNATURE_SKIP_DIRS = new Set([
|
|||
* re-checkout that restores the same size and timestamp. The byte hash busts on any
|
||||
* content change, so the fingerprint busts and the next launch stages the current
|
||||
* tree. The walk skips the heavy build, cache, and git directories the staging path
|
||||
* never ships, and records a symlink by its target text without following it. On a
|
||||
* read error the function returns a stable marker, so the fingerprint does not churn
|
||||
* while staging surfaces the real error. The walk runs only when the run carries
|
||||
* referenced projects (the multi-project sync path).
|
||||
* never ships, plus the project's own resolved Git-ignored paths, and records a
|
||||
* symlink by its target text without following it. On a read error the function
|
||||
* returns a stable marker, so the fingerprint does not churn while staging
|
||||
* surfaces the real error. The walk runs only when the run carries referenced
|
||||
* projects (the multi-project sync path).
|
||||
*
|
||||
* `ignoreResolution` is the ONE resolution `resolveReferencedSourceIgnore`
|
||||
* computed for this project — the same one the sandbox lane and the SSH lane
|
||||
* consume. A `failed` resolution skips the walk entirely and returns a stable
|
||||
* marker instead, because a failed project is not staged and its bytes are not
|
||||
* read anywhere.
|
||||
*/
|
||||
async function referencedSourceContentSignature(localPath: string): Promise<string> {
|
||||
export async function referencedSourceContentSignature(
|
||||
localPath: string,
|
||||
ignoreResolution: ReferencedSourceIgnoreResolution,
|
||||
): Promise<string> {
|
||||
if (ignoreResolution.kind === "failed") {
|
||||
return `unreadable:${ignoreResolution.reason}`;
|
||||
}
|
||||
const isIgnoredByGitResolution = (relativePath: string): boolean =>
|
||||
ignoreResolution.kind === "git" &&
|
||||
ignoreResolution.ignoredPaths.some(
|
||||
(entry) => relativePath === entry || relativePath.startsWith(`${entry}/`),
|
||||
);
|
||||
const hash = createHash("sha256");
|
||||
const walk = async (relative: string): Promise<void> => {
|
||||
const current = relative ? path.join(localPath, relative) : localPath;
|
||||
|
|
@ -564,6 +589,9 @@ async function referencedSourceContentSignature(localPath: string): Promise<stri
|
|||
dirents.sort((left, right) => (left.name < right.name ? -1 : left.name > right.name ? 1 : 0));
|
||||
for (const dirent of dirents) {
|
||||
const next = relative ? path.posix.join(relative, dirent.name) : dirent.name;
|
||||
if (isIgnoredByGitResolution(next)) {
|
||||
continue;
|
||||
}
|
||||
if (dirent.isDirectory()) {
|
||||
if (REFERENCED_SOURCE_SIGNATURE_SKIP_DIRS.has(dirent.name)) {
|
||||
continue;
|
||||
|
|
@ -1545,32 +1573,56 @@ async function buildRuntime(input: {
|
|||
const additionalSourceRecords = (
|
||||
Array.isArray(realizationContext.additional) ? realizationContext.additional : []
|
||||
).map((entry) => parseObject(entry));
|
||||
const additionalSources: SandboxAdditionalSource[] = additionalSourceRecords
|
||||
.map((entry) => ({ localPath: asString(entry.path, ""), projectId: asString(entry.projectId, "") }))
|
||||
const additionalSourceCandidates = additionalSourceRecords
|
||||
.map((entry) => ({
|
||||
localPath: asString(entry.path, ""),
|
||||
projectId: asString(entry.projectId, ""),
|
||||
projectWorkspaceId: asString(entry.projectWorkspaceId, ""),
|
||||
repoUrl: asString(entry.repoUrl, ""),
|
||||
repoRef: asString(entry.repoRef, ""),
|
||||
}))
|
||||
.filter((entry) => entry.localPath.length > 0 && entry.projectId.length > 0);
|
||||
// Resolve each referenced project's Git-ignored paths ONCE, here, before any
|
||||
// staging site runs. The sandbox lane, the SSH lane, and the content signature
|
||||
// below all consume this SAME resolution per project, so they can never apply
|
||||
// a different exclusion set to the same project. See
|
||||
// `resolveReferencedSourceIgnore` for the fail-closed rules.
|
||||
const additionalSourcesWithIgnore = await Promise.all(
|
||||
additionalSourceCandidates.map(async (entry) => ({
|
||||
...entry,
|
||||
ignoreResolution: await resolveReferencedSourceIgnore(entry.localPath),
|
||||
})),
|
||||
);
|
||||
const additionalSources: SandboxAdditionalSource[] = additionalSourcesWithIgnore.map((entry) => ({
|
||||
localPath: entry.localPath,
|
||||
projectId: entry.projectId,
|
||||
ignoreResolution: entry.ignoreResolution,
|
||||
}));
|
||||
// Stable identity of the referenced-project set for the session fingerprint.
|
||||
// The staged-runtime cache reuses already-staged referenced-project trees on a
|
||||
// compatible resume, so the fingerprint must change when the set OR a project's
|
||||
// pinned checkout changes. Without this, a resume reuses a stale staged tree.
|
||||
// Fold in each project's id, host path, workspace id, and pinned ref; sort by
|
||||
// projectId so the identity depends on the set, not the record order.
|
||||
const additionalSourcesIdentityBase = additionalSourceRecords
|
||||
const additionalSourcesIdentityBase = additionalSourcesWithIgnore
|
||||
.map((entry) => ({
|
||||
projectId: asString(entry.projectId, ""),
|
||||
localPath: asString(entry.path, ""),
|
||||
projectWorkspaceId: asString(entry.projectWorkspaceId, ""),
|
||||
repoUrl: asString(entry.repoUrl, ""),
|
||||
repoRef: asString(entry.repoRef, ""),
|
||||
projectId: entry.projectId,
|
||||
localPath: entry.localPath,
|
||||
projectWorkspaceId: entry.projectWorkspaceId,
|
||||
repoUrl: entry.repoUrl,
|
||||
repoRef: entry.repoRef,
|
||||
ignoreResolution: entry.ignoreResolution,
|
||||
}))
|
||||
.filter((entry) => entry.localPath.length > 0 && entry.projectId.length > 0)
|
||||
.sort((a, b) => (a.projectId < b.projectId ? -1 : a.projectId > b.projectId ? 1 : 0));
|
||||
// Metadata alone does not change on a content-only checkout change (same host
|
||||
// path and pinned ref, new file bytes). Fold in each tree's content signature so
|
||||
// a file add, remove, or edit busts the fingerprint and the resume re-stages.
|
||||
// The signature reads the same `ignoreResolution` the staging sites above use,
|
||||
// so it never disagrees with what was actually shipped.
|
||||
const additionalSourcesIdentity = await Promise.all(
|
||||
additionalSourcesIdentityBase.map(async (entry) => ({
|
||||
additionalSourcesIdentityBase.map(async ({ ignoreResolution, ...entry }) => ({
|
||||
...entry,
|
||||
contentSignature: await referencedSourceContentSignature(entry.localPath),
|
||||
contentSignature: await referencedSourceContentSignature(entry.localPath, ignoreResolution),
|
||||
})),
|
||||
);
|
||||
// Referenced-project workspace hints exposed to the agent through PAPERCLIP_WORKSPACES_JSON. The
|
||||
|
|
|
|||
|
|
@ -356,9 +356,9 @@ describe("command managed runtime", () => {
|
|||
adapterKey: "claude",
|
||||
workspaceLocalDir: localWorkspaceDir,
|
||||
additionalSources: [
|
||||
{ localPath: goodOne, projectId: "one" },
|
||||
{ localPath: path.join(rootDir, "missing"), projectId: "broken" },
|
||||
{ localPath: goodTwo, projectId: "two" },
|
||||
{ localPath: goodOne, projectId: "one", ignoreResolution: { kind: "other" } },
|
||||
{ localPath: path.join(rootDir, "missing"), projectId: "broken", ignoreResolution: { kind: "other" } },
|
||||
{ localPath: goodTwo, projectId: "two", ignoreResolution: { kind: "other" } },
|
||||
],
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -20,8 +20,12 @@ import type {
|
|||
AdditionalSourceStagingFailure,
|
||||
SandboxAdditionalSource,
|
||||
} from "./sandbox-managed-runtime.js";
|
||||
export {
|
||||
resolveReferencedSourceIgnore,
|
||||
} from "./sandbox-managed-runtime.js";
|
||||
export type {
|
||||
AdditionalSourceStagingFailure,
|
||||
ReferencedSourceIgnoreResolution,
|
||||
SandboxAdditionalSource,
|
||||
} from "./sandbox-managed-runtime.js";
|
||||
import {
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ import {
|
|||
integrateImportedGitHead,
|
||||
isMissingGitPrerequisiteError,
|
||||
readGitWorkspaceSnapshot,
|
||||
readReferencedSourceGitIgnoredPaths,
|
||||
runLocalGit,
|
||||
sanitizeGitRemoteUrl,
|
||||
setExpensiveWorkspaceGitExecutor,
|
||||
|
|
@ -534,6 +535,113 @@ describe("git workspace sync", () => {
|
|||
.rejects.toThrow(/Failed to merge concurrent remote git histories/);
|
||||
expect(await git(repo, ["rev-parse", "HEAD"])).toBe(currentHead);
|
||||
});
|
||||
|
||||
describe("readReferencedSourceGitIgnoredPaths", () => {
|
||||
it("returns null for a directory that is not a Git work tree", async () => {
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-referenced-nogit-"));
|
||||
cleanupDirs.push(rootDir);
|
||||
const plainDir = path.join(rootDir, "plain");
|
||||
await mkdir(plainDir, { recursive: true });
|
||||
await writeFile(path.join(plainDir, "file.txt"), "body\n", "utf8");
|
||||
|
||||
await expect(readReferencedSourceGitIgnoredPaths(plainDir)).resolves.toBeNull();
|
||||
});
|
||||
|
||||
it("reads the repository top level and the ignored paths of a Git work tree", async () => {
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-referenced-git-"));
|
||||
cleanupDirs.push(rootDir);
|
||||
const repo = await createRepo(rootDir);
|
||||
await writeFile(path.join(repo, ".gitignore"), "secret.env\nbuild/\n", "utf8");
|
||||
await writeFile(path.join(repo, "secret.env"), "TOKEN=abc\n", "utf8");
|
||||
await mkdir(path.join(repo, "build"), { recursive: true });
|
||||
await writeFile(path.join(repo, "build", "out.js"), "artifact\n", "utf8");
|
||||
|
||||
const scan = await readReferencedSourceGitIgnoredPaths(repo);
|
||||
expect(scan?.toplevel).toBe(await git(repo, ["rev-parse", "--show-toplevel"]));
|
||||
expect(scan?.ignoredPaths).toEqual(["build", "secret.env"]);
|
||||
});
|
||||
|
||||
it("preserves trailing whitespace in an ignored path entry", async () => {
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-referenced-trailing-ws-"));
|
||||
cleanupDirs.push(rootDir);
|
||||
const repo = await createRepo(rootDir);
|
||||
// A wildcard pattern avoids the separate rule that git trims an
|
||||
// unescaped trailing space in a .gitignore pattern itself; the trailing
|
||||
// space under test lives in the matched FILE name, not the pattern.
|
||||
const paddedName = "secret.env ";
|
||||
await writeFile(path.join(repo, ".gitignore"), "secret.env*\n", "utf8");
|
||||
await writeFile(path.join(repo, paddedName), "TOKEN=abc\n", "utf8");
|
||||
|
||||
const scan = await readReferencedSourceGitIgnoredPaths(repo);
|
||||
expect(scan?.ignoredPaths).toEqual([paddedName]);
|
||||
});
|
||||
|
||||
it("routes both scan commands through the registered scheduler instead of spawning git directly", async () => {
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-referenced-scheduler-"));
|
||||
cleanupDirs.push(rootDir);
|
||||
const repo = await createRepo(rootDir);
|
||||
await writeFile(path.join(repo, ".gitignore"), "build/\n", "utf8");
|
||||
await mkdir(path.join(repo, "build"), { recursive: true });
|
||||
await writeFile(path.join(repo, "build", "out.js"), "artifact\n", "utf8");
|
||||
|
||||
const operations: string[] = [];
|
||||
setExpensiveWorkspaceGitExecutor(async (input) => {
|
||||
operations.push(input.operation);
|
||||
return await runLocalGit(input.localDir, [...input.args], {
|
||||
timeout: input.timeout,
|
||||
maxBuffer: input.maxBuffer,
|
||||
env: input.env,
|
||||
});
|
||||
});
|
||||
|
||||
const scan = await readReferencedSourceGitIgnoredPaths(repo);
|
||||
|
||||
expect(scan?.ignoredPaths).toEqual(["build"]);
|
||||
// Both the toplevel probe and the ignored-paths read go through the SAME
|
||||
// process-wide admission seam the anchor workspace's expensive reads
|
||||
// use. A host process that bounds concurrent scans there also bounds
|
||||
// referenced-project scans, so a run with many referenced projects
|
||||
// cannot spawn one unbounded Git process per project.
|
||||
expect(operations.sort()).toEqual(["referenced_source.ignored_files", "referenced_source.toplevel"]);
|
||||
});
|
||||
|
||||
it("carries the hardened arguments and does not inherit a poisoned GIT_CONFIG_GLOBAL", async () => {
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-referenced-hardened-env-"));
|
||||
cleanupDirs.push(rootDir);
|
||||
const repo = await createRepo(rootDir);
|
||||
const badGlobalConfig = path.join(rootDir, "bad-global-gitconfig");
|
||||
await writeFile(badGlobalConfig, "this is not valid git config syntax [[[\n", "utf8");
|
||||
|
||||
const priorGlobal = process.env.GIT_CONFIG_GLOBAL;
|
||||
process.env.GIT_CONFIG_GLOBAL = badGlobalConfig;
|
||||
try {
|
||||
// A plain invocation inherits the poisoned global config and fails to parse it.
|
||||
await expect(execFile("git", ["-C", repo, "status", "--porcelain"])).rejects.toThrow();
|
||||
// The hardened helper does not inherit GIT_CONFIG_GLOBAL from this process's
|
||||
// environment, so it succeeds regardless.
|
||||
await expect(readReferencedSourceGitIgnoredPaths(repo)).resolves.toMatchObject({ ignoredPaths: [] });
|
||||
} finally {
|
||||
if (priorGlobal === undefined) delete process.env.GIT_CONFIG_GLOBAL;
|
||||
else process.env.GIT_CONFIG_GLOBAL = priorGlobal;
|
||||
}
|
||||
});
|
||||
|
||||
it("neutralizes a repository-local core.fsmonitor hook", async () => {
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-referenced-fsmonitor-"));
|
||||
cleanupDirs.push(rootDir);
|
||||
const repo = await createRepo(rootDir);
|
||||
const markerPath = path.join(rootDir, "pwned.txt");
|
||||
// A malicious repository-local config: a non-boolean `core.fsmonitor` value
|
||||
// is a hook COMMAND Git runs on every status-like read. `--no-optional-locks`
|
||||
// alone does not stop this; only the command-line `-c core.fsmonitor=false`
|
||||
// override does, because command-line config wins over repository config.
|
||||
await git(repo, ["config", "core.fsmonitor", `sh -c 'touch ${markerPath}; printf 1'`]);
|
||||
|
||||
await readReferencedSourceGitIgnoredPaths(repo);
|
||||
|
||||
await expect(stat(markerPath)).rejects.toThrow();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("sanitizeGitRemoteUrl", () => {
|
||||
|
|
|
|||
|
|
@ -23,6 +23,15 @@ export interface ExpensiveWorkspaceGitInput {
|
|||
operation: string;
|
||||
timeout: number;
|
||||
maxBuffer: number;
|
||||
/**
|
||||
* Optional environment override for the invocation. Absent for the anchor
|
||||
* workspace's own full-tree walks (they inherit the process environment, a
|
||||
* directory this process already controls). A referenced-project scan sets
|
||||
* this to its hardened environment (see {@link buildHardenedGitEnv}), so a
|
||||
* host executor that honors it still runs the read hardened even though it
|
||||
* dispatches through the same seam as the anchor's reads.
|
||||
*/
|
||||
env?: NodeJS.ProcessEnv;
|
||||
}
|
||||
|
||||
export type ExpensiveWorkspaceGitExecutor = (
|
||||
|
|
@ -69,6 +78,7 @@ export async function runLocalGit(
|
|||
options: {
|
||||
timeout?: number;
|
||||
maxBuffer?: number;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
} = {},
|
||||
): Promise<GitCommandResult> {
|
||||
return await new Promise<GitCommandResult>((resolve, reject) => {
|
||||
|
|
@ -78,6 +88,7 @@ export async function runLocalGit(
|
|||
{
|
||||
timeout: options.timeout ?? 15_000,
|
||||
maxBuffer: options.maxBuffer ?? 1024 * 128,
|
||||
env: options.env ?? process.env,
|
||||
},
|
||||
(error, stdout, stderr) => {
|
||||
if (error) {
|
||||
|
|
@ -97,7 +108,7 @@ async function runExpensiveWorkspaceGit(
|
|||
localDir: string,
|
||||
args: string[],
|
||||
operation: string,
|
||||
options: { timeout: number; maxBuffer: number },
|
||||
options: { timeout: number; maxBuffer: number; env?: NodeJS.ProcessEnv },
|
||||
): Promise<GitCommandResult> {
|
||||
if (expensiveWorkspaceGitExecutor) {
|
||||
return await expensiveWorkspaceGitExecutor({
|
||||
|
|
@ -106,6 +117,7 @@ async function runExpensiveWorkspaceGit(
|
|||
operation,
|
||||
timeout: options.timeout,
|
||||
maxBuffer: options.maxBuffer,
|
||||
env: options.env,
|
||||
});
|
||||
}
|
||||
return await runLocalGit(localDir, args, options);
|
||||
|
|
@ -168,6 +180,143 @@ export async function readGitWorkspaceSnapshot(localDir: string): Promise<GitWor
|
|||
}
|
||||
}
|
||||
|
||||
/** The `git status --ignored` output for one directory, read by {@link readReferencedSourceGitIgnoredPaths}. */
|
||||
export interface ReferencedSourceGitIgnoreScan {
|
||||
/** The absolute repository top level `git rev-parse --show-toplevel` reports. */
|
||||
toplevel: string;
|
||||
/** Ignored paths, relative to `toplevel`, trailing slashes stripped, sorted. */
|
||||
ignoredPaths: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the environment for a hardened, read-only Git invocation against a
|
||||
* directory this process does not control (a referenced project, not the
|
||||
* anchor workspace). Two protections apply:
|
||||
*
|
||||
* - Drop every inherited `GIT_*` variable, so an already-set override in this
|
||||
* process's own environment cannot change how the read-only command runs.
|
||||
* - Point the global config file at `/dev/null` (in addition to the
|
||||
* command-line `GIT_CONFIG_NOSYSTEM=1` the caller sets), so neither this
|
||||
* host's global nor system Git configuration can add a setting the
|
||||
* read-only command was not built to expect.
|
||||
*
|
||||
* This does not defend against the directory's OWN repository-local
|
||||
* configuration; the command-line `-c core.fsmonitor=false` override in
|
||||
* {@link runHardenedReadOnlyGit} does that instead, because command-line
|
||||
* config always wins over repository-local config.
|
||||
*/
|
||||
function buildHardenedGitEnv(): NodeJS.ProcessEnv {
|
||||
const env: NodeJS.ProcessEnv = {};
|
||||
for (const [key, value] of Object.entries(process.env)) {
|
||||
if (key.startsWith("GIT_") || value === undefined) continue;
|
||||
env[key] = value;
|
||||
}
|
||||
env.GIT_CONFIG_NOSYSTEM = "1";
|
||||
env.GIT_CONFIG_GLOBAL = "/dev/null";
|
||||
return env;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a read-only Git command against a directory this process does not
|
||||
* control, hardened against a hostile repository-local configuration, and
|
||||
* dispatched through {@link runExpensiveWorkspaceGit} — the SAME process-wide
|
||||
* admission seam the anchor workspace's expensive full-tree reads use. A host
|
||||
* process that registers a bounded scheduler there (see
|
||||
* `setExpensiveWorkspaceGitExecutor`) governs referenced-project scans too, so
|
||||
* a run with many referenced projects cannot spawn one unbounded Git process
|
||||
* per project; each request queues behind the same concurrency limit.
|
||||
*
|
||||
* Every call still carries `--no-optional-locks` (never blocks on, or is
|
||||
* blocked by, a concurrent Git process in the directory) and
|
||||
* `-c core.fsmonitor=false` (neutralizes a repository-local `core.fsmonitor`
|
||||
* setting that would otherwise run an arbitrary configured program on this
|
||||
* read). See {@link buildHardenedGitEnv} for the paired environment hardening,
|
||||
* carried through the executor's optional `env` field so hardening survives
|
||||
* the hop through a host-registered scheduler.
|
||||
*/
|
||||
async function runHardenedReadOnlyGit(
|
||||
localDir: string,
|
||||
args: string[],
|
||||
operation: string,
|
||||
options: { timeout: number; maxBuffer: number },
|
||||
): Promise<GitCommandResult> {
|
||||
return await runExpensiveWorkspaceGit(
|
||||
localDir,
|
||||
["-c", "core.fsmonitor=false", "--no-optional-locks", ...args],
|
||||
operation,
|
||||
{ timeout: options.timeout, maxBuffer: options.maxBuffer, env: buildHardenedGitEnv() },
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* True when a failed `git` invocation failed specifically because `localDir`
|
||||
* is not inside a Git work tree — Git's own "not a git repository" fatal
|
||||
* error. Distinguishes the expected non-Git case from a real failure (a
|
||||
* timeout, a permissions error, a corrupt repository), which must still
|
||||
* surface as a failure and never look like "no Git tree here".
|
||||
*/
|
||||
function isNotAGitRepositoryError(error: unknown): boolean {
|
||||
const stderr = error && typeof error === "object" && "stderr" in error ? String((error as { stderr: unknown }).stderr) : "";
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
return /not a git repository/i.test(stderr) || /not a git repository/i.test(message);
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the Git-ignored paths of a referenced-project host directory, for the
|
||||
* staging path to exclude them (see `resolveReferencedSourceIgnore` in
|
||||
* `sandbox-managed-runtime.ts`). Every command runs through
|
||||
* {@link runHardenedReadOnlyGit}, because the directory is a host checkout the
|
||||
* staging code does not control, unlike the anchor workspace.
|
||||
*
|
||||
* Returns `null` when `localDir` is not a Git work tree — the caller keeps
|
||||
* today's fixed excludes for that case. Throws on any other Git error, a
|
||||
* timeout, or malformed output, so the caller can fail closed and skip
|
||||
* staging that one project instead of shipping it unfiltered.
|
||||
*/
|
||||
export async function readReferencedSourceGitIgnoredPaths(
|
||||
localDir: string,
|
||||
): Promise<ReferencedSourceGitIgnoreScan | null> {
|
||||
let toplevel: string;
|
||||
try {
|
||||
const toplevelResult = await runHardenedReadOnlyGit(
|
||||
localDir,
|
||||
["rev-parse", "--show-toplevel"],
|
||||
"referenced_source.toplevel",
|
||||
{ timeout: 15_000, maxBuffer: 64 * 1024 },
|
||||
);
|
||||
toplevel = toplevelResult.stdout.trim();
|
||||
} catch (error) {
|
||||
if (isNotAGitRepositoryError(error)) {
|
||||
return null;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
if (!toplevel) {
|
||||
throw new Error(`git rev-parse --show-toplevel returned an empty path for ${localDir}`);
|
||||
}
|
||||
|
||||
const ignoredResult = await runHardenedReadOnlyGit(
|
||||
localDir,
|
||||
["status", "--ignored", "--porcelain=v1", "-z", "--untracked-files=normal"],
|
||||
"referenced_source.ignored_files",
|
||||
{ timeout: 60_000, maxBuffer: 16 * 1024 * 1024 },
|
||||
);
|
||||
// Do not trim each entry: `git status -z` already delimits entries with a
|
||||
// NUL byte, so a leading or trailing space in an entry is part of the path
|
||||
// itself, not padding to remove. A length check finds the one genuinely
|
||||
// empty entry `-z` appends after the last NUL, without eating a real path's
|
||||
// own leading or trailing whitespace.
|
||||
const ignoredPaths = ignoredResult.stdout
|
||||
.split("\0")
|
||||
.filter((entry) => entry.length > 0)
|
||||
.filter((entry) => entry.startsWith("!! "))
|
||||
.map((entry) => entry.slice(3).replace(/\/+$/, ""))
|
||||
.filter((entry) => entry.length > 0)
|
||||
.sort((left, right) => left.localeCompare(right));
|
||||
|
||||
return { toplevel, ignoredPaths };
|
||||
}
|
||||
|
||||
// scp-like ssh remote (`user@host:path`). The syntax has no password slot, so
|
||||
// it cannot embed a secret. Conservative shape: exactly one `@`, no colon in
|
||||
// the user segment (a colon there could smuggle credential-looking material),
|
||||
|
|
|
|||
|
|
@ -125,9 +125,9 @@ describe("remote managed runtime", () => {
|
|||
workspaceRemoteDir: "/app",
|
||||
syncWorkspace: false,
|
||||
additionalSources: [
|
||||
{ localPath: firstDir, projectId: "first" },
|
||||
{ localPath: brokenDir, projectId: "broken" },
|
||||
{ localPath: secondDir, projectId: "second" },
|
||||
{ localPath: firstDir, projectId: "first", ignoreResolution: { kind: "other" } },
|
||||
{ localPath: brokenDir, projectId: "broken", ignoreResolution: { kind: "other" } },
|
||||
{ localPath: secondDir, projectId: "second", ignoreResolution: { kind: "other" } },
|
||||
],
|
||||
});
|
||||
|
||||
|
|
@ -171,8 +171,8 @@ describe("remote managed runtime", () => {
|
|||
workspaceRemoteDir: "/app",
|
||||
syncWorkspace: false,
|
||||
additionalSources: [
|
||||
{ localPath: "relative/referenced", projectId: "relative" },
|
||||
{ localPath: healthyDir, projectId: "healthy" },
|
||||
{ localPath: "relative/referenced", projectId: "relative", ignoreResolution: { kind: "other" } },
|
||||
{ localPath: healthyDir, projectId: "healthy", ignoreResolution: { kind: "other" } },
|
||||
],
|
||||
});
|
||||
|
||||
|
|
@ -183,4 +183,83 @@ describe("remote managed runtime", () => {
|
|||
localDir: "relative/referenced",
|
||||
}));
|
||||
});
|
||||
|
||||
it("passes a project's resolved Git-ignored paths to the SSH exclude list, escaped", async () => {
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-remote-runtime-ignore-"));
|
||||
cleanupDirs.push(rootDir);
|
||||
const workspaceDir = path.join(rootDir, "workspace");
|
||||
const projectDir = path.join(rootDir, "referenced-project");
|
||||
await mkdir(workspaceDir, { recursive: true });
|
||||
|
||||
await prepareRemoteManagedRuntime({
|
||||
spec: {
|
||||
host: "127.0.0.1",
|
||||
port: 2222,
|
||||
username: "fixture",
|
||||
remoteWorkspacePath: "/app",
|
||||
remoteCwd: "/app",
|
||||
privateKey: "PRIVATE KEY",
|
||||
knownHosts: "KNOWN HOSTS",
|
||||
strictHostKeyChecking: true,
|
||||
},
|
||||
runId: "run-ignore",
|
||||
adapterKey: "codex",
|
||||
workspaceLocalDir: workspaceDir,
|
||||
workspaceRemoteDir: "/app",
|
||||
syncWorkspace: false,
|
||||
additionalSources: [
|
||||
{
|
||||
localPath: projectDir,
|
||||
projectId: "proj",
|
||||
ignoreResolution: { kind: "git", ignoredPaths: ["secret.env", "build", "weird[1].txt"] },
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const call = syncDirectoryToSsh.mock.calls.find((entry) => entry[0].localDir === projectDir);
|
||||
expect(call).toBeDefined();
|
||||
const exclude = (call![0] as { exclude?: string[] }).exclude ?? [];
|
||||
// The resolved ignored paths ride the exclude list, glob-escaped, on top of
|
||||
// the fixed heavy-directory excludes the SSH lane already applies.
|
||||
expect(exclude).toContain("secret.env");
|
||||
expect(exclude).toContain("build");
|
||||
expect(exclude).toContain("weird\\[1].txt");
|
||||
expect(exclude).toContain("node_modules");
|
||||
});
|
||||
|
||||
it("skips a project whose ignore resolution failed without ever calling syncDirectoryToSsh for it", async () => {
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-remote-runtime-failed-"));
|
||||
cleanupDirs.push(rootDir);
|
||||
const workspaceDir = path.join(rootDir, "workspace");
|
||||
const healthyDir = path.join(rootDir, "referenced-healthy");
|
||||
const failedDir = path.join(rootDir, "referenced-failed");
|
||||
await mkdir(workspaceDir, { recursive: true });
|
||||
|
||||
const prepared = await prepareRemoteManagedRuntime({
|
||||
spec: {
|
||||
host: "127.0.0.1",
|
||||
port: 2222,
|
||||
username: "fixture",
|
||||
remoteWorkspacePath: "/app",
|
||||
remoteCwd: "/app",
|
||||
privateKey: "PRIVATE KEY",
|
||||
knownHosts: "KNOWN HOSTS",
|
||||
strictHostKeyChecking: true,
|
||||
},
|
||||
runId: "run-failed",
|
||||
adapterKey: "codex",
|
||||
workspaceLocalDir: workspaceDir,
|
||||
workspaceRemoteDir: "/app",
|
||||
syncWorkspace: false,
|
||||
additionalSources: [
|
||||
{ localPath: healthyDir, projectId: "healthy", ignoreResolution: { kind: "other" } },
|
||||
{ localPath: failedDir, projectId: "failed", ignoreResolution: { kind: "failed", reason: "git status timed out" } },
|
||||
],
|
||||
});
|
||||
|
||||
// Fail closed: the failed project never reaches the transfer at all — no
|
||||
// bytes are sent for it — while the healthy project still stages.
|
||||
expect(Object.keys(prepared.additionalSourceDirs)).toEqual(["healthy"]);
|
||||
expect(syncDirectoryToSsh).not.toHaveBeenCalledWith(expect.objectContaining({ localDir: failedDir }));
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -7,13 +7,19 @@ import {
|
|||
restoreWorkspaceFromSshExecution,
|
||||
syncDirectoryToSsh,
|
||||
} from "./ssh.js";
|
||||
import type {
|
||||
SandboxAdditionalSource,
|
||||
SandboxManagedRuntimeAssetRestoreContext,
|
||||
import {
|
||||
mergeExcludes,
|
||||
referencedSourceIgnoreExcludeEntries,
|
||||
type SandboxAdditionalSource,
|
||||
type SandboxManagedRuntimeAssetRestoreContext,
|
||||
} from "./sandbox-managed-runtime.js";
|
||||
import { captureDirectorySnapshot } from "./workspace-restore-merge.js";
|
||||
import type { RuntimeProgressSink } from "./runtime-progress.js";
|
||||
|
||||
// The fixed heavy-directory excludes every referenced project drops,
|
||||
// regardless of its ignore resolution. A `git`-resolved project additionally
|
||||
// drops its own resolved ignored paths (see `referencedSourceIgnoreExcludeEntries`
|
||||
// and the per-project merge below); an `other` project keeps only this set.
|
||||
const REMOTE_ADDITIONAL_SOURCE_HEAVY_DIR_EXCLUDES = [
|
||||
"node_modules",
|
||||
"vendor",
|
||||
|
|
@ -179,7 +185,7 @@ export async function prepareRemoteManagedRuntime(input: {
|
|||
// the other projects continue (no workspace restore, unlike an asset failure).
|
||||
const additionalSourceDirs: Record<string, string> = {};
|
||||
for (const source of input.additionalSources ?? []) {
|
||||
const { localPath, projectId } = source;
|
||||
const { localPath, projectId, ignoreResolution } = source;
|
||||
try {
|
||||
if (!path.posix.isAbsolute(localPath)) {
|
||||
throw new Error(`additional source localPath is not an absolute path: ${localPath}`);
|
||||
|
|
@ -192,12 +198,21 @@ export async function prepareRemoteManagedRuntime(input: {
|
|||
) {
|
||||
throw new Error(`additional source projectId is not a simple path segment: ${projectId}`);
|
||||
}
|
||||
// Fail closed: a project whose ignore resolution failed is not staged at
|
||||
// all — the existing per-project skip-and-warn path below handles it.
|
||||
if (ignoreResolution.kind === "failed") {
|
||||
throw new Error(`referenced project ignore resolution failed: ${ignoreResolution.reason}`);
|
||||
}
|
||||
const remoteDir = path.posix.join(runtimeRootDir, `project-${projectId}`);
|
||||
const exclude = mergeExcludes(
|
||||
REMOTE_ADDITIONAL_SOURCE_HEAVY_DIR_EXCLUDES,
|
||||
referencedSourceIgnoreExcludeEntries(ignoreResolution),
|
||||
);
|
||||
await syncDirectoryToSsh({
|
||||
spec: input.spec,
|
||||
localDir: localPath,
|
||||
remoteDir,
|
||||
exclude: REMOTE_ADDITIONAL_SOURCE_HEAVY_DIR_EXCLUDES,
|
||||
exclude,
|
||||
onProgress: input.onProgress,
|
||||
progressLabel: `project-${projectId}`,
|
||||
});
|
||||
|
|
|
|||
|
|
@ -257,7 +257,11 @@ describe("sandbox native file sync", () => {
|
|||
adapterKey: "test-adapter",
|
||||
client,
|
||||
workspaceLocalDir: localWorkspaceDir,
|
||||
additionalSources: projects.map((project) => ({ localPath: project.localDir, projectId: project.projectId })),
|
||||
additionalSources: projects.map((project) => ({
|
||||
localPath: project.localDir,
|
||||
projectId: project.projectId,
|
||||
ignoreResolution: { kind: "other" },
|
||||
})),
|
||||
});
|
||||
|
||||
// Each project lands in its OWN `project-<projectId>` directory under the
|
||||
|
|
@ -320,9 +324,9 @@ describe("sandbox native file sync", () => {
|
|||
client,
|
||||
workspaceLocalDir: localWorkspaceDir,
|
||||
additionalSources: [
|
||||
{ localPath: goodDir, projectId: "good-a" },
|
||||
{ localPath: path.join(rootDir, "does-not-exist"), projectId: "broken" },
|
||||
{ localPath: goodDir, projectId: "good-b" },
|
||||
{ localPath: goodDir, projectId: "good-a", ignoreResolution: { kind: "other" } },
|
||||
{ localPath: path.join(rootDir, "does-not-exist"), projectId: "broken", ignoreResolution: { kind: "other" } },
|
||||
{ localPath: goodDir, projectId: "good-b", ignoreResolution: { kind: "other" } },
|
||||
],
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -9,8 +9,12 @@ import { resetLocalGitIndexToHead } from "./git-workspace-sync.js";
|
|||
|
||||
import {
|
||||
assertSyncOperationsConfined,
|
||||
escapeTarExcludeLiteral,
|
||||
mirrorDirectory,
|
||||
prepareSandboxManagedRuntime,
|
||||
resolveReferencedSourceIgnore,
|
||||
type PreparedSandboxManagedRuntime,
|
||||
type ReferencedSourceIgnoreResolution,
|
||||
type SandboxManagedRuntimeAsset,
|
||||
type SandboxManagedRuntimeClient,
|
||||
type SandboxSyncOperation,
|
||||
|
|
@ -160,6 +164,64 @@ async function git(cwd: string, args: string[]): Promise<string> {
|
|||
return stdout.trim();
|
||||
}
|
||||
|
||||
// A minimal committed Git repository, for the referenced-project ignore tests.
|
||||
async function initGitRepo(repoDir: string): Promise<void> {
|
||||
await mkdir(repoDir, { recursive: true });
|
||||
await git(repoDir, ["init", "-q"]);
|
||||
await git(repoDir, ["config", "user.name", "Paperclip Test"]);
|
||||
await git(repoDir, ["config", "user.email", "test@paperclip.dev"]);
|
||||
await writeFile(path.join(repoDir, "README.md"), "root\n", "utf8");
|
||||
await git(repoDir, ["add", "README.md"]);
|
||||
await git(repoDir, ["commit", "-qm", "base"]);
|
||||
}
|
||||
|
||||
// A `CommandManagedRuntimeRunner` that runs real shell commands on the host
|
||||
// filesystem (host FS stands in for the sandbox FS), exposing no native
|
||||
// `syncIn` — staging rides the real base64/tar fallback, the same transport a
|
||||
// provider without native sync uses. Unlike the in-file fake clients, this
|
||||
// fallback DOES build a real tarball via `createTarballFromDirectory` for a
|
||||
// `directory`-kind mapping, so it is the one that genuinely applies `exclude`.
|
||||
function makeInlineSpawnRunner(): CommandManagedRuntimeRunner {
|
||||
return {
|
||||
execute: (input) =>
|
||||
new Promise<RunProcessResult>((resolve) => {
|
||||
const startedAt = new Date().toISOString();
|
||||
const command =
|
||||
input.command === "sh" ? "/bin/sh" : input.command === "bash" ? "/bin/bash" : input.command;
|
||||
const child = spawn(command, input.args ?? [], { cwd: input.cwd, env: { ...process.env, ...input.env } });
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
child.stdout.on("data", (chunk) => { stdout += chunk.toString("utf8"); });
|
||||
child.stderr.on("data", (chunk) => { stderr += chunk.toString("utf8"); });
|
||||
child.on("error", () => resolve({ exitCode: 127, signal: null, timedOut: false, stdout, stderr, pid: null, startedAt }));
|
||||
child.on("close", (code) => resolve({ exitCode: code ?? 0, signal: null, timedOut: false, stdout, stderr, pid: child.pid ?? null, startedAt }));
|
||||
if (input.stdin != null) child.stdin.write(input.stdin);
|
||||
child.stdin.end();
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
// Stage ONE referenced project end-to-end (real tar fallback, real `exclude`
|
||||
// filtering) through the full `prepareCommandManagedRuntime` seam, for the
|
||||
// referenced-project ignore regression tests.
|
||||
async function stageOneReferencedProject(
|
||||
referencedDir: string,
|
||||
ignoreResolution: ReferencedSourceIgnoreResolution,
|
||||
): Promise<PreparedSandboxManagedRuntime> {
|
||||
const rootDir = path.dirname(referencedDir);
|
||||
const localWorkspaceDir = path.join(rootDir, "local-workspace");
|
||||
const remoteWorkspaceDir = path.join(rootDir, "remote-workspace");
|
||||
await mkdir(localWorkspaceDir, { recursive: true });
|
||||
await writeFile(path.join(localWorkspaceDir, "README.md"), "anchor\n", "utf8");
|
||||
return await prepareCommandManagedRuntime({
|
||||
runner: makeInlineSpawnRunner(),
|
||||
spec: { remoteCwd: remoteWorkspaceDir, timeoutMs: 30_000 },
|
||||
adapterKey: "test-adapter",
|
||||
workspaceLocalDir: localWorkspaceDir,
|
||||
additionalSources: [{ localPath: referencedDir, projectId: "proj", ignoreResolution }],
|
||||
});
|
||||
}
|
||||
|
||||
async function listTarMembers(rootDir: string, name: string, bytes: Buffer): Promise<string[]> {
|
||||
const tarPath = path.join(rootDir, name);
|
||||
await writeFile(tarPath, bytes);
|
||||
|
|
@ -2087,7 +2149,7 @@ describe("sandbox managed runtime", () => {
|
|||
adapterKey: "test-adapter",
|
||||
client,
|
||||
workspaceLocalDir: localWorkspaceDir,
|
||||
additionalSources: [{ localPath: referencedDir, projectId: "proj-first" }],
|
||||
additionalSources: [{ localPath: referencedDir, projectId: "proj-first", ignoreResolution: { kind: "other" } }],
|
||||
});
|
||||
|
||||
const referencedMapping = captured
|
||||
|
|
@ -2169,9 +2231,9 @@ describe("sandbox managed runtime", () => {
|
|||
adapterKey: "test-adapter",
|
||||
workspaceLocalDir: localWorkspaceDir,
|
||||
additionalSources: [
|
||||
{ localPath: first, projectId: "proj-first" },
|
||||
{ localPath: path.join(rootDir, "referenced-missing"), projectId: "proj-missing" },
|
||||
{ localPath: second, projectId: "proj-second" },
|
||||
{ localPath: first, projectId: "proj-first", ignoreResolution: { kind: "other" } },
|
||||
{ localPath: path.join(rootDir, "referenced-missing"), projectId: "proj-missing", ignoreResolution: { kind: "other" } },
|
||||
{ localPath: second, projectId: "proj-second", ignoreResolution: { kind: "other" } },
|
||||
],
|
||||
});
|
||||
|
||||
|
|
@ -2213,6 +2275,150 @@ describe("sandbox managed runtime", () => {
|
|||
}
|
||||
});
|
||||
|
||||
describe("resolveReferencedSourceIgnore", () => {
|
||||
it("re-relativizes root-relative ignored paths to a nested localPath", async () => {
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-ignore-nested-"));
|
||||
cleanupDirs.push(rootDir);
|
||||
const repo = path.join(rootDir, "repo");
|
||||
await initGitRepo(repo);
|
||||
// An ignored entry OUTSIDE the referenced project's localPath, and two
|
||||
// ignored entries inside it (one top-level, one nested).
|
||||
await writeFile(path.join(repo, ".gitignore"), "outside-secret.env\npackages/app/secret.env\npackages/app/build/\n", "utf8");
|
||||
await writeFile(path.join(repo, "outside-secret.env"), "outside\n", "utf8");
|
||||
const localPath = path.join(repo, "packages", "app");
|
||||
// Commit a tracked file under `localPath` first. Otherwise the whole
|
||||
// `packages/` directory is untracked, and `git status` collapses it to
|
||||
// one `packages/` line instead of reporting entries inside it
|
||||
// individually — the fixture needs the individual entries.
|
||||
await mkdir(localPath, { recursive: true });
|
||||
await writeFile(path.join(localPath, "index.ts"), "export {};\n", "utf8");
|
||||
await git(repo, ["add", "packages/app/index.ts"]);
|
||||
await git(repo, ["commit", "-qm", "add app"]);
|
||||
await mkdir(path.join(localPath, "build"), { recursive: true });
|
||||
await writeFile(path.join(localPath, "secret.env"), "TOKEN=abc\n", "utf8");
|
||||
await writeFile(path.join(localPath, "build", "out.js"), "artifact\n", "utf8");
|
||||
|
||||
const resolution = await resolveReferencedSourceIgnore(localPath);
|
||||
|
||||
// Only the entries under `localPath` apply, re-relativized to it — the
|
||||
// sibling `outside-secret.env` never appears, and the prefix
|
||||
// `packages/app/` is stripped.
|
||||
expect(resolution).toEqual({ kind: "git", ignoredPaths: ["build", "secret.env"] });
|
||||
});
|
||||
|
||||
it("keeps today's fixed excludes for a non-Git source", async () => {
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-ignore-nongit-"));
|
||||
cleanupDirs.push(rootDir);
|
||||
const plainDir = path.join(rootDir, "plain-project");
|
||||
await mkdir(plainDir, { recursive: true });
|
||||
await writeFile(path.join(plainDir, "file.txt"), "body\n", "utf8");
|
||||
|
||||
await expect(resolveReferencedSourceIgnore(plainDir)).resolves.toEqual({ kind: "other" });
|
||||
});
|
||||
|
||||
it("fails closed on a real Git error instead of returning an unfiltered result", async () => {
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-ignore-fail-"));
|
||||
cleanupDirs.push(rootDir);
|
||||
const repo = path.join(rootDir, "repo");
|
||||
await initGitRepo(repo);
|
||||
// Corrupt the index so `git rev-parse --show-toplevel` still succeeds but
|
||||
// `git status --ignored` fails with a real error (not "not a git repository").
|
||||
await writeFile(path.join(repo, ".git", "index"), "not a valid index\n", "utf8");
|
||||
|
||||
const resolution = await resolveReferencedSourceIgnore(repo);
|
||||
|
||||
expect(resolution.kind).toBe("failed");
|
||||
expect((resolution as { reason: string }).reason.length).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
|
||||
it("escapes glob metacharacters so a literal ignored path never over-excludes a sibling", async () => {
|
||||
// A bare tar `--exclude` pattern treats `*`/`?`/`[` as globs. Without
|
||||
// escaping, an ignored file named `secret[1].txt` would exclude the
|
||||
// unrelated sibling `secret1.txt` too (both match the glob `secret[1].txt`,
|
||||
// whose `[1]` is a one-character class matching the literal digit `1`).
|
||||
expect(escapeTarExcludeLiteral("secret[1].txt")).toBe("secret\\[1].txt");
|
||||
expect(escapeTarExcludeLiteral("wildcard*name")).toBe("wildcard\\*name");
|
||||
expect(escapeTarExcludeLiteral("question?mark")).toBe("question\\?mark");
|
||||
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-ignore-glob-"));
|
||||
cleanupDirs.push(rootDir);
|
||||
const referencedDir = path.join(rootDir, "referenced-project");
|
||||
await initGitRepo(referencedDir);
|
||||
// The gitignore pattern itself escapes `[` and `]` (gitignore patterns are
|
||||
// globs too), so it ignores ONLY the literal file `secret[1].txt`.
|
||||
await writeFile(path.join(referencedDir, ".gitignore"), "secret\\[1\\].txt\n", "utf8");
|
||||
await writeFile(path.join(referencedDir, "secret[1].txt"), "ignored\n", "utf8");
|
||||
// A sibling that would ALSO match the UNESCAPED tar exclude glob
|
||||
// `secret[1].txt` (its `[1]` is a one-character class matching `1`), if the
|
||||
// staging path failed to escape the ignored entry before passing it to tar.
|
||||
await writeFile(path.join(referencedDir, "secret1.txt"), "must stay\n", "utf8");
|
||||
|
||||
const ignoreResolution = await resolveReferencedSourceIgnore(referencedDir);
|
||||
expect(ignoreResolution).toEqual({ kind: "git", ignoredPaths: ["secret[1].txt"] });
|
||||
|
||||
const prepared = await stageOneReferencedProject(referencedDir, ignoreResolution);
|
||||
|
||||
const stagedDir = prepared.additionalSourceDirs["proj"]!;
|
||||
await expect(readFile(path.join(stagedDir, "secret[1].txt"), "utf8")).rejects.toMatchObject({ code: "ENOENT" });
|
||||
await expect(readFile(path.join(stagedDir, "secret1.txt"), "utf8")).resolves.toBe("must stay\n");
|
||||
});
|
||||
|
||||
it("never ships a Git-ignored secret in a referenced project's staged tree", async () => {
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-ignore-secret-"));
|
||||
cleanupDirs.push(rootDir);
|
||||
const referencedDir = path.join(rootDir, "referenced-project");
|
||||
await initGitRepo(referencedDir);
|
||||
await writeFile(path.join(referencedDir, ".gitignore"), "secret.env\n", "utf8");
|
||||
await writeFile(path.join(referencedDir, "secret.env"), "TOKEN=abc\n", "utf8");
|
||||
await writeFile(path.join(referencedDir, "tracked.md"), "kept\n", "utf8");
|
||||
|
||||
const ignoreResolution = await resolveReferencedSourceIgnore(referencedDir);
|
||||
expect(ignoreResolution).toEqual({ kind: "git", ignoredPaths: ["secret.env"] });
|
||||
|
||||
const prepared = await stageOneReferencedProject(referencedDir, ignoreResolution);
|
||||
|
||||
const stagedDir = prepared.additionalSourceDirs["proj"]!;
|
||||
await expect(readFile(path.join(stagedDir, "secret.env"), "utf8")).rejects.toMatchObject({ code: "ENOENT" });
|
||||
await expect(readFile(path.join(stagedDir, "tracked.md"), "utf8")).resolves.toBe("kept\n");
|
||||
});
|
||||
|
||||
it("stages no bytes for a project whose ignore resolution failed, and stages the rest", async () => {
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-ignore-failed-skip-"));
|
||||
cleanupDirs.push(rootDir);
|
||||
const localWorkspaceDir = path.join(rootDir, "local-workspace");
|
||||
const remoteWorkspaceDir = path.join(rootDir, "remote-workspace");
|
||||
const healthyDir = path.join(rootDir, "referenced-healthy");
|
||||
const failedDir = path.join(rootDir, "referenced-failed");
|
||||
await mkdir(localWorkspaceDir, { recursive: true });
|
||||
await mkdir(healthyDir, { recursive: true });
|
||||
await mkdir(failedDir, { recursive: true });
|
||||
await writeFile(path.join(localWorkspaceDir, "README.md"), "anchor\n", "utf8");
|
||||
await writeFile(path.join(healthyDir, "notes.md"), "healthy\n", "utf8");
|
||||
await writeFile(path.join(failedDir, "should-never-ship.txt"), "must not stage\n", "utf8");
|
||||
|
||||
const prepared = await prepareCommandManagedRuntime({
|
||||
runner: makeInlineSpawnRunner(),
|
||||
spec: { remoteCwd: remoteWorkspaceDir, timeoutMs: 30_000 },
|
||||
adapterKey: "test-adapter",
|
||||
workspaceLocalDir: localWorkspaceDir,
|
||||
additionalSources: [
|
||||
{ localPath: healthyDir, projectId: "healthy", ignoreResolution: { kind: "other" } },
|
||||
{ localPath: failedDir, projectId: "failed", ignoreResolution: { kind: "failed", reason: "boom: git status timed out" } },
|
||||
],
|
||||
});
|
||||
|
||||
// The failed project is not staged at all (fail closed) — no bytes reach the
|
||||
// sandbox for it — and is recorded as a first-class failure. The healthy
|
||||
// project stages normally.
|
||||
expect(Object.keys(prepared.additionalSourceDirs)).toEqual(["healthy"]);
|
||||
expect(prepared.additionalSourceFailures.map((failure) => failure.projectId)).toEqual(["failed"]);
|
||||
expect(prepared.additionalSourceFailures[0]!.error).toContain("boom: git status timed out");
|
||||
const runtimeRootDir = path.posix.join(remoteWorkspaceDir, ".paperclip-runtime", "test-adapter");
|
||||
await expect(readFile(path.join(runtimeRootDir, "project-failed", "should-never-ship.txt"), "utf8")).rejects
|
||||
.toMatchObject({ code: "ENOENT" });
|
||||
});
|
||||
|
||||
it("builds the workspace tarball inside one host pack span for a usual workspace sync", async () => {
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-pack-span-"));
|
||||
cleanupDirs.push(rootDir);
|
||||
|
|
@ -2655,8 +2861,8 @@ describe("sandbox managed runtime inbound coordinator", () => {
|
|||
syncWorkspace: false,
|
||||
workspaceLocalDir: workspaceDir,
|
||||
additionalSources: [
|
||||
{ localPath: dirOf("good"), projectId: "good" },
|
||||
{ localPath: dirOf("bad"), projectId: "bad" },
|
||||
{ localPath: dirOf("good"), projectId: "good", ignoreResolution: { kind: "other" } },
|
||||
{ localPath: dirOf("bad"), projectId: "bad", ignoreResolution: { kind: "other" } },
|
||||
],
|
||||
});
|
||||
|
||||
|
|
@ -2748,7 +2954,7 @@ describe("sandbox managed runtime inbound coordinator", () => {
|
|||
client,
|
||||
workspaceLocalDir: workspaceDir,
|
||||
assets: [{ key: "home", localDir: dirOf("home") }],
|
||||
additionalSources: [{ localPath: dirOf("proj"), projectId: "proj-1" }],
|
||||
additionalSources: [{ localPath: dirOf("proj"), projectId: "proj-1", ignoreResolution: { kind: "other" } }],
|
||||
runtimeSpan,
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ import {
|
|||
GIT_ARCHIVE_EXCLUDES,
|
||||
integrateImportedGitHead,
|
||||
readGitWorkspaceSnapshot,
|
||||
readReferencedSourceGitIgnoredPaths,
|
||||
resetLocalGitIndexToHead,
|
||||
withShallowGitWorkspaceClone,
|
||||
} from "./git-workspace-sync.js";
|
||||
|
|
@ -135,6 +136,27 @@ export interface SandboxManagedRuntimeAsset {
|
|||
restore?: (ctx: SandboxManagedRuntimeAssetRestoreContext) => Promise<void>;
|
||||
}
|
||||
|
||||
/**
|
||||
* How a referenced project's Git-ignored paths were resolved, computed once
|
||||
* per project by `resolveReferencedSourceIgnore` before staging starts. The
|
||||
* sandbox lane, the SSH lane, and the content-signature walk each consume
|
||||
* this ONE resolution, so the three sites never drift apart.
|
||||
*
|
||||
* - `git`: `localPath` is a Git work tree. `ignoredPaths` are its ignored
|
||||
* entries, already re-relativized to `localPath` (see
|
||||
* `resolveReferencedSourceIgnore`).
|
||||
* - `other`: `localPath` is not a Git work tree. The staging path keeps
|
||||
* today's fixed heavy-directory excludes.
|
||||
* - `failed`: the Git read failed, timed out, or returned output the
|
||||
* resolver could not parse or safely re-relativize. The project is NOT
|
||||
* staged (fail closed) — every site records it as a per-project failure
|
||||
* instead of shipping it unfiltered.
|
||||
*/
|
||||
export type ReferencedSourceIgnoreResolution =
|
||||
| { kind: "git"; ignoredPaths: string[] }
|
||||
| { kind: "other" }
|
||||
| { kind: "failed"; reason: string };
|
||||
|
||||
/**
|
||||
* A referenced (additional) project to stage into the run sandbox as a plain,
|
||||
* read-only tree. `localPath` is the host checkout directory. Upstream code
|
||||
|
|
@ -145,10 +167,113 @@ export interface SandboxManagedRuntimeAsset {
|
|||
* Additional sources are plain trees only. They never carry the anchor
|
||||
* workspace's git-history, overlay, or `.paperclip-runtime` preservation
|
||||
* semantics — those stay anchor-only.
|
||||
*
|
||||
* `ignoreResolution` is required so every construction site must supply it
|
||||
* explicitly — a caller cannot default to the unfiltered legacy behavior by
|
||||
* omission. Resolve it once per project with `resolveReferencedSourceIgnore`.
|
||||
*/
|
||||
export interface SandboxAdditionalSource {
|
||||
localPath: string;
|
||||
projectId: string;
|
||||
ignoreResolution: ReferencedSourceIgnoreResolution;
|
||||
}
|
||||
|
||||
/**
|
||||
* Escape tar `--exclude` glob metacharacters (`*`, `?`, `[`) in a literal
|
||||
* path, so a Git-ignored path that happens to contain one of them is matched
|
||||
* literally instead of as a pattern. Without this, a repository-controlled
|
||||
* path containing e.g. `*` could exclude unrelated sibling files that
|
||||
* happen to match the resulting glob. GNU tar and bsdtar both honor a
|
||||
* backslash as a `fnmatch` escape character, so this is not command
|
||||
* injection — `createTarballFromDirectory` and the SSH tar equivalent both
|
||||
* pass `--exclude` values as argument-vector entries, never through a shell.
|
||||
*/
|
||||
export function escapeTarExcludeLiteral(entry: string): string {
|
||||
return entry.replace(/\\/g, "\\\\").replace(/([*?[])/g, "\\$1");
|
||||
}
|
||||
|
||||
/**
|
||||
* The tar `--exclude` entries a referenced project's resolved ignore set
|
||||
* contributes, on top of the fixed heavy-directory excludes every site
|
||||
* already applies. Empty for `other` (today's fixed excludes are enough)
|
||||
* and for `failed` (the project is not staged at all, so no exclude list
|
||||
* matters).
|
||||
*/
|
||||
export function referencedSourceIgnoreExcludeEntries(resolution: ReferencedSourceIgnoreResolution): string[] {
|
||||
return resolution.kind === "git" ? resolution.ignoredPaths.map(escapeTarExcludeLiteral) : [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute the relative position of `localPath` under a Git `toplevel`
|
||||
* directory, as a POSIX path with no leading or trailing slash. Returns `""`
|
||||
* when `localPath` IS the toplevel. Returns `null` when the relation is not a
|
||||
* plain descendant — `localPath` escapes upward from `toplevel`, resolves to
|
||||
* an absolute/rooted result (a different filesystem root), or the two paths
|
||||
* are otherwise not comparable. The caller treats `null` as a resolution
|
||||
* failure (fail closed), never as "nothing to exclude".
|
||||
*/
|
||||
function relativizeUnderGitToplevel(input: { toplevel: string; localPath: string }): string | null {
|
||||
const toplevel = path.resolve(input.toplevel);
|
||||
const localPath = path.resolve(input.localPath);
|
||||
const relative = path.relative(toplevel, localPath);
|
||||
if (relative === "") return "";
|
||||
if (path.isAbsolute(relative)) return null;
|
||||
if (relative === ".." || relative.startsWith(`..${path.sep}`)) return null;
|
||||
return relative.split(path.sep).join("/");
|
||||
}
|
||||
|
||||
/**
|
||||
* Re-relativize root-relative ignored paths (as `git status --ignored`
|
||||
* reports them, from the repository toplevel) to `offset`, the position of
|
||||
* the referenced project's `localPath` under that toplevel. Keeps only the
|
||||
* entries that are `offset` itself or a descendant of it — an ignored path
|
||||
* elsewhere in the repository does not apply to this project's staged tree —
|
||||
* and strips the `offset` prefix so the result matches the tar member
|
||||
* namespace, which is `localPath`-relative.
|
||||
*/
|
||||
function reRelativizeIgnoredPathsToLocalPath(input: { ignoredPaths: string[]; offset: string }): string[] {
|
||||
if (input.offset === "") {
|
||||
return [...input.ignoredPaths];
|
||||
}
|
||||
const prefix = `${input.offset}/`;
|
||||
return input.ignoredPaths
|
||||
.filter((entry) => entry.startsWith(prefix))
|
||||
.map((entry) => entry.slice(prefix.length))
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve a referenced project's Git-ignored paths ONCE, before any staging
|
||||
* site runs. Called once per project (see `execute.ts`); the sandbox lane,
|
||||
* the SSH lane, and the content-signature walk all consume this one result,
|
||||
* so they can never apply a different exclusion set to the same project.
|
||||
*
|
||||
* Fails closed: a Git read error, a timeout, malformed output, or a `localPath`
|
||||
* that is not a plain descendant of its own Git toplevel all return `failed`,
|
||||
* never an empty ignore list — an empty list means "resolved, nothing extra to
|
||||
* exclude", which is a different claim than "the resolution did not run".
|
||||
*/
|
||||
export async function resolveReferencedSourceIgnore(localPath: string): Promise<ReferencedSourceIgnoreResolution> {
|
||||
let scan: Awaited<ReturnType<typeof readReferencedSourceGitIgnoredPaths>>;
|
||||
try {
|
||||
scan = await readReferencedSourceGitIgnoredPaths(localPath);
|
||||
} catch (error) {
|
||||
return { kind: "failed", reason: error instanceof Error ? error.message : String(error) };
|
||||
}
|
||||
if (!scan) {
|
||||
return { kind: "other" };
|
||||
}
|
||||
const offset = relativizeUnderGitToplevel({ toplevel: scan.toplevel, localPath });
|
||||
if (offset === null) {
|
||||
return {
|
||||
kind: "failed",
|
||||
reason: `referenced project path is not a descendant of its own Git top level: ${localPath} under ${scan.toplevel}`,
|
||||
};
|
||||
}
|
||||
return {
|
||||
kind: "git",
|
||||
ignoredPaths: reRelativizeIgnoredPathsToLocalPath({ ignoredPaths: scan.ignoredPaths, offset }),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -681,7 +806,7 @@ async function emitRuntimeStatus(
|
|||
await Promise.resolve(sink({ phase, message })).catch(() => undefined);
|
||||
}
|
||||
|
||||
function mergeExcludes(...groups: Array<string[] | undefined>): string[] {
|
||||
export function mergeExcludes(...groups: Array<string[] | undefined>): string[] {
|
||||
return [...new Set(groups.flatMap((group) => group ?? []))];
|
||||
}
|
||||
|
||||
|
|
@ -850,8 +975,11 @@ export async function prepareSandboxManagedRuntime(input: {
|
|||
const additionalSourceFailures: AdditionalSourceStagingFailure[] = [];
|
||||
// Additional projects stage as plain trees. Drop the heavy build/cache dirs a
|
||||
// reference tree does not need, and `.git` — additional sources never carry
|
||||
// git-history semantics (anchor-only).
|
||||
const additionalSourceExclude = mergeExcludes(SANDBOX_WORKSPACE_HEAVY_DIR_EXCLUDES, [".git"]);
|
||||
// git-history semantics (anchor-only). Each project also drops its OWN
|
||||
// resolved Git-ignored paths (or keeps this fixed set as-is for a non-Git
|
||||
// source) — see `resolveReferencedSourceIgnore` and the per-project merge
|
||||
// below.
|
||||
const additionalSourceBaseExclude = mergeExcludes(SANDBOX_WORKSPACE_HEAVY_DIR_EXCLUDES, [".git"]);
|
||||
|
||||
// Every delegated post-upload command (extract/wipe/remove-deleted/asset merge)
|
||||
// must run under the run-specific timeout (`spec.timeoutMs`), not the provider
|
||||
|
|
@ -1122,7 +1250,7 @@ export async function prepareSandboxManagedRuntime(input: {
|
|||
inboundTaskIsRequired.push(false);
|
||||
inboundTasks.push(() =>
|
||||
runStepSpan(`stage.project.${source.projectId}`, async () => {
|
||||
const { localPath, projectId } = source;
|
||||
const { localPath, projectId, ignoreResolution } = source;
|
||||
const label = `project-${projectId}`;
|
||||
try {
|
||||
if (!path.posix.isAbsolute(localPath)) {
|
||||
|
|
@ -1136,14 +1264,24 @@ export async function prepareSandboxManagedRuntime(input: {
|
|||
) {
|
||||
throw new Error(`additional source projectId is not a simple path segment: ${projectId}`);
|
||||
}
|
||||
// Fail closed: a project whose ignore resolution failed is not staged
|
||||
// at all. Shipping it with only the fixed heavy-directory excludes
|
||||
// would defeat the resolution's purpose.
|
||||
if (ignoreResolution.kind === "failed") {
|
||||
throw new Error(`referenced project ignore resolution failed: ${ignoreResolution.reason}`);
|
||||
}
|
||||
const remoteProjectDir = path.posix.join(runtimeRootDir, label);
|
||||
const exclude = mergeExcludes(
|
||||
additionalSourceBaseExclude,
|
||||
referencedSourceIgnoreExcludeEntries(ignoreResolution),
|
||||
);
|
||||
await emitRuntimeStatus(input.onRuntimeProgress, "config_sync", "Syncing referenced project to environment");
|
||||
await stageConfinedSyncIn({
|
||||
files: [{
|
||||
sourcePath: localPath,
|
||||
targetPath: remoteProjectDir,
|
||||
kind: "directory",
|
||||
exclude: additionalSourceExclude,
|
||||
exclude,
|
||||
access: "ro",
|
||||
}],
|
||||
sourceRoots: [localPath],
|
||||
|
|
|
|||
|
|
@ -882,6 +882,11 @@ setExpensiveWorkspaceGitExecutor(async (input) => {
|
|||
timeoutMs: input.timeout,
|
||||
maxStdoutBytes: input.maxBuffer,
|
||||
maxStderrBytes: input.maxBuffer,
|
||||
// Absent for the anchor workspace's own reads (they inherit the process
|
||||
// environment, a directory this process already controls). A
|
||||
// referenced-project scan sets this to its hardened environment, so the
|
||||
// hardening survives the hop through this shared scheduler.
|
||||
env: input.env,
|
||||
});
|
||||
return { stdout: result.stdout, stderr: result.stderr };
|
||||
});
|
||||
|
|
|
|||
Loading…
Reference in New Issue