feat(server): surface referenced-project sync warnings and enable multi-project sync by default (#10473)

## Thinking Path

> - Paperclip helps people manage AI agent work for a company
> - Workspace sync must keep one primary path and any referenced project
paths in step
> - A partial sync must not fail in silence
> - Operators also need a clear signal when the feature uses the new
default path
> - This pull request surfaces referenced-project warnings on the run
and turns the feature flag default on
> - The benefit is better visibility and a live multi-project sync path
by default

## Linked Issues or Issue Description

This pull request completes the multi-project workspace sync go-live
work.

Related pull requests:

- Refs: #10380
- Refs: #10448
- Refs: #10469

## What Changed

- Surface referenced-project warnings on the run when a project drops
during authorization or resolution.
- Record a structured failure reason for each dropped referenced
project.
- Emit one structured log line at run preparation with the requested
count, the synced count, and the failure reasons.
- Flip the workspace sync kill-switch default to on when the env value
is unset.
- Keep the primary workspace path unchanged.

## Verification

- `tsc --noEmit` passed in the server package.
- `heartbeat-referenced-projects.test.ts` and
`heartbeat-project-env.test.ts` passed.
- `workspace-runtime.test.ts` passed.
- `adapter-utils` runtime tests passed for sandbox, command, remote, and
file sync paths.

## Risks

- The new default can expose the feature to more runs if an operator
does not set the env override.
- The new surfaced warnings can change operator visible run output.
- The structured log line can add noise if a run has many referenced
project failures.

## Model Used

- OpenAI Codex, GPT-5, tool use, large context window.

## Checklist

- [x] I have included a thinking path that traces from project context
to this change
- [x] I have specified the model used with version and capability
details
- [x] I have checked ROADMAP.md and confirmed this PR does not duplicate
planned core work
- [x] I have searched GitHub for duplicate or related PRs and linked
them above
- [x] I have either linked existing issues with `Fixes: #` / `Closes #`
/ `Refs #` or described the issue in-PR following the relevant issue
template
- [x] I have not referenced internal or instance-local Paperclip issues
or links
- [x] My branch name describes the change and contains no internal
Paperclip ticket id or instance-derived details
- [x] I have run tests locally and they pass
- [x] I have added or updated tests where applicable
- [x] I have updated relevant documentation to reflect my changes
- [x] I have considered and documented any risks above
- [x] All Paperclip CI gates are green
- [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups
- [x] I will address all Greptile and reviewer comments before
requesting merge

---------

Co-authored-by: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Nicky Leach 2026-07-29 17:12:22 -07:00 committed by GitHub
parent 6a267e0328
commit a93a74f91a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 257 additions and 21 deletions

View File

@ -6,6 +6,7 @@ import { buildSkillMentionHref } from "@paperclipai/shared";
import {
LOW_TRUST_REVIEW_PRESET,
applyRunScopedMentionedSkillKeys,
buildReferencedProjectRunObservability,
buildRunWorkspaceHints,
extractMentionedSkillIdsFromSources,
resolveAdditionalProjectWorkspace,
@ -835,7 +836,7 @@ describe("resolveAdditionalRunWorkspaces", () => {
const result = await resolveAdditionalRunWorkspaces(issueId, null, options);
expect(result).toEqual({ additionalWorkspaces: [], warnings: [] });
expect(result).toEqual({ additionalWorkspaces: [], warnings: [], failures: [] });
// The flag-off path must be inert: no mention lookup and no workspace resolution run.
expect(mentionLookups).toBe(0);
expect(workspaceResolves).toBe(0);
@ -872,6 +873,48 @@ describe("resolveAdditionalRunWorkspaces", () => {
expect(result.warnings).toHaveLength(1);
expect(result.warnings[0]).toContain("project-a");
expect(result.warnings[0]).toContain("clone exploded");
expect(result.failures).toEqual([{ projectId: "project-a", reason: "resolution" }]);
});
it("aggregates per-layer warnings onto the run's surfaced warnings", async () => {
// Two valid projects sync; one is dropped by authorization and one fails to resolve. The run
// still resolves, and both dropped projects are named in the surfaced warnings, one per layer.
const options = buildOptions({
mentionedIds: ["project-a", "project-b", "denied-project", "broken-project"],
access: {
decide: async (input) =>
input.scope?.projectId === "denied-project"
? { allowed: false, action: "project:read", reason: "deny_no_grant", explanation: "denied" }
: allowDecision,
},
resolveProjectWorkspace: async (project) => {
if (project.projectId === "broken-project") {
throw new Error("clone exploded");
}
return {
cwd: `/managed/${project.projectId}`,
projectId: project.projectId,
workspaceId: null,
repoUrl: null,
repoRef: null,
};
},
});
const result = await resolveAdditionalRunWorkspaces(issueId, null, options);
expect(result.additionalWorkspaces.map((workspace) => workspace.projectId)).toEqual([
"project-a",
"project-b",
]);
// Each dropped project is named in a surfaced warning, across both layers.
expect(result.warnings.some((warning) => warning.includes("denied-project"))).toBe(true);
expect(result.warnings.some((warning) => warning.includes("broken-project"))).toBe(true);
// The structured failures carry the layer that dropped each project.
expect(result.failures).toEqual([
{ projectId: "denied-project", reason: "authorization" },
{ projectId: "broken-project", reason: "resolution" },
]);
});
it("skips referenced-project work on a remote target and warns when the issue mentions a project", async () => {
@ -903,11 +946,31 @@ describe("resolveAdditionalRunWorkspaces", () => {
expect(result.additionalWorkspaces).toEqual([]);
expect(result.warnings).toHaveLength(1);
expect(result.warnings[0]).toContain("local execution target");
// The remote path drops every referenced project at the staging layer. Each dropped project
// keeps a structured failure so the requested-vs-synced accounting counts the whole set and the
// run still emits its structured sync log.
expect(result.failures).toEqual([
{ projectId: "project-a", reason: "staging" },
{ projectId: "project-b", reason: "staging" },
]);
// The remote path must skip the per-project clone work whose result the target cannot receive.
expect(mentionLookups).toBe(1);
expect(workspaceResolves).toBe(0);
});
it("drops a duplicate remote referenced mention to a single staging failure and ignores the anchor", async () => {
const options = buildOptions({
executionTargetIsRemote: true,
mentionedIds: ["anchor-project", "project-a", "project-a"],
});
const result = await resolveAdditionalRunWorkspaces(issueId, "anchor-project", options);
// The anchor is not a referenced drop, and a repeated mention counts once.
expect(result.failures).toEqual([{ projectId: "project-a", reason: "staging" }]);
expect(result.warnings).toHaveLength(1);
});
it("stays silent on a remote target when the issue mentions no referenced project", async () => {
const options = buildOptions({
executionTargetIsRemote: true,
@ -916,7 +979,40 @@ describe("resolveAdditionalRunWorkspaces", () => {
const result = await resolveAdditionalRunWorkspaces(issueId, null, options);
expect(result).toEqual({ additionalWorkspaces: [], warnings: [] });
expect(result).toEqual({ additionalWorkspaces: [], warnings: [], failures: [] });
});
});
describe("buildReferencedProjectRunObservability", () => {
it("emits referenced_projects_requested vs referenced_projects_synced with failure reason", () => {
const observability = buildReferencedProjectRunObservability({
syncedProjectIds: ["project-a"],
failures: [
{ projectId: "project-b", reason: "authorization" },
{ projectId: "project-c", reason: "resolution" },
{ projectId: "project-d", reason: "staging" },
],
});
// Requested is the synced count plus every dropped project, so the counts reconcile.
expect(observability.referenced_projects_requested).toBe(4);
expect(observability.referenced_projects_synced).toBe(1);
expect(observability.referenced_project_failures).toEqual([
{ project_id: "project-b", reason: "authorization" },
{ project_id: "project-c", reason: "resolution" },
{ project_id: "project-d", reason: "staging" },
]);
});
it("emits zero counts and no failures when no project is referenced", () => {
const observability = buildReferencedProjectRunObservability({
syncedProjectIds: [],
failures: [],
});
expect(observability.referenced_projects_requested).toBe(0);
expect(observability.referenced_projects_synced).toBe(0);
expect(observability.referenced_project_failures).toEqual([]);
});
});

View File

@ -33,11 +33,15 @@ const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
describe("multi-project workspace sync kill-switch", () => {
it("is OFF by default and enabled only by truthy env values", () => {
expect(isMultiProjectWorkspaceSyncEnabled({})).toBe(false);
it("is ON by default and disabled only by an explicit false env value", () => {
// Default ON: an unset value resolves the feature live (go-live default).
expect(isMultiProjectWorkspaceSyncEnabled({})).toBe(true);
// The kill switch: an explicit false value (the rollback path) disables it.
expect(isMultiProjectWorkspaceSyncEnabled({ [MULTI_PROJECT_WORKSPACE_SYNC_ENV]: "" })).toBe(false);
expect(isMultiProjectWorkspaceSyncEnabled({ [MULTI_PROJECT_WORKSPACE_SYNC_ENV]: "false" })).toBe(false);
expect(isMultiProjectWorkspaceSyncEnabled({ [MULTI_PROJECT_WORKSPACE_SYNC_ENV]: "0" })).toBe(false);
expect(isMultiProjectWorkspaceSyncEnabled({ [MULTI_PROJECT_WORKSPACE_SYNC_ENV]: "off" })).toBe(false);
// Any other value keeps the feature on.
expect(isMultiProjectWorkspaceSyncEnabled({ [MULTI_PROJECT_WORKSPACE_SYNC_ENV]: "true" })).toBe(true);
expect(isMultiProjectWorkspaceSyncEnabled({ [MULTI_PROJECT_WORKSPACE_SYNC_ENV]: "1" })).toBe(true);
expect(isMultiProjectWorkspaceSyncEnabled({ [MULTI_PROJECT_WORKSPACE_SYNC_ENV]: "on" })).toBe(true);

View File

@ -2272,10 +2272,19 @@ export type ResolvedWorkspaceForRun = {
* workspace only, exactly as before.
*/
additionalWorkspaces: ResolvedAdditionalWorkspace[];
/**
* Structured record of every referenced project that the run dropped or failed, paired with the
* layer that dropped it. Run preparation reads this to emit the requested-vs-synced observability
* log. The human-readable form of each drop already rides {@link ResolvedWorkspaceForRun.warnings}.
*/
referencedProjectFailures: ReferencedProjectFailure[];
};
/** The anchor workspace shape, before the additional referenced workspaces are attached. */
type ResolvedAnchorWorkspaceForRun = Omit<ResolvedWorkspaceForRun, "additionalWorkspaces">;
type ResolvedAnchorWorkspaceForRun = Omit<
ResolvedWorkspaceForRun,
"additionalWorkspaces" | "referencedProjectFailures"
>;
/**
* Build the plural workspace list that a run exposes to the agent through the
@ -2319,17 +2328,40 @@ export function prioritizeProjectWorkspaceCandidatesForRun<T extends ProjectWork
}
/**
* Environment flag (kill-switch, default OFF) that gates whether run preparation
* Environment flag (kill-switch, default ON) that gates whether run preparation
* consumes the multi-project referenced-project set produced by
* {@link resolveRunReferencedProjects}. While unset/off, a run materializes only the
* anchor project's workspace exactly as before the referenced set is inert.
* {@link resolveRunReferencedProjects}. The feature is live by default: an unset
* value resolves ON. An operator disables the feature with an explicit false
* value (`"false"`, `"0"`, `"off"`, or `""`). While off, a run materializes only
* the anchor project's workspace exactly as before the referenced set is inert.
*/
export const MULTI_PROJECT_WORKSPACE_SYNC_ENV = "PAPERCLIP_MULTI_PROJECT_WORKSPACE_SYNC";
/**
* True when an environment value explicitly turns a flag off. An unset value is
* not false the caller decides the unset default. This is the inverse of
* {@link isTruthyRuntimeEnvValue} for the kill-switch words plus the empty string.
*/
function isFalsyRuntimeEnvValue(value: string | undefined): boolean {
if (value === undefined) {
return false;
}
const normalized = value.trim().toLowerCase();
return (
normalized === "" ||
normalized === "false" ||
normalized === "0" ||
normalized === "off" ||
normalized === "no"
);
}
export function isMultiProjectWorkspaceSyncEnabled(
env: Record<string, string | undefined> = process.env,
): boolean {
return isTruthyRuntimeEnvValue(env[MULTI_PROJECT_WORKSPACE_SYNC_ENV]);
// Default ON: an unset value is not false, so the feature is live unless an
// operator sets an explicit false value as the kill switch (rollback path).
return !isFalsyRuntimeEnvValue(env[MULTI_PROJECT_WORKSPACE_SYNC_ENV]);
}
/**
@ -2374,6 +2406,23 @@ export interface RunReferencedProject {
project: RunReferencedProjectRecord;
}
/**
* The layer that dropped or failed a referenced project. The run surfacing and the
* observability log both use these values as the per-failure reason:
* - `authorization`: the run actor is not authorized to read the project.
* - `resolution`: the project could not be brought into the run locally (unknown or
* unavailable project, cap exceeded, or a workspace clone/prepare failure).
* - `staging`: the project resolved but failed to stage into the run sandbox (the
* downstream remote path; see `sandbox-managed-runtime`).
*/
export type ReferencedProjectFailureReason = "authorization" | "resolution" | "staging";
/** One referenced project that a run dropped or failed, with the layer that caused it. */
export interface ReferencedProjectFailure {
projectId: string;
reason: ReferencedProjectFailureReason;
}
export interface ResolvedRunReferencedProjects {
/** The anchor (primary) project — retains the existing git-worktree run path; never re-authorized here. */
anchor: RunReferencedProject | null;
@ -2381,6 +2430,8 @@ export interface ResolvedRunReferencedProjects {
additional: RunReferencedProject[];
/** Human-readable warnings for every referenced project that was dropped (unavailable, unauthorized, or capped). */
warnings: string[];
/** Structured record of every dropped referenced project, paired with the layer that dropped it. */
failures: ReferencedProjectFailure[];
}
export interface ResolveRunReferencedProjectsOptions {
@ -2430,6 +2481,7 @@ export async function resolveRunReferencedProjects(
): Promise<ResolvedRunReferencedProjects> {
const { companyId, actor, issues, projects, access } = opts;
const warnings: string[] = [];
const failures: ReferencedProjectFailure[] = [];
const cap = Math.max(0, opts.maxAdditionalProjects ?? MAX_RUN_REFERENCED_ADDITIONAL_PROJECTS);
// The evaluation cap bounds candidate hydration + authorization fan-out. It is always at least the
// admitted cap so the admitted cap stays reachable in the normal (non-flood) case.
@ -2485,6 +2537,7 @@ export async function resolveRunReferencedProjects(
const project = byId.get(projectId);
if (!project) {
warnings.push(`Referenced project ${projectId} was skipped because it is not available in this company.`);
failures.push({ projectId, reason: "resolution" });
continue;
}
availableCandidates.push({ projectId, project });
@ -2538,6 +2591,7 @@ export async function resolveRunReferencedProjects(
if (!allowed) {
warnings.push(`Referenced project ${projectId} was skipped because it is not authorized for this run.`);
failures.push({ projectId, reason: "authorization" });
continue;
}
@ -2560,7 +2614,20 @@ export async function resolveRunReferencedProjects(
);
}
return { anchor, additional, warnings };
// Record every capped or unevaluated candidate as a per-project resolution failure so the run
// surfacing and the observability log can reconcile requested against synced. The evaluated
// candidates past the admitted cap carry a known projectId; the candidates the fan-out cap
// dropped before hydration carry their id from the ordered mention set.
if (capReachedAtIndex !== null) {
for (let index = capReachedAtIndex; index < candidates.length; index++) {
failures.push({ projectId: candidates[index]!.projectId, reason: "resolution" });
}
}
for (const projectId of allCandidateIds.slice(hydrationCursor)) {
failures.push({ projectId, reason: "resolution" });
}
return { anchor, additional, warnings, failures };
}
export interface ResolveAdditionalRunWorkspacesOptions {
@ -2599,9 +2666,13 @@ export async function resolveAdditionalRunWorkspaces(
issueId: string | null,
anchorProjectId: string | null,
opts: ResolveAdditionalRunWorkspacesOptions,
): Promise<{ additionalWorkspaces: ResolvedAdditionalWorkspace[]; warnings: string[] }> {
): Promise<{
additionalWorkspaces: ResolvedAdditionalWorkspace[];
warnings: string[];
failures: ReferencedProjectFailure[];
}> {
if (!opts.enabled || !issueId) {
return { additionalWorkspaces: [], warnings: [] };
return { additionalWorkspaces: [], warnings: [], failures: [] };
}
// A referenced project realizes as a local directory only. A remote execution target (ssh,
@ -2615,14 +2686,22 @@ export async function resolveAdditionalRunWorkspaces(
const mentionedIds = await opts.issues.findMentionedProjectIds(issueId, {
includeCommentBodies: true,
});
const hasReferencedMention = mentionedIds.some((projectId) => projectId !== anchorProjectId);
// Each distinct non-anchor mention is a referenced project this remote run drops. A remote
// target has no path to receive the referenced tree, so the run drops the whole set at the
// staging layer. Record one failure per dropped project so the requested-vs-synced accounting
// counts the whole referenced set and the run still emits its structured sync log.
const droppedProjectIds = [
...new Set(mentionedIds.filter((projectId) => projectId !== anchorProjectId)),
];
return {
additionalWorkspaces: [],
warnings: hasReferencedMention
? [
"Referenced-project workspaces are available only on a local execution target. This run uses a remote execution target, so no referenced-project workspace was attached.",
]
: [],
warnings:
droppedProjectIds.length > 0
? [
"Referenced-project workspaces are available only on a local execution target. This run uses a remote execution target, so no referenced-project workspace was attached.",
]
: [],
failures: droppedProjectIds.map((projectId) => ({ projectId, reason: "staging" as const })),
};
}
@ -2638,6 +2717,7 @@ export async function resolveAdditionalRunWorkspaces(
const additionalWorkspaces: ResolvedAdditionalWorkspace[] = [];
const warnings = [...referenced.warnings];
const failures = [...referenced.failures];
for (const project of referenced.additional) {
try {
additionalWorkspaces.push(await opts.resolveProjectWorkspace(project));
@ -2646,10 +2726,44 @@ export async function resolveAdditionalRunWorkspaces(
warnings.push(
`Referenced project ${project.projectId} was skipped because its workspace could not be prepared: ${reason}`,
);
failures.push({ projectId: project.projectId, reason: "resolution" });
}
}
return { additionalWorkspaces, warnings };
return { additionalWorkspaces, warnings, failures };
}
/** Structured fields for the one requested-vs-synced observability log a run emits at run prep. */
export interface ReferencedProjectRunObservability {
referenced_projects_requested: number;
referenced_projects_synced: number;
referenced_project_failures: Array<{
project_id: string;
reason: ReferencedProjectFailureReason;
}>;
}
/**
* Build the requested-vs-synced observability fields for a run's referenced-project set.
*
* A run requests one referenced project per authorized mention and syncs the projects that resolve.
* The requested count is the synced count plus every dropped project, so the two counts and the
* per-failure reasons together account for the whole referenced set. The human-readable warning for
* each drop rides the run's surfaced warnings channel; this function produces only the structured
* log fields, so a run emits exactly one line with a stable field shape.
*/
export function buildReferencedProjectRunObservability(input: {
syncedProjectIds: readonly string[];
failures: readonly ReferencedProjectFailure[];
}): ReferencedProjectRunObservability {
return {
referenced_projects_requested: input.syncedProjectIds.length + input.failures.length,
referenced_projects_synced: input.syncedProjectIds.length,
referenced_project_failures: input.failures.map((failure) => ({
project_id: failure.projectId,
reason: failure.reason,
})),
};
}
function readNonEmptyString(value: unknown): string | null {
@ -8080,11 +8194,11 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
): Promise<ResolvedWorkspaceForRun> {
const anchor = await resolveAnchorWorkspaceForRun(agent, context, previousSessionParams, opts);
if (!isMultiProjectWorkspaceSyncEnabled()) {
return { ...anchor, additionalWorkspaces: [] };
return { ...anchor, additionalWorkspaces: [], referencedProjectFailures: [] };
}
const issueId = readNonEmptyString(context.issueId) ?? readNonEmptyString(context.taskId);
const { additionalWorkspaces, warnings } = await resolveAdditionalRunWorkspaces(
const { additionalWorkspaces, warnings, failures } = await resolveAdditionalRunWorkspaces(
issueId,
anchor.projectId,
{
@ -8111,6 +8225,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
return {
...anchor,
additionalWorkspaces,
referencedProjectFailures: failures,
warnings: warnings.length > 0 ? [...anchor.warnings, ...warnings] : anchor.warnings,
};
}
@ -13657,6 +13772,27 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
})(),
};
context.paperclipWorkspaces = buildRunWorkspaceHints(resolvedWorkspace);
// Emit exactly one requested-vs-synced observability line for the referenced-project set. A run
// with no referenced project stays silent, so this adds no noise to the anchor-only default. The
// per-drop human warning already rides `runtimeWorkspaceWarnings`; this line carries the counts
// and the per-failure reason for a partial sync.
const referencedProjectObservability = buildReferencedProjectRunObservability({
syncedProjectIds: resolvedWorkspace.additionalWorkspaces.map(
(additional) => additional.projectId,
),
failures: resolvedWorkspace.referencedProjectFailures,
});
if (referencedProjectObservability.referenced_projects_requested > 0) {
logger.info(
{
runId: run.id,
companyId: agent.companyId,
issueId: issueRef?.id ?? null,
...referencedProjectObservability,
},
"run referenced-project sync",
);
}
// The wake payload is built before the execution workspace is resolved, so
// attach the branch pin here; the shared wake-prompt renderer surfaces it as
// a one-time "stay on this branch" hint on non-resumed sessions.