diff --git a/server/src/__tests__/heartbeat-project-env.test.ts b/server/src/__tests__/heartbeat-project-env.test.ts index def19721e0..7ac3644bca 100644 --- a/server/src/__tests__/heartbeat-project-env.test.ts +++ b/server/src/__tests__/heartbeat-project-env.test.ts @@ -6,6 +6,7 @@ import { buildSkillMentionHref } from "@paperclipai/shared"; import { LOW_TRUST_REVIEW_PRESET, applyRunScopedMentionedSkillKeys, + buildReferencedProjectRunObservability, buildRunWorkspaceHints, extractMentionedSkillIdsFromSources, resolveAdditionalProjectWorkspace, @@ -835,7 +836,7 @@ describe("resolveAdditionalRunWorkspaces", () => { const result = await resolveAdditionalRunWorkspaces(issueId, null, options); - expect(result).toEqual({ additionalWorkspaces: [], warnings: [] }); + expect(result).toEqual({ additionalWorkspaces: [], warnings: [], failures: [] }); // The flag-off path must be inert: no mention lookup and no workspace resolution run. expect(mentionLookups).toBe(0); expect(workspaceResolves).toBe(0); @@ -872,6 +873,48 @@ describe("resolveAdditionalRunWorkspaces", () => { expect(result.warnings).toHaveLength(1); expect(result.warnings[0]).toContain("project-a"); expect(result.warnings[0]).toContain("clone exploded"); + expect(result.failures).toEqual([{ projectId: "project-a", reason: "resolution" }]); + }); + + it("aggregates per-layer warnings onto the run's surfaced warnings", async () => { + // Two valid projects sync; one is dropped by authorization and one fails to resolve. The run + // still resolves, and both dropped projects are named in the surfaced warnings, one per layer. + const options = buildOptions({ + mentionedIds: ["project-a", "project-b", "denied-project", "broken-project"], + access: { + decide: async (input) => + input.scope?.projectId === "denied-project" + ? { allowed: false, action: "project:read", reason: "deny_no_grant", explanation: "denied" } + : allowDecision, + }, + resolveProjectWorkspace: async (project) => { + if (project.projectId === "broken-project") { + throw new Error("clone exploded"); + } + return { + cwd: `/managed/${project.projectId}`, + projectId: project.projectId, + workspaceId: null, + repoUrl: null, + repoRef: null, + }; + }, + }); + + const result = await resolveAdditionalRunWorkspaces(issueId, null, options); + + expect(result.additionalWorkspaces.map((workspace) => workspace.projectId)).toEqual([ + "project-a", + "project-b", + ]); + // Each dropped project is named in a surfaced warning, across both layers. + expect(result.warnings.some((warning) => warning.includes("denied-project"))).toBe(true); + expect(result.warnings.some((warning) => warning.includes("broken-project"))).toBe(true); + // The structured failures carry the layer that dropped each project. + expect(result.failures).toEqual([ + { projectId: "denied-project", reason: "authorization" }, + { projectId: "broken-project", reason: "resolution" }, + ]); }); it("skips referenced-project work on a remote target and warns when the issue mentions a project", async () => { @@ -903,11 +946,31 @@ describe("resolveAdditionalRunWorkspaces", () => { expect(result.additionalWorkspaces).toEqual([]); expect(result.warnings).toHaveLength(1); expect(result.warnings[0]).toContain("local execution target"); + // The remote path drops every referenced project at the staging layer. Each dropped project + // keeps a structured failure so the requested-vs-synced accounting counts the whole set and the + // run still emits its structured sync log. + expect(result.failures).toEqual([ + { projectId: "project-a", reason: "staging" }, + { projectId: "project-b", reason: "staging" }, + ]); // The remote path must skip the per-project clone work whose result the target cannot receive. expect(mentionLookups).toBe(1); expect(workspaceResolves).toBe(0); }); + it("drops a duplicate remote referenced mention to a single staging failure and ignores the anchor", async () => { + const options = buildOptions({ + executionTargetIsRemote: true, + mentionedIds: ["anchor-project", "project-a", "project-a"], + }); + + const result = await resolveAdditionalRunWorkspaces(issueId, "anchor-project", options); + + // The anchor is not a referenced drop, and a repeated mention counts once. + expect(result.failures).toEqual([{ projectId: "project-a", reason: "staging" }]); + expect(result.warnings).toHaveLength(1); + }); + it("stays silent on a remote target when the issue mentions no referenced project", async () => { const options = buildOptions({ executionTargetIsRemote: true, @@ -916,7 +979,40 @@ describe("resolveAdditionalRunWorkspaces", () => { const result = await resolveAdditionalRunWorkspaces(issueId, null, options); - expect(result).toEqual({ additionalWorkspaces: [], warnings: [] }); + expect(result).toEqual({ additionalWorkspaces: [], warnings: [], failures: [] }); + }); +}); + +describe("buildReferencedProjectRunObservability", () => { + it("emits referenced_projects_requested vs referenced_projects_synced with failure reason", () => { + const observability = buildReferencedProjectRunObservability({ + syncedProjectIds: ["project-a"], + failures: [ + { projectId: "project-b", reason: "authorization" }, + { projectId: "project-c", reason: "resolution" }, + { projectId: "project-d", reason: "staging" }, + ], + }); + + // Requested is the synced count plus every dropped project, so the counts reconcile. + expect(observability.referenced_projects_requested).toBe(4); + expect(observability.referenced_projects_synced).toBe(1); + expect(observability.referenced_project_failures).toEqual([ + { project_id: "project-b", reason: "authorization" }, + { project_id: "project-c", reason: "resolution" }, + { project_id: "project-d", reason: "staging" }, + ]); + }); + + it("emits zero counts and no failures when no project is referenced", () => { + const observability = buildReferencedProjectRunObservability({ + syncedProjectIds: [], + failures: [], + }); + + expect(observability.referenced_projects_requested).toBe(0); + expect(observability.referenced_projects_synced).toBe(0); + expect(observability.referenced_project_failures).toEqual([]); }); }); diff --git a/server/src/__tests__/heartbeat-referenced-projects.test.ts b/server/src/__tests__/heartbeat-referenced-projects.test.ts index 111501b74d..30c9f642ee 100644 --- a/server/src/__tests__/heartbeat-referenced-projects.test.ts +++ b/server/src/__tests__/heartbeat-referenced-projects.test.ts @@ -33,11 +33,15 @@ const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; describe("multi-project workspace sync kill-switch", () => { - it("is OFF by default and enabled only by truthy env values", () => { - expect(isMultiProjectWorkspaceSyncEnabled({})).toBe(false); + it("is ON by default and disabled only by an explicit false env value", () => { + // Default ON: an unset value resolves the feature live (go-live default). + expect(isMultiProjectWorkspaceSyncEnabled({})).toBe(true); + // The kill switch: an explicit false value (the rollback path) disables it. expect(isMultiProjectWorkspaceSyncEnabled({ [MULTI_PROJECT_WORKSPACE_SYNC_ENV]: "" })).toBe(false); expect(isMultiProjectWorkspaceSyncEnabled({ [MULTI_PROJECT_WORKSPACE_SYNC_ENV]: "false" })).toBe(false); expect(isMultiProjectWorkspaceSyncEnabled({ [MULTI_PROJECT_WORKSPACE_SYNC_ENV]: "0" })).toBe(false); + expect(isMultiProjectWorkspaceSyncEnabled({ [MULTI_PROJECT_WORKSPACE_SYNC_ENV]: "off" })).toBe(false); + // Any other value keeps the feature on. expect(isMultiProjectWorkspaceSyncEnabled({ [MULTI_PROJECT_WORKSPACE_SYNC_ENV]: "true" })).toBe(true); expect(isMultiProjectWorkspaceSyncEnabled({ [MULTI_PROJECT_WORKSPACE_SYNC_ENV]: "1" })).toBe(true); expect(isMultiProjectWorkspaceSyncEnabled({ [MULTI_PROJECT_WORKSPACE_SYNC_ENV]: "on" })).toBe(true); diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 6886d54939..0946448a4f 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -2272,10 +2272,19 @@ export type ResolvedWorkspaceForRun = { * workspace only, exactly as before. */ additionalWorkspaces: ResolvedAdditionalWorkspace[]; + /** + * Structured record of every referenced project that the run dropped or failed, paired with the + * layer that dropped it. Run preparation reads this to emit the requested-vs-synced observability + * log. The human-readable form of each drop already rides {@link ResolvedWorkspaceForRun.warnings}. + */ + referencedProjectFailures: ReferencedProjectFailure[]; }; /** The anchor workspace shape, before the additional referenced workspaces are attached. */ -type ResolvedAnchorWorkspaceForRun = Omit; +type ResolvedAnchorWorkspaceForRun = Omit< + ResolvedWorkspaceForRun, + "additionalWorkspaces" | "referencedProjectFailures" +>; /** * Build the plural workspace list that a run exposes to the agent through the @@ -2319,17 +2328,40 @@ export function prioritizeProjectWorkspaceCandidatesForRun = process.env, ): boolean { - return isTruthyRuntimeEnvValue(env[MULTI_PROJECT_WORKSPACE_SYNC_ENV]); + // Default ON: an unset value is not false, so the feature is live unless an + // operator sets an explicit false value as the kill switch (rollback path). + return !isFalsyRuntimeEnvValue(env[MULTI_PROJECT_WORKSPACE_SYNC_ENV]); } /** @@ -2374,6 +2406,23 @@ export interface RunReferencedProject { project: RunReferencedProjectRecord; } +/** + * The layer that dropped or failed a referenced project. The run surfacing and the + * observability log both use these values as the per-failure reason: + * - `authorization`: the run actor is not authorized to read the project. + * - `resolution`: the project could not be brought into the run locally (unknown or + * unavailable project, cap exceeded, or a workspace clone/prepare failure). + * - `staging`: the project resolved but failed to stage into the run sandbox (the + * downstream remote path; see `sandbox-managed-runtime`). + */ +export type ReferencedProjectFailureReason = "authorization" | "resolution" | "staging"; + +/** One referenced project that a run dropped or failed, with the layer that caused it. */ +export interface ReferencedProjectFailure { + projectId: string; + reason: ReferencedProjectFailureReason; +} + export interface ResolvedRunReferencedProjects { /** The anchor (primary) project — retains the existing git-worktree run path; never re-authorized here. */ anchor: RunReferencedProject | null; @@ -2381,6 +2430,8 @@ export interface ResolvedRunReferencedProjects { additional: RunReferencedProject[]; /** Human-readable warnings for every referenced project that was dropped (unavailable, unauthorized, or capped). */ warnings: string[]; + /** Structured record of every dropped referenced project, paired with the layer that dropped it. */ + failures: ReferencedProjectFailure[]; } export interface ResolveRunReferencedProjectsOptions { @@ -2430,6 +2481,7 @@ export async function resolveRunReferencedProjects( ): Promise { const { companyId, actor, issues, projects, access } = opts; const warnings: string[] = []; + const failures: ReferencedProjectFailure[] = []; const cap = Math.max(0, opts.maxAdditionalProjects ?? MAX_RUN_REFERENCED_ADDITIONAL_PROJECTS); // The evaluation cap bounds candidate hydration + authorization fan-out. It is always at least the // admitted cap so the admitted cap stays reachable in the normal (non-flood) case. @@ -2485,6 +2537,7 @@ export async function resolveRunReferencedProjects( const project = byId.get(projectId); if (!project) { warnings.push(`Referenced project ${projectId} was skipped because it is not available in this company.`); + failures.push({ projectId, reason: "resolution" }); continue; } availableCandidates.push({ projectId, project }); @@ -2538,6 +2591,7 @@ export async function resolveRunReferencedProjects( if (!allowed) { warnings.push(`Referenced project ${projectId} was skipped because it is not authorized for this run.`); + failures.push({ projectId, reason: "authorization" }); continue; } @@ -2560,7 +2614,20 @@ export async function resolveRunReferencedProjects( ); } - return { anchor, additional, warnings }; + // Record every capped or unevaluated candidate as a per-project resolution failure so the run + // surfacing and the observability log can reconcile requested against synced. The evaluated + // candidates past the admitted cap carry a known projectId; the candidates the fan-out cap + // dropped before hydration carry their id from the ordered mention set. + if (capReachedAtIndex !== null) { + for (let index = capReachedAtIndex; index < candidates.length; index++) { + failures.push({ projectId: candidates[index]!.projectId, reason: "resolution" }); + } + } + for (const projectId of allCandidateIds.slice(hydrationCursor)) { + failures.push({ projectId, reason: "resolution" }); + } + + return { anchor, additional, warnings, failures }; } export interface ResolveAdditionalRunWorkspacesOptions { @@ -2599,9 +2666,13 @@ export async function resolveAdditionalRunWorkspaces( issueId: string | null, anchorProjectId: string | null, opts: ResolveAdditionalRunWorkspacesOptions, -): Promise<{ additionalWorkspaces: ResolvedAdditionalWorkspace[]; warnings: string[] }> { +): Promise<{ + additionalWorkspaces: ResolvedAdditionalWorkspace[]; + warnings: string[]; + failures: ReferencedProjectFailure[]; +}> { if (!opts.enabled || !issueId) { - return { additionalWorkspaces: [], warnings: [] }; + return { additionalWorkspaces: [], warnings: [], failures: [] }; } // A referenced project realizes as a local directory only. A remote execution target (ssh, @@ -2615,14 +2686,22 @@ export async function resolveAdditionalRunWorkspaces( const mentionedIds = await opts.issues.findMentionedProjectIds(issueId, { includeCommentBodies: true, }); - const hasReferencedMention = mentionedIds.some((projectId) => projectId !== anchorProjectId); + // Each distinct non-anchor mention is a referenced project this remote run drops. A remote + // target has no path to receive the referenced tree, so the run drops the whole set at the + // staging layer. Record one failure per dropped project so the requested-vs-synced accounting + // counts the whole referenced set and the run still emits its structured sync log. + const droppedProjectIds = [ + ...new Set(mentionedIds.filter((projectId) => projectId !== anchorProjectId)), + ]; return { additionalWorkspaces: [], - warnings: hasReferencedMention - ? [ - "Referenced-project workspaces are available only on a local execution target. This run uses a remote execution target, so no referenced-project workspace was attached.", - ] - : [], + warnings: + droppedProjectIds.length > 0 + ? [ + "Referenced-project workspaces are available only on a local execution target. This run uses a remote execution target, so no referenced-project workspace was attached.", + ] + : [], + failures: droppedProjectIds.map((projectId) => ({ projectId, reason: "staging" as const })), }; } @@ -2638,6 +2717,7 @@ export async function resolveAdditionalRunWorkspaces( const additionalWorkspaces: ResolvedAdditionalWorkspace[] = []; const warnings = [...referenced.warnings]; + const failures = [...referenced.failures]; for (const project of referenced.additional) { try { additionalWorkspaces.push(await opts.resolveProjectWorkspace(project)); @@ -2646,10 +2726,44 @@ export async function resolveAdditionalRunWorkspaces( warnings.push( `Referenced project ${project.projectId} was skipped because its workspace could not be prepared: ${reason}`, ); + failures.push({ projectId: project.projectId, reason: "resolution" }); } } - return { additionalWorkspaces, warnings }; + return { additionalWorkspaces, warnings, failures }; +} + +/** Structured fields for the one requested-vs-synced observability log a run emits at run prep. */ +export interface ReferencedProjectRunObservability { + referenced_projects_requested: number; + referenced_projects_synced: number; + referenced_project_failures: Array<{ + project_id: string; + reason: ReferencedProjectFailureReason; + }>; +} + +/** + * Build the requested-vs-synced observability fields for a run's referenced-project set. + * + * A run requests one referenced project per authorized mention and syncs the projects that resolve. + * The requested count is the synced count plus every dropped project, so the two counts and the + * per-failure reasons together account for the whole referenced set. The human-readable warning for + * each drop rides the run's surfaced warnings channel; this function produces only the structured + * log fields, so a run emits exactly one line with a stable field shape. + */ +export function buildReferencedProjectRunObservability(input: { + syncedProjectIds: readonly string[]; + failures: readonly ReferencedProjectFailure[]; +}): ReferencedProjectRunObservability { + return { + referenced_projects_requested: input.syncedProjectIds.length + input.failures.length, + referenced_projects_synced: input.syncedProjectIds.length, + referenced_project_failures: input.failures.map((failure) => ({ + project_id: failure.projectId, + reason: failure.reason, + })), + }; } function readNonEmptyString(value: unknown): string | null { @@ -8080,11 +8194,11 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) ): Promise { const anchor = await resolveAnchorWorkspaceForRun(agent, context, previousSessionParams, opts); if (!isMultiProjectWorkspaceSyncEnabled()) { - return { ...anchor, additionalWorkspaces: [] }; + return { ...anchor, additionalWorkspaces: [], referencedProjectFailures: [] }; } const issueId = readNonEmptyString(context.issueId) ?? readNonEmptyString(context.taskId); - const { additionalWorkspaces, warnings } = await resolveAdditionalRunWorkspaces( + const { additionalWorkspaces, warnings, failures } = await resolveAdditionalRunWorkspaces( issueId, anchor.projectId, { @@ -8111,6 +8225,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) return { ...anchor, additionalWorkspaces, + referencedProjectFailures: failures, warnings: warnings.length > 0 ? [...anchor.warnings, ...warnings] : anchor.warnings, }; } @@ -13657,6 +13772,27 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) })(), }; context.paperclipWorkspaces = buildRunWorkspaceHints(resolvedWorkspace); + // Emit exactly one requested-vs-synced observability line for the referenced-project set. A run + // with no referenced project stays silent, so this adds no noise to the anchor-only default. The + // per-drop human warning already rides `runtimeWorkspaceWarnings`; this line carries the counts + // and the per-failure reason for a partial sync. + const referencedProjectObservability = buildReferencedProjectRunObservability({ + syncedProjectIds: resolvedWorkspace.additionalWorkspaces.map( + (additional) => additional.projectId, + ), + failures: resolvedWorkspace.referencedProjectFailures, + }); + if (referencedProjectObservability.referenced_projects_requested > 0) { + logger.info( + { + runId: run.id, + companyId: agent.companyId, + issueId: issueRef?.id ?? null, + ...referencedProjectObservability, + }, + "run referenced-project sync", + ); + } // The wake payload is built before the execution workspace is resolved, so // attach the branch pin here; the shared wake-prompt renderer surfaces it as // a one-time "stay on this branch" hint on non-resumed sessions.