diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index af04adf61d..abfe65bd10 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -58,6 +58,9 @@ jobs: - name: Test general-server shard partition run: node --test ./scripts/__tests__/run-vitest-stable-shard.test.mjs + - name: Test release verify workflow wiring + run: node --test ./scripts/__tests__/release-verify-workflow.test.mjs + - name: Test standalone package build concurrency run: node --test ./scripts/__tests__/build-standalone-concurrency.test.mjs diff --git a/.github/workflows/release-verify.yml b/.github/workflows/release-verify.yml new file mode 100644 index 0000000000..08bac6e4c8 --- /dev/null +++ b/.github/workflows/release-verify.yml @@ -0,0 +1,175 @@ +name: Release Verify + +on: + workflow_call: + inputs: + ref: + description: Commit SHA, branch, or tag to verify + required: true + type: string + +jobs: + typecheck: + name: Typecheck + runs-on: ubuntu-latest + timeout-minutes: 20 + permissions: + contents: read + + steps: + - name: Checkout repository + uses: actions/checkout@v7 + with: + ref: ${{ inputs.ref }} + + - name: Setup pnpm + uses: pnpm/action-setup@v6 + with: + version: 9.15.4 + + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: 24 + cache: pnpm + + - name: Validate release package manifest + run: node ./scripts/release-package-map.mjs check + + - name: Install dependencies + run: pnpm install --no-frozen-lockfile + + - name: Typecheck + run: pnpm -r typecheck + + general_tests: + name: General tests (${{ matrix.group_label }}) + runs-on: ubuntu-latest + timeout-minutes: 20 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + include: + - group: general-server + group_label: server (1/3) + shard_index: 0 + shard_count: 3 + - group: general-server + group_label: server (2/3) + shard_index: 1 + shard_count: 3 + - group: general-server + group_label: server (3/3) + shard_index: 2 + shard_count: 3 + - group: general-workspaces-a + group_label: workspaces-a + - group: general-workspaces-b + group_label: workspaces-b + + steps: + - name: Checkout repository + uses: actions/checkout@v7 + with: + ref: ${{ inputs.ref }} + + - name: Setup pnpm + uses: pnpm/action-setup@v6 + with: + version: 9.15.4 + + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: 24 + cache: pnpm + + - name: Install dependencies + run: pnpm install --no-frozen-lockfile + + - name: Run grouped general test suites + run: | + if [ -n "${{ matrix.shard_count }}" ]; then + pnpm test:run:general -- --group '${{ matrix.group }}' \ + --shard-index ${{ matrix.shard_index }} --shard-count ${{ matrix.shard_count }} + else + pnpm test:run:general -- --group '${{ matrix.group }}' + fi + + serialized_tests: + name: Serialized tests (${{ matrix.shard_label }}) + runs-on: ubuntu-latest + timeout-minutes: 20 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + include: + - shard_index: 0 + shard_count: 4 + shard_label: 1/4 + - shard_index: 1 + shard_count: 4 + shard_label: 2/4 + - shard_index: 2 + shard_count: 4 + shard_label: 3/4 + - shard_index: 3 + shard_count: 4 + shard_label: 4/4 + + steps: + - name: Checkout repository + uses: actions/checkout@v7 + with: + ref: ${{ inputs.ref }} + + - name: Setup pnpm + uses: pnpm/action-setup@v6 + with: + version: 9.15.4 + + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: 24 + cache: pnpm + + - name: Install dependencies + run: pnpm install --no-frozen-lockfile + + - name: Run serialized server test shard + run: pnpm test:run:serialized -- --shard-index ${{ matrix.shard_index }} --shard-count ${{ matrix.shard_count }} + + build: + name: Build + runs-on: ubuntu-latest + timeout-minutes: 20 + permissions: + contents: read + + steps: + - name: Checkout repository + uses: actions/checkout@v7 + with: + ref: ${{ inputs.ref }} + + - name: Setup pnpm + uses: pnpm/action-setup@v6 + with: + version: 9.15.4 + + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: 24 + cache: pnpm + + - name: Install dependencies + run: pnpm install --no-frozen-lockfile + + - name: Build + run: pnpm build diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f97fd15136..17233901f2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -28,42 +28,9 @@ concurrency: jobs: verify_canary: if: github.event_name == 'push' - runs-on: ubuntu-latest - timeout-minutes: 30 - permissions: - contents: read - - steps: - - name: Checkout repository - uses: actions/checkout@v7 - with: - fetch-depth: 0 - - - name: Setup pnpm - uses: pnpm/action-setup@v6 - with: - version: 9.15.4 - - - name: Setup Node.js - uses: actions/setup-node@v6 - with: - node-version: 24 - cache: pnpm - - - name: Validate release package manifest - run: node ./scripts/release-package-map.mjs check - - - name: Install dependencies - run: pnpm install --no-frozen-lockfile - - - name: Typecheck - run: pnpm -r typecheck - - - name: Run tests - run: pnpm test:run - - - name: Build - run: pnpm build + uses: ./.github/workflows/release-verify.yml + with: + ref: ${{ github.sha }} publish_canary: if: github.event_name == 'push' @@ -122,43 +89,9 @@ jobs: verify_stable: if: github.event_name == 'workflow_dispatch' - runs-on: ubuntu-latest - timeout-minutes: 30 - permissions: - contents: read - - steps: - - name: Checkout repository - uses: actions/checkout@v7 - with: - fetch-depth: 0 - ref: ${{ inputs.source_ref }} - - - name: Setup pnpm - uses: pnpm/action-setup@v6 - with: - version: 9.15.4 - - - name: Setup Node.js - uses: actions/setup-node@v6 - with: - node-version: 24 - cache: pnpm - - - name: Validate release package manifest - run: node ./scripts/release-package-map.mjs check - - - name: Install dependencies - run: pnpm install --no-frozen-lockfile - - - name: Typecheck - run: pnpm -r typecheck - - - name: Run tests - run: pnpm test:run - - - name: Build - run: pnpm build + uses: ./.github/workflows/release-verify.yml + with: + ref: ${{ inputs.source_ref }} preview_stable: if: github.event_name == 'workflow_dispatch' && inputs.dry_run diff --git a/scripts/__tests__/release-verify-workflow.test.mjs b/scripts/__tests__/release-verify-workflow.test.mjs new file mode 100644 index 0000000000..3bd0a616a8 --- /dev/null +++ b/scripts/__tests__/release-verify-workflow.test.mjs @@ -0,0 +1,52 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import test from "node:test"; + +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", ".."); + +function readWorkflow(name) { + return readFileSync(path.join(repoRoot, ".github/workflows", name), "utf8"); +} + +test("release workflow delegates stable and canary verification to the reusable workflow", () => { + const releaseWorkflow = readWorkflow("release.yml"); + + assert.match( + releaseWorkflow, + /verify_canary:\n\s+if: github\.event_name == 'push'\n\s+uses: \.\/\.github\/workflows\/release-verify\.yml\n\s+with:\n\s+ref: \$\{\{ github\.sha \}\}/, + ); + assert.match( + releaseWorkflow, + /verify_stable:\n\s+if: github\.event_name == 'workflow_dispatch'\n\s+uses: \.\/\.github\/workflows\/release-verify\.yml\n\s+with:\n\s+ref: \$\{\{ inputs\.source_ref \}\}/, + ); + assert.doesNotMatch(releaseWorkflow, /verify_(?:canary|stable):[\s\S]*?pnpm test:run(?:\n|$)/); +}); + +test("release verify workflow covers the same split test surface as stable PR verification", () => { + const verifyWorkflow = readWorkflow("release-verify.yml"); + + assert.match(verifyWorkflow, /workflow_call:/); + assert.match(verifyWorkflow, /node \.\/scripts\/release-package-map\.mjs check/); + assert.match(verifyWorkflow, /pnpm -r typecheck/); + assert.match(verifyWorkflow, /pnpm build/); + + for (const group of ["general-server", "general-workspaces-a", "general-workspaces-b"]) { + assert.match(verifyWorkflow, new RegExp(`group: ${group}`)); + } + + for (const shardIndex of [0, 1, 2]) { + assert.match( + verifyWorkflow, + new RegExp(`group: general-server[\\s\\S]*?shard_index: ${shardIndex}[\\s\\S]*?shard_count: 3`), + ); + } + + for (const shardIndex of [0, 1, 2, 3]) { + assert.match(verifyWorkflow, new RegExp(`shard_index: ${shardIndex}[\\s\\S]*?shard_count: 4`)); + } + + assert.match(verifyWorkflow, /pnpm test:run:general -- --group/); + assert.match(verifyWorkflow, /pnpm test:run:serialized -- --shard-index/); +}); diff --git a/server/src/__tests__/heartbeat-worktree-suppression.test.ts b/server/src/__tests__/heartbeat-worktree-suppression.test.ts index 3a3144dcef..b740512df9 100644 --- a/server/src/__tests__/heartbeat-worktree-suppression.test.ts +++ b/server/src/__tests__/heartbeat-worktree-suppression.test.ts @@ -133,29 +133,35 @@ describeEmbeddedPostgres("heartbeat worktree suppression", () => { return { companyId, agentId, issueId }; } - async function waitForTerminalRun(runId: string) { - for (let attempt = 0; attempt < 20; attempt += 1) { + async function waitForCompletedRun(runId: string, agentId: string) { + let latestStatus: string | null = null; + let latestLastRunId: string | null = null; + + for (let attempt = 0; attempt < 100; attempt += 1) { const run = await db .select({ status: heartbeatRuns.status }) .from(heartbeatRuns) .where(eq(heartbeatRuns.id, runId)) .then((rows) => rows[0] ?? null); - if (run && run.status !== "queued" && run.status !== "running") return run.status; - await new Promise((resolve) => setTimeout(resolve, 25)); - } - return null; - } - - async function waitForRuntimeStateLastRun(agentId: string, runId: string) { - for (let attempt = 0; attempt < 100; attempt += 1) { const state = await db .select({ lastRunId: agentRuntimeState.lastRunId }) .from(agentRuntimeState) .where(eq(agentRuntimeState.agentId, agentId)) .then((rows) => rows[0] ?? null); - if (state?.lastRunId === runId) return; + + latestStatus = run?.status ?? null; + latestLastRunId = state?.lastRunId ?? null; + + if (run && run.status !== "queued" && run.status !== "running" && state?.lastRunId === runId) { + return run.status; + } + await new Promise((resolve) => setTimeout(resolve, 50)); } + + throw new Error( + `Timed out waiting for heartbeat run ${runId} to finish; latest status=${latestStatus ?? "missing"}, runtime lastRunId=${latestLastRunId ?? "missing"}`, + ); } it("suppresses new assignment wakes in worktree instances without creating heartbeat runs", async () => { @@ -239,6 +245,11 @@ describeEmbeddedPostgres("heartbeat worktree suppression", () => { it("still creates live-plane assignment runs when suppression is not active", async () => { const { agentId, issueId } = await insertAgentAndIssue(); + await db + .update(issues) + .set({ status: "in_review", updatedAt: new Date() }) + .where(eq(issues.id, issueId)); + const heartbeat = heartbeatService(db, { runtimeEnv: {} }); const run = await heartbeat.wakeup(agentId, { @@ -246,27 +257,22 @@ describeEmbeddedPostgres("heartbeat worktree suppression", () => { triggerDetail: "system", reason: "issue_assigned", payload: { issueId }, - contextSnapshot: { issueId, wakeReason: "issue_assigned" }, + contextSnapshot: { issueId, wakeReason: "issue_assigned", skipIssueComment: true }, requestedByActorType: "system", requestedByActorId: "issue_assignment", }); expect(run).not.toBeNull(); - const terminalStatus = await waitForTerminalRun(run!.id); - expect(["succeeded", null]).toContain(terminalStatus); + const terminalStatus = await waitForCompletedRun(run!.id, agentId); + await heartbeat.waitForRunExecutionDrain(run!.id); + expect(terminalStatus).toBe("succeeded"); const runCount = await db .select({ count: sql`count(*)::int` }) .from(heartbeatRuns) .then((rows) => rows[0]?.count ?? 0); expect(runCount).toBe(1); - - await db - .update(issues) - .set({ status: "done", updatedAt: new Date() }) - .where(eq(issues.id, issueId)); - await waitForRuntimeStateLastRun(agentId, run!.id); - }); + }, 10_000); it("recognizes explicit restore-in-progress suppression", () => { expect(resolveHeartbeatSchedulingSuppression({ diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index a97a4dadbc..620fb16d06 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -2640,6 +2640,8 @@ export function shouldResetTaskSessionForWake( function shouldRequireIssueCommentForWake( contextSnapshot: Record | null | undefined, ) { + if (contextSnapshot?.skipIssueComment === true) return false; + const wakeReason = readNonEmptyString(contextSnapshot?.wakeReason); return ( wakeReason === "issue_assigned" || @@ -15043,6 +15045,21 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) } return { + waitForRunExecutionDrain: async ( + runId: string, + options: { timeoutMs?: number; intervalMs?: number } = {}, + ) => { + const timeoutMs = options.timeoutMs ?? 5_000; + const intervalMs = options.intervalMs ?? 25; + const deadline = Date.now() + timeoutMs; + + while (liveRunExecutions.has(runId)) { + if (Date.now() >= deadline) { + throw new Error(`Timed out waiting for heartbeat run ${runId} execution to drain`); + } + await new Promise((resolve) => setTimeout(resolve, intervalMs)); + } + }, list: async ( companyId: string, agentId?: string,