From a03f6eae3b0cc64fc20819c178c396b8b055979d Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Thu, 10 Sep 2026 19:51:48 -0700 Subject: [PATCH 1/2] test(ci): isolate chat fixtures and verify the real shard CLI Co-Authored-By: Paperclip --- doc/RELEASE-AUTOMATION-SETUP.md | 4 +- .../chat-channels.integration.test.ts | 21 ++++++- .../src/__tests__/vitest-chat-shards.test.ts | 60 +++++++++++++++++++ 3 files changed, 83 insertions(+), 2 deletions(-) create mode 100644 server/src/__tests__/vitest-chat-shards.test.ts diff --git a/doc/RELEASE-AUTOMATION-SETUP.md b/doc/RELEASE-AUTOMATION-SETUP.md index dcafc9bd67..510c464e3f 100644 --- a/doc/RELEASE-AUTOMATION-SETUP.md +++ b/doc/RELEASE-AUTOMATION-SETUP.md @@ -365,7 +365,9 @@ See [GitHub cache access restrictions](https://docs.github.com/en/actions/refere Release verification runs the large chat integration file on three independent runners. Four other server shards cover every remaining general server file. The ordinary local test command and trusted PR workflow keep their complete -`general-server` group. No application test assertions or fixtures change. +`general-server` group. Each chat case shuts down its services and pauses its own still-active endpoints +after assertions. This keeps workers in later cases from claiming earlier +fixtures in the shared test database. Application assertions stay unchanged. Each chat job collects active tests with Vitest, groups cases by source line, and balances those groups by case count. Parameterized cases and loop-generated diff --git a/server/src/__tests__/chat-channels.integration.test.ts b/server/src/__tests__/chat-channels.integration.test.ts index b565ed645d..4de026c40e 100644 --- a/server/src/__tests__/chat-channels.integration.test.ts +++ b/server/src/__tests__/chat-channels.integration.test.ts @@ -32,7 +32,7 @@ import { or, sql, } from "drizzle-orm"; -import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { agents, agentWakeupRequests, @@ -1022,8 +1022,26 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { rmSync(secretsTmpDir, { recursive: true, force: true }); }); + // Services scan this file's shared database. Retire each case's fixtures + // after its assertions so another case (or shard order) cannot claim them. + const fixtureCompanies = new Set(); + const fixtureServices = new Set(); + afterEach(async () => { + try { + await Promise.all([...fixtureServices].map((service) => service.shutdown())); + } finally { + if (fixtureCompanies.size > 0) { + await db.update(chatEndpoints).set({ status: "paused" }) + .where(and(inArray(chatEndpoints.companyId, [...fixtureCompanies]), eq(chatEndpoints.status, "active"))); + } + fixtureServices.clear(); + fixtureCompanies.clear(); + } + }); + async function seedCompany() { const companyId = randomUUID(); + fixtureCompanies.add(companyId); const assignedAgentId = randomUUID(); const replacementAgentId = randomUUID(); await db.insert(companies).values({ @@ -1193,6 +1211,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { runtime: runtime as unknown as ChatSdkRuntime, ...serviceOverrides, }); + fixtureServices.add(service); return { cancelRun, runtime, service, wakeup }; } diff --git a/server/src/__tests__/vitest-chat-shards.test.ts b/server/src/__tests__/vitest-chat-shards.test.ts new file mode 100644 index 0000000000..dd929f1dd8 --- /dev/null +++ b/server/src/__tests__/vitest-chat-shards.test.ts @@ -0,0 +1,60 @@ +import { spawnSync } from "node:child_process"; +import { mkdtempSync, mkdirSync, readFileSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { expect, it } from "vitest"; + +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../.."); + +it("runs every active nested/parameterized fixture case exactly once through the real chat shard CLI", () => { + const root = realpathSync(mkdtempSync(path.join(os.tmpdir(), "pc-shards-"))); + try { + const tests = path.join(root, "server/src/__tests__"); + mkdirSync(tests, { recursive: true }); + symlinkSync(path.join(repoRoot, "node_modules"), path.join(root, "node_modules"), "junction"); + writeFileSync(path.join(root, "package.json"), JSON.stringify({ private: true })); + writeFileSync(path.join(root, "vitest.config.mjs"), `export default { + test: { projects: [{ test: { name: "@paperclipai/server", root: ${JSON.stringify(path.join(root, "server"))}, + include: ["src/**/*.test.ts"], pool: "forks", maxWorkers: 1 } }] } + };`); + const trace = path.join(root, "executed.jsonl"); + const fixture = path.join(tests, "chat-channels.integration.test.ts"); + writeFileSync(fixture, `import { appendFileSync } from "node:fs"; + import { afterEach, beforeEach, describe, expect, it } from "vitest"; + let active = false; + beforeEach(() => { expect(active).toBe(false); active = true; }); + afterEach(() => { active = false; }); + function record(id) { expect(active).toBe(true); appendFileSync(${JSON.stringify(trace)}, JSON.stringify(id) + "\\n"); } + it("top-level", () => record("top")); + describe("nested", () => { + it("first", () => record("nested-first")); + it("second", () => record("nested-second")); + it.each(["a", "b", "c", "d"])("parameter %s", (value) => record(value)); + it.skip("intentionally skipped", () => { throw new Error("must stay skipped"); }); + });`); + const run = (index: number, count: number) => spawnSync(process.execPath, [ + path.join(repoRoot, "scripts/run-vitest-stable.mjs"), "--mode", "general", "--group", "general-chat", + "--shard-index", String(index), "--shard-count", String(count), + ], { cwd: root, env: { ...process.env, CI: "true" }, encoding: "utf8", timeout: 45_000, maxBuffer: 4 * 1024 * 1024 }); + for (const index of [0, 1]) { + const result = run(index, 2); + expect(result.error, result.stderr).toBeUndefined(); + expect(result.status, result.stdout + result.stderr).toBe(0); + expect(result.stdout).toContain("exact filter coverage verified"); + } + const executed = readFileSync(trace, "utf8").trim().split("\n").map((line) => JSON.parse(line)); + expect(executed.sort()).toEqual(["a", "b", "c", "d", "nested-first", "nested-second", "top"]); + + // A real assertion failure must still fail the wrapper after successful + // collection and filter validation. + writeFileSync(fixture, 'import { it } from "vitest"; it("fails", () => { throw new Error("fixture failure"); });'); + const failed = run(0, 1); + expect(failed.error, failed.stderr).toBeUndefined(); + expect(failed.stdout).toContain("exact filter coverage verified"); + expect(failed.status).not.toBe(0); + expect(failed.stdout + failed.stderr).toContain("fixture failure"); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}, 120_000); From 9066878f18a77dd502568989eef0a05ced990d55 Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Thu, 10 Sep 2026 19:57:54 -0700 Subject: [PATCH 2/2] test(ci): retire chat milestone bindings between cases Co-Authored-By: Paperclip --- doc/RELEASE-AUTOMATION-SETUP.md | 5 +++-- server/src/__tests__/chat-channels.integration.test.ts | 4 ++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/doc/RELEASE-AUTOMATION-SETUP.md b/doc/RELEASE-AUTOMATION-SETUP.md index 510c464e3f..3eaf5ceac2 100644 --- a/doc/RELEASE-AUTOMATION-SETUP.md +++ b/doc/RELEASE-AUTOMATION-SETUP.md @@ -365,8 +365,9 @@ See [GitHub cache access restrictions](https://docs.github.com/en/actions/refere Release verification runs the large chat integration file on three independent runners. Four other server shards cover every remaining general server file. The ordinary local test command and trusted PR workflow keep their complete -`general-server` group. Each chat case shuts down its services and pauses its own still-active endpoints -after assertions. This keeps workers in later cases from claiming earlier +`general-server` group. Each chat case shuts down its services, pauses its own +still-active endpoints, and retires its active/waiting conversations after +assertions. This keeps workers in later cases from claiming earlier fixtures in the shared test database. Application assertions stay unchanged. Each chat job collects active tests with Vitest, groups cases by source line, diff --git a/server/src/__tests__/chat-channels.integration.test.ts b/server/src/__tests__/chat-channels.integration.test.ts index 4de026c40e..61ecfec616 100644 --- a/server/src/__tests__/chat-channels.integration.test.ts +++ b/server/src/__tests__/chat-channels.integration.test.ts @@ -1033,6 +1033,10 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { if (fixtureCompanies.size > 0) { await db.update(chatEndpoints).set({ status: "paused" }) .where(and(inArray(chatEndpoints.companyId, [...fixtureCompanies]), eq(chatEndpoints.status, "active"))); + // The milestone scanner also considers paused endpoints while their + // conversations are active. Retire those bindings after assertions. + await db.update(chatConversations).set({ state: "completed" }) + .where(and(inArray(chatConversations.companyId, [...fixtureCompanies]), inArray(chatConversations.state, ["active", "waiting"]))); } fixtureServices.clear(); fixtureCompanies.clear();