test(ci): isolate chat fixtures and verify the real shard CLI

Co-Authored-By: Paperclip <noreply@paperclip.ing>

doc/RELEASE-AUTOMATION-SETUP.md

@@ -365,7 +365,9 @@ See [GitHub cache access restrictions](https://docs.github.com/en/actions/refere
 Release verification runs the large chat integration file on three independent
 runners. Four other server shards cover every remaining general server file.
 The ordinary local test command and trusted PR workflow keep their complete
-`general-server` group. No application test assertions or fixtures change.
+`general-server` group. Each chat case shuts down its services and pauses its own still-active endpoints
+after assertions. This keeps workers in later cases from claiming earlier
+fixtures in the shared test database. Application assertions stay unchanged.
 
 Each chat job collects active tests with Vitest, groups cases by source line,
 and balances those groups by case count. Parameterized cases and loop-generated

server/src/__tests__/chat-channels.integration.test.ts

@@ -32,7 +32,7 @@ import {
   or,
   sql,
 } from "drizzle-orm";
-import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
+import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
 import {
   agents,
   agentWakeupRequests,
@@ -1022,8 +1022,26 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
     rmSync(secretsTmpDir, { recursive: true, force: true });
   });
 
+  // Services scan this file's shared database. Retire each case's fixtures
+  // after its assertions so another case (or shard order) cannot claim them.
+  const fixtureCompanies = new Set<string>();
+  const fixtureServices = new Set<ChatChannelService>();
+  afterEach(async () => {
+    try {
+      await Promise.all([...fixtureServices].map((service) => service.shutdown()));
+    } finally {
+      if (fixtureCompanies.size > 0) {
+        await db.update(chatEndpoints).set({ status: "paused" })
+          .where(and(inArray(chatEndpoints.companyId, [...fixtureCompanies]), eq(chatEndpoints.status, "active")));
+      }
+      fixtureServices.clear();
+      fixtureCompanies.clear();
+    }
+  });
+
   async function seedCompany() {
     const companyId = randomUUID();
+    fixtureCompanies.add(companyId);
     const assignedAgentId = randomUUID();
     const replacementAgentId = randomUUID();
     await db.insert(companies).values({
@@ -1193,6 +1211,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
       runtime: runtime as unknown as ChatSdkRuntime,
       ...serviceOverrides,
     });
+    fixtureServices.add(service);
     return { cancelRun, runtime, service, wakeup };
   }
 

server/src/__tests__/vitest-chat-shards.test.ts

@@ -0,0 +1,60 @@
+import { spawnSync } from "node:child_process";
+import { mkdtempSync, mkdirSync, readFileSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { expect, it } from "vitest";
+
+const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../..");
+
+it("runs every active nested/parameterized fixture case exactly once through the real chat shard CLI", () => {
+  const root = realpathSync(mkdtempSync(path.join(os.tmpdir(), "pc-shards-")));
+  try {
+    const tests = path.join(root, "server/src/__tests__");
+    mkdirSync(tests, { recursive: true });
+    symlinkSync(path.join(repoRoot, "node_modules"), path.join(root, "node_modules"), "junction");
+    writeFileSync(path.join(root, "package.json"), JSON.stringify({ private: true }));
+    writeFileSync(path.join(root, "vitest.config.mjs"), `export default {
+      test: { projects: [{ test: { name: "@paperclipai/server", root: ${JSON.stringify(path.join(root, "server"))},
+        include: ["src/**/*.test.ts"], pool: "forks", maxWorkers: 1 } }] }
+    };`);
+    const trace = path.join(root, "executed.jsonl");
+    const fixture = path.join(tests, "chat-channels.integration.test.ts");
+    writeFileSync(fixture, `import { appendFileSync } from "node:fs";
+      import { afterEach, beforeEach, describe, expect, it } from "vitest";
+      let active = false;
+      beforeEach(() => { expect(active).toBe(false); active = true; });
+      afterEach(() => { active = false; });
+      function record(id) { expect(active).toBe(true); appendFileSync(${JSON.stringify(trace)}, JSON.stringify(id) + "\\n"); }
+      it("top-level", () => record("top"));
+      describe("nested", () => {
+        it("first", () => record("nested-first"));
+        it("second", () => record("nested-second"));
+        it.each(["a", "b", "c", "d"])("parameter %s", (value) => record(value));
+        it.skip("intentionally skipped", () => { throw new Error("must stay skipped"); });
+      });`);
+    const run = (index: number, count: number) => spawnSync(process.execPath, [
+      path.join(repoRoot, "scripts/run-vitest-stable.mjs"), "--mode", "general", "--group", "general-chat",
+      "--shard-index", String(index), "--shard-count", String(count),
+    ], { cwd: root, env: { ...process.env, CI: "true" }, encoding: "utf8", timeout: 45_000, maxBuffer: 4 * 1024 * 1024 });
+    for (const index of [0, 1]) {
+      const result = run(index, 2);
+      expect(result.error, result.stderr).toBeUndefined();
+      expect(result.status, result.stdout + result.stderr).toBe(0);
+      expect(result.stdout).toContain("exact filter coverage verified");
+    }
+    const executed = readFileSync(trace, "utf8").trim().split("\n").map((line) => JSON.parse(line));
+    expect(executed.sort()).toEqual(["a", "b", "c", "d", "nested-first", "nested-second", "top"]);
+
+    // A real assertion failure must still fail the wrapper after successful
+    // collection and filter validation.
+    writeFileSync(fixture, 'import { it } from "vitest"; it("fails", () => { throw new Error("fixture failure"); });');
+    const failed = run(0, 1);
+    expect(failed.error, failed.stderr).toBeUndefined();
+    expect(failed.stdout).toContain("exact filter coverage verified");
+    expect(failed.status).not.toBe(0);
+    expect(failed.stdout + failed.stderr).toContain("fixture failure");
+  } finally {
+    rmSync(root, { recursive: true, force: true });
+  }
+}, 120_000);