fix(ci): split release chat verification into test shards (#13198)
Split release chat verification into three validated test-line shards and balance other server suites across five runners using the measured native Runner integration cost. Retire each chat case's fixtures after assertions, preserve complete test coverage, and exercise the real shard CLI in PR tests. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Devin Foley
GitHub
parent
9e970df4c5
commit
42961b6ef1
|
|
@ -58,30 +58,41 @@ jobs:
|
|||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
# Five-way server split, matching pr-trusted.yml. Three shards sat
|
||||
# at 17-19 minutes against this job's 20-minute cap after the
|
||||
# server suite grew on 2026-09-10, and every canary that evening
|
||||
# died on the timeout instead of reporting a verdict.
|
||||
- group: general-server
|
||||
# Split the long chat file by collected test locations, and balance
|
||||
# the remaining server files across five runners. Normal PR/local
|
||||
# invocations retain their complete general-server group.
|
||||
- group: general-server-without-chat
|
||||
group_label: server (1/5)
|
||||
shard_index: 0
|
||||
shard_count: 5
|
||||
- group: general-server
|
||||
- group: general-server-without-chat
|
||||
group_label: server (2/5)
|
||||
shard_index: 1
|
||||
shard_count: 5
|
||||
- group: general-server
|
||||
- group: general-server-without-chat
|
||||
group_label: server (3/5)
|
||||
shard_index: 2
|
||||
shard_count: 5
|
||||
- group: general-server
|
||||
- group: general-server-without-chat
|
||||
group_label: server (4/5)
|
||||
shard_index: 3
|
||||
shard_count: 5
|
||||
- group: general-server
|
||||
- group: general-server-without-chat
|
||||
group_label: server (5/5)
|
||||
shard_index: 4
|
||||
shard_count: 5
|
||||
- group: general-chat
|
||||
group_label: chat (1/3)
|
||||
shard_index: 0
|
||||
shard_count: 3
|
||||
- group: general-chat
|
||||
group_label: chat (2/3)
|
||||
shard_index: 1
|
||||
shard_count: 3
|
||||
- group: general-chat
|
||||
group_label: chat (3/3)
|
||||
shard_index: 2
|
||||
shard_count: 3
|
||||
# Keep parity with pr.yml: workspaces-a is split with Vitest's
|
||||
# native --shard because the ui project dominates the lane.
|
||||
- group: general-workspaces-a
|
||||
|
|
|
|||
|
|
@ -359,3 +359,35 @@ default branch) are master here. A workflow with authority to execute arbitrary
|
|||
code on master can affect verification directly and is already trusted. The
|
||||
cache contains dependency build artifacts, not credentials or workspace output.
|
||||
See [GitHub cache access restrictions](https://docs.github.com/en/actions/reference/workflows-and-actions/dependency-caching#restrictions-for-accessing-a-cache).
|
||||
|
||||
## Chat integration test shards
|
||||
|
||||
Release verification runs the large chat integration file on three independent
|
||||
runners. Five other server shards cover every remaining general server file.
|
||||
The ordinary local test command and trusted PR workflow keep their complete
|
||||
`general-server` group. Each chat case shuts down its services, pauses its own
|
||||
still-active endpoints, and retires its active/waiting conversations after
|
||||
assertions. This keeps workers in later cases from claiming earlier
|
||||
fixtures in the shared test database. Application assertions stay unchanged.
|
||||
|
||||
Each chat job collects active tests with Vitest, groups cases by source line,
|
||||
and balances those groups by case count. Parameterized cases and loop-generated
|
||||
cases on one line stay together. The job re-collects with the exact line filters
|
||||
it will execute and fails if the selected case identities differ. Hooks and test
|
||||
execution remain sequential inside each runner with its own temporary home.
|
||||
|
||||
Run one shard locally with:
|
||||
|
||||
```sh
|
||||
pnpm test:run:general -- --group general-chat --shard-index 0 --shard-count 3
|
||||
```
|
||||
|
||||
Use indexes 0, 1, and 2 to run the complete chat suite. The CLI validates that
|
||||
each shard has work and that collection includes usable source locations. A
|
||||
Vitest collection or filtering change fails verification instead of dropping
|
||||
tests. Splitting adds three release-verification jobs and repeats collection and
|
||||
fixture setup; it does not make a single test faster.
|
||||
|
||||
The file-duration manifest also records the native Codex Runner integration
|
||||
suite's measured import and execution cost, so the existing file balancer
|
||||
accounts for it in both ordinary PR and release verification.
|
||||
|
|
|
|||
|
|
@ -206,28 +206,16 @@ test("release verify workflow covers the same split test surface as stable PR ve
|
|||
assert.match(buildJob, /persist-credentials: false/);
|
||||
assert.doesNotMatch(buildJob, /cache: pnpm/);
|
||||
|
||||
for (const group of [
|
||||
"general-server",
|
||||
"general-workspaces-a",
|
||||
"general-workspaces-b",
|
||||
]) {
|
||||
for (const group of ["general-server-without-chat", "general-chat", "general-workspaces-a", "general-workspaces-b"]) {
|
||||
assert.match(verifyWorkflow, new RegExp(`group: ${group}`));
|
||||
}
|
||||
|
||||
for (const shardIndex of [0, 1, 2, 3, 4]) {
|
||||
assert.match(
|
||||
verifyWorkflow,
|
||||
new RegExp(
|
||||
`group: general-server[\\s\\S]*?shard_index: ${shardIndex}[\\s\\S]*?shard_count: 5`,
|
||||
),
|
||||
);
|
||||
for (const [group, count] of [["general-server-without-chat", 5], ["general-chat", 3]]) {
|
||||
const rows = [...verifyWorkflow.matchAll(new RegExp(`group: ${group}\\n\\s+group_label: [^\\n]+\\n\\s+shard_index: (\\d+)\\n\\s+shard_count: (\\d+)`, "g"))];
|
||||
assert.deepEqual(rows.map((row) => [Number(row[1]), Number(row[2])]),
|
||||
Array.from({ length: count }, (_, index) => [index, count]));
|
||||
}
|
||||
|
||||
for (const shardIndex of [0, 1, 2, 3, 4]) {
|
||||
assert.match(
|
||||
verifyWorkflow,
|
||||
new RegExp(`shard_index: ${shardIndex}[\\s\\S]*?shard_count: 5`),
|
||||
);
|
||||
assert.match(verifyWorkflow, new RegExp(`shard_index: ${shardIndex}[\\s\\S]*?shard_count: 5`));
|
||||
}
|
||||
|
||||
// workspaces-a splits with Vitest native --shard in pr.yml; release
|
||||
|
|
|
|||
|
|
@ -10,6 +10,8 @@ import {
|
|||
partitionGeneralServerSuites,
|
||||
} from "../general-server-shard.mjs";
|
||||
|
||||
import { assertSelectedTests, partitionTestLines } from "../test-line-shard.mjs";
|
||||
|
||||
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "..");
|
||||
const script = path.join(repoRoot, "scripts", "run-vitest-stable.mjs");
|
||||
const durationsManifest = path.join(repoRoot, "scripts", "general-server-shard-durations.json");
|
||||
|
|
@ -264,3 +266,53 @@ test("the real shard partition is duration-balanced", () => {
|
|||
`shard weight spread ${maxTotal - minTotal}ms exceeds heaviest suite ${heaviest}ms: ${totals.join(", ")}`,
|
||||
);
|
||||
});
|
||||
|
||||
|
||||
test("release server shards plus the dedicated chat file cover the original server group exactly", () => {
|
||||
const full = dryRunJson(["--mode", "general", "--group", "general-server", "--shard-index", "0", "--shard-count", "1"]);
|
||||
const shards = Array.from({ length: 5 }, (_, index) => dryRunJson([
|
||||
"--mode", "general", "--group", "general-server-without-chat",
|
||||
"--shard-index", String(index), "--shard-count", "5",
|
||||
]));
|
||||
const files = shards.flatMap((shard) => shard.selectedGeneralServerSuites);
|
||||
const chat = "server/src/__tests__/chat-channels.integration.test.ts";
|
||||
assert.ok(!files.includes(chat));
|
||||
assert.deepEqual([...files, chat].sort(), full.selectedGeneralServerSuites.sort());
|
||||
assert.equal(new Set(files).size, files.length);
|
||||
const defaultRun = dryRunJson([]);
|
||||
assert.ok(defaultRun.generalServerSuiteCount === full.generalServerSuiteCount);
|
||||
});
|
||||
|
||||
const lineShardFile = path.join(repoRoot, "server/src/__tests__/chat-channels.integration.test.ts");
|
||||
const caseAt = (line, name) => ({ name, file: lineShardFile, projectName: "@paperclipai/server", location: { line, column: 3 } });
|
||||
|
||||
test("test-line shards cover nested and parameterized cases exactly once without splitting a source line", () => {
|
||||
const cases = [caseAt(10, "suite > nested > first"), caseAt(10, "suite > nested > second"),
|
||||
caseAt(20, "same name"), caseAt(30, "same name"), caseAt(40, "last"), caseAt(50, "new case")];
|
||||
const shards = partitionTestLines(cases, 3, lineShardFile);
|
||||
assert.deepEqual(shards.map((shard) => shard.tests.length), [2, 2, 2]);
|
||||
assert.equal(shards.filter((shard) => shard.lines.includes(10)).length, 1);
|
||||
assert.equal(shards.find((shard) => shard.lines.includes(10)).tests.length, 2);
|
||||
assert.equal(shards.flatMap((shard) => shard.lines).length, 5);
|
||||
assert.deepEqual(shards.flatMap((shard) => shard.tests).sort((a, b) => a.location.line - b.location.line), cases);
|
||||
assert.deepEqual(partitionTestLines([...cases].reverse(), 3, lineShardFile).map((shard) => shard.lines), shards.map((shard) => shard.lines));
|
||||
});
|
||||
|
||||
test("line-shard collection rejects empty, foreign, or unlocated tests and invalid shard counts", () => {
|
||||
const good = caseAt(10, "valid");
|
||||
for (const input of [[], null, [{ ...good, file: "/another.test.ts" }], [{ ...good, projectName: "wrong" }],
|
||||
[{ ...good, location: undefined }], [{ ...good, location: { line: 0 } }], [{ ...good, name: "" }]]) {
|
||||
assert.throws(() => partitionTestLines(input, 1, lineShardFile));
|
||||
}
|
||||
for (const count of [0, -1, 1.5, Infinity, 2]) assert.throws(() => partitionTestLines([good], count, lineShardFile));
|
||||
});
|
||||
|
||||
test("filtered collection must match the exact assigned case identities, including duplicates", () => {
|
||||
const expected = [caseAt(10, "same"), caseAt(10, "same"), caseAt(20, "nested > case")];
|
||||
assertSelectedTests(expected, [...expected].reverse(), lineShardFile);
|
||||
for (const actual of [expected.slice(1), [...expected, caseAt(30, "extra")],
|
||||
[expected[0], expected[1], caseAt(20, "renamed")],
|
||||
[expected[0], expected[1], caseAt(21, "nested > case")]]) {
|
||||
assert.throws(() => assertSelectedTests(expected, actual, lineShardFile));
|
||||
}
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,7 +1,9 @@
|
|||
{
|
||||
"$comment": "Per-suite wall-clock durations (ms) for the general-server vitest lane, used by scripts/general-server-shard.mjs to balance suites across the PR shard matrix. Sampled from a real PR run of .github/workflows/pr.yml (actions run 32708351172, 2026-08-24) by diffing consecutive per-suite completion timestamps in the 'Run grouped general test suites' logs \u2014 that captures each suite's true serial cost (import + collect + tests), not just the vitest-reported test time. Suites missing here get the median weight, so the manifest only needs occasional refreshes.",
|
||||
"$chatSample": "chat-channels.integration.test.ts: actions run 34405038082, job 102646337040, 2026-09-09. The first suite completed at 21:19:52.9026416Z after Vitest RUN at 21:09:07.5491992Z: 645354ms rounded up, including startup/import/collection; the 985 tests themselves took 629654ms. All 2972 tests in the shard passed, but the job exceeded its unchanged 20-minute bound during cleanup. Recording this missing heavy-suite weight lets the existing LPT partition reserve one of the existing five shards without changing test coverage, isolation, or deadlines.",
|
||||
"$nativeRunnerSample": "native-codex-runner.integration.test.ts: actions run 34555686996, job 103127786254, 2026-09-11. Consecutive suite completions at 02:50:34.2830368Z and 02:55:07.9837588Z give 273701ms including import/collection (test body 270773ms). This previously unweighted suite made one four-way server shard take 14m38s; recording its cost lets the existing LPT partition balance it in both PR and release runs.",
|
||||
"durations": {
|
||||
"server/src/services/native-runtime/native-codex-runner.integration.test.ts": 273701,
|
||||
"server/src/__tests__/access-service.test.ts": 4757,
|
||||
"server/src/__tests__/access-validators.test.ts": 645,
|
||||
"server/src/__tests__/activity-log-responsible-user.test.ts": 4407,
|
||||
|
|
|
|||
|
|
@ -1,11 +1,13 @@
|
|||
#!/usr/bin/env node
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { mkdirSync, mkdtempSync, readdirSync, realpathSync, statSync } from "node:fs";
|
||||
import { mkdirSync, mkdtempSync, readdirSync, readFileSync, realpathSync, statSync } from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { loadShardDurations, selectGeneralServerShard } from "./general-server-shard.mjs";
|
||||
|
||||
import { assertSelectedTests, partitionTestLines } from "./test-line-shard.mjs";
|
||||
|
||||
const repoRoot = process.cwd();
|
||||
const scriptsDir = path.dirname(fileURLToPath(import.meta.url));
|
||||
const generalServerShardDurations = loadShardDurations(
|
||||
|
|
@ -66,11 +68,15 @@ const serializedModeName = "serialized";
|
|||
const generalModeName = "general";
|
||||
const allModeName = "all";
|
||||
const generalServerGroupName = "general-server";
|
||||
const generalServerWithoutChatGroupName = "general-server-without-chat";
|
||||
const generalChatGroupName = "general-chat";
|
||||
const chatSuite = "server/src/__tests__/chat-channels.integration.test.ts";
|
||||
const generalWorkspacesAGroupName = "general-workspaces-a";
|
||||
const generalWorkspacesBGroupName = "general-workspaces-b";
|
||||
const generalWorkspacesAProjects = ["@paperclipai/ui", "paperclipai"];
|
||||
const generalWorkspacesBProjects = nonServerProjects.filter((project) => !generalWorkspacesAProjects.includes(project));
|
||||
const generalGroupNames = [generalServerGroupName, generalWorkspacesAGroupName, generalWorkspacesBGroupName];
|
||||
const allowedGeneralGroupNames = [...generalGroupNames, generalServerWithoutChatGroupName, generalChatGroupName];
|
||||
const serializedServerVitestArgs = [
|
||||
"--no-file-parallelism",
|
||||
"--maxWorkers=1",
|
||||
|
|
@ -216,10 +222,10 @@ function parseCliOptions(argv) {
|
|||
const shardAllowed =
|
||||
mode === serializedModeName ||
|
||||
(mode === generalModeName &&
|
||||
(group === generalServerGroupName || group === generalWorkspacesAGroupName));
|
||||
([generalServerGroupName, generalServerWithoutChatGroupName, generalChatGroupName, generalWorkspacesAGroupName].includes(group)));
|
||||
if (!shardAllowed && shardIndex !== null) {
|
||||
fail(
|
||||
"--shard-index/--shard-count are only valid with --mode serialized, --mode general --group general-server, or --mode general --group general-workspaces-a.",
|
||||
"--shard-index/--shard-count are only valid with serialized mode or a shardable general server/chat/workspaces-a group.",
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -227,8 +233,8 @@ function parseCliOptions(argv) {
|
|||
fail("--group is only valid with --mode general.");
|
||||
}
|
||||
|
||||
if (group !== null && !generalGroupNames.includes(group)) {
|
||||
fail(`Unknown group "${group}". Expected one of: ${generalGroupNames.join(", ")}.`);
|
||||
if (group !== null && !allowedGeneralGroupNames.includes(group)) {
|
||||
fail(`Unknown group "${group}". Expected one of: ${allowedGeneralGroupNames.join(", ")}.`);
|
||||
}
|
||||
|
||||
if (shardIndex !== null) {
|
||||
|
|
@ -271,7 +277,7 @@ function selectSerializedSuites(routeTests, shardIndex, shardCount) {
|
|||
return shardFiles.map((file) => byRepoPath.get(file));
|
||||
}
|
||||
|
||||
function runVitest(args, label) {
|
||||
function runVitest(args, label, testShard = null) {
|
||||
console.log(`\n[test:run] ${label}`);
|
||||
invocationIndex += 1;
|
||||
const tempRootParent = process.platform === "win32" ? os.tmpdir() : "/tmp";
|
||||
|
|
@ -291,6 +297,25 @@ function runVitest(args, label) {
|
|||
};
|
||||
mkdirSync(env.PAPERCLIP_HOME, { recursive: true });
|
||||
mkdirSync(env.TMPDIR, { recursive: true });
|
||||
if (testShard) {
|
||||
const collect = (filters, name) => {
|
||||
const output = path.join(testRoot, `${name}.json`);
|
||||
const result = spawnSync("pnpm", ["exec", "vitest", "list", ...sourceOnlyVitestArgs,
|
||||
...filters, "--allowOnly=false", "--includeTaskLocation", `--json=${output}`], {
|
||||
cwd: repoRoot, env, stdio: "inherit",
|
||||
});
|
||||
if (result.error || result.status !== 0) fail(`Vitest collection failed: ${result.error?.message ?? result.status}`);
|
||||
return JSON.parse(readFileSync(output, "utf8"));
|
||||
};
|
||||
const collected = collect(args, "all");
|
||||
const file = path.resolve(repoRoot, chatSuite);
|
||||
const selected = partitionTestLines(collected, testShard.count, file)[testShard.index];
|
||||
const filters = selected.lines.map((line) => `${chatSuite}:${line}`);
|
||||
args = [...args.filter((arg) => arg !== chatSuite), ...filters];
|
||||
assertSelectedTests(selected.tests, collect(args, "selected"), file);
|
||||
console.log(`[test:run] chat shard ${testShard.index + 1}/${testShard.count}: ${selected.tests.length}/${collected.length} tests, ${selected.lines.length} source lines; exact filter coverage verified`);
|
||||
args.push("--allowOnly=false");
|
||||
}
|
||||
const result = spawnSync("pnpm", ["exec", "vitest", "run", ...sourceOnlyVitestArgs, ...args], {
|
||||
cwd: repoRoot,
|
||||
env,
|
||||
|
|
@ -325,16 +350,23 @@ function runProjectGroup(projects, groupName, shardIndex = null, shardCount = nu
|
|||
}
|
||||
|
||||
function runGeneralGroup(routeTests, groupName, shardIndex = null, shardCount = null) {
|
||||
if (groupName === generalServerGroupName) {
|
||||
if (groupName === generalChatGroupName) {
|
||||
runVitest(["--project", "@paperclipai/server", ...serializedServerVitestArgs, chatSuite],
|
||||
"chat integration test shard", { index: shardIndex ?? 0, count: shardCount ?? 1 });
|
||||
return;
|
||||
}
|
||||
if (groupName === generalServerGroupName || groupName === generalServerWithoutChatGroupName) {
|
||||
const withoutChat = groupName === generalServerWithoutChatGroupName;
|
||||
const files = withoutChat ? generalServerTestFiles.filter((file) => file !== chatSuite) : generalServerTestFiles;
|
||||
if (shardCount !== null && shardCount > 1) {
|
||||
const shardFiles = selectGeneralServerShard(
|
||||
generalServerTestFiles,
|
||||
files,
|
||||
shardIndex,
|
||||
shardCount,
|
||||
generalServerShardDurations,
|
||||
);
|
||||
console.log(
|
||||
`\n[test:run] general-server shard ${shardIndex + 1}/${shardCount} running ${shardFiles.length} of ${generalServerTestFiles.length} suites`,
|
||||
`\n[test:run] general-server shard ${shardIndex + 1}/${shardCount} running ${shardFiles.length} of ${files.length} suites`,
|
||||
);
|
||||
if (shardFiles.length === 0) {
|
||||
return;
|
||||
|
|
@ -353,6 +385,7 @@ function runGeneralGroup(routeTests, groupName, shardIndex = null, shardCount =
|
|||
}
|
||||
|
||||
const excludeRouteArgs = routeTests.flatMap((file) => ["--exclude", file.serverPath]);
|
||||
if (withoutChat) excludeRouteArgs.push("--exclude", "src/__tests__/chat-channels.integration.test.ts");
|
||||
runVitest(
|
||||
[
|
||||
"--project",
|
||||
|
|
@ -436,16 +469,16 @@ if (options.dryRun) {
|
|||
shardIndex: options.shardIndex,
|
||||
shardCount: options.shardCount,
|
||||
group: options.group,
|
||||
availableGeneralGroups: generalGroupNames,
|
||||
availableGeneralGroups: allowedGeneralGroupNames,
|
||||
serializedSuiteCount: routeTests.length,
|
||||
selectedSerializedSuites: serializedSuites.map((routeTest) => routeTest.repoPath),
|
||||
generalServerSuiteCount: generalServerTestFiles.length,
|
||||
selectedGeneralServerSuites:
|
||||
options.mode === generalModeName &&
|
||||
options.group === generalServerGroupName &&
|
||||
[generalServerGroupName, generalServerWithoutChatGroupName].includes(options.group) &&
|
||||
options.shardCount !== null
|
||||
? selectGeneralServerShard(
|
||||
generalServerTestFiles,
|
||||
options.group === generalServerWithoutChatGroupName ? generalServerTestFiles.filter((file) => file !== chatSuite) : generalServerTestFiles,
|
||||
options.shardIndex,
|
||||
options.shardCount,
|
||||
generalServerShardDurations,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,45 @@
|
|||
import assert from "node:assert/strict";
|
||||
import path from "node:path";
|
||||
|
||||
function validateTests(tests, file) {
|
||||
assert.ok(Array.isArray(tests) && tests.length > 0, "Vitest must collect at least one test");
|
||||
for (const test of tests) {
|
||||
assert.equal(test.projectName, "@paperclipai/server", "unexpected test project");
|
||||
assert.equal(path.resolve(test.file), path.resolve(file), "unexpected test file");
|
||||
assert.ok(typeof test.name === "string" && test.name.length > 0, "missing test name");
|
||||
assert.ok(Number.isSafeInteger(test.location?.line) && test.location.line > 0, "missing test source line");
|
||||
}
|
||||
}
|
||||
|
||||
// Keep all cases registered on one source line together, including it.each
|
||||
// and loop-generated cases. Balance by collected case count, not line count.
|
||||
export function partitionTestLines(tests, count, file) {
|
||||
validateTests(tests, file);
|
||||
assert.ok(Number.isSafeInteger(count) && count > 0, "invalid shard count");
|
||||
const byLine = new Map();
|
||||
for (const test of tests) {
|
||||
const line = test.location.line;
|
||||
if (!byLine.has(line)) byLine.set(line, []);
|
||||
byLine.get(line).push(test);
|
||||
}
|
||||
assert.ok(byLine.size >= count, "each shard must contain a source line");
|
||||
const groups = [...byLine].sort((a, b) => b[1].length - a[1].length || a[0] - b[0]);
|
||||
const shards = Array.from({ length: count }, () => ({ lines: [], tests: [] }));
|
||||
for (const [line, cases] of groups) {
|
||||
const shard = shards.reduce((best, next) => next.tests.length < best.tests.length ? next : best);
|
||||
shard.lines.push(line);
|
||||
shard.tests.push(...cases);
|
||||
}
|
||||
for (const shard of shards) shard.lines.sort((a, b) => a - b);
|
||||
return shards;
|
||||
}
|
||||
|
||||
// Re-collect using the exact filters passed to the subsequent test run. A
|
||||
// Vitest filtering change must fail here instead of silently dropping cases.
|
||||
export function assertSelectedTests(expected, actual, file) {
|
||||
validateTests(actual, file);
|
||||
const identities = (tests) => tests.map((test) => JSON.stringify([
|
||||
test.projectName, path.resolve(test.file), test.location.line, test.name,
|
||||
])).sort();
|
||||
assert.deepEqual(identities(actual), identities(expected), "Vitest filters must select exactly the assigned tests");
|
||||
}
|
||||
|
|
@ -32,7 +32,7 @@ import {
|
|||
or,
|
||||
sql,
|
||||
} from "drizzle-orm";
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
agents,
|
||||
agentWakeupRequests,
|
||||
|
|
@ -1022,8 +1022,30 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
|
|||
rmSync(secretsTmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
// Services scan this file's shared database. Retire each case's fixtures
|
||||
// after its assertions so another case (or shard order) cannot claim them.
|
||||
const fixtureCompanies = new Set<string>();
|
||||
const fixtureServices = new Set<ChatChannelService>();
|
||||
afterEach(async () => {
|
||||
try {
|
||||
await Promise.all([...fixtureServices].map((service) => service.shutdown()));
|
||||
} finally {
|
||||
if (fixtureCompanies.size > 0) {
|
||||
await db.update(chatEndpoints).set({ status: "paused" })
|
||||
.where(and(inArray(chatEndpoints.companyId, [...fixtureCompanies]), eq(chatEndpoints.status, "active")));
|
||||
// The milestone scanner also considers paused endpoints while their
|
||||
// conversations are active. Retire those bindings after assertions.
|
||||
await db.update(chatConversations).set({ state: "completed" })
|
||||
.where(and(inArray(chatConversations.companyId, [...fixtureCompanies]), inArray(chatConversations.state, ["active", "waiting"])));
|
||||
}
|
||||
fixtureServices.clear();
|
||||
fixtureCompanies.clear();
|
||||
}
|
||||
});
|
||||
|
||||
async function seedCompany() {
|
||||
const companyId = randomUUID();
|
||||
fixtureCompanies.add(companyId);
|
||||
const assignedAgentId = randomUUID();
|
||||
const replacementAgentId = randomUUID();
|
||||
await db.insert(companies).values({
|
||||
|
|
@ -1193,6 +1215,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
|
|||
runtime: runtime as unknown as ChatSdkRuntime,
|
||||
...serviceOverrides,
|
||||
});
|
||||
fixtureServices.add(service);
|
||||
return { cancelRun, runtime, service, wakeup };
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,60 @@
|
|||
import { spawnSync } from "node:child_process";
|
||||
import { mkdtempSync, mkdirSync, readFileSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { expect, it } from "vitest";
|
||||
|
||||
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../..");
|
||||
|
||||
it("runs every active nested/parameterized fixture case exactly once through the real chat shard CLI", () => {
|
||||
const root = realpathSync(mkdtempSync(path.join(os.tmpdir(), "pc-shards-")));
|
||||
try {
|
||||
const tests = path.join(root, "server/src/__tests__");
|
||||
mkdirSync(tests, { recursive: true });
|
||||
symlinkSync(path.join(repoRoot, "node_modules"), path.join(root, "node_modules"), "junction");
|
||||
writeFileSync(path.join(root, "package.json"), JSON.stringify({ private: true }));
|
||||
writeFileSync(path.join(root, "vitest.config.mjs"), `export default {
|
||||
test: { projects: [{ test: { name: "@paperclipai/server", root: ${JSON.stringify(path.join(root, "server"))},
|
||||
include: ["src/**/*.test.ts"], pool: "forks", maxWorkers: 1 } }] }
|
||||
};`);
|
||||
const trace = path.join(root, "executed.jsonl");
|
||||
const fixture = path.join(tests, "chat-channels.integration.test.ts");
|
||||
writeFileSync(fixture, `import { appendFileSync } from "node:fs";
|
||||
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||
let active = false;
|
||||
beforeEach(() => { expect(active).toBe(false); active = true; });
|
||||
afterEach(() => { active = false; });
|
||||
function record(id) { expect(active).toBe(true); appendFileSync(${JSON.stringify(trace)}, JSON.stringify(id) + "\\n"); }
|
||||
it("top-level", () => record("top"));
|
||||
describe("nested", () => {
|
||||
it("first", () => record("nested-first"));
|
||||
it("second", () => record("nested-second"));
|
||||
it.each(["a", "b", "c", "d"])("parameter %s", (value) => record(value));
|
||||
it.skip("intentionally skipped", () => { throw new Error("must stay skipped"); });
|
||||
});`);
|
||||
const run = (index: number, count: number) => spawnSync(process.execPath, [
|
||||
path.join(repoRoot, "scripts/run-vitest-stable.mjs"), "--mode", "general", "--group", "general-chat",
|
||||
"--shard-index", String(index), "--shard-count", String(count),
|
||||
], { cwd: root, env: { ...process.env, CI: "true" }, encoding: "utf8", timeout: 45_000, maxBuffer: 4 * 1024 * 1024 });
|
||||
for (const index of [0, 1]) {
|
||||
const result = run(index, 2);
|
||||
expect(result.error, result.stderr).toBeUndefined();
|
||||
expect(result.status, result.stdout + result.stderr).toBe(0);
|
||||
expect(result.stdout).toContain("exact filter coverage verified");
|
||||
}
|
||||
const executed = readFileSync(trace, "utf8").trim().split("\n").map((line) => JSON.parse(line));
|
||||
expect(executed.sort()).toEqual(["a", "b", "c", "d", "nested-first", "nested-second", "top"]);
|
||||
|
||||
// A real assertion failure must still fail the wrapper after successful
|
||||
// collection and filter validation.
|
||||
writeFileSync(fixture, 'import { it } from "vitest"; it("fails", () => { throw new Error("fixture failure"); });');
|
||||
const failed = run(0, 1);
|
||||
expect(failed.error, failed.stderr).toBeUndefined();
|
||||
expect(failed.stdout).toContain("exact filter coverage verified");
|
||||
expect(failed.status).not.toBe(0);
|
||||
expect(failed.stdout + failed.stderr).toContain("fixture failure");
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
}, 120_000);
|
||||
Loading…
Reference in New Issue