From bfb98aff5dc1f74e08f5ad021758eba8084eda40 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Tue, 1 Sep 2026 04:46:04 -0500 Subject: [PATCH] test(runner): add credential-free acceptance foundation (#12652) ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work. > - Paperclip supports direct adapters and an experimental native Runner path. > - These paths need one stable compatibility matrix. > - The matrix must not launch providers or load credentials during normal tests. > - Result handling must reject incomplete output and sensitive values. > - This pull request adds a credential-free acceptance foundation. > - The benefit is a reviewable contract for later end-to-end executors. ## Linked Issues or Issue Description **What existing behavior does this improve?** This improves verification for direct adapters and Paperclip Runner providers. **Subsystem affected** Cross-cutting test infrastructure for adapters, the server runtime, and the task thread. **Current behavior** The repository has subsystem tests. It does not have one declarative matrix for direct and native compatibility. **Proposed behavior** Add a pure acceptance catalog, result validator, redaction helpers, and failure classification. Keep all execution authority outside this change. **Reason and benefit** The matrix makes legacy isolation and native recovery requirements explicit. The helpers let later executors report safe and complete results. **Breaking changes** None. This change does not alter production runtime selection or start any provider. ## What Changed - Add a catalog for built-in direct adapters and qualified native provider profiles. - Add compatibility cases for runtime selection, task threads, questions, and flag-change recovery. - Add pure redaction and transient-failure classification helpers. - Add fail-closed Markdown and JUnit report aggregation. - Add isolated test and type-check commands. - Document the credential-free boundary and deferred live execution work. ## Verification GitHub Actions must run: - `pnpm test:runner-acceptance` - `pnpm test:runner-acceptance:typecheck` - The repository test, type-check, build, policy, and security gates. No local test command was run. The repository owner requested GitHub-only verification. ## Risks Low production risk. The change adds test-only files and root scripts. The catalog can drift when a built-in adapter changes. Its validation fails closed on that drift. ## Model Used OpenAI Codex with the GPT-5 agent model. The work used high reasoning, repository inspection, tool use, and parallel code review. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [ ] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --- package.json | 2 + .../src/live/live-session.test.ts | 3 +- tests/runner-acceptance/README.md | 70 ++++ tests/runner-acceptance/catalog.test.ts | 64 ++++ tests/runner-acceptance/catalog.ts | 328 ++++++++++++++++++ tests/runner-acceptance/failure-classifier.ts | 28 ++ tests/runner-acceptance/redaction.ts | 118 +++++++ tests/runner-acceptance/report.test.ts | 189 ++++++++++ tests/runner-acceptance/report.ts | 316 +++++++++++++++++ tests/runner-acceptance/support.test.ts | 62 ++++ tests/runner-acceptance/tsconfig.json | 13 + tests/runner-acceptance/types.ts | 85 +++++ tests/runner-acceptance/vitest.config.ts | 10 + 13 files changed, 1287 insertions(+), 1 deletion(-) create mode 100644 tests/runner-acceptance/README.md create mode 100644 tests/runner-acceptance/catalog.test.ts create mode 100644 tests/runner-acceptance/catalog.ts create mode 100644 tests/runner-acceptance/failure-classifier.ts create mode 100644 tests/runner-acceptance/redaction.ts create mode 100644 tests/runner-acceptance/report.test.ts create mode 100644 tests/runner-acceptance/report.ts create mode 100644 tests/runner-acceptance/support.test.ts create mode 100644 tests/runner-acceptance/tsconfig.json create mode 100644 tests/runner-acceptance/types.ts create mode 100644 tests/runner-acceptance/vitest.config.ts diff --git a/package.json b/package.json index 67c699d3b2..1e73bb19d3 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,8 @@ "test": "pnpm run test:run", "test:watch": "pnpm run preflight:workspace-links && vitest", "test:run": "pnpm run preflight:workspace-links && node scripts/run-vitest-stable.mjs", + "test:runner-acceptance": "vitest run --config tests/runner-acceptance/vitest.config.ts", + "test:runner-acceptance:typecheck": "tsc -p tests/runner-acceptance/tsconfig.json", "test:run:general": "pnpm run preflight:workspace-links && pnpm --filter @paperclipai/plugin-sdk ensure-build-deps && node scripts/run-vitest-stable.mjs --mode general", "test:run:serialized": "pnpm run preflight:workspace-links && pnpm --filter @paperclipai/plugin-sdk ensure-build-deps && node scripts/run-vitest-stable.mjs --mode serialized", "db:generate": "pnpm --filter @paperclipai/db generate", diff --git a/packages/paperclip-runner/src/live/live-session.test.ts b/packages/paperclip-runner/src/live/live-session.test.ts index 4881dc4851..12ec300ebf 100644 --- a/packages/paperclip-runner/src/live/live-session.test.ts +++ b/packages/paperclip-runner/src/live/live-session.test.ts @@ -1118,6 +1118,7 @@ describe("Capability live runnerd and Codex session", () => { ])); await resumedService.shutdown(resumed.id, "test complete"); }, - 15_000, + // CI exercises two real process generations here and can exceed the unit default under load. + 30_000, ); }); diff --git a/tests/runner-acceptance/README.md b/tests/runner-acceptance/README.md new file mode 100644 index 0000000000..04339d3176 --- /dev/null +++ b/tests/runner-acceptance/README.md @@ -0,0 +1,70 @@ +# Credential-free Runner acceptance foundation + +This directory defines the reviewable, deterministic foundation for Runner +acceptance checks. It contains no launcher and grants no authority to start +Paperclip, a provider process, a browser, a remote environment, or a billable +model request. + +The catalog covers: + +- every current built-in direct adapter except the explicitly deferred Pi + adapter; +- the external-plugin direct-adapter compatibility contract; +- Paperclip Runner with Codex; +- Paperclip Runner with the qualified OpenCode model; and +- Paperclip Runner with the qualified ACPX Claude and Codex profiles. + +The direct-adapter cells assert the legacy boundary: runnerd does not start, +native records are not created, direct finalization remains authoritative, and +classic task controls remain available. Native cells assert persisted provider +identity, runtime authority, structured-question behavior, and recovery of an +already-recorded run after the rollout flag changes. + +Registered compatibility-only adapters, including the retired `acpx_local` +entry, remain in the direct catalog so a future selection check can prove they +never fall into native execution. Their presence is not a claim that the +adapter can start a provider session. + +## Commands + +Run the isolated unit suite: + +```sh +pnpm test:runner-acceptance +``` + +Check the standalone TypeScript boundary: + +```sh +pnpm test:runner-acceptance:typecheck +``` + +These commands are credential-free. They validate the catalog and pure support +utilities; they do not claim that a provider was contacted. + +## Result boundary + +Future fixture executors may emit `paperclip.runner-acceptance.result/v1` +objects and pass them to `buildRunnerAcceptanceReport`. Results must name a +catalog cell, report every expected assertion, carry successful redaction, and +contain no sensitive-looking structured values. The aggregator produces only +in-memory normalized data plus optional Markdown or JUnit strings. It does not +write evidence, screenshots, history, or public reports. + +## Deliberate exclusions + +This foundation does not include: + +- Pi or any Pi transitive package; +- Claude Managed or AWS AgentCore; +- Daytona, remote images, live provider execution, or cost accounting; +- provider-key loading, secret references, auth-file discovery, or tracked env + templates; +- screenshots, traces, raw logs, evidence archives, dashboards, history, or + publication workflows; or +- the eval kernel, scenario explorer, browser SDK, or package/runtime + replacement work. + +Add an executor only in a later, explicitly authorized change. Keep its launch +authority separate from this catalog, and make any billable or secret-bearing +mode opt-in and independently reviewed. diff --git a/tests/runner-acceptance/catalog.test.ts b/tests/runner-acceptance/catalog.test.ts new file mode 100644 index 0000000000..ca9850c678 --- /dev/null +++ b/tests/runner-acceptance/catalog.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, it } from "vitest"; + +import { QUALIFIED_ACPX_PROFILES } from "../../packages/paperclip-runner/src/drivers/acpx/qualified-profiles.js"; +import { QUALIFIED_OPENCODE_MODEL } from "../../packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.js"; +import { + directAcceptanceProfiles, + nativeAcceptanceProfiles, + runnerAcceptanceCases, + runnerAcceptanceMatrix, + validateRunnerAcceptanceCatalog, +} from "./catalog.js"; + +describe("credential-free Runner acceptance catalog", () => { + it("covers direct adapters without activating the native runtime", () => { + expect(directAcceptanceProfiles.length).toBeGreaterThan(10); + expect(directAcceptanceProfiles.every((profile) => + profile.generation === "direct" + && profile.expectedRuntimeMode === "legacy" + && profile.invariants.includes("runnerd.start_count=0") + && profile.invariants.includes("native_record.count=0"))).toBe(true); + expect(directAcceptanceProfiles.some(({ adapterType }) => adapterType === "process")).toBe(true); + expect(directAcceptanceProfiles.some(({ adapterType }) => adapterType === "http")).toBe(true); + expect(directAcceptanceProfiles.some(({ adapterScope }) => + adapterScope === "external_plugin_contract")).toBe(true); + }); + + it("contains only the approved native providers and ACPX profiles", () => { + expect(nativeAcceptanceProfiles.map(({ id }) => id)).toEqual([ + "runner-codex", + "runner-opencode", + "runner-acpx-claude", + "runner-acpx-codex", + ]); + expect(nativeAcceptanceProfiles.find(({ id }) => id === "runner-opencode")?.model) + .toBe(QUALIFIED_OPENCODE_MODEL); + expect(nativeAcceptanceProfiles.find(({ id }) => id === "runner-acpx-claude")?.model) + .toBe(QUALIFIED_ACPX_PROFILES.claude.qualificationModel); + expect(nativeAcceptanceProfiles.find(({ id }) => id === "runner-acpx-codex")?.model) + .toBe(QUALIFIED_ACPX_PROFILES.codex.qualificationModel); + expect(nativeAcceptanceProfiles.some(({ adapterConfig }) => + adapterConfig.acpxAgent === "pi")).toBe(false); + expect(directAcceptanceProfiles.some(({ adapterType }) => + adapterType === "pi_local")).toBe(false); + }); + + it("contains no launch authority, provider credentials, or managed providers", () => { + const serialized = JSON.stringify({ + profiles: [...directAcceptanceProfiles, ...nativeAcceptanceProfiles], + cases: runnerAcceptanceCases, + }); + expect(serialized).not.toMatch(/claude_managed|aws_agentcore|daytona/i); + expect(serialized).not.toMatch(/api.?key|secret_ref|authorization/i); + }); + + it("builds a stable, unique matrix and applies recovery only to native profiles", () => { + expect(validateRunnerAcceptanceCatalog()).toEqual(runnerAcceptanceMatrix); + expect(new Set(runnerAcceptanceMatrix.map(({ id }) => id)).size) + .toBe(runnerAcceptanceMatrix.length); + expect(runnerAcceptanceMatrix + .filter(({ acceptanceCase }) => acceptanceCase.id === "flagged-native-recovery") + .every(({ profile }) => profile.generation === "native")) + .toBe(true); + }); +}); diff --git a/tests/runner-acceptance/catalog.ts b/tests/runner-acceptance/catalog.ts new file mode 100644 index 0000000000..57978474dd --- /dev/null +++ b/tests/runner-acceptance/catalog.ts @@ -0,0 +1,328 @@ +import { createHash } from "node:crypto"; + +import { DEFAULT_CODEX_LOCAL_MODEL } from "../../packages/adapters/codex-local/src/index.js"; +import { QUALIFIED_ACPX_PROFILES } from "../../packages/paperclip-runner/src/drivers/acpx/qualified-profiles.js"; +import { QUALIFIED_OPENCODE_MODEL } from "../../packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.js"; +import { BUILTIN_ADAPTER_TYPES } from "../../server/src/adapters/builtin-adapter-types.js"; + +import { findSensitiveValue } from "./redaction.js"; +import type { + JsonValue, + RunnerAcceptanceCase, + RunnerAcceptanceCell, + RunnerAcceptanceProfile, +} from "./types.js"; + +const excludedBuiltInAdapterTypes = new Set(["paperclip_runner", "pi_local"]); + +const directBuiltInAdapterTypes = [ + "acpx_local", + "claude_local", + "codex_local", + "cursor_cloud", + "cursor", + "gemini_local", + "grok_local", + "hermes_gateway", + "hermes_local", + "kimi_local", + "openclaw_gateway", + "opencode_local", + "process", + "http", +] as const; + +function directProfile(adapterType: string): RunnerAcceptanceProfile { + return { + id: `direct-${adapterType.replaceAll("_", "-")}`, + label: `Direct ${adapterType}`, + generation: "direct", + adapterScope: "built_in", + adapterType, + expectedRuntimeMode: "legacy", + provider: null, + model: null, + adapterConfig: {}, + invariants: [ + "runtime.mode=legacy", + "runnerd.start_count=0", + "native_record.count=0", + "finalization.path=direct_adapter", + "task_controls.mode=classic", + ], + }; +} + +function nativeProfile(input: { + id: string; + label: string; + provider: "codex" | "opencode" | "acpx"; + model: string; + adapterConfig: Record; +}): RunnerAcceptanceProfile { + return { + ...input, + generation: "native", + adapterScope: "built_in", + adapterType: "paperclip_runner", + expectedRuntimeMode: "native", + invariants: [ + "runtime.mode=native", + "runnerd.start_count=1", + "native_record.count>=1", + "finalization.path=native", + "provider.persisted=true", + ], + }; +} + +export const directAcceptanceProfiles: readonly RunnerAcceptanceProfile[] = [ + ...directBuiltInAdapterTypes.map(directProfile), + { + id: "direct-external-plugin-contract", + label: "Direct external plugin contract", + generation: "direct", + adapterScope: "external_plugin_contract", + adapterType: "external_plugin:*", + expectedRuntimeMode: "legacy", + provider: null, + model: null, + adapterConfig: {}, + invariants: [ + "runtime.mode=legacy", + "runnerd.start_count=0", + "native_record.count=0", + "finalization.path=plugin_adapter", + "task_controls.mode=classic", + ], + }, +] as const; + +export const nativeAcceptanceProfiles: readonly RunnerAcceptanceProfile[] = [ + nativeProfile({ + id: "runner-codex", + label: "Paperclip Runner Codex", + provider: "codex", + model: DEFAULT_CODEX_LOCAL_MODEL, + adapterConfig: { + provider: "codex", + codexPermissionMode: "untrusted", + lifecycleMode: "per_turn", + }, + }), + nativeProfile({ + id: "runner-opencode", + label: "Paperclip Runner OpenCode", + provider: "opencode", + model: QUALIFIED_OPENCODE_MODEL, + adapterConfig: { + provider: "opencode", + model: QUALIFIED_OPENCODE_MODEL, + opencodePermissionMode: "ask", + lifecycleMode: "per_turn", + }, + }), + nativeProfile({ + id: "runner-acpx-claude", + label: "Paperclip Runner ACPX Claude", + provider: "acpx", + model: QUALIFIED_ACPX_PROFILES.claude.qualificationModel, + adapterConfig: { + provider: "acpx", + acpxAgent: "claude", + model: QUALIFIED_ACPX_PROFILES.claude.qualificationModel, + acpxPermissionMode: "approve-reads", + lifecycleMode: "per_turn", + }, + }), + nativeProfile({ + id: "runner-acpx-codex", + label: "Paperclip Runner ACPX Codex", + provider: "acpx", + model: QUALIFIED_ACPX_PROFILES.codex.qualificationModel, + adapterConfig: { + provider: "acpx", + acpxAgent: "codex", + model: QUALIFIED_ACPX_PROFILES.codex.qualificationModel, + acpxPermissionMode: "approve-reads", + lifecycleMode: "per_turn", + }, + }), +] as const; + +export const runnerAcceptanceProfiles: readonly RunnerAcceptanceProfile[] = [ + ...directAcceptanceProfiles, + ...nativeAcceptanceProfiles, +]; + +export const runnerAcceptanceCases: readonly RunnerAcceptanceCase[] = [ + { + id: "runtime-selection", + label: "Runtime selection remains isolated", + revision: 1, + appliesTo: "all", + assertions: [ + "runtime_selection.is_stable", + "status_authority.matches_runtime", + "execution_occurs_once", + ], + }, + { + id: "task-thread-baseline", + label: "Task thread preserves baseline behavior", + revision: 1, + appliesTo: "all", + assertions: [ + "task_thread.active_state", + "task_thread.settled_state", + "task_thread.empty_transcript_state", + "task_thread.classic_interface_state", + ], + }, + { + id: "question-round-trip", + label: "Structured question round trip", + revision: 1, + appliesTo: "all", + assertions: [ + "question.rendered_once", + "question.response_validated", + "question.resume_occurs_once", + "finalization.byte_stable", + ], + }, + { + id: "flagged-native-recovery", + label: "Persisted native recovery after flag change", + revision: 1, + appliesTo: "native", + assertions: [ + "fresh_start.flag_required", + "persisted_run.readable_when_disabled", + "persisted_run.recoverable_when_disabled", + "provider.identity_does_not_drift", + ], + }, +] as const; + +function suiteDefinitionHash() { + return createHash("sha256") + .update(JSON.stringify({ + profiles: runnerAcceptanceProfiles.map((profile) => ({ + id: profile.id, + adapterType: profile.adapterType, + generation: profile.generation, + provider: profile.provider, + model: profile.model, + adapterConfig: profile.adapterConfig, + invariants: profile.invariants, + })), + cases: runnerAcceptanceCases.map((acceptanceCase) => ({ + id: acceptanceCase.id, + revision: acceptanceCase.revision, + appliesTo: acceptanceCase.appliesTo, + assertions: acceptanceCase.assertions, + })), + })) + .digest("hex"); +} + +export const runnerAcceptanceSuiteDefinitionHash = suiteDefinitionHash(); + +export function buildRunnerAcceptanceMatrix(): RunnerAcceptanceCell[] { + return runnerAcceptanceProfiles.flatMap((profile) => + runnerAcceptanceCases + .filter((acceptanceCase) => + acceptanceCase.appliesTo === "all" + || acceptanceCase.appliesTo === profile.generation) + .map((acceptanceCase) => ({ + id: `${profile.id}.${acceptanceCase.id}`, + suiteId: "runner-compatibility" as const, + suiteDefinitionHash: runnerAcceptanceSuiteDefinitionHash, + profile, + acceptanceCase, + assertions: [...new Set([ + ...profile.invariants, + ...acceptanceCase.assertions, + ])], + })), + ); +} + +function duplicates(values: readonly string[]) { + const seen = new Set(); + return values.filter((value) => { + if (seen.has(value)) return true; + seen.add(value); + return false; + }); +} + +function assertSafeConfig(value: JsonValue, path = "adapterConfig") { + if (typeof value === "string") { + const leak = findSensitiveValue(value); + if (leak) throw new Error(`Acceptance catalog contains ${leak} at ${path}`); + return; + } + if (Array.isArray(value)) { + value.forEach((entry, index) => assertSafeConfig(entry, `${path}[${index}]`)); + return; + } + if (!value || typeof value !== "object") return; + for (const [key, entry] of Object.entries(value)) { + if (/(?:authorization|api.?key|token|password|secret|credential|env)/i.test(key)) { + throw new Error(`Acceptance catalog contains prohibited field ${path}.${key}`); + } + assertSafeConfig(entry, `${path}.${key}`); + } +} + +export function validateRunnerAcceptanceCatalog() { + const duplicateProfileIds = duplicates(runnerAcceptanceProfiles.map(({ id }) => id)); + if (duplicateProfileIds.length > 0) { + throw new Error(`Duplicate runner acceptance profile ids: ${duplicateProfileIds.join(", ")}`); + } + const duplicateCaseIds = duplicates(runnerAcceptanceCases.map(({ id }) => id)); + if (duplicateCaseIds.length > 0) { + throw new Error(`Duplicate runner acceptance case ids: ${duplicateCaseIds.join(", ")}`); + } + + const expectedDirectBuiltIns = new Set( + [...BUILTIN_ADAPTER_TYPES].filter((type) => !excludedBuiltInAdapterTypes.has(type)), + ); + const catalogedDirectBuiltIns = new Set( + directAcceptanceProfiles + .filter(({ adapterScope }) => adapterScope === "built_in") + .map(({ adapterType }) => adapterType), + ); + const missing = [...expectedDirectBuiltIns].filter((type) => !catalogedDirectBuiltIns.has(type)); + const unexpected = [...catalogedDirectBuiltIns].filter((type) => !expectedDirectBuiltIns.has(type)); + if (missing.length > 0 || unexpected.length > 0) { + throw new Error( + `Direct adapter catalog drift; missing=${missing.join(",") || "none"}; unexpected=${unexpected.join(",") || "none"}`, + ); + } + + for (const profile of runnerAcceptanceProfiles) { + assertSafeConfig(profile.adapterConfig); + if (profile.adapterType === "pi_local") { + throw new Error("Pi is outside the acceptance foundation boundary"); + } + if ( + profile.provider === "acpx" + && profile.adapterConfig.acpxAgent !== "claude" + && profile.adapterConfig.acpxAgent !== "codex" + ) { + throw new Error(`Unqualified ACPX profile ${profile.id}`); + } + } + + const matrix = buildRunnerAcceptanceMatrix(); + const duplicateCellIds = duplicates(matrix.map(({ id }) => id)); + if (duplicateCellIds.length > 0) { + throw new Error(`Duplicate runner acceptance cell ids: ${duplicateCellIds.join(", ")}`); + } + return matrix; +} + +export const runnerAcceptanceMatrix = validateRunnerAcceptanceCatalog(); diff --git a/tests/runner-acceptance/failure-classifier.ts b/tests/runner-acceptance/failure-classifier.ts new file mode 100644 index 0000000000..b76b0741e6 --- /dev/null +++ b/tests/runner-acceptance/failure-classifier.ts @@ -0,0 +1,28 @@ +import type { FailureClass } from "./types.js"; + +const TRANSIENT = + /(?:\b429\b|\b5\d\d\b|rate.?limit|ECONN(?:RESET|REFUSED)|socket hang up|network (?:error|interruption|timeout)|service unavailable|(?:provider|server|bootstrap|browser|webserver|health|connection|harness).*(?:temporar|timed? out|timeout|closed|failed|unavailable|interrupt|reset|refused|start|connect)|(?:timed? out|timeout).*(?:provider|server|bootstrap|browser|webserver|health|connection|harness))/i; +const PERMANENT = + /(?:unauthorized|forbidden|qualification|model.*(?:unsupported|incompatible)|artifact.*incompatible|runner_remote_.*(?:incompatible|unavailable)|provider.*unsupported)/i; +const CANDIDATE = + /(?:assertion|matcher|expected.*observed|marker|issue status|run status|runtime mode|wrong output|missing output)/i; + +export function classifyFailure(error: unknown): FailureClass { + const message = error instanceof Error + ? `${error.name}: ${error.message}` + : String(error); + if (/secret.*(?:leak|plaintext|redaction)/i.test(message)) return "secret_leak"; + if (/cleanup|teardown|lease.*release/i.test(message)) { + return TRANSIENT.test(message) + ? "transient_infrastructure" + : "cleanup_failure"; + } + if (PERMANENT.test(message)) return "permanent_infrastructure"; + if (TRANSIENT.test(message)) return "transient_infrastructure"; + if (CANDIDATE.test(message)) return "candidate_failure"; + return "candidate_failure"; +} + +export function shouldRetryFailure(failureClass: FailureClass) { + return failureClass === "transient_infrastructure"; +} diff --git a/tests/runner-acceptance/redaction.ts b/tests/runner-acceptance/redaction.ts new file mode 100644 index 0000000000..becf0ca9aa --- /dev/null +++ b/tests/runner-acceptance/redaction.ts @@ -0,0 +1,118 @@ +import { redactDiagnosticText } from "../../packages/adapter-utils/src/command-redaction.js"; + +const SECRET_SHAPES = [ + /\bsk-ant-[A-Za-z0-9_-]{16,}\b/g, + /\bsk-(?:proj-)?[A-Za-z0-9_-]{16,}\b/g, + /\bopenrouter[-_]?(?:api)?[-_]?key["'=:\s]+[A-Za-z0-9._-]{12,}\b/gi, +] as const; +const SENSITIVE_KEY = /(?:authorization|api.?key|token|password|secret|credential|private.?key)/i; + +export function normalizedSensitiveValues( + values: readonly (string | undefined)[], +) { + return [...new Set( + values + .map((value) => value?.trim()) + .filter((value): value is string => Boolean(value)), + )].sort((left, right) => right.length - left.length); +} + +function redactKnownValuesAndShapes( + value: string, + sensitiveValues: readonly string[], +) { + let redacted = value; + for (const sensitiveValue of normalizedSensitiveValues(sensitiveValues)) { + redacted = redacted.split(sensitiveValue).join("[REDACTED]"); + } + for (const pattern of SECRET_SHAPES) { + pattern.lastIndex = 0; + redacted = redacted.replace(pattern, "[REDACTED_SECRET_SHAPE]"); + } + return redacted; +} + +export function redactText( + value: string, + sensitiveValues: readonly string[] = [], +) { + return redactKnownValuesAndShapes( + redactDiagnosticText(value, "[REDACTED]"), + sensitiveValues, + ); +} + +export function findSensitiveValue( + value: string | Buffer, + sensitiveValues: readonly string[] = [], +): string | null { + const text = Buffer.isBuffer(value) ? value.toString("utf8") : value; + for (const sensitiveValue of normalizedSensitiveValues(sensitiveValues)) { + if (text.includes(sensitiveValue)) return "exact sensitive value"; + } + for (const pattern of SECRET_SHAPES) { + pattern.lastIndex = 0; + if (pattern.test(text)) return "secret-shaped value"; + } + return null; +} + +export function findSensitiveJsonValue( + value: unknown, + sensitiveValues: readonly string[] = [], +): string | null { + if (typeof value === "string") return findSensitiveValue(value, sensitiveValues); + if (Array.isArray(value)) { + for (const entry of value) { + const leak = findSensitiveJsonValue(entry, sensitiveValues); + if (leak) return leak; + } + return null; + } + if (value && typeof value === "object") { + for (const [key, entry] of Object.entries(value)) { + if ( + SENSITIVE_KEY.test(key) + && entry !== null + && entry !== undefined + && entry !== "[REDACTED]" + && entry !== "[REDACTED_SECRET_SHAPE]" + ) { + return `sensitive field ${key}`; + } + const leak = findSensitiveJsonValue(entry, sensitiveValues); + if (leak) return leak; + } + } + return null; +} + +export function sanitizeJson( + value: unknown, + sensitiveValues: readonly string[] = [], +): unknown { + if (typeof value === "string") return redactText(value, sensitiveValues); + if (Array.isArray(value)) { + return value.map((entry) => sanitizeJson(entry, sensitiveValues)); + } + if (value && typeof value === "object") { + return Object.fromEntries( + Object.entries(value).map(([key, entry]) => [ + key, + SENSITIVE_KEY.test(key) && entry !== null && entry !== undefined + ? "[REDACTED]" + : sanitizeJson(entry, sensitiveValues), + ]), + ); + } + return value; +} + +export function assertSensitiveValueFree( + value: string | Buffer, + sensitiveValues: readonly string[], + label: string, +) { + const leak = findSensitiveValue(value, sensitiveValues); + if (leak) throw new Error(`Secret leak in ${label}: ${leak}`); +} diff --git a/tests/runner-acceptance/report.test.ts b/tests/runner-acceptance/report.test.ts new file mode 100644 index 0000000000..e8c5bf7071 --- /dev/null +++ b/tests/runner-acceptance/report.test.ts @@ -0,0 +1,189 @@ +import { describe, expect, it } from "vitest"; + +import { runnerAcceptanceMatrix } from "./catalog.js"; +import { + buildRunnerAcceptanceReport, + renderRunnerAcceptanceJUnit, + renderRunnerAcceptanceMarkdown, +} from "./report.js"; +import type { RunnerAcceptanceCell, RunnerAcceptanceResult } from "./types.js"; + +function passingResult( + cell: RunnerAcceptanceCell, + attempt = 1, +): RunnerAcceptanceResult { + return { + schema: "paperclip.runner-acceptance.result/v1", + cellId: cell.id, + attempt, + status: "passed", + startedAt: "2026-09-01T00:00:00.000Z", + finishedAt: `2026-09-01T00:00:0${attempt}.000Z`, + durationMs: attempt * 1_000, + redaction: "passed", + assertions: cell.assertions.map((id) => ({ id, passed: true })), + }; +} + +describe("Runner acceptance report", () => { + const cells = runnerAcceptanceMatrix.slice(0, 2); + + it("selects the newest attempt and reports missing cells without publishing evidence", () => { + const retry = passingResult(cells[0]!, 2); + const report = buildRunnerAcceptanceReport({ + cells, + generatedAt: "2026-09-01T00:01:00.000Z", + results: [ + { + ...passingResult(cells[0]!), + status: "failed", + failureClass: "transient_infrastructure", + error: "connection timed out", + }, + retry, + ], + }); + + expect(report).toMatchObject({ + selected: 2, + passed: 1, + failed: 1, + retries: 1, + }); + expect(report.results[0]).toMatchObject({ attempt: 2, valid: true }); + expect(report.results[1]).toMatchObject({ + attempt: 0, + valid: false, + error: "No result was supplied", + }); + }); + + it("fails closed on incomplete assertions or unsafe structured values", () => { + const cell = cells[0]!; + const incomplete = passingResult(cell); + const report = buildRunnerAcceptanceReport({ + cells: [cell], + results: [{ + ...incomplete, + error: "redacted diagnostic", + assertions: incomplete.assertions.slice(1), + }], + }); + expect(report.failed).toBe(1); + expect(report.results[0]?.validationErrors).toContain( + `missing assertion ${cell.assertions[0]}`, + ); + + const unsafeResult = { + ...passingResult(cell), + diagnostic: { accessToken: "not-a-real-sensitive-value" }, + } as unknown as RunnerAcceptanceResult; + const unsafe = buildRunnerAcceptanceReport({ + cells: [cell], + results: [unsafeResult], + }); + expect(unsafe.failed).toBe(1); + expect(unsafe.results[0]?.validationErrors).toContain( + "unsafe result payload: sensitive field accessToken", + ); + expect(JSON.stringify(unsafe.results[0])).not.toContain( + "not-a-real-sensitive-value", + ); + expect(unsafe.results[0]).not.toHaveProperty("diagnostic"); + }); + + it("fails closed when assertions are missing or are not an array", () => { + const cell = cells[0]!; + const withoutAssertions = { ...passingResult(cell) } as Record; + delete withoutAssertions.assertions; + const missing = buildRunnerAcceptanceReport({ + cells: [cell], + results: [withoutAssertions], + }); + const nonArray = buildRunnerAcceptanceReport({ + cells: [cell], + results: [{ ...passingResult(cell), assertions: { passed: true } }], + }); + + for (const report of [missing, nonArray]) { + expect(report).toMatchObject({ selected: 1, passed: 0, failed: 1 }); + expect(report.results[0]).toMatchObject({ + valid: false, + assertions: [], + }); + expect(report.results[0]?.validationErrors).toEqual(expect.arrayContaining([ + "assertions must be an array", + `missing assertion ${cell.assertions[0]}`, + ])); + } + }); + + it("normalizes invalid scalar and assertion types without throwing", () => { + const cell = cells[0]!; + const report = buildRunnerAcceptanceReport({ + cells: [cell], + results: [{ + schema: 1, + cellId: cell.id, + attempt: "1", + status: true, + failureClass: 42, + error: { message: "failed" }, + startedAt: 1, + finishedAt: null, + durationMs: "1000", + redaction: false, + assertions: [ + null, + { id: cell.assertions[0], passed: "yes", detail: 42 }, + ], + }], + }); + + expect(report).toMatchObject({ selected: 1, passed: 0, failed: 1 }); + expect(report.results[0]).toMatchObject({ + attempt: 0, + status: "failed", + startedAt: "", + finishedAt: "", + durationMs: 0, + redaction: "failed", + valid: false, + }); + expect(report.results[0]?.validationErrors).toEqual(expect.arrayContaining([ + "unsupported result schema", + "attempt must be a positive safe integer", + "duration must be a non-negative finite number", + "timestamps must be valid ISO-compatible values", + "status must be passed or failed", + "failureClass is unsupported", + "error must be a string when present", + "redaction did not pass", + "assertion 0 must be an object", + `assertion ${cell.assertions[0]} passed must be a boolean`, + `assertion ${cell.assertions[0]} detail must be a string when present`, + ])); + }); + + it("renders deterministic Markdown and JUnit summaries", () => { + const report = buildRunnerAcceptanceReport({ + cells, + results: cells.map((cell) => passingResult(cell)), + generatedAt: "2026-09-01T00:01:00.000Z", + }); + const markdown = renderRunnerAcceptanceMarkdown(report); + const junit = renderRunnerAcceptanceJUnit(report); + + expect(markdown).toContain("Passed: 2/2"); + expect(markdown).toContain(cells[0]!.id); + expect(junit).toContain('tests="2" failures="0"'); + expect(junit).toContain('classname="runner-acceptance"'); + }); + + it("rejects results for cells outside the declared catalog", () => { + expect(() => buildRunnerAcceptanceReport({ + cells, + results: [{ ...passingResult(cells[0]!), cellId: "unknown.cell" }], + })).toThrow("unknown acceptance cells"); + }); +}); diff --git a/tests/runner-acceptance/report.ts b/tests/runner-acceptance/report.ts new file mode 100644 index 0000000000..8a99adc2a9 --- /dev/null +++ b/tests/runner-acceptance/report.ts @@ -0,0 +1,316 @@ +import { runnerAcceptanceMatrix } from "./catalog.js"; +import { findSensitiveJsonValue, redactText } from "./redaction.js"; +import type { + AggregatedAcceptanceResult, + FailureClass, + RunnerAcceptanceCell, + RunnerAcceptanceReport, + RunnerAcceptanceResult, +} from "./types.js"; + +const RESULT_SCHEMA = "paperclip.runner-acceptance.result/v1" as const; +const ALLOWED_FAILURE_CLASSES = new Set([ + "candidate_failure", + "transient_infrastructure", + "permanent_infrastructure", + "secret_leak", + "cleanup_failure", +]); + +function record(value: unknown): Record | null { + return value !== null && typeof value === "object" && !Array.isArray(value) + ? value as Record + : null; +} + +function field(value: Record, key: string): unknown { + try { + return value[key]; + } catch { + return undefined; + } +} + +function textField(value: Record, key: string): string | null { + const candidate = field(value, key); + return typeof candidate === "string" ? candidate : null; +} + +function numericField(value: Record, key: string): number | null { + const candidate = field(value, key); + return typeof candidate === "number" && Number.isFinite(candidate) + ? candidate + : null; +} + +function safeSensitivePayloadError(value: unknown): string | null { + try { + const leak = findSensitiveJsonValue(value); + return leak ? `unsafe result payload: ${leak}` : null; + } catch { + return "result payload could not be inspected safely"; + } +} + +function validationErrors( + value: unknown, + cell: RunnerAcceptanceCell, +) { + const errors: string[] = []; + const result = record(value); + if (!result) return ["result must be an object"]; + + if (field(result, "schema") !== RESULT_SCHEMA) { + errors.push("unsupported result schema"); + } + if (textField(result, "cellId") !== cell.id) { + errors.push("result cellId does not match the acceptance cell"); + } + const attempt = numericField(result, "attempt"); + if (attempt === null || !Number.isSafeInteger(attempt) || attempt < 1) { + errors.push("attempt must be a positive safe integer"); + } + const durationMs = numericField(result, "durationMs"); + if (durationMs === null || durationMs < 0) { + errors.push("duration must be a non-negative finite number"); + } + const startedAt = textField(result, "startedAt"); + const finishedAt = textField(result, "finishedAt"); + if ( + startedAt === null + || finishedAt === null + || !Number.isFinite(Date.parse(startedAt)) + || !Number.isFinite(Date.parse(finishedAt)) + ) { + errors.push("timestamps must be valid ISO-compatible values"); + } + const status = field(result, "status"); + if (status !== "passed" && status !== "failed") { + errors.push("status must be passed or failed"); + } + const failureClass = field(result, "failureClass"); + if (failureClass !== undefined && !ALLOWED_FAILURE_CLASSES.has(failureClass as FailureClass)) { + errors.push("failureClass is unsupported"); + } + const error = field(result, "error"); + if (error !== undefined && typeof error !== "string") { + errors.push("error must be a string when present"); + } + if (field(result, "redaction") !== "passed") errors.push("redaction did not pass"); + const sensitivePayloadError = safeSensitivePayloadError(value); + if (sensitivePayloadError) errors.push(sensitivePayloadError); + + const rawAssertions = field(result, "assertions"); + const assertions = Array.isArray(rawAssertions) ? rawAssertions : []; + if (!Array.isArray(rawAssertions)) errors.push("assertions must be an array"); + + const outcomes = new Map(); + let duplicateAssertionId = false; + for (const [index, rawAssertion] of assertions.entries()) { + const assertion = record(rawAssertion); + if (!assertion) { + errors.push(`assertion ${index} must be an object`); + continue; + } + const id = textField(assertion, "id"); + if (!id || id.trim().length === 0) { + errors.push(`assertion ${index} id must be a non-empty string`); + continue; + } + const passed = field(assertion, "passed"); + if (typeof passed !== "boolean") { + errors.push(`assertion ${id} passed must be a boolean`); + } + const detail = field(assertion, "detail"); + if (detail !== undefined && typeof detail !== "string") { + errors.push(`assertion ${id} detail must be a string when present`); + } + if (outcomes.has(id)) duplicateAssertionId = true; + else outcomes.set(id, { passed: passed === true }); + } + if (duplicateAssertionId) errors.push("duplicate assertion ids"); + + for (const assertionId of cell.assertions) { + const outcome = outcomes.get(assertionId); + if (!outcome) errors.push(`missing assertion ${assertionId}`); + else if (!outcome.passed) errors.push(`failed assertion ${assertionId}`); + } + const unknownAssertions = [...outcomes.keys()] + .filter((id) => !cell.assertions.includes(id)); + if (unknownAssertions.length > 0) { + errors.push(`unknown assertions: ${redactText(unknownAssertions.join(", "))}`); + } + if (status !== "passed") { + errors.push( + typeof error === "string" && error.length > 0 + ? redactText(error) + : typeof failureClass === "string" && ALLOWED_FAILURE_CLASSES.has(failureClass as FailureClass) + ? failureClass + : "acceptance cell failed", + ); + } + return errors; +} + +function normalizedResult( + value: unknown, + cell: RunnerAcceptanceCell, +): RunnerAcceptanceResult { + const result = record(value) ?? {}; + const attempt = numericField(result, "attempt"); + const durationMs = numericField(result, "durationMs"); + const status = field(result, "status"); + const failureClass = field(result, "failureClass"); + const error = textField(result, "error"); + const startedAt = textField(result, "startedAt"); + const finishedAt = textField(result, "finishedAt"); + const rawAssertions = field(result, "assertions"); + const assertions = Array.isArray(rawAssertions) ? rawAssertions : []; + return { + schema: RESULT_SCHEMA, + cellId: cell.id, + attempt: attempt !== null && Number.isSafeInteger(attempt) && attempt >= 1 + ? attempt + : 0, + status: status === "passed" ? "passed" : "failed", + ...(typeof failureClass === "string" && ALLOWED_FAILURE_CLASSES.has(failureClass as FailureClass) + ? { failureClass: failureClass as FailureClass } + : {}), + ...(error ? { error: redactText(error) } : {}), + startedAt: startedAt === null ? "" : redactText(startedAt), + finishedAt: finishedAt === null ? "" : redactText(finishedAt), + durationMs: durationMs !== null && durationMs >= 0 + ? durationMs + : 0, + redaction: field(result, "redaction") === "passed" ? "passed" : "failed", + assertions: cell.assertions.flatMap((id) => { + const assertion = assertions + .map(record) + .find((candidate) => candidate && textField(candidate, "id") === id); + return assertion + ? [{ + id, + passed: field(assertion, "passed") === true, + ...(typeof field(assertion, "detail") === "string" + ? { detail: redactText(field(assertion, "detail") as string) } + : {}), + }] + : []; + }), + }; +} + +function missingResult(cell: RunnerAcceptanceCell, generatedAt: string): AggregatedAcceptanceResult { + return { + schema: "paperclip.runner-acceptance.result/v1", + cellId: cell.id, + attempt: 0, + status: "failed", + failureClass: "permanent_infrastructure", + error: "No result was supplied", + startedAt: generatedAt, + finishedAt: generatedAt, + durationMs: 0, + redaction: "passed", + assertions: [], + valid: false, + validationErrors: ["No result was supplied"], + }; +} + +export function buildRunnerAcceptanceReport(input: { + results: readonly unknown[]; + cells?: readonly RunnerAcceptanceCell[]; + generatedAt?: string; +}): RunnerAcceptanceReport { + const cells = input.cells ?? runnerAcceptanceMatrix; + const generatedAt = input.generatedAt ?? new Date().toISOString(); + const candidates = new Map>(); + for (const [index, value] of input.results.entries()) { + const result = record(value); + const cellId = result ? textField(result, "cellId") : null; + const candidateCellId = cellId ?? `malformed-result-${index + 1}`; + candidates.set(candidateCellId, [ + ...(candidates.get(candidateCellId) ?? []), + { index, value }, + ]); + } + + const selected = cells.map((cell) => { + const attempts = (candidates.get(cell.id) ?? []).sort((left, right) => { + const leftRecord = record(left.value) ?? {}; + const rightRecord = record(right.value) ?? {}; + const leftAttempt = numericField(leftRecord, "attempt") ?? 0; + const rightAttempt = numericField(rightRecord, "attempt") ?? 0; + const leftFinishedAt = textField(leftRecord, "finishedAt"); + const rightFinishedAt = textField(rightRecord, "finishedAt"); + const leftFinishedTime = leftFinishedAt === null ? 0 : Date.parse(leftFinishedAt) || 0; + const rightFinishedTime = rightFinishedAt === null ? 0 : Date.parse(rightFinishedAt) || 0; + return rightAttempt - leftAttempt + || rightFinishedTime - leftFinishedTime + || left.index - right.index; + }); + const candidate = attempts[0]; + if (!candidate) return missingResult(cell, generatedAt); + const errors = validationErrors(candidate.value, cell); + return { + ...normalizedResult(candidate.value, cell), + valid: errors.length === 0, + validationErrors: errors, + } satisfies AggregatedAcceptanceResult; + }); + + const knownCellIds = new Set(cells.map(({ id }) => id)); + const unknownCellIds = [...candidates.keys()].filter((id) => !knownCellIds.has(id)); + if (unknownCellIds.length > 0) { + throw new Error(`Results reference unknown acceptance cells: ${unknownCellIds.join(", ")}`); + } + + return { + schema: "paperclip.runner-acceptance.report/v1", + generatedAt, + suiteDefinitionHash: cells[0]?.suiteDefinitionHash ?? "empty", + selected: selected.length, + passed: selected.filter(({ valid }) => valid).length, + failed: selected.filter(({ valid }) => !valid).length, + retries: selected.reduce((sum, result) => sum + Math.max(0, result.attempt - 1), 0), + results: selected, + }; +} + +function escapeMarkdown(value: string) { + return value.replaceAll("|", "\\|").replaceAll("\n", " "); +} + +export function renderRunnerAcceptanceMarkdown(report: RunnerAcceptanceReport) { + return [ + "# Runner acceptance", + "", + `Passed: ${report.passed}/${report.selected}`, + "", + "| Cell | Attempt | Result | Duration | Detail |", + "|---|---:|---|---:|---|", + ...report.results.map((result) => + `| ${escapeMarkdown(result.cellId)} | ${result.attempt} | ${result.valid ? "pass" : "fail"} | ${Math.round(result.durationMs / 1000)}s | ${escapeMarkdown(result.validationErrors.join("; ") || "ok")} |`), + "", + ].join("\n"); +} + +function xml(value: unknown) { + return String(value ?? "") + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll('"', """) + .replaceAll("'", "'"); +} + +export function renderRunnerAcceptanceJUnit(report: RunnerAcceptanceReport) { + const cases = report.results.map((result) => { + const failure = result.valid + ? "" + : ``; + return `${failure}`; + }).join(""); + return `${cases}\n`; +} diff --git a/tests/runner-acceptance/support.test.ts b/tests/runner-acceptance/support.test.ts new file mode 100644 index 0000000000..78e793cbbc --- /dev/null +++ b/tests/runner-acceptance/support.test.ts @@ -0,0 +1,62 @@ +import { describe, expect, it } from "vitest"; + +import { classifyFailure, shouldRetryFailure } from "./failure-classifier.js"; +import { + findSensitiveJsonValue, + findSensitiveValue, + normalizedSensitiveValues, + redactText, + sanitizeJson, +} from "./redaction.js"; + +describe("Runner acceptance failure classification", () => { + it.each([ + ["assertion runtime mode expected legacy observed native", "candidate_failure"], + ["provider connection timed out", "transient_infrastructure"], + ["provider unsupported by this runner", "permanent_infrastructure"], + ["secret redaction leak in result", "secret_leak"], + ["teardown left a process running", "cleanup_failure"], + ] as const)("classifies %s", (message, expected) => { + expect(classifyFailure(new Error(message))).toBe(expected); + }); + + it("retries only transient infrastructure failures", () => { + expect(shouldRetryFailure("transient_infrastructure")).toBe(true); + expect(shouldRetryFailure("candidate_failure")).toBe(false); + expect(shouldRetryFailure("secret_leak")).toBe(false); + }); +}); + +describe("Runner acceptance redaction", () => { + const fakeSensitiveValue = ["fixture", "sensitive", "value"].join("-"); + const secretShapedValue = ["sk", "proj", "fixturevalue1234567890"].join("-"); + + it("deduplicates and orders known sensitive values longest first", () => { + expect(normalizedSensitiveValues([" short ", undefined, "long-value", "short"])) + .toEqual(["long-value", "short"]); + }); + + it("redacts exact values, secret shapes, and structured sensitive fields", () => { + expect(redactText( + `value=${fakeSensitiveValue} shaped=${secretShapedValue}`, + [fakeSensitiveValue], + )).toBe("value=[REDACTED] shaped=[REDACTED]"); + expect(sanitizeJson({ + nested: { accessToken: fakeSensitiveValue }, + detail: `received ${secretShapedValue}`, + }, [fakeSensitiveValue])).toEqual({ + nested: { accessToken: "[REDACTED]" }, + detail: "received [REDACTED]", + }); + }); + + it("detects exact, shaped, and sensitive-key leaks without scanning files", () => { + expect(findSensitiveValue(fakeSensitiveValue, [fakeSensitiveValue])) + .toBe("exact sensitive value"); + expect(findSensitiveValue(secretShapedValue)).toBe("secret-shaped value"); + expect(findSensitiveJsonValue({ password: "fixture-password" })) + .toBe("sensitive field password"); + expect(findSensitiveJsonValue({ schema: "paperclip.runner-acceptance.result/v1" })) + .toBeNull(); + }); +}); diff --git a/tests/runner-acceptance/tsconfig.json b/tests/runner-acceptance/tsconfig.json new file mode 100644 index 0000000000..dceb2c2d5f --- /dev/null +++ b/tests/runner-acceptance/tsconfig.json @@ -0,0 +1,13 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "noEmit": true, + "rootDir": "../..", + "types": ["node"], + "typeRoots": [ + "../../cli/node_modules/@types", + "../../server/node_modules/@types" + ] + }, + "include": ["./**/*.ts"] +} diff --git a/tests/runner-acceptance/types.ts b/tests/runner-acceptance/types.ts new file mode 100644 index 0000000000..1a7997d3c7 --- /dev/null +++ b/tests/runner-acceptance/types.ts @@ -0,0 +1,85 @@ +export type JsonValue = + | null + | boolean + | number + | string + | JsonValue[] + | { [key: string]: JsonValue }; + +export type RunnerGeneration = "direct" | "native"; +export type ExpectedRuntimeMode = "legacy" | "native"; +export type NativeProvider = "codex" | "opencode" | "acpx"; +export type QualifiedAcpxAgent = "claude" | "codex"; + +export interface RunnerAcceptanceProfile { + id: string; + label: string; + generation: RunnerGeneration; + adapterScope: "built_in" | "external_plugin_contract"; + adapterType: string; + expectedRuntimeMode: ExpectedRuntimeMode; + provider: NativeProvider | null; + model: string | null; + adapterConfig: Readonly>; + invariants: readonly string[]; +} + +export interface RunnerAcceptanceCase { + id: string; + label: string; + revision: number; + appliesTo: "all" | RunnerGeneration; + assertions: readonly string[]; +} + +export interface RunnerAcceptanceCell { + id: string; + suiteId: "runner-compatibility"; + suiteDefinitionHash: string; + profile: RunnerAcceptanceProfile; + acceptanceCase: RunnerAcceptanceCase; + assertions: readonly string[]; +} + +export type FailureClass = + | "candidate_failure" + | "transient_infrastructure" + | "permanent_infrastructure" + | "secret_leak" + | "cleanup_failure"; + +export interface AcceptanceAssertionResult { + id: string; + passed: boolean; + detail?: string; +} + +export interface RunnerAcceptanceResult { + schema: "paperclip.runner-acceptance.result/v1"; + cellId: string; + attempt: number; + status: "passed" | "failed"; + failureClass?: FailureClass; + error?: string; + startedAt: string; + finishedAt: string; + durationMs: number; + redaction: "passed" | "failed"; + assertions: readonly AcceptanceAssertionResult[]; +} + +export interface AggregatedAcceptanceResult extends RunnerAcceptanceResult { + valid: boolean; + validationErrors: readonly string[]; +} + +export interface RunnerAcceptanceReport { + schema: "paperclip.runner-acceptance.report/v1"; + generatedAt: string; + suiteDefinitionHash: string; + selected: number; + passed: number; + failed: number; + retries: number; + results: readonly AggregatedAcceptanceResult[]; +} diff --git a/tests/runner-acceptance/vitest.config.ts b/tests/runner-acceptance/vitest.config.ts new file mode 100644 index 0000000000..43dd349591 --- /dev/null +++ b/tests/runner-acceptance/vitest.config.ts @@ -0,0 +1,10 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + root: import.meta.dirname, + test: { + name: "runner-acceptance", + include: ["**/*.test.ts"], + environment: "node", + }, +});