From 4af55ba6bdc3fb76c35cd73d169e9f2f200ed6e7 Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Mon, 17 Aug 2026 16:31:17 -0700 Subject: [PATCH] test(release-smoke): follow the onboarding wizard's chat-first rewrite (#11565) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - The release channel system publishes a nightly build only after the release smoke suite passes against the newest canary > - The scheduled nightly run has failed every night since August 12, so no nightly, and therefore no beta candidate, has shipped for six days > - The failures are stale test locators, not a product regression: the chat-first onboarding rewrite (#11101) changed wizard copy and the post-launch destination > - This pull request updates the smoke spec to match the current wizard > - The benefit is a green nightly lane and an unblocked beta promotion ## Linked Issues or Issue Description **What happened?** The scheduled `Release` nightly run fails in `smoke_nightly / smoke` every night since 2026-08-12. The failing spec is `tests/release-smoke/docker-auth-onboarding.spec.ts`. Four assertions no longer match the product after the chat-first onboarding rewrite (#11101): - The step-1 heading is now "Name your organization", not "Name your company". - The step-4 hire button is now "Connect", not "Give it a heartbeat". - The seeded first task is now titled "Paperclip onboarding". - A successful launch navigates to the seeded task's thread (`/issues/`), not `/dashboard`. **Expected behavior** The smoke suite passes against a canary that contains the current onboarding wizard, and the nightly lane publishes again. **Steps to reproduce** Run `.github/workflows/release-smoke.yml` against `paperclipai@canary` (any version at or after the rewrite), or dispatch `release.yml` with `channel: nightly`. Example red runs: 32014452506 (Aug 17), 31938284401 (Aug 16). **Paperclip version or commit** `2026.817.0-canary.12` Related (not duplicates): #11190 updated this same spec for the mission-first wizard; this PR is the follow-up for the chat-first rewrite that landed after it. ## What Changed - Update the step-1 wizard heading locator to "Name your organization". - Update the step-4 hire button locator to "Connect" and reword the step comment. - Update `FIRST_TASK_TITLE` to "Paperclip onboarding" (the wizard's current `DEFAULT_TASK_TITLE`). - Assert the post-launch URL is the seeded task's thread (`/issues/`), not `/dashboard`. ## Verification - Local run of the exact CI harness: `scripts/docker-onboard-smoke.sh` with `PAPERCLIPAI_VERSION=2026.817.0-canary.12`, then `pnpm run test:release-smoke` against the container — 1 passed. - The suite's later API assertions (company, CEO agent, mission goal, seeded issue assignment, assignment-sourced heartbeat run) all pass unchanged against the current canary. ## Risks - Low risk: test-only change; no product code is touched. - The spec remains copy-coupled to the wizard. If wizard copy churn continues, a follow-up could add stable `data-testid` hooks to the wizard so the smoke spec stops breaking on wording changes. ## Model Used Claude Fable 5 (Claude Code) ## Pre-submission checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template --- .../docker-auth-onboarding.spec.ts | 36 +++++++++++-------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/tests/release-smoke/docker-auth-onboarding.spec.ts b/tests/release-smoke/docker-auth-onboarding.spec.ts index f094e9e537..dd02b0e40f 100644 --- a/tests/release-smoke/docker-auth-onboarding.spec.ts +++ b/tests/release-smoke/docker-auth-onboarding.spec.ts @@ -14,7 +14,7 @@ const MISSION = "Ship a reliable release smoke suite for Paperclip."; const AGENT_NAME = "CEO"; // Seeded by the wizard's launch step (DEFAULT_TASK_TITLE in // ui/src/components/OnboardingWizard.tsx). -const FIRST_TASK_TITLE = "Hire your first engineer and create a hiring plan"; +const FIRST_TASK_TITLE = "Paperclip onboarding"; async function signIn(page: Page) { await page.goto("/"); @@ -28,7 +28,7 @@ async function signIn(page: Page) { } async function openOnboarding(page: Page) { - const wizardHeading = page.locator("h3", { hasText: "Name your company" }); + const wizardHeading = page.locator("h3", { hasText: "Name your organization" }); const startButton = page.getByRole("button", { name: "Start Onboarding" }); await expect(wizardHeading.or(startButton)).toBeVisible({ timeout: 20_000 }); @@ -67,24 +67,24 @@ test.describe("Docker authenticated onboarding smoke", () => { await leadNameInput.fill(AGENT_NAME); await page.getByRole("button", { name: "Next" }).click(); - // Step 4: keep the default adapter and hire the lead. The adapter - // environment test runs inside the smoke container, where no agent CLIs - // are installed; an unhealthy report is expected and must not block the - // hire. Allow generous time for the env probe + hire + auto-approval. - const heartbeatButton = page.getByRole("button", { - name: "Give it a heartbeat", - }); - await expect(heartbeatButton).toBeVisible({ timeout: 10_000 }); - await expect(heartbeatButton).toBeEnabled({ timeout: 30_000 }); - await heartbeatButton.click(); + // Step 4: keep the default adapter and connect (hire) the lead. The + // adapter environment check runs inside the smoke container, where no + // agent CLIs are installed; an unhealthy report is expected and must not + // block the hire. Allow generous time for the env probe + hire + + // auto-approval. + const connectButton = page.getByRole("button", { name: "Connect" }); + await expect(connectButton).toBeVisible({ timeout: 10_000 }); + await expect(connectButton).toBeEnabled({ timeout: 30_000 }); + await connectButton.click(); // Step 5: review, then launch. "Get started" provisions the onboarding - // goal/project and navigates to the dashboard only on success. + // goal/project/first task and, only on success, drops the user into the + // seeded first task's thread (not the dashboard). const getStartedButton = page.getByRole("button", { name: "Get started" }); await expect(getStartedButton).toBeVisible({ timeout: 60_000 }); await expect(getStartedButton).toBeEnabled({ timeout: 10_000 }); await getStartedButton.click(); - await expect(page).toHaveURL(/\/dashboard/, { timeout: 30_000 }); + await expect(page).toHaveURL(/\/issues\//, { timeout: 30_000 }); const baseUrl = new URL(page.url()).origin; @@ -135,6 +135,7 @@ test.describe("Docker authenticated onboarding smoke", () => { expect(issuesRes.ok()).toBe(true); const issues = (await issuesRes.json()) as Array<{ id: string; + identifier: string | null; title: string; assigneeAgentId: string | null; }>; @@ -142,6 +143,13 @@ test.describe("Docker authenticated onboarding smoke", () => { expect(seededIssue).toBeTruthy(); expect(seededIssue!.assigneeAgentId).toBe(ceoAgent!.id); + // The launch must have landed on the seeded task itself, not merely on + // some issue route. + const seededRef = seededIssue!.identifier ?? seededIssue!.id; + expect(new URL(page.url()).pathname.endsWith(`/issues/${seededRef}`)).toBe( + true + ); + await expect.poll( async () => { const runsRes = await page.request.get(