diff --git a/.github/workflows/evals.yml b/.github/workflows/evals.yml index 2611678dc..f2c5cb232 100644 --- a/.github/workflows/evals.yml +++ b/.github/workflows/evals.yml @@ -216,24 +216,6 @@ jobs: || { echo "ERROR: plan-ceo-review SKILL.md missing 'name: plan-ceo-review' frontmatter" >&2; exit 1; } echo "skill registry OK" - # TEMP DEBUG (remove after diagnosing pty-smoke discovery): the smoke fails - # with "Unknown command: /office-hours" even after retries (not a race). - # Capture what claude actually discovers in CI to fix the registry precisely. - - name: Debug skill discovery (pty smoke) - if: matrix.suite.name == 'e2e-pty-plan-smoke' - env: - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - run: | - set +e - echo "=== claude version + binary ==="; command -v claude; claude --version 2>&1 | head -3 - echo "=== HOME/CLAUDE/XDG env ==="; env | grep -iE '^(HOME|CLAUDE|XDG)=' || true - echo "=== ~/.claude/skills (top) ==="; ls -la "$HOME/.claude/skills" 2>&1 | head -30 - echo "=== office-hours SKILL.md resolves? ==="; head -3 "$HOME/.claude/skills/office-hours/SKILL.md" 2>&1 - echo "=== project .claude in cwd? ==="; ls -la "$GITHUB_WORKSPACE/.claude" 2>&1 | head - echo "=== claude -p probe (same discovery path) ===" - cd "$GITHUB_WORKSPACE" - timeout 90 claude -p "Reply with exactly the word READY and nothing else if the /office-hours slash command is available to you, otherwise reply MISSING." 2>&1 | tail -10 - - name: Run ${{ matrix.suite.name }} env: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} diff --git a/test/helpers/claude-pty-runner.ts b/test/helpers/claude-pty-runner.ts index d4e4ffef0..2f507c66c 100644 --- a/test/helpers/claude-pty-runner.ts +++ b/test/helpers/claude-pty-runner.ts @@ -1605,7 +1605,7 @@ export async function runPlanSkillObservation(opts: { // command. await Bun.sleep(3000); } - let since = session.mark(); + const since = session.mark(); session.send(`/${opts.skillName}\r`); const budgetMs = opts.timeoutMs ?? 180_000; @@ -1619,7 +1619,6 @@ export async function runPlanSkillObservation(opts: { // even if the current state is 'working'. let proseAUQEverObserved = false; let waitingEverObserved = false; - let unknownCmdRetries = 0; const JUDGE_AFTER_MS = 60_000; const JUDGE_INTERVAL_MS = 30_000; while (Date.now() - start < budgetMs) { @@ -1635,24 +1634,9 @@ export async function runPlanSkillObservation(opts: { }; } if (visible.includes('Unknown command:')) { - // Skill-discovery race: in a cold CI container the overlay-FS scan of - // the symlinked ~/.claude/skills registry can finish AFTER the boot - // grace, so the first `/skill` send reaches claude before the skill is - // indexed and is rejected as unknown. (Reproduces only in CI — passes - // locally where claude is warm and discovery is instant.) Re-send the - // command a few times, re-marking first so the stale "Unknown command:" - // line in scrollback can't immediately re-trip this check, before - // concluding the skill is genuinely unregistered. - if (unknownCmdRetries < 3) { - unknownCmdRetries++; - await Bun.sleep(6000); - since = session.mark(); - session.send(`/${opts.skillName}\r`); - continue; - } return { outcome: 'exited', - summary: `claude rejected /${opts.skillName} as unknown command after ${unknownCmdRetries} retries (skill not registered in this cwd)`, + summary: `claude rejected /${opts.skillName} as unknown command (skill not registered in this cwd)`, evidence: visible.slice(-2000), elapsedMs: Date.now() - startedAt, };