diff --git a/scripts/resolvers/testing.ts b/scripts/resolvers/testing.ts index 592382bdd..0f6f896e3 100644 --- a/scripts/resolvers/testing.ts +++ b/scripts/resolvers/testing.ts @@ -3,41 +3,70 @@ import type { TemplateContext } from './types'; export function generateTestBootstrap(_ctx: TemplateContext): string { return `## Test Framework Bootstrap -**Detect existing test framework and project runtime:** +**Read the project's CLAUDE.md (and TESTING.md if present) FIRST.** If it documents a test command, the project already told you: no detection, no bootstrap. Skip the rest of bootstrap and use that command in Step 5. + +**Otherwise gather markers. Every marker below is EVIDENCE for the question you ask — never a command to run blind.** A marker tells you which ecosystem you're in and which command to OFFER. It does not tell you the command works. Do not execute a candidate test command to "check" it: a probe on a project that never had that runner fails loudly and teaches you nothing, and installing a second framework over a working one is worse. \`\`\`bash setopt +o nomatch 2>/dev/null || true # zsh compat -# Detect project runtime -[ -f Gemfile ] && echo "RUNTIME:ruby" +# Definitive ecosystem markers (presence = ecosystem, NOT a command to run) +[ -f manage.py ] && echo "RUNTIME:python FRAMEWORK:django MARKER:manage.py" +{ [ -f pyproject.toml ] || [ -f pytest.ini ] || [ -f tox.ini ] || [ -f setup.cfg ] || [ -f requirements.txt ]; } && echo "RUNTIME:python" +[ -f Gemfile ] || [ -f Rakefile ] || [ -f .rspec ] && echo "RUNTIME:ruby" [ -f package.json ] && echo "RUNTIME:node" -[ -f requirements.txt ] || [ -f pyproject.toml ] && echo "RUNTIME:python" [ -f go.mod ] && echo "RUNTIME:go" [ -f Cargo.toml ] && echo "RUNTIME:rust" [ -f composer.json ] && echo "RUNTIME:php" [ -f mix.exs ] && echo "RUNTIME:elixir" +[ -f pom.xml ] && echo "RUNTIME:jvm BUILD:maven" +{ [ -f build.gradle ] || [ -f build.gradle.kts ]; } && echo "RUNTIME:jvm BUILD:gradle" # Detect sub-frameworks [ -f Gemfile ] && grep -q "rails" Gemfile 2>/dev/null && echo "FRAMEWORK:rails" [ -f package.json ] && grep -q '"next"' package.json 2>/dev/null && echo "FRAMEWORK:nextjs" -# Check for existing test infrastructure -ls jest.config.* vitest.config.* playwright.config.* .rspec pytest.ini pyproject.toml phpunit.xml 2>/dev/null -ls -d test/ tests/ spec/ __tests__/ cypress/ e2e/ 2>/dev/null +# Existing test path — config files, declared scripts, AND test FILES. +# A project with real tests and no config file is the common miss. +ls jest.config.* vitest.config.* playwright.config.* .rspec pytest.ini tox.ini phpunit.xml* 2>/dev/null +[ -f package.json ] && grep -q '"test"[[:space:]]*:' package.json && echo "SCRIPT:package.json test" +[ -f Makefile ] && grep -qE '^(test|check):' Makefile && echo "TARGET:make test" +[ -f pyproject.toml ] && grep -q "pytest" pyproject.toml && echo "CONFIG:pyproject pytest" +git ls-files | grep -cE '(^|/)(tests?|spec|__tests__)/|(^|/)tests?\\.py$|(^|/)test_[^/]+\\.py$|_test\\.(go|py|rb|ts|js|exs)$|\\.(test|spec)\\.[jt]sx?$|_spec\\.rb$|Test\\.(java|kt)$' | sed 's/^/TESTFILES:/' +# Rust keeps unit tests inside src/, so file names alone miss them +[ -f Cargo.toml ] && git grep -lF '#[test]' -- 'src' >/dev/null 2>&1 && echo "TESTS:rust in-source" # Check opt-out marker [ -f .gstack/no-test-bootstrap ] && echo "BOOTSTRAP_DECLINED" \`\`\` -**If test framework detected** (config files or test directories found): -Print "Test framework detected: {name} ({N} existing tests). Skipping bootstrap." +Map the markers to the command you will OFFER — never to one you run on a guess: + +| Marker | Ecosystem | Candidate command to offer | +|--------|-----------|----------------------------| +| \`manage.py\` | Django | \`python manage.py test\` (or \`pytest\` when pytest-django is in the deps) | +| \`pytest.ini\` / \`tox.ini\` / pytest in \`pyproject.toml\` / \`test_*.py\` | Python | \`pytest\` | +| \`go.mod\` (+ any \`*_test.go\`) | Go | \`go test ./...\` | +| \`Cargo.toml\` | Rust | \`cargo test\` | +| \`pom.xml\` | JVM (Maven) | \`mvn test\` | +| \`build.gradle\` / \`build.gradle.kts\` | JVM (Gradle) | \`./gradlew test\` | +| \`Gemfile\` / \`Rakefile\` / \`.rspec\` | Ruby | \`bundle exec rspec\`, \`bin/rails test\`, or \`rake test\` | +| \`mix.exs\` | Elixir | \`mix test\` | +| \`composer.json\` | PHP | \`composer test\` or \`./vendor/bin/phpunit\` | +| \`package.json\` with a \`test\` script | Node | that script, run with the package manager the lockfile names | +| \`Makefile\` with a \`test:\` target | any | \`make test\` | + +**If ANY existing-test evidence appears** (a config file, a declared test script or make target, a nonzero \`TESTFILES:\` count, or \`TESTS:rust in-source\`): the project has tests. **Do NOT bootstrap.** Print "Existing tests detected: {the evidence}." Then get the command the same way Step 5 does — CLAUDE.md/TESTING.md if documented, otherwise AskUserQuestion offering the candidates from the table above plus "Other", and persist the answer to CLAUDE.md's \`## Testing\` section so it is never asked again. When the ecosystem ships a runner (Django, Go, Rust, Elixir, Maven/Gradle), that runner is the candidate — never install a second framework beside a working one. Read 2-3 existing test files to learn conventions (naming, imports, assertion style, setup patterns). Store conventions as prose context for use in Phase 8e.5 or Step 7. **Skip the rest of bootstrap.** +Absent config files and absent \`tests/\` directories are NOT evidence of "no tests": Django keeps tests in \`/tests.py\`, Go in \`*_test.go\` beside the source, Rust in \`#[test]\` blocks inside \`src/\`. A green \`python manage.py test\` with no \`pytest.ini\` is a tested project, not a bootstrap candidate. + **If BOOTSTRAP_DECLINED** appears: Print "Test bootstrap previously declined — skipping." **Skip the rest of bootstrap.** -**If NO runtime detected** (no config files found): Use AskUserQuestion: +**If NO ecosystem marker matched:** Use AskUserQuestion: "I couldn't detect your project's language. What runtime are you using?" Options: A) Node.js/TypeScript B) Ruby/Rails C) Python D) Go E) Rust F) PHP G) Elixir H) This project doesn't need tests. +If the runtime you need isn't listed, offer "Other" and take the runtime plus the test command as free text. If user picks H → write \`.gstack/no-test-bootstrap\` and continue without tests. -**If runtime detected but no test framework — bootstrap:** +**If an ecosystem matched but there is no existing-test evidence at all — bootstrap:** ### B2. Research best practices @@ -53,7 +82,9 @@ If WebSearch is unavailable, use this built-in knowledge table: | Node.js | vitest + @testing-library | jest + @testing-library | | Next.js | vitest + @testing-library/react + playwright | jest + cypress | | Python | pytest + pytest-cov | unittest | +| Django | pytest + pytest-django | Django's built-in \`manage.py test\` (unittest) | | Go | stdlib testing + testify | stdlib only | +| JVM (Maven/Gradle) | JUnit 5 + AssertJ | JUnit 5 only | | Rust | cargo test (built-in) + mockall | — | | PHP | phpunit + mockery | pest | | Elixir | ExUnit (built-in) + ex_machina | — | @@ -202,15 +233,20 @@ Before analyzing coverage, detect the project's test framework: \`\`\`bash setopt +o nomatch 2>/dev/null || true # zsh compat -# Detect project runtime -[ -f Gemfile ] && echo "RUNTIME:ruby" +# Detect project runtime (markers are evidence, not commands to run blind) +[ -f manage.py ] && echo "RUNTIME:python FRAMEWORK:django" +{ [ -f pyproject.toml ] || [ -f pytest.ini ] || [ -f tox.ini ] || [ -f setup.cfg ] || [ -f requirements.txt ]; } && echo "RUNTIME:python" +[ -f Gemfile ] || [ -f Rakefile ] || [ -f .rspec ] && echo "RUNTIME:ruby" [ -f package.json ] && echo "RUNTIME:node" -[ -f requirements.txt ] || [ -f pyproject.toml ] && echo "RUNTIME:python" [ -f go.mod ] && echo "RUNTIME:go" [ -f Cargo.toml ] && echo "RUNTIME:rust" -# Check for existing test infrastructure -ls jest.config.* vitest.config.* playwright.config.* cypress.config.* .rspec pytest.ini phpunit.xml 2>/dev/null -ls -d test/ tests/ spec/ __tests__/ cypress/ e2e/ 2>/dev/null +[ -f pom.xml ] && echo "RUNTIME:jvm BUILD:maven" +{ [ -f build.gradle ] || [ -f build.gradle.kts ]; } && echo "RUNTIME:jvm BUILD:gradle" +# Check for existing test infrastructure — config files, scripts, AND test files +ls jest.config.* vitest.config.* playwright.config.* cypress.config.* .rspec pytest.ini tox.ini phpunit.xml 2>/dev/null +[ -f package.json ] && grep -q '"test"[[:space:]]*:' package.json && echo "SCRIPT:package.json test" +[ -f Makefile ] && grep -qE '^(test|check):' Makefile && echo "TARGET:make test" +git ls-files | grep -cE '(^|/)(tests?|spec|__tests__)/|(^|/)tests?\\.py$|(^|/)test_[^/]+\\.py$|_test\\.(go|py|rb|ts|js|exs)$|\\.(test|spec)\\.[jt]sx?$|_spec\\.rb$|Test\\.(java|kt)$' | sed 's/^/TESTFILES:/' \`\`\` 3. **If no framework detected:**${mode === 'ship' ? ' falls through to the Test Framework Bootstrap step (Step 4) which handles full setup.' : ' still produce the coverage diagram, but skip test generation.'}`); diff --git a/test/ship-test-detection-markers.test.ts b/test/ship-test-detection-markers.test.ts new file mode 100644 index 000000000..fb0c80f88 --- /dev/null +++ b/test/ship-test-detection-markers.test.ts @@ -0,0 +1,113 @@ +/** + * Regression: /ship Step 4 test-framework detection was blind to Django. + * + * A real Django project (manage.py + /tests.py, green `python manage.py + * test`, no pytest.ini and no tests/ directory) read as "no test framework", + * so /ship bootstrapped pytest on top of a working suite. Same blindness hit + * any config-less-but-tested project (Go *_test.go, in-source Rust #[test], + * package.json with only a test script). + * + * These run the resolver's OWN emitted detection block against fixtures, so + * the shell is checked, not just the prose — and the test is independent of + * generated-SKILL.md regen state. + * + * Ported from time-attack/gstack commit e3259078 (GStack 2), adapted from the + * fork's generated-markdown target to our resolver source of truth. + */ +import { describe, test, expect } from 'bun:test'; +import { execFileSync } from 'node:child_process'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; + +import { generateTestBootstrap } from '../scripts/resolvers/testing'; + +const section = generateTestBootstrap({} as never); + +/** The detection block the skill tells the agent to run (first bash fence). */ +function detectionScript(): string { + const open = section.indexOf('```bash\n'); + expect(open).toBeGreaterThan(-1); + const start = open + '```bash\n'.length; + const end = section.indexOf('```', start); + return section.slice(start, end); +} + +/** Run the detection block in a throwaway git repo laid out by `files`. */ +function detect(files: Record): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ship-detect-')); + try { + for (const [rel, body] of Object.entries(files)) { + const abs = path.join(dir, rel); + fs.mkdirSync(path.dirname(abs), { recursive: true }); + fs.writeFileSync(abs, body); + } + const script = path.join(dir, '.detect.sh'); + fs.writeFileSync(script, detectionScript()); + const git = (...args: string[]) => execFileSync('git', args, { cwd: dir }); + git('init', '-q', '.'); + git('add', '-A'); + git('-c', 'user.email=t@t', '-c', 'user.name=t', 'commit', '-qm', 'fixture'); + // The block's last line is a `[ -f marker ] && echo`, so a clean project + // exits 1 by design — read stdout, don't trust the status. + return execFileSync('bash', [script], { cwd: dir, encoding: 'utf-8' }); + } catch (err: unknown) { + const e = err as { stdout?: string }; + if (typeof e.stdout === 'string') return e.stdout; + throw err; + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +} + +const testFileCount = (out: string): number => + Number(/^TESTFILES:(\d+)$/m.exec(out)?.[1] ?? -1); + +describe('/ship Step 4 detection is multi-ecosystem', () => { + test('Django project reports manage.py and its existing tests', () => { + const out = detect({ + 'manage.py': '#!/usr/bin/env python\n', + 'requirements.txt': 'Django==5.0\n', + 'polls/tests.py': 'from django.test import TestCase\n', + }); + expect(out).toContain('MARKER:manage.py'); + expect(out).toContain('FRAMEWORK:django'); + expect(testFileCount(out)).toBeGreaterThan(0); + }); + + test('config-less projects with tests still report test evidence', () => { + expect(testFileCount(detect({ + 'go.mod': 'module x\n', + 'x_test.go': 'package x\n', + }))).toBeGreaterThan(0); + + const node = detect({ + 'package.json': '{"name":"n","scripts":{"test":"node --test"}}\n', + }); + expect(node).toContain('SCRIPT:package.json test'); + + const rust = detect({ + 'Cargo.toml': '[package]\nname="x"\n', + 'src/lib.rs': '#[cfg(test)]\nmod t{ #[test] fn x(){} }\n', + }); + expect(rust).toContain('TESTS:rust in-source'); + }); + + test('a genuinely untested project reports no test evidence', () => { + const out = detect({ 'go.mod': 'module x\n', 'x.go': 'package x\n' }); + expect(out).toContain('RUNTIME:go'); + expect(testFileCount(out)).toBe(0); + expect(out).not.toContain('TESTS:'); + }); + + test('every ecosystem marker maps to a command to OFFER, not to run blind', () => { + expect(section).toContain('never a command to run blind'); + for (const marker of ['manage.py', 'go.mod', 'Cargo.toml', 'pom.xml', + 'build.gradle', 'mix.exs', 'composer.json', 'Gemfile', 'pytest.ini']) { + expect(section).toContain(marker); + } + // The ask-and-persist contract, not a hardcoded project command. + expect(section).toContain('AskUserQuestion'); + expect(section).toContain('persist the answer to CLAUDE.md'); + }); +});