mirror of https://github.com/garrytan/gstack.git
fix(ship): test-command detection was blind to Django and config-less-but-tested projects
The Test Framework Bootstrap detected Python only via requirements.txt or pyproject.toml and treated missing config files as no-tests, so a green 'python manage.py test' Django app, a Go project with *_test.go beside the source, in-source Rust #[test] blocks, or a package.json with only a test script all got offered a SECOND test framework over a working one. Detection now enumerates definitive per-ecosystem markers (manage.py, tox.ini/setup.cfg, pom.xml/gradle, Makefile test targets, a tracked-file test census, in-source Rust tests) as EVIDENCE for the question it asks — never a command to run blind — preserving the read-CLAUDE.md-or-ask contract, with a marker→candidate-command table and ask-once persistence. The shared coverage-audit detection block gains the same markers. Test runs the resolver's emitted detection bash against Django / Go / Rust / Node fixtures in throwaway git repos. Ported from time-attack/gstack commit e3259078 (GStack 2). Co-authored-by: Sina Matian <sina@time-attack.dev> Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
parent
5b7b39e662
commit
da2fed9bc8
|
|
@ -3,41 +3,70 @@ import type { TemplateContext } from './types';
|
|||
export function generateTestBootstrap(_ctx: TemplateContext): string {
|
||||
return `## Test Framework Bootstrap
|
||||
|
||||
**Detect existing test framework and project runtime:**
|
||||
**Read the project's CLAUDE.md (and TESTING.md if present) FIRST.** If it documents a test command, the project already told you: no detection, no bootstrap. Skip the rest of bootstrap and use that command in Step 5.
|
||||
|
||||
**Otherwise gather markers. Every marker below is EVIDENCE for the question you ask — never a command to run blind.** A marker tells you which ecosystem you're in and which command to OFFER. It does not tell you the command works. Do not execute a candidate test command to "check" it: a probe on a project that never had that runner fails loudly and teaches you nothing, and installing a second framework over a working one is worse.
|
||||
|
||||
\`\`\`bash
|
||||
setopt +o nomatch 2>/dev/null || true # zsh compat
|
||||
# Detect project runtime
|
||||
[ -f Gemfile ] && echo "RUNTIME:ruby"
|
||||
# Definitive ecosystem markers (presence = ecosystem, NOT a command to run)
|
||||
[ -f manage.py ] && echo "RUNTIME:python FRAMEWORK:django MARKER:manage.py"
|
||||
{ [ -f pyproject.toml ] || [ -f pytest.ini ] || [ -f tox.ini ] || [ -f setup.cfg ] || [ -f requirements.txt ]; } && echo "RUNTIME:python"
|
||||
[ -f Gemfile ] || [ -f Rakefile ] || [ -f .rspec ] && echo "RUNTIME:ruby"
|
||||
[ -f package.json ] && echo "RUNTIME:node"
|
||||
[ -f requirements.txt ] || [ -f pyproject.toml ] && echo "RUNTIME:python"
|
||||
[ -f go.mod ] && echo "RUNTIME:go"
|
||||
[ -f Cargo.toml ] && echo "RUNTIME:rust"
|
||||
[ -f composer.json ] && echo "RUNTIME:php"
|
||||
[ -f mix.exs ] && echo "RUNTIME:elixir"
|
||||
[ -f pom.xml ] && echo "RUNTIME:jvm BUILD:maven"
|
||||
{ [ -f build.gradle ] || [ -f build.gradle.kts ]; } && echo "RUNTIME:jvm BUILD:gradle"
|
||||
# Detect sub-frameworks
|
||||
[ -f Gemfile ] && grep -q "rails" Gemfile 2>/dev/null && echo "FRAMEWORK:rails"
|
||||
[ -f package.json ] && grep -q '"next"' package.json 2>/dev/null && echo "FRAMEWORK:nextjs"
|
||||
# Check for existing test infrastructure
|
||||
ls jest.config.* vitest.config.* playwright.config.* .rspec pytest.ini pyproject.toml phpunit.xml 2>/dev/null
|
||||
ls -d test/ tests/ spec/ __tests__/ cypress/ e2e/ 2>/dev/null
|
||||
# Existing test path — config files, declared scripts, AND test FILES.
|
||||
# A project with real tests and no config file is the common miss.
|
||||
ls jest.config.* vitest.config.* playwright.config.* .rspec pytest.ini tox.ini phpunit.xml* 2>/dev/null
|
||||
[ -f package.json ] && grep -q '"test"[[:space:]]*:' package.json && echo "SCRIPT:package.json test"
|
||||
[ -f Makefile ] && grep -qE '^(test|check):' Makefile && echo "TARGET:make test"
|
||||
[ -f pyproject.toml ] && grep -q "pytest" pyproject.toml && echo "CONFIG:pyproject pytest"
|
||||
git ls-files | grep -cE '(^|/)(tests?|spec|__tests__)/|(^|/)tests?\\.py$|(^|/)test_[^/]+\\.py$|_test\\.(go|py|rb|ts|js|exs)$|\\.(test|spec)\\.[jt]sx?$|_spec\\.rb$|Test\\.(java|kt)$' | sed 's/^/TESTFILES:/'
|
||||
# Rust keeps unit tests inside src/, so file names alone miss them
|
||||
[ -f Cargo.toml ] && git grep -lF '#[test]' -- 'src' >/dev/null 2>&1 && echo "TESTS:rust in-source"
|
||||
# Check opt-out marker
|
||||
[ -f .gstack/no-test-bootstrap ] && echo "BOOTSTRAP_DECLINED"
|
||||
\`\`\`
|
||||
|
||||
**If test framework detected** (config files or test directories found):
|
||||
Print "Test framework detected: {name} ({N} existing tests). Skipping bootstrap."
|
||||
Map the markers to the command you will OFFER — never to one you run on a guess:
|
||||
|
||||
| Marker | Ecosystem | Candidate command to offer |
|
||||
|--------|-----------|----------------------------|
|
||||
| \`manage.py\` | Django | \`python manage.py test\` (or \`pytest\` when pytest-django is in the deps) |
|
||||
| \`pytest.ini\` / \`tox.ini\` / pytest in \`pyproject.toml\` / \`test_*.py\` | Python | \`pytest\` |
|
||||
| \`go.mod\` (+ any \`*_test.go\`) | Go | \`go test ./...\` |
|
||||
| \`Cargo.toml\` | Rust | \`cargo test\` |
|
||||
| \`pom.xml\` | JVM (Maven) | \`mvn test\` |
|
||||
| \`build.gradle\` / \`build.gradle.kts\` | JVM (Gradle) | \`./gradlew test\` |
|
||||
| \`Gemfile\` / \`Rakefile\` / \`.rspec\` | Ruby | \`bundle exec rspec\`, \`bin/rails test\`, or \`rake test\` |
|
||||
| \`mix.exs\` | Elixir | \`mix test\` |
|
||||
| \`composer.json\` | PHP | \`composer test\` or \`./vendor/bin/phpunit\` |
|
||||
| \`package.json\` with a \`test\` script | Node | that script, run with the package manager the lockfile names |
|
||||
| \`Makefile\` with a \`test:\` target | any | \`make test\` |
|
||||
|
||||
**If ANY existing-test evidence appears** (a config file, a declared test script or make target, a nonzero \`TESTFILES:\` count, or \`TESTS:rust in-source\`): the project has tests. **Do NOT bootstrap.** Print "Existing tests detected: {the evidence}." Then get the command the same way Step 5 does — CLAUDE.md/TESTING.md if documented, otherwise AskUserQuestion offering the candidates from the table above plus "Other", and persist the answer to CLAUDE.md's \`## Testing\` section so it is never asked again. When the ecosystem ships a runner (Django, Go, Rust, Elixir, Maven/Gradle), that runner is the candidate — never install a second framework beside a working one.
|
||||
Read 2-3 existing test files to learn conventions (naming, imports, assertion style, setup patterns).
|
||||
Store conventions as prose context for use in Phase 8e.5 or Step 7. **Skip the rest of bootstrap.**
|
||||
|
||||
Absent config files and absent \`tests/\` directories are NOT evidence of "no tests": Django keeps tests in \`<app>/tests.py\`, Go in \`*_test.go\` beside the source, Rust in \`#[test]\` blocks inside \`src/\`. A green \`python manage.py test\` with no \`pytest.ini\` is a tested project, not a bootstrap candidate.
|
||||
|
||||
**If BOOTSTRAP_DECLINED** appears: Print "Test bootstrap previously declined — skipping." **Skip the rest of bootstrap.**
|
||||
|
||||
**If NO runtime detected** (no config files found): Use AskUserQuestion:
|
||||
**If NO ecosystem marker matched:** Use AskUserQuestion:
|
||||
"I couldn't detect your project's language. What runtime are you using?"
|
||||
Options: A) Node.js/TypeScript B) Ruby/Rails C) Python D) Go E) Rust F) PHP G) Elixir H) This project doesn't need tests.
|
||||
If the runtime you need isn't listed, offer "Other" and take the runtime plus the test command as free text.
|
||||
If user picks H → write \`.gstack/no-test-bootstrap\` and continue without tests.
|
||||
|
||||
**If runtime detected but no test framework — bootstrap:**
|
||||
**If an ecosystem matched but there is no existing-test evidence at all — bootstrap:**
|
||||
|
||||
### B2. Research best practices
|
||||
|
||||
|
|
@ -53,7 +82,9 @@ If WebSearch is unavailable, use this built-in knowledge table:
|
|||
| Node.js | vitest + @testing-library | jest + @testing-library |
|
||||
| Next.js | vitest + @testing-library/react + playwright | jest + cypress |
|
||||
| Python | pytest + pytest-cov | unittest |
|
||||
| Django | pytest + pytest-django | Django's built-in \`manage.py test\` (unittest) |
|
||||
| Go | stdlib testing + testify | stdlib only |
|
||||
| JVM (Maven/Gradle) | JUnit 5 + AssertJ | JUnit 5 only |
|
||||
| Rust | cargo test (built-in) + mockall | — |
|
||||
| PHP | phpunit + mockery | pest |
|
||||
| Elixir | ExUnit (built-in) + ex_machina | — |
|
||||
|
|
@ -202,15 +233,20 @@ Before analyzing coverage, detect the project's test framework:
|
|||
|
||||
\`\`\`bash
|
||||
setopt +o nomatch 2>/dev/null || true # zsh compat
|
||||
# Detect project runtime
|
||||
[ -f Gemfile ] && echo "RUNTIME:ruby"
|
||||
# Detect project runtime (markers are evidence, not commands to run blind)
|
||||
[ -f manage.py ] && echo "RUNTIME:python FRAMEWORK:django"
|
||||
{ [ -f pyproject.toml ] || [ -f pytest.ini ] || [ -f tox.ini ] || [ -f setup.cfg ] || [ -f requirements.txt ]; } && echo "RUNTIME:python"
|
||||
[ -f Gemfile ] || [ -f Rakefile ] || [ -f .rspec ] && echo "RUNTIME:ruby"
|
||||
[ -f package.json ] && echo "RUNTIME:node"
|
||||
[ -f requirements.txt ] || [ -f pyproject.toml ] && echo "RUNTIME:python"
|
||||
[ -f go.mod ] && echo "RUNTIME:go"
|
||||
[ -f Cargo.toml ] && echo "RUNTIME:rust"
|
||||
# Check for existing test infrastructure
|
||||
ls jest.config.* vitest.config.* playwright.config.* cypress.config.* .rspec pytest.ini phpunit.xml 2>/dev/null
|
||||
ls -d test/ tests/ spec/ __tests__/ cypress/ e2e/ 2>/dev/null
|
||||
[ -f pom.xml ] && echo "RUNTIME:jvm BUILD:maven"
|
||||
{ [ -f build.gradle ] || [ -f build.gradle.kts ]; } && echo "RUNTIME:jvm BUILD:gradle"
|
||||
# Check for existing test infrastructure — config files, scripts, AND test files
|
||||
ls jest.config.* vitest.config.* playwright.config.* cypress.config.* .rspec pytest.ini tox.ini phpunit.xml 2>/dev/null
|
||||
[ -f package.json ] && grep -q '"test"[[:space:]]*:' package.json && echo "SCRIPT:package.json test"
|
||||
[ -f Makefile ] && grep -qE '^(test|check):' Makefile && echo "TARGET:make test"
|
||||
git ls-files | grep -cE '(^|/)(tests?|spec|__tests__)/|(^|/)tests?\\.py$|(^|/)test_[^/]+\\.py$|_test\\.(go|py|rb|ts|js|exs)$|\\.(test|spec)\\.[jt]sx?$|_spec\\.rb$|Test\\.(java|kt)$' | sed 's/^/TESTFILES:/'
|
||||
\`\`\`
|
||||
|
||||
3. **If no framework detected:**${mode === 'ship' ? ' falls through to the Test Framework Bootstrap step (Step 4) which handles full setup.' : ' still produce the coverage diagram, but skip test generation.'}`);
|
||||
|
|
|
|||
|
|
@ -0,0 +1,113 @@
|
|||
/**
|
||||
* Regression: /ship Step 4 test-framework detection was blind to Django.
|
||||
*
|
||||
* A real Django project (manage.py + <app>/tests.py, green `python manage.py
|
||||
* test`, no pytest.ini and no tests/ directory) read as "no test framework",
|
||||
* so /ship bootstrapped pytest on top of a working suite. Same blindness hit
|
||||
* any config-less-but-tested project (Go *_test.go, in-source Rust #[test],
|
||||
* package.json with only a test script).
|
||||
*
|
||||
* These run the resolver's OWN emitted detection block against fixtures, so
|
||||
* the shell is checked, not just the prose — and the test is independent of
|
||||
* generated-SKILL.md regen state.
|
||||
*
|
||||
* Ported from time-attack/gstack commit e3259078 (GStack 2), adapted from the
|
||||
* fork's generated-markdown target to our resolver source of truth.
|
||||
*/
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
|
||||
import { generateTestBootstrap } from '../scripts/resolvers/testing';
|
||||
|
||||
const section = generateTestBootstrap({} as never);
|
||||
|
||||
/** The detection block the skill tells the agent to run (first bash fence). */
|
||||
function detectionScript(): string {
|
||||
const open = section.indexOf('```bash\n');
|
||||
expect(open).toBeGreaterThan(-1);
|
||||
const start = open + '```bash\n'.length;
|
||||
const end = section.indexOf('```', start);
|
||||
return section.slice(start, end);
|
||||
}
|
||||
|
||||
/** Run the detection block in a throwaway git repo laid out by `files`. */
|
||||
function detect(files: Record<string, string>): string {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ship-detect-'));
|
||||
try {
|
||||
for (const [rel, body] of Object.entries(files)) {
|
||||
const abs = path.join(dir, rel);
|
||||
fs.mkdirSync(path.dirname(abs), { recursive: true });
|
||||
fs.writeFileSync(abs, body);
|
||||
}
|
||||
const script = path.join(dir, '.detect.sh');
|
||||
fs.writeFileSync(script, detectionScript());
|
||||
const git = (...args: string[]) => execFileSync('git', args, { cwd: dir });
|
||||
git('init', '-q', '.');
|
||||
git('add', '-A');
|
||||
git('-c', 'user.email=t@t', '-c', 'user.name=t', 'commit', '-qm', 'fixture');
|
||||
// The block's last line is a `[ -f marker ] && echo`, so a clean project
|
||||
// exits 1 by design — read stdout, don't trust the status.
|
||||
return execFileSync('bash', [script], { cwd: dir, encoding: 'utf-8' });
|
||||
} catch (err: unknown) {
|
||||
const e = err as { stdout?: string };
|
||||
if (typeof e.stdout === 'string') return e.stdout;
|
||||
throw err;
|
||||
} finally {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
const testFileCount = (out: string): number =>
|
||||
Number(/^TESTFILES:(\d+)$/m.exec(out)?.[1] ?? -1);
|
||||
|
||||
describe('/ship Step 4 detection is multi-ecosystem', () => {
|
||||
test('Django project reports manage.py and its existing tests', () => {
|
||||
const out = detect({
|
||||
'manage.py': '#!/usr/bin/env python\n',
|
||||
'requirements.txt': 'Django==5.0\n',
|
||||
'polls/tests.py': 'from django.test import TestCase\n',
|
||||
});
|
||||
expect(out).toContain('MARKER:manage.py');
|
||||
expect(out).toContain('FRAMEWORK:django');
|
||||
expect(testFileCount(out)).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
test('config-less projects with tests still report test evidence', () => {
|
||||
expect(testFileCount(detect({
|
||||
'go.mod': 'module x\n',
|
||||
'x_test.go': 'package x\n',
|
||||
}))).toBeGreaterThan(0);
|
||||
|
||||
const node = detect({
|
||||
'package.json': '{"name":"n","scripts":{"test":"node --test"}}\n',
|
||||
});
|
||||
expect(node).toContain('SCRIPT:package.json test');
|
||||
|
||||
const rust = detect({
|
||||
'Cargo.toml': '[package]\nname="x"\n',
|
||||
'src/lib.rs': '#[cfg(test)]\nmod t{ #[test] fn x(){} }\n',
|
||||
});
|
||||
expect(rust).toContain('TESTS:rust in-source');
|
||||
});
|
||||
|
||||
test('a genuinely untested project reports no test evidence', () => {
|
||||
const out = detect({ 'go.mod': 'module x\n', 'x.go': 'package x\n' });
|
||||
expect(out).toContain('RUNTIME:go');
|
||||
expect(testFileCount(out)).toBe(0);
|
||||
expect(out).not.toContain('TESTS:');
|
||||
});
|
||||
|
||||
test('every ecosystem marker maps to a command to OFFER, not to run blind', () => {
|
||||
expect(section).toContain('never a command to run blind');
|
||||
for (const marker of ['manage.py', 'go.mod', 'Cargo.toml', 'pom.xml',
|
||||
'build.gradle', 'mix.exs', 'composer.json', 'Gemfile', 'pytest.ini']) {
|
||||
expect(section).toContain(marker);
|
||||
}
|
||||
// The ask-and-persist contract, not a hardcoded project command.
|
||||
expect(section).toContain('AskUserQuestion');
|
||||
expect(section).toContain('persist the answer to CLAUDE.md');
|
||||
});
|
||||
});
|
||||
Loading…
Reference in New Issue