diff --git a/benchmark-models/SKILL.md b/benchmark-models/SKILL.md index 6a9d62616..80bcbd416 100644 --- a/benchmark-models/SKILL.md +++ b/benchmark-models/SKILL.md @@ -5,7 +5,7 @@ version: 1.0.0 description: Cross-model benchmark for gstack skills. (gstack) triggers: - cross model benchmark - - compare claude gpt gemini + - compare claude gpt gemini agy - benchmark skill across models - which model should I use allowed-tools: @@ -20,7 +20,7 @@ allowed-tools: ## When to invoke this skill Runs the same prompt through Claude, -GPT (via Codex CLI), and Gemini side-by-side — compares latency, tokens, cost, +GPT (via Codex CLI), Gemini, and Agy side-by-side — compares latency, tokens, cost, and optionally quality via LLM judge. Answers "which model is actually best for this skill?" with data instead of vibes. Separate from /benchmark, which measures web page performance. Use when: "benchmark models", "compare models", @@ -578,12 +578,12 @@ If C: ask for the path. Verify it exists. Use as positional argument. ## Step 2: Choose providers ```bash -"$BIN" --prompt "unused, dry-run" --models claude,gpt,gemini --dry-run +"$BIN" --prompt "unused, dry-run" --models claude,gpt,gemini,agy --dry-run ``` Show the dry-run output. The "Adapter availability" section tells the user which providers will actually run (OK) vs skip (NOT READY — remediation hint included). -If ALL three show NOT READY: stop with a clear message — benchmark can't run without at least one authed provider. Suggest `claude login`, `codex login`, or `gemini login` / `export GOOGLE_API_KEY`. +If ALL four show NOT READY: stop with a clear message — benchmark can't run without at least one authenticated provider. Suggest `claude login`, `codex login`, `gemini login` / `export GOOGLE_API_KEY`, or logging in through `agy`. If at least one is OK: AskUserQuestion: - **Simplify:** "Which models should we include? The dry-run above showed which are authed. Unauthed ones will be skipped cleanly — they won't abort the batch." diff --git a/benchmark-models/SKILL.md.tmpl b/benchmark-models/SKILL.md.tmpl index 034cda182..dc40ddbf4 100644 --- a/benchmark-models/SKILL.md.tmpl +++ b/benchmark-models/SKILL.md.tmpl @@ -4,7 +4,7 @@ preamble-tier: 1 version: 1.0.0 description: | Cross-model benchmark for gstack skills. Runs the same prompt through Claude, - GPT (via Codex CLI), and Gemini side-by-side — compares latency, tokens, cost, + GPT (via Codex CLI), Gemini, and Agy side-by-side — compares latency, tokens, cost, and optionally quality via LLM judge. Answers "which model is actually best for this skill?" with data instead of vibes. Separate from /benchmark, which measures web page performance. Use when: "benchmark models", "compare models", @@ -15,7 +15,7 @@ voice-triggers: - "which model is best" triggers: - cross model benchmark - - compare claude gpt gemini + - compare claude gpt gemini agy - benchmark skill across models - which model should I use allowed-tools: @@ -69,12 +69,12 @@ If C: ask for the path. Verify it exists. Use as positional argument. ## Step 2: Choose providers ```bash -"$BIN" --prompt "unused, dry-run" --models claude,gpt,gemini --dry-run +"$BIN" --prompt "unused, dry-run" --models claude,gpt,gemini,agy --dry-run ``` Show the dry-run output. The "Adapter availability" section tells the user which providers will actually run (OK) vs skip (NOT READY — remediation hint included). -If ALL three show NOT READY: stop with a clear message — benchmark can't run without at least one authed provider. Suggest `claude login`, `codex login`, or `gemini login` / `export GOOGLE_API_KEY`. +If ALL four show NOT READY: stop with a clear message — benchmark can't run without at least one authenticated provider. Suggest `claude login`, `codex login`, `gemini login` / `export GOOGLE_API_KEY`, or logging in through `agy`. If at least one is OK: AskUserQuestion: - **Simplify:** "Which models should we include? The dry-run above showed which are authed. Unauthed ones will be skipped cleanly — they won't abort the batch." diff --git a/scripts/proactive-suggestions.json b/scripts/proactive-suggestions.json index d08c60853..466192738 100644 --- a/scripts/proactive-suggestions.json +++ b/scripts/proactive-suggestions.json @@ -15,7 +15,7 @@ }, "benchmark-models": { "lead": "Cross-model benchmark for gstack skills.", - "routing": "Runs the same prompt through Claude,\nGPT (via Codex CLI), and Gemini side-by-side — compares latency, tokens, cost,\nand optionally quality via LLM judge. Answers \"which model is actually best\nfor this skill?\" with data instead of vibes. Separate from /benchmark, which\nmeasures web page performance. Use when: \"benchmark models\", \"compare models\",\n\"which model is best for X\", \"cross-model comparison\", \"model shootout\".", + "routing": "Runs the same prompt through Claude,\nGPT (via Codex CLI), Gemini, and Agy side-by-side — compares latency, tokens, cost,\nand optionally quality via LLM judge. Answers \"which model is actually best\nfor this skill?\" with data instead of vibes. Separate from /benchmark, which\nmeasures web page performance. Use when: \"benchmark models\", \"compare models\",\n\"which model is best for X\", \"cross-model comparison\", \"model shootout\".", "voice_line": "Voice triggers (speech-to-text aliases): \"compare models\", \"model shootout\", \"which model is best\"." }, "browse": { diff --git a/setup b/setup index dfd0e160b..5e5e6bc00 100755 --- a/setup +++ b/setup @@ -1304,10 +1304,18 @@ if [ "$INSTALL_AGY" -eq 1 ]; then mkdir -p "$AGY_SKILLS_SOURCE" create_agy_runtime_root "$SOURCE_GSTACK_DIR" "$AGY_GSTACK_SOURCE" link_agy_skill_dirs "$SOURCE_GSTACK_DIR" "$AGY_SKILLS_SOURCE" - # Register/import the plugin into Agy - if command -v agy >/dev/null 2>&1; then - echo "Registering gstack plugin with Agy..." - agy plugin install "$AGY_GSTACK_SOURCE" >/dev/null 2>&1 || agy plugin import "$AGY_GSTACK_SOURCE" --force >/dev/null 2>&1 || true + # Explicit Agy setup is incomplete unless the CLI can register the plugin. + if ! command -v agy >/dev/null 2>&1; then + echo "gstack setup failed: agy CLI not found on PATH; install Agy, then rerun './setup --host agy'." >&2 + exit 1 + fi + echo "Registering gstack plugin with Agy..." + if ! agy plugin install "$AGY_GSTACK_SOURCE"; then + echo "Agy plugin install failed; trying import --force..." >&2 + if ! agy plugin import "$AGY_GSTACK_SOURCE" --force; then + echo "gstack setup failed: Agy could not register the plugin at $AGY_GSTACK_SOURCE." >&2 + exit 1 + fi fi echo "gstack ready (agy)." echo " browse: $BROWSE_BIN" diff --git a/test/gen-skill-docs.test.ts b/test/gen-skill-docs.test.ts index 5e580b7d2..c06b69f8e 100644 --- a/test/gen-skill-docs.test.ts +++ b/test/gen-skill-docs.test.ts @@ -2439,6 +2439,9 @@ describe('setup script validation', () => { expect(setupContent).toContain('qa/references'); expect(setupContent).toContain('dx-hall-of-fame.md'); expect(setupContent).toContain('agy plugin install'); + expect(setupContent).toContain('agy CLI not found on PATH'); + expect(setupContent).toContain('Agy could not register the plugin'); + expect(setupContent).not.toContain('agy plugin import "$AGY_GSTACK_SOURCE" --force >/dev/null 2>&1 || true'); }); test('setup regenerates deleted agy skills and derives the plugin version from VERSION', () => { diff --git a/test/helpers/providers/agy.ts b/test/helpers/providers/agy.ts index 3a50efe6d..c37129f08 100644 --- a/test/helpers/providers/agy.ts +++ b/test/helpers/providers/agy.ts @@ -30,8 +30,16 @@ export class AgyAdapter implements ProviderAdapter { async run(opts: RunOpts): Promise { const start = Date.now(); - const args = ['--print', opts.prompt, '--dangerously-skip-permissions']; + // Benchmarks must never grant an agent write/tool approval in the caller's + // workdir. Plan mode plus the terminal sandbox is Agy's read-only boundary. + const args = ['--print', opts.prompt, '--mode', 'plan', '--sandbox']; if (opts.model) args.push('--model', opts.model); + if (opts.extraArgs?.includes('--dangerously-skip-permissions')) { + return this.emptyResult(0, { + code: 'unknown', + reason: '--dangerously-skip-permissions is not allowed in benchmarks', + }, opts.model); + } if (opts.extraArgs) args.push(...opts.extraArgs); try { diff --git a/test/skill-e2e-benchmark-providers.test.ts b/test/skill-e2e-benchmark-providers.test.ts index 446762d11..bd60c71cf 100644 --- a/test/skill-e2e-benchmark-providers.test.ts +++ b/test/skill-e2e-benchmark-providers.test.ts @@ -42,6 +42,20 @@ const gpt = new GptAdapter(); const gemini = new GeminiAdapter(); const agy = new AgyAdapter(); +test('agy adapter enforces sandboxed plan mode and rejects permission bypass', async () => { + const source = fs.readFileSync(path.join(import.meta.dir, 'helpers/providers/agy.ts'), 'utf8'); + expect(source).toContain("'--mode', 'plan', '--sandbox'"); + expect(source).not.toContain("opts.prompt, '--dangerously-skip-permissions'"); + + const result = await agy.run({ + prompt: PROMPT, + workdir: os.tmpdir(), + timeoutMs: 1000, + extraArgs: ['--dangerously-skip-permissions'], + }); + expect(result.error?.reason).toContain('not allowed'); +}); + // Use a temp working directory so provider CLIs can't accidentally touch the repo. // Created in beforeAll / cleaned in afterAll so concurrent CI runs don't leak. let workdir: string;