fix: harden local Agy integration

benchmark-models/SKILL.md

@@ -5,7 +5,7 @@ version: 1.0.0
 description: Cross-model benchmark for gstack skills. (gstack)
 triggers:
   - cross model benchmark
-  - compare claude gpt gemini
+  - compare claude gpt gemini agy
   - benchmark skill across models
   - which model should I use
 allowed-tools:
@@ -20,7 +20,7 @@ allowed-tools:
 ## When to invoke this skill
 
 Runs the same prompt through Claude,
-GPT (via Codex CLI), and Gemini side-by-side — compares latency, tokens, cost,
+GPT (via Codex CLI), Gemini, and Agy side-by-side — compares latency, tokens, cost,
 and optionally quality via LLM judge. Answers "which model is actually best
 for this skill?" with data instead of vibes. Separate from /benchmark, which
 measures web page performance. Use when: "benchmark models", "compare models",
@@ -578,12 +578,12 @@ If C: ask for the path. Verify it exists. Use as positional argument.
 ## Step 2: Choose providers
 
 ```bash
-"$BIN" --prompt "unused, dry-run" --models claude,gpt,gemini --dry-run
+"$BIN" --prompt "unused, dry-run" --models claude,gpt,gemini,agy --dry-run
 ```
 
 Show the dry-run output. The "Adapter availability" section tells the user which providers will actually run (OK) vs skip (NOT READY — remediation hint included).
 
-If ALL three show NOT READY: stop with a clear message — benchmark can't run without at least one authed provider. Suggest `claude login`, `codex login`, or `gemini login` / `export GOOGLE_API_KEY`.
+If ALL four show NOT READY: stop with a clear message — benchmark can't run without at least one authenticated provider. Suggest `claude login`, `codex login`, `gemini login` / `export GOOGLE_API_KEY`, or logging in through `agy`.
 
 If at least one is OK: AskUserQuestion:
 - **Simplify:** "Which models should we include? The dry-run above showed which are authed. Unauthed ones will be skipped cleanly — they won't abort the batch."

benchmark-models/SKILL.md.tmpl

@@ -4,7 +4,7 @@ preamble-tier: 1
 version: 1.0.0
 description: |
   Cross-model benchmark for gstack skills. Runs the same prompt through Claude,
-  GPT (via Codex CLI), and Gemini side-by-side — compares latency, tokens, cost,
+  GPT (via Codex CLI), Gemini, and Agy side-by-side — compares latency, tokens, cost,
   and optionally quality via LLM judge. Answers "which model is actually best
   for this skill?" with data instead of vibes. Separate from /benchmark, which
   measures web page performance. Use when: "benchmark models", "compare models",
@@ -15,7 +15,7 @@ voice-triggers:
   - "which model is best"
 triggers:
   - cross model benchmark
-  - compare claude gpt gemini
+  - compare claude gpt gemini agy
   - benchmark skill across models
   - which model should I use
 allowed-tools:
@@ -69,12 +69,12 @@ If C: ask for the path. Verify it exists. Use as positional argument.
 ## Step 2: Choose providers
 
 ```bash
-"$BIN" --prompt "unused, dry-run" --models claude,gpt,gemini --dry-run
+"$BIN" --prompt "unused, dry-run" --models claude,gpt,gemini,agy --dry-run
 ```
 
 Show the dry-run output. The "Adapter availability" section tells the user which providers will actually run (OK) vs skip (NOT READY — remediation hint included).
 
-If ALL three show NOT READY: stop with a clear message — benchmark can't run without at least one authed provider. Suggest `claude login`, `codex login`, or `gemini login` / `export GOOGLE_API_KEY`.
+If ALL four show NOT READY: stop with a clear message — benchmark can't run without at least one authenticated provider. Suggest `claude login`, `codex login`, `gemini login` / `export GOOGLE_API_KEY`, or logging in through `agy`.
 
 If at least one is OK: AskUserQuestion:
 - **Simplify:** "Which models should we include? The dry-run above showed which are authed. Unauthed ones will be skipped cleanly — they won't abort the batch."

scripts/proactive-suggestions.json

@@ -15,7 +15,7 @@
     },
     "benchmark-models": {
       "lead": "Cross-model benchmark for gstack skills.",
-      "routing": "Runs the same prompt through Claude,\nGPT (via Codex CLI), and Gemini side-by-side — compares latency, tokens, cost,\nand optionally quality via LLM judge. Answers \"which model is actually best\nfor this skill?\" with data instead of vibes. Separate from /benchmark, which\nmeasures web page performance. Use when: \"benchmark models\", \"compare models\",\n\"which model is best for X\", \"cross-model comparison\", \"model shootout\".",
+      "routing": "Runs the same prompt through Claude,\nGPT (via Codex CLI), Gemini, and Agy side-by-side — compares latency, tokens, cost,\nand optionally quality via LLM judge. Answers \"which model is actually best\nfor this skill?\" with data instead of vibes. Separate from /benchmark, which\nmeasures web page performance. Use when: \"benchmark models\", \"compare models\",\n\"which model is best for X\", \"cross-model comparison\", \"model shootout\".",
       "voice_line": "Voice triggers (speech-to-text aliases): \"compare models\", \"model shootout\", \"which model is best\"."
     },
     "browse": {

setup

@@ -1304,10 +1304,18 @@ if [ "$INSTALL_AGY" -eq 1 ]; then
   mkdir -p "$AGY_SKILLS_SOURCE"
   create_agy_runtime_root "$SOURCE_GSTACK_DIR" "$AGY_GSTACK_SOURCE"
   link_agy_skill_dirs "$SOURCE_GSTACK_DIR" "$AGY_SKILLS_SOURCE"
-  # Register/import the plugin into Agy
+  # Explicit Agy setup is incomplete unless the CLI can register the plugin.
-  if command -v agy >/dev/null 2>&1; then
+  if ! command -v agy >/dev/null 2>&1; then
-    echo "Registering gstack plugin with Agy..."
+    echo "gstack setup failed: agy CLI not found on PATH; install Agy, then rerun './setup --host agy'." >&2
-    agy plugin install "$AGY_GSTACK_SOURCE" >/dev/null 2>&1 || agy plugin import "$AGY_GSTACK_SOURCE" --force >/dev/null 2>&1 || true
+    exit 1
+  fi
+  echo "Registering gstack plugin with Agy..."
+  if ! agy plugin install "$AGY_GSTACK_SOURCE"; then
+    echo "Agy plugin install failed; trying import --force..." >&2
+    if ! agy plugin import "$AGY_GSTACK_SOURCE" --force; then
+      echo "gstack setup failed: Agy could not register the plugin at $AGY_GSTACK_SOURCE." >&2
+      exit 1
+    fi
   fi
   echo "gstack ready (agy)."
   echo "  browse: $BROWSE_BIN"

test/gen-skill-docs.test.ts

@@ -2439,6 +2439,9 @@ describe('setup script validation', () => {
     expect(setupContent).toContain('qa/references');
     expect(setupContent).toContain('dx-hall-of-fame.md');
     expect(setupContent).toContain('agy plugin install');
+    expect(setupContent).toContain('agy CLI not found on PATH');
+    expect(setupContent).toContain('Agy could not register the plugin');
+    expect(setupContent).not.toContain('agy plugin import "$AGY_GSTACK_SOURCE" --force >/dev/null 2>&1 || true');
   });
 
   test('setup regenerates deleted agy skills and derives the plugin version from VERSION', () => {

test/helpers/providers/agy.ts

@@ -30,8 +30,16 @@ export class AgyAdapter implements ProviderAdapter {
 
   async run(opts: RunOpts): Promise<RunResult> {
     const start = Date.now();
-    const args = ['--print', opts.prompt, '--dangerously-skip-permissions'];
+    // Benchmarks must never grant an agent write/tool approval in the caller's
+    // workdir. Plan mode plus the terminal sandbox is Agy's read-only boundary.
+    const args = ['--print', opts.prompt, '--mode', 'plan', '--sandbox'];
     if (opts.model) args.push('--model', opts.model);
+    if (opts.extraArgs?.includes('--dangerously-skip-permissions')) {
+      return this.emptyResult(0, {
+        code: 'unknown',
+        reason: '--dangerously-skip-permissions is not allowed in benchmarks',
+      }, opts.model);
+    }
     if (opts.extraArgs) args.push(...opts.extraArgs);
 
     try {

test/skill-e2e-benchmark-providers.test.ts

@@ -42,6 +42,20 @@ const gpt = new GptAdapter();
 const gemini = new GeminiAdapter();
 const agy = new AgyAdapter();
 
+test('agy adapter enforces sandboxed plan mode and rejects permission bypass', async () => {
+  const source = fs.readFileSync(path.join(import.meta.dir, 'helpers/providers/agy.ts'), 'utf8');
+  expect(source).toContain("'--mode', 'plan', '--sandbox'");
+  expect(source).not.toContain("opts.prompt, '--dangerously-skip-permissions'");
+
+  const result = await agy.run({
+    prompt: PROMPT,
+    workdir: os.tmpdir(),
+    timeoutMs: 1000,
+    extraArgs: ['--dangerously-skip-permissions'],
+  });
+  expect(result.error?.reason).toContain('not allowed');
+});
+
 // Use a temp working directory so provider CLIs can't accidentally touch the repo.
 // Created in beforeAll / cleaned in afterAll so concurrent CI runs don't leak.
 let workdir: string;