mirror of https://github.com/garrytan/gstack.git
feat: migrate eval storage to project-scoped paths
Move eval results and E2E run artifacts from ~/.gstack-dev/evals/ to ~/.gstack/projects/$SLUG/evals/ so each project's eval history lives alongside its other gstack data. Falls back to legacy path if slug detection fails. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
f4b8f2592f
commit
f3151839d8
|
|
@ -15,10 +15,11 @@ import {
|
||||||
findPreviousRun,
|
findPreviousRun,
|
||||||
compareEvalResults,
|
compareEvalResults,
|
||||||
formatComparison,
|
formatComparison,
|
||||||
|
getProjectEvalDir,
|
||||||
} from '../test/helpers/eval-store';
|
} from '../test/helpers/eval-store';
|
||||||
import type { EvalResult } from '../test/helpers/eval-store';
|
import type { EvalResult } from '../test/helpers/eval-store';
|
||||||
|
|
||||||
const EVAL_DIR = path.join(os.homedir(), '.gstack-dev', 'evals');
|
const EVAL_DIR = getProjectEvalDir();
|
||||||
|
|
||||||
function loadResult(filepath: string): EvalResult {
|
function loadResult(filepath: string): EvalResult {
|
||||||
// Resolve relative to EVAL_DIR if not absolute
|
// Resolve relative to EVAL_DIR if not absolute
|
||||||
|
|
|
||||||
|
|
@ -8,8 +8,9 @@
|
||||||
import * as fs from 'fs';
|
import * as fs from 'fs';
|
||||||
import * as path from 'path';
|
import * as path from 'path';
|
||||||
import * as os from 'os';
|
import * as os from 'os';
|
||||||
|
import { getProjectEvalDir } from '../test/helpers/eval-store';
|
||||||
|
|
||||||
const EVAL_DIR = path.join(os.homedir(), '.gstack-dev', 'evals');
|
const EVAL_DIR = getProjectEvalDir();
|
||||||
|
|
||||||
// Parse args
|
// Parse args
|
||||||
const args = process.argv.slice(2);
|
const args = process.argv.slice(2);
|
||||||
|
|
|
||||||
|
|
@ -9,8 +9,9 @@ import * as fs from 'fs';
|
||||||
import * as path from 'path';
|
import * as path from 'path';
|
||||||
import * as os from 'os';
|
import * as os from 'os';
|
||||||
import type { EvalResult } from '../test/helpers/eval-store';
|
import type { EvalResult } from '../test/helpers/eval-store';
|
||||||
|
import { getProjectEvalDir } from '../test/helpers/eval-store';
|
||||||
|
|
||||||
const EVAL_DIR = path.join(os.homedir(), '.gstack-dev', 'evals');
|
const EVAL_DIR = getProjectEvalDir();
|
||||||
|
|
||||||
let files: string[];
|
let files: string[];
|
||||||
try {
|
try {
|
||||||
|
|
|
||||||
|
|
@ -2,7 +2,7 @@
|
||||||
* Eval result persistence and comparison.
|
* Eval result persistence and comparison.
|
||||||
*
|
*
|
||||||
* EvalCollector accumulates test results, writes them to
|
* EvalCollector accumulates test results, writes them to
|
||||||
* ~/.gstack-dev/evals/{version}-{branch}-{tier}-{timestamp}.json,
|
* ~/.gstack/projects/$SLUG/evals/{version}-{branch}-{tier}-{timestamp}.json,
|
||||||
* prints a summary table, and auto-compares with the previous run.
|
* prints a summary table, and auto-compares with the previous run.
|
||||||
*
|
*
|
||||||
* Comparison functions are exported for reuse by the eval:compare CLI.
|
* Comparison functions are exported for reuse by the eval:compare CLI.
|
||||||
|
|
@ -14,7 +14,32 @@ import * as os from 'os';
|
||||||
import { spawnSync } from 'child_process';
|
import { spawnSync } from 'child_process';
|
||||||
|
|
||||||
const SCHEMA_VERSION = 1;
|
const SCHEMA_VERSION = 1;
|
||||||
const DEFAULT_EVAL_DIR = path.join(os.homedir(), '.gstack-dev', 'evals');
|
const LEGACY_EVAL_DIR = path.join(os.homedir(), '.gstack-dev', 'evals');
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Detect project-scoped eval dir via gstack-slug.
|
||||||
|
* Falls back to legacy ~/.gstack-dev/evals/ if slug detection fails.
|
||||||
|
*/
|
||||||
|
export function getProjectEvalDir(): string {
|
||||||
|
try {
|
||||||
|
// Try repo-local gstack-slug first, then global install
|
||||||
|
const localSlug = spawnSync('bash', ['-c', '.claude/skills/gstack/bin/gstack-slug 2>/dev/null || ~/.claude/skills/gstack/bin/gstack-slug 2>/dev/null'], {
|
||||||
|
stdio: 'pipe', timeout: 3000,
|
||||||
|
});
|
||||||
|
const output = localSlug.stdout?.toString().trim();
|
||||||
|
if (output) {
|
||||||
|
const slugMatch = output.match(/^SLUG=(.+)$/m);
|
||||||
|
if (slugMatch && slugMatch[1]) {
|
||||||
|
const dir = path.join(os.homedir(), '.gstack', 'projects', slugMatch[1], 'evals');
|
||||||
|
fs.mkdirSync(dir, { recursive: true });
|
||||||
|
return dir;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch { /* fall through */ }
|
||||||
|
return LEGACY_EVAL_DIR;
|
||||||
|
}
|
||||||
|
|
||||||
|
const DEFAULT_EVAL_DIR = getProjectEvalDir();
|
||||||
|
|
||||||
// --- Interfaces ---
|
// --- Interfaces ---
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -9,9 +9,11 @@
|
||||||
import * as fs from 'fs';
|
import * as fs from 'fs';
|
||||||
import * as path from 'path';
|
import * as path from 'path';
|
||||||
import * as os from 'os';
|
import * as os from 'os';
|
||||||
|
import { getProjectEvalDir } from './eval-store';
|
||||||
|
|
||||||
const GSTACK_DEV_DIR = path.join(os.homedir(), '.gstack-dev');
|
const GSTACK_DEV_DIR = path.join(os.homedir(), '.gstack-dev');
|
||||||
const HEARTBEAT_PATH = path.join(GSTACK_DEV_DIR, 'e2e-live.json');
|
const HEARTBEAT_PATH = path.join(GSTACK_DEV_DIR, 'e2e-live.json'); // heartbeat stays global
|
||||||
|
const PROJECT_DIR = path.dirname(getProjectEvalDir()); // ~/.gstack/projects/$SLUG/
|
||||||
|
|
||||||
/** Sanitize test name for use as filename: strip leading slashes, replace / with - */
|
/** Sanitize test name for use as filename: strip leading slashes, replace / with - */
|
||||||
export function sanitizeTestName(name: string): string {
|
export function sanitizeTestName(name: string): string {
|
||||||
|
|
@ -144,7 +146,7 @@ export async function runSkillTest(options: {
|
||||||
const safeName = testName ? sanitizeTestName(testName) : null;
|
const safeName = testName ? sanitizeTestName(testName) : null;
|
||||||
if (runId) {
|
if (runId) {
|
||||||
try {
|
try {
|
||||||
runDir = path.join(GSTACK_DEV_DIR, 'e2e-runs', runId);
|
runDir = path.join(PROJECT_DIR, 'e2e-runs', runId);
|
||||||
fs.mkdirSync(runDir, { recursive: true });
|
fs.mkdirSync(runDir, { recursive: true });
|
||||||
} catch { /* non-fatal */ }
|
} catch { /* non-fatal */ }
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue