mirror of https://github.com/garrytan/gstack.git
fix(memory): isolate transcript ingest by repo trust
This commit is contained in:
Sinabina
parent
7c9df1c568
commit
6249074d53
|
|
@ -520,13 +520,65 @@ function* walkGstackArtifacts(ctx: WalkContext): Generator<{ path: string; type:
|
|||
}
|
||||
|
||||
function* walkAllSources(ctx: WalkContext): Generator<{ path: string; type: MemoryType }> {
|
||||
if (ctx.args.sources.has("transcript")) {
|
||||
if (ctx.args.sources.has("transcript") && transcriptIngestMode() !== "off") {
|
||||
yield* walkClaudeCodeProjects(ctx);
|
||||
yield* walkCodexSessions(ctx);
|
||||
}
|
||||
yield* walkGstackArtifacts(ctx);
|
||||
}
|
||||
|
||||
function transcriptIngestMode(): string {
|
||||
try {
|
||||
const raw = readFileSync(join(GSTACK_HOME, "config.yaml"), "utf-8");
|
||||
const match = raw.match(/^transcript_ingest_mode:\s*([^#\s]+)\s*(?:#.*)?$/m);
|
||||
return match?.[1]?.toLowerCase() || "";
|
||||
} catch {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
let cachedCurrentRepoRemote: string | undefined;
|
||||
function currentRepoRemote(): string {
|
||||
if (cachedCurrentRepoRemote !== undefined) return cachedCurrentRepoRemote;
|
||||
try {
|
||||
cachedCurrentRepoRemote = canonicalizeRemote(
|
||||
execFileSync("git", ["-C", process.cwd(), "remote", "get-url", "origin"], {
|
||||
encoding: "utf-8",
|
||||
timeout: 5_000,
|
||||
stdio: ["ignore", "pipe", "ignore"],
|
||||
}).trim(),
|
||||
);
|
||||
} catch {
|
||||
cachedCurrentRepoRemote = "";
|
||||
}
|
||||
return cachedCurrentRepoRemote;
|
||||
}
|
||||
|
||||
let cachedRepoPolicy: Record<string, unknown> | undefined;
|
||||
function repoTrustTier(remote: string): string {
|
||||
if (!remote) return "unset";
|
||||
try {
|
||||
cachedRepoPolicy ??= JSON.parse(
|
||||
readFileSync(join(GSTACK_HOME, "gbrain-repo-policy.json"), "utf-8"),
|
||||
) as Record<string, unknown>;
|
||||
const tier = cachedRepoPolicy[remote];
|
||||
return typeof tier === "string" ? tier : "unset";
|
||||
} catch {
|
||||
cachedRepoPolicy = {};
|
||||
return "unset";
|
||||
}
|
||||
}
|
||||
|
||||
function transcriptSourceId(): string {
|
||||
const remote = currentRepoRemote();
|
||||
const readable = repoSlug(remote)
|
||||
.replace(/[^a-z0-9]+/gi, "-")
|
||||
.replace(/^-+|-+$/g, "")
|
||||
.toLowerCase() || "current-repo";
|
||||
const hash = createHash("sha256").update(remote || process.cwd()).digest("hex").slice(0, 8);
|
||||
return `transcripts-${readable.slice(0, 11)}-${hash}`;
|
||||
}
|
||||
|
||||
// ── Renderers ──────────────────────────────────────────────────────────────
|
||||
|
||||
interface ParsedSession {
|
||||
|
|
@ -1171,6 +1223,16 @@ function preparePages(
|
|||
skippedUnattributed++;
|
||||
continue;
|
||||
}
|
||||
const currentRemote = currentRepoRemote();
|
||||
const tier = repoTrustTier(page.git_remote || "");
|
||||
const explicitlyUnattributed = (!page.git_remote || page.git_remote === "_unattributed") && args.includeUnattributed;
|
||||
if (
|
||||
!explicitlyUnattributed &&
|
||||
(!currentRemote || page.git_remote !== currentRemote || tier === "deny" || tier === "read-only")
|
||||
) {
|
||||
skippedUnattributed++;
|
||||
continue;
|
||||
}
|
||||
if (page.partial) partialPages++;
|
||||
} else {
|
||||
page = buildArtifactPage(path, type);
|
||||
|
|
@ -1409,6 +1471,7 @@ export function resolveImportTimeoutMs(
|
|||
function runGbrainImport(
|
||||
stagingDir: string,
|
||||
timeoutMs: number,
|
||||
sourceId?: string,
|
||||
): Promise<{ status: number | null; stdout: string; stderr: string; timedOut: boolean }> {
|
||||
installSignalForwarder();
|
||||
return new Promise((resolve) => {
|
||||
|
|
@ -1416,7 +1479,9 @@ function runGbrainImport(
|
|||
// inside Next.js / Prisma / Rails projects with their own
|
||||
// .env.local (codex review #7 — defense in depth on top of the
|
||||
// parent gstack-gbrain-sync seeding the bun grandchild's env).
|
||||
const child = spawnGbrainAsync(["import", stagingDir, "--no-embed", "--json"]);
|
||||
const importArgs = ["import", stagingDir, "--no-embed", "--json"];
|
||||
if (sourceId) importArgs.push("--source-id", sourceId);
|
||||
const child = spawnGbrainAsync(importArgs);
|
||||
_activeImportChild = child;
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
|
|
@ -1690,7 +1755,10 @@ async function ingestPass(args: CliArgs): Promise<BulkResult> {
|
|||
// spawn, parent termination orphans the gbrain process (observed
|
||||
// during 2026-05-10 cold-run testing — gbrain kept running 15 min
|
||||
// after the orchestrator timed out).
|
||||
const importResult = await runGbrainImport(stagingDir, resolveImportTimeoutMs());
|
||||
const sourceId = args.sources.size === 1 && args.sources.has("transcript")
|
||||
? transcriptSourceId()
|
||||
: undefined;
|
||||
const importResult = await runGbrainImport(stagingDir, resolveImportTimeoutMs(), sourceId);
|
||||
|
||||
const stdout = importResult.stdout || "";
|
||||
const stderr = importResult.stderr || "";
|
||||
|
|
@ -1893,6 +1961,25 @@ function printBulkResult(r: BulkResult, args: CliArgs): void {
|
|||
}
|
||||
}
|
||||
|
||||
async function runScopedIngest(args: CliArgs): Promise<BulkResult> {
|
||||
const hasTranscripts = args.sources.has("transcript");
|
||||
const artifactTypes = new Set([...args.sources].filter((type) => type !== "transcript"));
|
||||
if (!hasTranscripts || artifactTypes.size === 0) return ingestPass(args);
|
||||
|
||||
const transcriptResult = await ingestPass({ ...args, sources: new Set<MemoryType>(["transcript"]) });
|
||||
const artifactResult = await ingestPass({ ...args, sources: artifactTypes });
|
||||
return {
|
||||
written: transcriptResult.written + artifactResult.written,
|
||||
skipped_secret: transcriptResult.skipped_secret + artifactResult.skipped_secret,
|
||||
skipped_dedup: transcriptResult.skipped_dedup + artifactResult.skipped_dedup,
|
||||
skipped_unattributed: transcriptResult.skipped_unattributed + artifactResult.skipped_unattributed,
|
||||
failed: transcriptResult.failed + artifactResult.failed,
|
||||
duration_ms: transcriptResult.duration_ms + artifactResult.duration_ms,
|
||||
partial_pages: transcriptResult.partial_pages + artifactResult.partial_pages,
|
||||
system_error: transcriptResult.system_error || artifactResult.system_error,
|
||||
};
|
||||
}
|
||||
|
||||
// ── Entry point ────────────────────────────────────────────────────────────
|
||||
|
||||
async function main(): Promise<void> {
|
||||
|
|
@ -1913,7 +2000,7 @@ async function main(): Promise<void> {
|
|||
if (args.mode === "incremental" && args.quiet) {
|
||||
// Steady-state fast path: log nothing unless changes happen.
|
||||
const t0 = Date.now();
|
||||
const result = await ingestPass(args);
|
||||
const result = await runScopedIngest(args);
|
||||
const dt = Date.now() - t0;
|
||||
if (result.written > 0 || result.failed > 0) {
|
||||
console.error(`[memory-ingest] ${result.written} written, ${result.failed} failed in ${dt}ms`);
|
||||
|
|
@ -1924,7 +2011,7 @@ async function main(): Promise<void> {
|
|||
return;
|
||||
}
|
||||
|
||||
const result = await ingestPass(args);
|
||||
const result = await runScopedIngest(args);
|
||||
printBulkResult(result, args);
|
||||
if (result.system_error) process.exit(1);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -362,6 +362,7 @@ EOF
|
|||
DIR="\${2:-}"
|
||||
NO_EMBED=0
|
||||
JSON=0
|
||||
echo "argv=\$*" >> "\$ARGS_LOG"
|
||||
shift 2 || true
|
||||
for arg in "\$@"; do
|
||||
case "\$arg" in
|
||||
|
|
@ -766,3 +767,71 @@ exit 0
|
|||
rmSync(home, { recursive: true, force: true });
|
||||
});
|
||||
});
|
||||
|
||||
describe("gstack-memory-ingest transcript boundaries (#2140)", () => {
|
||||
it("honors off mode before invoking gbrain", () => {
|
||||
const home = makeTestHome();
|
||||
const gstackHome = join(home, ".gstack");
|
||||
mkdirSync(gstackHome, { recursive: true });
|
||||
writeFileSync(join(gstackHome, "config.yaml"), "transcript_ingest_mode: off\n");
|
||||
writeClaudeCodeSession(
|
||||
home,
|
||||
"current",
|
||||
"off-session",
|
||||
`${JSON.stringify({ type: "user", sessionId: "off-session", cwd: process.cwd(), timestamp: new Date().toISOString(), message: { role: "user", content: "private" } })}\n`,
|
||||
);
|
||||
const fake = installFakeGbrain(home);
|
||||
const r = runScript(["--bulk", "--quiet", "--sources", "transcript"], {
|
||||
HOME: home,
|
||||
GSTACK_HOME: gstackHome,
|
||||
PATH: `${fake.binDir}:${process.env.PATH}`,
|
||||
});
|
||||
expect(r.exitCode).toBe(0);
|
||||
expect(existsSync(fake.argsFile)).toBe(false);
|
||||
rmSync(home, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("imports only the trusted current repo into a dedicated transcript source", () => {
|
||||
const home = makeTestHome();
|
||||
const gstackHome = join(home, ".gstack");
|
||||
mkdirSync(gstackHome, { recursive: true });
|
||||
const currentRemote = spawnSync("git", ["remote", "get-url", "origin"], { encoding: "utf-8" }).stdout.trim();
|
||||
const currentKey = currentRemote.replace(/^[^:]+:\/\//, "").replace(/^[^@]+@/, "").replace(":", "/").replace(/\.git$/, "").toLowerCase();
|
||||
const repos = [
|
||||
["denied", "https://github.com/probe/denied.git"],
|
||||
["readonly", "https://github.com/probe/readonly.git"],
|
||||
["other", "https://github.com/probe/other.git"],
|
||||
];
|
||||
for (const [name, remote] of repos) {
|
||||
const cwd = join(home, name);
|
||||
mkdirSync(cwd, { recursive: true });
|
||||
spawnSync("git", ["init", "-q"], { cwd });
|
||||
spawnSync("git", ["remote", "add", "origin", remote], { cwd });
|
||||
writeClaudeCodeSession(home, name, `${name}-session`, `${JSON.stringify({ type: "user", sessionId: `${name}-session`, cwd, timestamp: new Date().toISOString(), message: { role: "user", content: name } })}\n`);
|
||||
}
|
||||
writeClaudeCodeSession(home, "current", "current-session", `${JSON.stringify({ type: "user", sessionId: "current-session", cwd: process.cwd(), timestamp: new Date().toISOString(), message: { role: "user", content: "current" } })}\n`);
|
||||
writeFileSync(join(gstackHome, "gbrain-repo-policy.json"), JSON.stringify({
|
||||
_schema_version: 2,
|
||||
[currentKey]: "read-write",
|
||||
"github.com/probe/denied": "deny",
|
||||
"github.com/probe/readonly": "read-only",
|
||||
"github.com/probe/other": "read-write",
|
||||
}));
|
||||
const fake = installFakeGbrain(home);
|
||||
const r = runScript(["--bulk", "--quiet", "--sources", "transcript"], {
|
||||
HOME: home,
|
||||
GSTACK_HOME: gstackHome,
|
||||
PATH: `${fake.binDir}:${process.env.PATH}`,
|
||||
});
|
||||
expect(r.exitCode).toBe(0);
|
||||
const stagedPages = readFileSync(fake.stagingListFile, "utf-8").trim().split("\n").filter((line) => line.endsWith(".md"));
|
||||
expect(stagedPages).toHaveLength(1);
|
||||
const args = readFileSync(fake.argsFile, "utf-8");
|
||||
expect(args).toContain("--source-id transcripts-");
|
||||
expect(args).not.toContain("--source-id default");
|
||||
const sourceId = args.match(/--source-id\s+(\S+)/)?.[1] || "";
|
||||
expect(sourceId).toMatch(/^[a-z0-9]+(?:-[a-z0-9]+)*$/);
|
||||
expect(sourceId.length).toBeLessThanOrEqual(32);
|
||||
rmSync(home, { recursive: true, force: true });
|
||||
});
|
||||
});
|
||||
|
|
|
|||
Loading…
Reference in New Issue