diff --git a/bin/gstack-memory-ingest.ts b/bin/gstack-memory-ingest.ts index 653d4069a..e407ed093 100644 --- a/bin/gstack-memory-ingest.ts +++ b/bin/gstack-memory-ingest.ts @@ -1782,6 +1782,49 @@ async function ingestPass(args: CliArgs): Promise { ); failed += failedSources.size; + // Reconcile gbrain's own accounting against what we staged. Without this, + // a batch that gbrain never SAW is indistinguishable from a batch that + // succeeded: readNewFailures() only reports PER-FILE failures, so when + // `gbrain import` collects zero files it writes nothing to + // sync-failures.jsonl, failedSources is empty, and every prepared file + // gets state-recorded as ingested. The pass then reports "N written" + // while the brain gained nothing — and because state now says "done", + // no future run retries. Silent, permanent data loss. + // + // Observed cause: `gbrain import` honours .gitignore, and + // `gstack-artifacts-init` writes `.gitignore = "*"` into $GSTACK_HOME. + // makeStagingDir() stages under $GSTACK_HOME, so on any machine that has + // run artifacts-init, collect_files returns 0 for every batch. + // + // `skipped` counts content_hash no-ops, which ARE successful landings. + const expectedLandings = prep.prepared.length - failedSources.size; + const accountedLandings = + (importJson.imported ?? 0) + (importJson.skipped ?? 0); + if (accountedLandings < expectedLandings) { + const collected = + importJson.total_files !== undefined + ? ` gbrain collected ${importJson.total_files} file(s) from the staging dir.` + : ""; + const msg = + `gbrain import accounted for ${accountedLandings} of ${expectedLandings} staged page(s) ` + + `(imported=${importJson.imported ?? 0}, unchanged=${importJson.skipped ?? 0}).${collected} ` + + `Refusing to advance state — the unaccounted pages would be marked ingested without ` + + `landing in the brain. If the count is 0, check whether ${stagingDir} is inside a git ` + + `repo that ignores it (gbrain import honours .gitignore).`; + console.error(`[memory-ingest] ERR: ${msg}`); + failed += prep.prepared.length; + return { + written: 0, + skipped_secret: prep.skippedSecret, + skipped_dedup: prep.skippedDedup, + skipped_unattributed: prep.skippedUnattributed, + failed, + duration_ms: Date.now() - t0, + partial_pages: prep.partialPages, + system_error: msg, + }; + } + // Phase 3: state recording. Only files that landed in gbrain get // their mtime+sha256 stamped. Failed source paths are deliberately // left un-state'd so the next run re-prepares them and gbrain's diff --git a/test/gstack-memory-ingest.test.ts b/test/gstack-memory-ingest.test.ts index fef9070c4..039beefad 100644 --- a/test/gstack-memory-ingest.test.ts +++ b/test/gstack-memory-ingest.test.ts @@ -330,7 +330,7 @@ describe("gstack-memory-ingest --limit", () => { */ function installFakeGbrain( home: string, - opts: { failingPaths?: string[] } = {}, + opts: { failingPaths?: string[]; collectNothing?: boolean } = {}, ): { binDir: string; logFile: string; argsFile: string; stagingListFile: string } { const binDir = join(home, "fake-bin"); mkdirSync(binDir, { recursive: true }); @@ -392,6 +392,13 @@ EOF else TOTAL=0 fi + # collectNothing: simulate gbrain walking the staging dir and finding + # nothing — the real-world shape when .gitignore hides every staged file + # from collect_files. Crucially this writes NO sync-failures.jsonl entry, + # because there is no per-file failure: gbrain never saw the files. + if [ "${opts.collectNothing ? "1" : "0"}" = "1" ]; then + TOTAL=0 + fi ERRORS=0 if [ -n "\$FAILING_LIST" ]; then ERRORS=\$(echo "\$FAILING_LIST" | tr '|' '\\n' | wc -l | tr -d ' ') @@ -470,6 +477,51 @@ describe("gstack-memory-ingest writer (gbrain v0.20+ batch `import` interface)", expect(stagedList).toMatch(/^\.\/transcripts\/claude-code\/.+\.md$/m); }); + // Silent-data-loss regression: gbrain accepts the import call, exits 0, and + // reports imported=0 because collect_files found nothing in the staging dir + // (real-world cause: gstack-artifacts-init writes `.gitignore = "*"` into + // $GSTACK_HOME, and `gbrain import` honours .gitignore, so every file staged + // under $GSTACK_HOME is invisible to it). + // + // No per-file failure is written to sync-failures.jsonl — gbrain never SAW + // the files — so readNewFailures returns empty. Before the reconciliation + // check, that made a total loss indistinguishable from success: every + // prepared file got state-recorded as ingested and the pass reported + // "N written". State then said "done", so no later run ever retried. + it("refuses to advance state when gbrain imports fewer pages than were staged", () => { + const home = makeTestHome(); + const gstackHome = join(home, ".gstack"); + mkdirSync(gstackHome, { recursive: true }); + const { binDir, logFile } = installFakeGbrain(home, { collectNothing: true }); + + const session = + `{"type":"user","message":{"role":"user","content":"hi"},"timestamp":"2026-05-01T00:00:00Z","cwd":"/tmp/foo"}\n` + + `{"type":"assistant","message":{"role":"assistant","content":"hello"},"timestamp":"2026-05-01T00:00:01Z"}\n`; + writeClaudeCodeSession(home, "tmp-foo", "abc123", session); + + const r = runScript(["--bulk", "--include-unattributed", "--quiet"], { + HOME: home, + GSTACK_HOME: gstackHome, + PATH: `${binDir}:${process.env.PATH || ""}`, + }); + + // gbrain WAS called — this is not a "gbrain missing" path. + expect(existsSync(logFile)).toBe(true); + + // The pass must not claim success. + expect(r.stderr).toMatch(/\[memory-ingest\] ERR:.*accounted for 0 of 1 staged page/); + expect(r.stderr).toMatch(/Refusing to advance state/); + expect(r.stdout).not.toMatch(/written:\s+1/); + + // The critical assertion: state must NOT mark the session ingested, or the + // next run skips it forever and the transcript is lost silently. + const statePath = join(gstackHome, ".transcript-ingest-state.json"); + if (existsSync(statePath)) { + const state = JSON.parse(readFileSync(statePath, "utf-8")); + expect(Object.keys(state.sessions || {}).length).toBe(0); + } + }); + // Originally landed in v1.32.0.0 (PR #1411) on the per-file `gbrain put` // path. Postgres rejects 0x00 in UTF-8 text columns. Some Claude Code // transcripts contain NUL inside user-pasted content or tool output. The