diff --git a/bin/gstack-memory-ingest.ts b/bin/gstack-memory-ingest.ts index 653d4069a..52047f5b4 100644 --- a/bin/gstack-memory-ingest.ts +++ b/bin/gstack-memory-ingest.ts @@ -598,33 +598,56 @@ function parseTranscriptJsonl(path: string): ParsedSession | null { let messageCount = 0; let toolCalls = 0; const bodyParts: string[] = []; + const seenMessages = new Set(); + const appendMessage = (role: string, content: string): void => { + const normalizedRole = (role || "user").toLowerCase(); + if (normalizedRole !== "user" && normalizedRole !== "assistant") return; + const dedupeKey = `${normalizedRole}\0${content}`; + if (seenMessages.has(dedupeKey)) return; + seenMessages.add(dedupeKey); + bodyParts.push(`## ${normalizedRole.charAt(0).toUpperCase() + normalizedRole.slice(1)}\n\n${content}`); + messageCount++; + }; + for (const rec of parsedLines) { + if (isCodex && rec?.payload) { + const payload = rec.payload; + if (payload?.type === "message" && payload.role) { + const content = extractContentText(payload); + if (content) appendMessage(payload.role, content); + } else if (payload?.type === "user_message" || payload?.type === "agent_message") { + const content = extractContentText(payload); + if (content) { + appendMessage(payload.type === "agent_message" ? "assistant" : "user", content); + } + } else if (payload?.type === "function_call") { + toolCalls++; + const tool = payload.name || "tool"; + bodyParts.push(`### Tool call: ${tool}`); + } else if (payload?.message) { + const msg = payload.message; + const role = msg.role || payload.role || "user"; + const content = extractContentText(msg); + if (content) appendMessage(role, content); + } + continue; + } + if (rec?.type === "user" || rec?.message?.role === "user") { const content = extractContentText(rec); if (content) { - bodyParts.push(`## User\n\n${content}`); - messageCount++; + appendMessage("user", content); } } else if (rec?.type === "assistant" || rec?.message?.role === "assistant") { const content = extractContentText(rec); if (content) { - bodyParts.push(`## Assistant\n\n${content}`); - messageCount++; + appendMessage("assistant", content); } } else if (rec?.type === "tool" || rec?.tool_use_id || rec?.tool_call) { toolCalls++; // Collapse to one-line summary const tool = rec?.name || rec?.tool || rec?.tool_call?.name || "tool"; bodyParts.push(`### Tool call: ${tool}`); - } else if (isCodex && rec?.payload?.message) { - // Codex shape: each record has payload.message - const msg = rec.payload.message; - const role = msg.role || "user"; - const content = extractContentText(msg); - if (content) { - bodyParts.push(`## ${role.charAt(0).toUpperCase() + role.slice(1)}\n\n${content}`); - messageCount++; - } } } @@ -645,18 +668,20 @@ function parseTranscriptJsonl(path: string): ParsedSession | null { function extractContentText(rec: any): string { if (!rec) return ""; + if (typeof rec === "string") return rec; if (typeof rec.content === "string") return rec.content; if (typeof rec.text === "string") return rec.text; + if (typeof rec.message === "string") return rec.message; if (typeof rec.message?.content === "string") return rec.message.content; if (Array.isArray(rec.message?.content)) { return rec.message.content - .map((c: any) => (typeof c === "string" ? c : c?.text || "")) + .map((c: any) => (typeof c === "string" ? c : c?.text || c?.content || "")) .filter(Boolean) .join("\n"); } if (Array.isArray(rec.content)) { return rec.content - .map((c: any) => (typeof c === "string" ? c : c?.text || "")) + .map((c: any) => (typeof c === "string" ? c : c?.text || c?.content || "")) .filter(Boolean) .join("\n"); } diff --git a/test/gstack-memory-ingest.test.ts b/test/gstack-memory-ingest.test.ts index fef9070c4..e1ed6543a 100644 --- a/test/gstack-memory-ingest.test.ts +++ b/test/gstack-memory-ingest.test.ts @@ -542,6 +542,105 @@ esac rmSync(home, { recursive: true, force: true }); }); + it("renders current Codex JSONL message and function-call records into the staged body", () => { + const home = makeTestHome(); + const gstackHome = join(home, ".gstack"); + mkdirSync(gstackHome, { recursive: true }); + + const binDir = join(home, "fake-bin"); + mkdirSync(binDir, { recursive: true }); + const stagingCopy = join(home, "staging-copy"); + const script = `#!/usr/bin/env bash +case "\${1:-}" in + --help|-h) echo "Usage: gbrain "; echo "Commands:"; echo " import Import"; exit 0 ;; + import) + DIR="\${2:-}" + cp -R "\$DIR" "${stagingCopy}" 2>/dev/null || true + if [[ " \$* " == *" --json "* ]]; then + echo '{"status":"success","duration_s":0.1,"imported":1,"skipped":0,"errors":0,"chunks":1,"total_files":1}' + fi + exit 0 ;; + *) echo "unknown"; exit 2 ;; +esac +`; + const binPath = join(binDir, "gbrain"); + writeFileSync(binPath, script, "utf-8"); + chmodSync(binPath, 0o755); + + const today = new Date(); + const ymd = `${today.getFullYear()}-${String(today.getMonth() + 1).padStart(2, "0")}-${String(today.getDate()).padStart(2, "0")}`; + const session = + `${JSON.stringify({ + type: "session_meta", + payload: { id: "019ecf48-4ec1-7d41-abb8-0b9dd51d4ffa", cwd: "/tmp/codex-current" }, + timestamp: "2026-06-16T07:15:20.277Z", + })}\n` + + `${JSON.stringify({ + type: "response_item", + payload: { + type: "message", + role: "developer", + content: [{ type: "input_text", text: "Do not ingest developer instructions into session memory." }], + }, + timestamp: "2026-06-16T07:15:21.000Z", + })}\n` + + `${JSON.stringify({ + type: "response_item", + payload: { + type: "message", + role: "user", + content: [{ type: "input_text", text: "how can I know if this session is sync in gbrain?" }], + }, + timestamp: "2026-06-16T08:48:49.152Z", + })}\n` + + `${JSON.stringify({ + type: "event_msg", + payload: { + type: "agent_message", + message: "The transcript page exists, but exact text search is not working yet.", + }, + timestamp: "2026-06-16T09:00:00.000Z", + })}\n` + + `${JSON.stringify({ + type: "response_item", + payload: { + type: "function_call", + name: "exec_command", + arguments: "{\"cmd\":\"gbrain search\"}", + call_id: "call_current", + }, + timestamp: "2026-06-16T09:00:01.000Z", + })}\n`; + writeCodexSession(home, ymd, session); + + const r = runScript(["--bulk", "--include-unattributed", "--quiet"], { + HOME: home, + GSTACK_HOME: gstackHome, + PATH: `${binDir}:${process.env.PATH || ""}`, + }); + expect(r.exitCode).toBe(0); + expect(existsSync(stagingCopy)).toBe(true); + + const findMd = spawnSync("find", [stagingCopy, "-name", "*.md", "-type", "f"], { + encoding: "utf-8", + }); + const mdPaths = (findMd.stdout || "").trim().split("\n").filter(Boolean); + expect(mdPaths.length).toBeGreaterThan(0); + const body = readFileSync(mdPaths[0], "utf-8"); + + expect(body).toContain("message_count: 2"); + expect(body).toContain("tool_calls: 1"); + expect(body).toContain("## User"); + expect(body).toContain("how can I know if this session is sync in gbrain?"); + expect(body).toContain("## Assistant"); + expect(body).toContain("exact text search is not working yet"); + expect(body).toContain("### Tool call: exec_command"); + expect(body).not.toContain("## Developer"); + expect(body).not.toContain("Do not ingest developer instructions"); + + rmSync(home, { recursive: true, force: true }); + }); + it("injects title/type/tags into the staged page's YAML frontmatter", () => { const home = makeTestHome(); const gstackHome = join(home, ".gstack");