From 17eaaaf73e419dcd0dab95045fc6e6d3f6998591 Mon Sep 17 00:00:00 2001 From: Luis Esteban Acevedo Ladino Date: Fri, 24 Apr 2026 07:25:04 -0500 Subject: [PATCH 1/2] feat: add native gemini provider --- apps/web/.env.example | 6 + apps/web/package.json | 1 + .../src/agent/__tests__/system-prompt.test.ts | 40 ++ .../agent/providers/__tests__/gemini.test.ts | 502 ++++++++++++++++++ .../agent/providers/__tests__/index.test.ts | 12 + apps/web/src/agent/providers/gemini.ts | 240 +++++++++ apps/web/src/agent/providers/index.ts | 3 + apps/web/src/agent/system-prompt.ts | 8 +- .../tools/__tests__/transcribe-video.test.ts | 140 ++++- .../src/agent/tools/transcribe-video.tool.ts | 35 +- bun.lock | 3 + .../archive-report.md | 59 ++ .../design.md | 155 ++++++ .../exploration.md | 0 .../proposal.md | 65 +++ .../specs/agent-session-shell/spec.md | 97 ++++ .../2026-04-24-gemini-adapter-fase-1/tasks.md | 28 + .../verify-report.md | 116 ++++ openspec/specs/agent-session-shell/spec.md | 68 ++- 19 files changed, 1572 insertions(+), 6 deletions(-) create mode 100644 apps/web/src/agent/providers/__tests__/gemini.test.ts create mode 100644 apps/web/src/agent/providers/gemini.ts create mode 100644 openspec/changes/archive/2026-04-24-gemini-adapter-fase-1/archive-report.md create mode 100644 openspec/changes/archive/2026-04-24-gemini-adapter-fase-1/design.md rename openspec/changes/{gemini-adapter-fase-1 => archive/2026-04-24-gemini-adapter-fase-1}/exploration.md (100%) create mode 100644 openspec/changes/archive/2026-04-24-gemini-adapter-fase-1/proposal.md create mode 100644 openspec/changes/archive/2026-04-24-gemini-adapter-fase-1/specs/agent-session-shell/spec.md create mode 100644 openspec/changes/archive/2026-04-24-gemini-adapter-fase-1/tasks.md create mode 100644 openspec/changes/archive/2026-04-24-gemini-adapter-fase-1/verify-report.md diff --git a/apps/web/.env.example b/apps/web/.env.example index 77ed9805..c5b16905 100644 --- a/apps/web/.env.example +++ b/apps/web/.env.example @@ -25,3 +25,9 @@ LLM_PROVIDER=openai-compatible LLM_API_KEY=your_openai_api_key_here LLM_MODEL=gpt-4o # LLM_BASE_URL=https://api.openai.com/v1 + +# Gemini provider (set LLM_PROVIDER=gemini to activate) +# LLM_PROVIDER=gemini +# LLM_API_KEY=your_gemini_api_key_here +# LLM_MODEL=gemini-2.0-flash +# Note: LLM_BASE_URL is ignored when using the gemini provider diff --git a/apps/web/package.json b/apps/web/package.json index 0b8a704d..b8f38950 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -42,6 +42,7 @@ "cmdk": "^1.0.0", "culori": "^4.0.2", "drizzle-orm": "^0.44.2", + "@google/generative-ai": "^0.24.0", "embla-carousel-react": "^8.5.1", "eventemitter3": "^5.0.1", "feed": "^5.1.0", diff --git a/apps/web/src/agent/__tests__/system-prompt.test.ts b/apps/web/src/agent/__tests__/system-prompt.test.ts index 5c83fe82..b20d2b2a 100644 --- a/apps/web/src/agent/__tests__/system-prompt.test.ts +++ b/apps/web/src/agent/__tests__/system-prompt.test.ts @@ -37,6 +37,46 @@ describe("buildSystemPrompt", () => { expect(prompt).toContain("NeuralCut video editor"); }); + test("includes asset internal id in media listing", () => { + const context: AgentContext = { + projectId: "proj-1", + activeSceneId: "scene-A", + mediaAssets: [ + { id: "m1", name: "intro.mp4", type: "video", duration: 30 }, + ], + playbackTimeMs: 0, + }; + + const prompt = buildSystemPrompt(context); + + expect(prompt).toContain("[id: m1]"); + }); + + test("includes assetId instruction when media assets are present", () => { + const context: AgentContext = { + projectId: "proj-1", + activeSceneId: "scene-A", + mediaAssets: [ + { id: "m1", name: "conclu.mov", type: "video", duration: 60 }, + ], + playbackTimeMs: 0, + }; + + const prompt = buildSystemPrompt(context); + + expect(prompt).toContain("IMPORTANT"); + expect(prompt).toContain("assetId"); + expect(prompt).toContain('internal "id" value'); + expect(prompt).toContain("NOT the filename"); + }); + + test("omits assetId instruction when no media assets", () => { + const prompt = buildSystemPrompt(BASE_CONTEXT); + + expect(prompt).not.toContain("IMPORTANT"); + expect(prompt).not.toContain("assetId"); + }); + test("is valid when no media assets are loaded", () => { const context: AgentContext = { projectId: "proj-2", diff --git a/apps/web/src/agent/providers/__tests__/gemini.test.ts b/apps/web/src/agent/providers/__tests__/gemini.test.ts new file mode 100644 index 00000000..8e5cd3be --- /dev/null +++ b/apps/web/src/agent/providers/__tests__/gemini.test.ts @@ -0,0 +1,502 @@ +import { beforeEach, describe, expect, jest, test } from "bun:test"; +import { SchemaType, type GenerateContentResult } from "@google/generative-ai"; +import type { ProviderConfig } from "@/agent/providers/types"; +import type { ToolSchema } from "@/agent/types"; + +// --------------------------------------------------------------------------- +// Mock the Google Generative AI SDK before the adapter module loads +// --------------------------------------------------------------------------- + +const mockGenerateContent = jest.fn(); + +jest.mock("@google/generative-ai", () => ({ + GoogleGenerativeAI: class MockGoogleGenerativeAI { + getGenerativeModel(_opts: { model: string }) { + return { generateContent: mockGenerateContent }; + } + constructor(_opts: { apiKey: string }) {} + }, + SchemaType: { + STRING: "string", + NUMBER: "number", + BOOLEAN: "boolean", + OBJECT: "object", + }, +})); + +// Import after mock is set up +import { + GeminiAdapter, + toGeminiContents, + toGeminiTools, + fromGeminiResponse, +} from "@/agent/providers/gemini"; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +const TEST_CONFIG: ProviderConfig = { + provider: "gemini", + apiKey: "test-gemini-key", + model: "gemini-2.0-flash", +}; + +const SAMPLE_TOOLS: ToolSchema[] = [ + { + name: "transcribe_video", + description: "Transcribes a video asset", + parameters: [ + { key: "assetId", type: "string", required: true }, + { key: "language", type: "string", required: false }, + ], + }, +]; + +function makeGeminiResponse(overrides: { + text?: string; + functionCalls?: Array<{ name: string; args: Record }>; +}): GenerateContentResult { + const parts: Array> = []; + + if (overrides.text !== undefined) { + parts.push({ text: overrides.text }); + } + + if (overrides.functionCalls) { + for (const fc of overrides.functionCalls) { + parts.push({ functionCall: { name: fc.name, args: fc.args } }); + } + } + + return { + response: { + candidates: [ + { + content: { parts }, + }, + ], + }, + } as unknown as GenerateContentResult; +} + +function makeChatMessage( + role: "user" | "assistant" | "system" | "tool_result", + content: string, + extra?: Record, +) { + return { + id: `msg-${Math.random().toString(36).slice(2)}`, + role, + content, + timestamp: Date.now(), + ...extra, + }; +} + +// --------------------------------------------------------------------------- +// Tests: toGeminiContents (Task 4.1) +// --------------------------------------------------------------------------- + +describe("toGeminiContents", () => { + test("maps user message to Gemini user role", () => { + const messages = [makeChatMessage("user", "Hello")]; + const contents = toGeminiContents(messages); + + expect(contents).toHaveLength(1); + expect(contents[0].role).toBe("user"); + expect(contents[0].parts).toEqual([{ text: "Hello" }]); + }); + + test("maps assistant message to Gemini model role", () => { + const messages = [makeChatMessage("assistant", "Hi there!")]; + const contents = toGeminiContents(messages); + + expect(contents).toHaveLength(1); + expect(contents[0].role).toBe("model"); + expect(contents[0].parts).toEqual([{ text: "Hi there!" }]); + }); + + test("maps assistant with toolCalls to model role with functionCall parts", () => { + const messages = [ + makeChatMessage("assistant", "Let me check.", { + toolCalls: [ + { + id: "tc_1", + name: "transcribe_video", + args: { assetId: "vid-123" }, + }, + ], + }), + ]; + const contents = toGeminiContents(messages); + + expect(contents).toHaveLength(1); + expect(contents[0].role).toBe("model"); + expect(contents[0].parts).toEqual([ + { text: "Let me check." }, + { + functionCall: { + name: "transcribe_video", + args: { assetId: "vid-123" }, + }, + }, + ]); + }); + + test("maps tool_result to function role with functionResponse", () => { + const messages = [ + makeChatMessage("assistant", "", { + toolCalls: [ + { + id: "tc_1", + name: "transcribe_video", + args: { assetId: "vid-123" }, + }, + ], + }), + makeChatMessage("tool_result", '{"transcript": "Hello world"}', { + toolCallId: "tc_1", + }), + ]; + const contents = toGeminiContents(messages); + + // First: model (assistant with toolCalls) + expect(contents[0].role).toBe("model"); + // Second: function (tool_result) + expect(contents[1].role).toBe("function"); + expect(contents[1].parts).toEqual([ + { + functionResponse: { + name: "transcribe_video", + response: { transcript: "Hello world" }, + }, + }, + ]); + }); + + test("excludes system messages from contents", () => { + const messages = [ + makeChatMessage("system", "You are a video editor."), + makeChatMessage("user", "Hello"), + ]; + const contents = toGeminiContents(messages); + + expect(contents).toHaveLength(1); + expect(contents[0].role).toBe("user"); + }); + + test("handles tool_result with non-JSON content by wrapping in object", () => { + const messages = [ + makeChatMessage("assistant", "", { + toolCalls: [ + { id: "tc_1", name: "some_tool", args: {} }, + ], + }), + makeChatMessage("tool_result", "plain text result", { + toolCallId: "tc_1", + }), + ]; + const contents = toGeminiContents(messages); + + expect(contents[1].parts).toEqual([ + { + functionResponse: { + name: "some_tool", + response: { result: "plain text result" }, + }, + }, + ]); + }); +}); + +// --------------------------------------------------------------------------- +// Tests: toGeminiTools (Task 4.2) +// --------------------------------------------------------------------------- + +describe("toGeminiTools", () => { + test("converts a single tool with parameters", () => { + const declarations = toGeminiTools(SAMPLE_TOOLS); + + expect(declarations).toHaveLength(1); + expect(declarations[0].name).toBe("transcribe_video"); + expect(declarations[0].description).toBe("Transcribes a video asset"); + expect(declarations[0].parameters?.type).toBe(SchemaType.OBJECT); + expect(declarations[0].parameters?.properties).toHaveProperty("assetId"); + expect(declarations[0].parameters?.required).toEqual(["assetId"]); + }); + + test("maps type strings to Gemini SchemaType values", () => { + const tools: ToolSchema[] = [ + { + name: "multi_param_tool", + description: "Tool with all param types", + parameters: [ + { key: "name", type: "string", required: true }, + { key: "count", type: "number", required: false }, + { key: "active", type: "boolean", required: false }, + { key: "meta", type: "object", required: false }, + ], + }, + ]; + + const declarations = toGeminiTools(tools); + const props = declarations[0].parameters?.properties as Record< + string, + { type: string } + >; + + expect(props.name.type).toBe(SchemaType.STRING); + expect(props.count.type).toBe(SchemaType.NUMBER); + expect(props.active.type).toBe(SchemaType.BOOLEAN); + expect(props.meta.type).toBe(SchemaType.OBJECT); + }); + + test("returns empty array when no tools provided", () => { + expect(toGeminiTools([])).toEqual([]); + }); +}); + +// --------------------------------------------------------------------------- +// Tests: fromGeminiResponse (Task 4.3) +// --------------------------------------------------------------------------- + +describe("fromGeminiResponse", () => { + test("extracts text-only response with no toolCalls", () => { + const response = makeGeminiResponse({ text: "Hello from Gemini!" }); + const result = fromGeminiResponse(response); + + expect(result.content).toBe("Hello from Gemini!"); + expect(result.toolCalls).toBeUndefined(); + }); + + test("maps function-call response to toolCalls with non-empty synthesized IDs", () => { + const response = makeGeminiResponse({ + text: undefined, + functionCalls: [ + { name: "transcribe_video", args: { assetId: "vid-1" } }, + ], + }); + const result = fromGeminiResponse(response); + + expect(result.content).toBe(""); + expect(result.toolCalls).toHaveLength(1); + expect(result.toolCalls?.[0].name).toBe("transcribe_video"); + expect(result.toolCalls?.[0].args).toEqual({ assetId: "vid-1" }); + // Non-empty synthesized ID + expect(result.toolCalls?.[0].id).toBeTruthy(); + expect(typeof result.toolCalls?.[0].id).toBe("string"); + }); + + test("two calls produce different IDs", () => { + const response1 = makeGeminiResponse({ + functionCalls: [{ name: "tool_a", args: {} }], + }); + const response2 = makeGeminiResponse({ + functionCalls: [{ name: "tool_a", args: {} }], + }); + + const result1 = fromGeminiResponse(response1); + const result2 = fromGeminiResponse(response2); + + expect(result1.toolCalls?.[0].id).not.toBe( + result2.toolCalls?.[0].id, + ); + }); + + test("handles mixed text and function-call parts", () => { + const response = makeGeminiResponse({ + text: "I'll look that up.", + functionCalls: [{ name: "search", args: { q: "test" } }], + }); + const result = fromGeminiResponse(response); + + expect(result.content).toBe("I'll look that up."); + expect(result.toolCalls).toHaveLength(1); + }); +}); + +// --------------------------------------------------------------------------- +// Tests: error paths (Task 4.4) +// --------------------------------------------------------------------------- + +describe("error paths", () => { + test("throws on empty candidates", () => { + const response = { + response: { candidates: [] }, + }; + + expect(() => fromGeminiResponse(response as any)).toThrow( + "Empty response from Gemini provider", + ); + }); + + test("throws on null candidates", () => { + const response = { + response: { candidates: null }, + }; + + expect(() => fromGeminiResponse(response as any)).toThrow( + "Empty response from Gemini provider", + ); + }); + + test("SDK error propagates from chat()", async () => { + mockGenerateContent.mockRejectedValueOnce( + new Error("API key not valid"), + ); + + const adapter = new GeminiAdapter(TEST_CONFIG); + + expect( + adapter.chat({ + messages: [makeChatMessage("user", "Hello")], + systemPrompt: "", + tools: [], + }), + ).rejects.toThrow("API key not valid"); + }); +}); + +// --------------------------------------------------------------------------- +// Tests: full chat() round-trip (Task 4.6) +// --------------------------------------------------------------------------- + +describe("GeminiAdapter.chat()", () => { + beforeEach(() => { + mockGenerateContent.mockReset(); + }); + + test("calls generateContent with correct contents and no tools", async () => { + mockGenerateContent.mockResolvedValueOnce( + makeGeminiResponse({ text: "Hi!" }), + ); + + const adapter = new GeminiAdapter(TEST_CONFIG); + const result = await adapter.chat({ + messages: [makeChatMessage("user", "Hello")], + systemPrompt: "You are helpful.", + tools: [], + }); + + expect(result.content).toBe("Hi!"); + expect(result.toolCalls).toBeUndefined(); + + const callArgs = mockGenerateContent.mock.calls[0][0]; + expect(callArgs.contents).toHaveLength(1); + expect(callArgs.contents[0].role).toBe("user"); + expect(callArgs.systemInstruction).toBe("You are helpful."); + expect(callArgs.tools).toBeUndefined(); + }); + + test("calls generateContent with tools and systemInstruction", async () => { + mockGenerateContent.mockResolvedValueOnce( + makeGeminiResponse({ + functionCalls: [ + { + name: "transcribe_video", + args: { assetId: "vid-1" }, + }, + ], + }), + ); + + const adapter = new GeminiAdapter(TEST_CONFIG); + const result = await adapter.chat({ + messages: [makeChatMessage("user", "Transcribe this")], + systemPrompt: "Video editing assistant", + tools: SAMPLE_TOOLS, + }); + + expect(result.toolCalls).toHaveLength(1); + expect(result.toolCalls?.[0].name).toBe("transcribe_video"); + + const callArgs = mockGenerateContent.mock.calls[0][0]; + expect(callArgs.systemInstruction).toBe("Video editing assistant"); + expect(callArgs.tools).toHaveLength(1); + expect(callArgs.tools[0].functionDeclarations).toHaveLength(1); + }); + + test("passes multi-turn conversation through to Gemini", async () => { + mockGenerateContent.mockResolvedValueOnce( + makeGeminiResponse({ text: "Sure!" }), + ); + + const adapter = new GeminiAdapter(TEST_CONFIG); + await adapter.chat({ + messages: [ + makeChatMessage("user", "First question"), + makeChatMessage("assistant", "First answer"), + makeChatMessage("user", "Second question"), + ], + systemPrompt: "", + tools: [], + }); + + const callArgs = mockGenerateContent.mock.calls[0][0]; + expect(callArgs.contents).toHaveLength(3); + expect(callArgs.contents[0].role).toBe("user"); + expect(callArgs.contents[1].role).toBe("model"); + expect(callArgs.contents[2].role).toBe("user"); + }); + + test("does not include systemInstruction when systemPrompt is empty", async () => { + mockGenerateContent.mockResolvedValueOnce( + makeGeminiResponse({ text: "OK" }), + ); + + const adapter = new GeminiAdapter(TEST_CONFIG); + await adapter.chat({ + messages: [makeChatMessage("user", "Hi")], + systemPrompt: "", + tools: [], + }); + + const callArgs = mockGenerateContent.mock.calls[0][0]; + expect(callArgs.systemInstruction).toBeUndefined(); + }); + + test("ignores baseUrl in config — Gemini still works", async () => { + mockGenerateContent.mockResolvedValueOnce( + makeGeminiResponse({ text: "Hello!" }), + ); + + const configWithBaseUrl: ProviderConfig = { + provider: "gemini", + apiKey: "test-gemini-key", + model: "gemini-2.0-flash", + baseUrl: "https://should-be-ignored.example.com", + }; + + const adapter = new GeminiAdapter(configWithBaseUrl); + const result = await adapter.chat({ + messages: [makeChatMessage("user", "Hi")], + systemPrompt: "", + tools: [], + }); + + expect(result.content).toBe("Hello!"); + expect(result.toolCalls).toBeUndefined(); + }); + + test("returns text when tools are provided but Gemini responds with text only", async () => { + mockGenerateContent.mockResolvedValueOnce( + makeGeminiResponse({ text: "I don't need any tools for this." }), + ); + + const adapter = new GeminiAdapter(TEST_CONFIG); + const result = await adapter.chat({ + messages: [makeChatMessage("user", "Just chat")], + systemPrompt: "You are helpful.", + tools: SAMPLE_TOOLS, + }); + + expect(result.content).toBe("I don't need any tools for this."); + expect(result.toolCalls).toBeUndefined(); + + // Verify tools were still sent to Gemini + const callArgs = mockGenerateContent.mock.calls[0][0]; + expect(callArgs.tools).toBeDefined(); + expect(callArgs.tools[0].functionDeclarations).toHaveLength(1); + }); +}); diff --git a/apps/web/src/agent/providers/__tests__/index.test.ts b/apps/web/src/agent/providers/__tests__/index.test.ts index 913475a5..73478897 100644 --- a/apps/web/src/agent/providers/__tests__/index.test.ts +++ b/apps/web/src/agent/providers/__tests__/index.test.ts @@ -1,6 +1,7 @@ import { describe, expect, test } from "bun:test"; import { createProvider } from "@/agent/providers/index"; import { OpenAICompatibleAdapter } from "@/agent/providers/openai-compatible"; +import { GeminiAdapter } from "@/agent/providers/gemini"; import type { ProviderConfig } from "@/agent/providers/types"; describe("createProvider", () => { @@ -28,6 +29,17 @@ describe("createProvider", () => { // Adapter was constructed without throwing — config accepted }); + test("returns GeminiAdapter for gemini provider", () => { + const config: ProviderConfig = { + provider: "gemini", + apiKey: "test-gemini-key", + model: "gemini-2.0-flash", + }; + + const adapter = createProvider(config); + expect(adapter).toBeInstanceOf(GeminiAdapter); + }); + test("throws Error for unknown provider", () => { const config: ProviderConfig = { provider: "unknown-provider", diff --git a/apps/web/src/agent/providers/gemini.ts b/apps/web/src/agent/providers/gemini.ts new file mode 100644 index 00000000..33961e3e --- /dev/null +++ b/apps/web/src/agent/providers/gemini.ts @@ -0,0 +1,240 @@ +import { + GoogleGenerativeAI, + type Content, + type FunctionDeclaration, + type FunctionDeclarationSchema, + type GenerateContentResult, + SchemaType, +} from "@google/generative-ai"; +import type { ChatMessage, ToolCall, ToolSchema } from "@/agent/types"; +import type { + ProviderAdapter, + ProviderConfig, + ProviderResponse, +} from "./types"; + +// --------------------------------------------------------------------------- +// Internal conversion helpers +// --------------------------------------------------------------------------- + +/** + * Maps internal type strings to Gemini SchemaType values. + */ +const TYPE_MAP: Record = { + string: SchemaType.STRING, + number: SchemaType.NUMBER, + boolean: SchemaType.BOOLEAN, + object: SchemaType.OBJECT, +}; + +/** + * Converts internal ChatMessages into Gemini Content[]. + * + * - `user` → role "user", text part + * - `assistant` → role "model", text part + functionCall parts from toolCalls + * - `tool_result`→ role "function", functionResponse part + * - `system` → skipped (handled separately via systemInstruction) + */ +export function toGeminiContents(messages: ChatMessage[]): Content[] { + const contents: Content[] = []; + + for (const msg of messages) { + if (msg.role === "system") { + continue; + } + + if (msg.role === "user") { + contents.push({ + role: "user", + parts: [{ text: msg.content }], + }); + } else if (msg.role === "assistant") { + const parts: Content["parts"] = []; + + if (msg.content) { + parts.push({ text: msg.content }); + } + + if (msg.toolCalls && msg.toolCalls.length > 0) { + for (const tc of msg.toolCalls) { + parts.push({ + functionCall: { name: tc.name, args: tc.args }, + }); + } + } + + contents.push({ role: "model", parts }); + } else if (msg.role === "tool_result") { + // Gemini uses the function name to correlate responses, not IDs. + // We parse the content as JSON for the response object; fall back + // to wrapping it if it's not valid JSON. + let responseData: object; + try { + responseData = JSON.parse(msg.content) as object; + } catch { + responseData = { result: msg.content }; + } + + // Derive the function name from the toolCallId → find the matching + // tool name in prior messages. If not found, use a placeholder. + const toolName = findToolNameForResult(messages, msg.toolCallId); + + contents.push({ + role: "function", + parts: [ + { + functionResponse: { + name: toolName, + response: responseData, + }, + }, + ], + }); + } + } + + return contents; +} + +/** + * Walks backward through messages to find the function name associated with + * a toolCallId. This is needed because Gemini's functionResponse uses names, + * not IDs. + */ +function findToolNameForResult( + messages: ChatMessage[], + toolCallId?: string, +): string { + if (!toolCallId) return "unknown"; + + for (let i = messages.length - 1; i >= 0; i--) { + const msg = messages[i]; + if (msg.role === "assistant" && msg.toolCalls) { + const match = msg.toolCalls.find((tc) => tc.id === toolCallId); + if (match) return match.name; + } + } + + return "unknown"; +} + +/** + * Converts internal ToolSchema[] into Gemini FunctionDeclaration[]. + * + * Maps internal type strings ("string", "number", etc.) to Gemini SchemaType + * enum values. Returns an empty array when no tools are provided. + */ +export function toGeminiTools(tools: ToolSchema[]): FunctionDeclaration[] { + if (tools.length === 0) return []; + + return tools.map((tool) => { + const properties: FunctionDeclarationSchema["properties"] = {}; + + for (const param of tool.parameters) { + const geminiType = TYPE_MAP[param.type] ?? SchemaType.STRING; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (properties as Record)[param.key] = { type: geminiType }; + } + + const requiredParams = tool.parameters + .filter((p) => p.required) + .map((p) => p.key); + + return { + name: tool.name, + description: tool.description, + parameters: { + type: SchemaType.OBJECT, + properties, + ...(requiredParams.length > 0 && { required: requiredParams }), + }, + }; + }); +} + +/** + * Converts a Gemini GenerateContentResult into the provider-agnostic + * ProviderResponse shape. + * + * - Text parts are concatenated into `content`. + * - FunctionCall parts are mapped to `toolCalls[]` with synthesized UUIDs. + * - Throws if no candidates are present. + */ +export function fromGeminiResponse( + response: GenerateContentResult, +): ProviderResponse { + const candidates = response.response.candidates; + if (!candidates || candidates.length === 0) { + throw new Error("Empty response from Gemini provider"); + } + + const parts = candidates[0].content?.parts ?? []; + const textParts: string[] = []; + const toolCalls: ToolCall[] = []; + + for (const part of parts) { + if ("text" in part && part.text !== undefined) { + textParts.push(part.text); + } + if ("functionCall" in part && part.functionCall) { + toolCalls.push({ + id: crypto.randomUUID(), + name: part.functionCall.name, + args: (part.functionCall.args as Record) ?? {}, + }); + } + } + + const content = textParts.join(""); + const result: ProviderResponse = { content }; + + if (toolCalls.length > 0) { + result.toolCalls = toolCalls; + } + + return result; +} + +// --------------------------------------------------------------------------- +// Adapter implementation +// --------------------------------------------------------------------------- + +/** + * Gemini adapter using @google/generative-ai SDK. + * + * Uses the stateless `generateContent()` API — full history is sent with each + * call, matching the existing route contract. System prompts are mapped to + * Gemini's `systemInstruction` field, not as user messages. + */ +export class GeminiAdapter implements ProviderAdapter { + private model; + + constructor(config: ProviderConfig) { + const genAI = new GoogleGenerativeAI(config.apiKey); + this.model = genAI.getGenerativeModel({ model: config.model }); + // config.baseUrl is intentionally ignored for the Gemini provider. + } + + async chat(params: { + messages: ChatMessage[]; + systemPrompt: string; + tools: ToolSchema[]; + }): Promise { + const contents = toGeminiContents(params.messages); + const functionDeclarations = toGeminiTools(params.tools); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const request: any = { contents }; + + if (params.systemPrompt) { + request.systemInstruction = params.systemPrompt; + } + + if (functionDeclarations.length > 0) { + request.tools = [{ functionDeclarations }]; + } + + const result = await this.model.generateContent(request); + return fromGeminiResponse(result); + } +} diff --git a/apps/web/src/agent/providers/index.ts b/apps/web/src/agent/providers/index.ts index 5bbb2cdb..8939f6ad 100644 --- a/apps/web/src/agent/providers/index.ts +++ b/apps/web/src/agent/providers/index.ts @@ -1,5 +1,6 @@ import type { ProviderAdapter, ProviderConfig } from "./types"; import { OpenAICompatibleAdapter } from "./openai-compatible"; +import { GeminiAdapter } from "./gemini"; /** * Factory — resolves a ProviderConfig to a concrete adapter. @@ -12,6 +13,8 @@ export function createProvider(config: ProviderConfig): ProviderAdapter { switch (config.provider) { case "openai-compatible": return new OpenAICompatibleAdapter(config); + case "gemini": + return new GeminiAdapter(config); default: throw new Error(`Unknown LLM provider: ${config.provider}`); } diff --git a/apps/web/src/agent/system-prompt.ts b/apps/web/src/agent/system-prompt.ts index 26b4229c..37407700 100644 --- a/apps/web/src/agent/system-prompt.ts +++ b/apps/web/src/agent/system-prompt.ts @@ -17,7 +17,7 @@ export function buildSystemPrompt( ): string { const mediaSection = context.mediaAssets.length > 0 - ? `Active media assets:\n${context.mediaAssets.map((m) => `- ${m.name} (${m.type}, ${m.duration}s)`).join("\n")}` + ? `Active media assets:\n${context.mediaAssets.map((m) => `- [id: ${m.id}] ${m.name} (${m.type}, ${m.duration}s)`).join("\n")}` : "No media assets loaded."; const parts = [ @@ -28,6 +28,12 @@ export function buildSystemPrompt( mediaSection, ]; + if (context.mediaAssets.length > 0) { + parts.push( + 'IMPORTANT: When calling tools that accept an "assetId" parameter, always use the internal "id" value (e.g., "v1"), NOT the filename or display name.', + ); + } + if (tools && tools.length > 0) { const toolList = tools .map((t) => `- ${t.name}: ${t.description}`) diff --git a/apps/web/src/agent/tools/__tests__/transcribe-video.test.ts b/apps/web/src/agent/tools/__tests__/transcribe-video.test.ts index af52caea..9b1c474d 100644 --- a/apps/web/src/agent/tools/__tests__/transcribe-video.test.ts +++ b/apps/web/src/agent/tools/__tests__/transcribe-video.test.ts @@ -227,7 +227,7 @@ describe("transcribe_video tool — error branches", () => { const result = await tool.execute({ assetId: "v99" }, context); expect(result).toEqual({ - error: "Could not access file for asset v99", + error: 'No asset found with id or name "v99". Available ids: [v1]. Available names: [intro.mp4].', }); }); @@ -282,3 +282,141 @@ describe("transcribe_video tool — error branches", () => { expect(result).toEqual({ error: "Worker not initialized" }); }); }); + +describe("transcribe_video tool — name-based assetId fallback", () => { + beforeEach(() => { + mockResolveAssetFile.mockClear(); + mockGetAssetHasAudio.mockClear(); + mockDecodeAudio.mockClear(); + mockTranscribe.mockClear(); + }); + + test("resolves by filename when no id match (exact name)", async () => { + const fakeFile = new File([], "conclu.mov", { type: "video/quicktime" }); + const context = makeContext({ + mediaAssets: [ + fakeVideoAsset({ id: "abc123", name: "conclu.mov" }), + ], + }); + + mockResolveAssetFile.mockReturnValue(fakeFile); + mockGetAssetHasAudio.mockReturnValue(true); + mockDecodeAudio.mockResolvedValue({ + samples: new Float32Array(16000), + sampleRate: 16000, + }); + mockTranscribe.mockResolvedValue(fakeTranscriptionResult()); + + const tool = toolRegistry.get("transcribe_video"); + const result = await tool.execute({ assetId: "conclu.mov" }, context); + + expect(result).toEqual( + expect.objectContaining({ assetName: "conclu.mov" }), + ); + // Resolved asset must pass its internal id to the adapter + expect(mockResolveAssetFile).toHaveBeenCalledWith("abc123"); + expect(mockGetAssetHasAudio).toHaveBeenCalledWith("abc123"); + }); + + test("resolves by case-insensitive name when no exact id or name match", async () => { + const fakeFile = new File([], "Conclu.MOV", { type: "video/quicktime" }); + const context = makeContext({ + mediaAssets: [ + fakeVideoAsset({ id: "abc123", name: "Conclu.MOV" }), + ], + }); + + mockResolveAssetFile.mockReturnValue(fakeFile); + mockGetAssetHasAudio.mockReturnValue(true); + mockDecodeAudio.mockResolvedValue({ + samples: new Float32Array(16000), + sampleRate: 16000, + }); + mockTranscribe.mockResolvedValue(fakeTranscriptionResult()); + + const tool = toolRegistry.get("transcribe_video"); + const result = await tool.execute({ assetId: "conclu.mov" }, context); + + expect(result).toEqual( + expect.objectContaining({ assetName: "Conclu.MOV" }), + ); + expect(mockResolveAssetFile).toHaveBeenCalledWith("abc123"); + }); + + test("returns ambiguous error when two assets share same name", async () => { + const context = makeContext({ + mediaAssets: [ + fakeVideoAsset({ id: "v1", name: "clip.mp4" }), + fakeVideoAsset({ id: "v2", name: "clip.mp4" }), + ], + }); + + const tool = toolRegistry.get("transcribe_video"); + const result = await tool.execute({ assetId: "clip.mp4" }, context); + + expect(result).toEqual({ + error: 'Ambiguous asset name "clip.mp4" matches multiple assets (clip.mp4, clip.mp4). Specify the internal id: v1, v2.', + }); + }); + + test("returns ambiguous error for case-insensitive duplicates", async () => { + const context = makeContext({ + mediaAssets: [ + fakeVideoAsset({ id: "v1", name: "Clip.MP4" }), + fakeVideoAsset({ id: "v2", name: "CLIP.MP4" }), + ], + }); + + const tool = toolRegistry.get("transcribe_video"); + const result = await tool.execute({ assetId: "clip.mp4" }, context); + + expect(result).toEqual({ + error: 'Ambiguous asset name "clip.mp4" matches multiple assets (Clip.MP4, CLIP.MP4). Specify the internal id: v1, v2.', + }); + }); + + test("prefers exact id match over name match", async () => { + const fakeFile = new File([], "intro.mp4", { type: "video/mp4" }); + // An asset whose id happens to equal another asset's name + const context = makeContext({ + mediaAssets: [ + fakeVideoAsset({ id: "v1", name: "intro.mp4" }), + fakeVideoAsset({ id: "intro.mp4", name: "outro.mp4" }), + ], + }); + + mockResolveAssetFile.mockReturnValue(fakeFile); + mockGetAssetHasAudio.mockReturnValue(true); + mockDecodeAudio.mockResolvedValue({ + samples: new Float32Array(16000), + sampleRate: 16000, + }); + mockTranscribe.mockResolvedValue(fakeTranscriptionResult()); + + const tool = toolRegistry.get("transcribe_video"); + const result = await tool.execute({ assetId: "intro.mp4" }, context); + + // "intro.mp4" matches v1 by id AND the second asset by id exactly + // id lookup is first, so it resolves the second asset (id="intro.mp4") + expect(result).toEqual( + expect.objectContaining({ assetName: "outro.mp4" }), + ); + expect(mockResolveAssetFile).toHaveBeenCalledWith("intro.mp4"); + }); + + test("returns not-found error with available ids and names for totally unknown assetId", async () => { + const context = makeContext({ + mediaAssets: [ + fakeVideoAsset({ id: "v1", name: "intro.mp4" }), + fakeVideoAsset({ id: "v2", name: "outro.mp4" }), + ], + }); + + const tool = toolRegistry.get("transcribe_video"); + const result = await tool.execute({ assetId: "nonexistent.mov" }, context); + + expect(result).toEqual({ + error: 'No asset found with id or name "nonexistent.mov". Available ids: [v1, v2]. Available names: [intro.mp4, outro.mp4].', + }); + }); +}); diff --git a/apps/web/src/agent/tools/transcribe-video.tool.ts b/apps/web/src/agent/tools/transcribe-video.tool.ts index 805cd8f0..e121e7ae 100644 --- a/apps/web/src/agent/tools/transcribe-video.tool.ts +++ b/apps/web/src/agent/tools/transcribe-video.tool.ts @@ -57,10 +57,43 @@ const transcribeVideoTool: ToolDefinition = { let targetAsset: (typeof mediaCandidates)[number] | undefined; if (typedArgs.assetId) { + // 1. Exact internal-id match targetAsset = mediaCandidates.find((a) => a.id === typedArgs.assetId); + + // 2. Fallback: exact name match (LLM sometimes passes filename) if (!targetAsset) { + const nameMatches = mediaCandidates.filter( + (a) => a.name === typedArgs.assetId, + ); + if (nameMatches.length === 1) { + targetAsset = nameMatches[0]; + } else if (nameMatches.length > 1) { + return { + error: `Ambiguous asset name "${typedArgs.assetId}" matches multiple assets (${nameMatches.map((a) => a.name).join(", ")}). Specify the internal id: ${nameMatches.map((a) => `${a.id}`).join(", ")}.`, + }; + } + } + + // 3. Case-insensitive fallback (only if unambiguous) + if (!targetAsset) { + const lower = typedArgs.assetId.toLowerCase(); + const ciMatches = mediaCandidates.filter( + (a) => a.name.toLowerCase() === lower, + ); + if (ciMatches.length === 1) { + targetAsset = ciMatches[0]; + } else if (ciMatches.length > 1) { + return { + error: `Ambiguous asset name "${typedArgs.assetId}" matches multiple assets (${ciMatches.map((a) => a.name).join(", ")}). Specify the internal id: ${ciMatches.map((a) => `${a.id}`).join(", ")}.`, + }; + } + } + + if (!targetAsset) { + const ids = mediaCandidates.map((a) => a.id).join(", "); + const names = mediaCandidates.map((a) => a.name).join(", "); return { - error: `Could not access file for asset ${typedArgs.assetId}`, + error: `No asset found with id or name "${typedArgs.assetId}". Available ids: [${ids}]. Available names: [${names}].`, }; } } else { diff --git a/bun.lock b/bun.lock index a39f2c2f..f1dd7d86 100644 --- a/bun.lock +++ b/bun.lock @@ -20,6 +20,7 @@ "name": "@opencut/web", "version": "0.1.0", "dependencies": { + "@google/generative-ai": "^0.24.0", "@hello-pangea/dnd": "^18.0.1", "@hugeicons/core-free-icons": "^3.3.0", "@hugeicons/react": "^1.1.6", @@ -348,6 +349,8 @@ "@floating-ui/utils": ["@floating-ui/utils@0.2.10", "", {}, "sha512-aGTxbpbg8/b5JfU1HXSrbH3wXZuLPJcNEcZQFMxLs3oSzgtVu6nFPkbbGGUvBcUjKV2YyB9Wxxabo+HEH9tcRQ=="], + "@google/generative-ai": ["@google/generative-ai@0.24.1", "", {}, "sha512-MqO+MLfM6kjxcKoy0p1wRzG3b4ZZXtPI+z2IE26UogS2Cm/XHO+7gGRBh6gcJsOiIVoH93UwKvW4HdgiOZCy9Q=="], + "@hello-pangea/dnd": ["@hello-pangea/dnd@18.0.1", "", { "dependencies": { "@babel/runtime": "^7.26.7", "css-box-model": "^1.2.1", "raf-schd": "^4.0.3", "react-redux": "^9.2.0", "redux": "^5.0.1" }, "peerDependencies": { "react": "^18.0.0 || ^19.0.0", "react-dom": "^18.0.0 || ^19.0.0" } }, "sha512-xojVWG8s/TGrKT1fC8K2tIWeejJYTAeJuj36zM//yEm/ZrnZUSFGS15BpO+jGZT1ybWvyXmeDJwPYb4dhWlbZQ=="], "@hugeicons/core-free-icons": ["@hugeicons/core-free-icons@3.3.0", "", {}, "sha512-qYyr4JQ2eQIHTSTbITvnJvs6ERNK64D9gpwZnf2IyuG0exzqfyABLO/oTB71FB3RZPfu1GbwycdiGSo46apjMQ=="], diff --git a/openspec/changes/archive/2026-04-24-gemini-adapter-fase-1/archive-report.md b/openspec/changes/archive/2026-04-24-gemini-adapter-fase-1/archive-report.md new file mode 100644 index 00000000..3046c859 --- /dev/null +++ b/openspec/changes/archive/2026-04-24-gemini-adapter-fase-1/archive-report.md @@ -0,0 +1,59 @@ +# Archive Report: gemini-adapter-fase-1 + +**Change**: gemini-adapter-fase-1 +**Archived**: 2026-04-24 +**Verification Verdict**: PASS — 14/14 tasks complete, 12/12 spec scenarios compliant, 48/48 tests passing, 0 TypeScript errors + +## Scope Summary + +This change adds **native Gemini provider support for normal chat and tool calling only**. Video upload/file understanding remains deferred to a future change. The `GeminiAdapter` implements the existing `ProviderAdapter` seam via `@google/generative-ai` SDK, supporting stateless `generateContent()` with system prompt mapping, tool schema conversion, and synthesized tool-call IDs. The OpenAI-compatible path is untouched. + +## Engram Artifact Traceability + +| Artifact | Observation ID | +|----------|---------------| +| Proposal | #1016 | +| Spec (delta) | #1019 | +| Design | #1018 | +| Tasks | #1020 | +| Apply Progress | #1021 | +| Verify Report | #1024 | + +## OpenSpec Archive + +**Moved to**: `openspec/changes/archive/2026-04-24-gemini-adapter-fase-1/` + +### Archive Contents +- proposal.md ✅ +- specs/ (agent-session-shell delta) ✅ +- design.md ✅ +- tasks.md ✅ (14/14 tasks complete) +- verify-report.md ✅ +- exploration.md ✅ + +## Specs Synced + +| Domain | Action | Details | +|--------|--------|---------| +| agent-session-shell | Updated | 2 requirements modified (Provider Adapter Interface, Provider Configuration), 2 requirements added (Gemini Free-Form Chat, Gemini Tool Calling), 0 removed | + +### Modified Requirements +- **Provider Adapter Interface**: Expanded factory to support `openai-compatible` and `gemini`; added tool-call ID synthesis mandate; added Gemini adapter selection scenario +- **Provider Configuration**: Added `LLM_PROVIDER` valid value constraint (`openai-compatible`, `gemini`); added Gemini-specific `LLM_BASE_URL` ignoring behavior; added unknown-provider and Gemini-config scenarios + +### Added Requirements +- **Gemini Free-Form Chat**: Plain text chat through Gemini `generateContent()` with system prompt mapped to `systemInstruction` +- **Gemini Tool Calling**: Tool schema → Gemini function declaration conversion, function call response → `ToolCall[]` with synthesized IDs + +## Source of Truth Updated +- `openspec/specs/agent-session-shell/spec.md` — now reflects Gemini provider support + +## Deferred Scope (explicitly out of scope) +- Video/file upload or direct file understanding +- `@asset` references, broader multimodal inputs, image/video reasoning +- Streaming UX or adapter contract changes for streaming +- Safety/grounding/thinking configuration beyond defaults + +## SDD Cycle Complete +The change has been fully explored, proposed, specified, designed, implemented, verified, and archived. +Ready for the next change. diff --git a/openspec/changes/archive/2026-04-24-gemini-adapter-fase-1/design.md b/openspec/changes/archive/2026-04-24-gemini-adapter-fase-1/design.md new file mode 100644 index 00000000..282e6c5d --- /dev/null +++ b/openspec/changes/archive/2026-04-24-gemini-adapter-fase-1/design.md @@ -0,0 +1,155 @@ +# Design: Gemini Adapter — Phase 1 + +## Technical Approach + +Implement a stateless `GeminiAdapter` class behind the existing `ProviderAdapter` seam, following the exact same pattern as `OpenAICompatibleAdapter`. The adapter converts internal `ChatMessage[]`/`ToolSchema[]` into Gemini's `generateContent()` wire format and parses responses back into `ProviderResponse`. A new `"gemini"` case in `createProvider()` activates it when `LLM_PROVIDER=gemini`. Zero changes to orchestrator, route, or client contracts. + +## Architecture Decisions + +### Decision: SDK choice + +| Option | Tradeoff | Decision | +|--------|----------|----------| +| `@google/generative-ai` | Established, stable, v0.21+, `GoogleGenerativeAI` class, `generateContent()` stateless API | ✅ Selected | +| `@google/genai` | Newer unified SDK, less battle-tested, API may still shift | ❌ Rejected | + +**Rationale**: `@google/generative-ai` is the most widely adopted Gemini SDK for Node/TS. It supports `systemInstruction`, `functionDeclaration`, and stateless `generateContent()` — exactly what we need. Consistent with using the mature `openai` SDK for the existing adapter. + +### Decision: Stateless generateContent vs stateful startChat + +| Option | Tradeoff | Decision | +|--------|----------|----------| +| `model.generateContent(contents, config)` | Stateless, full history each call — matches current adapter contract | ✅ Selected | +| `chat.sendMessage()` via `startChat()` | Stateful session, server-managed history | ❌ Rejected | + +**Rationale**: Current architecture POSTs full history from client on every turn. Stateless `generateContent()` maps 1:1 to this pattern. No session management needed. + +### Decision: Tool call ID synthesis + +| Option | Tradeoff | Decision | +|--------|----------|----------| +| `crypto.randomUUID()` | Built-in, zero deps, unique | ✅ Selected | +| `nanoid()` (already a dep) | Shorter IDs, also unique | ❌ Rejected | + +**Rationale**: `crypto.randomUUID()` is universally available in Node/Bun, no import needed, and produces standard UUIDs matching OpenAI's `call_xxx` format semantically. Keeps the adapter self-contained. + +### Decision: System prompt placement + +| Option | Tradeoff | Decision | +|--------|----------|----------| +| `systemInstruction` on `GenerateContentRequest` | Gemini-native, separate from contents | ✅ Selected | +| Prepend as `user` message | Works but dilutes system/user boundary | ❌ Rejected | + +**Rationale**: Gemini SDK supports `systemInstruction` as a first-class config field. Using it keeps system prompt semantically correct and matches Gemini best practices. + +## Data Flow + +``` +Route (chat/route.ts) + │ + ├─ createProvider(config) → GeminiAdapter instance + │ + └─ adapter.chat({ messages, systemPrompt, tools }) + │ + ├─ toGeminiContents(messages) → Content[] (user/model/function) + ├─ toGeminiTools(tools) → FunctionDeclaration[] + │ + └─ model.generateContent({ contents, tools, systemInstruction }) + │ + └─ fromGeminiResponse(response) → ProviderResponse + │ + ├─ content: candidate.content.parts[].text + └─ toolCalls: candidate.content.parts[] + .filter(functionCall) + .map → { id: uuid(), name, args } +``` + +### Message mapping + +| Internal role | Gemini role | Notes | +|---------------|-------------|-------| +| `system` | `systemInstruction` (config) | Not in contents array | +| `user` | `user` part | Direct text mapping | +| `assistant` | `model` part | Includes `functionCall` parts for tool calls | +| `tool_result` | `function` role | Uses `functionResponse: { name, response }` keyed by tool name (not ID) | + +### Type mapping (internal → Gemini) + +| Internal type | Gemini type | +|---------------|-------------| +| `"string"` | `SchemaType.STRING` | +| `"number"` | `SchemaType.NUMBER` | +| `"boolean"` | `SchemaType.BOOLEAN` | +| `"object"` | `SchemaType.OBJECT` | + +## File Changes + +| File | Action | Description | +|------|--------|-------------| +| `apps/web/src/agent/providers/gemini.ts` | Create | GeminiAdapter + conversion helpers (`toGeminiContents`, `toGeminiTools`, `fromGeminiResponse`) | +| `apps/web/src/agent/providers/index.ts` | Modify | Add `"gemini"` case to factory switch | +| `apps/web/src/agent/providers/__tests__/gemini.test.ts` | Create | Conversion tests, error paths, tool call ID synthesis | +| `apps/web/src/agent/providers/__tests__/index.test.ts` | Modify | Add factory test for `"gemini"` → `GeminiAdapter` | +| `apps/web/package.json` | Modify | Add `@google/generative-ai` dependency | +| `apps/web/.env.example` | Modify | Add Gemini example config lines | + +## Interfaces / Contracts + +No interface changes. The adapter implements the existing `ProviderAdapter`: + +```ts +// Reused as-is from providers/types.ts +interface ProviderAdapter { + chat(params: { + messages: ChatMessage[]; + systemPrompt: string; + tools: ToolSchema[]; + }): Promise; +} +``` + +### Internal helper signatures (gemini.ts) + +```ts +function toGeminiContents( + messages: ChatMessage[], +): GenerateContentRequest["contents"]; + +function toGeminiTools( + tools: ToolSchema[], +): FunctionDeclaration[]; + +function fromGeminiResponse( + response: GenerateContentResult, +): ProviderResponse; +``` + +### Tool call ID — name collision handling + +When mapping `tool_result` back to Gemini's `functionResponse`, Gemini keys by function **name** not ID. If multiple calls to the same tool exist in history, we group them by name and send the last result per name. This is acceptable because: + +1. Current tool set (`transcribe_video`) is single-invocation per turn +2. Multi-call same-tool scenarios are deferred to a later change +3. Orchestrator already handles the ID→name correlation client-side + +## Testing Strategy + +| Layer | What to Test | Approach | +|-------|-------------|----------| +| Unit | `toGeminiContents` mapping (user, assistant, tool_result, assistant+toolCalls) | Mock SDK, inspect call args | +| Unit | `toGeminiTools` type casing (string→STRING, etc.) | Direct assertion on output | +| Unit | `fromGeminiResponse` content extraction | Construct Gemini response shape, assert `ProviderResponse` | +| Unit | Tool call ID synthesis (non-empty, unique) | Multiple calls produce different IDs | +| Unit | Error: empty candidates → throws | Assert error message | +| Unit | Error: SDK throws → propagates | Assert error bubbles up | +| Unit | Factory: `"gemini"` → `GeminiAdapter` | Instance check | +| Integration | Full `chat()` round-trip with mocked SDK | End-to-end within adapter | + +## Migration / Rollout + +No migration required. Additive change — setting `LLM_PROVIDER=gemini` activates the new adapter. Existing `openai-compatible` default is untouched. + +## Open Questions + +- [ ] Confirm `@google/generative-ai` latest stable version at impl time (currently ^0.21) +- [ ] Verify Gemini model naming convention user will use (e.g. `gemini-2.0-flash` vs `models/gemini-2.0-flash`) diff --git a/openspec/changes/gemini-adapter-fase-1/exploration.md b/openspec/changes/archive/2026-04-24-gemini-adapter-fase-1/exploration.md similarity index 100% rename from openspec/changes/gemini-adapter-fase-1/exploration.md rename to openspec/changes/archive/2026-04-24-gemini-adapter-fase-1/exploration.md diff --git a/openspec/changes/archive/2026-04-24-gemini-adapter-fase-1/proposal.md b/openspec/changes/archive/2026-04-24-gemini-adapter-fase-1/proposal.md new file mode 100644 index 00000000..6719a596 --- /dev/null +++ b/openspec/changes/archive/2026-04-24-gemini-adapter-fase-1/proposal.md @@ -0,0 +1,65 @@ +# Proposal: Gemini Adapter — Phase 1 + +## Intent + +Add Gemini as a native provider behind the existing provider-agnostic LLM seam so the agent can use Gemini for plain chat and tool calling without changing orchestrator or route contracts. + +## Scope + +### In Scope +- Native `GeminiAdapter` implementing the current `ProviderAdapter` +- Gemini free-form chat using existing stateless full-history flow +- Gemini tool calling, with `transcribe_video` as the first real tool +- Config support for `LLM_PROVIDER=gemini`, `LLM_API_KEY`, `LLM_MODEL` +- Tests and env/docs updates while preserving the OpenAI-compatible path + +### Out of Scope +- Video/file upload or direct file understanding +- `@asset` references, broader multimodal inputs, and image/video reasoning +- Streaming UX or adapter contract changes for streaming +- Safety/grounding/thinking configuration beyond defaults + +## Capabilities + +### New Capabilities +- None + +### Modified Capabilities +- `agent-session-shell`: expand provider selection/configuration so the existing provider adapter contract also supports native Gemini chat and function calling + +## Approach + +Implement a native Gemini SDK-backed adapter and register it in the provider factory. Keep the adapter stateless by mapping full internal message history into Gemini `generateContent()` requests, converting tool schemas/results at the adapter boundary, and synthesizing tool call IDs so internal contracts stay unchanged. + +## Affected Areas + +| Area | Impact | Description | +|------|--------|-------------| +| `apps/web/src/agent/providers/gemini.ts` | New | Gemini request/response/tool conversion | +| `apps/web/src/agent/providers/index.ts` | Modified | Factory registration for `gemini` | +| `apps/web/src/agent/providers/__tests__/gemini.test.ts` | New | Adapter conversion/error tests | +| `apps/web/src/agent/providers/__tests__/index.test.ts` | Modified | Factory coverage | +| `apps/web/package.json` | Modified | Gemini SDK dependency | +| `apps/web/.env.example` | Modified | Gemini env configuration | + +## Risks + +| Risk | Likelihood | Mitigation | +|------|------------|------------| +| Gemini has no tool call IDs | Med | Synthesize unique IDs inside adapter | +| Gemini type casing differs from internal schema | Med | Centralize schema mapping tests | +| Gemini system prompt placement differs | Low | Map system prompt explicitly to Gemini config | + +## Rollback Plan + +Remove the Gemini adapter, factory branch, dependency, and env docs; keep `openai-compatible` as the unchanged default path. + +## Dependencies + +- Gemini JS SDK selection to be verified during implementation + +## Success Criteria + +- [ ] `LLM_PROVIDER=gemini` resolves a native Gemini adapter without breaking `openai-compatible` +- [ ] Gemini returns normal assistant text through the existing route/orchestrator flow +- [ ] Gemini can emit structured tool calls and execute `transcribe_video` through the existing loop diff --git a/openspec/changes/archive/2026-04-24-gemini-adapter-fase-1/specs/agent-session-shell/spec.md b/openspec/changes/archive/2026-04-24-gemini-adapter-fase-1/specs/agent-session-shell/spec.md new file mode 100644 index 00000000..ea9fbc2e --- /dev/null +++ b/openspec/changes/archive/2026-04-24-gemini-adapter-fase-1/specs/agent-session-shell/spec.md @@ -0,0 +1,97 @@ +# Delta for agent-session-shell + +## MODIFIED Requirements + +### Requirement: Provider Adapter Interface + +The system SHALL define a server-side provider adapter interface with a single method `chat(params: ProviderChatParams): Promise`. `ProviderChatParams` MUST include `messages`, `systemPrompt`, and `toolSchemas` in a provider-agnostic internal format. `ProviderChatResponse` MUST include `content: string` and `toolCalls?: ProviderToolCall[]` where `ProviderToolCall { id, name, args }`. Each concrete adapter converts internal tool schemas to its own wire format. The factory MUST support at least `openai-compatible` and `gemini` providers. Adapters whose native API does not assign IDs to tool calls MUST synthesize unique IDs so every returned `ToolCall.id` is a non-empty string. + +(Previously: Phase 1 MAY ship exactly one adapter (OpenAI-compatible) as long as the interface contract remains provider-agnostic.) + +#### Scenario: Adapter translates tool schemas to wire format + +- GIVEN `transcribe_video` has internal parameters `[{ key: "assetId", type: "string", required: false }]` +- WHEN the adapter receives tool schemas before calling the provider +- THEN it converts them to its provider-specific wire format + +#### Scenario: Adapter selected by config — OpenAI-compatible + +- GIVEN `LLM_PROVIDER` is set to `"openai-compatible"` +- WHEN the route resolves the adapter +- THEN the OpenAI-compatible adapter is returned + +#### Scenario: Adapter selected by config — Gemini + +- GIVEN `LLM_PROVIDER` is set to `"gemini"` +- WHEN the route resolves the adapter +- THEN the Gemini adapter is returned + +#### Scenario: Gemini adapter synthesizes tool-call IDs + +- GIVEN the Gemini API returns a function call response with no call ID +- WHEN the Gemini adapter maps the response to `ProviderResponse` +- THEN each `ToolCall` in `toolCalls` has a non-empty `id` field generated by the adapter + +### Requirement: Provider Configuration + +Server-only environment variables `LLM_PROVIDER` (required), `LLM_MODEL` (required), `LLM_API_KEY` (required), and `LLM_BASE_URL` (optional) MUST determine which adapter and model the route uses. `LLM_PROVIDER` MUST accept at least `"openai-compatible"` and `"gemini"`. When `LLM_PROVIDER` is `"gemini"`, `LLM_BASE_URL` MUST be ignored. Configuration MUST be read server-side only — never exposed to the client. + +(Previously: Provider config listed env vars but did not constrain valid provider values or describe Gemini-specific behavior.) + +#### Scenario: Missing required configuration + +- GIVEN `LLM_API_KEY` is not set in the environment +- WHEN the route attempts to initialize the provider +- THEN the call returns 502 with a missing-configuration error + +#### Scenario: Unknown provider + +- GIVEN `LLM_PROVIDER` is set to `"unknown-provider"` +- WHEN the route resolves the adapter +- THEN the factory throws `"Unknown LLM provider: unknown-provider"` + +#### Scenario: Gemini config ignores base URL + +- GIVEN `LLM_PROVIDER` is `"gemini"` and `LLM_BASE_URL` is set +- WHEN the Gemini adapter is created +- THEN `LLM_BASE_URL` is not used in the API call + +## ADDED Requirements + +### Requirement: Gemini Free-Form Chat + +When the Gemini adapter receives a request with no tool schemas (or tools array is empty), it MUST send messages to the Gemini `generateContent` API and return a `ProviderResponse` with `content` set to the assistant text and `toolCalls` undefined. + +#### Scenario: Plain chat through Gemini + +- GIVEN `LLM_PROVIDER` is `"gemini"` and messages contain `[{ role: "user", content: "Hello" }]` +- WHEN the adapter calls Gemini and receives a text-only response +- THEN `ProviderResponse.content` is the assistant text and `toolCalls` is undefined + +#### Scenario: System prompt mapping + +- GIVEN the adapter receives `systemPrompt: "You are a video editor"` +- WHEN it builds the Gemini request +- THEN the system prompt is mapped to Gemini's system instruction field, not as a user message + +### Requirement: Gemini Tool Calling + +When the Gemini adapter receives a request with tool schemas, it MUST convert internal `ToolSchema[]` to Gemini function declarations, call `generateContent` with declarations attached, and if Gemini returns function call parts, map each to a `ToolCall { id, name, args }` with a synthesized non-empty `id`. + +#### Scenario: Gemini returns function calls + +- GIVEN `transcribe_video` is in tool schemas and Gemini responds with a function call part `{ name: "transcribe_video", args: {} }` +- WHEN the adapter maps the response +- THEN `toolCalls` contains one `ToolCall` with `name: "transcribe_video"`, `args: {}`, and a synthesized `id` + +#### Scenario: Gemini returns text despite tools + +- GIVEN tool schemas are provided but Gemini responds with only text +- WHEN the adapter maps the response +- THEN `toolCalls` is undefined and `content` holds the text + +#### Scenario: Tool schema type mapping + +- GIVEN an internal tool schema has parameters `[{ key: "assetId", type: "string", required: true }]` +- WHEN the adapter converts to Gemini function declarations +- THEN the parameter appears in `FunctionDeclaration.parameters.properties` with type `"string"` diff --git a/openspec/changes/archive/2026-04-24-gemini-adapter-fase-1/tasks.md b/openspec/changes/archive/2026-04-24-gemini-adapter-fase-1/tasks.md new file mode 100644 index 00000000..499ab63e --- /dev/null +++ b/openspec/changes/archive/2026-04-24-gemini-adapter-fase-1/tasks.md @@ -0,0 +1,28 @@ +# Tasks: Gemini Adapter — Phase 1 + +## Phase 1: Foundation + +- [x] 1.1 Add `@google/generative-ai` dependency to `apps/web/package.json` (pin to latest stable `^0.21` or newer) +- [x] 1.2 Add Gemini env config lines to `apps/web/.env.example`: `LLM_PROVIDER=gemini`, `LLM_API_KEY`, `LLM_MODEL=gemini-2.0-flash`, comment noting `LLM_BASE_URL` is ignored for gemini + +## Phase 2: Core Adapter + +- [x] 2.1 Create `apps/web/src/agent/providers/gemini.ts` — import `GoogleGenerativeAI` from `@google/generative-ai`, `SchemaType` for type mapping +- [x] 2.2 Implement `toGeminiContents(messages: ChatMessage[])` — map `user→user`, `assistant→model` (with `functionCall` parts), `tool_result→function` role with `functionResponse`; skip `system` role (handled separately) +- [x] 2.3 Implement `toGeminiTools(tools: ToolSchema[])` — convert each `ToolSchema` to a `FunctionDeclaration` with `SchemaType` property types (`string→STRING`, `number→NUMBER`, `boolean→BOOLEAN`, `object→OBJECT`); return empty array when no tools +- [x] 2.4 Implement `fromGeminiResponse(response)` — extract `candidate.content.parts`, pull text into `content`, filter `functionCall` parts into `toolCalls[]` with `id: crypto.randomUUID()`, `name`, `args`; throw on empty candidates +- [x] 2.5 Implement `GeminiAdapter` class — constructor takes `ProviderConfig`, creates `GoogleGenerativeAI` + `getGenerativeModel({ model, systemInstruction })`; `chat()` calls `model.generateContent({ contents, tools })` then returns `fromGeminiResponse()`; ignore `config.baseUrl` + +## Phase 3: Factory Integration + +- [x] 3.1 Add `"gemini"` case to `createProvider()` switch in `apps/web/src/agent/providers/index.ts` — import and return `new GeminiAdapter(config)` +- [x] 3.2 Verify existing `openai-compatible` factory case is untouched and all existing tests still pass + +## Phase 4: Tests + +- [x] 4.1 Create `apps/web/src/agent/providers/__tests__/gemini.test.ts` — test `toGeminiContents`: user message, assistant with toolCalls, tool_result mapping, system message excluded from contents +- [x] 4.2 Test `toGeminiTools`: single tool with parameters, type casing (`string→STRING` etc.), empty tools returns empty array +- [x] 4.3 Test `fromGeminiResponse`: text-only response → `{ content, toolCalls: undefined }`; function-call response → `toolCalls` with non-empty synthesized IDs; two calls produce different IDs +- [x] 4.4 Test error paths: empty candidates → throws; SDK error → propagates +- [x] 4.5 Add factory test to `apps/web/src/agent/providers/__tests__/index.test.ts`: `"gemini"` config → `GeminiAdapter` instance; unknown provider still throws +- [x] 4.6 Test full `chat()` round-trip with mocked `GoogleGenerativeAI` — verify `generateContent` called with correct contents, tools, and systemInstruction diff --git a/openspec/changes/archive/2026-04-24-gemini-adapter-fase-1/verify-report.md b/openspec/changes/archive/2026-04-24-gemini-adapter-fase-1/verify-report.md new file mode 100644 index 00000000..8f4d6e73 --- /dev/null +++ b/openspec/changes/archive/2026-04-24-gemini-adapter-fase-1/verify-report.md @@ -0,0 +1,116 @@ +# Verification Report + +**Change**: gemini-adapter-fase-1 +**Version**: N/A +**Mode**: Standard + +--- + +### Completeness + +| Metric | Value | +|--------|-------| +| Tasks total | 14 | +| Tasks complete | 14 | +| Tasks incomplete | 0 | + +All tasks in `openspec/changes/gemini-adapter-fase-1/tasks.md` are marked complete, including the post-verify fixes. + +--- + +### Build & Tests Execution + +**Build**: ➖ Skipped + +```text +No explicit `rules.verify.build_command` is configured in `openspec/config.yaml`. +Project instructions also say never build after changes, so verification used TypeScript type-checking instead of `next build`. +``` + +**Type Check**: ✅ Passed + +```text +Command: bunx tsc --noEmit + +(no output) +``` + +**Tests**: ✅ 48 passed / ❌ 0 failed / ⚠️ 0 skipped + +```text +Command: bun test src/agent/providers/__tests__ src/app/api/agent/chat/__tests__/route.test.ts + +bun test v1.3.13 +48 pass +0 fail +106 expect() calls +Ran 48 tests across 4 files. +``` + +**Coverage**: ➖ Not available + +--- + +### Spec Compliance Matrix + +| Requirement | Scenario | Test | Result | +|-------------|----------|------|--------| +| Provider Adapter Interface | Adapter translates tool schemas to wire format | `src/agent/providers/__tests__/gemini.test.ts > converts a single tool with parameters` + `maps type strings to Gemini SchemaType values` | ✅ COMPLIANT | +| Provider Adapter Interface | Adapter selected by config — OpenAI-compatible | `src/agent/providers/__tests__/index.test.ts > returns OpenAICompatibleAdapter for openai-compatible provider` | ✅ COMPLIANT | +| Provider Adapter Interface | Adapter selected by config — Gemini | `src/agent/providers/__tests__/index.test.ts > returns GeminiAdapter for gemini provider` | ✅ COMPLIANT | +| Provider Adapter Interface | Gemini adapter synthesizes tool-call IDs | `src/agent/providers/__tests__/gemini.test.ts > maps function-call response to toolCalls with non-empty synthesized IDs` | ✅ COMPLIANT | +| Provider Configuration | Missing required configuration | `src/app/api/agent/chat/__tests__/route.test.ts > returns 502 when LLM_API_KEY is missing` | ✅ COMPLIANT | +| Provider Configuration | Unknown provider | `src/agent/providers/__tests__/index.test.ts > throws Error for unknown provider` | ✅ COMPLIANT | +| Provider Configuration | Gemini config ignores base URL | `src/agent/providers/__tests__/gemini.test.ts > ignores baseUrl in config — Gemini still works` | ✅ COMPLIANT | +| Gemini Free-Form Chat | Plain chat through Gemini | `src/agent/providers/__tests__/gemini.test.ts > calls generateContent with correct contents and no tools` | ✅ COMPLIANT | +| Gemini Free-Form Chat | System prompt mapping | `src/agent/providers/__tests__/gemini.test.ts > calls generateContent with correct contents and no tools` | ✅ COMPLIANT | +| Gemini Tool Calling | Gemini returns function calls | `src/agent/providers/__tests__/gemini.test.ts > calls generateContent with tools and systemInstruction` | ✅ COMPLIANT | +| Gemini Tool Calling | Gemini returns text despite tools | `src/agent/providers/__tests__/gemini.test.ts > returns text when tools are provided but Gemini responds with text only` | ✅ COMPLIANT | +| Gemini Tool Calling | Tool schema type mapping | `src/agent/providers/__tests__/gemini.test.ts > maps type strings to Gemini SchemaType values` | ✅ COMPLIANT | + +**Compliance summary**: 12/12 scenarios compliant + +--- + +### Correctness (Static — Structural Evidence) + +| Requirement | Status | Notes | +|------------|--------|-------| +| Provider Adapter Interface | ✅ Implemented | `apps/web/src/agent/providers/types.ts` keeps the provider-agnostic `chat()` seam, `index.ts` supports both providers, and `gemini.ts` synthesizes tool-call IDs with `crypto.randomUUID()`. | +| Provider Configuration | ✅ Implemented | `apps/web/src/app/api/agent/chat/route.ts` reads `LLM_PROVIDER`, `LLM_API_KEY`, `LLM_MODEL`, and `LLM_BASE_URL` server-side only; `GeminiAdapter` intentionally ignores `baseUrl`. | +| Gemini Free-Form Chat | ✅ Implemented | `GeminiAdapter.chat()` maps messages into Gemini contents, sets `systemInstruction` when present, and returns text-only responses without `toolCalls`. | +| Gemini Tool Calling | ✅ Implemented | `toGeminiTools()` converts internal tool schemas to Gemini declarations and `fromGeminiResponse()` maps function-call parts back into provider `toolCalls`. | +| Explicit non-goals remain deferred | ✅ Implemented | The touched files remain additive only; no streaming, multimodal upload, `@asset`, or contract changes were introduced. | + +--- + +### Coherence (Design) + +| Decision | Followed? | Notes | +|----------|-----------|-------| +| Use `@google/generative-ai` SDK | ✅ Yes | Added in `apps/web/package.json`; adapter imports `GoogleGenerativeAI` and `SchemaType`. | +| Preserve stateless provider seam | ✅ Yes | `GeminiAdapter.chat()` sends full history through one `generateContent()` call with no session state. | +| Synthesize tool-call IDs inside adapter | ✅ Yes | `fromGeminiResponse()` generates IDs for Gemini function calls. | +| Keep OpenAI-compatible path intact | ✅ Yes | `createProvider()` still returns `OpenAICompatibleAdapter` unchanged for the original provider. | +| Map system prompt via Gemini-native field | ✅ Yes | `GeminiAdapter.chat()` sets `request.systemInstruction` instead of injecting a fake user message. | + +--- + +### Issues Found + +**CRITICAL** (must fix before archive): +None + +**WARNING** (should fix): +None + +**SUGGESTION** (nice to have): +None + +--- + +### Verdict + +PASS + +All planned tasks are complete, the changed slice now type-checks, targeted runtime tests pass, and every spec scenario has passing behavioral evidence. diff --git a/openspec/specs/agent-session-shell/spec.md b/openspec/specs/agent-session-shell/spec.md index 1ddfac14..76727084 100644 --- a/openspec/specs/agent-session-shell/spec.md +++ b/openspec/specs/agent-session-shell/spec.md @@ -110,7 +110,7 @@ The system SHALL define types in `apps/web/src/agent/types.ts`: `ChatMessage { i ### Requirement: Provider Adapter Interface -The system SHALL define a server-side provider adapter interface with a single method `chat(params: ProviderChatParams): Promise`. `ProviderChatParams` MUST include `messages`, `systemPrompt`, and `toolSchemas` in a provider-agnostic internal format. `ProviderChatResponse` MUST include `content: string` and `toolCalls?: ProviderToolCall[]` where `ProviderToolCall { id, name, args }`. Each concrete adapter converts internal tool schemas to its own wire format. Phase 1 MAY ship exactly one adapter (OpenAI-compatible) as long as the interface contract remains provider-agnostic. +The system SHALL define a server-side provider adapter interface with a single method `chat(params: ProviderChatParams): Promise`. `ProviderChatParams` MUST include `messages`, `systemPrompt`, and `toolSchemas` in a provider-agnostic internal format. `ProviderChatResponse` MUST include `content: string` and `toolCalls?: ProviderToolCall[]` where `ProviderToolCall { id, name, args }`. Each concrete adapter converts internal tool schemas to its own wire format. The factory MUST support at least `openai-compatible` and `gemini` providers. Adapters whose native API does not assign IDs to tool calls MUST synthesize unique IDs so every returned `ToolCall.id` is a non-empty string. #### Scenario: Adapter translates tool schemas to wire format @@ -118,15 +118,27 @@ The system SHALL define a server-side provider adapter interface with a single m - WHEN the adapter receives tool schemas before calling the provider - THEN it converts them to its provider-specific wire format -#### Scenario: Adapter selected by config +#### Scenario: Adapter selected by config — OpenAI-compatible - GIVEN `LLM_PROVIDER` is set to `"openai-compatible"` - WHEN the route resolves the adapter - THEN the OpenAI-compatible adapter is returned +#### Scenario: Adapter selected by config — Gemini + +- GIVEN `LLM_PROVIDER` is set to `"gemini"` +- WHEN the route resolves the adapter +- THEN the Gemini adapter is returned + +#### Scenario: Gemini adapter synthesizes tool-call IDs + +- GIVEN the Gemini API returns a function call response with no call ID +- WHEN the Gemini adapter maps the response to `ProviderResponse` +- THEN each `ToolCall` in `toolCalls` has a non-empty `id` field generated by the adapter + ### Requirement: Provider Configuration -Server-only environment variables `LLM_PROVIDER` (required), `LLM_MODEL` (required), `LLM_API_KEY` (required), and `LLM_BASE_URL` (optional) MUST determine which adapter and model the route uses. Configuration MUST be read server-side only — never exposed to the client. +Server-only environment variables `LLM_PROVIDER` (required), `LLM_MODEL` (required), `LLM_API_KEY` (required), and `LLM_BASE_URL` (optional) MUST determine which adapter and model the route uses. `LLM_PROVIDER` MUST accept at least `"openai-compatible"` and `"gemini"`. When `LLM_PROVIDER` is `"gemini"`, `LLM_BASE_URL` MUST be ignored. Configuration MUST be read server-side only — never exposed to the client. #### Scenario: Missing required configuration @@ -134,6 +146,18 @@ Server-only environment variables `LLM_PROVIDER` (required), `LLM_MODEL` (requir - WHEN the route attempts to initialize the provider - THEN the call returns 502 with a missing-configuration error +#### Scenario: Unknown provider + +- GIVEN `LLM_PROVIDER` is set to `"unknown-provider"` +- WHEN the route resolves the adapter +- THEN the factory throws `"Unknown LLM provider: unknown-provider"` + +#### Scenario: Gemini config ignores base URL + +- GIVEN `LLM_PROVIDER` is `"gemini"` and `LLM_BASE_URL` is set +- WHEN the Gemini adapter is created +- THEN `LLM_BASE_URL` is not used in the API call + ### Requirement: Tool Argument Validation Before executing any tool call, the orchestrator MUST validate that each argument key declared `required: true` in the tool's `parameters` is present in the call args and is of the declared type. If validation fails, the tool MUST NOT execute; instead the orchestrator MUST produce a `ToolResult` with `error` describing the missing or malformed argument. @@ -166,3 +190,41 @@ The orchestrator loop MUST enforce a hard iteration cap of **8**. An iteration i - GIVEN the provider returns a final text response after 2 tool-call iterations - WHEN the orchestrator processes the response - THEN no cap message is appended and status is `"idle"` + +### Requirement: Gemini Free-Form Chat + +When the Gemini adapter receives a request with no tool schemas (or tools array is empty), it MUST send messages to the Gemini `generateContent` API and return a `ProviderResponse` with `content` set to the assistant text and `toolCalls` undefined. + +#### Scenario: Plain chat through Gemini + +- GIVEN `LLM_PROVIDER` is `"gemini"` and messages contain `[{ role: "user", content: "Hello" }]` +- WHEN the adapter calls Gemini and receives a text-only response +- THEN `ProviderResponse.content` is the assistant text and `toolCalls` is undefined + +#### Scenario: System prompt mapping + +- GIVEN the adapter receives `systemPrompt: "You are a video editor"` +- WHEN it builds the Gemini request +- THEN the system prompt is mapped to Gemini's system instruction field, not as a user message + +### Requirement: Gemini Tool Calling + +When the Gemini adapter receives a request with tool schemas, it MUST convert internal `ToolSchema[]` to Gemini function declarations, call `generateContent` with declarations attached, and if Gemini returns function call parts, map each to a `ToolCall { id, name, args }` with a synthesized non-empty `id`. + +#### Scenario: Gemini returns function calls + +- GIVEN `transcribe_video` is in tool schemas and Gemini responds with a function call part `{ name: "transcribe_video", args: {} }` +- WHEN the adapter maps the response +- THEN `toolCalls` contains one `ToolCall` with `name: "transcribe_video"`, `args: {}`, and a synthesized `id` + +#### Scenario: Gemini returns text despite tools + +- GIVEN tool schemas are provided but Gemini responds with only text +- WHEN the adapter maps the response +- THEN `toolCalls` is undefined and `content` holds the text + +#### Scenario: Tool schema type mapping + +- GIVEN an internal tool schema has parameters `[{ key: "assetId", type: "string", required: true }]` +- WHEN the adapter converts to Gemini function declarations +- THEN the parameter appears in `FunctionDeclaration.parameters.properties` with type `"string"` From ee532c2647300e136f1a0336eb97701a97461b81 Mon Sep 17 00:00:00 2001 From: Luis Esteban Acevedo Ladino Date: Fri, 24 Apr 2026 09:34:38 -0500 Subject: [PATCH 2/2] docs --- docs/agent-tool-specs.md | 451 ++++++++++++++++++ docs/agent-tools.md | 324 +++++++++++++ .../exploration.md | 292 ++++++++++++ .../gemini-analyze-video-fase-1/proposal.md | 62 +++ 4 files changed, 1129 insertions(+) create mode 100644 docs/agent-tool-specs.md create mode 100644 docs/agent-tools.md create mode 100644 openspec/changes/gemini-analyze-video-fase-1/exploration.md create mode 100644 openspec/changes/gemini-analyze-video-fase-1/proposal.md diff --git a/docs/agent-tool-specs.md b/docs/agent-tool-specs.md new file mode 100644 index 00000000..57db4d62 --- /dev/null +++ b/docs/agent-tool-specs.md @@ -0,0 +1,451 @@ +# Agent Tool Specs — NeuralCut + +Este documento define las primitives iniciales del agente. Cada tool debe ser pequeña, accionable y componible. El agente puede resolver features complejas combinando estas tools, pero ninguna tool debe intentar “hacer todo”. + +## Principios generales + +- Las tools reciben datos estructurados, no instrucciones libres. +- Las tools deben validar inputs y devolver errores claros. +- Las tools que modifican el editor deben usar los comandos/managers existentes para preservar undo/redo cuando aplique. +- El agente debe usar `list_project_assets` y `list_timeline` antes de editar cuando no tenga IDs concretos. +- Gemini se usa para comprensión multimodal; la edición real se ejecuta con tools determinísticas. + +--- + +## 1. `list_project_assets` + +### Propósito +Listar assets disponibles en el proyecto para que el agente sepa qué archivos existen y cuáles están usados en el timeline. + +### Input +```ts +{ + filter?: "all" | "used" | "unused"; + type?: "all" | "video" | "audio" | "image"; +} +``` + +### Output +```ts +{ + assets: Array<{ + id: string; + name: string; + type: "video" | "audio" | "image"; + duration?: number; + usedInTimeline: boolean; + }>; +} +``` + +### Requirements +- MUST return all loaded project assets by default. +- MUST support filtering by usage: `used`, `unused`, `all`. +- MUST support filtering by asset type. +- MUST include stable internal `id` values. +- MUST NOT mutate editor state. + +### Errors +- If no project is loaded, return `{ error: "No active project" }`. + +--- + +## 2. `list_timeline` + +### Propósito +Listar el estado actual del timeline para que el agente pueda referenciar clips, textos, stickers y otros elementos por `trackId`/`elementId`. + +### Input +```ts +{} +``` + +### Output +```ts +{ + tracks: Array<{ + trackId: string; + type: "main" | "overlay" | "audio" | "text" | "effect"; + elements: Array<{ + elementId: string; + type: string; + assetId?: string; + name?: string; + start: number; + end: number; + }>; + }>; +} +``` + +### Requirements +- MUST return a structured timeline summary. +- MUST include `trackId` and `elementId` for editable elements. +- MUST include timing in seconds or a clearly documented unit. +- MUST NOT mutate editor state. + +### Errors +- If no active scene/timeline exists, return `{ error: "No active timeline" }`. + +--- + +## 3. `load_asset_context` + +### Propósito +Cargar un asset en el contexto multimodal del agente usando Gemini. Esta es la base para que el agente entienda video/audio/imagen antes de editar. + +### Input +```ts +{ + assetId: string; +} +``` + +### Output +```ts +{ + assetId: string; + status: "loaded" | "processing"; + cached: boolean; + provider: "gemini"; + fileUri?: string; + summary?: string; +} +``` + +### Requirements +- MUST resolve the asset by internal `assetId`. +- MAY support fallback by filename if unambiguous, but internal ID is preferred. +- MUST upload/process the asset with Gemini when not cached. +- MUST cache the provider file reference by `assetId` to avoid repeated uploads. +- MUST keep API keys server-side. +- MUST support video first; audio/image support may be added if Gemini path supports it cleanly. +- MUST NOT edit the timeline. + +### Cache behavior +- Cache is application/orchestrator infrastructure, not necessarily a visible LLM tool. +- Cache should store at least: + ```ts + { + assetId: string; + provider: "gemini"; + fileUri: string; + status: "active" | "processing" | "failed"; + createdAt: number; + } + ``` + +### Errors +- Unknown asset: `{ error: "Asset not found" }`. +- Unsupported type: `{ error: "Unsupported asset type" }`. +- Upload/processing failure: `{ error: "Failed to load asset context" }` with safe details. + +### Non-goals +- No editing actions. +- No `@asset` UI autocomplete. +- No streaming progress in the first implementation. + +--- + +## 4. `cut_segment` + +### Propósito +Cortar o remover un rango de tiempo del timeline. + +### Input +```ts +{ + start: number; + end: number; + mode: "remove" | "keep"; +} +``` + +### Output +```ts +{ + success: boolean; + affectedElements: string[]; +} +``` + +### Requirements +- MUST validate `start < end`. +- MUST operate on timeline time, using the editor’s canonical time unit internally. +- MUST use existing timeline/command infrastructure when possible. +- `remove` MUST remove the selected time range. +- `keep` MUST preserve only the selected time range and remove outside material, if feasible in current editor model. +- MUST preserve undo/redo behavior if the editor supports it for the operation. + +### Errors +- Invalid range: `{ error: "Invalid time range" }`. +- Empty timeline: `{ error: "No timeline content" }`. +- Unsupported mode: `{ error: "Unsupported cut mode" }`. + +--- + +## 5. `add_text` + +### Propósito +Agregar texto visual al timeline. Esta primitive cubre títulos, hooks, labels y subtítulos básicos. + +### Input +```ts +{ + text: string; + start: number; + end: number; + position: "top" | "center" | "bottom"; + style?: "plain" | "subtitle" | "hook" | "label"; +} +``` + +### Output +```ts +{ + elementId: string; + trackId: string; +} +``` + +### Requirements +- MUST validate non-empty `text`. +- MUST validate `start < end`. +- MUST create a text element using existing timeline APIs. +- MUST map `position` to a sensible default placement. +- SHOULD provide simple style presets, but implementation may start with defaults. +- MUST NOT generate full subtitles automatically; that is a flow using transcript/context + repeated `add_text` calls. + +### Errors +- Empty text: `{ error: "Text is required" }`. +- Invalid range: `{ error: "Invalid time range" }`. + +--- + +## 6. `update_text` + +### Propósito +Editar un texto existente en el timeline. + +### Input +```ts +{ + trackId: string; + elementId: string; + text?: string; + start?: number; + end?: number; + position?: "top" | "center" | "bottom"; +} +``` + +### Output +```ts +{ + success: boolean; + elementId: string; +} +``` + +### Requirements +- MUST find an existing text element by `trackId` + `elementId`. +- MUST only update provided fields. +- MUST validate timing if `start`/`end` are provided. +- MUST preserve undo/redo behavior if supported. + +### Errors +- Missing element: `{ error: "Text element not found" }`. +- Wrong element type: `{ error: "Element is not text" }`. +- Invalid range: `{ error: "Invalid time range" }`. + +--- + +## 7. `add_media_to_timeline` + +### Propósito +Agregar un asset existente al timeline. + +### Input +```ts +{ + assetId: string; + startTime: number; + trackType: "main" | "overlay" | "audio"; +} +``` + +### Output +```ts +{ + elementId: string; + trackId: string; +} +``` + +### Requirements +- MUST resolve asset by `assetId`. +- MUST validate asset type compatibility with `trackType`. +- MUST insert the element using existing timeline APIs. +- SHOULD choose a sensible track when one is not obvious, but first version requires explicit `trackType`. + +### Errors +- Asset not found: `{ error: "Asset not found" }`. +- Invalid track type: `{ error: "Invalid track type for asset" }`. + +--- + +## 8. `delete_element` + +### Propósito +Eliminar un elemento específico del timeline. + +### Input +```ts +{ + trackId: string; + elementId: string; +} +``` + +### Output +```ts +{ + success: boolean; +} +``` + +### Requirements +- MUST find element by `trackId` + `elementId`. +- MUST remove only that element. +- MUST preserve undo/redo behavior if supported. + +### Errors +- Missing element: `{ error: "Element not found" }`. + +--- + +## 9. `set_volume` + +### Propósito +Ajustar volumen de un elemento de audio o video. + +### Input +```ts +{ + trackId: string; + elementId: string; + volume: number; +} +``` + +### Output +```ts +{ + success: boolean; + volume: number; +} +``` + +### Requirements +- MUST validate `volume` in range `0..1`. +- MUST only apply to elements that support audio volume. +- MUST preserve undo/redo behavior if supported. + +### Errors +- Invalid volume: `{ error: "Volume must be between 0 and 1" }`. +- Unsupported element: `{ error: "Element does not support volume" }`. + +--- + +## 10. `add_sticker` + +### Propósito +Agregar un sticker existente al timeline. + +### Input +```ts +{ + stickerId: string; + start: number; + end: number; + position: "top-left" | "top-right" | "center" | "bottom-left" | "bottom-right"; +} +``` + +### Output +```ts +{ + elementId: string; + trackId: string; +} +``` + +### Requirements +- MUST resolve sticker by `stickerId`. +- MUST validate `start < end`. +- MUST create a timeline element using existing sticker/timeline APIs. +- MUST map `position` to sensible coordinates. + +### Errors +- Sticker not found: `{ error: "Sticker not found" }`. +- Invalid range: `{ error: "Invalid time range" }`. + +--- + +## 11. `apply_effect` + +### Propósito +Aplicar un efecto existente a un clip. En el estado actual del repo, el efecto real disponible parece ser `blur`. + +### Input +```ts +{ + trackId: string; + elementId: string; + effectType: "blur"; + params?: { + intensity?: number; + }; +} +``` + +### Output +```ts +{ + effectId: string; + elementId: string; +} +``` + +### Requirements +- MUST validate the element is visual and supports effects. +- MUST validate `effectType` exists in the effects registry. +- MUST apply default params when `params` are omitted. +- MUST validate `intensity` if provided. +- MUST preserve undo/redo behavior if supported. + +### Errors +- Effect not found: `{ error: "Effect not found" }`. +- Unsupported element: `{ error: "Element does not support effects" }`. +- Invalid params: `{ error: "Invalid effect parameters" }`. + +--- + +## Existing/secondary tool: `transcribe_video` + +### Propósito +Transcribir audio usando Whisper local. Es útil para captions y edición por texto, pero no debe ser la primitive principal de comprensión si `load_asset_context` con Gemini está disponible. + +### Input +```ts +{ + assetId?: string; + language?: string; + modelId?: string; +} +``` + +### Estado +- Implementada. +- Debe mantenerse como secundaria. +- Puede ser usada por flujos de subtítulos. diff --git a/docs/agent-tools.md b/docs/agent-tools.md new file mode 100644 index 00000000..85a3f506 --- /dev/null +++ b/docs/agent-tools.md @@ -0,0 +1,324 @@ +# Agent Tools — Lista propuesta + +Este documento resume las tools propuestas para el agente de NeuralCut. La idea es mantenerlas **primitivas, acotadas y componibles**, no crear una tool por cada feature de marketing. + +## Principio + +El agente debe combinar primitives simples: + +- primero entiende el proyecto/assets, +- luego decide qué acción hacer, +- después ejecuta operaciones concretas del editor. + +No queremos tools gigantes tipo “hazme un reel completo”. Eso debe ser un flujo del agente usando varias tools pequeñas. + +--- + +## Tools de contexto y percepción + +### `list_project_assets` + +Lista assets conocidos del proyecto. + +```ts +{ + filter?: "all" | "used" | "unused"; + type?: "all" | "video" | "audio" | "image"; +} +``` + +Uso: +- saber qué archivos existen, +- distinguir assets usados/no usados, +- decidir qué asset cargar o editar. + +--- + +### `list_timeline` + +Lista el estado actual del timeline. + +```ts +{} +``` + +Uso: +- saber qué clips/textos/stickers/effects existen, +- obtener `trackId` y `elementId`, +- preparar operaciones como cortar, borrar o editar. + +--- + +### `load_asset_context` + +Carga un asset en el contexto multimodal del agente usando Gemini. + +```ts +{ + assetId: string; +} +``` + +Uso: +- subir/procesar video, audio o imagen con Gemini, +- dejar el asset disponible para razonamiento posterior, +- cachear la referencia para no subirlo repetidamente. + +> Esta reemplaza la idea anterior de `analyze_video({ question })` o `analyze_asset({ analysisType })`. La tool no “pregunta”; solo carga el asset al contexto. + +--- + +### `get_asset_context` *(infra interna, no tool visible)* + +Recupera información/cache de un asset ya cargado. + +```ts +{ + assetId: string; +} +``` + +Uso: +- reutilizar análisis/contexto previo, +- evitar re-upload, +- permitir que el agente sepa si un asset ya está disponible para Gemini. + +> Esta NO debería exponerse necesariamente como tool al LLM. Es más sano tratarla como infraestructura interna del agente/orquestador: `load_asset_context` carga y cachea; el orquestador/store puede consultar el cache e inyectar ese estado en el prompt/contexto sin hacer que el modelo llame una tool administrativa. + +--- + +## Tools de edición primitivas + +### `cut_segment` + +Corta o remueve un rango de tiempo del timeline. + +```ts +{ + start: number; + end: number; + mode: "remove" | "keep"; +} +``` + +Uso: +- quitar silencios, +- quitar errores, +- recortar intro/outro, +- construir highlights. + +--- + +### `add_media_to_timeline` + +Agrega un asset existente al timeline. + +```ts +{ + assetId: string; + startTime: number; + trackType: "main" | "overlay" | "audio"; +} +``` + +Uso: +- insertar clips, +- agregar música, +- poner b-roll, +- agregar imágenes/logos. + +--- + +### `delete_element` + +Elimina un elemento específico del timeline. + +```ts +{ + trackId: string; + elementId: string; +} +``` + +Uso: +- borrar clips, +- borrar textos, +- borrar stickers, +- borrar efectos standalone si aplica. + +--- + +### `add_text` + +Agrega texto visual al timeline. + +```ts +{ + text: string; + start: number; + end: number; + position: "top" | "center" | "bottom"; + style?: "plain" | "subtitle" | "hook" | "label"; +} +``` + +Uso: +- subtítulos, +- hooks, +- títulos, +- labels, +- texto explicativo. + +--- + +### `update_text` + +Actualiza un texto existente. + +```ts +{ + trackId: string; + elementId: string; + text?: string; + start?: number; + end?: number; + position?: "top" | "center" | "bottom"; +} +``` + +Uso: +- corregir texto, +- cambiar timing, +- mover captions/hooks. + +--- + +### `add_sticker` + +Agrega un sticker al timeline. + +```ts +{ + stickerId: string; + start: number; + end: number; + position: "top-left" | "top-right" | "center" | "bottom-left" | "bottom-right"; +} +``` + +Uso: +- agregar elementos visuales simples, +- reacciones, +- énfasis gráfico. + +--- + +### `set_volume` + +Ajusta volumen de un elemento de audio/video. + +```ts +{ + trackId: string; + elementId: string; + volume: number; // 0–1 +} +``` + +Uso: +- bajar música, +- subir voz, +- mutear clip, +- balance básico. + +--- + +### `apply_effect` + +Aplica un efecto existente a un clip. + +```ts +{ + trackId: string; + elementId: string; + effectType: "blur"; + params?: { + intensity?: number; + }; +} +``` + +Uso: +- aplicar blur. + +> El repo actualmente tiene infraestructura de efectos, pero el efecto real registrado parece ser solo `blur`. Corrección de color/LUTs todavía no está disponible. + +--- + +## Tools existentes o en standby + +### `transcribe_video` + +Transcribe audio de video/audio usando Whisper local. + +```ts +{ + assetId?: string; + language?: string; + modelId?: string; +} +``` + +Uso: +- subtítulos, +- captions, +- búsqueda de frases exactas, +- edición por texto. + +Estado: +- implementada, +- útil como herramienta secundaria, +- no debe ser la tool principal para entender el video si Gemini puede cargar el asset multimodalmente. + +--- + +## Ideas descartadas o diferidas + +### `analyze_video({ question })` + +Descartada como primitive principal porque mete “chat dentro de la tool”. Mejor usar `load_asset_context` y dejar que el agente razone con el contexto cargado. + +### `analyze_asset({ analysisType })` + +Descartada porque `analysisType` introduce categorías artificiales. Gemini debe cargar el asset completo al contexto; el agente decide qué hacer después. + +### `remove_silences` + +Diferida/no prioritaria porque se puede expresar como flujo con `cut_segment`. + +### `generate_subtitles` + +Diferida porque se puede construir con `transcribe_video` + `add_text`. + +### `color_correct` / `apply_lut` + +Diferida porque el repo actual no parece tener corrección de color/LUTs implementados todavía. + +--- + +## Orden recomendado + +1. `list_project_assets` +2. `list_timeline` +3. `load_asset_context` +4. `cut_segment` +5. `add_text` +6. `add_media_to_timeline` +7. `delete_element` +8. `set_volume` +9. `add_sticker` +10. `apply_effect` + +Infra interna asociada: + +- `get_asset_context` / cache lookup para `assetId -> Gemini fileUri/status/summary`, no necesariamente visible para el LLM. diff --git a/openspec/changes/gemini-analyze-video-fase-1/exploration.md b/openspec/changes/gemini-analyze-video-fase-1/exploration.md new file mode 100644 index 00000000..8429f8fe --- /dev/null +++ b/openspec/changes/gemini-analyze-video-fase-1/exploration.md @@ -0,0 +1,292 @@ +# Exploration: `analyze_video` — Gemini Video Understanding (Phase 1) + +## Current State + +### Shipped infrastructure + +Three completed changes underpin this work: + +| Change | What shipped | +|--------|-------------| +| `infra-habilitadora-fase-1` | Full agent pipeline: orchestrator (client-side), tool registry, mock tool, context adapter, system prompt, chat UI | +| `gemini-adapter-fase-1` | `GeminiAdapter` behind `ProviderAdapter` seam — text-only `chat()` via `@google/generative-ai` v0.24.0 | +| `transcribe-video-fase-1` | `transcribe_video` tool — Whisper-based, client-side execution, returns `TranscriptionToolResult` with segments + timestamps. Fully implemented and verified. | + +### How the agent pipeline works today + +``` +User message → ChatPanel → orchestrator.run(messages, context) + → POST /api/agent/chat (server) + → createProvider(config) → GeminiAdapter.chat({ messages, systemPrompt, tools }) + → model.generateContent({ contents, systemInstruction, tools }) + → { content, toolCalls? } + → resolveToolCalls(toolCalls, context) (client) + → toolRegistry.get(name).execute(args, context) + → ToolResult → appended as tool_result message + → Loop back to POST (max 8 iterations) +``` + +**Key architectural facts**: +- **Server route** (`route.ts`): Stateless, holds `LLM_API_KEY`, creates adapter, calls `chat()`. Exposes tool schemas to the LLM but NEVER executes tools. +- **Client orchestrator**: Holds `AgentContext`, resolves tool calls, executes tools in-browser where `EditorCore` is accessible. +- **Tools are client-side only**: `ToolDefinition.execute()` runs in the browser, has access to `EditorContextAdapter.resolveAssetFile()` and `EditorCore`. +- **Tool schemas are duplicated**: `route.ts` has `providerToolDefs` (stub `execute`) for the LLM; `toolRegistry` has real definitions for execution. + +### Gemini SDK already installed + +`@google/generative-ai` v0.24.0 is in `apps/web/package.json`. The `GeminiAdapter` uses `GoogleGenerativeAI` → `getGenerativeModel()` → `generateContent()`. The SDK also exports: + +- **`GoogleAIFileManager`** from `@google/generative-ai/server` — server-side file upload/management +- **`FileState`** enum — `PROCESSING`, `ACTIVE`, `FAILED` +- **`FileDataPart`** — `{ fileData: { fileUri, mimeType } }` content part for `generateContent()` + +### EditorContextAdapter + +Already provides the sanctioned path for tools to obtain a `File`: + +```ts +EditorContextAdapter.resolveAssetFile(assetId?: string): File | null +EditorContextAdapter.getAssetHasAudio(assetId: string): boolean | undefined +``` + +--- + +## Affected Areas + +| File | Why affected | +|------|-------------| +| `apps/web/src/agent/tools/analyze-video.tool.ts` | **NEW** — `analyze_video` tool definition (client-side) | +| `apps/web/src/app/api/agent/analyze-video/route.ts` | **NEW** — server route: upload video to Gemini File API + poll + query | +| `apps/web/src/agent/orchestrator.ts` | Add side-effect import of new tool | +| `apps/web/src/app/api/agent/chat/route.ts` | Add `analyze_video` to `providerToolDefs`, optionally remove/demote `transcribe_video` | +| `apps/web/src/agent/system-prompt.ts` | May update guidance to prefer `analyze_video` | +| `apps/web/src/components/editor/panels/chat/message-bubble.tsx` | May add rendering for `AnalyzeVideoResult` shape | + +**Unchanged** (confirmed safe): +- `providers/gemini.ts` — adapter handles text chat; video goes through a separate route using `GoogleAIFileManager` +- `providers/types.ts` — `ProviderAdapter` contract unchanged +- `agent/types.ts` — `AgentContext`, `ToolDefinition` unchanged +- `agent/context.ts` — `resolveAssetFile()` already returns `File` +- `tools/transcribe-video.tool.ts` — stays registered, just de-emphasized in tool list + +--- + +## Critical Constraint: File API Is Server-Side Only + +The Gemini Files API requires the API key and can only be called server-side. The current architecture has tools executing **client-side**. This means `analyze_video` cannot be a pure client-side tool like `transcribe_video`. It needs a **server route** to handle the upload-to-Gemini flow. + +### Proposed flow + +``` +LLM decides: analyze_video({ assetId: "v1", question: "What happens in this video?" }) + → Client orchestrator resolves tool call + → tool.execute(args, context) + → EditorContextAdapter.resolveAssetFile("v1") → File blob + → POST /api/agent/analyze-video (FormData: video file + question + assetId) + → Server: GoogleAIFileManager.uploadFile(buffer, { mimeType, displayName }) + → Server: poll file.state until ACTIVE (or timeout) + → Server: model.generateContent([ question, { fileData: { fileUri, mimeType } }]) + → Server: return { answer, duration, ... } + → Tool returns structured AnalyzeVideoResult +``` + +### Why not inline base64? + +Gemini supports `InlineDataPart` with base64-encoded video, but: +- **Size limit**: Inline data is practical only for files under ~20MB raw (~27MB base64). Real video projects regularly exceed this. +- **No reuse**: The file is sent every call. The File API gives a `fileUri` that persists (up to 48h free / 7d paid), enabling reuse. +- **Processing state**: The File API handles video decoding, audio extraction, and indexing server-side. Inline data forces the model to process on every call. + +**Verdict**: Use the File API for production readiness. Phase 1 uses the File API exclusively. + +--- + +## Approaches + +### 1. Dedicated `/api/agent/analyze-video` Route (Recommended) + +Create a new server route that handles the full upload→poll→query flow. The client-side tool sends the video file as FormData, the server does everything Gemini-side, and returns a structured answer. + +- **Pros**: + - Clean separation: upload+analyze logic is entirely server-side (no API key leakage) + - Reuses existing `GeminiAdapter` pattern (same SDK, same config) + - The tool on the client side is thin — just FormData construction and response parsing + - Natural place for future caching (fileUri by assetId) + - Can report processing progress via streaming in Phase 2 +- **Cons**: + - New route to maintain + - Large FormData upload adds latency (video files can be hundreds of MB) + - Polling loop blocks the server request (video processing can take 30s-5min for long videos) +- **Effort**: Medium + +### 2. Extend `/api/agent/chat` Route for Multimodal + +Add multimodal content support to the existing chat route. The client sends video content parts alongside text messages. The server handles upload transparently within the adapter. + +- **Pros**: + - Single route, single contract + - More "correct" architecturally — the chat route becomes multimodal +- **Cons**: + - **Major refactor of the adapter contract** — `ProviderAdapter.chat()` currently takes text-only `ChatMessage[]`. Adding binary content parts requires interface changes to `ChatMessage`, `ProviderAdapter`, and both adapters. + - Breaks the clean text-only abstraction that `route.ts` depends on + - The upload/poll cycle doesn't fit neatly into the synchronous `chat()` call + - Cross-contaminates text chat and file upload concerns +- **Effort**: High + +### 3. Upload-Then-Chat Two-Step + +Split into two routes: `POST /api/agent/upload-video` (upload + poll → return fileUri) and `POST /api/agent/chat` (modified to accept fileUri references). The client tool orchestrates both calls. + +- **Pros**: + - Separation of upload and query concerns + - fileUri can be cached client-side and reused across queries without re-uploading +- **Cons**: + - Two round trips for a single tool call (upload, then query) + - Client must manage fileUri lifecycle (when to re-upload) + - More complex tool execution logic + - The chat route still needs multimodal content support (same con as Approach 2) +- **Effort**: Medium-High + +--- + +## Recommendation + +**Approach 1: Dedicated `/api/agent/analyze-video` route.** + +Rationale: +1. **Minimal contract changes**: No changes to `ProviderAdapter`, `ChatMessage`, or the existing chat route. The new route is self-contained. +2. **API key safety**: All Gemini interaction happens server-side. The client tool only sends a FormData blob. +3. **Matches the existing pattern**: Just as `transcribe_video` is a client tool that calls services, `analyze_video` is a client tool that calls a server endpoint. The server endpoint is the Gemini File API equivalent of the transcription service. +4. **Cache-friendly**: The route can return the `fileUri` in the response, and in Phase 2, we add a KV cache (Upstash Redis is already installed) keyed by `assetId` to skip re-uploads. +5. **Thin scope**: The tool is `analyze_video({ assetId?, question })` → answer. No editing actions, no timeline mutations. + +--- + +## Recommended Tool Design + +```typescript +// analyze-video.tool.ts + +interface AnalyzeVideoArgs { + assetId?: string; + question: string; +} + +interface AnalyzeVideoResult { + assetName: string; + question: string; + answer: string; + duration: number; +} + +// OR error: +interface AnalyzeVideoError { + error: string; +} +``` + +### Server route contract + +``` +POST /api/agent/analyze-video +Content-Type: multipart/form-data + +Fields: + - video: File (the video blob) + - question: string + - assetId: string (for display/logging) + - mimeType: string (e.g., "video/mp4") + +Response: + { answer: string, duration: number } + OR { error: string } +``` + +### Processing on the server + +``` +1. Parse FormData → extract video buffer + question + mimeType +2. GoogleAIFileManager.uploadFile(buffer, { mimeType, displayName: assetId }) +3. Poll file.state every 5s until ACTIVE (max 120s timeout) +4. model.generateContent([ + { text: question }, + { fileData: { fileUri: file.uri, mimeType: file.mimeType } } + ]) +5. Return { answer: response.text(), duration } +``` + +### Asset resolution + +Reuses the same pattern as `transcribe_video`: +1. If `assetId` provided → resolve that specific asset +2. If only one video/audio asset → use it +3. If multiple → return error asking user to specify + +### De-emphasizing `transcribe_video` + +In `route.ts`, replace `transcribe_video` in `providerToolDefs` with `analyze_video`. The tool stays registered in the client-side registry (available if needed) but the LLM no longer sees it in its tool list. + +```typescript +// route.ts — updated providerToolDefs +const providerToolDefs: ToolDefinition[] = [ + { + name: "analyze_video", + description: "Analyzes a video using Gemini multimodal understanding. Can answer questions about visual content, audio, speech, and overall composition. Returns a structured answer.", + parameters: [ + { key: "assetId", type: "string", required: false }, + { key: "question", type: "string", required: true }, + ], + execute: async () => ({}), // stub — never called server-side + }, +]; +``` + +--- + +## Caching Considerations (Phase 1 vs Phase 2) + +### Phase 1 (this change): No caching +- Every `analyze_video` call uploads the video fresh +- Simple, predictable, no state management +- Downside: redundant uploads for repeated questions about the same video + +### Phase 2: File URI cache via Upstash Redis +- `Upstash Redis` is already a project dependency (`@upstash/redis`, `@upstash/ratelimit`) +- Key: `gemini-file:{sha256hash}` or `gemini-file:{assetId}:{fileSize}` +- Value: `{ fileUri, mimeType, expiresAt }` +- On each call: check cache → if valid, skip upload → use cached fileUri +- Need to handle: file mutation (re-upload if asset changes), TTL alignment with Gemini's expiration + +--- + +## Risks + +1. **Upload size and latency**: Video files can be 100MB+. Uploading to the Gemini File API from a server route adds significant latency. The Next.js route has a default body size limit that may need configuration. **Mitigation**: Phase 1 sets a reasonable max file size (e.g., 500MB). Configure `bodySizeLimit` in the Next.js route config. + +2. **Processing polling blocks the request**: After upload, Gemini needs to process the video (decode frames, extract audio, index). For long videos this can take minutes. The server route holds open during polling. **Mitigation**: Set a polling timeout (e.g., 120s). Return a clear error if processing exceeds the limit. Phase 2 can add async processing with a status check endpoint. + +3. **SDK uploadFile takes file path, not Buffer**: The `GoogleAIFileManager.uploadFile()` API officially takes a `filePath: string`. The docs mention `Buffer` support in the API review, but the TypeScript types may not reflect this. **Mitigation**: If `uploadFile` doesn't accept Buffer directly, use the raw Gemini REST API (`POST /upload/v1beta/files`) with `fetch` for the upload, then use the SDK for `generateContent`. OR write to `/tmp` (works in Node.js but not ideal in serverless). + +4. **`@google/generative-ai` is "deprecated"**: Context7 marks this SDK as deprecated, superseded by `@google/genai`. The newer SDK may have better File API support. **Mitigation**: Phase 1 ships with `@google/generative-ai` (already installed). Note SDK migration as Phase 2 tech debt. The File API behavior is identical across SDKs — only the client library wrapper differs. + +5. **Model availability**: Video understanding requires Gemini 2.5 Flash or Pro (or 1.5 Pro). If the user's `LLM_MODEL` is set to a text-only model, the analyze route needs a separate model config. **Mitigation**: The analyze route reads `LLM_MODEL` but can fall back to `gemini-2.5-flash` if the configured model doesn't support multimodal. Or add a separate `GEMINI_VIDEO_MODEL` env var. + +6. **Serverless timeout**: Next.js on Cloudflare (the deployment target) has request timeout limits. Video processing may exceed these. **Mitigation**: Phase 1 targets local development first. For Cloudflare deployment, use a background worker or queue system in Phase 2. + +7. **API key exposure surface**: The new route handles file uploads and must validate inputs carefully to prevent abuse. **Mitigation**: Use the existing `@upstash/ratelimit` for rate limiting. Validate file MIME type server-side. + +8. **No structured timestamps in Phase 1**: Gemini's response about a video includes natural language time references ("at 0:45 the camera pans") but not structured timestamp objects. **Mitigation**: Accept natural language in Phase 1. Phase 2 can add prompt engineering or response parsing for structured timestamps. + +--- + +## Ready for Proposal + +**Yes.** The exploration identifies: +- One new client-side tool file (`analyze-video.tool.ts`) +- One new server route (`/api/agent/analyze-video/route.ts`) +- Minor wiring changes (orchestrator import, route tool defs update) +- Clear de-emphasis path for `transcribe_video` +- No changes to `ProviderAdapter`, `ChatMessage`, or existing adapter code +- Clear Phase 2 boundary (caching, structured timestamps, progress reporting, async processing) + +**Next**: Run `sdd-propose` with this exploration as input. diff --git a/openspec/changes/gemini-analyze-video-fase-1/proposal.md b/openspec/changes/gemini-analyze-video-fase-1/proposal.md new file mode 100644 index 00000000..b0911042 --- /dev/null +++ b/openspec/changes/gemini-analyze-video-fase-1/proposal.md @@ -0,0 +1,62 @@ +# Proposal: Gemini Analyze Video Phase 1 + +## Intent + +Make Gemini video understanding the primary path for video questions so the agent can inspect the actual selected video, not just a transcript. Keep the API key server-side and avoid widening the existing provider/chat contract. + +## Scope + +### In Scope +- Add tool `analyze_video` with args `{ assetId?: string, question: string }`. +- Send the resolved video file to a new server route that uploads/polls Gemini Files API and returns a natural-language answer with optional timestamp hints. +- Expose `analyze_video` as the primary video-understanding tool and de-emphasize `transcribe_video` in LLM-visible tool definitions/prompting. + +### Out of Scope +- Editing actions or timeline mutations. +- `@asset` UI, streaming/progress UX, and robust caching for large videos. + +## Capabilities + +### New Capabilities +- None + +### Modified Capabilities +- `agent-session-shell`: tool registry/API proxy behavior now prioritizes `analyze_video` for video understanding instead of `transcribe_video` as the primary exposed tool. +- `agent-context-bridge`: system prompt guidance must describe `analyze_video` as the preferred tool for questions about loaded video assets. + +## Approach + +Use a dedicated `POST /api/agent/analyze-video` route. The client tool resolves the asset `File`, sends `FormData`, and the server performs Gemini native upload → processing poll → multimodal query. Keep `ProviderAdapter` text-chat only; keep `transcribe_video` registered as standby. + +## Affected Areas + +| Area | Impact | Description | +|------|--------|-------------| +| `apps/web/src/agent/tools/analyze-video.tool.ts` | New | Client tool contract and file resolution | +| `apps/web/src/app/api/agent/analyze-video/route.ts` | New | Server-side Gemini Files API flow | +| `apps/web/src/app/api/agent/chat/route.ts` | Modified | LLM-visible tool definitions prioritize `analyze_video` | +| `apps/web/src/agent/system-prompt.ts` | Modified | Prompt guidance prefers video analysis over transcription | +| `apps/web/src/agent/orchestrator.ts` | Modified | Register new tool | + +## Risks + +| Risk | Likelihood | Mitigation | +|------|------------|------------| +| Large upload / long processing | Med | Timeout, clear errors, phase-2 progress/caching | +| SDK File API friction | Med | Fallback to raw REST upload if needed | +| Wrong model lacks video support | Low | Route-level multimodal-capable model fallback | + +## Rollback Plan + +Remove `analyze_video` wiring, restore `transcribe_video` as the primary exposed tool, and delete the dedicated analyze route. + +## Dependencies + +- Existing Gemini server SDK and server-side `LLM_API_KEY` +- Existing `EditorContextAdapter.resolveAssetFile()` asset resolution path + +## Success Criteria + +- [ ] The LLM can call `analyze_video({ assetId?, question })` and receive a natural-language answer about the real video file. +- [ ] API key remains server-only; no client-side Gemini upload logic is introduced. +- [ ] `transcribe_video` remains available as secondary/standby, not the primary exposed video tool.