From 17eaaaf73e419dcd0dab95045fc6e6d3f6998591 Mon Sep 17 00:00:00 2001 From: Luis Esteban Acevedo Ladino Date: Fri, 24 Apr 2026 07:25:04 -0500 Subject: [PATCH] feat: add native gemini provider --- apps/web/.env.example | 6 + apps/web/package.json | 1 + .../src/agent/__tests__/system-prompt.test.ts | 40 ++ .../agent/providers/__tests__/gemini.test.ts | 502 ++++++++++++++++++ .../agent/providers/__tests__/index.test.ts | 12 + apps/web/src/agent/providers/gemini.ts | 240 +++++++++ apps/web/src/agent/providers/index.ts | 3 + apps/web/src/agent/system-prompt.ts | 8 +- .../tools/__tests__/transcribe-video.test.ts | 140 ++++- .../src/agent/tools/transcribe-video.tool.ts | 35 +- bun.lock | 3 + .../archive-report.md | 59 ++ .../design.md | 155 ++++++ .../exploration.md | 0 .../proposal.md | 65 +++ .../specs/agent-session-shell/spec.md | 97 ++++ .../2026-04-24-gemini-adapter-fase-1/tasks.md | 28 + .../verify-report.md | 116 ++++ openspec/specs/agent-session-shell/spec.md | 68 ++- 19 files changed, 1572 insertions(+), 6 deletions(-) create mode 100644 apps/web/src/agent/providers/__tests__/gemini.test.ts create mode 100644 apps/web/src/agent/providers/gemini.ts create mode 100644 openspec/changes/archive/2026-04-24-gemini-adapter-fase-1/archive-report.md create mode 100644 openspec/changes/archive/2026-04-24-gemini-adapter-fase-1/design.md rename openspec/changes/{gemini-adapter-fase-1 => archive/2026-04-24-gemini-adapter-fase-1}/exploration.md (100%) create mode 100644 openspec/changes/archive/2026-04-24-gemini-adapter-fase-1/proposal.md create mode 100644 openspec/changes/archive/2026-04-24-gemini-adapter-fase-1/specs/agent-session-shell/spec.md create mode 100644 openspec/changes/archive/2026-04-24-gemini-adapter-fase-1/tasks.md create mode 100644 openspec/changes/archive/2026-04-24-gemini-adapter-fase-1/verify-report.md diff --git a/apps/web/.env.example b/apps/web/.env.example index 77ed9805..c5b16905 100644 --- a/apps/web/.env.example +++ b/apps/web/.env.example @@ -25,3 +25,9 @@ LLM_PROVIDER=openai-compatible LLM_API_KEY=your_openai_api_key_here LLM_MODEL=gpt-4o # LLM_BASE_URL=https://api.openai.com/v1 + +# Gemini provider (set LLM_PROVIDER=gemini to activate) +# LLM_PROVIDER=gemini +# LLM_API_KEY=your_gemini_api_key_here +# LLM_MODEL=gemini-2.0-flash +# Note: LLM_BASE_URL is ignored when using the gemini provider diff --git a/apps/web/package.json b/apps/web/package.json index 0b8a704d..b8f38950 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -42,6 +42,7 @@ "cmdk": "^1.0.0", "culori": "^4.0.2", "drizzle-orm": "^0.44.2", + "@google/generative-ai": "^0.24.0", "embla-carousel-react": "^8.5.1", "eventemitter3": "^5.0.1", "feed": "^5.1.0", diff --git a/apps/web/src/agent/__tests__/system-prompt.test.ts b/apps/web/src/agent/__tests__/system-prompt.test.ts index 5c83fe82..b20d2b2a 100644 --- a/apps/web/src/agent/__tests__/system-prompt.test.ts +++ b/apps/web/src/agent/__tests__/system-prompt.test.ts @@ -37,6 +37,46 @@ describe("buildSystemPrompt", () => { expect(prompt).toContain("NeuralCut video editor"); }); + test("includes asset internal id in media listing", () => { + const context: AgentContext = { + projectId: "proj-1", + activeSceneId: "scene-A", + mediaAssets: [ + { id: "m1", name: "intro.mp4", type: "video", duration: 30 }, + ], + playbackTimeMs: 0, + }; + + const prompt = buildSystemPrompt(context); + + expect(prompt).toContain("[id: m1]"); + }); + + test("includes assetId instruction when media assets are present", () => { + const context: AgentContext = { + projectId: "proj-1", + activeSceneId: "scene-A", + mediaAssets: [ + { id: "m1", name: "conclu.mov", type: "video", duration: 60 }, + ], + playbackTimeMs: 0, + }; + + const prompt = buildSystemPrompt(context); + + expect(prompt).toContain("IMPORTANT"); + expect(prompt).toContain("assetId"); + expect(prompt).toContain('internal "id" value'); + expect(prompt).toContain("NOT the filename"); + }); + + test("omits assetId instruction when no media assets", () => { + const prompt = buildSystemPrompt(BASE_CONTEXT); + + expect(prompt).not.toContain("IMPORTANT"); + expect(prompt).not.toContain("assetId"); + }); + test("is valid when no media assets are loaded", () => { const context: AgentContext = { projectId: "proj-2", diff --git a/apps/web/src/agent/providers/__tests__/gemini.test.ts b/apps/web/src/agent/providers/__tests__/gemini.test.ts new file mode 100644 index 00000000..8e5cd3be --- /dev/null +++ b/apps/web/src/agent/providers/__tests__/gemini.test.ts @@ -0,0 +1,502 @@ +import { beforeEach, describe, expect, jest, test } from "bun:test"; +import { SchemaType, type GenerateContentResult } from "@google/generative-ai"; +import type { ProviderConfig } from "@/agent/providers/types"; +import type { ToolSchema } from "@/agent/types"; + +// --------------------------------------------------------------------------- +// Mock the Google Generative AI SDK before the adapter module loads +// --------------------------------------------------------------------------- + +const mockGenerateContent = jest.fn(); + +jest.mock("@google/generative-ai", () => ({ + GoogleGenerativeAI: class MockGoogleGenerativeAI { + getGenerativeModel(_opts: { model: string }) { + return { generateContent: mockGenerateContent }; + } + constructor(_opts: { apiKey: string }) {} + }, + SchemaType: { + STRING: "string", + NUMBER: "number", + BOOLEAN: "boolean", + OBJECT: "object", + }, +})); + +// Import after mock is set up +import { + GeminiAdapter, + toGeminiContents, + toGeminiTools, + fromGeminiResponse, +} from "@/agent/providers/gemini"; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +const TEST_CONFIG: ProviderConfig = { + provider: "gemini", + apiKey: "test-gemini-key", + model: "gemini-2.0-flash", +}; + +const SAMPLE_TOOLS: ToolSchema[] = [ + { + name: "transcribe_video", + description: "Transcribes a video asset", + parameters: [ + { key: "assetId", type: "string", required: true }, + { key: "language", type: "string", required: false }, + ], + }, +]; + +function makeGeminiResponse(overrides: { + text?: string; + functionCalls?: Array<{ name: string; args: Record }>; +}): GenerateContentResult { + const parts: Array> = []; + + if (overrides.text !== undefined) { + parts.push({ text: overrides.text }); + } + + if (overrides.functionCalls) { + for (const fc of overrides.functionCalls) { + parts.push({ functionCall: { name: fc.name, args: fc.args } }); + } + } + + return { + response: { + candidates: [ + { + content: { parts }, + }, + ], + }, + } as unknown as GenerateContentResult; +} + +function makeChatMessage( + role: "user" | "assistant" | "system" | "tool_result", + content: string, + extra?: Record, +) { + return { + id: `msg-${Math.random().toString(36).slice(2)}`, + role, + content, + timestamp: Date.now(), + ...extra, + }; +} + +// --------------------------------------------------------------------------- +// Tests: toGeminiContents (Task 4.1) +// --------------------------------------------------------------------------- + +describe("toGeminiContents", () => { + test("maps user message to Gemini user role", () => { + const messages = [makeChatMessage("user", "Hello")]; + const contents = toGeminiContents(messages); + + expect(contents).toHaveLength(1); + expect(contents[0].role).toBe("user"); + expect(contents[0].parts).toEqual([{ text: "Hello" }]); + }); + + test("maps assistant message to Gemini model role", () => { + const messages = [makeChatMessage("assistant", "Hi there!")]; + const contents = toGeminiContents(messages); + + expect(contents).toHaveLength(1); + expect(contents[0].role).toBe("model"); + expect(contents[0].parts).toEqual([{ text: "Hi there!" }]); + }); + + test("maps assistant with toolCalls to model role with functionCall parts", () => { + const messages = [ + makeChatMessage("assistant", "Let me check.", { + toolCalls: [ + { + id: "tc_1", + name: "transcribe_video", + args: { assetId: "vid-123" }, + }, + ], + }), + ]; + const contents = toGeminiContents(messages); + + expect(contents).toHaveLength(1); + expect(contents[0].role).toBe("model"); + expect(contents[0].parts).toEqual([ + { text: "Let me check." }, + { + functionCall: { + name: "transcribe_video", + args: { assetId: "vid-123" }, + }, + }, + ]); + }); + + test("maps tool_result to function role with functionResponse", () => { + const messages = [ + makeChatMessage("assistant", "", { + toolCalls: [ + { + id: "tc_1", + name: "transcribe_video", + args: { assetId: "vid-123" }, + }, + ], + }), + makeChatMessage("tool_result", '{"transcript": "Hello world"}', { + toolCallId: "tc_1", + }), + ]; + const contents = toGeminiContents(messages); + + // First: model (assistant with toolCalls) + expect(contents[0].role).toBe("model"); + // Second: function (tool_result) + expect(contents[1].role).toBe("function"); + expect(contents[1].parts).toEqual([ + { + functionResponse: { + name: "transcribe_video", + response: { transcript: "Hello world" }, + }, + }, + ]); + }); + + test("excludes system messages from contents", () => { + const messages = [ + makeChatMessage("system", "You are a video editor."), + makeChatMessage("user", "Hello"), + ]; + const contents = toGeminiContents(messages); + + expect(contents).toHaveLength(1); + expect(contents[0].role).toBe("user"); + }); + + test("handles tool_result with non-JSON content by wrapping in object", () => { + const messages = [ + makeChatMessage("assistant", "", { + toolCalls: [ + { id: "tc_1", name: "some_tool", args: {} }, + ], + }), + makeChatMessage("tool_result", "plain text result", { + toolCallId: "tc_1", + }), + ]; + const contents = toGeminiContents(messages); + + expect(contents[1].parts).toEqual([ + { + functionResponse: { + name: "some_tool", + response: { result: "plain text result" }, + }, + }, + ]); + }); +}); + +// --------------------------------------------------------------------------- +// Tests: toGeminiTools (Task 4.2) +// --------------------------------------------------------------------------- + +describe("toGeminiTools", () => { + test("converts a single tool with parameters", () => { + const declarations = toGeminiTools(SAMPLE_TOOLS); + + expect(declarations).toHaveLength(1); + expect(declarations[0].name).toBe("transcribe_video"); + expect(declarations[0].description).toBe("Transcribes a video asset"); + expect(declarations[0].parameters?.type).toBe(SchemaType.OBJECT); + expect(declarations[0].parameters?.properties).toHaveProperty("assetId"); + expect(declarations[0].parameters?.required).toEqual(["assetId"]); + }); + + test("maps type strings to Gemini SchemaType values", () => { + const tools: ToolSchema[] = [ + { + name: "multi_param_tool", + description: "Tool with all param types", + parameters: [ + { key: "name", type: "string", required: true }, + { key: "count", type: "number", required: false }, + { key: "active", type: "boolean", required: false }, + { key: "meta", type: "object", required: false }, + ], + }, + ]; + + const declarations = toGeminiTools(tools); + const props = declarations[0].parameters?.properties as Record< + string, + { type: string } + >; + + expect(props.name.type).toBe(SchemaType.STRING); + expect(props.count.type).toBe(SchemaType.NUMBER); + expect(props.active.type).toBe(SchemaType.BOOLEAN); + expect(props.meta.type).toBe(SchemaType.OBJECT); + }); + + test("returns empty array when no tools provided", () => { + expect(toGeminiTools([])).toEqual([]); + }); +}); + +// --------------------------------------------------------------------------- +// Tests: fromGeminiResponse (Task 4.3) +// --------------------------------------------------------------------------- + +describe("fromGeminiResponse", () => { + test("extracts text-only response with no toolCalls", () => { + const response = makeGeminiResponse({ text: "Hello from Gemini!" }); + const result = fromGeminiResponse(response); + + expect(result.content).toBe("Hello from Gemini!"); + expect(result.toolCalls).toBeUndefined(); + }); + + test("maps function-call response to toolCalls with non-empty synthesized IDs", () => { + const response = makeGeminiResponse({ + text: undefined, + functionCalls: [ + { name: "transcribe_video", args: { assetId: "vid-1" } }, + ], + }); + const result = fromGeminiResponse(response); + + expect(result.content).toBe(""); + expect(result.toolCalls).toHaveLength(1); + expect(result.toolCalls?.[0].name).toBe("transcribe_video"); + expect(result.toolCalls?.[0].args).toEqual({ assetId: "vid-1" }); + // Non-empty synthesized ID + expect(result.toolCalls?.[0].id).toBeTruthy(); + expect(typeof result.toolCalls?.[0].id).toBe("string"); + }); + + test("two calls produce different IDs", () => { + const response1 = makeGeminiResponse({ + functionCalls: [{ name: "tool_a", args: {} }], + }); + const response2 = makeGeminiResponse({ + functionCalls: [{ name: "tool_a", args: {} }], + }); + + const result1 = fromGeminiResponse(response1); + const result2 = fromGeminiResponse(response2); + + expect(result1.toolCalls?.[0].id).not.toBe( + result2.toolCalls?.[0].id, + ); + }); + + test("handles mixed text and function-call parts", () => { + const response = makeGeminiResponse({ + text: "I'll look that up.", + functionCalls: [{ name: "search", args: { q: "test" } }], + }); + const result = fromGeminiResponse(response); + + expect(result.content).toBe("I'll look that up."); + expect(result.toolCalls).toHaveLength(1); + }); +}); + +// --------------------------------------------------------------------------- +// Tests: error paths (Task 4.4) +// --------------------------------------------------------------------------- + +describe("error paths", () => { + test("throws on empty candidates", () => { + const response = { + response: { candidates: [] }, + }; + + expect(() => fromGeminiResponse(response as any)).toThrow( + "Empty response from Gemini provider", + ); + }); + + test("throws on null candidates", () => { + const response = { + response: { candidates: null }, + }; + + expect(() => fromGeminiResponse(response as any)).toThrow( + "Empty response from Gemini provider", + ); + }); + + test("SDK error propagates from chat()", async () => { + mockGenerateContent.mockRejectedValueOnce( + new Error("API key not valid"), + ); + + const adapter = new GeminiAdapter(TEST_CONFIG); + + expect( + adapter.chat({ + messages: [makeChatMessage("user", "Hello")], + systemPrompt: "", + tools: [], + }), + ).rejects.toThrow("API key not valid"); + }); +}); + +// --------------------------------------------------------------------------- +// Tests: full chat() round-trip (Task 4.6) +// --------------------------------------------------------------------------- + +describe("GeminiAdapter.chat()", () => { + beforeEach(() => { + mockGenerateContent.mockReset(); + }); + + test("calls generateContent with correct contents and no tools", async () => { + mockGenerateContent.mockResolvedValueOnce( + makeGeminiResponse({ text: "Hi!" }), + ); + + const adapter = new GeminiAdapter(TEST_CONFIG); + const result = await adapter.chat({ + messages: [makeChatMessage("user", "Hello")], + systemPrompt: "You are helpful.", + tools: [], + }); + + expect(result.content).toBe("Hi!"); + expect(result.toolCalls).toBeUndefined(); + + const callArgs = mockGenerateContent.mock.calls[0][0]; + expect(callArgs.contents).toHaveLength(1); + expect(callArgs.contents[0].role).toBe("user"); + expect(callArgs.systemInstruction).toBe("You are helpful."); + expect(callArgs.tools).toBeUndefined(); + }); + + test("calls generateContent with tools and systemInstruction", async () => { + mockGenerateContent.mockResolvedValueOnce( + makeGeminiResponse({ + functionCalls: [ + { + name: "transcribe_video", + args: { assetId: "vid-1" }, + }, + ], + }), + ); + + const adapter = new GeminiAdapter(TEST_CONFIG); + const result = await adapter.chat({ + messages: [makeChatMessage("user", "Transcribe this")], + systemPrompt: "Video editing assistant", + tools: SAMPLE_TOOLS, + }); + + expect(result.toolCalls).toHaveLength(1); + expect(result.toolCalls?.[0].name).toBe("transcribe_video"); + + const callArgs = mockGenerateContent.mock.calls[0][0]; + expect(callArgs.systemInstruction).toBe("Video editing assistant"); + expect(callArgs.tools).toHaveLength(1); + expect(callArgs.tools[0].functionDeclarations).toHaveLength(1); + }); + + test("passes multi-turn conversation through to Gemini", async () => { + mockGenerateContent.mockResolvedValueOnce( + makeGeminiResponse({ text: "Sure!" }), + ); + + const adapter = new GeminiAdapter(TEST_CONFIG); + await adapter.chat({ + messages: [ + makeChatMessage("user", "First question"), + makeChatMessage("assistant", "First answer"), + makeChatMessage("user", "Second question"), + ], + systemPrompt: "", + tools: [], + }); + + const callArgs = mockGenerateContent.mock.calls[0][0]; + expect(callArgs.contents).toHaveLength(3); + expect(callArgs.contents[0].role).toBe("user"); + expect(callArgs.contents[1].role).toBe("model"); + expect(callArgs.contents[2].role).toBe("user"); + }); + + test("does not include systemInstruction when systemPrompt is empty", async () => { + mockGenerateContent.mockResolvedValueOnce( + makeGeminiResponse({ text: "OK" }), + ); + + const adapter = new GeminiAdapter(TEST_CONFIG); + await adapter.chat({ + messages: [makeChatMessage("user", "Hi")], + systemPrompt: "", + tools: [], + }); + + const callArgs = mockGenerateContent.mock.calls[0][0]; + expect(callArgs.systemInstruction).toBeUndefined(); + }); + + test("ignores baseUrl in config — Gemini still works", async () => { + mockGenerateContent.mockResolvedValueOnce( + makeGeminiResponse({ text: "Hello!" }), + ); + + const configWithBaseUrl: ProviderConfig = { + provider: "gemini", + apiKey: "test-gemini-key", + model: "gemini-2.0-flash", + baseUrl: "https://should-be-ignored.example.com", + }; + + const adapter = new GeminiAdapter(configWithBaseUrl); + const result = await adapter.chat({ + messages: [makeChatMessage("user", "Hi")], + systemPrompt: "", + tools: [], + }); + + expect(result.content).toBe("Hello!"); + expect(result.toolCalls).toBeUndefined(); + }); + + test("returns text when tools are provided but Gemini responds with text only", async () => { + mockGenerateContent.mockResolvedValueOnce( + makeGeminiResponse({ text: "I don't need any tools for this." }), + ); + + const adapter = new GeminiAdapter(TEST_CONFIG); + const result = await adapter.chat({ + messages: [makeChatMessage("user", "Just chat")], + systemPrompt: "You are helpful.", + tools: SAMPLE_TOOLS, + }); + + expect(result.content).toBe("I don't need any tools for this."); + expect(result.toolCalls).toBeUndefined(); + + // Verify tools were still sent to Gemini + const callArgs = mockGenerateContent.mock.calls[0][0]; + expect(callArgs.tools).toBeDefined(); + expect(callArgs.tools[0].functionDeclarations).toHaveLength(1); + }); +}); diff --git a/apps/web/src/agent/providers/__tests__/index.test.ts b/apps/web/src/agent/providers/__tests__/index.test.ts index 913475a5..73478897 100644 --- a/apps/web/src/agent/providers/__tests__/index.test.ts +++ b/apps/web/src/agent/providers/__tests__/index.test.ts @@ -1,6 +1,7 @@ import { describe, expect, test } from "bun:test"; import { createProvider } from "@/agent/providers/index"; import { OpenAICompatibleAdapter } from "@/agent/providers/openai-compatible"; +import { GeminiAdapter } from "@/agent/providers/gemini"; import type { ProviderConfig } from "@/agent/providers/types"; describe("createProvider", () => { @@ -28,6 +29,17 @@ describe("createProvider", () => { // Adapter was constructed without throwing — config accepted }); + test("returns GeminiAdapter for gemini provider", () => { + const config: ProviderConfig = { + provider: "gemini", + apiKey: "test-gemini-key", + model: "gemini-2.0-flash", + }; + + const adapter = createProvider(config); + expect(adapter).toBeInstanceOf(GeminiAdapter); + }); + test("throws Error for unknown provider", () => { const config: ProviderConfig = { provider: "unknown-provider", diff --git a/apps/web/src/agent/providers/gemini.ts b/apps/web/src/agent/providers/gemini.ts new file mode 100644 index 00000000..33961e3e --- /dev/null +++ b/apps/web/src/agent/providers/gemini.ts @@ -0,0 +1,240 @@ +import { + GoogleGenerativeAI, + type Content, + type FunctionDeclaration, + type FunctionDeclarationSchema, + type GenerateContentResult, + SchemaType, +} from "@google/generative-ai"; +import type { ChatMessage, ToolCall, ToolSchema } from "@/agent/types"; +import type { + ProviderAdapter, + ProviderConfig, + ProviderResponse, +} from "./types"; + +// --------------------------------------------------------------------------- +// Internal conversion helpers +// --------------------------------------------------------------------------- + +/** + * Maps internal type strings to Gemini SchemaType values. + */ +const TYPE_MAP: Record = { + string: SchemaType.STRING, + number: SchemaType.NUMBER, + boolean: SchemaType.BOOLEAN, + object: SchemaType.OBJECT, +}; + +/** + * Converts internal ChatMessages into Gemini Content[]. + * + * - `user` → role "user", text part + * - `assistant` → role "model", text part + functionCall parts from toolCalls + * - `tool_result`→ role "function", functionResponse part + * - `system` → skipped (handled separately via systemInstruction) + */ +export function toGeminiContents(messages: ChatMessage[]): Content[] { + const contents: Content[] = []; + + for (const msg of messages) { + if (msg.role === "system") { + continue; + } + + if (msg.role === "user") { + contents.push({ + role: "user", + parts: [{ text: msg.content }], + }); + } else if (msg.role === "assistant") { + const parts: Content["parts"] = []; + + if (msg.content) { + parts.push({ text: msg.content }); + } + + if (msg.toolCalls && msg.toolCalls.length > 0) { + for (const tc of msg.toolCalls) { + parts.push({ + functionCall: { name: tc.name, args: tc.args }, + }); + } + } + + contents.push({ role: "model", parts }); + } else if (msg.role === "tool_result") { + // Gemini uses the function name to correlate responses, not IDs. + // We parse the content as JSON for the response object; fall back + // to wrapping it if it's not valid JSON. + let responseData: object; + try { + responseData = JSON.parse(msg.content) as object; + } catch { + responseData = { result: msg.content }; + } + + // Derive the function name from the toolCallId → find the matching + // tool name in prior messages. If not found, use a placeholder. + const toolName = findToolNameForResult(messages, msg.toolCallId); + + contents.push({ + role: "function", + parts: [ + { + functionResponse: { + name: toolName, + response: responseData, + }, + }, + ], + }); + } + } + + return contents; +} + +/** + * Walks backward through messages to find the function name associated with + * a toolCallId. This is needed because Gemini's functionResponse uses names, + * not IDs. + */ +function findToolNameForResult( + messages: ChatMessage[], + toolCallId?: string, +): string { + if (!toolCallId) return "unknown"; + + for (let i = messages.length - 1; i >= 0; i--) { + const msg = messages[i]; + if (msg.role === "assistant" && msg.toolCalls) { + const match = msg.toolCalls.find((tc) => tc.id === toolCallId); + if (match) return match.name; + } + } + + return "unknown"; +} + +/** + * Converts internal ToolSchema[] into Gemini FunctionDeclaration[]. + * + * Maps internal type strings ("string", "number", etc.) to Gemini SchemaType + * enum values. Returns an empty array when no tools are provided. + */ +export function toGeminiTools(tools: ToolSchema[]): FunctionDeclaration[] { + if (tools.length === 0) return []; + + return tools.map((tool) => { + const properties: FunctionDeclarationSchema["properties"] = {}; + + for (const param of tool.parameters) { + const geminiType = TYPE_MAP[param.type] ?? SchemaType.STRING; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (properties as Record)[param.key] = { type: geminiType }; + } + + const requiredParams = tool.parameters + .filter((p) => p.required) + .map((p) => p.key); + + return { + name: tool.name, + description: tool.description, + parameters: { + type: SchemaType.OBJECT, + properties, + ...(requiredParams.length > 0 && { required: requiredParams }), + }, + }; + }); +} + +/** + * Converts a Gemini GenerateContentResult into the provider-agnostic + * ProviderResponse shape. + * + * - Text parts are concatenated into `content`. + * - FunctionCall parts are mapped to `toolCalls[]` with synthesized UUIDs. + * - Throws if no candidates are present. + */ +export function fromGeminiResponse( + response: GenerateContentResult, +): ProviderResponse { + const candidates = response.response.candidates; + if (!candidates || candidates.length === 0) { + throw new Error("Empty response from Gemini provider"); + } + + const parts = candidates[0].content?.parts ?? []; + const textParts: string[] = []; + const toolCalls: ToolCall[] = []; + + for (const part of parts) { + if ("text" in part && part.text !== undefined) { + textParts.push(part.text); + } + if ("functionCall" in part && part.functionCall) { + toolCalls.push({ + id: crypto.randomUUID(), + name: part.functionCall.name, + args: (part.functionCall.args as Record) ?? {}, + }); + } + } + + const content = textParts.join(""); + const result: ProviderResponse = { content }; + + if (toolCalls.length > 0) { + result.toolCalls = toolCalls; + } + + return result; +} + +// --------------------------------------------------------------------------- +// Adapter implementation +// --------------------------------------------------------------------------- + +/** + * Gemini adapter using @google/generative-ai SDK. + * + * Uses the stateless `generateContent()` API — full history is sent with each + * call, matching the existing route contract. System prompts are mapped to + * Gemini's `systemInstruction` field, not as user messages. + */ +export class GeminiAdapter implements ProviderAdapter { + private model; + + constructor(config: ProviderConfig) { + const genAI = new GoogleGenerativeAI(config.apiKey); + this.model = genAI.getGenerativeModel({ model: config.model }); + // config.baseUrl is intentionally ignored for the Gemini provider. + } + + async chat(params: { + messages: ChatMessage[]; + systemPrompt: string; + tools: ToolSchema[]; + }): Promise { + const contents = toGeminiContents(params.messages); + const functionDeclarations = toGeminiTools(params.tools); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const request: any = { contents }; + + if (params.systemPrompt) { + request.systemInstruction = params.systemPrompt; + } + + if (functionDeclarations.length > 0) { + request.tools = [{ functionDeclarations }]; + } + + const result = await this.model.generateContent(request); + return fromGeminiResponse(result); + } +} diff --git a/apps/web/src/agent/providers/index.ts b/apps/web/src/agent/providers/index.ts index 5bbb2cdb..8939f6ad 100644 --- a/apps/web/src/agent/providers/index.ts +++ b/apps/web/src/agent/providers/index.ts @@ -1,5 +1,6 @@ import type { ProviderAdapter, ProviderConfig } from "./types"; import { OpenAICompatibleAdapter } from "./openai-compatible"; +import { GeminiAdapter } from "./gemini"; /** * Factory — resolves a ProviderConfig to a concrete adapter. @@ -12,6 +13,8 @@ export function createProvider(config: ProviderConfig): ProviderAdapter { switch (config.provider) { case "openai-compatible": return new OpenAICompatibleAdapter(config); + case "gemini": + return new GeminiAdapter(config); default: throw new Error(`Unknown LLM provider: ${config.provider}`); } diff --git a/apps/web/src/agent/system-prompt.ts b/apps/web/src/agent/system-prompt.ts index 26b4229c..37407700 100644 --- a/apps/web/src/agent/system-prompt.ts +++ b/apps/web/src/agent/system-prompt.ts @@ -17,7 +17,7 @@ export function buildSystemPrompt( ): string { const mediaSection = context.mediaAssets.length > 0 - ? `Active media assets:\n${context.mediaAssets.map((m) => `- ${m.name} (${m.type}, ${m.duration}s)`).join("\n")}` + ? `Active media assets:\n${context.mediaAssets.map((m) => `- [id: ${m.id}] ${m.name} (${m.type}, ${m.duration}s)`).join("\n")}` : "No media assets loaded."; const parts = [ @@ -28,6 +28,12 @@ export function buildSystemPrompt( mediaSection, ]; + if (context.mediaAssets.length > 0) { + parts.push( + 'IMPORTANT: When calling tools that accept an "assetId" parameter, always use the internal "id" value (e.g., "v1"), NOT the filename or display name.', + ); + } + if (tools && tools.length > 0) { const toolList = tools .map((t) => `- ${t.name}: ${t.description}`) diff --git a/apps/web/src/agent/tools/__tests__/transcribe-video.test.ts b/apps/web/src/agent/tools/__tests__/transcribe-video.test.ts index af52caea..9b1c474d 100644 --- a/apps/web/src/agent/tools/__tests__/transcribe-video.test.ts +++ b/apps/web/src/agent/tools/__tests__/transcribe-video.test.ts @@ -227,7 +227,7 @@ describe("transcribe_video tool — error branches", () => { const result = await tool.execute({ assetId: "v99" }, context); expect(result).toEqual({ - error: "Could not access file for asset v99", + error: 'No asset found with id or name "v99". Available ids: [v1]. Available names: [intro.mp4].', }); }); @@ -282,3 +282,141 @@ describe("transcribe_video tool — error branches", () => { expect(result).toEqual({ error: "Worker not initialized" }); }); }); + +describe("transcribe_video tool — name-based assetId fallback", () => { + beforeEach(() => { + mockResolveAssetFile.mockClear(); + mockGetAssetHasAudio.mockClear(); + mockDecodeAudio.mockClear(); + mockTranscribe.mockClear(); + }); + + test("resolves by filename when no id match (exact name)", async () => { + const fakeFile = new File([], "conclu.mov", { type: "video/quicktime" }); + const context = makeContext({ + mediaAssets: [ + fakeVideoAsset({ id: "abc123", name: "conclu.mov" }), + ], + }); + + mockResolveAssetFile.mockReturnValue(fakeFile); + mockGetAssetHasAudio.mockReturnValue(true); + mockDecodeAudio.mockResolvedValue({ + samples: new Float32Array(16000), + sampleRate: 16000, + }); + mockTranscribe.mockResolvedValue(fakeTranscriptionResult()); + + const tool = toolRegistry.get("transcribe_video"); + const result = await tool.execute({ assetId: "conclu.mov" }, context); + + expect(result).toEqual( + expect.objectContaining({ assetName: "conclu.mov" }), + ); + // Resolved asset must pass its internal id to the adapter + expect(mockResolveAssetFile).toHaveBeenCalledWith("abc123"); + expect(mockGetAssetHasAudio).toHaveBeenCalledWith("abc123"); + }); + + test("resolves by case-insensitive name when no exact id or name match", async () => { + const fakeFile = new File([], "Conclu.MOV", { type: "video/quicktime" }); + const context = makeContext({ + mediaAssets: [ + fakeVideoAsset({ id: "abc123", name: "Conclu.MOV" }), + ], + }); + + mockResolveAssetFile.mockReturnValue(fakeFile); + mockGetAssetHasAudio.mockReturnValue(true); + mockDecodeAudio.mockResolvedValue({ + samples: new Float32Array(16000), + sampleRate: 16000, + }); + mockTranscribe.mockResolvedValue(fakeTranscriptionResult()); + + const tool = toolRegistry.get("transcribe_video"); + const result = await tool.execute({ assetId: "conclu.mov" }, context); + + expect(result).toEqual( + expect.objectContaining({ assetName: "Conclu.MOV" }), + ); + expect(mockResolveAssetFile).toHaveBeenCalledWith("abc123"); + }); + + test("returns ambiguous error when two assets share same name", async () => { + const context = makeContext({ + mediaAssets: [ + fakeVideoAsset({ id: "v1", name: "clip.mp4" }), + fakeVideoAsset({ id: "v2", name: "clip.mp4" }), + ], + }); + + const tool = toolRegistry.get("transcribe_video"); + const result = await tool.execute({ assetId: "clip.mp4" }, context); + + expect(result).toEqual({ + error: 'Ambiguous asset name "clip.mp4" matches multiple assets (clip.mp4, clip.mp4). Specify the internal id: v1, v2.', + }); + }); + + test("returns ambiguous error for case-insensitive duplicates", async () => { + const context = makeContext({ + mediaAssets: [ + fakeVideoAsset({ id: "v1", name: "Clip.MP4" }), + fakeVideoAsset({ id: "v2", name: "CLIP.MP4" }), + ], + }); + + const tool = toolRegistry.get("transcribe_video"); + const result = await tool.execute({ assetId: "clip.mp4" }, context); + + expect(result).toEqual({ + error: 'Ambiguous asset name "clip.mp4" matches multiple assets (Clip.MP4, CLIP.MP4). Specify the internal id: v1, v2.', + }); + }); + + test("prefers exact id match over name match", async () => { + const fakeFile = new File([], "intro.mp4", { type: "video/mp4" }); + // An asset whose id happens to equal another asset's name + const context = makeContext({ + mediaAssets: [ + fakeVideoAsset({ id: "v1", name: "intro.mp4" }), + fakeVideoAsset({ id: "intro.mp4", name: "outro.mp4" }), + ], + }); + + mockResolveAssetFile.mockReturnValue(fakeFile); + mockGetAssetHasAudio.mockReturnValue(true); + mockDecodeAudio.mockResolvedValue({ + samples: new Float32Array(16000), + sampleRate: 16000, + }); + mockTranscribe.mockResolvedValue(fakeTranscriptionResult()); + + const tool = toolRegistry.get("transcribe_video"); + const result = await tool.execute({ assetId: "intro.mp4" }, context); + + // "intro.mp4" matches v1 by id AND the second asset by id exactly + // id lookup is first, so it resolves the second asset (id="intro.mp4") + expect(result).toEqual( + expect.objectContaining({ assetName: "outro.mp4" }), + ); + expect(mockResolveAssetFile).toHaveBeenCalledWith("intro.mp4"); + }); + + test("returns not-found error with available ids and names for totally unknown assetId", async () => { + const context = makeContext({ + mediaAssets: [ + fakeVideoAsset({ id: "v1", name: "intro.mp4" }), + fakeVideoAsset({ id: "v2", name: "outro.mp4" }), + ], + }); + + const tool = toolRegistry.get("transcribe_video"); + const result = await tool.execute({ assetId: "nonexistent.mov" }, context); + + expect(result).toEqual({ + error: 'No asset found with id or name "nonexistent.mov". Available ids: [v1, v2]. Available names: [intro.mp4, outro.mp4].', + }); + }); +}); diff --git a/apps/web/src/agent/tools/transcribe-video.tool.ts b/apps/web/src/agent/tools/transcribe-video.tool.ts index 805cd8f0..e121e7ae 100644 --- a/apps/web/src/agent/tools/transcribe-video.tool.ts +++ b/apps/web/src/agent/tools/transcribe-video.tool.ts @@ -57,10 +57,43 @@ const transcribeVideoTool: ToolDefinition = { let targetAsset: (typeof mediaCandidates)[number] | undefined; if (typedArgs.assetId) { + // 1. Exact internal-id match targetAsset = mediaCandidates.find((a) => a.id === typedArgs.assetId); + + // 2. Fallback: exact name match (LLM sometimes passes filename) if (!targetAsset) { + const nameMatches = mediaCandidates.filter( + (a) => a.name === typedArgs.assetId, + ); + if (nameMatches.length === 1) { + targetAsset = nameMatches[0]; + } else if (nameMatches.length > 1) { + return { + error: `Ambiguous asset name "${typedArgs.assetId}" matches multiple assets (${nameMatches.map((a) => a.name).join(", ")}). Specify the internal id: ${nameMatches.map((a) => `${a.id}`).join(", ")}.`, + }; + } + } + + // 3. Case-insensitive fallback (only if unambiguous) + if (!targetAsset) { + const lower = typedArgs.assetId.toLowerCase(); + const ciMatches = mediaCandidates.filter( + (a) => a.name.toLowerCase() === lower, + ); + if (ciMatches.length === 1) { + targetAsset = ciMatches[0]; + } else if (ciMatches.length > 1) { + return { + error: `Ambiguous asset name "${typedArgs.assetId}" matches multiple assets (${ciMatches.map((a) => a.name).join(", ")}). Specify the internal id: ${ciMatches.map((a) => `${a.id}`).join(", ")}.`, + }; + } + } + + if (!targetAsset) { + const ids = mediaCandidates.map((a) => a.id).join(", "); + const names = mediaCandidates.map((a) => a.name).join(", "); return { - error: `Could not access file for asset ${typedArgs.assetId}`, + error: `No asset found with id or name "${typedArgs.assetId}". Available ids: [${ids}]. Available names: [${names}].`, }; } } else { diff --git a/bun.lock b/bun.lock index a39f2c2f..f1dd7d86 100644 --- a/bun.lock +++ b/bun.lock @@ -20,6 +20,7 @@ "name": "@opencut/web", "version": "0.1.0", "dependencies": { + "@google/generative-ai": "^0.24.0", "@hello-pangea/dnd": "^18.0.1", "@hugeicons/core-free-icons": "^3.3.0", "@hugeicons/react": "^1.1.6", @@ -348,6 +349,8 @@ "@floating-ui/utils": ["@floating-ui/utils@0.2.10", "", {}, "sha512-aGTxbpbg8/b5JfU1HXSrbH3wXZuLPJcNEcZQFMxLs3oSzgtVu6nFPkbbGGUvBcUjKV2YyB9Wxxabo+HEH9tcRQ=="], + "@google/generative-ai": ["@google/generative-ai@0.24.1", "", {}, "sha512-MqO+MLfM6kjxcKoy0p1wRzG3b4ZZXtPI+z2IE26UogS2Cm/XHO+7gGRBh6gcJsOiIVoH93UwKvW4HdgiOZCy9Q=="], + "@hello-pangea/dnd": ["@hello-pangea/dnd@18.0.1", "", { "dependencies": { "@babel/runtime": "^7.26.7", "css-box-model": "^1.2.1", "raf-schd": "^4.0.3", "react-redux": "^9.2.0", "redux": "^5.0.1" }, "peerDependencies": { "react": "^18.0.0 || ^19.0.0", "react-dom": "^18.0.0 || ^19.0.0" } }, "sha512-xojVWG8s/TGrKT1fC8K2tIWeejJYTAeJuj36zM//yEm/ZrnZUSFGS15BpO+jGZT1ybWvyXmeDJwPYb4dhWlbZQ=="], "@hugeicons/core-free-icons": ["@hugeicons/core-free-icons@3.3.0", "", {}, "sha512-qYyr4JQ2eQIHTSTbITvnJvs6ERNK64D9gpwZnf2IyuG0exzqfyABLO/oTB71FB3RZPfu1GbwycdiGSo46apjMQ=="], diff --git a/openspec/changes/archive/2026-04-24-gemini-adapter-fase-1/archive-report.md b/openspec/changes/archive/2026-04-24-gemini-adapter-fase-1/archive-report.md new file mode 100644 index 00000000..3046c859 --- /dev/null +++ b/openspec/changes/archive/2026-04-24-gemini-adapter-fase-1/archive-report.md @@ -0,0 +1,59 @@ +# Archive Report: gemini-adapter-fase-1 + +**Change**: gemini-adapter-fase-1 +**Archived**: 2026-04-24 +**Verification Verdict**: PASS — 14/14 tasks complete, 12/12 spec scenarios compliant, 48/48 tests passing, 0 TypeScript errors + +## Scope Summary + +This change adds **native Gemini provider support for normal chat and tool calling only**. Video upload/file understanding remains deferred to a future change. The `GeminiAdapter` implements the existing `ProviderAdapter` seam via `@google/generative-ai` SDK, supporting stateless `generateContent()` with system prompt mapping, tool schema conversion, and synthesized tool-call IDs. The OpenAI-compatible path is untouched. + +## Engram Artifact Traceability + +| Artifact | Observation ID | +|----------|---------------| +| Proposal | #1016 | +| Spec (delta) | #1019 | +| Design | #1018 | +| Tasks | #1020 | +| Apply Progress | #1021 | +| Verify Report | #1024 | + +## OpenSpec Archive + +**Moved to**: `openspec/changes/archive/2026-04-24-gemini-adapter-fase-1/` + +### Archive Contents +- proposal.md ✅ +- specs/ (agent-session-shell delta) ✅ +- design.md ✅ +- tasks.md ✅ (14/14 tasks complete) +- verify-report.md ✅ +- exploration.md ✅ + +## Specs Synced + +| Domain | Action | Details | +|--------|--------|---------| +| agent-session-shell | Updated | 2 requirements modified (Provider Adapter Interface, Provider Configuration), 2 requirements added (Gemini Free-Form Chat, Gemini Tool Calling), 0 removed | + +### Modified Requirements +- **Provider Adapter Interface**: Expanded factory to support `openai-compatible` and `gemini`; added tool-call ID synthesis mandate; added Gemini adapter selection scenario +- **Provider Configuration**: Added `LLM_PROVIDER` valid value constraint (`openai-compatible`, `gemini`); added Gemini-specific `LLM_BASE_URL` ignoring behavior; added unknown-provider and Gemini-config scenarios + +### Added Requirements +- **Gemini Free-Form Chat**: Plain text chat through Gemini `generateContent()` with system prompt mapped to `systemInstruction` +- **Gemini Tool Calling**: Tool schema → Gemini function declaration conversion, function call response → `ToolCall[]` with synthesized IDs + +## Source of Truth Updated +- `openspec/specs/agent-session-shell/spec.md` — now reflects Gemini provider support + +## Deferred Scope (explicitly out of scope) +- Video/file upload or direct file understanding +- `@asset` references, broader multimodal inputs, image/video reasoning +- Streaming UX or adapter contract changes for streaming +- Safety/grounding/thinking configuration beyond defaults + +## SDD Cycle Complete +The change has been fully explored, proposed, specified, designed, implemented, verified, and archived. +Ready for the next change. diff --git a/openspec/changes/archive/2026-04-24-gemini-adapter-fase-1/design.md b/openspec/changes/archive/2026-04-24-gemini-adapter-fase-1/design.md new file mode 100644 index 00000000..282e6c5d --- /dev/null +++ b/openspec/changes/archive/2026-04-24-gemini-adapter-fase-1/design.md @@ -0,0 +1,155 @@ +# Design: Gemini Adapter — Phase 1 + +## Technical Approach + +Implement a stateless `GeminiAdapter` class behind the existing `ProviderAdapter` seam, following the exact same pattern as `OpenAICompatibleAdapter`. The adapter converts internal `ChatMessage[]`/`ToolSchema[]` into Gemini's `generateContent()` wire format and parses responses back into `ProviderResponse`. A new `"gemini"` case in `createProvider()` activates it when `LLM_PROVIDER=gemini`. Zero changes to orchestrator, route, or client contracts. + +## Architecture Decisions + +### Decision: SDK choice + +| Option | Tradeoff | Decision | +|--------|----------|----------| +| `@google/generative-ai` | Established, stable, v0.21+, `GoogleGenerativeAI` class, `generateContent()` stateless API | ✅ Selected | +| `@google/genai` | Newer unified SDK, less battle-tested, API may still shift | ❌ Rejected | + +**Rationale**: `@google/generative-ai` is the most widely adopted Gemini SDK for Node/TS. It supports `systemInstruction`, `functionDeclaration`, and stateless `generateContent()` — exactly what we need. Consistent with using the mature `openai` SDK for the existing adapter. + +### Decision: Stateless generateContent vs stateful startChat + +| Option | Tradeoff | Decision | +|--------|----------|----------| +| `model.generateContent(contents, config)` | Stateless, full history each call — matches current adapter contract | ✅ Selected | +| `chat.sendMessage()` via `startChat()` | Stateful session, server-managed history | ❌ Rejected | + +**Rationale**: Current architecture POSTs full history from client on every turn. Stateless `generateContent()` maps 1:1 to this pattern. No session management needed. + +### Decision: Tool call ID synthesis + +| Option | Tradeoff | Decision | +|--------|----------|----------| +| `crypto.randomUUID()` | Built-in, zero deps, unique | ✅ Selected | +| `nanoid()` (already a dep) | Shorter IDs, also unique | ❌ Rejected | + +**Rationale**: `crypto.randomUUID()` is universally available in Node/Bun, no import needed, and produces standard UUIDs matching OpenAI's `call_xxx` format semantically. Keeps the adapter self-contained. + +### Decision: System prompt placement + +| Option | Tradeoff | Decision | +|--------|----------|----------| +| `systemInstruction` on `GenerateContentRequest` | Gemini-native, separate from contents | ✅ Selected | +| Prepend as `user` message | Works but dilutes system/user boundary | ❌ Rejected | + +**Rationale**: Gemini SDK supports `systemInstruction` as a first-class config field. Using it keeps system prompt semantically correct and matches Gemini best practices. + +## Data Flow + +``` +Route (chat/route.ts) + │ + ├─ createProvider(config) → GeminiAdapter instance + │ + └─ adapter.chat({ messages, systemPrompt, tools }) + │ + ├─ toGeminiContents(messages) → Content[] (user/model/function) + ├─ toGeminiTools(tools) → FunctionDeclaration[] + │ + └─ model.generateContent({ contents, tools, systemInstruction }) + │ + └─ fromGeminiResponse(response) → ProviderResponse + │ + ├─ content: candidate.content.parts[].text + └─ toolCalls: candidate.content.parts[] + .filter(functionCall) + .map → { id: uuid(), name, args } +``` + +### Message mapping + +| Internal role | Gemini role | Notes | +|---------------|-------------|-------| +| `system` | `systemInstruction` (config) | Not in contents array | +| `user` | `user` part | Direct text mapping | +| `assistant` | `model` part | Includes `functionCall` parts for tool calls | +| `tool_result` | `function` role | Uses `functionResponse: { name, response }` keyed by tool name (not ID) | + +### Type mapping (internal → Gemini) + +| Internal type | Gemini type | +|---------------|-------------| +| `"string"` | `SchemaType.STRING` | +| `"number"` | `SchemaType.NUMBER` | +| `"boolean"` | `SchemaType.BOOLEAN` | +| `"object"` | `SchemaType.OBJECT` | + +## File Changes + +| File | Action | Description | +|------|--------|-------------| +| `apps/web/src/agent/providers/gemini.ts` | Create | GeminiAdapter + conversion helpers (`toGeminiContents`, `toGeminiTools`, `fromGeminiResponse`) | +| `apps/web/src/agent/providers/index.ts` | Modify | Add `"gemini"` case to factory switch | +| `apps/web/src/agent/providers/__tests__/gemini.test.ts` | Create | Conversion tests, error paths, tool call ID synthesis | +| `apps/web/src/agent/providers/__tests__/index.test.ts` | Modify | Add factory test for `"gemini"` → `GeminiAdapter` | +| `apps/web/package.json` | Modify | Add `@google/generative-ai` dependency | +| `apps/web/.env.example` | Modify | Add Gemini example config lines | + +## Interfaces / Contracts + +No interface changes. The adapter implements the existing `ProviderAdapter`: + +```ts +// Reused as-is from providers/types.ts +interface ProviderAdapter { + chat(params: { + messages: ChatMessage[]; + systemPrompt: string; + tools: ToolSchema[]; + }): Promise; +} +``` + +### Internal helper signatures (gemini.ts) + +```ts +function toGeminiContents( + messages: ChatMessage[], +): GenerateContentRequest["contents"]; + +function toGeminiTools( + tools: ToolSchema[], +): FunctionDeclaration[]; + +function fromGeminiResponse( + response: GenerateContentResult, +): ProviderResponse; +``` + +### Tool call ID — name collision handling + +When mapping `tool_result` back to Gemini's `functionResponse`, Gemini keys by function **name** not ID. If multiple calls to the same tool exist in history, we group them by name and send the last result per name. This is acceptable because: + +1. Current tool set (`transcribe_video`) is single-invocation per turn +2. Multi-call same-tool scenarios are deferred to a later change +3. Orchestrator already handles the ID→name correlation client-side + +## Testing Strategy + +| Layer | What to Test | Approach | +|-------|-------------|----------| +| Unit | `toGeminiContents` mapping (user, assistant, tool_result, assistant+toolCalls) | Mock SDK, inspect call args | +| Unit | `toGeminiTools` type casing (string→STRING, etc.) | Direct assertion on output | +| Unit | `fromGeminiResponse` content extraction | Construct Gemini response shape, assert `ProviderResponse` | +| Unit | Tool call ID synthesis (non-empty, unique) | Multiple calls produce different IDs | +| Unit | Error: empty candidates → throws | Assert error message | +| Unit | Error: SDK throws → propagates | Assert error bubbles up | +| Unit | Factory: `"gemini"` → `GeminiAdapter` | Instance check | +| Integration | Full `chat()` round-trip with mocked SDK | End-to-end within adapter | + +## Migration / Rollout + +No migration required. Additive change — setting `LLM_PROVIDER=gemini` activates the new adapter. Existing `openai-compatible` default is untouched. + +## Open Questions + +- [ ] Confirm `@google/generative-ai` latest stable version at impl time (currently ^0.21) +- [ ] Verify Gemini model naming convention user will use (e.g. `gemini-2.0-flash` vs `models/gemini-2.0-flash`) diff --git a/openspec/changes/gemini-adapter-fase-1/exploration.md b/openspec/changes/archive/2026-04-24-gemini-adapter-fase-1/exploration.md similarity index 100% rename from openspec/changes/gemini-adapter-fase-1/exploration.md rename to openspec/changes/archive/2026-04-24-gemini-adapter-fase-1/exploration.md diff --git a/openspec/changes/archive/2026-04-24-gemini-adapter-fase-1/proposal.md b/openspec/changes/archive/2026-04-24-gemini-adapter-fase-1/proposal.md new file mode 100644 index 00000000..6719a596 --- /dev/null +++ b/openspec/changes/archive/2026-04-24-gemini-adapter-fase-1/proposal.md @@ -0,0 +1,65 @@ +# Proposal: Gemini Adapter — Phase 1 + +## Intent + +Add Gemini as a native provider behind the existing provider-agnostic LLM seam so the agent can use Gemini for plain chat and tool calling without changing orchestrator or route contracts. + +## Scope + +### In Scope +- Native `GeminiAdapter` implementing the current `ProviderAdapter` +- Gemini free-form chat using existing stateless full-history flow +- Gemini tool calling, with `transcribe_video` as the first real tool +- Config support for `LLM_PROVIDER=gemini`, `LLM_API_KEY`, `LLM_MODEL` +- Tests and env/docs updates while preserving the OpenAI-compatible path + +### Out of Scope +- Video/file upload or direct file understanding +- `@asset` references, broader multimodal inputs, and image/video reasoning +- Streaming UX or adapter contract changes for streaming +- Safety/grounding/thinking configuration beyond defaults + +## Capabilities + +### New Capabilities +- None + +### Modified Capabilities +- `agent-session-shell`: expand provider selection/configuration so the existing provider adapter contract also supports native Gemini chat and function calling + +## Approach + +Implement a native Gemini SDK-backed adapter and register it in the provider factory. Keep the adapter stateless by mapping full internal message history into Gemini `generateContent()` requests, converting tool schemas/results at the adapter boundary, and synthesizing tool call IDs so internal contracts stay unchanged. + +## Affected Areas + +| Area | Impact | Description | +|------|--------|-------------| +| `apps/web/src/agent/providers/gemini.ts` | New | Gemini request/response/tool conversion | +| `apps/web/src/agent/providers/index.ts` | Modified | Factory registration for `gemini` | +| `apps/web/src/agent/providers/__tests__/gemini.test.ts` | New | Adapter conversion/error tests | +| `apps/web/src/agent/providers/__tests__/index.test.ts` | Modified | Factory coverage | +| `apps/web/package.json` | Modified | Gemini SDK dependency | +| `apps/web/.env.example` | Modified | Gemini env configuration | + +## Risks + +| Risk | Likelihood | Mitigation | +|------|------------|------------| +| Gemini has no tool call IDs | Med | Synthesize unique IDs inside adapter | +| Gemini type casing differs from internal schema | Med | Centralize schema mapping tests | +| Gemini system prompt placement differs | Low | Map system prompt explicitly to Gemini config | + +## Rollback Plan + +Remove the Gemini adapter, factory branch, dependency, and env docs; keep `openai-compatible` as the unchanged default path. + +## Dependencies + +- Gemini JS SDK selection to be verified during implementation + +## Success Criteria + +- [ ] `LLM_PROVIDER=gemini` resolves a native Gemini adapter without breaking `openai-compatible` +- [ ] Gemini returns normal assistant text through the existing route/orchestrator flow +- [ ] Gemini can emit structured tool calls and execute `transcribe_video` through the existing loop diff --git a/openspec/changes/archive/2026-04-24-gemini-adapter-fase-1/specs/agent-session-shell/spec.md b/openspec/changes/archive/2026-04-24-gemini-adapter-fase-1/specs/agent-session-shell/spec.md new file mode 100644 index 00000000..ea9fbc2e --- /dev/null +++ b/openspec/changes/archive/2026-04-24-gemini-adapter-fase-1/specs/agent-session-shell/spec.md @@ -0,0 +1,97 @@ +# Delta for agent-session-shell + +## MODIFIED Requirements + +### Requirement: Provider Adapter Interface + +The system SHALL define a server-side provider adapter interface with a single method `chat(params: ProviderChatParams): Promise`. `ProviderChatParams` MUST include `messages`, `systemPrompt`, and `toolSchemas` in a provider-agnostic internal format. `ProviderChatResponse` MUST include `content: string` and `toolCalls?: ProviderToolCall[]` where `ProviderToolCall { id, name, args }`. Each concrete adapter converts internal tool schemas to its own wire format. The factory MUST support at least `openai-compatible` and `gemini` providers. Adapters whose native API does not assign IDs to tool calls MUST synthesize unique IDs so every returned `ToolCall.id` is a non-empty string. + +(Previously: Phase 1 MAY ship exactly one adapter (OpenAI-compatible) as long as the interface contract remains provider-agnostic.) + +#### Scenario: Adapter translates tool schemas to wire format + +- GIVEN `transcribe_video` has internal parameters `[{ key: "assetId", type: "string", required: false }]` +- WHEN the adapter receives tool schemas before calling the provider +- THEN it converts them to its provider-specific wire format + +#### Scenario: Adapter selected by config — OpenAI-compatible + +- GIVEN `LLM_PROVIDER` is set to `"openai-compatible"` +- WHEN the route resolves the adapter +- THEN the OpenAI-compatible adapter is returned + +#### Scenario: Adapter selected by config — Gemini + +- GIVEN `LLM_PROVIDER` is set to `"gemini"` +- WHEN the route resolves the adapter +- THEN the Gemini adapter is returned + +#### Scenario: Gemini adapter synthesizes tool-call IDs + +- GIVEN the Gemini API returns a function call response with no call ID +- WHEN the Gemini adapter maps the response to `ProviderResponse` +- THEN each `ToolCall` in `toolCalls` has a non-empty `id` field generated by the adapter + +### Requirement: Provider Configuration + +Server-only environment variables `LLM_PROVIDER` (required), `LLM_MODEL` (required), `LLM_API_KEY` (required), and `LLM_BASE_URL` (optional) MUST determine which adapter and model the route uses. `LLM_PROVIDER` MUST accept at least `"openai-compatible"` and `"gemini"`. When `LLM_PROVIDER` is `"gemini"`, `LLM_BASE_URL` MUST be ignored. Configuration MUST be read server-side only — never exposed to the client. + +(Previously: Provider config listed env vars but did not constrain valid provider values or describe Gemini-specific behavior.) + +#### Scenario: Missing required configuration + +- GIVEN `LLM_API_KEY` is not set in the environment +- WHEN the route attempts to initialize the provider +- THEN the call returns 502 with a missing-configuration error + +#### Scenario: Unknown provider + +- GIVEN `LLM_PROVIDER` is set to `"unknown-provider"` +- WHEN the route resolves the adapter +- THEN the factory throws `"Unknown LLM provider: unknown-provider"` + +#### Scenario: Gemini config ignores base URL + +- GIVEN `LLM_PROVIDER` is `"gemini"` and `LLM_BASE_URL` is set +- WHEN the Gemini adapter is created +- THEN `LLM_BASE_URL` is not used in the API call + +## ADDED Requirements + +### Requirement: Gemini Free-Form Chat + +When the Gemini adapter receives a request with no tool schemas (or tools array is empty), it MUST send messages to the Gemini `generateContent` API and return a `ProviderResponse` with `content` set to the assistant text and `toolCalls` undefined. + +#### Scenario: Plain chat through Gemini + +- GIVEN `LLM_PROVIDER` is `"gemini"` and messages contain `[{ role: "user", content: "Hello" }]` +- WHEN the adapter calls Gemini and receives a text-only response +- THEN `ProviderResponse.content` is the assistant text and `toolCalls` is undefined + +#### Scenario: System prompt mapping + +- GIVEN the adapter receives `systemPrompt: "You are a video editor"` +- WHEN it builds the Gemini request +- THEN the system prompt is mapped to Gemini's system instruction field, not as a user message + +### Requirement: Gemini Tool Calling + +When the Gemini adapter receives a request with tool schemas, it MUST convert internal `ToolSchema[]` to Gemini function declarations, call `generateContent` with declarations attached, and if Gemini returns function call parts, map each to a `ToolCall { id, name, args }` with a synthesized non-empty `id`. + +#### Scenario: Gemini returns function calls + +- GIVEN `transcribe_video` is in tool schemas and Gemini responds with a function call part `{ name: "transcribe_video", args: {} }` +- WHEN the adapter maps the response +- THEN `toolCalls` contains one `ToolCall` with `name: "transcribe_video"`, `args: {}`, and a synthesized `id` + +#### Scenario: Gemini returns text despite tools + +- GIVEN tool schemas are provided but Gemini responds with only text +- WHEN the adapter maps the response +- THEN `toolCalls` is undefined and `content` holds the text + +#### Scenario: Tool schema type mapping + +- GIVEN an internal tool schema has parameters `[{ key: "assetId", type: "string", required: true }]` +- WHEN the adapter converts to Gemini function declarations +- THEN the parameter appears in `FunctionDeclaration.parameters.properties` with type `"string"` diff --git a/openspec/changes/archive/2026-04-24-gemini-adapter-fase-1/tasks.md b/openspec/changes/archive/2026-04-24-gemini-adapter-fase-1/tasks.md new file mode 100644 index 00000000..499ab63e --- /dev/null +++ b/openspec/changes/archive/2026-04-24-gemini-adapter-fase-1/tasks.md @@ -0,0 +1,28 @@ +# Tasks: Gemini Adapter — Phase 1 + +## Phase 1: Foundation + +- [x] 1.1 Add `@google/generative-ai` dependency to `apps/web/package.json` (pin to latest stable `^0.21` or newer) +- [x] 1.2 Add Gemini env config lines to `apps/web/.env.example`: `LLM_PROVIDER=gemini`, `LLM_API_KEY`, `LLM_MODEL=gemini-2.0-flash`, comment noting `LLM_BASE_URL` is ignored for gemini + +## Phase 2: Core Adapter + +- [x] 2.1 Create `apps/web/src/agent/providers/gemini.ts` — import `GoogleGenerativeAI` from `@google/generative-ai`, `SchemaType` for type mapping +- [x] 2.2 Implement `toGeminiContents(messages: ChatMessage[])` — map `user→user`, `assistant→model` (with `functionCall` parts), `tool_result→function` role with `functionResponse`; skip `system` role (handled separately) +- [x] 2.3 Implement `toGeminiTools(tools: ToolSchema[])` — convert each `ToolSchema` to a `FunctionDeclaration` with `SchemaType` property types (`string→STRING`, `number→NUMBER`, `boolean→BOOLEAN`, `object→OBJECT`); return empty array when no tools +- [x] 2.4 Implement `fromGeminiResponse(response)` — extract `candidate.content.parts`, pull text into `content`, filter `functionCall` parts into `toolCalls[]` with `id: crypto.randomUUID()`, `name`, `args`; throw on empty candidates +- [x] 2.5 Implement `GeminiAdapter` class — constructor takes `ProviderConfig`, creates `GoogleGenerativeAI` + `getGenerativeModel({ model, systemInstruction })`; `chat()` calls `model.generateContent({ contents, tools })` then returns `fromGeminiResponse()`; ignore `config.baseUrl` + +## Phase 3: Factory Integration + +- [x] 3.1 Add `"gemini"` case to `createProvider()` switch in `apps/web/src/agent/providers/index.ts` — import and return `new GeminiAdapter(config)` +- [x] 3.2 Verify existing `openai-compatible` factory case is untouched and all existing tests still pass + +## Phase 4: Tests + +- [x] 4.1 Create `apps/web/src/agent/providers/__tests__/gemini.test.ts` — test `toGeminiContents`: user message, assistant with toolCalls, tool_result mapping, system message excluded from contents +- [x] 4.2 Test `toGeminiTools`: single tool with parameters, type casing (`string→STRING` etc.), empty tools returns empty array +- [x] 4.3 Test `fromGeminiResponse`: text-only response → `{ content, toolCalls: undefined }`; function-call response → `toolCalls` with non-empty synthesized IDs; two calls produce different IDs +- [x] 4.4 Test error paths: empty candidates → throws; SDK error → propagates +- [x] 4.5 Add factory test to `apps/web/src/agent/providers/__tests__/index.test.ts`: `"gemini"` config → `GeminiAdapter` instance; unknown provider still throws +- [x] 4.6 Test full `chat()` round-trip with mocked `GoogleGenerativeAI` — verify `generateContent` called with correct contents, tools, and systemInstruction diff --git a/openspec/changes/archive/2026-04-24-gemini-adapter-fase-1/verify-report.md b/openspec/changes/archive/2026-04-24-gemini-adapter-fase-1/verify-report.md new file mode 100644 index 00000000..8f4d6e73 --- /dev/null +++ b/openspec/changes/archive/2026-04-24-gemini-adapter-fase-1/verify-report.md @@ -0,0 +1,116 @@ +# Verification Report + +**Change**: gemini-adapter-fase-1 +**Version**: N/A +**Mode**: Standard + +--- + +### Completeness + +| Metric | Value | +|--------|-------| +| Tasks total | 14 | +| Tasks complete | 14 | +| Tasks incomplete | 0 | + +All tasks in `openspec/changes/gemini-adapter-fase-1/tasks.md` are marked complete, including the post-verify fixes. + +--- + +### Build & Tests Execution + +**Build**: ➖ Skipped + +```text +No explicit `rules.verify.build_command` is configured in `openspec/config.yaml`. +Project instructions also say never build after changes, so verification used TypeScript type-checking instead of `next build`. +``` + +**Type Check**: ✅ Passed + +```text +Command: bunx tsc --noEmit + +(no output) +``` + +**Tests**: ✅ 48 passed / ❌ 0 failed / ⚠️ 0 skipped + +```text +Command: bun test src/agent/providers/__tests__ src/app/api/agent/chat/__tests__/route.test.ts + +bun test v1.3.13 +48 pass +0 fail +106 expect() calls +Ran 48 tests across 4 files. +``` + +**Coverage**: ➖ Not available + +--- + +### Spec Compliance Matrix + +| Requirement | Scenario | Test | Result | +|-------------|----------|------|--------| +| Provider Adapter Interface | Adapter translates tool schemas to wire format | `src/agent/providers/__tests__/gemini.test.ts > converts a single tool with parameters` + `maps type strings to Gemini SchemaType values` | ✅ COMPLIANT | +| Provider Adapter Interface | Adapter selected by config — OpenAI-compatible | `src/agent/providers/__tests__/index.test.ts > returns OpenAICompatibleAdapter for openai-compatible provider` | ✅ COMPLIANT | +| Provider Adapter Interface | Adapter selected by config — Gemini | `src/agent/providers/__tests__/index.test.ts > returns GeminiAdapter for gemini provider` | ✅ COMPLIANT | +| Provider Adapter Interface | Gemini adapter synthesizes tool-call IDs | `src/agent/providers/__tests__/gemini.test.ts > maps function-call response to toolCalls with non-empty synthesized IDs` | ✅ COMPLIANT | +| Provider Configuration | Missing required configuration | `src/app/api/agent/chat/__tests__/route.test.ts > returns 502 when LLM_API_KEY is missing` | ✅ COMPLIANT | +| Provider Configuration | Unknown provider | `src/agent/providers/__tests__/index.test.ts > throws Error for unknown provider` | ✅ COMPLIANT | +| Provider Configuration | Gemini config ignores base URL | `src/agent/providers/__tests__/gemini.test.ts > ignores baseUrl in config — Gemini still works` | ✅ COMPLIANT | +| Gemini Free-Form Chat | Plain chat through Gemini | `src/agent/providers/__tests__/gemini.test.ts > calls generateContent with correct contents and no tools` | ✅ COMPLIANT | +| Gemini Free-Form Chat | System prompt mapping | `src/agent/providers/__tests__/gemini.test.ts > calls generateContent with correct contents and no tools` | ✅ COMPLIANT | +| Gemini Tool Calling | Gemini returns function calls | `src/agent/providers/__tests__/gemini.test.ts > calls generateContent with tools and systemInstruction` | ✅ COMPLIANT | +| Gemini Tool Calling | Gemini returns text despite tools | `src/agent/providers/__tests__/gemini.test.ts > returns text when tools are provided but Gemini responds with text only` | ✅ COMPLIANT | +| Gemini Tool Calling | Tool schema type mapping | `src/agent/providers/__tests__/gemini.test.ts > maps type strings to Gemini SchemaType values` | ✅ COMPLIANT | + +**Compliance summary**: 12/12 scenarios compliant + +--- + +### Correctness (Static — Structural Evidence) + +| Requirement | Status | Notes | +|------------|--------|-------| +| Provider Adapter Interface | ✅ Implemented | `apps/web/src/agent/providers/types.ts` keeps the provider-agnostic `chat()` seam, `index.ts` supports both providers, and `gemini.ts` synthesizes tool-call IDs with `crypto.randomUUID()`. | +| Provider Configuration | ✅ Implemented | `apps/web/src/app/api/agent/chat/route.ts` reads `LLM_PROVIDER`, `LLM_API_KEY`, `LLM_MODEL`, and `LLM_BASE_URL` server-side only; `GeminiAdapter` intentionally ignores `baseUrl`. | +| Gemini Free-Form Chat | ✅ Implemented | `GeminiAdapter.chat()` maps messages into Gemini contents, sets `systemInstruction` when present, and returns text-only responses without `toolCalls`. | +| Gemini Tool Calling | ✅ Implemented | `toGeminiTools()` converts internal tool schemas to Gemini declarations and `fromGeminiResponse()` maps function-call parts back into provider `toolCalls`. | +| Explicit non-goals remain deferred | ✅ Implemented | The touched files remain additive only; no streaming, multimodal upload, `@asset`, or contract changes were introduced. | + +--- + +### Coherence (Design) + +| Decision | Followed? | Notes | +|----------|-----------|-------| +| Use `@google/generative-ai` SDK | ✅ Yes | Added in `apps/web/package.json`; adapter imports `GoogleGenerativeAI` and `SchemaType`. | +| Preserve stateless provider seam | ✅ Yes | `GeminiAdapter.chat()` sends full history through one `generateContent()` call with no session state. | +| Synthesize tool-call IDs inside adapter | ✅ Yes | `fromGeminiResponse()` generates IDs for Gemini function calls. | +| Keep OpenAI-compatible path intact | ✅ Yes | `createProvider()` still returns `OpenAICompatibleAdapter` unchanged for the original provider. | +| Map system prompt via Gemini-native field | ✅ Yes | `GeminiAdapter.chat()` sets `request.systemInstruction` instead of injecting a fake user message. | + +--- + +### Issues Found + +**CRITICAL** (must fix before archive): +None + +**WARNING** (should fix): +None + +**SUGGESTION** (nice to have): +None + +--- + +### Verdict + +PASS + +All planned tasks are complete, the changed slice now type-checks, targeted runtime tests pass, and every spec scenario has passing behavioral evidence. diff --git a/openspec/specs/agent-session-shell/spec.md b/openspec/specs/agent-session-shell/spec.md index 1ddfac14..76727084 100644 --- a/openspec/specs/agent-session-shell/spec.md +++ b/openspec/specs/agent-session-shell/spec.md @@ -110,7 +110,7 @@ The system SHALL define types in `apps/web/src/agent/types.ts`: `ChatMessage { i ### Requirement: Provider Adapter Interface -The system SHALL define a server-side provider adapter interface with a single method `chat(params: ProviderChatParams): Promise`. `ProviderChatParams` MUST include `messages`, `systemPrompt`, and `toolSchemas` in a provider-agnostic internal format. `ProviderChatResponse` MUST include `content: string` and `toolCalls?: ProviderToolCall[]` where `ProviderToolCall { id, name, args }`. Each concrete adapter converts internal tool schemas to its own wire format. Phase 1 MAY ship exactly one adapter (OpenAI-compatible) as long as the interface contract remains provider-agnostic. +The system SHALL define a server-side provider adapter interface with a single method `chat(params: ProviderChatParams): Promise`. `ProviderChatParams` MUST include `messages`, `systemPrompt`, and `toolSchemas` in a provider-agnostic internal format. `ProviderChatResponse` MUST include `content: string` and `toolCalls?: ProviderToolCall[]` where `ProviderToolCall { id, name, args }`. Each concrete adapter converts internal tool schemas to its own wire format. The factory MUST support at least `openai-compatible` and `gemini` providers. Adapters whose native API does not assign IDs to tool calls MUST synthesize unique IDs so every returned `ToolCall.id` is a non-empty string. #### Scenario: Adapter translates tool schemas to wire format @@ -118,15 +118,27 @@ The system SHALL define a server-side provider adapter interface with a single m - WHEN the adapter receives tool schemas before calling the provider - THEN it converts them to its provider-specific wire format -#### Scenario: Adapter selected by config +#### Scenario: Adapter selected by config — OpenAI-compatible - GIVEN `LLM_PROVIDER` is set to `"openai-compatible"` - WHEN the route resolves the adapter - THEN the OpenAI-compatible adapter is returned +#### Scenario: Adapter selected by config — Gemini + +- GIVEN `LLM_PROVIDER` is set to `"gemini"` +- WHEN the route resolves the adapter +- THEN the Gemini adapter is returned + +#### Scenario: Gemini adapter synthesizes tool-call IDs + +- GIVEN the Gemini API returns a function call response with no call ID +- WHEN the Gemini adapter maps the response to `ProviderResponse` +- THEN each `ToolCall` in `toolCalls` has a non-empty `id` field generated by the adapter + ### Requirement: Provider Configuration -Server-only environment variables `LLM_PROVIDER` (required), `LLM_MODEL` (required), `LLM_API_KEY` (required), and `LLM_BASE_URL` (optional) MUST determine which adapter and model the route uses. Configuration MUST be read server-side only — never exposed to the client. +Server-only environment variables `LLM_PROVIDER` (required), `LLM_MODEL` (required), `LLM_API_KEY` (required), and `LLM_BASE_URL` (optional) MUST determine which adapter and model the route uses. `LLM_PROVIDER` MUST accept at least `"openai-compatible"` and `"gemini"`. When `LLM_PROVIDER` is `"gemini"`, `LLM_BASE_URL` MUST be ignored. Configuration MUST be read server-side only — never exposed to the client. #### Scenario: Missing required configuration @@ -134,6 +146,18 @@ Server-only environment variables `LLM_PROVIDER` (required), `LLM_MODEL` (requir - WHEN the route attempts to initialize the provider - THEN the call returns 502 with a missing-configuration error +#### Scenario: Unknown provider + +- GIVEN `LLM_PROVIDER` is set to `"unknown-provider"` +- WHEN the route resolves the adapter +- THEN the factory throws `"Unknown LLM provider: unknown-provider"` + +#### Scenario: Gemini config ignores base URL + +- GIVEN `LLM_PROVIDER` is `"gemini"` and `LLM_BASE_URL` is set +- WHEN the Gemini adapter is created +- THEN `LLM_BASE_URL` is not used in the API call + ### Requirement: Tool Argument Validation Before executing any tool call, the orchestrator MUST validate that each argument key declared `required: true` in the tool's `parameters` is present in the call args and is of the declared type. If validation fails, the tool MUST NOT execute; instead the orchestrator MUST produce a `ToolResult` with `error` describing the missing or malformed argument. @@ -166,3 +190,41 @@ The orchestrator loop MUST enforce a hard iteration cap of **8**. An iteration i - GIVEN the provider returns a final text response after 2 tool-call iterations - WHEN the orchestrator processes the response - THEN no cap message is appended and status is `"idle"` + +### Requirement: Gemini Free-Form Chat + +When the Gemini adapter receives a request with no tool schemas (or tools array is empty), it MUST send messages to the Gemini `generateContent` API and return a `ProviderResponse` with `content` set to the assistant text and `toolCalls` undefined. + +#### Scenario: Plain chat through Gemini + +- GIVEN `LLM_PROVIDER` is `"gemini"` and messages contain `[{ role: "user", content: "Hello" }]` +- WHEN the adapter calls Gemini and receives a text-only response +- THEN `ProviderResponse.content` is the assistant text and `toolCalls` is undefined + +#### Scenario: System prompt mapping + +- GIVEN the adapter receives `systemPrompt: "You are a video editor"` +- WHEN it builds the Gemini request +- THEN the system prompt is mapped to Gemini's system instruction field, not as a user message + +### Requirement: Gemini Tool Calling + +When the Gemini adapter receives a request with tool schemas, it MUST convert internal `ToolSchema[]` to Gemini function declarations, call `generateContent` with declarations attached, and if Gemini returns function call parts, map each to a `ToolCall { id, name, args }` with a synthesized non-empty `id`. + +#### Scenario: Gemini returns function calls + +- GIVEN `transcribe_video` is in tool schemas and Gemini responds with a function call part `{ name: "transcribe_video", args: {} }` +- WHEN the adapter maps the response +- THEN `toolCalls` contains one `ToolCall` with `name: "transcribe_video"`, `args: {}`, and a synthesized `id` + +#### Scenario: Gemini returns text despite tools + +- GIVEN tool schemas are provided but Gemini responds with only text +- WHEN the adapter maps the response +- THEN `toolCalls` is undefined and `content` holds the text + +#### Scenario: Tool schema type mapping + +- GIVEN an internal tool schema has parameters `[{ key: "assetId", type: "string", required: true }]` +- WHEN the adapter converts to Gemini function declarations +- THEN the parameter appears in `FunctionDeclaration.parameters.properties` with type `"string"`