feat: add native gemini provider

This commit is contained in:
Luis Esteban Acevedo Ladino 2026-04-24 07:25:04 -05:00
parent c07e33b91d
commit 17eaaaf73e
19 changed files with 1572 additions and 6 deletions

View File

@ -25,3 +25,9 @@ LLM_PROVIDER=openai-compatible
LLM_API_KEY=your_openai_api_key_here
LLM_MODEL=gpt-4o
# LLM_BASE_URL=https://api.openai.com/v1
# Gemini provider (set LLM_PROVIDER=gemini to activate)
# LLM_PROVIDER=gemini
# LLM_API_KEY=your_gemini_api_key_here
# LLM_MODEL=gemini-2.0-flash
# Note: LLM_BASE_URL is ignored when using the gemini provider

View File

@ -42,6 +42,7 @@
"cmdk": "^1.0.0",
"culori": "^4.0.2",
"drizzle-orm": "^0.44.2",
"@google/generative-ai": "^0.24.0",
"embla-carousel-react": "^8.5.1",
"eventemitter3": "^5.0.1",
"feed": "^5.1.0",

View File

@ -37,6 +37,46 @@ describe("buildSystemPrompt", () => {
expect(prompt).toContain("NeuralCut video editor");
});
test("includes asset internal id in media listing", () => {
const context: AgentContext = {
projectId: "proj-1",
activeSceneId: "scene-A",
mediaAssets: [
{ id: "m1", name: "intro.mp4", type: "video", duration: 30 },
],
playbackTimeMs: 0,
};
const prompt = buildSystemPrompt(context);
expect(prompt).toContain("[id: m1]");
});
test("includes assetId instruction when media assets are present", () => {
const context: AgentContext = {
projectId: "proj-1",
activeSceneId: "scene-A",
mediaAssets: [
{ id: "m1", name: "conclu.mov", type: "video", duration: 60 },
],
playbackTimeMs: 0,
};
const prompt = buildSystemPrompt(context);
expect(prompt).toContain("IMPORTANT");
expect(prompt).toContain("assetId");
expect(prompt).toContain('internal "id" value');
expect(prompt).toContain("NOT the filename");
});
test("omits assetId instruction when no media assets", () => {
const prompt = buildSystemPrompt(BASE_CONTEXT);
expect(prompt).not.toContain("IMPORTANT");
expect(prompt).not.toContain("assetId");
});
test("is valid when no media assets are loaded", () => {
const context: AgentContext = {
projectId: "proj-2",

View File

@ -0,0 +1,502 @@
import { beforeEach, describe, expect, jest, test } from "bun:test";
import { SchemaType, type GenerateContentResult } from "@google/generative-ai";
import type { ProviderConfig } from "@/agent/providers/types";
import type { ToolSchema } from "@/agent/types";
// ---------------------------------------------------------------------------
// Mock the Google Generative AI SDK before the adapter module loads
// ---------------------------------------------------------------------------
const mockGenerateContent = jest.fn();
jest.mock("@google/generative-ai", () => ({
GoogleGenerativeAI: class MockGoogleGenerativeAI {
getGenerativeModel(_opts: { model: string }) {
return { generateContent: mockGenerateContent };
}
constructor(_opts: { apiKey: string }) {}
},
SchemaType: {
STRING: "string",
NUMBER: "number",
BOOLEAN: "boolean",
OBJECT: "object",
},
}));
// Import after mock is set up
import {
GeminiAdapter,
toGeminiContents,
toGeminiTools,
fromGeminiResponse,
} from "@/agent/providers/gemini";
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
const TEST_CONFIG: ProviderConfig = {
provider: "gemini",
apiKey: "test-gemini-key",
model: "gemini-2.0-flash",
};
const SAMPLE_TOOLS: ToolSchema[] = [
{
name: "transcribe_video",
description: "Transcribes a video asset",
parameters: [
{ key: "assetId", type: "string", required: true },
{ key: "language", type: "string", required: false },
],
},
];
function makeGeminiResponse(overrides: {
text?: string;
functionCalls?: Array<{ name: string; args: Record<string, unknown> }>;
}): GenerateContentResult {
const parts: Array<Record<string, unknown>> = [];
if (overrides.text !== undefined) {
parts.push({ text: overrides.text });
}
if (overrides.functionCalls) {
for (const fc of overrides.functionCalls) {
parts.push({ functionCall: { name: fc.name, args: fc.args } });
}
}
return {
response: {
candidates: [
{
content: { parts },
},
],
},
} as unknown as GenerateContentResult;
}
function makeChatMessage(
role: "user" | "assistant" | "system" | "tool_result",
content: string,
extra?: Record<string, unknown>,
) {
return {
id: `msg-${Math.random().toString(36).slice(2)}`,
role,
content,
timestamp: Date.now(),
...extra,
};
}
// ---------------------------------------------------------------------------
// Tests: toGeminiContents (Task 4.1)
// ---------------------------------------------------------------------------
describe("toGeminiContents", () => {
test("maps user message to Gemini user role", () => {
const messages = [makeChatMessage("user", "Hello")];
const contents = toGeminiContents(messages);
expect(contents).toHaveLength(1);
expect(contents[0].role).toBe("user");
expect(contents[0].parts).toEqual([{ text: "Hello" }]);
});
test("maps assistant message to Gemini model role", () => {
const messages = [makeChatMessage("assistant", "Hi there!")];
const contents = toGeminiContents(messages);
expect(contents).toHaveLength(1);
expect(contents[0].role).toBe("model");
expect(contents[0].parts).toEqual([{ text: "Hi there!" }]);
});
test("maps assistant with toolCalls to model role with functionCall parts", () => {
const messages = [
makeChatMessage("assistant", "Let me check.", {
toolCalls: [
{
id: "tc_1",
name: "transcribe_video",
args: { assetId: "vid-123" },
},
],
}),
];
const contents = toGeminiContents(messages);
expect(contents).toHaveLength(1);
expect(contents[0].role).toBe("model");
expect(contents[0].parts).toEqual([
{ text: "Let me check." },
{
functionCall: {
name: "transcribe_video",
args: { assetId: "vid-123" },
},
},
]);
});
test("maps tool_result to function role with functionResponse", () => {
const messages = [
makeChatMessage("assistant", "", {
toolCalls: [
{
id: "tc_1",
name: "transcribe_video",
args: { assetId: "vid-123" },
},
],
}),
makeChatMessage("tool_result", '{"transcript": "Hello world"}', {
toolCallId: "tc_1",
}),
];
const contents = toGeminiContents(messages);
// First: model (assistant with toolCalls)
expect(contents[0].role).toBe("model");
// Second: function (tool_result)
expect(contents[1].role).toBe("function");
expect(contents[1].parts).toEqual([
{
functionResponse: {
name: "transcribe_video",
response: { transcript: "Hello world" },
},
},
]);
});
test("excludes system messages from contents", () => {
const messages = [
makeChatMessage("system", "You are a video editor."),
makeChatMessage("user", "Hello"),
];
const contents = toGeminiContents(messages);
expect(contents).toHaveLength(1);
expect(contents[0].role).toBe("user");
});
test("handles tool_result with non-JSON content by wrapping in object", () => {
const messages = [
makeChatMessage("assistant", "", {
toolCalls: [
{ id: "tc_1", name: "some_tool", args: {} },
],
}),
makeChatMessage("tool_result", "plain text result", {
toolCallId: "tc_1",
}),
];
const contents = toGeminiContents(messages);
expect(contents[1].parts).toEqual([
{
functionResponse: {
name: "some_tool",
response: { result: "plain text result" },
},
},
]);
});
});
// ---------------------------------------------------------------------------
// Tests: toGeminiTools (Task 4.2)
// ---------------------------------------------------------------------------
describe("toGeminiTools", () => {
test("converts a single tool with parameters", () => {
const declarations = toGeminiTools(SAMPLE_TOOLS);
expect(declarations).toHaveLength(1);
expect(declarations[0].name).toBe("transcribe_video");
expect(declarations[0].description).toBe("Transcribes a video asset");
expect(declarations[0].parameters?.type).toBe(SchemaType.OBJECT);
expect(declarations[0].parameters?.properties).toHaveProperty("assetId");
expect(declarations[0].parameters?.required).toEqual(["assetId"]);
});
test("maps type strings to Gemini SchemaType values", () => {
const tools: ToolSchema[] = [
{
name: "multi_param_tool",
description: "Tool with all param types",
parameters: [
{ key: "name", type: "string", required: true },
{ key: "count", type: "number", required: false },
{ key: "active", type: "boolean", required: false },
{ key: "meta", type: "object", required: false },
],
},
];
const declarations = toGeminiTools(tools);
const props = declarations[0].parameters?.properties as Record<
string,
{ type: string }
>;
expect(props.name.type).toBe(SchemaType.STRING);
expect(props.count.type).toBe(SchemaType.NUMBER);
expect(props.active.type).toBe(SchemaType.BOOLEAN);
expect(props.meta.type).toBe(SchemaType.OBJECT);
});
test("returns empty array when no tools provided", () => {
expect(toGeminiTools([])).toEqual([]);
});
});
// ---------------------------------------------------------------------------
// Tests: fromGeminiResponse (Task 4.3)
// ---------------------------------------------------------------------------
describe("fromGeminiResponse", () => {
test("extracts text-only response with no toolCalls", () => {
const response = makeGeminiResponse({ text: "Hello from Gemini!" });
const result = fromGeminiResponse(response);
expect(result.content).toBe("Hello from Gemini!");
expect(result.toolCalls).toBeUndefined();
});
test("maps function-call response to toolCalls with non-empty synthesized IDs", () => {
const response = makeGeminiResponse({
text: undefined,
functionCalls: [
{ name: "transcribe_video", args: { assetId: "vid-1" } },
],
});
const result = fromGeminiResponse(response);
expect(result.content).toBe("");
expect(result.toolCalls).toHaveLength(1);
expect(result.toolCalls?.[0].name).toBe("transcribe_video");
expect(result.toolCalls?.[0].args).toEqual({ assetId: "vid-1" });
// Non-empty synthesized ID
expect(result.toolCalls?.[0].id).toBeTruthy();
expect(typeof result.toolCalls?.[0].id).toBe("string");
});
test("two calls produce different IDs", () => {
const response1 = makeGeminiResponse({
functionCalls: [{ name: "tool_a", args: {} }],
});
const response2 = makeGeminiResponse({
functionCalls: [{ name: "tool_a", args: {} }],
});
const result1 = fromGeminiResponse(response1);
const result2 = fromGeminiResponse(response2);
expect(result1.toolCalls?.[0].id).not.toBe(
result2.toolCalls?.[0].id,
);
});
test("handles mixed text and function-call parts", () => {
const response = makeGeminiResponse({
text: "I'll look that up.",
functionCalls: [{ name: "search", args: { q: "test" } }],
});
const result = fromGeminiResponse(response);
expect(result.content).toBe("I'll look that up.");
expect(result.toolCalls).toHaveLength(1);
});
});
// ---------------------------------------------------------------------------
// Tests: error paths (Task 4.4)
// ---------------------------------------------------------------------------
describe("error paths", () => {
test("throws on empty candidates", () => {
const response = {
response: { candidates: [] },
};
expect(() => fromGeminiResponse(response as any)).toThrow(
"Empty response from Gemini provider",
);
});
test("throws on null candidates", () => {
const response = {
response: { candidates: null },
};
expect(() => fromGeminiResponse(response as any)).toThrow(
"Empty response from Gemini provider",
);
});
test("SDK error propagates from chat()", async () => {
mockGenerateContent.mockRejectedValueOnce(
new Error("API key not valid"),
);
const adapter = new GeminiAdapter(TEST_CONFIG);
expect(
adapter.chat({
messages: [makeChatMessage("user", "Hello")],
systemPrompt: "",
tools: [],
}),
).rejects.toThrow("API key not valid");
});
});
// ---------------------------------------------------------------------------
// Tests: full chat() round-trip (Task 4.6)
// ---------------------------------------------------------------------------
describe("GeminiAdapter.chat()", () => {
beforeEach(() => {
mockGenerateContent.mockReset();
});
test("calls generateContent with correct contents and no tools", async () => {
mockGenerateContent.mockResolvedValueOnce(
makeGeminiResponse({ text: "Hi!" }),
);
const adapter = new GeminiAdapter(TEST_CONFIG);
const result = await adapter.chat({
messages: [makeChatMessage("user", "Hello")],
systemPrompt: "You are helpful.",
tools: [],
});
expect(result.content).toBe("Hi!");
expect(result.toolCalls).toBeUndefined();
const callArgs = mockGenerateContent.mock.calls[0][0];
expect(callArgs.contents).toHaveLength(1);
expect(callArgs.contents[0].role).toBe("user");
expect(callArgs.systemInstruction).toBe("You are helpful.");
expect(callArgs.tools).toBeUndefined();
});
test("calls generateContent with tools and systemInstruction", async () => {
mockGenerateContent.mockResolvedValueOnce(
makeGeminiResponse({
functionCalls: [
{
name: "transcribe_video",
args: { assetId: "vid-1" },
},
],
}),
);
const adapter = new GeminiAdapter(TEST_CONFIG);
const result = await adapter.chat({
messages: [makeChatMessage("user", "Transcribe this")],
systemPrompt: "Video editing assistant",
tools: SAMPLE_TOOLS,
});
expect(result.toolCalls).toHaveLength(1);
expect(result.toolCalls?.[0].name).toBe("transcribe_video");
const callArgs = mockGenerateContent.mock.calls[0][0];
expect(callArgs.systemInstruction).toBe("Video editing assistant");
expect(callArgs.tools).toHaveLength(1);
expect(callArgs.tools[0].functionDeclarations).toHaveLength(1);
});
test("passes multi-turn conversation through to Gemini", async () => {
mockGenerateContent.mockResolvedValueOnce(
makeGeminiResponse({ text: "Sure!" }),
);
const adapter = new GeminiAdapter(TEST_CONFIG);
await adapter.chat({
messages: [
makeChatMessage("user", "First question"),
makeChatMessage("assistant", "First answer"),
makeChatMessage("user", "Second question"),
],
systemPrompt: "",
tools: [],
});
const callArgs = mockGenerateContent.mock.calls[0][0];
expect(callArgs.contents).toHaveLength(3);
expect(callArgs.contents[0].role).toBe("user");
expect(callArgs.contents[1].role).toBe("model");
expect(callArgs.contents[2].role).toBe("user");
});
test("does not include systemInstruction when systemPrompt is empty", async () => {
mockGenerateContent.mockResolvedValueOnce(
makeGeminiResponse({ text: "OK" }),
);
const adapter = new GeminiAdapter(TEST_CONFIG);
await adapter.chat({
messages: [makeChatMessage("user", "Hi")],
systemPrompt: "",
tools: [],
});
const callArgs = mockGenerateContent.mock.calls[0][0];
expect(callArgs.systemInstruction).toBeUndefined();
});
test("ignores baseUrl in config — Gemini still works", async () => {
mockGenerateContent.mockResolvedValueOnce(
makeGeminiResponse({ text: "Hello!" }),
);
const configWithBaseUrl: ProviderConfig = {
provider: "gemini",
apiKey: "test-gemini-key",
model: "gemini-2.0-flash",
baseUrl: "https://should-be-ignored.example.com",
};
const adapter = new GeminiAdapter(configWithBaseUrl);
const result = await adapter.chat({
messages: [makeChatMessage("user", "Hi")],
systemPrompt: "",
tools: [],
});
expect(result.content).toBe("Hello!");
expect(result.toolCalls).toBeUndefined();
});
test("returns text when tools are provided but Gemini responds with text only", async () => {
mockGenerateContent.mockResolvedValueOnce(
makeGeminiResponse({ text: "I don't need any tools for this." }),
);
const adapter = new GeminiAdapter(TEST_CONFIG);
const result = await adapter.chat({
messages: [makeChatMessage("user", "Just chat")],
systemPrompt: "You are helpful.",
tools: SAMPLE_TOOLS,
});
expect(result.content).toBe("I don't need any tools for this.");
expect(result.toolCalls).toBeUndefined();
// Verify tools were still sent to Gemini
const callArgs = mockGenerateContent.mock.calls[0][0];
expect(callArgs.tools).toBeDefined();
expect(callArgs.tools[0].functionDeclarations).toHaveLength(1);
});
});

View File

@ -1,6 +1,7 @@
import { describe, expect, test } from "bun:test";
import { createProvider } from "@/agent/providers/index";
import { OpenAICompatibleAdapter } from "@/agent/providers/openai-compatible";
import { GeminiAdapter } from "@/agent/providers/gemini";
import type { ProviderConfig } from "@/agent/providers/types";
describe("createProvider", () => {
@ -28,6 +29,17 @@ describe("createProvider", () => {
// Adapter was constructed without throwing — config accepted
});
test("returns GeminiAdapter for gemini provider", () => {
const config: ProviderConfig = {
provider: "gemini",
apiKey: "test-gemini-key",
model: "gemini-2.0-flash",
};
const adapter = createProvider(config);
expect(adapter).toBeInstanceOf(GeminiAdapter);
});
test("throws Error for unknown provider", () => {
const config: ProviderConfig = {
provider: "unknown-provider",

View File

@ -0,0 +1,240 @@
import {
GoogleGenerativeAI,
type Content,
type FunctionDeclaration,
type FunctionDeclarationSchema,
type GenerateContentResult,
SchemaType,
} from "@google/generative-ai";
import type { ChatMessage, ToolCall, ToolSchema } from "@/agent/types";
import type {
ProviderAdapter,
ProviderConfig,
ProviderResponse,
} from "./types";
// ---------------------------------------------------------------------------
// Internal conversion helpers
// ---------------------------------------------------------------------------
/**
* Maps internal type strings to Gemini SchemaType values.
*/
const TYPE_MAP: Record<string, SchemaType> = {
string: SchemaType.STRING,
number: SchemaType.NUMBER,
boolean: SchemaType.BOOLEAN,
object: SchemaType.OBJECT,
};
/**
* Converts internal ChatMessages into Gemini Content[].
*
* - `user` role "user", text part
* - `assistant` role "model", text part + functionCall parts from toolCalls
* - `tool_result` role "function", functionResponse part
* - `system` skipped (handled separately via systemInstruction)
*/
export function toGeminiContents(messages: ChatMessage[]): Content[] {
const contents: Content[] = [];
for (const msg of messages) {
if (msg.role === "system") {
continue;
}
if (msg.role === "user") {
contents.push({
role: "user",
parts: [{ text: msg.content }],
});
} else if (msg.role === "assistant") {
const parts: Content["parts"] = [];
if (msg.content) {
parts.push({ text: msg.content });
}
if (msg.toolCalls && msg.toolCalls.length > 0) {
for (const tc of msg.toolCalls) {
parts.push({
functionCall: { name: tc.name, args: tc.args },
});
}
}
contents.push({ role: "model", parts });
} else if (msg.role === "tool_result") {
// Gemini uses the function name to correlate responses, not IDs.
// We parse the content as JSON for the response object; fall back
// to wrapping it if it's not valid JSON.
let responseData: object;
try {
responseData = JSON.parse(msg.content) as object;
} catch {
responseData = { result: msg.content };
}
// Derive the function name from the toolCallId → find the matching
// tool name in prior messages. If not found, use a placeholder.
const toolName = findToolNameForResult(messages, msg.toolCallId);
contents.push({
role: "function",
parts: [
{
functionResponse: {
name: toolName,
response: responseData,
},
},
],
});
}
}
return contents;
}
/**
* Walks backward through messages to find the function name associated with
* a toolCallId. This is needed because Gemini's functionResponse uses names,
* not IDs.
*/
function findToolNameForResult(
messages: ChatMessage[],
toolCallId?: string,
): string {
if (!toolCallId) return "unknown";
for (let i = messages.length - 1; i >= 0; i--) {
const msg = messages[i];
if (msg.role === "assistant" && msg.toolCalls) {
const match = msg.toolCalls.find((tc) => tc.id === toolCallId);
if (match) return match.name;
}
}
return "unknown";
}
/**
* Converts internal ToolSchema[] into Gemini FunctionDeclaration[].
*
* Maps internal type strings ("string", "number", etc.) to Gemini SchemaType
* enum values. Returns an empty array when no tools are provided.
*/
export function toGeminiTools(tools: ToolSchema[]): FunctionDeclaration[] {
if (tools.length === 0) return [];
return tools.map((tool) => {
const properties: FunctionDeclarationSchema["properties"] = {};
for (const param of tool.parameters) {
const geminiType = TYPE_MAP[param.type] ?? SchemaType.STRING;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
(properties as Record<string, any>)[param.key] = { type: geminiType };
}
const requiredParams = tool.parameters
.filter((p) => p.required)
.map((p) => p.key);
return {
name: tool.name,
description: tool.description,
parameters: {
type: SchemaType.OBJECT,
properties,
...(requiredParams.length > 0 && { required: requiredParams }),
},
};
});
}
/**
* Converts a Gemini GenerateContentResult into the provider-agnostic
* ProviderResponse shape.
*
* - Text parts are concatenated into `content`.
* - FunctionCall parts are mapped to `toolCalls[]` with synthesized UUIDs.
* - Throws if no candidates are present.
*/
export function fromGeminiResponse(
response: GenerateContentResult,
): ProviderResponse {
const candidates = response.response.candidates;
if (!candidates || candidates.length === 0) {
throw new Error("Empty response from Gemini provider");
}
const parts = candidates[0].content?.parts ?? [];
const textParts: string[] = [];
const toolCalls: ToolCall[] = [];
for (const part of parts) {
if ("text" in part && part.text !== undefined) {
textParts.push(part.text);
}
if ("functionCall" in part && part.functionCall) {
toolCalls.push({
id: crypto.randomUUID(),
name: part.functionCall.name,
args: (part.functionCall.args as Record<string, unknown>) ?? {},
});
}
}
const content = textParts.join("");
const result: ProviderResponse = { content };
if (toolCalls.length > 0) {
result.toolCalls = toolCalls;
}
return result;
}
// ---------------------------------------------------------------------------
// Adapter implementation
// ---------------------------------------------------------------------------
/**
* Gemini adapter using @google/generative-ai SDK.
*
* Uses the stateless `generateContent()` API full history is sent with each
* call, matching the existing route contract. System prompts are mapped to
* Gemini's `systemInstruction` field, not as user messages.
*/
export class GeminiAdapter implements ProviderAdapter {
private model;
constructor(config: ProviderConfig) {
const genAI = new GoogleGenerativeAI(config.apiKey);
this.model = genAI.getGenerativeModel({ model: config.model });
// config.baseUrl is intentionally ignored for the Gemini provider.
}
async chat(params: {
messages: ChatMessage[];
systemPrompt: string;
tools: ToolSchema[];
}): Promise<ProviderResponse> {
const contents = toGeminiContents(params.messages);
const functionDeclarations = toGeminiTools(params.tools);
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const request: any = { contents };
if (params.systemPrompt) {
request.systemInstruction = params.systemPrompt;
}
if (functionDeclarations.length > 0) {
request.tools = [{ functionDeclarations }];
}
const result = await this.model.generateContent(request);
return fromGeminiResponse(result);
}
}

View File

@ -1,5 +1,6 @@
import type { ProviderAdapter, ProviderConfig } from "./types";
import { OpenAICompatibleAdapter } from "./openai-compatible";
import { GeminiAdapter } from "./gemini";
/**
* Factory resolves a ProviderConfig to a concrete adapter.
@ -12,6 +13,8 @@ export function createProvider(config: ProviderConfig): ProviderAdapter {
switch (config.provider) {
case "openai-compatible":
return new OpenAICompatibleAdapter(config);
case "gemini":
return new GeminiAdapter(config);
default:
throw new Error(`Unknown LLM provider: ${config.provider}`);
}

View File

@ -17,7 +17,7 @@ export function buildSystemPrompt(
): string {
const mediaSection =
context.mediaAssets.length > 0
? `Active media assets:\n${context.mediaAssets.map((m) => `- ${m.name} (${m.type}, ${m.duration}s)`).join("\n")}`
? `Active media assets:\n${context.mediaAssets.map((m) => `- [id: ${m.id}] ${m.name} (${m.type}, ${m.duration}s)`).join("\n")}`
: "No media assets loaded.";
const parts = [
@ -28,6 +28,12 @@ export function buildSystemPrompt(
mediaSection,
];
if (context.mediaAssets.length > 0) {
parts.push(
'IMPORTANT: When calling tools that accept an "assetId" parameter, always use the internal "id" value (e.g., "v1"), NOT the filename or display name.',
);
}
if (tools && tools.length > 0) {
const toolList = tools
.map((t) => `- ${t.name}: ${t.description}`)

View File

@ -227,7 +227,7 @@ describe("transcribe_video tool — error branches", () => {
const result = await tool.execute({ assetId: "v99" }, context);
expect(result).toEqual({
error: "Could not access file for asset v99",
error: 'No asset found with id or name "v99". Available ids: [v1]. Available names: [intro.mp4].',
});
});
@ -282,3 +282,141 @@ describe("transcribe_video tool — error branches", () => {
expect(result).toEqual({ error: "Worker not initialized" });
});
});
describe("transcribe_video tool — name-based assetId fallback", () => {
beforeEach(() => {
mockResolveAssetFile.mockClear();
mockGetAssetHasAudio.mockClear();
mockDecodeAudio.mockClear();
mockTranscribe.mockClear();
});
test("resolves by filename when no id match (exact name)", async () => {
const fakeFile = new File([], "conclu.mov", { type: "video/quicktime" });
const context = makeContext({
mediaAssets: [
fakeVideoAsset({ id: "abc123", name: "conclu.mov" }),
],
});
mockResolveAssetFile.mockReturnValue(fakeFile);
mockGetAssetHasAudio.mockReturnValue(true);
mockDecodeAudio.mockResolvedValue({
samples: new Float32Array(16000),
sampleRate: 16000,
});
mockTranscribe.mockResolvedValue(fakeTranscriptionResult());
const tool = toolRegistry.get("transcribe_video");
const result = await tool.execute({ assetId: "conclu.mov" }, context);
expect(result).toEqual(
expect.objectContaining({ assetName: "conclu.mov" }),
);
// Resolved asset must pass its internal id to the adapter
expect(mockResolveAssetFile).toHaveBeenCalledWith("abc123");
expect(mockGetAssetHasAudio).toHaveBeenCalledWith("abc123");
});
test("resolves by case-insensitive name when no exact id or name match", async () => {
const fakeFile = new File([], "Conclu.MOV", { type: "video/quicktime" });
const context = makeContext({
mediaAssets: [
fakeVideoAsset({ id: "abc123", name: "Conclu.MOV" }),
],
});
mockResolveAssetFile.mockReturnValue(fakeFile);
mockGetAssetHasAudio.mockReturnValue(true);
mockDecodeAudio.mockResolvedValue({
samples: new Float32Array(16000),
sampleRate: 16000,
});
mockTranscribe.mockResolvedValue(fakeTranscriptionResult());
const tool = toolRegistry.get("transcribe_video");
const result = await tool.execute({ assetId: "conclu.mov" }, context);
expect(result).toEqual(
expect.objectContaining({ assetName: "Conclu.MOV" }),
);
expect(mockResolveAssetFile).toHaveBeenCalledWith("abc123");
});
test("returns ambiguous error when two assets share same name", async () => {
const context = makeContext({
mediaAssets: [
fakeVideoAsset({ id: "v1", name: "clip.mp4" }),
fakeVideoAsset({ id: "v2", name: "clip.mp4" }),
],
});
const tool = toolRegistry.get("transcribe_video");
const result = await tool.execute({ assetId: "clip.mp4" }, context);
expect(result).toEqual({
error: 'Ambiguous asset name "clip.mp4" matches multiple assets (clip.mp4, clip.mp4). Specify the internal id: v1, v2.',
});
});
test("returns ambiguous error for case-insensitive duplicates", async () => {
const context = makeContext({
mediaAssets: [
fakeVideoAsset({ id: "v1", name: "Clip.MP4" }),
fakeVideoAsset({ id: "v2", name: "CLIP.MP4" }),
],
});
const tool = toolRegistry.get("transcribe_video");
const result = await tool.execute({ assetId: "clip.mp4" }, context);
expect(result).toEqual({
error: 'Ambiguous asset name "clip.mp4" matches multiple assets (Clip.MP4, CLIP.MP4). Specify the internal id: v1, v2.',
});
});
test("prefers exact id match over name match", async () => {
const fakeFile = new File([], "intro.mp4", { type: "video/mp4" });
// An asset whose id happens to equal another asset's name
const context = makeContext({
mediaAssets: [
fakeVideoAsset({ id: "v1", name: "intro.mp4" }),
fakeVideoAsset({ id: "intro.mp4", name: "outro.mp4" }),
],
});
mockResolveAssetFile.mockReturnValue(fakeFile);
mockGetAssetHasAudio.mockReturnValue(true);
mockDecodeAudio.mockResolvedValue({
samples: new Float32Array(16000),
sampleRate: 16000,
});
mockTranscribe.mockResolvedValue(fakeTranscriptionResult());
const tool = toolRegistry.get("transcribe_video");
const result = await tool.execute({ assetId: "intro.mp4" }, context);
// "intro.mp4" matches v1 by id AND the second asset by id exactly
// id lookup is first, so it resolves the second asset (id="intro.mp4")
expect(result).toEqual(
expect.objectContaining({ assetName: "outro.mp4" }),
);
expect(mockResolveAssetFile).toHaveBeenCalledWith("intro.mp4");
});
test("returns not-found error with available ids and names for totally unknown assetId", async () => {
const context = makeContext({
mediaAssets: [
fakeVideoAsset({ id: "v1", name: "intro.mp4" }),
fakeVideoAsset({ id: "v2", name: "outro.mp4" }),
],
});
const tool = toolRegistry.get("transcribe_video");
const result = await tool.execute({ assetId: "nonexistent.mov" }, context);
expect(result).toEqual({
error: 'No asset found with id or name "nonexistent.mov". Available ids: [v1, v2]. Available names: [intro.mp4, outro.mp4].',
});
});
});

View File

@ -57,10 +57,43 @@ const transcribeVideoTool: ToolDefinition = {
let targetAsset: (typeof mediaCandidates)[number] | undefined;
if (typedArgs.assetId) {
// 1. Exact internal-id match
targetAsset = mediaCandidates.find((a) => a.id === typedArgs.assetId);
// 2. Fallback: exact name match (LLM sometimes passes filename)
if (!targetAsset) {
const nameMatches = mediaCandidates.filter(
(a) => a.name === typedArgs.assetId,
);
if (nameMatches.length === 1) {
targetAsset = nameMatches[0];
} else if (nameMatches.length > 1) {
return {
error: `Ambiguous asset name "${typedArgs.assetId}" matches multiple assets (${nameMatches.map((a) => a.name).join(", ")}). Specify the internal id: ${nameMatches.map((a) => `${a.id}`).join(", ")}.`,
};
}
}
// 3. Case-insensitive fallback (only if unambiguous)
if (!targetAsset) {
const lower = typedArgs.assetId.toLowerCase();
const ciMatches = mediaCandidates.filter(
(a) => a.name.toLowerCase() === lower,
);
if (ciMatches.length === 1) {
targetAsset = ciMatches[0];
} else if (ciMatches.length > 1) {
return {
error: `Ambiguous asset name "${typedArgs.assetId}" matches multiple assets (${ciMatches.map((a) => a.name).join(", ")}). Specify the internal id: ${ciMatches.map((a) => `${a.id}`).join(", ")}.`,
};
}
}
if (!targetAsset) {
const ids = mediaCandidates.map((a) => a.id).join(", ");
const names = mediaCandidates.map((a) => a.name).join(", ");
return {
error: `Could not access file for asset ${typedArgs.assetId}`,
error: `No asset found with id or name "${typedArgs.assetId}". Available ids: [${ids}]. Available names: [${names}].`,
};
}
} else {

View File

@ -20,6 +20,7 @@
"name": "@opencut/web",
"version": "0.1.0",
"dependencies": {
"@google/generative-ai": "^0.24.0",
"@hello-pangea/dnd": "^18.0.1",
"@hugeicons/core-free-icons": "^3.3.0",
"@hugeicons/react": "^1.1.6",
@ -348,6 +349,8 @@
"@floating-ui/utils": ["@floating-ui/utils@0.2.10", "", {}, "sha512-aGTxbpbg8/b5JfU1HXSrbH3wXZuLPJcNEcZQFMxLs3oSzgtVu6nFPkbbGGUvBcUjKV2YyB9Wxxabo+HEH9tcRQ=="],
"@google/generative-ai": ["@google/generative-ai@0.24.1", "", {}, "sha512-MqO+MLfM6kjxcKoy0p1wRzG3b4ZZXtPI+z2IE26UogS2Cm/XHO+7gGRBh6gcJsOiIVoH93UwKvW4HdgiOZCy9Q=="],
"@hello-pangea/dnd": ["@hello-pangea/dnd@18.0.1", "", { "dependencies": { "@babel/runtime": "^7.26.7", "css-box-model": "^1.2.1", "raf-schd": "^4.0.3", "react-redux": "^9.2.0", "redux": "^5.0.1" }, "peerDependencies": { "react": "^18.0.0 || ^19.0.0", "react-dom": "^18.0.0 || ^19.0.0" } }, "sha512-xojVWG8s/TGrKT1fC8K2tIWeejJYTAeJuj36zM//yEm/ZrnZUSFGS15BpO+jGZT1ybWvyXmeDJwPYb4dhWlbZQ=="],
"@hugeicons/core-free-icons": ["@hugeicons/core-free-icons@3.3.0", "", {}, "sha512-qYyr4JQ2eQIHTSTbITvnJvs6ERNK64D9gpwZnf2IyuG0exzqfyABLO/oTB71FB3RZPfu1GbwycdiGSo46apjMQ=="],

View File

@ -0,0 +1,59 @@
# Archive Report: gemini-adapter-fase-1
**Change**: gemini-adapter-fase-1
**Archived**: 2026-04-24
**Verification Verdict**: PASS — 14/14 tasks complete, 12/12 spec scenarios compliant, 48/48 tests passing, 0 TypeScript errors
## Scope Summary
This change adds **native Gemini provider support for normal chat and tool calling only**. Video upload/file understanding remains deferred to a future change. The `GeminiAdapter` implements the existing `ProviderAdapter` seam via `@google/generative-ai` SDK, supporting stateless `generateContent()` with system prompt mapping, tool schema conversion, and synthesized tool-call IDs. The OpenAI-compatible path is untouched.
## Engram Artifact Traceability
| Artifact | Observation ID |
|----------|---------------|
| Proposal | #1016 |
| Spec (delta) | #1019 |
| Design | #1018 |
| Tasks | #1020 |
| Apply Progress | #1021 |
| Verify Report | #1024 |
## OpenSpec Archive
**Moved to**: `openspec/changes/archive/2026-04-24-gemini-adapter-fase-1/`
### Archive Contents
- proposal.md ✅
- specs/ (agent-session-shell delta) ✅
- design.md ✅
- tasks.md ✅ (14/14 tasks complete)
- verify-report.md ✅
- exploration.md ✅
## Specs Synced
| Domain | Action | Details |
|--------|--------|---------|
| agent-session-shell | Updated | 2 requirements modified (Provider Adapter Interface, Provider Configuration), 2 requirements added (Gemini Free-Form Chat, Gemini Tool Calling), 0 removed |
### Modified Requirements
- **Provider Adapter Interface**: Expanded factory to support `openai-compatible` and `gemini`; added tool-call ID synthesis mandate; added Gemini adapter selection scenario
- **Provider Configuration**: Added `LLM_PROVIDER` valid value constraint (`openai-compatible`, `gemini`); added Gemini-specific `LLM_BASE_URL` ignoring behavior; added unknown-provider and Gemini-config scenarios
### Added Requirements
- **Gemini Free-Form Chat**: Plain text chat through Gemini `generateContent()` with system prompt mapped to `systemInstruction`
- **Gemini Tool Calling**: Tool schema → Gemini function declaration conversion, function call response → `ToolCall[]` with synthesized IDs
## Source of Truth Updated
- `openspec/specs/agent-session-shell/spec.md` — now reflects Gemini provider support
## Deferred Scope (explicitly out of scope)
- Video/file upload or direct file understanding
- `@asset` references, broader multimodal inputs, image/video reasoning
- Streaming UX or adapter contract changes for streaming
- Safety/grounding/thinking configuration beyond defaults
## SDD Cycle Complete
The change has been fully explored, proposed, specified, designed, implemented, verified, and archived.
Ready for the next change.

View File

@ -0,0 +1,155 @@
# Design: Gemini Adapter — Phase 1
## Technical Approach
Implement a stateless `GeminiAdapter` class behind the existing `ProviderAdapter` seam, following the exact same pattern as `OpenAICompatibleAdapter`. The adapter converts internal `ChatMessage[]`/`ToolSchema[]` into Gemini's `generateContent()` wire format and parses responses back into `ProviderResponse`. A new `"gemini"` case in `createProvider()` activates it when `LLM_PROVIDER=gemini`. Zero changes to orchestrator, route, or client contracts.
## Architecture Decisions
### Decision: SDK choice
| Option | Tradeoff | Decision |
|--------|----------|----------|
| `@google/generative-ai` | Established, stable, v0.21+, `GoogleGenerativeAI` class, `generateContent()` stateless API | ✅ Selected |
| `@google/genai` | Newer unified SDK, less battle-tested, API may still shift | ❌ Rejected |
**Rationale**: `@google/generative-ai` is the most widely adopted Gemini SDK for Node/TS. It supports `systemInstruction`, `functionDeclaration`, and stateless `generateContent()` — exactly what we need. Consistent with using the mature `openai` SDK for the existing adapter.
### Decision: Stateless generateContent vs stateful startChat
| Option | Tradeoff | Decision |
|--------|----------|----------|
| `model.generateContent(contents, config)` | Stateless, full history each call — matches current adapter contract | ✅ Selected |
| `chat.sendMessage()` via `startChat()` | Stateful session, server-managed history | ❌ Rejected |
**Rationale**: Current architecture POSTs full history from client on every turn. Stateless `generateContent()` maps 1:1 to this pattern. No session management needed.
### Decision: Tool call ID synthesis
| Option | Tradeoff | Decision |
|--------|----------|----------|
| `crypto.randomUUID()` | Built-in, zero deps, unique | ✅ Selected |
| `nanoid()` (already a dep) | Shorter IDs, also unique | ❌ Rejected |
**Rationale**: `crypto.randomUUID()` is universally available in Node/Bun, no import needed, and produces standard UUIDs matching OpenAI's `call_xxx` format semantically. Keeps the adapter self-contained.
### Decision: System prompt placement
| Option | Tradeoff | Decision |
|--------|----------|----------|
| `systemInstruction` on `GenerateContentRequest` | Gemini-native, separate from contents | ✅ Selected |
| Prepend as `user` message | Works but dilutes system/user boundary | ❌ Rejected |
**Rationale**: Gemini SDK supports `systemInstruction` as a first-class config field. Using it keeps system prompt semantically correct and matches Gemini best practices.
## Data Flow
```
Route (chat/route.ts)
├─ createProvider(config) → GeminiAdapter instance
└─ adapter.chat({ messages, systemPrompt, tools })
├─ toGeminiContents(messages) → Content[] (user/model/function)
├─ toGeminiTools(tools) → FunctionDeclaration[]
└─ model.generateContent({ contents, tools, systemInstruction })
└─ fromGeminiResponse(response) → ProviderResponse
├─ content: candidate.content.parts[].text
└─ toolCalls: candidate.content.parts[]
.filter(functionCall)
.map → { id: uuid(), name, args }
```
### Message mapping
| Internal role | Gemini role | Notes |
|---------------|-------------|-------|
| `system` | `systemInstruction` (config) | Not in contents array |
| `user` | `user` part | Direct text mapping |
| `assistant` | `model` part | Includes `functionCall` parts for tool calls |
| `tool_result` | `function` role | Uses `functionResponse: { name, response }` keyed by tool name (not ID) |
### Type mapping (internal → Gemini)
| Internal type | Gemini type |
|---------------|-------------|
| `"string"` | `SchemaType.STRING` |
| `"number"` | `SchemaType.NUMBER` |
| `"boolean"` | `SchemaType.BOOLEAN` |
| `"object"` | `SchemaType.OBJECT` |
## File Changes
| File | Action | Description |
|------|--------|-------------|
| `apps/web/src/agent/providers/gemini.ts` | Create | GeminiAdapter + conversion helpers (`toGeminiContents`, `toGeminiTools`, `fromGeminiResponse`) |
| `apps/web/src/agent/providers/index.ts` | Modify | Add `"gemini"` case to factory switch |
| `apps/web/src/agent/providers/__tests__/gemini.test.ts` | Create | Conversion tests, error paths, tool call ID synthesis |
| `apps/web/src/agent/providers/__tests__/index.test.ts` | Modify | Add factory test for `"gemini"``GeminiAdapter` |
| `apps/web/package.json` | Modify | Add `@google/generative-ai` dependency |
| `apps/web/.env.example` | Modify | Add Gemini example config lines |
## Interfaces / Contracts
No interface changes. The adapter implements the existing `ProviderAdapter`:
```ts
// Reused as-is from providers/types.ts
interface ProviderAdapter {
chat(params: {
messages: ChatMessage[];
systemPrompt: string;
tools: ToolSchema[];
}): Promise<ProviderResponse>;
}
```
### Internal helper signatures (gemini.ts)
```ts
function toGeminiContents(
messages: ChatMessage[],
): GenerateContentRequest["contents"];
function toGeminiTools(
tools: ToolSchema[],
): FunctionDeclaration[];
function fromGeminiResponse(
response: GenerateContentResult,
): ProviderResponse;
```
### Tool call ID — name collision handling
When mapping `tool_result` back to Gemini's `functionResponse`, Gemini keys by function **name** not ID. If multiple calls to the same tool exist in history, we group them by name and send the last result per name. This is acceptable because:
1. Current tool set (`transcribe_video`) is single-invocation per turn
2. Multi-call same-tool scenarios are deferred to a later change
3. Orchestrator already handles the ID→name correlation client-side
## Testing Strategy
| Layer | What to Test | Approach |
|-------|-------------|----------|
| Unit | `toGeminiContents` mapping (user, assistant, tool_result, assistant+toolCalls) | Mock SDK, inspect call args |
| Unit | `toGeminiTools` type casing (string→STRING, etc.) | Direct assertion on output |
| Unit | `fromGeminiResponse` content extraction | Construct Gemini response shape, assert `ProviderResponse` |
| Unit | Tool call ID synthesis (non-empty, unique) | Multiple calls produce different IDs |
| Unit | Error: empty candidates → throws | Assert error message |
| Unit | Error: SDK throws → propagates | Assert error bubbles up |
| Unit | Factory: `"gemini"``GeminiAdapter` | Instance check |
| Integration | Full `chat()` round-trip with mocked SDK | End-to-end within adapter |
## Migration / Rollout
No migration required. Additive change — setting `LLM_PROVIDER=gemini` activates the new adapter. Existing `openai-compatible` default is untouched.
## Open Questions
- [ ] Confirm `@google/generative-ai` latest stable version at impl time (currently ^0.21)
- [ ] Verify Gemini model naming convention user will use (e.g. `gemini-2.0-flash` vs `models/gemini-2.0-flash`)

View File

@ -0,0 +1,65 @@
# Proposal: Gemini Adapter — Phase 1
## Intent
Add Gemini as a native provider behind the existing provider-agnostic LLM seam so the agent can use Gemini for plain chat and tool calling without changing orchestrator or route contracts.
## Scope
### In Scope
- Native `GeminiAdapter` implementing the current `ProviderAdapter`
- Gemini free-form chat using existing stateless full-history flow
- Gemini tool calling, with `transcribe_video` as the first real tool
- Config support for `LLM_PROVIDER=gemini`, `LLM_API_KEY`, `LLM_MODEL`
- Tests and env/docs updates while preserving the OpenAI-compatible path
### Out of Scope
- Video/file upload or direct file understanding
- `@asset` references, broader multimodal inputs, and image/video reasoning
- Streaming UX or adapter contract changes for streaming
- Safety/grounding/thinking configuration beyond defaults
## Capabilities
### New Capabilities
- None
### Modified Capabilities
- `agent-session-shell`: expand provider selection/configuration so the existing provider adapter contract also supports native Gemini chat and function calling
## Approach
Implement a native Gemini SDK-backed adapter and register it in the provider factory. Keep the adapter stateless by mapping full internal message history into Gemini `generateContent()` requests, converting tool schemas/results at the adapter boundary, and synthesizing tool call IDs so internal contracts stay unchanged.
## Affected Areas
| Area | Impact | Description |
|------|--------|-------------|
| `apps/web/src/agent/providers/gemini.ts` | New | Gemini request/response/tool conversion |
| `apps/web/src/agent/providers/index.ts` | Modified | Factory registration for `gemini` |
| `apps/web/src/agent/providers/__tests__/gemini.test.ts` | New | Adapter conversion/error tests |
| `apps/web/src/agent/providers/__tests__/index.test.ts` | Modified | Factory coverage |
| `apps/web/package.json` | Modified | Gemini SDK dependency |
| `apps/web/.env.example` | Modified | Gemini env configuration |
## Risks
| Risk | Likelihood | Mitigation |
|------|------------|------------|
| Gemini has no tool call IDs | Med | Synthesize unique IDs inside adapter |
| Gemini type casing differs from internal schema | Med | Centralize schema mapping tests |
| Gemini system prompt placement differs | Low | Map system prompt explicitly to Gemini config |
## Rollback Plan
Remove the Gemini adapter, factory branch, dependency, and env docs; keep `openai-compatible` as the unchanged default path.
## Dependencies
- Gemini JS SDK selection to be verified during implementation
## Success Criteria
- [ ] `LLM_PROVIDER=gemini` resolves a native Gemini adapter without breaking `openai-compatible`
- [ ] Gemini returns normal assistant text through the existing route/orchestrator flow
- [ ] Gemini can emit structured tool calls and execute `transcribe_video` through the existing loop

View File

@ -0,0 +1,97 @@
# Delta for agent-session-shell
## MODIFIED Requirements
### Requirement: Provider Adapter Interface
The system SHALL define a server-side provider adapter interface with a single method `chat(params: ProviderChatParams): Promise<ProviderChatResponse>`. `ProviderChatParams` MUST include `messages`, `systemPrompt`, and `toolSchemas` in a provider-agnostic internal format. `ProviderChatResponse` MUST include `content: string` and `toolCalls?: ProviderToolCall[]` where `ProviderToolCall { id, name, args }`. Each concrete adapter converts internal tool schemas to its own wire format. The factory MUST support at least `openai-compatible` and `gemini` providers. Adapters whose native API does not assign IDs to tool calls MUST synthesize unique IDs so every returned `ToolCall.id` is a non-empty string.
(Previously: Phase 1 MAY ship exactly one adapter (OpenAI-compatible) as long as the interface contract remains provider-agnostic.)
#### Scenario: Adapter translates tool schemas to wire format
- GIVEN `transcribe_video` has internal parameters `[{ key: "assetId", type: "string", required: false }]`
- WHEN the adapter receives tool schemas before calling the provider
- THEN it converts them to its provider-specific wire format
#### Scenario: Adapter selected by config — OpenAI-compatible
- GIVEN `LLM_PROVIDER` is set to `"openai-compatible"`
- WHEN the route resolves the adapter
- THEN the OpenAI-compatible adapter is returned
#### Scenario: Adapter selected by config — Gemini
- GIVEN `LLM_PROVIDER` is set to `"gemini"`
- WHEN the route resolves the adapter
- THEN the Gemini adapter is returned
#### Scenario: Gemini adapter synthesizes tool-call IDs
- GIVEN the Gemini API returns a function call response with no call ID
- WHEN the Gemini adapter maps the response to `ProviderResponse`
- THEN each `ToolCall` in `toolCalls` has a non-empty `id` field generated by the adapter
### Requirement: Provider Configuration
Server-only environment variables `LLM_PROVIDER` (required), `LLM_MODEL` (required), `LLM_API_KEY` (required), and `LLM_BASE_URL` (optional) MUST determine which adapter and model the route uses. `LLM_PROVIDER` MUST accept at least `"openai-compatible"` and `"gemini"`. When `LLM_PROVIDER` is `"gemini"`, `LLM_BASE_URL` MUST be ignored. Configuration MUST be read server-side only — never exposed to the client.
(Previously: Provider config listed env vars but did not constrain valid provider values or describe Gemini-specific behavior.)
#### Scenario: Missing required configuration
- GIVEN `LLM_API_KEY` is not set in the environment
- WHEN the route attempts to initialize the provider
- THEN the call returns 502 with a missing-configuration error
#### Scenario: Unknown provider
- GIVEN `LLM_PROVIDER` is set to `"unknown-provider"`
- WHEN the route resolves the adapter
- THEN the factory throws `"Unknown LLM provider: unknown-provider"`
#### Scenario: Gemini config ignores base URL
- GIVEN `LLM_PROVIDER` is `"gemini"` and `LLM_BASE_URL` is set
- WHEN the Gemini adapter is created
- THEN `LLM_BASE_URL` is not used in the API call
## ADDED Requirements
### Requirement: Gemini Free-Form Chat
When the Gemini adapter receives a request with no tool schemas (or tools array is empty), it MUST send messages to the Gemini `generateContent` API and return a `ProviderResponse` with `content` set to the assistant text and `toolCalls` undefined.
#### Scenario: Plain chat through Gemini
- GIVEN `LLM_PROVIDER` is `"gemini"` and messages contain `[{ role: "user", content: "Hello" }]`
- WHEN the adapter calls Gemini and receives a text-only response
- THEN `ProviderResponse.content` is the assistant text and `toolCalls` is undefined
#### Scenario: System prompt mapping
- GIVEN the adapter receives `systemPrompt: "You are a video editor"`
- WHEN it builds the Gemini request
- THEN the system prompt is mapped to Gemini's system instruction field, not as a user message
### Requirement: Gemini Tool Calling
When the Gemini adapter receives a request with tool schemas, it MUST convert internal `ToolSchema[]` to Gemini function declarations, call `generateContent` with declarations attached, and if Gemini returns function call parts, map each to a `ToolCall { id, name, args }` with a synthesized non-empty `id`.
#### Scenario: Gemini returns function calls
- GIVEN `transcribe_video` is in tool schemas and Gemini responds with a function call part `{ name: "transcribe_video", args: {} }`
- WHEN the adapter maps the response
- THEN `toolCalls` contains one `ToolCall` with `name: "transcribe_video"`, `args: {}`, and a synthesized `id`
#### Scenario: Gemini returns text despite tools
- GIVEN tool schemas are provided but Gemini responds with only text
- WHEN the adapter maps the response
- THEN `toolCalls` is undefined and `content` holds the text
#### Scenario: Tool schema type mapping
- GIVEN an internal tool schema has parameters `[{ key: "assetId", type: "string", required: true }]`
- WHEN the adapter converts to Gemini function declarations
- THEN the parameter appears in `FunctionDeclaration.parameters.properties` with type `"string"`

View File

@ -0,0 +1,28 @@
# Tasks: Gemini Adapter — Phase 1
## Phase 1: Foundation
- [x] 1.1 Add `@google/generative-ai` dependency to `apps/web/package.json` (pin to latest stable `^0.21` or newer)
- [x] 1.2 Add Gemini env config lines to `apps/web/.env.example`: `LLM_PROVIDER=gemini`, `LLM_API_KEY`, `LLM_MODEL=gemini-2.0-flash`, comment noting `LLM_BASE_URL` is ignored for gemini
## Phase 2: Core Adapter
- [x] 2.1 Create `apps/web/src/agent/providers/gemini.ts` — import `GoogleGenerativeAI` from `@google/generative-ai`, `SchemaType` for type mapping
- [x] 2.2 Implement `toGeminiContents(messages: ChatMessage[])` — map `user→user`, `assistant→model` (with `functionCall` parts), `tool_result→function` role with `functionResponse`; skip `system` role (handled separately)
- [x] 2.3 Implement `toGeminiTools(tools: ToolSchema[])` — convert each `ToolSchema` to a `FunctionDeclaration` with `SchemaType` property types (`string→STRING`, `number→NUMBER`, `boolean→BOOLEAN`, `object→OBJECT`); return empty array when no tools
- [x] 2.4 Implement `fromGeminiResponse(response)` — extract `candidate.content.parts`, pull text into `content`, filter `functionCall` parts into `toolCalls[]` with `id: crypto.randomUUID()`, `name`, `args`; throw on empty candidates
- [x] 2.5 Implement `GeminiAdapter` class — constructor takes `ProviderConfig`, creates `GoogleGenerativeAI` + `getGenerativeModel({ model, systemInstruction })`; `chat()` calls `model.generateContent({ contents, tools })` then returns `fromGeminiResponse()`; ignore `config.baseUrl`
## Phase 3: Factory Integration
- [x] 3.1 Add `"gemini"` case to `createProvider()` switch in `apps/web/src/agent/providers/index.ts` — import and return `new GeminiAdapter(config)`
- [x] 3.2 Verify existing `openai-compatible` factory case is untouched and all existing tests still pass
## Phase 4: Tests
- [x] 4.1 Create `apps/web/src/agent/providers/__tests__/gemini.test.ts` — test `toGeminiContents`: user message, assistant with toolCalls, tool_result mapping, system message excluded from contents
- [x] 4.2 Test `toGeminiTools`: single tool with parameters, type casing (`string→STRING` etc.), empty tools returns empty array
- [x] 4.3 Test `fromGeminiResponse`: text-only response → `{ content, toolCalls: undefined }`; function-call response → `toolCalls` with non-empty synthesized IDs; two calls produce different IDs
- [x] 4.4 Test error paths: empty candidates → throws; SDK error → propagates
- [x] 4.5 Add factory test to `apps/web/src/agent/providers/__tests__/index.test.ts`: `"gemini"` config → `GeminiAdapter` instance; unknown provider still throws
- [x] 4.6 Test full `chat()` round-trip with mocked `GoogleGenerativeAI` — verify `generateContent` called with correct contents, tools, and systemInstruction

View File

@ -0,0 +1,116 @@
# Verification Report
**Change**: gemini-adapter-fase-1
**Version**: N/A
**Mode**: Standard
---
### Completeness
| Metric | Value |
|--------|-------|
| Tasks total | 14 |
| Tasks complete | 14 |
| Tasks incomplete | 0 |
All tasks in `openspec/changes/gemini-adapter-fase-1/tasks.md` are marked complete, including the post-verify fixes.
---
### Build & Tests Execution
**Build**: Skipped
```text
No explicit `rules.verify.build_command` is configured in `openspec/config.yaml`.
Project instructions also say never build after changes, so verification used TypeScript type-checking instead of `next build`.
```
**Type Check**: ✅ Passed
```text
Command: bunx tsc --noEmit
(no output)
```
**Tests**: ✅ 48 passed / ❌ 0 failed / ⚠️ 0 skipped
```text
Command: bun test src/agent/providers/__tests__ src/app/api/agent/chat/__tests__/route.test.ts
bun test v1.3.13
48 pass
0 fail
106 expect() calls
Ran 48 tests across 4 files.
```
**Coverage**: Not available
---
### Spec Compliance Matrix
| Requirement | Scenario | Test | Result |
|-------------|----------|------|--------|
| Provider Adapter Interface | Adapter translates tool schemas to wire format | `src/agent/providers/__tests__/gemini.test.ts > converts a single tool with parameters` + `maps type strings to Gemini SchemaType values` | ✅ COMPLIANT |
| Provider Adapter Interface | Adapter selected by config — OpenAI-compatible | `src/agent/providers/__tests__/index.test.ts > returns OpenAICompatibleAdapter for openai-compatible provider` | ✅ COMPLIANT |
| Provider Adapter Interface | Adapter selected by config — Gemini | `src/agent/providers/__tests__/index.test.ts > returns GeminiAdapter for gemini provider` | ✅ COMPLIANT |
| Provider Adapter Interface | Gemini adapter synthesizes tool-call IDs | `src/agent/providers/__tests__/gemini.test.ts > maps function-call response to toolCalls with non-empty synthesized IDs` | ✅ COMPLIANT |
| Provider Configuration | Missing required configuration | `src/app/api/agent/chat/__tests__/route.test.ts > returns 502 when LLM_API_KEY is missing` | ✅ COMPLIANT |
| Provider Configuration | Unknown provider | `src/agent/providers/__tests__/index.test.ts > throws Error for unknown provider` | ✅ COMPLIANT |
| Provider Configuration | Gemini config ignores base URL | `src/agent/providers/__tests__/gemini.test.ts > ignores baseUrl in config — Gemini still works` | ✅ COMPLIANT |
| Gemini Free-Form Chat | Plain chat through Gemini | `src/agent/providers/__tests__/gemini.test.ts > calls generateContent with correct contents and no tools` | ✅ COMPLIANT |
| Gemini Free-Form Chat | System prompt mapping | `src/agent/providers/__tests__/gemini.test.ts > calls generateContent with correct contents and no tools` | ✅ COMPLIANT |
| Gemini Tool Calling | Gemini returns function calls | `src/agent/providers/__tests__/gemini.test.ts > calls generateContent with tools and systemInstruction` | ✅ COMPLIANT |
| Gemini Tool Calling | Gemini returns text despite tools | `src/agent/providers/__tests__/gemini.test.ts > returns text when tools are provided but Gemini responds with text only` | ✅ COMPLIANT |
| Gemini Tool Calling | Tool schema type mapping | `src/agent/providers/__tests__/gemini.test.ts > maps type strings to Gemini SchemaType values` | ✅ COMPLIANT |
**Compliance summary**: 12/12 scenarios compliant
---
### Correctness (Static — Structural Evidence)
| Requirement | Status | Notes |
|------------|--------|-------|
| Provider Adapter Interface | ✅ Implemented | `apps/web/src/agent/providers/types.ts` keeps the provider-agnostic `chat()` seam, `index.ts` supports both providers, and `gemini.ts` synthesizes tool-call IDs with `crypto.randomUUID()`. |
| Provider Configuration | ✅ Implemented | `apps/web/src/app/api/agent/chat/route.ts` reads `LLM_PROVIDER`, `LLM_API_KEY`, `LLM_MODEL`, and `LLM_BASE_URL` server-side only; `GeminiAdapter` intentionally ignores `baseUrl`. |
| Gemini Free-Form Chat | ✅ Implemented | `GeminiAdapter.chat()` maps messages into Gemini contents, sets `systemInstruction` when present, and returns text-only responses without `toolCalls`. |
| Gemini Tool Calling | ✅ Implemented | `toGeminiTools()` converts internal tool schemas to Gemini declarations and `fromGeminiResponse()` maps function-call parts back into provider `toolCalls`. |
| Explicit non-goals remain deferred | ✅ Implemented | The touched files remain additive only; no streaming, multimodal upload, `@asset`, or contract changes were introduced. |
---
### Coherence (Design)
| Decision | Followed? | Notes |
|----------|-----------|-------|
| Use `@google/generative-ai` SDK | ✅ Yes | Added in `apps/web/package.json`; adapter imports `GoogleGenerativeAI` and `SchemaType`. |
| Preserve stateless provider seam | ✅ Yes | `GeminiAdapter.chat()` sends full history through one `generateContent()` call with no session state. |
| Synthesize tool-call IDs inside adapter | ✅ Yes | `fromGeminiResponse()` generates IDs for Gemini function calls. |
| Keep OpenAI-compatible path intact | ✅ Yes | `createProvider()` still returns `OpenAICompatibleAdapter` unchanged for the original provider. |
| Map system prompt via Gemini-native field | ✅ Yes | `GeminiAdapter.chat()` sets `request.systemInstruction` instead of injecting a fake user message. |
---
### Issues Found
**CRITICAL** (must fix before archive):
None
**WARNING** (should fix):
None
**SUGGESTION** (nice to have):
None
---
### Verdict
PASS
All planned tasks are complete, the changed slice now type-checks, targeted runtime tests pass, and every spec scenario has passing behavioral evidence.

View File

@ -110,7 +110,7 @@ The system SHALL define types in `apps/web/src/agent/types.ts`: `ChatMessage { i
### Requirement: Provider Adapter Interface
The system SHALL define a server-side provider adapter interface with a single method `chat(params: ProviderChatParams): Promise<ProviderChatResponse>`. `ProviderChatParams` MUST include `messages`, `systemPrompt`, and `toolSchemas` in a provider-agnostic internal format. `ProviderChatResponse` MUST include `content: string` and `toolCalls?: ProviderToolCall[]` where `ProviderToolCall { id, name, args }`. Each concrete adapter converts internal tool schemas to its own wire format. Phase 1 MAY ship exactly one adapter (OpenAI-compatible) as long as the interface contract remains provider-agnostic.
The system SHALL define a server-side provider adapter interface with a single method `chat(params: ProviderChatParams): Promise<ProviderChatResponse>`. `ProviderChatParams` MUST include `messages`, `systemPrompt`, and `toolSchemas` in a provider-agnostic internal format. `ProviderChatResponse` MUST include `content: string` and `toolCalls?: ProviderToolCall[]` where `ProviderToolCall { id, name, args }`. Each concrete adapter converts internal tool schemas to its own wire format. The factory MUST support at least `openai-compatible` and `gemini` providers. Adapters whose native API does not assign IDs to tool calls MUST synthesize unique IDs so every returned `ToolCall.id` is a non-empty string.
#### Scenario: Adapter translates tool schemas to wire format
@ -118,15 +118,27 @@ The system SHALL define a server-side provider adapter interface with a single m
- WHEN the adapter receives tool schemas before calling the provider
- THEN it converts them to its provider-specific wire format
#### Scenario: Adapter selected by config
#### Scenario: Adapter selected by config — OpenAI-compatible
- GIVEN `LLM_PROVIDER` is set to `"openai-compatible"`
- WHEN the route resolves the adapter
- THEN the OpenAI-compatible adapter is returned
#### Scenario: Adapter selected by config — Gemini
- GIVEN `LLM_PROVIDER` is set to `"gemini"`
- WHEN the route resolves the adapter
- THEN the Gemini adapter is returned
#### Scenario: Gemini adapter synthesizes tool-call IDs
- GIVEN the Gemini API returns a function call response with no call ID
- WHEN the Gemini adapter maps the response to `ProviderResponse`
- THEN each `ToolCall` in `toolCalls` has a non-empty `id` field generated by the adapter
### Requirement: Provider Configuration
Server-only environment variables `LLM_PROVIDER` (required), `LLM_MODEL` (required), `LLM_API_KEY` (required), and `LLM_BASE_URL` (optional) MUST determine which adapter and model the route uses. Configuration MUST be read server-side only — never exposed to the client.
Server-only environment variables `LLM_PROVIDER` (required), `LLM_MODEL` (required), `LLM_API_KEY` (required), and `LLM_BASE_URL` (optional) MUST determine which adapter and model the route uses. `LLM_PROVIDER` MUST accept at least `"openai-compatible"` and `"gemini"`. When `LLM_PROVIDER` is `"gemini"`, `LLM_BASE_URL` MUST be ignored. Configuration MUST be read server-side only — never exposed to the client.
#### Scenario: Missing required configuration
@ -134,6 +146,18 @@ Server-only environment variables `LLM_PROVIDER` (required), `LLM_MODEL` (requir
- WHEN the route attempts to initialize the provider
- THEN the call returns 502 with a missing-configuration error
#### Scenario: Unknown provider
- GIVEN `LLM_PROVIDER` is set to `"unknown-provider"`
- WHEN the route resolves the adapter
- THEN the factory throws `"Unknown LLM provider: unknown-provider"`
#### Scenario: Gemini config ignores base URL
- GIVEN `LLM_PROVIDER` is `"gemini"` and `LLM_BASE_URL` is set
- WHEN the Gemini adapter is created
- THEN `LLM_BASE_URL` is not used in the API call
### Requirement: Tool Argument Validation
Before executing any tool call, the orchestrator MUST validate that each argument key declared `required: true` in the tool's `parameters` is present in the call args and is of the declared type. If validation fails, the tool MUST NOT execute; instead the orchestrator MUST produce a `ToolResult` with `error` describing the missing or malformed argument.
@ -166,3 +190,41 @@ The orchestrator loop MUST enforce a hard iteration cap of **8**. An iteration i
- GIVEN the provider returns a final text response after 2 tool-call iterations
- WHEN the orchestrator processes the response
- THEN no cap message is appended and status is `"idle"`
### Requirement: Gemini Free-Form Chat
When the Gemini adapter receives a request with no tool schemas (or tools array is empty), it MUST send messages to the Gemini `generateContent` API and return a `ProviderResponse` with `content` set to the assistant text and `toolCalls` undefined.
#### Scenario: Plain chat through Gemini
- GIVEN `LLM_PROVIDER` is `"gemini"` and messages contain `[{ role: "user", content: "Hello" }]`
- WHEN the adapter calls Gemini and receives a text-only response
- THEN `ProviderResponse.content` is the assistant text and `toolCalls` is undefined
#### Scenario: System prompt mapping
- GIVEN the adapter receives `systemPrompt: "You are a video editor"`
- WHEN it builds the Gemini request
- THEN the system prompt is mapped to Gemini's system instruction field, not as a user message
### Requirement: Gemini Tool Calling
When the Gemini adapter receives a request with tool schemas, it MUST convert internal `ToolSchema[]` to Gemini function declarations, call `generateContent` with declarations attached, and if Gemini returns function call parts, map each to a `ToolCall { id, name, args }` with a synthesized non-empty `id`.
#### Scenario: Gemini returns function calls
- GIVEN `transcribe_video` is in tool schemas and Gemini responds with a function call part `{ name: "transcribe_video", args: {} }`
- WHEN the adapter maps the response
- THEN `toolCalls` contains one `ToolCall` with `name: "transcribe_video"`, `args: {}`, and a synthesized `id`
#### Scenario: Gemini returns text despite tools
- GIVEN tool schemas are provided but Gemini responds with only text
- WHEN the adapter maps the response
- THEN `toolCalls` is undefined and `content` holds the text
#### Scenario: Tool schema type mapping
- GIVEN an internal tool schema has parameters `[{ key: "assetId", type: "string", required: true }]`
- WHEN the adapter converts to Gemini function declarations
- THEN the parameter appears in `FunctionDeclaration.parameters.properties` with type `"string"`