feat: add native gemini provider
This commit is contained in:
parent
c07e33b91d
commit
17eaaaf73e
|
|
@ -25,3 +25,9 @@ LLM_PROVIDER=openai-compatible
|
|||
LLM_API_KEY=your_openai_api_key_here
|
||||
LLM_MODEL=gpt-4o
|
||||
# LLM_BASE_URL=https://api.openai.com/v1
|
||||
|
||||
# Gemini provider (set LLM_PROVIDER=gemini to activate)
|
||||
# LLM_PROVIDER=gemini
|
||||
# LLM_API_KEY=your_gemini_api_key_here
|
||||
# LLM_MODEL=gemini-2.0-flash
|
||||
# Note: LLM_BASE_URL is ignored when using the gemini provider
|
||||
|
|
|
|||
|
|
@ -42,6 +42,7 @@
|
|||
"cmdk": "^1.0.0",
|
||||
"culori": "^4.0.2",
|
||||
"drizzle-orm": "^0.44.2",
|
||||
"@google/generative-ai": "^0.24.0",
|
||||
"embla-carousel-react": "^8.5.1",
|
||||
"eventemitter3": "^5.0.1",
|
||||
"feed": "^5.1.0",
|
||||
|
|
|
|||
|
|
@ -37,6 +37,46 @@ describe("buildSystemPrompt", () => {
|
|||
expect(prompt).toContain("NeuralCut video editor");
|
||||
});
|
||||
|
||||
test("includes asset internal id in media listing", () => {
|
||||
const context: AgentContext = {
|
||||
projectId: "proj-1",
|
||||
activeSceneId: "scene-A",
|
||||
mediaAssets: [
|
||||
{ id: "m1", name: "intro.mp4", type: "video", duration: 30 },
|
||||
],
|
||||
playbackTimeMs: 0,
|
||||
};
|
||||
|
||||
const prompt = buildSystemPrompt(context);
|
||||
|
||||
expect(prompt).toContain("[id: m1]");
|
||||
});
|
||||
|
||||
test("includes assetId instruction when media assets are present", () => {
|
||||
const context: AgentContext = {
|
||||
projectId: "proj-1",
|
||||
activeSceneId: "scene-A",
|
||||
mediaAssets: [
|
||||
{ id: "m1", name: "conclu.mov", type: "video", duration: 60 },
|
||||
],
|
||||
playbackTimeMs: 0,
|
||||
};
|
||||
|
||||
const prompt = buildSystemPrompt(context);
|
||||
|
||||
expect(prompt).toContain("IMPORTANT");
|
||||
expect(prompt).toContain("assetId");
|
||||
expect(prompt).toContain('internal "id" value');
|
||||
expect(prompt).toContain("NOT the filename");
|
||||
});
|
||||
|
||||
test("omits assetId instruction when no media assets", () => {
|
||||
const prompt = buildSystemPrompt(BASE_CONTEXT);
|
||||
|
||||
expect(prompt).not.toContain("IMPORTANT");
|
||||
expect(prompt).not.toContain("assetId");
|
||||
});
|
||||
|
||||
test("is valid when no media assets are loaded", () => {
|
||||
const context: AgentContext = {
|
||||
projectId: "proj-2",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,502 @@
|
|||
import { beforeEach, describe, expect, jest, test } from "bun:test";
|
||||
import { SchemaType, type GenerateContentResult } from "@google/generative-ai";
|
||||
import type { ProviderConfig } from "@/agent/providers/types";
|
||||
import type { ToolSchema } from "@/agent/types";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Mock the Google Generative AI SDK before the adapter module loads
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const mockGenerateContent = jest.fn();
|
||||
|
||||
jest.mock("@google/generative-ai", () => ({
|
||||
GoogleGenerativeAI: class MockGoogleGenerativeAI {
|
||||
getGenerativeModel(_opts: { model: string }) {
|
||||
return { generateContent: mockGenerateContent };
|
||||
}
|
||||
constructor(_opts: { apiKey: string }) {}
|
||||
},
|
||||
SchemaType: {
|
||||
STRING: "string",
|
||||
NUMBER: "number",
|
||||
BOOLEAN: "boolean",
|
||||
OBJECT: "object",
|
||||
},
|
||||
}));
|
||||
|
||||
// Import after mock is set up
|
||||
import {
|
||||
GeminiAdapter,
|
||||
toGeminiContents,
|
||||
toGeminiTools,
|
||||
fromGeminiResponse,
|
||||
} from "@/agent/providers/gemini";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const TEST_CONFIG: ProviderConfig = {
|
||||
provider: "gemini",
|
||||
apiKey: "test-gemini-key",
|
||||
model: "gemini-2.0-flash",
|
||||
};
|
||||
|
||||
const SAMPLE_TOOLS: ToolSchema[] = [
|
||||
{
|
||||
name: "transcribe_video",
|
||||
description: "Transcribes a video asset",
|
||||
parameters: [
|
||||
{ key: "assetId", type: "string", required: true },
|
||||
{ key: "language", type: "string", required: false },
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
function makeGeminiResponse(overrides: {
|
||||
text?: string;
|
||||
functionCalls?: Array<{ name: string; args: Record<string, unknown> }>;
|
||||
}): GenerateContentResult {
|
||||
const parts: Array<Record<string, unknown>> = [];
|
||||
|
||||
if (overrides.text !== undefined) {
|
||||
parts.push({ text: overrides.text });
|
||||
}
|
||||
|
||||
if (overrides.functionCalls) {
|
||||
for (const fc of overrides.functionCalls) {
|
||||
parts.push({ functionCall: { name: fc.name, args: fc.args } });
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
response: {
|
||||
candidates: [
|
||||
{
|
||||
content: { parts },
|
||||
},
|
||||
],
|
||||
},
|
||||
} as unknown as GenerateContentResult;
|
||||
}
|
||||
|
||||
function makeChatMessage(
|
||||
role: "user" | "assistant" | "system" | "tool_result",
|
||||
content: string,
|
||||
extra?: Record<string, unknown>,
|
||||
) {
|
||||
return {
|
||||
id: `msg-${Math.random().toString(36).slice(2)}`,
|
||||
role,
|
||||
content,
|
||||
timestamp: Date.now(),
|
||||
...extra,
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Tests: toGeminiContents (Task 4.1)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("toGeminiContents", () => {
|
||||
test("maps user message to Gemini user role", () => {
|
||||
const messages = [makeChatMessage("user", "Hello")];
|
||||
const contents = toGeminiContents(messages);
|
||||
|
||||
expect(contents).toHaveLength(1);
|
||||
expect(contents[0].role).toBe("user");
|
||||
expect(contents[0].parts).toEqual([{ text: "Hello" }]);
|
||||
});
|
||||
|
||||
test("maps assistant message to Gemini model role", () => {
|
||||
const messages = [makeChatMessage("assistant", "Hi there!")];
|
||||
const contents = toGeminiContents(messages);
|
||||
|
||||
expect(contents).toHaveLength(1);
|
||||
expect(contents[0].role).toBe("model");
|
||||
expect(contents[0].parts).toEqual([{ text: "Hi there!" }]);
|
||||
});
|
||||
|
||||
test("maps assistant with toolCalls to model role with functionCall parts", () => {
|
||||
const messages = [
|
||||
makeChatMessage("assistant", "Let me check.", {
|
||||
toolCalls: [
|
||||
{
|
||||
id: "tc_1",
|
||||
name: "transcribe_video",
|
||||
args: { assetId: "vid-123" },
|
||||
},
|
||||
],
|
||||
}),
|
||||
];
|
||||
const contents = toGeminiContents(messages);
|
||||
|
||||
expect(contents).toHaveLength(1);
|
||||
expect(contents[0].role).toBe("model");
|
||||
expect(contents[0].parts).toEqual([
|
||||
{ text: "Let me check." },
|
||||
{
|
||||
functionCall: {
|
||||
name: "transcribe_video",
|
||||
args: { assetId: "vid-123" },
|
||||
},
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
test("maps tool_result to function role with functionResponse", () => {
|
||||
const messages = [
|
||||
makeChatMessage("assistant", "", {
|
||||
toolCalls: [
|
||||
{
|
||||
id: "tc_1",
|
||||
name: "transcribe_video",
|
||||
args: { assetId: "vid-123" },
|
||||
},
|
||||
],
|
||||
}),
|
||||
makeChatMessage("tool_result", '{"transcript": "Hello world"}', {
|
||||
toolCallId: "tc_1",
|
||||
}),
|
||||
];
|
||||
const contents = toGeminiContents(messages);
|
||||
|
||||
// First: model (assistant with toolCalls)
|
||||
expect(contents[0].role).toBe("model");
|
||||
// Second: function (tool_result)
|
||||
expect(contents[1].role).toBe("function");
|
||||
expect(contents[1].parts).toEqual([
|
||||
{
|
||||
functionResponse: {
|
||||
name: "transcribe_video",
|
||||
response: { transcript: "Hello world" },
|
||||
},
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
test("excludes system messages from contents", () => {
|
||||
const messages = [
|
||||
makeChatMessage("system", "You are a video editor."),
|
||||
makeChatMessage("user", "Hello"),
|
||||
];
|
||||
const contents = toGeminiContents(messages);
|
||||
|
||||
expect(contents).toHaveLength(1);
|
||||
expect(contents[0].role).toBe("user");
|
||||
});
|
||||
|
||||
test("handles tool_result with non-JSON content by wrapping in object", () => {
|
||||
const messages = [
|
||||
makeChatMessage("assistant", "", {
|
||||
toolCalls: [
|
||||
{ id: "tc_1", name: "some_tool", args: {} },
|
||||
],
|
||||
}),
|
||||
makeChatMessage("tool_result", "plain text result", {
|
||||
toolCallId: "tc_1",
|
||||
}),
|
||||
];
|
||||
const contents = toGeminiContents(messages);
|
||||
|
||||
expect(contents[1].parts).toEqual([
|
||||
{
|
||||
functionResponse: {
|
||||
name: "some_tool",
|
||||
response: { result: "plain text result" },
|
||||
},
|
||||
},
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Tests: toGeminiTools (Task 4.2)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("toGeminiTools", () => {
|
||||
test("converts a single tool with parameters", () => {
|
||||
const declarations = toGeminiTools(SAMPLE_TOOLS);
|
||||
|
||||
expect(declarations).toHaveLength(1);
|
||||
expect(declarations[0].name).toBe("transcribe_video");
|
||||
expect(declarations[0].description).toBe("Transcribes a video asset");
|
||||
expect(declarations[0].parameters?.type).toBe(SchemaType.OBJECT);
|
||||
expect(declarations[0].parameters?.properties).toHaveProperty("assetId");
|
||||
expect(declarations[0].parameters?.required).toEqual(["assetId"]);
|
||||
});
|
||||
|
||||
test("maps type strings to Gemini SchemaType values", () => {
|
||||
const tools: ToolSchema[] = [
|
||||
{
|
||||
name: "multi_param_tool",
|
||||
description: "Tool with all param types",
|
||||
parameters: [
|
||||
{ key: "name", type: "string", required: true },
|
||||
{ key: "count", type: "number", required: false },
|
||||
{ key: "active", type: "boolean", required: false },
|
||||
{ key: "meta", type: "object", required: false },
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
const declarations = toGeminiTools(tools);
|
||||
const props = declarations[0].parameters?.properties as Record<
|
||||
string,
|
||||
{ type: string }
|
||||
>;
|
||||
|
||||
expect(props.name.type).toBe(SchemaType.STRING);
|
||||
expect(props.count.type).toBe(SchemaType.NUMBER);
|
||||
expect(props.active.type).toBe(SchemaType.BOOLEAN);
|
||||
expect(props.meta.type).toBe(SchemaType.OBJECT);
|
||||
});
|
||||
|
||||
test("returns empty array when no tools provided", () => {
|
||||
expect(toGeminiTools([])).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Tests: fromGeminiResponse (Task 4.3)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("fromGeminiResponse", () => {
|
||||
test("extracts text-only response with no toolCalls", () => {
|
||||
const response = makeGeminiResponse({ text: "Hello from Gemini!" });
|
||||
const result = fromGeminiResponse(response);
|
||||
|
||||
expect(result.content).toBe("Hello from Gemini!");
|
||||
expect(result.toolCalls).toBeUndefined();
|
||||
});
|
||||
|
||||
test("maps function-call response to toolCalls with non-empty synthesized IDs", () => {
|
||||
const response = makeGeminiResponse({
|
||||
text: undefined,
|
||||
functionCalls: [
|
||||
{ name: "transcribe_video", args: { assetId: "vid-1" } },
|
||||
],
|
||||
});
|
||||
const result = fromGeminiResponse(response);
|
||||
|
||||
expect(result.content).toBe("");
|
||||
expect(result.toolCalls).toHaveLength(1);
|
||||
expect(result.toolCalls?.[0].name).toBe("transcribe_video");
|
||||
expect(result.toolCalls?.[0].args).toEqual({ assetId: "vid-1" });
|
||||
// Non-empty synthesized ID
|
||||
expect(result.toolCalls?.[0].id).toBeTruthy();
|
||||
expect(typeof result.toolCalls?.[0].id).toBe("string");
|
||||
});
|
||||
|
||||
test("two calls produce different IDs", () => {
|
||||
const response1 = makeGeminiResponse({
|
||||
functionCalls: [{ name: "tool_a", args: {} }],
|
||||
});
|
||||
const response2 = makeGeminiResponse({
|
||||
functionCalls: [{ name: "tool_a", args: {} }],
|
||||
});
|
||||
|
||||
const result1 = fromGeminiResponse(response1);
|
||||
const result2 = fromGeminiResponse(response2);
|
||||
|
||||
expect(result1.toolCalls?.[0].id).not.toBe(
|
||||
result2.toolCalls?.[0].id,
|
||||
);
|
||||
});
|
||||
|
||||
test("handles mixed text and function-call parts", () => {
|
||||
const response = makeGeminiResponse({
|
||||
text: "I'll look that up.",
|
||||
functionCalls: [{ name: "search", args: { q: "test" } }],
|
||||
});
|
||||
const result = fromGeminiResponse(response);
|
||||
|
||||
expect(result.content).toBe("I'll look that up.");
|
||||
expect(result.toolCalls).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Tests: error paths (Task 4.4)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("error paths", () => {
|
||||
test("throws on empty candidates", () => {
|
||||
const response = {
|
||||
response: { candidates: [] },
|
||||
};
|
||||
|
||||
expect(() => fromGeminiResponse(response as any)).toThrow(
|
||||
"Empty response from Gemini provider",
|
||||
);
|
||||
});
|
||||
|
||||
test("throws on null candidates", () => {
|
||||
const response = {
|
||||
response: { candidates: null },
|
||||
};
|
||||
|
||||
expect(() => fromGeminiResponse(response as any)).toThrow(
|
||||
"Empty response from Gemini provider",
|
||||
);
|
||||
});
|
||||
|
||||
test("SDK error propagates from chat()", async () => {
|
||||
mockGenerateContent.mockRejectedValueOnce(
|
||||
new Error("API key not valid"),
|
||||
);
|
||||
|
||||
const adapter = new GeminiAdapter(TEST_CONFIG);
|
||||
|
||||
expect(
|
||||
adapter.chat({
|
||||
messages: [makeChatMessage("user", "Hello")],
|
||||
systemPrompt: "",
|
||||
tools: [],
|
||||
}),
|
||||
).rejects.toThrow("API key not valid");
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Tests: full chat() round-trip (Task 4.6)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("GeminiAdapter.chat()", () => {
|
||||
beforeEach(() => {
|
||||
mockGenerateContent.mockReset();
|
||||
});
|
||||
|
||||
test("calls generateContent with correct contents and no tools", async () => {
|
||||
mockGenerateContent.mockResolvedValueOnce(
|
||||
makeGeminiResponse({ text: "Hi!" }),
|
||||
);
|
||||
|
||||
const adapter = new GeminiAdapter(TEST_CONFIG);
|
||||
const result = await adapter.chat({
|
||||
messages: [makeChatMessage("user", "Hello")],
|
||||
systemPrompt: "You are helpful.",
|
||||
tools: [],
|
||||
});
|
||||
|
||||
expect(result.content).toBe("Hi!");
|
||||
expect(result.toolCalls).toBeUndefined();
|
||||
|
||||
const callArgs = mockGenerateContent.mock.calls[0][0];
|
||||
expect(callArgs.contents).toHaveLength(1);
|
||||
expect(callArgs.contents[0].role).toBe("user");
|
||||
expect(callArgs.systemInstruction).toBe("You are helpful.");
|
||||
expect(callArgs.tools).toBeUndefined();
|
||||
});
|
||||
|
||||
test("calls generateContent with tools and systemInstruction", async () => {
|
||||
mockGenerateContent.mockResolvedValueOnce(
|
||||
makeGeminiResponse({
|
||||
functionCalls: [
|
||||
{
|
||||
name: "transcribe_video",
|
||||
args: { assetId: "vid-1" },
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
const adapter = new GeminiAdapter(TEST_CONFIG);
|
||||
const result = await adapter.chat({
|
||||
messages: [makeChatMessage("user", "Transcribe this")],
|
||||
systemPrompt: "Video editing assistant",
|
||||
tools: SAMPLE_TOOLS,
|
||||
});
|
||||
|
||||
expect(result.toolCalls).toHaveLength(1);
|
||||
expect(result.toolCalls?.[0].name).toBe("transcribe_video");
|
||||
|
||||
const callArgs = mockGenerateContent.mock.calls[0][0];
|
||||
expect(callArgs.systemInstruction).toBe("Video editing assistant");
|
||||
expect(callArgs.tools).toHaveLength(1);
|
||||
expect(callArgs.tools[0].functionDeclarations).toHaveLength(1);
|
||||
});
|
||||
|
||||
test("passes multi-turn conversation through to Gemini", async () => {
|
||||
mockGenerateContent.mockResolvedValueOnce(
|
||||
makeGeminiResponse({ text: "Sure!" }),
|
||||
);
|
||||
|
||||
const adapter = new GeminiAdapter(TEST_CONFIG);
|
||||
await adapter.chat({
|
||||
messages: [
|
||||
makeChatMessage("user", "First question"),
|
||||
makeChatMessage("assistant", "First answer"),
|
||||
makeChatMessage("user", "Second question"),
|
||||
],
|
||||
systemPrompt: "",
|
||||
tools: [],
|
||||
});
|
||||
|
||||
const callArgs = mockGenerateContent.mock.calls[0][0];
|
||||
expect(callArgs.contents).toHaveLength(3);
|
||||
expect(callArgs.contents[0].role).toBe("user");
|
||||
expect(callArgs.contents[1].role).toBe("model");
|
||||
expect(callArgs.contents[2].role).toBe("user");
|
||||
});
|
||||
|
||||
test("does not include systemInstruction when systemPrompt is empty", async () => {
|
||||
mockGenerateContent.mockResolvedValueOnce(
|
||||
makeGeminiResponse({ text: "OK" }),
|
||||
);
|
||||
|
||||
const adapter = new GeminiAdapter(TEST_CONFIG);
|
||||
await adapter.chat({
|
||||
messages: [makeChatMessage("user", "Hi")],
|
||||
systemPrompt: "",
|
||||
tools: [],
|
||||
});
|
||||
|
||||
const callArgs = mockGenerateContent.mock.calls[0][0];
|
||||
expect(callArgs.systemInstruction).toBeUndefined();
|
||||
});
|
||||
|
||||
test("ignores baseUrl in config — Gemini still works", async () => {
|
||||
mockGenerateContent.mockResolvedValueOnce(
|
||||
makeGeminiResponse({ text: "Hello!" }),
|
||||
);
|
||||
|
||||
const configWithBaseUrl: ProviderConfig = {
|
||||
provider: "gemini",
|
||||
apiKey: "test-gemini-key",
|
||||
model: "gemini-2.0-flash",
|
||||
baseUrl: "https://should-be-ignored.example.com",
|
||||
};
|
||||
|
||||
const adapter = new GeminiAdapter(configWithBaseUrl);
|
||||
const result = await adapter.chat({
|
||||
messages: [makeChatMessage("user", "Hi")],
|
||||
systemPrompt: "",
|
||||
tools: [],
|
||||
});
|
||||
|
||||
expect(result.content).toBe("Hello!");
|
||||
expect(result.toolCalls).toBeUndefined();
|
||||
});
|
||||
|
||||
test("returns text when tools are provided but Gemini responds with text only", async () => {
|
||||
mockGenerateContent.mockResolvedValueOnce(
|
||||
makeGeminiResponse({ text: "I don't need any tools for this." }),
|
||||
);
|
||||
|
||||
const adapter = new GeminiAdapter(TEST_CONFIG);
|
||||
const result = await adapter.chat({
|
||||
messages: [makeChatMessage("user", "Just chat")],
|
||||
systemPrompt: "You are helpful.",
|
||||
tools: SAMPLE_TOOLS,
|
||||
});
|
||||
|
||||
expect(result.content).toBe("I don't need any tools for this.");
|
||||
expect(result.toolCalls).toBeUndefined();
|
||||
|
||||
// Verify tools were still sent to Gemini
|
||||
const callArgs = mockGenerateContent.mock.calls[0][0];
|
||||
expect(callArgs.tools).toBeDefined();
|
||||
expect(callArgs.tools[0].functionDeclarations).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
import { describe, expect, test } from "bun:test";
|
||||
import { createProvider } from "@/agent/providers/index";
|
||||
import { OpenAICompatibleAdapter } from "@/agent/providers/openai-compatible";
|
||||
import { GeminiAdapter } from "@/agent/providers/gemini";
|
||||
import type { ProviderConfig } from "@/agent/providers/types";
|
||||
|
||||
describe("createProvider", () => {
|
||||
|
|
@ -28,6 +29,17 @@ describe("createProvider", () => {
|
|||
// Adapter was constructed without throwing — config accepted
|
||||
});
|
||||
|
||||
test("returns GeminiAdapter for gemini provider", () => {
|
||||
const config: ProviderConfig = {
|
||||
provider: "gemini",
|
||||
apiKey: "test-gemini-key",
|
||||
model: "gemini-2.0-flash",
|
||||
};
|
||||
|
||||
const adapter = createProvider(config);
|
||||
expect(adapter).toBeInstanceOf(GeminiAdapter);
|
||||
});
|
||||
|
||||
test("throws Error for unknown provider", () => {
|
||||
const config: ProviderConfig = {
|
||||
provider: "unknown-provider",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,240 @@
|
|||
import {
|
||||
GoogleGenerativeAI,
|
||||
type Content,
|
||||
type FunctionDeclaration,
|
||||
type FunctionDeclarationSchema,
|
||||
type GenerateContentResult,
|
||||
SchemaType,
|
||||
} from "@google/generative-ai";
|
||||
import type { ChatMessage, ToolCall, ToolSchema } from "@/agent/types";
|
||||
import type {
|
||||
ProviderAdapter,
|
||||
ProviderConfig,
|
||||
ProviderResponse,
|
||||
} from "./types";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Internal conversion helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Maps internal type strings to Gemini SchemaType values.
|
||||
*/
|
||||
const TYPE_MAP: Record<string, SchemaType> = {
|
||||
string: SchemaType.STRING,
|
||||
number: SchemaType.NUMBER,
|
||||
boolean: SchemaType.BOOLEAN,
|
||||
object: SchemaType.OBJECT,
|
||||
};
|
||||
|
||||
/**
|
||||
* Converts internal ChatMessages into Gemini Content[].
|
||||
*
|
||||
* - `user` → role "user", text part
|
||||
* - `assistant` → role "model", text part + functionCall parts from toolCalls
|
||||
* - `tool_result`→ role "function", functionResponse part
|
||||
* - `system` → skipped (handled separately via systemInstruction)
|
||||
*/
|
||||
export function toGeminiContents(messages: ChatMessage[]): Content[] {
|
||||
const contents: Content[] = [];
|
||||
|
||||
for (const msg of messages) {
|
||||
if (msg.role === "system") {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (msg.role === "user") {
|
||||
contents.push({
|
||||
role: "user",
|
||||
parts: [{ text: msg.content }],
|
||||
});
|
||||
} else if (msg.role === "assistant") {
|
||||
const parts: Content["parts"] = [];
|
||||
|
||||
if (msg.content) {
|
||||
parts.push({ text: msg.content });
|
||||
}
|
||||
|
||||
if (msg.toolCalls && msg.toolCalls.length > 0) {
|
||||
for (const tc of msg.toolCalls) {
|
||||
parts.push({
|
||||
functionCall: { name: tc.name, args: tc.args },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
contents.push({ role: "model", parts });
|
||||
} else if (msg.role === "tool_result") {
|
||||
// Gemini uses the function name to correlate responses, not IDs.
|
||||
// We parse the content as JSON for the response object; fall back
|
||||
// to wrapping it if it's not valid JSON.
|
||||
let responseData: object;
|
||||
try {
|
||||
responseData = JSON.parse(msg.content) as object;
|
||||
} catch {
|
||||
responseData = { result: msg.content };
|
||||
}
|
||||
|
||||
// Derive the function name from the toolCallId → find the matching
|
||||
// tool name in prior messages. If not found, use a placeholder.
|
||||
const toolName = findToolNameForResult(messages, msg.toolCallId);
|
||||
|
||||
contents.push({
|
||||
role: "function",
|
||||
parts: [
|
||||
{
|
||||
functionResponse: {
|
||||
name: toolName,
|
||||
response: responseData,
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return contents;
|
||||
}
|
||||
|
||||
/**
|
||||
* Walks backward through messages to find the function name associated with
|
||||
* a toolCallId. This is needed because Gemini's functionResponse uses names,
|
||||
* not IDs.
|
||||
*/
|
||||
function findToolNameForResult(
|
||||
messages: ChatMessage[],
|
||||
toolCallId?: string,
|
||||
): string {
|
||||
if (!toolCallId) return "unknown";
|
||||
|
||||
for (let i = messages.length - 1; i >= 0; i--) {
|
||||
const msg = messages[i];
|
||||
if (msg.role === "assistant" && msg.toolCalls) {
|
||||
const match = msg.toolCalls.find((tc) => tc.id === toolCallId);
|
||||
if (match) return match.name;
|
||||
}
|
||||
}
|
||||
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts internal ToolSchema[] into Gemini FunctionDeclaration[].
|
||||
*
|
||||
* Maps internal type strings ("string", "number", etc.) to Gemini SchemaType
|
||||
* enum values. Returns an empty array when no tools are provided.
|
||||
*/
|
||||
export function toGeminiTools(tools: ToolSchema[]): FunctionDeclaration[] {
|
||||
if (tools.length === 0) return [];
|
||||
|
||||
return tools.map((tool) => {
|
||||
const properties: FunctionDeclarationSchema["properties"] = {};
|
||||
|
||||
for (const param of tool.parameters) {
|
||||
const geminiType = TYPE_MAP[param.type] ?? SchemaType.STRING;
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
(properties as Record<string, any>)[param.key] = { type: geminiType };
|
||||
}
|
||||
|
||||
const requiredParams = tool.parameters
|
||||
.filter((p) => p.required)
|
||||
.map((p) => p.key);
|
||||
|
||||
return {
|
||||
name: tool.name,
|
||||
description: tool.description,
|
||||
parameters: {
|
||||
type: SchemaType.OBJECT,
|
||||
properties,
|
||||
...(requiredParams.length > 0 && { required: requiredParams }),
|
||||
},
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts a Gemini GenerateContentResult into the provider-agnostic
|
||||
* ProviderResponse shape.
|
||||
*
|
||||
* - Text parts are concatenated into `content`.
|
||||
* - FunctionCall parts are mapped to `toolCalls[]` with synthesized UUIDs.
|
||||
* - Throws if no candidates are present.
|
||||
*/
|
||||
export function fromGeminiResponse(
|
||||
response: GenerateContentResult,
|
||||
): ProviderResponse {
|
||||
const candidates = response.response.candidates;
|
||||
if (!candidates || candidates.length === 0) {
|
||||
throw new Error("Empty response from Gemini provider");
|
||||
}
|
||||
|
||||
const parts = candidates[0].content?.parts ?? [];
|
||||
const textParts: string[] = [];
|
||||
const toolCalls: ToolCall[] = [];
|
||||
|
||||
for (const part of parts) {
|
||||
if ("text" in part && part.text !== undefined) {
|
||||
textParts.push(part.text);
|
||||
}
|
||||
if ("functionCall" in part && part.functionCall) {
|
||||
toolCalls.push({
|
||||
id: crypto.randomUUID(),
|
||||
name: part.functionCall.name,
|
||||
args: (part.functionCall.args as Record<string, unknown>) ?? {},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const content = textParts.join("");
|
||||
const result: ProviderResponse = { content };
|
||||
|
||||
if (toolCalls.length > 0) {
|
||||
result.toolCalls = toolCalls;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Adapter implementation
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Gemini adapter using @google/generative-ai SDK.
|
||||
*
|
||||
* Uses the stateless `generateContent()` API — full history is sent with each
|
||||
* call, matching the existing route contract. System prompts are mapped to
|
||||
* Gemini's `systemInstruction` field, not as user messages.
|
||||
*/
|
||||
export class GeminiAdapter implements ProviderAdapter {
|
||||
private model;
|
||||
|
||||
constructor(config: ProviderConfig) {
|
||||
const genAI = new GoogleGenerativeAI(config.apiKey);
|
||||
this.model = genAI.getGenerativeModel({ model: config.model });
|
||||
// config.baseUrl is intentionally ignored for the Gemini provider.
|
||||
}
|
||||
|
||||
async chat(params: {
|
||||
messages: ChatMessage[];
|
||||
systemPrompt: string;
|
||||
tools: ToolSchema[];
|
||||
}): Promise<ProviderResponse> {
|
||||
const contents = toGeminiContents(params.messages);
|
||||
const functionDeclarations = toGeminiTools(params.tools);
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const request: any = { contents };
|
||||
|
||||
if (params.systemPrompt) {
|
||||
request.systemInstruction = params.systemPrompt;
|
||||
}
|
||||
|
||||
if (functionDeclarations.length > 0) {
|
||||
request.tools = [{ functionDeclarations }];
|
||||
}
|
||||
|
||||
const result = await this.model.generateContent(request);
|
||||
return fromGeminiResponse(result);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,6 @@
|
|||
import type { ProviderAdapter, ProviderConfig } from "./types";
|
||||
import { OpenAICompatibleAdapter } from "./openai-compatible";
|
||||
import { GeminiAdapter } from "./gemini";
|
||||
|
||||
/**
|
||||
* Factory — resolves a ProviderConfig to a concrete adapter.
|
||||
|
|
@ -12,6 +13,8 @@ export function createProvider(config: ProviderConfig): ProviderAdapter {
|
|||
switch (config.provider) {
|
||||
case "openai-compatible":
|
||||
return new OpenAICompatibleAdapter(config);
|
||||
case "gemini":
|
||||
return new GeminiAdapter(config);
|
||||
default:
|
||||
throw new Error(`Unknown LLM provider: ${config.provider}`);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ export function buildSystemPrompt(
|
|||
): string {
|
||||
const mediaSection =
|
||||
context.mediaAssets.length > 0
|
||||
? `Active media assets:\n${context.mediaAssets.map((m) => `- ${m.name} (${m.type}, ${m.duration}s)`).join("\n")}`
|
||||
? `Active media assets:\n${context.mediaAssets.map((m) => `- [id: ${m.id}] ${m.name} (${m.type}, ${m.duration}s)`).join("\n")}`
|
||||
: "No media assets loaded.";
|
||||
|
||||
const parts = [
|
||||
|
|
@ -28,6 +28,12 @@ export function buildSystemPrompt(
|
|||
mediaSection,
|
||||
];
|
||||
|
||||
if (context.mediaAssets.length > 0) {
|
||||
parts.push(
|
||||
'IMPORTANT: When calling tools that accept an "assetId" parameter, always use the internal "id" value (e.g., "v1"), NOT the filename or display name.',
|
||||
);
|
||||
}
|
||||
|
||||
if (tools && tools.length > 0) {
|
||||
const toolList = tools
|
||||
.map((t) => `- ${t.name}: ${t.description}`)
|
||||
|
|
|
|||
|
|
@ -227,7 +227,7 @@ describe("transcribe_video tool — error branches", () => {
|
|||
|
||||
const result = await tool.execute({ assetId: "v99" }, context);
|
||||
expect(result).toEqual({
|
||||
error: "Could not access file for asset v99",
|
||||
error: 'No asset found with id or name "v99". Available ids: [v1]. Available names: [intro.mp4].',
|
||||
});
|
||||
});
|
||||
|
||||
|
|
@ -282,3 +282,141 @@ describe("transcribe_video tool — error branches", () => {
|
|||
expect(result).toEqual({ error: "Worker not initialized" });
|
||||
});
|
||||
});
|
||||
|
||||
describe("transcribe_video tool — name-based assetId fallback", () => {
|
||||
beforeEach(() => {
|
||||
mockResolveAssetFile.mockClear();
|
||||
mockGetAssetHasAudio.mockClear();
|
||||
mockDecodeAudio.mockClear();
|
||||
mockTranscribe.mockClear();
|
||||
});
|
||||
|
||||
test("resolves by filename when no id match (exact name)", async () => {
|
||||
const fakeFile = new File([], "conclu.mov", { type: "video/quicktime" });
|
||||
const context = makeContext({
|
||||
mediaAssets: [
|
||||
fakeVideoAsset({ id: "abc123", name: "conclu.mov" }),
|
||||
],
|
||||
});
|
||||
|
||||
mockResolveAssetFile.mockReturnValue(fakeFile);
|
||||
mockGetAssetHasAudio.mockReturnValue(true);
|
||||
mockDecodeAudio.mockResolvedValue({
|
||||
samples: new Float32Array(16000),
|
||||
sampleRate: 16000,
|
||||
});
|
||||
mockTranscribe.mockResolvedValue(fakeTranscriptionResult());
|
||||
|
||||
const tool = toolRegistry.get("transcribe_video");
|
||||
const result = await tool.execute({ assetId: "conclu.mov" }, context);
|
||||
|
||||
expect(result).toEqual(
|
||||
expect.objectContaining({ assetName: "conclu.mov" }),
|
||||
);
|
||||
// Resolved asset must pass its internal id to the adapter
|
||||
expect(mockResolveAssetFile).toHaveBeenCalledWith("abc123");
|
||||
expect(mockGetAssetHasAudio).toHaveBeenCalledWith("abc123");
|
||||
});
|
||||
|
||||
test("resolves by case-insensitive name when no exact id or name match", async () => {
|
||||
const fakeFile = new File([], "Conclu.MOV", { type: "video/quicktime" });
|
||||
const context = makeContext({
|
||||
mediaAssets: [
|
||||
fakeVideoAsset({ id: "abc123", name: "Conclu.MOV" }),
|
||||
],
|
||||
});
|
||||
|
||||
mockResolveAssetFile.mockReturnValue(fakeFile);
|
||||
mockGetAssetHasAudio.mockReturnValue(true);
|
||||
mockDecodeAudio.mockResolvedValue({
|
||||
samples: new Float32Array(16000),
|
||||
sampleRate: 16000,
|
||||
});
|
||||
mockTranscribe.mockResolvedValue(fakeTranscriptionResult());
|
||||
|
||||
const tool = toolRegistry.get("transcribe_video");
|
||||
const result = await tool.execute({ assetId: "conclu.mov" }, context);
|
||||
|
||||
expect(result).toEqual(
|
||||
expect.objectContaining({ assetName: "Conclu.MOV" }),
|
||||
);
|
||||
expect(mockResolveAssetFile).toHaveBeenCalledWith("abc123");
|
||||
});
|
||||
|
||||
test("returns ambiguous error when two assets share same name", async () => {
|
||||
const context = makeContext({
|
||||
mediaAssets: [
|
||||
fakeVideoAsset({ id: "v1", name: "clip.mp4" }),
|
||||
fakeVideoAsset({ id: "v2", name: "clip.mp4" }),
|
||||
],
|
||||
});
|
||||
|
||||
const tool = toolRegistry.get("transcribe_video");
|
||||
const result = await tool.execute({ assetId: "clip.mp4" }, context);
|
||||
|
||||
expect(result).toEqual({
|
||||
error: 'Ambiguous asset name "clip.mp4" matches multiple assets (clip.mp4, clip.mp4). Specify the internal id: v1, v2.',
|
||||
});
|
||||
});
|
||||
|
||||
test("returns ambiguous error for case-insensitive duplicates", async () => {
|
||||
const context = makeContext({
|
||||
mediaAssets: [
|
||||
fakeVideoAsset({ id: "v1", name: "Clip.MP4" }),
|
||||
fakeVideoAsset({ id: "v2", name: "CLIP.MP4" }),
|
||||
],
|
||||
});
|
||||
|
||||
const tool = toolRegistry.get("transcribe_video");
|
||||
const result = await tool.execute({ assetId: "clip.mp4" }, context);
|
||||
|
||||
expect(result).toEqual({
|
||||
error: 'Ambiguous asset name "clip.mp4" matches multiple assets (Clip.MP4, CLIP.MP4). Specify the internal id: v1, v2.',
|
||||
});
|
||||
});
|
||||
|
||||
test("prefers exact id match over name match", async () => {
|
||||
const fakeFile = new File([], "intro.mp4", { type: "video/mp4" });
|
||||
// An asset whose id happens to equal another asset's name
|
||||
const context = makeContext({
|
||||
mediaAssets: [
|
||||
fakeVideoAsset({ id: "v1", name: "intro.mp4" }),
|
||||
fakeVideoAsset({ id: "intro.mp4", name: "outro.mp4" }),
|
||||
],
|
||||
});
|
||||
|
||||
mockResolveAssetFile.mockReturnValue(fakeFile);
|
||||
mockGetAssetHasAudio.mockReturnValue(true);
|
||||
mockDecodeAudio.mockResolvedValue({
|
||||
samples: new Float32Array(16000),
|
||||
sampleRate: 16000,
|
||||
});
|
||||
mockTranscribe.mockResolvedValue(fakeTranscriptionResult());
|
||||
|
||||
const tool = toolRegistry.get("transcribe_video");
|
||||
const result = await tool.execute({ assetId: "intro.mp4" }, context);
|
||||
|
||||
// "intro.mp4" matches v1 by id AND the second asset by id exactly
|
||||
// id lookup is first, so it resolves the second asset (id="intro.mp4")
|
||||
expect(result).toEqual(
|
||||
expect.objectContaining({ assetName: "outro.mp4" }),
|
||||
);
|
||||
expect(mockResolveAssetFile).toHaveBeenCalledWith("intro.mp4");
|
||||
});
|
||||
|
||||
test("returns not-found error with available ids and names for totally unknown assetId", async () => {
|
||||
const context = makeContext({
|
||||
mediaAssets: [
|
||||
fakeVideoAsset({ id: "v1", name: "intro.mp4" }),
|
||||
fakeVideoAsset({ id: "v2", name: "outro.mp4" }),
|
||||
],
|
||||
});
|
||||
|
||||
const tool = toolRegistry.get("transcribe_video");
|
||||
const result = await tool.execute({ assetId: "nonexistent.mov" }, context);
|
||||
|
||||
expect(result).toEqual({
|
||||
error: 'No asset found with id or name "nonexistent.mov". Available ids: [v1, v2]. Available names: [intro.mp4, outro.mp4].',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -57,10 +57,43 @@ const transcribeVideoTool: ToolDefinition = {
|
|||
let targetAsset: (typeof mediaCandidates)[number] | undefined;
|
||||
|
||||
if (typedArgs.assetId) {
|
||||
// 1. Exact internal-id match
|
||||
targetAsset = mediaCandidates.find((a) => a.id === typedArgs.assetId);
|
||||
|
||||
// 2. Fallback: exact name match (LLM sometimes passes filename)
|
||||
if (!targetAsset) {
|
||||
const nameMatches = mediaCandidates.filter(
|
||||
(a) => a.name === typedArgs.assetId,
|
||||
);
|
||||
if (nameMatches.length === 1) {
|
||||
targetAsset = nameMatches[0];
|
||||
} else if (nameMatches.length > 1) {
|
||||
return {
|
||||
error: `Ambiguous asset name "${typedArgs.assetId}" matches multiple assets (${nameMatches.map((a) => a.name).join(", ")}). Specify the internal id: ${nameMatches.map((a) => `${a.id}`).join(", ")}.`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Case-insensitive fallback (only if unambiguous)
|
||||
if (!targetAsset) {
|
||||
const lower = typedArgs.assetId.toLowerCase();
|
||||
const ciMatches = mediaCandidates.filter(
|
||||
(a) => a.name.toLowerCase() === lower,
|
||||
);
|
||||
if (ciMatches.length === 1) {
|
||||
targetAsset = ciMatches[0];
|
||||
} else if (ciMatches.length > 1) {
|
||||
return {
|
||||
error: `Ambiguous asset name "${typedArgs.assetId}" matches multiple assets (${ciMatches.map((a) => a.name).join(", ")}). Specify the internal id: ${ciMatches.map((a) => `${a.id}`).join(", ")}.`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
if (!targetAsset) {
|
||||
const ids = mediaCandidates.map((a) => a.id).join(", ");
|
||||
const names = mediaCandidates.map((a) => a.name).join(", ");
|
||||
return {
|
||||
error: `Could not access file for asset ${typedArgs.assetId}`,
|
||||
error: `No asset found with id or name "${typedArgs.assetId}". Available ids: [${ids}]. Available names: [${names}].`,
|
||||
};
|
||||
}
|
||||
} else {
|
||||
|
|
|
|||
3
bun.lock
3
bun.lock
|
|
@ -20,6 +20,7 @@
|
|||
"name": "@opencut/web",
|
||||
"version": "0.1.0",
|
||||
"dependencies": {
|
||||
"@google/generative-ai": "^0.24.0",
|
||||
"@hello-pangea/dnd": "^18.0.1",
|
||||
"@hugeicons/core-free-icons": "^3.3.0",
|
||||
"@hugeicons/react": "^1.1.6",
|
||||
|
|
@ -348,6 +349,8 @@
|
|||
|
||||
"@floating-ui/utils": ["@floating-ui/utils@0.2.10", "", {}, "sha512-aGTxbpbg8/b5JfU1HXSrbH3wXZuLPJcNEcZQFMxLs3oSzgtVu6nFPkbbGGUvBcUjKV2YyB9Wxxabo+HEH9tcRQ=="],
|
||||
|
||||
"@google/generative-ai": ["@google/generative-ai@0.24.1", "", {}, "sha512-MqO+MLfM6kjxcKoy0p1wRzG3b4ZZXtPI+z2IE26UogS2Cm/XHO+7gGRBh6gcJsOiIVoH93UwKvW4HdgiOZCy9Q=="],
|
||||
|
||||
"@hello-pangea/dnd": ["@hello-pangea/dnd@18.0.1", "", { "dependencies": { "@babel/runtime": "^7.26.7", "css-box-model": "^1.2.1", "raf-schd": "^4.0.3", "react-redux": "^9.2.0", "redux": "^5.0.1" }, "peerDependencies": { "react": "^18.0.0 || ^19.0.0", "react-dom": "^18.0.0 || ^19.0.0" } }, "sha512-xojVWG8s/TGrKT1fC8K2tIWeejJYTAeJuj36zM//yEm/ZrnZUSFGS15BpO+jGZT1ybWvyXmeDJwPYb4dhWlbZQ=="],
|
||||
|
||||
"@hugeicons/core-free-icons": ["@hugeicons/core-free-icons@3.3.0", "", {}, "sha512-qYyr4JQ2eQIHTSTbITvnJvs6ERNK64D9gpwZnf2IyuG0exzqfyABLO/oTB71FB3RZPfu1GbwycdiGSo46apjMQ=="],
|
||||
|
|
|
|||
|
|
@ -0,0 +1,59 @@
|
|||
# Archive Report: gemini-adapter-fase-1
|
||||
|
||||
**Change**: gemini-adapter-fase-1
|
||||
**Archived**: 2026-04-24
|
||||
**Verification Verdict**: PASS — 14/14 tasks complete, 12/12 spec scenarios compliant, 48/48 tests passing, 0 TypeScript errors
|
||||
|
||||
## Scope Summary
|
||||
|
||||
This change adds **native Gemini provider support for normal chat and tool calling only**. Video upload/file understanding remains deferred to a future change. The `GeminiAdapter` implements the existing `ProviderAdapter` seam via `@google/generative-ai` SDK, supporting stateless `generateContent()` with system prompt mapping, tool schema conversion, and synthesized tool-call IDs. The OpenAI-compatible path is untouched.
|
||||
|
||||
## Engram Artifact Traceability
|
||||
|
||||
| Artifact | Observation ID |
|
||||
|----------|---------------|
|
||||
| Proposal | #1016 |
|
||||
| Spec (delta) | #1019 |
|
||||
| Design | #1018 |
|
||||
| Tasks | #1020 |
|
||||
| Apply Progress | #1021 |
|
||||
| Verify Report | #1024 |
|
||||
|
||||
## OpenSpec Archive
|
||||
|
||||
**Moved to**: `openspec/changes/archive/2026-04-24-gemini-adapter-fase-1/`
|
||||
|
||||
### Archive Contents
|
||||
- proposal.md ✅
|
||||
- specs/ (agent-session-shell delta) ✅
|
||||
- design.md ✅
|
||||
- tasks.md ✅ (14/14 tasks complete)
|
||||
- verify-report.md ✅
|
||||
- exploration.md ✅
|
||||
|
||||
## Specs Synced
|
||||
|
||||
| Domain | Action | Details |
|
||||
|--------|--------|---------|
|
||||
| agent-session-shell | Updated | 2 requirements modified (Provider Adapter Interface, Provider Configuration), 2 requirements added (Gemini Free-Form Chat, Gemini Tool Calling), 0 removed |
|
||||
|
||||
### Modified Requirements
|
||||
- **Provider Adapter Interface**: Expanded factory to support `openai-compatible` and `gemini`; added tool-call ID synthesis mandate; added Gemini adapter selection scenario
|
||||
- **Provider Configuration**: Added `LLM_PROVIDER` valid value constraint (`openai-compatible`, `gemini`); added Gemini-specific `LLM_BASE_URL` ignoring behavior; added unknown-provider and Gemini-config scenarios
|
||||
|
||||
### Added Requirements
|
||||
- **Gemini Free-Form Chat**: Plain text chat through Gemini `generateContent()` with system prompt mapped to `systemInstruction`
|
||||
- **Gemini Tool Calling**: Tool schema → Gemini function declaration conversion, function call response → `ToolCall[]` with synthesized IDs
|
||||
|
||||
## Source of Truth Updated
|
||||
- `openspec/specs/agent-session-shell/spec.md` — now reflects Gemini provider support
|
||||
|
||||
## Deferred Scope (explicitly out of scope)
|
||||
- Video/file upload or direct file understanding
|
||||
- `@asset` references, broader multimodal inputs, image/video reasoning
|
||||
- Streaming UX or adapter contract changes for streaming
|
||||
- Safety/grounding/thinking configuration beyond defaults
|
||||
|
||||
## SDD Cycle Complete
|
||||
The change has been fully explored, proposed, specified, designed, implemented, verified, and archived.
|
||||
Ready for the next change.
|
||||
|
|
@ -0,0 +1,155 @@
|
|||
# Design: Gemini Adapter — Phase 1
|
||||
|
||||
## Technical Approach
|
||||
|
||||
Implement a stateless `GeminiAdapter` class behind the existing `ProviderAdapter` seam, following the exact same pattern as `OpenAICompatibleAdapter`. The adapter converts internal `ChatMessage[]`/`ToolSchema[]` into Gemini's `generateContent()` wire format and parses responses back into `ProviderResponse`. A new `"gemini"` case in `createProvider()` activates it when `LLM_PROVIDER=gemini`. Zero changes to orchestrator, route, or client contracts.
|
||||
|
||||
## Architecture Decisions
|
||||
|
||||
### Decision: SDK choice
|
||||
|
||||
| Option | Tradeoff | Decision |
|
||||
|--------|----------|----------|
|
||||
| `@google/generative-ai` | Established, stable, v0.21+, `GoogleGenerativeAI` class, `generateContent()` stateless API | ✅ Selected |
|
||||
| `@google/genai` | Newer unified SDK, less battle-tested, API may still shift | ❌ Rejected |
|
||||
|
||||
**Rationale**: `@google/generative-ai` is the most widely adopted Gemini SDK for Node/TS. It supports `systemInstruction`, `functionDeclaration`, and stateless `generateContent()` — exactly what we need. Consistent with using the mature `openai` SDK for the existing adapter.
|
||||
|
||||
### Decision: Stateless generateContent vs stateful startChat
|
||||
|
||||
| Option | Tradeoff | Decision |
|
||||
|--------|----------|----------|
|
||||
| `model.generateContent(contents, config)` | Stateless, full history each call — matches current adapter contract | ✅ Selected |
|
||||
| `chat.sendMessage()` via `startChat()` | Stateful session, server-managed history | ❌ Rejected |
|
||||
|
||||
**Rationale**: Current architecture POSTs full history from client on every turn. Stateless `generateContent()` maps 1:1 to this pattern. No session management needed.
|
||||
|
||||
### Decision: Tool call ID synthesis
|
||||
|
||||
| Option | Tradeoff | Decision |
|
||||
|--------|----------|----------|
|
||||
| `crypto.randomUUID()` | Built-in, zero deps, unique | ✅ Selected |
|
||||
| `nanoid()` (already a dep) | Shorter IDs, also unique | ❌ Rejected |
|
||||
|
||||
**Rationale**: `crypto.randomUUID()` is universally available in Node/Bun, no import needed, and produces standard UUIDs matching OpenAI's `call_xxx` format semantically. Keeps the adapter self-contained.
|
||||
|
||||
### Decision: System prompt placement
|
||||
|
||||
| Option | Tradeoff | Decision |
|
||||
|--------|----------|----------|
|
||||
| `systemInstruction` on `GenerateContentRequest` | Gemini-native, separate from contents | ✅ Selected |
|
||||
| Prepend as `user` message | Works but dilutes system/user boundary | ❌ Rejected |
|
||||
|
||||
**Rationale**: Gemini SDK supports `systemInstruction` as a first-class config field. Using it keeps system prompt semantically correct and matches Gemini best practices.
|
||||
|
||||
## Data Flow
|
||||
|
||||
```
|
||||
Route (chat/route.ts)
|
||||
│
|
||||
├─ createProvider(config) → GeminiAdapter instance
|
||||
│
|
||||
└─ adapter.chat({ messages, systemPrompt, tools })
|
||||
│
|
||||
├─ toGeminiContents(messages) → Content[] (user/model/function)
|
||||
├─ toGeminiTools(tools) → FunctionDeclaration[]
|
||||
│
|
||||
└─ model.generateContent({ contents, tools, systemInstruction })
|
||||
│
|
||||
└─ fromGeminiResponse(response) → ProviderResponse
|
||||
│
|
||||
├─ content: candidate.content.parts[].text
|
||||
└─ toolCalls: candidate.content.parts[]
|
||||
.filter(functionCall)
|
||||
.map → { id: uuid(), name, args }
|
||||
```
|
||||
|
||||
### Message mapping
|
||||
|
||||
| Internal role | Gemini role | Notes |
|
||||
|---------------|-------------|-------|
|
||||
| `system` | `systemInstruction` (config) | Not in contents array |
|
||||
| `user` | `user` part | Direct text mapping |
|
||||
| `assistant` | `model` part | Includes `functionCall` parts for tool calls |
|
||||
| `tool_result` | `function` role | Uses `functionResponse: { name, response }` keyed by tool name (not ID) |
|
||||
|
||||
### Type mapping (internal → Gemini)
|
||||
|
||||
| Internal type | Gemini type |
|
||||
|---------------|-------------|
|
||||
| `"string"` | `SchemaType.STRING` |
|
||||
| `"number"` | `SchemaType.NUMBER` |
|
||||
| `"boolean"` | `SchemaType.BOOLEAN` |
|
||||
| `"object"` | `SchemaType.OBJECT` |
|
||||
|
||||
## File Changes
|
||||
|
||||
| File | Action | Description |
|
||||
|------|--------|-------------|
|
||||
| `apps/web/src/agent/providers/gemini.ts` | Create | GeminiAdapter + conversion helpers (`toGeminiContents`, `toGeminiTools`, `fromGeminiResponse`) |
|
||||
| `apps/web/src/agent/providers/index.ts` | Modify | Add `"gemini"` case to factory switch |
|
||||
| `apps/web/src/agent/providers/__tests__/gemini.test.ts` | Create | Conversion tests, error paths, tool call ID synthesis |
|
||||
| `apps/web/src/agent/providers/__tests__/index.test.ts` | Modify | Add factory test for `"gemini"` → `GeminiAdapter` |
|
||||
| `apps/web/package.json` | Modify | Add `@google/generative-ai` dependency |
|
||||
| `apps/web/.env.example` | Modify | Add Gemini example config lines |
|
||||
|
||||
## Interfaces / Contracts
|
||||
|
||||
No interface changes. The adapter implements the existing `ProviderAdapter`:
|
||||
|
||||
```ts
|
||||
// Reused as-is from providers/types.ts
|
||||
interface ProviderAdapter {
|
||||
chat(params: {
|
||||
messages: ChatMessage[];
|
||||
systemPrompt: string;
|
||||
tools: ToolSchema[];
|
||||
}): Promise<ProviderResponse>;
|
||||
}
|
||||
```
|
||||
|
||||
### Internal helper signatures (gemini.ts)
|
||||
|
||||
```ts
|
||||
function toGeminiContents(
|
||||
messages: ChatMessage[],
|
||||
): GenerateContentRequest["contents"];
|
||||
|
||||
function toGeminiTools(
|
||||
tools: ToolSchema[],
|
||||
): FunctionDeclaration[];
|
||||
|
||||
function fromGeminiResponse(
|
||||
response: GenerateContentResult,
|
||||
): ProviderResponse;
|
||||
```
|
||||
|
||||
### Tool call ID — name collision handling
|
||||
|
||||
When mapping `tool_result` back to Gemini's `functionResponse`, Gemini keys by function **name** not ID. If multiple calls to the same tool exist in history, we group them by name and send the last result per name. This is acceptable because:
|
||||
|
||||
1. Current tool set (`transcribe_video`) is single-invocation per turn
|
||||
2. Multi-call same-tool scenarios are deferred to a later change
|
||||
3. Orchestrator already handles the ID→name correlation client-side
|
||||
|
||||
## Testing Strategy
|
||||
|
||||
| Layer | What to Test | Approach |
|
||||
|-------|-------------|----------|
|
||||
| Unit | `toGeminiContents` mapping (user, assistant, tool_result, assistant+toolCalls) | Mock SDK, inspect call args |
|
||||
| Unit | `toGeminiTools` type casing (string→STRING, etc.) | Direct assertion on output |
|
||||
| Unit | `fromGeminiResponse` content extraction | Construct Gemini response shape, assert `ProviderResponse` |
|
||||
| Unit | Tool call ID synthesis (non-empty, unique) | Multiple calls produce different IDs |
|
||||
| Unit | Error: empty candidates → throws | Assert error message |
|
||||
| Unit | Error: SDK throws → propagates | Assert error bubbles up |
|
||||
| Unit | Factory: `"gemini"` → `GeminiAdapter` | Instance check |
|
||||
| Integration | Full `chat()` round-trip with mocked SDK | End-to-end within adapter |
|
||||
|
||||
## Migration / Rollout
|
||||
|
||||
No migration required. Additive change — setting `LLM_PROVIDER=gemini` activates the new adapter. Existing `openai-compatible` default is untouched.
|
||||
|
||||
## Open Questions
|
||||
|
||||
- [ ] Confirm `@google/generative-ai` latest stable version at impl time (currently ^0.21)
|
||||
- [ ] Verify Gemini model naming convention user will use (e.g. `gemini-2.0-flash` vs `models/gemini-2.0-flash`)
|
||||
|
|
@ -0,0 +1,65 @@
|
|||
# Proposal: Gemini Adapter — Phase 1
|
||||
|
||||
## Intent
|
||||
|
||||
Add Gemini as a native provider behind the existing provider-agnostic LLM seam so the agent can use Gemini for plain chat and tool calling without changing orchestrator or route contracts.
|
||||
|
||||
## Scope
|
||||
|
||||
### In Scope
|
||||
- Native `GeminiAdapter` implementing the current `ProviderAdapter`
|
||||
- Gemini free-form chat using existing stateless full-history flow
|
||||
- Gemini tool calling, with `transcribe_video` as the first real tool
|
||||
- Config support for `LLM_PROVIDER=gemini`, `LLM_API_KEY`, `LLM_MODEL`
|
||||
- Tests and env/docs updates while preserving the OpenAI-compatible path
|
||||
|
||||
### Out of Scope
|
||||
- Video/file upload or direct file understanding
|
||||
- `@asset` references, broader multimodal inputs, and image/video reasoning
|
||||
- Streaming UX or adapter contract changes for streaming
|
||||
- Safety/grounding/thinking configuration beyond defaults
|
||||
|
||||
## Capabilities
|
||||
|
||||
### New Capabilities
|
||||
- None
|
||||
|
||||
### Modified Capabilities
|
||||
- `agent-session-shell`: expand provider selection/configuration so the existing provider adapter contract also supports native Gemini chat and function calling
|
||||
|
||||
## Approach
|
||||
|
||||
Implement a native Gemini SDK-backed adapter and register it in the provider factory. Keep the adapter stateless by mapping full internal message history into Gemini `generateContent()` requests, converting tool schemas/results at the adapter boundary, and synthesizing tool call IDs so internal contracts stay unchanged.
|
||||
|
||||
## Affected Areas
|
||||
|
||||
| Area | Impact | Description |
|
||||
|------|--------|-------------|
|
||||
| `apps/web/src/agent/providers/gemini.ts` | New | Gemini request/response/tool conversion |
|
||||
| `apps/web/src/agent/providers/index.ts` | Modified | Factory registration for `gemini` |
|
||||
| `apps/web/src/agent/providers/__tests__/gemini.test.ts` | New | Adapter conversion/error tests |
|
||||
| `apps/web/src/agent/providers/__tests__/index.test.ts` | Modified | Factory coverage |
|
||||
| `apps/web/package.json` | Modified | Gemini SDK dependency |
|
||||
| `apps/web/.env.example` | Modified | Gemini env configuration |
|
||||
|
||||
## Risks
|
||||
|
||||
| Risk | Likelihood | Mitigation |
|
||||
|------|------------|------------|
|
||||
| Gemini has no tool call IDs | Med | Synthesize unique IDs inside adapter |
|
||||
| Gemini type casing differs from internal schema | Med | Centralize schema mapping tests |
|
||||
| Gemini system prompt placement differs | Low | Map system prompt explicitly to Gemini config |
|
||||
|
||||
## Rollback Plan
|
||||
|
||||
Remove the Gemini adapter, factory branch, dependency, and env docs; keep `openai-compatible` as the unchanged default path.
|
||||
|
||||
## Dependencies
|
||||
|
||||
- Gemini JS SDK selection to be verified during implementation
|
||||
|
||||
## Success Criteria
|
||||
|
||||
- [ ] `LLM_PROVIDER=gemini` resolves a native Gemini adapter without breaking `openai-compatible`
|
||||
- [ ] Gemini returns normal assistant text through the existing route/orchestrator flow
|
||||
- [ ] Gemini can emit structured tool calls and execute `transcribe_video` through the existing loop
|
||||
|
|
@ -0,0 +1,97 @@
|
|||
# Delta for agent-session-shell
|
||||
|
||||
## MODIFIED Requirements
|
||||
|
||||
### Requirement: Provider Adapter Interface
|
||||
|
||||
The system SHALL define a server-side provider adapter interface with a single method `chat(params: ProviderChatParams): Promise<ProviderChatResponse>`. `ProviderChatParams` MUST include `messages`, `systemPrompt`, and `toolSchemas` in a provider-agnostic internal format. `ProviderChatResponse` MUST include `content: string` and `toolCalls?: ProviderToolCall[]` where `ProviderToolCall { id, name, args }`. Each concrete adapter converts internal tool schemas to its own wire format. The factory MUST support at least `openai-compatible` and `gemini` providers. Adapters whose native API does not assign IDs to tool calls MUST synthesize unique IDs so every returned `ToolCall.id` is a non-empty string.
|
||||
|
||||
(Previously: Phase 1 MAY ship exactly one adapter (OpenAI-compatible) as long as the interface contract remains provider-agnostic.)
|
||||
|
||||
#### Scenario: Adapter translates tool schemas to wire format
|
||||
|
||||
- GIVEN `transcribe_video` has internal parameters `[{ key: "assetId", type: "string", required: false }]`
|
||||
- WHEN the adapter receives tool schemas before calling the provider
|
||||
- THEN it converts them to its provider-specific wire format
|
||||
|
||||
#### Scenario: Adapter selected by config — OpenAI-compatible
|
||||
|
||||
- GIVEN `LLM_PROVIDER` is set to `"openai-compatible"`
|
||||
- WHEN the route resolves the adapter
|
||||
- THEN the OpenAI-compatible adapter is returned
|
||||
|
||||
#### Scenario: Adapter selected by config — Gemini
|
||||
|
||||
- GIVEN `LLM_PROVIDER` is set to `"gemini"`
|
||||
- WHEN the route resolves the adapter
|
||||
- THEN the Gemini adapter is returned
|
||||
|
||||
#### Scenario: Gemini adapter synthesizes tool-call IDs
|
||||
|
||||
- GIVEN the Gemini API returns a function call response with no call ID
|
||||
- WHEN the Gemini adapter maps the response to `ProviderResponse`
|
||||
- THEN each `ToolCall` in `toolCalls` has a non-empty `id` field generated by the adapter
|
||||
|
||||
### Requirement: Provider Configuration
|
||||
|
||||
Server-only environment variables `LLM_PROVIDER` (required), `LLM_MODEL` (required), `LLM_API_KEY` (required), and `LLM_BASE_URL` (optional) MUST determine which adapter and model the route uses. `LLM_PROVIDER` MUST accept at least `"openai-compatible"` and `"gemini"`. When `LLM_PROVIDER` is `"gemini"`, `LLM_BASE_URL` MUST be ignored. Configuration MUST be read server-side only — never exposed to the client.
|
||||
|
||||
(Previously: Provider config listed env vars but did not constrain valid provider values or describe Gemini-specific behavior.)
|
||||
|
||||
#### Scenario: Missing required configuration
|
||||
|
||||
- GIVEN `LLM_API_KEY` is not set in the environment
|
||||
- WHEN the route attempts to initialize the provider
|
||||
- THEN the call returns 502 with a missing-configuration error
|
||||
|
||||
#### Scenario: Unknown provider
|
||||
|
||||
- GIVEN `LLM_PROVIDER` is set to `"unknown-provider"`
|
||||
- WHEN the route resolves the adapter
|
||||
- THEN the factory throws `"Unknown LLM provider: unknown-provider"`
|
||||
|
||||
#### Scenario: Gemini config ignores base URL
|
||||
|
||||
- GIVEN `LLM_PROVIDER` is `"gemini"` and `LLM_BASE_URL` is set
|
||||
- WHEN the Gemini adapter is created
|
||||
- THEN `LLM_BASE_URL` is not used in the API call
|
||||
|
||||
## ADDED Requirements
|
||||
|
||||
### Requirement: Gemini Free-Form Chat
|
||||
|
||||
When the Gemini adapter receives a request with no tool schemas (or tools array is empty), it MUST send messages to the Gemini `generateContent` API and return a `ProviderResponse` with `content` set to the assistant text and `toolCalls` undefined.
|
||||
|
||||
#### Scenario: Plain chat through Gemini
|
||||
|
||||
- GIVEN `LLM_PROVIDER` is `"gemini"` and messages contain `[{ role: "user", content: "Hello" }]`
|
||||
- WHEN the adapter calls Gemini and receives a text-only response
|
||||
- THEN `ProviderResponse.content` is the assistant text and `toolCalls` is undefined
|
||||
|
||||
#### Scenario: System prompt mapping
|
||||
|
||||
- GIVEN the adapter receives `systemPrompt: "You are a video editor"`
|
||||
- WHEN it builds the Gemini request
|
||||
- THEN the system prompt is mapped to Gemini's system instruction field, not as a user message
|
||||
|
||||
### Requirement: Gemini Tool Calling
|
||||
|
||||
When the Gemini adapter receives a request with tool schemas, it MUST convert internal `ToolSchema[]` to Gemini function declarations, call `generateContent` with declarations attached, and if Gemini returns function call parts, map each to a `ToolCall { id, name, args }` with a synthesized non-empty `id`.
|
||||
|
||||
#### Scenario: Gemini returns function calls
|
||||
|
||||
- GIVEN `transcribe_video` is in tool schemas and Gemini responds with a function call part `{ name: "transcribe_video", args: {} }`
|
||||
- WHEN the adapter maps the response
|
||||
- THEN `toolCalls` contains one `ToolCall` with `name: "transcribe_video"`, `args: {}`, and a synthesized `id`
|
||||
|
||||
#### Scenario: Gemini returns text despite tools
|
||||
|
||||
- GIVEN tool schemas are provided but Gemini responds with only text
|
||||
- WHEN the adapter maps the response
|
||||
- THEN `toolCalls` is undefined and `content` holds the text
|
||||
|
||||
#### Scenario: Tool schema type mapping
|
||||
|
||||
- GIVEN an internal tool schema has parameters `[{ key: "assetId", type: "string", required: true }]`
|
||||
- WHEN the adapter converts to Gemini function declarations
|
||||
- THEN the parameter appears in `FunctionDeclaration.parameters.properties` with type `"string"`
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
# Tasks: Gemini Adapter — Phase 1
|
||||
|
||||
## Phase 1: Foundation
|
||||
|
||||
- [x] 1.1 Add `@google/generative-ai` dependency to `apps/web/package.json` (pin to latest stable `^0.21` or newer)
|
||||
- [x] 1.2 Add Gemini env config lines to `apps/web/.env.example`: `LLM_PROVIDER=gemini`, `LLM_API_KEY`, `LLM_MODEL=gemini-2.0-flash`, comment noting `LLM_BASE_URL` is ignored for gemini
|
||||
|
||||
## Phase 2: Core Adapter
|
||||
|
||||
- [x] 2.1 Create `apps/web/src/agent/providers/gemini.ts` — import `GoogleGenerativeAI` from `@google/generative-ai`, `SchemaType` for type mapping
|
||||
- [x] 2.2 Implement `toGeminiContents(messages: ChatMessage[])` — map `user→user`, `assistant→model` (with `functionCall` parts), `tool_result→function` role with `functionResponse`; skip `system` role (handled separately)
|
||||
- [x] 2.3 Implement `toGeminiTools(tools: ToolSchema[])` — convert each `ToolSchema` to a `FunctionDeclaration` with `SchemaType` property types (`string→STRING`, `number→NUMBER`, `boolean→BOOLEAN`, `object→OBJECT`); return empty array when no tools
|
||||
- [x] 2.4 Implement `fromGeminiResponse(response)` — extract `candidate.content.parts`, pull text into `content`, filter `functionCall` parts into `toolCalls[]` with `id: crypto.randomUUID()`, `name`, `args`; throw on empty candidates
|
||||
- [x] 2.5 Implement `GeminiAdapter` class — constructor takes `ProviderConfig`, creates `GoogleGenerativeAI` + `getGenerativeModel({ model, systemInstruction })`; `chat()` calls `model.generateContent({ contents, tools })` then returns `fromGeminiResponse()`; ignore `config.baseUrl`
|
||||
|
||||
## Phase 3: Factory Integration
|
||||
|
||||
- [x] 3.1 Add `"gemini"` case to `createProvider()` switch in `apps/web/src/agent/providers/index.ts` — import and return `new GeminiAdapter(config)`
|
||||
- [x] 3.2 Verify existing `openai-compatible` factory case is untouched and all existing tests still pass
|
||||
|
||||
## Phase 4: Tests
|
||||
|
||||
- [x] 4.1 Create `apps/web/src/agent/providers/__tests__/gemini.test.ts` — test `toGeminiContents`: user message, assistant with toolCalls, tool_result mapping, system message excluded from contents
|
||||
- [x] 4.2 Test `toGeminiTools`: single tool with parameters, type casing (`string→STRING` etc.), empty tools returns empty array
|
||||
- [x] 4.3 Test `fromGeminiResponse`: text-only response → `{ content, toolCalls: undefined }`; function-call response → `toolCalls` with non-empty synthesized IDs; two calls produce different IDs
|
||||
- [x] 4.4 Test error paths: empty candidates → throws; SDK error → propagates
|
||||
- [x] 4.5 Add factory test to `apps/web/src/agent/providers/__tests__/index.test.ts`: `"gemini"` config → `GeminiAdapter` instance; unknown provider still throws
|
||||
- [x] 4.6 Test full `chat()` round-trip with mocked `GoogleGenerativeAI` — verify `generateContent` called with correct contents, tools, and systemInstruction
|
||||
|
|
@ -0,0 +1,116 @@
|
|||
# Verification Report
|
||||
|
||||
**Change**: gemini-adapter-fase-1
|
||||
**Version**: N/A
|
||||
**Mode**: Standard
|
||||
|
||||
---
|
||||
|
||||
### Completeness
|
||||
|
||||
| Metric | Value |
|
||||
|--------|-------|
|
||||
| Tasks total | 14 |
|
||||
| Tasks complete | 14 |
|
||||
| Tasks incomplete | 0 |
|
||||
|
||||
All tasks in `openspec/changes/gemini-adapter-fase-1/tasks.md` are marked complete, including the post-verify fixes.
|
||||
|
||||
---
|
||||
|
||||
### Build & Tests Execution
|
||||
|
||||
**Build**: ➖ Skipped
|
||||
|
||||
```text
|
||||
No explicit `rules.verify.build_command` is configured in `openspec/config.yaml`.
|
||||
Project instructions also say never build after changes, so verification used TypeScript type-checking instead of `next build`.
|
||||
```
|
||||
|
||||
**Type Check**: ✅ Passed
|
||||
|
||||
```text
|
||||
Command: bunx tsc --noEmit
|
||||
|
||||
(no output)
|
||||
```
|
||||
|
||||
**Tests**: ✅ 48 passed / ❌ 0 failed / ⚠️ 0 skipped
|
||||
|
||||
```text
|
||||
Command: bun test src/agent/providers/__tests__ src/app/api/agent/chat/__tests__/route.test.ts
|
||||
|
||||
bun test v1.3.13
|
||||
48 pass
|
||||
0 fail
|
||||
106 expect() calls
|
||||
Ran 48 tests across 4 files.
|
||||
```
|
||||
|
||||
**Coverage**: ➖ Not available
|
||||
|
||||
---
|
||||
|
||||
### Spec Compliance Matrix
|
||||
|
||||
| Requirement | Scenario | Test | Result |
|
||||
|-------------|----------|------|--------|
|
||||
| Provider Adapter Interface | Adapter translates tool schemas to wire format | `src/agent/providers/__tests__/gemini.test.ts > converts a single tool with parameters` + `maps type strings to Gemini SchemaType values` | ✅ COMPLIANT |
|
||||
| Provider Adapter Interface | Adapter selected by config — OpenAI-compatible | `src/agent/providers/__tests__/index.test.ts > returns OpenAICompatibleAdapter for openai-compatible provider` | ✅ COMPLIANT |
|
||||
| Provider Adapter Interface | Adapter selected by config — Gemini | `src/agent/providers/__tests__/index.test.ts > returns GeminiAdapter for gemini provider` | ✅ COMPLIANT |
|
||||
| Provider Adapter Interface | Gemini adapter synthesizes tool-call IDs | `src/agent/providers/__tests__/gemini.test.ts > maps function-call response to toolCalls with non-empty synthesized IDs` | ✅ COMPLIANT |
|
||||
| Provider Configuration | Missing required configuration | `src/app/api/agent/chat/__tests__/route.test.ts > returns 502 when LLM_API_KEY is missing` | ✅ COMPLIANT |
|
||||
| Provider Configuration | Unknown provider | `src/agent/providers/__tests__/index.test.ts > throws Error for unknown provider` | ✅ COMPLIANT |
|
||||
| Provider Configuration | Gemini config ignores base URL | `src/agent/providers/__tests__/gemini.test.ts > ignores baseUrl in config — Gemini still works` | ✅ COMPLIANT |
|
||||
| Gemini Free-Form Chat | Plain chat through Gemini | `src/agent/providers/__tests__/gemini.test.ts > calls generateContent with correct contents and no tools` | ✅ COMPLIANT |
|
||||
| Gemini Free-Form Chat | System prompt mapping | `src/agent/providers/__tests__/gemini.test.ts > calls generateContent with correct contents and no tools` | ✅ COMPLIANT |
|
||||
| Gemini Tool Calling | Gemini returns function calls | `src/agent/providers/__tests__/gemini.test.ts > calls generateContent with tools and systemInstruction` | ✅ COMPLIANT |
|
||||
| Gemini Tool Calling | Gemini returns text despite tools | `src/agent/providers/__tests__/gemini.test.ts > returns text when tools are provided but Gemini responds with text only` | ✅ COMPLIANT |
|
||||
| Gemini Tool Calling | Tool schema type mapping | `src/agent/providers/__tests__/gemini.test.ts > maps type strings to Gemini SchemaType values` | ✅ COMPLIANT |
|
||||
|
||||
**Compliance summary**: 12/12 scenarios compliant
|
||||
|
||||
---
|
||||
|
||||
### Correctness (Static — Structural Evidence)
|
||||
|
||||
| Requirement | Status | Notes |
|
||||
|------------|--------|-------|
|
||||
| Provider Adapter Interface | ✅ Implemented | `apps/web/src/agent/providers/types.ts` keeps the provider-agnostic `chat()` seam, `index.ts` supports both providers, and `gemini.ts` synthesizes tool-call IDs with `crypto.randomUUID()`. |
|
||||
| Provider Configuration | ✅ Implemented | `apps/web/src/app/api/agent/chat/route.ts` reads `LLM_PROVIDER`, `LLM_API_KEY`, `LLM_MODEL`, and `LLM_BASE_URL` server-side only; `GeminiAdapter` intentionally ignores `baseUrl`. |
|
||||
| Gemini Free-Form Chat | ✅ Implemented | `GeminiAdapter.chat()` maps messages into Gemini contents, sets `systemInstruction` when present, and returns text-only responses without `toolCalls`. |
|
||||
| Gemini Tool Calling | ✅ Implemented | `toGeminiTools()` converts internal tool schemas to Gemini declarations and `fromGeminiResponse()` maps function-call parts back into provider `toolCalls`. |
|
||||
| Explicit non-goals remain deferred | ✅ Implemented | The touched files remain additive only; no streaming, multimodal upload, `@asset`, or contract changes were introduced. |
|
||||
|
||||
---
|
||||
|
||||
### Coherence (Design)
|
||||
|
||||
| Decision | Followed? | Notes |
|
||||
|----------|-----------|-------|
|
||||
| Use `@google/generative-ai` SDK | ✅ Yes | Added in `apps/web/package.json`; adapter imports `GoogleGenerativeAI` and `SchemaType`. |
|
||||
| Preserve stateless provider seam | ✅ Yes | `GeminiAdapter.chat()` sends full history through one `generateContent()` call with no session state. |
|
||||
| Synthesize tool-call IDs inside adapter | ✅ Yes | `fromGeminiResponse()` generates IDs for Gemini function calls. |
|
||||
| Keep OpenAI-compatible path intact | ✅ Yes | `createProvider()` still returns `OpenAICompatibleAdapter` unchanged for the original provider. |
|
||||
| Map system prompt via Gemini-native field | ✅ Yes | `GeminiAdapter.chat()` sets `request.systemInstruction` instead of injecting a fake user message. |
|
||||
|
||||
---
|
||||
|
||||
### Issues Found
|
||||
|
||||
**CRITICAL** (must fix before archive):
|
||||
None
|
||||
|
||||
**WARNING** (should fix):
|
||||
None
|
||||
|
||||
**SUGGESTION** (nice to have):
|
||||
None
|
||||
|
||||
---
|
||||
|
||||
### Verdict
|
||||
|
||||
PASS
|
||||
|
||||
All planned tasks are complete, the changed slice now type-checks, targeted runtime tests pass, and every spec scenario has passing behavioral evidence.
|
||||
|
|
@ -110,7 +110,7 @@ The system SHALL define types in `apps/web/src/agent/types.ts`: `ChatMessage { i
|
|||
|
||||
### Requirement: Provider Adapter Interface
|
||||
|
||||
The system SHALL define a server-side provider adapter interface with a single method `chat(params: ProviderChatParams): Promise<ProviderChatResponse>`. `ProviderChatParams` MUST include `messages`, `systemPrompt`, and `toolSchemas` in a provider-agnostic internal format. `ProviderChatResponse` MUST include `content: string` and `toolCalls?: ProviderToolCall[]` where `ProviderToolCall { id, name, args }`. Each concrete adapter converts internal tool schemas to its own wire format. Phase 1 MAY ship exactly one adapter (OpenAI-compatible) as long as the interface contract remains provider-agnostic.
|
||||
The system SHALL define a server-side provider adapter interface with a single method `chat(params: ProviderChatParams): Promise<ProviderChatResponse>`. `ProviderChatParams` MUST include `messages`, `systemPrompt`, and `toolSchemas` in a provider-agnostic internal format. `ProviderChatResponse` MUST include `content: string` and `toolCalls?: ProviderToolCall[]` where `ProviderToolCall { id, name, args }`. Each concrete adapter converts internal tool schemas to its own wire format. The factory MUST support at least `openai-compatible` and `gemini` providers. Adapters whose native API does not assign IDs to tool calls MUST synthesize unique IDs so every returned `ToolCall.id` is a non-empty string.
|
||||
|
||||
#### Scenario: Adapter translates tool schemas to wire format
|
||||
|
||||
|
|
@ -118,15 +118,27 @@ The system SHALL define a server-side provider adapter interface with a single m
|
|||
- WHEN the adapter receives tool schemas before calling the provider
|
||||
- THEN it converts them to its provider-specific wire format
|
||||
|
||||
#### Scenario: Adapter selected by config
|
||||
#### Scenario: Adapter selected by config — OpenAI-compatible
|
||||
|
||||
- GIVEN `LLM_PROVIDER` is set to `"openai-compatible"`
|
||||
- WHEN the route resolves the adapter
|
||||
- THEN the OpenAI-compatible adapter is returned
|
||||
|
||||
#### Scenario: Adapter selected by config — Gemini
|
||||
|
||||
- GIVEN `LLM_PROVIDER` is set to `"gemini"`
|
||||
- WHEN the route resolves the adapter
|
||||
- THEN the Gemini adapter is returned
|
||||
|
||||
#### Scenario: Gemini adapter synthesizes tool-call IDs
|
||||
|
||||
- GIVEN the Gemini API returns a function call response with no call ID
|
||||
- WHEN the Gemini adapter maps the response to `ProviderResponse`
|
||||
- THEN each `ToolCall` in `toolCalls` has a non-empty `id` field generated by the adapter
|
||||
|
||||
### Requirement: Provider Configuration
|
||||
|
||||
Server-only environment variables `LLM_PROVIDER` (required), `LLM_MODEL` (required), `LLM_API_KEY` (required), and `LLM_BASE_URL` (optional) MUST determine which adapter and model the route uses. Configuration MUST be read server-side only — never exposed to the client.
|
||||
Server-only environment variables `LLM_PROVIDER` (required), `LLM_MODEL` (required), `LLM_API_KEY` (required), and `LLM_BASE_URL` (optional) MUST determine which adapter and model the route uses. `LLM_PROVIDER` MUST accept at least `"openai-compatible"` and `"gemini"`. When `LLM_PROVIDER` is `"gemini"`, `LLM_BASE_URL` MUST be ignored. Configuration MUST be read server-side only — never exposed to the client.
|
||||
|
||||
#### Scenario: Missing required configuration
|
||||
|
||||
|
|
@ -134,6 +146,18 @@ Server-only environment variables `LLM_PROVIDER` (required), `LLM_MODEL` (requir
|
|||
- WHEN the route attempts to initialize the provider
|
||||
- THEN the call returns 502 with a missing-configuration error
|
||||
|
||||
#### Scenario: Unknown provider
|
||||
|
||||
- GIVEN `LLM_PROVIDER` is set to `"unknown-provider"`
|
||||
- WHEN the route resolves the adapter
|
||||
- THEN the factory throws `"Unknown LLM provider: unknown-provider"`
|
||||
|
||||
#### Scenario: Gemini config ignores base URL
|
||||
|
||||
- GIVEN `LLM_PROVIDER` is `"gemini"` and `LLM_BASE_URL` is set
|
||||
- WHEN the Gemini adapter is created
|
||||
- THEN `LLM_BASE_URL` is not used in the API call
|
||||
|
||||
### Requirement: Tool Argument Validation
|
||||
|
||||
Before executing any tool call, the orchestrator MUST validate that each argument key declared `required: true` in the tool's `parameters` is present in the call args and is of the declared type. If validation fails, the tool MUST NOT execute; instead the orchestrator MUST produce a `ToolResult` with `error` describing the missing or malformed argument.
|
||||
|
|
@ -166,3 +190,41 @@ The orchestrator loop MUST enforce a hard iteration cap of **8**. An iteration i
|
|||
- GIVEN the provider returns a final text response after 2 tool-call iterations
|
||||
- WHEN the orchestrator processes the response
|
||||
- THEN no cap message is appended and status is `"idle"`
|
||||
|
||||
### Requirement: Gemini Free-Form Chat
|
||||
|
||||
When the Gemini adapter receives a request with no tool schemas (or tools array is empty), it MUST send messages to the Gemini `generateContent` API and return a `ProviderResponse` with `content` set to the assistant text and `toolCalls` undefined.
|
||||
|
||||
#### Scenario: Plain chat through Gemini
|
||||
|
||||
- GIVEN `LLM_PROVIDER` is `"gemini"` and messages contain `[{ role: "user", content: "Hello" }]`
|
||||
- WHEN the adapter calls Gemini and receives a text-only response
|
||||
- THEN `ProviderResponse.content` is the assistant text and `toolCalls` is undefined
|
||||
|
||||
#### Scenario: System prompt mapping
|
||||
|
||||
- GIVEN the adapter receives `systemPrompt: "You are a video editor"`
|
||||
- WHEN it builds the Gemini request
|
||||
- THEN the system prompt is mapped to Gemini's system instruction field, not as a user message
|
||||
|
||||
### Requirement: Gemini Tool Calling
|
||||
|
||||
When the Gemini adapter receives a request with tool schemas, it MUST convert internal `ToolSchema[]` to Gemini function declarations, call `generateContent` with declarations attached, and if Gemini returns function call parts, map each to a `ToolCall { id, name, args }` with a synthesized non-empty `id`.
|
||||
|
||||
#### Scenario: Gemini returns function calls
|
||||
|
||||
- GIVEN `transcribe_video` is in tool schemas and Gemini responds with a function call part `{ name: "transcribe_video", args: {} }`
|
||||
- WHEN the adapter maps the response
|
||||
- THEN `toolCalls` contains one `ToolCall` with `name: "transcribe_video"`, `args: {}`, and a synthesized `id`
|
||||
|
||||
#### Scenario: Gemini returns text despite tools
|
||||
|
||||
- GIVEN tool schemas are provided but Gemini responds with only text
|
||||
- WHEN the adapter maps the response
|
||||
- THEN `toolCalls` is undefined and `content` holds the text
|
||||
|
||||
#### Scenario: Tool schema type mapping
|
||||
|
||||
- GIVEN an internal tool schema has parameters `[{ key: "assetId", type: "string", required: true }]`
|
||||
- WHEN the adapter converts to Gemini function declarations
|
||||
- THEN the parameter appears in `FunctionDeclaration.parameters.properties` with type `"string"`
|
||||
|
|
|
|||
Loading…
Reference in New Issue