Merge pull request #1 from luisacevelad/ft/gemini-provider
Ft/gemini provider
This commit is contained in:
luisacevelad
GitHub
commit
e176c6e094
|
|
@ -25,3 +25,9 @@ LLM_PROVIDER=openai-compatible
|
|||
LLM_API_KEY=your_openai_api_key_here
|
||||
LLM_MODEL=gpt-4o
|
||||
# LLM_BASE_URL=https://api.openai.com/v1
|
||||
|
||||
# Gemini provider (set LLM_PROVIDER=gemini to activate)
|
||||
# LLM_PROVIDER=gemini
|
||||
# LLM_API_KEY=your_gemini_api_key_here
|
||||
# LLM_MODEL=gemini-2.0-flash
|
||||
# Note: LLM_BASE_URL is ignored when using the gemini provider
|
||||
|
|
|
|||
|
|
@ -42,6 +42,7 @@
|
|||
"cmdk": "^1.0.0",
|
||||
"culori": "^4.0.2",
|
||||
"drizzle-orm": "^0.44.2",
|
||||
"@google/generative-ai": "^0.24.0",
|
||||
"embla-carousel-react": "^8.5.1",
|
||||
"eventemitter3": "^5.0.1",
|
||||
"feed": "^5.1.0",
|
||||
|
|
|
|||
|
|
@ -37,6 +37,46 @@ describe("buildSystemPrompt", () => {
|
|||
expect(prompt).toContain("NeuralCut video editor");
|
||||
});
|
||||
|
||||
test("includes asset internal id in media listing", () => {
|
||||
const context: AgentContext = {
|
||||
projectId: "proj-1",
|
||||
activeSceneId: "scene-A",
|
||||
mediaAssets: [
|
||||
{ id: "m1", name: "intro.mp4", type: "video", duration: 30 },
|
||||
],
|
||||
playbackTimeMs: 0,
|
||||
};
|
||||
|
||||
const prompt = buildSystemPrompt(context);
|
||||
|
||||
expect(prompt).toContain("[id: m1]");
|
||||
});
|
||||
|
||||
test("includes assetId instruction when media assets are present", () => {
|
||||
const context: AgentContext = {
|
||||
projectId: "proj-1",
|
||||
activeSceneId: "scene-A",
|
||||
mediaAssets: [
|
||||
{ id: "m1", name: "conclu.mov", type: "video", duration: 60 },
|
||||
],
|
||||
playbackTimeMs: 0,
|
||||
};
|
||||
|
||||
const prompt = buildSystemPrompt(context);
|
||||
|
||||
expect(prompt).toContain("IMPORTANT");
|
||||
expect(prompt).toContain("assetId");
|
||||
expect(prompt).toContain('internal "id" value');
|
||||
expect(prompt).toContain("NOT the filename");
|
||||
});
|
||||
|
||||
test("omits assetId instruction when no media assets", () => {
|
||||
const prompt = buildSystemPrompt(BASE_CONTEXT);
|
||||
|
||||
expect(prompt).not.toContain("IMPORTANT");
|
||||
expect(prompt).not.toContain("assetId");
|
||||
});
|
||||
|
||||
test("is valid when no media assets are loaded", () => {
|
||||
const context: AgentContext = {
|
||||
projectId: "proj-2",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,502 @@
|
|||
import { beforeEach, describe, expect, jest, test } from "bun:test";
|
||||
import { SchemaType, type GenerateContentResult } from "@google/generative-ai";
|
||||
import type { ProviderConfig } from "@/agent/providers/types";
|
||||
import type { ToolSchema } from "@/agent/types";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Mock the Google Generative AI SDK before the adapter module loads
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const mockGenerateContent = jest.fn();
|
||||
|
||||
jest.mock("@google/generative-ai", () => ({
|
||||
GoogleGenerativeAI: class MockGoogleGenerativeAI {
|
||||
getGenerativeModel(_opts: { model: string }) {
|
||||
return { generateContent: mockGenerateContent };
|
||||
}
|
||||
constructor(_opts: { apiKey: string }) {}
|
||||
},
|
||||
SchemaType: {
|
||||
STRING: "string",
|
||||
NUMBER: "number",
|
||||
BOOLEAN: "boolean",
|
||||
OBJECT: "object",
|
||||
},
|
||||
}));
|
||||
|
||||
// Import after mock is set up
|
||||
import {
|
||||
GeminiAdapter,
|
||||
toGeminiContents,
|
||||
toGeminiTools,
|
||||
fromGeminiResponse,
|
||||
} from "@/agent/providers/gemini";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const TEST_CONFIG: ProviderConfig = {
|
||||
provider: "gemini",
|
||||
apiKey: "test-gemini-key",
|
||||
model: "gemini-2.0-flash",
|
||||
};
|
||||
|
||||
const SAMPLE_TOOLS: ToolSchema[] = [
|
||||
{
|
||||
name: "transcribe_video",
|
||||
description: "Transcribes a video asset",
|
||||
parameters: [
|
||||
{ key: "assetId", type: "string", required: true },
|
||||
{ key: "language", type: "string", required: false },
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
function makeGeminiResponse(overrides: {
|
||||
text?: string;
|
||||
functionCalls?: Array<{ name: string; args: Record<string, unknown> }>;
|
||||
}): GenerateContentResult {
|
||||
const parts: Array<Record<string, unknown>> = [];
|
||||
|
||||
if (overrides.text !== undefined) {
|
||||
parts.push({ text: overrides.text });
|
||||
}
|
||||
|
||||
if (overrides.functionCalls) {
|
||||
for (const fc of overrides.functionCalls) {
|
||||
parts.push({ functionCall: { name: fc.name, args: fc.args } });
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
response: {
|
||||
candidates: [
|
||||
{
|
||||
content: { parts },
|
||||
},
|
||||
],
|
||||
},
|
||||
} as unknown as GenerateContentResult;
|
||||
}
|
||||
|
||||
function makeChatMessage(
|
||||
role: "user" | "assistant" | "system" | "tool_result",
|
||||
content: string,
|
||||
extra?: Record<string, unknown>,
|
||||
) {
|
||||
return {
|
||||
id: `msg-${Math.random().toString(36).slice(2)}`,
|
||||
role,
|
||||
content,
|
||||
timestamp: Date.now(),
|
||||
...extra,
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Tests: toGeminiContents (Task 4.1)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("toGeminiContents", () => {
|
||||
test("maps user message to Gemini user role", () => {
|
||||
const messages = [makeChatMessage("user", "Hello")];
|
||||
const contents = toGeminiContents(messages);
|
||||
|
||||
expect(contents).toHaveLength(1);
|
||||
expect(contents[0].role).toBe("user");
|
||||
expect(contents[0].parts).toEqual([{ text: "Hello" }]);
|
||||
});
|
||||
|
||||
test("maps assistant message to Gemini model role", () => {
|
||||
const messages = [makeChatMessage("assistant", "Hi there!")];
|
||||
const contents = toGeminiContents(messages);
|
||||
|
||||
expect(contents).toHaveLength(1);
|
||||
expect(contents[0].role).toBe("model");
|
||||
expect(contents[0].parts).toEqual([{ text: "Hi there!" }]);
|
||||
});
|
||||
|
||||
test("maps assistant with toolCalls to model role with functionCall parts", () => {
|
||||
const messages = [
|
||||
makeChatMessage("assistant", "Let me check.", {
|
||||
toolCalls: [
|
||||
{
|
||||
id: "tc_1",
|
||||
name: "transcribe_video",
|
||||
args: { assetId: "vid-123" },
|
||||
},
|
||||
],
|
||||
}),
|
||||
];
|
||||
const contents = toGeminiContents(messages);
|
||||
|
||||
expect(contents).toHaveLength(1);
|
||||
expect(contents[0].role).toBe("model");
|
||||
expect(contents[0].parts).toEqual([
|
||||
{ text: "Let me check." },
|
||||
{
|
||||
functionCall: {
|
||||
name: "transcribe_video",
|
||||
args: { assetId: "vid-123" },
|
||||
},
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
test("maps tool_result to function role with functionResponse", () => {
|
||||
const messages = [
|
||||
makeChatMessage("assistant", "", {
|
||||
toolCalls: [
|
||||
{
|
||||
id: "tc_1",
|
||||
name: "transcribe_video",
|
||||
args: { assetId: "vid-123" },
|
||||
},
|
||||
],
|
||||
}),
|
||||
makeChatMessage("tool_result", '{"transcript": "Hello world"}', {
|
||||
toolCallId: "tc_1",
|
||||
}),
|
||||
];
|
||||
const contents = toGeminiContents(messages);
|
||||
|
||||
// First: model (assistant with toolCalls)
|
||||
expect(contents[0].role).toBe("model");
|
||||
// Second: function (tool_result)
|
||||
expect(contents[1].role).toBe("function");
|
||||
expect(contents[1].parts).toEqual([
|
||||
{
|
||||
functionResponse: {
|
||||
name: "transcribe_video",
|
||||
response: { transcript: "Hello world" },
|
||||
},
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
test("excludes system messages from contents", () => {
|
||||
const messages = [
|
||||
makeChatMessage("system", "You are a video editor."),
|
||||
makeChatMessage("user", "Hello"),
|
||||
];
|
||||
const contents = toGeminiContents(messages);
|
||||
|
||||
expect(contents).toHaveLength(1);
|
||||
expect(contents[0].role).toBe("user");
|
||||
});
|
||||
|
||||
test("handles tool_result with non-JSON content by wrapping in object", () => {
|
||||
const messages = [
|
||||
makeChatMessage("assistant", "", {
|
||||
toolCalls: [
|
||||
{ id: "tc_1", name: "some_tool", args: {} },
|
||||
],
|
||||
}),
|
||||
makeChatMessage("tool_result", "plain text result", {
|
||||
toolCallId: "tc_1",
|
||||
}),
|
||||
];
|
||||
const contents = toGeminiContents(messages);
|
||||
|
||||
expect(contents[1].parts).toEqual([
|
||||
{
|
||||
functionResponse: {
|
||||
name: "some_tool",
|
||||
response: { result: "plain text result" },
|
||||
},
|
||||
},
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Tests: toGeminiTools (Task 4.2)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("toGeminiTools", () => {
|
||||
test("converts a single tool with parameters", () => {
|
||||
const declarations = toGeminiTools(SAMPLE_TOOLS);
|
||||
|
||||
expect(declarations).toHaveLength(1);
|
||||
expect(declarations[0].name).toBe("transcribe_video");
|
||||
expect(declarations[0].description).toBe("Transcribes a video asset");
|
||||
expect(declarations[0].parameters?.type).toBe(SchemaType.OBJECT);
|
||||
expect(declarations[0].parameters?.properties).toHaveProperty("assetId");
|
||||
expect(declarations[0].parameters?.required).toEqual(["assetId"]);
|
||||
});
|
||||
|
||||
test("maps type strings to Gemini SchemaType values", () => {
|
||||
const tools: ToolSchema[] = [
|
||||
{
|
||||
name: "multi_param_tool",
|
||||
description: "Tool with all param types",
|
||||
parameters: [
|
||||
{ key: "name", type: "string", required: true },
|
||||
{ key: "count", type: "number", required: false },
|
||||
{ key: "active", type: "boolean", required: false },
|
||||
{ key: "meta", type: "object", required: false },
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
const declarations = toGeminiTools(tools);
|
||||
const props = declarations[0].parameters?.properties as Record<
|
||||
string,
|
||||
{ type: string }
|
||||
>;
|
||||
|
||||
expect(props.name.type).toBe(SchemaType.STRING);
|
||||
expect(props.count.type).toBe(SchemaType.NUMBER);
|
||||
expect(props.active.type).toBe(SchemaType.BOOLEAN);
|
||||
expect(props.meta.type).toBe(SchemaType.OBJECT);
|
||||
});
|
||||
|
||||
test("returns empty array when no tools provided", () => {
|
||||
expect(toGeminiTools([])).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Tests: fromGeminiResponse (Task 4.3)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("fromGeminiResponse", () => {
|
||||
test("extracts text-only response with no toolCalls", () => {
|
||||
const response = makeGeminiResponse({ text: "Hello from Gemini!" });
|
||||
const result = fromGeminiResponse(response);
|
||||
|
||||
expect(result.content).toBe("Hello from Gemini!");
|
||||
expect(result.toolCalls).toBeUndefined();
|
||||
});
|
||||
|
||||
test("maps function-call response to toolCalls with non-empty synthesized IDs", () => {
|
||||
const response = makeGeminiResponse({
|
||||
text: undefined,
|
||||
functionCalls: [
|
||||
{ name: "transcribe_video", args: { assetId: "vid-1" } },
|
||||
],
|
||||
});
|
||||
const result = fromGeminiResponse(response);
|
||||
|
||||
expect(result.content).toBe("");
|
||||
expect(result.toolCalls).toHaveLength(1);
|
||||
expect(result.toolCalls?.[0].name).toBe("transcribe_video");
|
||||
expect(result.toolCalls?.[0].args).toEqual({ assetId: "vid-1" });
|
||||
// Non-empty synthesized ID
|
||||
expect(result.toolCalls?.[0].id).toBeTruthy();
|
||||
expect(typeof result.toolCalls?.[0].id).toBe("string");
|
||||
});
|
||||
|
||||
test("two calls produce different IDs", () => {
|
||||
const response1 = makeGeminiResponse({
|
||||
functionCalls: [{ name: "tool_a", args: {} }],
|
||||
});
|
||||
const response2 = makeGeminiResponse({
|
||||
functionCalls: [{ name: "tool_a", args: {} }],
|
||||
});
|
||||
|
||||
const result1 = fromGeminiResponse(response1);
|
||||
const result2 = fromGeminiResponse(response2);
|
||||
|
||||
expect(result1.toolCalls?.[0].id).not.toBe(
|
||||
result2.toolCalls?.[0].id,
|
||||
);
|
||||
});
|
||||
|
||||
test("handles mixed text and function-call parts", () => {
|
||||
const response = makeGeminiResponse({
|
||||
text: "I'll look that up.",
|
||||
functionCalls: [{ name: "search", args: { q: "test" } }],
|
||||
});
|
||||
const result = fromGeminiResponse(response);
|
||||
|
||||
expect(result.content).toBe("I'll look that up.");
|
||||
expect(result.toolCalls).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Tests: error paths (Task 4.4)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("error paths", () => {
|
||||
test("throws on empty candidates", () => {
|
||||
const response = {
|
||||
response: { candidates: [] },
|
||||
};
|
||||
|
||||
expect(() => fromGeminiResponse(response as any)).toThrow(
|
||||
"Empty response from Gemini provider",
|
||||
);
|
||||
});
|
||||
|
||||
test("throws on null candidates", () => {
|
||||
const response = {
|
||||
response: { candidates: null },
|
||||
};
|
||||
|
||||
expect(() => fromGeminiResponse(response as any)).toThrow(
|
||||
"Empty response from Gemini provider",
|
||||
);
|
||||
});
|
||||
|
||||
test("SDK error propagates from chat()", async () => {
|
||||
mockGenerateContent.mockRejectedValueOnce(
|
||||
new Error("API key not valid"),
|
||||
);
|
||||
|
||||
const adapter = new GeminiAdapter(TEST_CONFIG);
|
||||
|
||||
expect(
|
||||
adapter.chat({
|
||||
messages: [makeChatMessage("user", "Hello")],
|
||||
systemPrompt: "",
|
||||
tools: [],
|
||||
}),
|
||||
).rejects.toThrow("API key not valid");
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Tests: full chat() round-trip (Task 4.6)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("GeminiAdapter.chat()", () => {
|
||||
beforeEach(() => {
|
||||
mockGenerateContent.mockReset();
|
||||
});
|
||||
|
||||
test("calls generateContent with correct contents and no tools", async () => {
|
||||
mockGenerateContent.mockResolvedValueOnce(
|
||||
makeGeminiResponse({ text: "Hi!" }),
|
||||
);
|
||||
|
||||
const adapter = new GeminiAdapter(TEST_CONFIG);
|
||||
const result = await adapter.chat({
|
||||
messages: [makeChatMessage("user", "Hello")],
|
||||
systemPrompt: "You are helpful.",
|
||||
tools: [],
|
||||
});
|
||||
|
||||
expect(result.content).toBe("Hi!");
|
||||
expect(result.toolCalls).toBeUndefined();
|
||||
|
||||
const callArgs = mockGenerateContent.mock.calls[0][0];
|
||||
expect(callArgs.contents).toHaveLength(1);
|
||||
expect(callArgs.contents[0].role).toBe("user");
|
||||
expect(callArgs.systemInstruction).toBe("You are helpful.");
|
||||
expect(callArgs.tools).toBeUndefined();
|
||||
});
|
||||
|
||||
test("calls generateContent with tools and systemInstruction", async () => {
|
||||
mockGenerateContent.mockResolvedValueOnce(
|
||||
makeGeminiResponse({
|
||||
functionCalls: [
|
||||
{
|
||||
name: "transcribe_video",
|
||||
args: { assetId: "vid-1" },
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
const adapter = new GeminiAdapter(TEST_CONFIG);
|
||||
const result = await adapter.chat({
|
||||
messages: [makeChatMessage("user", "Transcribe this")],
|
||||
systemPrompt: "Video editing assistant",
|
||||
tools: SAMPLE_TOOLS,
|
||||
});
|
||||
|
||||
expect(result.toolCalls).toHaveLength(1);
|
||||
expect(result.toolCalls?.[0].name).toBe("transcribe_video");
|
||||
|
||||
const callArgs = mockGenerateContent.mock.calls[0][0];
|
||||
expect(callArgs.systemInstruction).toBe("Video editing assistant");
|
||||
expect(callArgs.tools).toHaveLength(1);
|
||||
expect(callArgs.tools[0].functionDeclarations).toHaveLength(1);
|
||||
});
|
||||
|
||||
test("passes multi-turn conversation through to Gemini", async () => {
|
||||
mockGenerateContent.mockResolvedValueOnce(
|
||||
makeGeminiResponse({ text: "Sure!" }),
|
||||
);
|
||||
|
||||
const adapter = new GeminiAdapter(TEST_CONFIG);
|
||||
await adapter.chat({
|
||||
messages: [
|
||||
makeChatMessage("user", "First question"),
|
||||
makeChatMessage("assistant", "First answer"),
|
||||
makeChatMessage("user", "Second question"),
|
||||
],
|
||||
systemPrompt: "",
|
||||
tools: [],
|
||||
});
|
||||
|
||||
const callArgs = mockGenerateContent.mock.calls[0][0];
|
||||
expect(callArgs.contents).toHaveLength(3);
|
||||
expect(callArgs.contents[0].role).toBe("user");
|
||||
expect(callArgs.contents[1].role).toBe("model");
|
||||
expect(callArgs.contents[2].role).toBe("user");
|
||||
});
|
||||
|
||||
test("does not include systemInstruction when systemPrompt is empty", async () => {
|
||||
mockGenerateContent.mockResolvedValueOnce(
|
||||
makeGeminiResponse({ text: "OK" }),
|
||||
);
|
||||
|
||||
const adapter = new GeminiAdapter(TEST_CONFIG);
|
||||
await adapter.chat({
|
||||
messages: [makeChatMessage("user", "Hi")],
|
||||
systemPrompt: "",
|
||||
tools: [],
|
||||
});
|
||||
|
||||
const callArgs = mockGenerateContent.mock.calls[0][0];
|
||||
expect(callArgs.systemInstruction).toBeUndefined();
|
||||
});
|
||||
|
||||
test("ignores baseUrl in config — Gemini still works", async () => {
|
||||
mockGenerateContent.mockResolvedValueOnce(
|
||||
makeGeminiResponse({ text: "Hello!" }),
|
||||
);
|
||||
|
||||
const configWithBaseUrl: ProviderConfig = {
|
||||
provider: "gemini",
|
||||
apiKey: "test-gemini-key",
|
||||
model: "gemini-2.0-flash",
|
||||
baseUrl: "https://should-be-ignored.example.com",
|
||||
};
|
||||
|
||||
const adapter = new GeminiAdapter(configWithBaseUrl);
|
||||
const result = await adapter.chat({
|
||||
messages: [makeChatMessage("user", "Hi")],
|
||||
systemPrompt: "",
|
||||
tools: [],
|
||||
});
|
||||
|
||||
expect(result.content).toBe("Hello!");
|
||||
expect(result.toolCalls).toBeUndefined();
|
||||
});
|
||||
|
||||
test("returns text when tools are provided but Gemini responds with text only", async () => {
|
||||
mockGenerateContent.mockResolvedValueOnce(
|
||||
makeGeminiResponse({ text: "I don't need any tools for this." }),
|
||||
);
|
||||
|
||||
const adapter = new GeminiAdapter(TEST_CONFIG);
|
||||
const result = await adapter.chat({
|
||||
messages: [makeChatMessage("user", "Just chat")],
|
||||
systemPrompt: "You are helpful.",
|
||||
tools: SAMPLE_TOOLS,
|
||||
});
|
||||
|
||||
expect(result.content).toBe("I don't need any tools for this.");
|
||||
expect(result.toolCalls).toBeUndefined();
|
||||
|
||||
// Verify tools were still sent to Gemini
|
||||
const callArgs = mockGenerateContent.mock.calls[0][0];
|
||||
expect(callArgs.tools).toBeDefined();
|
||||
expect(callArgs.tools[0].functionDeclarations).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
import { describe, expect, test } from "bun:test";
|
||||
import { createProvider } from "@/agent/providers/index";
|
||||
import { OpenAICompatibleAdapter } from "@/agent/providers/openai-compatible";
|
||||
import { GeminiAdapter } from "@/agent/providers/gemini";
|
||||
import type { ProviderConfig } from "@/agent/providers/types";
|
||||
|
||||
describe("createProvider", () => {
|
||||
|
|
@ -28,6 +29,17 @@ describe("createProvider", () => {
|
|||
// Adapter was constructed without throwing — config accepted
|
||||
});
|
||||
|
||||
test("returns GeminiAdapter for gemini provider", () => {
|
||||
const config: ProviderConfig = {
|
||||
provider: "gemini",
|
||||
apiKey: "test-gemini-key",
|
||||
model: "gemini-2.0-flash",
|
||||
};
|
||||
|
||||
const adapter = createProvider(config);
|
||||
expect(adapter).toBeInstanceOf(GeminiAdapter);
|
||||
});
|
||||
|
||||
test("throws Error for unknown provider", () => {
|
||||
const config: ProviderConfig = {
|
||||
provider: "unknown-provider",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,240 @@
|
|||
import {
|
||||
GoogleGenerativeAI,
|
||||
type Content,
|
||||
type FunctionDeclaration,
|
||||
type FunctionDeclarationSchema,
|
||||
type GenerateContentResult,
|
||||
SchemaType,
|
||||
} from "@google/generative-ai";
|
||||
import type { ChatMessage, ToolCall, ToolSchema } from "@/agent/types";
|
||||
import type {
|
||||
ProviderAdapter,
|
||||
ProviderConfig,
|
||||
ProviderResponse,
|
||||
} from "./types";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Internal conversion helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Maps internal type strings to Gemini SchemaType values.
|
||||
*/
|
||||
const TYPE_MAP: Record<string, SchemaType> = {
|
||||
string: SchemaType.STRING,
|
||||
number: SchemaType.NUMBER,
|
||||
boolean: SchemaType.BOOLEAN,
|
||||
object: SchemaType.OBJECT,
|
||||
};
|
||||
|
||||
/**
|
||||
* Converts internal ChatMessages into Gemini Content[].
|
||||
*
|
||||
* - `user` → role "user", text part
|
||||
* - `assistant` → role "model", text part + functionCall parts from toolCalls
|
||||
* - `tool_result`→ role "function", functionResponse part
|
||||
* - `system` → skipped (handled separately via systemInstruction)
|
||||
*/
|
||||
export function toGeminiContents(messages: ChatMessage[]): Content[] {
|
||||
const contents: Content[] = [];
|
||||
|
||||
for (const msg of messages) {
|
||||
if (msg.role === "system") {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (msg.role === "user") {
|
||||
contents.push({
|
||||
role: "user",
|
||||
parts: [{ text: msg.content }],
|
||||
});
|
||||
} else if (msg.role === "assistant") {
|
||||
const parts: Content["parts"] = [];
|
||||
|
||||
if (msg.content) {
|
||||
parts.push({ text: msg.content });
|
||||
}
|
||||
|
||||
if (msg.toolCalls && msg.toolCalls.length > 0) {
|
||||
for (const tc of msg.toolCalls) {
|
||||
parts.push({
|
||||
functionCall: { name: tc.name, args: tc.args },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
contents.push({ role: "model", parts });
|
||||
} else if (msg.role === "tool_result") {
|
||||
// Gemini uses the function name to correlate responses, not IDs.
|
||||
// We parse the content as JSON for the response object; fall back
|
||||
// to wrapping it if it's not valid JSON.
|
||||
let responseData: object;
|
||||
try {
|
||||
responseData = JSON.parse(msg.content) as object;
|
||||
} catch {
|
||||
responseData = { result: msg.content };
|
||||
}
|
||||
|
||||
// Derive the function name from the toolCallId → find the matching
|
||||
// tool name in prior messages. If not found, use a placeholder.
|
||||
const toolName = findToolNameForResult(messages, msg.toolCallId);
|
||||
|
||||
contents.push({
|
||||
role: "function",
|
||||
parts: [
|
||||
{
|
||||
functionResponse: {
|
||||
name: toolName,
|
||||
response: responseData,
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return contents;
|
||||
}
|
||||
|
||||
/**
|
||||
* Walks backward through messages to find the function name associated with
|
||||
* a toolCallId. This is needed because Gemini's functionResponse uses names,
|
||||
* not IDs.
|
||||
*/
|
||||
function findToolNameForResult(
|
||||
messages: ChatMessage[],
|
||||
toolCallId?: string,
|
||||
): string {
|
||||
if (!toolCallId) return "unknown";
|
||||
|
||||
for (let i = messages.length - 1; i >= 0; i--) {
|
||||
const msg = messages[i];
|
||||
if (msg.role === "assistant" && msg.toolCalls) {
|
||||
const match = msg.toolCalls.find((tc) => tc.id === toolCallId);
|
||||
if (match) return match.name;
|
||||
}
|
||||
}
|
||||
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts internal ToolSchema[] into Gemini FunctionDeclaration[].
|
||||
*
|
||||
* Maps internal type strings ("string", "number", etc.) to Gemini SchemaType
|
||||
* enum values. Returns an empty array when no tools are provided.
|
||||
*/
|
||||
export function toGeminiTools(tools: ToolSchema[]): FunctionDeclaration[] {
|
||||
if (tools.length === 0) return [];
|
||||
|
||||
return tools.map((tool) => {
|
||||
const properties: FunctionDeclarationSchema["properties"] = {};
|
||||
|
||||
for (const param of tool.parameters) {
|
||||
const geminiType = TYPE_MAP[param.type] ?? SchemaType.STRING;
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
(properties as Record<string, any>)[param.key] = { type: geminiType };
|
||||
}
|
||||
|
||||
const requiredParams = tool.parameters
|
||||
.filter((p) => p.required)
|
||||
.map((p) => p.key);
|
||||
|
||||
return {
|
||||
name: tool.name,
|
||||
description: tool.description,
|
||||
parameters: {
|
||||
type: SchemaType.OBJECT,
|
||||
properties,
|
||||
...(requiredParams.length > 0 && { required: requiredParams }),
|
||||
},
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts a Gemini GenerateContentResult into the provider-agnostic
|
||||
* ProviderResponse shape.
|
||||
*
|
||||
* - Text parts are concatenated into `content`.
|
||||
* - FunctionCall parts are mapped to `toolCalls[]` with synthesized UUIDs.
|
||||
* - Throws if no candidates are present.
|
||||
*/
|
||||
export function fromGeminiResponse(
|
||||
response: GenerateContentResult,
|
||||
): ProviderResponse {
|
||||
const candidates = response.response.candidates;
|
||||
if (!candidates || candidates.length === 0) {
|
||||
throw new Error("Empty response from Gemini provider");
|
||||
}
|
||||
|
||||
const parts = candidates[0].content?.parts ?? [];
|
||||
const textParts: string[] = [];
|
||||
const toolCalls: ToolCall[] = [];
|
||||
|
||||
for (const part of parts) {
|
||||
if ("text" in part && part.text !== undefined) {
|
||||
textParts.push(part.text);
|
||||
}
|
||||
if ("functionCall" in part && part.functionCall) {
|
||||
toolCalls.push({
|
||||
id: crypto.randomUUID(),
|
||||
name: part.functionCall.name,
|
||||
args: (part.functionCall.args as Record<string, unknown>) ?? {},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const content = textParts.join("");
|
||||
const result: ProviderResponse = { content };
|
||||
|
||||
if (toolCalls.length > 0) {
|
||||
result.toolCalls = toolCalls;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Adapter implementation
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Gemini adapter using @google/generative-ai SDK.
|
||||
*
|
||||
* Uses the stateless `generateContent()` API — full history is sent with each
|
||||
* call, matching the existing route contract. System prompts are mapped to
|
||||
* Gemini's `systemInstruction` field, not as user messages.
|
||||
*/
|
||||
export class GeminiAdapter implements ProviderAdapter {
|
||||
private model;
|
||||
|
||||
constructor(config: ProviderConfig) {
|
||||
const genAI = new GoogleGenerativeAI(config.apiKey);
|
||||
this.model = genAI.getGenerativeModel({ model: config.model });
|
||||
// config.baseUrl is intentionally ignored for the Gemini provider.
|
||||
}
|
||||
|
||||
async chat(params: {
|
||||
messages: ChatMessage[];
|
||||
systemPrompt: string;
|
||||
tools: ToolSchema[];
|
||||
}): Promise<ProviderResponse> {
|
||||
const contents = toGeminiContents(params.messages);
|
||||
const functionDeclarations = toGeminiTools(params.tools);
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const request: any = { contents };
|
||||
|
||||
if (params.systemPrompt) {
|
||||
request.systemInstruction = params.systemPrompt;
|
||||
}
|
||||
|
||||
if (functionDeclarations.length > 0) {
|
||||
request.tools = [{ functionDeclarations }];
|
||||
}
|
||||
|
||||
const result = await this.model.generateContent(request);
|
||||
return fromGeminiResponse(result);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,6 @@
|
|||
import type { ProviderAdapter, ProviderConfig } from "./types";
|
||||
import { OpenAICompatibleAdapter } from "./openai-compatible";
|
||||
import { GeminiAdapter } from "./gemini";
|
||||
|
||||
/**
|
||||
* Factory — resolves a ProviderConfig to a concrete adapter.
|
||||
|
|
@ -12,6 +13,8 @@ export function createProvider(config: ProviderConfig): ProviderAdapter {
|
|||
switch (config.provider) {
|
||||
case "openai-compatible":
|
||||
return new OpenAICompatibleAdapter(config);
|
||||
case "gemini":
|
||||
return new GeminiAdapter(config);
|
||||
default:
|
||||
throw new Error(`Unknown LLM provider: ${config.provider}`);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ export function buildSystemPrompt(
|
|||
): string {
|
||||
const mediaSection =
|
||||
context.mediaAssets.length > 0
|
||||
? `Active media assets:\n${context.mediaAssets.map((m) => `- ${m.name} (${m.type}, ${m.duration}s)`).join("\n")}`
|
||||
? `Active media assets:\n${context.mediaAssets.map((m) => `- [id: ${m.id}] ${m.name} (${m.type}, ${m.duration}s)`).join("\n")}`
|
||||
: "No media assets loaded.";
|
||||
|
||||
const parts = [
|
||||
|
|
@ -28,6 +28,12 @@ export function buildSystemPrompt(
|
|||
mediaSection,
|
||||
];
|
||||
|
||||
if (context.mediaAssets.length > 0) {
|
||||
parts.push(
|
||||
'IMPORTANT: When calling tools that accept an "assetId" parameter, always use the internal "id" value (e.g., "v1"), NOT the filename or display name.',
|
||||
);
|
||||
}
|
||||
|
||||
if (tools && tools.length > 0) {
|
||||
const toolList = tools
|
||||
.map((t) => `- ${t.name}: ${t.description}`)
|
||||
|
|
|
|||
|
|
@ -227,7 +227,7 @@ describe("transcribe_video tool — error branches", () => {
|
|||
|
||||
const result = await tool.execute({ assetId: "v99" }, context);
|
||||
expect(result).toEqual({
|
||||
error: "Could not access file for asset v99",
|
||||
error: 'No asset found with id or name "v99". Available ids: [v1]. Available names: [intro.mp4].',
|
||||
});
|
||||
});
|
||||
|
||||
|
|
@ -282,3 +282,141 @@ describe("transcribe_video tool — error branches", () => {
|
|||
expect(result).toEqual({ error: "Worker not initialized" });
|
||||
});
|
||||
});
|
||||
|
||||
describe("transcribe_video tool — name-based assetId fallback", () => {
|
||||
beforeEach(() => {
|
||||
mockResolveAssetFile.mockClear();
|
||||
mockGetAssetHasAudio.mockClear();
|
||||
mockDecodeAudio.mockClear();
|
||||
mockTranscribe.mockClear();
|
||||
});
|
||||
|
||||
test("resolves by filename when no id match (exact name)", async () => {
|
||||
const fakeFile = new File([], "conclu.mov", { type: "video/quicktime" });
|
||||
const context = makeContext({
|
||||
mediaAssets: [
|
||||
fakeVideoAsset({ id: "abc123", name: "conclu.mov" }),
|
||||
],
|
||||
});
|
||||
|
||||
mockResolveAssetFile.mockReturnValue(fakeFile);
|
||||
mockGetAssetHasAudio.mockReturnValue(true);
|
||||
mockDecodeAudio.mockResolvedValue({
|
||||
samples: new Float32Array(16000),
|
||||
sampleRate: 16000,
|
||||
});
|
||||
mockTranscribe.mockResolvedValue(fakeTranscriptionResult());
|
||||
|
||||
const tool = toolRegistry.get("transcribe_video");
|
||||
const result = await tool.execute({ assetId: "conclu.mov" }, context);
|
||||
|
||||
expect(result).toEqual(
|
||||
expect.objectContaining({ assetName: "conclu.mov" }),
|
||||
);
|
||||
// Resolved asset must pass its internal id to the adapter
|
||||
expect(mockResolveAssetFile).toHaveBeenCalledWith("abc123");
|
||||
expect(mockGetAssetHasAudio).toHaveBeenCalledWith("abc123");
|
||||
});
|
||||
|
||||
test("resolves by case-insensitive name when no exact id or name match", async () => {
|
||||
const fakeFile = new File([], "Conclu.MOV", { type: "video/quicktime" });
|
||||
const context = makeContext({
|
||||
mediaAssets: [
|
||||
fakeVideoAsset({ id: "abc123", name: "Conclu.MOV" }),
|
||||
],
|
||||
});
|
||||
|
||||
mockResolveAssetFile.mockReturnValue(fakeFile);
|
||||
mockGetAssetHasAudio.mockReturnValue(true);
|
||||
mockDecodeAudio.mockResolvedValue({
|
||||
samples: new Float32Array(16000),
|
||||
sampleRate: 16000,
|
||||
});
|
||||
mockTranscribe.mockResolvedValue(fakeTranscriptionResult());
|
||||
|
||||
const tool = toolRegistry.get("transcribe_video");
|
||||
const result = await tool.execute({ assetId: "conclu.mov" }, context);
|
||||
|
||||
expect(result).toEqual(
|
||||
expect.objectContaining({ assetName: "Conclu.MOV" }),
|
||||
);
|
||||
expect(mockResolveAssetFile).toHaveBeenCalledWith("abc123");
|
||||
});
|
||||
|
||||
test("returns ambiguous error when two assets share same name", async () => {
|
||||
const context = makeContext({
|
||||
mediaAssets: [
|
||||
fakeVideoAsset({ id: "v1", name: "clip.mp4" }),
|
||||
fakeVideoAsset({ id: "v2", name: "clip.mp4" }),
|
||||
],
|
||||
});
|
||||
|
||||
const tool = toolRegistry.get("transcribe_video");
|
||||
const result = await tool.execute({ assetId: "clip.mp4" }, context);
|
||||
|
||||
expect(result).toEqual({
|
||||
error: 'Ambiguous asset name "clip.mp4" matches multiple assets (clip.mp4, clip.mp4). Specify the internal id: v1, v2.',
|
||||
});
|
||||
});
|
||||
|
||||
test("returns ambiguous error for case-insensitive duplicates", async () => {
|
||||
const context = makeContext({
|
||||
mediaAssets: [
|
||||
fakeVideoAsset({ id: "v1", name: "Clip.MP4" }),
|
||||
fakeVideoAsset({ id: "v2", name: "CLIP.MP4" }),
|
||||
],
|
||||
});
|
||||
|
||||
const tool = toolRegistry.get("transcribe_video");
|
||||
const result = await tool.execute({ assetId: "clip.mp4" }, context);
|
||||
|
||||
expect(result).toEqual({
|
||||
error: 'Ambiguous asset name "clip.mp4" matches multiple assets (Clip.MP4, CLIP.MP4). Specify the internal id: v1, v2.',
|
||||
});
|
||||
});
|
||||
|
||||
test("prefers exact id match over name match", async () => {
|
||||
const fakeFile = new File([], "intro.mp4", { type: "video/mp4" });
|
||||
// An asset whose id happens to equal another asset's name
|
||||
const context = makeContext({
|
||||
mediaAssets: [
|
||||
fakeVideoAsset({ id: "v1", name: "intro.mp4" }),
|
||||
fakeVideoAsset({ id: "intro.mp4", name: "outro.mp4" }),
|
||||
],
|
||||
});
|
||||
|
||||
mockResolveAssetFile.mockReturnValue(fakeFile);
|
||||
mockGetAssetHasAudio.mockReturnValue(true);
|
||||
mockDecodeAudio.mockResolvedValue({
|
||||
samples: new Float32Array(16000),
|
||||
sampleRate: 16000,
|
||||
});
|
||||
mockTranscribe.mockResolvedValue(fakeTranscriptionResult());
|
||||
|
||||
const tool = toolRegistry.get("transcribe_video");
|
||||
const result = await tool.execute({ assetId: "intro.mp4" }, context);
|
||||
|
||||
// "intro.mp4" matches v1 by id AND the second asset by id exactly
|
||||
// id lookup is first, so it resolves the second asset (id="intro.mp4")
|
||||
expect(result).toEqual(
|
||||
expect.objectContaining({ assetName: "outro.mp4" }),
|
||||
);
|
||||
expect(mockResolveAssetFile).toHaveBeenCalledWith("intro.mp4");
|
||||
});
|
||||
|
||||
test("returns not-found error with available ids and names for totally unknown assetId", async () => {
|
||||
const context = makeContext({
|
||||
mediaAssets: [
|
||||
fakeVideoAsset({ id: "v1", name: "intro.mp4" }),
|
||||
fakeVideoAsset({ id: "v2", name: "outro.mp4" }),
|
||||
],
|
||||
});
|
||||
|
||||
const tool = toolRegistry.get("transcribe_video");
|
||||
const result = await tool.execute({ assetId: "nonexistent.mov" }, context);
|
||||
|
||||
expect(result).toEqual({
|
||||
error: 'No asset found with id or name "nonexistent.mov". Available ids: [v1, v2]. Available names: [intro.mp4, outro.mp4].',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -57,10 +57,43 @@ const transcribeVideoTool: ToolDefinition = {
|
|||
let targetAsset: (typeof mediaCandidates)[number] | undefined;
|
||||
|
||||
if (typedArgs.assetId) {
|
||||
// 1. Exact internal-id match
|
||||
targetAsset = mediaCandidates.find((a) => a.id === typedArgs.assetId);
|
||||
|
||||
// 2. Fallback: exact name match (LLM sometimes passes filename)
|
||||
if (!targetAsset) {
|
||||
const nameMatches = mediaCandidates.filter(
|
||||
(a) => a.name === typedArgs.assetId,
|
||||
);
|
||||
if (nameMatches.length === 1) {
|
||||
targetAsset = nameMatches[0];
|
||||
} else if (nameMatches.length > 1) {
|
||||
return {
|
||||
error: `Ambiguous asset name "${typedArgs.assetId}" matches multiple assets (${nameMatches.map((a) => a.name).join(", ")}). Specify the internal id: ${nameMatches.map((a) => `${a.id}`).join(", ")}.`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Case-insensitive fallback (only if unambiguous)
|
||||
if (!targetAsset) {
|
||||
const lower = typedArgs.assetId.toLowerCase();
|
||||
const ciMatches = mediaCandidates.filter(
|
||||
(a) => a.name.toLowerCase() === lower,
|
||||
);
|
||||
if (ciMatches.length === 1) {
|
||||
targetAsset = ciMatches[0];
|
||||
} else if (ciMatches.length > 1) {
|
||||
return {
|
||||
error: `Ambiguous asset name "${typedArgs.assetId}" matches multiple assets (${ciMatches.map((a) => a.name).join(", ")}). Specify the internal id: ${ciMatches.map((a) => `${a.id}`).join(", ")}.`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
if (!targetAsset) {
|
||||
const ids = mediaCandidates.map((a) => a.id).join(", ");
|
||||
const names = mediaCandidates.map((a) => a.name).join(", ");
|
||||
return {
|
||||
error: `Could not access file for asset ${typedArgs.assetId}`,
|
||||
error: `No asset found with id or name "${typedArgs.assetId}". Available ids: [${ids}]. Available names: [${names}].`,
|
||||
};
|
||||
}
|
||||
} else {
|
||||
|
|
|
|||
3
bun.lock
3
bun.lock
|
|
@ -20,6 +20,7 @@
|
|||
"name": "@opencut/web",
|
||||
"version": "0.1.0",
|
||||
"dependencies": {
|
||||
"@google/generative-ai": "^0.24.0",
|
||||
"@hello-pangea/dnd": "^18.0.1",
|
||||
"@hugeicons/core-free-icons": "^3.3.0",
|
||||
"@hugeicons/react": "^1.1.6",
|
||||
|
|
@ -348,6 +349,8 @@
|
|||
|
||||
"@floating-ui/utils": ["@floating-ui/utils@0.2.10", "", {}, "sha512-aGTxbpbg8/b5JfU1HXSrbH3wXZuLPJcNEcZQFMxLs3oSzgtVu6nFPkbbGGUvBcUjKV2YyB9Wxxabo+HEH9tcRQ=="],
|
||||
|
||||
"@google/generative-ai": ["@google/generative-ai@0.24.1", "", {}, "sha512-MqO+MLfM6kjxcKoy0p1wRzG3b4ZZXtPI+z2IE26UogS2Cm/XHO+7gGRBh6gcJsOiIVoH93UwKvW4HdgiOZCy9Q=="],
|
||||
|
||||
"@hello-pangea/dnd": ["@hello-pangea/dnd@18.0.1", "", { "dependencies": { "@babel/runtime": "^7.26.7", "css-box-model": "^1.2.1", "raf-schd": "^4.0.3", "react-redux": "^9.2.0", "redux": "^5.0.1" }, "peerDependencies": { "react": "^18.0.0 || ^19.0.0", "react-dom": "^18.0.0 || ^19.0.0" } }, "sha512-xojVWG8s/TGrKT1fC8K2tIWeejJYTAeJuj36zM//yEm/ZrnZUSFGS15BpO+jGZT1ybWvyXmeDJwPYb4dhWlbZQ=="],
|
||||
|
||||
"@hugeicons/core-free-icons": ["@hugeicons/core-free-icons@3.3.0", "", {}, "sha512-qYyr4JQ2eQIHTSTbITvnJvs6ERNK64D9gpwZnf2IyuG0exzqfyABLO/oTB71FB3RZPfu1GbwycdiGSo46apjMQ=="],
|
||||
|
|
|
|||
|
|
@ -0,0 +1,451 @@
|
|||
# Agent Tool Specs — NeuralCut
|
||||
|
||||
Este documento define las primitives iniciales del agente. Cada tool debe ser pequeña, accionable y componible. El agente puede resolver features complejas combinando estas tools, pero ninguna tool debe intentar “hacer todo”.
|
||||
|
||||
## Principios generales
|
||||
|
||||
- Las tools reciben datos estructurados, no instrucciones libres.
|
||||
- Las tools deben validar inputs y devolver errores claros.
|
||||
- Las tools que modifican el editor deben usar los comandos/managers existentes para preservar undo/redo cuando aplique.
|
||||
- El agente debe usar `list_project_assets` y `list_timeline` antes de editar cuando no tenga IDs concretos.
|
||||
- Gemini se usa para comprensión multimodal; la edición real se ejecuta con tools determinísticas.
|
||||
|
||||
---
|
||||
|
||||
## 1. `list_project_assets`
|
||||
|
||||
### Propósito
|
||||
Listar assets disponibles en el proyecto para que el agente sepa qué archivos existen y cuáles están usados en el timeline.
|
||||
|
||||
### Input
|
||||
```ts
|
||||
{
|
||||
filter?: "all" | "used" | "unused";
|
||||
type?: "all" | "video" | "audio" | "image";
|
||||
}
|
||||
```
|
||||
|
||||
### Output
|
||||
```ts
|
||||
{
|
||||
assets: Array<{
|
||||
id: string;
|
||||
name: string;
|
||||
type: "video" | "audio" | "image";
|
||||
duration?: number;
|
||||
usedInTimeline: boolean;
|
||||
}>;
|
||||
}
|
||||
```
|
||||
|
||||
### Requirements
|
||||
- MUST return all loaded project assets by default.
|
||||
- MUST support filtering by usage: `used`, `unused`, `all`.
|
||||
- MUST support filtering by asset type.
|
||||
- MUST include stable internal `id` values.
|
||||
- MUST NOT mutate editor state.
|
||||
|
||||
### Errors
|
||||
- If no project is loaded, return `{ error: "No active project" }`.
|
||||
|
||||
---
|
||||
|
||||
## 2. `list_timeline`
|
||||
|
||||
### Propósito
|
||||
Listar el estado actual del timeline para que el agente pueda referenciar clips, textos, stickers y otros elementos por `trackId`/`elementId`.
|
||||
|
||||
### Input
|
||||
```ts
|
||||
{}
|
||||
```
|
||||
|
||||
### Output
|
||||
```ts
|
||||
{
|
||||
tracks: Array<{
|
||||
trackId: string;
|
||||
type: "main" | "overlay" | "audio" | "text" | "effect";
|
||||
elements: Array<{
|
||||
elementId: string;
|
||||
type: string;
|
||||
assetId?: string;
|
||||
name?: string;
|
||||
start: number;
|
||||
end: number;
|
||||
}>;
|
||||
}>;
|
||||
}
|
||||
```
|
||||
|
||||
### Requirements
|
||||
- MUST return a structured timeline summary.
|
||||
- MUST include `trackId` and `elementId` for editable elements.
|
||||
- MUST include timing in seconds or a clearly documented unit.
|
||||
- MUST NOT mutate editor state.
|
||||
|
||||
### Errors
|
||||
- If no active scene/timeline exists, return `{ error: "No active timeline" }`.
|
||||
|
||||
---
|
||||
|
||||
## 3. `load_asset_context`
|
||||
|
||||
### Propósito
|
||||
Cargar un asset en el contexto multimodal del agente usando Gemini. Esta es la base para que el agente entienda video/audio/imagen antes de editar.
|
||||
|
||||
### Input
|
||||
```ts
|
||||
{
|
||||
assetId: string;
|
||||
}
|
||||
```
|
||||
|
||||
### Output
|
||||
```ts
|
||||
{
|
||||
assetId: string;
|
||||
status: "loaded" | "processing";
|
||||
cached: boolean;
|
||||
provider: "gemini";
|
||||
fileUri?: string;
|
||||
summary?: string;
|
||||
}
|
||||
```
|
||||
|
||||
### Requirements
|
||||
- MUST resolve the asset by internal `assetId`.
|
||||
- MAY support fallback by filename if unambiguous, but internal ID is preferred.
|
||||
- MUST upload/process the asset with Gemini when not cached.
|
||||
- MUST cache the provider file reference by `assetId` to avoid repeated uploads.
|
||||
- MUST keep API keys server-side.
|
||||
- MUST support video first; audio/image support may be added if Gemini path supports it cleanly.
|
||||
- MUST NOT edit the timeline.
|
||||
|
||||
### Cache behavior
|
||||
- Cache is application/orchestrator infrastructure, not necessarily a visible LLM tool.
|
||||
- Cache should store at least:
|
||||
```ts
|
||||
{
|
||||
assetId: string;
|
||||
provider: "gemini";
|
||||
fileUri: string;
|
||||
status: "active" | "processing" | "failed";
|
||||
createdAt: number;
|
||||
}
|
||||
```
|
||||
|
||||
### Errors
|
||||
- Unknown asset: `{ error: "Asset not found" }`.
|
||||
- Unsupported type: `{ error: "Unsupported asset type" }`.
|
||||
- Upload/processing failure: `{ error: "Failed to load asset context" }` with safe details.
|
||||
|
||||
### Non-goals
|
||||
- No editing actions.
|
||||
- No `@asset` UI autocomplete.
|
||||
- No streaming progress in the first implementation.
|
||||
|
||||
---
|
||||
|
||||
## 4. `cut_segment`
|
||||
|
||||
### Propósito
|
||||
Cortar o remover un rango de tiempo del timeline.
|
||||
|
||||
### Input
|
||||
```ts
|
||||
{
|
||||
start: number;
|
||||
end: number;
|
||||
mode: "remove" | "keep";
|
||||
}
|
||||
```
|
||||
|
||||
### Output
|
||||
```ts
|
||||
{
|
||||
success: boolean;
|
||||
affectedElements: string[];
|
||||
}
|
||||
```
|
||||
|
||||
### Requirements
|
||||
- MUST validate `start < end`.
|
||||
- MUST operate on timeline time, using the editor’s canonical time unit internally.
|
||||
- MUST use existing timeline/command infrastructure when possible.
|
||||
- `remove` MUST remove the selected time range.
|
||||
- `keep` MUST preserve only the selected time range and remove outside material, if feasible in current editor model.
|
||||
- MUST preserve undo/redo behavior if the editor supports it for the operation.
|
||||
|
||||
### Errors
|
||||
- Invalid range: `{ error: "Invalid time range" }`.
|
||||
- Empty timeline: `{ error: "No timeline content" }`.
|
||||
- Unsupported mode: `{ error: "Unsupported cut mode" }`.
|
||||
|
||||
---
|
||||
|
||||
## 5. `add_text`
|
||||
|
||||
### Propósito
|
||||
Agregar texto visual al timeline. Esta primitive cubre títulos, hooks, labels y subtítulos básicos.
|
||||
|
||||
### Input
|
||||
```ts
|
||||
{
|
||||
text: string;
|
||||
start: number;
|
||||
end: number;
|
||||
position: "top" | "center" | "bottom";
|
||||
style?: "plain" | "subtitle" | "hook" | "label";
|
||||
}
|
||||
```
|
||||
|
||||
### Output
|
||||
```ts
|
||||
{
|
||||
elementId: string;
|
||||
trackId: string;
|
||||
}
|
||||
```
|
||||
|
||||
### Requirements
|
||||
- MUST validate non-empty `text`.
|
||||
- MUST validate `start < end`.
|
||||
- MUST create a text element using existing timeline APIs.
|
||||
- MUST map `position` to a sensible default placement.
|
||||
- SHOULD provide simple style presets, but implementation may start with defaults.
|
||||
- MUST NOT generate full subtitles automatically; that is a flow using transcript/context + repeated `add_text` calls.
|
||||
|
||||
### Errors
|
||||
- Empty text: `{ error: "Text is required" }`.
|
||||
- Invalid range: `{ error: "Invalid time range" }`.
|
||||
|
||||
---
|
||||
|
||||
## 6. `update_text`
|
||||
|
||||
### Propósito
|
||||
Editar un texto existente en el timeline.
|
||||
|
||||
### Input
|
||||
```ts
|
||||
{
|
||||
trackId: string;
|
||||
elementId: string;
|
||||
text?: string;
|
||||
start?: number;
|
||||
end?: number;
|
||||
position?: "top" | "center" | "bottom";
|
||||
}
|
||||
```
|
||||
|
||||
### Output
|
||||
```ts
|
||||
{
|
||||
success: boolean;
|
||||
elementId: string;
|
||||
}
|
||||
```
|
||||
|
||||
### Requirements
|
||||
- MUST find an existing text element by `trackId` + `elementId`.
|
||||
- MUST only update provided fields.
|
||||
- MUST validate timing if `start`/`end` are provided.
|
||||
- MUST preserve undo/redo behavior if supported.
|
||||
|
||||
### Errors
|
||||
- Missing element: `{ error: "Text element not found" }`.
|
||||
- Wrong element type: `{ error: "Element is not text" }`.
|
||||
- Invalid range: `{ error: "Invalid time range" }`.
|
||||
|
||||
---
|
||||
|
||||
## 7. `add_media_to_timeline`
|
||||
|
||||
### Propósito
|
||||
Agregar un asset existente al timeline.
|
||||
|
||||
### Input
|
||||
```ts
|
||||
{
|
||||
assetId: string;
|
||||
startTime: number;
|
||||
trackType: "main" | "overlay" | "audio";
|
||||
}
|
||||
```
|
||||
|
||||
### Output
|
||||
```ts
|
||||
{
|
||||
elementId: string;
|
||||
trackId: string;
|
||||
}
|
||||
```
|
||||
|
||||
### Requirements
|
||||
- MUST resolve asset by `assetId`.
|
||||
- MUST validate asset type compatibility with `trackType`.
|
||||
- MUST insert the element using existing timeline APIs.
|
||||
- SHOULD choose a sensible track when one is not obvious, but first version requires explicit `trackType`.
|
||||
|
||||
### Errors
|
||||
- Asset not found: `{ error: "Asset not found" }`.
|
||||
- Invalid track type: `{ error: "Invalid track type for asset" }`.
|
||||
|
||||
---
|
||||
|
||||
## 8. `delete_element`
|
||||
|
||||
### Propósito
|
||||
Eliminar un elemento específico del timeline.
|
||||
|
||||
### Input
|
||||
```ts
|
||||
{
|
||||
trackId: string;
|
||||
elementId: string;
|
||||
}
|
||||
```
|
||||
|
||||
### Output
|
||||
```ts
|
||||
{
|
||||
success: boolean;
|
||||
}
|
||||
```
|
||||
|
||||
### Requirements
|
||||
- MUST find element by `trackId` + `elementId`.
|
||||
- MUST remove only that element.
|
||||
- MUST preserve undo/redo behavior if supported.
|
||||
|
||||
### Errors
|
||||
- Missing element: `{ error: "Element not found" }`.
|
||||
|
||||
---
|
||||
|
||||
## 9. `set_volume`
|
||||
|
||||
### Propósito
|
||||
Ajustar volumen de un elemento de audio o video.
|
||||
|
||||
### Input
|
||||
```ts
|
||||
{
|
||||
trackId: string;
|
||||
elementId: string;
|
||||
volume: number;
|
||||
}
|
||||
```
|
||||
|
||||
### Output
|
||||
```ts
|
||||
{
|
||||
success: boolean;
|
||||
volume: number;
|
||||
}
|
||||
```
|
||||
|
||||
### Requirements
|
||||
- MUST validate `volume` in range `0..1`.
|
||||
- MUST only apply to elements that support audio volume.
|
||||
- MUST preserve undo/redo behavior if supported.
|
||||
|
||||
### Errors
|
||||
- Invalid volume: `{ error: "Volume must be between 0 and 1" }`.
|
||||
- Unsupported element: `{ error: "Element does not support volume" }`.
|
||||
|
||||
---
|
||||
|
||||
## 10. `add_sticker`
|
||||
|
||||
### Propósito
|
||||
Agregar un sticker existente al timeline.
|
||||
|
||||
### Input
|
||||
```ts
|
||||
{
|
||||
stickerId: string;
|
||||
start: number;
|
||||
end: number;
|
||||
position: "top-left" | "top-right" | "center" | "bottom-left" | "bottom-right";
|
||||
}
|
||||
```
|
||||
|
||||
### Output
|
||||
```ts
|
||||
{
|
||||
elementId: string;
|
||||
trackId: string;
|
||||
}
|
||||
```
|
||||
|
||||
### Requirements
|
||||
- MUST resolve sticker by `stickerId`.
|
||||
- MUST validate `start < end`.
|
||||
- MUST create a timeline element using existing sticker/timeline APIs.
|
||||
- MUST map `position` to sensible coordinates.
|
||||
|
||||
### Errors
|
||||
- Sticker not found: `{ error: "Sticker not found" }`.
|
||||
- Invalid range: `{ error: "Invalid time range" }`.
|
||||
|
||||
---
|
||||
|
||||
## 11. `apply_effect`
|
||||
|
||||
### Propósito
|
||||
Aplicar un efecto existente a un clip. En el estado actual del repo, el efecto real disponible parece ser `blur`.
|
||||
|
||||
### Input
|
||||
```ts
|
||||
{
|
||||
trackId: string;
|
||||
elementId: string;
|
||||
effectType: "blur";
|
||||
params?: {
|
||||
intensity?: number;
|
||||
};
|
||||
}
|
||||
```
|
||||
|
||||
### Output
|
||||
```ts
|
||||
{
|
||||
effectId: string;
|
||||
elementId: string;
|
||||
}
|
||||
```
|
||||
|
||||
### Requirements
|
||||
- MUST validate the element is visual and supports effects.
|
||||
- MUST validate `effectType` exists in the effects registry.
|
||||
- MUST apply default params when `params` are omitted.
|
||||
- MUST validate `intensity` if provided.
|
||||
- MUST preserve undo/redo behavior if supported.
|
||||
|
||||
### Errors
|
||||
- Effect not found: `{ error: "Effect not found" }`.
|
||||
- Unsupported element: `{ error: "Element does not support effects" }`.
|
||||
- Invalid params: `{ error: "Invalid effect parameters" }`.
|
||||
|
||||
---
|
||||
|
||||
## Existing/secondary tool: `transcribe_video`
|
||||
|
||||
### Propósito
|
||||
Transcribir audio usando Whisper local. Es útil para captions y edición por texto, pero no debe ser la primitive principal de comprensión si `load_asset_context` con Gemini está disponible.
|
||||
|
||||
### Input
|
||||
```ts
|
||||
{
|
||||
assetId?: string;
|
||||
language?: string;
|
||||
modelId?: string;
|
||||
}
|
||||
```
|
||||
|
||||
### Estado
|
||||
- Implementada.
|
||||
- Debe mantenerse como secundaria.
|
||||
- Puede ser usada por flujos de subtítulos.
|
||||
|
|
@ -0,0 +1,324 @@
|
|||
# Agent Tools — Lista propuesta
|
||||
|
||||
Este documento resume las tools propuestas para el agente de NeuralCut. La idea es mantenerlas **primitivas, acotadas y componibles**, no crear una tool por cada feature de marketing.
|
||||
|
||||
## Principio
|
||||
|
||||
El agente debe combinar primitives simples:
|
||||
|
||||
- primero entiende el proyecto/assets,
|
||||
- luego decide qué acción hacer,
|
||||
- después ejecuta operaciones concretas del editor.
|
||||
|
||||
No queremos tools gigantes tipo “hazme un reel completo”. Eso debe ser un flujo del agente usando varias tools pequeñas.
|
||||
|
||||
---
|
||||
|
||||
## Tools de contexto y percepción
|
||||
|
||||
### `list_project_assets`
|
||||
|
||||
Lista assets conocidos del proyecto.
|
||||
|
||||
```ts
|
||||
{
|
||||
filter?: "all" | "used" | "unused";
|
||||
type?: "all" | "video" | "audio" | "image";
|
||||
}
|
||||
```
|
||||
|
||||
Uso:
|
||||
- saber qué archivos existen,
|
||||
- distinguir assets usados/no usados,
|
||||
- decidir qué asset cargar o editar.
|
||||
|
||||
---
|
||||
|
||||
### `list_timeline`
|
||||
|
||||
Lista el estado actual del timeline.
|
||||
|
||||
```ts
|
||||
{}
|
||||
```
|
||||
|
||||
Uso:
|
||||
- saber qué clips/textos/stickers/effects existen,
|
||||
- obtener `trackId` y `elementId`,
|
||||
- preparar operaciones como cortar, borrar o editar.
|
||||
|
||||
---
|
||||
|
||||
### `load_asset_context`
|
||||
|
||||
Carga un asset en el contexto multimodal del agente usando Gemini.
|
||||
|
||||
```ts
|
||||
{
|
||||
assetId: string;
|
||||
}
|
||||
```
|
||||
|
||||
Uso:
|
||||
- subir/procesar video, audio o imagen con Gemini,
|
||||
- dejar el asset disponible para razonamiento posterior,
|
||||
- cachear la referencia para no subirlo repetidamente.
|
||||
|
||||
> Esta reemplaza la idea anterior de `analyze_video({ question })` o `analyze_asset({ analysisType })`. La tool no “pregunta”; solo carga el asset al contexto.
|
||||
|
||||
---
|
||||
|
||||
### `get_asset_context` *(infra interna, no tool visible)*
|
||||
|
||||
Recupera información/cache de un asset ya cargado.
|
||||
|
||||
```ts
|
||||
{
|
||||
assetId: string;
|
||||
}
|
||||
```
|
||||
|
||||
Uso:
|
||||
- reutilizar análisis/contexto previo,
|
||||
- evitar re-upload,
|
||||
- permitir que el agente sepa si un asset ya está disponible para Gemini.
|
||||
|
||||
> Esta NO debería exponerse necesariamente como tool al LLM. Es más sano tratarla como infraestructura interna del agente/orquestador: `load_asset_context` carga y cachea; el orquestador/store puede consultar el cache e inyectar ese estado en el prompt/contexto sin hacer que el modelo llame una tool administrativa.
|
||||
|
||||
---
|
||||
|
||||
## Tools de edición primitivas
|
||||
|
||||
### `cut_segment`
|
||||
|
||||
Corta o remueve un rango de tiempo del timeline.
|
||||
|
||||
```ts
|
||||
{
|
||||
start: number;
|
||||
end: number;
|
||||
mode: "remove" | "keep";
|
||||
}
|
||||
```
|
||||
|
||||
Uso:
|
||||
- quitar silencios,
|
||||
- quitar errores,
|
||||
- recortar intro/outro,
|
||||
- construir highlights.
|
||||
|
||||
---
|
||||
|
||||
### `add_media_to_timeline`
|
||||
|
||||
Agrega un asset existente al timeline.
|
||||
|
||||
```ts
|
||||
{
|
||||
assetId: string;
|
||||
startTime: number;
|
||||
trackType: "main" | "overlay" | "audio";
|
||||
}
|
||||
```
|
||||
|
||||
Uso:
|
||||
- insertar clips,
|
||||
- agregar música,
|
||||
- poner b-roll,
|
||||
- agregar imágenes/logos.
|
||||
|
||||
---
|
||||
|
||||
### `delete_element`
|
||||
|
||||
Elimina un elemento específico del timeline.
|
||||
|
||||
```ts
|
||||
{
|
||||
trackId: string;
|
||||
elementId: string;
|
||||
}
|
||||
```
|
||||
|
||||
Uso:
|
||||
- borrar clips,
|
||||
- borrar textos,
|
||||
- borrar stickers,
|
||||
- borrar efectos standalone si aplica.
|
||||
|
||||
---
|
||||
|
||||
### `add_text`
|
||||
|
||||
Agrega texto visual al timeline.
|
||||
|
||||
```ts
|
||||
{
|
||||
text: string;
|
||||
start: number;
|
||||
end: number;
|
||||
position: "top" | "center" | "bottom";
|
||||
style?: "plain" | "subtitle" | "hook" | "label";
|
||||
}
|
||||
```
|
||||
|
||||
Uso:
|
||||
- subtítulos,
|
||||
- hooks,
|
||||
- títulos,
|
||||
- labels,
|
||||
- texto explicativo.
|
||||
|
||||
---
|
||||
|
||||
### `update_text`
|
||||
|
||||
Actualiza un texto existente.
|
||||
|
||||
```ts
|
||||
{
|
||||
trackId: string;
|
||||
elementId: string;
|
||||
text?: string;
|
||||
start?: number;
|
||||
end?: number;
|
||||
position?: "top" | "center" | "bottom";
|
||||
}
|
||||
```
|
||||
|
||||
Uso:
|
||||
- corregir texto,
|
||||
- cambiar timing,
|
||||
- mover captions/hooks.
|
||||
|
||||
---
|
||||
|
||||
### `add_sticker`
|
||||
|
||||
Agrega un sticker al timeline.
|
||||
|
||||
```ts
|
||||
{
|
||||
stickerId: string;
|
||||
start: number;
|
||||
end: number;
|
||||
position: "top-left" | "top-right" | "center" | "bottom-left" | "bottom-right";
|
||||
}
|
||||
```
|
||||
|
||||
Uso:
|
||||
- agregar elementos visuales simples,
|
||||
- reacciones,
|
||||
- énfasis gráfico.
|
||||
|
||||
---
|
||||
|
||||
### `set_volume`
|
||||
|
||||
Ajusta volumen de un elemento de audio/video.
|
||||
|
||||
```ts
|
||||
{
|
||||
trackId: string;
|
||||
elementId: string;
|
||||
volume: number; // 0–1
|
||||
}
|
||||
```
|
||||
|
||||
Uso:
|
||||
- bajar música,
|
||||
- subir voz,
|
||||
- mutear clip,
|
||||
- balance básico.
|
||||
|
||||
---
|
||||
|
||||
### `apply_effect`
|
||||
|
||||
Aplica un efecto existente a un clip.
|
||||
|
||||
```ts
|
||||
{
|
||||
trackId: string;
|
||||
elementId: string;
|
||||
effectType: "blur";
|
||||
params?: {
|
||||
intensity?: number;
|
||||
};
|
||||
}
|
||||
```
|
||||
|
||||
Uso:
|
||||
- aplicar blur.
|
||||
|
||||
> El repo actualmente tiene infraestructura de efectos, pero el efecto real registrado parece ser solo `blur`. Corrección de color/LUTs todavía no está disponible.
|
||||
|
||||
---
|
||||
|
||||
## Tools existentes o en standby
|
||||
|
||||
### `transcribe_video`
|
||||
|
||||
Transcribe audio de video/audio usando Whisper local.
|
||||
|
||||
```ts
|
||||
{
|
||||
assetId?: string;
|
||||
language?: string;
|
||||
modelId?: string;
|
||||
}
|
||||
```
|
||||
|
||||
Uso:
|
||||
- subtítulos,
|
||||
- captions,
|
||||
- búsqueda de frases exactas,
|
||||
- edición por texto.
|
||||
|
||||
Estado:
|
||||
- implementada,
|
||||
- útil como herramienta secundaria,
|
||||
- no debe ser la tool principal para entender el video si Gemini puede cargar el asset multimodalmente.
|
||||
|
||||
---
|
||||
|
||||
## Ideas descartadas o diferidas
|
||||
|
||||
### `analyze_video({ question })`
|
||||
|
||||
Descartada como primitive principal porque mete “chat dentro de la tool”. Mejor usar `load_asset_context` y dejar que el agente razone con el contexto cargado.
|
||||
|
||||
### `analyze_asset({ analysisType })`
|
||||
|
||||
Descartada porque `analysisType` introduce categorías artificiales. Gemini debe cargar el asset completo al contexto; el agente decide qué hacer después.
|
||||
|
||||
### `remove_silences`
|
||||
|
||||
Diferida/no prioritaria porque se puede expresar como flujo con `cut_segment`.
|
||||
|
||||
### `generate_subtitles`
|
||||
|
||||
Diferida porque se puede construir con `transcribe_video` + `add_text`.
|
||||
|
||||
### `color_correct` / `apply_lut`
|
||||
|
||||
Diferida porque el repo actual no parece tener corrección de color/LUTs implementados todavía.
|
||||
|
||||
---
|
||||
|
||||
## Orden recomendado
|
||||
|
||||
1. `list_project_assets`
|
||||
2. `list_timeline`
|
||||
3. `load_asset_context`
|
||||
4. `cut_segment`
|
||||
5. `add_text`
|
||||
6. `add_media_to_timeline`
|
||||
7. `delete_element`
|
||||
8. `set_volume`
|
||||
9. `add_sticker`
|
||||
10. `apply_effect`
|
||||
|
||||
Infra interna asociada:
|
||||
|
||||
- `get_asset_context` / cache lookup para `assetId -> Gemini fileUri/status/summary`, no necesariamente visible para el LLM.
|
||||
|
|
@ -0,0 +1,59 @@
|
|||
# Archive Report: gemini-adapter-fase-1
|
||||
|
||||
**Change**: gemini-adapter-fase-1
|
||||
**Archived**: 2026-04-24
|
||||
**Verification Verdict**: PASS — 14/14 tasks complete, 12/12 spec scenarios compliant, 48/48 tests passing, 0 TypeScript errors
|
||||
|
||||
## Scope Summary
|
||||
|
||||
This change adds **native Gemini provider support for normal chat and tool calling only**. Video upload/file understanding remains deferred to a future change. The `GeminiAdapter` implements the existing `ProviderAdapter` seam via `@google/generative-ai` SDK, supporting stateless `generateContent()` with system prompt mapping, tool schema conversion, and synthesized tool-call IDs. The OpenAI-compatible path is untouched.
|
||||
|
||||
## Engram Artifact Traceability
|
||||
|
||||
| Artifact | Observation ID |
|
||||
|----------|---------------|
|
||||
| Proposal | #1016 |
|
||||
| Spec (delta) | #1019 |
|
||||
| Design | #1018 |
|
||||
| Tasks | #1020 |
|
||||
| Apply Progress | #1021 |
|
||||
| Verify Report | #1024 |
|
||||
|
||||
## OpenSpec Archive
|
||||
|
||||
**Moved to**: `openspec/changes/archive/2026-04-24-gemini-adapter-fase-1/`
|
||||
|
||||
### Archive Contents
|
||||
- proposal.md ✅
|
||||
- specs/ (agent-session-shell delta) ✅
|
||||
- design.md ✅
|
||||
- tasks.md ✅ (14/14 tasks complete)
|
||||
- verify-report.md ✅
|
||||
- exploration.md ✅
|
||||
|
||||
## Specs Synced
|
||||
|
||||
| Domain | Action | Details |
|
||||
|--------|--------|---------|
|
||||
| agent-session-shell | Updated | 2 requirements modified (Provider Adapter Interface, Provider Configuration), 2 requirements added (Gemini Free-Form Chat, Gemini Tool Calling), 0 removed |
|
||||
|
||||
### Modified Requirements
|
||||
- **Provider Adapter Interface**: Expanded factory to support `openai-compatible` and `gemini`; added tool-call ID synthesis mandate; added Gemini adapter selection scenario
|
||||
- **Provider Configuration**: Added `LLM_PROVIDER` valid value constraint (`openai-compatible`, `gemini`); added Gemini-specific `LLM_BASE_URL` ignoring behavior; added unknown-provider and Gemini-config scenarios
|
||||
|
||||
### Added Requirements
|
||||
- **Gemini Free-Form Chat**: Plain text chat through Gemini `generateContent()` with system prompt mapped to `systemInstruction`
|
||||
- **Gemini Tool Calling**: Tool schema → Gemini function declaration conversion, function call response → `ToolCall[]` with synthesized IDs
|
||||
|
||||
## Source of Truth Updated
|
||||
- `openspec/specs/agent-session-shell/spec.md` — now reflects Gemini provider support
|
||||
|
||||
## Deferred Scope (explicitly out of scope)
|
||||
- Video/file upload or direct file understanding
|
||||
- `@asset` references, broader multimodal inputs, image/video reasoning
|
||||
- Streaming UX or adapter contract changes for streaming
|
||||
- Safety/grounding/thinking configuration beyond defaults
|
||||
|
||||
## SDD Cycle Complete
|
||||
The change has been fully explored, proposed, specified, designed, implemented, verified, and archived.
|
||||
Ready for the next change.
|
||||
|
|
@ -0,0 +1,155 @@
|
|||
# Design: Gemini Adapter — Phase 1
|
||||
|
||||
## Technical Approach
|
||||
|
||||
Implement a stateless `GeminiAdapter` class behind the existing `ProviderAdapter` seam, following the exact same pattern as `OpenAICompatibleAdapter`. The adapter converts internal `ChatMessage[]`/`ToolSchema[]` into Gemini's `generateContent()` wire format and parses responses back into `ProviderResponse`. A new `"gemini"` case in `createProvider()` activates it when `LLM_PROVIDER=gemini`. Zero changes to orchestrator, route, or client contracts.
|
||||
|
||||
## Architecture Decisions
|
||||
|
||||
### Decision: SDK choice
|
||||
|
||||
| Option | Tradeoff | Decision |
|
||||
|--------|----------|----------|
|
||||
| `@google/generative-ai` | Established, stable, v0.21+, `GoogleGenerativeAI` class, `generateContent()` stateless API | ✅ Selected |
|
||||
| `@google/genai` | Newer unified SDK, less battle-tested, API may still shift | ❌ Rejected |
|
||||
|
||||
**Rationale**: `@google/generative-ai` is the most widely adopted Gemini SDK for Node/TS. It supports `systemInstruction`, `functionDeclaration`, and stateless `generateContent()` — exactly what we need. Consistent with using the mature `openai` SDK for the existing adapter.
|
||||
|
||||
### Decision: Stateless generateContent vs stateful startChat
|
||||
|
||||
| Option | Tradeoff | Decision |
|
||||
|--------|----------|----------|
|
||||
| `model.generateContent(contents, config)` | Stateless, full history each call — matches current adapter contract | ✅ Selected |
|
||||
| `chat.sendMessage()` via `startChat()` | Stateful session, server-managed history | ❌ Rejected |
|
||||
|
||||
**Rationale**: Current architecture POSTs full history from client on every turn. Stateless `generateContent()` maps 1:1 to this pattern. No session management needed.
|
||||
|
||||
### Decision: Tool call ID synthesis
|
||||
|
||||
| Option | Tradeoff | Decision |
|
||||
|--------|----------|----------|
|
||||
| `crypto.randomUUID()` | Built-in, zero deps, unique | ✅ Selected |
|
||||
| `nanoid()` (already a dep) | Shorter IDs, also unique | ❌ Rejected |
|
||||
|
||||
**Rationale**: `crypto.randomUUID()` is universally available in Node/Bun, no import needed, and produces standard UUIDs matching OpenAI's `call_xxx` format semantically. Keeps the adapter self-contained.
|
||||
|
||||
### Decision: System prompt placement
|
||||
|
||||
| Option | Tradeoff | Decision |
|
||||
|--------|----------|----------|
|
||||
| `systemInstruction` on `GenerateContentRequest` | Gemini-native, separate from contents | ✅ Selected |
|
||||
| Prepend as `user` message | Works but dilutes system/user boundary | ❌ Rejected |
|
||||
|
||||
**Rationale**: Gemini SDK supports `systemInstruction` as a first-class config field. Using it keeps system prompt semantically correct and matches Gemini best practices.
|
||||
|
||||
## Data Flow
|
||||
|
||||
```
|
||||
Route (chat/route.ts)
|
||||
│
|
||||
├─ createProvider(config) → GeminiAdapter instance
|
||||
│
|
||||
└─ adapter.chat({ messages, systemPrompt, tools })
|
||||
│
|
||||
├─ toGeminiContents(messages) → Content[] (user/model/function)
|
||||
├─ toGeminiTools(tools) → FunctionDeclaration[]
|
||||
│
|
||||
└─ model.generateContent({ contents, tools, systemInstruction })
|
||||
│
|
||||
└─ fromGeminiResponse(response) → ProviderResponse
|
||||
│
|
||||
├─ content: candidate.content.parts[].text
|
||||
└─ toolCalls: candidate.content.parts[]
|
||||
.filter(functionCall)
|
||||
.map → { id: uuid(), name, args }
|
||||
```
|
||||
|
||||
### Message mapping
|
||||
|
||||
| Internal role | Gemini role | Notes |
|
||||
|---------------|-------------|-------|
|
||||
| `system` | `systemInstruction` (config) | Not in contents array |
|
||||
| `user` | `user` part | Direct text mapping |
|
||||
| `assistant` | `model` part | Includes `functionCall` parts for tool calls |
|
||||
| `tool_result` | `function` role | Uses `functionResponse: { name, response }` keyed by tool name (not ID) |
|
||||
|
||||
### Type mapping (internal → Gemini)
|
||||
|
||||
| Internal type | Gemini type |
|
||||
|---------------|-------------|
|
||||
| `"string"` | `SchemaType.STRING` |
|
||||
| `"number"` | `SchemaType.NUMBER` |
|
||||
| `"boolean"` | `SchemaType.BOOLEAN` |
|
||||
| `"object"` | `SchemaType.OBJECT` |
|
||||
|
||||
## File Changes
|
||||
|
||||
| File | Action | Description |
|
||||
|------|--------|-------------|
|
||||
| `apps/web/src/agent/providers/gemini.ts` | Create | GeminiAdapter + conversion helpers (`toGeminiContents`, `toGeminiTools`, `fromGeminiResponse`) |
|
||||
| `apps/web/src/agent/providers/index.ts` | Modify | Add `"gemini"` case to factory switch |
|
||||
| `apps/web/src/agent/providers/__tests__/gemini.test.ts` | Create | Conversion tests, error paths, tool call ID synthesis |
|
||||
| `apps/web/src/agent/providers/__tests__/index.test.ts` | Modify | Add factory test for `"gemini"` → `GeminiAdapter` |
|
||||
| `apps/web/package.json` | Modify | Add `@google/generative-ai` dependency |
|
||||
| `apps/web/.env.example` | Modify | Add Gemini example config lines |
|
||||
|
||||
## Interfaces / Contracts
|
||||
|
||||
No interface changes. The adapter implements the existing `ProviderAdapter`:
|
||||
|
||||
```ts
|
||||
// Reused as-is from providers/types.ts
|
||||
interface ProviderAdapter {
|
||||
chat(params: {
|
||||
messages: ChatMessage[];
|
||||
systemPrompt: string;
|
||||
tools: ToolSchema[];
|
||||
}): Promise<ProviderResponse>;
|
||||
}
|
||||
```
|
||||
|
||||
### Internal helper signatures (gemini.ts)
|
||||
|
||||
```ts
|
||||
function toGeminiContents(
|
||||
messages: ChatMessage[],
|
||||
): GenerateContentRequest["contents"];
|
||||
|
||||
function toGeminiTools(
|
||||
tools: ToolSchema[],
|
||||
): FunctionDeclaration[];
|
||||
|
||||
function fromGeminiResponse(
|
||||
response: GenerateContentResult,
|
||||
): ProviderResponse;
|
||||
```
|
||||
|
||||
### Tool call ID — name collision handling
|
||||
|
||||
When mapping `tool_result` back to Gemini's `functionResponse`, Gemini keys by function **name** not ID. If multiple calls to the same tool exist in history, we group them by name and send the last result per name. This is acceptable because:
|
||||
|
||||
1. Current tool set (`transcribe_video`) is single-invocation per turn
|
||||
2. Multi-call same-tool scenarios are deferred to a later change
|
||||
3. Orchestrator already handles the ID→name correlation client-side
|
||||
|
||||
## Testing Strategy
|
||||
|
||||
| Layer | What to Test | Approach |
|
||||
|-------|-------------|----------|
|
||||
| Unit | `toGeminiContents` mapping (user, assistant, tool_result, assistant+toolCalls) | Mock SDK, inspect call args |
|
||||
| Unit | `toGeminiTools` type casing (string→STRING, etc.) | Direct assertion on output |
|
||||
| Unit | `fromGeminiResponse` content extraction | Construct Gemini response shape, assert `ProviderResponse` |
|
||||
| Unit | Tool call ID synthesis (non-empty, unique) | Multiple calls produce different IDs |
|
||||
| Unit | Error: empty candidates → throws | Assert error message |
|
||||
| Unit | Error: SDK throws → propagates | Assert error bubbles up |
|
||||
| Unit | Factory: `"gemini"` → `GeminiAdapter` | Instance check |
|
||||
| Integration | Full `chat()` round-trip with mocked SDK | End-to-end within adapter |
|
||||
|
||||
## Migration / Rollout
|
||||
|
||||
No migration required. Additive change — setting `LLM_PROVIDER=gemini` activates the new adapter. Existing `openai-compatible` default is untouched.
|
||||
|
||||
## Open Questions
|
||||
|
||||
- [ ] Confirm `@google/generative-ai` latest stable version at impl time (currently ^0.21)
|
||||
- [ ] Verify Gemini model naming convention user will use (e.g. `gemini-2.0-flash` vs `models/gemini-2.0-flash`)
|
||||
|
|
@ -0,0 +1,65 @@
|
|||
# Proposal: Gemini Adapter — Phase 1
|
||||
|
||||
## Intent
|
||||
|
||||
Add Gemini as a native provider behind the existing provider-agnostic LLM seam so the agent can use Gemini for plain chat and tool calling without changing orchestrator or route contracts.
|
||||
|
||||
## Scope
|
||||
|
||||
### In Scope
|
||||
- Native `GeminiAdapter` implementing the current `ProviderAdapter`
|
||||
- Gemini free-form chat using existing stateless full-history flow
|
||||
- Gemini tool calling, with `transcribe_video` as the first real tool
|
||||
- Config support for `LLM_PROVIDER=gemini`, `LLM_API_KEY`, `LLM_MODEL`
|
||||
- Tests and env/docs updates while preserving the OpenAI-compatible path
|
||||
|
||||
### Out of Scope
|
||||
- Video/file upload or direct file understanding
|
||||
- `@asset` references, broader multimodal inputs, and image/video reasoning
|
||||
- Streaming UX or adapter contract changes for streaming
|
||||
- Safety/grounding/thinking configuration beyond defaults
|
||||
|
||||
## Capabilities
|
||||
|
||||
### New Capabilities
|
||||
- None
|
||||
|
||||
### Modified Capabilities
|
||||
- `agent-session-shell`: expand provider selection/configuration so the existing provider adapter contract also supports native Gemini chat and function calling
|
||||
|
||||
## Approach
|
||||
|
||||
Implement a native Gemini SDK-backed adapter and register it in the provider factory. Keep the adapter stateless by mapping full internal message history into Gemini `generateContent()` requests, converting tool schemas/results at the adapter boundary, and synthesizing tool call IDs so internal contracts stay unchanged.
|
||||
|
||||
## Affected Areas
|
||||
|
||||
| Area | Impact | Description |
|
||||
|------|--------|-------------|
|
||||
| `apps/web/src/agent/providers/gemini.ts` | New | Gemini request/response/tool conversion |
|
||||
| `apps/web/src/agent/providers/index.ts` | Modified | Factory registration for `gemini` |
|
||||
| `apps/web/src/agent/providers/__tests__/gemini.test.ts` | New | Adapter conversion/error tests |
|
||||
| `apps/web/src/agent/providers/__tests__/index.test.ts` | Modified | Factory coverage |
|
||||
| `apps/web/package.json` | Modified | Gemini SDK dependency |
|
||||
| `apps/web/.env.example` | Modified | Gemini env configuration |
|
||||
|
||||
## Risks
|
||||
|
||||
| Risk | Likelihood | Mitigation |
|
||||
|------|------------|------------|
|
||||
| Gemini has no tool call IDs | Med | Synthesize unique IDs inside adapter |
|
||||
| Gemini type casing differs from internal schema | Med | Centralize schema mapping tests |
|
||||
| Gemini system prompt placement differs | Low | Map system prompt explicitly to Gemini config |
|
||||
|
||||
## Rollback Plan
|
||||
|
||||
Remove the Gemini adapter, factory branch, dependency, and env docs; keep `openai-compatible` as the unchanged default path.
|
||||
|
||||
## Dependencies
|
||||
|
||||
- Gemini JS SDK selection to be verified during implementation
|
||||
|
||||
## Success Criteria
|
||||
|
||||
- [ ] `LLM_PROVIDER=gemini` resolves a native Gemini adapter without breaking `openai-compatible`
|
||||
- [ ] Gemini returns normal assistant text through the existing route/orchestrator flow
|
||||
- [ ] Gemini can emit structured tool calls and execute `transcribe_video` through the existing loop
|
||||
|
|
@ -0,0 +1,97 @@
|
|||
# Delta for agent-session-shell
|
||||
|
||||
## MODIFIED Requirements
|
||||
|
||||
### Requirement: Provider Adapter Interface
|
||||
|
||||
The system SHALL define a server-side provider adapter interface with a single method `chat(params: ProviderChatParams): Promise<ProviderChatResponse>`. `ProviderChatParams` MUST include `messages`, `systemPrompt`, and `toolSchemas` in a provider-agnostic internal format. `ProviderChatResponse` MUST include `content: string` and `toolCalls?: ProviderToolCall[]` where `ProviderToolCall { id, name, args }`. Each concrete adapter converts internal tool schemas to its own wire format. The factory MUST support at least `openai-compatible` and `gemini` providers. Adapters whose native API does not assign IDs to tool calls MUST synthesize unique IDs so every returned `ToolCall.id` is a non-empty string.
|
||||
|
||||
(Previously: Phase 1 MAY ship exactly one adapter (OpenAI-compatible) as long as the interface contract remains provider-agnostic.)
|
||||
|
||||
#### Scenario: Adapter translates tool schemas to wire format
|
||||
|
||||
- GIVEN `transcribe_video` has internal parameters `[{ key: "assetId", type: "string", required: false }]`
|
||||
- WHEN the adapter receives tool schemas before calling the provider
|
||||
- THEN it converts them to its provider-specific wire format
|
||||
|
||||
#### Scenario: Adapter selected by config — OpenAI-compatible
|
||||
|
||||
- GIVEN `LLM_PROVIDER` is set to `"openai-compatible"`
|
||||
- WHEN the route resolves the adapter
|
||||
- THEN the OpenAI-compatible adapter is returned
|
||||
|
||||
#### Scenario: Adapter selected by config — Gemini
|
||||
|
||||
- GIVEN `LLM_PROVIDER` is set to `"gemini"`
|
||||
- WHEN the route resolves the adapter
|
||||
- THEN the Gemini adapter is returned
|
||||
|
||||
#### Scenario: Gemini adapter synthesizes tool-call IDs
|
||||
|
||||
- GIVEN the Gemini API returns a function call response with no call ID
|
||||
- WHEN the Gemini adapter maps the response to `ProviderResponse`
|
||||
- THEN each `ToolCall` in `toolCalls` has a non-empty `id` field generated by the adapter
|
||||
|
||||
### Requirement: Provider Configuration
|
||||
|
||||
Server-only environment variables `LLM_PROVIDER` (required), `LLM_MODEL` (required), `LLM_API_KEY` (required), and `LLM_BASE_URL` (optional) MUST determine which adapter and model the route uses. `LLM_PROVIDER` MUST accept at least `"openai-compatible"` and `"gemini"`. When `LLM_PROVIDER` is `"gemini"`, `LLM_BASE_URL` MUST be ignored. Configuration MUST be read server-side only — never exposed to the client.
|
||||
|
||||
(Previously: Provider config listed env vars but did not constrain valid provider values or describe Gemini-specific behavior.)
|
||||
|
||||
#### Scenario: Missing required configuration
|
||||
|
||||
- GIVEN `LLM_API_KEY` is not set in the environment
|
||||
- WHEN the route attempts to initialize the provider
|
||||
- THEN the call returns 502 with a missing-configuration error
|
||||
|
||||
#### Scenario: Unknown provider
|
||||
|
||||
- GIVEN `LLM_PROVIDER` is set to `"unknown-provider"`
|
||||
- WHEN the route resolves the adapter
|
||||
- THEN the factory throws `"Unknown LLM provider: unknown-provider"`
|
||||
|
||||
#### Scenario: Gemini config ignores base URL
|
||||
|
||||
- GIVEN `LLM_PROVIDER` is `"gemini"` and `LLM_BASE_URL` is set
|
||||
- WHEN the Gemini adapter is created
|
||||
- THEN `LLM_BASE_URL` is not used in the API call
|
||||
|
||||
## ADDED Requirements
|
||||
|
||||
### Requirement: Gemini Free-Form Chat
|
||||
|
||||
When the Gemini adapter receives a request with no tool schemas (or tools array is empty), it MUST send messages to the Gemini `generateContent` API and return a `ProviderResponse` with `content` set to the assistant text and `toolCalls` undefined.
|
||||
|
||||
#### Scenario: Plain chat through Gemini
|
||||
|
||||
- GIVEN `LLM_PROVIDER` is `"gemini"` and messages contain `[{ role: "user", content: "Hello" }]`
|
||||
- WHEN the adapter calls Gemini and receives a text-only response
|
||||
- THEN `ProviderResponse.content` is the assistant text and `toolCalls` is undefined
|
||||
|
||||
#### Scenario: System prompt mapping
|
||||
|
||||
- GIVEN the adapter receives `systemPrompt: "You are a video editor"`
|
||||
- WHEN it builds the Gemini request
|
||||
- THEN the system prompt is mapped to Gemini's system instruction field, not as a user message
|
||||
|
||||
### Requirement: Gemini Tool Calling
|
||||
|
||||
When the Gemini adapter receives a request with tool schemas, it MUST convert internal `ToolSchema[]` to Gemini function declarations, call `generateContent` with declarations attached, and if Gemini returns function call parts, map each to a `ToolCall { id, name, args }` with a synthesized non-empty `id`.
|
||||
|
||||
#### Scenario: Gemini returns function calls
|
||||
|
||||
- GIVEN `transcribe_video` is in tool schemas and Gemini responds with a function call part `{ name: "transcribe_video", args: {} }`
|
||||
- WHEN the adapter maps the response
|
||||
- THEN `toolCalls` contains one `ToolCall` with `name: "transcribe_video"`, `args: {}`, and a synthesized `id`
|
||||
|
||||
#### Scenario: Gemini returns text despite tools
|
||||
|
||||
- GIVEN tool schemas are provided but Gemini responds with only text
|
||||
- WHEN the adapter maps the response
|
||||
- THEN `toolCalls` is undefined and `content` holds the text
|
||||
|
||||
#### Scenario: Tool schema type mapping
|
||||
|
||||
- GIVEN an internal tool schema has parameters `[{ key: "assetId", type: "string", required: true }]`
|
||||
- WHEN the adapter converts to Gemini function declarations
|
||||
- THEN the parameter appears in `FunctionDeclaration.parameters.properties` with type `"string"`
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
# Tasks: Gemini Adapter — Phase 1
|
||||
|
||||
## Phase 1: Foundation
|
||||
|
||||
- [x] 1.1 Add `@google/generative-ai` dependency to `apps/web/package.json` (pin to latest stable `^0.21` or newer)
|
||||
- [x] 1.2 Add Gemini env config lines to `apps/web/.env.example`: `LLM_PROVIDER=gemini`, `LLM_API_KEY`, `LLM_MODEL=gemini-2.0-flash`, comment noting `LLM_BASE_URL` is ignored for gemini
|
||||
|
||||
## Phase 2: Core Adapter
|
||||
|
||||
- [x] 2.1 Create `apps/web/src/agent/providers/gemini.ts` — import `GoogleGenerativeAI` from `@google/generative-ai`, `SchemaType` for type mapping
|
||||
- [x] 2.2 Implement `toGeminiContents(messages: ChatMessage[])` — map `user→user`, `assistant→model` (with `functionCall` parts), `tool_result→function` role with `functionResponse`; skip `system` role (handled separately)
|
||||
- [x] 2.3 Implement `toGeminiTools(tools: ToolSchema[])` — convert each `ToolSchema` to a `FunctionDeclaration` with `SchemaType` property types (`string→STRING`, `number→NUMBER`, `boolean→BOOLEAN`, `object→OBJECT`); return empty array when no tools
|
||||
- [x] 2.4 Implement `fromGeminiResponse(response)` — extract `candidate.content.parts`, pull text into `content`, filter `functionCall` parts into `toolCalls[]` with `id: crypto.randomUUID()`, `name`, `args`; throw on empty candidates
|
||||
- [x] 2.5 Implement `GeminiAdapter` class — constructor takes `ProviderConfig`, creates `GoogleGenerativeAI` + `getGenerativeModel({ model, systemInstruction })`; `chat()` calls `model.generateContent({ contents, tools })` then returns `fromGeminiResponse()`; ignore `config.baseUrl`
|
||||
|
||||
## Phase 3: Factory Integration
|
||||
|
||||
- [x] 3.1 Add `"gemini"` case to `createProvider()` switch in `apps/web/src/agent/providers/index.ts` — import and return `new GeminiAdapter(config)`
|
||||
- [x] 3.2 Verify existing `openai-compatible` factory case is untouched and all existing tests still pass
|
||||
|
||||
## Phase 4: Tests
|
||||
|
||||
- [x] 4.1 Create `apps/web/src/agent/providers/__tests__/gemini.test.ts` — test `toGeminiContents`: user message, assistant with toolCalls, tool_result mapping, system message excluded from contents
|
||||
- [x] 4.2 Test `toGeminiTools`: single tool with parameters, type casing (`string→STRING` etc.), empty tools returns empty array
|
||||
- [x] 4.3 Test `fromGeminiResponse`: text-only response → `{ content, toolCalls: undefined }`; function-call response → `toolCalls` with non-empty synthesized IDs; two calls produce different IDs
|
||||
- [x] 4.4 Test error paths: empty candidates → throws; SDK error → propagates
|
||||
- [x] 4.5 Add factory test to `apps/web/src/agent/providers/__tests__/index.test.ts`: `"gemini"` config → `GeminiAdapter` instance; unknown provider still throws
|
||||
- [x] 4.6 Test full `chat()` round-trip with mocked `GoogleGenerativeAI` — verify `generateContent` called with correct contents, tools, and systemInstruction
|
||||
|
|
@ -0,0 +1,116 @@
|
|||
# Verification Report
|
||||
|
||||
**Change**: gemini-adapter-fase-1
|
||||
**Version**: N/A
|
||||
**Mode**: Standard
|
||||
|
||||
---
|
||||
|
||||
### Completeness
|
||||
|
||||
| Metric | Value |
|
||||
|--------|-------|
|
||||
| Tasks total | 14 |
|
||||
| Tasks complete | 14 |
|
||||
| Tasks incomplete | 0 |
|
||||
|
||||
All tasks in `openspec/changes/gemini-adapter-fase-1/tasks.md` are marked complete, including the post-verify fixes.
|
||||
|
||||
---
|
||||
|
||||
### Build & Tests Execution
|
||||
|
||||
**Build**: ➖ Skipped
|
||||
|
||||
```text
|
||||
No explicit `rules.verify.build_command` is configured in `openspec/config.yaml`.
|
||||
Project instructions also say never build after changes, so verification used TypeScript type-checking instead of `next build`.
|
||||
```
|
||||
|
||||
**Type Check**: ✅ Passed
|
||||
|
||||
```text
|
||||
Command: bunx tsc --noEmit
|
||||
|
||||
(no output)
|
||||
```
|
||||
|
||||
**Tests**: ✅ 48 passed / ❌ 0 failed / ⚠️ 0 skipped
|
||||
|
||||
```text
|
||||
Command: bun test src/agent/providers/__tests__ src/app/api/agent/chat/__tests__/route.test.ts
|
||||
|
||||
bun test v1.3.13
|
||||
48 pass
|
||||
0 fail
|
||||
106 expect() calls
|
||||
Ran 48 tests across 4 files.
|
||||
```
|
||||
|
||||
**Coverage**: ➖ Not available
|
||||
|
||||
---
|
||||
|
||||
### Spec Compliance Matrix
|
||||
|
||||
| Requirement | Scenario | Test | Result |
|
||||
|-------------|----------|------|--------|
|
||||
| Provider Adapter Interface | Adapter translates tool schemas to wire format | `src/agent/providers/__tests__/gemini.test.ts > converts a single tool with parameters` + `maps type strings to Gemini SchemaType values` | ✅ COMPLIANT |
|
||||
| Provider Adapter Interface | Adapter selected by config — OpenAI-compatible | `src/agent/providers/__tests__/index.test.ts > returns OpenAICompatibleAdapter for openai-compatible provider` | ✅ COMPLIANT |
|
||||
| Provider Adapter Interface | Adapter selected by config — Gemini | `src/agent/providers/__tests__/index.test.ts > returns GeminiAdapter for gemini provider` | ✅ COMPLIANT |
|
||||
| Provider Adapter Interface | Gemini adapter synthesizes tool-call IDs | `src/agent/providers/__tests__/gemini.test.ts > maps function-call response to toolCalls with non-empty synthesized IDs` | ✅ COMPLIANT |
|
||||
| Provider Configuration | Missing required configuration | `src/app/api/agent/chat/__tests__/route.test.ts > returns 502 when LLM_API_KEY is missing` | ✅ COMPLIANT |
|
||||
| Provider Configuration | Unknown provider | `src/agent/providers/__tests__/index.test.ts > throws Error for unknown provider` | ✅ COMPLIANT |
|
||||
| Provider Configuration | Gemini config ignores base URL | `src/agent/providers/__tests__/gemini.test.ts > ignores baseUrl in config — Gemini still works` | ✅ COMPLIANT |
|
||||
| Gemini Free-Form Chat | Plain chat through Gemini | `src/agent/providers/__tests__/gemini.test.ts > calls generateContent with correct contents and no tools` | ✅ COMPLIANT |
|
||||
| Gemini Free-Form Chat | System prompt mapping | `src/agent/providers/__tests__/gemini.test.ts > calls generateContent with correct contents and no tools` | ✅ COMPLIANT |
|
||||
| Gemini Tool Calling | Gemini returns function calls | `src/agent/providers/__tests__/gemini.test.ts > calls generateContent with tools and systemInstruction` | ✅ COMPLIANT |
|
||||
| Gemini Tool Calling | Gemini returns text despite tools | `src/agent/providers/__tests__/gemini.test.ts > returns text when tools are provided but Gemini responds with text only` | ✅ COMPLIANT |
|
||||
| Gemini Tool Calling | Tool schema type mapping | `src/agent/providers/__tests__/gemini.test.ts > maps type strings to Gemini SchemaType values` | ✅ COMPLIANT |
|
||||
|
||||
**Compliance summary**: 12/12 scenarios compliant
|
||||
|
||||
---
|
||||
|
||||
### Correctness (Static — Structural Evidence)
|
||||
|
||||
| Requirement | Status | Notes |
|
||||
|------------|--------|-------|
|
||||
| Provider Adapter Interface | ✅ Implemented | `apps/web/src/agent/providers/types.ts` keeps the provider-agnostic `chat()` seam, `index.ts` supports both providers, and `gemini.ts` synthesizes tool-call IDs with `crypto.randomUUID()`. |
|
||||
| Provider Configuration | ✅ Implemented | `apps/web/src/app/api/agent/chat/route.ts` reads `LLM_PROVIDER`, `LLM_API_KEY`, `LLM_MODEL`, and `LLM_BASE_URL` server-side only; `GeminiAdapter` intentionally ignores `baseUrl`. |
|
||||
| Gemini Free-Form Chat | ✅ Implemented | `GeminiAdapter.chat()` maps messages into Gemini contents, sets `systemInstruction` when present, and returns text-only responses without `toolCalls`. |
|
||||
| Gemini Tool Calling | ✅ Implemented | `toGeminiTools()` converts internal tool schemas to Gemini declarations and `fromGeminiResponse()` maps function-call parts back into provider `toolCalls`. |
|
||||
| Explicit non-goals remain deferred | ✅ Implemented | The touched files remain additive only; no streaming, multimodal upload, `@asset`, or contract changes were introduced. |
|
||||
|
||||
---
|
||||
|
||||
### Coherence (Design)
|
||||
|
||||
| Decision | Followed? | Notes |
|
||||
|----------|-----------|-------|
|
||||
| Use `@google/generative-ai` SDK | ✅ Yes | Added in `apps/web/package.json`; adapter imports `GoogleGenerativeAI` and `SchemaType`. |
|
||||
| Preserve stateless provider seam | ✅ Yes | `GeminiAdapter.chat()` sends full history through one `generateContent()` call with no session state. |
|
||||
| Synthesize tool-call IDs inside adapter | ✅ Yes | `fromGeminiResponse()` generates IDs for Gemini function calls. |
|
||||
| Keep OpenAI-compatible path intact | ✅ Yes | `createProvider()` still returns `OpenAICompatibleAdapter` unchanged for the original provider. |
|
||||
| Map system prompt via Gemini-native field | ✅ Yes | `GeminiAdapter.chat()` sets `request.systemInstruction` instead of injecting a fake user message. |
|
||||
|
||||
---
|
||||
|
||||
### Issues Found
|
||||
|
||||
**CRITICAL** (must fix before archive):
|
||||
None
|
||||
|
||||
**WARNING** (should fix):
|
||||
None
|
||||
|
||||
**SUGGESTION** (nice to have):
|
||||
None
|
||||
|
||||
---
|
||||
|
||||
### Verdict
|
||||
|
||||
PASS
|
||||
|
||||
All planned tasks are complete, the changed slice now type-checks, targeted runtime tests pass, and every spec scenario has passing behavioral evidence.
|
||||
|
|
@ -0,0 +1,292 @@
|
|||
# Exploration: `analyze_video` — Gemini Video Understanding (Phase 1)
|
||||
|
||||
## Current State
|
||||
|
||||
### Shipped infrastructure
|
||||
|
||||
Three completed changes underpin this work:
|
||||
|
||||
| Change | What shipped |
|
||||
|--------|-------------|
|
||||
| `infra-habilitadora-fase-1` | Full agent pipeline: orchestrator (client-side), tool registry, mock tool, context adapter, system prompt, chat UI |
|
||||
| `gemini-adapter-fase-1` | `GeminiAdapter` behind `ProviderAdapter` seam — text-only `chat()` via `@google/generative-ai` v0.24.0 |
|
||||
| `transcribe-video-fase-1` | `transcribe_video` tool — Whisper-based, client-side execution, returns `TranscriptionToolResult` with segments + timestamps. Fully implemented and verified. |
|
||||
|
||||
### How the agent pipeline works today
|
||||
|
||||
```
|
||||
User message → ChatPanel → orchestrator.run(messages, context)
|
||||
→ POST /api/agent/chat (server)
|
||||
→ createProvider(config) → GeminiAdapter.chat({ messages, systemPrompt, tools })
|
||||
→ model.generateContent({ contents, systemInstruction, tools })
|
||||
→ { content, toolCalls? }
|
||||
→ resolveToolCalls(toolCalls, context) (client)
|
||||
→ toolRegistry.get(name).execute(args, context)
|
||||
→ ToolResult → appended as tool_result message
|
||||
→ Loop back to POST (max 8 iterations)
|
||||
```
|
||||
|
||||
**Key architectural facts**:
|
||||
- **Server route** (`route.ts`): Stateless, holds `LLM_API_KEY`, creates adapter, calls `chat()`. Exposes tool schemas to the LLM but NEVER executes tools.
|
||||
- **Client orchestrator**: Holds `AgentContext`, resolves tool calls, executes tools in-browser where `EditorCore` is accessible.
|
||||
- **Tools are client-side only**: `ToolDefinition.execute()` runs in the browser, has access to `EditorContextAdapter.resolveAssetFile()` and `EditorCore`.
|
||||
- **Tool schemas are duplicated**: `route.ts` has `providerToolDefs` (stub `execute`) for the LLM; `toolRegistry` has real definitions for execution.
|
||||
|
||||
### Gemini SDK already installed
|
||||
|
||||
`@google/generative-ai` v0.24.0 is in `apps/web/package.json`. The `GeminiAdapter` uses `GoogleGenerativeAI` → `getGenerativeModel()` → `generateContent()`. The SDK also exports:
|
||||
|
||||
- **`GoogleAIFileManager`** from `@google/generative-ai/server` — server-side file upload/management
|
||||
- **`FileState`** enum — `PROCESSING`, `ACTIVE`, `FAILED`
|
||||
- **`FileDataPart`** — `{ fileData: { fileUri, mimeType } }` content part for `generateContent()`
|
||||
|
||||
### EditorContextAdapter
|
||||
|
||||
Already provides the sanctioned path for tools to obtain a `File`:
|
||||
|
||||
```ts
|
||||
EditorContextAdapter.resolveAssetFile(assetId?: string): File | null
|
||||
EditorContextAdapter.getAssetHasAudio(assetId: string): boolean | undefined
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Affected Areas
|
||||
|
||||
| File | Why affected |
|
||||
|------|-------------|
|
||||
| `apps/web/src/agent/tools/analyze-video.tool.ts` | **NEW** — `analyze_video` tool definition (client-side) |
|
||||
| `apps/web/src/app/api/agent/analyze-video/route.ts` | **NEW** — server route: upload video to Gemini File API + poll + query |
|
||||
| `apps/web/src/agent/orchestrator.ts` | Add side-effect import of new tool |
|
||||
| `apps/web/src/app/api/agent/chat/route.ts` | Add `analyze_video` to `providerToolDefs`, optionally remove/demote `transcribe_video` |
|
||||
| `apps/web/src/agent/system-prompt.ts` | May update guidance to prefer `analyze_video` |
|
||||
| `apps/web/src/components/editor/panels/chat/message-bubble.tsx` | May add rendering for `AnalyzeVideoResult` shape |
|
||||
|
||||
**Unchanged** (confirmed safe):
|
||||
- `providers/gemini.ts` — adapter handles text chat; video goes through a separate route using `GoogleAIFileManager`
|
||||
- `providers/types.ts` — `ProviderAdapter` contract unchanged
|
||||
- `agent/types.ts` — `AgentContext`, `ToolDefinition` unchanged
|
||||
- `agent/context.ts` — `resolveAssetFile()` already returns `File`
|
||||
- `tools/transcribe-video.tool.ts` — stays registered, just de-emphasized in tool list
|
||||
|
||||
---
|
||||
|
||||
## Critical Constraint: File API Is Server-Side Only
|
||||
|
||||
The Gemini Files API requires the API key and can only be called server-side. The current architecture has tools executing **client-side**. This means `analyze_video` cannot be a pure client-side tool like `transcribe_video`. It needs a **server route** to handle the upload-to-Gemini flow.
|
||||
|
||||
### Proposed flow
|
||||
|
||||
```
|
||||
LLM decides: analyze_video({ assetId: "v1", question: "What happens in this video?" })
|
||||
→ Client orchestrator resolves tool call
|
||||
→ tool.execute(args, context)
|
||||
→ EditorContextAdapter.resolveAssetFile("v1") → File blob
|
||||
→ POST /api/agent/analyze-video (FormData: video file + question + assetId)
|
||||
→ Server: GoogleAIFileManager.uploadFile(buffer, { mimeType, displayName })
|
||||
→ Server: poll file.state until ACTIVE (or timeout)
|
||||
→ Server: model.generateContent([ question, { fileData: { fileUri, mimeType } }])
|
||||
→ Server: return { answer, duration, ... }
|
||||
→ Tool returns structured AnalyzeVideoResult
|
||||
```
|
||||
|
||||
### Why not inline base64?
|
||||
|
||||
Gemini supports `InlineDataPart` with base64-encoded video, but:
|
||||
- **Size limit**: Inline data is practical only for files under ~20MB raw (~27MB base64). Real video projects regularly exceed this.
|
||||
- **No reuse**: The file is sent every call. The File API gives a `fileUri` that persists (up to 48h free / 7d paid), enabling reuse.
|
||||
- **Processing state**: The File API handles video decoding, audio extraction, and indexing server-side. Inline data forces the model to process on every call.
|
||||
|
||||
**Verdict**: Use the File API for production readiness. Phase 1 uses the File API exclusively.
|
||||
|
||||
---
|
||||
|
||||
## Approaches
|
||||
|
||||
### 1. Dedicated `/api/agent/analyze-video` Route (Recommended)
|
||||
|
||||
Create a new server route that handles the full upload→poll→query flow. The client-side tool sends the video file as FormData, the server does everything Gemini-side, and returns a structured answer.
|
||||
|
||||
- **Pros**:
|
||||
- Clean separation: upload+analyze logic is entirely server-side (no API key leakage)
|
||||
- Reuses existing `GeminiAdapter` pattern (same SDK, same config)
|
||||
- The tool on the client side is thin — just FormData construction and response parsing
|
||||
- Natural place for future caching (fileUri by assetId)
|
||||
- Can report processing progress via streaming in Phase 2
|
||||
- **Cons**:
|
||||
- New route to maintain
|
||||
- Large FormData upload adds latency (video files can be hundreds of MB)
|
||||
- Polling loop blocks the server request (video processing can take 30s-5min for long videos)
|
||||
- **Effort**: Medium
|
||||
|
||||
### 2. Extend `/api/agent/chat` Route for Multimodal
|
||||
|
||||
Add multimodal content support to the existing chat route. The client sends video content parts alongside text messages. The server handles upload transparently within the adapter.
|
||||
|
||||
- **Pros**:
|
||||
- Single route, single contract
|
||||
- More "correct" architecturally — the chat route becomes multimodal
|
||||
- **Cons**:
|
||||
- **Major refactor of the adapter contract** — `ProviderAdapter.chat()` currently takes text-only `ChatMessage[]`. Adding binary content parts requires interface changes to `ChatMessage`, `ProviderAdapter`, and both adapters.
|
||||
- Breaks the clean text-only abstraction that `route.ts` depends on
|
||||
- The upload/poll cycle doesn't fit neatly into the synchronous `chat()` call
|
||||
- Cross-contaminates text chat and file upload concerns
|
||||
- **Effort**: High
|
||||
|
||||
### 3. Upload-Then-Chat Two-Step
|
||||
|
||||
Split into two routes: `POST /api/agent/upload-video` (upload + poll → return fileUri) and `POST /api/agent/chat` (modified to accept fileUri references). The client tool orchestrates both calls.
|
||||
|
||||
- **Pros**:
|
||||
- Separation of upload and query concerns
|
||||
- fileUri can be cached client-side and reused across queries without re-uploading
|
||||
- **Cons**:
|
||||
- Two round trips for a single tool call (upload, then query)
|
||||
- Client must manage fileUri lifecycle (when to re-upload)
|
||||
- More complex tool execution logic
|
||||
- The chat route still needs multimodal content support (same con as Approach 2)
|
||||
- **Effort**: Medium-High
|
||||
|
||||
---
|
||||
|
||||
## Recommendation
|
||||
|
||||
**Approach 1: Dedicated `/api/agent/analyze-video` route.**
|
||||
|
||||
Rationale:
|
||||
1. **Minimal contract changes**: No changes to `ProviderAdapter`, `ChatMessage`, or the existing chat route. The new route is self-contained.
|
||||
2. **API key safety**: All Gemini interaction happens server-side. The client tool only sends a FormData blob.
|
||||
3. **Matches the existing pattern**: Just as `transcribe_video` is a client tool that calls services, `analyze_video` is a client tool that calls a server endpoint. The server endpoint is the Gemini File API equivalent of the transcription service.
|
||||
4. **Cache-friendly**: The route can return the `fileUri` in the response, and in Phase 2, we add a KV cache (Upstash Redis is already installed) keyed by `assetId` to skip re-uploads.
|
||||
5. **Thin scope**: The tool is `analyze_video({ assetId?, question })` → answer. No editing actions, no timeline mutations.
|
||||
|
||||
---
|
||||
|
||||
## Recommended Tool Design
|
||||
|
||||
```typescript
|
||||
// analyze-video.tool.ts
|
||||
|
||||
interface AnalyzeVideoArgs {
|
||||
assetId?: string;
|
||||
question: string;
|
||||
}
|
||||
|
||||
interface AnalyzeVideoResult {
|
||||
assetName: string;
|
||||
question: string;
|
||||
answer: string;
|
||||
duration: number;
|
||||
}
|
||||
|
||||
// OR error:
|
||||
interface AnalyzeVideoError {
|
||||
error: string;
|
||||
}
|
||||
```
|
||||
|
||||
### Server route contract
|
||||
|
||||
```
|
||||
POST /api/agent/analyze-video
|
||||
Content-Type: multipart/form-data
|
||||
|
||||
Fields:
|
||||
- video: File (the video blob)
|
||||
- question: string
|
||||
- assetId: string (for display/logging)
|
||||
- mimeType: string (e.g., "video/mp4")
|
||||
|
||||
Response:
|
||||
{ answer: string, duration: number }
|
||||
OR { error: string }
|
||||
```
|
||||
|
||||
### Processing on the server
|
||||
|
||||
```
|
||||
1. Parse FormData → extract video buffer + question + mimeType
|
||||
2. GoogleAIFileManager.uploadFile(buffer, { mimeType, displayName: assetId })
|
||||
3. Poll file.state every 5s until ACTIVE (max 120s timeout)
|
||||
4. model.generateContent([
|
||||
{ text: question },
|
||||
{ fileData: { fileUri: file.uri, mimeType: file.mimeType } }
|
||||
])
|
||||
5. Return { answer: response.text(), duration }
|
||||
```
|
||||
|
||||
### Asset resolution
|
||||
|
||||
Reuses the same pattern as `transcribe_video`:
|
||||
1. If `assetId` provided → resolve that specific asset
|
||||
2. If only one video/audio asset → use it
|
||||
3. If multiple → return error asking user to specify
|
||||
|
||||
### De-emphasizing `transcribe_video`
|
||||
|
||||
In `route.ts`, replace `transcribe_video` in `providerToolDefs` with `analyze_video`. The tool stays registered in the client-side registry (available if needed) but the LLM no longer sees it in its tool list.
|
||||
|
||||
```typescript
|
||||
// route.ts — updated providerToolDefs
|
||||
const providerToolDefs: ToolDefinition[] = [
|
||||
{
|
||||
name: "analyze_video",
|
||||
description: "Analyzes a video using Gemini multimodal understanding. Can answer questions about visual content, audio, speech, and overall composition. Returns a structured answer.",
|
||||
parameters: [
|
||||
{ key: "assetId", type: "string", required: false },
|
||||
{ key: "question", type: "string", required: true },
|
||||
],
|
||||
execute: async () => ({}), // stub — never called server-side
|
||||
},
|
||||
];
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Caching Considerations (Phase 1 vs Phase 2)
|
||||
|
||||
### Phase 1 (this change): No caching
|
||||
- Every `analyze_video` call uploads the video fresh
|
||||
- Simple, predictable, no state management
|
||||
- Downside: redundant uploads for repeated questions about the same video
|
||||
|
||||
### Phase 2: File URI cache via Upstash Redis
|
||||
- `Upstash Redis` is already a project dependency (`@upstash/redis`, `@upstash/ratelimit`)
|
||||
- Key: `gemini-file:{sha256hash}` or `gemini-file:{assetId}:{fileSize}`
|
||||
- Value: `{ fileUri, mimeType, expiresAt }`
|
||||
- On each call: check cache → if valid, skip upload → use cached fileUri
|
||||
- Need to handle: file mutation (re-upload if asset changes), TTL alignment with Gemini's expiration
|
||||
|
||||
---
|
||||
|
||||
## Risks
|
||||
|
||||
1. **Upload size and latency**: Video files can be 100MB+. Uploading to the Gemini File API from a server route adds significant latency. The Next.js route has a default body size limit that may need configuration. **Mitigation**: Phase 1 sets a reasonable max file size (e.g., 500MB). Configure `bodySizeLimit` in the Next.js route config.
|
||||
|
||||
2. **Processing polling blocks the request**: After upload, Gemini needs to process the video (decode frames, extract audio, index). For long videos this can take minutes. The server route holds open during polling. **Mitigation**: Set a polling timeout (e.g., 120s). Return a clear error if processing exceeds the limit. Phase 2 can add async processing with a status check endpoint.
|
||||
|
||||
3. **SDK uploadFile takes file path, not Buffer**: The `GoogleAIFileManager.uploadFile()` API officially takes a `filePath: string`. The docs mention `Buffer` support in the API review, but the TypeScript types may not reflect this. **Mitigation**: If `uploadFile` doesn't accept Buffer directly, use the raw Gemini REST API (`POST /upload/v1beta/files`) with `fetch` for the upload, then use the SDK for `generateContent`. OR write to `/tmp` (works in Node.js but not ideal in serverless).
|
||||
|
||||
4. **`@google/generative-ai` is "deprecated"**: Context7 marks this SDK as deprecated, superseded by `@google/genai`. The newer SDK may have better File API support. **Mitigation**: Phase 1 ships with `@google/generative-ai` (already installed). Note SDK migration as Phase 2 tech debt. The File API behavior is identical across SDKs — only the client library wrapper differs.
|
||||
|
||||
5. **Model availability**: Video understanding requires Gemini 2.5 Flash or Pro (or 1.5 Pro). If the user's `LLM_MODEL` is set to a text-only model, the analyze route needs a separate model config. **Mitigation**: The analyze route reads `LLM_MODEL` but can fall back to `gemini-2.5-flash` if the configured model doesn't support multimodal. Or add a separate `GEMINI_VIDEO_MODEL` env var.
|
||||
|
||||
6. **Serverless timeout**: Next.js on Cloudflare (the deployment target) has request timeout limits. Video processing may exceed these. **Mitigation**: Phase 1 targets local development first. For Cloudflare deployment, use a background worker or queue system in Phase 2.
|
||||
|
||||
7. **API key exposure surface**: The new route handles file uploads and must validate inputs carefully to prevent abuse. **Mitigation**: Use the existing `@upstash/ratelimit` for rate limiting. Validate file MIME type server-side.
|
||||
|
||||
8. **No structured timestamps in Phase 1**: Gemini's response about a video includes natural language time references ("at 0:45 the camera pans") but not structured timestamp objects. **Mitigation**: Accept natural language in Phase 1. Phase 2 can add prompt engineering or response parsing for structured timestamps.
|
||||
|
||||
---
|
||||
|
||||
## Ready for Proposal
|
||||
|
||||
**Yes.** The exploration identifies:
|
||||
- One new client-side tool file (`analyze-video.tool.ts`)
|
||||
- One new server route (`/api/agent/analyze-video/route.ts`)
|
||||
- Minor wiring changes (orchestrator import, route tool defs update)
|
||||
- Clear de-emphasis path for `transcribe_video`
|
||||
- No changes to `ProviderAdapter`, `ChatMessage`, or existing adapter code
|
||||
- Clear Phase 2 boundary (caching, structured timestamps, progress reporting, async processing)
|
||||
|
||||
**Next**: Run `sdd-propose` with this exploration as input.
|
||||
|
|
@ -0,0 +1,62 @@
|
|||
# Proposal: Gemini Analyze Video Phase 1
|
||||
|
||||
## Intent
|
||||
|
||||
Make Gemini video understanding the primary path for video questions so the agent can inspect the actual selected video, not just a transcript. Keep the API key server-side and avoid widening the existing provider/chat contract.
|
||||
|
||||
## Scope
|
||||
|
||||
### In Scope
|
||||
- Add tool `analyze_video` with args `{ assetId?: string, question: string }`.
|
||||
- Send the resolved video file to a new server route that uploads/polls Gemini Files API and returns a natural-language answer with optional timestamp hints.
|
||||
- Expose `analyze_video` as the primary video-understanding tool and de-emphasize `transcribe_video` in LLM-visible tool definitions/prompting.
|
||||
|
||||
### Out of Scope
|
||||
- Editing actions or timeline mutations.
|
||||
- `@asset` UI, streaming/progress UX, and robust caching for large videos.
|
||||
|
||||
## Capabilities
|
||||
|
||||
### New Capabilities
|
||||
- None
|
||||
|
||||
### Modified Capabilities
|
||||
- `agent-session-shell`: tool registry/API proxy behavior now prioritizes `analyze_video` for video understanding instead of `transcribe_video` as the primary exposed tool.
|
||||
- `agent-context-bridge`: system prompt guidance must describe `analyze_video` as the preferred tool for questions about loaded video assets.
|
||||
|
||||
## Approach
|
||||
|
||||
Use a dedicated `POST /api/agent/analyze-video` route. The client tool resolves the asset `File`, sends `FormData`, and the server performs Gemini native upload → processing poll → multimodal query. Keep `ProviderAdapter` text-chat only; keep `transcribe_video` registered as standby.
|
||||
|
||||
## Affected Areas
|
||||
|
||||
| Area | Impact | Description |
|
||||
|------|--------|-------------|
|
||||
| `apps/web/src/agent/tools/analyze-video.tool.ts` | New | Client tool contract and file resolution |
|
||||
| `apps/web/src/app/api/agent/analyze-video/route.ts` | New | Server-side Gemini Files API flow |
|
||||
| `apps/web/src/app/api/agent/chat/route.ts` | Modified | LLM-visible tool definitions prioritize `analyze_video` |
|
||||
| `apps/web/src/agent/system-prompt.ts` | Modified | Prompt guidance prefers video analysis over transcription |
|
||||
| `apps/web/src/agent/orchestrator.ts` | Modified | Register new tool |
|
||||
|
||||
## Risks
|
||||
|
||||
| Risk | Likelihood | Mitigation |
|
||||
|------|------------|------------|
|
||||
| Large upload / long processing | Med | Timeout, clear errors, phase-2 progress/caching |
|
||||
| SDK File API friction | Med | Fallback to raw REST upload if needed |
|
||||
| Wrong model lacks video support | Low | Route-level multimodal-capable model fallback |
|
||||
|
||||
## Rollback Plan
|
||||
|
||||
Remove `analyze_video` wiring, restore `transcribe_video` as the primary exposed tool, and delete the dedicated analyze route.
|
||||
|
||||
## Dependencies
|
||||
|
||||
- Existing Gemini server SDK and server-side `LLM_API_KEY`
|
||||
- Existing `EditorContextAdapter.resolveAssetFile()` asset resolution path
|
||||
|
||||
## Success Criteria
|
||||
|
||||
- [ ] The LLM can call `analyze_video({ assetId?, question })` and receive a natural-language answer about the real video file.
|
||||
- [ ] API key remains server-only; no client-side Gemini upload logic is introduced.
|
||||
- [ ] `transcribe_video` remains available as secondary/standby, not the primary exposed video tool.
|
||||
|
|
@ -110,7 +110,7 @@ The system SHALL define types in `apps/web/src/agent/types.ts`: `ChatMessage { i
|
|||
|
||||
### Requirement: Provider Adapter Interface
|
||||
|
||||
The system SHALL define a server-side provider adapter interface with a single method `chat(params: ProviderChatParams): Promise<ProviderChatResponse>`. `ProviderChatParams` MUST include `messages`, `systemPrompt`, and `toolSchemas` in a provider-agnostic internal format. `ProviderChatResponse` MUST include `content: string` and `toolCalls?: ProviderToolCall[]` where `ProviderToolCall { id, name, args }`. Each concrete adapter converts internal tool schemas to its own wire format. Phase 1 MAY ship exactly one adapter (OpenAI-compatible) as long as the interface contract remains provider-agnostic.
|
||||
The system SHALL define a server-side provider adapter interface with a single method `chat(params: ProviderChatParams): Promise<ProviderChatResponse>`. `ProviderChatParams` MUST include `messages`, `systemPrompt`, and `toolSchemas` in a provider-agnostic internal format. `ProviderChatResponse` MUST include `content: string` and `toolCalls?: ProviderToolCall[]` where `ProviderToolCall { id, name, args }`. Each concrete adapter converts internal tool schemas to its own wire format. The factory MUST support at least `openai-compatible` and `gemini` providers. Adapters whose native API does not assign IDs to tool calls MUST synthesize unique IDs so every returned `ToolCall.id` is a non-empty string.
|
||||
|
||||
#### Scenario: Adapter translates tool schemas to wire format
|
||||
|
||||
|
|
@ -118,15 +118,27 @@ The system SHALL define a server-side provider adapter interface with a single m
|
|||
- WHEN the adapter receives tool schemas before calling the provider
|
||||
- THEN it converts them to its provider-specific wire format
|
||||
|
||||
#### Scenario: Adapter selected by config
|
||||
#### Scenario: Adapter selected by config — OpenAI-compatible
|
||||
|
||||
- GIVEN `LLM_PROVIDER` is set to `"openai-compatible"`
|
||||
- WHEN the route resolves the adapter
|
||||
- THEN the OpenAI-compatible adapter is returned
|
||||
|
||||
#### Scenario: Adapter selected by config — Gemini
|
||||
|
||||
- GIVEN `LLM_PROVIDER` is set to `"gemini"`
|
||||
- WHEN the route resolves the adapter
|
||||
- THEN the Gemini adapter is returned
|
||||
|
||||
#### Scenario: Gemini adapter synthesizes tool-call IDs
|
||||
|
||||
- GIVEN the Gemini API returns a function call response with no call ID
|
||||
- WHEN the Gemini adapter maps the response to `ProviderResponse`
|
||||
- THEN each `ToolCall` in `toolCalls` has a non-empty `id` field generated by the adapter
|
||||
|
||||
### Requirement: Provider Configuration
|
||||
|
||||
Server-only environment variables `LLM_PROVIDER` (required), `LLM_MODEL` (required), `LLM_API_KEY` (required), and `LLM_BASE_URL` (optional) MUST determine which adapter and model the route uses. Configuration MUST be read server-side only — never exposed to the client.
|
||||
Server-only environment variables `LLM_PROVIDER` (required), `LLM_MODEL` (required), `LLM_API_KEY` (required), and `LLM_BASE_URL` (optional) MUST determine which adapter and model the route uses. `LLM_PROVIDER` MUST accept at least `"openai-compatible"` and `"gemini"`. When `LLM_PROVIDER` is `"gemini"`, `LLM_BASE_URL` MUST be ignored. Configuration MUST be read server-side only — never exposed to the client.
|
||||
|
||||
#### Scenario: Missing required configuration
|
||||
|
||||
|
|
@ -134,6 +146,18 @@ Server-only environment variables `LLM_PROVIDER` (required), `LLM_MODEL` (requir
|
|||
- WHEN the route attempts to initialize the provider
|
||||
- THEN the call returns 502 with a missing-configuration error
|
||||
|
||||
#### Scenario: Unknown provider
|
||||
|
||||
- GIVEN `LLM_PROVIDER` is set to `"unknown-provider"`
|
||||
- WHEN the route resolves the adapter
|
||||
- THEN the factory throws `"Unknown LLM provider: unknown-provider"`
|
||||
|
||||
#### Scenario: Gemini config ignores base URL
|
||||
|
||||
- GIVEN `LLM_PROVIDER` is `"gemini"` and `LLM_BASE_URL` is set
|
||||
- WHEN the Gemini adapter is created
|
||||
- THEN `LLM_BASE_URL` is not used in the API call
|
||||
|
||||
### Requirement: Tool Argument Validation
|
||||
|
||||
Before executing any tool call, the orchestrator MUST validate that each argument key declared `required: true` in the tool's `parameters` is present in the call args and is of the declared type. If validation fails, the tool MUST NOT execute; instead the orchestrator MUST produce a `ToolResult` with `error` describing the missing or malformed argument.
|
||||
|
|
@ -166,3 +190,41 @@ The orchestrator loop MUST enforce a hard iteration cap of **8**. An iteration i
|
|||
- GIVEN the provider returns a final text response after 2 tool-call iterations
|
||||
- WHEN the orchestrator processes the response
|
||||
- THEN no cap message is appended and status is `"idle"`
|
||||
|
||||
### Requirement: Gemini Free-Form Chat
|
||||
|
||||
When the Gemini adapter receives a request with no tool schemas (or tools array is empty), it MUST send messages to the Gemini `generateContent` API and return a `ProviderResponse` with `content` set to the assistant text and `toolCalls` undefined.
|
||||
|
||||
#### Scenario: Plain chat through Gemini
|
||||
|
||||
- GIVEN `LLM_PROVIDER` is `"gemini"` and messages contain `[{ role: "user", content: "Hello" }]`
|
||||
- WHEN the adapter calls Gemini and receives a text-only response
|
||||
- THEN `ProviderResponse.content` is the assistant text and `toolCalls` is undefined
|
||||
|
||||
#### Scenario: System prompt mapping
|
||||
|
||||
- GIVEN the adapter receives `systemPrompt: "You are a video editor"`
|
||||
- WHEN it builds the Gemini request
|
||||
- THEN the system prompt is mapped to Gemini's system instruction field, not as a user message
|
||||
|
||||
### Requirement: Gemini Tool Calling
|
||||
|
||||
When the Gemini adapter receives a request with tool schemas, it MUST convert internal `ToolSchema[]` to Gemini function declarations, call `generateContent` with declarations attached, and if Gemini returns function call parts, map each to a `ToolCall { id, name, args }` with a synthesized non-empty `id`.
|
||||
|
||||
#### Scenario: Gemini returns function calls
|
||||
|
||||
- GIVEN `transcribe_video` is in tool schemas and Gemini responds with a function call part `{ name: "transcribe_video", args: {} }`
|
||||
- WHEN the adapter maps the response
|
||||
- THEN `toolCalls` contains one `ToolCall` with `name: "transcribe_video"`, `args: {}`, and a synthesized `id`
|
||||
|
||||
#### Scenario: Gemini returns text despite tools
|
||||
|
||||
- GIVEN tool schemas are provided but Gemini responds with only text
|
||||
- WHEN the adapter maps the response
|
||||
- THEN `toolCalls` is undefined and `content` holds the text
|
||||
|
||||
#### Scenario: Tool schema type mapping
|
||||
|
||||
- GIVEN an internal tool schema has parameters `[{ key: "assetId", type: "string", required: true }]`
|
||||
- WHEN the adapter converts to Gemini function declarations
|
||||
- THEN the parameter appears in `FunctionDeclaration.parameters.properties` with type `"string"`
|
||||
|
|
|
|||
Loading…
Reference in New Issue