diff --git a/apps/web/src/agent/__tests__/context-mapper.test.ts b/apps/web/src/agent/__tests__/context-mapper.test.ts index 663015d7..35d2a16b 100644 --- a/apps/web/src/agent/__tests__/context-mapper.test.ts +++ b/apps/web/src/agent/__tests__/context-mapper.test.ts @@ -121,6 +121,7 @@ describe("buildContextFromEditorState (context mapper)", () => { id: "text-1", type: "text", name: "Caption", + content: "Hello world", startTime: 3, duration: 2, }, @@ -151,9 +152,31 @@ describe("buildContextFromEditorState (context mapper)", () => { }); expect(ctx.timelineTracks).toEqual([ + { + trackId: "text-track", + type: "text", + position: 0, + visualLayer: 1, + isVisualLayer: true, + stacking: "top", + elements: [ + { + elementId: "text-1", + type: "text", + name: "Caption", + content: "Hello world", + start: 3, + end: 5, + }, + ], + }, { trackId: "main-track", type: "main", + position: 1, + visualLayer: 0, + isVisualLayer: true, + stacking: "main", elements: [ { elementId: "clip-1", @@ -165,22 +188,13 @@ describe("buildContextFromEditorState (context mapper)", () => { }, ], }, - { - trackId: "text-track", - type: "text", - elements: [ - { - elementId: "text-1", - type: "text", - name: "Caption", - start: 3, - end: 5, - }, - ], - }, { trackId: "audio-track", type: "audio", + position: 2, + visualLayer: null, + isVisualLayer: false, + stacking: "audio", elements: [ { elementId: "music-1", diff --git a/apps/web/src/agent/__tests__/orchestrator.test.ts b/apps/web/src/agent/__tests__/orchestrator.test.ts index fc70aaf1..d4511d1e 100644 --- a/apps/web/src/agent/__tests__/orchestrator.test.ts +++ b/apps/web/src/agent/__tests__/orchestrator.test.ts @@ -14,8 +14,8 @@ import type { AgentContext, ToolDefinition } from "@/agent/types"; // --------------------------------------------------------------------------- // Mock the transitive WASM dependency chain before loading the orchestrator. // -// The orchestrator imports transcribe-video.tool which imports @/agent/context -// which imports @/core → opencut-wasm (WASM binary that can't load in test env). +// The orchestrator imports tools that depend on @/agent/context, which imports +// @/core → opencut-wasm (WASM binary that can't load in test env). // // We mock @/agent/context to cut the entire WASM chain. // We also mock @/lib/media/audio and @/services/transcription/service to prevent diff --git a/apps/web/src/agent/__tests__/system-prompt.test.ts b/apps/web/src/agent/__tests__/system-prompt.test.ts index b20d2b2a..66bb7f15 100644 --- a/apps/web/src/agent/__tests__/system-prompt.test.ts +++ b/apps/web/src/agent/__tests__/system-prompt.test.ts @@ -134,6 +134,31 @@ describe("buildSystemPrompt", () => { expect(prompt).toContain("plain text"); }); + test("instructs Gemini to analyze loaded fileData as actual media", () => { + const tools = [ + { name: "load_context", description: "Loads Gemini media context" }, + ]; + + const prompt = buildSystemPrompt(BASE_CONTEXT, tools); + + expect(prompt).toContain("attached fileData"); + expect(prompt).toContain("actual video/audio/image content"); + expect(prompt).toContain("visual and audio questions directly"); + }); + + test("instructs Gemini to load media before refusing visual questions", () => { + const tools = [ + { name: "list_timeline", description: "Lists timeline" }, + { name: "load_context", description: "Loads Gemini media context" }, + ]; + + const prompt = buildSystemPrompt(BASE_CONTEXT, tools); + + expect(prompt).toContain("never answer that you cannot see or hear"); + expect(prompt).toContain("call list_project_assets or list_timeline"); + expect(prompt).toContain("then call load_context"); + }); + test("lists multiple tools when provided", () => { const tools = [ { name: "transcribe_video", description: "Transcribes audio" }, diff --git a/apps/web/src/agent/context-mapper.ts b/apps/web/src/agent/context-mapper.ts index 160caade..4ba62dcb 100644 --- a/apps/web/src/agent/context-mapper.ts +++ b/apps/web/src/agent/context-mapper.ts @@ -55,17 +55,41 @@ function buildTimelineTracks( } const tracks: AgentTimelineTrack[] = []; + const overlayTracks = scene.tracks.overlay ?? []; + let position = 0; - if (scene.tracks.main) { - tracks.push(toTimelineTrack(scene.tracks.main, "main")); + for (let index = 0; index < overlayTracks.length; index++) { + const track = overlayTracks[index]; + tracks.push( + toTimelineTrack(track, mapOverlayTrackType(track.type), { + position: position++, + visualLayer: overlayTracks.length - index, + isVisualLayer: true, + stacking: index === 0 ? "top" : "above_main", + }), + ); } - for (const track of scene.tracks.overlay ?? []) { - tracks.push(toTimelineTrack(track, mapOverlayTrackType(track.type))); + if (scene.tracks.main) { + tracks.push( + toTimelineTrack(scene.tracks.main, "main", { + position: position++, + visualLayer: 0, + isVisualLayer: true, + stacking: "main", + }), + ); } for (const track of scene.tracks.audio ?? []) { - tracks.push(toTimelineTrack(track, "audio")); + tracks.push( + toTimelineTrack(track, "audio", { + position: position++, + visualLayer: null, + isVisualLayer: false, + stacking: "audio", + }), + ); } return tracks; @@ -74,10 +98,15 @@ function buildTimelineTracks( function toTimelineTrack( track: TrackInput, type: AgentTimelineTrack["type"], + stacking: Pick< + AgentTimelineTrack, + "position" | "visualLayer" | "isVisualLayer" | "stacking" + >, ): AgentTimelineTrack { return { trackId: track.id ?? "", type, + ...stacking, elements: (track.elements ?? []) .filter(hasTimelineElementShape) .map((element) => ({ @@ -85,6 +114,7 @@ function toTimelineTrack( type: element.type, ...(hasMediaId(element) ? { assetId: element.mediaId } : {}), ...(element.name ? { name: element.name } : {}), + ...(hasTextContent(element) ? { content: element.content } : {}), start: element.startTime, end: element.startTime + element.duration, })), @@ -131,6 +161,15 @@ function hasMediaId(element: unknown): element is { mediaId: string } { ); } +function hasTextContent(element: unknown): element is { content: string } { + return ( + typeof element === "object" && + element !== null && + "content" in element && + typeof element.content === "string" + ); +} + function hasTimelineElementShape(element: unknown): element is { id: string; type: string; diff --git a/apps/web/src/agent/orchestrator.ts b/apps/web/src/agent/orchestrator.ts index 1b5db501..43855d33 100644 --- a/apps/web/src/agent/orchestrator.ts +++ b/apps/web/src/agent/orchestrator.ts @@ -6,13 +6,12 @@ import type { ToolResult, } from "@/agent/types"; import { toolRegistry } from "@/agent/tools/registry"; +import "@/agent/tools/load-context.tool"; import "@/agent/tools/list-project-assets.tool"; import "@/agent/tools/list-timeline.tool"; -import "@/agent/tools/transcribe-video.tool"; import { useChatStore } from "@/stores/chat-store"; import { useAgentStore } from "@/stores/agent-store"; -/** Hard cap on multi-turn iterations (user-specified). */ const MAX_ITERATIONS = 8; interface APIResponse { @@ -20,17 +19,6 @@ interface APIResponse { toolCalls?: ToolCall[]; } -/** - * Client-side orchestrator — multi-turn loop (v2). - * - * 1. Sets agent status → sending - * 2. Loop (max MAX_ITERATIONS): - * a. POST current message history to /api/agent/chat - * b. If toolCalls → validate args, resolve via registry, append results → loop - * c. If no toolCalls → append final assistant message → break - * 3. On max-iteration cap → append limit message, idle - * 4. Error → chatStore.error + error status - */ export async function run( messages: ChatMessage[], context: AgentContext, @@ -41,7 +29,6 @@ export async function run( agentStore.setContext(context); agentStore.setStatus("sending"); - // Working copy of messages that grows with each turn const workingMessages = [...messages]; try { @@ -58,13 +45,24 @@ export async function run( }); if (!response.ok) { - throw new Error(`API error: ${response.status}`); + let detail = ""; + try { + const errBody = await response.json(); + detail = errBody?.error ?? JSON.stringify(errBody); + } catch { + detail = response.statusText; + } + throw new Error( + `API error ${response.status}: ${detail}`, + ); } const data: APIResponse = await response.json(); - // No tool calls → final answer, done if (!data.toolCalls || data.toolCalls.length === 0) { + if (!data.content || data.content.trim().length === 0) { + throw new Error("Empty response from provider"); + } chatStore.addMessage({ role: "assistant", content: data.content, @@ -72,10 +70,8 @@ export async function run( break; } - // Tool calls present → validate, resolve, append, continue agentStore.setStatus("processing"); - // Append assistant message (with toolCalls) to chat + working history chatStore.addMessage({ role: "assistant", content: data.content, @@ -89,10 +85,8 @@ export async function run( timestamp: Date.now(), }); - // Resolve each tool call (validates args before execution) const toolResults = await resolveToolCalls(data.toolCalls, context); - // Append tool results to chat + working history for (const tr of toolResults) { const content = tr.error ? `Error in ${tr.name}: ${tr.error}` @@ -112,7 +106,6 @@ export async function run( }); } - // If this was the last allowed iteration, mark cap hit if (iterations >= MAX_ITERATIONS) { hitCap = true; } @@ -131,6 +124,7 @@ export async function run( } catch (error) { const message = error instanceof Error ? error.message : "Unknown orchestrator error"; + console.error("[orchestrator] Error:", message); chatStore.setError(message); agentStore.setStatus("error"); } diff --git a/apps/web/src/agent/providers/__tests__/gemini.test.ts b/apps/web/src/agent/providers/__tests__/gemini.test.ts index 8e5cd3be..59558316 100644 --- a/apps/web/src/agent/providers/__tests__/gemini.test.ts +++ b/apps/web/src/agent/providers/__tests__/gemini.test.ts @@ -55,7 +55,11 @@ const SAMPLE_TOOLS: ToolSchema[] = [ function makeGeminiResponse(overrides: { text?: string; - functionCalls?: Array<{ name: string; args: Record }>; + functionCalls?: Array<{ + name: string; + args: Record; + thoughtSignature?: string; + }>; }): GenerateContentResult { const parts: Array> = []; @@ -65,7 +69,12 @@ function makeGeminiResponse(overrides: { if (overrides.functionCalls) { for (const fc of overrides.functionCalls) { - parts.push({ functionCall: { name: fc.name, args: fc.args } }); + parts.push({ + functionCall: { name: fc.name, args: fc.args }, + ...(fc.thoughtSignature + ? { thoughtSignature: fc.thoughtSignature } + : {}), + }); } } @@ -133,7 +142,7 @@ describe("toGeminiContents", () => { expect(contents).toHaveLength(1); expect(contents[0].role).toBe("model"); - expect(contents[0].parts).toEqual([ + expect(contents[0].parts as unknown).toEqual([ { text: "Let me check." }, { functionCall: { @@ -144,6 +153,33 @@ describe("toGeminiContents", () => { ]); }); + test("preserves Gemini thought signatures on assistant tool calls", () => { + const messages = [ + makeChatMessage("assistant", "", { + toolCalls: [ + { + id: "tc_1", + name: "load_context", + args: { targetType: "asset", assetId: "asset-1" }, + thoughtSignature: "sig-abc", + }, + ], + }), + ]; + + const contents = toGeminiContents(messages); + + expect(contents[0].parts as unknown).toEqual([ + { + functionCall: { + name: "load_context", + args: { targetType: "asset", assetId: "asset-1" }, + }, + thoughtSignature: "sig-abc", + }, + ]); + }); + test("maps tool_result to function role with functionResponse", () => { const messages = [ makeChatMessage("assistant", "", { @@ -189,9 +225,7 @@ describe("toGeminiContents", () => { test("handles tool_result with non-JSON content by wrapping in object", () => { const messages = [ makeChatMessage("assistant", "", { - toolCalls: [ - { id: "tc_1", name: "some_tool", args: {} }, - ], + toolCalls: [{ id: "tc_1", name: "some_tool", args: {} }], }), makeChatMessage("tool_result", "plain text result", { toolCallId: "tc_1", @@ -208,6 +242,46 @@ describe("toGeminiContents", () => { }, ]); }); + + test("adds fileData part for loaded media context tool results", () => { + const messages = [ + makeChatMessage("assistant", "", { + toolCalls: [{ id: "tc_1", name: "load_context", args: {} }], + }), + makeChatMessage( + "tool_result", + JSON.stringify({ + context: { + kind: "media", + assetName: "intro.mp4", + fileUri: "gemini://files/abc", + mimeType: "video/mp4", + }, + }), + { toolCallId: "tc_1" }, + ), + ]; + + const contents = toGeminiContents(messages); + + expect(contents[1].role).toBe("function"); + expect(contents[2]).toEqual({ + role: "user", + parts: [ + { + text: expect.stringContaining( + "The attached fileData is the actual media file", + ), + }, + { + fileData: { + fileUri: "gemini://files/abc", + mimeType: "video/mp4", + }, + }, + ], + }); + }); }); // --------------------------------------------------------------------------- @@ -274,7 +348,11 @@ describe("fromGeminiResponse", () => { const response = makeGeminiResponse({ text: undefined, functionCalls: [ - { name: "transcribe_video", args: { assetId: "vid-1" } }, + { + name: "transcribe_video", + args: { assetId: "vid-1" }, + thoughtSignature: "sig-123", + }, ], }); const result = fromGeminiResponse(response); @@ -283,6 +361,7 @@ describe("fromGeminiResponse", () => { expect(result.toolCalls).toHaveLength(1); expect(result.toolCalls?.[0].name).toBe("transcribe_video"); expect(result.toolCalls?.[0].args).toEqual({ assetId: "vid-1" }); + expect(result.toolCalls?.[0].thoughtSignature).toBe("sig-123"); // Non-empty synthesized ID expect(result.toolCalls?.[0].id).toBeTruthy(); expect(typeof result.toolCalls?.[0].id).toBe("string"); @@ -299,9 +378,7 @@ describe("fromGeminiResponse", () => { const result1 = fromGeminiResponse(response1); const result2 = fromGeminiResponse(response2); - expect(result1.toolCalls?.[0].id).not.toBe( - result2.toolCalls?.[0].id, - ); + expect(result1.toolCalls?.[0].id).not.toBe(result2.toolCalls?.[0].id); }); test("handles mixed text and function-call parts", () => { @@ -342,9 +419,7 @@ describe("error paths", () => { }); test("SDK error propagates from chat()", async () => { - mockGenerateContent.mockRejectedValueOnce( - new Error("API key not valid"), - ); + mockGenerateContent.mockRejectedValueOnce(new Error("API key not valid")); const adapter = new GeminiAdapter(TEST_CONFIG); diff --git a/apps/web/src/agent/providers/gemini.ts b/apps/web/src/agent/providers/gemini.ts index 33961e3e..ec1b32c5 100644 --- a/apps/web/src/agent/providers/gemini.ts +++ b/apps/web/src/agent/providers/gemini.ts @@ -59,6 +59,9 @@ export function toGeminiContents(messages: ChatMessage[]): Content[] { for (const tc of msg.toolCalls) { parts.push({ functionCall: { name: tc.name, args: tc.args }, + ...(tc.thoughtSignature + ? { thoughtSignature: tc.thoughtSignature } + : {}), }); } } @@ -90,12 +93,63 @@ export function toGeminiContents(messages: ChatMessage[]): Content[] { }, ], }); + + const loadedMediaContext = getLoadedMediaContext(responseData); + if (loadedMediaContext) { + contents.push({ + role: "user", + parts: [ + { + text: `Loaded media context from ${toolName}: ${loadedMediaContext.assetName}. The attached fileData is the actual media file, not just metadata. Use Gemini's multimodal video/audio/visual understanding to answer questions about visible objects, colors, scenes, speech, silence, and timestamps. Do not claim you only have metadata for this loaded file.`, + }, + { + fileData: { + fileUri: loadedMediaContext.fileUri, + mimeType: loadedMediaContext.mimeType, + }, + }, + ], + }); + } } } return contents; } +function getLoadedMediaContext(responseData: object): { + assetName: string; + fileUri: string; + mimeType: string; +} | null { + const context = (responseData as { context?: unknown }).context; + if (!context || typeof context !== "object") { + return null; + } + + const maybeMediaContext = context as { + kind?: unknown; + assetName?: unknown; + fileUri?: unknown; + mimeType?: unknown; + }; + + if ( + maybeMediaContext.kind !== "media" || + typeof maybeMediaContext.assetName !== "string" || + typeof maybeMediaContext.fileUri !== "string" || + typeof maybeMediaContext.mimeType !== "string" + ) { + return null; + } + + return { + assetName: maybeMediaContext.assetName, + fileUri: maybeMediaContext.fileUri, + mimeType: maybeMediaContext.mimeType, + }; +} + /** * Walks backward through messages to find the function name associated with * a toolCallId. This is needed because Gemini's functionResponse uses names, @@ -177,10 +231,13 @@ export function fromGeminiResponse( textParts.push(part.text); } if ("functionCall" in part && part.functionCall) { + const thoughtSignature = (part as { thoughtSignature?: unknown }) + .thoughtSignature; toolCalls.push({ id: crypto.randomUUID(), name: part.functionCall.name, args: (part.functionCall.args as Record) ?? {}, + ...(typeof thoughtSignature === "string" ? { thoughtSignature } : {}), }); } } diff --git a/apps/web/src/agent/system-prompt.ts b/apps/web/src/agent/system-prompt.ts index 37407700..2cf31b65 100644 --- a/apps/web/src/agent/system-prompt.ts +++ b/apps/web/src/agent/system-prompt.ts @@ -40,6 +40,8 @@ export function buildSystemPrompt( .join("\n"); parts.push( `Available tools:\n${toolList}`, + 'For questions about what is visible or audible in a media asset, never answer that you cannot see or hear the media if load_context is available. First infer the asset from the active assets or timeline; if needed call list_project_assets or list_timeline, then call load_context with the discovered internal id or timeline element ids.', + 'If load_context has loaded media and the conversation contains an attached fileData part, treat it as the actual video/audio/image content. You may answer visual and audio questions directly from that loaded media without calling extraction tools unless the user asks for a separate extraction workflow.', "When you need to perform an action matching one of these tools, call the appropriate tool. For all other requests, respond directly in plain text.", ); } diff --git a/apps/web/src/agent/tools/__tests__/list-timeline.test.ts b/apps/web/src/agent/tools/__tests__/list-timeline.test.ts index 6e308790..28295c5a 100644 --- a/apps/web/src/agent/tools/__tests__/list-timeline.test.ts +++ b/apps/web/src/agent/tools/__tests__/list-timeline.test.ts @@ -13,6 +13,10 @@ function makeContext(overrides: Partial = {}): AgentContext { { trackId: "main-track", type: "main", + position: 0, + visualLayer: 0, + isVisualLayer: true, + stacking: "main", elements: [ { elementId: "clip-1", @@ -44,6 +48,10 @@ describe("list_timeline tool", () => { { trackId: "main-track", type: "main", + position: 0, + visualLayer: 0, + isVisualLayer: true, + stacking: "main", elements: [ { elementId: "clip-1", diff --git a/apps/web/src/agent/tools/__tests__/load-context.test.ts b/apps/web/src/agent/tools/__tests__/load-context.test.ts new file mode 100644 index 00000000..99cbaaad --- /dev/null +++ b/apps/web/src/agent/tools/__tests__/load-context.test.ts @@ -0,0 +1,283 @@ +import { beforeEach, describe, expect, mock, test } from "bun:test"; +import type { AgentContext } from "@/agent/types"; +import { toolRegistry } from "@/agent/tools/registry"; + +const mockResolveAssetFile = mock<(assetId?: string) => File | null>(); + +mock.module("@/agent/context", () => ({ + EditorContextAdapter: { + resolveAssetFile: mockResolveAssetFile, + }, +})); + +await import("@/agent/tools/load-context.tool"); + +const originalFetch = globalThis.fetch; + +function makeContext(overrides: Partial = {}): AgentContext { + return { + projectId: "proj-1", + activeSceneId: "scene-A", + mediaAssets: [ + { + id: "asset-video", + name: "intro.mp4", + type: "video", + duration: 30, + usedInTimeline: true, + }, + ], + timelineTracks: [ + { + trackId: "text-track", + type: "text", + position: 0, + visualLayer: 1, + isVisualLayer: true, + stacking: "top", + elements: [ + { + elementId: "caption-1", + type: "text", + name: "Caption", + content: "Hello Gemini", + start: 1, + end: 3, + }, + ], + }, + { + trackId: "main-track", + type: "main", + position: 1, + visualLayer: 0, + isVisualLayer: true, + stacking: "main", + elements: [ + { + elementId: "clip-1", + type: "video", + assetId: "asset-video", + name: "Intro clip", + start: 5, + end: 15, + }, + ], + }, + ], + playbackTimeMs: 0, + ...overrides, + }; +} + +function mockGeminiUploadResponse() { + globalThis.fetch = mock(() => + Promise.resolve( + new Response( + JSON.stringify({ + provider: "gemini", + status: "loaded", + fileName: "files/abc", + fileUri: "gemini://files/abc", + mimeType: "video/mp4", + displayName: "intro.mp4", + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ), + ), + ) as unknown as typeof fetch; +} + +describe("load_context tool", () => { + beforeEach(() => { + mockResolveAssetFile.mockReset(); + globalThis.fetch = originalFetch; + }); + + test("is registered in the tool registry", () => { + expect(toolRegistry.has("load_context")).toBe(true); + }); + + test("loads text timeline elements as exact structured context", async () => { + const tool = toolRegistry.get("load_context"); + const result = await tool.execute( + { + targetType: "timeline_element", + trackId: "text-track", + elementId: "caption-1", + }, + makeContext(), + ); + + expect(result).toEqual({ + targetType: "timeline_element", + id: "caption-1", + status: "loaded", + cached: false, + provider: "gemini", + context: { + kind: "text", + trackId: "text-track", + elementId: "caption-1", + type: "text", + name: "Caption", + content: "Hello Gemini", + start: 1, + end: 3, + }, + }); + expect(mockResolveAssetFile).not.toHaveBeenCalled(); + }); + + test("uploads media assets through the server endpoint", async () => { + const file = new File(["fake"], "intro.mp4", { type: "video/mp4" }); + mockResolveAssetFile.mockReturnValue(file); + mockGeminiUploadResponse(); + + const tool = toolRegistry.get("load_context"); + const result = await tool.execute( + { targetType: "asset", assetId: "asset-video" }, + makeContext(), + ); + + expect(result).toEqual({ + targetType: "asset", + id: "asset-video", + status: "loaded", + cached: false, + provider: "gemini", + context: { + kind: "media", + assetId: "asset-video", + assetName: "intro.mp4", + assetType: "video", + duration: 30, + fileUri: "gemini://files/abc", + mimeType: "video/mp4", + fileName: "files/abc", + displayName: "intro.mp4", + }, + }); + expect(mockResolveAssetFile).toHaveBeenCalledWith("asset-video"); + }); + + test("sends inferred MIME type when the file type is octet-stream", async () => { + const file = new File(["fake"], "stored-asset", { + type: "application/octet-stream", + }); + mockResolveAssetFile.mockReturnValue(file); + + const uploaded: { mimeType: FormDataEntryValue | null } = { mimeType: null }; + globalThis.fetch = mock((_, init) => { + uploaded.mimeType = + init?.body instanceof FormData ? init.body.get("mimeType") : null; + return Promise.resolve( + new Response( + JSON.stringify({ + provider: "gemini", + status: "loaded", + fileName: "files/abc", + fileUri: "gemini://files/abc", + mimeType: "video/quicktime", + displayName: "conclu.mov", + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ), + ); + }) as unknown as typeof fetch; + + const tool = toolRegistry.get("load_context"); + const result = await tool.execute( + { targetType: "asset", assetId: "asset-video-octet" }, + makeContext({ + mediaAssets: [ + { + id: "asset-video-octet", + name: "conclu.mov", + type: "video", + duration: 30, + }, + ], + }), + ); + + expect(uploaded.mimeType).toBe("video/quicktime"); + expect(result).toEqual( + expect.objectContaining({ + context: expect.objectContaining({ mimeType: "video/quicktime" }), + }), + ); + }); + + test("loads media context for timeline elements with assetId", async () => { + const file = new File(["fake"], "intro.mp4", { type: "video/mp4" }); + mockResolveAssetFile.mockReturnValue(file); + mockGeminiUploadResponse(); + + const tool = toolRegistry.get("load_context"); + const result = await tool.execute( + { + targetType: "timeline_element", + trackId: "main-track", + elementId: "clip-1", + }, + makeContext({ + mediaAssets: [ + { + id: "asset-video-2", + name: "other.mp4", + type: "video", + duration: 20, + }, + ], + timelineTracks: [ + { + trackId: "main-track", + type: "main", + position: 0, + visualLayer: 0, + isVisualLayer: true, + stacking: "main", + elements: [ + { + elementId: "clip-1", + type: "video", + assetId: "asset-video-2", + start: 5, + end: 15, + }, + ], + }, + ], + }), + ); + + expect(result).toEqual( + expect.objectContaining({ + targetType: "asset", + id: "asset-video-2", + context: expect.objectContaining({ + kind: "media", + assetId: "asset-video-2", + element: { + trackId: "main-track", + elementId: "clip-1", + type: "video", + start: 5, + end: 15, + }, + }), + }), + ); + }); + + test("validates missing timeline identifiers", async () => { + const tool = toolRegistry.get("load_context"); + const result = await tool.execute( + { targetType: "timeline_element", elementId: "caption-1" }, + makeContext(), + ); + + expect(result).toEqual({ error: "trackId and elementId are required" }); + }); +}); diff --git a/apps/web/src/agent/tools/list-timeline.tool.ts b/apps/web/src/agent/tools/list-timeline.tool.ts index 0dcf3796..6176980e 100644 --- a/apps/web/src/agent/tools/list-timeline.tool.ts +++ b/apps/web/src/agent/tools/list-timeline.tool.ts @@ -12,7 +12,7 @@ export type ListTimelineResult = { const listTimelineTool: ToolDefinition = { name: "list_timeline", description: - "Lists the active timeline as structured tracks and editable elements with trackId, elementId, type, assetId, name, start, and end times.", + "Lists the active timeline as structured tracks and editable elements with layer metadata. position is timeline row order from top to bottom; visualLayer is render stacking where higher renders above lower, and audio tracks have null visualLayer.", parameters: [], execute: async ( _args: Record, diff --git a/apps/web/src/agent/tools/load-context.tool.ts b/apps/web/src/agent/tools/load-context.tool.ts new file mode 100644 index 00000000..f63937da --- /dev/null +++ b/apps/web/src/agent/tools/load-context.tool.ts @@ -0,0 +1,391 @@ +import type { + AgentContext, + AgentTimelineTrack, + ToolDefinition, +} from "@/agent/types"; +import { EditorContextAdapter } from "@/agent/context"; +import { toolRegistry } from "@/agent/tools/registry"; + +type LoadContextTargetType = "asset" | "timeline_element"; + +export type LoadContextArgs = { + targetType?: LoadContextTargetType; + id?: string; + assetId?: string; + trackId?: string; + elementId?: string; +}; + +export type LoadContextResult = { + targetType: LoadContextTargetType; + id: string; + status: "loaded" | "processing"; + cached: boolean; + provider: "gemini"; + context: TextContext | MediaContext; +}; + +type TextContext = { + kind: "text"; + trackId: string; + elementId: string; + type: string; + name?: string; + content: string; + start: number; + end: number; +}; + +type MediaContext = { + kind: "media"; + assetId: string; + assetName: string; + assetType: string; + duration: number; + fileUri: string; + mimeType: string; + fileName?: string; + displayName?: string; + element?: { + trackId: string; + elementId: string; + type: string; + start: number; + end: number; + }; +}; + +type UploadedGeminiFile = { + provider: "gemini"; + status: "loaded" | "processing"; + fileName?: string; + fileUri: string; + mimeType: string; + displayName?: string; +}; + +const assetContextCache = new Map(); +const textContextCache = new Map(); + +const loadContextTool: ToolDefinition = { + name: "load_context", + description: + "Loads relevant Gemini context for a project asset or timeline element. Media assets are uploaded through the server-side Gemini Files API; text/caption elements are loaded as exact structured text.", + parameters: [ + { key: "targetType", type: "string", required: true }, + { key: "id", type: "string", required: false }, + { key: "assetId", type: "string", required: false }, + { key: "trackId", type: "string", required: false }, + { key: "elementId", type: "string", required: false }, + ], + execute: async ( + args: Record, + context: AgentContext, + ): Promise => { + const typedArgs = args as LoadContextArgs; + + if (typedArgs.targetType === "asset") { + return loadAssetContext(typedArgs, context); + } + + if (typedArgs.targetType === "timeline_element") { + return loadTimelineElementContext(typedArgs, context); + } + + return { error: "Unsupported context target type" }; + }, +}; + +async function loadAssetContext( + args: LoadContextArgs, + context: AgentContext, + element?: TimelineElementRef, +): Promise { + const assetId = args.assetId ?? args.id ?? element?.assetId; + if (!assetId) { + return { error: "Asset id is required" }; + } + + const asset = context.mediaAssets.find( + (mediaAsset) => mediaAsset.id === assetId, + ); + if (!asset) { + return { error: "Asset not found" }; + } + + if (!isSupportedMediaAssetType(asset.type)) { + return { error: "Unsupported asset type" }; + } + + const cached = assetContextCache.get(asset.id); + if (cached) { + return { + targetType: "asset", + id: asset.id, + status: cached.status, + cached: true, + provider: "gemini", + context: toMediaContext({ asset, uploaded: cached, element }), + }; + } + + const file = EditorContextAdapter.resolveAssetFile(asset.id); + if (!file) { + return { error: `Could not access file for asset ${asset.id}` }; + } + + const uploaded = await uploadFileToGemini({ file, asset }); + if ("error" in uploaded) { + return uploaded; + } + + assetContextCache.set(asset.id, uploaded); + + return { + targetType: "asset", + id: asset.id, + status: uploaded.status, + cached: false, + provider: "gemini", + context: toMediaContext({ asset, uploaded, element }), + }; +} + +async function loadTimelineElementContext( + args: LoadContextArgs, + context: AgentContext, +): Promise { + const trackId = args.trackId; + const elementId = args.elementId ?? args.id; + + if (!trackId || !elementId) { + return { error: "trackId and elementId are required" }; + } + + const element = findTimelineElement( + context.timelineTracks, + trackId, + elementId, + ); + if (!element) { + return { error: "Timeline element not found" }; + } + + if (element.element.content !== undefined) { + const cacheKey = `${trackId}:${elementId}`; + const cached = textContextCache.get(cacheKey); + if (cached) { + return { + targetType: "timeline_element", + id: elementId, + status: "loaded", + cached: true, + provider: "gemini", + context: cached, + }; + } + + const textContext: TextContext = { + kind: "text", + trackId, + elementId, + type: element.element.type, + ...(element.element.name ? { name: element.element.name } : {}), + content: element.element.content, + start: element.element.start, + end: element.element.end, + }; + + textContextCache.set(cacheKey, textContext); + + return { + targetType: "timeline_element", + id: elementId, + status: "loaded", + cached: false, + provider: "gemini", + context: textContext, + }; + } + + if (element.element.assetId) { + return loadAssetContext( + { targetType: "asset", assetId: element.element.assetId }, + context, + { + assetId: element.element.assetId, + trackId, + elementId, + type: element.element.type, + start: element.element.start, + end: element.element.end, + }, + ); + } + + return { error: "Timeline element has no loadable context" }; +} + +async function uploadFileToGemini({ + file, + asset, +}: { + file: File; + asset: AgentContext["mediaAssets"][number]; +}): Promise { + const mimeType = resolveGeminiMimeType({ + fileName: file.name, + displayName: asset.name, + fileType: file.type, + assetType: asset.type, + }); + + if (!mimeType) { + return { error: `Unsupported MIME type for asset ${asset.id}` }; + } + + const formData = new FormData(); + formData.set("file", file); + formData.set("displayName", asset.name); + formData.set("mimeType", mimeType); + + const response = await fetch("/api/agent/context/load", { + method: "POST", + body: formData, + }); + + const data = (await response.json()) as Partial & { + error?: string; + }; + + if (!response.ok) { + return { error: data.error ?? "Failed to load context" }; + } + + if (!data.fileUri || !data.mimeType || !data.status) { + return { error: "Invalid context load response" }; + } + + return { + provider: "gemini", + status: data.status, + fileName: data.fileName, + fileUri: data.fileUri, + mimeType: data.mimeType, + displayName: data.displayName, + }; +} + +function resolveGeminiMimeType({ + fileName, + displayName, + fileType, + assetType, +}: { + fileName: string; + displayName: string; + fileType?: string; + assetType: string; +}): string | null { + if (fileType && fileType !== "application/octet-stream") { + return fileType; + } + + const mimeType = inferMimeTypeFromName(displayName) ?? inferMimeTypeFromName(fileName); + if (mimeType) { + return mimeType; + } + + if (assetType === "image") { + return "image/png"; + } + + return null; +} + +function inferMimeTypeFromName(name: string): string | null { + const extension = name.split(".").pop()?.toLowerCase(); + return extension ? (MIME_BY_EXTENSION[extension] ?? null) : null; +} + +const MIME_BY_EXTENSION: Record = { + avi: "video/x-msvideo", + flac: "audio/flac", + gif: "image/gif", + jpeg: "image/jpeg", + jpg: "image/jpeg", + m4a: "audio/mp4", + mov: "video/quicktime", + mp3: "audio/mpeg", + mp4: "video/mp4", + mpeg: "video/mpeg", + mpg: "video/mpeg", + png: "image/png", + wav: "audio/wav", + webm: "video/webm", + webp: "image/webp", +}; + +function toMediaContext({ + asset, + uploaded, + element, +}: { + asset: AgentContext["mediaAssets"][number]; + uploaded: UploadedGeminiFile; + element?: TimelineElementRef; +}): MediaContext { + return { + kind: "media", + assetId: asset.id, + assetName: asset.name, + assetType: asset.type, + duration: asset.duration, + fileUri: uploaded.fileUri, + mimeType: uploaded.mimeType, + ...(uploaded.fileName ? { fileName: uploaded.fileName } : {}), + ...(uploaded.displayName ? { displayName: uploaded.displayName } : {}), + ...(element + ? { + element: { + trackId: element.trackId, + elementId: element.elementId, + type: element.type, + start: element.start, + end: element.end, + }, + } + : {}), + }; +} + +function findTimelineElement( + tracks: AgentTimelineTrack[] | undefined, + trackId: string, + elementId: string, +): { + track: AgentTimelineTrack; + element: AgentTimelineTrack["elements"][number]; +} | null { + const track = tracks?.find((candidate) => candidate.trackId === trackId); + const element = track?.elements.find( + (candidate) => candidate.elementId === elementId, + ); + + return track && element ? { track, element } : null; +} + +function isSupportedMediaAssetType(type: string): boolean { + return type === "video" || type === "audio" || type === "image"; +} + +type TimelineElementRef = { + assetId: string; + trackId: string; + elementId: string; + type: string; + start: number; + end: number; +}; + +toolRegistry.register("load_context", loadContextTool); diff --git a/apps/web/src/agent/types.ts b/apps/web/src/agent/types.ts index 197a1979..a34d3bad 100644 --- a/apps/web/src/agent/types.ts +++ b/apps/web/src/agent/types.ts @@ -19,6 +19,7 @@ export interface ToolCall { id: string; name: string; args: Record; + thoughtSignature?: string; } export interface ToolResult { @@ -45,11 +46,18 @@ export interface AgentContext { export type AgentTimelineTrack = { trackId: string; type: "main" | "overlay" | "audio" | "text" | "effect"; + /** Timeline row position, top-to-bottom. 0 is the top visible row. */ + position: number; + /** Visual stacking layer. Higher numbers render above lower numbers. Audio is null. */ + visualLayer: number | null; + isVisualLayer: boolean; + stacking: "top" | "above_main" | "main" | "audio"; elements: Array<{ elementId: string; type: string; assetId?: string; name?: string; + content?: string; start: number; end: number; }>; diff --git a/apps/web/src/app/api/agent/chat/__tests__/route.test.ts b/apps/web/src/app/api/agent/chat/__tests__/route.test.ts index 39da3450..6b8bd5e1 100644 --- a/apps/web/src/app/api/agent/chat/__tests__/route.test.ts +++ b/apps/web/src/app/api/agent/chat/__tests__/route.test.ts @@ -40,11 +40,16 @@ const VALID_CONTEXT = { | Array<{ trackId: string; type: "main" | "overlay" | "audio" | "text" | "effect"; + position: number; + visualLayer: number | null; + isVisualLayer: boolean; + stacking: "top" | "above_main" | "main" | "audio"; elements: Array<{ elementId: string; type: string; assetId?: string; name?: string; + content?: string; start: number; end: number; }>; @@ -150,7 +155,11 @@ describe("POST /api/agent/chat", () => { expect(callArgs.tools).toHaveLength(3); expect( callArgs.tools.map((tool) => (tool as { name: string }).name), - ).toEqual(["list_project_assets", "list_timeline", "transcribe_video"]); + ).toEqual([ + "load_context", + "list_project_assets", + "list_timeline", + ]); }); // ----------------------------------------------------------------------- diff --git a/apps/web/src/app/api/agent/chat/route.ts b/apps/web/src/app/api/agent/chat/route.ts index 75edf7b5..cf42895b 100644 --- a/apps/web/src/app/api/agent/chat/route.ts +++ b/apps/web/src/app/api/agent/chat/route.ts @@ -1,16 +1,27 @@ import { type NextRequest, NextResponse } from "next/server"; import { z } from "zod"; -import type { AgentContext, ToolDefinition, ToolSchema } from "@/agent/types"; +import type { AgentContext, ToolSchema } from "@/agent/types"; import { buildSystemPrompt } from "@/agent/system-prompt"; import { createProvider } from "@/agent/providers"; import type { ProviderConfig } from "@/agent/providers"; -import { toToolSchemas } from "@/agent/tools/registry"; // --------------------------------------------------------------------------- -// Provider-facing tool schemas (execution is client-side) -// echo_context is intentionally excluded — only user-facing tools are exposed. +// Provider-facing tool schemas. Execution is client-side. +// echo_context is intentionally excluded; only user-facing tools are exposed. // --------------------------------------------------------------------------- -const providerToolDefs: ToolDefinition[] = [ +const providerToolSchemas: ToolSchema[] = [ + { + name: "load_context", + description: + "Loads the actual Gemini multimodal context for a project asset or timeline element. Use this before answering questions about visible objects, colors, scenes, speech, silence, or timestamps in media. Use targetType='asset' with assetId/id for media, or targetType='timeline_element' with trackId and elementId/id for captions, text, or timeline media elements.", + parameters: [ + { key: "targetType", type: "string", required: true }, + { key: "id", type: "string", required: false }, + { key: "assetId", type: "string", required: false }, + { key: "trackId", type: "string", required: false }, + { key: "elementId", type: "string", required: false }, + ], + }, { name: "list_project_assets", description: @@ -19,25 +30,12 @@ const providerToolDefs: ToolDefinition[] = [ { key: "filter", type: "string", required: false }, { key: "type", type: "string", required: false }, ], - execute: async () => ({}), // stub — never called server-side }, { name: "list_timeline", description: - "Lists the active timeline as structured tracks and editable elements with trackId, elementId, type, assetId, name, start, and end times.", + "Lists the active timeline as structured tracks and editable elements with layer metadata. Tracks include position (top-to-bottom timeline row), visualLayer (higher renders above lower, null for audio), isVisualLayer, and stacking. Use this to understand which clips visually cover others and to discover ids before load_context.", parameters: [], - execute: async () => ({}), // stub — never called server-side - }, - { - name: "transcribe_video", - description: - "Transcribes the audio of a video or audio asset using Whisper. Returns structured transcript with segments and timestamps.", - parameters: [ - { key: "assetId", type: "string", required: false }, - { key: "language", type: "string", required: false }, - { key: "modelId", type: "string", required: false }, - ], - execute: async () => ({}), // stub — never called server-side }, ]; @@ -48,6 +46,7 @@ const toolCallSchema = z.object({ id: z.string(), name: z.string(), args: z.record(z.string(), z.unknown()), + thoughtSignature: z.string().optional(), }); const chatRequestSchema = z.object({ @@ -78,12 +77,17 @@ const chatRequestSchema = z.object({ z.object({ trackId: z.string(), type: z.enum(["main", "overlay", "audio", "text", "effect"]), + position: z.number(), + visualLayer: z.number().nullable(), + isVisualLayer: z.boolean(), + stacking: z.enum(["top", "above_main", "main", "audio"]), elements: z.array( z.object({ elementId: z.string(), type: z.string(), assetId: z.string().optional(), name: z.string().optional(), + content: z.string().optional(), start: z.number(), end: z.number(), }), @@ -103,8 +107,15 @@ export async function POST(request: NextRequest) { const result = chatRequestSchema.safeParse(body); if (!result.success) { + const fieldErrors = result.error.flatten().fieldErrors; + if (process.env.NODE_ENV === "development") { + console.error( + "[agent/chat] Validation failed:", + JSON.stringify({ fieldErrors }, null, 2), + ); + } return NextResponse.json( - { error: "Invalid input", details: result.error.flatten().fieldErrors }, + { error: "Invalid input", details: fieldErrors }, { status: 400 }, ); } @@ -130,8 +141,7 @@ export async function POST(request: NextRequest) { const config: ProviderConfig = { provider, apiKey, model, baseUrl }; // Build system prompt with tool guidance - const toolSchemas: ToolSchema[] = toToolSchemas(providerToolDefs); - const systemPrompt = buildSystemPrompt(context, providerToolDefs); + const systemPrompt = buildSystemPrompt(context, providerToolSchemas); // Delegate to provider adapter try { @@ -139,7 +149,7 @@ export async function POST(request: NextRequest) { const response = await adapter.chat({ messages, systemPrompt, - tools: toolSchemas, + tools: providerToolSchemas, }); return NextResponse.json({ @@ -147,7 +157,19 @@ export async function POST(request: NextRequest) { ...(response.toolCalls && { toolCalls: response.toolCalls }), }); } catch (error) { + const status = (error as { status?: number }).status; + const message = + error instanceof Error ? error.message : "Unknown provider error"; console.error("[agent/chat] Provider error:", error); - return NextResponse.json({ error: "LLM provider error" }, { status: 502 }); + return NextResponse.json( + { + error: "LLM provider error", + ...(status ? { providerStatus: status } : {}), + ...(process.env.NODE_ENV === "development" + ? { detail: message } + : {}), + }, + { status: 502 }, + ); } } diff --git a/apps/web/src/app/api/agent/context/load/route.ts b/apps/web/src/app/api/agent/context/load/route.ts new file mode 100644 index 00000000..5707ab19 --- /dev/null +++ b/apps/web/src/app/api/agent/context/load/route.ts @@ -0,0 +1,137 @@ +import { type NextRequest, NextResponse } from "next/server"; +import { FileState, GoogleAIFileManager } from "@google/generative-ai/server"; + +const MAX_PROCESSING_POLLS = 12; +const PROCESSING_POLL_INTERVAL_MS = 5000; + +export async function POST(request: NextRequest) { + const apiKey = process.env.GEMINI_API_KEY ?? process.env.LLM_API_KEY; + if (!apiKey) { + return NextResponse.json( + { + error: + "Server configuration error: missing GEMINI_API_KEY or LLM_API_KEY", + }, + { status: 502 }, + ); + } + + const formData = await request.formData(); + const file = formData.get("file"); + const displayName = formData.get("displayName"); + const requestedMimeType = formData.get("mimeType"); + + if (!(file instanceof File)) { + return NextResponse.json({ error: "File is required" }, { status: 400 }); + } + + try { + const mimeType = resolveGeminiMimeType({ + fileName: file.name, + fileType: file.type, + displayName: typeof displayName === "string" ? displayName : undefined, + requestedMimeType: + typeof requestedMimeType === "string" ? requestedMimeType : undefined, + }); + + if (!mimeType) { + return NextResponse.json( + { error: "Unsupported MIME type" }, + { status: 400 }, + ); + } + + const fileManager = new GoogleAIFileManager(apiKey); + const uploadResult = await fileManager.uploadFile( + Buffer.from(await file.arrayBuffer()), + { + mimeType, + displayName: + typeof displayName === "string" && displayName + ? displayName + : file.name, + }, + ); + + let uploadedFile = uploadResult.file; + for (let i = 0; i < MAX_PROCESSING_POLLS; i++) { + if (uploadedFile.state !== FileState.PROCESSING) { + break; + } + + await wait(PROCESSING_POLL_INTERVAL_MS); + uploadedFile = await fileManager.getFile(uploadedFile.name); + } + + if (uploadedFile.state === FileState.FAILED) { + return NextResponse.json( + { error: "Failed to load context" }, + { status: 502 }, + ); + } + + return NextResponse.json({ + provider: "gemini", + status: + uploadedFile.state === FileState.PROCESSING ? "processing" : "loaded", + fileName: uploadedFile.name, + fileUri: uploadedFile.uri, + mimeType: uploadedFile.mimeType || mimeType, + displayName: uploadedFile.displayName, + }); + } catch (error) { + console.error("[agent/context/load] Gemini upload error:", error); + return NextResponse.json( + { error: "Failed to load context" }, + { status: 502 }, + ); + } +} + +function wait(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function resolveGeminiMimeType({ + fileName, + fileType, + displayName, + requestedMimeType, +}: { + fileName: string; + fileType?: string; + displayName?: string; + requestedMimeType?: string; +}): string | null { + for (const candidate of [requestedMimeType, fileType]) { + if (candidate && candidate !== "application/octet-stream") { + return candidate; + } + } + + const mimeType = inferMimeTypeFromName(displayName) ?? inferMimeTypeFromName(fileName); + return mimeType ?? null; +} + +function inferMimeTypeFromName(name?: string): string | null { + const extension = name?.split(".").pop()?.toLowerCase(); + return extension ? (MIME_BY_EXTENSION[extension] ?? null) : null; +} + +const MIME_BY_EXTENSION: Record = { + avi: "video/x-msvideo", + flac: "audio/flac", + gif: "image/gif", + jpeg: "image/jpeg", + jpg: "image/jpeg", + m4a: "audio/mp4", + mov: "video/quicktime", + mp3: "audio/mpeg", + mp4: "video/mp4", + mpeg: "video/mpeg", + mpg: "video/mpeg", + png: "image/png", + wav: "audio/wav", + webm: "video/webm", + webp: "image/webp", +};