RomanNum3ral
/
OpenCut
mirror of https://github.com/OpenCut-app/OpenCut.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #5 from luisacevelad/ft/load-context 

Ft/load context
This commit is contained in:
luisacevelad 2026-04-24 15:00:00 -05:00 committed by GitHub
parent 532f7ab18a 59e40e9047
commit 9cd5a6816e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
16 changed files with 1230 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
apps/web/src/agent/__tests__/context-mapper.test.ts
View File

@ -121,6 +121,7 @@ describe("buildContextFromEditorState (context mapper)", () => {
id: "text-1",
type: "text",
name: "Caption",
content: "Hello world",
startTime: 3,
duration: 2,
},
@ -151,9 +152,31 @@ describe("buildContextFromEditorState (context mapper)", () => {
});
expect(ctx.timelineTracks).toEqual([
{
trackId: "text-track",
type: "text",
position: 0,
visualLayer: 1,
isVisualLayer: true,
stacking: "top",
elements: [
{
elementId: "text-1",
type: "text",
name: "Caption",
content: "Hello world",
start: 3,
end: 5,
},
],
},
{
trackId: "main-track",
type: "main",
position: 1,
visualLayer: 0,
isVisualLayer: true,
stacking: "main",
elements: [
{
elementId: "clip-1",
@ -165,22 +188,13 @@ describe("buildContextFromEditorState (context mapper)", () => {
},
],
},
{
trackId: "text-track",
type: "text",
elements: [
{
elementId: "text-1",
type: "text",
name: "Caption",
start: 3,
end: 5,
},
],
},
{
trackId: "audio-track",
type: "audio",
position: 2,
visualLayer: null,
isVisualLayer: false,
stacking: "audio",
elements: [
{
elementId: "music-1",

4
apps/web/src/agent/__tests__/orchestrator.test.ts
View File

@ -14,8 +14,8 @@ import type { AgentContext, ToolDefinition } from "@/agent/types";
// ---------------------------------------------------------------------------
// Mock the transitive WASM dependency chain before loading the orchestrator.
//
// The orchestrator imports transcribe-video.tool which imports @/agent/context
// which imports @/core → opencut-wasm (WASM binary that can't load in test env).
// The orchestrator imports tools that depend on @/agent/context, which imports
// @/core → opencut-wasm (WASM binary that can't load in test env).
//
// We mock @/agent/context to cut the entire WASM chain.
// We also mock @/lib/media/audio and @/services/transcription/service to prevent

25
apps/web/src/agent/__tests__/system-prompt.test.ts
View File

@ -134,6 +134,31 @@ describe("buildSystemPrompt", () => {
expect(prompt).toContain("plain text");
});
test("instructs Gemini to analyze loaded fileData as actual media", () => {
const tools = [
{ name: "load_context", description: "Loads Gemini media context" },
];
const prompt = buildSystemPrompt(BASE_CONTEXT, tools);
expect(prompt).toContain("attached fileData");
expect(prompt).toContain("actual video/audio/image content");
expect(prompt).toContain("visual and audio questions directly");
});
test("instructs Gemini to load media before refusing visual questions", () => {
const tools = [
{ name: "list_timeline", description: "Lists timeline" },
{ name: "load_context", description: "Loads Gemini media context" },
];
const prompt = buildSystemPrompt(BASE_CONTEXT, tools);
expect(prompt).toContain("never answer that you cannot see or hear");
expect(prompt).toContain("call list_project_assets or list_timeline");
expect(prompt).toContain("then call load_context");
});
test("lists multiple tools when provided", () => {
const tools = [
{ name: "transcribe_video", description: "Transcribes audio" },

49
apps/web/src/agent/context-mapper.ts
View File

@ -55,17 +55,41 @@ function buildTimelineTracks(
}
const tracks: AgentTimelineTrack[] = [];
const overlayTracks = scene.tracks.overlay ?? [];
let position = 0;
if (scene.tracks.main) {
tracks.push(toTimelineTrack(scene.tracks.main, "main"));
for (let index = 0; index < overlayTracks.length; index++) {
const track = overlayTracks[index];
tracks.push(
toTimelineTrack(track, mapOverlayTrackType(track.type), {
position: position++,
visualLayer: overlayTracks.length - index,
isVisualLayer: true,
stacking: index === 0 ? "top" : "above_main",
}),
);
}
for (const track of scene.tracks.overlay ?? []) {
tracks.push(toTimelineTrack(track, mapOverlayTrackType(track.type)));
if (scene.tracks.main) {
tracks.push(
toTimelineTrack(scene.tracks.main, "main", {
position: position++,
visualLayer: 0,
isVisualLayer: true,
stacking: "main",
}),
);
}
for (const track of scene.tracks.audio ?? []) {
tracks.push(toTimelineTrack(track, "audio"));
tracks.push(
toTimelineTrack(track, "audio", {
position: position++,
visualLayer: null,
isVisualLayer: false,
stacking: "audio",
}),
);
}
return tracks;
@ -74,10 +98,15 @@ function buildTimelineTracks(
function toTimelineTrack(
track: TrackInput,
type: AgentTimelineTrack["type"],
stacking: Pick<
AgentTimelineTrack,
"position" | "visualLayer" | "isVisualLayer" | "stacking"
>,
): AgentTimelineTrack {
return {
trackId: track.id ?? "",
type,
...stacking,
elements: (track.elements ?? [])
.filter(hasTimelineElementShape)
.map((element) => ({
@ -85,6 +114,7 @@ function toTimelineTrack(
type: element.type,
...(hasMediaId(element) ? { assetId: element.mediaId } : {}),
...(element.name ? { name: element.name } : {}),
...(hasTextContent(element) ? { content: element.content } : {}),
start: element.startTime,
end: element.startTime + element.duration,
})),
@ -131,6 +161,15 @@ function hasMediaId(element: unknown): element is { mediaId: string } {
);
}
function hasTextContent(element: unknown): element is { content: string } {
return (
typeof element === "object" &&
element !== null &&
"content" in element &&
typeof element.content === "string"
);
}
function hasTimelineElementShape(element: unknown): element is {
id: string;
type: string;

36
apps/web/src/agent/orchestrator.ts
View File

@ -6,13 +6,12 @@ import type {
ToolResult,
} from "@/agent/types";
import { toolRegistry } from "@/agent/tools/registry";
import "@/agent/tools/load-context.tool";
import "@/agent/tools/list-project-assets.tool";
import "@/agent/tools/list-timeline.tool";
import "@/agent/tools/transcribe-video.tool";
import { useChatStore } from "@/stores/chat-store";
import { useAgentStore } from "@/stores/agent-store";
/** Hard cap on multi-turn iterations (user-specified). */
const MAX_ITERATIONS = 8;
interface APIResponse {
@ -20,17 +19,6 @@ interface APIResponse {
toolCalls?: ToolCall[];
}
/**
* Client-side orchestrator — multi-turn loop (v2).
*
* 1. Sets agent status → sending
* 2. Loop (max MAX_ITERATIONS):
* a. POST current message history to /api/agent/chat
* b. If toolCalls → validate args, resolve via registry, append results → loop
* c. If no toolCalls → append final assistant message → break
* 3. On max-iteration cap → append limit message, idle
* 4. Error → chatStore.error + error status
*/
export async function run(
messages: ChatMessage[],
context: AgentContext,
@ -41,7 +29,6 @@ export async function run(
agentStore.setContext(context);
agentStore.setStatus("sending");
// Working copy of messages that grows with each turn
const workingMessages = [...messages];
try {
@ -58,13 +45,24 @@ export async function run(
});
if (!response.ok) {
throw new Error(`API error: ${response.status}`);
let detail = "";
try {
const errBody = await response.json();
detail = errBody?.error ?? JSON.stringify(errBody);
} catch {
detail = response.statusText;
}
throw new Error(
`API error ${response.status}: ${detail}`,
);
}
const data: APIResponse = await response.json();
// No tool calls → final answer, done
if (!data.toolCalls || data.toolCalls.length === 0) {
if (!data.content || data.content.trim().length === 0) {
throw new Error("Empty response from provider");
}
chatStore.addMessage({
role: "assistant",
content: data.content,
@ -72,10 +70,8 @@ export async function run(
break;
}
// Tool calls present → validate, resolve, append, continue
agentStore.setStatus("processing");
// Append assistant message (with toolCalls) to chat + working history
chatStore.addMessage({
role: "assistant",
content: data.content,
@ -89,10 +85,8 @@ export async function run(
timestamp: Date.now(),
});
// Resolve each tool call (validates args before execution)
const toolResults = await resolveToolCalls(data.toolCalls, context);
// Append tool results to chat + working history
for (const tr of toolResults) {
const content = tr.error
? `Error in ${tr.name}: ${tr.error}`
@ -112,7 +106,6 @@ export async function run(
});
}
// If this was the last allowed iteration, mark cap hit
if (iterations >= MAX_ITERATIONS) {
hitCap = true;
}
@ -131,6 +124,7 @@ export async function run(
} catch (error) {
const message =
error instanceof Error ? error.message : "Unknown orchestrator error";
console.error("[orchestrator] Error:", message);
chatStore.setError(message);
agentStore.setStatus("error");
}

139
apps/web/src/agent/providers/__tests__/gemini.test.ts
View File

@ -55,7 +55,11 @@ const SAMPLE_TOOLS: ToolSchema[] = [
function makeGeminiResponse(overrides: {
text?: string;
functionCalls?: Array<{ name: string; args: Record<string, unknown> }>;
functionCalls?: Array<{
name: string;
args: Record<string, unknown>;
thoughtSignature?: string;
}>;
}): GenerateContentResult {
const parts: Array<Record<string, unknown>> = [];
@ -65,7 +69,12 @@ function makeGeminiResponse(overrides: {
if (overrides.functionCalls) {
for (const fc of overrides.functionCalls) {
parts.push({ functionCall: { name: fc.name, args: fc.args } });
parts.push({
functionCall: { name: fc.name, args: fc.args },
...(fc.thoughtSignature
? { thoughtSignature: fc.thoughtSignature }
: {}),
});
}
}
@ -133,7 +142,7 @@ describe("toGeminiContents", () => {
expect(contents).toHaveLength(1);
expect(contents[0].role).toBe("model");
expect(contents[0].parts).toEqual([
expect(contents[0].parts as unknown).toEqual([
{ text: "Let me check." },
{
functionCall: {
@ -144,6 +153,33 @@ describe("toGeminiContents", () => {
]);
});
test("preserves Gemini thought signatures on assistant tool calls", () => {
const messages = [
makeChatMessage("assistant", "", {
toolCalls: [
{
id: "tc_1",
name: "load_context",
args: { targetType: "asset", assetId: "asset-1" },
thoughtSignature: "sig-abc",
},
],
}),
];
const contents = toGeminiContents(messages);
expect(contents[0].parts as unknown).toEqual([
{
functionCall: {
name: "load_context",
args: { targetType: "asset", assetId: "asset-1" },
},
thoughtSignature: "sig-abc",
},
]);
});
test("maps tool_result to function role with functionResponse", () => {
const messages = [
makeChatMessage("assistant", "", {
@ -189,9 +225,7 @@ describe("toGeminiContents", () => {
test("handles tool_result with non-JSON content by wrapping in object", () => {
const messages = [
makeChatMessage("assistant", "", {
toolCalls: [
{ id: "tc_1", name: "some_tool", args: {} },
],
toolCalls: [{ id: "tc_1", name: "some_tool", args: {} }],
}),
makeChatMessage("tool_result", "plain text result", {
toolCallId: "tc_1",
@ -208,6 +242,46 @@ describe("toGeminiContents", () => {
},
]);
});
test("adds fileData part for loaded media context tool results", () => {
const messages = [
makeChatMessage("assistant", "", {
toolCalls: [{ id: "tc_1", name: "load_context", args: {} }],
}),
makeChatMessage(
"tool_result",
JSON.stringify({
context: {
kind: "media",
assetName: "intro.mp4",
fileUri: "gemini://files/abc",
mimeType: "video/mp4",
},
}),
{ toolCallId: "tc_1" },
),
];
const contents = toGeminiContents(messages);
expect(contents[1].role).toBe("function");
expect(contents[2]).toEqual({
role: "user",
parts: [
{
text: expect.stringContaining(
"The attached fileData is the actual media file",
),
},
{
fileData: {
fileUri: "gemini://files/abc",
mimeType: "video/mp4",
},
},
],
});
});
});
// ---------------------------------------------------------------------------
@ -274,7 +348,11 @@ describe("fromGeminiResponse", () => {
const response = makeGeminiResponse({
text: undefined,
functionCalls: [
{ name: "transcribe_video", args: { assetId: "vid-1" } },
{
name: "transcribe_video",
args: { assetId: "vid-1" },
thoughtSignature: "sig-123",
},
],
});
const result = fromGeminiResponse(response);
@ -283,6 +361,7 @@ describe("fromGeminiResponse", () => {
expect(result.toolCalls).toHaveLength(1);
expect(result.toolCalls?.[0].name).toBe("transcribe_video");
expect(result.toolCalls?.[0].args).toEqual({ assetId: "vid-1" });
expect(result.toolCalls?.[0].thoughtSignature).toBe("sig-123");
// Non-empty synthesized ID
expect(result.toolCalls?.[0].id).toBeTruthy();
expect(typeof result.toolCalls?.[0].id).toBe("string");
@ -299,9 +378,7 @@ describe("fromGeminiResponse", () => {
const result1 = fromGeminiResponse(response1);
const result2 = fromGeminiResponse(response2);
expect(result1.toolCalls?.[0].id).not.toBe(
result2.toolCalls?.[0].id,
);
expect(result1.toolCalls?.[0].id).not.toBe(result2.toolCalls?.[0].id);
});
test("handles mixed text and function-call parts", () => {
@ -342,9 +419,7 @@ describe("error paths", () => {
});
test("SDK error propagates from chat()", async () => {
mockGenerateContent.mockRejectedValueOnce(
new Error("API key not valid"),
);
mockGenerateContent.mockRejectedValueOnce(new Error("API key not valid"));
const adapter = new GeminiAdapter(TEST_CONFIG);
@ -499,4 +574,42 @@ describe("GeminiAdapter.chat()", () => {
expect(callArgs.tools).toBeDefined();
expect(callArgs.tools[0].functionDeclarations).toHaveLength(1);
});
test("retries transient provider failures before returning content", async () => {
const providerError = Object.assign(new Error("Service Unavailable"), {
status: 503,
});
mockGenerateContent
.mockRejectedValueOnce(providerError)
.mockRejectedValueOnce(providerError)
.mockResolvedValueOnce(makeGeminiResponse({ text: "Recovered" }));
const adapter = new GeminiAdapter(TEST_CONFIG);
const result = await adapter.chat({
messages: [makeChatMessage("user", "Hi")],
systemPrompt: "",
tools: [],
});
expect(result.content).toBe("Recovered");
expect(mockGenerateContent).toHaveBeenCalledTimes(3);
});
test("does not retry non-transient provider failures", async () => {
const providerError = Object.assign(new Error("Bad request"), {
status: 400,
});
mockGenerateContent.mockRejectedValueOnce(providerError);
const adapter = new GeminiAdapter(TEST_CONFIG);
await expect(
adapter.chat({
messages: [makeChatMessage("user", "Hi")],
systemPrompt: "",
tools: [],
}),
).rejects.toThrow("Bad request");
expect(mockGenerateContent).toHaveBeenCalledTimes(1);
});
});

106
apps/web/src/agent/providers/gemini.ts
View File

@ -27,6 +27,51 @@ const TYPE_MAP: Record<string, SchemaType> = {
object: SchemaType.OBJECT,
};
const GEMINI_MAX_ATTEMPTS = 5;
const GEMINI_RETRY_BASE_DELAY_MS = process.env.NODE_ENV === "test" ? 0 : 1000;
const GEMINI_RETRYABLE_STATUSES = new Set([429, 500, 502, 503, 504]);
function getProviderStatus(error: unknown): number | null {
const status = (error as { status?: unknown })?.status;
if (typeof status === "number") return status;
const message = error instanceof Error ? error.message : String(error);
const match = message.match(/\[(\d{3})\s/);
return match ? Number(match[1]) : null;
}
function isRetryableProviderError(error: unknown): boolean {
const status = getProviderStatus(error);
return status !== null && GEMINI_RETRYABLE_STATUSES.has(status);
}
function wait(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
async function withGeminiRetry<T>(operation: () => Promise<T>): Promise<T> {
let lastError: unknown;
for (let attempt = 1; attempt <= GEMINI_MAX_ATTEMPTS; attempt++) {
try {
return await operation();
} catch (error) {
lastError = error;
if (attempt === GEMINI_MAX_ATTEMPTS || !isRetryableProviderError(error)) {
throw error;
}
const backoffMs = GEMINI_RETRY_BASE_DELAY_MS * 2 ** (attempt - 1);
const jitterMs =
GEMINI_RETRY_BASE_DELAY_MS === 0 ? 0 : Math.random() * 250;
await wait(backoffMs + jitterMs);
}
}
throw lastError;
}
/**
* Converts internal ChatMessages into Gemini Content[].
*
@ -59,6 +104,9 @@ export function toGeminiContents(messages: ChatMessage[]): Content[] {
for (const tc of msg.toolCalls) {
parts.push({
functionCall: { name: tc.name, args: tc.args },
...(tc.thoughtSignature
? { thoughtSignature: tc.thoughtSignature }
: {}),
});
}
}
@ -90,12 +138,63 @@ export function toGeminiContents(messages: ChatMessage[]): Content[] {
},
],
});
const loadedMediaContext = getLoadedMediaContext(responseData);
if (loadedMediaContext) {
contents.push({
role: "user",
parts: [
{
text: `Loaded media context from ${toolName}: ${loadedMediaContext.assetName}. The attached fileData is the actual media file, not just metadata. Use Gemini's multimodal video/audio/visual understanding to answer questions about visible objects, colors, scenes, speech, silence, and timestamps. Do not claim you only have metadata for this loaded file.`,
},
{
fileData: {
fileUri: loadedMediaContext.fileUri,
mimeType: loadedMediaContext.mimeType,
},
},
],
});
}
}
}
return contents;
}
function getLoadedMediaContext(responseData: object): {
assetName: string;
fileUri: string;
mimeType: string;
} | null {
const context = (responseData as { context?: unknown }).context;
if (!context || typeof context !== "object") {
return null;
}
const maybeMediaContext = context as {
kind?: unknown;
assetName?: unknown;
fileUri?: unknown;
mimeType?: unknown;
};
if (
maybeMediaContext.kind !== "media" ||
typeof maybeMediaContext.assetName !== "string" ||
typeof maybeMediaContext.fileUri !== "string" ||
typeof maybeMediaContext.mimeType !== "string"
) {
return null;
}
return {
assetName: maybeMediaContext.assetName,
fileUri: maybeMediaContext.fileUri,
mimeType: maybeMediaContext.mimeType,
};
}
/**
* Walks backward through messages to find the function name associated with
* a toolCallId. This is needed because Gemini's functionResponse uses names,
@ -177,10 +276,13 @@ export function fromGeminiResponse(
textParts.push(part.text);
}
if ("functionCall" in part && part.functionCall) {
const thoughtSignature = (part as { thoughtSignature?: unknown })
.thoughtSignature;
toolCalls.push({
id: crypto.randomUUID(),
name: part.functionCall.name,
args: (part.functionCall.args as Record<string, unknown>) ?? {},
...(typeof thoughtSignature === "string" ? { thoughtSignature } : {}),
});
}
}
@ -234,7 +336,9 @@ export class GeminiAdapter implements ProviderAdapter {
request.tools = [{ functionDeclarations }];
}
const result = await this.model.generateContent(request);
const result = await withGeminiRetry(() =>
this.model.generateContent(request),
);
return fromGeminiResponse(result);
}
}

2
apps/web/src/agent/system-prompt.ts
View File

@ -40,6 +40,8 @@ export function buildSystemPrompt(
.join("\n");
parts.push(
`Available tools:\n${toolList}`,
'For questions about what is visible or audible in a media asset, never answer that you cannot see or hear the media if load_context is available. First infer the asset from the active assets or timeline; if needed call list_project_assets or list_timeline, then call load_context with the discovered internal id or timeline element ids.',
'If load_context has loaded media and the conversation contains an attached fileData part, treat it as the actual video/audio/image content. You may answer visual and audio questions directly from that loaded media without calling extraction tools unless the user asks for a separate extraction workflow.',
"When you need to perform an action matching one of these tools, call the appropriate tool. For all other requests, respond directly in plain text.",
);
}

8
apps/web/src/agent/tools/__tests__/list-timeline.test.ts
View File

@ -13,6 +13,10 @@ function makeContext(overrides: Partial<AgentContext> = {}): AgentContext {
{
trackId: "main-track",
type: "main",
position: 0,
visualLayer: 0,
isVisualLayer: true,
stacking: "main",
elements: [
{
elementId: "clip-1",
@ -44,6 +48,10 @@ describe("list_timeline tool", () => {
{
trackId: "main-track",
type: "main",
position: 0,
visualLayer: 0,
isVisualLayer: true,
stacking: "main",
elements: [
{
elementId: "clip-1",

283
apps/web/src/agent/tools/__tests__/load-context.test.ts Normal file
View File

@ -0,0 +1,283 @@
import { beforeEach, describe, expect, mock, test } from "bun:test";
import type { AgentContext } from "@/agent/types";
import { toolRegistry } from "@/agent/tools/registry";
const mockResolveAssetFile = mock<(assetId?: string) => File | null>();
mock.module("@/agent/context", () => ({
EditorContextAdapter: {
resolveAssetFile: mockResolveAssetFile,
},
}));
await import("@/agent/tools/load-context.tool");
const originalFetch = globalThis.fetch;
function makeContext(overrides: Partial<AgentContext> = {}): AgentContext {
return {
projectId: "proj-1",
activeSceneId: "scene-A",
mediaAssets: [
{
id: "asset-video",
name: "intro.mp4",
type: "video",
duration: 30,
usedInTimeline: true,
},
],
timelineTracks: [
{
trackId: "text-track",
type: "text",
position: 0,
visualLayer: 1,
isVisualLayer: true,
stacking: "top",
elements: [
{
elementId: "caption-1",
type: "text",
name: "Caption",
content: "Hello Gemini",
start: 1,
end: 3,
},
],
},
{
trackId: "main-track",
type: "main",
position: 1,
visualLayer: 0,
isVisualLayer: true,
stacking: "main",
elements: [
{
elementId: "clip-1",
type: "video",
assetId: "asset-video",
name: "Intro clip",
start: 5,
end: 15,
},
],
},
],
playbackTimeMs: 0,
...overrides,
};
}
function mockGeminiUploadResponse() {
globalThis.fetch = mock(() =>
Promise.resolve(
new Response(
JSON.stringify({
provider: "gemini",
status: "loaded",
fileName: "files/abc",
fileUri: "gemini://files/abc",
mimeType: "video/mp4",
displayName: "intro.mp4",
}),
{ status: 200, headers: { "Content-Type": "application/json" } },
),
),
) as unknown as typeof fetch;
}
describe("load_context tool", () => {
beforeEach(() => {
mockResolveAssetFile.mockReset();
globalThis.fetch = originalFetch;
});
test("is registered in the tool registry", () => {
expect(toolRegistry.has("load_context")).toBe(true);
});
test("loads text timeline elements as exact structured context", async () => {
const tool = toolRegistry.get("load_context");
const result = await tool.execute(
{
targetType: "timeline_element",
trackId: "text-track",
elementId: "caption-1",
},
makeContext(),
);
expect(result).toEqual({
targetType: "timeline_element",
id: "caption-1",
status: "loaded",
cached: false,
provider: "gemini",
context: {
kind: "text",
trackId: "text-track",
elementId: "caption-1",
type: "text",
name: "Caption",
content: "Hello Gemini",
start: 1,
end: 3,
},
});
expect(mockResolveAssetFile).not.toHaveBeenCalled();
});
test("uploads media assets through the server endpoint", async () => {
const file = new File(["fake"], "intro.mp4", { type: "video/mp4" });
mockResolveAssetFile.mockReturnValue(file);
mockGeminiUploadResponse();
const tool = toolRegistry.get("load_context");
const result = await tool.execute(
{ targetType: "asset", assetId: "asset-video" },
makeContext(),
);
expect(result).toEqual({
targetType: "asset",
id: "asset-video",
status: "loaded",
cached: false,
provider: "gemini",
context: {
kind: "media",
assetId: "asset-video",
assetName: "intro.mp4",
assetType: "video",
duration: 30,
fileUri: "gemini://files/abc",
mimeType: "video/mp4",
fileName: "files/abc",
displayName: "intro.mp4",
},
});
expect(mockResolveAssetFile).toHaveBeenCalledWith("asset-video");
});
test("sends inferred MIME type when the file type is octet-stream", async () => {
const file = new File(["fake"], "stored-asset", {
type: "application/octet-stream",
});
mockResolveAssetFile.mockReturnValue(file);
const uploaded: { mimeType: FormDataEntryValue | null } = { mimeType: null };
globalThis.fetch = mock((_, init) => {
uploaded.mimeType =
init?.body instanceof FormData ? init.body.get("mimeType") : null;
return Promise.resolve(
new Response(
JSON.stringify({
provider: "gemini",
status: "loaded",
fileName: "files/abc",
fileUri: "gemini://files/abc",
mimeType: "video/quicktime",
displayName: "conclu.mov",
}),
{ status: 200, headers: { "Content-Type": "application/json" } },
),
);
}) as unknown as typeof fetch;
const tool = toolRegistry.get("load_context");
const result = await tool.execute(
{ targetType: "asset", assetId: "asset-video-octet" },
makeContext({
mediaAssets: [
{
id: "asset-video-octet",
name: "conclu.mov",
type: "video",
duration: 30,
},
],
}),
);
expect(uploaded.mimeType).toBe("video/quicktime");
expect(result).toEqual(
expect.objectContaining({
context: expect.objectContaining({ mimeType: "video/quicktime" }),
}),
);
});
test("loads media context for timeline elements with assetId", async () => {
const file = new File(["fake"], "intro.mp4", { type: "video/mp4" });
mockResolveAssetFile.mockReturnValue(file);
mockGeminiUploadResponse();
const tool = toolRegistry.get("load_context");
const result = await tool.execute(
{
targetType: "timeline_element",
trackId: "main-track",
elementId: "clip-1",
},
makeContext({
mediaAssets: [
{
id: "asset-video-2",
name: "other.mp4",
type: "video",
duration: 20,
},
],
timelineTracks: [
{
trackId: "main-track",
type: "main",
position: 0,
visualLayer: 0,
isVisualLayer: true,
stacking: "main",
elements: [
{
elementId: "clip-1",
type: "video",
assetId: "asset-video-2",
start: 5,
end: 15,
},
],
},
],
}),
);
expect(result).toEqual(
expect.objectContaining({
targetType: "asset",
id: "asset-video-2",
context: expect.objectContaining({
kind: "media",
assetId: "asset-video-2",
element: {
trackId: "main-track",
elementId: "clip-1",
type: "video",
start: 5,
end: 15,
},
}),
}),
);
});
test("validates missing timeline identifiers", async () => {
const tool = toolRegistry.get("load_context");
const result = await tool.execute(
{ targetType: "timeline_element", elementId: "caption-1" },
makeContext(),
);
expect(result).toEqual({ error: "trackId and elementId are required" });
});
});

2
apps/web/src/agent/tools/list-timeline.tool.ts
View File

@ -12,7 +12,7 @@ export type ListTimelineResult = {
const listTimelineTool: ToolDefinition = {
name: "list_timeline",
description:
"Lists the active timeline as structured tracks and editable elements with trackId, elementId, type, assetId, name, start, and end times.",
"Lists the active timeline as structured tracks and editable elements with layer metadata. position is timeline row order from top to bottom; visualLayer is render stacking where higher renders above lower, and audio tracks have null visualLayer.",
parameters: [],
execute: async (
_args: Record<string, unknown>,

391
apps/web/src/agent/tools/load-context.tool.ts Normal file
View File

@ -0,0 +1,391 @@
import type {
AgentContext,
AgentTimelineTrack,
ToolDefinition,
} from "@/agent/types";
import { EditorContextAdapter } from "@/agent/context";
import { toolRegistry } from "@/agent/tools/registry";
type LoadContextTargetType = "asset" | "timeline_element";
export type LoadContextArgs = {
targetType?: LoadContextTargetType;
id?: string;
assetId?: string;
trackId?: string;
elementId?: string;
};
export type LoadContextResult = {
targetType: LoadContextTargetType;
id: string;
status: "loaded" | "processing";
cached: boolean;
provider: "gemini";
context: TextContext | MediaContext;
};
type TextContext = {
kind: "text";
trackId: string;
elementId: string;
type: string;
name?: string;
content: string;
start: number;
end: number;
};
type MediaContext = {
kind: "media";
assetId: string;
assetName: string;
assetType: string;
duration: number;
fileUri: string;
mimeType: string;
fileName?: string;
displayName?: string;
element?: {
trackId: string;
elementId: string;
type: string;
start: number;
end: number;
};
};
type UploadedGeminiFile = {
provider: "gemini";
status: "loaded" | "processing";
fileName?: string;
fileUri: string;
mimeType: string;
displayName?: string;
};
const assetContextCache = new Map<string, UploadedGeminiFile>();
const textContextCache = new Map<string, TextContext>();
const loadContextTool: ToolDefinition = {
name: "load_context",
description:
"Loads relevant Gemini context for a project asset or timeline element. Media assets are uploaded through the server-side Gemini Files API; text/caption elements are loaded as exact structured text.",
parameters: [
{ key: "targetType", type: "string", required: true },
{ key: "id", type: "string", required: false },
{ key: "assetId", type: "string", required: false },
{ key: "trackId", type: "string", required: false },
{ key: "elementId", type: "string", required: false },
],
execute: async (
args: Record<string, unknown>,
context: AgentContext,
): Promise<LoadContextResult | { error: string }> => {
const typedArgs = args as LoadContextArgs;
if (typedArgs.targetType === "asset") {
return loadAssetContext(typedArgs, context);
}
if (typedArgs.targetType === "timeline_element") {
return loadTimelineElementContext(typedArgs, context);
}
return { error: "Unsupported context target type" };
},
};
async function loadAssetContext(
args: LoadContextArgs,
context: AgentContext,
element?: TimelineElementRef,
): Promise<LoadContextResult | { error: string }> {
const assetId = args.assetId ?? args.id ?? element?.assetId;
if (!assetId) {
return { error: "Asset id is required" };
}
const asset = context.mediaAssets.find(
(mediaAsset) => mediaAsset.id === assetId,
);
if (!asset) {
return { error: "Asset not found" };
}
if (!isSupportedMediaAssetType(asset.type)) {
return { error: "Unsupported asset type" };
}
const cached = assetContextCache.get(asset.id);
if (cached) {
return {
targetType: "asset",
id: asset.id,
status: cached.status,
cached: true,
provider: "gemini",
context: toMediaContext({ asset, uploaded: cached, element }),
};
}
const file = EditorContextAdapter.resolveAssetFile(asset.id);
if (!file) {
return { error: `Could not access file for asset ${asset.id}` };
}
const uploaded = await uploadFileToGemini({ file, asset });
if ("error" in uploaded) {
return uploaded;
}
assetContextCache.set(asset.id, uploaded);
return {
targetType: "asset",
id: asset.id,
status: uploaded.status,
cached: false,
provider: "gemini",
context: toMediaContext({ asset, uploaded, element }),
};
}
async function loadTimelineElementContext(
args: LoadContextArgs,
context: AgentContext,
): Promise<LoadContextResult | { error: string }> {
const trackId = args.trackId;
const elementId = args.elementId ?? args.id;
if (!trackId || !elementId) {
return { error: "trackId and elementId are required" };
}
const element = findTimelineElement(
context.timelineTracks,
trackId,
elementId,
);
if (!element) {
return { error: "Timeline element not found" };
}
if (element.element.content !== undefined) {
const cacheKey = `${trackId}:${elementId}`;
const cached = textContextCache.get(cacheKey);
if (cached) {
return {
targetType: "timeline_element",
id: elementId,
status: "loaded",
cached: true,
provider: "gemini",
context: cached,
};
}
const textContext: TextContext = {
kind: "text",
trackId,
elementId,
type: element.element.type,
...(element.element.name ? { name: element.element.name } : {}),
content: element.element.content,
start: element.element.start,
end: element.element.end,
};
textContextCache.set(cacheKey, textContext);
return {
targetType: "timeline_element",
id: elementId,
status: "loaded",
cached: false,
provider: "gemini",
context: textContext,
};
}
if (element.element.assetId) {
return loadAssetContext(
{ targetType: "asset", assetId: element.element.assetId },
context,
{
assetId: element.element.assetId,
trackId,
elementId,
type: element.element.type,
start: element.element.start,
end: element.element.end,
},
);
}
return { error: "Timeline element has no loadable context" };
}
async function uploadFileToGemini({
file,
asset,
}: {
file: File;
asset: AgentContext["mediaAssets"][number];
}): Promise<UploadedGeminiFile | { error: string }> {
const mimeType = resolveGeminiMimeType({
fileName: file.name,
displayName: asset.name,
fileType: file.type,
assetType: asset.type,
});
if (!mimeType) {
return { error: `Unsupported MIME type for asset ${asset.id}` };
}
const formData = new FormData();
formData.set("file", file);
formData.set("displayName", asset.name);
formData.set("mimeType", mimeType);
const response = await fetch("/api/agent/context/load", {
method: "POST",
body: formData,
});
const data = (await response.json()) as Partial<UploadedGeminiFile> & {
error?: string;
};
if (!response.ok) {
return { error: data.error ?? "Failed to load context" };
}
if (!data.fileUri || !data.mimeType || !data.status) {
return { error: "Invalid context load response" };
}
return {
provider: "gemini",
status: data.status,
fileName: data.fileName,
fileUri: data.fileUri,
mimeType: data.mimeType,
displayName: data.displayName,
};
}
function resolveGeminiMimeType({
fileName,
displayName,
fileType,
assetType,
}: {
fileName: string;
displayName: string;
fileType?: string;
assetType: string;
}): string | null {
if (fileType && fileType !== "application/octet-stream") {
return fileType;
}
const mimeType = inferMimeTypeFromName(displayName) ?? inferMimeTypeFromName(fileName);
if (mimeType) {
return mimeType;
}
if (assetType === "image") {
return "image/png";
}
return null;
}
function inferMimeTypeFromName(name: string): string | null {
const extension = name.split(".").pop()?.toLowerCase();
return extension ? (MIME_BY_EXTENSION[extension] ?? null) : null;
}
const MIME_BY_EXTENSION: Record<string, string> = {
avi: "video/x-msvideo",
flac: "audio/flac",
gif: "image/gif",
jpeg: "image/jpeg",
jpg: "image/jpeg",
m4a: "audio/mp4",
mov: "video/quicktime",
mp3: "audio/mpeg",
mp4: "video/mp4",
mpeg: "video/mpeg",
mpg: "video/mpeg",
png: "image/png",
wav: "audio/wav",
webm: "video/webm",
webp: "image/webp",
};
function toMediaContext({
asset,
uploaded,
element,
}: {
asset: AgentContext["mediaAssets"][number];
uploaded: UploadedGeminiFile;
element?: TimelineElementRef;
}): MediaContext {
return {
kind: "media",
assetId: asset.id,
assetName: asset.name,
assetType: asset.type,
duration: asset.duration,
fileUri: uploaded.fileUri,
mimeType: uploaded.mimeType,
...(uploaded.fileName ? { fileName: uploaded.fileName } : {}),
...(uploaded.displayName ? { displayName: uploaded.displayName } : {}),
...(element
? {
element: {
trackId: element.trackId,
elementId: element.elementId,
type: element.type,
start: element.start,
end: element.end,
},
}
: {}),
};
}
function findTimelineElement(
tracks: AgentTimelineTrack[] | undefined,
trackId: string,
elementId: string,
): {
track: AgentTimelineTrack;
element: AgentTimelineTrack["elements"][number];
} | null {
const track = tracks?.find((candidate) => candidate.trackId === trackId);
const element = track?.elements.find(
(candidate) => candidate.elementId === elementId,
);
return track && element ? { track, element } : null;
}
function isSupportedMediaAssetType(type: string): boolean {
return type === "video" || type === "audio" || type === "image";
}
type TimelineElementRef = {
assetId: string;
trackId: string;
elementId: string;
type: string;
start: number;
end: number;
};
toolRegistry.register("load_context", loadContextTool);

8
apps/web/src/agent/types.ts
View File

@ -19,6 +19,7 @@ export interface ToolCall {
id: string;
name: string;
args: Record<string, unknown>;
thoughtSignature?: string;
}
export interface ToolResult {
@ -45,11 +46,18 @@ export interface AgentContext {
export type AgentTimelineTrack = {
trackId: string;
type: "main" | "overlay" | "audio" | "text" | "effect";
/** Timeline row position, top-to-bottom. 0 is the top visible row. */
position: number;
/** Visual stacking layer. Higher numbers render above lower numbers. Audio is null. */
visualLayer: number | null;
isVisualLayer: boolean;
stacking: "top" | "above_main" | "main" | "audio";
elements: Array<{
elementId: string;
type: string;
assetId?: string;
name?: string;
content?: string;
start: number;
end: number;
}>;

11
apps/web/src/app/api/agent/chat/__tests__/route.test.ts
View File

@ -40,11 +40,16 @@ const VALID_CONTEXT = {
| Array<{
trackId: string;
type: "main" | "overlay" | "audio" | "text" | "effect";
position: number;
visualLayer: number | null;
isVisualLayer: boolean;
stacking: "top" | "above_main" | "main" | "audio";
elements: Array<{
elementId: string;
type: string;
assetId?: string;
name?: string;
content?: string;
start: number;
end: number;
}>;
@ -150,7 +155,11 @@ describe("POST /api/agent/chat", () => {
expect(callArgs.tools).toHaveLength(3);
expect(
callArgs.tools.map((tool) => (tool as { name: string }).name),
).toEqual(["list_project_assets", "list_timeline", "transcribe_video"]);
).toEqual([
"load_context",
"list_project_assets",
"list_timeline",
]);
});
// -----------------------------------------------------------------------

70
apps/web/src/app/api/agent/chat/route.ts
View File

@ -1,16 +1,27 @@
import { type NextRequest, NextResponse } from "next/server";
import { z } from "zod";
import type { AgentContext, ToolDefinition, ToolSchema } from "@/agent/types";
import type { AgentContext, ToolSchema } from "@/agent/types";
import { buildSystemPrompt } from "@/agent/system-prompt";
import { createProvider } from "@/agent/providers";
import type { ProviderConfig } from "@/agent/providers";
import { toToolSchemas } from "@/agent/tools/registry";
// ---------------------------------------------------------------------------
// Provider-facing tool schemas (execution is client-side)
// echo_context is intentionally excluded — only user-facing tools are exposed.
// Provider-facing tool schemas. Execution is client-side.
// echo_context is intentionally excluded; only user-facing tools are exposed.
// ---------------------------------------------------------------------------
const providerToolDefs: ToolDefinition[] = [
const providerToolSchemas: ToolSchema[] = [
{
name: "load_context",
description:
"Loads the actual Gemini multimodal context for a project asset or timeline element. Use this before answering questions about visible objects, colors, scenes, speech, silence, or timestamps in media. Use targetType='asset' with assetId/id for media, or targetType='timeline_element' with trackId and elementId/id for captions, text, or timeline media elements.",
parameters: [
{ key: "targetType", type: "string", required: true },
{ key: "id", type: "string", required: false },
{ key: "assetId", type: "string", required: false },
{ key: "trackId", type: "string", required: false },
{ key: "elementId", type: "string", required: false },
],
},
{
name: "list_project_assets",
description:
@ -19,25 +30,12 @@ const providerToolDefs: ToolDefinition[] = [
{ key: "filter", type: "string", required: false },
{ key: "type", type: "string", required: false },
],
execute: async () => ({}), // stub — never called server-side
},
{
name: "list_timeline",
description:
"Lists the active timeline as structured tracks and editable elements with trackId, elementId, type, assetId, name, start, and end times.",
"Lists the active timeline as structured tracks and editable elements with layer metadata. Tracks include position (top-to-bottom timeline row), visualLayer (higher renders above lower, null for audio), isVisualLayer, and stacking. Use this to understand which clips visually cover others and to discover ids before load_context.",
parameters: [],
execute: async () => ({}), // stub — never called server-side
},
{
name: "transcribe_video",
description:
"Transcribes the audio of a video or audio asset using Whisper. Returns structured transcript with segments and timestamps.",
parameters: [
{ key: "assetId", type: "string", required: false },
{ key: "language", type: "string", required: false },
{ key: "modelId", type: "string", required: false },
],
execute: async () => ({}), // stub — never called server-side
},
];
@ -48,6 +46,7 @@ const toolCallSchema = z.object({
id: z.string(),
name: z.string(),
args: z.record(z.string(), z.unknown()),
thoughtSignature: z.string().optional(),
});
const chatRequestSchema = z.object({
@ -78,12 +77,17 @@ const chatRequestSchema = z.object({
z.object({
trackId: z.string(),
type: z.enum(["main", "overlay", "audio", "text", "effect"]),
position: z.number(),
visualLayer: z.number().nullable(),
isVisualLayer: z.boolean(),
stacking: z.enum(["top", "above_main", "main", "audio"]),
elements: z.array(
z.object({
elementId: z.string(),
type: z.string(),
assetId: z.string().optional(),
name: z.string().optional(),
content: z.string().optional(),
start: z.number(),
end: z.number(),
}),
@ -103,8 +107,15 @@ export async function POST(request: NextRequest) {
const result = chatRequestSchema.safeParse(body);
if (!result.success) {
const fieldErrors = result.error.flatten().fieldErrors;
if (process.env.NODE_ENV === "development") {
console.error(
"[agent/chat] Validation failed:",
JSON.stringify({ fieldErrors }, null, 2),
);
}
return NextResponse.json(
{ error: "Invalid input", details: result.error.flatten().fieldErrors },
{ error: "Invalid input", details: fieldErrors },
{ status: 400 },
);
}
@ -130,8 +141,7 @@ export async function POST(request: NextRequest) {
const config: ProviderConfig = { provider, apiKey, model, baseUrl };
// Build system prompt with tool guidance
const toolSchemas: ToolSchema[] = toToolSchemas(providerToolDefs);
const systemPrompt = buildSystemPrompt(context, providerToolDefs);
const systemPrompt = buildSystemPrompt(context, providerToolSchemas);
// Delegate to provider adapter
try {
@ -139,7 +149,7 @@ export async function POST(request: NextRequest) {
const response = await adapter.chat({
messages,
systemPrompt,
tools: toolSchemas,
tools: providerToolSchemas,
});
return NextResponse.json({
@ -147,7 +157,19 @@ export async function POST(request: NextRequest) {
...(response.toolCalls && { toolCalls: response.toolCalls }),
});
} catch (error) {
const status = (error as { status?: number }).status;
const message =
error instanceof Error ? error.message : "Unknown provider error";
console.error("[agent/chat] Provider error:", error);
return NextResponse.json({ error: "LLM provider error" }, { status: 502 });
return NextResponse.json(
{
error: "LLM provider error",
...(status ? { providerStatus: status } : {}),
...(process.env.NODE_ENV === "development"
? { detail: message }
: {}),
},
{ status: 502 },
);
}
}

137
apps/web/src/app/api/agent/context/load/route.ts Normal file
View File

@ -0,0 +1,137 @@
import { type NextRequest, NextResponse } from "next/server";
import { FileState, GoogleAIFileManager } from "@google/generative-ai/server";
const MAX_PROCESSING_POLLS = 12;
const PROCESSING_POLL_INTERVAL_MS = 5000;
export async function POST(request: NextRequest) {
const apiKey = process.env.GEMINI_API_KEY ?? process.env.LLM_API_KEY;
if (!apiKey) {
return NextResponse.json(
{
error:
"Server configuration error: missing GEMINI_API_KEY or LLM_API_KEY",
},
{ status: 502 },
);
}
const formData = await request.formData();
const file = formData.get("file");
const displayName = formData.get("displayName");
const requestedMimeType = formData.get("mimeType");
if (!(file instanceof File)) {
return NextResponse.json({ error: "File is required" }, { status: 400 });
}
try {
const mimeType = resolveGeminiMimeType({
fileName: file.name,
fileType: file.type,
displayName: typeof displayName === "string" ? displayName : undefined,
requestedMimeType:
typeof requestedMimeType === "string" ? requestedMimeType : undefined,
});
if (!mimeType) {
return NextResponse.json(
{ error: "Unsupported MIME type" },
{ status: 400 },
);
}
const fileManager = new GoogleAIFileManager(apiKey);
const uploadResult = await fileManager.uploadFile(
Buffer.from(await file.arrayBuffer()),
{
mimeType,
displayName:
typeof displayName === "string" && displayName
? displayName
: file.name,
},
);
let uploadedFile = uploadResult.file;
for (let i = 0; i < MAX_PROCESSING_POLLS; i++) {
if (uploadedFile.state !== FileState.PROCESSING) {
break;
}
await wait(PROCESSING_POLL_INTERVAL_MS);
uploadedFile = await fileManager.getFile(uploadedFile.name);
}
if (uploadedFile.state === FileState.FAILED) {
return NextResponse.json(
{ error: "Failed to load context" },
{ status: 502 },
);
}
return NextResponse.json({
provider: "gemini",
status:
uploadedFile.state === FileState.PROCESSING ? "processing" : "loaded",
fileName: uploadedFile.name,
fileUri: uploadedFile.uri,
mimeType: uploadedFile.mimeType || mimeType,
displayName: uploadedFile.displayName,
});
} catch (error) {
console.error("[agent/context/load] Gemini upload error:", error);
return NextResponse.json(
{ error: "Failed to load context" },
{ status: 502 },
);
}
}
function wait(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
function resolveGeminiMimeType({
fileName,
fileType,
displayName,
requestedMimeType,
}: {
fileName: string;
fileType?: string;
displayName?: string;
requestedMimeType?: string;
}): string | null {
for (const candidate of [requestedMimeType, fileType]) {
if (candidate && candidate !== "application/octet-stream") {
return candidate;
}
}
const mimeType = inferMimeTypeFromName(displayName) ?? inferMimeTypeFromName(fileName);
return mimeType ?? null;
}
function inferMimeTypeFromName(name?: string): string | null {
const extension = name?.split(".").pop()?.toLowerCase();
return extension ? (MIME_BY_EXTENSION[extension] ?? null) : null;
}
const MIME_BY_EXTENSION: Record<string, string> = {
avi: "video/x-msvideo",
flac: "audio/flac",
gif: "image/gif",
jpeg: "image/jpeg",
jpg: "image/jpeg",
m4a: "audio/mp4",
mov: "video/quicktime",
mp3: "audio/mpeg",
mp4: "video/mp4",
mpeg: "video/mpeg",
mpg: "video/mpeg",
png: "image/png",
wav: "audio/wav",
webm: "video/webm",
webp: "image/webp",
};