ft: load context
This commit is contained in:
Luis Esteban Acevedo Ladino
parent
532f7ab18a
commit
1ac3d49562
|
|
@ -121,6 +121,7 @@ describe("buildContextFromEditorState (context mapper)", () => {
|
|||
id: "text-1",
|
||||
type: "text",
|
||||
name: "Caption",
|
||||
content: "Hello world",
|
||||
startTime: 3,
|
||||
duration: 2,
|
||||
},
|
||||
|
|
@ -151,9 +152,31 @@ describe("buildContextFromEditorState (context mapper)", () => {
|
|||
});
|
||||
|
||||
expect(ctx.timelineTracks).toEqual([
|
||||
{
|
||||
trackId: "text-track",
|
||||
type: "text",
|
||||
position: 0,
|
||||
visualLayer: 1,
|
||||
isVisualLayer: true,
|
||||
stacking: "top",
|
||||
elements: [
|
||||
{
|
||||
elementId: "text-1",
|
||||
type: "text",
|
||||
name: "Caption",
|
||||
content: "Hello world",
|
||||
start: 3,
|
||||
end: 5,
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
trackId: "main-track",
|
||||
type: "main",
|
||||
position: 1,
|
||||
visualLayer: 0,
|
||||
isVisualLayer: true,
|
||||
stacking: "main",
|
||||
elements: [
|
||||
{
|
||||
elementId: "clip-1",
|
||||
|
|
@ -165,22 +188,13 @@ describe("buildContextFromEditorState (context mapper)", () => {
|
|||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
trackId: "text-track",
|
||||
type: "text",
|
||||
elements: [
|
||||
{
|
||||
elementId: "text-1",
|
||||
type: "text",
|
||||
name: "Caption",
|
||||
start: 3,
|
||||
end: 5,
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
trackId: "audio-track",
|
||||
type: "audio",
|
||||
position: 2,
|
||||
visualLayer: null,
|
||||
isVisualLayer: false,
|
||||
stacking: "audio",
|
||||
elements: [
|
||||
{
|
||||
elementId: "music-1",
|
||||
|
|
|
|||
|
|
@ -14,8 +14,8 @@ import type { AgentContext, ToolDefinition } from "@/agent/types";
|
|||
// ---------------------------------------------------------------------------
|
||||
// Mock the transitive WASM dependency chain before loading the orchestrator.
|
||||
//
|
||||
// The orchestrator imports transcribe-video.tool which imports @/agent/context
|
||||
// which imports @/core → opencut-wasm (WASM binary that can't load in test env).
|
||||
// The orchestrator imports tools that depend on @/agent/context, which imports
|
||||
// @/core → opencut-wasm (WASM binary that can't load in test env).
|
||||
//
|
||||
// We mock @/agent/context to cut the entire WASM chain.
|
||||
// We also mock @/lib/media/audio and @/services/transcription/service to prevent
|
||||
|
|
|
|||
|
|
@ -134,6 +134,31 @@ describe("buildSystemPrompt", () => {
|
|||
expect(prompt).toContain("plain text");
|
||||
});
|
||||
|
||||
test("instructs Gemini to analyze loaded fileData as actual media", () => {
|
||||
const tools = [
|
||||
{ name: "load_context", description: "Loads Gemini media context" },
|
||||
];
|
||||
|
||||
const prompt = buildSystemPrompt(BASE_CONTEXT, tools);
|
||||
|
||||
expect(prompt).toContain("attached fileData");
|
||||
expect(prompt).toContain("actual video/audio/image content");
|
||||
expect(prompt).toContain("visual and audio questions directly");
|
||||
});
|
||||
|
||||
test("instructs Gemini to load media before refusing visual questions", () => {
|
||||
const tools = [
|
||||
{ name: "list_timeline", description: "Lists timeline" },
|
||||
{ name: "load_context", description: "Loads Gemini media context" },
|
||||
];
|
||||
|
||||
const prompt = buildSystemPrompt(BASE_CONTEXT, tools);
|
||||
|
||||
expect(prompt).toContain("never answer that you cannot see or hear");
|
||||
expect(prompt).toContain("call list_project_assets or list_timeline");
|
||||
expect(prompt).toContain("then call load_context");
|
||||
});
|
||||
|
||||
test("lists multiple tools when provided", () => {
|
||||
const tools = [
|
||||
{ name: "transcribe_video", description: "Transcribes audio" },
|
||||
|
|
|
|||
|
|
@ -55,17 +55,41 @@ function buildTimelineTracks(
|
|||
}
|
||||
|
||||
const tracks: AgentTimelineTrack[] = [];
|
||||
const overlayTracks = scene.tracks.overlay ?? [];
|
||||
let position = 0;
|
||||
|
||||
if (scene.tracks.main) {
|
||||
tracks.push(toTimelineTrack(scene.tracks.main, "main"));
|
||||
for (let index = 0; index < overlayTracks.length; index++) {
|
||||
const track = overlayTracks[index];
|
||||
tracks.push(
|
||||
toTimelineTrack(track, mapOverlayTrackType(track.type), {
|
||||
position: position++,
|
||||
visualLayer: overlayTracks.length - index,
|
||||
isVisualLayer: true,
|
||||
stacking: index === 0 ? "top" : "above_main",
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
for (const track of scene.tracks.overlay ?? []) {
|
||||
tracks.push(toTimelineTrack(track, mapOverlayTrackType(track.type)));
|
||||
if (scene.tracks.main) {
|
||||
tracks.push(
|
||||
toTimelineTrack(scene.tracks.main, "main", {
|
||||
position: position++,
|
||||
visualLayer: 0,
|
||||
isVisualLayer: true,
|
||||
stacking: "main",
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
for (const track of scene.tracks.audio ?? []) {
|
||||
tracks.push(toTimelineTrack(track, "audio"));
|
||||
tracks.push(
|
||||
toTimelineTrack(track, "audio", {
|
||||
position: position++,
|
||||
visualLayer: null,
|
||||
isVisualLayer: false,
|
||||
stacking: "audio",
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
return tracks;
|
||||
|
|
@ -74,10 +98,15 @@ function buildTimelineTracks(
|
|||
function toTimelineTrack(
|
||||
track: TrackInput,
|
||||
type: AgentTimelineTrack["type"],
|
||||
stacking: Pick<
|
||||
AgentTimelineTrack,
|
||||
"position" | "visualLayer" | "isVisualLayer" | "stacking"
|
||||
>,
|
||||
): AgentTimelineTrack {
|
||||
return {
|
||||
trackId: track.id ?? "",
|
||||
type,
|
||||
...stacking,
|
||||
elements: (track.elements ?? [])
|
||||
.filter(hasTimelineElementShape)
|
||||
.map((element) => ({
|
||||
|
|
@ -85,6 +114,7 @@ function toTimelineTrack(
|
|||
type: element.type,
|
||||
...(hasMediaId(element) ? { assetId: element.mediaId } : {}),
|
||||
...(element.name ? { name: element.name } : {}),
|
||||
...(hasTextContent(element) ? { content: element.content } : {}),
|
||||
start: element.startTime,
|
||||
end: element.startTime + element.duration,
|
||||
})),
|
||||
|
|
@ -131,6 +161,15 @@ function hasMediaId(element: unknown): element is { mediaId: string } {
|
|||
);
|
||||
}
|
||||
|
||||
function hasTextContent(element: unknown): element is { content: string } {
|
||||
return (
|
||||
typeof element === "object" &&
|
||||
element !== null &&
|
||||
"content" in element &&
|
||||
typeof element.content === "string"
|
||||
);
|
||||
}
|
||||
|
||||
function hasTimelineElementShape(element: unknown): element is {
|
||||
id: string;
|
||||
type: string;
|
||||
|
|
|
|||
|
|
@ -6,13 +6,12 @@ import type {
|
|||
ToolResult,
|
||||
} from "@/agent/types";
|
||||
import { toolRegistry } from "@/agent/tools/registry";
|
||||
import "@/agent/tools/load-context.tool";
|
||||
import "@/agent/tools/list-project-assets.tool";
|
||||
import "@/agent/tools/list-timeline.tool";
|
||||
import "@/agent/tools/transcribe-video.tool";
|
||||
import { useChatStore } from "@/stores/chat-store";
|
||||
import { useAgentStore } from "@/stores/agent-store";
|
||||
|
||||
/** Hard cap on multi-turn iterations (user-specified). */
|
||||
const MAX_ITERATIONS = 8;
|
||||
|
||||
interface APIResponse {
|
||||
|
|
@ -20,17 +19,6 @@ interface APIResponse {
|
|||
toolCalls?: ToolCall[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Client-side orchestrator — multi-turn loop (v2).
|
||||
*
|
||||
* 1. Sets agent status → sending
|
||||
* 2. Loop (max MAX_ITERATIONS):
|
||||
* a. POST current message history to /api/agent/chat
|
||||
* b. If toolCalls → validate args, resolve via registry, append results → loop
|
||||
* c. If no toolCalls → append final assistant message → break
|
||||
* 3. On max-iteration cap → append limit message, idle
|
||||
* 4. Error → chatStore.error + error status
|
||||
*/
|
||||
export async function run(
|
||||
messages: ChatMessage[],
|
||||
context: AgentContext,
|
||||
|
|
@ -41,7 +29,6 @@ export async function run(
|
|||
agentStore.setContext(context);
|
||||
agentStore.setStatus("sending");
|
||||
|
||||
// Working copy of messages that grows with each turn
|
||||
const workingMessages = [...messages];
|
||||
|
||||
try {
|
||||
|
|
@ -58,13 +45,24 @@ export async function run(
|
|||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`API error: ${response.status}`);
|
||||
let detail = "";
|
||||
try {
|
||||
const errBody = await response.json();
|
||||
detail = errBody?.error ?? JSON.stringify(errBody);
|
||||
} catch {
|
||||
detail = response.statusText;
|
||||
}
|
||||
throw new Error(
|
||||
`API error ${response.status}: ${detail}`,
|
||||
);
|
||||
}
|
||||
|
||||
const data: APIResponse = await response.json();
|
||||
|
||||
// No tool calls → final answer, done
|
||||
if (!data.toolCalls || data.toolCalls.length === 0) {
|
||||
if (!data.content || data.content.trim().length === 0) {
|
||||
throw new Error("Empty response from provider");
|
||||
}
|
||||
chatStore.addMessage({
|
||||
role: "assistant",
|
||||
content: data.content,
|
||||
|
|
@ -72,10 +70,8 @@ export async function run(
|
|||
break;
|
||||
}
|
||||
|
||||
// Tool calls present → validate, resolve, append, continue
|
||||
agentStore.setStatus("processing");
|
||||
|
||||
// Append assistant message (with toolCalls) to chat + working history
|
||||
chatStore.addMessage({
|
||||
role: "assistant",
|
||||
content: data.content,
|
||||
|
|
@ -89,10 +85,8 @@ export async function run(
|
|||
timestamp: Date.now(),
|
||||
});
|
||||
|
||||
// Resolve each tool call (validates args before execution)
|
||||
const toolResults = await resolveToolCalls(data.toolCalls, context);
|
||||
|
||||
// Append tool results to chat + working history
|
||||
for (const tr of toolResults) {
|
||||
const content = tr.error
|
||||
? `Error in ${tr.name}: ${tr.error}`
|
||||
|
|
@ -112,7 +106,6 @@ export async function run(
|
|||
});
|
||||
}
|
||||
|
||||
// If this was the last allowed iteration, mark cap hit
|
||||
if (iterations >= MAX_ITERATIONS) {
|
||||
hitCap = true;
|
||||
}
|
||||
|
|
@ -131,6 +124,7 @@ export async function run(
|
|||
} catch (error) {
|
||||
const message =
|
||||
error instanceof Error ? error.message : "Unknown orchestrator error";
|
||||
console.error("[orchestrator] Error:", message);
|
||||
chatStore.setError(message);
|
||||
agentStore.setStatus("error");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -55,7 +55,11 @@ const SAMPLE_TOOLS: ToolSchema[] = [
|
|||
|
||||
function makeGeminiResponse(overrides: {
|
||||
text?: string;
|
||||
functionCalls?: Array<{ name: string; args: Record<string, unknown> }>;
|
||||
functionCalls?: Array<{
|
||||
name: string;
|
||||
args: Record<string, unknown>;
|
||||
thoughtSignature?: string;
|
||||
}>;
|
||||
}): GenerateContentResult {
|
||||
const parts: Array<Record<string, unknown>> = [];
|
||||
|
||||
|
|
@ -65,7 +69,12 @@ function makeGeminiResponse(overrides: {
|
|||
|
||||
if (overrides.functionCalls) {
|
||||
for (const fc of overrides.functionCalls) {
|
||||
parts.push({ functionCall: { name: fc.name, args: fc.args } });
|
||||
parts.push({
|
||||
functionCall: { name: fc.name, args: fc.args },
|
||||
...(fc.thoughtSignature
|
||||
? { thoughtSignature: fc.thoughtSignature }
|
||||
: {}),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -133,7 +142,7 @@ describe("toGeminiContents", () => {
|
|||
|
||||
expect(contents).toHaveLength(1);
|
||||
expect(contents[0].role).toBe("model");
|
||||
expect(contents[0].parts).toEqual([
|
||||
expect(contents[0].parts as unknown).toEqual([
|
||||
{ text: "Let me check." },
|
||||
{
|
||||
functionCall: {
|
||||
|
|
@ -144,6 +153,33 @@ describe("toGeminiContents", () => {
|
|||
]);
|
||||
});
|
||||
|
||||
test("preserves Gemini thought signatures on assistant tool calls", () => {
|
||||
const messages = [
|
||||
makeChatMessage("assistant", "", {
|
||||
toolCalls: [
|
||||
{
|
||||
id: "tc_1",
|
||||
name: "load_context",
|
||||
args: { targetType: "asset", assetId: "asset-1" },
|
||||
thoughtSignature: "sig-abc",
|
||||
},
|
||||
],
|
||||
}),
|
||||
];
|
||||
|
||||
const contents = toGeminiContents(messages);
|
||||
|
||||
expect(contents[0].parts as unknown).toEqual([
|
||||
{
|
||||
functionCall: {
|
||||
name: "load_context",
|
||||
args: { targetType: "asset", assetId: "asset-1" },
|
||||
},
|
||||
thoughtSignature: "sig-abc",
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
test("maps tool_result to function role with functionResponse", () => {
|
||||
const messages = [
|
||||
makeChatMessage("assistant", "", {
|
||||
|
|
@ -189,9 +225,7 @@ describe("toGeminiContents", () => {
|
|||
test("handles tool_result with non-JSON content by wrapping in object", () => {
|
||||
const messages = [
|
||||
makeChatMessage("assistant", "", {
|
||||
toolCalls: [
|
||||
{ id: "tc_1", name: "some_tool", args: {} },
|
||||
],
|
||||
toolCalls: [{ id: "tc_1", name: "some_tool", args: {} }],
|
||||
}),
|
||||
makeChatMessage("tool_result", "plain text result", {
|
||||
toolCallId: "tc_1",
|
||||
|
|
@ -208,6 +242,46 @@ describe("toGeminiContents", () => {
|
|||
},
|
||||
]);
|
||||
});
|
||||
|
||||
test("adds fileData part for loaded media context tool results", () => {
|
||||
const messages = [
|
||||
makeChatMessage("assistant", "", {
|
||||
toolCalls: [{ id: "tc_1", name: "load_context", args: {} }],
|
||||
}),
|
||||
makeChatMessage(
|
||||
"tool_result",
|
||||
JSON.stringify({
|
||||
context: {
|
||||
kind: "media",
|
||||
assetName: "intro.mp4",
|
||||
fileUri: "gemini://files/abc",
|
||||
mimeType: "video/mp4",
|
||||
},
|
||||
}),
|
||||
{ toolCallId: "tc_1" },
|
||||
),
|
||||
];
|
||||
|
||||
const contents = toGeminiContents(messages);
|
||||
|
||||
expect(contents[1].role).toBe("function");
|
||||
expect(contents[2]).toEqual({
|
||||
role: "user",
|
||||
parts: [
|
||||
{
|
||||
text: expect.stringContaining(
|
||||
"The attached fileData is the actual media file",
|
||||
),
|
||||
},
|
||||
{
|
||||
fileData: {
|
||||
fileUri: "gemini://files/abc",
|
||||
mimeType: "video/mp4",
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
|
@ -274,7 +348,11 @@ describe("fromGeminiResponse", () => {
|
|||
const response = makeGeminiResponse({
|
||||
text: undefined,
|
||||
functionCalls: [
|
||||
{ name: "transcribe_video", args: { assetId: "vid-1" } },
|
||||
{
|
||||
name: "transcribe_video",
|
||||
args: { assetId: "vid-1" },
|
||||
thoughtSignature: "sig-123",
|
||||
},
|
||||
],
|
||||
});
|
||||
const result = fromGeminiResponse(response);
|
||||
|
|
@ -283,6 +361,7 @@ describe("fromGeminiResponse", () => {
|
|||
expect(result.toolCalls).toHaveLength(1);
|
||||
expect(result.toolCalls?.[0].name).toBe("transcribe_video");
|
||||
expect(result.toolCalls?.[0].args).toEqual({ assetId: "vid-1" });
|
||||
expect(result.toolCalls?.[0].thoughtSignature).toBe("sig-123");
|
||||
// Non-empty synthesized ID
|
||||
expect(result.toolCalls?.[0].id).toBeTruthy();
|
||||
expect(typeof result.toolCalls?.[0].id).toBe("string");
|
||||
|
|
@ -299,9 +378,7 @@ describe("fromGeminiResponse", () => {
|
|||
const result1 = fromGeminiResponse(response1);
|
||||
const result2 = fromGeminiResponse(response2);
|
||||
|
||||
expect(result1.toolCalls?.[0].id).not.toBe(
|
||||
result2.toolCalls?.[0].id,
|
||||
);
|
||||
expect(result1.toolCalls?.[0].id).not.toBe(result2.toolCalls?.[0].id);
|
||||
});
|
||||
|
||||
test("handles mixed text and function-call parts", () => {
|
||||
|
|
@ -342,9 +419,7 @@ describe("error paths", () => {
|
|||
});
|
||||
|
||||
test("SDK error propagates from chat()", async () => {
|
||||
mockGenerateContent.mockRejectedValueOnce(
|
||||
new Error("API key not valid"),
|
||||
);
|
||||
mockGenerateContent.mockRejectedValueOnce(new Error("API key not valid"));
|
||||
|
||||
const adapter = new GeminiAdapter(TEST_CONFIG);
|
||||
|
||||
|
|
|
|||
|
|
@ -59,6 +59,9 @@ export function toGeminiContents(messages: ChatMessage[]): Content[] {
|
|||
for (const tc of msg.toolCalls) {
|
||||
parts.push({
|
||||
functionCall: { name: tc.name, args: tc.args },
|
||||
...(tc.thoughtSignature
|
||||
? { thoughtSignature: tc.thoughtSignature }
|
||||
: {}),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
@ -90,12 +93,63 @@ export function toGeminiContents(messages: ChatMessage[]): Content[] {
|
|||
},
|
||||
],
|
||||
});
|
||||
|
||||
const loadedMediaContext = getLoadedMediaContext(responseData);
|
||||
if (loadedMediaContext) {
|
||||
contents.push({
|
||||
role: "user",
|
||||
parts: [
|
||||
{
|
||||
text: `Loaded media context from ${toolName}: ${loadedMediaContext.assetName}. The attached fileData is the actual media file, not just metadata. Use Gemini's multimodal video/audio/visual understanding to answer questions about visible objects, colors, scenes, speech, silence, and timestamps. Do not claim you only have metadata for this loaded file.`,
|
||||
},
|
||||
{
|
||||
fileData: {
|
||||
fileUri: loadedMediaContext.fileUri,
|
||||
mimeType: loadedMediaContext.mimeType,
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return contents;
|
||||
}
|
||||
|
||||
function getLoadedMediaContext(responseData: object): {
|
||||
assetName: string;
|
||||
fileUri: string;
|
||||
mimeType: string;
|
||||
} | null {
|
||||
const context = (responseData as { context?: unknown }).context;
|
||||
if (!context || typeof context !== "object") {
|
||||
return null;
|
||||
}
|
||||
|
||||
const maybeMediaContext = context as {
|
||||
kind?: unknown;
|
||||
assetName?: unknown;
|
||||
fileUri?: unknown;
|
||||
mimeType?: unknown;
|
||||
};
|
||||
|
||||
if (
|
||||
maybeMediaContext.kind !== "media" ||
|
||||
typeof maybeMediaContext.assetName !== "string" ||
|
||||
typeof maybeMediaContext.fileUri !== "string" ||
|
||||
typeof maybeMediaContext.mimeType !== "string"
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
assetName: maybeMediaContext.assetName,
|
||||
fileUri: maybeMediaContext.fileUri,
|
||||
mimeType: maybeMediaContext.mimeType,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Walks backward through messages to find the function name associated with
|
||||
* a toolCallId. This is needed because Gemini's functionResponse uses names,
|
||||
|
|
@ -177,10 +231,13 @@ export function fromGeminiResponse(
|
|||
textParts.push(part.text);
|
||||
}
|
||||
if ("functionCall" in part && part.functionCall) {
|
||||
const thoughtSignature = (part as { thoughtSignature?: unknown })
|
||||
.thoughtSignature;
|
||||
toolCalls.push({
|
||||
id: crypto.randomUUID(),
|
||||
name: part.functionCall.name,
|
||||
args: (part.functionCall.args as Record<string, unknown>) ?? {},
|
||||
...(typeof thoughtSignature === "string" ? { thoughtSignature } : {}),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -40,6 +40,8 @@ export function buildSystemPrompt(
|
|||
.join("\n");
|
||||
parts.push(
|
||||
`Available tools:\n${toolList}`,
|
||||
'For questions about what is visible or audible in a media asset, never answer that you cannot see or hear the media if load_context is available. First infer the asset from the active assets or timeline; if needed call list_project_assets or list_timeline, then call load_context with the discovered internal id or timeline element ids.',
|
||||
'If load_context has loaded media and the conversation contains an attached fileData part, treat it as the actual video/audio/image content. You may answer visual and audio questions directly from that loaded media without calling extraction tools unless the user asks for a separate extraction workflow.',
|
||||
"When you need to perform an action matching one of these tools, call the appropriate tool. For all other requests, respond directly in plain text.",
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,6 +13,10 @@ function makeContext(overrides: Partial<AgentContext> = {}): AgentContext {
|
|||
{
|
||||
trackId: "main-track",
|
||||
type: "main",
|
||||
position: 0,
|
||||
visualLayer: 0,
|
||||
isVisualLayer: true,
|
||||
stacking: "main",
|
||||
elements: [
|
||||
{
|
||||
elementId: "clip-1",
|
||||
|
|
@ -44,6 +48,10 @@ describe("list_timeline tool", () => {
|
|||
{
|
||||
trackId: "main-track",
|
||||
type: "main",
|
||||
position: 0,
|
||||
visualLayer: 0,
|
||||
isVisualLayer: true,
|
||||
stacking: "main",
|
||||
elements: [
|
||||
{
|
||||
elementId: "clip-1",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,283 @@
|
|||
import { beforeEach, describe, expect, mock, test } from "bun:test";
|
||||
import type { AgentContext } from "@/agent/types";
|
||||
import { toolRegistry } from "@/agent/tools/registry";
|
||||
|
||||
const mockResolveAssetFile = mock<(assetId?: string) => File | null>();
|
||||
|
||||
mock.module("@/agent/context", () => ({
|
||||
EditorContextAdapter: {
|
||||
resolveAssetFile: mockResolveAssetFile,
|
||||
},
|
||||
}));
|
||||
|
||||
await import("@/agent/tools/load-context.tool");
|
||||
|
||||
const originalFetch = globalThis.fetch;
|
||||
|
||||
function makeContext(overrides: Partial<AgentContext> = {}): AgentContext {
|
||||
return {
|
||||
projectId: "proj-1",
|
||||
activeSceneId: "scene-A",
|
||||
mediaAssets: [
|
||||
{
|
||||
id: "asset-video",
|
||||
name: "intro.mp4",
|
||||
type: "video",
|
||||
duration: 30,
|
||||
usedInTimeline: true,
|
||||
},
|
||||
],
|
||||
timelineTracks: [
|
||||
{
|
||||
trackId: "text-track",
|
||||
type: "text",
|
||||
position: 0,
|
||||
visualLayer: 1,
|
||||
isVisualLayer: true,
|
||||
stacking: "top",
|
||||
elements: [
|
||||
{
|
||||
elementId: "caption-1",
|
||||
type: "text",
|
||||
name: "Caption",
|
||||
content: "Hello Gemini",
|
||||
start: 1,
|
||||
end: 3,
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
trackId: "main-track",
|
||||
type: "main",
|
||||
position: 1,
|
||||
visualLayer: 0,
|
||||
isVisualLayer: true,
|
||||
stacking: "main",
|
||||
elements: [
|
||||
{
|
||||
elementId: "clip-1",
|
||||
type: "video",
|
||||
assetId: "asset-video",
|
||||
name: "Intro clip",
|
||||
start: 5,
|
||||
end: 15,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
playbackTimeMs: 0,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function mockGeminiUploadResponse() {
|
||||
globalThis.fetch = mock(() =>
|
||||
Promise.resolve(
|
||||
new Response(
|
||||
JSON.stringify({
|
||||
provider: "gemini",
|
||||
status: "loaded",
|
||||
fileName: "files/abc",
|
||||
fileUri: "gemini://files/abc",
|
||||
mimeType: "video/mp4",
|
||||
displayName: "intro.mp4",
|
||||
}),
|
||||
{ status: 200, headers: { "Content-Type": "application/json" } },
|
||||
),
|
||||
),
|
||||
) as unknown as typeof fetch;
|
||||
}
|
||||
|
||||
describe("load_context tool", () => {
|
||||
beforeEach(() => {
|
||||
mockResolveAssetFile.mockReset();
|
||||
globalThis.fetch = originalFetch;
|
||||
});
|
||||
|
||||
test("is registered in the tool registry", () => {
|
||||
expect(toolRegistry.has("load_context")).toBe(true);
|
||||
});
|
||||
|
||||
test("loads text timeline elements as exact structured context", async () => {
|
||||
const tool = toolRegistry.get("load_context");
|
||||
const result = await tool.execute(
|
||||
{
|
||||
targetType: "timeline_element",
|
||||
trackId: "text-track",
|
||||
elementId: "caption-1",
|
||||
},
|
||||
makeContext(),
|
||||
);
|
||||
|
||||
expect(result).toEqual({
|
||||
targetType: "timeline_element",
|
||||
id: "caption-1",
|
||||
status: "loaded",
|
||||
cached: false,
|
||||
provider: "gemini",
|
||||
context: {
|
||||
kind: "text",
|
||||
trackId: "text-track",
|
||||
elementId: "caption-1",
|
||||
type: "text",
|
||||
name: "Caption",
|
||||
content: "Hello Gemini",
|
||||
start: 1,
|
||||
end: 3,
|
||||
},
|
||||
});
|
||||
expect(mockResolveAssetFile).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test("uploads media assets through the server endpoint", async () => {
|
||||
const file = new File(["fake"], "intro.mp4", { type: "video/mp4" });
|
||||
mockResolveAssetFile.mockReturnValue(file);
|
||||
mockGeminiUploadResponse();
|
||||
|
||||
const tool = toolRegistry.get("load_context");
|
||||
const result = await tool.execute(
|
||||
{ targetType: "asset", assetId: "asset-video" },
|
||||
makeContext(),
|
||||
);
|
||||
|
||||
expect(result).toEqual({
|
||||
targetType: "asset",
|
||||
id: "asset-video",
|
||||
status: "loaded",
|
||||
cached: false,
|
||||
provider: "gemini",
|
||||
context: {
|
||||
kind: "media",
|
||||
assetId: "asset-video",
|
||||
assetName: "intro.mp4",
|
||||
assetType: "video",
|
||||
duration: 30,
|
||||
fileUri: "gemini://files/abc",
|
||||
mimeType: "video/mp4",
|
||||
fileName: "files/abc",
|
||||
displayName: "intro.mp4",
|
||||
},
|
||||
});
|
||||
expect(mockResolveAssetFile).toHaveBeenCalledWith("asset-video");
|
||||
});
|
||||
|
||||
test("sends inferred MIME type when the file type is octet-stream", async () => {
|
||||
const file = new File(["fake"], "stored-asset", {
|
||||
type: "application/octet-stream",
|
||||
});
|
||||
mockResolveAssetFile.mockReturnValue(file);
|
||||
|
||||
const uploaded: { mimeType: FormDataEntryValue | null } = { mimeType: null };
|
||||
globalThis.fetch = mock((_, init) => {
|
||||
uploaded.mimeType =
|
||||
init?.body instanceof FormData ? init.body.get("mimeType") : null;
|
||||
return Promise.resolve(
|
||||
new Response(
|
||||
JSON.stringify({
|
||||
provider: "gemini",
|
||||
status: "loaded",
|
||||
fileName: "files/abc",
|
||||
fileUri: "gemini://files/abc",
|
||||
mimeType: "video/quicktime",
|
||||
displayName: "conclu.mov",
|
||||
}),
|
||||
{ status: 200, headers: { "Content-Type": "application/json" } },
|
||||
),
|
||||
);
|
||||
}) as unknown as typeof fetch;
|
||||
|
||||
const tool = toolRegistry.get("load_context");
|
||||
const result = await tool.execute(
|
||||
{ targetType: "asset", assetId: "asset-video-octet" },
|
||||
makeContext({
|
||||
mediaAssets: [
|
||||
{
|
||||
id: "asset-video-octet",
|
||||
name: "conclu.mov",
|
||||
type: "video",
|
||||
duration: 30,
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
expect(uploaded.mimeType).toBe("video/quicktime");
|
||||
expect(result).toEqual(
|
||||
expect.objectContaining({
|
||||
context: expect.objectContaining({ mimeType: "video/quicktime" }),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
test("loads media context for timeline elements with assetId", async () => {
|
||||
const file = new File(["fake"], "intro.mp4", { type: "video/mp4" });
|
||||
mockResolveAssetFile.mockReturnValue(file);
|
||||
mockGeminiUploadResponse();
|
||||
|
||||
const tool = toolRegistry.get("load_context");
|
||||
const result = await tool.execute(
|
||||
{
|
||||
targetType: "timeline_element",
|
||||
trackId: "main-track",
|
||||
elementId: "clip-1",
|
||||
},
|
||||
makeContext({
|
||||
mediaAssets: [
|
||||
{
|
||||
id: "asset-video-2",
|
||||
name: "other.mp4",
|
||||
type: "video",
|
||||
duration: 20,
|
||||
},
|
||||
],
|
||||
timelineTracks: [
|
||||
{
|
||||
trackId: "main-track",
|
||||
type: "main",
|
||||
position: 0,
|
||||
visualLayer: 0,
|
||||
isVisualLayer: true,
|
||||
stacking: "main",
|
||||
elements: [
|
||||
{
|
||||
elementId: "clip-1",
|
||||
type: "video",
|
||||
assetId: "asset-video-2",
|
||||
start: 5,
|
||||
end: 15,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
expect(result).toEqual(
|
||||
expect.objectContaining({
|
||||
targetType: "asset",
|
||||
id: "asset-video-2",
|
||||
context: expect.objectContaining({
|
||||
kind: "media",
|
||||
assetId: "asset-video-2",
|
||||
element: {
|
||||
trackId: "main-track",
|
||||
elementId: "clip-1",
|
||||
type: "video",
|
||||
start: 5,
|
||||
end: 15,
|
||||
},
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
test("validates missing timeline identifiers", async () => {
|
||||
const tool = toolRegistry.get("load_context");
|
||||
const result = await tool.execute(
|
||||
{ targetType: "timeline_element", elementId: "caption-1" },
|
||||
makeContext(),
|
||||
);
|
||||
|
||||
expect(result).toEqual({ error: "trackId and elementId are required" });
|
||||
});
|
||||
});
|
||||
|
|
@ -12,7 +12,7 @@ export type ListTimelineResult = {
|
|||
const listTimelineTool: ToolDefinition = {
|
||||
name: "list_timeline",
|
||||
description:
|
||||
"Lists the active timeline as structured tracks and editable elements with trackId, elementId, type, assetId, name, start, and end times.",
|
||||
"Lists the active timeline as structured tracks and editable elements with layer metadata. position is timeline row order from top to bottom; visualLayer is render stacking where higher renders above lower, and audio tracks have null visualLayer.",
|
||||
parameters: [],
|
||||
execute: async (
|
||||
_args: Record<string, unknown>,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,391 @@
|
|||
import type {
|
||||
AgentContext,
|
||||
AgentTimelineTrack,
|
||||
ToolDefinition,
|
||||
} from "@/agent/types";
|
||||
import { EditorContextAdapter } from "@/agent/context";
|
||||
import { toolRegistry } from "@/agent/tools/registry";
|
||||
|
||||
type LoadContextTargetType = "asset" | "timeline_element";
|
||||
|
||||
export type LoadContextArgs = {
|
||||
targetType?: LoadContextTargetType;
|
||||
id?: string;
|
||||
assetId?: string;
|
||||
trackId?: string;
|
||||
elementId?: string;
|
||||
};
|
||||
|
||||
export type LoadContextResult = {
|
||||
targetType: LoadContextTargetType;
|
||||
id: string;
|
||||
status: "loaded" | "processing";
|
||||
cached: boolean;
|
||||
provider: "gemini";
|
||||
context: TextContext | MediaContext;
|
||||
};
|
||||
|
||||
type TextContext = {
|
||||
kind: "text";
|
||||
trackId: string;
|
||||
elementId: string;
|
||||
type: string;
|
||||
name?: string;
|
||||
content: string;
|
||||
start: number;
|
||||
end: number;
|
||||
};
|
||||
|
||||
type MediaContext = {
|
||||
kind: "media";
|
||||
assetId: string;
|
||||
assetName: string;
|
||||
assetType: string;
|
||||
duration: number;
|
||||
fileUri: string;
|
||||
mimeType: string;
|
||||
fileName?: string;
|
||||
displayName?: string;
|
||||
element?: {
|
||||
trackId: string;
|
||||
elementId: string;
|
||||
type: string;
|
||||
start: number;
|
||||
end: number;
|
||||
};
|
||||
};
|
||||
|
||||
type UploadedGeminiFile = {
|
||||
provider: "gemini";
|
||||
status: "loaded" | "processing";
|
||||
fileName?: string;
|
||||
fileUri: string;
|
||||
mimeType: string;
|
||||
displayName?: string;
|
||||
};
|
||||
|
||||
const assetContextCache = new Map<string, UploadedGeminiFile>();
|
||||
const textContextCache = new Map<string, TextContext>();
|
||||
|
||||
const loadContextTool: ToolDefinition = {
|
||||
name: "load_context",
|
||||
description:
|
||||
"Loads relevant Gemini context for a project asset or timeline element. Media assets are uploaded through the server-side Gemini Files API; text/caption elements are loaded as exact structured text.",
|
||||
parameters: [
|
||||
{ key: "targetType", type: "string", required: true },
|
||||
{ key: "id", type: "string", required: false },
|
||||
{ key: "assetId", type: "string", required: false },
|
||||
{ key: "trackId", type: "string", required: false },
|
||||
{ key: "elementId", type: "string", required: false },
|
||||
],
|
||||
execute: async (
|
||||
args: Record<string, unknown>,
|
||||
context: AgentContext,
|
||||
): Promise<LoadContextResult | { error: string }> => {
|
||||
const typedArgs = args as LoadContextArgs;
|
||||
|
||||
if (typedArgs.targetType === "asset") {
|
||||
return loadAssetContext(typedArgs, context);
|
||||
}
|
||||
|
||||
if (typedArgs.targetType === "timeline_element") {
|
||||
return loadTimelineElementContext(typedArgs, context);
|
||||
}
|
||||
|
||||
return { error: "Unsupported context target type" };
|
||||
},
|
||||
};
|
||||
|
||||
async function loadAssetContext(
|
||||
args: LoadContextArgs,
|
||||
context: AgentContext,
|
||||
element?: TimelineElementRef,
|
||||
): Promise<LoadContextResult | { error: string }> {
|
||||
const assetId = args.assetId ?? args.id ?? element?.assetId;
|
||||
if (!assetId) {
|
||||
return { error: "Asset id is required" };
|
||||
}
|
||||
|
||||
const asset = context.mediaAssets.find(
|
||||
(mediaAsset) => mediaAsset.id === assetId,
|
||||
);
|
||||
if (!asset) {
|
||||
return { error: "Asset not found" };
|
||||
}
|
||||
|
||||
if (!isSupportedMediaAssetType(asset.type)) {
|
||||
return { error: "Unsupported asset type" };
|
||||
}
|
||||
|
||||
const cached = assetContextCache.get(asset.id);
|
||||
if (cached) {
|
||||
return {
|
||||
targetType: "asset",
|
||||
id: asset.id,
|
||||
status: cached.status,
|
||||
cached: true,
|
||||
provider: "gemini",
|
||||
context: toMediaContext({ asset, uploaded: cached, element }),
|
||||
};
|
||||
}
|
||||
|
||||
const file = EditorContextAdapter.resolveAssetFile(asset.id);
|
||||
if (!file) {
|
||||
return { error: `Could not access file for asset ${asset.id}` };
|
||||
}
|
||||
|
||||
const uploaded = await uploadFileToGemini({ file, asset });
|
||||
if ("error" in uploaded) {
|
||||
return uploaded;
|
||||
}
|
||||
|
||||
assetContextCache.set(asset.id, uploaded);
|
||||
|
||||
return {
|
||||
targetType: "asset",
|
||||
id: asset.id,
|
||||
status: uploaded.status,
|
||||
cached: false,
|
||||
provider: "gemini",
|
||||
context: toMediaContext({ asset, uploaded, element }),
|
||||
};
|
||||
}
|
||||
|
||||
async function loadTimelineElementContext(
|
||||
args: LoadContextArgs,
|
||||
context: AgentContext,
|
||||
): Promise<LoadContextResult | { error: string }> {
|
||||
const trackId = args.trackId;
|
||||
const elementId = args.elementId ?? args.id;
|
||||
|
||||
if (!trackId || !elementId) {
|
||||
return { error: "trackId and elementId are required" };
|
||||
}
|
||||
|
||||
const element = findTimelineElement(
|
||||
context.timelineTracks,
|
||||
trackId,
|
||||
elementId,
|
||||
);
|
||||
if (!element) {
|
||||
return { error: "Timeline element not found" };
|
||||
}
|
||||
|
||||
if (element.element.content !== undefined) {
|
||||
const cacheKey = `${trackId}:${elementId}`;
|
||||
const cached = textContextCache.get(cacheKey);
|
||||
if (cached) {
|
||||
return {
|
||||
targetType: "timeline_element",
|
||||
id: elementId,
|
||||
status: "loaded",
|
||||
cached: true,
|
||||
provider: "gemini",
|
||||
context: cached,
|
||||
};
|
||||
}
|
||||
|
||||
const textContext: TextContext = {
|
||||
kind: "text",
|
||||
trackId,
|
||||
elementId,
|
||||
type: element.element.type,
|
||||
...(element.element.name ? { name: element.element.name } : {}),
|
||||
content: element.element.content,
|
||||
start: element.element.start,
|
||||
end: element.element.end,
|
||||
};
|
||||
|
||||
textContextCache.set(cacheKey, textContext);
|
||||
|
||||
return {
|
||||
targetType: "timeline_element",
|
||||
id: elementId,
|
||||
status: "loaded",
|
||||
cached: false,
|
||||
provider: "gemini",
|
||||
context: textContext,
|
||||
};
|
||||
}
|
||||
|
||||
if (element.element.assetId) {
|
||||
return loadAssetContext(
|
||||
{ targetType: "asset", assetId: element.element.assetId },
|
||||
context,
|
||||
{
|
||||
assetId: element.element.assetId,
|
||||
trackId,
|
||||
elementId,
|
||||
type: element.element.type,
|
||||
start: element.element.start,
|
||||
end: element.element.end,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
return { error: "Timeline element has no loadable context" };
|
||||
}
|
||||
|
||||
async function uploadFileToGemini({
|
||||
file,
|
||||
asset,
|
||||
}: {
|
||||
file: File;
|
||||
asset: AgentContext["mediaAssets"][number];
|
||||
}): Promise<UploadedGeminiFile | { error: string }> {
|
||||
const mimeType = resolveGeminiMimeType({
|
||||
fileName: file.name,
|
||||
displayName: asset.name,
|
||||
fileType: file.type,
|
||||
assetType: asset.type,
|
||||
});
|
||||
|
||||
if (!mimeType) {
|
||||
return { error: `Unsupported MIME type for asset ${asset.id}` };
|
||||
}
|
||||
|
||||
const formData = new FormData();
|
||||
formData.set("file", file);
|
||||
formData.set("displayName", asset.name);
|
||||
formData.set("mimeType", mimeType);
|
||||
|
||||
const response = await fetch("/api/agent/context/load", {
|
||||
method: "POST",
|
||||
body: formData,
|
||||
});
|
||||
|
||||
const data = (await response.json()) as Partial<UploadedGeminiFile> & {
|
||||
error?: string;
|
||||
};
|
||||
|
||||
if (!response.ok) {
|
||||
return { error: data.error ?? "Failed to load context" };
|
||||
}
|
||||
|
||||
if (!data.fileUri || !data.mimeType || !data.status) {
|
||||
return { error: "Invalid context load response" };
|
||||
}
|
||||
|
||||
return {
|
||||
provider: "gemini",
|
||||
status: data.status,
|
||||
fileName: data.fileName,
|
||||
fileUri: data.fileUri,
|
||||
mimeType: data.mimeType,
|
||||
displayName: data.displayName,
|
||||
};
|
||||
}
|
||||
|
||||
function resolveGeminiMimeType({
|
||||
fileName,
|
||||
displayName,
|
||||
fileType,
|
||||
assetType,
|
||||
}: {
|
||||
fileName: string;
|
||||
displayName: string;
|
||||
fileType?: string;
|
||||
assetType: string;
|
||||
}): string | null {
|
||||
if (fileType && fileType !== "application/octet-stream") {
|
||||
return fileType;
|
||||
}
|
||||
|
||||
const mimeType = inferMimeTypeFromName(displayName) ?? inferMimeTypeFromName(fileName);
|
||||
if (mimeType) {
|
||||
return mimeType;
|
||||
}
|
||||
|
||||
if (assetType === "image") {
|
||||
return "image/png";
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function inferMimeTypeFromName(name: string): string | null {
|
||||
const extension = name.split(".").pop()?.toLowerCase();
|
||||
return extension ? (MIME_BY_EXTENSION[extension] ?? null) : null;
|
||||
}
|
||||
|
||||
const MIME_BY_EXTENSION: Record<string, string> = {
|
||||
avi: "video/x-msvideo",
|
||||
flac: "audio/flac",
|
||||
gif: "image/gif",
|
||||
jpeg: "image/jpeg",
|
||||
jpg: "image/jpeg",
|
||||
m4a: "audio/mp4",
|
||||
mov: "video/quicktime",
|
||||
mp3: "audio/mpeg",
|
||||
mp4: "video/mp4",
|
||||
mpeg: "video/mpeg",
|
||||
mpg: "video/mpeg",
|
||||
png: "image/png",
|
||||
wav: "audio/wav",
|
||||
webm: "video/webm",
|
||||
webp: "image/webp",
|
||||
};
|
||||
|
||||
function toMediaContext({
|
||||
asset,
|
||||
uploaded,
|
||||
element,
|
||||
}: {
|
||||
asset: AgentContext["mediaAssets"][number];
|
||||
uploaded: UploadedGeminiFile;
|
||||
element?: TimelineElementRef;
|
||||
}): MediaContext {
|
||||
return {
|
||||
kind: "media",
|
||||
assetId: asset.id,
|
||||
assetName: asset.name,
|
||||
assetType: asset.type,
|
||||
duration: asset.duration,
|
||||
fileUri: uploaded.fileUri,
|
||||
mimeType: uploaded.mimeType,
|
||||
...(uploaded.fileName ? { fileName: uploaded.fileName } : {}),
|
||||
...(uploaded.displayName ? { displayName: uploaded.displayName } : {}),
|
||||
...(element
|
||||
? {
|
||||
element: {
|
||||
trackId: element.trackId,
|
||||
elementId: element.elementId,
|
||||
type: element.type,
|
||||
start: element.start,
|
||||
end: element.end,
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
function findTimelineElement(
|
||||
tracks: AgentTimelineTrack[] | undefined,
|
||||
trackId: string,
|
||||
elementId: string,
|
||||
): {
|
||||
track: AgentTimelineTrack;
|
||||
element: AgentTimelineTrack["elements"][number];
|
||||
} | null {
|
||||
const track = tracks?.find((candidate) => candidate.trackId === trackId);
|
||||
const element = track?.elements.find(
|
||||
(candidate) => candidate.elementId === elementId,
|
||||
);
|
||||
|
||||
return track && element ? { track, element } : null;
|
||||
}
|
||||
|
||||
function isSupportedMediaAssetType(type: string): boolean {
|
||||
return type === "video" || type === "audio" || type === "image";
|
||||
}
|
||||
|
||||
type TimelineElementRef = {
|
||||
assetId: string;
|
||||
trackId: string;
|
||||
elementId: string;
|
||||
type: string;
|
||||
start: number;
|
||||
end: number;
|
||||
};
|
||||
|
||||
toolRegistry.register("load_context", loadContextTool);
|
||||
|
|
@ -19,6 +19,7 @@ export interface ToolCall {
|
|||
id: string;
|
||||
name: string;
|
||||
args: Record<string, unknown>;
|
||||
thoughtSignature?: string;
|
||||
}
|
||||
|
||||
export interface ToolResult {
|
||||
|
|
@ -45,11 +46,18 @@ export interface AgentContext {
|
|||
export type AgentTimelineTrack = {
|
||||
trackId: string;
|
||||
type: "main" | "overlay" | "audio" | "text" | "effect";
|
||||
/** Timeline row position, top-to-bottom. 0 is the top visible row. */
|
||||
position: number;
|
||||
/** Visual stacking layer. Higher numbers render above lower numbers. Audio is null. */
|
||||
visualLayer: number | null;
|
||||
isVisualLayer: boolean;
|
||||
stacking: "top" | "above_main" | "main" | "audio";
|
||||
elements: Array<{
|
||||
elementId: string;
|
||||
type: string;
|
||||
assetId?: string;
|
||||
name?: string;
|
||||
content?: string;
|
||||
start: number;
|
||||
end: number;
|
||||
}>;
|
||||
|
|
|
|||
|
|
@ -40,11 +40,16 @@ const VALID_CONTEXT = {
|
|||
| Array<{
|
||||
trackId: string;
|
||||
type: "main" | "overlay" | "audio" | "text" | "effect";
|
||||
position: number;
|
||||
visualLayer: number | null;
|
||||
isVisualLayer: boolean;
|
||||
stacking: "top" | "above_main" | "main" | "audio";
|
||||
elements: Array<{
|
||||
elementId: string;
|
||||
type: string;
|
||||
assetId?: string;
|
||||
name?: string;
|
||||
content?: string;
|
||||
start: number;
|
||||
end: number;
|
||||
}>;
|
||||
|
|
@ -150,7 +155,11 @@ describe("POST /api/agent/chat", () => {
|
|||
expect(callArgs.tools).toHaveLength(3);
|
||||
expect(
|
||||
callArgs.tools.map((tool) => (tool as { name: string }).name),
|
||||
).toEqual(["list_project_assets", "list_timeline", "transcribe_video"]);
|
||||
).toEqual([
|
||||
"load_context",
|
||||
"list_project_assets",
|
||||
"list_timeline",
|
||||
]);
|
||||
});
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -1,16 +1,27 @@
|
|||
import { type NextRequest, NextResponse } from "next/server";
|
||||
import { z } from "zod";
|
||||
import type { AgentContext, ToolDefinition, ToolSchema } from "@/agent/types";
|
||||
import type { AgentContext, ToolSchema } from "@/agent/types";
|
||||
import { buildSystemPrompt } from "@/agent/system-prompt";
|
||||
import { createProvider } from "@/agent/providers";
|
||||
import type { ProviderConfig } from "@/agent/providers";
|
||||
import { toToolSchemas } from "@/agent/tools/registry";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Provider-facing tool schemas (execution is client-side)
|
||||
// echo_context is intentionally excluded — only user-facing tools are exposed.
|
||||
// Provider-facing tool schemas. Execution is client-side.
|
||||
// echo_context is intentionally excluded; only user-facing tools are exposed.
|
||||
// ---------------------------------------------------------------------------
|
||||
const providerToolDefs: ToolDefinition[] = [
|
||||
const providerToolSchemas: ToolSchema[] = [
|
||||
{
|
||||
name: "load_context",
|
||||
description:
|
||||
"Loads the actual Gemini multimodal context for a project asset or timeline element. Use this before answering questions about visible objects, colors, scenes, speech, silence, or timestamps in media. Use targetType='asset' with assetId/id for media, or targetType='timeline_element' with trackId and elementId/id for captions, text, or timeline media elements.",
|
||||
parameters: [
|
||||
{ key: "targetType", type: "string", required: true },
|
||||
{ key: "id", type: "string", required: false },
|
||||
{ key: "assetId", type: "string", required: false },
|
||||
{ key: "trackId", type: "string", required: false },
|
||||
{ key: "elementId", type: "string", required: false },
|
||||
],
|
||||
},
|
||||
{
|
||||
name: "list_project_assets",
|
||||
description:
|
||||
|
|
@ -19,25 +30,12 @@ const providerToolDefs: ToolDefinition[] = [
|
|||
{ key: "filter", type: "string", required: false },
|
||||
{ key: "type", type: "string", required: false },
|
||||
],
|
||||
execute: async () => ({}), // stub — never called server-side
|
||||
},
|
||||
{
|
||||
name: "list_timeline",
|
||||
description:
|
||||
"Lists the active timeline as structured tracks and editable elements with trackId, elementId, type, assetId, name, start, and end times.",
|
||||
"Lists the active timeline as structured tracks and editable elements with layer metadata. Tracks include position (top-to-bottom timeline row), visualLayer (higher renders above lower, null for audio), isVisualLayer, and stacking. Use this to understand which clips visually cover others and to discover ids before load_context.",
|
||||
parameters: [],
|
||||
execute: async () => ({}), // stub — never called server-side
|
||||
},
|
||||
{
|
||||
name: "transcribe_video",
|
||||
description:
|
||||
"Transcribes the audio of a video or audio asset using Whisper. Returns structured transcript with segments and timestamps.",
|
||||
parameters: [
|
||||
{ key: "assetId", type: "string", required: false },
|
||||
{ key: "language", type: "string", required: false },
|
||||
{ key: "modelId", type: "string", required: false },
|
||||
],
|
||||
execute: async () => ({}), // stub — never called server-side
|
||||
},
|
||||
];
|
||||
|
||||
|
|
@ -48,6 +46,7 @@ const toolCallSchema = z.object({
|
|||
id: z.string(),
|
||||
name: z.string(),
|
||||
args: z.record(z.string(), z.unknown()),
|
||||
thoughtSignature: z.string().optional(),
|
||||
});
|
||||
|
||||
const chatRequestSchema = z.object({
|
||||
|
|
@ -78,12 +77,17 @@ const chatRequestSchema = z.object({
|
|||
z.object({
|
||||
trackId: z.string(),
|
||||
type: z.enum(["main", "overlay", "audio", "text", "effect"]),
|
||||
position: z.number(),
|
||||
visualLayer: z.number().nullable(),
|
||||
isVisualLayer: z.boolean(),
|
||||
stacking: z.enum(["top", "above_main", "main", "audio"]),
|
||||
elements: z.array(
|
||||
z.object({
|
||||
elementId: z.string(),
|
||||
type: z.string(),
|
||||
assetId: z.string().optional(),
|
||||
name: z.string().optional(),
|
||||
content: z.string().optional(),
|
||||
start: z.number(),
|
||||
end: z.number(),
|
||||
}),
|
||||
|
|
@ -103,8 +107,15 @@ export async function POST(request: NextRequest) {
|
|||
const result = chatRequestSchema.safeParse(body);
|
||||
|
||||
if (!result.success) {
|
||||
const fieldErrors = result.error.flatten().fieldErrors;
|
||||
if (process.env.NODE_ENV === "development") {
|
||||
console.error(
|
||||
"[agent/chat] Validation failed:",
|
||||
JSON.stringify({ fieldErrors }, null, 2),
|
||||
);
|
||||
}
|
||||
return NextResponse.json(
|
||||
{ error: "Invalid input", details: result.error.flatten().fieldErrors },
|
||||
{ error: "Invalid input", details: fieldErrors },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
|
|
@ -130,8 +141,7 @@ export async function POST(request: NextRequest) {
|
|||
const config: ProviderConfig = { provider, apiKey, model, baseUrl };
|
||||
|
||||
// Build system prompt with tool guidance
|
||||
const toolSchemas: ToolSchema[] = toToolSchemas(providerToolDefs);
|
||||
const systemPrompt = buildSystemPrompt(context, providerToolDefs);
|
||||
const systemPrompt = buildSystemPrompt(context, providerToolSchemas);
|
||||
|
||||
// Delegate to provider adapter
|
||||
try {
|
||||
|
|
@ -139,7 +149,7 @@ export async function POST(request: NextRequest) {
|
|||
const response = await adapter.chat({
|
||||
messages,
|
||||
systemPrompt,
|
||||
tools: toolSchemas,
|
||||
tools: providerToolSchemas,
|
||||
});
|
||||
|
||||
return NextResponse.json({
|
||||
|
|
@ -147,7 +157,19 @@ export async function POST(request: NextRequest) {
|
|||
...(response.toolCalls && { toolCalls: response.toolCalls }),
|
||||
});
|
||||
} catch (error) {
|
||||
const status = (error as { status?: number }).status;
|
||||
const message =
|
||||
error instanceof Error ? error.message : "Unknown provider error";
|
||||
console.error("[agent/chat] Provider error:", error);
|
||||
return NextResponse.json({ error: "LLM provider error" }, { status: 502 });
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: "LLM provider error",
|
||||
...(status ? { providerStatus: status } : {}),
|
||||
...(process.env.NODE_ENV === "development"
|
||||
? { detail: message }
|
||||
: {}),
|
||||
},
|
||||
{ status: 502 },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,137 @@
|
|||
import { type NextRequest, NextResponse } from "next/server";
|
||||
import { FileState, GoogleAIFileManager } from "@google/generative-ai/server";
|
||||
|
||||
const MAX_PROCESSING_POLLS = 12;
|
||||
const PROCESSING_POLL_INTERVAL_MS = 5000;
|
||||
|
||||
export async function POST(request: NextRequest) {
|
||||
const apiKey = process.env.GEMINI_API_KEY ?? process.env.LLM_API_KEY;
|
||||
if (!apiKey) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error:
|
||||
"Server configuration error: missing GEMINI_API_KEY or LLM_API_KEY",
|
||||
},
|
||||
{ status: 502 },
|
||||
);
|
||||
}
|
||||
|
||||
const formData = await request.formData();
|
||||
const file = formData.get("file");
|
||||
const displayName = formData.get("displayName");
|
||||
const requestedMimeType = formData.get("mimeType");
|
||||
|
||||
if (!(file instanceof File)) {
|
||||
return NextResponse.json({ error: "File is required" }, { status: 400 });
|
||||
}
|
||||
|
||||
try {
|
||||
const mimeType = resolveGeminiMimeType({
|
||||
fileName: file.name,
|
||||
fileType: file.type,
|
||||
displayName: typeof displayName === "string" ? displayName : undefined,
|
||||
requestedMimeType:
|
||||
typeof requestedMimeType === "string" ? requestedMimeType : undefined,
|
||||
});
|
||||
|
||||
if (!mimeType) {
|
||||
return NextResponse.json(
|
||||
{ error: "Unsupported MIME type" },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
|
||||
const fileManager = new GoogleAIFileManager(apiKey);
|
||||
const uploadResult = await fileManager.uploadFile(
|
||||
Buffer.from(await file.arrayBuffer()),
|
||||
{
|
||||
mimeType,
|
||||
displayName:
|
||||
typeof displayName === "string" && displayName
|
||||
? displayName
|
||||
: file.name,
|
||||
},
|
||||
);
|
||||
|
||||
let uploadedFile = uploadResult.file;
|
||||
for (let i = 0; i < MAX_PROCESSING_POLLS; i++) {
|
||||
if (uploadedFile.state !== FileState.PROCESSING) {
|
||||
break;
|
||||
}
|
||||
|
||||
await wait(PROCESSING_POLL_INTERVAL_MS);
|
||||
uploadedFile = await fileManager.getFile(uploadedFile.name);
|
||||
}
|
||||
|
||||
if (uploadedFile.state === FileState.FAILED) {
|
||||
return NextResponse.json(
|
||||
{ error: "Failed to load context" },
|
||||
{ status: 502 },
|
||||
);
|
||||
}
|
||||
|
||||
return NextResponse.json({
|
||||
provider: "gemini",
|
||||
status:
|
||||
uploadedFile.state === FileState.PROCESSING ? "processing" : "loaded",
|
||||
fileName: uploadedFile.name,
|
||||
fileUri: uploadedFile.uri,
|
||||
mimeType: uploadedFile.mimeType || mimeType,
|
||||
displayName: uploadedFile.displayName,
|
||||
});
|
||||
} catch (error) {
|
||||
console.error("[agent/context/load] Gemini upload error:", error);
|
||||
return NextResponse.json(
|
||||
{ error: "Failed to load context" },
|
||||
{ status: 502 },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function wait(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
function resolveGeminiMimeType({
|
||||
fileName,
|
||||
fileType,
|
||||
displayName,
|
||||
requestedMimeType,
|
||||
}: {
|
||||
fileName: string;
|
||||
fileType?: string;
|
||||
displayName?: string;
|
||||
requestedMimeType?: string;
|
||||
}): string | null {
|
||||
for (const candidate of [requestedMimeType, fileType]) {
|
||||
if (candidate && candidate !== "application/octet-stream") {
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
|
||||
const mimeType = inferMimeTypeFromName(displayName) ?? inferMimeTypeFromName(fileName);
|
||||
return mimeType ?? null;
|
||||
}
|
||||
|
||||
function inferMimeTypeFromName(name?: string): string | null {
|
||||
const extension = name?.split(".").pop()?.toLowerCase();
|
||||
return extension ? (MIME_BY_EXTENSION[extension] ?? null) : null;
|
||||
}
|
||||
|
||||
const MIME_BY_EXTENSION: Record<string, string> = {
|
||||
avi: "video/x-msvideo",
|
||||
flac: "audio/flac",
|
||||
gif: "image/gif",
|
||||
jpeg: "image/jpeg",
|
||||
jpg: "image/jpeg",
|
||||
m4a: "audio/mp4",
|
||||
mov: "video/quicktime",
|
||||
mp3: "audio/mpeg",
|
||||
mp4: "video/mp4",
|
||||
mpeg: "video/mpeg",
|
||||
mpg: "video/mpeg",
|
||||
png: "image/png",
|
||||
wav: "audio/wav",
|
||||
webm: "video/webm",
|
||||
webp: "image/webp",
|
||||
};
|
||||
Loading…
Reference in New Issue