diff --git a/apps/web/src/agent/context.ts b/apps/web/src/agent/context.ts index bd268634..3ca8816e 100644 --- a/apps/web/src/agent/context.ts +++ b/apps/web/src/agent/context.ts @@ -4,13 +4,21 @@ import { buildContextFromEditorState } from "@/agent/context-mapper"; import { BatchCommand } from "@/lib/commands"; import { AddTrackCommand, InsertElementCommand } from "@/lib/commands/timeline"; import { DEFAULT_NEW_ELEMENT_DURATION } from "@/lib/timeline/creation"; -import { buildElementFromMedia } from "@/lib/timeline/element-utils"; +import { + buildElementFromMedia, + buildTextElement, +} from "@/lib/timeline/element-utils"; +import { DEFAULTS } from "@/lib/timeline/defaults"; import type { SceneTracks, + TextBackground, + TextElement, TimelineElement, TimelineTrack, TrackType, } from "@/lib/timeline"; +import type { TextStyleOverrides } from "@/agent/tools/add-text.tool"; +import type { UpdateTextArgs } from "@/agent/tools/update-text.tool"; import { canPlaceTimeSpansOnTrack } from "@/lib/timeline/placement/overlap"; import { validateElementTrackCompatibility } from "@/lib/timeline/placement"; import { findTrackInSceneTracks } from "@/lib/timeline/track-element-update"; @@ -367,6 +375,176 @@ export const EditorContextAdapter = { duration: ticksToSeconds(element.duration), }; }, + + addText({ + text, + start, + end, + position, + style = "plain", + color, + fontSize, + fontFamily, + fontWeight, + fontStyle, + textAlign, + letterSpacing, + positionX, + positionY, + background, + }: { + text: string; + start: number; + end: number; + position: "top" | "center" | "bottom"; + style?: "plain" | "subtitle" | "hook" | "label"; + } & TextStyleOverrides): + | { elementId: string; trackId: string } + | { error: string } { + const core = EditorCore.getInstance(); + const activeScene = core.scenes.getActiveSceneOrNull(); + if (!activeScene) { + return { error: "No active timeline" }; + } + + const startTicks = secondsToTicks(start); + const duration = secondsToTicks(end) - startTicks; + if (duration <= 0) { + return { error: "Invalid time range" }; + } + + const preset = getTextStylePreset(style); + const resolvedPosition = getTextPosition(position); + const overridePosition = + positionX !== undefined || positionY !== undefined + ? { + x: positionX ?? resolvedPosition.x, + y: positionY ?? resolvedPosition.y, + } + : resolvedPosition; + + const resolvedBackground = resolveBackground(preset.background, background); + + const element = buildTextElement({ + raw: { + ...preset, + ...(color !== undefined && { color }), + ...(fontSize !== undefined && { fontSize }), + ...(fontFamily !== undefined && { fontFamily }), + ...(fontWeight !== undefined && { fontWeight }), + ...(fontStyle !== undefined && { fontStyle }), + ...(textAlign !== undefined && { textAlign }), + ...(letterSpacing !== undefined && { letterSpacing }), + background: resolvedBackground, + name: text, + content: text, + duration, + transform: { + ...DEFAULTS.text.element.transform, + position: overridePosition, + }, + }, + startTime: startTicks, + }); + + const insertCommand = new InsertElementCommand({ + element, + placement: { mode: "auto", trackType: "text" }, + }); + core.command.execute({ command: insertCommand }); + + const trackId = insertCommand.getTrackId(); + if (!trackId) { + return { error: "Failed to add text" }; + } + + return { + elementId: insertCommand.getElementId(), + trackId, + }; + }, + + updateText({ + elementIds, + content, + color, + fontSize, + fontFamily, + fontWeight, + fontStyle, + textAlign, + letterSpacing, + positionX, + positionY, + background, + }: { + elementIds: string[]; + } & Omit): + | { + success: boolean; + updated: Array<{ elementId: string; trackId: string }>; + skipped: string[]; + } + | { error: string } { + const core = EditorCore.getInstance(); + const activeScene = core.scenes.getActiveSceneOrNull(); + if (!activeScene) { + return { error: "No active timeline" }; + } + + const resolved = findTimelineElementsWithTracksByIds({ + tracks: activeScene.tracks, + elementIds, + }); + + const textElements = resolved.filter( + ({ element }) => element.type === "text", + ); + const foundIds = new Set(textElements.map(({ element }) => element.id)); + const skipped = elementIds.filter((id) => !foundIds.has(id)); + + if (textElements.length === 0) { + return { + success: false, + updated: [], + skipped, + }; + } + + const updates: Array<{ + trackId: string; + elementId: string; + patch: Partial; + }> = []; + + for (const { element, track } of textElements) { + const patch = buildTextPatch(element, { + content, + color, + fontSize, + fontFamily, + fontWeight, + fontStyle, + textAlign, + letterSpacing, + positionX, + positionY, + background, + }); + updates.push({ trackId: track.id, elementId: element.id, patch }); + } + + core.timeline.updateElements({ updates }); + + return { + success: true, + updated: updates.map((u) => ({ + elementId: u.elementId, + trackId: u.trackId, + })), + skipped, + }; + }, }; function secondsToTicks(seconds: number): number { @@ -490,6 +668,78 @@ function getMaxElementDurationTicks({ ); } +function getTextPosition(position: "top" | "center" | "bottom"): { + x: number; + y: number; +} { + if (position === "top") { + return { x: 0, y: -35 }; + } + if (position === "bottom") { + return { x: 0, y: 35 }; + } + return { x: 0, y: 0 }; +} + +function resolveBackground( + presetBackground: TextBackground | undefined, + override: TextStyleOverrides["background"], +): TextBackground | undefined { + if (!override) return presetBackground; + const base = presetBackground ?? DEFAULTS.text.element.background; + return { + ...base, + enabled: override.enabled, + ...(override.color !== undefined && { color: override.color }), + ...(override.cornerRadius !== undefined && { + cornerRadius: override.cornerRadius, + }), + ...(override.padding !== undefined && { + paddingX: override.padding, + paddingY: override.padding, + }), + }; +} + +function getTextStylePreset(style: "plain" | "subtitle" | "hook" | "label") { + const noBackground = { + ...DEFAULTS.text.element.background, + enabled: false, + color: "transparent", + }; + + if (style === "subtitle") { + return { + fontSize: 5, + fontWeight: "bold" as const, + background: { + ...DEFAULTS.text.element.background, + enabled: true, + color: "#000000", + }, + }; + } + if (style === "hook") { + return { + fontSize: 8, + fontWeight: "bold" as const, + background: noBackground, + }; + } + if (style === "label") { + return { + fontSize: 5, + fontWeight: "bold" as const, + background: { + ...DEFAULTS.text.element.background, + enabled: true, + color: "#000000", + }, + }; + } + return { fontSize: 6, background: noBackground }; +} + function isAssetCompatibleWithRequestedTrack({ assetType, trackType, @@ -592,4 +842,56 @@ function findTimelineElementsWithTracksByIds({ return result; } +function buildTextPatch( + element: TimelineElement, + overrides: Omit, +): Partial { + if (element.type !== "text") return {}; + + const patch: Partial = {}; + + if (overrides.content !== undefined) patch.content = overrides.content; + if (overrides.color !== undefined) patch.color = overrides.color; + if (overrides.fontSize !== undefined) patch.fontSize = overrides.fontSize; + if (overrides.fontFamily !== undefined) + patch.fontFamily = overrides.fontFamily; + if (overrides.fontWeight !== undefined) + patch.fontWeight = overrides.fontWeight; + if (overrides.fontStyle !== undefined) patch.fontStyle = overrides.fontStyle; + if (overrides.textAlign !== undefined) patch.textAlign = overrides.textAlign; + if (overrides.letterSpacing !== undefined) + patch.letterSpacing = overrides.letterSpacing; + + if (overrides.positionX !== undefined || overrides.positionY !== undefined) { + const currentPos = element.transform.position; + patch.transform = { + ...element.transform, + position: { + x: overrides.positionX ?? currentPos.x, + y: overrides.positionY ?? currentPos.y, + }, + }; + } + + if (overrides.background !== undefined) { + const currentBg = element.background; + patch.background = { + ...currentBg, + enabled: overrides.background.enabled, + ...(overrides.background.color !== undefined && { + color: overrides.background.color, + }), + ...(overrides.background.cornerRadius !== undefined && { + cornerRadius: overrides.background.cornerRadius, + }), + ...(overrides.background.padding !== undefined && { + paddingX: overrides.background.padding, + paddingY: overrides.background.padding, + }), + }; + } + + return patch as Partial; +} + export { buildSystemPrompt } from "@/agent/system-prompt"; diff --git a/apps/web/src/agent/orchestrator.ts b/apps/web/src/agent/orchestrator.ts index 9896deb3..0beb96e2 100644 --- a/apps/web/src/agent/orchestrator.ts +++ b/apps/web/src/agent/orchestrator.ts @@ -6,14 +6,7 @@ import type { ToolResult, } from "@/agent/types"; import { toolRegistry } from "@/agent/tools/registry"; -import "@/agent/tools/load-context.tool"; -import "@/agent/tools/list-project-assets.tool"; -import "@/agent/tools/list-timeline.tool"; -import "@/agent/tools/split.tool"; -import "@/agent/tools/delete-timeline-elements.tool"; -import "@/agent/tools/move-timeline-elements.tool"; -import "@/agent/tools/add-media-to-timeline.tool"; -import "@/agent/tools/update-timeline-element-timing.tool"; +import "@/agent/tools"; import { useChatStore } from "@/stores/chat-store"; import { useAgentStore } from "@/stores/agent-store"; diff --git a/apps/web/src/agent/system-prompt.ts b/apps/web/src/agent/system-prompt.ts index 2cf31b65..992b8edc 100644 --- a/apps/web/src/agent/system-prompt.ts +++ b/apps/web/src/agent/system-prompt.ts @@ -40,8 +40,10 @@ export function buildSystemPrompt( .join("\n"); parts.push( `Available tools:\n${toolList}`, - 'For questions about what is visible or audible in a media asset, never answer that you cannot see or hear the media if load_context is available. First infer the asset from the active assets or timeline; if needed call list_project_assets or list_timeline, then call load_context with the discovered internal id or timeline element ids.', - 'If load_context has loaded media and the conversation contains an attached fileData part, treat it as the actual video/audio/image content. You may answer visual and audio questions directly from that loaded media without calling extraction tools unless the user asks for a separate extraction workflow.', + "For questions about what is visible or audible in a media asset, never answer that you cannot see or hear the media if load_context is available. First infer the asset from the active assets or timeline; if needed call list_project_assets or list_timeline, then call load_context with the discovered internal id or timeline element ids.", + "If load_context has loaded media and the conversation contains an attached fileData part, treat it as the actual video/audio/image content. You may answer visual and audio questions directly from that loaded media without calling extraction tools unless the user asks for a separate extraction workflow.", + "Only claim edits that were actually performed by tool calls in this conversation. Do not say you added, removed, cleaned, or updated text/subtitles unless the relevant add_text, update_text, delete_timeline_elements, or update_timeline_element_timing tool call succeeded.", + "When the user asks to add titles, hooks, labels, captions, subtitles, or visible text, call add_text. Do not add text proactively for unrelated edit requests such as cutting silence unless the user asks for text.", "When you need to perform an action matching one of these tools, call the appropriate tool. For all other requests, respond directly in plain text.", ); } diff --git a/apps/web/src/agent/tools/__tests__/add-text.test.ts b/apps/web/src/agent/tools/__tests__/add-text.test.ts new file mode 100644 index 00000000..c309e5a3 --- /dev/null +++ b/apps/web/src/agent/tools/__tests__/add-text.test.ts @@ -0,0 +1,202 @@ +import { beforeEach, describe, expect, mock, test } from "bun:test"; +import type { AgentContext } from "@/agent/types"; +import { toolRegistry } from "@/agent/tools/registry"; + +const mockAddText = mock( + (_args: { + text: string; + start: number; + end: number; + position: string; + style?: string; + color?: string; + fontSize?: number; + fontFamily?: string; + fontWeight?: string; + fontStyle?: string; + textAlign?: string; + letterSpacing?: number; + positionX?: number; + positionY?: number; + background?: { enabled: boolean; color?: string; cornerRadius?: number; padding?: number }; + }) => ({ + elementId: "text-1", + trackId: "text-track-1", + }), +); + +mock.module("@/agent/context", () => ({ + EditorContextAdapter: { + addText: mockAddText, + }, +})); + +await import("@/agent/tools/add-text.tool"); + +const context: AgentContext = { + projectId: "proj-1", + activeSceneId: "scene-A", + mediaAssets: [], + playbackTimeMs: 0, +}; + +describe("add_text tool", () => { + beforeEach(() => { + mockAddText.mockClear(); + }); + + test("is registered in the tool registry", () => { + expect(toolRegistry.has("add_text")).toBe(true); + }); + + test("adds text through the editor adapter", async () => { + const tool = toolRegistry.get("add_text"); + const result = await tool.execute( + { + text: "Hello", + start: 1, + end: 4, + position: "bottom", + style: "subtitle", + }, + context, + ); + + expect(mockAddText).toHaveBeenCalledWith({ + text: "Hello", + start: 1, + end: 4, + position: "bottom", + style: "subtitle", + }); + expect(result).toEqual({ elementId: "text-1", trackId: "text-track-1" }); + }); + + test("omits style when not provided", async () => { + const tool = toolRegistry.get("add_text"); + await tool.execute( + { text: "Hello", start: 1, end: 4, position: "center" }, + context, + ); + + expect(mockAddText).toHaveBeenCalledWith({ + text: "Hello", + start: 1, + end: 4, + position: "center", + }); + }); + + test("passes style overrides to the adapter", async () => { + const tool = toolRegistry.get("add_text"); + await tool.execute( + { + text: "Title", + start: 0, + end: 3, + position: "center", + style: "hook", + color: "#FF0000", + fontSize: 10, + fontFamily: "Inter", + fontWeight: "bold", + fontStyle: "italic", + textAlign: "center", + letterSpacing: 2, + positionX: 5, + positionY: -10, + background: { enabled: true, color: "#000000", cornerRadius: 8, padding: 10 }, + }, + context, + ); + + expect(mockAddText).toHaveBeenCalledWith({ + text: "Title", + start: 0, + end: 3, + position: "center", + style: "hook", + color: "#FF0000", + fontSize: 10, + fontFamily: "Inter", + fontWeight: "bold", + fontStyle: "italic", + textAlign: "center", + letterSpacing: 2, + positionX: 5, + positionY: -10, + background: { enabled: true, color: "#000000", cornerRadius: 8, padding: 10 }, + }); + }); + + test("validates add text arguments", async () => { + const tool = toolRegistry.get("add_text"); + + expect( + await tool.execute( + { text: "", start: 0, end: 1, position: "center" }, + context, + ), + ).toEqual({ error: "Text is required" }); + expect( + await tool.execute( + { text: "Hello", start: 1, end: 1, position: "center" }, + context, + ), + ).toEqual({ error: "Invalid time range" }); + expect( + await tool.execute( + { text: "Hello", start: 0, end: 1, position: "left" }, + context, + ), + ).toEqual({ error: "Invalid text position" }); + expect( + await tool.execute( + { text: "Hello", start: 0, end: 1, position: "center", style: "big" }, + context, + ), + ).toEqual({ error: "Invalid text style" }); + expect(mockAddText).not.toHaveBeenCalled(); + }); + + test("validates override arguments", async () => { + const tool = toolRegistry.get("add_text"); + + expect( + await tool.execute( + { text: "A", start: 0, end: 1, position: "center", fontWeight: "heavy" }, + context, + ), + ).toEqual({ error: "Invalid fontWeight" }); + + expect( + await tool.execute( + { text: "A", start: 0, end: 1, position: "center", fontStyle: "oblique" }, + context, + ), + ).toEqual({ error: "Invalid fontStyle" }); + + expect( + await tool.execute( + { text: "A", start: 0, end: 1, position: "center", textAlign: "justify" }, + context, + ), + ).toEqual({ error: "Invalid textAlign" }); + + expect( + await tool.execute( + { text: "A", start: 0, end: 1, position: "center", fontSize: -5 }, + context, + ), + ).toEqual({ error: "Invalid fontSize" }); + + expect( + await tool.execute( + { text: "A", start: 0, end: 1, position: "center", background: "yes" }, + context, + ), + ).toEqual({ error: "Invalid background" }); + + expect(mockAddText).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/web/src/agent/tools/__tests__/delete-timeline-elements.test.ts b/apps/web/src/agent/tools/__tests__/delete-timeline-elements.test.ts index e92c5e5e..685d3b3c 100644 --- a/apps/web/src/agent/tools/__tests__/delete-timeline-elements.test.ts +++ b/apps/web/src/agent/tools/__tests__/delete-timeline-elements.test.ts @@ -51,19 +51,28 @@ describe("delete_timeline_elements tool", () => { const tool = toolRegistry.get("delete_timeline_elements"); expect(await tool.execute({ elementIds: [] }, context)).toEqual({ - error: "Invalid element ids", + error: + 'elementIds must be a non-empty JSON array of strings, e.g. ["id1","id2"]', }); - expect(await tool.execute({ elementIds: ["clip-1", ""] }, context)).toEqual( + expect(await tool.execute({ elementIds: ["", " "] }, context)).toEqual( { - error: "Invalid element ids", + error: + 'elementIds must be a non-empty JSON array of strings, e.g. ["id1","id2"]', }, ); - expect(await tool.execute({ elementIds: ["clip-1", 2] }, context)).toEqual({ - error: "Invalid element ids", - }); - expect(await tool.execute({ elementIds: "clip-1" }, context)).toEqual({ - error: "Invalid element ids", + expect(await tool.execute({}, context)).toEqual({ + error: + 'elementIds must be a non-empty JSON array of strings, e.g. ["id1","id2"]', }); expect(mockDeleteTimelineElements).not.toHaveBeenCalled(); }); + + test("accepts comma-separated string as elementIds", async () => { + const tool = toolRegistry.get("delete_timeline_elements"); + + await tool.execute({ elementIds: "clip-1,clip-2" }, context); + expect(mockDeleteTimelineElements).toHaveBeenCalledWith({ + elementIds: ["clip-1", "clip-2"], + }); + }); }); diff --git a/apps/web/src/agent/tools/__tests__/move-timeline-elements.test.ts b/apps/web/src/agent/tools/__tests__/move-timeline-elements.test.ts index 43b2984e..860c67a7 100644 --- a/apps/web/src/agent/tools/__tests__/move-timeline-elements.test.ts +++ b/apps/web/src/agent/tools/__tests__/move-timeline-elements.test.ts @@ -67,7 +67,8 @@ describe("move_timeline_elements tool", () => { const tool = toolRegistry.get("move_timeline_elements"); expect(await tool.execute({ elementIds: [], start: 0 }, context)).toEqual({ - error: "Invalid element ids", + error: + 'elementIds must be a non-empty JSON array of strings, e.g. ["id1","id2"]', }); expect( await tool.execute({ elementIds: ["clip-1"], start: -1 }, context), diff --git a/apps/web/src/agent/tools/__tests__/update-text.test.ts b/apps/web/src/agent/tools/__tests__/update-text.test.ts new file mode 100644 index 00000000..f3add54c --- /dev/null +++ b/apps/web/src/agent/tools/__tests__/update-text.test.ts @@ -0,0 +1,206 @@ +import { beforeEach, describe, expect, mock, test } from "bun:test"; +import type { AgentContext } from "@/agent/types"; +import { toolRegistry } from "@/agent/tools/registry"; + +const mockUpdateText = mock( + (_args: { + elementIds: string[]; + content?: string; + color?: string; + fontSize?: number; + fontFamily?: string; + fontWeight?: string; + fontStyle?: string; + textAlign?: string; + letterSpacing?: number; + positionX?: number; + positionY?: number; + background?: { + enabled: boolean; + color?: string; + cornerRadius?: number; + padding?: number; + }; + }) => ({ + success: true, + updated: [ + { elementId: "el-1", trackId: "track-1" }, + { elementId: "el-2", trackId: "track-1" }, + ], + skipped: [], + }), +); + +mock.module("@/agent/context", () => ({ + EditorContextAdapter: { + updateText: mockUpdateText, + }, +})); + +await import("@/agent/tools/update-text.tool"); + +const context: AgentContext = { + projectId: "proj-1", + activeSceneId: "scene-A", + mediaAssets: [], + playbackTimeMs: 0, +}; + +describe("update_text tool", () => { + beforeEach(() => { + mockUpdateText.mockClear(); + }); + + test("is registered in the tool registry", () => { + expect(toolRegistry.has("update_text")).toBe(true); + }); + + test("updates text elements through the editor adapter", async () => { + const tool = toolRegistry.get("update_text"); + const result = await tool.execute( + { + elementIds: ["el-1", "el-2"], + color: "#FFFFFF", + fontSize: 10, + }, + context, + ); + + expect(mockUpdateText).toHaveBeenCalledWith({ + elementIds: ["el-1", "el-2"], + color: "#FFFFFF", + fontSize: 10, + }); + expect(result).toEqual({ + success: true, + updated: [ + { elementId: "el-1", trackId: "track-1" }, + { elementId: "el-2", trackId: "track-1" }, + ], + skipped: [], + }); + }); + + test("passes all overrides to the adapter", async () => { + const tool = toolRegistry.get("update_text"); + await tool.execute( + { + elementIds: ["el-1"], + content: "New text", + color: "#FF0000", + fontSize: 8, + fontFamily: "Arial", + fontWeight: "bold", + fontStyle: "italic", + textAlign: "center", + letterSpacing: 2, + positionX: 5, + positionY: -10, + background: { + enabled: true, + color: "#000000", + cornerRadius: 8, + padding: 10, + }, + }, + context, + ); + + expect(mockUpdateText).toHaveBeenCalledWith({ + elementIds: ["el-1"], + content: "New text", + color: "#FF0000", + fontSize: 8, + fontFamily: "Arial", + fontWeight: "bold", + fontStyle: "italic", + textAlign: "center", + letterSpacing: 2, + positionX: 5, + positionY: -10, + background: { + enabled: true, + color: "#000000", + cornerRadius: 8, + padding: 10, + }, + }); + }); + + test("validates elementIds is required and non-empty", async () => { + const tool = toolRegistry.get("update_text"); + + expect(await tool.execute({}, context)).toEqual({ + error: + 'elementIds must be a non-empty JSON array of strings, e.g. ["id1","id2"]', + }); + expect(await tool.execute({ elementIds: [] }, context)).toEqual({ + error: + 'elementIds must be a non-empty JSON array of strings, e.g. ["id1","id2"]', + }); + expect( + await tool.execute({ elementIds: ["", " "] }, context), + ).toEqual({ + error: + 'elementIds must be a non-empty JSON array of strings, e.g. ["id1","id2"]', + }); + expect(mockUpdateText).not.toHaveBeenCalled(); + }); + + test("accepts comma-separated string as elementIds", async () => { + const tool = toolRegistry.get("update_text"); + await tool.execute( + { + elementIds: "el-1,el-2, el-3", + fontFamily: "Playfair Display", + }, + context, + ); + + expect(mockUpdateText).toHaveBeenCalledWith({ + elementIds: ["el-1", "el-2", "el-3"], + fontFamily: "Playfair Display", + }); + }); + + test("validates override types", async () => { + const tool = toolRegistry.get("update_text"); + + expect( + await tool.execute( + { elementIds: ["el-1"], fontWeight: "heavy" }, + context, + ), + ).toEqual({ error: "Invalid fontWeight" }); + + expect( + await tool.execute( + { elementIds: ["el-1"], fontStyle: "oblique" }, + context, + ), + ).toEqual({ error: "Invalid fontStyle" }); + + expect( + await tool.execute( + { elementIds: ["el-1"], textAlign: "justify" }, + context, + ), + ).toEqual({ error: "Invalid textAlign" }); + + expect( + await tool.execute( + { elementIds: ["el-1"], fontSize: -5 }, + context, + ), + ).toEqual({ error: "Invalid fontSize" }); + + expect( + await tool.execute( + { elementIds: ["el-1"], background: "yes" }, + context, + ), + ).toEqual({ error: "Invalid background" }); + + expect(mockUpdateText).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/web/src/agent/tools/add-media-to-timeline.tool.ts b/apps/web/src/agent/tools/add-media-to-timeline.tool.ts index 2b135641..79f538e8 100644 --- a/apps/web/src/agent/tools/add-media-to-timeline.tool.ts +++ b/apps/web/src/agent/tools/add-media-to-timeline.tool.ts @@ -1,6 +1,7 @@ import { EditorContextAdapter } from "@/agent/context"; import type { AgentContext, ToolDefinition } from "@/agent/types"; import { toolRegistry } from "@/agent/tools/registry"; +import { addMediaToTimelineSchema } from "@/agent/tools/schemas"; type AddMediaTrackType = "main" | "overlay" | "audio"; @@ -17,15 +18,7 @@ export type AddMediaToTimelineResult = { }; const addMediaToTimelineTool: ToolDefinition = { - name: "add_media_to_timeline", - description: - "Adds an existing project media asset to the active timeline. Use list_project_assets first to discover assetId. startTime and optional duration are in timeline seconds. trackType chooses main video, overlay video, or audio placement.", - parameters: [ - { key: "assetId", type: "string", required: true }, - { key: "startTime", type: "number", required: true }, - { key: "trackType", type: "string", required: true }, - { key: "duration", type: "number", required: false }, - ], + ...addMediaToTimelineSchema, execute: async ( args: Record, _context: AgentContext, @@ -81,4 +74,4 @@ function isValidOptionalDuration( ); } -toolRegistry.register("add_media_to_timeline", addMediaToTimelineTool); +toolRegistry.register(addMediaToTimelineSchema.name, addMediaToTimelineTool); diff --git a/apps/web/src/agent/tools/add-text.tool.ts b/apps/web/src/agent/tools/add-text.tool.ts new file mode 100644 index 00000000..1e6f6d06 --- /dev/null +++ b/apps/web/src/agent/tools/add-text.tool.ts @@ -0,0 +1,159 @@ +import { EditorContextAdapter } from "@/agent/context"; +import type { AgentContext, ToolDefinition } from "@/agent/types"; +import { toolRegistry } from "@/agent/tools/registry"; +import { addTextSchema } from "@/agent/tools/schemas"; + +type TextPosition = "top" | "center" | "bottom"; +type TextStyle = "plain" | "subtitle" | "hook" | "label"; + +export type TextStyleOverrides = { + color?: string; + fontSize?: number; + fontFamily?: string; + fontWeight?: "normal" | "bold"; + fontStyle?: "normal" | "italic"; + textAlign?: "left" | "center" | "right"; + letterSpacing?: number; + positionX?: number; + positionY?: number; + background?: { + enabled: boolean; + color?: string; + cornerRadius?: number; + padding?: number; + }; +}; + +export type AddTextArgs = { + text: string; + start: number; + end: number; + position: TextPosition; + style?: TextStyle; +} & TextStyleOverrides; + +export type AddTextResult = { + elementId: string; + trackId: string; +}; + +const VALID_FONT_WEIGHTS = new Set(["normal", "bold"]); +const VALID_FONT_STYLES = new Set(["normal", "italic"]); +const VALID_TEXT_ALIGNS = new Set(["left", "center", "right"]); + +const addTextTool: ToolDefinition = { + ...addTextSchema, + execute: async ( + args: Record, + _context: AgentContext, + ): Promise => { + const { text, start, end, position, style } = args; + + if (!isValidText(text)) { + return { error: "Text is required" }; + } + if (!isValidTime(start) || !isValidTime(end) || start >= end) { + return { error: "Invalid time range" }; + } + if (!isValidPosition(position)) { + return { error: "Invalid text position" }; + } + if (!isValidOptionalStyle(style)) { + return { error: "Invalid text style" }; + } + if ( + args.fontWeight !== undefined && + !VALID_FONT_WEIGHTS.has(args.fontWeight as string) + ) { + return { error: "Invalid fontWeight" }; + } + if ( + args.fontStyle !== undefined && + !VALID_FONT_STYLES.has(args.fontStyle as string) + ) { + return { error: "Invalid fontStyle" }; + } + if ( + args.textAlign !== undefined && + !VALID_TEXT_ALIGNS.has(args.textAlign as string) + ) { + return { error: "Invalid textAlign" }; + } + if ( + args.fontSize !== undefined && + (typeof args.fontSize !== "number" || args.fontSize <= 0) + ) { + return { error: "Invalid fontSize" }; + } + if ( + args.letterSpacing !== undefined && + typeof args.letterSpacing !== "number" + ) { + return { error: "Invalid letterSpacing" }; + } + if (args.positionX !== undefined && typeof args.positionX !== "number") { + return { error: "Invalid positionX" }; + } + if (args.positionY !== undefined && typeof args.positionY !== "number") { + return { error: "Invalid positionY" }; + } + if (args.background !== undefined && typeof args.background !== "object") { + return { error: "Invalid background" }; + } + + const overrides: TextStyleOverrides = {}; + if (args.color !== undefined) overrides.color = args.color as string; + if (args.fontSize !== undefined) + overrides.fontSize = args.fontSize as number; + if (args.fontFamily !== undefined) + overrides.fontFamily = args.fontFamily as string; + if (args.fontWeight !== undefined) + overrides.fontWeight = args.fontWeight as "normal" | "bold"; + if (args.fontStyle !== undefined) + overrides.fontStyle = args.fontStyle as "normal" | "italic"; + if (args.textAlign !== undefined) + overrides.textAlign = args.textAlign as "left" | "center" | "right"; + if (args.letterSpacing !== undefined) + overrides.letterSpacing = args.letterSpacing as number; + if (args.positionX !== undefined) + overrides.positionX = args.positionX as number; + if (args.positionY !== undefined) + overrides.positionY = args.positionY as number; + if (args.background !== undefined) + overrides.background = + args.background as TextStyleOverrides["background"]; + + return EditorContextAdapter.addText({ + text, + start, + end, + position, + ...(style !== undefined && { style }), + ...overrides, + }); + }, +}; + +function isValidText(text: unknown): text is string { + return typeof text === "string" && text.trim().length > 0; +} + +function isValidTime(time: unknown): time is number { + return typeof time === "number" && Number.isFinite(time) && time >= 0; +} + +function isValidPosition(position: unknown): position is TextPosition { + return position === "top" || position === "center" || position === "bottom"; +} + +function isValidOptionalStyle(style: unknown): style is TextStyle | undefined { + return ( + style === undefined || + style === "plain" || + style === "subtitle" || + style === "hook" || + style === "label" + ); +} + +toolRegistry.register(addTextSchema.name, addTextTool); diff --git a/apps/web/src/agent/tools/delete-timeline-elements.tool.ts b/apps/web/src/agent/tools/delete-timeline-elements.tool.ts index 13922a3e..45d0f7dc 100644 --- a/apps/web/src/agent/tools/delete-timeline-elements.tool.ts +++ b/apps/web/src/agent/tools/delete-timeline-elements.tool.ts @@ -1,6 +1,8 @@ import { EditorContextAdapter } from "@/agent/context"; import type { AgentContext, ToolDefinition } from "@/agent/types"; import { toolRegistry } from "@/agent/tools/registry"; +import { resolveElementIds } from "@/agent/tools/resolve-element-ids"; +import { deleteTimelineElementsSchema } from "@/agent/tools/schemas"; export type DeleteTimelineElementsArgs = { elementIds: string[]; @@ -12,33 +14,25 @@ export type DeleteTimelineElementsResult = { }; const deleteTimelineElementsTool: ToolDefinition = { - name: "delete_timeline_elements", - description: - "Deletes one or more timeline elements by elementId. Use list_timeline first to discover exact elementIds. To delete a time range, split at the range boundaries first, then delete the isolated elementIds.", - parameters: [{ key: "elementIds", type: "string[]", required: true }], + ...deleteTimelineElementsSchema, execute: async ( args: Record, _context: AgentContext, ): Promise => { - const elementIds = args.elementIds; + const elementIds = resolveElementIds(args.elementIds); - if (!isValidElementIds(elementIds)) { - return { error: "Invalid element ids" }; + if (!elementIds) { + return { + error: + 'elementIds must be a non-empty JSON array of strings, e.g. ["id1","id2"]', + }; } return EditorContextAdapter.deleteTimelineElements({ elementIds }); }, }; -function isValidElementIds(elementIds: unknown): elementIds is string[] { - return ( - Array.isArray(elementIds) && - elementIds.length > 0 && - elementIds.every( - (elementId) => - typeof elementId === "string" && elementId.trim().length > 0, - ) - ); -} - -toolRegistry.register("delete_timeline_elements", deleteTimelineElementsTool); +toolRegistry.register( + deleteTimelineElementsSchema.name, + deleteTimelineElementsTool, +); diff --git a/apps/web/src/agent/tools/index.ts b/apps/web/src/agent/tools/index.ts new file mode 100644 index 00000000..8348956d --- /dev/null +++ b/apps/web/src/agent/tools/index.ts @@ -0,0 +1,22 @@ +/** + * Client-only barrel for agent tools. Importing this file registers every + * tool with `toolRegistry` via side effects. + * + * IMPORTANT: do NOT import this from server code. Tool modules transitively + * import EditorContextAdapter, which touches WASM/EditorCore (browser-only). + * Server code should import schemas from `./schemas` instead. + * + * To add a new tool: declare its schema in `./schemas.ts`, create the + * `*.tool.ts` file, then add a single side-effect import below. + */ + +import "@/agent/tools/load-context.tool"; +import "@/agent/tools/list-project-assets.tool"; +import "@/agent/tools/list-timeline.tool"; +import "@/agent/tools/split.tool"; +import "@/agent/tools/delete-timeline-elements.tool"; +import "@/agent/tools/move-timeline-elements.tool"; +import "@/agent/tools/add-media-to-timeline.tool"; +import "@/agent/tools/update-timeline-element-timing.tool"; +import "@/agent/tools/add-text.tool"; +import "@/agent/tools/update-text.tool"; diff --git a/apps/web/src/agent/tools/list-project-assets.tool.ts b/apps/web/src/agent/tools/list-project-assets.tool.ts index cabfb8ca..3c1aaa9c 100644 --- a/apps/web/src/agent/tools/list-project-assets.tool.ts +++ b/apps/web/src/agent/tools/list-project-assets.tool.ts @@ -1,5 +1,6 @@ import type { AgentContext, ToolDefinition } from "@/agent/types"; import { toolRegistry } from "@/agent/tools/registry"; +import { listProjectAssetsSchema } from "@/agent/tools/schemas"; type AssetTypeFilter = "all" | "video" | "audio" | "image"; type UsageFilter = "all" | "used" | "unused"; @@ -27,13 +28,7 @@ const assetTypes = new Set(["all", "video", "audio", "image"]); const usageFilters = new Set(["all", "used", "unused"]); const listProjectAssetsTool: ToolDefinition = { - name: "list_project_assets", - description: - "Lists project media assets with stable ids, type, duration, and whether each asset is used in the active timeline.", - parameters: [ - { key: "filter", type: "string", required: false }, - { key: "type", type: "string", required: false }, - ], + ...listProjectAssetsSchema, execute: async ( args: Record, context: AgentContext, @@ -82,4 +77,4 @@ function isSupportedAsset( ); } -toolRegistry.register("list_project_assets", listProjectAssetsTool); +toolRegistry.register(listProjectAssetsSchema.name, listProjectAssetsTool); diff --git a/apps/web/src/agent/tools/list-timeline.tool.ts b/apps/web/src/agent/tools/list-timeline.tool.ts index 789d2f8b..dc74897c 100644 --- a/apps/web/src/agent/tools/list-timeline.tool.ts +++ b/apps/web/src/agent/tools/list-timeline.tool.ts @@ -4,16 +4,14 @@ import type { ToolDefinition, } from "@/agent/types"; import { toolRegistry } from "@/agent/tools/registry"; +import { listTimelineSchema } from "@/agent/tools/schemas"; export type ListTimelineResult = { tracks: AgentTimelineTrack[]; }; const listTimelineTool: ToolDefinition = { - name: "list_timeline", - description: - "Lists the active timeline as structured tracks and editable elements with layer metadata. Element start/end values are in seconds. position is timeline row order from top to bottom; visualLayer is render stacking where higher renders above lower, and audio tracks have null visualLayer.", - parameters: [], + ...listTimelineSchema, execute: async ( _args: Record, context: AgentContext, @@ -26,4 +24,4 @@ const listTimelineTool: ToolDefinition = { }, }; -toolRegistry.register("list_timeline", listTimelineTool); +toolRegistry.register(listTimelineSchema.name, listTimelineTool); diff --git a/apps/web/src/agent/tools/load-context.tool.ts b/apps/web/src/agent/tools/load-context.tool.ts index f63937da..c6613474 100644 --- a/apps/web/src/agent/tools/load-context.tool.ts +++ b/apps/web/src/agent/tools/load-context.tool.ts @@ -5,6 +5,7 @@ import type { } from "@/agent/types"; import { EditorContextAdapter } from "@/agent/context"; import { toolRegistry } from "@/agent/tools/registry"; +import { loadContextSchema } from "@/agent/tools/schemas"; type LoadContextTargetType = "asset" | "timeline_element"; @@ -68,16 +69,7 @@ const assetContextCache = new Map(); const textContextCache = new Map(); const loadContextTool: ToolDefinition = { - name: "load_context", - description: - "Loads relevant Gemini context for a project asset or timeline element. Media assets are uploaded through the server-side Gemini Files API; text/caption elements are loaded as exact structured text.", - parameters: [ - { key: "targetType", type: "string", required: true }, - { key: "id", type: "string", required: false }, - { key: "assetId", type: "string", required: false }, - { key: "trackId", type: "string", required: false }, - { key: "elementId", type: "string", required: false }, - ], + ...loadContextSchema, execute: async ( args: Record, context: AgentContext, @@ -388,4 +380,4 @@ type TimelineElementRef = { end: number; }; -toolRegistry.register("load_context", loadContextTool); +toolRegistry.register(loadContextSchema.name, loadContextTool); diff --git a/apps/web/src/agent/tools/move-timeline-elements.tool.ts b/apps/web/src/agent/tools/move-timeline-elements.tool.ts index 0aa77ba0..ecc9258c 100644 --- a/apps/web/src/agent/tools/move-timeline-elements.tool.ts +++ b/apps/web/src/agent/tools/move-timeline-elements.tool.ts @@ -1,6 +1,8 @@ import { EditorContextAdapter } from "@/agent/context"; import type { AgentContext, ToolDefinition } from "@/agent/types"; import { toolRegistry } from "@/agent/tools/registry"; +import { resolveElementIds } from "@/agent/tools/resolve-element-ids"; +import { moveTimelineElementsSchema } from "@/agent/tools/schemas"; export type MoveTimelineElementsArgs = { elementIds: string[]; @@ -19,22 +21,19 @@ export type MoveTimelineElementsResult = { }; const moveTimelineElementsTool: ToolDefinition = { - name: "move_timeline_elements", - description: - "Moves one or more existing timeline elements to a new timeline start time in seconds. For multiple elements, the earliest selected element is moved to start and the others preserve their relative offsets. Optionally pass targetTrackId to move them to another compatible track.", - parameters: [ - { key: "elementIds", type: "string[]", required: true }, - { key: "start", type: "number", required: true }, - { key: "targetTrackId", type: "string", required: false }, - ], + ...moveTimelineElementsSchema, execute: async ( args: Record, _context: AgentContext, ): Promise => { - const { elementIds, start, targetTrackId } = args; + const { start, targetTrackId } = args; + const elementIds = resolveElementIds(args.elementIds); - if (!isValidElementIds(elementIds)) { - return { error: "Invalid element ids" }; + if (!elementIds) { + return { + error: + 'elementIds must be a non-empty JSON array of strings, e.g. ["id1","id2"]', + }; } if (!isValidStart(start)) { return { error: "Invalid start time" }; @@ -51,17 +50,6 @@ const moveTimelineElementsTool: ToolDefinition = { }, }; -function isValidElementIds(elementIds: unknown): elementIds is string[] { - return ( - Array.isArray(elementIds) && - elementIds.length > 0 && - elementIds.every( - (elementId) => - typeof elementId === "string" && elementId.trim().length > 0, - ) - ); -} - function isValidStart(start: unknown): start is number { return typeof start === "number" && Number.isFinite(start) && start >= 0; } @@ -75,4 +63,7 @@ function isValidOptionalTargetTrackId( ); } -toolRegistry.register("move_timeline_elements", moveTimelineElementsTool); +toolRegistry.register( + moveTimelineElementsSchema.name, + moveTimelineElementsTool, +); diff --git a/apps/web/src/agent/tools/resolve-element-ids.ts b/apps/web/src/agent/tools/resolve-element-ids.ts new file mode 100644 index 00000000..563244af --- /dev/null +++ b/apps/web/src/agent/tools/resolve-element-ids.ts @@ -0,0 +1,18 @@ +export function resolveElementIds(raw: unknown): string[] | null { + if (Array.isArray(raw)) { + const ids = raw.filter( + (id) => typeof id === "string" && id.trim().length > 0, + ); + return ids.length > 0 ? ids : null; + } + + if (typeof raw === "string" && raw.trim().length > 0) { + const ids = raw + .split(",") + .map((id) => id.trim()) + .filter((id) => id.length > 0); + return ids.length > 0 ? ids : null; + } + + return null; +} diff --git a/apps/web/src/agent/tools/schemas.ts b/apps/web/src/agent/tools/schemas.ts new file mode 100644 index 00000000..af76ed90 --- /dev/null +++ b/apps/web/src/agent/tools/schemas.ts @@ -0,0 +1,153 @@ +import type { ToolSchema } from "@/agent/types"; + +/** + * Single source of truth for agent tool schemas. + * + * Both the server route (provider-facing schema list) and the client tool + * definitions (`*.tool.ts`) import from here. This guarantees that the + * description and parameters the model sees are exactly what the executor + * validates against — no drift possible. + * + * Pure data: no side effects, no EditorCore imports. Safe for server use. + */ + +export const loadContextSchema: ToolSchema = { + name: "load_context", + description: + "Loads the actual Gemini multimodal context for a project asset or timeline element. Use this before answering questions about visible objects, colors, scenes, speech, silence, or timestamps in media. Use targetType='asset' with assetId/id for media, or targetType='timeline_element' with trackId and elementId/id for captions, text, or timeline media elements.", + parameters: [ + { key: "targetType", type: "string", required: true }, + { key: "id", type: "string", required: false }, + { key: "assetId", type: "string", required: false }, + { key: "trackId", type: "string", required: false }, + { key: "elementId", type: "string", required: false }, + ], +}; + +export const listProjectAssetsSchema: ToolSchema = { + name: "list_project_assets", + description: + "Lists project media assets with stable ids, type, duration, and whether each asset is used in the active timeline.", + parameters: [ + { key: "filter", type: "string", required: false }, + { key: "type", type: "string", required: false }, + ], +}; + +export const listTimelineSchema: ToolSchema = { + name: "list_timeline", + description: + "Lists the active timeline as structured tracks and editable elements with layer metadata. Element start/end values are in seconds. Tracks include position (top-to-bottom timeline row), visualLayer (higher renders above lower, null for audio), isVisualLayer, and stacking. Use this to understand which clips visually cover others and to discover ids before load_context.", + parameters: [], +}; + +export const splitSchema: ToolSchema = { + name: "split", + description: + "Splits timeline elements at one or more requested timeline times in seconds without deleting, trimming, or moving content. Use one time for a single cut, or multiple times to isolate ranges before separate edit/delete operations.", + parameters: [{ key: "times", type: "number[]", required: true }], +}; + +export const deleteTimelineElementsSchema: ToolSchema = { + name: "delete_timeline_elements", + description: + "Deletes one or more timeline elements by elementId. Use list_timeline first to discover exact elementIds. To delete a time range, split at the range boundaries first, then delete the isolated elementIds.", + parameters: [{ key: "elementIds", type: "string[]", required: true }], +}; + +export const moveTimelineElementsSchema: ToolSchema = { + name: "move_timeline_elements", + description: + "Moves one or more existing timeline elements to a new timeline start time in seconds. For multiple elements, the earliest selected element is moved to start and the others preserve their relative offsets. Optionally pass targetTrackId to move them to another compatible track.", + parameters: [ + { key: "elementIds", type: "string[]", required: true }, + { key: "start", type: "number", required: true }, + { key: "targetTrackId", type: "string", required: false }, + ], +}; + +export const addMediaToTimelineSchema: ToolSchema = { + name: "add_media_to_timeline", + description: + "Adds an existing project media asset to the active timeline. Use list_project_assets first to discover assetId. startTime and optional duration are in timeline seconds. trackType must be main, overlay, or audio.", + parameters: [ + { key: "assetId", type: "string", required: true }, + { key: "startTime", type: "number", required: true }, + { key: "trackType", type: "string", required: true }, + { key: "duration", type: "number", required: false }, + ], +}; + +export const updateTimelineElementTimingSchema: ToolSchema = { + name: "update_timeline_element_timing", + description: + "Updates an existing timeline element's timing. Use list_timeline first to discover elementId. start, end, and duration are timeline seconds; pass at least one of start, end, or duration.", + parameters: [ + { key: "elementId", type: "string", required: true }, + { key: "start", type: "number", required: false }, + { key: "end", type: "number", required: false }, + { key: "duration", type: "number", required: false }, + ], +}; + +export const addTextSchema: ToolSchema = { + name: "add_text", + description: + "Adds visual text to the active timeline. Use for titles, hooks, labels, and simple subtitle blocks. start and end are timeline seconds. position must be top, center, or bottom; style may be plain, subtitle, hook, or label. Override style defaults with color (hex), fontSize (scaled units), fontFamily, fontWeight (normal/bold), fontStyle (normal/italic), textAlign (left/center/right), letterSpacing, positionX/positionY (offset from -50 to 50), or background ({ enabled, color?, cornerRadius?, padding? }).", + parameters: [ + { key: "text", type: "string", required: true }, + { key: "start", type: "number", required: true }, + { key: "end", type: "number", required: true }, + { key: "position", type: "string", required: true }, + { key: "style", type: "string", required: false }, + { key: "color", type: "string", required: false }, + { key: "fontSize", type: "number", required: false }, + { key: "fontFamily", type: "string", required: false }, + { key: "fontWeight", type: "string", required: false }, + { key: "fontStyle", type: "string", required: false }, + { key: "textAlign", type: "string", required: false }, + { key: "letterSpacing", type: "number", required: false }, + { key: "positionX", type: "number", required: false }, + { key: "positionY", type: "number", required: false }, + { key: "background", type: "object", required: false }, + ], +}; + +export const updateTextSchema: ToolSchema = { + name: "update_text", + description: + "Updates visual properties of existing text elements. Pass elementIds (from list_timeline) and any combination of text style overrides. Only text elements are updated; non-text elements are skipped. All listed text elements receive the same overrides — use for bulk styling subtitle blocks.", + parameters: [ + // "array" (not "string[]") so the orchestrator validator stays + // tolerant of CSV-string fallback handled by resolveElementIds. + { key: "elementIds", type: "array", required: true }, + { key: "content", type: "string", required: false }, + { key: "color", type: "string", required: false }, + { key: "fontSize", type: "number", required: false }, + { key: "fontFamily", type: "string", required: false }, + { key: "fontWeight", type: "string", required: false }, + { key: "fontStyle", type: "string", required: false }, + { key: "textAlign", type: "string", required: false }, + { key: "letterSpacing", type: "number", required: false }, + { key: "positionX", type: "number", required: false }, + { key: "positionY", type: "number", required: false }, + { key: "background", type: "object", required: false }, + ], +}; + +/** + * The exact list of schemas exposed to the LLM. + * Excludes internal-only tools (transcribe_video, mock). + */ +export const providerToolSchemas: ToolSchema[] = [ + loadContextSchema, + listProjectAssetsSchema, + listTimelineSchema, + splitSchema, + deleteTimelineElementsSchema, + moveTimelineElementsSchema, + addMediaToTimelineSchema, + updateTimelineElementTimingSchema, + addTextSchema, + updateTextSchema, +]; diff --git a/apps/web/src/agent/tools/split.tool.ts b/apps/web/src/agent/tools/split.tool.ts index 4efa1f7a..1aead443 100644 --- a/apps/web/src/agent/tools/split.tool.ts +++ b/apps/web/src/agent/tools/split.tool.ts @@ -1,6 +1,7 @@ import { EditorContextAdapter } from "@/agent/context"; import type { AgentContext, ToolDefinition } from "@/agent/types"; import { toolRegistry } from "@/agent/tools/registry"; +import { splitSchema } from "@/agent/tools/schemas"; export type SplitArgs = { times: number[]; @@ -12,10 +13,7 @@ export type SplitResult = { }; const splitTool: ToolDefinition = { - name: "split", - description: - "Splits timeline elements at one or more requested times without deleting, trimming, or moving content. Use one time for a single cut, or multiple times to isolate ranges before separate edit/delete operations.", - parameters: [{ key: "times", type: "number[]", required: true }], + ...splitSchema, execute: async ( args: Record, _context: AgentContext, @@ -40,4 +38,4 @@ function isValidTimes(times: unknown): times is number[] { ); } -toolRegistry.register("split", splitTool); +toolRegistry.register(splitSchema.name, splitTool); diff --git a/apps/web/src/agent/tools/update-text.tool.ts b/apps/web/src/agent/tools/update-text.tool.ts new file mode 100644 index 00000000..c99217b7 --- /dev/null +++ b/apps/web/src/agent/tools/update-text.tool.ts @@ -0,0 +1,134 @@ +import { EditorContextAdapter } from "@/agent/context"; +import type { AgentContext, ToolDefinition } from "@/agent/types"; +import { toolRegistry } from "@/agent/tools/registry"; +import { resolveElementIds } from "@/agent/tools/resolve-element-ids"; +import { updateTextSchema } from "@/agent/tools/schemas"; + +export type UpdateTextArgs = { + elementIds: string[]; + content?: string; + color?: string; + fontSize?: number; + fontFamily?: string; + fontWeight?: "normal" | "bold"; + fontStyle?: "normal" | "italic"; + textAlign?: "left" | "center" | "right"; + letterSpacing?: number; + positionX?: number; + positionY?: number; + background?: { + enabled: boolean; + color?: string; + cornerRadius?: number; + padding?: number; + }; +}; + +export type UpdatedTextElement = { + elementId: string; + trackId: string; +}; + +export type UpdateTextResult = { + success: boolean; + updated: UpdatedTextElement[]; + skipped: string[]; +}; + +const VALID_FONT_WEIGHTS = new Set(["normal", "bold"]); +const VALID_FONT_STYLES = new Set(["normal", "italic"]); +const VALID_TEXT_ALIGNS = new Set(["left", "center", "right"]); + +const updateTextTool: ToolDefinition = { + ...updateTextSchema, + execute: async ( + args: Record, + _context: AgentContext, + ): Promise => { + const { elementIds, content, background } = args; + + const resolvedIds = resolveElementIds(elementIds); + if (resolvedIds === null) { + return { + error: + 'elementIds must be a non-empty JSON array of strings, e.g. ["id1","id2"]', + }; + } + + if ( + content !== undefined && + !(typeof content === "string" && content.trim().length > 0) + ) { + return { error: "content must be a non-empty string" }; + } + + if ( + args.fontWeight !== undefined && + !VALID_FONT_WEIGHTS.has(args.fontWeight as string) + ) { + return { error: "Invalid fontWeight" }; + } + if ( + args.fontStyle !== undefined && + !VALID_FONT_STYLES.has(args.fontStyle as string) + ) { + return { error: "Invalid fontStyle" }; + } + if ( + args.textAlign !== undefined && + !VALID_TEXT_ALIGNS.has(args.textAlign as string) + ) { + return { error: "Invalid textAlign" }; + } + if ( + args.fontSize !== undefined && + (typeof args.fontSize !== "number" || args.fontSize <= 0) + ) { + return { error: "Invalid fontSize" }; + } + if ( + args.letterSpacing !== undefined && + typeof args.letterSpacing !== "number" + ) { + return { error: "Invalid letterSpacing" }; + } + if (args.positionX !== undefined && typeof args.positionX !== "number") { + return { error: "Invalid positionX" }; + } + if (args.positionY !== undefined && typeof args.positionY !== "number") { + return { error: "Invalid positionY" }; + } + if (background !== undefined && typeof background !== "object") { + return { error: "Invalid background" }; + } + + const overrides: Omit = {}; + if (content !== undefined) overrides.content = content as string; + if (args.color !== undefined) overrides.color = args.color as string; + if (args.fontSize !== undefined) + overrides.fontSize = args.fontSize as number; + if (args.fontFamily !== undefined) + overrides.fontFamily = args.fontFamily as string; + if (args.fontWeight !== undefined) + overrides.fontWeight = args.fontWeight as "normal" | "bold"; + if (args.fontStyle !== undefined) + overrides.fontStyle = args.fontStyle as "normal" | "italic"; + if (args.textAlign !== undefined) + overrides.textAlign = args.textAlign as "left" | "center" | "right"; + if (args.letterSpacing !== undefined) + overrides.letterSpacing = args.letterSpacing as number; + if (args.positionX !== undefined) + overrides.positionX = args.positionX as number; + if (args.positionY !== undefined) + overrides.positionY = args.positionY as number; + if (background !== undefined) + overrides.background = background as UpdateTextArgs["background"]; + + return EditorContextAdapter.updateText({ + elementIds: resolvedIds, + ...overrides, + }); + }, +}; + +toolRegistry.register(updateTextSchema.name, updateTextTool); diff --git a/apps/web/src/agent/tools/update-timeline-element-timing.tool.ts b/apps/web/src/agent/tools/update-timeline-element-timing.tool.ts index bc8a5dee..efc9dbb6 100644 --- a/apps/web/src/agent/tools/update-timeline-element-timing.tool.ts +++ b/apps/web/src/agent/tools/update-timeline-element-timing.tool.ts @@ -1,6 +1,7 @@ import { EditorContextAdapter } from "@/agent/context"; import type { AgentContext, ToolDefinition } from "@/agent/types"; import { toolRegistry } from "@/agent/tools/registry"; +import { updateTimelineElementTimingSchema } from "@/agent/tools/schemas"; export type UpdateTimelineElementTimingArgs = { elementId: string; @@ -19,15 +20,7 @@ export type UpdateTimelineElementTimingResult = { }; const updateTimelineElementTimingTool: ToolDefinition = { - name: "update_timeline_element_timing", - description: - "Updates an existing timeline element's timing. Use list_timeline first to discover elementId. start, end, and duration are timeline seconds; pass at least one of start, end, or duration.", - parameters: [ - { key: "elementId", type: "string", required: true }, - { key: "start", type: "number", required: false }, - { key: "end", type: "number", required: false }, - { key: "duration", type: "number", required: false }, - ], + ...updateTimelineElementTimingSchema, execute: async ( args: Record, _context: AgentContext, @@ -87,6 +80,6 @@ function isValidOptionalDuration( } toolRegistry.register( - "update_timeline_element_timing", + updateTimelineElementTimingSchema.name, updateTimelineElementTimingTool, ); diff --git a/apps/web/src/app/api/agent/chat/__tests__/route.test.ts b/apps/web/src/app/api/agent/chat/__tests__/route.test.ts index 2f2f0a4f..1c2d2e61 100644 --- a/apps/web/src/app/api/agent/chat/__tests__/route.test.ts +++ b/apps/web/src/app/api/agent/chat/__tests__/route.test.ts @@ -152,7 +152,7 @@ describe("POST /api/agent/chat", () => { expect(callArgs.messages).toHaveLength(1); expect(callArgs.systemPrompt).toContain("NeuralCut"); - expect(callArgs.tools).toHaveLength(8); + expect(callArgs.tools).toHaveLength(10); expect( callArgs.tools.map((tool) => (tool as { name: string }).name), ).toEqual([ @@ -164,6 +164,8 @@ describe("POST /api/agent/chat", () => { "move_timeline_elements", "add_media_to_timeline", "update_timeline_element_timing", + "add_text", + "update_text", ]); }); diff --git a/apps/web/src/app/api/agent/chat/route.ts b/apps/web/src/app/api/agent/chat/route.ts index 72e02b43..1e906922 100644 --- a/apps/web/src/app/api/agent/chat/route.ts +++ b/apps/web/src/app/api/agent/chat/route.ts @@ -1,87 +1,10 @@ import { type NextRequest, NextResponse } from "next/server"; import { z } from "zod"; -import type { AgentContext, ToolSchema } from "@/agent/types"; +import type { AgentContext } from "@/agent/types"; import { buildSystemPrompt } from "@/agent/system-prompt"; import { createProvider } from "@/agent/providers"; import type { ProviderConfig } from "@/agent/providers"; - -// --------------------------------------------------------------------------- -// Provider-facing tool schemas. Execution is client-side. -// echo_context is intentionally excluded; only user-facing tools are exposed. -// --------------------------------------------------------------------------- -const providerToolSchemas: ToolSchema[] = [ - { - name: "load_context", - description: - "Loads the actual Gemini multimodal context for a project asset or timeline element. Use this before answering questions about visible objects, colors, scenes, speech, silence, or timestamps in media. Use targetType='asset' with assetId/id for media, or targetType='timeline_element' with trackId and elementId/id for captions, text, or timeline media elements.", - parameters: [ - { key: "targetType", type: "string", required: true }, - { key: "id", type: "string", required: false }, - { key: "assetId", type: "string", required: false }, - { key: "trackId", type: "string", required: false }, - { key: "elementId", type: "string", required: false }, - ], - }, - { - name: "list_project_assets", - description: - "Lists project media assets with stable ids, type, duration, and whether each asset is used in the active timeline.", - parameters: [ - { key: "filter", type: "string", required: false }, - { key: "type", type: "string", required: false }, - ], - }, - { - name: "list_timeline", - description: - "Lists the active timeline as structured tracks and editable elements with layer metadata. Element start/end values are in seconds. Tracks include position (top-to-bottom timeline row), visualLayer (higher renders above lower, null for audio), isVisualLayer, and stacking. Use this to understand which clips visually cover others and to discover ids before load_context.", - parameters: [], - }, - { - name: "split", - description: - "Splits timeline elements at one or more requested timeline times in seconds without deleting, trimming, or moving content. Use one time for a single cut, or multiple times to isolate ranges before separate edit/delete operations.", - parameters: [{ key: "times", type: "number[]", required: true }], - }, - { - name: "delete_timeline_elements", - description: - "Deletes one or more timeline elements by elementId. Use list_timeline first to discover exact elementIds. To delete a time range, split at the range boundaries first, then delete the isolated elementIds.", - parameters: [{ key: "elementIds", type: "string[]", required: true }], - }, - { - name: "move_timeline_elements", - description: - "Moves one or more existing timeline elements to a new timeline start time in seconds. For multiple elements, the earliest selected element is moved to start and the others preserve their relative offsets. Optionally pass targetTrackId to move them to another compatible track.", - parameters: [ - { key: "elementIds", type: "string[]", required: true }, - { key: "start", type: "number", required: true }, - { key: "targetTrackId", type: "string", required: false }, - ], - }, - { - name: "add_media_to_timeline", - description: - "Adds an existing project media asset to the active timeline. Use list_project_assets first to discover assetId. startTime and optional duration are in timeline seconds. trackType must be main, overlay, or audio.", - parameters: [ - { key: "assetId", type: "string", required: true }, - { key: "startTime", type: "number", required: true }, - { key: "trackType", type: "string", required: true }, - { key: "duration", type: "number", required: false }, - ], - }, - { - name: "update_timeline_element_timing", - description: - "Updates an existing timeline element's timing. Use list_timeline first to discover elementId. start, end, and duration are timeline seconds; pass at least one of start, end, or duration.", - parameters: [ - { key: "elementId", type: "string", required: true }, - { key: "start", type: "number", required: false }, - { key: "end", type: "number", required: false }, - { key: "duration", type: "number", required: false }, - ], - }, -]; +import { providerToolSchemas } from "@/agent/tools/schemas"; // --------------------------------------------------------------------------- // Zod validation schemas diff --git a/apps/web/src/components/editor/panels/chat/tool-formatters.ts b/apps/web/src/components/editor/panels/chat/tool-formatters.ts index 4715083d..c6c50db5 100644 --- a/apps/web/src/components/editor/panels/chat/tool-formatters.ts +++ b/apps/web/src/components/editor/panels/chat/tool-formatters.ts @@ -116,6 +116,37 @@ const TOOL_CALL_FORMATTERS: Record< description: parts.length > 0 ? parts.join(", ") : "No timing changes", }; }, + add_text: (args) => { + const parts: string[] = []; + if (args.color) parts.push(String(args.color)); + if (args.fontSize) parts.push(`size ${args.fontSize}`); + if (args.fontFamily) parts.push(String(args.fontFamily)); + if ((args.background as Record | undefined)?.enabled) + parts.push("bg"); + return { + label: "Add Text", + description: `${String(args.text ?? "text")} · ${formatSeconds(args.start)}-${formatSeconds(args.end)}${parts.length > 0 ? ` · ${parts.join(" ")}` : ""}`, + }; + }, + update_text: (args) => { + const raw = args.elementIds; + const ids = Array.isArray(raw) + ? raw.length + : typeof raw === "string" + ? raw.split(",").filter((id) => id.trim().length > 0).length + : 0; + const overrides: string[] = []; + if (args.content) overrides.push("text"); + if (args.color) overrides.push(String(args.color)); + if (args.fontSize) overrides.push(`size ${args.fontSize}`); + if (args.fontFamily) overrides.push(String(args.fontFamily)); + if ((args.background as Record | undefined)?.enabled) + overrides.push("bg"); + return { + label: "Update Text", + description: `${ids} element${ids !== 1 ? "s" : ""}${overrides.length > 0 ? ` · ${overrides.join(" ")}` : ""}`, + }; + }, load_context: (args) => { const targetType = String(args.targetType ?? "unknown"); const id = args.id ?? args.assetId ?? args.elementId; @@ -227,6 +258,33 @@ const TOOL_RESULT_FORMATTERS: Record< : "Failed", }; }, + add_text: (parsed) => { + const data = parsed as { + elementId?: string; + trackId?: string; + } | null; + if (!data) return null; + return { + label: "Text Added", + description: data.trackId ? `track ${data.trackId}` : "Inserted", + }; + }, + update_text: (parsed) => { + const data = parsed as { + success?: boolean; + updated?: Array<{ elementId: string }>; + skipped?: string[]; + } | null; + if (!data) return null; + const count = data.updated?.length ?? 0; + const skipped = data.skipped?.length ?? 0; + return { + label: "Text Updated", + description: data.success + ? `${count} updated${skipped > 0 ? `, ${skipped} skipped` : ""}` + : "Failed", + }; + }, load_context: (parsed) => { const data = parsed as { status?: string;