From 773e637fa6460547f1dc379a64cc05efb058acda Mon Sep 17 00:00:00 2001 From: Luis Esteban Acevedo Ladino Date: Sat, 25 Apr 2026 07:59:38 -0500 Subject: [PATCH] feat: add agent timeline placement tools --- apps/web/src/agent/context.ts | 453 +++++++++++++++++- apps/web/src/agent/orchestrator.ts | 3 + .../__tests__/add-media-to-timeline.test.ts | 100 ++++ .../__tests__/move-timeline-elements.test.ts | 95 ++++ .../update-timeline-element-timing.test.ts | 89 ++++ .../agent/tools/add-media-to-timeline.tool.ts | 84 ++++ .../tools/move-timeline-elements.tool.ts | 78 +++ .../update-timeline-element-timing.tool.ts | 92 ++++ .../api/agent/chat/__tests__/route.test.ts | 5 +- apps/web/src/app/api/agent/chat/route.ts | 32 ++ .../editor/panels/chat/tool-formatters.ts | 68 +++ docs/agent-tool-specs.md | 116 ++++- docs/agent-tools.md | 55 ++- propuesta_tecnica.md | 6 +- 14 files changed, 1261 insertions(+), 15 deletions(-) create mode 100644 apps/web/src/agent/tools/__tests__/add-media-to-timeline.test.ts create mode 100644 apps/web/src/agent/tools/__tests__/move-timeline-elements.test.ts create mode 100644 apps/web/src/agent/tools/__tests__/update-timeline-element-timing.test.ts create mode 100644 apps/web/src/agent/tools/add-media-to-timeline.tool.ts create mode 100644 apps/web/src/agent/tools/move-timeline-elements.tool.ts create mode 100644 apps/web/src/agent/tools/update-timeline-element-timing.tool.ts diff --git a/apps/web/src/agent/context.ts b/apps/web/src/agent/context.ts index 1c61921e..bd268634 100644 --- a/apps/web/src/agent/context.ts +++ b/apps/web/src/agent/context.ts @@ -1,7 +1,19 @@ import { EditorCore } from "@/core"; import { TICKS_PER_SECOND } from "@/lib/wasm"; import { buildContextFromEditorState } from "@/agent/context-mapper"; -import type { SceneTracks } from "@/lib/timeline"; +import { BatchCommand } from "@/lib/commands"; +import { AddTrackCommand, InsertElementCommand } from "@/lib/commands/timeline"; +import { DEFAULT_NEW_ELEMENT_DURATION } from "@/lib/timeline/creation"; +import { buildElementFromMedia } from "@/lib/timeline/element-utils"; +import type { + SceneTracks, + TimelineElement, + TimelineTrack, + TrackType, +} from "@/lib/timeline"; +import { canPlaceTimeSpansOnTrack } from "@/lib/timeline/placement/overlap"; +import { validateElementTrackCompatibility } from "@/lib/timeline/placement"; +import { findTrackInSceneTracks } from "@/lib/timeline/track-element-update"; /** * Thin adapter: the ONLY file in agent/ that imports from core/. @@ -105,12 +117,429 @@ export const EditorContextAdapter = { core.timeline.deleteElements({ elements }); return { success: true, deletedElements: requestedIds }; }, + + moveTimelineElements({ + elementIds, + start, + targetTrackId, + }: { + elementIds: string[]; + start: number; + targetTrackId?: string; + }): + | { + success: boolean; + movedElements: Array<{ + elementId: string; + trackId: string; + start: number; + end: number; + }>; + } + | { error: string } { + const core = EditorCore.getInstance(); + const activeScene = core.scenes.getActiveSceneOrNull(); + if (!activeScene) { + return { error: "No active timeline" }; + } + + if (!hasTimelineContent(activeScene.tracks)) { + return { error: "No timeline content" }; + } + + const requestedIds = [...new Set(elementIds)]; + const elements = findTimelineElementsWithTracksByIds({ + tracks: activeScene.tracks, + elementIds: requestedIds, + }); + const foundIds = new Set(elements.map(({ element }) => element.id)); + const missingIds = requestedIds.filter( + (elementId) => !foundIds.has(elementId), + ); + + if (missingIds.length > 0) { + return { error: `Timeline elements not found: ${missingIds.join(", ")}` }; + } + + for (const { element, track } of elements) { + const targetTrack = targetTrackId + ? findTrackInSceneTracks({ + tracks: activeScene.tracks, + trackId: targetTrackId, + }) + : track; + if (!targetTrack) { + return { error: `Target track not found: ${targetTrackId}` }; + } + + const validation = validateElementTrackCompatibility({ + element, + track: targetTrack, + }); + if (!validation.isValid) { + return { + error: + validation.errorMessage ?? + "Element cannot be placed on target track", + }; + } + } + + const earliestStart = Math.min( + ...elements.map(({ element }) => element.startTime), + ); + const delta = secondsToTicks(start) - earliestStart; + + for (const { element, track } of elements) { + core.timeline.moveElement({ + sourceTrackId: track.id, + targetTrackId: targetTrackId ?? track.id, + elementId: element.id, + newStartTime: element.startTime + delta, + }); + } + + const updatedScene = core.scenes.getActiveSceneOrNull(); + const movedElements = updatedScene + ? findTimelineElementsWithTracksByIds({ + tracks: updatedScene.tracks, + elementIds: requestedIds, + }).map(({ element, track }) => ({ + elementId: element.id, + trackId: track.id, + start: ticksToSeconds(element.startTime), + end: ticksToSeconds(element.startTime + element.duration), + })) + : []; + + return { success: true, movedElements }; + }, + + addMediaToTimeline({ + assetId, + startTime, + trackType, + duration: requestedDuration, + }: { + assetId: string; + startTime: number; + trackType: "main" | "overlay" | "audio"; + duration?: number; + }): { elementId: string; trackId: string } | { error: string } { + const core = EditorCore.getInstance(); + const activeScene = core.scenes.getActiveSceneOrNull(); + if (!activeScene) { + return { error: "No active timeline" }; + } + + const asset = core.media.getAssets().find((item) => item.id === assetId); + if (!asset) { + return { error: "Asset not found" }; + } + + if ( + !isAssetCompatibleWithRequestedTrack({ assetType: asset.type, trackType }) + ) { + return { error: "Invalid track type for asset" }; + } + + const startTimeTicks = secondsToTicks(startTime); + const duration = resolveRequestedElementDuration({ + assetDuration: asset.duration, + assetType: asset.type, + requestedDuration, + }); + if (duration === null) { + return { error: "Duration exceeds source duration" }; + } + const element = buildElementFromMedia({ + mediaId: asset.id, + mediaType: asset.type, + name: asset.name, + duration, + startTime: startTimeTicks, + buffer: + asset.type === "audio" + ? new AudioBuffer({ length: 1, sampleRate: 44100 }) + : undefined, + }); + + const placement = resolveMediaInsertionPlacement({ + tracks: activeScene.tracks, + trackType, + startTime: startTimeTicks, + duration, + }); + const insertCommand = new InsertElementCommand({ + element, + placement: { mode: "explicit", trackId: placement.trackId }, + }); + + if (placement.addTrackCommand) { + core.command.execute({ + command: new BatchCommand([placement.addTrackCommand, insertCommand]), + }); + } else { + core.command.execute({ command: insertCommand }); + } + + const insertedTrackId = insertCommand.getTrackId() ?? placement.trackId; + return { + elementId: insertCommand.getElementId(), + trackId: insertedTrackId, + }; + }, + + updateTimelineElementTiming({ + elementId, + start, + end, + duration, + }: { + elementId: string; + start?: number; + end?: number; + duration?: number; + }): + | { + success: boolean; + elementId: string; + trackId: string; + start: number; + end: number; + duration: number; + } + | { error: string } { + const core = EditorCore.getInstance(); + const activeScene = core.scenes.getActiveSceneOrNull(); + if (!activeScene) { + return { error: "No active timeline" }; + } + + if (!hasTimelineContent(activeScene.tracks)) { + return { error: "No timeline content" }; + } + + const [resolved] = findTimelineElementsWithTracksByIds({ + tracks: activeScene.tracks, + elementIds: [elementId], + }); + if (!resolved) { + return { error: `Timeline element not found: ${elementId}` }; + } + + const patch = buildTimingPatch({ + element: resolved.element, + start, + end, + duration, + }); + if ("error" in patch) { + return patch; + } + + core.timeline.updateElements({ + updates: [ + { + trackId: resolved.track.id, + elementId: resolved.element.id, + patch, + }, + ], + }); + + const updatedScene = core.scenes.getActiveSceneOrNull(); + const [updated] = updatedScene + ? findTimelineElementsWithTracksByIds({ + tracks: updatedScene.tracks, + elementIds: [elementId], + }) + : []; + const element = updated?.element ?? resolved.element; + const track = updated?.track ?? resolved.track; + + return { + success: true, + elementId: element.id, + trackId: track.id, + start: ticksToSeconds(element.startTime), + end: ticksToSeconds(element.startTime + element.duration), + duration: ticksToSeconds(element.duration), + }; + }, }; function secondsToTicks(seconds: number): number { return Math.round(seconds * TICKS_PER_SECOND); } +function ticksToSeconds(ticks: number): number { + return ticks / TICKS_PER_SECOND; +} + +function resolveRequestedElementDuration({ + assetDuration, + assetType, + requestedDuration, +}: { + assetDuration?: number; + assetType: "video" | "audio" | "image"; + requestedDuration?: number; +}): number | null { + if (requestedDuration === undefined) { + return assetDuration != null + ? Math.round(assetDuration * TICKS_PER_SECOND) + : DEFAULT_NEW_ELEMENT_DURATION; + } + + if ( + (assetType === "video" || assetType === "audio") && + assetDuration != null && + requestedDuration > assetDuration + ) { + return null; + } + + return secondsToTicks(requestedDuration); +} + +function buildTimingPatch({ + element, + start, + end, + duration, +}: { + element: TimelineElement; + start?: number; + end?: number; + duration?: number; +}): Partial | { error: string } { + const nextStart = + start === undefined ? element.startTime : secondsToTicks(start); + const nextDuration = resolveTimingDuration({ + element, + startTicks: nextStart, + end, + duration, + }); + if ("error" in nextDuration) { + return nextDuration; + } + + return { + ...(start !== undefined && { startTime: nextStart }), + ...(nextDuration.duration !== element.duration && { + duration: nextDuration.duration, + }), + }; +} + +function resolveTimingDuration({ + element, + startTicks, + end, + duration, +}: { + element: TimelineElement; + startTicks: number; + end?: number; + duration?: number; +}): { duration: number } | { error: string } { + if (duration !== undefined && end !== undefined) { + const durationTicks = secondsToTicks(duration); + const endDerivedDuration = secondsToTicks(end) - startTicks; + if (endDerivedDuration !== durationTicks) { + return { error: "Conflicting timing values" }; + } + } + + const nextDuration = + duration !== undefined + ? secondsToTicks(duration) + : end !== undefined + ? secondsToTicks(end) - startTicks + : element.duration; + + if (nextDuration <= 0) { + return { error: "Invalid time range" }; + } + + const maxDuration = getMaxElementDurationTicks({ element }); + if (maxDuration !== undefined && nextDuration > maxDuration) { + return { error: "Duration exceeds source duration" }; + } + + return { duration: nextDuration }; +} + +function getMaxElementDurationTicks({ + element, +}: { + element: TimelineElement; +}): number | undefined { + if ( + (element.type !== "video" && element.type !== "audio") || + typeof element.sourceDuration !== "number" + ) { + return undefined; + } + + return Math.max( + 0, + element.sourceDuration - element.trimStart - element.trimEnd, + ); +} + +function isAssetCompatibleWithRequestedTrack({ + assetType, + trackType, +}: { + assetType: "video" | "audio" | "image"; + trackType: "main" | "overlay" | "audio"; +}): boolean { + if (trackType === "audio") { + return assetType === "audio"; + } + return assetType === "video" || assetType === "image"; +} + +function resolveMediaInsertionPlacement({ + tracks, + trackType, + startTime, + duration, +}: { + tracks: SceneTracks; + trackType: "main" | "overlay" | "audio"; + startTime: number; + duration: number; +}): { trackId: string; addTrackCommand?: AddTrackCommand } { + if (trackType === "main") { + return { trackId: tracks.main.id }; + } + + const compatibleTrackType: TrackType = + trackType === "audio" ? "audio" : "video"; + const candidateTracks = trackType === "audio" ? tracks.audio : tracks.overlay; + const availableTrack = candidateTracks.find( + (track) => + track.type === compatibleTrackType && + canPlaceTimeSpansOnTrack({ + track, + timeSpans: [{ startTime, duration }], + }), + ); + if (availableTrack) { + return { trackId: availableTrack.id }; + } + + const addTrackCommand = new AddTrackCommand(compatibleTrackType); + return { + trackId: addTrackCommand.getTrackId(), + addTrackCommand, + }; +} + function hasTimelineContent(tracks: SceneTracks): boolean { return ( tracks.main.elements.length > 0 || @@ -141,4 +570,26 @@ function findTimelineElementsByIds({ return result; } +function findTimelineElementsWithTracksByIds({ + tracks, + elementIds, +}: { + tracks: SceneTracks; + elementIds: string[]; +}): Array<{ track: TimelineTrack; element: TimelineElement }> { + const requestedIds = new Set(elementIds); + const result: Array<{ track: TimelineTrack; element: TimelineElement }> = []; + const allTracks = [tracks.main, ...tracks.overlay, ...tracks.audio]; + + for (const track of allTracks) { + for (const element of track.elements) { + if (requestedIds.has(element.id)) { + result.push({ track, element }); + } + } + } + + return result; +} + export { buildSystemPrompt } from "@/agent/system-prompt"; diff --git a/apps/web/src/agent/orchestrator.ts b/apps/web/src/agent/orchestrator.ts index 6b2065a9..9896deb3 100644 --- a/apps/web/src/agent/orchestrator.ts +++ b/apps/web/src/agent/orchestrator.ts @@ -11,6 +11,9 @@ import "@/agent/tools/list-project-assets.tool"; import "@/agent/tools/list-timeline.tool"; import "@/agent/tools/split.tool"; import "@/agent/tools/delete-timeline-elements.tool"; +import "@/agent/tools/move-timeline-elements.tool"; +import "@/agent/tools/add-media-to-timeline.tool"; +import "@/agent/tools/update-timeline-element-timing.tool"; import { useChatStore } from "@/stores/chat-store"; import { useAgentStore } from "@/stores/agent-store"; diff --git a/apps/web/src/agent/tools/__tests__/add-media-to-timeline.test.ts b/apps/web/src/agent/tools/__tests__/add-media-to-timeline.test.ts new file mode 100644 index 00000000..9f49b892 --- /dev/null +++ b/apps/web/src/agent/tools/__tests__/add-media-to-timeline.test.ts @@ -0,0 +1,100 @@ +import { beforeEach, describe, expect, mock, test } from "bun:test"; +import type { AgentContext } from "@/agent/types"; +import { toolRegistry } from "@/agent/tools/registry"; + +const mockAddMediaToTimeline = mock( + (_args: { + assetId: string; + startTime: number; + trackType: string; + duration?: number; + }) => ({ + elementId: "element-1", + trackId: "main", + }), +); + +mock.module("@/agent/context", () => ({ + EditorContextAdapter: { + addMediaToTimeline: mockAddMediaToTimeline, + }, +})); + +await import("@/agent/tools/add-media-to-timeline.tool"); + +const context: AgentContext = { + projectId: "proj-1", + activeSceneId: "scene-A", + mediaAssets: [], + playbackTimeMs: 0, +}; + +describe("add_media_to_timeline tool", () => { + beforeEach(() => { + mockAddMediaToTimeline.mockClear(); + }); + + test("is registered in the tool registry", () => { + expect(toolRegistry.has("add_media_to_timeline")).toBe(true); + }); + + test("adds media through the editor adapter", async () => { + const tool = toolRegistry.get("add_media_to_timeline"); + const result = await tool.execute( + { assetId: "asset-1", startTime: 2, trackType: "main" }, + context, + ); + + expect(mockAddMediaToTimeline).toHaveBeenCalledWith({ + assetId: "asset-1", + startTime: 2, + trackType: "main", + }); + expect(result).toEqual({ elementId: "element-1", trackId: "main" }); + }); + + test("passes optional duration through the editor adapter", async () => { + const tool = toolRegistry.get("add_media_to_timeline"); + await tool.execute( + { assetId: "asset-1", startTime: 2, trackType: "overlay", duration: 6 }, + context, + ); + + expect(mockAddMediaToTimeline).toHaveBeenCalledWith({ + assetId: "asset-1", + startTime: 2, + trackType: "overlay", + duration: 6, + }); + }); + + test("validates add media arguments", async () => { + const tool = toolRegistry.get("add_media_to_timeline"); + + expect( + await tool.execute( + { assetId: "", startTime: 0, trackType: "main" }, + context, + ), + ).toEqual({ error: "Asset not found" }); + expect( + await tool.execute( + { assetId: "asset-1", startTime: -1, trackType: "main" }, + context, + ), + ).toEqual({ error: "Invalid start time" }); + expect( + await tool.execute( + { assetId: "asset-1", startTime: 0, trackType: "text" }, + context, + ), + ).toEqual({ error: "Invalid track type for asset" }); + expect( + await tool.execute( + { assetId: "asset-1", startTime: 0, trackType: "main", duration: 0 }, + context, + ), + ).toEqual({ error: "Invalid duration" }); + expect(mockAddMediaToTimeline).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/web/src/agent/tools/__tests__/move-timeline-elements.test.ts b/apps/web/src/agent/tools/__tests__/move-timeline-elements.test.ts new file mode 100644 index 00000000..43b2984e --- /dev/null +++ b/apps/web/src/agent/tools/__tests__/move-timeline-elements.test.ts @@ -0,0 +1,95 @@ +import { beforeEach, describe, expect, mock, test } from "bun:test"; +import type { AgentContext } from "@/agent/types"; +import { toolRegistry } from "@/agent/tools/registry"; + +const mockMoveTimelineElements = mock( + (_args: { elementIds: string[]; start: number; targetTrackId?: string }) => ({ + success: true, + movedElements: [{ elementId: "clip-1", trackId: "main", start: 4, end: 8 }], + }), +); + +mock.module("@/agent/context", () => ({ + EditorContextAdapter: { + moveTimelineElements: mockMoveTimelineElements, + }, +})); + +await import("@/agent/tools/move-timeline-elements.tool"); + +const context: AgentContext = { + projectId: "proj-1", + activeSceneId: "scene-A", + mediaAssets: [], + playbackTimeMs: 0, +}; + +describe("move_timeline_elements tool", () => { + beforeEach(() => { + mockMoveTimelineElements.mockClear(); + }); + + test("is registered in the tool registry", () => { + expect(toolRegistry.has("move_timeline_elements")).toBe(true); + }); + + test("moves timeline elements through the editor adapter", async () => { + const tool = toolRegistry.get("move_timeline_elements"); + const result = await tool.execute( + { elementIds: ["clip-1"], start: 4, targetTrackId: "overlay-1" }, + context, + ); + + expect(mockMoveTimelineElements).toHaveBeenCalledWith({ + elementIds: ["clip-1"], + start: 4, + targetTrackId: "overlay-1", + }); + expect(result).toEqual({ + success: true, + movedElements: [ + { elementId: "clip-1", trackId: "main", start: 4, end: 8 }, + ], + }); + }); + + test("omits targetTrackId when moving within current tracks", async () => { + const tool = toolRegistry.get("move_timeline_elements"); + await tool.execute({ elementIds: ["clip-1"], start: 4 }, context); + + expect(mockMoveTimelineElements).toHaveBeenCalledWith({ + elementIds: ["clip-1"], + start: 4, + }); + }); + + test("validates move arguments", async () => { + const tool = toolRegistry.get("move_timeline_elements"); + + expect(await tool.execute({ elementIds: [], start: 0 }, context)).toEqual({ + error: "Invalid element ids", + }); + expect( + await tool.execute({ elementIds: ["clip-1"], start: -1 }, context), + ).toEqual({ + error: "Invalid start time", + }); + expect( + await tool.execute( + { elementIds: ["clip-1"], start: Number.NaN }, + context, + ), + ).toEqual({ + error: "Invalid start time", + }); + expect( + await tool.execute( + { elementIds: ["clip-1"], start: 0, targetTrackId: "" }, + context, + ), + ).toEqual({ + error: "Invalid target track id", + }); + expect(mockMoveTimelineElements).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/web/src/agent/tools/__tests__/update-timeline-element-timing.test.ts b/apps/web/src/agent/tools/__tests__/update-timeline-element-timing.test.ts new file mode 100644 index 00000000..68adce5c --- /dev/null +++ b/apps/web/src/agent/tools/__tests__/update-timeline-element-timing.test.ts @@ -0,0 +1,89 @@ +import { beforeEach, describe, expect, mock, test } from "bun:test"; +import type { AgentContext } from "@/agent/types"; +import { toolRegistry } from "@/agent/tools/registry"; + +const mockUpdateTimelineElementTiming = mock( + (_args: { + elementId: string; + start?: number; + end?: number; + duration?: number; + }) => ({ + success: true, + elementId: "element-1", + trackId: "main", + start: 2, + end: 8, + duration: 6, + }), +); + +mock.module("@/agent/context", () => ({ + EditorContextAdapter: { + updateTimelineElementTiming: mockUpdateTimelineElementTiming, + }, +})); + +await import("@/agent/tools/update-timeline-element-timing.tool"); + +const context: AgentContext = { + projectId: "proj-1", + activeSceneId: "scene-A", + mediaAssets: [], + playbackTimeMs: 0, +}; + +describe("update_timeline_element_timing tool", () => { + beforeEach(() => { + mockUpdateTimelineElementTiming.mockClear(); + }); + + test("is registered in the tool registry", () => { + expect(toolRegistry.has("update_timeline_element_timing")).toBe(true); + }); + + test("updates timing through the editor adapter", async () => { + const tool = toolRegistry.get("update_timeline_element_timing"); + const result = await tool.execute( + { elementId: "element-1", start: 2, duration: 6 }, + context, + ); + + expect(mockUpdateTimelineElementTiming).toHaveBeenCalledWith({ + elementId: "element-1", + start: 2, + duration: 6, + }); + expect(result).toEqual({ + success: true, + elementId: "element-1", + trackId: "main", + start: 2, + end: 8, + duration: 6, + }); + }); + + test("validates timing update arguments", async () => { + const tool = toolRegistry.get("update_timeline_element_timing"); + + expect(await tool.execute({ elementId: "", duration: 1 }, context)).toEqual( + { + error: "Invalid element id", + }, + ); + expect(await tool.execute({ elementId: "element-1" }, context)).toEqual({ + error: "Invalid timing update", + }); + expect( + await tool.execute({ elementId: "element-1", start: -1 }, context), + ).toEqual({ error: "Invalid start time" }); + expect( + await tool.execute({ elementId: "element-1", end: 0 }, context), + ).toEqual({ error: "Invalid end time" }); + expect( + await tool.execute({ elementId: "element-1", duration: 0 }, context), + ).toEqual({ error: "Invalid duration" }); + expect(mockUpdateTimelineElementTiming).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/web/src/agent/tools/add-media-to-timeline.tool.ts b/apps/web/src/agent/tools/add-media-to-timeline.tool.ts new file mode 100644 index 00000000..2b135641 --- /dev/null +++ b/apps/web/src/agent/tools/add-media-to-timeline.tool.ts @@ -0,0 +1,84 @@ +import { EditorContextAdapter } from "@/agent/context"; +import type { AgentContext, ToolDefinition } from "@/agent/types"; +import { toolRegistry } from "@/agent/tools/registry"; + +type AddMediaTrackType = "main" | "overlay" | "audio"; + +export type AddMediaToTimelineArgs = { + assetId: string; + startTime: number; + trackType: AddMediaTrackType; + duration?: number; +}; + +export type AddMediaToTimelineResult = { + elementId: string; + trackId: string; +}; + +const addMediaToTimelineTool: ToolDefinition = { + name: "add_media_to_timeline", + description: + "Adds an existing project media asset to the active timeline. Use list_project_assets first to discover assetId. startTime and optional duration are in timeline seconds. trackType chooses main video, overlay video, or audio placement.", + parameters: [ + { key: "assetId", type: "string", required: true }, + { key: "startTime", type: "number", required: true }, + { key: "trackType", type: "string", required: true }, + { key: "duration", type: "number", required: false }, + ], + execute: async ( + args: Record, + _context: AgentContext, + ): Promise => { + const { assetId, startTime, trackType, duration } = args; + + if (!isValidAssetId(assetId)) { + return { error: "Asset not found" }; + } + if (!isValidStartTime(startTime)) { + return { error: "Invalid start time" }; + } + if (!isValidTrackType(trackType)) { + return { error: "Invalid track type for asset" }; + } + if (!isValidOptionalDuration(duration)) { + return { error: "Invalid duration" }; + } + + return EditorContextAdapter.addMediaToTimeline({ + assetId, + startTime, + trackType, + ...(duration !== undefined && { duration }), + }); + }, +}; + +function isValidAssetId(assetId: unknown): assetId is string { + return typeof assetId === "string" && assetId.trim().length > 0; +} + +function isValidStartTime(startTime: unknown): startTime is number { + return ( + typeof startTime === "number" && + Number.isFinite(startTime) && + startTime >= 0 + ); +} + +function isValidTrackType(trackType: unknown): trackType is AddMediaTrackType { + return ( + trackType === "main" || trackType === "overlay" || trackType === "audio" + ); +} + +function isValidOptionalDuration( + duration: unknown, +): duration is number | undefined { + return ( + duration === undefined || + (typeof duration === "number" && Number.isFinite(duration) && duration > 0) + ); +} + +toolRegistry.register("add_media_to_timeline", addMediaToTimelineTool); diff --git a/apps/web/src/agent/tools/move-timeline-elements.tool.ts b/apps/web/src/agent/tools/move-timeline-elements.tool.ts new file mode 100644 index 00000000..0aa77ba0 --- /dev/null +++ b/apps/web/src/agent/tools/move-timeline-elements.tool.ts @@ -0,0 +1,78 @@ +import { EditorContextAdapter } from "@/agent/context"; +import type { AgentContext, ToolDefinition } from "@/agent/types"; +import { toolRegistry } from "@/agent/tools/registry"; + +export type MoveTimelineElementsArgs = { + elementIds: string[]; + start: number; + targetTrackId?: string; +}; + +export type MoveTimelineElementsResult = { + success: boolean; + movedElements: Array<{ + elementId: string; + trackId: string; + start: number; + end: number; + }>; +}; + +const moveTimelineElementsTool: ToolDefinition = { + name: "move_timeline_elements", + description: + "Moves one or more existing timeline elements to a new timeline start time in seconds. For multiple elements, the earliest selected element is moved to start and the others preserve their relative offsets. Optionally pass targetTrackId to move them to another compatible track.", + parameters: [ + { key: "elementIds", type: "string[]", required: true }, + { key: "start", type: "number", required: true }, + { key: "targetTrackId", type: "string", required: false }, + ], + execute: async ( + args: Record, + _context: AgentContext, + ): Promise => { + const { elementIds, start, targetTrackId } = args; + + if (!isValidElementIds(elementIds)) { + return { error: "Invalid element ids" }; + } + if (!isValidStart(start)) { + return { error: "Invalid start time" }; + } + if (!isValidOptionalTargetTrackId(targetTrackId)) { + return { error: "Invalid target track id" }; + } + + return EditorContextAdapter.moveTimelineElements({ + elementIds, + start, + ...(targetTrackId !== undefined && { targetTrackId }), + }); + }, +}; + +function isValidElementIds(elementIds: unknown): elementIds is string[] { + return ( + Array.isArray(elementIds) && + elementIds.length > 0 && + elementIds.every( + (elementId) => + typeof elementId === "string" && elementId.trim().length > 0, + ) + ); +} + +function isValidStart(start: unknown): start is number { + return typeof start === "number" && Number.isFinite(start) && start >= 0; +} + +function isValidOptionalTargetTrackId( + targetTrackId: unknown, +): targetTrackId is string | undefined { + return ( + targetTrackId === undefined || + (typeof targetTrackId === "string" && targetTrackId.trim().length > 0) + ); +} + +toolRegistry.register("move_timeline_elements", moveTimelineElementsTool); diff --git a/apps/web/src/agent/tools/update-timeline-element-timing.tool.ts b/apps/web/src/agent/tools/update-timeline-element-timing.tool.ts new file mode 100644 index 00000000..bc8a5dee --- /dev/null +++ b/apps/web/src/agent/tools/update-timeline-element-timing.tool.ts @@ -0,0 +1,92 @@ +import { EditorContextAdapter } from "@/agent/context"; +import type { AgentContext, ToolDefinition } from "@/agent/types"; +import { toolRegistry } from "@/agent/tools/registry"; + +export type UpdateTimelineElementTimingArgs = { + elementId: string; + start?: number; + end?: number; + duration?: number; +}; + +export type UpdateTimelineElementTimingResult = { + success: boolean; + elementId: string; + trackId: string; + start: number; + end: number; + duration: number; +}; + +const updateTimelineElementTimingTool: ToolDefinition = { + name: "update_timeline_element_timing", + description: + "Updates an existing timeline element's timing. Use list_timeline first to discover elementId. start, end, and duration are timeline seconds; pass at least one of start, end, or duration.", + parameters: [ + { key: "elementId", type: "string", required: true }, + { key: "start", type: "number", required: false }, + { key: "end", type: "number", required: false }, + { key: "duration", type: "number", required: false }, + ], + execute: async ( + args: Record, + _context: AgentContext, + ): Promise => { + const { elementId, start, end, duration } = args; + + if (!isValidElementId(elementId)) { + return { error: "Invalid element id" }; + } + if (start === undefined && end === undefined && duration === undefined) { + return { error: "Invalid timing update" }; + } + if (!isValidOptionalStart(start)) { + return { error: "Invalid start time" }; + } + if (!isValidOptionalEnd(end)) { + return { error: "Invalid end time" }; + } + if (!isValidOptionalDuration(duration)) { + return { error: "Invalid duration" }; + } + + return EditorContextAdapter.updateTimelineElementTiming({ + elementId, + ...(start !== undefined && { start }), + ...(end !== undefined && { end }), + ...(duration !== undefined && { duration }), + }); + }, +}; + +function isValidElementId(elementId: unknown): elementId is string { + return typeof elementId === "string" && elementId.trim().length > 0; +} + +function isValidOptionalStart(start: unknown): start is number | undefined { + return ( + start === undefined || + (typeof start === "number" && Number.isFinite(start) && start >= 0) + ); +} + +function isValidOptionalEnd(end: unknown): end is number | undefined { + return ( + end === undefined || + (typeof end === "number" && Number.isFinite(end) && end > 0) + ); +} + +function isValidOptionalDuration( + duration: unknown, +): duration is number | undefined { + return ( + duration === undefined || + (typeof duration === "number" && Number.isFinite(duration) && duration > 0) + ); +} + +toolRegistry.register( + "update_timeline_element_timing", + updateTimelineElementTimingTool, +); diff --git a/apps/web/src/app/api/agent/chat/__tests__/route.test.ts b/apps/web/src/app/api/agent/chat/__tests__/route.test.ts index dd339a8a..2f2f0a4f 100644 --- a/apps/web/src/app/api/agent/chat/__tests__/route.test.ts +++ b/apps/web/src/app/api/agent/chat/__tests__/route.test.ts @@ -152,7 +152,7 @@ describe("POST /api/agent/chat", () => { expect(callArgs.messages).toHaveLength(1); expect(callArgs.systemPrompt).toContain("NeuralCut"); - expect(callArgs.tools).toHaveLength(5); + expect(callArgs.tools).toHaveLength(8); expect( callArgs.tools.map((tool) => (tool as { name: string }).name), ).toEqual([ @@ -161,6 +161,9 @@ describe("POST /api/agent/chat", () => { "list_timeline", "split", "delete_timeline_elements", + "move_timeline_elements", + "add_media_to_timeline", + "update_timeline_element_timing", ]); }); diff --git a/apps/web/src/app/api/agent/chat/route.ts b/apps/web/src/app/api/agent/chat/route.ts index 81fca83d..72e02b43 100644 --- a/apps/web/src/app/api/agent/chat/route.ts +++ b/apps/web/src/app/api/agent/chat/route.ts @@ -49,6 +49,38 @@ const providerToolSchemas: ToolSchema[] = [ "Deletes one or more timeline elements by elementId. Use list_timeline first to discover exact elementIds. To delete a time range, split at the range boundaries first, then delete the isolated elementIds.", parameters: [{ key: "elementIds", type: "string[]", required: true }], }, + { + name: "move_timeline_elements", + description: + "Moves one or more existing timeline elements to a new timeline start time in seconds. For multiple elements, the earliest selected element is moved to start and the others preserve their relative offsets. Optionally pass targetTrackId to move them to another compatible track.", + parameters: [ + { key: "elementIds", type: "string[]", required: true }, + { key: "start", type: "number", required: true }, + { key: "targetTrackId", type: "string", required: false }, + ], + }, + { + name: "add_media_to_timeline", + description: + "Adds an existing project media asset to the active timeline. Use list_project_assets first to discover assetId. startTime and optional duration are in timeline seconds. trackType must be main, overlay, or audio.", + parameters: [ + { key: "assetId", type: "string", required: true }, + { key: "startTime", type: "number", required: true }, + { key: "trackType", type: "string", required: true }, + { key: "duration", type: "number", required: false }, + ], + }, + { + name: "update_timeline_element_timing", + description: + "Updates an existing timeline element's timing. Use list_timeline first to discover elementId. start, end, and duration are timeline seconds; pass at least one of start, end, or duration.", + parameters: [ + { key: "elementId", type: "string", required: true }, + { key: "start", type: "number", required: false }, + { key: "end", type: "number", required: false }, + { key: "duration", type: "number", required: false }, + ], + }, ]; // --------------------------------------------------------------------------- diff --git a/apps/web/src/components/editor/panels/chat/tool-formatters.ts b/apps/web/src/components/editor/panels/chat/tool-formatters.ts index 7dab6717..4715083d 100644 --- a/apps/web/src/components/editor/panels/chat/tool-formatters.ts +++ b/apps/web/src/components/editor/panels/chat/tool-formatters.ts @@ -88,6 +88,34 @@ const TOOL_CALL_FORMATTERS: Record< : "No elements specified", }; }, + move_timeline_elements: (args) => { + const elementIds = args.elementIds as string[] | undefined; + const count = elementIds?.length ?? 0; + const start = formatSeconds(args.start); + return { + label: "Move", + description: `${count} element${count !== 1 ? "s" : ""} to ${start}`, + }; + }, + add_media_to_timeline: (args) => ({ + label: "Add Media", + description: `${String(args.assetId ?? "asset")} to ${String(args.trackType ?? "timeline")} at ${formatSeconds(args.startTime)}${typeof args.duration === "number" ? ` for ${formatSeconds(args.duration)}` : ""}`, + }), + update_timeline_element_timing: (args) => { + const parts = [ + typeof args.start === "number" + ? `start ${formatSeconds(args.start)}` + : null, + typeof args.end === "number" ? `end ${formatSeconds(args.end)}` : null, + typeof args.duration === "number" + ? `duration ${formatSeconds(args.duration)}` + : null, + ].filter(Boolean); + return { + label: "Update Timing", + description: parts.length > 0 ? parts.join(", ") : "No timing changes", + }; + }, load_context: (args) => { const targetType = String(args.targetType ?? "unknown"); const id = args.id ?? args.assetId ?? args.elementId; @@ -159,6 +187,46 @@ const TOOL_RESULT_FORMATTERS: Record< : "Failed", }; }, + move_timeline_elements: (parsed) => { + const data = parsed as { + success?: boolean; + movedElements?: unknown[]; + } | null; + if (!data) return null; + const count = data.movedElements?.length ?? 0; + return { + label: "Moved", + description: data.success + ? `${count} element${count !== 1 ? "s" : ""}` + : "Failed", + }; + }, + add_media_to_timeline: (parsed) => { + const data = parsed as { + elementId?: string; + trackId?: string; + } | null; + if (!data) return null; + return { + label: "Media Added", + description: data.trackId ? `track ${data.trackId}` : "Inserted", + }; + }, + update_timeline_element_timing: (parsed) => { + const data = parsed as { + success?: boolean; + start?: number; + end?: number; + duration?: number; + } | null; + if (!data) return null; + return { + label: "Timing Updated", + description: data.success + ? `${formatSeconds(data.start)}-${formatSeconds(data.end)} (${formatSeconds(data.duration)})` + : "Failed", + }; + }, load_context: (parsed) => { const data = parsed as { status?: string; diff --git a/docs/agent-tool-specs.md b/docs/agent-tool-specs.md index dacd1aca..8a67a16f 100644 --- a/docs/agent-tool-specs.md +++ b/docs/agent-tool-specs.md @@ -217,7 +217,52 @@ Eliminar uno o más elementos concretos del timeline por `elementId`. Para elimi --- -## 6. `add_text` +## 6. `move_timeline_elements` + +### Propósito +Mover uno o más elementos existentes del timeline a un nuevo tiempo inicial. Si se mueven varios elementos, el elemento más temprano queda en `start` y los demás conservan sus offsets relativos. Opcionalmente puede moverse el grupo a otra pista compatible con `targetTrackId`. + +### Input +```ts +{ + elementIds: string[]; + start: number; // timeline seconds + targetTrackId?: string; +} +``` + +### Output +```ts +{ + success: boolean; + movedElements: Array<{ + elementId: string; + trackId: string; + start: number; + end: number; + }>; +} +``` + +### Requirements +- MUST validate `elementIds` contains at least one non-empty string. +- MUST validate `start` is a finite non-negative timeline time in seconds. +- MUST use `list_timeline` first when exact `elementId` or `targetTrackId` values are unknown. +- MUST preserve relative offsets when moving multiple elements. +- MUST keep elements on their current tracks when `targetTrackId` is omitted. +- MUST fail without mutating if any requested element is missing or the target track is incompatible. +- MUST preserve undo/redo behavior if supported. + +### Errors +- Invalid ids: `{ error: "Invalid element ids" }`. +- Invalid start: `{ error: "Invalid start time" }`. +- Missing element: `{ error: "Timeline elements not found: " }`. +- Missing target track: `{ error: "Target track not found: " }`. +- Empty timeline: `{ error: "No timeline content" }`. + +--- + +## 7. `add_text` ### Propósito Agregar texto visual al timeline. Esta primitive cubre títulos, hooks, labels y subtítulos básicos. @@ -255,7 +300,7 @@ Agregar texto visual al timeline. Esta primitive cubre títulos, hooks, labels y --- -## 7. `update_text` +## 8. `update_text` ### Propósito Editar un texto existente en el timeline. @@ -293,7 +338,7 @@ Editar un texto existente en el timeline. --- -## 8. `add_media_to_timeline` +## 9. `add_media_to_timeline` ### Propósito Agregar un asset existente al timeline. @@ -304,6 +349,7 @@ Agregar un asset existente al timeline. assetId: string; startTime: number; trackType: "main" | "overlay" | "audio"; + duration?: number; // timeline seconds } ``` @@ -318,16 +364,72 @@ Agregar un asset existente al timeline. ### Requirements - MUST resolve asset by `assetId`. - MUST validate asset type compatibility with `trackType`. +- MUST validate optional `duration` as a positive timeline duration in seconds. - MUST insert the element using existing timeline APIs. +- MUST use full source duration for video/audio by default. +- MUST use editor default duration for images when `duration` is omitted. +- MUST reject requested video/audio duration beyond known source duration. - SHOULD choose a sensible track when one is not obvious, but first version requires explicit `trackType`. ### Errors - Asset not found: `{ error: "Asset not found" }`. - Invalid track type: `{ error: "Invalid track type for asset" }`. +- Invalid duration: `{ error: "Invalid duration" }`. +- Duration too long: `{ error: "Duration exceeds source duration" }`. --- -## 9. `delete_element` *(deprecated in favor of `delete_timeline_elements`)* +## 10. `update_timeline_element_timing` + +### Propósito +Actualizar el inicio, final o duración de un elemento existente del timeline sin cambiar el asset original. Esta primitive cubre pedidos como “hacé que esta foto dure 10 segundos” o “que este clip termine en 00:15”. + +### Input +```ts +{ + elementId: string; + start?: number; // timeline seconds + end?: number; // timeline seconds + duration?: number; // timeline seconds +} +``` + +### Output +```ts +{ + success: boolean; + elementId: string; + trackId: string; + start: number; + end: number; + duration: number; +} +``` + +### Requirements +- MUST validate `elementId` as a non-empty string. +- MUST require at least one of `start`, `end`, or `duration`. +- MUST accept all timing values in seconds and convert to the editor’s canonical unit internally. +- MUST use `list_timeline` first when exact `elementId` is unknown. +- MUST update only the timeline element, never the project asset. +- MUST preserve undo/redo behavior if supported. +- MUST reject conflicting `end` and `duration` values when both are provided. +- MUST reject video/audio duration beyond known source duration. + +### Errors +- Invalid id: `{ error: "Invalid element id" }`. +- Invalid update: `{ error: "Invalid timing update" }`. +- Invalid start: `{ error: "Invalid start time" }`. +- Invalid end: `{ error: "Invalid end time" }`. +- Invalid duration: `{ error: "Invalid duration" }`. +- Invalid range: `{ error: "Invalid time range" }`. +- Conflicting values: `{ error: "Conflicting timing values" }`. +- Missing element: `{ error: "Timeline element not found: " }`. +- Duration too long: `{ error: "Duration exceeds source duration" }`. + +--- + +## 11. `delete_element` *(deprecated in favor of `delete_timeline_elements`)* ### Propósito Eliminar un elemento específico del timeline. La implementación actual debe preferir `delete_timeline_elements` porque soporta borrado en lote y permite componer rangos después de `split`. @@ -357,7 +459,7 @@ Eliminar un elemento específico del timeline. La implementación actual debe pr --- -## 10. `set_volume` +## 12. `set_volume` ### Propósito Ajustar volumen de un elemento de audio o video. @@ -390,7 +492,7 @@ Ajustar volumen de un elemento de audio o video. --- -## 11. `add_sticker` +## 13. `add_sticker` ### Propósito Agregar un sticker existente al timeline. @@ -425,7 +527,7 @@ Agregar un sticker existente al timeline. --- -## 12. `apply_effect` +## 14. `apply_effect` ### Propósito Aplicar un efecto existente a un clip. En el estado actual del repo, el efecto real disponible parece ser `blur`. diff --git a/docs/agent-tools.md b/docs/agent-tools.md index bec0bbf3..4b100d89 100644 --- a/docs/agent-tools.md +++ b/docs/agent-tools.md @@ -119,6 +119,7 @@ Agrega un asset existente al timeline. assetId: string; startTime: number; trackType: "main" | "overlay" | "audio"; + duration?: number; // timeline seconds } ``` @@ -126,7 +127,29 @@ Uso: - insertar clips, - agregar música, - poner b-roll, -- agregar imágenes/logos. +- agregar imágenes/logos, +- definir cuánto dura una imagen o un tramo insertado. + +--- + +### `update_timeline_element_timing` + +Actualiza el inicio, final o duración de un elemento existente del timeline. + +```ts +{ + elementId: string; + start?: number; + end?: number; + duration?: number; +} +``` + +Uso: +- hacer que una foto dure más o menos, +- ajustar el final de un texto/sticker/imagen, +- mover el inicio y mantener o recalcular duración, +- recortar la duración visible de audio/video sin cambiar el asset original. --- @@ -150,6 +173,26 @@ Uso: --- +### `move_timeline_elements` + +Mueve uno o más elementos existentes del timeline a otro tiempo y, opcionalmente, a otra pista compatible. + +```ts +{ + elementIds: string[]; + start: number; // timeline seconds + targetTrackId?: string; +} +``` + +Uso: +- mover un clip al inicio, +- correr subtítulos unos segundos, +- reubicar b-roll en una pista superior, +- mantener un grupo sincronizado preservando offsets relativos. + +--- + ### `add_text` Agrega texto visual al timeline. @@ -315,11 +358,13 @@ Diferida porque el repo actual no parece tener corrección de color/LUTs impleme 3. `load_asset_context` 4. `split` 5. `delete_timeline_elements` -6. `add_text` +6. `move_timeline_elements` 7. `add_media_to_timeline` -8. `set_volume` -9. `add_sticker` -10. `apply_effect` +8. `update_timeline_element_timing` +9. `add_text` +10. `set_volume` +11. `add_sticker` +12. `apply_effect` Infra interna asociada: diff --git a/propuesta_tecnica.md b/propuesta_tecnica.md index 1b8fff8c..3ce25dcb 100644 --- a/propuesta_tecnica.md +++ b/propuesta_tecnica.md @@ -50,7 +50,7 @@ Decisiones vigentes: Roadmap vigente de alto nivel: 1. **Contexto y percepción:** `list_project_assets`, `list_timeline`, `load_asset_context`. -2. **Edición básica:** `split`, `delete_timeline_elements`, `add_text`, `add_media_to_timeline`. +2. **Edición básica:** `split`, `delete_timeline_elements`, `move_timeline_elements`, `add_media_to_timeline`, `update_timeline_element_timing`, `add_text`. 3. **Ajustes simples:** `set_volume`, `add_sticker`, `apply_effect`. 4. **UX avanzada:** referencias `@asset`, progreso de uploads/procesamiento, previews/aprobaciones. @@ -423,9 +423,11 @@ Estas son las tools que el equipo debe priorizar. Son suficientemente pequeñas | `load_asset_context` | Carga un asset en el contexto multimodal de Gemini y cachea la referencia | Prioridad crítica / killer feature | | `split` | Hace cortes puntuales en uno o más timestamps sin borrar contenido | Prioridad alta | | `delete_timeline_elements` | Borra uno o más elementos concretos del timeline por `elementId` | Prioridad alta | +| `move_timeline_elements` | Mueve elementos existentes a otro tiempo o pista compatible | Prioridad alta | | `add_text` | Agrega texto visual: hooks, títulos, labels, subtítulos básicos | Prioridad alta | | `update_text` | Modifica texto existente | Prioridad media | | `add_media_to_timeline` | Inserta video/audio/imagen existente al timeline | Prioridad alta | +| `update_timeline_element_timing` | Ajusta inicio, final o duración de un elemento existente | Prioridad alta | | `delete_element` | Reemplazada por `delete_timeline_elements` para soportar borrado en lote | Baja | | `set_volume` | Ajusta volumen de audio/video | Prioridad media | | `add_sticker` | Inserta sticker existente | Prioridad media | @@ -842,6 +844,8 @@ Esto no reemplaza la primera feature de producto; la **habilita**. **Should-have actualizado:** - Como usuario, puedo insertar assets existentes al timeline con `add_media_to_timeline`. - Como usuario, puedo borrar elementos específicos con `delete_timeline_elements`. +- Como usuario, puedo mover elementos existentes con `move_timeline_elements`. +- Como usuario, puedo ajustar cuánto dura un elemento existente con `update_timeline_element_timing`. - Como usuario, puedo ajustar volumen con `set_volume`. - Como usuario, puedo agregar stickers con `add_sticker`. - Como usuario, puedo aplicar efectos existentes con `apply_effect` (inicialmente `blur`).