From 99ed16854b6002d6340695c9463f020ab88e5cc9 Mon Sep 17 00:00:00 2001 From: Luis Esteban Acevedo Ladino Date: Fri, 24 Apr 2026 10:34:22 -0500 Subject: [PATCH] feat: add timeline listing tool --- .../agent/__tests__/context-mapper.test.ts | 113 +++++++++++++++++- apps/web/src/agent/context-mapper.ts | 79 +++++++++++- apps/web/src/agent/orchestrator.ts | 1 + .../tools/__tests__/list-timeline.test.ts | 78 ++++++++++++ .../web/src/agent/tools/list-timeline.tool.ts | 29 +++++ apps/web/src/agent/types.ts | 14 +++ .../api/agent/chat/__tests__/route.test.ts | 18 ++- apps/web/src/app/api/agent/chat/route.ts | 25 ++++ 8 files changed, 351 insertions(+), 6 deletions(-) create mode 100644 apps/web/src/agent/tools/__tests__/list-timeline.test.ts create mode 100644 apps/web/src/agent/tools/list-timeline.tool.ts diff --git a/apps/web/src/agent/__tests__/context-mapper.test.ts b/apps/web/src/agent/__tests__/context-mapper.test.ts index 585165df..663015d7 100644 --- a/apps/web/src/agent/__tests__/context-mapper.test.ts +++ b/apps/web/src/agent/__tests__/context-mapper.test.ts @@ -67,9 +67,13 @@ describe("buildContextFromEditorState (context mapper)", () => { activeScene: { id: "scene-A", tracks: { - main: { elements: [{ mediaId: "m1" }] }, - overlay: [{ elements: [{ mediaId: "m3" }] }], - audio: [{ elements: [{ mediaId: "m2" }] }], + main: { id: "main", type: "video", elements: [{ mediaId: "m1" }] }, + overlay: [ + { id: "overlay", type: "video", elements: [{ mediaId: "m3" }] }, + ], + audio: [ + { id: "audio", type: "audio", elements: [{ mediaId: "m2" }] }, + ], }, }, assets: [ @@ -88,6 +92,109 @@ describe("buildContextFromEditorState (context mapper)", () => { ]); }); + test("maps active timeline tracks for agent tools", () => { + const ctx = buildContextFromEditorState({ + project: { metadata: { id: "proj-1" } }, + activeScene: { + id: "scene-A", + tracks: { + main: { + id: "main-track", + type: "video", + elements: [ + { + id: "clip-1", + type: "video", + mediaId: "m1", + name: "Intro", + startTime: 2, + duration: 5, + }, + ], + }, + overlay: [ + { + id: "text-track", + type: "text", + elements: [ + { + id: "text-1", + type: "text", + name: "Caption", + startTime: 3, + duration: 2, + }, + ], + }, + ], + audio: [ + { + id: "audio-track", + type: "audio", + elements: [ + { + id: "music-1", + type: "audio", + mediaId: "m2", + name: "Music", + startTime: 0, + duration: 10, + }, + ], + }, + ], + }, + }, + assets: [], + currentTimeTicks: 0, + ticksPerSecond: 100, + }); + + expect(ctx.timelineTracks).toEqual([ + { + trackId: "main-track", + type: "main", + elements: [ + { + elementId: "clip-1", + type: "video", + assetId: "m1", + name: "Intro", + start: 2, + end: 7, + }, + ], + }, + { + trackId: "text-track", + type: "text", + elements: [ + { + elementId: "text-1", + type: "text", + name: "Caption", + start: 3, + end: 5, + }, + ], + }, + { + trackId: "audio-track", + type: "audio", + elements: [ + { + elementId: "music-1", + type: "audio", + assetId: "m2", + name: "Music", + start: 0, + end: 10, + }, + ], + }, + ]); + }); + test("calculates playbackTimeMs from ticks correctly", () => { const ctx = buildContextFromEditorState({ project: { metadata: { id: "p" } }, diff --git a/apps/web/src/agent/context-mapper.ts b/apps/web/src/agent/context-mapper.ts index 30a913c0..160caade 100644 --- a/apps/web/src/agent/context-mapper.ts +++ b/apps/web/src/agent/context-mapper.ts @@ -1,4 +1,4 @@ -import type { AgentContext } from "@/agent/types"; +import type { AgentContext, AgentTimelineTrack } from "@/agent/types"; /** * Pure data-mapping function extracted from EditorContextAdapter. @@ -25,6 +25,7 @@ export function buildContextFromEditorState(params: { duration: a.duration ?? 0, usedInTimeline: usedMediaIds.has(a.id), })), + timelineTracks: buildTimelineTracks(params.activeScene), playbackTimeMs: Math.round( (params.currentTimeTicks / params.ticksPerSecond) * 1000, ), @@ -41,9 +42,63 @@ type ActiveSceneInput = { }; type TrackInput = { + id?: string; + type?: string; elements?: unknown[]; }; +function buildTimelineTracks( + scene: ActiveSceneInput | null, +): AgentTimelineTrack[] | undefined { + if (!scene?.tracks) { + return undefined; + } + + const tracks: AgentTimelineTrack[] = []; + + if (scene.tracks.main) { + tracks.push(toTimelineTrack(scene.tracks.main, "main")); + } + + for (const track of scene.tracks.overlay ?? []) { + tracks.push(toTimelineTrack(track, mapOverlayTrackType(track.type))); + } + + for (const track of scene.tracks.audio ?? []) { + tracks.push(toTimelineTrack(track, "audio")); + } + + return tracks; +} + +function toTimelineTrack( + track: TrackInput, + type: AgentTimelineTrack["type"], +): AgentTimelineTrack { + return { + trackId: track.id ?? "", + type, + elements: (track.elements ?? []) + .filter(hasTimelineElementShape) + .map((element) => ({ + elementId: element.id, + type: element.type, + ...(hasMediaId(element) ? { assetId: element.mediaId } : {}), + ...(element.name ? { name: element.name } : {}), + start: element.startTime, + end: element.startTime + element.duration, + })), + }; +} + +function mapOverlayTrackType( + type: string | undefined, +): AgentTimelineTrack["type"] { + if (type === "text") return "text"; + if (type === "effect") return "effect"; + return "overlay"; +} + function collectUsedMediaIds(scene: ActiveSceneInput | null): Set { const ids = new Set(); if (!scene?.tracks) { @@ -75,3 +130,25 @@ function hasMediaId(element: unknown): element is { mediaId: string } { typeof element.mediaId === "string" ); } + +function hasTimelineElementShape(element: unknown): element is { + id: string; + type: string; + name?: string; + startTime: number; + duration: number; +} { + return ( + typeof element === "object" && + element !== null && + "id" in element && + "type" in element && + "startTime" in element && + "duration" in element && + typeof element.id === "string" && + typeof element.type === "string" && + (!("name" in element) || typeof element.name === "string") && + typeof element.startTime === "number" && + typeof element.duration === "number" + ); +} diff --git a/apps/web/src/agent/orchestrator.ts b/apps/web/src/agent/orchestrator.ts index 400704de..1b5db501 100644 --- a/apps/web/src/agent/orchestrator.ts +++ b/apps/web/src/agent/orchestrator.ts @@ -7,6 +7,7 @@ import type { } from "@/agent/types"; import { toolRegistry } from "@/agent/tools/registry"; import "@/agent/tools/list-project-assets.tool"; +import "@/agent/tools/list-timeline.tool"; import "@/agent/tools/transcribe-video.tool"; import { useChatStore } from "@/stores/chat-store"; import { useAgentStore } from "@/stores/agent-store"; diff --git a/apps/web/src/agent/tools/__tests__/list-timeline.test.ts b/apps/web/src/agent/tools/__tests__/list-timeline.test.ts new file mode 100644 index 00000000..6e308790 --- /dev/null +++ b/apps/web/src/agent/tools/__tests__/list-timeline.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, test } from "bun:test"; +import type { AgentContext } from "@/agent/types"; +import { toolRegistry } from "@/agent/tools/registry"; + +await import("@/agent/tools/list-timeline.tool"); + +function makeContext(overrides: Partial = {}): AgentContext { + return { + projectId: "proj-1", + activeSceneId: "scene-A", + mediaAssets: [], + timelineTracks: [ + { + trackId: "main-track", + type: "main", + elements: [ + { + elementId: "clip-1", + type: "video", + assetId: "m1", + name: "Intro", + start: 0, + end: 10, + }, + ], + }, + ], + playbackTimeMs: 0, + ...overrides, + }; +} + +describe("list_timeline tool", () => { + test("is registered in the tool registry", () => { + expect(toolRegistry.has("list_timeline")).toBe(true); + }); + + test("returns active timeline tracks", async () => { + const tool = toolRegistry.get("list_timeline"); + const result = await tool.execute({}, makeContext()); + + expect(result).toEqual({ + tracks: [ + { + trackId: "main-track", + type: "main", + elements: [ + { + elementId: "clip-1", + type: "video", + assetId: "m1", + name: "Intro", + start: 0, + end: 10, + }, + ], + }, + ], + }); + }); + + test("returns no active timeline error without active scene", async () => { + const tool = toolRegistry.get("list_timeline"); + const result = await tool.execute({}, makeContext({ activeSceneId: null })); + + expect(result).toEqual({ error: "No active timeline" }); + }); + + test("returns no active timeline error without mapped tracks", async () => { + const tool = toolRegistry.get("list_timeline"); + const result = await tool.execute( + {}, + makeContext({ timelineTracks: undefined }), + ); + + expect(result).toEqual({ error: "No active timeline" }); + }); +}); diff --git a/apps/web/src/agent/tools/list-timeline.tool.ts b/apps/web/src/agent/tools/list-timeline.tool.ts new file mode 100644 index 00000000..0dcf3796 --- /dev/null +++ b/apps/web/src/agent/tools/list-timeline.tool.ts @@ -0,0 +1,29 @@ +import type { + AgentContext, + AgentTimelineTrack, + ToolDefinition, +} from "@/agent/types"; +import { toolRegistry } from "@/agent/tools/registry"; + +export type ListTimelineResult = { + tracks: AgentTimelineTrack[]; +}; + +const listTimelineTool: ToolDefinition = { + name: "list_timeline", + description: + "Lists the active timeline as structured tracks and editable elements with trackId, elementId, type, assetId, name, start, and end times.", + parameters: [], + execute: async ( + _args: Record, + context: AgentContext, + ): Promise => { + if (!context.activeSceneId || !context.timelineTracks) { + return { error: "No active timeline" }; + } + + return { tracks: context.timelineTracks }; + }, +}; + +toolRegistry.register("list_timeline", listTimelineTool); diff --git a/apps/web/src/agent/types.ts b/apps/web/src/agent/types.ts index 6513dedf..197a1979 100644 --- a/apps/web/src/agent/types.ts +++ b/apps/web/src/agent/types.ts @@ -38,9 +38,23 @@ export interface AgentContext { duration: number; usedInTimeline?: boolean; }>; + timelineTracks?: AgentTimelineTrack[]; playbackTimeMs: number; } +export type AgentTimelineTrack = { + trackId: string; + type: "main" | "overlay" | "audio" | "text" | "effect"; + elements: Array<{ + elementId: string; + type: string; + assetId?: string; + name?: string; + start: number; + end: number; + }>; +}; + export interface ToolDefinition { name: string; description: string; diff --git a/apps/web/src/app/api/agent/chat/__tests__/route.test.ts b/apps/web/src/app/api/agent/chat/__tests__/route.test.ts index ae1ec10f..39da3450 100644 --- a/apps/web/src/app/api/agent/chat/__tests__/route.test.ts +++ b/apps/web/src/app/api/agent/chat/__tests__/route.test.ts @@ -36,6 +36,20 @@ const VALID_CONTEXT = { duration: number; usedInTimeline?: boolean; }>, + timelineTracks: undefined as + | Array<{ + trackId: string; + type: "main" | "overlay" | "audio" | "text" | "effect"; + elements: Array<{ + elementId: string; + type: string; + assetId?: string; + name?: string; + start: number; + end: number; + }>; + }> + | undefined, playbackTimeMs: 0, }; @@ -133,10 +147,10 @@ describe("POST /api/agent/chat", () => { expect(callArgs.messages).toHaveLength(1); expect(callArgs.systemPrompt).toContain("NeuralCut"); - expect(callArgs.tools).toHaveLength(2); + expect(callArgs.tools).toHaveLength(3); expect( callArgs.tools.map((tool) => (tool as { name: string }).name), - ).toEqual(["list_project_assets", "transcribe_video"]); + ).toEqual(["list_project_assets", "list_timeline", "transcribe_video"]); }); // ----------------------------------------------------------------------- diff --git a/apps/web/src/app/api/agent/chat/route.ts b/apps/web/src/app/api/agent/chat/route.ts index 2144b764..75edf7b5 100644 --- a/apps/web/src/app/api/agent/chat/route.ts +++ b/apps/web/src/app/api/agent/chat/route.ts @@ -21,6 +21,13 @@ const providerToolDefs: ToolDefinition[] = [ ], execute: async () => ({}), // stub — never called server-side }, + { + name: "list_timeline", + description: + "Lists the active timeline as structured tracks and editable elements with trackId, elementId, type, assetId, name, start, and end times.", + parameters: [], + execute: async () => ({}), // stub — never called server-side + }, { name: "transcribe_video", description: @@ -66,6 +73,24 @@ const chatRequestSchema = z.object({ usedInTimeline: z.boolean().optional(), }), ), + timelineTracks: z + .array( + z.object({ + trackId: z.string(), + type: z.enum(["main", "overlay", "audio", "text", "effect"]), + elements: z.array( + z.object({ + elementId: z.string(), + type: z.string(), + assetId: z.string().optional(), + name: z.string().optional(), + start: z.number(), + end: z.number(), + }), + ), + }), + ) + .optional(), playbackTimeMs: z.number(), }) satisfies z.ZodType, });