RomanNum3ral
/
OpenCut
mirror of https://github.com/OpenCut-app/OpenCut.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat: add timeline listing tool

This commit is contained in:
Luis Esteban Acevedo Ladino 2026-04-24 10:34:22 -05:00
parent b8be9db18b
commit 99ed16854b
8 changed files with 351 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

113
apps/web/src/agent/__tests__/context-mapper.test.ts
View File

@ -67,9 +67,13 @@ describe("buildContextFromEditorState (context mapper)", () => {
activeScene: {
id: "scene-A",
tracks: {
main: { elements: [{ mediaId: "m1" }] },
overlay: [{ elements: [{ mediaId: "m3" }] }],
audio: [{ elements: [{ mediaId: "m2" }] }],
main: { id: "main", type: "video", elements: [{ mediaId: "m1" }] },
overlay: [
{ id: "overlay", type: "video", elements: [{ mediaId: "m3" }] },
],
audio: [
{ id: "audio", type: "audio", elements: [{ mediaId: "m2" }] },
],
},
},
assets: [
@ -88,6 +92,109 @@ describe("buildContextFromEditorState (context mapper)", () => {
]);
});
test("maps active timeline tracks for agent tools", () => {
const ctx = buildContextFromEditorState({
project: { metadata: { id: "proj-1" } },
activeScene: {
id: "scene-A",
tracks: {
main: {
id: "main-track",
type: "video",
elements: [
{
id: "clip-1",
type: "video",
mediaId: "m1",
name: "Intro",
startTime: 2,
duration: 5,
},
],
},
overlay: [
{
id: "text-track",
type: "text",
elements: [
{
id: "text-1",
type: "text",
name: "Caption",
startTime: 3,
duration: 2,
},
],
},
],
audio: [
{
id: "audio-track",
type: "audio",
elements: [
{
id: "music-1",
type: "audio",
mediaId: "m2",
name: "Music",
startTime: 0,
duration: 10,
},
],
},
],
},
},
assets: [],
currentTimeTicks: 0,
ticksPerSecond: 100,
});
expect(ctx.timelineTracks).toEqual([
{
trackId: "main-track",
type: "main",
elements: [
{
elementId: "clip-1",
type: "video",
assetId: "m1",
name: "Intro",
start: 2,
end: 7,
},
],
},
{
trackId: "text-track",
type: "text",
elements: [
{
elementId: "text-1",
type: "text",
name: "Caption",
start: 3,
end: 5,
},
],
},
{
trackId: "audio-track",
type: "audio",
elements: [
{
elementId: "music-1",
type: "audio",
assetId: "m2",
name: "Music",
start: 0,
end: 10,
},
],
},
]);
});
test("calculates playbackTimeMs from ticks correctly", () => {
const ctx = buildContextFromEditorState({
project: { metadata: { id: "p" } },

79
apps/web/src/agent/context-mapper.ts
View File

@ -1,4 +1,4 @@
import type { AgentContext } from "@/agent/types";
import type { AgentContext, AgentTimelineTrack } from "@/agent/types";
/**
* Pure data-mapping function extracted from EditorContextAdapter.
@ -25,6 +25,7 @@ export function buildContextFromEditorState(params: {
duration: a.duration ?? 0,
usedInTimeline: usedMediaIds.has(a.id),
})),
timelineTracks: buildTimelineTracks(params.activeScene),
playbackTimeMs: Math.round(
(params.currentTimeTicks / params.ticksPerSecond) * 1000,
),
@ -41,9 +42,63 @@ type ActiveSceneInput = {
};
type TrackInput = {
id?: string;
type?: string;
elements?: unknown[];
};
function buildTimelineTracks(
scene: ActiveSceneInput | null,
): AgentTimelineTrack[] | undefined {
if (!scene?.tracks) {
return undefined;
}
const tracks: AgentTimelineTrack[] = [];
if (scene.tracks.main) {
tracks.push(toTimelineTrack(scene.tracks.main, "main"));
}
for (const track of scene.tracks.overlay ?? []) {
tracks.push(toTimelineTrack(track, mapOverlayTrackType(track.type)));
}
for (const track of scene.tracks.audio ?? []) {
tracks.push(toTimelineTrack(track, "audio"));
}
return tracks;
}
function toTimelineTrack(
track: TrackInput,
type: AgentTimelineTrack["type"],
): AgentTimelineTrack {
return {
trackId: track.id ?? "",
type,
elements: (track.elements ?? [])
.filter(hasTimelineElementShape)
.map((element) => ({
elementId: element.id,
type: element.type,
...(hasMediaId(element) ? { assetId: element.mediaId } : {}),
...(element.name ? { name: element.name } : {}),
start: element.startTime,
end: element.startTime + element.duration,
})),
};
}
function mapOverlayTrackType(
type: string | undefined,
): AgentTimelineTrack["type"] {
if (type === "text") return "text";
if (type === "effect") return "effect";
return "overlay";
}
function collectUsedMediaIds(scene: ActiveSceneInput | null): Set<string> {
const ids = new Set<string>();
if (!scene?.tracks) {
@ -75,3 +130,25 @@ function hasMediaId(element: unknown): element is { mediaId: string } {
typeof element.mediaId === "string"
);
}
function hasTimelineElementShape(element: unknown): element is {
id: string;
type: string;
name?: string;
startTime: number;
duration: number;
} {
return (
typeof element === "object" &&
element !== null &&
"id" in element &&
"type" in element &&
"startTime" in element &&
"duration" in element &&
typeof element.id === "string" &&
typeof element.type === "string" &&
(!("name" in element) || typeof element.name === "string") &&
typeof element.startTime === "number" &&
typeof element.duration === "number"
);
}

1
apps/web/src/agent/orchestrator.ts
View File

@ -7,6 +7,7 @@ import type {
} from "@/agent/types";
import { toolRegistry } from "@/agent/tools/registry";
import "@/agent/tools/list-project-assets.tool";
import "@/agent/tools/list-timeline.tool";
import "@/agent/tools/transcribe-video.tool";
import { useChatStore } from "@/stores/chat-store";
import { useAgentStore } from "@/stores/agent-store";

78
apps/web/src/agent/tools/__tests__/list-timeline.test.ts Normal file
View File

@ -0,0 +1,78 @@
import { describe, expect, test } from "bun:test";
import type { AgentContext } from "@/agent/types";
import { toolRegistry } from "@/agent/tools/registry";
await import("@/agent/tools/list-timeline.tool");
function makeContext(overrides: Partial<AgentContext> = {}): AgentContext {
return {
projectId: "proj-1",
activeSceneId: "scene-A",
mediaAssets: [],
timelineTracks: [
{
trackId: "main-track",
type: "main",
elements: [
{
elementId: "clip-1",
type: "video",
assetId: "m1",
name: "Intro",
start: 0,
end: 10,
},
],
},
],
playbackTimeMs: 0,
...overrides,
};
}
describe("list_timeline tool", () => {
test("is registered in the tool registry", () => {
expect(toolRegistry.has("list_timeline")).toBe(true);
});
test("returns active timeline tracks", async () => {
const tool = toolRegistry.get("list_timeline");
const result = await tool.execute({}, makeContext());
expect(result).toEqual({
tracks: [
{
trackId: "main-track",
type: "main",
elements: [
{
elementId: "clip-1",
type: "video",
assetId: "m1",
name: "Intro",
start: 0,
end: 10,
},
],
},
],
});
});
test("returns no active timeline error without active scene", async () => {
const tool = toolRegistry.get("list_timeline");
const result = await tool.execute({}, makeContext({ activeSceneId: null }));
expect(result).toEqual({ error: "No active timeline" });
});
test("returns no active timeline error without mapped tracks", async () => {
const tool = toolRegistry.get("list_timeline");
const result = await tool.execute(
{},
makeContext({ timelineTracks: undefined }),
);
expect(result).toEqual({ error: "No active timeline" });
});
});

29
apps/web/src/agent/tools/list-timeline.tool.ts Normal file
View File

@ -0,0 +1,29 @@
import type {
AgentContext,
AgentTimelineTrack,
ToolDefinition,
} from "@/agent/types";
import { toolRegistry } from "@/agent/tools/registry";
export type ListTimelineResult = {
tracks: AgentTimelineTrack[];
};
const listTimelineTool: ToolDefinition = {
name: "list_timeline",
description:
"Lists the active timeline as structured tracks and editable elements with trackId, elementId, type, assetId, name, start, and end times.",
parameters: [],
execute: async (
_args: Record<string, unknown>,
context: AgentContext,
): Promise<ListTimelineResult | { error: string }> => {
if (!context.activeSceneId || !context.timelineTracks) {
return { error: "No active timeline" };
}
return { tracks: context.timelineTracks };
},
};
toolRegistry.register("list_timeline", listTimelineTool);

14
apps/web/src/agent/types.ts
View File

@ -38,9 +38,23 @@ export interface AgentContext {
duration: number;
usedInTimeline?: boolean;
}>;
timelineTracks?: AgentTimelineTrack[];
playbackTimeMs: number;
}
export type AgentTimelineTrack = {
trackId: string;
type: "main" | "overlay" | "audio" | "text" | "effect";
elements: Array<{
elementId: string;
type: string;
assetId?: string;
name?: string;
start: number;
end: number;
}>;
};
export interface ToolDefinition {
name: string;
description: string;

18
apps/web/src/app/api/agent/chat/__tests__/route.test.ts
View File

@ -36,6 +36,20 @@ const VALID_CONTEXT = {
duration: number;
usedInTimeline?: boolean;
}>,
timelineTracks: undefined as
| Array<{
trackId: string;
type: "main" | "overlay" | "audio" | "text" | "effect";
elements: Array<{
elementId: string;
type: string;
assetId?: string;
name?: string;
start: number;
end: number;
}>;
}>
| undefined,
playbackTimeMs: 0,
};
@ -133,10 +147,10 @@ describe("POST /api/agent/chat", () => {
expect(callArgs.messages).toHaveLength(1);
expect(callArgs.systemPrompt).toContain("NeuralCut");
expect(callArgs.tools).toHaveLength(2);
expect(callArgs.tools).toHaveLength(3);
expect(
callArgs.tools.map((tool) => (tool as { name: string }).name),
).toEqual(["list_project_assets", "transcribe_video"]);
).toEqual(["list_project_assets", "list_timeline", "transcribe_video"]);
});
// -----------------------------------------------------------------------

25
apps/web/src/app/api/agent/chat/route.ts
View File

@ -21,6 +21,13 @@ const providerToolDefs: ToolDefinition[] = [
],
execute: async () => ({}), // stub — never called server-side
},
{
name: "list_timeline",
description:
"Lists the active timeline as structured tracks and editable elements with trackId, elementId, type, assetId, name, start, and end times.",
parameters: [],
execute: async () => ({}), // stub — never called server-side
},
{
name: "transcribe_video",
description:
@ -66,6 +73,24 @@ const chatRequestSchema = z.object({
usedInTimeline: z.boolean().optional(),
}),
),
timelineTracks: z
.array(
z.object({
trackId: z.string(),
type: z.enum(["main", "overlay", "audio", "text", "effect"]),
elements: z.array(
z.object({
elementId: z.string(),
type: z.string(),
assetId: z.string().optional(),
name: z.string().optional(),
start: z.number(),
end: z.number(),
}),
),
}),
)
.optional(),
playbackTimeMs: z.number(),
}) satisfies z.ZodType<AgentContext>,
});