RomanNum3ral
/
OpenCut
mirror of https://github.com/OpenCut-app/OpenCut.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat: add project assets listing tool

This commit is contained in:
Luis Esteban Acevedo Ladino 2026-04-24 10:14:54 -05:00
parent 3293d35a59
commit 37ca2fbc66
8 changed files with 328 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
apps/web/src/agent/__tests__/context-mapper.test.ts
View File

@ -22,12 +22,14 @@ describe("buildContextFromEditorState (context mapper)", () => {
name: "intro.mp4",
type: "video",
duration: 30,
usedInTimeline: false,
});
expect(ctx.mediaAssets[1]).toEqual({
id: "m2",
name: "bgm.mp3",
type: "audio",
duration: 180,
usedInTimeline: false,
});
expect(ctx.playbackTimeMs).toBe(15000);
});
@ -59,6 +61,33 @@ describe("buildContextFromEditorState (context mapper)", () => {
expect(ctx.mediaAssets[0].duration).toBe(0);
});
test("marks media assets used by the active timeline", () => {
const ctx = buildContextFromEditorState({
project: { metadata: { id: "proj-1" } },
activeScene: {
id: "scene-A",
tracks: {
main: { elements: [{ mediaId: "m1" }] },
overlay: [{ elements: [{ mediaId: "m3" }] }],
audio: [{ elements: [{ mediaId: "m2" }] }],
},
},
assets: [
{ id: "m1", name: "intro.mp4", type: "video", duration: 30 },
{ id: "m2", name: "bgm.mp3", type: "audio", duration: 180 },
{ id: "m4", name: "unused.png", type: "image" },
],
currentTimeTicks: 0,
ticksPerSecond: 100,
});
expect(ctx.mediaAssets.map((asset) => asset.usedInTimeline)).toEqual([
true,
true,
false,
]);
});
test("calculates playbackTimeMs from ticks correctly", () => {
const ctx = buildContextFromEditorState({
project: { metadata: { id: "p" } },
@ -76,9 +105,7 @@ describe("buildContextFromEditorState (context mapper)", () => {
const ctx = buildContextFromEditorState({
project: { metadata: { id: "proj-1" } },
activeScene: { id: "s1" },
assets: [
{ id: "m1", name: "a.mp4", type: "video", duration: 10 },
],
assets: [{ id: "m1", name: "a.mp4", type: "video", duration: 10 }],
currentTimeTicks: 100,
ticksPerSecond: 100,
});

50
apps/web/src/agent/context-mapper.ts
View File

@ -8,11 +8,13 @@ import type { AgentContext } from "@/agent/types";
*/
export function buildContextFromEditorState(params: {
project: { metadata: { id: string } } | null;
activeScene: { id: string } | null;
activeScene: ActiveSceneInput | null;
assets: Array<{ id: string; name: string; type: string; duration?: number }>;
currentTimeTicks: number;
ticksPerSecond: number;
}): AgentContext {
const usedMediaIds = collectUsedMediaIds(params.activeScene);
return {
projectId: params.project?.metadata.id ?? null,
activeSceneId: params.activeScene?.id ?? null,
@ -21,9 +23,55 @@ export function buildContextFromEditorState(params: {
name: a.name,
type: a.type,
duration: a.duration ?? 0,
usedInTimeline: usedMediaIds.has(a.id),
})),
playbackTimeMs: Math.round(
(params.currentTimeTicks / params.ticksPerSecond) * 1000,
),
};
}
type ActiveSceneInput = {
id: string;
tracks?: {
main?: TrackInput;
overlay?: TrackInput[];
audio?: TrackInput[];
};
};
type TrackInput = {
elements?: unknown[];
};
function collectUsedMediaIds(scene: ActiveSceneInput | null): Set<string> {
const ids = new Set<string>();
if (!scene?.tracks) {
return ids;
}
const tracks = [
scene.tracks.main,
...(scene.tracks.overlay ?? []),
...(scene.tracks.audio ?? []),
].filter((track): track is TrackInput => Boolean(track));
for (const track of tracks) {
for (const element of track.elements ?? []) {
if (hasMediaId(element)) {
ids.add(element.mediaId);
}
}
}
return ids;
}
function hasMediaId(element: unknown): element is { mediaId: string } {
return (
typeof element === "object" &&
element !== null &&
"mediaId" in element &&
typeof element.mediaId === "string"
);
}

1
apps/web/src/agent/orchestrator.ts
View File

@ -6,6 +6,7 @@ import type {
ToolResult,
} from "@/agent/types";
import { toolRegistry } from "@/agent/tools/registry";
import "@/agent/tools/list-project-assets.tool";
import "@/agent/tools/transcribe-video.tool";
import { useChatStore } from "@/stores/chat-store";
import { useAgentStore } from "@/stores/agent-store";

124
apps/web/src/agent/tools/__tests__/list-project-assets.test.ts Normal file
View File

@ -0,0 +1,124 @@
import { describe, expect, test } from "bun:test";
import type { AgentContext } from "@/agent/types";
import { toolRegistry } from "@/agent/tools/registry";
await import("@/agent/tools/list-project-assets.tool");
function makeContext(overrides: Partial<AgentContext> = {}): AgentContext {
return {
projectId: "proj-1",
activeSceneId: "scene-A",
mediaAssets: [
{
id: "v1",
name: "clip.mp4",
type: "video",
duration: 30,
usedInTimeline: true,
},
{
id: "a1",
name: "music.mp3",
type: "audio",
duration: 120,
usedInTimeline: false,
},
{
id: "i1",
name: "logo.png",
type: "image",
duration: 0,
usedInTimeline: false,
},
],
playbackTimeMs: 0,
...overrides,
};
}
describe("list_project_assets tool", () => {
test("is registered in the tool registry", () => {
expect(toolRegistry.has("list_project_assets")).toBe(true);
});
test("returns all project assets by default", async () => {
const tool = toolRegistry.get("list_project_assets");
const result = await tool.execute({}, makeContext());
expect(result).toEqual({
assets: [
{
id: "v1",
name: "clip.mp4",
type: "video",
duration: 30,
usedInTimeline: true,
},
{
id: "a1",
name: "music.mp3",
type: "audio",
duration: 120,
usedInTimeline: false,
},
{
id: "i1",
name: "logo.png",
type: "image",
usedInTimeline: false,
},
],
});
});
test("filters by usage and type", async () => {
const tool = toolRegistry.get("list_project_assets");
const used = await tool.execute({ filter: "used" }, makeContext());
const unusedAudio = await tool.execute(
{ filter: "unused", type: "audio" },
makeContext(),
);
expect(used).toEqual({
assets: [
{
id: "v1",
name: "clip.mp4",
type: "video",
duration: 30,
usedInTimeline: true,
},
],
});
expect(unusedAudio).toEqual({
assets: [
{
id: "a1",
name: "music.mp3",
type: "audio",
duration: 120,
usedInTimeline: false,
},
],
});
});
test("returns no active project error", async () => {
const tool = toolRegistry.get("list_project_assets");
const result = await tool.execute({}, makeContext({ projectId: null }));
expect(result).toEqual({ error: "No active project" });
});
test("validates filters", async () => {
const tool = toolRegistry.get("list_project_assets");
expect(await tool.execute({ filter: "bad" }, makeContext())).toEqual({
error: "Invalid asset usage filter",
});
expect(await tool.execute({ type: "document" }, makeContext())).toEqual({
error: "Invalid asset type filter",
});
});
});

85
apps/web/src/agent/tools/list-project-assets.tool.ts Normal file
View File

@ -0,0 +1,85 @@
import type { AgentContext, ToolDefinition } from "@/agent/types";
import { toolRegistry } from "@/agent/tools/registry";
type AssetTypeFilter = "all" | "video" | "audio" | "image";
type UsageFilter = "all" | "used" | "unused";
export type ListProjectAssetsArgs = {
filter?: UsageFilter;
type?: AssetTypeFilter;
};
export type ListProjectAssetsResult = {
assets: Array<{
id: string;
name: string;
type: "video" | "audio" | "image";
duration?: number;
usedInTimeline: boolean;
}>;
};
type SupportedAsset = AgentContext["mediaAssets"][number] & {
type: "video" | "audio" | "image";
};
const assetTypes = new Set<AssetTypeFilter>(["all", "video", "audio", "image"]);
const usageFilters = new Set<UsageFilter>(["all", "used", "unused"]);
const listProjectAssetsTool: ToolDefinition = {
name: "list_project_assets",
description:
"Lists project media assets with stable ids, type, duration, and whether each asset is used in the active timeline.",
parameters: [
{ key: "filter", type: "string", required: false },
{ key: "type", type: "string", required: false },
],
execute: async (
args: Record<string, unknown>,
context: AgentContext,
): Promise<ListProjectAssetsResult | { error: string }> => {
if (!context.projectId) {
return { error: "No active project" };
}
const filter = (args.filter ?? "all") as UsageFilter;
const type = (args.type ?? "all") as AssetTypeFilter;
if (!usageFilters.has(filter)) {
return { error: "Invalid asset usage filter" };
}
if (!assetTypes.has(type)) {
return { error: "Invalid asset type filter" };
}
const assets = context.mediaAssets
.filter(isSupportedAsset)
.filter((asset) => type === "all" || asset.type === type)
.filter((asset) => {
const usedInTimeline = asset.usedInTimeline ?? false;
if (filter === "used") return usedInTimeline;
if (filter === "unused") return !usedInTimeline;
return true;
})
.map((asset) => ({
id: asset.id,
name: asset.name,
type: asset.type,
...(asset.duration > 0 ? { duration: asset.duration } : {}),
usedInTimeline: asset.usedInTimeline ?? false,
}));
return { assets };
},
};
function isSupportedAsset(
asset: AgentContext["mediaAssets"][number],
): asset is SupportedAsset {
return (
asset.type === "video" || asset.type === "audio" || asset.type === "image"
);
}
toolRegistry.register("list_project_assets", listProjectAssetsTool);

1
apps/web/src/agent/types.ts
View File

@ -36,6 +36,7 @@ export interface AgentContext {
name: string;
type: string;
duration: number;
usedInTimeline?: boolean;
}>;
playbackTimeMs: number;
}

29
apps/web/src/app/api/agent/chat/__tests__/route.test.ts
View File

@ -29,12 +29,25 @@ function makeRequest(body: unknown): NextRequest {
const VALID_CONTEXT = {
projectId: null,
activeSceneId: null,
mediaAssets: [] as Array<{ id: string; name: string; type: string; duration: number }>,
mediaAssets: [] as Array<{
id: string;
name: string;
type: string;
duration: number;
usedInTimeline?: boolean;
}>,
playbackTimeMs: 0,
};
const VALID_BODY = {
messages: [{ id: "msg_1", role: "user" as const, content: "Hello", timestamp: Date.now() }],
messages: [
{
id: "msg_1",
role: "user" as const,
content: "Hello",
timestamp: Date.now(),
},
],
context: VALID_CONTEXT,
};
@ -49,7 +62,12 @@ describe("POST /api/agent/chat", () => {
mockChat.mockReset();
// Save and set LLM_* env vars
for (const key of ["LLM_PROVIDER", "LLM_API_KEY", "LLM_MODEL", "LLM_BASE_URL"]) {
for (const key of [
"LLM_PROVIDER",
"LLM_API_KEY",
"LLM_MODEL",
"LLM_BASE_URL",
]) {
savedEnv[key] = process.env[key];
}
process.env.LLM_PROVIDER = "openai-compatible";
@ -115,7 +133,10 @@ describe("POST /api/agent/chat", () => {
expect(callArgs.messages).toHaveLength(1);
expect(callArgs.systemPrompt).toContain("NeuralCut");
expect(callArgs.tools).toHaveLength(1);
expect(callArgs.tools).toHaveLength(2);
expect(
callArgs.tools.map((tool) => (tool as { name: string }).name),
).toEqual(["list_project_assets", "transcribe_video"]);
});
// -----------------------------------------------------------------------

18
apps/web/src/app/api/agent/chat/route.ts
View File

@ -8,9 +8,19 @@ import { toToolSchemas } from "@/agent/tools/registry";
// ---------------------------------------------------------------------------
// Provider-facing tool schemas (execution is client-side)
// echo_context is intentionally excluded — only transcribe_video is exposed.
// echo_context is intentionally excluded — only user-facing tools are exposed.
// ---------------------------------------------------------------------------
const providerToolDefs: ToolDefinition[] = [
{
name: "list_project_assets",
description:
"Lists project media assets with stable ids, type, duration, and whether each asset is used in the active timeline.",
parameters: [
{ key: "filter", type: "string", required: false },
{ key: "type", type: "string", required: false },
],
execute: async () => ({}), // stub — never called server-side
},
{
name: "transcribe_video",
description:
@ -53,6 +63,7 @@ const chatRequestSchema = z.object({
name: z.string(),
type: z.string(),
duration: z.number(),
usedInTimeline: z.boolean().optional(),
}),
),
playbackTimeMs: z.number(),
@ -112,9 +123,6 @@ export async function POST(request: NextRequest) {
});
} catch (error) {
console.error("[agent/chat] Provider error:", error);
return NextResponse.json(
{ error: "LLM provider error" },
{ status: 502 },
);
return NextResponse.json({ error: "LLM provider error" }, { status: 502 });
}
}