diff --git a/apps/web/src/agent/__tests__/context-mapper.test.ts b/apps/web/src/agent/__tests__/context-mapper.test.ts index d78ad24d..585165df 100644 --- a/apps/web/src/agent/__tests__/context-mapper.test.ts +++ b/apps/web/src/agent/__tests__/context-mapper.test.ts @@ -22,12 +22,14 @@ describe("buildContextFromEditorState (context mapper)", () => { name: "intro.mp4", type: "video", duration: 30, + usedInTimeline: false, }); expect(ctx.mediaAssets[1]).toEqual({ id: "m2", name: "bgm.mp3", type: "audio", duration: 180, + usedInTimeline: false, }); expect(ctx.playbackTimeMs).toBe(15000); }); @@ -59,6 +61,33 @@ describe("buildContextFromEditorState (context mapper)", () => { expect(ctx.mediaAssets[0].duration).toBe(0); }); + test("marks media assets used by the active timeline", () => { + const ctx = buildContextFromEditorState({ + project: { metadata: { id: "proj-1" } }, + activeScene: { + id: "scene-A", + tracks: { + main: { elements: [{ mediaId: "m1" }] }, + overlay: [{ elements: [{ mediaId: "m3" }] }], + audio: [{ elements: [{ mediaId: "m2" }] }], + }, + }, + assets: [ + { id: "m1", name: "intro.mp4", type: "video", duration: 30 }, + { id: "m2", name: "bgm.mp3", type: "audio", duration: 180 }, + { id: "m4", name: "unused.png", type: "image" }, + ], + currentTimeTicks: 0, + ticksPerSecond: 100, + }); + + expect(ctx.mediaAssets.map((asset) => asset.usedInTimeline)).toEqual([ + true, + true, + false, + ]); + }); + test("calculates playbackTimeMs from ticks correctly", () => { const ctx = buildContextFromEditorState({ project: { metadata: { id: "p" } }, @@ -76,9 +105,7 @@ describe("buildContextFromEditorState (context mapper)", () => { const ctx = buildContextFromEditorState({ project: { metadata: { id: "proj-1" } }, activeScene: { id: "s1" }, - assets: [ - { id: "m1", name: "a.mp4", type: "video", duration: 10 }, - ], + assets: [{ id: "m1", name: "a.mp4", type: "video", duration: 10 }], currentTimeTicks: 100, ticksPerSecond: 100, }); diff --git a/apps/web/src/agent/context-mapper.ts b/apps/web/src/agent/context-mapper.ts index 53c8fd46..30a913c0 100644 --- a/apps/web/src/agent/context-mapper.ts +++ b/apps/web/src/agent/context-mapper.ts @@ -8,11 +8,13 @@ import type { AgentContext } from "@/agent/types"; */ export function buildContextFromEditorState(params: { project: { metadata: { id: string } } | null; - activeScene: { id: string } | null; + activeScene: ActiveSceneInput | null; assets: Array<{ id: string; name: string; type: string; duration?: number }>; currentTimeTicks: number; ticksPerSecond: number; }): AgentContext { + const usedMediaIds = collectUsedMediaIds(params.activeScene); + return { projectId: params.project?.metadata.id ?? null, activeSceneId: params.activeScene?.id ?? null, @@ -21,9 +23,55 @@ export function buildContextFromEditorState(params: { name: a.name, type: a.type, duration: a.duration ?? 0, + usedInTimeline: usedMediaIds.has(a.id), })), playbackTimeMs: Math.round( (params.currentTimeTicks / params.ticksPerSecond) * 1000, ), }; } + +type ActiveSceneInput = { + id: string; + tracks?: { + main?: TrackInput; + overlay?: TrackInput[]; + audio?: TrackInput[]; + }; +}; + +type TrackInput = { + elements?: unknown[]; +}; + +function collectUsedMediaIds(scene: ActiveSceneInput | null): Set { + const ids = new Set(); + if (!scene?.tracks) { + return ids; + } + + const tracks = [ + scene.tracks.main, + ...(scene.tracks.overlay ?? []), + ...(scene.tracks.audio ?? []), + ].filter((track): track is TrackInput => Boolean(track)); + + for (const track of tracks) { + for (const element of track.elements ?? []) { + if (hasMediaId(element)) { + ids.add(element.mediaId); + } + } + } + + return ids; +} + +function hasMediaId(element: unknown): element is { mediaId: string } { + return ( + typeof element === "object" && + element !== null && + "mediaId" in element && + typeof element.mediaId === "string" + ); +} diff --git a/apps/web/src/agent/orchestrator.ts b/apps/web/src/agent/orchestrator.ts index 89625958..400704de 100644 --- a/apps/web/src/agent/orchestrator.ts +++ b/apps/web/src/agent/orchestrator.ts @@ -6,6 +6,7 @@ import type { ToolResult, } from "@/agent/types"; import { toolRegistry } from "@/agent/tools/registry"; +import "@/agent/tools/list-project-assets.tool"; import "@/agent/tools/transcribe-video.tool"; import { useChatStore } from "@/stores/chat-store"; import { useAgentStore } from "@/stores/agent-store"; diff --git a/apps/web/src/agent/tools/__tests__/list-project-assets.test.ts b/apps/web/src/agent/tools/__tests__/list-project-assets.test.ts new file mode 100644 index 00000000..1e4f7c9e --- /dev/null +++ b/apps/web/src/agent/tools/__tests__/list-project-assets.test.ts @@ -0,0 +1,124 @@ +import { describe, expect, test } from "bun:test"; +import type { AgentContext } from "@/agent/types"; +import { toolRegistry } from "@/agent/tools/registry"; + +await import("@/agent/tools/list-project-assets.tool"); + +function makeContext(overrides: Partial = {}): AgentContext { + return { + projectId: "proj-1", + activeSceneId: "scene-A", + mediaAssets: [ + { + id: "v1", + name: "clip.mp4", + type: "video", + duration: 30, + usedInTimeline: true, + }, + { + id: "a1", + name: "music.mp3", + type: "audio", + duration: 120, + usedInTimeline: false, + }, + { + id: "i1", + name: "logo.png", + type: "image", + duration: 0, + usedInTimeline: false, + }, + ], + playbackTimeMs: 0, + ...overrides, + }; +} + +describe("list_project_assets tool", () => { + test("is registered in the tool registry", () => { + expect(toolRegistry.has("list_project_assets")).toBe(true); + }); + + test("returns all project assets by default", async () => { + const tool = toolRegistry.get("list_project_assets"); + const result = await tool.execute({}, makeContext()); + + expect(result).toEqual({ + assets: [ + { + id: "v1", + name: "clip.mp4", + type: "video", + duration: 30, + usedInTimeline: true, + }, + { + id: "a1", + name: "music.mp3", + type: "audio", + duration: 120, + usedInTimeline: false, + }, + { + id: "i1", + name: "logo.png", + type: "image", + usedInTimeline: false, + }, + ], + }); + }); + + test("filters by usage and type", async () => { + const tool = toolRegistry.get("list_project_assets"); + + const used = await tool.execute({ filter: "used" }, makeContext()); + const unusedAudio = await tool.execute( + { filter: "unused", type: "audio" }, + makeContext(), + ); + + expect(used).toEqual({ + assets: [ + { + id: "v1", + name: "clip.mp4", + type: "video", + duration: 30, + usedInTimeline: true, + }, + ], + }); + expect(unusedAudio).toEqual({ + assets: [ + { + id: "a1", + name: "music.mp3", + type: "audio", + duration: 120, + usedInTimeline: false, + }, + ], + }); + }); + + test("returns no active project error", async () => { + const tool = toolRegistry.get("list_project_assets"); + const result = await tool.execute({}, makeContext({ projectId: null })); + + expect(result).toEqual({ error: "No active project" }); + }); + + test("validates filters", async () => { + const tool = toolRegistry.get("list_project_assets"); + + expect(await tool.execute({ filter: "bad" }, makeContext())).toEqual({ + error: "Invalid asset usage filter", + }); + expect(await tool.execute({ type: "document" }, makeContext())).toEqual({ + error: "Invalid asset type filter", + }); + }); +}); diff --git a/apps/web/src/agent/tools/list-project-assets.tool.ts b/apps/web/src/agent/tools/list-project-assets.tool.ts new file mode 100644 index 00000000..cabfb8ca --- /dev/null +++ b/apps/web/src/agent/tools/list-project-assets.tool.ts @@ -0,0 +1,85 @@ +import type { AgentContext, ToolDefinition } from "@/agent/types"; +import { toolRegistry } from "@/agent/tools/registry"; + +type AssetTypeFilter = "all" | "video" | "audio" | "image"; +type UsageFilter = "all" | "used" | "unused"; + +export type ListProjectAssetsArgs = { + filter?: UsageFilter; + type?: AssetTypeFilter; +}; + +export type ListProjectAssetsResult = { + assets: Array<{ + id: string; + name: string; + type: "video" | "audio" | "image"; + duration?: number; + usedInTimeline: boolean; + }>; +}; + +type SupportedAsset = AgentContext["mediaAssets"][number] & { + type: "video" | "audio" | "image"; +}; + +const assetTypes = new Set(["all", "video", "audio", "image"]); +const usageFilters = new Set(["all", "used", "unused"]); + +const listProjectAssetsTool: ToolDefinition = { + name: "list_project_assets", + description: + "Lists project media assets with stable ids, type, duration, and whether each asset is used in the active timeline.", + parameters: [ + { key: "filter", type: "string", required: false }, + { key: "type", type: "string", required: false }, + ], + execute: async ( + args: Record, + context: AgentContext, + ): Promise => { + if (!context.projectId) { + return { error: "No active project" }; + } + + const filter = (args.filter ?? "all") as UsageFilter; + const type = (args.type ?? "all") as AssetTypeFilter; + + if (!usageFilters.has(filter)) { + return { error: "Invalid asset usage filter" }; + } + + if (!assetTypes.has(type)) { + return { error: "Invalid asset type filter" }; + } + + const assets = context.mediaAssets + .filter(isSupportedAsset) + .filter((asset) => type === "all" || asset.type === type) + .filter((asset) => { + const usedInTimeline = asset.usedInTimeline ?? false; + if (filter === "used") return usedInTimeline; + if (filter === "unused") return !usedInTimeline; + return true; + }) + .map((asset) => ({ + id: asset.id, + name: asset.name, + type: asset.type, + ...(asset.duration > 0 ? { duration: asset.duration } : {}), + usedInTimeline: asset.usedInTimeline ?? false, + })); + + return { assets }; + }, +}; + +function isSupportedAsset( + asset: AgentContext["mediaAssets"][number], +): asset is SupportedAsset { + return ( + asset.type === "video" || asset.type === "audio" || asset.type === "image" + ); +} + +toolRegistry.register("list_project_assets", listProjectAssetsTool); diff --git a/apps/web/src/agent/types.ts b/apps/web/src/agent/types.ts index 9aa14835..6513dedf 100644 --- a/apps/web/src/agent/types.ts +++ b/apps/web/src/agent/types.ts @@ -36,6 +36,7 @@ export interface AgentContext { name: string; type: string; duration: number; + usedInTimeline?: boolean; }>; playbackTimeMs: number; } diff --git a/apps/web/src/app/api/agent/chat/__tests__/route.test.ts b/apps/web/src/app/api/agent/chat/__tests__/route.test.ts index 8a182d31..ae1ec10f 100644 --- a/apps/web/src/app/api/agent/chat/__tests__/route.test.ts +++ b/apps/web/src/app/api/agent/chat/__tests__/route.test.ts @@ -29,12 +29,25 @@ function makeRequest(body: unknown): NextRequest { const VALID_CONTEXT = { projectId: null, activeSceneId: null, - mediaAssets: [] as Array<{ id: string; name: string; type: string; duration: number }>, + mediaAssets: [] as Array<{ + id: string; + name: string; + type: string; + duration: number; + usedInTimeline?: boolean; + }>, playbackTimeMs: 0, }; const VALID_BODY = { - messages: [{ id: "msg_1", role: "user" as const, content: "Hello", timestamp: Date.now() }], + messages: [ + { + id: "msg_1", + role: "user" as const, + content: "Hello", + timestamp: Date.now(), + }, + ], context: VALID_CONTEXT, }; @@ -49,7 +62,12 @@ describe("POST /api/agent/chat", () => { mockChat.mockReset(); // Save and set LLM_* env vars - for (const key of ["LLM_PROVIDER", "LLM_API_KEY", "LLM_MODEL", "LLM_BASE_URL"]) { + for (const key of [ + "LLM_PROVIDER", + "LLM_API_KEY", + "LLM_MODEL", + "LLM_BASE_URL", + ]) { savedEnv[key] = process.env[key]; } process.env.LLM_PROVIDER = "openai-compatible"; @@ -115,7 +133,10 @@ describe("POST /api/agent/chat", () => { expect(callArgs.messages).toHaveLength(1); expect(callArgs.systemPrompt).toContain("NeuralCut"); - expect(callArgs.tools).toHaveLength(1); + expect(callArgs.tools).toHaveLength(2); + expect( + callArgs.tools.map((tool) => (tool as { name: string }).name), + ).toEqual(["list_project_assets", "transcribe_video"]); }); // ----------------------------------------------------------------------- diff --git a/apps/web/src/app/api/agent/chat/route.ts b/apps/web/src/app/api/agent/chat/route.ts index a791eebb..2144b764 100644 --- a/apps/web/src/app/api/agent/chat/route.ts +++ b/apps/web/src/app/api/agent/chat/route.ts @@ -8,9 +8,19 @@ import { toToolSchemas } from "@/agent/tools/registry"; // --------------------------------------------------------------------------- // Provider-facing tool schemas (execution is client-side) -// echo_context is intentionally excluded — only transcribe_video is exposed. +// echo_context is intentionally excluded — only user-facing tools are exposed. // --------------------------------------------------------------------------- const providerToolDefs: ToolDefinition[] = [ + { + name: "list_project_assets", + description: + "Lists project media assets with stable ids, type, duration, and whether each asset is used in the active timeline.", + parameters: [ + { key: "filter", type: "string", required: false }, + { key: "type", type: "string", required: false }, + ], + execute: async () => ({}), // stub — never called server-side + }, { name: "transcribe_video", description: @@ -53,6 +63,7 @@ const chatRequestSchema = z.object({ name: z.string(), type: z.string(), duration: z.number(), + usedInTimeline: z.boolean().optional(), }), ), playbackTimeMs: z.number(), @@ -112,9 +123,6 @@ export async function POST(request: NextRequest) { }); } catch (error) { console.error("[agent/chat] Provider error:", error); - return NextResponse.json( - { error: "LLM provider error" }, - { status: 502 }, - ); + return NextResponse.json({ error: "LLM provider error" }, { status: 502 }); } }