From a93c45586bee265f5af014457ea34b77d4012052 Mon Sep 17 00:00:00 2001 From: Luis Esteban Acevedo Ladino Date: Thu, 23 Apr 2026 07:52:25 -0500 Subject: [PATCH] feat: add conversational agent infrastructure shell --- .../agent/__tests__/context-mapper.test.ts | 102 ++++++++++ .../agent/__tests__/import-boundary.test.ts | 49 +++++ .../src/agent/__tests__/orchestrator.test.ts | 164 ++++++++++++++++ .../src/agent/__tests__/system-prompt.test.ts | 59 ++++++ .../agent/__tests__/types-contract.test.ts | 147 +++++++++++++++ apps/web/src/agent/context-mapper.ts | 29 +++ apps/web/src/agent/context.ts | 23 +++ apps/web/src/agent/orchestrator.ts | 109 +++++++++++ apps/web/src/agent/system-prompt.ts | 16 ++ .../agent/tools/__tests__/mock-tool.test.ts | 73 ++++++++ apps/web/src/agent/tools/mock.tool.ts | 19 ++ apps/web/src/agent/tools/registry.ts | 6 + apps/web/src/agent/types.ts | 49 +++++ .../api/agent/chat/__tests__/route.test.ts | 67 +++++++ apps/web/src/app/api/agent/chat/route.ts | 38 ++++ apps/web/src/app/editor/[project_id]/page.tsx | 37 +++- .../editor/panels/chat/chat-input.tsx | 62 +++++++ .../components/editor/panels/chat/index.tsx | 109 +++++++++++ .../editor/panels/chat/message-bubble.tsx | 32 ++++ .../src/stores/__tests__/agent-store.test.ts | 84 +++++++++ .../__tests__/chat-panel-behavior.test.ts | 175 ++++++++++++++++++ .../src/stores/__tests__/chat-store.test.ts | 90 +++++++++ .../__tests__/right-panel-store.test.ts | 42 +++++ apps/web/src/stores/agent-store.ts | 30 +++ apps/web/src/stores/chat-store.ts | 44 +++++ apps/web/src/stores/right-panel-store.ts | 13 ++ .../design.md | 135 ++++++++++++++ .../exploration.md | 0 .../proposal.md | 0 .../specs/agent-context-bridge/spec.md | 66 +++++++ .../specs/agent-session-shell/spec.md | 95 ++++++++++ .../specs/editor-chat-panel/spec.md | 76 ++++++++ .../tasks.md | 50 +++++ .../verify-report.md | 154 +++++++++++++++ openspec/specs/agent-context-bridge/spec.md | 66 +++++++ openspec/specs/agent-session-shell/spec.md | 95 ++++++++++ openspec/specs/editor-chat-panel/spec.md | 76 ++++++++ 37 files changed, 2480 insertions(+), 1 deletion(-) create mode 100644 apps/web/src/agent/__tests__/context-mapper.test.ts create mode 100644 apps/web/src/agent/__tests__/import-boundary.test.ts create mode 100644 apps/web/src/agent/__tests__/orchestrator.test.ts create mode 100644 apps/web/src/agent/__tests__/system-prompt.test.ts create mode 100644 apps/web/src/agent/__tests__/types-contract.test.ts create mode 100644 apps/web/src/agent/context-mapper.ts create mode 100644 apps/web/src/agent/context.ts create mode 100644 apps/web/src/agent/orchestrator.ts create mode 100644 apps/web/src/agent/system-prompt.ts create mode 100644 apps/web/src/agent/tools/__tests__/mock-tool.test.ts create mode 100644 apps/web/src/agent/tools/mock.tool.ts create mode 100644 apps/web/src/agent/tools/registry.ts create mode 100644 apps/web/src/agent/types.ts create mode 100644 apps/web/src/app/api/agent/chat/__tests__/route.test.ts create mode 100644 apps/web/src/app/api/agent/chat/route.ts create mode 100644 apps/web/src/components/editor/panels/chat/chat-input.tsx create mode 100644 apps/web/src/components/editor/panels/chat/index.tsx create mode 100644 apps/web/src/components/editor/panels/chat/message-bubble.tsx create mode 100644 apps/web/src/stores/__tests__/agent-store.test.ts create mode 100644 apps/web/src/stores/__tests__/chat-panel-behavior.test.ts create mode 100644 apps/web/src/stores/__tests__/chat-store.test.ts create mode 100644 apps/web/src/stores/__tests__/right-panel-store.test.ts create mode 100644 apps/web/src/stores/agent-store.ts create mode 100644 apps/web/src/stores/chat-store.ts create mode 100644 apps/web/src/stores/right-panel-store.ts create mode 100644 openspec/changes/archive/2026-04-22-infra-habilitadora-fase-1/design.md rename openspec/changes/{infra-habilitadora-fase-1 => archive/2026-04-22-infra-habilitadora-fase-1}/exploration.md (100%) rename openspec/changes/{infra-habilitadora-fase-1 => archive/2026-04-22-infra-habilitadora-fase-1}/proposal.md (100%) create mode 100644 openspec/changes/archive/2026-04-22-infra-habilitadora-fase-1/specs/agent-context-bridge/spec.md create mode 100644 openspec/changes/archive/2026-04-22-infra-habilitadora-fase-1/specs/agent-session-shell/spec.md create mode 100644 openspec/changes/archive/2026-04-22-infra-habilitadora-fase-1/specs/editor-chat-panel/spec.md create mode 100644 openspec/changes/archive/2026-04-22-infra-habilitadora-fase-1/tasks.md create mode 100644 openspec/changes/archive/2026-04-22-infra-habilitadora-fase-1/verify-report.md create mode 100644 openspec/specs/agent-context-bridge/spec.md create mode 100644 openspec/specs/agent-session-shell/spec.md create mode 100644 openspec/specs/editor-chat-panel/spec.md diff --git a/apps/web/src/agent/__tests__/context-mapper.test.ts b/apps/web/src/agent/__tests__/context-mapper.test.ts new file mode 100644 index 00000000..d78ad24d --- /dev/null +++ b/apps/web/src/agent/__tests__/context-mapper.test.ts @@ -0,0 +1,102 @@ +import { describe, expect, test } from "bun:test"; +import { buildContextFromEditorState } from "@/agent/context-mapper"; + +describe("buildContextFromEditorState (context mapper)", () => { + test("maps active media populated from editor", () => { + const ctx = buildContextFromEditorState({ + project: { metadata: { id: "proj-1" } }, + activeScene: { id: "scene-A" }, + assets: [ + { id: "m1", name: "intro.mp4", type: "video", duration: 30 }, + { id: "m2", name: "bgm.mp3", type: "audio", duration: 180 }, + ], + currentTimeTicks: 1500, + ticksPerSecond: 100, + }); + + expect(ctx.projectId).toBe("proj-1"); + expect(ctx.activeSceneId).toBe("scene-A"); + expect(ctx.mediaAssets).toHaveLength(2); + expect(ctx.mediaAssets[0]).toEqual({ + id: "m1", + name: "intro.mp4", + type: "video", + duration: 30, + }); + expect(ctx.mediaAssets[1]).toEqual({ + id: "m2", + name: "bgm.mp3", + type: "audio", + duration: 180, + }); + expect(ctx.playbackTimeMs).toBe(15000); + }); + + test("returns null projectId when no project loaded", () => { + const ctx = buildContextFromEditorState({ + project: null, + activeScene: null, + assets: [], + currentTimeTicks: 0, + ticksPerSecond: 100, + }); + + expect(ctx.projectId).toBeNull(); + expect(ctx.activeSceneId).toBeNull(); + expect(ctx.mediaAssets).toEqual([]); + expect(ctx.playbackTimeMs).toBe(0); + }); + + test("handles assets with undefined duration", () => { + const ctx = buildContextFromEditorState({ + project: { metadata: { id: "proj-2" } }, + activeScene: { id: "scene-B" }, + assets: [{ id: "m1", name: "unknown.dat", type: "other" }], + currentTimeTicks: 500, + ticksPerSecond: 100, + }); + + expect(ctx.mediaAssets[0].duration).toBe(0); + }); + + test("calculates playbackTimeMs from ticks correctly", () => { + const ctx = buildContextFromEditorState({ + project: { metadata: { id: "p" } }, + activeScene: null, + assets: [], + currentTimeTicks: 3000, + ticksPerSecond: 100, + }); + + // 3000 ticks / 100 tps = 30 seconds = 30000ms + expect(ctx.playbackTimeMs).toBe(30000); + }); + + test("result is JSON-serializable", () => { + const ctx = buildContextFromEditorState({ + project: { metadata: { id: "proj-1" } }, + activeScene: { id: "s1" }, + assets: [ + { id: "m1", name: "a.mp4", type: "video", duration: 10 }, + ], + currentTimeTicks: 100, + ticksPerSecond: 100, + }); + + const json = JSON.stringify(ctx); + const parsed = JSON.parse(json); + expect(parsed).toEqual(ctx); + }); + + test("handles empty assets list", () => { + const ctx = buildContextFromEditorState({ + project: { metadata: { id: "proj-1" } }, + activeScene: null, + assets: [], + currentTimeTicks: 0, + ticksPerSecond: 100, + }); + + expect(ctx.mediaAssets).toEqual([]); + }); +}); diff --git a/apps/web/src/agent/__tests__/import-boundary.test.ts b/apps/web/src/agent/__tests__/import-boundary.test.ts new file mode 100644 index 00000000..4a0e9fb2 --- /dev/null +++ b/apps/web/src/agent/__tests__/import-boundary.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, test } from "bun:test"; +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { globSync } from "node:fs"; + +/** + * Static analysis test: verifies the import boundary constraint. + * Only context.ts may import from @/core within the agent/ directory. + */ +describe("agent import boundary", () => { + const AGENT_DIR = resolve(import.meta.dir, ".."); + const CORE_IMPORT_PATTERN = /from\s+["']@\/core/; + + function getAgentTsFiles(): string[] { + return globSync("**/*.ts", { cwd: AGENT_DIR }).filter( + (f) => !f.includes("__tests__"), + ); + } + + test("only context.ts imports from @/core in agent/", () => { + const files = getAgentTsFiles(); + const violators: string[] = []; + + for (const file of files) { + const fullPath = resolve(AGENT_DIR, file); + const content = readFileSync(fullPath, "utf-8"); + + if (CORE_IMPORT_PATTERN.test(content)) { + if (!file.endsWith("context.ts")) { + violators.push(file); + } + } + } + + expect(violators).toEqual([]); + }); + + test("context.ts does import from @/core (boundary file exists)", () => { + const contextPath = resolve(AGENT_DIR, "context.ts"); + const content = readFileSync(contextPath, "utf-8"); + expect(CORE_IMPORT_PATTERN.test(content)).toBe(true); + }); + + test("orchestrator does not import from @/core", () => { + const orchestratorPath = resolve(AGENT_DIR, "orchestrator.ts"); + const content = readFileSync(orchestratorPath, "utf-8"); + expect(CORE_IMPORT_PATTERN.test(content)).toBe(false); + }); +}); diff --git a/apps/web/src/agent/__tests__/orchestrator.test.ts b/apps/web/src/agent/__tests__/orchestrator.test.ts new file mode 100644 index 00000000..55bbed0b --- /dev/null +++ b/apps/web/src/agent/__tests__/orchestrator.test.ts @@ -0,0 +1,164 @@ +import { + afterEach, + beforeEach, + describe, + expect, + test, +} from "bun:test"; +import { run } from "@/agent/orchestrator"; +import { useChatStore } from "@/stores/chat-store"; +import { useAgentStore } from "@/stores/agent-store"; +import type { AgentContext, ChatMessage } from "@/agent/types"; + +const MOCK_CONTEXT: AgentContext = { + projectId: "proj-1", + activeSceneId: "scene-A", + mediaAssets: [], + playbackTimeMs: 0, +}; + +const originalFetch = globalThis.fetch; + +function mockFetchResponse(data: unknown, status = 200) { + return Promise.resolve( + new Response(JSON.stringify(data), { + status, + headers: { "Content-Type": "application/json" }, + }), + ); +} + +function setMockFetch(fn: (input: RequestInfo | URL, init?: RequestInit) => Promise) { + globalThis.fetch = fn as typeof fetch; +} + +describe("orchestrator", () => { + beforeEach(() => { + useChatStore.getState().clearMessages(); + useAgentStore.getState().reset(); + }); + + afterEach(() => { + globalThis.fetch = originalFetch; + }); + + test("happy path — no tool calls", async () => { + setMockFetch(() => + mockFetchResponse({ + content: "Hello! How can I help?", + })); + + const messages: ChatMessage[] = [ + { id: "1", role: "user", content: "Hi", timestamp: Date.now() }, + ]; + + await run(messages, MOCK_CONTEXT); + + const chatState = useChatStore.getState(); + const agentState = useAgentStore.getState(); + + // Assistant message appended + expect(chatState.messages).toHaveLength(1); + expect(chatState.messages[0].role).toBe("assistant"); + expect(chatState.messages[0].content).toBe("Hello! How can I help?"); + + // Loading cleared + expect(chatState.loading).toBe(false); + expect(chatState.error).toBeNull(); + + // Agent state reset + expect(agentState.status).toBe("idle"); + expect(agentState.activeTool).toBeNull(); + expect(agentState.context).toEqual(MOCK_CONTEXT); + }); + + test("tool call resolution — echo_context executes", async () => { + setMockFetch(() => + mockFetchResponse({ + content: "Let me check your editor context.", + toolCalls: [ + { id: "tc_1", name: "echo_context", args: {} }, + ], + })); + + const messages: ChatMessage[] = [ + { id: "1", role: "user", content: "Show me context", timestamp: Date.now() }, + ]; + + await run(messages, MOCK_CONTEXT); + + const chatState = useChatStore.getState(); + const agentState = useAgentStore.getState(); + + // Assistant message + tool result + expect(chatState.messages).toHaveLength(2); + expect(chatState.messages[0].role).toBe("assistant"); + expect(chatState.messages[1].role).toBe("tool_result"); + + // Tool result should contain echo_context output + const toolResult = JSON.parse(chatState.messages[1].content); + expect(toolResult.projectId).toBe("proj-1"); + + // States cleaned up + expect(chatState.loading).toBe(false); + expect(agentState.status).toBe("idle"); + expect(agentState.context).toEqual(MOCK_CONTEXT); + }); + + test("error path — API failure sets error and loading cleared", async () => { + setMockFetch(() => + mockFetchResponse({ error: "fail" }, 500)); + + const messages: ChatMessage[] = [ + { id: "1", role: "user", content: "Hi", timestamp: Date.now() }, + ]; + + await run(messages, MOCK_CONTEXT); + + const chatState = useChatStore.getState(); + const agentState = useAgentStore.getState(); + + // Error set + expect(chatState.error).toBeTruthy(); + expect(chatState.loading).toBe(false); + + // Agent in error state + expect(agentState.status).toBe("error"); + + // No assistant message added + expect(chatState.messages).toHaveLength(0); + }); + + test("error path — network failure sets error", async () => { + setMockFetch(() => Promise.reject(new Error("Network error"))); + + const messages: ChatMessage[] = [ + { id: "1", role: "user", content: "Hi", timestamp: Date.now() }, + ]; + + await run(messages, MOCK_CONTEXT); + + const chatState = useChatStore.getState(); + expect(chatState.error).toBe("Network error"); + expect(chatState.loading).toBe(false); + expect(useAgentStore.getState().status).toBe("error"); + }); + + test("context is stored in agentStore", async () => { + setMockFetch(() => + mockFetchResponse({ content: "ok" })); + + const context: AgentContext = { + projectId: "special-proj", + activeSceneId: "scene-X", + mediaAssets: [{ id: "m1", name: "a.mp4", type: "video", duration: 10 }], + playbackTimeMs: 1234, + }; + + await run([], context); + + const { context: stored } = useAgentStore.getState(); + expect(stored.projectId).toBe("special-proj"); + expect(stored.mediaAssets).toHaveLength(1); + }); +}); diff --git a/apps/web/src/agent/__tests__/system-prompt.test.ts b/apps/web/src/agent/__tests__/system-prompt.test.ts new file mode 100644 index 00000000..87956013 --- /dev/null +++ b/apps/web/src/agent/__tests__/system-prompt.test.ts @@ -0,0 +1,59 @@ +import { describe, expect, test } from "bun:test"; +import { buildSystemPrompt } from "@/agent/system-prompt"; +import type { AgentContext } from "@/agent/types"; + +describe("buildSystemPrompt", () => { + test("includes media assets when present", () => { + const context: AgentContext = { + projectId: "proj-1", + activeSceneId: "scene-A", + mediaAssets: [ + { id: "m1", name: "intro.mp4", type: "video", duration: 30 }, + { id: "m2", name: "bgm.mp3", type: "audio", duration: 180 }, + ], + playbackTimeMs: 15000, + }; + + const prompt = buildSystemPrompt(context); + + expect(prompt).toContain("intro.mp4"); + expect(prompt).toContain("bgm.mp3"); + expect(prompt).toContain("video, 30s"); + expect(prompt).toContain("audio, 180s"); + expect(prompt).toContain("Project: proj-1"); + expect(prompt).toContain("Active scene: scene-A"); + expect(prompt).toContain("Playback position: 15000ms"); + expect(prompt).toContain("NeuralCut video editor"); + }); + + test("is valid when no media assets are loaded", () => { + const context: AgentContext = { + projectId: "proj-2", + activeSceneId: null, + mediaAssets: [], + playbackTimeMs: 0, + }; + + const prompt = buildSystemPrompt(context); + + expect(prompt).toContain("No media assets loaded."); + expect(prompt).toContain("Project: proj-2"); + expect(prompt).toContain("No active scene"); + expect(prompt).toContain("Playback position: 0ms"); + }); + + test("is valid when all fields are null/empty", () => { + const context: AgentContext = { + projectId: null, + activeSceneId: null, + mediaAssets: [], + playbackTimeMs: 0, + }; + + const prompt = buildSystemPrompt(context); + + expect(prompt).toContain("No project loaded"); + expect(prompt).toContain("No active scene"); + expect(prompt).toContain("No media assets loaded."); + }); +}); diff --git a/apps/web/src/agent/__tests__/types-contract.test.ts b/apps/web/src/agent/__tests__/types-contract.test.ts new file mode 100644 index 00000000..68a16145 --- /dev/null +++ b/apps/web/src/agent/__tests__/types-contract.test.ts @@ -0,0 +1,147 @@ +import { describe, expect, test } from "bun:test"; +import type { + ExecutionState, + ChatMessage, + ToolCall, + ToolResult, + AgentContext, + ToolDefinition, +} from "@/agent/types"; + +describe("agent/types contracts", () => { + test("ExecutionState accepts all valid states", () => { + const states: ExecutionState[] = [ + "idle", + "sending", + "processing", + "responding", + "error", + ]; + expect(states).toHaveLength(5); + for (const s of states) { + expect(typeof s).toBe("string"); + } + }); + + test("ChatMessage shape is constructable", () => { + const msg: ChatMessage = { + id: "msg-1", + role: "user", + content: "Hello", + timestamp: Date.now(), + }; + expect(msg.id).toBe("msg-1"); + expect(msg.role).toBe("user"); + expect(msg.content).toBe("Hello"); + expect(typeof msg.timestamp).toBe("number"); + }); + + test("ChatMessage with toolCalls is constructable", () => { + const msg: ChatMessage = { + id: "msg-2", + role: "assistant", + content: "Let me check", + timestamp: Date.now(), + toolCalls: [{ id: "tc-1", name: "echo_context", args: {} }], + }; + expect(msg.toolCalls).toHaveLength(1); + expect(msg.toolCalls?.[0].name).toBe("echo_context"); + }); + + test("ToolCall shape is constructable", () => { + const tc: ToolCall = { + id: "tc-1", + name: "some_tool", + args: { key: "value" }, + }; + expect(tc.id).toBe("tc-1"); + expect(tc.name).toBe("some_tool"); + expect(tc.args).toEqual({ key: "value" }); + }); + + test("ToolResult shape is constructable", () => { + const tr: ToolResult = { + toolCallId: "tc-1", + name: "some_tool", + result: { data: 42 }, + }; + expect(tr.toolCallId).toBe("tc-1"); + expect(tr.result).toEqual({ data: 42 }); + expect(tr.error).toBeUndefined(); + }); + + test("ToolResult with error is constructable", () => { + const tr: ToolResult = { + toolCallId: "tc-2", + name: "failing_tool", + result: null, + error: "Execution failed", + }; + expect(tr.error).toBe("Execution failed"); + }); + + test("AgentContext shape is constructable with media", () => { + const ctx: AgentContext = { + projectId: "proj-1", + activeSceneId: "scene-A", + mediaAssets: [ + { id: "m1", name: "clip.mp4", type: "video", duration: 30 }, + ], + playbackTimeMs: 5000, + }; + expect(ctx.projectId).toBe("proj-1"); + expect(ctx.mediaAssets).toHaveLength(1); + expect(ctx.playbackTimeMs).toBe(5000); + }); + + test("AgentContext shape with null fields is constructable", () => { + const ctx: AgentContext = { + projectId: null, + activeSceneId: null, + mediaAssets: [], + playbackTimeMs: 0, + }; + expect(ctx.projectId).toBeNull(); + expect(ctx.activeSceneId).toBeNull(); + expect(ctx.mediaAssets).toEqual([]); + }); + + test("ToolDefinition shape is constructable", () => { + const def: ToolDefinition = { + name: "test_tool", + description: "A test tool", + parameters: [{ key: "input", type: "string", required: true }], + execute: async () => ({ ok: true }), + }; + expect(def.name).toBe("test_tool"); + expect(def.parameters).toHaveLength(1); + expect(typeof def.execute).toBe("function"); + }); + + test("all types are JSON-serializable", () => { + const msg: ChatMessage = { + id: "m1", + role: "user", + content: "hi", + timestamp: 123, + }; + const ctx: AgentContext = { + projectId: "p1", + activeSceneId: null, + mediaAssets: [], + playbackTimeMs: 0, + }; + const tc: ToolCall = { id: "t1", name: "tool", args: {} }; + const tr: ToolResult = { toolCallId: "t1", name: "tool", result: 42 }; + + // None of these should throw on serialization + expect(() => JSON.stringify(msg)).not.toThrow(); + expect(() => JSON.stringify(ctx)).not.toThrow(); + expect(() => JSON.stringify(tc)).not.toThrow(); + expect(() => JSON.stringify(tr)).not.toThrow(); + + // Round-trip preserves data + expect(JSON.parse(JSON.stringify(ctx))).toEqual(ctx); + expect(JSON.parse(JSON.stringify(tc))).toEqual(tc); + }); +}); diff --git a/apps/web/src/agent/context-mapper.ts b/apps/web/src/agent/context-mapper.ts new file mode 100644 index 00000000..53c8fd46 --- /dev/null +++ b/apps/web/src/agent/context-mapper.ts @@ -0,0 +1,29 @@ +import type { AgentContext } from "@/agent/types"; + +/** + * Pure data-mapping function extracted from EditorContextAdapter. + * WASM-free — testable in bun:test without loading EditorCore. + * + * Takes pre-extracted editor state and maps it to an AgentContext POJO. + */ +export function buildContextFromEditorState(params: { + project: { metadata: { id: string } } | null; + activeScene: { id: string } | null; + assets: Array<{ id: string; name: string; type: string; duration?: number }>; + currentTimeTicks: number; + ticksPerSecond: number; +}): AgentContext { + return { + projectId: params.project?.metadata.id ?? null, + activeSceneId: params.activeScene?.id ?? null, + mediaAssets: params.assets.map((a) => ({ + id: a.id, + name: a.name, + type: a.type, + duration: a.duration ?? 0, + })), + playbackTimeMs: Math.round( + (params.currentTimeTicks / params.ticksPerSecond) * 1000, + ), + }; +} diff --git a/apps/web/src/agent/context.ts b/apps/web/src/agent/context.ts new file mode 100644 index 00000000..a7606947 --- /dev/null +++ b/apps/web/src/agent/context.ts @@ -0,0 +1,23 @@ +import { EditorCore } from "@/core"; +import { TICKS_PER_SECOND } from "@/lib/wasm"; +import { buildContextFromEditorState } from "@/agent/context-mapper"; + +/** + * Thin adapter: the ONLY file in agent/ that imports from core/. + * Returns a plain AgentContext POJO so tools never touch EditorCore directly. + * Data mapping logic is extracted to context-mapper.ts for WASM-free testability. + */ +export const EditorContextAdapter = { + getContext() { + const core = EditorCore.getInstance(); + return buildContextFromEditorState({ + project: core.project.getActiveOrNull(), + activeScene: core.scenes.getActiveSceneOrNull(), + assets: core.media.getAssets(), + currentTimeTicks: core.playback.getCurrentTime(), + ticksPerSecond: TICKS_PER_SECOND, + }); + }, +}; + +export { buildSystemPrompt } from "@/agent/system-prompt"; diff --git a/apps/web/src/agent/orchestrator.ts b/apps/web/src/agent/orchestrator.ts new file mode 100644 index 00000000..04298782 --- /dev/null +++ b/apps/web/src/agent/orchestrator.ts @@ -0,0 +1,109 @@ +import type { + AgentContext, + ChatMessage, + ToolCall, + ToolResult, +} from "@/agent/types"; +import { toolRegistry } from "@/agent/tools/registry"; +import "@/agent/tools/mock.tool"; +import { useChatStore } from "@/stores/chat-store"; +import { useAgentStore } from "@/stores/agent-store"; + +interface APIResponse { + content: string; + toolCalls?: ToolCall[]; +} + +/** + * Client-side orchestrator — single-pass only (v1). + * + * 1. Sets agent status → sends + * 2. POST to /api/agent/chat + * 3. Resolves any tool calls via the registry + * 4. Appends assistant message + tool results to chatStore + * 5. Returns to idle (or error) + */ +export async function run( + messages: ChatMessage[], + context: AgentContext, +): Promise { + const agentStore = useAgentStore.getState(); + const chatStore = useChatStore.getState(); + + agentStore.setContext(context); + agentStore.setStatus("sending"); + + try { + const response = await fetch("/api/agent/chat", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ messages, context }), + }); + + if (!response.ok) { + throw new Error(`API error: ${response.status}`); + } + + const data: APIResponse = await response.json(); + + // Resolve tool calls if present + let toolResults: ToolResult[] = []; + if (data.toolCalls && data.toolCalls.length > 0) { + agentStore.setStatus("processing"); + toolResults = await resolveToolCalls(data.toolCalls, context); + } + + // Append assistant response + chatStore.addMessage({ + role: "assistant", + content: data.content, + toolCalls: data.toolCalls, + }); + + // Append tool results as separate messages + for (const tr of toolResults) { + chatStore.addMessage({ + role: "tool_result", + content: tr.error + ? `Error in ${tr.name}: ${tr.error}` + : JSON.stringify(tr.result), + }); + } + + agentStore.setStatus("idle"); + chatStore.setLoading(false); + } catch (error) { + const message = + error instanceof Error ? error.message : "Unknown orchestrator error"; + chatStore.setError(message); + agentStore.setStatus("error"); + } +} + +async function resolveToolCalls( + toolCalls: ToolCall[], + context: AgentContext, +): Promise { + const agentStore = useAgentStore.getState(); + const results: ToolResult[] = []; + + for (const tc of toolCalls) { + agentStore.setActiveTool(tc.name); + + try { + const tool = toolRegistry.get(tc.name); + const result = await tool.execute(tc.args, context); + results.push({ toolCallId: tc.id, name: tc.name, result }); + } catch (error) { + results.push({ + toolCallId: tc.id, + name: tc.name, + result: null, + error: error instanceof Error ? error.message : "Tool execution failed", + }); + } + } + + agentStore.setActiveTool(null); + return results; +} diff --git a/apps/web/src/agent/system-prompt.ts b/apps/web/src/agent/system-prompt.ts new file mode 100644 index 00000000..e263cd86 --- /dev/null +++ b/apps/web/src/agent/system-prompt.ts @@ -0,0 +1,16 @@ +import type { AgentContext } from "@/agent/types"; + +export function buildSystemPrompt(context: AgentContext): string { + const mediaSection = + context.mediaAssets.length > 0 + ? `Active media assets:\n${context.mediaAssets.map((m) => `- ${m.name} (${m.type}, ${m.duration}s)`).join("\n")}` + : "No media assets loaded."; + + return [ + "You are an AI assistant embedded in the NeuralCut video editor.", + `Project: ${context.projectId ?? "No project loaded"}`, + `Active scene: ${context.activeSceneId ?? "No active scene"}`, + `Playback position: ${context.playbackTimeMs}ms`, + mediaSection, + ].join("\n"); +} diff --git a/apps/web/src/agent/tools/__tests__/mock-tool.test.ts b/apps/web/src/agent/tools/__tests__/mock-tool.test.ts new file mode 100644 index 00000000..ee99b300 --- /dev/null +++ b/apps/web/src/agent/tools/__tests__/mock-tool.test.ts @@ -0,0 +1,73 @@ +import { describe, expect, test } from "bun:test"; +import "@/agent/tools/mock.tool"; +import { toolRegistry } from "@/agent/tools/registry"; +import type { AgentContext } from "@/agent/types"; + +describe("echo_context tool", () => { + test("is registered in the tool registry", () => { + expect(toolRegistry.has("echo_context")).toBe(true); + }); + + test("returns context summary with media assets", async () => { + const context: AgentContext = { + projectId: "proj-1", + activeSceneId: "scene-A", + mediaAssets: [ + { id: "m1", name: "clip.mp4", type: "video", duration: 120 }, + { id: "m2", name: "song.mp3", type: "audio", duration: 45 }, + ], + playbackTimeMs: 5000, + }; + + const tool = toolRegistry.get("echo_context"); + const result = await tool.execute({}, context); + + expect(result).toEqual({ + projectId: "proj-1", + activeSceneId: "scene-A", + mediaCount: 2, + mediaNames: ["clip.mp4", "song.mp3"], + playbackTimeMs: 5000, + }); + }); + + test("returns valid result with no media assets", async () => { + const context: AgentContext = { + projectId: null, + activeSceneId: null, + mediaAssets: [], + playbackTimeMs: 0, + }; + + const tool = toolRegistry.get("echo_context"); + const result = await tool.execute({}, context); + + expect(result).toEqual({ + projectId: null, + activeSceneId: null, + mediaCount: 0, + mediaNames: [], + playbackTimeMs: 0, + }); + }); + + test("ignores unused args", async () => { + const context: AgentContext = { + projectId: "proj-2", + activeSceneId: null, + mediaAssets: [], + playbackTimeMs: 0, + }; + + const tool = toolRegistry.get("echo_context"); + const result = await tool.execute({ extra: "ignored" }, context); + + expect(result).toEqual({ + projectId: "proj-2", + activeSceneId: null, + mediaCount: 0, + mediaNames: [], + playbackTimeMs: 0, + }); + }); +}); diff --git a/apps/web/src/agent/tools/mock.tool.ts b/apps/web/src/agent/tools/mock.tool.ts new file mode 100644 index 00000000..eaecdd06 --- /dev/null +++ b/apps/web/src/agent/tools/mock.tool.ts @@ -0,0 +1,19 @@ +import type { AgentContext, ToolDefinition } from "@/agent/types"; +import { toolRegistry } from "@/agent/tools/registry"; + +const echoContextTool: ToolDefinition = { + name: "echo_context", + description: "Returns a summary of the current editor context for debugging", + parameters: [], + execute: async (_args, context: AgentContext) => { + return { + projectId: context.projectId, + activeSceneId: context.activeSceneId, + mediaCount: context.mediaAssets.length, + mediaNames: context.mediaAssets.map((m) => m.name), + playbackTimeMs: context.playbackTimeMs, + }; + }, +}; + +toolRegistry.register("echo_context", echoContextTool); diff --git a/apps/web/src/agent/tools/registry.ts b/apps/web/src/agent/tools/registry.ts new file mode 100644 index 00000000..ae024e77 --- /dev/null +++ b/apps/web/src/agent/tools/registry.ts @@ -0,0 +1,6 @@ +import { DefinitionRegistry } from "@/lib/registry"; +import type { ToolDefinition } from "@/agent/types"; + +export const toolRegistry = new DefinitionRegistry( + "tool", +); diff --git a/apps/web/src/agent/types.ts b/apps/web/src/agent/types.ts new file mode 100644 index 00000000..5510a97b --- /dev/null +++ b/apps/web/src/agent/types.ts @@ -0,0 +1,49 @@ +export type ExecutionState = + | "idle" + | "sending" + | "processing" + | "responding" + | "error"; + +export interface ChatMessage { + id: string; + role: "user" | "assistant" | "tool_result"; + content: string; + toolCalls?: ToolCall[]; + timestamp: number; +} + +export interface ToolCall { + id: string; + name: string; + args: Record; +} + +export interface ToolResult { + toolCallId: string; + name: string; + result: unknown; + error?: string; +} + +export interface AgentContext { + projectId: string | null; + activeSceneId: string | null; + mediaAssets: Array<{ + id: string; + name: string; + type: string; + duration: number; + }>; + playbackTimeMs: number; +} + +export interface ToolDefinition { + name: string; + description: string; + parameters: Array<{ key: string; type: string; required: boolean }>; + execute: ( + args: Record, + context: AgentContext, + ) => Promise; +} diff --git a/apps/web/src/app/api/agent/chat/__tests__/route.test.ts b/apps/web/src/app/api/agent/chat/__tests__/route.test.ts new file mode 100644 index 00000000..edeffb83 --- /dev/null +++ b/apps/web/src/app/api/agent/chat/__tests__/route.test.ts @@ -0,0 +1,67 @@ +import { describe, expect, test } from "bun:test"; +import { POST } from "@/app/api/agent/chat/route"; +import { NextRequest } from "next/server"; + +function makeRequest(body: unknown): NextRequest { + return new NextRequest("http://localhost/api/agent/chat", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); +} + +describe("POST /api/agent/chat", () => { + test("returns mock response for valid input", async () => { + const req = makeRequest({ + messages: [{ role: "user", content: "Hello" }], + }); + + const res = await POST(req); + expect(res.status).toBe(200); + + const data = await res.json(); + expect(data.content).toBe("Let me check your editor context."); + expect(data.toolCalls).toHaveLength(1); + expect(data.toolCalls[0].name).toBe("echo_context"); + expect(data.toolCalls[0].id).toBe("mock_tc_1"); + }); + + test("returns 400 for invalid input — missing messages", async () => { + const req = makeRequest({}); + + const res = await POST(req); + expect(res.status).toBe(400); + + const data = await res.json(); + expect(data.error).toBe("Invalid input"); + expect(data.details).toBeDefined(); + }); + + test("returns 400 for invalid input — bad role", async () => { + const req = makeRequest({ + messages: [{ role: "invalid_role", content: "test" }], + }); + + const res = await POST(req); + expect(res.status).toBe(400); + }); + + test("accepts valid input with optional context", async () => { + const req = makeRequest({ + messages: [{ role: "user", content: "Test" }], + context: { projectId: "p1" }, + }); + + const res = await POST(req); + expect(res.status).toBe(200); + }); + + test("returns 400 for non-array messages", async () => { + const req = makeRequest({ + messages: "not an array", + }); + + const res = await POST(req); + expect(res.status).toBe(400); + }); +}); diff --git a/apps/web/src/app/api/agent/chat/route.ts b/apps/web/src/app/api/agent/chat/route.ts new file mode 100644 index 00000000..dc87c2f2 --- /dev/null +++ b/apps/web/src/app/api/agent/chat/route.ts @@ -0,0 +1,38 @@ +import { type NextRequest, NextResponse } from "next/server"; +import { z } from "zod"; + +const chatRequestSchema = z.object({ + messages: z.array( + z.object({ + role: z.enum(["user", "assistant", "tool_result"]), + content: z.string(), + }), + ), + context: z.record(z.string(), z.unknown()).optional(), +}); + +export async function POST(request: NextRequest) { + const body = await request.json(); + const result = chatRequestSchema.safeParse(body); + + if (!result.success) { + return NextResponse.json( + { error: "Invalid input", details: result.error.flatten().fieldErrors }, + { status: 400 }, + ); + } + + // Phase 1: canned mock response with a tool call to validate the full pipeline + const mockResponse = { + content: "Let me check your editor context.", + toolCalls: [ + { + id: "mock_tc_1", + name: "echo_context", + args: {}, + }, + ], + }; + + return NextResponse.json(mockResponse); +} diff --git a/apps/web/src/app/editor/[project_id]/page.tsx b/apps/web/src/app/editor/[project_id]/page.tsx index a3a7afa5..b6bf5f62 100644 --- a/apps/web/src/app/editor/[project_id]/page.tsx +++ b/apps/web/src/app/editor/[project_id]/page.tsx @@ -8,6 +8,7 @@ import { } from "@/components/ui/resizable"; import { AssetsPanel } from "@/components/editor/panels/assets"; import { PropertiesPanel } from "@/components/editor/panels/properties"; +import { ChatPanel } from "@/components/editor/panels/chat"; import { Timeline } from "@/components/editor/panels/timeline"; import { PreviewPanel } from "@/components/editor/panels/preview"; import { EditorHeader } from "@/components/editor/editor-header"; @@ -15,6 +16,7 @@ import { EditorProvider } from "@/components/providers/editor-provider"; import { Onboarding } from "@/components/editor/onboarding"; import { MigrationDialog } from "@/components/editor/dialogs/migration-dialog"; import { usePanelStore } from "@/stores/panel-store"; +import { useRightPanelStore } from "@/stores/right-panel-store"; import { usePasteMedia } from "@/hooks/use-paste-media"; import { MobileGate } from "@/components/editor/mobile-gate"; import { useState } from "react"; @@ -67,6 +69,39 @@ function DegradedRendererBanner() { ); } +function RightPanel() { + const activeTab = useRightPanelStore((s) => s.activeTab); + const setActiveTab = useRightPanelStore((s) => s.setActiveTab); + + return ( +
+
+
+ + +
+
+
+ {activeTab === "properties" ? : } +
+
+ ); +} + function EditorLayout() { usePasteMedia(); const { panels, setPanel } = usePanelStore(); @@ -122,7 +157,7 @@ function EditorLayout() { maxSize={40} className="min-w-0" > - + diff --git a/apps/web/src/components/editor/panels/chat/chat-input.tsx b/apps/web/src/components/editor/panels/chat/chat-input.tsx new file mode 100644 index 00000000..ac6e82d6 --- /dev/null +++ b/apps/web/src/components/editor/panels/chat/chat-input.tsx @@ -0,0 +1,62 @@ +"use client"; + +import { useState, useCallback, type KeyboardEvent } from "react"; +import { HugeiconsIcon } from "@hugeicons/react"; +import { Sent02Icon } from "@hugeicons/core-free-icons"; +import { Button } from "@/components/ui/button"; +import { Spinner } from "@/components/ui/spinner"; + +interface ChatInputProps { + onSend: (content: string) => void; + disabled: boolean; +} + +export function ChatInput({ onSend, disabled }: ChatInputProps) { + const [value, setValue] = useState(""); + + const handleSend = useCallback(() => { + const trimmed = value.trim(); + if (!trimmed || disabled) return; + onSend(trimmed); + setValue(""); + }, [value, disabled, onSend]); + + const handleKeyDown = useCallback( + (e: KeyboardEvent) => { + if (e.key === "Enter" && !e.shiftKey) { + e.preventDefault(); + handleSend(); + } + }, + [handleSend], + ); + + return ( +
+
+