From e5889937065fa2db0b5e705655c8716b518f3648 Mon Sep 17 00:00:00 2001 From: Luis Esteban Acevedo Ladino Date: Thu, 30 Apr 2026 12:45:19 -0500 Subject: [PATCH] feat: add plan mode with read-only analysis, user approval, and structured execution MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduces a plan/execute mode system for the AI agent. In plan mode the agent is read-only — it analyzes footage, asks clarifying questions, and submits a structured step-by-step plan. The user reviews and approves before the agent switches to execute mode to apply edits. New tools: submit_plan, ask_user, request_plan_approval, update_plan_step. UI: plan panel checklist, mode transition modal, question prompt with options, and Enter/Shift+Enter mode toggle in chat input. --- apps/web/src/agent/orchestrator.ts | 51 ++++++- apps/web/src/agent/system-prompt.ts | 72 ++++++++- apps/web/src/agent/tools/ask-user.tool.ts | 44 ++++++ apps/web/src/agent/tools/index.ts | 4 + .../agent/tools/request-plan-approval.tool.ts | 46 ++++++ apps/web/src/agent/tools/schemas.ts | 43 ++++++ apps/web/src/agent/tools/submit-plan.tool.ts | 42 +++++ .../src/agent/tools/update-plan-step.tool.ts | 57 +++++++ apps/web/src/agent/types.ts | 36 ++++- apps/web/src/app/api/agent/chat/route.ts | 7 +- .../editor/panels/chat/ask-user-prompt.tsx | 93 ++++++++++++ .../editor/panels/chat/chat-input.tsx | 91 +++++++++-- .../components/editor/panels/chat/index.tsx | 19 ++- .../panels/chat/mode-transition-modal.tsx | 105 +++++++++++++ .../editor/panels/chat/plan-panel.tsx | 143 ++++++++++++++++++ apps/web/src/stores/agent-store.ts | 23 ++- apps/web/src/stores/plan-store.ts | 90 +++++++++++ 17 files changed, 932 insertions(+), 34 deletions(-) create mode 100644 apps/web/src/agent/tools/ask-user.tool.ts create mode 100644 apps/web/src/agent/tools/request-plan-approval.tool.ts create mode 100644 apps/web/src/agent/tools/submit-plan.tool.ts create mode 100644 apps/web/src/agent/tools/update-plan-step.tool.ts create mode 100644 apps/web/src/components/editor/panels/chat/ask-user-prompt.tsx create mode 100644 apps/web/src/components/editor/panels/chat/mode-transition-modal.tsx create mode 100644 apps/web/src/components/editor/panels/chat/plan-panel.tsx create mode 100644 apps/web/src/stores/plan-store.ts diff --git a/apps/web/src/agent/orchestrator.ts b/apps/web/src/agent/orchestrator.ts index dc73dab2..840e1635 100644 --- a/apps/web/src/agent/orchestrator.ts +++ b/apps/web/src/agent/orchestrator.ts @@ -33,6 +33,14 @@ const WRITE_TOOLS = new Set([ "update_keyframe_curve", ]); +const PLAN_ONLY_TOOLS = new Set([ + "submit_plan", + "ask_user", + "request_plan_approval", +]); + +const EXECUTE_ONLY_TOOLS = new Set(["update_plan_step"]); + interface APIResponse { content: string; toolCalls?: ToolCall[]; @@ -57,10 +65,16 @@ export async function run( while (iterations < MAX_ITERATIONS) { iterations++; + const liveMode = useAgentStore.getState().mode; + const liveContext: AgentContext = { ...context, mode: liveMode }; + const response = await fetch("/api/agent/chat", { method: "POST", headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ messages: workingMessages, context }), + body: JSON.stringify({ + messages: workingMessages, + context: liveContext, + }), }); if (!response.ok) { @@ -207,12 +221,43 @@ async function resolveToolCalls( ): Promise { const agentStore = useAgentStore.getState(); const results: ToolResult[] = []; + const currentMode = useAgentStore.getState().mode; for (const tc of toolCalls) { useAgentStore.getState().setActiveTool(tc.name); - const currentMode = useAgentStore.getState().permissionMode; - if (currentMode === "ask" && WRITE_TOOLS.has(tc.name)) { + if (currentMode === "plan" && WRITE_TOOLS.has(tc.name)) { + results.push({ + toolCallId: tc.id, + name: tc.name, + result: null, + error: `Cannot use '${tc.name}' in plan mode. This tool modifies the timeline. Use submit_plan to finalize your plan, then request_plan_approval to switch to execute mode.`, + }); + continue; + } + + if (currentMode === "plan" && EXECUTE_ONLY_TOOLS.has(tc.name)) { + results.push({ + toolCallId: tc.id, + name: tc.name, + result: null, + error: `Cannot use '${tc.name}' in plan mode. This tool is only available during execution.`, + }); + continue; + } + + if (currentMode === "execute" && PLAN_ONLY_TOOLS.has(tc.name)) { + results.push({ + toolCallId: tc.id, + name: tc.name, + result: null, + error: `Cannot use '${tc.name}' in execute mode. This tool is only available during planning.`, + }); + continue; + } + + const currentPermissionMode = useAgentStore.getState().permissionMode; + if (currentPermissionMode === "ask" && WRITE_TOOLS.has(tc.name)) { const approved = await requestApproval(tc); if (!approved) { results.push({ diff --git a/apps/web/src/agent/system-prompt.ts b/apps/web/src/agent/system-prompt.ts index 348fd867..5dd5449e 100644 --- a/apps/web/src/agent/system-prompt.ts +++ b/apps/web/src/agent/system-prompt.ts @@ -1,20 +1,70 @@ -import type { AgentContext } from "@/agent/types"; +import type { AgentContext, AgentMode } from "@/agent/types"; export interface ToolSummary { name: string; description: string; } -/** - * Builds the system prompt for the LLM from the editor context. - * - * Pure function — no client-only imports (no EditorCore, WASM, browser APIs). - * Consumable server-side in the API route. - */ +const PLAN_MODE_INSTRUCTIONS = ` +## CURRENT MODE: PLAN (READ-ONLY) + +You are in PLAN mode. You CANNOT make any edits to the timeline. You can only: +- Read and analyze the project (list_project_assets, list_timeline, get_element, list_effects, get_effect, list_keyframes, list_animatable_properties) +- Load media into your context (load_context) +- Discover skills (list_skills, load_skill) +- Ask the user questions (ask_user) +- Submit your plan (submit_plan) + +## YOUR WORKFLOW IN PLAN MODE: + +1. **Analyze**: Use read-only tools to understand what media and timeline state exist. Use load_context to actually see/hear the footage. +2. **Discover**: If the user's request matches a known editing pattern, call list_skills to find relevant techniques. +3. **Clarify**: If anything is ambiguous, use ask_user to get clarification before planning. +4. **Plan**: Once you understand the footage and the goal, call submit_plan with a structured step-by-step plan. + +## PLAN QUALITY RULES: + +- Each step must reference SPECIFIC tools and SPECIFIC values (timestamps, element IDs after discovery, effect params) +- Steps should be ordered by dependency (split before delete, add text before animate, etc.) +- If a skill was loaded, reference its techniques but ADAPT them to the actual footage — never copy blindly +- Include timing estimates where possible +- Be honest about limitations — if the footage doesn't support a technique, say so + +## WHEN TO SUBMIT: + +Submit the plan when: +- You have enough context about the media content +- All clarifying questions have been answered +- You have a concrete, actionable plan + +After submitting, the user will review and either approve (switching to execute mode) or request changes. +`; + +const EXECUTE_MODE_INSTRUCTIONS = ` +## CURRENT MODE: EXECUTE + +You are in EXECUTE mode. You can read AND write to the timeline. Follow the approved plan step by step. + +## EXECUTION RULES: + +1. Follow the plan steps in order. Each step lists the tools to use. +2. After completing a step, call update_plan_step to mark it done. +3. If a step fails, note the error in update_plan_step and decide whether to continue or skip. +4. If you discover the plan needs adjustment mid-execution, explain what changed and why. +5. When all steps are done, provide a summary of what was accomplished. + +## IF NO ACTIVE PLAN: + +Execute the user's request directly. For complex edits (3+ tool calls), consider using submit_plan first to get user approval. +`; + export function buildSystemPrompt( context: AgentContext, tools?: ToolSummary[], + mode?: AgentMode, ): string { + const activeMode = mode ?? context.mode ?? "execute"; + const mediaSection = context.mediaAssets.length > 0 ? `Active media assets:\n${context.mediaAssets.map((m) => `- [id: ${m.id}] ${m.name} (${m.type}, ${m.duration}s)`).join("\n")}` @@ -45,9 +95,15 @@ export function buildSystemPrompt( "Only claim edits that were actually performed by tool calls in this conversation. Do not say you added, removed, cleaned, or updated text/subtitles unless the relevant add_text, update_text, delete_timeline_elements, or update_timeline_element_timing tool call succeeded.", "When the user asks to add titles, hooks, labels, captions, subtitles, or visible text, call add_text. Do not add text proactively for unrelated edit requests such as cutting silence unless the user asks for text.", "When you need to perform an action matching one of these tools, call the appropriate tool. For all other requests, respond directly in plain text.", - "SKILLS: You have editing skills available — pre-built recipe workflows for common editing patterns. When the user asks for a complex edit (viral video, pitch, specific style, etc.), call list_skills to discover relevant skills, then call load_skill with the matching skillId to get full step-by-step instructions. Follow those instructions precisely using the standard editing tools. If the user's request doesn't match any skill, proceed with your own editing approach.", + "SKILLS: You have editing skills available — pre-built technique libraries for common editing patterns. When the user asks for a complex edit (viral video, pitch, specific style, etc.), call list_skills to discover relevant skills, then call load_skill with the matching skillId to get technique definitions. Adapt those techniques to the actual footage content. If the user's request doesn't match any skill, proceed with your own editing approach.", ); } + if (activeMode === "plan") { + parts.push(PLAN_MODE_INSTRUCTIONS); + } else { + parts.push(EXECUTE_MODE_INSTRUCTIONS); + } + return parts.join("\n"); } diff --git a/apps/web/src/agent/tools/ask-user.tool.ts b/apps/web/src/agent/tools/ask-user.tool.ts new file mode 100644 index 00000000..2746b59b --- /dev/null +++ b/apps/web/src/agent/tools/ask-user.tool.ts @@ -0,0 +1,44 @@ +import type { AgentContext, ToolDefinition } from "@/agent/types"; +import { toolRegistry } from "@/agent/tools/registry"; +import { askUserSchema } from "@/agent/tools/schemas"; +import { usePlanStore } from "@/stores/plan-store"; +import { nanoid } from "nanoid"; + +const askUserTool: ToolDefinition = { + ...askUserSchema, + execute: async ( + args: Record, + _context: AgentContext, + ): Promise<{ questionId: string; answer: string } | { error: string }> => { + const { question, options } = args as { + question: string; + options?: Array<{ label: string; description?: string }>; + }; + + if (!question || typeof question !== "string") { + return { error: "question is required and must be a string" }; + } + + const questionId = nanoid(); + const planStore = usePlanStore.getState(); + + planStore.setPendingQuestion({ + id: questionId, + question, + options: options && Array.isArray(options) ? options : undefined, + }); + + return new Promise((resolve) => { + const unsubscribe = usePlanStore.subscribe((state) => { + if (state.pendingQuestion?.answer) { + unsubscribe(); + const answer = state.pendingQuestion.answer; + usePlanStore.getState().setPendingQuestion(null); + resolve({ questionId, answer }); + } + }); + }); + }, +}; + +toolRegistry.register(askUserSchema.name, askUserTool); diff --git a/apps/web/src/agent/tools/index.ts b/apps/web/src/agent/tools/index.ts index 3cd7819b..9b6b71dc 100644 --- a/apps/web/src/agent/tools/index.ts +++ b/apps/web/src/agent/tools/index.ts @@ -38,3 +38,7 @@ import "@/agent/tools/update-keyframe-curve.tool"; import "@/agent/tools/list-animatable-properties.tool"; import "@/agent/skills/list-skills.tool"; import "@/agent/skills/load-skill.tool"; +import "@/agent/tools/submit-plan.tool"; +import "@/agent/tools/ask-user.tool"; +import "@/agent/tools/request-plan-approval.tool"; +import "@/agent/tools/update-plan-step.tool"; diff --git a/apps/web/src/agent/tools/request-plan-approval.tool.ts b/apps/web/src/agent/tools/request-plan-approval.tool.ts new file mode 100644 index 00000000..a75773d5 --- /dev/null +++ b/apps/web/src/agent/tools/request-plan-approval.tool.ts @@ -0,0 +1,46 @@ +import type { AgentContext, ToolDefinition } from "@/agent/types"; +import { toolRegistry } from "@/agent/tools/registry"; +import { requestPlanApprovalSchema } from "@/agent/tools/schemas"; +import { useAgentStore } from "@/stores/agent-store"; +import { usePlanStore } from "@/stores/plan-store"; +import type { AgentMode } from "@/agent/types"; + +const requestPlanApprovalTool: ToolDefinition = { + ...requestPlanApprovalSchema, + execute: async ( + _args: Record, + _context: AgentContext, + ): Promise<{ approved: boolean; mode: AgentMode } | { error: string }> => { + const planStore = usePlanStore.getState(); + if (!planStore.plan) { + return { + error: "No plan has been submitted yet. Call submit_plan first.", + }; + } + + const agentStore = useAgentStore.getState(); + if (agentStore.mode !== "plan") { + return { error: "Already in execute mode. No transition needed." }; + } + + planStore.updatePlanStatus("approved"); + + return new Promise((resolve) => { + useAgentStore.getState().setPendingModeTransition({ + targetMode: "execute", + resolve: (approved) => { + useAgentStore.getState().setPendingModeTransition(null); + if (approved) { + useAgentStore.getState().setMode("execute"); + } + resolve({ + approved, + mode: approved ? "execute" : "plan", + }); + }, + }); + }); + }, +}; + +toolRegistry.register(requestPlanApprovalSchema.name, requestPlanApprovalTool); diff --git a/apps/web/src/agent/tools/schemas.ts b/apps/web/src/agent/tools/schemas.ts index dbba40dd..69a4825a 100644 --- a/apps/web/src/agent/tools/schemas.ts +++ b/apps/web/src/agent/tools/schemas.ts @@ -305,6 +305,45 @@ export const loadSkillSchema: ToolSchema = { parameters: [{ key: "skillId", type: "string", required: true }], }; +export const submitPlanSchema: ToolSchema = { + name: "submit_plan", + description: + "Submits a structured editing plan with steps. Use ONLY in plan mode after analyzing the footage and understanding the user's request. Each step should describe a specific action with the tools to use. After submission, the user reviews and approves or requests changes before execution begins.", + parameters: [ + { key: "summary", type: "string", required: true }, + { key: "steps", type: "array", required: true }, + { key: "questions", type: "array", required: false }, + ], +}; + +export const askUserSchema: ToolSchema = { + name: "ask_user", + description: + "Asks the user a question during plan mode. Use when you need clarification about their intent, preferences, or content details before finalizing the plan. Optionally provide quick-reply options for common answers. The user can also type a free-form response.", + parameters: [ + { key: "question", type: "string", required: true }, + { key: "options", type: "array", required: false }, + ], +}; + +export const requestPlanApprovalSchema: ToolSchema = { + name: "request_plan_approval", + description: + "Requests user approval to switch from plan mode to execute mode. The user sees a modal with 'Continue planning' and 'Start editing' options. ALWAYS requires explicit user approval — there is no auto-approve. Use this when the plan is ready for execution.", + parameters: [], +}; + +export const updatePlanStepSchema: ToolSchema = { + name: "update_plan_step", + description: + "Updates the status of a plan step during execution. Call this after completing each step to track progress. The plan panel updates in real-time. Use status 'done' for successful completion, 'skipped' if the step was unnecessary, or 'in_progress' if starting.", + parameters: [ + { key: "stepId", type: "string", required: true }, + { key: "status", type: "string", required: true }, + { key: "result", type: "string", required: false }, + ], +}; + /** * The exact list of schemas exposed to the LLM. * Excludes internal-only tools (transcribe_video, mock). @@ -338,4 +377,8 @@ export const providerToolSchemas: ToolSchema[] = [ listAnimatablePropertiesSchema, listSkillsSchema, loadSkillSchema, + submitPlanSchema, + askUserSchema, + requestPlanApprovalSchema, + updatePlanStepSchema, ]; diff --git a/apps/web/src/agent/tools/submit-plan.tool.ts b/apps/web/src/agent/tools/submit-plan.tool.ts new file mode 100644 index 00000000..5926a5e7 --- /dev/null +++ b/apps/web/src/agent/tools/submit-plan.tool.ts @@ -0,0 +1,42 @@ +import type { AgentContext, ToolDefinition } from "@/agent/types"; +import { toolRegistry } from "@/agent/tools/registry"; +import { submitPlanSchema } from "@/agent/tools/schemas"; +import { usePlanStore, createPlanFromSteps } from "@/stores/plan-store"; + +const submitPlanTool: ToolDefinition = { + ...submitPlanSchema, + execute: async ( + args: Record, + _context: AgentContext, + ): Promise<{ planId: string; stepCount: number } | { error: string }> => { + const { summary, steps, questions } = args as { + summary: string; + steps: Array<{ description: string; tools: string[] }>; + questions?: string[]; + }; + + if (!summary || typeof summary !== "string") { + return { error: "summary is required and must be a string" }; + } + + if (!Array.isArray(steps) || steps.length === 0) { + return { error: "steps must be a non-empty array" }; + } + + for (const [i, step] of steps.entries()) { + if (!step.description || typeof step.description !== "string") { + return { error: `Step ${i + 1} must have a description string` }; + } + if (!Array.isArray(step.tools)) { + return { error: `Step ${i + 1} must have a tools array` }; + } + } + + const plan = createPlanFromSteps(summary, steps, questions); + usePlanStore.getState().setPlan(plan); + + return { planId: plan.id, stepCount: plan.steps.length }; + }, +}; + +toolRegistry.register(submitPlanSchema.name, submitPlanTool); diff --git a/apps/web/src/agent/tools/update-plan-step.tool.ts b/apps/web/src/agent/tools/update-plan-step.tool.ts new file mode 100644 index 00000000..beb837dc --- /dev/null +++ b/apps/web/src/agent/tools/update-plan-step.tool.ts @@ -0,0 +1,57 @@ +import type { AgentContext, ToolDefinition } from "@/agent/types"; +import { toolRegistry } from "@/agent/tools/registry"; +import { updatePlanStepSchema } from "@/agent/tools/schemas"; +import { usePlanStore } from "@/stores/plan-store"; +import type { PlanStepStatus } from "@/agent/types"; + +const VALID_STATUSES: PlanStepStatus[] = [ + "pending", + "in_progress", + "done", + "skipped", +]; + +const updatePlanStepTool: ToolDefinition = { + ...updatePlanStepSchema, + execute: async ( + args: Record, + _context: AgentContext, + ): + | { success: boolean; stepId: string; status: string } + | { error: string } => { + const { stepId, status, result } = args as { + stepId: string; + status: string; + result?: string; + }; + + if (!stepId || typeof stepId !== "string") { + return { error: "stepId is required" }; + } + + if (!VALID_STATUSES.includes(status as PlanStepStatus)) { + return { + error: `Invalid status: ${status}. Must be one of: ${VALID_STATUSES.join(", ")}`, + }; + } + + const planStore = usePlanStore.getState(); + const plan = planStore.plan; + if (!plan) { + return { error: "No active plan" }; + } + + const step = plan.steps.find((s) => s.id === stepId); + if (!step) { + return { + error: `Step not found: ${stepId}. Available: ${plan.steps.map((s) => s.id).join(", ")}`, + }; + } + + planStore.updateStepStatus(stepId, status as PlanStepStatus, result); + + return { success: true, stepId, status }; + }, +}; + +toolRegistry.register(updatePlanStepSchema.name, updatePlanStepTool); diff --git a/apps/web/src/agent/types.ts b/apps/web/src/agent/types.ts index 3c4fe233..35221793 100644 --- a/apps/web/src/agent/types.ts +++ b/apps/web/src/agent/types.ts @@ -5,12 +5,45 @@ export type ExecutionState = | "responding" | "error"; +export type AgentMode = "plan" | "execute"; + +export type PlanStepStatus = "pending" | "in_progress" | "done" | "skipped"; +export type PlanStatus = + | "drafting" + | "awaiting_approval" + | "approved" + | "executing" + | "completed"; + +export interface PlanStep { + id: string; + description: string; + tools: string[]; + status: PlanStepStatus; + result?: string; +} + +export interface Plan { + id: string; + summary: string; + steps: PlanStep[]; + questions?: string[]; + status: PlanStatus; + createdAt: number; +} + +export interface AgentQuestion { + id: string; + question: string; + options?: Array<{ label: string; description?: string }>; + answer?: string; +} + export interface ChatMessage { id: string; role: "system" | "user" | "assistant" | "tool_result"; content: string; toolCalls?: ToolCall[]; - /** Associates a tool_result message with the ToolCall it answers. */ toolCallId?: string; timestamp: number; } @@ -41,6 +74,7 @@ export interface AgentContext { }>; timelineTracks?: AgentTimelineTrack[]; playbackTimeMs: number; + mode?: AgentMode; } export type AgentTimelineTrack = { diff --git a/apps/web/src/app/api/agent/chat/route.ts b/apps/web/src/app/api/agent/chat/route.ts index 1e906922..5291fd55 100644 --- a/apps/web/src/app/api/agent/chat/route.ts +++ b/apps/web/src/app/api/agent/chat/route.ts @@ -63,6 +63,7 @@ const chatRequestSchema = z.object({ ) .optional(), playbackTimeMs: z.number(), + mode: z.enum(["plan", "execute"]).optional(), }) satisfies z.ZodType, }); @@ -89,7 +90,8 @@ export async function POST(request: NextRequest) { const { messages, context } = result.data; - // Resolve provider config from LLM_* env vars + const mode = context.mode; + const provider = process.env.LLM_PROVIDER; const apiKey = process.env.LLM_API_KEY; const model = process.env.LLM_MODEL; @@ -107,8 +109,7 @@ export async function POST(request: NextRequest) { const config: ProviderConfig = { provider, apiKey, model, baseUrl }; - // Build system prompt with tool guidance - const systemPrompt = buildSystemPrompt(context, providerToolSchemas); + const systemPrompt = buildSystemPrompt(context, providerToolSchemas, mode); // Delegate to provider adapter try { diff --git a/apps/web/src/components/editor/panels/chat/ask-user-prompt.tsx b/apps/web/src/components/editor/panels/chat/ask-user-prompt.tsx new file mode 100644 index 00000000..2a8cb2fb --- /dev/null +++ b/apps/web/src/components/editor/panels/chat/ask-user-prompt.tsx @@ -0,0 +1,93 @@ +"use client"; + +import { useState, useCallback } from "react"; +import { HugeiconsIcon } from "@hugeicons/react"; +import { QuestionExchangeIcon } from "@hugeicons/core-free-icons"; +import { Button } from "@/components/ui/button"; +import { usePlanStore } from "@/stores/plan-store"; + +export function AskUserPrompt() { + const pendingQuestion = usePlanStore((s) => s.pendingQuestion); + const answerQuestion = usePlanStore((s) => s.answerQuestion); + const [freeForm, setFreeForm] = useState(""); + + const handleOptionClick = useCallback( + (label: string) => { + answerQuestion(label); + setFreeForm(""); + }, + [answerQuestion], + ); + + const handleFreeFormSubmit = useCallback(() => { + const trimmed = freeForm.trim(); + if (!trimmed) return; + answerQuestion(trimmed); + setFreeForm(""); + }, [freeForm, answerQuestion]); + + const handleKeyDown = useCallback( + (e: React.KeyboardEvent) => { + if (e.key === "Enter" && !e.shiftKey) { + e.preventDefault(); + handleFreeFormSubmit(); + } + }, + [handleFreeFormSubmit], + ); + + if (!pendingQuestion || pendingQuestion.answer) return null; + + return ( +
+
+ + + Agent question + +
+

{pendingQuestion.question}

+ + {pendingQuestion.options && pendingQuestion.options.length > 0 && ( +
+ {pendingQuestion.options.map((opt) => ( + + ))} +
+ )} + +
+ setFreeForm(e.target.value)} + onKeyDown={handleKeyDown} + placeholder="Type your answer..." + className="border-border bg-input flex-1 rounded-md border px-2 py-1 text-xs outline-none focus:border-blue-500/50" + /> + +
+
+ ); +} diff --git a/apps/web/src/components/editor/panels/chat/chat-input.tsx b/apps/web/src/components/editor/panels/chat/chat-input.tsx index f7218e3b..9213ae56 100644 --- a/apps/web/src/components/editor/panels/chat/chat-input.tsx +++ b/apps/web/src/components/editor/panels/chat/chat-input.tsx @@ -6,11 +6,15 @@ import { Sent02Icon, FlashIcon, Shield01Icon, + PlanChartIcon, + Edit02Icon, } from "@hugeicons/core-free-icons"; import { Button } from "@/components/ui/button"; import { Spinner } from "@/components/ui/spinner"; import { useAgentStore, type PermissionMode } from "@/stores/agent-store"; +import { usePlanStore } from "@/stores/plan-store"; import { cn } from "@/utils/ui"; +import type { AgentMode } from "@/agent/types"; interface ChatInputProps { onSend: (content: string) => void; @@ -22,31 +26,51 @@ export function ChatInput({ onSend, disabled }: ChatInputProps) { const permissionMode = useAgentStore((s) => s.permissionMode); const setPermissionMode = useAgentStore((s) => s.setPermissionMode); const pendingApproval = useAgentStore((s) => s.pendingApproval); + const mode = useAgentStore((s) => s.mode); + const setMode = useAgentStore((s) => s.setMode); + const pendingTransition = useAgentStore((s) => s.pendingModeTransition); + const plan = usePlanStore((s) => s.plan); - const handleSend = useCallback(() => { - const trimmed = value.trim(); - if (!trimmed || disabled) return; - onSend(trimmed); - setValue(""); - }, [value, disabled, onSend]); + const handleSend = useCallback( + (forcedMode?: AgentMode) => { + const trimmed = value.trim(); + if (!trimmed || disabled) return; + const targetMode = forcedMode ?? mode; + if (targetMode !== mode) { + setMode(targetMode); + } + onSend(trimmed); + setValue(""); + }, + [value, disabled, onSend, mode, setMode], + ); const handleKeyDown = useCallback( (e: KeyboardEvent) => { if (e.key === "Enter" && !e.shiftKey) { e.preventDefault(); - handleSend(); + handleSend("execute"); + } + if (e.key === "Enter" && e.shiftKey) { + e.preventDefault(); + handleSend("plan"); } }, [handleSend], ); const toggleMode = useCallback(() => { - const next: PermissionMode = - permissionMode === "skip" ? "ask" : "skip"; + const next: AgentMode = mode === "execute" ? "plan" : "execute"; + setMode(next); + }, [mode, setMode]); + + const togglePermission = useCallback(() => { + const next: PermissionMode = permissionMode === "skip" ? "ask" : "skip"; setPermissionMode(next); }, [permissionMode, setPermissionMode]); const isAsk = permissionMode === "ask"; + const isPlanMode = mode === "plan"; return (
@@ -55,16 +79,38 @@ export function ChatInput({ onSend, disabled }: ChatInputProps) { value={value} onChange={(e) => setValue(e.target.value)} onKeyDown={handleKeyDown} - disabled={disabled} - placeholder="Ask the assistant..." + disabled={disabled || !!pendingTransition} + placeholder={ + isPlanMode + ? "Describe what you want (plan mode)..." + : "Ask the assistant..." + } rows={1} className="border-border bg-input focus-visible:border-primary/50 focus-visible:ring-primary/20 flex-1 resize-none rounded-md border px-3 py-2 text-sm outline-none focus-visible:ring-2 disabled:cursor-not-allowed disabled:opacity-50" /> +
+
+ + Enter = edit mode |{" "} + Shift+Enter = plan mode + + {plan && ( + + + {isPlanMode ? "PLANNING" : "EXECUTING"} + + + )} +
); } diff --git a/apps/web/src/components/editor/panels/chat/index.tsx b/apps/web/src/components/editor/panels/chat/index.tsx index e9164d97..ee74c0a9 100644 --- a/apps/web/src/components/editor/panels/chat/index.tsx +++ b/apps/web/src/components/editor/panels/chat/index.tsx @@ -8,9 +8,13 @@ import { Button } from "@/components/ui/button"; import { Spinner } from "@/components/ui/spinner"; import { useChatStore } from "@/stores/chat-store"; import { useAgentStore } from "@/stores/agent-store"; +import { usePlanStore } from "@/stores/plan-store"; import { MessageBubble } from "./message-bubble"; import { ChatInput } from "./chat-input"; import { ToolPermissionRequest } from "./tool-permission-request"; +import { PlanPanel } from "./plan-panel"; +import { ModeTransitionModal } from "./mode-transition-modal"; +import { AskUserPrompt } from "./ask-user-prompt"; import { run as orchestratorRun } from "@/agent/orchestrator"; import { EditorContextAdapter } from "@/agent/context"; @@ -21,16 +25,16 @@ export function ChatPanel() { const sendMessage = useChatStore((s) => s.sendMessage); const setError = useChatStore((s) => s.setError); const pendingApproval = useAgentStore((s) => s.pendingApproval); + const pendingTransition = useAgentStore((s) => s.pendingModeTransition); + const plan = usePlanStore((s) => s.plan); const scrollRef = useRef(null); - // Auto-scroll on new messages or loading state change - // biome-ignore lint/correctness/useExhaustiveDependencies: re-scroll when messages or loading change useEffect(() => { if (scrollRef.current) { scrollRef.current.scrollTop = scrollRef.current.scrollHeight; } - }, [messages, loading]); + }); const handleSend = useCallback( async (content: string) => { @@ -52,6 +56,12 @@ export function ChatPanel() { return (
+ {plan && ( +
+ +
+ )} + {messages.length === 0 && !loading && !error ? (
)} + {pendingApproval && ( )} @@ -110,6 +121,8 @@ export function ChatPanel() { )} + + {pendingTransition && }
); } diff --git a/apps/web/src/components/editor/panels/chat/mode-transition-modal.tsx b/apps/web/src/components/editor/panels/chat/mode-transition-modal.tsx new file mode 100644 index 00000000..b5c74ab7 --- /dev/null +++ b/apps/web/src/components/editor/panels/chat/mode-transition-modal.tsx @@ -0,0 +1,105 @@ +"use client"; + +import { useCallback } from "react"; +import { HugeiconsIcon } from "@hugeicons/react"; +import { PlanChartIcon, Edit02Icon } from "@hugeicons/core-free-icons"; +import { Button } from "@/components/ui/button"; +import { useAgentStore } from "@/stores/agent-store"; +import { usePlanStore } from "@/stores/plan-store"; +import { cn } from "@/utils/ui"; + +export function ModeTransitionModal() { + const pendingTransition = useAgentStore((s) => s.pendingModeTransition); + const plan = usePlanStore((s) => s.plan); + + const handleContinuePlanning = useCallback(() => { + if (!pendingTransition) return; + pendingTransition.resolve(false); + }, [pendingTransition]); + + const handleStartEditing = useCallback(() => { + if (!pendingTransition) return; + pendingTransition.resolve(true); + }, [pendingTransition]); + + if (!pendingTransition) return null; + + const doneCount = plan?.steps.filter((s) => s.status === "done").length ?? 0; + const totalCount = plan?.steps.length ?? 0; + + return ( +
+
+
+
+

Plan ready for execution

+ {plan && ( +

+ {plan.summary} — {totalCount} step{totalCount !== 1 ? "s" : ""} + {doneCount > 0 && ` (${doneCount} already done)`} +

+ )} +
+ + {plan && ( +
+
    + {plan.steps.map((step) => ( +
  • + + + {step.description} + +
    • + ))} +
+
+ )} + +
+ + +
+
+
+
+ ); +} diff --git a/apps/web/src/components/editor/panels/chat/plan-panel.tsx b/apps/web/src/components/editor/panels/chat/plan-panel.tsx new file mode 100644 index 00000000..f8f11e42 --- /dev/null +++ b/apps/web/src/components/editor/panels/chat/plan-panel.tsx @@ -0,0 +1,143 @@ +"use client"; + +import { HugeiconsIcon } from "@hugeicons/react"; +import { + CheckmarkCircle02Icon, + CancelCircleIcon, + Loading03Icon, + MinusSign01Icon, + PlanChartIcon, +} from "@hugeicons/core-free-icons"; +import { ScrollArea } from "@/components/ui/scroll-area"; +import { usePlanStore } from "@/stores/plan-store"; +import { useAgentStore } from "@/stores/agent-store"; +import { cn } from "@/utils/ui"; +import type { PlanStep, PlanStepStatus } from "@/agent/types"; + +const STATUS_CONFIG: Record< + PlanStepStatus, + { icon: typeof CheckmarkCircle02Icon; className: string } +> = { + pending: { + icon: MinusSign01Icon, + className: "text-muted-foreground", + }, + in_progress: { + icon: Loading03Icon, + className: "text-blue-400 animate-spin", + }, + done: { + icon: CheckmarkCircle02Icon, + className: "text-emerald-400", + }, + skipped: { + icon: CancelCircleIcon, + className: "text-muted-foreground/50", + }, +}; + +function StepItem({ step }: { step: PlanStep }) { + const config = STATUS_CONFIG[step.status]; + + return ( +
+ +
+ + {step.description} + + {step.result && ( + + {step.result} + + )} +
+
+ ); +} + +export function PlanPanel() { + const plan = usePlanStore((s) => s.plan); + const mode = useAgentStore((s) => s.mode); + + if (!plan) { + return ( +
+ +

+ {mode === "plan" + ? "Analyzing footage to build a plan..." + : "No active plan"} +

+
+ ); + } + + const doneCount = plan.steps.filter((s) => s.status === "done").length; + const totalCount = plan.steps.length; + const progress = totalCount > 0 ? (doneCount / totalCount) * 100 : 0; + + return ( +
+
+
+ Plan + + {plan.summary} + +
+
+
+
+
+ + {doneCount}/{totalCount} + +
+
+ + +
+ {plan.steps.map((step) => ( + + ))} +
+ + + {plan.questions && plan.questions.length > 0 && ( +
+ + Open questions + +
    + {plan.questions.map((q) => ( +
  • + {q} +
    • + ))} +
+
+ )} +
+ ); +} diff --git a/apps/web/src/stores/agent-store.ts b/apps/web/src/stores/agent-store.ts index 6367ce80..91d65844 100644 --- a/apps/web/src/stores/agent-store.ts +++ b/apps/web/src/stores/agent-store.ts @@ -1,5 +1,10 @@ import { create } from "zustand"; -import type { AgentContext, ExecutionState, ToolCall } from "@/agent/types"; +import type { + AgentContext, + AgentMode, + ExecutionState, + ToolCall, +} from "@/agent/types"; export type PermissionMode = "ask" | "skip"; @@ -8,6 +13,11 @@ export interface PendingToolApproval { resolve: (approved: boolean) => void; } +export interface PendingModeTransition { + targetMode: AgentMode; + resolve: (approved: boolean) => void; +} + const DEFAULT_CONTEXT: AgentContext = { projectId: null, activeSceneId: null, @@ -21,11 +31,15 @@ interface AgentState { context: AgentContext; permissionMode: PermissionMode; pendingApproval: PendingToolApproval | null; + mode: AgentMode; + pendingModeTransition: PendingModeTransition | null; setStatus: (status: ExecutionState) => void; setActiveTool: (tool: string | null) => void; setContext: (context: AgentContext) => void; setPermissionMode: (mode: PermissionMode) => void; setPendingApproval: (pending: PendingToolApproval | null) => void; + setMode: (mode: AgentMode) => void; + setPendingModeTransition: (transition: PendingModeTransition | null) => void; reset: () => void; } @@ -35,16 +49,23 @@ export const useAgentStore = create()((set) => ({ context: DEFAULT_CONTEXT, permissionMode: "skip", pendingApproval: null, + mode: "execute", + pendingModeTransition: null, setStatus: (status) => set({ status }), setActiveTool: (tool) => set({ activeTool: tool }), setContext: (context) => set({ context }), setPermissionMode: (mode) => set({ permissionMode: mode }), setPendingApproval: (pending) => set({ pendingApproval: pending }), + setMode: (mode) => set({ mode }), + setPendingModeTransition: (transition) => + set({ pendingModeTransition: transition }), reset: () => set({ status: "idle", activeTool: null, context: DEFAULT_CONTEXT, pendingApproval: null, + mode: "execute", + pendingModeTransition: null, }), })); diff --git a/apps/web/src/stores/plan-store.ts b/apps/web/src/stores/plan-store.ts new file mode 100644 index 00000000..4f0e0577 --- /dev/null +++ b/apps/web/src/stores/plan-store.ts @@ -0,0 +1,90 @@ +import { create } from "zustand"; +import { nanoid } from "nanoid"; +import type { + AgentQuestion, + Plan, + PlanStepStatus, + PlanStatus, +} from "@/agent/types"; + +interface PlanState { + plan: Plan | null; + pendingQuestion: AgentQuestion | null; + modeTransitionPending: boolean; + + setPlan: (plan: Plan) => void; + updatePlanStatus: (status: PlanStatus) => void; + updateStepStatus: ( + stepId: string, + status: PlanStepStatus, + result?: string, + ) => void; + clearPlan: () => void; + + setPendingQuestion: (question: AgentQuestion | null) => void; + answerQuestion: (answer: string) => void; + + setModeTransitionPending: (pending: boolean) => void; +} + +export const usePlanStore = create()((set, get) => ({ + plan: null, + pendingQuestion: null, + modeTransitionPending: false, + + setPlan: (plan) => set({ plan }), + + updatePlanStatus: (status) => { + const current = get().plan; + if (!current) return; + set({ plan: { ...current, status } }); + }, + + updateStepStatus: (stepId, status, result) => { + const current = get().plan; + if (!current) return; + set({ + plan: { + ...current, + steps: current.steps.map((s) => + s.id === stepId + ? { ...s, status, ...(result !== undefined && { result }) } + : s, + ), + }, + }); + }, + + clearPlan: () => set({ plan: null, pendingQuestion: null }), + + setPendingQuestion: (question) => set({ pendingQuestion: question }), + + answerQuestion: (answer) => { + const current = get().pendingQuestion; + if (!current) return; + set({ pendingQuestion: { ...current, answer } }); + }, + + setModeTransitionPending: (pending) => + set({ modeTransitionPending: pending }), +})); + +export function createPlanFromSteps( + summary: string, + steps: Array<{ description: string; tools: string[] }>, + questions?: string[], +): Plan { + return { + id: nanoid(), + summary, + steps: steps.map((s) => ({ + id: nanoid(), + description: s.description, + tools: s.tools, + status: "pending" as PlanStepStatus, + })), + questions, + status: "awaiting_approval" as PlanStatus, + createdAt: Date.now(), + }; +}