From e5889937065fa2db0b5e705655c8716b518f3648 Mon Sep 17 00:00:00 2001 From: Luis Esteban Acevedo Ladino Date: Thu, 30 Apr 2026 12:45:19 -0500 Subject: [PATCH 1/6] feat: add plan mode with read-only analysis, user approval, and structured execution MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduces a plan/execute mode system for the AI agent. In plan mode the agent is read-only — it analyzes footage, asks clarifying questions, and submits a structured step-by-step plan. The user reviews and approves before the agent switches to execute mode to apply edits. New tools: submit_plan, ask_user, request_plan_approval, update_plan_step. UI: plan panel checklist, mode transition modal, question prompt with options, and Enter/Shift+Enter mode toggle in chat input. --- apps/web/src/agent/orchestrator.ts | 51 ++++++- apps/web/src/agent/system-prompt.ts | 72 ++++++++- apps/web/src/agent/tools/ask-user.tool.ts | 44 ++++++ apps/web/src/agent/tools/index.ts | 4 + .../agent/tools/request-plan-approval.tool.ts | 46 ++++++ apps/web/src/agent/tools/schemas.ts | 43 ++++++ apps/web/src/agent/tools/submit-plan.tool.ts | 42 +++++ .../src/agent/tools/update-plan-step.tool.ts | 57 +++++++ apps/web/src/agent/types.ts | 36 ++++- apps/web/src/app/api/agent/chat/route.ts | 7 +- .../editor/panels/chat/ask-user-prompt.tsx | 93 ++++++++++++ .../editor/panels/chat/chat-input.tsx | 91 +++++++++-- .../components/editor/panels/chat/index.tsx | 19 ++- .../panels/chat/mode-transition-modal.tsx | 105 +++++++++++++ .../editor/panels/chat/plan-panel.tsx | 143 ++++++++++++++++++ apps/web/src/stores/agent-store.ts | 23 ++- apps/web/src/stores/plan-store.ts | 90 +++++++++++ 17 files changed, 932 insertions(+), 34 deletions(-) create mode 100644 apps/web/src/agent/tools/ask-user.tool.ts create mode 100644 apps/web/src/agent/tools/request-plan-approval.tool.ts create mode 100644 apps/web/src/agent/tools/submit-plan.tool.ts create mode 100644 apps/web/src/agent/tools/update-plan-step.tool.ts create mode 100644 apps/web/src/components/editor/panels/chat/ask-user-prompt.tsx create mode 100644 apps/web/src/components/editor/panels/chat/mode-transition-modal.tsx create mode 100644 apps/web/src/components/editor/panels/chat/plan-panel.tsx create mode 100644 apps/web/src/stores/plan-store.ts diff --git a/apps/web/src/agent/orchestrator.ts b/apps/web/src/agent/orchestrator.ts index dc73dab2..840e1635 100644 --- a/apps/web/src/agent/orchestrator.ts +++ b/apps/web/src/agent/orchestrator.ts @@ -33,6 +33,14 @@ const WRITE_TOOLS = new Set([ "update_keyframe_curve", ]); +const PLAN_ONLY_TOOLS = new Set([ + "submit_plan", + "ask_user", + "request_plan_approval", +]); + +const EXECUTE_ONLY_TOOLS = new Set(["update_plan_step"]); + interface APIResponse { content: string; toolCalls?: ToolCall[]; @@ -57,10 +65,16 @@ export async function run( while (iterations < MAX_ITERATIONS) { iterations++; + const liveMode = useAgentStore.getState().mode; + const liveContext: AgentContext = { ...context, mode: liveMode }; + const response = await fetch("/api/agent/chat", { method: "POST", headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ messages: workingMessages, context }), + body: JSON.stringify({ + messages: workingMessages, + context: liveContext, + }), }); if (!response.ok) { @@ -207,12 +221,43 @@ async function resolveToolCalls( ): Promise { const agentStore = useAgentStore.getState(); const results: ToolResult[] = []; + const currentMode = useAgentStore.getState().mode; for (const tc of toolCalls) { useAgentStore.getState().setActiveTool(tc.name); - const currentMode = useAgentStore.getState().permissionMode; - if (currentMode === "ask" && WRITE_TOOLS.has(tc.name)) { + if (currentMode === "plan" && WRITE_TOOLS.has(tc.name)) { + results.push({ + toolCallId: tc.id, + name: tc.name, + result: null, + error: `Cannot use '${tc.name}' in plan mode. This tool modifies the timeline. Use submit_plan to finalize your plan, then request_plan_approval to switch to execute mode.`, + }); + continue; + } + + if (currentMode === "plan" && EXECUTE_ONLY_TOOLS.has(tc.name)) { + results.push({ + toolCallId: tc.id, + name: tc.name, + result: null, + error: `Cannot use '${tc.name}' in plan mode. This tool is only available during execution.`, + }); + continue; + } + + if (currentMode === "execute" && PLAN_ONLY_TOOLS.has(tc.name)) { + results.push({ + toolCallId: tc.id, + name: tc.name, + result: null, + error: `Cannot use '${tc.name}' in execute mode. This tool is only available during planning.`, + }); + continue; + } + + const currentPermissionMode = useAgentStore.getState().permissionMode; + if (currentPermissionMode === "ask" && WRITE_TOOLS.has(tc.name)) { const approved = await requestApproval(tc); if (!approved) { results.push({ diff --git a/apps/web/src/agent/system-prompt.ts b/apps/web/src/agent/system-prompt.ts index 348fd867..5dd5449e 100644 --- a/apps/web/src/agent/system-prompt.ts +++ b/apps/web/src/agent/system-prompt.ts @@ -1,20 +1,70 @@ -import type { AgentContext } from "@/agent/types"; +import type { AgentContext, AgentMode } from "@/agent/types"; export interface ToolSummary { name: string; description: string; } -/** - * Builds the system prompt for the LLM from the editor context. - * - * Pure function — no client-only imports (no EditorCore, WASM, browser APIs). - * Consumable server-side in the API route. - */ +const PLAN_MODE_INSTRUCTIONS = ` +## CURRENT MODE: PLAN (READ-ONLY) + +You are in PLAN mode. You CANNOT make any edits to the timeline. You can only: +- Read and analyze the project (list_project_assets, list_timeline, get_element, list_effects, get_effect, list_keyframes, list_animatable_properties) +- Load media into your context (load_context) +- Discover skills (list_skills, load_skill) +- Ask the user questions (ask_user) +- Submit your plan (submit_plan) + +## YOUR WORKFLOW IN PLAN MODE: + +1. **Analyze**: Use read-only tools to understand what media and timeline state exist. Use load_context to actually see/hear the footage. +2. **Discover**: If the user's request matches a known editing pattern, call list_skills to find relevant techniques. +3. **Clarify**: If anything is ambiguous, use ask_user to get clarification before planning. +4. **Plan**: Once you understand the footage and the goal, call submit_plan with a structured step-by-step plan. + +## PLAN QUALITY RULES: + +- Each step must reference SPECIFIC tools and SPECIFIC values (timestamps, element IDs after discovery, effect params) +- Steps should be ordered by dependency (split before delete, add text before animate, etc.) +- If a skill was loaded, reference its techniques but ADAPT them to the actual footage — never copy blindly +- Include timing estimates where possible +- Be honest about limitations — if the footage doesn't support a technique, say so + +## WHEN TO SUBMIT: + +Submit the plan when: +- You have enough context about the media content +- All clarifying questions have been answered +- You have a concrete, actionable plan + +After submitting, the user will review and either approve (switching to execute mode) or request changes. +`; + +const EXECUTE_MODE_INSTRUCTIONS = ` +## CURRENT MODE: EXECUTE + +You are in EXECUTE mode. You can read AND write to the timeline. Follow the approved plan step by step. + +## EXECUTION RULES: + +1. Follow the plan steps in order. Each step lists the tools to use. +2. After completing a step, call update_plan_step to mark it done. +3. If a step fails, note the error in update_plan_step and decide whether to continue or skip. +4. If you discover the plan needs adjustment mid-execution, explain what changed and why. +5. When all steps are done, provide a summary of what was accomplished. + +## IF NO ACTIVE PLAN: + +Execute the user's request directly. For complex edits (3+ tool calls), consider using submit_plan first to get user approval. +`; + export function buildSystemPrompt( context: AgentContext, tools?: ToolSummary[], + mode?: AgentMode, ): string { + const activeMode = mode ?? context.mode ?? "execute"; + const mediaSection = context.mediaAssets.length > 0 ? `Active media assets:\n${context.mediaAssets.map((m) => `- [id: ${m.id}] ${m.name} (${m.type}, ${m.duration}s)`).join("\n")}` @@ -45,9 +95,15 @@ export function buildSystemPrompt( "Only claim edits that were actually performed by tool calls in this conversation. Do not say you added, removed, cleaned, or updated text/subtitles unless the relevant add_text, update_text, delete_timeline_elements, or update_timeline_element_timing tool call succeeded.", "When the user asks to add titles, hooks, labels, captions, subtitles, or visible text, call add_text. Do not add text proactively for unrelated edit requests such as cutting silence unless the user asks for text.", "When you need to perform an action matching one of these tools, call the appropriate tool. For all other requests, respond directly in plain text.", - "SKILLS: You have editing skills available — pre-built recipe workflows for common editing patterns. When the user asks for a complex edit (viral video, pitch, specific style, etc.), call list_skills to discover relevant skills, then call load_skill with the matching skillId to get full step-by-step instructions. Follow those instructions precisely using the standard editing tools. If the user's request doesn't match any skill, proceed with your own editing approach.", + "SKILLS: You have editing skills available — pre-built technique libraries for common editing patterns. When the user asks for a complex edit (viral video, pitch, specific style, etc.), call list_skills to discover relevant skills, then call load_skill with the matching skillId to get technique definitions. Adapt those techniques to the actual footage content. If the user's request doesn't match any skill, proceed with your own editing approach.", ); } + if (activeMode === "plan") { + parts.push(PLAN_MODE_INSTRUCTIONS); + } else { + parts.push(EXECUTE_MODE_INSTRUCTIONS); + } + return parts.join("\n"); } diff --git a/apps/web/src/agent/tools/ask-user.tool.ts b/apps/web/src/agent/tools/ask-user.tool.ts new file mode 100644 index 00000000..2746b59b --- /dev/null +++ b/apps/web/src/agent/tools/ask-user.tool.ts @@ -0,0 +1,44 @@ +import type { AgentContext, ToolDefinition } from "@/agent/types"; +import { toolRegistry } from "@/agent/tools/registry"; +import { askUserSchema } from "@/agent/tools/schemas"; +import { usePlanStore } from "@/stores/plan-store"; +import { nanoid } from "nanoid"; + +const askUserTool: ToolDefinition = { + ...askUserSchema, + execute: async ( + args: Record, + _context: AgentContext, + ): Promise<{ questionId: string; answer: string } | { error: string }> => { + const { question, options } = args as { + question: string; + options?: Array<{ label: string; description?: string }>; + }; + + if (!question || typeof question !== "string") { + return { error: "question is required and must be a string" }; + } + + const questionId = nanoid(); + const planStore = usePlanStore.getState(); + + planStore.setPendingQuestion({ + id: questionId, + question, + options: options && Array.isArray(options) ? options : undefined, + }); + + return new Promise((resolve) => { + const unsubscribe = usePlanStore.subscribe((state) => { + if (state.pendingQuestion?.answer) { + unsubscribe(); + const answer = state.pendingQuestion.answer; + usePlanStore.getState().setPendingQuestion(null); + resolve({ questionId, answer }); + } + }); + }); + }, +}; + +toolRegistry.register(askUserSchema.name, askUserTool); diff --git a/apps/web/src/agent/tools/index.ts b/apps/web/src/agent/tools/index.ts index 3cd7819b..9b6b71dc 100644 --- a/apps/web/src/agent/tools/index.ts +++ b/apps/web/src/agent/tools/index.ts @@ -38,3 +38,7 @@ import "@/agent/tools/update-keyframe-curve.tool"; import "@/agent/tools/list-animatable-properties.tool"; import "@/agent/skills/list-skills.tool"; import "@/agent/skills/load-skill.tool"; +import "@/agent/tools/submit-plan.tool"; +import "@/agent/tools/ask-user.tool"; +import "@/agent/tools/request-plan-approval.tool"; +import "@/agent/tools/update-plan-step.tool"; diff --git a/apps/web/src/agent/tools/request-plan-approval.tool.ts b/apps/web/src/agent/tools/request-plan-approval.tool.ts new file mode 100644 index 00000000..a75773d5 --- /dev/null +++ b/apps/web/src/agent/tools/request-plan-approval.tool.ts @@ -0,0 +1,46 @@ +import type { AgentContext, ToolDefinition } from "@/agent/types"; +import { toolRegistry } from "@/agent/tools/registry"; +import { requestPlanApprovalSchema } from "@/agent/tools/schemas"; +import { useAgentStore } from "@/stores/agent-store"; +import { usePlanStore } from "@/stores/plan-store"; +import type { AgentMode } from "@/agent/types"; + +const requestPlanApprovalTool: ToolDefinition = { + ...requestPlanApprovalSchema, + execute: async ( + _args: Record, + _context: AgentContext, + ): Promise<{ approved: boolean; mode: AgentMode } | { error: string }> => { + const planStore = usePlanStore.getState(); + if (!planStore.plan) { + return { + error: "No plan has been submitted yet. Call submit_plan first.", + }; + } + + const agentStore = useAgentStore.getState(); + if (agentStore.mode !== "plan") { + return { error: "Already in execute mode. No transition needed." }; + } + + planStore.updatePlanStatus("approved"); + + return new Promise((resolve) => { + useAgentStore.getState().setPendingModeTransition({ + targetMode: "execute", + resolve: (approved) => { + useAgentStore.getState().setPendingModeTransition(null); + if (approved) { + useAgentStore.getState().setMode("execute"); + } + resolve({ + approved, + mode: approved ? "execute" : "plan", + }); + }, + }); + }); + }, +}; + +toolRegistry.register(requestPlanApprovalSchema.name, requestPlanApprovalTool); diff --git a/apps/web/src/agent/tools/schemas.ts b/apps/web/src/agent/tools/schemas.ts index dbba40dd..69a4825a 100644 --- a/apps/web/src/agent/tools/schemas.ts +++ b/apps/web/src/agent/tools/schemas.ts @@ -305,6 +305,45 @@ export const loadSkillSchema: ToolSchema = { parameters: [{ key: "skillId", type: "string", required: true }], }; +export const submitPlanSchema: ToolSchema = { + name: "submit_plan", + description: + "Submits a structured editing plan with steps. Use ONLY in plan mode after analyzing the footage and understanding the user's request. Each step should describe a specific action with the tools to use. After submission, the user reviews and approves or requests changes before execution begins.", + parameters: [ + { key: "summary", type: "string", required: true }, + { key: "steps", type: "array", required: true }, + { key: "questions", type: "array", required: false }, + ], +}; + +export const askUserSchema: ToolSchema = { + name: "ask_user", + description: + "Asks the user a question during plan mode. Use when you need clarification about their intent, preferences, or content details before finalizing the plan. Optionally provide quick-reply options for common answers. The user can also type a free-form response.", + parameters: [ + { key: "question", type: "string", required: true }, + { key: "options", type: "array", required: false }, + ], +}; + +export const requestPlanApprovalSchema: ToolSchema = { + name: "request_plan_approval", + description: + "Requests user approval to switch from plan mode to execute mode. The user sees a modal with 'Continue planning' and 'Start editing' options. ALWAYS requires explicit user approval — there is no auto-approve. Use this when the plan is ready for execution.", + parameters: [], +}; + +export const updatePlanStepSchema: ToolSchema = { + name: "update_plan_step", + description: + "Updates the status of a plan step during execution. Call this after completing each step to track progress. The plan panel updates in real-time. Use status 'done' for successful completion, 'skipped' if the step was unnecessary, or 'in_progress' if starting.", + parameters: [ + { key: "stepId", type: "string", required: true }, + { key: "status", type: "string", required: true }, + { key: "result", type: "string", required: false }, + ], +}; + /** * The exact list of schemas exposed to the LLM. * Excludes internal-only tools (transcribe_video, mock). @@ -338,4 +377,8 @@ export const providerToolSchemas: ToolSchema[] = [ listAnimatablePropertiesSchema, listSkillsSchema, loadSkillSchema, + submitPlanSchema, + askUserSchema, + requestPlanApprovalSchema, + updatePlanStepSchema, ]; diff --git a/apps/web/src/agent/tools/submit-plan.tool.ts b/apps/web/src/agent/tools/submit-plan.tool.ts new file mode 100644 index 00000000..5926a5e7 --- /dev/null +++ b/apps/web/src/agent/tools/submit-plan.tool.ts @@ -0,0 +1,42 @@ +import type { AgentContext, ToolDefinition } from "@/agent/types"; +import { toolRegistry } from "@/agent/tools/registry"; +import { submitPlanSchema } from "@/agent/tools/schemas"; +import { usePlanStore, createPlanFromSteps } from "@/stores/plan-store"; + +const submitPlanTool: ToolDefinition = { + ...submitPlanSchema, + execute: async ( + args: Record, + _context: AgentContext, + ): Promise<{ planId: string; stepCount: number } | { error: string }> => { + const { summary, steps, questions } = args as { + summary: string; + steps: Array<{ description: string; tools: string[] }>; + questions?: string[]; + }; + + if (!summary || typeof summary !== "string") { + return { error: "summary is required and must be a string" }; + } + + if (!Array.isArray(steps) || steps.length === 0) { + return { error: "steps must be a non-empty array" }; + } + + for (const [i, step] of steps.entries()) { + if (!step.description || typeof step.description !== "string") { + return { error: `Step ${i + 1} must have a description string` }; + } + if (!Array.isArray(step.tools)) { + return { error: `Step ${i + 1} must have a tools array` }; + } + } + + const plan = createPlanFromSteps(summary, steps, questions); + usePlanStore.getState().setPlan(plan); + + return { planId: plan.id, stepCount: plan.steps.length }; + }, +}; + +toolRegistry.register(submitPlanSchema.name, submitPlanTool); diff --git a/apps/web/src/agent/tools/update-plan-step.tool.ts b/apps/web/src/agent/tools/update-plan-step.tool.ts new file mode 100644 index 00000000..beb837dc --- /dev/null +++ b/apps/web/src/agent/tools/update-plan-step.tool.ts @@ -0,0 +1,57 @@ +import type { AgentContext, ToolDefinition } from "@/agent/types"; +import { toolRegistry } from "@/agent/tools/registry"; +import { updatePlanStepSchema } from "@/agent/tools/schemas"; +import { usePlanStore } from "@/stores/plan-store"; +import type { PlanStepStatus } from "@/agent/types"; + +const VALID_STATUSES: PlanStepStatus[] = [ + "pending", + "in_progress", + "done", + "skipped", +]; + +const updatePlanStepTool: ToolDefinition = { + ...updatePlanStepSchema, + execute: async ( + args: Record, + _context: AgentContext, + ): + | { success: boolean; stepId: string; status: string } + | { error: string } => { + const { stepId, status, result } = args as { + stepId: string; + status: string; + result?: string; + }; + + if (!stepId || typeof stepId !== "string") { + return { error: "stepId is required" }; + } + + if (!VALID_STATUSES.includes(status as PlanStepStatus)) { + return { + error: `Invalid status: ${status}. Must be one of: ${VALID_STATUSES.join(", ")}`, + }; + } + + const planStore = usePlanStore.getState(); + const plan = planStore.plan; + if (!plan) { + return { error: "No active plan" }; + } + + const step = plan.steps.find((s) => s.id === stepId); + if (!step) { + return { + error: `Step not found: ${stepId}. Available: ${plan.steps.map((s) => s.id).join(", ")}`, + }; + } + + planStore.updateStepStatus(stepId, status as PlanStepStatus, result); + + return { success: true, stepId, status }; + }, +}; + +toolRegistry.register(updatePlanStepSchema.name, updatePlanStepTool); diff --git a/apps/web/src/agent/types.ts b/apps/web/src/agent/types.ts index 3c4fe233..35221793 100644 --- a/apps/web/src/agent/types.ts +++ b/apps/web/src/agent/types.ts @@ -5,12 +5,45 @@ export type ExecutionState = | "responding" | "error"; +export type AgentMode = "plan" | "execute"; + +export type PlanStepStatus = "pending" | "in_progress" | "done" | "skipped"; +export type PlanStatus = + | "drafting" + | "awaiting_approval" + | "approved" + | "executing" + | "completed"; + +export interface PlanStep { + id: string; + description: string; + tools: string[]; + status: PlanStepStatus; + result?: string; +} + +export interface Plan { + id: string; + summary: string; + steps: PlanStep[]; + questions?: string[]; + status: PlanStatus; + createdAt: number; +} + +export interface AgentQuestion { + id: string; + question: string; + options?: Array<{ label: string; description?: string }>; + answer?: string; +} + export interface ChatMessage { id: string; role: "system" | "user" | "assistant" | "tool_result"; content: string; toolCalls?: ToolCall[]; - /** Associates a tool_result message with the ToolCall it answers. */ toolCallId?: string; timestamp: number; } @@ -41,6 +74,7 @@ export interface AgentContext { }>; timelineTracks?: AgentTimelineTrack[]; playbackTimeMs: number; + mode?: AgentMode; } export type AgentTimelineTrack = { diff --git a/apps/web/src/app/api/agent/chat/route.ts b/apps/web/src/app/api/agent/chat/route.ts index 1e906922..5291fd55 100644 --- a/apps/web/src/app/api/agent/chat/route.ts +++ b/apps/web/src/app/api/agent/chat/route.ts @@ -63,6 +63,7 @@ const chatRequestSchema = z.object({ ) .optional(), playbackTimeMs: z.number(), + mode: z.enum(["plan", "execute"]).optional(), }) satisfies z.ZodType, }); @@ -89,7 +90,8 @@ export async function POST(request: NextRequest) { const { messages, context } = result.data; - // Resolve provider config from LLM_* env vars + const mode = context.mode; + const provider = process.env.LLM_PROVIDER; const apiKey = process.env.LLM_API_KEY; const model = process.env.LLM_MODEL; @@ -107,8 +109,7 @@ export async function POST(request: NextRequest) { const config: ProviderConfig = { provider, apiKey, model, baseUrl }; - // Build system prompt with tool guidance - const systemPrompt = buildSystemPrompt(context, providerToolSchemas); + const systemPrompt = buildSystemPrompt(context, providerToolSchemas, mode); // Delegate to provider adapter try { diff --git a/apps/web/src/components/editor/panels/chat/ask-user-prompt.tsx b/apps/web/src/components/editor/panels/chat/ask-user-prompt.tsx new file mode 100644 index 00000000..2a8cb2fb --- /dev/null +++ b/apps/web/src/components/editor/panels/chat/ask-user-prompt.tsx @@ -0,0 +1,93 @@ +"use client"; + +import { useState, useCallback } from "react"; +import { HugeiconsIcon } from "@hugeicons/react"; +import { QuestionExchangeIcon } from "@hugeicons/core-free-icons"; +import { Button } from "@/components/ui/button"; +import { usePlanStore } from "@/stores/plan-store"; + +export function AskUserPrompt() { + const pendingQuestion = usePlanStore((s) => s.pendingQuestion); + const answerQuestion = usePlanStore((s) => s.answerQuestion); + const [freeForm, setFreeForm] = useState(""); + + const handleOptionClick = useCallback( + (label: string) => { + answerQuestion(label); + setFreeForm(""); + }, + [answerQuestion], + ); + + const handleFreeFormSubmit = useCallback(() => { + const trimmed = freeForm.trim(); + if (!trimmed) return; + answerQuestion(trimmed); + setFreeForm(""); + }, [freeForm, answerQuestion]); + + const handleKeyDown = useCallback( + (e: React.KeyboardEvent) => { + if (e.key === "Enter" && !e.shiftKey) { + e.preventDefault(); + handleFreeFormSubmit(); + } + }, + [handleFreeFormSubmit], + ); + + if (!pendingQuestion || pendingQuestion.answer) return null; + + return ( +
+
+ + + Agent question + +
+

{pendingQuestion.question}

+ + {pendingQuestion.options && pendingQuestion.options.length > 0 && ( +
+ {pendingQuestion.options.map((opt) => ( + + ))} +
+ )} + +
+ setFreeForm(e.target.value)} + onKeyDown={handleKeyDown} + placeholder="Type your answer..." + className="border-border bg-input flex-1 rounded-md border px-2 py-1 text-xs outline-none focus:border-blue-500/50" + /> + +
+
+ ); +} diff --git a/apps/web/src/components/editor/panels/chat/chat-input.tsx b/apps/web/src/components/editor/panels/chat/chat-input.tsx index f7218e3b..9213ae56 100644 --- a/apps/web/src/components/editor/panels/chat/chat-input.tsx +++ b/apps/web/src/components/editor/panels/chat/chat-input.tsx @@ -6,11 +6,15 @@ import { Sent02Icon, FlashIcon, Shield01Icon, + PlanChartIcon, + Edit02Icon, } from "@hugeicons/core-free-icons"; import { Button } from "@/components/ui/button"; import { Spinner } from "@/components/ui/spinner"; import { useAgentStore, type PermissionMode } from "@/stores/agent-store"; +import { usePlanStore } from "@/stores/plan-store"; import { cn } from "@/utils/ui"; +import type { AgentMode } from "@/agent/types"; interface ChatInputProps { onSend: (content: string) => void; @@ -22,31 +26,51 @@ export function ChatInput({ onSend, disabled }: ChatInputProps) { const permissionMode = useAgentStore((s) => s.permissionMode); const setPermissionMode = useAgentStore((s) => s.setPermissionMode); const pendingApproval = useAgentStore((s) => s.pendingApproval); + const mode = useAgentStore((s) => s.mode); + const setMode = useAgentStore((s) => s.setMode); + const pendingTransition = useAgentStore((s) => s.pendingModeTransition); + const plan = usePlanStore((s) => s.plan); - const handleSend = useCallback(() => { - const trimmed = value.trim(); - if (!trimmed || disabled) return; - onSend(trimmed); - setValue(""); - }, [value, disabled, onSend]); + const handleSend = useCallback( + (forcedMode?: AgentMode) => { + const trimmed = value.trim(); + if (!trimmed || disabled) return; + const targetMode = forcedMode ?? mode; + if (targetMode !== mode) { + setMode(targetMode); + } + onSend(trimmed); + setValue(""); + }, + [value, disabled, onSend, mode, setMode], + ); const handleKeyDown = useCallback( (e: KeyboardEvent) => { if (e.key === "Enter" && !e.shiftKey) { e.preventDefault(); - handleSend(); + handleSend("execute"); + } + if (e.key === "Enter" && e.shiftKey) { + e.preventDefault(); + handleSend("plan"); } }, [handleSend], ); const toggleMode = useCallback(() => { - const next: PermissionMode = - permissionMode === "skip" ? "ask" : "skip"; + const next: AgentMode = mode === "execute" ? "plan" : "execute"; + setMode(next); + }, [mode, setMode]); + + const togglePermission = useCallback(() => { + const next: PermissionMode = permissionMode === "skip" ? "ask" : "skip"; setPermissionMode(next); }, [permissionMode, setPermissionMode]); const isAsk = permissionMode === "ask"; + const isPlanMode = mode === "plan"; return (
@@ -55,16 +79,38 @@ export function ChatInput({ onSend, disabled }: ChatInputProps) { value={value} onChange={(e) => setValue(e.target.value)} onKeyDown={handleKeyDown} - disabled={disabled} - placeholder="Ask the assistant..." + disabled={disabled || !!pendingTransition} + placeholder={ + isPlanMode + ? "Describe what you want (plan mode)..." + : "Ask the assistant..." + } rows={1} className="border-border bg-input focus-visible:border-primary/50 focus-visible:ring-primary/20 flex-1 resize-none rounded-md border px-3 py-2 text-sm outline-none focus-visible:ring-2 disabled:cursor-not-allowed disabled:opacity-50" /> +
+
+ + Enter = edit mode |{" "} + Shift+Enter = plan mode + + {plan && ( + + + {isPlanMode ? "PLANNING" : "EXECUTING"} + + + )} +
); } diff --git a/apps/web/src/components/editor/panels/chat/index.tsx b/apps/web/src/components/editor/panels/chat/index.tsx index e9164d97..ee74c0a9 100644 --- a/apps/web/src/components/editor/panels/chat/index.tsx +++ b/apps/web/src/components/editor/panels/chat/index.tsx @@ -8,9 +8,13 @@ import { Button } from "@/components/ui/button"; import { Spinner } from "@/components/ui/spinner"; import { useChatStore } from "@/stores/chat-store"; import { useAgentStore } from "@/stores/agent-store"; +import { usePlanStore } from "@/stores/plan-store"; import { MessageBubble } from "./message-bubble"; import { ChatInput } from "./chat-input"; import { ToolPermissionRequest } from "./tool-permission-request"; +import { PlanPanel } from "./plan-panel"; +import { ModeTransitionModal } from "./mode-transition-modal"; +import { AskUserPrompt } from "./ask-user-prompt"; import { run as orchestratorRun } from "@/agent/orchestrator"; import { EditorContextAdapter } from "@/agent/context"; @@ -21,16 +25,16 @@ export function ChatPanel() { const sendMessage = useChatStore((s) => s.sendMessage); const setError = useChatStore((s) => s.setError); const pendingApproval = useAgentStore((s) => s.pendingApproval); + const pendingTransition = useAgentStore((s) => s.pendingModeTransition); + const plan = usePlanStore((s) => s.plan); const scrollRef = useRef(null); - // Auto-scroll on new messages or loading state change - // biome-ignore lint/correctness/useExhaustiveDependencies: re-scroll when messages or loading change useEffect(() => { if (scrollRef.current) { scrollRef.current.scrollTop = scrollRef.current.scrollHeight; } - }, [messages, loading]); + }); const handleSend = useCallback( async (content: string) => { @@ -52,6 +56,12 @@ export function ChatPanel() { return (
+ {plan && ( +
+ +
+ )} + {messages.length === 0 && !loading && !error ? (
)} + {pendingApproval && ( )} @@ -110,6 +121,8 @@ export function ChatPanel() { )} + + {pendingTransition && }
); } diff --git a/apps/web/src/components/editor/panels/chat/mode-transition-modal.tsx b/apps/web/src/components/editor/panels/chat/mode-transition-modal.tsx new file mode 100644 index 00000000..b5c74ab7 --- /dev/null +++ b/apps/web/src/components/editor/panels/chat/mode-transition-modal.tsx @@ -0,0 +1,105 @@ +"use client"; + +import { useCallback } from "react"; +import { HugeiconsIcon } from "@hugeicons/react"; +import { PlanChartIcon, Edit02Icon } from "@hugeicons/core-free-icons"; +import { Button } from "@/components/ui/button"; +import { useAgentStore } from "@/stores/agent-store"; +import { usePlanStore } from "@/stores/plan-store"; +import { cn } from "@/utils/ui"; + +export function ModeTransitionModal() { + const pendingTransition = useAgentStore((s) => s.pendingModeTransition); + const plan = usePlanStore((s) => s.plan); + + const handleContinuePlanning = useCallback(() => { + if (!pendingTransition) return; + pendingTransition.resolve(false); + }, [pendingTransition]); + + const handleStartEditing = useCallback(() => { + if (!pendingTransition) return; + pendingTransition.resolve(true); + }, [pendingTransition]); + + if (!pendingTransition) return null; + + const doneCount = plan?.steps.filter((s) => s.status === "done").length ?? 0; + const totalCount = plan?.steps.length ?? 0; + + return ( +
+
+
+
+

Plan ready for execution

+ {plan && ( +

+ {plan.summary} — {totalCount} step{totalCount !== 1 ? "s" : ""} + {doneCount > 0 && ` (${doneCount} already done)`} +

+ )} +
+ + {plan && ( +
+
    + {plan.steps.map((step) => ( +
  • + + + {step.description} + +
    • + ))} +
+
+ )} + +
+ + +
+
+
+
+ ); +} diff --git a/apps/web/src/components/editor/panels/chat/plan-panel.tsx b/apps/web/src/components/editor/panels/chat/plan-panel.tsx new file mode 100644 index 00000000..f8f11e42 --- /dev/null +++ b/apps/web/src/components/editor/panels/chat/plan-panel.tsx @@ -0,0 +1,143 @@ +"use client"; + +import { HugeiconsIcon } from "@hugeicons/react"; +import { + CheckmarkCircle02Icon, + CancelCircleIcon, + Loading03Icon, + MinusSign01Icon, + PlanChartIcon, +} from "@hugeicons/core-free-icons"; +import { ScrollArea } from "@/components/ui/scroll-area"; +import { usePlanStore } from "@/stores/plan-store"; +import { useAgentStore } from "@/stores/agent-store"; +import { cn } from "@/utils/ui"; +import type { PlanStep, PlanStepStatus } from "@/agent/types"; + +const STATUS_CONFIG: Record< + PlanStepStatus, + { icon: typeof CheckmarkCircle02Icon; className: string } +> = { + pending: { + icon: MinusSign01Icon, + className: "text-muted-foreground", + }, + in_progress: { + icon: Loading03Icon, + className: "text-blue-400 animate-spin", + }, + done: { + icon: CheckmarkCircle02Icon, + className: "text-emerald-400", + }, + skipped: { + icon: CancelCircleIcon, + className: "text-muted-foreground/50", + }, +}; + +function StepItem({ step }: { step: PlanStep }) { + const config = STATUS_CONFIG[step.status]; + + return ( +
+ +
+ + {step.description} + + {step.result && ( + + {step.result} + + )} +
+
+ ); +} + +export function PlanPanel() { + const plan = usePlanStore((s) => s.plan); + const mode = useAgentStore((s) => s.mode); + + if (!plan) { + return ( +
+ +

+ {mode === "plan" + ? "Analyzing footage to build a plan..." + : "No active plan"} +

+
+ ); + } + + const doneCount = plan.steps.filter((s) => s.status === "done").length; + const totalCount = plan.steps.length; + const progress = totalCount > 0 ? (doneCount / totalCount) * 100 : 0; + + return ( +
+
+
+ Plan + + {plan.summary} + +
+
+
+
+
+ + {doneCount}/{totalCount} + +
+
+ + +
+ {plan.steps.map((step) => ( + + ))} +
+ + + {plan.questions && plan.questions.length > 0 && ( +
+ + Open questions + +
    + {plan.questions.map((q) => ( +
  • + {q} +
    • + ))} +
+
+ )} +
+ ); +} diff --git a/apps/web/src/stores/agent-store.ts b/apps/web/src/stores/agent-store.ts index 6367ce80..91d65844 100644 --- a/apps/web/src/stores/agent-store.ts +++ b/apps/web/src/stores/agent-store.ts @@ -1,5 +1,10 @@ import { create } from "zustand"; -import type { AgentContext, ExecutionState, ToolCall } from "@/agent/types"; +import type { + AgentContext, + AgentMode, + ExecutionState, + ToolCall, +} from "@/agent/types"; export type PermissionMode = "ask" | "skip"; @@ -8,6 +13,11 @@ export interface PendingToolApproval { resolve: (approved: boolean) => void; } +export interface PendingModeTransition { + targetMode: AgentMode; + resolve: (approved: boolean) => void; +} + const DEFAULT_CONTEXT: AgentContext = { projectId: null, activeSceneId: null, @@ -21,11 +31,15 @@ interface AgentState { context: AgentContext; permissionMode: PermissionMode; pendingApproval: PendingToolApproval | null; + mode: AgentMode; + pendingModeTransition: PendingModeTransition | null; setStatus: (status: ExecutionState) => void; setActiveTool: (tool: string | null) => void; setContext: (context: AgentContext) => void; setPermissionMode: (mode: PermissionMode) => void; setPendingApproval: (pending: PendingToolApproval | null) => void; + setMode: (mode: AgentMode) => void; + setPendingModeTransition: (transition: PendingModeTransition | null) => void; reset: () => void; } @@ -35,16 +49,23 @@ export const useAgentStore = create()((set) => ({ context: DEFAULT_CONTEXT, permissionMode: "skip", pendingApproval: null, + mode: "execute", + pendingModeTransition: null, setStatus: (status) => set({ status }), setActiveTool: (tool) => set({ activeTool: tool }), setContext: (context) => set({ context }), setPermissionMode: (mode) => set({ permissionMode: mode }), setPendingApproval: (pending) => set({ pendingApproval: pending }), + setMode: (mode) => set({ mode }), + setPendingModeTransition: (transition) => + set({ pendingModeTransition: transition }), reset: () => set({ status: "idle", activeTool: null, context: DEFAULT_CONTEXT, pendingApproval: null, + mode: "execute", + pendingModeTransition: null, }), })); diff --git a/apps/web/src/stores/plan-store.ts b/apps/web/src/stores/plan-store.ts new file mode 100644 index 00000000..4f0e0577 --- /dev/null +++ b/apps/web/src/stores/plan-store.ts @@ -0,0 +1,90 @@ +import { create } from "zustand"; +import { nanoid } from "nanoid"; +import type { + AgentQuestion, + Plan, + PlanStepStatus, + PlanStatus, +} from "@/agent/types"; + +interface PlanState { + plan: Plan | null; + pendingQuestion: AgentQuestion | null; + modeTransitionPending: boolean; + + setPlan: (plan: Plan) => void; + updatePlanStatus: (status: PlanStatus) => void; + updateStepStatus: ( + stepId: string, + status: PlanStepStatus, + result?: string, + ) => void; + clearPlan: () => void; + + setPendingQuestion: (question: AgentQuestion | null) => void; + answerQuestion: (answer: string) => void; + + setModeTransitionPending: (pending: boolean) => void; +} + +export const usePlanStore = create()((set, get) => ({ + plan: null, + pendingQuestion: null, + modeTransitionPending: false, + + setPlan: (plan) => set({ plan }), + + updatePlanStatus: (status) => { + const current = get().plan; + if (!current) return; + set({ plan: { ...current, status } }); + }, + + updateStepStatus: (stepId, status, result) => { + const current = get().plan; + if (!current) return; + set({ + plan: { + ...current, + steps: current.steps.map((s) => + s.id === stepId + ? { ...s, status, ...(result !== undefined && { result }) } + : s, + ), + }, + }); + }, + + clearPlan: () => set({ plan: null, pendingQuestion: null }), + + setPendingQuestion: (question) => set({ pendingQuestion: question }), + + answerQuestion: (answer) => { + const current = get().pendingQuestion; + if (!current) return; + set({ pendingQuestion: { ...current, answer } }); + }, + + setModeTransitionPending: (pending) => + set({ modeTransitionPending: pending }), +})); + +export function createPlanFromSteps( + summary: string, + steps: Array<{ description: string; tools: string[] }>, + questions?: string[], +): Plan { + return { + id: nanoid(), + summary, + steps: steps.map((s) => ({ + id: nanoid(), + description: s.description, + tools: s.tools, + status: "pending" as PlanStepStatus, + })), + questions, + status: "awaiting_approval" as PlanStatus, + createdAt: Date.now(), + }; +} From c8a8e49dc784b98a821ba49eec64e78188166ef7 Mon Sep 17 00:00:00 2001 From: Luis Esteban Acevedo Ladino Date: Fri, 1 May 2026 17:17:23 -0500 Subject: [PATCH 2/6] fix: use correct MinusSignIcon export from hugeicons --- apps/web/src/components/editor/panels/chat/plan-panel.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/web/src/components/editor/panels/chat/plan-panel.tsx b/apps/web/src/components/editor/panels/chat/plan-panel.tsx index f8f11e42..37ec6f6d 100644 --- a/apps/web/src/components/editor/panels/chat/plan-panel.tsx +++ b/apps/web/src/components/editor/panels/chat/plan-panel.tsx @@ -5,7 +5,7 @@ import { CheckmarkCircle02Icon, CancelCircleIcon, Loading03Icon, - MinusSign01Icon, + MinusSignIcon, PlanChartIcon, } from "@hugeicons/core-free-icons"; import { ScrollArea } from "@/components/ui/scroll-area"; @@ -19,7 +19,7 @@ const STATUS_CONFIG: Record< { icon: typeof CheckmarkCircle02Icon; className: string } > = { pending: { - icon: MinusSign01Icon, + icon: MinusSignIcon, className: "text-muted-foreground", }, in_progress: { From 786501d226fceeb30e9c80c0fecae673894d72d8 Mon Sep 17 00:00:00 2001 From: Luis Esteban Acevedo Ladino Date: Fri, 1 May 2026 17:20:07 -0500 Subject: [PATCH 3/6] fix: use correct hugeicons exports (CheckListIcon, QuestionIcon) --- .../web/src/components/editor/panels/chat/ask-user-prompt.tsx | 4 ++-- apps/web/src/components/editor/panels/chat/chat-input.tsx | 4 ++-- .../components/editor/panels/chat/mode-transition-modal.tsx | 4 ++-- apps/web/src/components/editor/panels/chat/plan-panel.tsx | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/apps/web/src/components/editor/panels/chat/ask-user-prompt.tsx b/apps/web/src/components/editor/panels/chat/ask-user-prompt.tsx index 2a8cb2fb..6fb7b0c8 100644 --- a/apps/web/src/components/editor/panels/chat/ask-user-prompt.tsx +++ b/apps/web/src/components/editor/panels/chat/ask-user-prompt.tsx @@ -2,7 +2,7 @@ import { useState, useCallback } from "react"; import { HugeiconsIcon } from "@hugeicons/react"; -import { QuestionExchangeIcon } from "@hugeicons/core-free-icons"; +import { QuestionIcon } from "@hugeicons/core-free-icons"; import { Button } from "@/components/ui/button"; import { usePlanStore } from "@/stores/plan-store"; @@ -42,7 +42,7 @@ export function AskUserPrompt() {
diff --git a/apps/web/src/components/editor/panels/chat/chat-input.tsx b/apps/web/src/components/editor/panels/chat/chat-input.tsx index 9213ae56..f18a2975 100644 --- a/apps/web/src/components/editor/panels/chat/chat-input.tsx +++ b/apps/web/src/components/editor/panels/chat/chat-input.tsx @@ -6,7 +6,7 @@ import { Sent02Icon, FlashIcon, Shield01Icon, - PlanChartIcon, + CheckListIcon, Edit02Icon, } from "@hugeicons/core-free-icons"; import { Button } from "@/components/ui/button"; @@ -101,7 +101,7 @@ export function ChatInput({ onSend, disabled }: ChatInputProps) { } > diff --git a/apps/web/src/components/editor/panels/chat/mode-transition-modal.tsx b/apps/web/src/components/editor/panels/chat/mode-transition-modal.tsx index b5c74ab7..9289938b 100644 --- a/apps/web/src/components/editor/panels/chat/mode-transition-modal.tsx +++ b/apps/web/src/components/editor/panels/chat/mode-transition-modal.tsx @@ -2,7 +2,7 @@ import { useCallback } from "react"; import { HugeiconsIcon } from "@hugeicons/react"; -import { PlanChartIcon, Edit02Icon } from "@hugeicons/core-free-icons"; +import { CheckListIcon, Edit02Icon } from "@hugeicons/core-free-icons"; import { Button } from "@/components/ui/button"; import { useAgentStore } from "@/stores/agent-store"; import { usePlanStore } from "@/stores/plan-store"; @@ -91,7 +91,7 @@ export function ModeTransitionModal() { className="w-full" > diff --git a/apps/web/src/components/editor/panels/chat/plan-panel.tsx b/apps/web/src/components/editor/panels/chat/plan-panel.tsx index 37ec6f6d..e5046e66 100644 --- a/apps/web/src/components/editor/panels/chat/plan-panel.tsx +++ b/apps/web/src/components/editor/panels/chat/plan-panel.tsx @@ -6,7 +6,7 @@ import { CancelCircleIcon, Loading03Icon, MinusSignIcon, - PlanChartIcon, + CheckListIcon, } from "@hugeicons/core-free-icons"; import { ScrollArea } from "@/components/ui/scroll-area"; import { usePlanStore } from "@/stores/plan-store"; @@ -77,7 +77,7 @@ export function PlanPanel() { return (
From 6d313cfdc4a8dc55785e349c4bef2c96356c8315 Mon Sep 17 00:00:00 2001 From: Luis Esteban Acevedo Ladino Date: Fri, 1 May 2026 17:43:02 -0500 Subject: [PATCH 4/6] docs: update agent tools inventory with skills and plan mode tools --- docs/agent-tools-inventory.md | 36 ++++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/docs/agent-tools-inventory.md b/docs/agent-tools-inventory.md index 8bfda5f9..04c4ece1 100644 --- a/docs/agent-tools-inventory.md +++ b/docs/agent-tools-inventory.md @@ -1,6 +1,6 @@ # NeuralCut — Agent Tools Inventory -Complete inventory of all tools, effects, masks, and animatable properties available to the video editor agent. +Complete inventory of all tools, skills, effects, masks, and animatable properties available to the video editor agent. **Source of truth**: `apps/web/src/agent/tools/schemas.ts` — the `providerToolSchemas` array defines what gets sent to the LLM. **Execution**: `apps/web/src/agent/context.ts` (EditorContextAdapter). @@ -8,7 +8,7 @@ Complete inventory of all tools, effects, masks, and animatable properties avail --- -## 25 Provider-Facing Tools +## 31 Provider-Facing Tools ### Context & Perception (read-only) @@ -74,6 +74,23 @@ Complete inventory of all tools, effects, masks, and animatable properties avail | Tool | Description | Schema | Implementation | |------|-------------|--------|----------------| | `undo` | Undoes the last editing action. Returns remaining undo depth. | `schemas.ts:195` | `undo.tool.ts` | +| `redo` | Redoes the last undone action. Only works after an undo. Returns whether there are more actions to redo. | `schemas.ts:174` | `redo.tool.ts` | + +### Skills (recipe-based editing) + +| Tool | Description | Schema | Implementation | +|------|-------------|--------|----------------| +| `list_skills` | Lists available editing skill recipes — pre-built workflows for common patterns like viral shorts, pitch videos. Returns skill id, name, and description. | `schemas.ts:294` | `skills/list-skills.tool.ts` | +| `load_skill` | Loads full instructions for a specific skill by id. Contains recipe with sections, timing rules, text styles, effect parameters, and quality checklist. | `schemas.ts:301` | `skills/load-skill.tool.ts` | + +### Plan Mode (structured workflow) + +| Tool | Description | Schema | Implementation | +|------|-------------|--------|----------------| +| `submit_plan` | Submits a structured editing plan with steps. Use ONLY in plan mode after analyzing footage. Each step describes a specific action with tools to use. User reviews and approves before execution. | `schemas.ts:308` | `submit-plan.tool.ts` | +| `ask_user` | Asks the user a question during plan mode. Use when clarification is needed about intent, preferences, or content details. Optionally provides quick-reply options. | `schemas.ts:319` | `ask-user.tool.ts` | +| `request_plan_approval` | Requests user approval to switch from plan mode to execute mode. Shows modal with 'Continue planning' and 'Start editing' options. Requires explicit user approval. | `schemas.ts:329` | `request-plan-approval.tool.ts` | +| `update_plan_step` | Updates the status of a plan step during execution. Call after completing each step to track progress. Use status 'done', 'skipped', or 'in_progress'. | `schemas.ts:336` | `update-plan-step.tool.ts` | --- @@ -84,7 +101,16 @@ Complete inventory of all tools, effects, masks, and animatable properties avail | `transcribe_video` | Transcribes audio using Whisper (local). Returns structured transcript with segments/timestamps. Registered in toolRegistry but excluded from `providerToolSchemas` (`schemas.ts:296-297`). Used internally by the agent orchestrator. | `transcribe-video.tool.ts` | | `echo_context` | Debug tool that returns a summary of the current editor context. Registered in toolRegistry but not in `providerToolSchemas`. | `mock.tool.ts` | -> **Note**: `redo` has a schema (`schemas.ts:174`) and implementation (`redo.tool.ts`) but is **NOT** included in `providerToolSchemas`. +--- + +## 2 Built-in Skills + +Defined in `apps/web/src/agent/skills/builtin/`. + +| Skill ID | Name | Description | +|----------|------|-------------| +| `viral-short` | Viral Short | Creates a short-form viral video with hook, clips, text overlays, and effects | +| `pitch-video` | Pitch Video | Creates a professional pitch video with intro, key points, transitions, and outro | --- @@ -200,6 +226,8 @@ Dynamic path types for extension: | `apps/web/src/agent/tools/schemas.ts` | Single source of truth for all tool schemas + `providerToolSchemas` array | | `apps/web/src/agent/tools/index.ts` | Barrel that registers all tools via side-effect imports | | `apps/web/src/agent/tools/registry.ts` | Generic `DefinitionRegistry` | +| `apps/web/src/agent/skills/index.ts` | Barrel that registers all skills via side-effect imports | +| `apps/web/src/agent/skills/registry.ts` | Skill definition registry | | `apps/web/src/agent/orchestrator.ts` | Agent loop: sends to LLM, resolves tool calls, up to 20 iterations | | `apps/web/src/agent/context.ts` | `EditorContextAdapter` — the ONLY file that imports EditorCore. All tool execution logic. | | `apps/web/src/agent/system-prompt.ts` | Builds system prompt from context + tool summaries | @@ -214,3 +242,5 @@ Dynamic path types for extension: 4. **Agent must call `list_project_assets` and `list_timeline` before editing** when it lacks concrete IDs. 5. **Gemini handles multimodal understanding**; editing is deterministic via tools. 6. **`split` does NOT delete** — compose `split` + `delete_timeline_elements` for range removal. +7. **Plan mode** enables structured workflow: submit_plan → request_plan_approval → update_plan_step for execution tracking. +8. **Skills** provide pre-built recipes for common editing patterns, composable with standard tools. \ No newline at end of file From 3e912159e1bb72ecc5e5d4a066d7b3f195eeccd3 Mon Sep 17 00:00:00 2001 From: Luis Esteban Acevedo Ladino Date: Fri, 1 May 2026 17:53:57 -0500 Subject: [PATCH 5/6] fix: require plan approval before complex edits --- apps/web/src/agent/orchestrator.ts | 34 +++++----- apps/web/src/agent/system-prompt.ts | 18 +++++- .../agent/tools/request-plan-approval.tool.ts | 12 ++-- apps/web/src/agent/tools/schemas.ts | 6 +- apps/web/src/agent/tools/submit-plan.tool.ts | 63 ++++++++++++++++--- .../panels/chat/mode-transition-modal.tsx | 4 +- 6 files changed, 99 insertions(+), 38 deletions(-) diff --git a/apps/web/src/agent/orchestrator.ts b/apps/web/src/agent/orchestrator.ts index 840e1635..df4a800d 100644 --- a/apps/web/src/agent/orchestrator.ts +++ b/apps/web/src/agent/orchestrator.ts @@ -9,6 +9,7 @@ import { toolRegistry } from "@/agent/tools/registry"; import "@/agent/tools"; import { useChatStore } from "@/stores/chat-store"; import { useAgentStore } from "@/stores/agent-store"; +import { usePlanStore } from "@/stores/plan-store"; const MAX_ITERATIONS = 20; @@ -33,12 +34,6 @@ const WRITE_TOOLS = new Set([ "update_keyframe_curve", ]); -const PLAN_ONLY_TOOLS = new Set([ - "submit_plan", - "ask_user", - "request_plan_approval", -]); - const EXECUTE_ONLY_TOOLS = new Set(["update_plan_step"]); interface APIResponse { @@ -221,10 +216,25 @@ async function resolveToolCalls( ): Promise { const agentStore = useAgentStore.getState(); const results: ToolResult[] = []; - const currentMode = useAgentStore.getState().mode; for (const tc of toolCalls) { useAgentStore.getState().setActiveTool(tc.name); + const currentMode = useAgentStore.getState().mode; + const currentPlan = usePlanStore.getState().plan; + + if ( + WRITE_TOOLS.has(tc.name) && + currentPlan?.status === "awaiting_approval" + ) { + results.push({ + toolCallId: tc.id, + name: tc.name, + result: null, + error: + "Cannot edit while a plan is awaiting user approval. Call request_plan_approval and wait for the user to choose Go edit.", + }); + continue; + } if (currentMode === "plan" && WRITE_TOOLS.has(tc.name)) { results.push({ @@ -246,16 +256,6 @@ async function resolveToolCalls( continue; } - if (currentMode === "execute" && PLAN_ONLY_TOOLS.has(tc.name)) { - results.push({ - toolCallId: tc.id, - name: tc.name, - result: null, - error: `Cannot use '${tc.name}' in execute mode. This tool is only available during planning.`, - }); - continue; - } - const currentPermissionMode = useAgentStore.getState().permissionMode; if (currentPermissionMode === "ask" && WRITE_TOOLS.has(tc.name)) { const approved = await requestApproval(tc); diff --git a/apps/web/src/agent/system-prompt.ts b/apps/web/src/agent/system-prompt.ts index 5dd5449e..88f22752 100644 --- a/apps/web/src/agent/system-prompt.ts +++ b/apps/web/src/agent/system-prompt.ts @@ -21,6 +21,7 @@ You are in PLAN mode. You CANNOT make any edits to the timeline. You can only: 2. **Discover**: If the user's request matches a known editing pattern, call list_skills to find relevant techniques. 3. **Clarify**: If anything is ambiguous, use ask_user to get clarification before planning. 4. **Plan**: Once you understand the footage and the goal, call submit_plan with a structured step-by-step plan. +5. **Approval**: Immediately after submit_plan succeeds, call request_plan_approval. Do NOT edit until the user approves. ## PLAN QUALITY RULES: @@ -37,13 +38,24 @@ Submit the plan when: - All clarifying questions have been answered - You have a concrete, actionable plan -After submitting, the user will review and either approve (switching to execute mode) or request changes. +After submitting, you MUST ask for approval through request_plan_approval. The user will choose Keep planning or Go edit. `; const EXECUTE_MODE_INSTRUCTIONS = ` ## CURRENT MODE: EXECUTE -You are in EXECUTE mode. You can read AND write to the timeline. Follow the approved plan step by step. +You are in EXECUTE mode. You can read AND write to the timeline, but complex editing requests still require a plan first. + +## MANDATORY PLANNING GATE + +For any complex editing request that would require multiple timeline edits (cuts, moving media, adding titles/effects, or touching 3+ tools), you MUST: + +1. Analyze the project with read-only tools. +2. Call submit_plan with a concise structured checklist. +3. Call request_plan_approval. +4. Wait for the user to choose Go edit. + +Do NOT perform write tools for complex edits before approval. This is required even if the current mode is EXECUTE. ## EXECUTION RULES: @@ -55,7 +67,7 @@ You are in EXECUTE mode. You can read AND write to the timeline. Follow the appr ## IF NO ACTIVE PLAN: -Execute the user's request directly. For complex edits (3+ tool calls), consider using submit_plan first to get user approval. +Simple one-shot edits may execute directly. Complex edits must go through submit_plan + request_plan_approval first. `; export function buildSystemPrompt( diff --git a/apps/web/src/agent/tools/request-plan-approval.tool.ts b/apps/web/src/agent/tools/request-plan-approval.tool.ts index a75773d5..e00b415c 100644 --- a/apps/web/src/agent/tools/request-plan-approval.tool.ts +++ b/apps/web/src/agent/tools/request-plan-approval.tool.ts @@ -18,12 +18,8 @@ const requestPlanApprovalTool: ToolDefinition = { }; } - const agentStore = useAgentStore.getState(); - if (agentStore.mode !== "plan") { - return { error: "Already in execute mode. No transition needed." }; - } - - planStore.updatePlanStatus("approved"); + useAgentStore.getState().setMode("plan"); + planStore.updatePlanStatus("awaiting_approval"); return new Promise((resolve) => { useAgentStore.getState().setPendingModeTransition({ @@ -32,6 +28,10 @@ const requestPlanApprovalTool: ToolDefinition = { useAgentStore.getState().setPendingModeTransition(null); if (approved) { useAgentStore.getState().setMode("execute"); + usePlanStore.getState().updatePlanStatus("executing"); + } else { + useAgentStore.getState().setMode("plan"); + usePlanStore.getState().updatePlanStatus("awaiting_approval"); } resolve({ approved, diff --git a/apps/web/src/agent/tools/schemas.ts b/apps/web/src/agent/tools/schemas.ts index 69a4825a..175579c9 100644 --- a/apps/web/src/agent/tools/schemas.ts +++ b/apps/web/src/agent/tools/schemas.ts @@ -308,7 +308,7 @@ export const loadSkillSchema: ToolSchema = { export const submitPlanSchema: ToolSchema = { name: "submit_plan", description: - "Submits a structured editing plan with steps. Use ONLY in plan mode after analyzing the footage and understanding the user's request. Each step should describe a specific action with the tools to use. After submission, the user reviews and approves or requests changes before execution begins.", + "Submits a structured editing plan with steps. Use this before complex edits, even from execute mode. Each step should describe a specific action with the tools to use. After submission, call request_plan_approval and wait for the user before any write tools.", parameters: [ { key: "summary", type: "string", required: true }, { key: "steps", type: "array", required: true }, @@ -319,7 +319,7 @@ export const submitPlanSchema: ToolSchema = { export const askUserSchema: ToolSchema = { name: "ask_user", description: - "Asks the user a question during plan mode. Use when you need clarification about their intent, preferences, or content details before finalizing the plan. Optionally provide quick-reply options for common answers. The user can also type a free-form response.", + "Asks the user a question before or during planning. Use when you need clarification about their intent, preferences, or content details before finalizing the plan. Optionally provide quick-reply options for common answers. The user can also type a free-form response.", parameters: [ { key: "question", type: "string", required: true }, { key: "options", type: "array", required: false }, @@ -329,7 +329,7 @@ export const askUserSchema: ToolSchema = { export const requestPlanApprovalSchema: ToolSchema = { name: "request_plan_approval", description: - "Requests user approval to switch from plan mode to execute mode. The user sees a modal with 'Continue planning' and 'Start editing' options. ALWAYS requires explicit user approval — there is no auto-approve. Use this when the plan is ready for execution.", + "Requests user approval to switch from planning to editing. The user sees a modal with Keep planning and Go edit options. ALWAYS requires explicit user approval — there is no auto-approve. Use immediately after submit_plan succeeds.", parameters: [], }; diff --git a/apps/web/src/agent/tools/submit-plan.tool.ts b/apps/web/src/agent/tools/submit-plan.tool.ts index 5926a5e7..b2668936 100644 --- a/apps/web/src/agent/tools/submit-plan.tool.ts +++ b/apps/web/src/agent/tools/submit-plan.tool.ts @@ -2,16 +2,26 @@ import type { AgentContext, ToolDefinition } from "@/agent/types"; import { toolRegistry } from "@/agent/tools/registry"; import { submitPlanSchema } from "@/agent/tools/schemas"; import { usePlanStore, createPlanFromSteps } from "@/stores/plan-store"; +import { useAgentStore } from "@/stores/agent-store"; + +type SubmittedStep = { description: string; tools: string[] }; const submitPlanTool: ToolDefinition = { ...submitPlanSchema, execute: async ( args: Record, _context: AgentContext, - ): Promise<{ planId: string; stepCount: number } | { error: string }> => { + ): Promise< + | { + planId: string; + stepCount: number; + nextAction: "request_plan_approval"; + } + | { error: string } + > => { const { summary, steps, questions } = args as { summary: string; - steps: Array<{ description: string; tools: string[] }>; + steps: unknown; questions?: string[]; }; @@ -19,11 +29,14 @@ const submitPlanTool: ToolDefinition = { return { error: "summary is required and must be a string" }; } - if (!Array.isArray(steps) || steps.length === 0) { - return { error: "steps must be a non-empty array" }; + const normalizedSteps = normalizeSteps(steps); + if (normalizedSteps.length === 0) { + return { + error: "steps must be a non-empty array or a numbered list string", + }; } - for (const [i, step] of steps.entries()) { + for (const [i, step] of normalizedSteps.entries()) { if (!step.description || typeof step.description !== "string") { return { error: `Step ${i + 1} must have a description string` }; } @@ -32,11 +45,47 @@ const submitPlanTool: ToolDefinition = { } } - const plan = createPlanFromSteps(summary, steps, questions); + const plan = createPlanFromSteps(summary, normalizedSteps, questions); usePlanStore.getState().setPlan(plan); + useAgentStore.getState().setMode("plan"); - return { planId: plan.id, stepCount: plan.steps.length }; + return { + planId: plan.id, + stepCount: plan.steps.length, + nextAction: "request_plan_approval", + }; }, }; toolRegistry.register(submitPlanSchema.name, submitPlanTool); + +function normalizeSteps(steps: unknown): SubmittedStep[] { + if (Array.isArray(steps)) { + return steps + .map((step) => { + if (typeof step === "string") { + return { description: step, tools: [] }; + } + if (typeof step !== "object" || step === null) return null; + const raw = step as { description?: unknown; tools?: unknown }; + return { + description: + typeof raw.description === "string" ? raw.description : "", + tools: Array.isArray(raw.tools) + ? raw.tools.filter( + (tool): tool is string => typeof tool === "string", + ) + : [], + }; + }) + .filter((step): step is SubmittedStep => step !== null); + } + + if (typeof steps !== "string") return []; + + return steps + .split("\n") + .map((line) => line.replace(/^\s*\d+[.)-]?\s*/, "").trim()) + .filter(Boolean) + .map((description) => ({ description, tools: [] })); +} diff --git a/apps/web/src/components/editor/panels/chat/mode-transition-modal.tsx b/apps/web/src/components/editor/panels/chat/mode-transition-modal.tsx index 9289938b..1f2c14af 100644 --- a/apps/web/src/components/editor/panels/chat/mode-transition-modal.tsx +++ b/apps/web/src/components/editor/panels/chat/mode-transition-modal.tsx @@ -82,7 +82,7 @@ export function ModeTransitionModal() { className="mr-2 size-4" strokeWidth={1.5} /> - Start editing + Go edit
From cea87833e44bedf75e75187f759691d205b71473 Mon Sep 17 00:00:00 2001 From: Luis Esteban Acevedo Ladino Date: Fri, 1 May 2026 19:03:47 -0500 Subject: [PATCH 6/6] fix: make plan approval an inline chat card --- apps/web/src/agent/system-prompt.ts | 10 +- apps/web/src/agent/tools/schemas.ts | 4 +- apps/web/src/agent/tools/submit-plan.tool.ts | 30 ++++- .../editor/panels/chat/chat-input.tsx | 112 ++++++++---------- .../components/editor/panels/chat/index.tsx | 15 +-- .../panels/chat/mode-transition-modal.tsx | 105 ---------------- .../editor/panels/chat/plan-approval-card.tsx | 88 ++++++++++++++ 7 files changed, 170 insertions(+), 194 deletions(-) delete mode 100644 apps/web/src/components/editor/panels/chat/mode-transition-modal.tsx create mode 100644 apps/web/src/components/editor/panels/chat/plan-approval-card.tsx diff --git a/apps/web/src/agent/system-prompt.ts b/apps/web/src/agent/system-prompt.ts index 88f22752..213e0a13 100644 --- a/apps/web/src/agent/system-prompt.ts +++ b/apps/web/src/agent/system-prompt.ts @@ -20,8 +20,7 @@ You are in PLAN mode. You CANNOT make any edits to the timeline. You can only: 1. **Analyze**: Use read-only tools to understand what media and timeline state exist. Use load_context to actually see/hear the footage. 2. **Discover**: If the user's request matches a known editing pattern, call list_skills to find relevant techniques. 3. **Clarify**: If anything is ambiguous, use ask_user to get clarification before planning. -4. **Plan**: Once you understand the footage and the goal, call submit_plan with a structured step-by-step plan. -5. **Approval**: Immediately after submit_plan succeeds, call request_plan_approval. Do NOT edit until the user approves. +4. **Plan**: Once you understand the footage and the goal, call submit_plan with a structured step-by-step plan. This shows the user a plan card with Go edit / Keep planning and waits for their choice. ## PLAN QUALITY RULES: @@ -38,7 +37,7 @@ Submit the plan when: - All clarifying questions have been answered - You have a concrete, actionable plan -After submitting, you MUST ask for approval through request_plan_approval. The user will choose Keep planning or Go edit. +After submit_plan returns approved=true, continue with the plan in execute mode. If it returns approved=false, keep discussing and refining the plan. `; const EXECUTE_MODE_INSTRUCTIONS = ` @@ -52,8 +51,7 @@ For any complex editing request that would require multiple timeline edits (cuts 1. Analyze the project with read-only tools. 2. Call submit_plan with a concise structured checklist. -3. Call request_plan_approval. -4. Wait for the user to choose Go edit. +3. Wait for the user to choose Go edit. Do NOT perform write tools for complex edits before approval. This is required even if the current mode is EXECUTE. @@ -67,7 +65,7 @@ Do NOT perform write tools for complex edits before approval. This is required e ## IF NO ACTIVE PLAN: -Simple one-shot edits may execute directly. Complex edits must go through submit_plan + request_plan_approval first. +Simple one-shot edits may execute directly. Complex edits must go through submit_plan first. `; export function buildSystemPrompt( diff --git a/apps/web/src/agent/tools/schemas.ts b/apps/web/src/agent/tools/schemas.ts index 175579c9..1e819a9e 100644 --- a/apps/web/src/agent/tools/schemas.ts +++ b/apps/web/src/agent/tools/schemas.ts @@ -308,7 +308,7 @@ export const loadSkillSchema: ToolSchema = { export const submitPlanSchema: ToolSchema = { name: "submit_plan", description: - "Submits a structured editing plan with steps. Use this before complex edits, even from execute mode. Each step should describe a specific action with the tools to use. After submission, call request_plan_approval and wait for the user before any write tools.", + "Submits a structured editing plan with steps and waits for user approval through a plan card with Go edit and Keep planning actions. Use this before complex edits, even from execute mode. Each step should describe a specific action with the tools to use. If approved=true is returned, continue executing the plan. If approved=false is returned, keep refining the plan with the user.", parameters: [ { key: "summary", type: "string", required: true }, { key: "steps", type: "array", required: true }, @@ -329,7 +329,7 @@ export const askUserSchema: ToolSchema = { export const requestPlanApprovalSchema: ToolSchema = { name: "request_plan_approval", description: - "Requests user approval to switch from planning to editing. The user sees a modal with Keep planning and Go edit options. ALWAYS requires explicit user approval — there is no auto-approve. Use immediately after submit_plan succeeds.", + "Requests user approval to switch from planning to editing for an already submitted plan. Usually you do not need this because submit_plan already waits for approval. The user sees a plan card with Keep planning and Go edit options.", parameters: [], }; diff --git a/apps/web/src/agent/tools/submit-plan.tool.ts b/apps/web/src/agent/tools/submit-plan.tool.ts index b2668936..65b028fc 100644 --- a/apps/web/src/agent/tools/submit-plan.tool.ts +++ b/apps/web/src/agent/tools/submit-plan.tool.ts @@ -3,6 +3,7 @@ import { toolRegistry } from "@/agent/tools/registry"; import { submitPlanSchema } from "@/agent/tools/schemas"; import { usePlanStore, createPlanFromSteps } from "@/stores/plan-store"; import { useAgentStore } from "@/stores/agent-store"; +import type { AgentMode } from "@/agent/types"; type SubmittedStep = { description: string; tools: string[] }; @@ -15,7 +16,8 @@ const submitPlanTool: ToolDefinition = { | { planId: string; stepCount: number; - nextAction: "request_plan_approval"; + approved: boolean; + mode: AgentMode; } | { error: string } > => { @@ -49,11 +51,27 @@ const submitPlanTool: ToolDefinition = { usePlanStore.getState().setPlan(plan); useAgentStore.getState().setMode("plan"); - return { - planId: plan.id, - stepCount: plan.steps.length, - nextAction: "request_plan_approval", - }; + return new Promise((resolve) => { + useAgentStore.getState().setPendingModeTransition({ + targetMode: "execute", + resolve: (approved) => { + useAgentStore.getState().setPendingModeTransition(null); + if (approved) { + useAgentStore.getState().setMode("execute"); + usePlanStore.getState().updatePlanStatus("executing"); + } else { + useAgentStore.getState().setMode("plan"); + usePlanStore.getState().updatePlanStatus("awaiting_approval"); + } + resolve({ + planId: plan.id, + stepCount: plan.steps.length, + approved, + mode: approved ? "execute" : "plan", + }); + }, + }); + }); }, }; diff --git a/apps/web/src/components/editor/panels/chat/chat-input.tsx b/apps/web/src/components/editor/panels/chat/chat-input.tsx index f18a2975..91084b34 100644 --- a/apps/web/src/components/editor/panels/chat/chat-input.tsx +++ b/apps/web/src/components/editor/panels/chat/chat-input.tsx @@ -1,6 +1,6 @@ "use client"; -import { useState, useCallback, type KeyboardEvent } from "react"; +import { useState, useCallback, useRef, type KeyboardEvent } from "react"; import { HugeiconsIcon } from "@hugeicons/react"; import { Sent02Icon, @@ -12,9 +12,7 @@ import { import { Button } from "@/components/ui/button"; import { Spinner } from "@/components/ui/spinner"; import { useAgentStore, type PermissionMode } from "@/stores/agent-store"; -import { usePlanStore } from "@/stores/plan-store"; import { cn } from "@/utils/ui"; -import type { AgentMode } from "@/agent/types"; interface ChatInputProps { onSend: (content: string) => void; @@ -23,46 +21,48 @@ interface ChatInputProps { export function ChatInput({ onSend, disabled }: ChatInputProps) { const [value, setValue] = useState(""); + const shiftHandledRef = useRef(false); const permissionMode = useAgentStore((s) => s.permissionMode); const setPermissionMode = useAgentStore((s) => s.setPermissionMode); const pendingApproval = useAgentStore((s) => s.pendingApproval); const mode = useAgentStore((s) => s.mode); const setMode = useAgentStore((s) => s.setMode); const pendingTransition = useAgentStore((s) => s.pendingModeTransition); - const plan = usePlanStore((s) => s.plan); + const isPlanMode = mode === "plan"; + const toggleMode = useCallback(() => { + setMode(isPlanMode ? "execute" : "plan"); + }, [isPlanMode, setMode]); - const handleSend = useCallback( - (forcedMode?: AgentMode) => { - const trimmed = value.trim(); - if (!trimmed || disabled) return; - const targetMode = forcedMode ?? mode; - if (targetMode !== mode) { - setMode(targetMode); - } - onSend(trimmed); - setValue(""); - }, - [value, disabled, onSend, mode, setMode], - ); + const handleSend = useCallback(() => { + const trimmed = value.trim(); + if (!trimmed || disabled) return; + onSend(trimmed); + setValue(""); + }, [value, disabled, onSend]); const handleKeyDown = useCallback( (e: KeyboardEvent) => { + if (e.key === "Shift") { + if (!shiftHandledRef.current) { + shiftHandledRef.current = true; + toggleMode(); + } + return; + } + if (e.key === "Enter" && !e.shiftKey) { e.preventDefault(); - handleSend("execute"); - } - if (e.key === "Enter" && e.shiftKey) { - e.preventDefault(); - handleSend("plan"); + handleSend(); } }, - [handleSend], + [handleSend, toggleMode], ); - const toggleMode = useCallback(() => { - const next: AgentMode = mode === "execute" ? "plan" : "execute"; - setMode(next); - }, [mode, setMode]); + const handleKeyUp = useCallback((e: KeyboardEvent) => { + if (e.key === "Shift") { + shiftHandledRef.current = false; + } + }, []); const togglePermission = useCallback(() => { const next: PermissionMode = permissionMode === "skip" ? "ask" : "skip"; @@ -70,15 +70,36 @@ export function ChatInput({ onSend, disabled }: ChatInputProps) { }, [permissionMode, setPermissionMode]); const isAsk = permissionMode === "ask"; - const isPlanMode = mode === "plan"; return (
+
+ + + Press Shift to switch + +