Merge pull request #10 from luisacevelad/feat/plan-mode

Feat/plan mode
This commit is contained in:
luisacevelad 2026-05-01 19:46:05 -05:00 committed by GitHub
commit 09c28cce72
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
18 changed files with 997 additions and 32 deletions

View File

@ -9,6 +9,7 @@ import { toolRegistry } from "@/agent/tools/registry";
import "@/agent/tools";
import { useChatStore } from "@/stores/chat-store";
import { useAgentStore } from "@/stores/agent-store";
import { usePlanStore } from "@/stores/plan-store";
const MAX_ITERATIONS = 20;
@ -33,6 +34,8 @@ const WRITE_TOOLS = new Set([
"update_keyframe_curve",
]);
const EXECUTE_ONLY_TOOLS = new Set(["update_plan_step"]);
interface APIResponse {
content: string;
toolCalls?: ToolCall[];
@ -57,10 +60,16 @@ export async function run(
while (iterations < MAX_ITERATIONS) {
iterations++;
const liveMode = useAgentStore.getState().mode;
const liveContext: AgentContext = { ...context, mode: liveMode };
const response = await fetch("/api/agent/chat", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ messages: workingMessages, context }),
body: JSON.stringify({
messages: workingMessages,
context: liveContext,
}),
});
if (!response.ok) {
@ -210,9 +219,45 @@ async function resolveToolCalls(
for (const tc of toolCalls) {
useAgentStore.getState().setActiveTool(tc.name);
const currentMode = useAgentStore.getState().mode;
const currentPlan = usePlanStore.getState().plan;
const currentMode = useAgentStore.getState().permissionMode;
if (currentMode === "ask" && WRITE_TOOLS.has(tc.name)) {
if (
WRITE_TOOLS.has(tc.name) &&
currentPlan?.status === "awaiting_approval"
) {
results.push({
toolCallId: tc.id,
name: tc.name,
result: null,
error:
"Cannot edit while a plan is awaiting user approval. Call request_plan_approval and wait for the user to choose Go edit.",
});
continue;
}
if (currentMode === "plan" && WRITE_TOOLS.has(tc.name)) {
results.push({
toolCallId: tc.id,
name: tc.name,
result: null,
error: `Cannot use '${tc.name}' in plan mode. This tool modifies the timeline. Use submit_plan to finalize your plan, then request_plan_approval to switch to execute mode.`,
});
continue;
}
if (currentMode === "plan" && EXECUTE_ONLY_TOOLS.has(tc.name)) {
results.push({
toolCallId: tc.id,
name: tc.name,
result: null,
error: `Cannot use '${tc.name}' in plan mode. This tool is only available during execution.`,
});
continue;
}
const currentPermissionMode = useAgentStore.getState().permissionMode;
if (currentPermissionMode === "ask" && WRITE_TOOLS.has(tc.name)) {
const approved = await requestApproval(tc);
if (!approved) {
results.push({

View File

@ -1,20 +1,80 @@
import type { AgentContext } from "@/agent/types";
import type { AgentContext, AgentMode } from "@/agent/types";
export interface ToolSummary {
name: string;
description: string;
}
/**
* Builds the system prompt for the LLM from the editor context.
*
* Pure function no client-only imports (no EditorCore, WASM, browser APIs).
* Consumable server-side in the API route.
*/
const PLAN_MODE_INSTRUCTIONS = `
## CURRENT MODE: PLAN (READ-ONLY)
You are in PLAN mode. You CANNOT make any edits to the timeline. You can only:
- Read and analyze the project (list_project_assets, list_timeline, get_element, list_effects, get_effect, list_keyframes, list_animatable_properties)
- Load media into your context (load_context)
- Discover skills (list_skills, load_skill)
- Ask the user questions (ask_user)
- Submit your plan (submit_plan)
## YOUR WORKFLOW IN PLAN MODE:
1. **Analyze**: Use read-only tools to understand what media and timeline state exist. Use load_context to actually see/hear the footage.
2. **Discover**: If the user's request matches a known editing pattern, call list_skills to find relevant techniques.
3. **Clarify**: If anything is ambiguous, use ask_user to get clarification before planning.
4. **Plan**: Once you understand the footage and the goal, call submit_plan with a structured step-by-step plan. This shows the user a plan card with Go edit / Keep planning and waits for their choice.
## PLAN QUALITY RULES:
- Each step must reference SPECIFIC tools and SPECIFIC values (timestamps, element IDs after discovery, effect params)
- Steps should be ordered by dependency (split before delete, add text before animate, etc.)
- If a skill was loaded, reference its techniques but ADAPT them to the actual footage never copy blindly
- Include timing estimates where possible
- Be honest about limitations if the footage doesn't support a technique, say so
## WHEN TO SUBMIT:
Submit the plan when:
- You have enough context about the media content
- All clarifying questions have been answered
- You have a concrete, actionable plan
After submit_plan returns approved=true, continue with the plan in execute mode. If it returns approved=false, keep discussing and refining the plan.
`;
const EXECUTE_MODE_INSTRUCTIONS = `
## CURRENT MODE: EXECUTE
You are in EXECUTE mode. You can read AND write to the timeline, but complex editing requests still require a plan first.
## MANDATORY PLANNING GATE
For any complex editing request that would require multiple timeline edits (cuts, moving media, adding titles/effects, or touching 3+ tools), you MUST:
1. Analyze the project with read-only tools.
2. Call submit_plan with a concise structured checklist.
3. Wait for the user to choose Go edit.
Do NOT perform write tools for complex edits before approval. This is required even if the current mode is EXECUTE.
## EXECUTION RULES:
1. Follow the plan steps in order. Each step lists the tools to use.
2. After completing a step, call update_plan_step to mark it done.
3. If a step fails, note the error in update_plan_step and decide whether to continue or skip.
4. If you discover the plan needs adjustment mid-execution, explain what changed and why.
5. When all steps are done, provide a summary of what was accomplished.
## IF NO ACTIVE PLAN:
Simple one-shot edits may execute directly. Complex edits must go through submit_plan first.
`;
export function buildSystemPrompt(
context: AgentContext,
tools?: ToolSummary[],
mode?: AgentMode,
): string {
const activeMode = mode ?? context.mode ?? "execute";
const mediaSection =
context.mediaAssets.length > 0
? `Active media assets:\n${context.mediaAssets.map((m) => `- [id: ${m.id}] ${m.name} (${m.type}, ${m.duration}s)`).join("\n")}`
@ -45,9 +105,15 @@ export function buildSystemPrompt(
"Only claim edits that were actually performed by tool calls in this conversation. Do not say you added, removed, cleaned, or updated text/subtitles unless the relevant add_text, update_text, delete_timeline_elements, or update_timeline_element_timing tool call succeeded.",
"When the user asks to add titles, hooks, labels, captions, subtitles, or visible text, call add_text. Do not add text proactively for unrelated edit requests such as cutting silence unless the user asks for text.",
"When you need to perform an action matching one of these tools, call the appropriate tool. For all other requests, respond directly in plain text.",
"SKILLS: You have editing skills available — pre-built recipe workflows for common editing patterns. When the user asks for a complex edit (viral video, pitch, specific style, etc.), call list_skills to discover relevant skills, then call load_skill with the matching skillId to get full step-by-step instructions. Follow those instructions precisely using the standard editing tools. If the user's request doesn't match any skill, proceed with your own editing approach.",
"SKILLS: You have editing skills available — pre-built technique libraries for common editing patterns. When the user asks for a complex edit (viral video, pitch, specific style, etc.), call list_skills to discover relevant skills, then call load_skill with the matching skillId to get technique definitions. Adapt those techniques to the actual footage content. If the user's request doesn't match any skill, proceed with your own editing approach.",
);
}
if (activeMode === "plan") {
parts.push(PLAN_MODE_INSTRUCTIONS);
} else {
parts.push(EXECUTE_MODE_INSTRUCTIONS);
}
return parts.join("\n");
}

View File

@ -0,0 +1,44 @@
import type { AgentContext, ToolDefinition } from "@/agent/types";
import { toolRegistry } from "@/agent/tools/registry";
import { askUserSchema } from "@/agent/tools/schemas";
import { usePlanStore } from "@/stores/plan-store";
import { nanoid } from "nanoid";
const askUserTool: ToolDefinition = {
...askUserSchema,
execute: async (
args: Record<string, unknown>,
_context: AgentContext,
): Promise<{ questionId: string; answer: string } | { error: string }> => {
const { question, options } = args as {
question: string;
options?: Array<{ label: string; description?: string }>;
};
if (!question || typeof question !== "string") {
return { error: "question is required and must be a string" };
}
const questionId = nanoid();
const planStore = usePlanStore.getState();
planStore.setPendingQuestion({
id: questionId,
question,
options: options && Array.isArray(options) ? options : undefined,
});
return new Promise((resolve) => {
const unsubscribe = usePlanStore.subscribe((state) => {
if (state.pendingQuestion?.answer) {
unsubscribe();
const answer = state.pendingQuestion.answer;
usePlanStore.getState().setPendingQuestion(null);
resolve({ questionId, answer });
}
});
});
},
};
toolRegistry.register(askUserSchema.name, askUserTool);

View File

@ -38,3 +38,7 @@ import "@/agent/tools/update-keyframe-curve.tool";
import "@/agent/tools/list-animatable-properties.tool";
import "@/agent/skills/list-skills.tool";
import "@/agent/skills/load-skill.tool";
import "@/agent/tools/submit-plan.tool";
import "@/agent/tools/ask-user.tool";
import "@/agent/tools/request-plan-approval.tool";
import "@/agent/tools/update-plan-step.tool";

View File

@ -0,0 +1,46 @@
import type { AgentContext, ToolDefinition } from "@/agent/types";
import { toolRegistry } from "@/agent/tools/registry";
import { requestPlanApprovalSchema } from "@/agent/tools/schemas";
import { useAgentStore } from "@/stores/agent-store";
import { usePlanStore } from "@/stores/plan-store";
import type { AgentMode } from "@/agent/types";
const requestPlanApprovalTool: ToolDefinition = {
...requestPlanApprovalSchema,
execute: async (
_args: Record<string, unknown>,
_context: AgentContext,
): Promise<{ approved: boolean; mode: AgentMode } | { error: string }> => {
const planStore = usePlanStore.getState();
if (!planStore.plan) {
return {
error: "No plan has been submitted yet. Call submit_plan first.",
};
}
useAgentStore.getState().setMode("plan");
planStore.updatePlanStatus("awaiting_approval");
return new Promise((resolve) => {
useAgentStore.getState().setPendingModeTransition({
targetMode: "execute",
resolve: (approved) => {
useAgentStore.getState().setPendingModeTransition(null);
if (approved) {
useAgentStore.getState().setMode("execute");
usePlanStore.getState().updatePlanStatus("executing");
} else {
useAgentStore.getState().setMode("plan");
usePlanStore.getState().updatePlanStatus("awaiting_approval");
}
resolve({
approved,
mode: approved ? "execute" : "plan",
});
},
});
});
},
};
toolRegistry.register(requestPlanApprovalSchema.name, requestPlanApprovalTool);

View File

@ -305,6 +305,45 @@ export const loadSkillSchema: ToolSchema = {
parameters: [{ key: "skillId", type: "string", required: true }],
};
export const submitPlanSchema: ToolSchema = {
name: "submit_plan",
description:
"Submits a structured editing plan with steps and waits for user approval through a plan card with Go edit and Keep planning actions. Use this before complex edits, even from execute mode. Each step should describe a specific action with the tools to use. If approved=true is returned, continue executing the plan. If approved=false is returned, keep refining the plan with the user.",
parameters: [
{ key: "summary", type: "string", required: true },
{ key: "steps", type: "array", required: true },
{ key: "questions", type: "array", required: false },
],
};
export const askUserSchema: ToolSchema = {
name: "ask_user",
description:
"Asks the user a question before or during planning. Use when you need clarification about their intent, preferences, or content details before finalizing the plan. Optionally provide quick-reply options for common answers. The user can also type a free-form response.",
parameters: [
{ key: "question", type: "string", required: true },
{ key: "options", type: "array", required: false },
],
};
export const requestPlanApprovalSchema: ToolSchema = {
name: "request_plan_approval",
description:
"Requests user approval to switch from planning to editing for an already submitted plan. Usually you do not need this because submit_plan already waits for approval. The user sees a plan card with Keep planning and Go edit options.",
parameters: [],
};
export const updatePlanStepSchema: ToolSchema = {
name: "update_plan_step",
description:
"Updates the status of a plan step during execution. Call this after completing each step to track progress. The plan panel updates in real-time. Use status 'done' for successful completion, 'skipped' if the step was unnecessary, or 'in_progress' if starting.",
parameters: [
{ key: "stepId", type: "string", required: true },
{ key: "status", type: "string", required: true },
{ key: "result", type: "string", required: false },
],
};
/**
* The exact list of schemas exposed to the LLM.
* Excludes internal-only tools (transcribe_video, mock).
@ -338,4 +377,8 @@ export const providerToolSchemas: ToolSchema[] = [
listAnimatablePropertiesSchema,
listSkillsSchema,
loadSkillSchema,
submitPlanSchema,
askUserSchema,
requestPlanApprovalSchema,
updatePlanStepSchema,
];

View File

@ -0,0 +1,109 @@
import type { AgentContext, ToolDefinition } from "@/agent/types";
import { toolRegistry } from "@/agent/tools/registry";
import { submitPlanSchema } from "@/agent/tools/schemas";
import { usePlanStore, createPlanFromSteps } from "@/stores/plan-store";
import { useAgentStore } from "@/stores/agent-store";
import type { AgentMode } from "@/agent/types";
type SubmittedStep = { description: string; tools: string[] };
const submitPlanTool: ToolDefinition = {
...submitPlanSchema,
execute: async (
args: Record<string, unknown>,
_context: AgentContext,
): Promise<
| {
planId: string;
stepCount: number;
approved: boolean;
mode: AgentMode;
}
| { error: string }
> => {
const { summary, steps, questions } = args as {
summary: string;
steps: unknown;
questions?: string[];
};
if (!summary || typeof summary !== "string") {
return { error: "summary is required and must be a string" };
}
const normalizedSteps = normalizeSteps(steps);
if (normalizedSteps.length === 0) {
return {
error: "steps must be a non-empty array or a numbered list string",
};
}
for (const [i, step] of normalizedSteps.entries()) {
if (!step.description || typeof step.description !== "string") {
return { error: `Step ${i + 1} must have a description string` };
}
if (!Array.isArray(step.tools)) {
return { error: `Step ${i + 1} must have a tools array` };
}
}
const plan = createPlanFromSteps(summary, normalizedSteps, questions);
usePlanStore.getState().setPlan(plan);
useAgentStore.getState().setMode("plan");
return new Promise((resolve) => {
useAgentStore.getState().setPendingModeTransition({
targetMode: "execute",
resolve: (approved) => {
useAgentStore.getState().setPendingModeTransition(null);
if (approved) {
useAgentStore.getState().setMode("execute");
usePlanStore.getState().updatePlanStatus("executing");
} else {
useAgentStore.getState().setMode("plan");
usePlanStore.getState().updatePlanStatus("awaiting_approval");
}
resolve({
planId: plan.id,
stepCount: plan.steps.length,
approved,
mode: approved ? "execute" : "plan",
});
},
});
});
},
};
toolRegistry.register(submitPlanSchema.name, submitPlanTool);
function normalizeSteps(steps: unknown): SubmittedStep[] {
if (Array.isArray(steps)) {
return steps
.map((step) => {
if (typeof step === "string") {
return { description: step, tools: [] };
}
if (typeof step !== "object" || step === null) return null;
const raw = step as { description?: unknown; tools?: unknown };
return {
description:
typeof raw.description === "string" ? raw.description : "",
tools: Array.isArray(raw.tools)
? raw.tools.filter(
(tool): tool is string => typeof tool === "string",
)
: [],
};
})
.filter((step): step is SubmittedStep => step !== null);
}
if (typeof steps !== "string") return [];
return steps
.split("\n")
.map((line) => line.replace(/^\s*\d+[.)-]?\s*/, "").trim())
.filter(Boolean)
.map((description) => ({ description, tools: [] }));
}

View File

@ -0,0 +1,57 @@
import type { AgentContext, ToolDefinition } from "@/agent/types";
import { toolRegistry } from "@/agent/tools/registry";
import { updatePlanStepSchema } from "@/agent/tools/schemas";
import { usePlanStore } from "@/stores/plan-store";
import type { PlanStepStatus } from "@/agent/types";
const VALID_STATUSES: PlanStepStatus[] = [
"pending",
"in_progress",
"done",
"skipped",
];
const updatePlanStepTool: ToolDefinition = {
...updatePlanStepSchema,
execute: async (
args: Record<string, unknown>,
_context: AgentContext,
):
| { success: boolean; stepId: string; status: string }
| { error: string } => {
const { stepId, status, result } = args as {
stepId: string;
status: string;
result?: string;
};
if (!stepId || typeof stepId !== "string") {
return { error: "stepId is required" };
}
if (!VALID_STATUSES.includes(status as PlanStepStatus)) {
return {
error: `Invalid status: ${status}. Must be one of: ${VALID_STATUSES.join(", ")}`,
};
}
const planStore = usePlanStore.getState();
const plan = planStore.plan;
if (!plan) {
return { error: "No active plan" };
}
const step = plan.steps.find((s) => s.id === stepId);
if (!step) {
return {
error: `Step not found: ${stepId}. Available: ${plan.steps.map((s) => s.id).join(", ")}`,
};
}
planStore.updateStepStatus(stepId, status as PlanStepStatus, result);
return { success: true, stepId, status };
},
};
toolRegistry.register(updatePlanStepSchema.name, updatePlanStepTool);

View File

@ -5,12 +5,45 @@ export type ExecutionState =
| "responding"
| "error";
export type AgentMode = "plan" | "execute";
export type PlanStepStatus = "pending" | "in_progress" | "done" | "skipped";
export type PlanStatus =
| "drafting"
| "awaiting_approval"
| "approved"
| "executing"
| "completed";
export interface PlanStep {
id: string;
description: string;
tools: string[];
status: PlanStepStatus;
result?: string;
}
export interface Plan {
id: string;
summary: string;
steps: PlanStep[];
questions?: string[];
status: PlanStatus;
createdAt: number;
}
export interface AgentQuestion {
id: string;
question: string;
options?: Array<{ label: string; description?: string }>;
answer?: string;
}
export interface ChatMessage {
id: string;
role: "system" | "user" | "assistant" | "tool_result";
content: string;
toolCalls?: ToolCall[];
/** Associates a tool_result message with the ToolCall it answers. */
toolCallId?: string;
timestamp: number;
}
@ -41,6 +74,7 @@ export interface AgentContext {
}>;
timelineTracks?: AgentTimelineTrack[];
playbackTimeMs: number;
mode?: AgentMode;
}
export type AgentTimelineTrack = {

View File

@ -63,6 +63,7 @@ const chatRequestSchema = z.object({
)
.optional(),
playbackTimeMs: z.number(),
mode: z.enum(["plan", "execute"]).optional(),
}) satisfies z.ZodType<AgentContext>,
});
@ -89,7 +90,8 @@ export async function POST(request: NextRequest) {
const { messages, context } = result.data;
// Resolve provider config from LLM_* env vars
const mode = context.mode;
const provider = process.env.LLM_PROVIDER;
const apiKey = process.env.LLM_API_KEY;
const model = process.env.LLM_MODEL;
@ -107,8 +109,7 @@ export async function POST(request: NextRequest) {
const config: ProviderConfig = { provider, apiKey, model, baseUrl };
// Build system prompt with tool guidance
const systemPrompt = buildSystemPrompt(context, providerToolSchemas);
const systemPrompt = buildSystemPrompt(context, providerToolSchemas, mode);
// Delegate to provider adapter
try {

View File

@ -0,0 +1,93 @@
"use client";
import { useState, useCallback } from "react";
import { HugeiconsIcon } from "@hugeicons/react";
import { QuestionIcon } from "@hugeicons/core-free-icons";
import { Button } from "@/components/ui/button";
import { usePlanStore } from "@/stores/plan-store";
export function AskUserPrompt() {
const pendingQuestion = usePlanStore((s) => s.pendingQuestion);
const answerQuestion = usePlanStore((s) => s.answerQuestion);
const [freeForm, setFreeForm] = useState("");
const handleOptionClick = useCallback(
(label: string) => {
answerQuestion(label);
setFreeForm("");
},
[answerQuestion],
);
const handleFreeFormSubmit = useCallback(() => {
const trimmed = freeForm.trim();
if (!trimmed) return;
answerQuestion(trimmed);
setFreeForm("");
}, [freeForm, answerQuestion]);
const handleKeyDown = useCallback(
(e: React.KeyboardEvent) => {
if (e.key === "Enter" && !e.shiftKey) {
e.preventDefault();
handleFreeFormSubmit();
}
},
[handleFreeFormSubmit],
);
if (!pendingQuestion || pendingQuestion.answer) return null;
return (
<div className="flex flex-col gap-2 rounded-md border border-blue-500/30 bg-blue-500/5 px-3 py-2.5">
<div className="flex items-center gap-2">
<HugeiconsIcon
icon={QuestionIcon}
className="size-4 shrink-0 text-blue-400"
strokeWidth={1.5}
/>
<span className="text-xs font-medium text-blue-400">
Agent question
</span>
</div>
<p className="text-xs text-foreground/90">{pendingQuestion.question}</p>
{pendingQuestion.options && pendingQuestion.options.length > 0 && (
<div className="flex flex-wrap gap-1.5">
{pendingQuestion.options.map((opt) => (
<Button
key={opt.label}
variant="outline"
size="sm"
onClick={() => handleOptionClick(opt.label)}
className="text-xs"
title={opt.description}
>
{opt.label}
</Button>
))}
</div>
)}
<div className="flex items-center gap-1.5">
<input
type="text"
value={freeForm}
onChange={(e) => setFreeForm(e.target.value)}
onKeyDown={handleKeyDown}
placeholder="Type your answer..."
className="border-border bg-input flex-1 rounded-md border px-2 py-1 text-xs outline-none focus:border-blue-500/50"
/>
<Button
variant="secondary"
size="sm"
onClick={handleFreeFormSubmit}
disabled={!freeForm.trim()}
className="text-xs"
>
Send
</Button>
</div>
</div>
);
}

View File

@ -1,11 +1,13 @@
"use client";
import { useState, useCallback, type KeyboardEvent } from "react";
import { useState, useCallback, useRef, type KeyboardEvent } from "react";
import { HugeiconsIcon } from "@hugeicons/react";
import {
Sent02Icon,
FlashIcon,
Shield01Icon,
CheckListIcon,
Edit02Icon,
} from "@hugeicons/core-free-icons";
import { Button } from "@/components/ui/button";
import { Spinner } from "@/components/ui/spinner";
@ -19,9 +21,17 @@ interface ChatInputProps {
export function ChatInput({ onSend, disabled }: ChatInputProps) {
const [value, setValue] = useState("");
const shiftHandledRef = useRef(false);
const permissionMode = useAgentStore((s) => s.permissionMode);
const setPermissionMode = useAgentStore((s) => s.setPermissionMode);
const pendingApproval = useAgentStore((s) => s.pendingApproval);
const mode = useAgentStore((s) => s.mode);
const setMode = useAgentStore((s) => s.setMode);
const pendingTransition = useAgentStore((s) => s.pendingModeTransition);
const isPlanMode = mode === "plan";
const toggleMode = useCallback(() => {
setMode(isPlanMode ? "execute" : "plan");
}, [isPlanMode, setMode]);
const handleSend = useCallback(() => {
const trimmed = value.trim();
@ -32,17 +42,30 @@ export function ChatInput({ onSend, disabled }: ChatInputProps) {
const handleKeyDown = useCallback(
(e: KeyboardEvent<HTMLTextAreaElement>) => {
if (e.key === "Shift") {
if (!shiftHandledRef.current) {
shiftHandledRef.current = true;
toggleMode();
}
return;
}
if (e.key === "Enter" && !e.shiftKey) {
e.preventDefault();
handleSend();
}
},
[handleSend],
[handleSend, toggleMode],
);
const toggleMode = useCallback(() => {
const next: PermissionMode =
permissionMode === "skip" ? "ask" : "skip";
const handleKeyUp = useCallback((e: KeyboardEvent<HTMLTextAreaElement>) => {
if (e.key === "Shift") {
shiftHandledRef.current = false;
}
}, []);
const togglePermission = useCallback(() => {
const next: PermissionMode = permissionMode === "skip" ? "ask" : "skip";
setPermissionMode(next);
}, [permissionMode, setPermissionMode]);
@ -50,21 +73,47 @@ export function ChatInput({ onSend, disabled }: ChatInputProps) {
return (
<div className="border-t p-3">
<div className="mb-2 flex items-center justify-between gap-2">
<button
type="button"
onClick={toggleMode}
disabled={!!pendingApproval || !!pendingTransition}
className={cn(
"bg-primary/10 text-primary border-primary/25 inline-flex items-center gap-1.5 rounded-full border px-2.5 py-1 text-[11px] font-semibold uppercase tracking-wide transition-colors",
"disabled:cursor-not-allowed disabled:opacity-50",
)}
>
<HugeiconsIcon
icon={isPlanMode ? CheckListIcon : Edit02Icon}
className="size-3.5"
strokeWidth={2}
/>
{isPlanMode ? "Plan mode" : "Edit mode"}
</button>
<span className="text-muted-foreground text-[10px]">
Press Shift to switch
</span>
</div>
<div className="flex items-end gap-2">
<textarea
value={value}
onChange={(e) => setValue(e.target.value)}
onKeyDown={handleKeyDown}
disabled={disabled}
placeholder="Ask the assistant..."
onKeyUp={handleKeyUp}
disabled={disabled || !!pendingTransition}
placeholder={
isPlanMode
? "Describe what you want (plan mode)..."
: "Ask the assistant..."
}
rows={1}
className="border-border bg-input focus-visible:border-primary/50 focus-visible:ring-primary/20 flex-1 resize-none rounded-md border px-3 py-2 text-sm outline-none focus-visible:ring-2 disabled:cursor-not-allowed disabled:opacity-50"
/>
<Button
variant={isAsk ? "default" : "secondary"}
size="icon"
onClick={toggleMode}
disabled={!!pendingApproval}
onClick={togglePermission}
disabled={!!pendingApproval || !!pendingTransition}
aria-label={isAsk ? "Ask permissions" : "Skip permissions"}
title={isAsk ? "Ask permissions" : "Skip permissions"}
>
@ -78,7 +127,7 @@ export function ChatInput({ onSend, disabled }: ChatInputProps) {
variant="secondary"
size="icon"
onClick={handleSend}
disabled={disabled || !value.trim()}
disabled={disabled || !value.trim() || !!pendingTransition}
aria-label="Send message"
>
{disabled ? (

View File

@ -11,6 +11,8 @@ import { useAgentStore } from "@/stores/agent-store";
import { MessageBubble } from "./message-bubble";
import { ChatInput } from "./chat-input";
import { ToolPermissionRequest } from "./tool-permission-request";
import { AskUserPrompt } from "./ask-user-prompt";
import { PlanApprovalCard } from "./plan-approval-card";
import { run as orchestratorRun } from "@/agent/orchestrator";
import { EditorContextAdapter } from "@/agent/context";
@ -24,13 +26,11 @@ export function ChatPanel() {
const scrollRef = useRef<HTMLDivElement>(null);
// Auto-scroll on new messages or loading state change
// biome-ignore lint/correctness/useExhaustiveDependencies: re-scroll when messages or loading change
useEffect(() => {
if (scrollRef.current) {
scrollRef.current.scrollTop = scrollRef.current.scrollHeight;
}
}, [messages, loading]);
});
const handleSend = useCallback(
async (content: string) => {
@ -80,6 +80,8 @@ export function ChatPanel() {
</span>
</div>
)}
<PlanApprovalCard />
<AskUserPrompt />
{pendingApproval && (
<ToolPermissionRequest pending={pendingApproval} />
)}

View File

@ -0,0 +1,88 @@
"use client";
import { useCallback } from "react";
import { HugeiconsIcon } from "@hugeicons/react";
import {
CheckListIcon,
Edit02Icon,
ReloadIcon,
} from "@hugeicons/core-free-icons";
import { Button } from "@/components/ui/button";
import { useAgentStore } from "@/stores/agent-store";
import { usePlanStore } from "@/stores/plan-store";
export function PlanApprovalCard() {
const pendingTransition = useAgentStore((s) => s.pendingModeTransition);
const plan = usePlanStore((s) => s.plan);
const handleKeepPlanning = useCallback(() => {
if (!pendingTransition) return;
pendingTransition.resolve(false);
}, [pendingTransition]);
const handleGoEdit = useCallback(() => {
if (!pendingTransition) return;
pendingTransition.resolve(true);
}, [pendingTransition]);
if (!pendingTransition || !plan) return null;
return (
<div className="border-primary/30 bg-primary/5 flex flex-col gap-3 rounded-lg border p-3 shadow-sm">
<div className="flex items-start gap-2">
<div className="bg-primary/15 text-primary mt-0.5 rounded-md p-1.5">
<HugeiconsIcon
icon={CheckListIcon}
className="size-4"
strokeWidth={2}
/>
</div>
<div className="flex min-w-0 flex-1 flex-col gap-1">
<div className="flex items-center justify-between gap-2">
<span className="text-sm font-semibold">Plan ready</span>
<span className="text-primary text-[10px] font-medium uppercase tracking-wide">
Plan mode
</span>
</div>
<p className="text-muted-foreground text-xs leading-relaxed">
{plan.summary}
</p>
</div>
</div>
<div className="bg-background/70 rounded-md border p-2">
<ul className="flex flex-col gap-1.5">
{plan.steps.map((step, index) => (
<li key={step.id} className="flex items-start gap-2 text-xs">
<span className="bg-primary/10 text-primary mt-0.5 flex size-4 shrink-0 items-center justify-center rounded-full text-[10px] font-medium">
{index + 1}
</span>
<span className="leading-relaxed">{step.description}</span>
</li>
))}
</ul>
</div>
<div className="flex items-center gap-2">
<Button size="sm" onClick={handleGoEdit} className="h-8 flex-1 text-xs">
<HugeiconsIcon icon={Edit02Icon} className="mr-1.5 size-3.5" />
Go edit
</Button>
<Button
variant="outline"
size="sm"
onClick={handleKeepPlanning}
className="h-8 flex-1 text-xs"
>
<HugeiconsIcon icon={ReloadIcon} className="mr-1.5 size-3.5" />
Keep planning
</Button>
</div>
<p className="text-muted-foreground text-[10px]">
Go edit switches to edit mode and lets the agent execute the plan. Keep
planning keeps writes blocked.
</p>
</div>
);
}

View File

@ -0,0 +1,143 @@
"use client";
import { HugeiconsIcon } from "@hugeicons/react";
import {
CheckmarkCircle02Icon,
CancelCircleIcon,
Loading03Icon,
MinusSignIcon,
CheckListIcon,
} from "@hugeicons/core-free-icons";
import { ScrollArea } from "@/components/ui/scroll-area";
import { usePlanStore } from "@/stores/plan-store";
import { useAgentStore } from "@/stores/agent-store";
import { cn } from "@/utils/ui";
import type { PlanStep, PlanStepStatus } from "@/agent/types";
const STATUS_CONFIG: Record<
PlanStepStatus,
{ icon: typeof CheckmarkCircle02Icon; className: string }
> = {
pending: {
icon: MinusSignIcon,
className: "text-muted-foreground",
},
in_progress: {
icon: Loading03Icon,
className: "text-blue-400 animate-spin",
},
done: {
icon: CheckmarkCircle02Icon,
className: "text-emerald-400",
},
skipped: {
icon: CancelCircleIcon,
className: "text-muted-foreground/50",
},
};
function StepItem({ step }: { step: PlanStep }) {
const config = STATUS_CONFIG[step.status];
return (
<div
className={cn(
"flex items-start gap-2 rounded-md px-2 py-1.5 text-xs",
step.status === "skipped" && "opacity-50",
)}
>
<HugeiconsIcon
icon={config.icon}
className={cn("mt-0.5 size-3.5 shrink-0", config.className)}
strokeWidth={2}
/>
<div className="flex flex-col gap-0.5">
<span
className={cn(
step.status === "done" && "text-muted-foreground line-through",
)}
>
{step.description}
</span>
{step.result && (
<span className="text-muted-foreground text-[10px]">
{step.result}
</span>
)}
</div>
</div>
);
}
export function PlanPanel() {
const plan = usePlanStore((s) => s.plan);
const mode = useAgentStore((s) => s.mode);
if (!plan) {
return (
<div className="flex h-full flex-col items-center justify-center gap-2 p-4">
<HugeiconsIcon
icon={CheckListIcon}
className="text-muted-foreground/50 size-8"
strokeWidth={1}
/>
<p className="text-muted-foreground text-center text-xs">
{mode === "plan"
? "Analyzing footage to build a plan..."
: "No active plan"}
</p>
</div>
);
}
const doneCount = plan.steps.filter((s) => s.status === "done").length;
const totalCount = plan.steps.length;
const progress = totalCount > 0 ? (doneCount / totalCount) * 100 : 0;
return (
<div className="flex h-full flex-col">
<div className="flex items-center justify-between border-b px-3 py-2">
<div className="flex flex-col">
<span className="text-xs font-medium">Plan</span>
<span className="text-muted-foreground text-[10px]">
{plan.summary}
</span>
</div>
<div className="flex items-center gap-1.5">
<div className="bg-muted h-1.5 w-16 overflow-hidden rounded-full">
<div
className="bg-emerald-400 h-full rounded-full transition-all duration-300"
style={{ width: `${progress}%` }}
/>
</div>
<span className="text-muted-foreground text-[10px] tabular-nums">
{doneCount}/{totalCount}
</span>
</div>
</div>
<ScrollArea className="flex-1 p-2">
<div className="flex flex-col gap-0.5">
{plan.steps.map((step) => (
<StepItem key={step.id} step={step} />
))}
</div>
</ScrollArea>
{plan.questions && plan.questions.length > 0 && (
<div className="border-t px-3 py-2">
<span className="text-muted-foreground text-[10px] font-medium uppercase">
Open questions
</span>
<ul className="mt-1 flex flex-col gap-0.5">
{plan.questions.map((q) => (
<li key={q} className="text-muted-foreground text-[11px]">
{q}
</li>
))}
</ul>
</div>
)}
</div>
);
}

View File

@ -1,5 +1,10 @@
import { create } from "zustand";
import type { AgentContext, ExecutionState, ToolCall } from "@/agent/types";
import type {
AgentContext,
AgentMode,
ExecutionState,
ToolCall,
} from "@/agent/types";
export type PermissionMode = "ask" | "skip";
@ -8,6 +13,11 @@ export interface PendingToolApproval {
resolve: (approved: boolean) => void;
}
export interface PendingModeTransition {
targetMode: AgentMode;
resolve: (approved: boolean) => void;
}
const DEFAULT_CONTEXT: AgentContext = {
projectId: null,
activeSceneId: null,
@ -21,11 +31,15 @@ interface AgentState {
context: AgentContext;
permissionMode: PermissionMode;
pendingApproval: PendingToolApproval | null;
mode: AgentMode;
pendingModeTransition: PendingModeTransition | null;
setStatus: (status: ExecutionState) => void;
setActiveTool: (tool: string | null) => void;
setContext: (context: AgentContext) => void;
setPermissionMode: (mode: PermissionMode) => void;
setPendingApproval: (pending: PendingToolApproval | null) => void;
setMode: (mode: AgentMode) => void;
setPendingModeTransition: (transition: PendingModeTransition | null) => void;
reset: () => void;
}
@ -35,16 +49,23 @@ export const useAgentStore = create<AgentState>()((set) => ({
context: DEFAULT_CONTEXT,
permissionMode: "skip",
pendingApproval: null,
mode: "execute",
pendingModeTransition: null,
setStatus: (status) => set({ status }),
setActiveTool: (tool) => set({ activeTool: tool }),
setContext: (context) => set({ context }),
setPermissionMode: (mode) => set({ permissionMode: mode }),
setPendingApproval: (pending) => set({ pendingApproval: pending }),
setMode: (mode) => set({ mode }),
setPendingModeTransition: (transition) =>
set({ pendingModeTransition: transition }),
reset: () =>
set({
status: "idle",
activeTool: null,
context: DEFAULT_CONTEXT,
pendingApproval: null,
mode: "execute",
pendingModeTransition: null,
}),
}));

View File

@ -0,0 +1,90 @@
import { create } from "zustand";
import { nanoid } from "nanoid";
import type {
AgentQuestion,
Plan,
PlanStepStatus,
PlanStatus,
} from "@/agent/types";
interface PlanState {
plan: Plan | null;
pendingQuestion: AgentQuestion | null;
modeTransitionPending: boolean;
setPlan: (plan: Plan) => void;
updatePlanStatus: (status: PlanStatus) => void;
updateStepStatus: (
stepId: string,
status: PlanStepStatus,
result?: string,
) => void;
clearPlan: () => void;
setPendingQuestion: (question: AgentQuestion | null) => void;
answerQuestion: (answer: string) => void;
setModeTransitionPending: (pending: boolean) => void;
}
export const usePlanStore = create<PlanState>()((set, get) => ({
plan: null,
pendingQuestion: null,
modeTransitionPending: false,
setPlan: (plan) => set({ plan }),
updatePlanStatus: (status) => {
const current = get().plan;
if (!current) return;
set({ plan: { ...current, status } });
},
updateStepStatus: (stepId, status, result) => {
const current = get().plan;
if (!current) return;
set({
plan: {
...current,
steps: current.steps.map((s) =>
s.id === stepId
? { ...s, status, ...(result !== undefined && { result }) }
: s,
),
},
});
},
clearPlan: () => set({ plan: null, pendingQuestion: null }),
setPendingQuestion: (question) => set({ pendingQuestion: question }),
answerQuestion: (answer) => {
const current = get().pendingQuestion;
if (!current) return;
set({ pendingQuestion: { ...current, answer } });
},
setModeTransitionPending: (pending) =>
set({ modeTransitionPending: pending }),
}));
export function createPlanFromSteps(
summary: string,
steps: Array<{ description: string; tools: string[] }>,
questions?: string[],
): Plan {
return {
id: nanoid(),
summary,
steps: steps.map((s) => ({
id: nanoid(),
description: s.description,
tools: s.tools,
status: "pending" as PlanStepStatus,
})),
questions,
status: "awaiting_approval" as PlanStatus,
createdAt: Date.now(),
};
}

View File

@ -1,6 +1,6 @@
# NeuralCut — Agent Tools Inventory
Complete inventory of all tools, effects, masks, and animatable properties available to the video editor agent.
Complete inventory of all tools, skills, effects, masks, and animatable properties available to the video editor agent.
**Source of truth**: `apps/web/src/agent/tools/schemas.ts` — the `providerToolSchemas` array defines what gets sent to the LLM.
**Execution**: `apps/web/src/agent/context.ts` (EditorContextAdapter).
@ -8,7 +8,7 @@ Complete inventory of all tools, effects, masks, and animatable properties avail
---
## 25 Provider-Facing Tools
## 31 Provider-Facing Tools
### Context & Perception (read-only)
@ -74,6 +74,23 @@ Complete inventory of all tools, effects, masks, and animatable properties avail
| Tool | Description | Schema | Implementation |
|------|-------------|--------|----------------|
| `undo` | Undoes the last editing action. Returns remaining undo depth. | `schemas.ts:195` | `undo.tool.ts` |
| `redo` | Redoes the last undone action. Only works after an undo. Returns whether there are more actions to redo. | `schemas.ts:174` | `redo.tool.ts` |
### Skills (recipe-based editing)
| Tool | Description | Schema | Implementation |
|------|-------------|--------|----------------|
| `list_skills` | Lists available editing skill recipes — pre-built workflows for common patterns like viral shorts, pitch videos. Returns skill id, name, and description. | `schemas.ts:294` | `skills/list-skills.tool.ts` |
| `load_skill` | Loads full instructions for a specific skill by id. Contains recipe with sections, timing rules, text styles, effect parameters, and quality checklist. | `schemas.ts:301` | `skills/load-skill.tool.ts` |
### Plan Mode (structured workflow)
| Tool | Description | Schema | Implementation |
|------|-------------|--------|----------------|
| `submit_plan` | Submits a structured editing plan with steps. Use ONLY in plan mode after analyzing footage. Each step describes a specific action with tools to use. User reviews and approves before execution. | `schemas.ts:308` | `submit-plan.tool.ts` |
| `ask_user` | Asks the user a question during plan mode. Use when clarification is needed about intent, preferences, or content details. Optionally provides quick-reply options. | `schemas.ts:319` | `ask-user.tool.ts` |
| `request_plan_approval` | Requests user approval to switch from plan mode to execute mode. Shows modal with 'Continue planning' and 'Start editing' options. Requires explicit user approval. | `schemas.ts:329` | `request-plan-approval.tool.ts` |
| `update_plan_step` | Updates the status of a plan step during execution. Call after completing each step to track progress. Use status 'done', 'skipped', or 'in_progress'. | `schemas.ts:336` | `update-plan-step.tool.ts` |
---
@ -84,7 +101,16 @@ Complete inventory of all tools, effects, masks, and animatable properties avail
| `transcribe_video` | Transcribes audio using Whisper (local). Returns structured transcript with segments/timestamps. Registered in toolRegistry but excluded from `providerToolSchemas` (`schemas.ts:296-297`). Used internally by the agent orchestrator. | `transcribe-video.tool.ts` |
| `echo_context` | Debug tool that returns a summary of the current editor context. Registered in toolRegistry but not in `providerToolSchemas`. | `mock.tool.ts` |
> **Note**: `redo` has a schema (`schemas.ts:174`) and implementation (`redo.tool.ts`) but is **NOT** included in `providerToolSchemas`.
---
## 2 Built-in Skills
Defined in `apps/web/src/agent/skills/builtin/`.
| Skill ID | Name | Description |
|----------|------|-------------|
| `viral-short` | Viral Short | Creates a short-form viral video with hook, clips, text overlays, and effects |
| `pitch-video` | Pitch Video | Creates a professional pitch video with intro, key points, transitions, and outro |
---
@ -200,6 +226,8 @@ Dynamic path types for extension:
| `apps/web/src/agent/tools/schemas.ts` | Single source of truth for all tool schemas + `providerToolSchemas` array |
| `apps/web/src/agent/tools/index.ts` | Barrel that registers all tools via side-effect imports |
| `apps/web/src/agent/tools/registry.ts` | Generic `DefinitionRegistry<string, ToolDefinition>` |
| `apps/web/src/agent/skills/index.ts` | Barrel that registers all skills via side-effect imports |
| `apps/web/src/agent/skills/registry.ts` | Skill definition registry |
| `apps/web/src/agent/orchestrator.ts` | Agent loop: sends to LLM, resolves tool calls, up to 20 iterations |
| `apps/web/src/agent/context.ts` | `EditorContextAdapter` — the ONLY file that imports EditorCore. All tool execution logic. |
| `apps/web/src/agent/system-prompt.ts` | Builds system prompt from context + tool summaries |
@ -214,3 +242,5 @@ Dynamic path types for extension:
4. **Agent must call `list_project_assets` and `list_timeline` before editing** when it lacks concrete IDs.
5. **Gemini handles multimodal understanding**; editing is deterministic via tools.
6. **`split` does NOT delete** — compose `split` + `delete_timeline_elements` for range removal.
7. **Plan mode** enables structured workflow: submit_plan → request_plan_approval → update_plan_step for execution tracking.
8. **Skills** provide pre-built recipes for common editing patterns, composable with standard tools.