From 4c5e151822f4911174a8d4fdde1d3e5efe3bbe6d Mon Sep 17 00:00:00 2001 From: Luis Esteban Acevedo Ladino Date: Fri, 24 Apr 2026 20:10:12 -0500 Subject: [PATCH] ft: add agent split tool --- .../agent/__tests__/context-mapper.test.ts | 10 +- apps/web/src/agent/context-mapper.ts | 21 ++-- apps/web/src/agent/context.ts | 34 +++++++ apps/web/src/agent/orchestrator.ts | 12 ++- .../agent/providers/__tests__/gemini.test.ts | 25 ++++- apps/web/src/agent/providers/gemini.ts | 37 ++++--- .../src/agent/tools/__tests__/split.test.ts | 69 +++++++++++++ .../web/src/agent/tools/list-timeline.tool.ts | 2 +- apps/web/src/agent/tools/split.tool.ts | 43 +++++++++ apps/web/src/agent/types.ts | 2 + .../api/agent/chat/__tests__/route.test.ts | 21 +++- apps/web/src/app/api/agent/chat/route.ts | 41 ++++++-- .../web/src/core/managers/timeline-manager.ts | 30 +++--- .../lib/commands/timeline/element/index.ts | 1 + .../lib/commands/timeline/element/split.ts | 96 +++++++++++++++++++ docs/agent-tool-specs.md | 20 ++-- docs/agent-tools.md | 23 ++--- propuesta_tecnica.md | 24 ++--- 18 files changed, 428 insertions(+), 83 deletions(-) create mode 100644 apps/web/src/agent/tools/__tests__/split.test.ts create mode 100644 apps/web/src/agent/tools/split.tool.ts create mode 100644 apps/web/src/lib/commands/timeline/element/split.ts diff --git a/apps/web/src/agent/__tests__/context-mapper.test.ts b/apps/web/src/agent/__tests__/context-mapper.test.ts index 35d2a16b..bb7d19ea 100644 --- a/apps/web/src/agent/__tests__/context-mapper.test.ts +++ b/apps/web/src/agent/__tests__/context-mapper.test.ts @@ -107,8 +107,8 @@ describe("buildContextFromEditorState (context mapper)", () => { type: "video", mediaId: "m1", name: "Intro", - startTime: 2, - duration: 5, + startTime: 200, + duration: 500, }, ], }, @@ -122,8 +122,8 @@ describe("buildContextFromEditorState (context mapper)", () => { type: "text", name: "Caption", content: "Hello world", - startTime: 3, - duration: 2, + startTime: 300, + duration: 200, }, ], }, @@ -139,7 +139,7 @@ describe("buildContextFromEditorState (context mapper)", () => { mediaId: "m2", name: "Music", startTime: 0, - duration: 10, + duration: 1000, }, ], }, diff --git a/apps/web/src/agent/context-mapper.ts b/apps/web/src/agent/context-mapper.ts index 4ba62dcb..09aef1fe 100644 --- a/apps/web/src/agent/context-mapper.ts +++ b/apps/web/src/agent/context-mapper.ts @@ -25,7 +25,10 @@ export function buildContextFromEditorState(params: { duration: a.duration ?? 0, usedInTimeline: usedMediaIds.has(a.id), })), - timelineTracks: buildTimelineTracks(params.activeScene), + timelineTracks: buildTimelineTracks( + params.activeScene, + params.ticksPerSecond, + ), playbackTimeMs: Math.round( (params.currentTimeTicks / params.ticksPerSecond) * 1000, ), @@ -49,6 +52,7 @@ type TrackInput = { function buildTimelineTracks( scene: ActiveSceneInput | null, + ticksPerSecond: number, ): AgentTimelineTrack[] | undefined { if (!scene?.tracks) { return undefined; @@ -61,7 +65,7 @@ function buildTimelineTracks( for (let index = 0; index < overlayTracks.length; index++) { const track = overlayTracks[index]; tracks.push( - toTimelineTrack(track, mapOverlayTrackType(track.type), { + toTimelineTrack(track, mapOverlayTrackType(track.type), ticksPerSecond, { position: position++, visualLayer: overlayTracks.length - index, isVisualLayer: true, @@ -72,7 +76,7 @@ function buildTimelineTracks( if (scene.tracks.main) { tracks.push( - toTimelineTrack(scene.tracks.main, "main", { + toTimelineTrack(scene.tracks.main, "main", ticksPerSecond, { position: position++, visualLayer: 0, isVisualLayer: true, @@ -83,7 +87,7 @@ function buildTimelineTracks( for (const track of scene.tracks.audio ?? []) { tracks.push( - toTimelineTrack(track, "audio", { + toTimelineTrack(track, "audio", ticksPerSecond, { position: position++, visualLayer: null, isVisualLayer: false, @@ -98,6 +102,7 @@ function buildTimelineTracks( function toTimelineTrack( track: TrackInput, type: AgentTimelineTrack["type"], + ticksPerSecond: number, stacking: Pick< AgentTimelineTrack, "position" | "visualLayer" | "isVisualLayer" | "stacking" @@ -115,12 +120,16 @@ function toTimelineTrack( ...(hasMediaId(element) ? { assetId: element.mediaId } : {}), ...(element.name ? { name: element.name } : {}), ...(hasTextContent(element) ? { content: element.content } : {}), - start: element.startTime, - end: element.startTime + element.duration, + start: toSeconds(element.startTime, ticksPerSecond), + end: toSeconds(element.startTime + element.duration, ticksPerSecond), })), }; } +function toSeconds(ticks: number, ticksPerSecond: number): number { + return ticks / ticksPerSecond; +} + function mapOverlayTrackType( type: string | undefined, ): AgentTimelineTrack["type"] { diff --git a/apps/web/src/agent/context.ts b/apps/web/src/agent/context.ts index d419f0cc..8b4710a0 100644 --- a/apps/web/src/agent/context.ts +++ b/apps/web/src/agent/context.ts @@ -1,6 +1,7 @@ import { EditorCore } from "@/core"; import { TICKS_PER_SECOND } from "@/lib/wasm"; import { buildContextFromEditorState } from "@/agent/context-mapper"; +import type { SceneTracks } from "@/lib/timeline"; /** * Thin adapter: the ONLY file in agent/ that imports from core/. @@ -50,6 +51,39 @@ export const EditorContextAdapter = { const asset = core.media.getAssets().find((a) => a.id === assetId); return asset?.hasAudio; }, + + splitTimeline({ + times, + }: { + times: number[]; + }): { success: boolean; affectedElements: string[] } | { error: string } { + const core = EditorCore.getInstance(); + const activeScene = core.scenes.getActiveSceneOrNull(); + if (!activeScene) { + return { error: "No active timeline" }; + } + + if (!hasTimelineContent(activeScene.tracks)) { + return { error: "No timeline content" }; + } + + const affectedElements = core.timeline.split({ + times: times.map((time) => secondsToTicks(time)), + }); + return { success: true, affectedElements }; + }, }; +function secondsToTicks(seconds: number): number { + return Math.round(seconds * TICKS_PER_SECOND); +} + +function hasTimelineContent(tracks: SceneTracks): boolean { + return ( + tracks.main.elements.length > 0 || + tracks.overlay.some((track) => track.elements.length > 0) || + tracks.audio.some((track) => track.elements.length > 0) + ); +} + export { buildSystemPrompt } from "@/agent/system-prompt"; diff --git a/apps/web/src/agent/orchestrator.ts b/apps/web/src/agent/orchestrator.ts index 43855d33..a3164e82 100644 --- a/apps/web/src/agent/orchestrator.ts +++ b/apps/web/src/agent/orchestrator.ts @@ -9,6 +9,7 @@ import { toolRegistry } from "@/agent/tools/registry"; import "@/agent/tools/load-context.tool"; import "@/agent/tools/list-project-assets.tool"; import "@/agent/tools/list-timeline.tool"; +import "@/agent/tools/split.tool"; import { useChatStore } from "@/stores/chat-store"; import { useAgentStore } from "@/stores/agent-store"; @@ -52,9 +53,7 @@ export async function run( } catch { detail = response.statusText; } - throw new Error( - `API error ${response.status}: ${detail}`, - ); + throw new Error(`API error ${response.status}: ${detail}`); } const data: APIResponse = await response.json(); @@ -161,6 +160,13 @@ function validateToolArgs( if (param.type === "boolean" && actualType !== "boolean") { return `Argument "${param.key}" must be a boolean, got ${actualType}`; } + if ( + param.type === "number[]" && + (!Array.isArray(value) || + !value.every((item) => typeof item === "number")) + ) { + return `Argument "${param.key}" must be an array of numbers`; + } } return null; diff --git a/apps/web/src/agent/providers/__tests__/gemini.test.ts b/apps/web/src/agent/providers/__tests__/gemini.test.ts index 691f0840..2b4e30e3 100644 --- a/apps/web/src/agent/providers/__tests__/gemini.test.ts +++ b/apps/web/src/agent/providers/__tests__/gemini.test.ts @@ -21,6 +21,7 @@ jest.mock("@google/generative-ai", () => ({ NUMBER: "number", BOOLEAN: "boolean", OBJECT: "object", + ARRAY: "array", }, })); @@ -308,6 +309,7 @@ describe("toGeminiTools", () => { parameters: [ { key: "name", type: "string", required: true }, { key: "count", type: "number", required: false }, + { key: "times", type: "number[]", required: false }, { key: "active", type: "boolean", required: false }, { key: "meta", type: "object", required: false }, ], @@ -317,11 +319,13 @@ describe("toGeminiTools", () => { const declarations = toGeminiTools(tools); const props = declarations[0].parameters?.properties as Record< string, - { type: string } + { type: string; items?: { type: string } } >; expect(props.name.type).toBe(SchemaType.STRING); expect(props.count.type).toBe(SchemaType.NUMBER); + expect(props.times.type).toBe(SchemaType.ARRAY); + expect(props.times.items?.type).toBe(SchemaType.NUMBER); expect(props.active.type).toBe(SchemaType.BOOLEAN); expect(props.meta.type).toBe(SchemaType.OBJECT); }); @@ -612,4 +616,23 @@ describe("GeminiAdapter.chat()", () => { expect(mockGenerateContent).toHaveBeenCalledTimes(1); }); + + test("does not retry depleted billing credits", async () => { + const providerError = Object.assign( + new Error("Your prepayment credits are depleted. Manage billing."), + { status: 429 }, + ); + mockGenerateContent.mockRejectedValueOnce(providerError); + + const adapter = new GeminiAdapter(TEST_CONFIG); + await expect( + adapter.chat({ + messages: [makeChatMessage("user", "Hi")], + systemPrompt: "", + tools: [], + }), + ).rejects.toThrow("prepayment credits are depleted"); + + expect(mockGenerateContent).toHaveBeenCalledTimes(1); + }); }); diff --git a/apps/web/src/agent/providers/gemini.ts b/apps/web/src/agent/providers/gemini.ts index 5afc2b1e..a4a7006e 100644 --- a/apps/web/src/agent/providers/gemini.ts +++ b/apps/web/src/agent/providers/gemini.ts @@ -4,6 +4,7 @@ import { type FunctionDeclaration, type FunctionDeclarationSchema, type GenerateContentResult, + type Schema, SchemaType, } from "@google/generative-ai"; import type { ChatMessage, ToolCall, ToolSchema } from "@/agent/types"; @@ -17,16 +18,6 @@ import type { // Internal conversion helpers // --------------------------------------------------------------------------- -/** - * Maps internal type strings to Gemini SchemaType values. - */ -const TYPE_MAP: Record = { - string: SchemaType.STRING, - number: SchemaType.NUMBER, - boolean: SchemaType.BOOLEAN, - object: SchemaType.OBJECT, -}; - const GEMINI_MAX_ATTEMPTS = 5; const GEMINI_RETRY_BASE_DELAY_MS = process.env.NODE_ENV === "test" ? 0 : 1000; const GEMINI_RETRYABLE_STATUSES = new Set([429, 500, 502, 503, 504]); @@ -42,9 +33,20 @@ function getProviderStatus(error: unknown): number | null { function isRetryableProviderError(error: unknown): boolean { const status = getProviderStatus(error); + if (status === 429 && isBillingOrQuotaExhaustedError(error)) { + return false; + } + return status !== null && GEMINI_RETRYABLE_STATUSES.has(status); } +function isBillingOrQuotaExhaustedError(error: unknown): boolean { + const message = error instanceof Error ? error.message : String(error); + return /prepayment credits are depleted|billing|quota.*(?:depleted|exhausted)/i.test( + message, + ); +} + function wait(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } @@ -230,9 +232,7 @@ export function toGeminiTools(tools: ToolSchema[]): FunctionDeclaration[] { const properties: FunctionDeclarationSchema["properties"] = {}; for (const param of tool.parameters) { - const geminiType = TYPE_MAP[param.type] ?? SchemaType.STRING; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - (properties as Record)[param.key] = { type: geminiType }; + properties[param.key] = toGeminiParameterSchema(param.type); } const requiredParams = tool.parameters @@ -251,6 +251,17 @@ export function toGeminiTools(tools: ToolSchema[]): FunctionDeclaration[] { }); } +function toGeminiParameterSchema(type: string): Schema { + if (type === "number") return { type: SchemaType.NUMBER }; + if (type === "boolean") return { type: SchemaType.BOOLEAN }; + if (type === "object") return { type: SchemaType.OBJECT, properties: {} }; + if (type === "number[]") { + return { type: SchemaType.ARRAY, items: { type: SchemaType.NUMBER } }; + } + + return { type: SchemaType.STRING }; +} + /** * Converts a Gemini GenerateContentResult into the provider-agnostic * ProviderResponse shape. diff --git a/apps/web/src/agent/tools/__tests__/split.test.ts b/apps/web/src/agent/tools/__tests__/split.test.ts new file mode 100644 index 00000000..261348f9 --- /dev/null +++ b/apps/web/src/agent/tools/__tests__/split.test.ts @@ -0,0 +1,69 @@ +import { beforeEach, describe, expect, mock, test } from "bun:test"; +import type { AgentContext } from "@/agent/types"; +import { toolRegistry } from "@/agent/tools/registry"; + +const mockSplitTimeline = mock((_args: { times: number[] }) => ({ + success: true, + affectedElements: ["clip-1", "clip-2"], +})); + +mock.module("@/agent/context", () => ({ + EditorContextAdapter: { + splitTimeline: mockSplitTimeline, + }, +})); + +await import("@/agent/tools/split.tool"); + +const context: AgentContext = { + projectId: "proj-1", + activeSceneId: "scene-A", + mediaAssets: [], + playbackTimeMs: 0, +}; + +describe("split tool", () => { + beforeEach(() => { + mockSplitTimeline.mockClear(); + }); + + test("is registered in the tool registry", () => { + expect(toolRegistry.has("split")).toBe(true); + }); + + test("splits the timeline through the editor adapter", async () => { + const tool = toolRegistry.get("split"); + const result = await tool.execute({ times: [2] }, context); + + expect(mockSplitTimeline).toHaveBeenCalledWith({ times: [2] }); + expect(result).toEqual({ + success: true, + affectedElements: ["clip-1", "clip-2"], + }); + }); + + test("supports multiple split times", async () => { + const tool = toolRegistry.get("split"); + await tool.execute({ times: [2, 5, 8] }, context); + + expect(mockSplitTimeline).toHaveBeenCalledWith({ times: [2, 5, 8] }); + }); + + test("validates split times", async () => { + const tool = toolRegistry.get("split"); + + expect(await tool.execute({ times: [] }, context)).toEqual({ + error: "Invalid split times", + }); + expect(await tool.execute({ times: [2, Number.NaN] }, context)).toEqual({ + error: "Invalid split times", + }); + expect(await tool.execute({ times: [-1] }, context)).toEqual({ + error: "Invalid split times", + }); + expect(await tool.execute({ times: "2" }, context)).toEqual({ + error: "Invalid split times", + }); + expect(mockSplitTimeline).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/web/src/agent/tools/list-timeline.tool.ts b/apps/web/src/agent/tools/list-timeline.tool.ts index 6176980e..789d2f8b 100644 --- a/apps/web/src/agent/tools/list-timeline.tool.ts +++ b/apps/web/src/agent/tools/list-timeline.tool.ts @@ -12,7 +12,7 @@ export type ListTimelineResult = { const listTimelineTool: ToolDefinition = { name: "list_timeline", description: - "Lists the active timeline as structured tracks and editable elements with layer metadata. position is timeline row order from top to bottom; visualLayer is render stacking where higher renders above lower, and audio tracks have null visualLayer.", + "Lists the active timeline as structured tracks and editable elements with layer metadata. Element start/end values are in seconds. position is timeline row order from top to bottom; visualLayer is render stacking where higher renders above lower, and audio tracks have null visualLayer.", parameters: [], execute: async ( _args: Record, diff --git a/apps/web/src/agent/tools/split.tool.ts b/apps/web/src/agent/tools/split.tool.ts new file mode 100644 index 00000000..4efa1f7a --- /dev/null +++ b/apps/web/src/agent/tools/split.tool.ts @@ -0,0 +1,43 @@ +import { EditorContextAdapter } from "@/agent/context"; +import type { AgentContext, ToolDefinition } from "@/agent/types"; +import { toolRegistry } from "@/agent/tools/registry"; + +export type SplitArgs = { + times: number[]; +}; + +export type SplitResult = { + success: boolean; + affectedElements: string[]; +}; + +const splitTool: ToolDefinition = { + name: "split", + description: + "Splits timeline elements at one or more requested times without deleting, trimming, or moving content. Use one time for a single cut, or multiple times to isolate ranges before separate edit/delete operations.", + parameters: [{ key: "times", type: "number[]", required: true }], + execute: async ( + args: Record, + _context: AgentContext, + ): Promise => { + const times = args.times; + + if (!isValidTimes(times)) { + return { error: "Invalid split times" }; + } + + return EditorContextAdapter.splitTimeline({ times }); + }, +}; + +function isValidTimes(times: unknown): times is number[] { + return ( + Array.isArray(times) && + times.length > 0 && + times.every( + (time) => typeof time === "number" && Number.isFinite(time) && time >= 0, + ) + ); +} + +toolRegistry.register("split", splitTool); diff --git a/apps/web/src/agent/types.ts b/apps/web/src/agent/types.ts index a34d3bad..c8fa448f 100644 --- a/apps/web/src/agent/types.ts +++ b/apps/web/src/agent/types.ts @@ -58,7 +58,9 @@ export type AgentTimelineTrack = { assetId?: string; name?: string; content?: string; + /** Timeline start in seconds. */ start: number; + /** Timeline end in seconds. */ end: number; }>; }; diff --git a/apps/web/src/app/api/agent/chat/__tests__/route.test.ts b/apps/web/src/app/api/agent/chat/__tests__/route.test.ts index 6b8bd5e1..f9ed25b1 100644 --- a/apps/web/src/app/api/agent/chat/__tests__/route.test.ts +++ b/apps/web/src/app/api/agent/chat/__tests__/route.test.ts @@ -152,13 +152,14 @@ describe("POST /api/agent/chat", () => { expect(callArgs.messages).toHaveLength(1); expect(callArgs.systemPrompt).toContain("NeuralCut"); - expect(callArgs.tools).toHaveLength(3); + expect(callArgs.tools).toHaveLength(4); expect( callArgs.tools.map((tool) => (tool as { name: string }).name), ).toEqual([ "load_context", "list_project_assets", "list_timeline", + "split", ]); }); @@ -240,4 +241,22 @@ describe("POST /api/agent/chat", () => { const data = await res.json(); expect(data.error).toBe("LLM provider error"); }); + + test("returns clear error when Gemini credits are depleted", async () => { + mockChat.mockRejectedValueOnce( + Object.assign( + new Error( + "[GoogleGenerativeAI Error]: [429 Too Many Requests] Your prepayment credits are depleted.", + ), + { status: 429 }, + ), + ); + + const res = await POST(makeRequest(VALID_BODY)); + expect(res.status).toBe(402); + + const data = await res.json(); + expect(data.error).toContain("Gemini credits are depleted"); + expect(data.providerStatus).toBe(429); + }); }); diff --git a/apps/web/src/app/api/agent/chat/route.ts b/apps/web/src/app/api/agent/chat/route.ts index cf42895b..cba16b5b 100644 --- a/apps/web/src/app/api/agent/chat/route.ts +++ b/apps/web/src/app/api/agent/chat/route.ts @@ -34,9 +34,15 @@ const providerToolSchemas: ToolSchema[] = [ { name: "list_timeline", description: - "Lists the active timeline as structured tracks and editable elements with layer metadata. Tracks include position (top-to-bottom timeline row), visualLayer (higher renders above lower, null for audio), isVisualLayer, and stacking. Use this to understand which clips visually cover others and to discover ids before load_context.", + "Lists the active timeline as structured tracks and editable elements with layer metadata. Element start/end values are in seconds. Tracks include position (top-to-bottom timeline row), visualLayer (higher renders above lower, null for audio), isVisualLayer, and stacking. Use this to understand which clips visually cover others and to discover ids before load_context.", parameters: [], }, + { + name: "split", + description: + "Splits timeline elements at one or more requested timeline times in seconds without deleting, trimming, or moving content. Use one time for a single cut, or multiple times to isolate ranges before separate edit/delete operations.", + parameters: [{ key: "times", type: "number[]", required: true }], + }, ]; // --------------------------------------------------------------------------- @@ -160,16 +166,39 @@ export async function POST(request: NextRequest) { const status = (error as { status?: number }).status; const message = error instanceof Error ? error.message : "Unknown provider error"; + const providerError = toProviderErrorResponse({ message, status }); console.error("[agent/chat] Provider error:", error); return NextResponse.json( { - error: "LLM provider error", + error: providerError.error, ...(status ? { providerStatus: status } : {}), - ...(process.env.NODE_ENV === "development" - ? { detail: message } - : {}), + ...(process.env.NODE_ENV === "development" ? { detail: message } : {}), }, - { status: 502 }, + { status: providerError.status }, ); } } + +function toProviderErrorResponse({ + message, + status, +}: { + message: string; + status?: number; +}): { error: string; status: number } { + if (status === 429 && isBillingOrQuotaExhaustedMessage(message)) { + return { + status: 402, + error: + "Gemini credits are depleted. Add credits in AI Studio or switch LLM_PROVIDER/LLM_MODEL in .env.local.", + }; + } + + return { status: 502, error: "LLM provider error" }; +} + +function isBillingOrQuotaExhaustedMessage(message: string): boolean { + return /prepayment credits are depleted|billing|quota.*(?:depleted|exhausted)/i.test( + message, + ); +} diff --git a/apps/web/src/core/managers/timeline-manager.ts b/apps/web/src/core/managers/timeline-manager.ts index 70cd0439..fd7f9dcc 100644 --- a/apps/web/src/core/managers/timeline-manager.ts +++ b/apps/web/src/core/managers/timeline-manager.ts @@ -8,9 +8,7 @@ import type { RetimeConfig, } from "@/lib/timeline"; import { calculateTotalDuration } from "@/lib/timeline"; -import { - findTrackInSceneTracks, -} from "@/lib/timeline/track-element-update"; +import { findTrackInSceneTracks } from "@/lib/timeline/track-element-update"; import { canElementBeHidden, canElementHaveAudio, @@ -39,6 +37,7 @@ import { DuplicateElementsCommand, UpdateElementsCommand, SplitElementsCommand, + SplitCommand, MoveElementCommand, TracksSnapshotCommand, UpsertKeyframeCommand, @@ -200,6 +199,12 @@ export class TimelineManager { return command.getRightSideElements(); } + split({ times }: { times: number[] }): string[] { + const command = new SplitCommand(times); + this.editor.command.execute({ command }); + return command.getAffectedElements(); + } + getTotalDuration(): number { const activeScene = this.editor.scenes.getActiveSceneOrNull(); if (!activeScene) { @@ -580,14 +585,7 @@ export class TimelineManager { } const commands = keyframes.map( - ({ - trackId, - elementId, - propertyPath, - componentKey, - keyframeId, - patch, - }) => + ({ trackId, elementId, propertyPath, componentKey, keyframeId, patch }) => new UpdateScalarKeyframeCurveCommand({ trackId, elementId, @@ -711,7 +709,9 @@ export class TimelineManager { private applyPreviewOverlay(tracks: SceneTracks): SceneTracks { if (this.previewOverlay.size === 0) return tracks; - const applyTrackOverlay = (track: TTrack): TTrack => { + const applyTrackOverlay = ( + track: TTrack, + ): TTrack => { const hasOverlay = track.elements.some((element) => this.previewOverlay.has(element.id), ); @@ -803,7 +803,11 @@ export class TimelineManager { } getPreviewTracks(): SceneTracks | null { - return this.previewTracks ?? this.editor.scenes.getActiveSceneOrNull()?.tracks ?? null; + return ( + this.previewTracks ?? + this.editor.scenes.getActiveSceneOrNull()?.tracks ?? + null + ); } subscribe(listener: () => void): () => void { diff --git a/apps/web/src/lib/commands/timeline/element/index.ts b/apps/web/src/lib/commands/timeline/element/index.ts index ff365801..6920decf 100644 --- a/apps/web/src/lib/commands/timeline/element/index.ts +++ b/apps/web/src/lib/commands/timeline/element/index.ts @@ -2,6 +2,7 @@ export { InsertElementCommand } from "./insert-element"; export { DeleteElementsCommand } from "./delete-elements"; export { DuplicateElementsCommand } from "./duplicate-elements"; export { SplitElementsCommand } from "./split-elements"; +export { SplitCommand } from "./split"; export { UpdateElementsCommand } from "./update-elements"; export { ToggleSourceAudioSeparationCommand } from "./toggle-source-audio-separation"; export { MoveElementCommand } from "./move-elements"; diff --git a/apps/web/src/lib/commands/timeline/element/split.ts b/apps/web/src/lib/commands/timeline/element/split.ts new file mode 100644 index 00000000..45cced4e --- /dev/null +++ b/apps/web/src/lib/commands/timeline/element/split.ts @@ -0,0 +1,96 @@ +import { EditorCore } from "@/core"; +import { Command, type CommandResult } from "@/lib/commands/base-command"; +import type { SceneTracks } from "@/lib/timeline"; +import { SplitElementsCommand } from "./split-elements"; + +export class SplitCommand extends Command { + private savedState: SceneTracks | null = null; + private affectedElements: string[] = []; + + constructor(private readonly times: number[]) { + super(); + } + + getAffectedElements(): string[] { + return this.affectedElements; + } + + execute(): CommandResult | undefined { + const editor = EditorCore.getInstance(); + this.savedState = editor.scenes.getActiveScene().tracks; + const affectedElements = new Set(); + let selectedRightSide: Array<{ trackId: string; elementId: string }> = []; + + for (const time of uniqueSortedTimes(this.times)) { + const tracks = editor.scenes.getActiveScene().tracks; + const targets = findElementsIntersectingTime({ tracks, time }); + for (const target of targets) { + affectedElements.add(target.elementId); + } + + const rightSide = executeSplit({ elements: targets, splitTime: time }); + if (rightSide.length > 0) { + selectedRightSide = rightSide; + } + for (const target of rightSide) { + affectedElements.add(target.elementId); + } + } + + this.affectedElements = [...affectedElements]; + return this.affectedElements.length > 0 + ? { select: selectedRightSide } + : undefined; + } + + undo(): void { + if (!this.savedState) { + return; + } + + EditorCore.getInstance().timeline.updateTracks(this.savedState); + } +} + +function uniqueSortedTimes(times: number[]): number[] { + return [...new Set(times)].sort((a, b) => a - b); +} + +function executeSplit({ + elements, + splitTime, +}: { + elements: Array<{ trackId: string; elementId: string }>; + splitTime: number; +}): Array<{ trackId: string; elementId: string }> { + if (elements.length === 0) { + return []; + } + + const command = new SplitElementsCommand({ elements, splitTime }); + command.execute(); + return command.getRightSideElements(); +} + +function findElementsIntersectingTime({ + tracks, + time, +}: { + tracks: SceneTracks; + time: number; +}): Array<{ trackId: string; elementId: string }> { + const result: Array<{ trackId: string; elementId: string }> = []; + const allTracks = [tracks.main, ...tracks.overlay, ...tracks.audio]; + + for (const track of allTracks) { + for (const element of track.elements) { + const elementStart = element.startTime; + const elementEnd = element.startTime + element.duration; + if (time > elementStart && time < elementEnd) { + result.push({ trackId: track.id, elementId: element.id }); + } + } + } + + return result; +} diff --git a/docs/agent-tool-specs.md b/docs/agent-tool-specs.md index 57db4d62..4789c909 100644 --- a/docs/agent-tool-specs.md +++ b/docs/agent-tool-specs.md @@ -147,17 +147,15 @@ Cargar un asset en el contexto multimodal del agente usando Gemini. Esta es la b --- -## 4. `cut_segment` +## 4. `split` ### Propósito -Cortar o remover un rango de tiempo del timeline. +Dividir elementos del timeline en uno o más puntos de tiempo, sin borrar contenido. Esta primitive equivale a hacer cortes/splits puntuales en el editor; eliminar material debe ser una tool separada y puede componerse después de hacer splits. ### Input ```ts { - start: number; - end: number; - mode: "remove" | "keep"; + times: number[]; // timeline seconds } ``` @@ -170,17 +168,17 @@ Cortar o remover un rango de tiempo del timeline. ``` ### Requirements -- MUST validate `start < end`. -- MUST operate on timeline time, using the editor’s canonical time unit internally. +- MUST validate `times` contains at least one finite number. +- MUST accept timeline times in seconds and convert to the editor’s canonical time unit internally. - MUST use existing timeline/command infrastructure when possible. -- `remove` MUST remove the selected time range. -- `keep` MUST preserve only the selected time range and remove outside material, if feasible in current editor model. +- MUST split every timeline element intersecting each requested time when the split point falls strictly inside the element. +- MUST NOT delete, trim away, or move timeline content. +- MUST be idempotent at existing boundaries: if a requested time already equals an element boundary, it MUST NOT create a duplicate split there. - MUST preserve undo/redo behavior if the editor supports it for the operation. ### Errors -- Invalid range: `{ error: "Invalid time range" }`. +- Invalid times: `{ error: "Invalid split times" }`. - Empty timeline: `{ error: "No timeline content" }`. -- Unsupported mode: `{ error: "Unsupported cut mode" }`. --- diff --git a/docs/agent-tools.md b/docs/agent-tools.md index 85a3f506..83020e1b 100644 --- a/docs/agent-tools.md +++ b/docs/agent-tools.md @@ -89,23 +89,24 @@ Uso: ## Tools de edición primitivas -### `cut_segment` +### `split` -Corta o remueve un rango de tiempo del timeline. +Hace splits/cortes en uno o más puntos de tiempo del timeline, sin borrar contenido. ```ts { - start: number; - end: number; - mode: "remove" | "keep"; + times: number[]; // timeline seconds } ``` Uso: -- quitar silencios, -- quitar errores, -- recortar intro/outro, -- construir highlights. +- cortar en un punto específico, +- aislar una sección antes de borrarla con otra tool usando dos tiempos, +- preparar edición por rangos, +- separar intro/outro sin eliminar nada, +- dejarle al agente una primitive composable para flujos más grandes. + +> `split` NO elimina material. Si el usuario pide “eliminá esta parte”, el agente debe componer `split` con una tool de borrado cuando exista. --- @@ -294,7 +295,7 @@ Descartada porque `analysisType` introduce categorías artificiales. Gemini debe ### `remove_silences` -Diferida/no prioritaria porque se puede expresar como flujo con `cut_segment`. +Diferida/no prioritaria porque se puede expresar como flujo con `split` + tool de borrado. ### `generate_subtitles` @@ -311,7 +312,7 @@ Diferida porque el repo actual no parece tener corrección de color/LUTs impleme 1. `list_project_assets` 2. `list_timeline` 3. `load_asset_context` -4. `cut_segment` +4. `split` 5. `add_text` 6. `add_media_to_timeline` 7. `delete_element` diff --git a/propuesta_tecnica.md b/propuesta_tecnica.md index 1efc129b..32a68039 100644 --- a/propuesta_tecnica.md +++ b/propuesta_tecnica.md @@ -50,7 +50,7 @@ Decisiones vigentes: Roadmap vigente de alto nivel: 1. **Contexto y percepción:** `list_project_assets`, `list_timeline`, `load_asset_context`. -2. **Edición básica:** `cut_segment`, `add_text`, `add_media_to_timeline`, `delete_element`. +2. **Edición básica:** `split`, `add_text`, `add_media_to_timeline`, `delete_element`. 3. **Ajustes simples:** `set_volume`, `add_sticker`, `apply_effect`. 4. **UX avanzada:** referencias `@asset`, progreso de uploads/procesamiento, previews/aprobaciones. @@ -421,7 +421,7 @@ Estas son las tools que el equipo debe priorizar. Son suficientemente pequeñas | `list_project_assets` | Lista assets del proyecto, usados/no usados y por tipo | Prioridad alta | | `list_timeline` | Devuelve resumen estructurado del timeline con `trackId`/`elementId` | Prioridad alta | | `load_asset_context` | Carga un asset en el contexto multimodal de Gemini y cachea la referencia | Prioridad crítica / killer feature | -| `cut_segment` | Corta/remueve rangos de tiempo | Prioridad alta | +| `split` | Hace cortes puntuales en uno o más timestamps sin borrar contenido | Prioridad alta | | `add_text` | Agrega texto visual: hooks, títulos, labels, subtítulos básicos | Prioridad alta | | `update_text` | Modifica texto existente | Prioridad media | | `add_media_to_timeline` | Inserta video/audio/imagen existente al timeline | Prioridad alta | @@ -481,16 +481,16 @@ La siguiente lista se conserva como material de ideación, pero no debe tomarse ``` **Implementación (TS → FFmpeg o Rust WASM):** frame diff analysis. FFmpeg scene filter para rápido, Rust WASM para precisión. -#### 7.4. `cut_segment` +#### 7.4. `split` ```typescript { - name: "cut_segment", - description: "Cut a segment from the video and add it to the timeline", - input: { video_id: string, start: number, end: number, track?: number }, - output: { clip_id: string, position: number } + name: "split", + description: "Split timeline elements at one or more timestamps without deleting content", + input: { times: number[] /* timeline seconds */ }, + output: { success: boolean, affectedElements: string[] } } ``` -**Implementación (TS → Zustand store):** manipula el `timelineStore` de NeuralCut directamente desde el agente en TS. +**Implementación (TS → comandos del editor):** ejecuta splits determinísticos usando la infraestructura de timeline/commands para preservar undo/redo. #### 7.5. `concat_segments` ```typescript @@ -671,7 +671,7 @@ los mejores momentos y subtítulos amarillos" │ ▼ 4. Para cada highlight: - - cut_segment(start, end) + - split({ times: [start, end] }) │ ▼ 5. concat_segments(ids) ──────► Reel de ~30s @@ -716,7 +716,7 @@ Usuario: "quita la parte donde me equivoco" Agente: watch_video("where does the person mess up?") → identifica timestamp 1:23-1:31 → pide confirmación - → cut_segment para remover + → split({ times: [start, end] }) + tool de borrado ``` --- @@ -809,7 +809,7 @@ Esto no reemplaza la primera feature de producto; la **habilita**. | **2** | 1 semana | Ideación + diseño | Wireframes del panel de chat. Diseño de la capa de abstracción. Decisión de scope de tools. | | **3** | 1 semana | Infraestructura habilitadora | ChatPanel mínimo funcional, `chatStore`/`agentStore`, endpoint `/api/agent/chat`, orquestador básico, registro de tools, tipos compartidos, integración con video activo. | | **4** | 1 semana | Primer vertical slice real | `transcribe_video` conectada end-to-end con provider real o local, respuesta visible en chat y timestamps persistidos en estado. | -| **5** | 1 semana | Tier 1 tools | detect_silences, detect_scenes, cut_segment, concat_segments sobre la base ya creada. | +| **5** | 1 semana | Tier 1 tools | detect_silences, detect_scenes, split, concat_segments sobre la base ya creada. | | **6** | 1 semana | Video understanding | watch_video + take_screenshot + describe_scene. Primera demo "wow". | | **7** | 1 semana | Creative tools | generate_subtitles, apply_lut, detect_faces. | | **8** | 1 semana | Reframe + beats | auto_reframe (MediaPipe), detect_beats (Essentia). | @@ -835,7 +835,7 @@ Esto no reemplaza la primera feature de producto; la **habilita**. - Como agente, puedo listar assets del proyecto y saber cuáles están en el timeline. - Como agente, puedo cargar un asset con Gemini usando `load_asset_context` para entenderlo multimodalmente. - Como usuario, puedo preguntarle al agente sobre el contenido del video después de que el asset esté cargado en contexto. -- Como usuario, puedo pedir cortes básicos por timestamp usando `cut_segment`. +- Como usuario, puedo pedir cortes básicos por timestamp usando `split`. - Como usuario, puedo pedir texto visual básico usando `add_text`. **Should-have actualizado:**