ft: add agent split tool

This commit is contained in:
Luis Esteban Acevedo Ladino 2026-04-24 20:10:12 -05:00
parent 9cd5a6816e
commit 4c5e151822
18 changed files with 428 additions and 83 deletions

View File

@ -107,8 +107,8 @@ describe("buildContextFromEditorState (context mapper)", () => {
type: "video",
mediaId: "m1",
name: "Intro",
startTime: 2,
duration: 5,
startTime: 200,
duration: 500,
},
],
},
@ -122,8 +122,8 @@ describe("buildContextFromEditorState (context mapper)", () => {
type: "text",
name: "Caption",
content: "Hello world",
startTime: 3,
duration: 2,
startTime: 300,
duration: 200,
},
],
},
@ -139,7 +139,7 @@ describe("buildContextFromEditorState (context mapper)", () => {
mediaId: "m2",
name: "Music",
startTime: 0,
duration: 10,
duration: 1000,
},
],
},

View File

@ -25,7 +25,10 @@ export function buildContextFromEditorState(params: {
duration: a.duration ?? 0,
usedInTimeline: usedMediaIds.has(a.id),
})),
timelineTracks: buildTimelineTracks(params.activeScene),
timelineTracks: buildTimelineTracks(
params.activeScene,
params.ticksPerSecond,
),
playbackTimeMs: Math.round(
(params.currentTimeTicks / params.ticksPerSecond) * 1000,
),
@ -49,6 +52,7 @@ type TrackInput = {
function buildTimelineTracks(
scene: ActiveSceneInput | null,
ticksPerSecond: number,
): AgentTimelineTrack[] | undefined {
if (!scene?.tracks) {
return undefined;
@ -61,7 +65,7 @@ function buildTimelineTracks(
for (let index = 0; index < overlayTracks.length; index++) {
const track = overlayTracks[index];
tracks.push(
toTimelineTrack(track, mapOverlayTrackType(track.type), {
toTimelineTrack(track, mapOverlayTrackType(track.type), ticksPerSecond, {
position: position++,
visualLayer: overlayTracks.length - index,
isVisualLayer: true,
@ -72,7 +76,7 @@ function buildTimelineTracks(
if (scene.tracks.main) {
tracks.push(
toTimelineTrack(scene.tracks.main, "main", {
toTimelineTrack(scene.tracks.main, "main", ticksPerSecond, {
position: position++,
visualLayer: 0,
isVisualLayer: true,
@ -83,7 +87,7 @@ function buildTimelineTracks(
for (const track of scene.tracks.audio ?? []) {
tracks.push(
toTimelineTrack(track, "audio", {
toTimelineTrack(track, "audio", ticksPerSecond, {
position: position++,
visualLayer: null,
isVisualLayer: false,
@ -98,6 +102,7 @@ function buildTimelineTracks(
function toTimelineTrack(
track: TrackInput,
type: AgentTimelineTrack["type"],
ticksPerSecond: number,
stacking: Pick<
AgentTimelineTrack,
"position" | "visualLayer" | "isVisualLayer" | "stacking"
@ -115,12 +120,16 @@ function toTimelineTrack(
...(hasMediaId(element) ? { assetId: element.mediaId } : {}),
...(element.name ? { name: element.name } : {}),
...(hasTextContent(element) ? { content: element.content } : {}),
start: element.startTime,
end: element.startTime + element.duration,
start: toSeconds(element.startTime, ticksPerSecond),
end: toSeconds(element.startTime + element.duration, ticksPerSecond),
})),
};
}
function toSeconds(ticks: number, ticksPerSecond: number): number {
return ticks / ticksPerSecond;
}
function mapOverlayTrackType(
type: string | undefined,
): AgentTimelineTrack["type"] {

View File

@ -1,6 +1,7 @@
import { EditorCore } from "@/core";
import { TICKS_PER_SECOND } from "@/lib/wasm";
import { buildContextFromEditorState } from "@/agent/context-mapper";
import type { SceneTracks } from "@/lib/timeline";
/**
* Thin adapter: the ONLY file in agent/ that imports from core/.
@ -50,6 +51,39 @@ export const EditorContextAdapter = {
const asset = core.media.getAssets().find((a) => a.id === assetId);
return asset?.hasAudio;
},
splitTimeline({
times,
}: {
times: number[];
}): { success: boolean; affectedElements: string[] } | { error: string } {
const core = EditorCore.getInstance();
const activeScene = core.scenes.getActiveSceneOrNull();
if (!activeScene) {
return { error: "No active timeline" };
}
if (!hasTimelineContent(activeScene.tracks)) {
return { error: "No timeline content" };
}
const affectedElements = core.timeline.split({
times: times.map((time) => secondsToTicks(time)),
});
return { success: true, affectedElements };
},
};
function secondsToTicks(seconds: number): number {
return Math.round(seconds * TICKS_PER_SECOND);
}
function hasTimelineContent(tracks: SceneTracks): boolean {
return (
tracks.main.elements.length > 0 ||
tracks.overlay.some((track) => track.elements.length > 0) ||
tracks.audio.some((track) => track.elements.length > 0)
);
}
export { buildSystemPrompt } from "@/agent/system-prompt";

View File

@ -9,6 +9,7 @@ import { toolRegistry } from "@/agent/tools/registry";
import "@/agent/tools/load-context.tool";
import "@/agent/tools/list-project-assets.tool";
import "@/agent/tools/list-timeline.tool";
import "@/agent/tools/split.tool";
import { useChatStore } from "@/stores/chat-store";
import { useAgentStore } from "@/stores/agent-store";
@ -52,9 +53,7 @@ export async function run(
} catch {
detail = response.statusText;
}
throw new Error(
`API error ${response.status}: ${detail}`,
);
throw new Error(`API error ${response.status}: ${detail}`);
}
const data: APIResponse = await response.json();
@ -161,6 +160,13 @@ function validateToolArgs(
if (param.type === "boolean" && actualType !== "boolean") {
return `Argument "${param.key}" must be a boolean, got ${actualType}`;
}
if (
param.type === "number[]" &&
(!Array.isArray(value) ||
!value.every((item) => typeof item === "number"))
) {
return `Argument "${param.key}" must be an array of numbers`;
}
}
return null;

View File

@ -21,6 +21,7 @@ jest.mock("@google/generative-ai", () => ({
NUMBER: "number",
BOOLEAN: "boolean",
OBJECT: "object",
ARRAY: "array",
},
}));
@ -308,6 +309,7 @@ describe("toGeminiTools", () => {
parameters: [
{ key: "name", type: "string", required: true },
{ key: "count", type: "number", required: false },
{ key: "times", type: "number[]", required: false },
{ key: "active", type: "boolean", required: false },
{ key: "meta", type: "object", required: false },
],
@ -317,11 +319,13 @@ describe("toGeminiTools", () => {
const declarations = toGeminiTools(tools);
const props = declarations[0].parameters?.properties as Record<
string,
{ type: string }
{ type: string; items?: { type: string } }
>;
expect(props.name.type).toBe(SchemaType.STRING);
expect(props.count.type).toBe(SchemaType.NUMBER);
expect(props.times.type).toBe(SchemaType.ARRAY);
expect(props.times.items?.type).toBe(SchemaType.NUMBER);
expect(props.active.type).toBe(SchemaType.BOOLEAN);
expect(props.meta.type).toBe(SchemaType.OBJECT);
});
@ -612,4 +616,23 @@ describe("GeminiAdapter.chat()", () => {
expect(mockGenerateContent).toHaveBeenCalledTimes(1);
});
test("does not retry depleted billing credits", async () => {
const providerError = Object.assign(
new Error("Your prepayment credits are depleted. Manage billing."),
{ status: 429 },
);
mockGenerateContent.mockRejectedValueOnce(providerError);
const adapter = new GeminiAdapter(TEST_CONFIG);
await expect(
adapter.chat({
messages: [makeChatMessage("user", "Hi")],
systemPrompt: "",
tools: [],
}),
).rejects.toThrow("prepayment credits are depleted");
expect(mockGenerateContent).toHaveBeenCalledTimes(1);
});
});

View File

@ -4,6 +4,7 @@ import {
type FunctionDeclaration,
type FunctionDeclarationSchema,
type GenerateContentResult,
type Schema,
SchemaType,
} from "@google/generative-ai";
import type { ChatMessage, ToolCall, ToolSchema } from "@/agent/types";
@ -17,16 +18,6 @@ import type {
// Internal conversion helpers
// ---------------------------------------------------------------------------
/**
* Maps internal type strings to Gemini SchemaType values.
*/
const TYPE_MAP: Record<string, SchemaType> = {
string: SchemaType.STRING,
number: SchemaType.NUMBER,
boolean: SchemaType.BOOLEAN,
object: SchemaType.OBJECT,
};
const GEMINI_MAX_ATTEMPTS = 5;
const GEMINI_RETRY_BASE_DELAY_MS = process.env.NODE_ENV === "test" ? 0 : 1000;
const GEMINI_RETRYABLE_STATUSES = new Set([429, 500, 502, 503, 504]);
@ -42,9 +33,20 @@ function getProviderStatus(error: unknown): number | null {
function isRetryableProviderError(error: unknown): boolean {
const status = getProviderStatus(error);
if (status === 429 && isBillingOrQuotaExhaustedError(error)) {
return false;
}
return status !== null && GEMINI_RETRYABLE_STATUSES.has(status);
}
function isBillingOrQuotaExhaustedError(error: unknown): boolean {
const message = error instanceof Error ? error.message : String(error);
return /prepayment credits are depleted|billing|quota.*(?:depleted|exhausted)/i.test(
message,
);
}
function wait(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
@ -230,9 +232,7 @@ export function toGeminiTools(tools: ToolSchema[]): FunctionDeclaration[] {
const properties: FunctionDeclarationSchema["properties"] = {};
for (const param of tool.parameters) {
const geminiType = TYPE_MAP[param.type] ?? SchemaType.STRING;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
(properties as Record<string, any>)[param.key] = { type: geminiType };
properties[param.key] = toGeminiParameterSchema(param.type);
}
const requiredParams = tool.parameters
@ -251,6 +251,17 @@ export function toGeminiTools(tools: ToolSchema[]): FunctionDeclaration[] {
});
}
function toGeminiParameterSchema(type: string): Schema {
if (type === "number") return { type: SchemaType.NUMBER };
if (type === "boolean") return { type: SchemaType.BOOLEAN };
if (type === "object") return { type: SchemaType.OBJECT, properties: {} };
if (type === "number[]") {
return { type: SchemaType.ARRAY, items: { type: SchemaType.NUMBER } };
}
return { type: SchemaType.STRING };
}
/**
* Converts a Gemini GenerateContentResult into the provider-agnostic
* ProviderResponse shape.

View File

@ -0,0 +1,69 @@
import { beforeEach, describe, expect, mock, test } from "bun:test";
import type { AgentContext } from "@/agent/types";
import { toolRegistry } from "@/agent/tools/registry";
const mockSplitTimeline = mock((_args: { times: number[] }) => ({
success: true,
affectedElements: ["clip-1", "clip-2"],
}));
mock.module("@/agent/context", () => ({
EditorContextAdapter: {
splitTimeline: mockSplitTimeline,
},
}));
await import("@/agent/tools/split.tool");
const context: AgentContext = {
projectId: "proj-1",
activeSceneId: "scene-A",
mediaAssets: [],
playbackTimeMs: 0,
};
describe("split tool", () => {
beforeEach(() => {
mockSplitTimeline.mockClear();
});
test("is registered in the tool registry", () => {
expect(toolRegistry.has("split")).toBe(true);
});
test("splits the timeline through the editor adapter", async () => {
const tool = toolRegistry.get("split");
const result = await tool.execute({ times: [2] }, context);
expect(mockSplitTimeline).toHaveBeenCalledWith({ times: [2] });
expect(result).toEqual({
success: true,
affectedElements: ["clip-1", "clip-2"],
});
});
test("supports multiple split times", async () => {
const tool = toolRegistry.get("split");
await tool.execute({ times: [2, 5, 8] }, context);
expect(mockSplitTimeline).toHaveBeenCalledWith({ times: [2, 5, 8] });
});
test("validates split times", async () => {
const tool = toolRegistry.get("split");
expect(await tool.execute({ times: [] }, context)).toEqual({
error: "Invalid split times",
});
expect(await tool.execute({ times: [2, Number.NaN] }, context)).toEqual({
error: "Invalid split times",
});
expect(await tool.execute({ times: [-1] }, context)).toEqual({
error: "Invalid split times",
});
expect(await tool.execute({ times: "2" }, context)).toEqual({
error: "Invalid split times",
});
expect(mockSplitTimeline).not.toHaveBeenCalled();
});
});

View File

@ -12,7 +12,7 @@ export type ListTimelineResult = {
const listTimelineTool: ToolDefinition = {
name: "list_timeline",
description:
"Lists the active timeline as structured tracks and editable elements with layer metadata. position is timeline row order from top to bottom; visualLayer is render stacking where higher renders above lower, and audio tracks have null visualLayer.",
"Lists the active timeline as structured tracks and editable elements with layer metadata. Element start/end values are in seconds. position is timeline row order from top to bottom; visualLayer is render stacking where higher renders above lower, and audio tracks have null visualLayer.",
parameters: [],
execute: async (
_args: Record<string, unknown>,

View File

@ -0,0 +1,43 @@
import { EditorContextAdapter } from "@/agent/context";
import type { AgentContext, ToolDefinition } from "@/agent/types";
import { toolRegistry } from "@/agent/tools/registry";
export type SplitArgs = {
times: number[];
};
export type SplitResult = {
success: boolean;
affectedElements: string[];
};
const splitTool: ToolDefinition = {
name: "split",
description:
"Splits timeline elements at one or more requested times without deleting, trimming, or moving content. Use one time for a single cut, or multiple times to isolate ranges before separate edit/delete operations.",
parameters: [{ key: "times", type: "number[]", required: true }],
execute: async (
args: Record<string, unknown>,
_context: AgentContext,
): Promise<SplitResult | { error: string }> => {
const times = args.times;
if (!isValidTimes(times)) {
return { error: "Invalid split times" };
}
return EditorContextAdapter.splitTimeline({ times });
},
};
function isValidTimes(times: unknown): times is number[] {
return (
Array.isArray(times) &&
times.length > 0 &&
times.every(
(time) => typeof time === "number" && Number.isFinite(time) && time >= 0,
)
);
}
toolRegistry.register("split", splitTool);

View File

@ -58,7 +58,9 @@ export type AgentTimelineTrack = {
assetId?: string;
name?: string;
content?: string;
/** Timeline start in seconds. */
start: number;
/** Timeline end in seconds. */
end: number;
}>;
};

View File

@ -152,13 +152,14 @@ describe("POST /api/agent/chat", () => {
expect(callArgs.messages).toHaveLength(1);
expect(callArgs.systemPrompt).toContain("NeuralCut");
expect(callArgs.tools).toHaveLength(3);
expect(callArgs.tools).toHaveLength(4);
expect(
callArgs.tools.map((tool) => (tool as { name: string }).name),
).toEqual([
"load_context",
"list_project_assets",
"list_timeline",
"split",
]);
});
@ -240,4 +241,22 @@ describe("POST /api/agent/chat", () => {
const data = await res.json();
expect(data.error).toBe("LLM provider error");
});
test("returns clear error when Gemini credits are depleted", async () => {
mockChat.mockRejectedValueOnce(
Object.assign(
new Error(
"[GoogleGenerativeAI Error]: [429 Too Many Requests] Your prepayment credits are depleted.",
),
{ status: 429 },
),
);
const res = await POST(makeRequest(VALID_BODY));
expect(res.status).toBe(402);
const data = await res.json();
expect(data.error).toContain("Gemini credits are depleted");
expect(data.providerStatus).toBe(429);
});
});

View File

@ -34,9 +34,15 @@ const providerToolSchemas: ToolSchema[] = [
{
name: "list_timeline",
description:
"Lists the active timeline as structured tracks and editable elements with layer metadata. Tracks include position (top-to-bottom timeline row), visualLayer (higher renders above lower, null for audio), isVisualLayer, and stacking. Use this to understand which clips visually cover others and to discover ids before load_context.",
"Lists the active timeline as structured tracks and editable elements with layer metadata. Element start/end values are in seconds. Tracks include position (top-to-bottom timeline row), visualLayer (higher renders above lower, null for audio), isVisualLayer, and stacking. Use this to understand which clips visually cover others and to discover ids before load_context.",
parameters: [],
},
{
name: "split",
description:
"Splits timeline elements at one or more requested timeline times in seconds without deleting, trimming, or moving content. Use one time for a single cut, or multiple times to isolate ranges before separate edit/delete operations.",
parameters: [{ key: "times", type: "number[]", required: true }],
},
];
// ---------------------------------------------------------------------------
@ -160,16 +166,39 @@ export async function POST(request: NextRequest) {
const status = (error as { status?: number }).status;
const message =
error instanceof Error ? error.message : "Unknown provider error";
const providerError = toProviderErrorResponse({ message, status });
console.error("[agent/chat] Provider error:", error);
return NextResponse.json(
{
error: "LLM provider error",
error: providerError.error,
...(status ? { providerStatus: status } : {}),
...(process.env.NODE_ENV === "development"
? { detail: message }
: {}),
...(process.env.NODE_ENV === "development" ? { detail: message } : {}),
},
{ status: 502 },
{ status: providerError.status },
);
}
}
function toProviderErrorResponse({
message,
status,
}: {
message: string;
status?: number;
}): { error: string; status: number } {
if (status === 429 && isBillingOrQuotaExhaustedMessage(message)) {
return {
status: 402,
error:
"Gemini credits are depleted. Add credits in AI Studio or switch LLM_PROVIDER/LLM_MODEL in .env.local.",
};
}
return { status: 502, error: "LLM provider error" };
}
function isBillingOrQuotaExhaustedMessage(message: string): boolean {
return /prepayment credits are depleted|billing|quota.*(?:depleted|exhausted)/i.test(
message,
);
}

View File

@ -8,9 +8,7 @@ import type {
RetimeConfig,
} from "@/lib/timeline";
import { calculateTotalDuration } from "@/lib/timeline";
import {
findTrackInSceneTracks,
} from "@/lib/timeline/track-element-update";
import { findTrackInSceneTracks } from "@/lib/timeline/track-element-update";
import {
canElementBeHidden,
canElementHaveAudio,
@ -39,6 +37,7 @@ import {
DuplicateElementsCommand,
UpdateElementsCommand,
SplitElementsCommand,
SplitCommand,
MoveElementCommand,
TracksSnapshotCommand,
UpsertKeyframeCommand,
@ -200,6 +199,12 @@ export class TimelineManager {
return command.getRightSideElements();
}
split({ times }: { times: number[] }): string[] {
const command = new SplitCommand(times);
this.editor.command.execute({ command });
return command.getAffectedElements();
}
getTotalDuration(): number {
const activeScene = this.editor.scenes.getActiveSceneOrNull();
if (!activeScene) {
@ -580,14 +585,7 @@ export class TimelineManager {
}
const commands = keyframes.map(
({
trackId,
elementId,
propertyPath,
componentKey,
keyframeId,
patch,
}) =>
({ trackId, elementId, propertyPath, componentKey, keyframeId, patch }) =>
new UpdateScalarKeyframeCurveCommand({
trackId,
elementId,
@ -711,7 +709,9 @@ export class TimelineManager {
private applyPreviewOverlay(tracks: SceneTracks): SceneTracks {
if (this.previewOverlay.size === 0) return tracks;
const applyTrackOverlay = <TTrack extends TimelineTrack>(track: TTrack): TTrack => {
const applyTrackOverlay = <TTrack extends TimelineTrack>(
track: TTrack,
): TTrack => {
const hasOverlay = track.elements.some((element) =>
this.previewOverlay.has(element.id),
);
@ -803,7 +803,11 @@ export class TimelineManager {
}
getPreviewTracks(): SceneTracks | null {
return this.previewTracks ?? this.editor.scenes.getActiveSceneOrNull()?.tracks ?? null;
return (
this.previewTracks ??
this.editor.scenes.getActiveSceneOrNull()?.tracks ??
null
);
}
subscribe(listener: () => void): () => void {

View File

@ -2,6 +2,7 @@ export { InsertElementCommand } from "./insert-element";
export { DeleteElementsCommand } from "./delete-elements";
export { DuplicateElementsCommand } from "./duplicate-elements";
export { SplitElementsCommand } from "./split-elements";
export { SplitCommand } from "./split";
export { UpdateElementsCommand } from "./update-elements";
export { ToggleSourceAudioSeparationCommand } from "./toggle-source-audio-separation";
export { MoveElementCommand } from "./move-elements";

View File

@ -0,0 +1,96 @@
import { EditorCore } from "@/core";
import { Command, type CommandResult } from "@/lib/commands/base-command";
import type { SceneTracks } from "@/lib/timeline";
import { SplitElementsCommand } from "./split-elements";
export class SplitCommand extends Command {
private savedState: SceneTracks | null = null;
private affectedElements: string[] = [];
constructor(private readonly times: number[]) {
super();
}
getAffectedElements(): string[] {
return this.affectedElements;
}
execute(): CommandResult | undefined {
const editor = EditorCore.getInstance();
this.savedState = editor.scenes.getActiveScene().tracks;
const affectedElements = new Set<string>();
let selectedRightSide: Array<{ trackId: string; elementId: string }> = [];
for (const time of uniqueSortedTimes(this.times)) {
const tracks = editor.scenes.getActiveScene().tracks;
const targets = findElementsIntersectingTime({ tracks, time });
for (const target of targets) {
affectedElements.add(target.elementId);
}
const rightSide = executeSplit({ elements: targets, splitTime: time });
if (rightSide.length > 0) {
selectedRightSide = rightSide;
}
for (const target of rightSide) {
affectedElements.add(target.elementId);
}
}
this.affectedElements = [...affectedElements];
return this.affectedElements.length > 0
? { select: selectedRightSide }
: undefined;
}
undo(): void {
if (!this.savedState) {
return;
}
EditorCore.getInstance().timeline.updateTracks(this.savedState);
}
}
function uniqueSortedTimes(times: number[]): number[] {
return [...new Set(times)].sort((a, b) => a - b);
}
function executeSplit({
elements,
splitTime,
}: {
elements: Array<{ trackId: string; elementId: string }>;
splitTime: number;
}): Array<{ trackId: string; elementId: string }> {
if (elements.length === 0) {
return [];
}
const command = new SplitElementsCommand({ elements, splitTime });
command.execute();
return command.getRightSideElements();
}
function findElementsIntersectingTime({
tracks,
time,
}: {
tracks: SceneTracks;
time: number;
}): Array<{ trackId: string; elementId: string }> {
const result: Array<{ trackId: string; elementId: string }> = [];
const allTracks = [tracks.main, ...tracks.overlay, ...tracks.audio];
for (const track of allTracks) {
for (const element of track.elements) {
const elementStart = element.startTime;
const elementEnd = element.startTime + element.duration;
if (time > elementStart && time < elementEnd) {
result.push({ trackId: track.id, elementId: element.id });
}
}
}
return result;
}

View File

@ -147,17 +147,15 @@ Cargar un asset en el contexto multimodal del agente usando Gemini. Esta es la b
---
## 4. `cut_segment`
## 4. `split`
### Propósito
Cortar o remover un rango de tiempo del timeline.
Dividir elementos del timeline en uno o más puntos de tiempo, sin borrar contenido. Esta primitive equivale a hacer cortes/splits puntuales en el editor; eliminar material debe ser una tool separada y puede componerse después de hacer splits.
### Input
```ts
{
start: number;
end: number;
mode: "remove" | "keep";
times: number[]; // timeline seconds
}
```
@ -170,17 +168,17 @@ Cortar o remover un rango de tiempo del timeline.
```
### Requirements
- MUST validate `start < end`.
- MUST operate on timeline time, using the editors canonical time unit internally.
- MUST validate `times` contains at least one finite number.
- MUST accept timeline times in seconds and convert to the editors canonical time unit internally.
- MUST use existing timeline/command infrastructure when possible.
- `remove` MUST remove the selected time range.
- `keep` MUST preserve only the selected time range and remove outside material, if feasible in current editor model.
- MUST split every timeline element intersecting each requested time when the split point falls strictly inside the element.
- MUST NOT delete, trim away, or move timeline content.
- MUST be idempotent at existing boundaries: if a requested time already equals an element boundary, it MUST NOT create a duplicate split there.
- MUST preserve undo/redo behavior if the editor supports it for the operation.
### Errors
- Invalid range: `{ error: "Invalid time range" }`.
- Invalid times: `{ error: "Invalid split times" }`.
- Empty timeline: `{ error: "No timeline content" }`.
- Unsupported mode: `{ error: "Unsupported cut mode" }`.
---

View File

@ -89,23 +89,24 @@ Uso:
## Tools de edición primitivas
### `cut_segment`
### `split`
Corta o remueve un rango de tiempo del timeline.
Hace splits/cortes en uno o más puntos de tiempo del timeline, sin borrar contenido.
```ts
{
start: number;
end: number;
mode: "remove" | "keep";
times: number[]; // timeline seconds
}
```
Uso:
- quitar silencios,
- quitar errores,
- recortar intro/outro,
- construir highlights.
- cortar en un punto específico,
- aislar una sección antes de borrarla con otra tool usando dos tiempos,
- preparar edición por rangos,
- separar intro/outro sin eliminar nada,
- dejarle al agente una primitive composable para flujos más grandes.
> `split` NO elimina material. Si el usuario pide “eliminá esta parte”, el agente debe componer `split` con una tool de borrado cuando exista.
---
@ -294,7 +295,7 @@ Descartada porque `analysisType` introduce categorías artificiales. Gemini debe
### `remove_silences`
Diferida/no prioritaria porque se puede expresar como flujo con `cut_segment`.
Diferida/no prioritaria porque se puede expresar como flujo con `split` + tool de borrado.
### `generate_subtitles`
@ -311,7 +312,7 @@ Diferida porque el repo actual no parece tener corrección de color/LUTs impleme
1. `list_project_assets`
2. `list_timeline`
3. `load_asset_context`
4. `cut_segment`
4. `split`
5. `add_text`
6. `add_media_to_timeline`
7. `delete_element`

View File

@ -50,7 +50,7 @@ Decisiones vigentes:
Roadmap vigente de alto nivel:
1. **Contexto y percepción:** `list_project_assets`, `list_timeline`, `load_asset_context`.
2. **Edición básica:** `cut_segment`, `add_text`, `add_media_to_timeline`, `delete_element`.
2. **Edición básica:** `split`, `add_text`, `add_media_to_timeline`, `delete_element`.
3. **Ajustes simples:** `set_volume`, `add_sticker`, `apply_effect`.
4. **UX avanzada:** referencias `@asset`, progreso de uploads/procesamiento, previews/aprobaciones.
@ -421,7 +421,7 @@ Estas son las tools que el equipo debe priorizar. Son suficientemente pequeñas
| `list_project_assets` | Lista assets del proyecto, usados/no usados y por tipo | Prioridad alta |
| `list_timeline` | Devuelve resumen estructurado del timeline con `trackId`/`elementId` | Prioridad alta |
| `load_asset_context` | Carga un asset en el contexto multimodal de Gemini y cachea la referencia | Prioridad crítica / killer feature |
| `cut_segment` | Corta/remueve rangos de tiempo | Prioridad alta |
| `split` | Hace cortes puntuales en uno o más timestamps sin borrar contenido | Prioridad alta |
| `add_text` | Agrega texto visual: hooks, títulos, labels, subtítulos básicos | Prioridad alta |
| `update_text` | Modifica texto existente | Prioridad media |
| `add_media_to_timeline` | Inserta video/audio/imagen existente al timeline | Prioridad alta |
@ -481,16 +481,16 @@ La siguiente lista se conserva como material de ideación, pero no debe tomarse
```
**Implementación (TS → FFmpeg o Rust WASM):** frame diff analysis. FFmpeg scene filter para rápido, Rust WASM para precisión.
#### 7.4. `cut_segment`
#### 7.4. `split`
```typescript
{
name: "cut_segment",
description: "Cut a segment from the video and add it to the timeline",
input: { video_id: string, start: number, end: number, track?: number },
output: { clip_id: string, position: number }
name: "split",
description: "Split timeline elements at one or more timestamps without deleting content",
input: { times: number[] /* timeline seconds */ },
output: { success: boolean, affectedElements: string[] }
}
```
**Implementación (TS → Zustand store):** manipula el `timelineStore` de NeuralCut directamente desde el agente en TS.
**Implementación (TS → comandos del editor):** ejecuta splits determinísticos usando la infraestructura de timeline/commands para preservar undo/redo.
#### 7.5. `concat_segments`
```typescript
@ -671,7 +671,7 @@ los mejores momentos y subtítulos amarillos"
4. Para cada highlight:
- cut_segment(start, end)
- split({ times: [start, end] })
5. concat_segments(ids) ──────► Reel de ~30s
@ -716,7 +716,7 @@ Usuario: "quita la parte donde me equivoco"
Agente: watch_video("where does the person mess up?")
→ identifica timestamp 1:23-1:31
→ pide confirmación
cut_segment para remover
split({ times: [start, end] }) + tool de borrado
```
---
@ -809,7 +809,7 @@ Esto no reemplaza la primera feature de producto; la **habilita**.
| **2** | 1 semana | Ideación + diseño | Wireframes del panel de chat. Diseño de la capa de abstracción. Decisión de scope de tools. |
| **3** | 1 semana | Infraestructura habilitadora | ChatPanel mínimo funcional, `chatStore`/`agentStore`, endpoint `/api/agent/chat`, orquestador básico, registro de tools, tipos compartidos, integración con video activo. |
| **4** | 1 semana | Primer vertical slice real | `transcribe_video` conectada end-to-end con provider real o local, respuesta visible en chat y timestamps persistidos en estado. |
| **5** | 1 semana | Tier 1 tools | detect_silences, detect_scenes, cut_segment, concat_segments sobre la base ya creada. |
| **5** | 1 semana | Tier 1 tools | detect_silences, detect_scenes, split, concat_segments sobre la base ya creada. |
| **6** | 1 semana | Video understanding | watch_video + take_screenshot + describe_scene. Primera demo "wow". |
| **7** | 1 semana | Creative tools | generate_subtitles, apply_lut, detect_faces. |
| **8** | 1 semana | Reframe + beats | auto_reframe (MediaPipe), detect_beats (Essentia). |
@ -835,7 +835,7 @@ Esto no reemplaza la primera feature de producto; la **habilita**.
- Como agente, puedo listar assets del proyecto y saber cuáles están en el timeline.
- Como agente, puedo cargar un asset con Gemini usando `load_asset_context` para entenderlo multimodalmente.
- Como usuario, puedo preguntarle al agente sobre el contenido del video después de que el asset esté cargado en contexto.
- Como usuario, puedo pedir cortes básicos por timestamp usando `cut_segment`.
- Como usuario, puedo pedir cortes básicos por timestamp usando `split`.
- Como usuario, puedo pedir texto visual básico usando `add_text`.
**Should-have actualizado:**