ft: add agent split tool
This commit is contained in:
parent
9cd5a6816e
commit
4c5e151822
|
|
@ -107,8 +107,8 @@ describe("buildContextFromEditorState (context mapper)", () => {
|
|||
type: "video",
|
||||
mediaId: "m1",
|
||||
name: "Intro",
|
||||
startTime: 2,
|
||||
duration: 5,
|
||||
startTime: 200,
|
||||
duration: 500,
|
||||
},
|
||||
],
|
||||
},
|
||||
|
|
@ -122,8 +122,8 @@ describe("buildContextFromEditorState (context mapper)", () => {
|
|||
type: "text",
|
||||
name: "Caption",
|
||||
content: "Hello world",
|
||||
startTime: 3,
|
||||
duration: 2,
|
||||
startTime: 300,
|
||||
duration: 200,
|
||||
},
|
||||
],
|
||||
},
|
||||
|
|
@ -139,7 +139,7 @@ describe("buildContextFromEditorState (context mapper)", () => {
|
|||
mediaId: "m2",
|
||||
name: "Music",
|
||||
startTime: 0,
|
||||
duration: 10,
|
||||
duration: 1000,
|
||||
},
|
||||
],
|
||||
},
|
||||
|
|
|
|||
|
|
@ -25,7 +25,10 @@ export function buildContextFromEditorState(params: {
|
|||
duration: a.duration ?? 0,
|
||||
usedInTimeline: usedMediaIds.has(a.id),
|
||||
})),
|
||||
timelineTracks: buildTimelineTracks(params.activeScene),
|
||||
timelineTracks: buildTimelineTracks(
|
||||
params.activeScene,
|
||||
params.ticksPerSecond,
|
||||
),
|
||||
playbackTimeMs: Math.round(
|
||||
(params.currentTimeTicks / params.ticksPerSecond) * 1000,
|
||||
),
|
||||
|
|
@ -49,6 +52,7 @@ type TrackInput = {
|
|||
|
||||
function buildTimelineTracks(
|
||||
scene: ActiveSceneInput | null,
|
||||
ticksPerSecond: number,
|
||||
): AgentTimelineTrack[] | undefined {
|
||||
if (!scene?.tracks) {
|
||||
return undefined;
|
||||
|
|
@ -61,7 +65,7 @@ function buildTimelineTracks(
|
|||
for (let index = 0; index < overlayTracks.length; index++) {
|
||||
const track = overlayTracks[index];
|
||||
tracks.push(
|
||||
toTimelineTrack(track, mapOverlayTrackType(track.type), {
|
||||
toTimelineTrack(track, mapOverlayTrackType(track.type), ticksPerSecond, {
|
||||
position: position++,
|
||||
visualLayer: overlayTracks.length - index,
|
||||
isVisualLayer: true,
|
||||
|
|
@ -72,7 +76,7 @@ function buildTimelineTracks(
|
|||
|
||||
if (scene.tracks.main) {
|
||||
tracks.push(
|
||||
toTimelineTrack(scene.tracks.main, "main", {
|
||||
toTimelineTrack(scene.tracks.main, "main", ticksPerSecond, {
|
||||
position: position++,
|
||||
visualLayer: 0,
|
||||
isVisualLayer: true,
|
||||
|
|
@ -83,7 +87,7 @@ function buildTimelineTracks(
|
|||
|
||||
for (const track of scene.tracks.audio ?? []) {
|
||||
tracks.push(
|
||||
toTimelineTrack(track, "audio", {
|
||||
toTimelineTrack(track, "audio", ticksPerSecond, {
|
||||
position: position++,
|
||||
visualLayer: null,
|
||||
isVisualLayer: false,
|
||||
|
|
@ -98,6 +102,7 @@ function buildTimelineTracks(
|
|||
function toTimelineTrack(
|
||||
track: TrackInput,
|
||||
type: AgentTimelineTrack["type"],
|
||||
ticksPerSecond: number,
|
||||
stacking: Pick<
|
||||
AgentTimelineTrack,
|
||||
"position" | "visualLayer" | "isVisualLayer" | "stacking"
|
||||
|
|
@ -115,12 +120,16 @@ function toTimelineTrack(
|
|||
...(hasMediaId(element) ? { assetId: element.mediaId } : {}),
|
||||
...(element.name ? { name: element.name } : {}),
|
||||
...(hasTextContent(element) ? { content: element.content } : {}),
|
||||
start: element.startTime,
|
||||
end: element.startTime + element.duration,
|
||||
start: toSeconds(element.startTime, ticksPerSecond),
|
||||
end: toSeconds(element.startTime + element.duration, ticksPerSecond),
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
function toSeconds(ticks: number, ticksPerSecond: number): number {
|
||||
return ticks / ticksPerSecond;
|
||||
}
|
||||
|
||||
function mapOverlayTrackType(
|
||||
type: string | undefined,
|
||||
): AgentTimelineTrack["type"] {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import { EditorCore } from "@/core";
|
||||
import { TICKS_PER_SECOND } from "@/lib/wasm";
|
||||
import { buildContextFromEditorState } from "@/agent/context-mapper";
|
||||
import type { SceneTracks } from "@/lib/timeline";
|
||||
|
||||
/**
|
||||
* Thin adapter: the ONLY file in agent/ that imports from core/.
|
||||
|
|
@ -50,6 +51,39 @@ export const EditorContextAdapter = {
|
|||
const asset = core.media.getAssets().find((a) => a.id === assetId);
|
||||
return asset?.hasAudio;
|
||||
},
|
||||
|
||||
splitTimeline({
|
||||
times,
|
||||
}: {
|
||||
times: number[];
|
||||
}): { success: boolean; affectedElements: string[] } | { error: string } {
|
||||
const core = EditorCore.getInstance();
|
||||
const activeScene = core.scenes.getActiveSceneOrNull();
|
||||
if (!activeScene) {
|
||||
return { error: "No active timeline" };
|
||||
}
|
||||
|
||||
if (!hasTimelineContent(activeScene.tracks)) {
|
||||
return { error: "No timeline content" };
|
||||
}
|
||||
|
||||
const affectedElements = core.timeline.split({
|
||||
times: times.map((time) => secondsToTicks(time)),
|
||||
});
|
||||
return { success: true, affectedElements };
|
||||
},
|
||||
};
|
||||
|
||||
function secondsToTicks(seconds: number): number {
|
||||
return Math.round(seconds * TICKS_PER_SECOND);
|
||||
}
|
||||
|
||||
function hasTimelineContent(tracks: SceneTracks): boolean {
|
||||
return (
|
||||
tracks.main.elements.length > 0 ||
|
||||
tracks.overlay.some((track) => track.elements.length > 0) ||
|
||||
tracks.audio.some((track) => track.elements.length > 0)
|
||||
);
|
||||
}
|
||||
|
||||
export { buildSystemPrompt } from "@/agent/system-prompt";
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import { toolRegistry } from "@/agent/tools/registry";
|
|||
import "@/agent/tools/load-context.tool";
|
||||
import "@/agent/tools/list-project-assets.tool";
|
||||
import "@/agent/tools/list-timeline.tool";
|
||||
import "@/agent/tools/split.tool";
|
||||
import { useChatStore } from "@/stores/chat-store";
|
||||
import { useAgentStore } from "@/stores/agent-store";
|
||||
|
||||
|
|
@ -52,9 +53,7 @@ export async function run(
|
|||
} catch {
|
||||
detail = response.statusText;
|
||||
}
|
||||
throw new Error(
|
||||
`API error ${response.status}: ${detail}`,
|
||||
);
|
||||
throw new Error(`API error ${response.status}: ${detail}`);
|
||||
}
|
||||
|
||||
const data: APIResponse = await response.json();
|
||||
|
|
@ -161,6 +160,13 @@ function validateToolArgs(
|
|||
if (param.type === "boolean" && actualType !== "boolean") {
|
||||
return `Argument "${param.key}" must be a boolean, got ${actualType}`;
|
||||
}
|
||||
if (
|
||||
param.type === "number[]" &&
|
||||
(!Array.isArray(value) ||
|
||||
!value.every((item) => typeof item === "number"))
|
||||
) {
|
||||
return `Argument "${param.key}" must be an array of numbers`;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ jest.mock("@google/generative-ai", () => ({
|
|||
NUMBER: "number",
|
||||
BOOLEAN: "boolean",
|
||||
OBJECT: "object",
|
||||
ARRAY: "array",
|
||||
},
|
||||
}));
|
||||
|
||||
|
|
@ -308,6 +309,7 @@ describe("toGeminiTools", () => {
|
|||
parameters: [
|
||||
{ key: "name", type: "string", required: true },
|
||||
{ key: "count", type: "number", required: false },
|
||||
{ key: "times", type: "number[]", required: false },
|
||||
{ key: "active", type: "boolean", required: false },
|
||||
{ key: "meta", type: "object", required: false },
|
||||
],
|
||||
|
|
@ -317,11 +319,13 @@ describe("toGeminiTools", () => {
|
|||
const declarations = toGeminiTools(tools);
|
||||
const props = declarations[0].parameters?.properties as Record<
|
||||
string,
|
||||
{ type: string }
|
||||
{ type: string; items?: { type: string } }
|
||||
>;
|
||||
|
||||
expect(props.name.type).toBe(SchemaType.STRING);
|
||||
expect(props.count.type).toBe(SchemaType.NUMBER);
|
||||
expect(props.times.type).toBe(SchemaType.ARRAY);
|
||||
expect(props.times.items?.type).toBe(SchemaType.NUMBER);
|
||||
expect(props.active.type).toBe(SchemaType.BOOLEAN);
|
||||
expect(props.meta.type).toBe(SchemaType.OBJECT);
|
||||
});
|
||||
|
|
@ -612,4 +616,23 @@ describe("GeminiAdapter.chat()", () => {
|
|||
|
||||
expect(mockGenerateContent).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
test("does not retry depleted billing credits", async () => {
|
||||
const providerError = Object.assign(
|
||||
new Error("Your prepayment credits are depleted. Manage billing."),
|
||||
{ status: 429 },
|
||||
);
|
||||
mockGenerateContent.mockRejectedValueOnce(providerError);
|
||||
|
||||
const adapter = new GeminiAdapter(TEST_CONFIG);
|
||||
await expect(
|
||||
adapter.chat({
|
||||
messages: [makeChatMessage("user", "Hi")],
|
||||
systemPrompt: "",
|
||||
tools: [],
|
||||
}),
|
||||
).rejects.toThrow("prepayment credits are depleted");
|
||||
|
||||
expect(mockGenerateContent).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import {
|
|||
type FunctionDeclaration,
|
||||
type FunctionDeclarationSchema,
|
||||
type GenerateContentResult,
|
||||
type Schema,
|
||||
SchemaType,
|
||||
} from "@google/generative-ai";
|
||||
import type { ChatMessage, ToolCall, ToolSchema } from "@/agent/types";
|
||||
|
|
@ -17,16 +18,6 @@ import type {
|
|||
// Internal conversion helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Maps internal type strings to Gemini SchemaType values.
|
||||
*/
|
||||
const TYPE_MAP: Record<string, SchemaType> = {
|
||||
string: SchemaType.STRING,
|
||||
number: SchemaType.NUMBER,
|
||||
boolean: SchemaType.BOOLEAN,
|
||||
object: SchemaType.OBJECT,
|
||||
};
|
||||
|
||||
const GEMINI_MAX_ATTEMPTS = 5;
|
||||
const GEMINI_RETRY_BASE_DELAY_MS = process.env.NODE_ENV === "test" ? 0 : 1000;
|
||||
const GEMINI_RETRYABLE_STATUSES = new Set([429, 500, 502, 503, 504]);
|
||||
|
|
@ -42,9 +33,20 @@ function getProviderStatus(error: unknown): number | null {
|
|||
|
||||
function isRetryableProviderError(error: unknown): boolean {
|
||||
const status = getProviderStatus(error);
|
||||
if (status === 429 && isBillingOrQuotaExhaustedError(error)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return status !== null && GEMINI_RETRYABLE_STATUSES.has(status);
|
||||
}
|
||||
|
||||
function isBillingOrQuotaExhaustedError(error: unknown): boolean {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
return /prepayment credits are depleted|billing|quota.*(?:depleted|exhausted)/i.test(
|
||||
message,
|
||||
);
|
||||
}
|
||||
|
||||
function wait(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
|
@ -230,9 +232,7 @@ export function toGeminiTools(tools: ToolSchema[]): FunctionDeclaration[] {
|
|||
const properties: FunctionDeclarationSchema["properties"] = {};
|
||||
|
||||
for (const param of tool.parameters) {
|
||||
const geminiType = TYPE_MAP[param.type] ?? SchemaType.STRING;
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
(properties as Record<string, any>)[param.key] = { type: geminiType };
|
||||
properties[param.key] = toGeminiParameterSchema(param.type);
|
||||
}
|
||||
|
||||
const requiredParams = tool.parameters
|
||||
|
|
@ -251,6 +251,17 @@ export function toGeminiTools(tools: ToolSchema[]): FunctionDeclaration[] {
|
|||
});
|
||||
}
|
||||
|
||||
function toGeminiParameterSchema(type: string): Schema {
|
||||
if (type === "number") return { type: SchemaType.NUMBER };
|
||||
if (type === "boolean") return { type: SchemaType.BOOLEAN };
|
||||
if (type === "object") return { type: SchemaType.OBJECT, properties: {} };
|
||||
if (type === "number[]") {
|
||||
return { type: SchemaType.ARRAY, items: { type: SchemaType.NUMBER } };
|
||||
}
|
||||
|
||||
return { type: SchemaType.STRING };
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts a Gemini GenerateContentResult into the provider-agnostic
|
||||
* ProviderResponse shape.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,69 @@
|
|||
import { beforeEach, describe, expect, mock, test } from "bun:test";
|
||||
import type { AgentContext } from "@/agent/types";
|
||||
import { toolRegistry } from "@/agent/tools/registry";
|
||||
|
||||
const mockSplitTimeline = mock((_args: { times: number[] }) => ({
|
||||
success: true,
|
||||
affectedElements: ["clip-1", "clip-2"],
|
||||
}));
|
||||
|
||||
mock.module("@/agent/context", () => ({
|
||||
EditorContextAdapter: {
|
||||
splitTimeline: mockSplitTimeline,
|
||||
},
|
||||
}));
|
||||
|
||||
await import("@/agent/tools/split.tool");
|
||||
|
||||
const context: AgentContext = {
|
||||
projectId: "proj-1",
|
||||
activeSceneId: "scene-A",
|
||||
mediaAssets: [],
|
||||
playbackTimeMs: 0,
|
||||
};
|
||||
|
||||
describe("split tool", () => {
|
||||
beforeEach(() => {
|
||||
mockSplitTimeline.mockClear();
|
||||
});
|
||||
|
||||
test("is registered in the tool registry", () => {
|
||||
expect(toolRegistry.has("split")).toBe(true);
|
||||
});
|
||||
|
||||
test("splits the timeline through the editor adapter", async () => {
|
||||
const tool = toolRegistry.get("split");
|
||||
const result = await tool.execute({ times: [2] }, context);
|
||||
|
||||
expect(mockSplitTimeline).toHaveBeenCalledWith({ times: [2] });
|
||||
expect(result).toEqual({
|
||||
success: true,
|
||||
affectedElements: ["clip-1", "clip-2"],
|
||||
});
|
||||
});
|
||||
|
||||
test("supports multiple split times", async () => {
|
||||
const tool = toolRegistry.get("split");
|
||||
await tool.execute({ times: [2, 5, 8] }, context);
|
||||
|
||||
expect(mockSplitTimeline).toHaveBeenCalledWith({ times: [2, 5, 8] });
|
||||
});
|
||||
|
||||
test("validates split times", async () => {
|
||||
const tool = toolRegistry.get("split");
|
||||
|
||||
expect(await tool.execute({ times: [] }, context)).toEqual({
|
||||
error: "Invalid split times",
|
||||
});
|
||||
expect(await tool.execute({ times: [2, Number.NaN] }, context)).toEqual({
|
||||
error: "Invalid split times",
|
||||
});
|
||||
expect(await tool.execute({ times: [-1] }, context)).toEqual({
|
||||
error: "Invalid split times",
|
||||
});
|
||||
expect(await tool.execute({ times: "2" }, context)).toEqual({
|
||||
error: "Invalid split times",
|
||||
});
|
||||
expect(mockSplitTimeline).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
|
@ -12,7 +12,7 @@ export type ListTimelineResult = {
|
|||
const listTimelineTool: ToolDefinition = {
|
||||
name: "list_timeline",
|
||||
description:
|
||||
"Lists the active timeline as structured tracks and editable elements with layer metadata. position is timeline row order from top to bottom; visualLayer is render stacking where higher renders above lower, and audio tracks have null visualLayer.",
|
||||
"Lists the active timeline as structured tracks and editable elements with layer metadata. Element start/end values are in seconds. position is timeline row order from top to bottom; visualLayer is render stacking where higher renders above lower, and audio tracks have null visualLayer.",
|
||||
parameters: [],
|
||||
execute: async (
|
||||
_args: Record<string, unknown>,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,43 @@
|
|||
import { EditorContextAdapter } from "@/agent/context";
|
||||
import type { AgentContext, ToolDefinition } from "@/agent/types";
|
||||
import { toolRegistry } from "@/agent/tools/registry";
|
||||
|
||||
export type SplitArgs = {
|
||||
times: number[];
|
||||
};
|
||||
|
||||
export type SplitResult = {
|
||||
success: boolean;
|
||||
affectedElements: string[];
|
||||
};
|
||||
|
||||
const splitTool: ToolDefinition = {
|
||||
name: "split",
|
||||
description:
|
||||
"Splits timeline elements at one or more requested times without deleting, trimming, or moving content. Use one time for a single cut, or multiple times to isolate ranges before separate edit/delete operations.",
|
||||
parameters: [{ key: "times", type: "number[]", required: true }],
|
||||
execute: async (
|
||||
args: Record<string, unknown>,
|
||||
_context: AgentContext,
|
||||
): Promise<SplitResult | { error: string }> => {
|
||||
const times = args.times;
|
||||
|
||||
if (!isValidTimes(times)) {
|
||||
return { error: "Invalid split times" };
|
||||
}
|
||||
|
||||
return EditorContextAdapter.splitTimeline({ times });
|
||||
},
|
||||
};
|
||||
|
||||
function isValidTimes(times: unknown): times is number[] {
|
||||
return (
|
||||
Array.isArray(times) &&
|
||||
times.length > 0 &&
|
||||
times.every(
|
||||
(time) => typeof time === "number" && Number.isFinite(time) && time >= 0,
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
toolRegistry.register("split", splitTool);
|
||||
|
|
@ -58,7 +58,9 @@ export type AgentTimelineTrack = {
|
|||
assetId?: string;
|
||||
name?: string;
|
||||
content?: string;
|
||||
/** Timeline start in seconds. */
|
||||
start: number;
|
||||
/** Timeline end in seconds. */
|
||||
end: number;
|
||||
}>;
|
||||
};
|
||||
|
|
|
|||
|
|
@ -152,13 +152,14 @@ describe("POST /api/agent/chat", () => {
|
|||
|
||||
expect(callArgs.messages).toHaveLength(1);
|
||||
expect(callArgs.systemPrompt).toContain("NeuralCut");
|
||||
expect(callArgs.tools).toHaveLength(3);
|
||||
expect(callArgs.tools).toHaveLength(4);
|
||||
expect(
|
||||
callArgs.tools.map((tool) => (tool as { name: string }).name),
|
||||
).toEqual([
|
||||
"load_context",
|
||||
"list_project_assets",
|
||||
"list_timeline",
|
||||
"split",
|
||||
]);
|
||||
});
|
||||
|
||||
|
|
@ -240,4 +241,22 @@ describe("POST /api/agent/chat", () => {
|
|||
const data = await res.json();
|
||||
expect(data.error).toBe("LLM provider error");
|
||||
});
|
||||
|
||||
test("returns clear error when Gemini credits are depleted", async () => {
|
||||
mockChat.mockRejectedValueOnce(
|
||||
Object.assign(
|
||||
new Error(
|
||||
"[GoogleGenerativeAI Error]: [429 Too Many Requests] Your prepayment credits are depleted.",
|
||||
),
|
||||
{ status: 429 },
|
||||
),
|
||||
);
|
||||
|
||||
const res = await POST(makeRequest(VALID_BODY));
|
||||
expect(res.status).toBe(402);
|
||||
|
||||
const data = await res.json();
|
||||
expect(data.error).toContain("Gemini credits are depleted");
|
||||
expect(data.providerStatus).toBe(429);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -34,9 +34,15 @@ const providerToolSchemas: ToolSchema[] = [
|
|||
{
|
||||
name: "list_timeline",
|
||||
description:
|
||||
"Lists the active timeline as structured tracks and editable elements with layer metadata. Tracks include position (top-to-bottom timeline row), visualLayer (higher renders above lower, null for audio), isVisualLayer, and stacking. Use this to understand which clips visually cover others and to discover ids before load_context.",
|
||||
"Lists the active timeline as structured tracks and editable elements with layer metadata. Element start/end values are in seconds. Tracks include position (top-to-bottom timeline row), visualLayer (higher renders above lower, null for audio), isVisualLayer, and stacking. Use this to understand which clips visually cover others and to discover ids before load_context.",
|
||||
parameters: [],
|
||||
},
|
||||
{
|
||||
name: "split",
|
||||
description:
|
||||
"Splits timeline elements at one or more requested timeline times in seconds without deleting, trimming, or moving content. Use one time for a single cut, or multiple times to isolate ranges before separate edit/delete operations.",
|
||||
parameters: [{ key: "times", type: "number[]", required: true }],
|
||||
},
|
||||
];
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
|
@ -160,16 +166,39 @@ export async function POST(request: NextRequest) {
|
|||
const status = (error as { status?: number }).status;
|
||||
const message =
|
||||
error instanceof Error ? error.message : "Unknown provider error";
|
||||
const providerError = toProviderErrorResponse({ message, status });
|
||||
console.error("[agent/chat] Provider error:", error);
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: "LLM provider error",
|
||||
error: providerError.error,
|
||||
...(status ? { providerStatus: status } : {}),
|
||||
...(process.env.NODE_ENV === "development"
|
||||
? { detail: message }
|
||||
: {}),
|
||||
...(process.env.NODE_ENV === "development" ? { detail: message } : {}),
|
||||
},
|
||||
{ status: 502 },
|
||||
{ status: providerError.status },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function toProviderErrorResponse({
|
||||
message,
|
||||
status,
|
||||
}: {
|
||||
message: string;
|
||||
status?: number;
|
||||
}): { error: string; status: number } {
|
||||
if (status === 429 && isBillingOrQuotaExhaustedMessage(message)) {
|
||||
return {
|
||||
status: 402,
|
||||
error:
|
||||
"Gemini credits are depleted. Add credits in AI Studio or switch LLM_PROVIDER/LLM_MODEL in .env.local.",
|
||||
};
|
||||
}
|
||||
|
||||
return { status: 502, error: "LLM provider error" };
|
||||
}
|
||||
|
||||
function isBillingOrQuotaExhaustedMessage(message: string): boolean {
|
||||
return /prepayment credits are depleted|billing|quota.*(?:depleted|exhausted)/i.test(
|
||||
message,
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,9 +8,7 @@ import type {
|
|||
RetimeConfig,
|
||||
} from "@/lib/timeline";
|
||||
import { calculateTotalDuration } from "@/lib/timeline";
|
||||
import {
|
||||
findTrackInSceneTracks,
|
||||
} from "@/lib/timeline/track-element-update";
|
||||
import { findTrackInSceneTracks } from "@/lib/timeline/track-element-update";
|
||||
import {
|
||||
canElementBeHidden,
|
||||
canElementHaveAudio,
|
||||
|
|
@ -39,6 +37,7 @@ import {
|
|||
DuplicateElementsCommand,
|
||||
UpdateElementsCommand,
|
||||
SplitElementsCommand,
|
||||
SplitCommand,
|
||||
MoveElementCommand,
|
||||
TracksSnapshotCommand,
|
||||
UpsertKeyframeCommand,
|
||||
|
|
@ -200,6 +199,12 @@ export class TimelineManager {
|
|||
return command.getRightSideElements();
|
||||
}
|
||||
|
||||
split({ times }: { times: number[] }): string[] {
|
||||
const command = new SplitCommand(times);
|
||||
this.editor.command.execute({ command });
|
||||
return command.getAffectedElements();
|
||||
}
|
||||
|
||||
getTotalDuration(): number {
|
||||
const activeScene = this.editor.scenes.getActiveSceneOrNull();
|
||||
if (!activeScene) {
|
||||
|
|
@ -580,14 +585,7 @@ export class TimelineManager {
|
|||
}
|
||||
|
||||
const commands = keyframes.map(
|
||||
({
|
||||
trackId,
|
||||
elementId,
|
||||
propertyPath,
|
||||
componentKey,
|
||||
keyframeId,
|
||||
patch,
|
||||
}) =>
|
||||
({ trackId, elementId, propertyPath, componentKey, keyframeId, patch }) =>
|
||||
new UpdateScalarKeyframeCurveCommand({
|
||||
trackId,
|
||||
elementId,
|
||||
|
|
@ -711,7 +709,9 @@ export class TimelineManager {
|
|||
private applyPreviewOverlay(tracks: SceneTracks): SceneTracks {
|
||||
if (this.previewOverlay.size === 0) return tracks;
|
||||
|
||||
const applyTrackOverlay = <TTrack extends TimelineTrack>(track: TTrack): TTrack => {
|
||||
const applyTrackOverlay = <TTrack extends TimelineTrack>(
|
||||
track: TTrack,
|
||||
): TTrack => {
|
||||
const hasOverlay = track.elements.some((element) =>
|
||||
this.previewOverlay.has(element.id),
|
||||
);
|
||||
|
|
@ -803,7 +803,11 @@ export class TimelineManager {
|
|||
}
|
||||
|
||||
getPreviewTracks(): SceneTracks | null {
|
||||
return this.previewTracks ?? this.editor.scenes.getActiveSceneOrNull()?.tracks ?? null;
|
||||
return (
|
||||
this.previewTracks ??
|
||||
this.editor.scenes.getActiveSceneOrNull()?.tracks ??
|
||||
null
|
||||
);
|
||||
}
|
||||
|
||||
subscribe(listener: () => void): () => void {
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ export { InsertElementCommand } from "./insert-element";
|
|||
export { DeleteElementsCommand } from "./delete-elements";
|
||||
export { DuplicateElementsCommand } from "./duplicate-elements";
|
||||
export { SplitElementsCommand } from "./split-elements";
|
||||
export { SplitCommand } from "./split";
|
||||
export { UpdateElementsCommand } from "./update-elements";
|
||||
export { ToggleSourceAudioSeparationCommand } from "./toggle-source-audio-separation";
|
||||
export { MoveElementCommand } from "./move-elements";
|
||||
|
|
|
|||
|
|
@ -0,0 +1,96 @@
|
|||
import { EditorCore } from "@/core";
|
||||
import { Command, type CommandResult } from "@/lib/commands/base-command";
|
||||
import type { SceneTracks } from "@/lib/timeline";
|
||||
import { SplitElementsCommand } from "./split-elements";
|
||||
|
||||
export class SplitCommand extends Command {
|
||||
private savedState: SceneTracks | null = null;
|
||||
private affectedElements: string[] = [];
|
||||
|
||||
constructor(private readonly times: number[]) {
|
||||
super();
|
||||
}
|
||||
|
||||
getAffectedElements(): string[] {
|
||||
return this.affectedElements;
|
||||
}
|
||||
|
||||
execute(): CommandResult | undefined {
|
||||
const editor = EditorCore.getInstance();
|
||||
this.savedState = editor.scenes.getActiveScene().tracks;
|
||||
const affectedElements = new Set<string>();
|
||||
let selectedRightSide: Array<{ trackId: string; elementId: string }> = [];
|
||||
|
||||
for (const time of uniqueSortedTimes(this.times)) {
|
||||
const tracks = editor.scenes.getActiveScene().tracks;
|
||||
const targets = findElementsIntersectingTime({ tracks, time });
|
||||
for (const target of targets) {
|
||||
affectedElements.add(target.elementId);
|
||||
}
|
||||
|
||||
const rightSide = executeSplit({ elements: targets, splitTime: time });
|
||||
if (rightSide.length > 0) {
|
||||
selectedRightSide = rightSide;
|
||||
}
|
||||
for (const target of rightSide) {
|
||||
affectedElements.add(target.elementId);
|
||||
}
|
||||
}
|
||||
|
||||
this.affectedElements = [...affectedElements];
|
||||
return this.affectedElements.length > 0
|
||||
? { select: selectedRightSide }
|
||||
: undefined;
|
||||
}
|
||||
|
||||
undo(): void {
|
||||
if (!this.savedState) {
|
||||
return;
|
||||
}
|
||||
|
||||
EditorCore.getInstance().timeline.updateTracks(this.savedState);
|
||||
}
|
||||
}
|
||||
|
||||
function uniqueSortedTimes(times: number[]): number[] {
|
||||
return [...new Set(times)].sort((a, b) => a - b);
|
||||
}
|
||||
|
||||
function executeSplit({
|
||||
elements,
|
||||
splitTime,
|
||||
}: {
|
||||
elements: Array<{ trackId: string; elementId: string }>;
|
||||
splitTime: number;
|
||||
}): Array<{ trackId: string; elementId: string }> {
|
||||
if (elements.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const command = new SplitElementsCommand({ elements, splitTime });
|
||||
command.execute();
|
||||
return command.getRightSideElements();
|
||||
}
|
||||
|
||||
function findElementsIntersectingTime({
|
||||
tracks,
|
||||
time,
|
||||
}: {
|
||||
tracks: SceneTracks;
|
||||
time: number;
|
||||
}): Array<{ trackId: string; elementId: string }> {
|
||||
const result: Array<{ trackId: string; elementId: string }> = [];
|
||||
const allTracks = [tracks.main, ...tracks.overlay, ...tracks.audio];
|
||||
|
||||
for (const track of allTracks) {
|
||||
for (const element of track.elements) {
|
||||
const elementStart = element.startTime;
|
||||
const elementEnd = element.startTime + element.duration;
|
||||
if (time > elementStart && time < elementEnd) {
|
||||
result.push({ trackId: track.id, elementId: element.id });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
|
@ -147,17 +147,15 @@ Cargar un asset en el contexto multimodal del agente usando Gemini. Esta es la b
|
|||
|
||||
---
|
||||
|
||||
## 4. `cut_segment`
|
||||
## 4. `split`
|
||||
|
||||
### Propósito
|
||||
Cortar o remover un rango de tiempo del timeline.
|
||||
Dividir elementos del timeline en uno o más puntos de tiempo, sin borrar contenido. Esta primitive equivale a hacer cortes/splits puntuales en el editor; eliminar material debe ser una tool separada y puede componerse después de hacer splits.
|
||||
|
||||
### Input
|
||||
```ts
|
||||
{
|
||||
start: number;
|
||||
end: number;
|
||||
mode: "remove" | "keep";
|
||||
times: number[]; // timeline seconds
|
||||
}
|
||||
```
|
||||
|
||||
|
|
@ -170,17 +168,17 @@ Cortar o remover un rango de tiempo del timeline.
|
|||
```
|
||||
|
||||
### Requirements
|
||||
- MUST validate `start < end`.
|
||||
- MUST operate on timeline time, using the editor’s canonical time unit internally.
|
||||
- MUST validate `times` contains at least one finite number.
|
||||
- MUST accept timeline times in seconds and convert to the editor’s canonical time unit internally.
|
||||
- MUST use existing timeline/command infrastructure when possible.
|
||||
- `remove` MUST remove the selected time range.
|
||||
- `keep` MUST preserve only the selected time range and remove outside material, if feasible in current editor model.
|
||||
- MUST split every timeline element intersecting each requested time when the split point falls strictly inside the element.
|
||||
- MUST NOT delete, trim away, or move timeline content.
|
||||
- MUST be idempotent at existing boundaries: if a requested time already equals an element boundary, it MUST NOT create a duplicate split there.
|
||||
- MUST preserve undo/redo behavior if the editor supports it for the operation.
|
||||
|
||||
### Errors
|
||||
- Invalid range: `{ error: "Invalid time range" }`.
|
||||
- Invalid times: `{ error: "Invalid split times" }`.
|
||||
- Empty timeline: `{ error: "No timeline content" }`.
|
||||
- Unsupported mode: `{ error: "Unsupported cut mode" }`.
|
||||
|
||||
---
|
||||
|
||||
|
|
|
|||
|
|
@ -89,23 +89,24 @@ Uso:
|
|||
|
||||
## Tools de edición primitivas
|
||||
|
||||
### `cut_segment`
|
||||
### `split`
|
||||
|
||||
Corta o remueve un rango de tiempo del timeline.
|
||||
Hace splits/cortes en uno o más puntos de tiempo del timeline, sin borrar contenido.
|
||||
|
||||
```ts
|
||||
{
|
||||
start: number;
|
||||
end: number;
|
||||
mode: "remove" | "keep";
|
||||
times: number[]; // timeline seconds
|
||||
}
|
||||
```
|
||||
|
||||
Uso:
|
||||
- quitar silencios,
|
||||
- quitar errores,
|
||||
- recortar intro/outro,
|
||||
- construir highlights.
|
||||
- cortar en un punto específico,
|
||||
- aislar una sección antes de borrarla con otra tool usando dos tiempos,
|
||||
- preparar edición por rangos,
|
||||
- separar intro/outro sin eliminar nada,
|
||||
- dejarle al agente una primitive composable para flujos más grandes.
|
||||
|
||||
> `split` NO elimina material. Si el usuario pide “eliminá esta parte”, el agente debe componer `split` con una tool de borrado cuando exista.
|
||||
|
||||
---
|
||||
|
||||
|
|
@ -294,7 +295,7 @@ Descartada porque `analysisType` introduce categorías artificiales. Gemini debe
|
|||
|
||||
### `remove_silences`
|
||||
|
||||
Diferida/no prioritaria porque se puede expresar como flujo con `cut_segment`.
|
||||
Diferida/no prioritaria porque se puede expresar como flujo con `split` + tool de borrado.
|
||||
|
||||
### `generate_subtitles`
|
||||
|
||||
|
|
@ -311,7 +312,7 @@ Diferida porque el repo actual no parece tener corrección de color/LUTs impleme
|
|||
1. `list_project_assets`
|
||||
2. `list_timeline`
|
||||
3. `load_asset_context`
|
||||
4. `cut_segment`
|
||||
4. `split`
|
||||
5. `add_text`
|
||||
6. `add_media_to_timeline`
|
||||
7. `delete_element`
|
||||
|
|
|
|||
|
|
@ -50,7 +50,7 @@ Decisiones vigentes:
|
|||
Roadmap vigente de alto nivel:
|
||||
|
||||
1. **Contexto y percepción:** `list_project_assets`, `list_timeline`, `load_asset_context`.
|
||||
2. **Edición básica:** `cut_segment`, `add_text`, `add_media_to_timeline`, `delete_element`.
|
||||
2. **Edición básica:** `split`, `add_text`, `add_media_to_timeline`, `delete_element`.
|
||||
3. **Ajustes simples:** `set_volume`, `add_sticker`, `apply_effect`.
|
||||
4. **UX avanzada:** referencias `@asset`, progreso de uploads/procesamiento, previews/aprobaciones.
|
||||
|
||||
|
|
@ -421,7 +421,7 @@ Estas son las tools que el equipo debe priorizar. Son suficientemente pequeñas
|
|||
| `list_project_assets` | Lista assets del proyecto, usados/no usados y por tipo | Prioridad alta |
|
||||
| `list_timeline` | Devuelve resumen estructurado del timeline con `trackId`/`elementId` | Prioridad alta |
|
||||
| `load_asset_context` | Carga un asset en el contexto multimodal de Gemini y cachea la referencia | Prioridad crítica / killer feature |
|
||||
| `cut_segment` | Corta/remueve rangos de tiempo | Prioridad alta |
|
||||
| `split` | Hace cortes puntuales en uno o más timestamps sin borrar contenido | Prioridad alta |
|
||||
| `add_text` | Agrega texto visual: hooks, títulos, labels, subtítulos básicos | Prioridad alta |
|
||||
| `update_text` | Modifica texto existente | Prioridad media |
|
||||
| `add_media_to_timeline` | Inserta video/audio/imagen existente al timeline | Prioridad alta |
|
||||
|
|
@ -481,16 +481,16 @@ La siguiente lista se conserva como material de ideación, pero no debe tomarse
|
|||
```
|
||||
**Implementación (TS → FFmpeg o Rust WASM):** frame diff analysis. FFmpeg scene filter para rápido, Rust WASM para precisión.
|
||||
|
||||
#### 7.4. `cut_segment`
|
||||
#### 7.4. `split`
|
||||
```typescript
|
||||
{
|
||||
name: "cut_segment",
|
||||
description: "Cut a segment from the video and add it to the timeline",
|
||||
input: { video_id: string, start: number, end: number, track?: number },
|
||||
output: { clip_id: string, position: number }
|
||||
name: "split",
|
||||
description: "Split timeline elements at one or more timestamps without deleting content",
|
||||
input: { times: number[] /* timeline seconds */ },
|
||||
output: { success: boolean, affectedElements: string[] }
|
||||
}
|
||||
```
|
||||
**Implementación (TS → Zustand store):** manipula el `timelineStore` de NeuralCut directamente desde el agente en TS.
|
||||
**Implementación (TS → comandos del editor):** ejecuta splits determinísticos usando la infraestructura de timeline/commands para preservar undo/redo.
|
||||
|
||||
#### 7.5. `concat_segments`
|
||||
```typescript
|
||||
|
|
@ -671,7 +671,7 @@ los mejores momentos y subtítulos amarillos"
|
|||
│
|
||||
▼
|
||||
4. Para cada highlight:
|
||||
- cut_segment(start, end)
|
||||
- split({ times: [start, end] })
|
||||
│
|
||||
▼
|
||||
5. concat_segments(ids) ──────► Reel de ~30s
|
||||
|
|
@ -716,7 +716,7 @@ Usuario: "quita la parte donde me equivoco"
|
|||
Agente: watch_video("where does the person mess up?")
|
||||
→ identifica timestamp 1:23-1:31
|
||||
→ pide confirmación
|
||||
→ cut_segment para remover
|
||||
→ split({ times: [start, end] }) + tool de borrado
|
||||
```
|
||||
|
||||
---
|
||||
|
|
@ -809,7 +809,7 @@ Esto no reemplaza la primera feature de producto; la **habilita**.
|
|||
| **2** | 1 semana | Ideación + diseño | Wireframes del panel de chat. Diseño de la capa de abstracción. Decisión de scope de tools. |
|
||||
| **3** | 1 semana | Infraestructura habilitadora | ChatPanel mínimo funcional, `chatStore`/`agentStore`, endpoint `/api/agent/chat`, orquestador básico, registro de tools, tipos compartidos, integración con video activo. |
|
||||
| **4** | 1 semana | Primer vertical slice real | `transcribe_video` conectada end-to-end con provider real o local, respuesta visible en chat y timestamps persistidos en estado. |
|
||||
| **5** | 1 semana | Tier 1 tools | detect_silences, detect_scenes, cut_segment, concat_segments sobre la base ya creada. |
|
||||
| **5** | 1 semana | Tier 1 tools | detect_silences, detect_scenes, split, concat_segments sobre la base ya creada. |
|
||||
| **6** | 1 semana | Video understanding | watch_video + take_screenshot + describe_scene. Primera demo "wow". |
|
||||
| **7** | 1 semana | Creative tools | generate_subtitles, apply_lut, detect_faces. |
|
||||
| **8** | 1 semana | Reframe + beats | auto_reframe (MediaPipe), detect_beats (Essentia). |
|
||||
|
|
@ -835,7 +835,7 @@ Esto no reemplaza la primera feature de producto; la **habilita**.
|
|||
- Como agente, puedo listar assets del proyecto y saber cuáles están en el timeline.
|
||||
- Como agente, puedo cargar un asset con Gemini usando `load_asset_context` para entenderlo multimodalmente.
|
||||
- Como usuario, puedo preguntarle al agente sobre el contenido del video después de que el asset esté cargado en contexto.
|
||||
- Como usuario, puedo pedir cortes básicos por timestamp usando `cut_segment`.
|
||||
- Como usuario, puedo pedir cortes básicos por timestamp usando `split`.
|
||||
- Como usuario, puedo pedir texto visual básico usando `add_text`.
|
||||
|
||||
**Should-have actualizado:**
|
||||
|
|
|
|||
Loading…
Reference in New Issue