feat: add get_element and update_clip agent tools
Add two new agent tools for deep element inspection and property mutation: - get_element: returns full type-specific metadata for any timeline element (video, image, text, graphic, sticker, audio, effect) - update_clip: mutates any visual property on an element including masks, opacity, transform, blendMode, hidden, volume, muted, and name Masks are handled as a nested property with add/update/remove actions, aligned with the data model where masks are properties of elements rather than standalone timeline entities.
This commit is contained in:
Luis Esteban Acevedo Ladino
parent
9775f6f30c
commit
b48062a5f7
|
|
@ -24,6 +24,11 @@ import { canPlaceTimeSpansOnTrack } from "@/lib/timeline/placement/overlap";
|
|||
import { validateElementTrackCompatibility } from "@/lib/timeline/placement";
|
||||
import { findTrackInSceneTracks } from "@/lib/timeline/track-element-update";
|
||||
import { effectsRegistry } from "@/lib/effects";
|
||||
import { masksRegistry, buildDefaultMaskInstance } from "@/lib/masks";
|
||||
import type { MaskType } from "@/lib/masks/types";
|
||||
import type { MaskableElement } from "@/lib/timeline";
|
||||
import { isMaskableElement } from "@/lib/timeline/element-utils";
|
||||
import type { BlendMode, Transform } from "@/lib/rendering";
|
||||
|
||||
/**
|
||||
* Thin adapter: the ONLY file in agent/ that imports from core/.
|
||||
|
|
@ -648,7 +653,8 @@ export const EditorContextAdapter = {
|
|||
return { error: "Element is not an effect" };
|
||||
}
|
||||
|
||||
const effectElement = resolved.element as import("@/lib/timeline/types").EffectElement;
|
||||
const effectElement =
|
||||
resolved.element as import("@/lib/timeline/types").EffectElement;
|
||||
const definition = effectsRegistry.get(effectElement.effectType);
|
||||
if (!definition) {
|
||||
return { error: `Effect not found: ${effectElement.effectType}` };
|
||||
|
|
@ -666,7 +672,9 @@ export const EditorContextAdapter = {
|
|||
{
|
||||
trackId: resolved.track.id,
|
||||
elementId,
|
||||
patch: { params: mergedParams } as Partial<import("@/lib/timeline/types").TimelineElement>,
|
||||
patch: { params: mergedParams } as Partial<
|
||||
import("@/lib/timeline/types").TimelineElement
|
||||
>,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
|
@ -677,12 +685,264 @@ export const EditorContextAdapter = {
|
|||
appliedParams: mergedParams,
|
||||
};
|
||||
},
|
||||
|
||||
getElement({
|
||||
elementId,
|
||||
}: {
|
||||
elementId: string;
|
||||
}): Record<string, unknown> | { error: string } {
|
||||
const core = EditorCore.getInstance();
|
||||
const activeScene = core.scenes.getActiveSceneOrNull();
|
||||
if (!activeScene) {
|
||||
return { error: "No active timeline" };
|
||||
}
|
||||
|
||||
const [resolved] = findTimelineElementsWithTracksByIds({
|
||||
tracks: activeScene.tracks,
|
||||
elementIds: [elementId],
|
||||
});
|
||||
if (!resolved) {
|
||||
return { error: `Element not found: ${elementId}` };
|
||||
}
|
||||
|
||||
return serializeElement(resolved.element, resolved.track.id);
|
||||
},
|
||||
|
||||
updateClip({
|
||||
elementId,
|
||||
name,
|
||||
mask,
|
||||
opacity,
|
||||
positionX,
|
||||
positionY,
|
||||
rotation,
|
||||
scaleX,
|
||||
scaleY,
|
||||
blendMode,
|
||||
hidden,
|
||||
volume,
|
||||
muted,
|
||||
}: {
|
||||
elementId: string;
|
||||
name?: string;
|
||||
mask?: {
|
||||
action: "add" | "update" | "remove";
|
||||
maskType?: string;
|
||||
params?: Record<string, number | string | boolean>;
|
||||
};
|
||||
opacity?: number;
|
||||
positionX?: number;
|
||||
positionY?: number;
|
||||
rotation?: number;
|
||||
scaleX?: number;
|
||||
scaleY?: number;
|
||||
blendMode?: string;
|
||||
hidden?: boolean;
|
||||
volume?: number;
|
||||
muted?: boolean;
|
||||
}):
|
||||
| { success: boolean; elementId: string; applied: Record<string, unknown> }
|
||||
| { error: string } {
|
||||
const core = EditorCore.getInstance();
|
||||
const activeScene = core.scenes.getActiveSceneOrNull();
|
||||
if (!activeScene) {
|
||||
return { error: "No active timeline" };
|
||||
}
|
||||
|
||||
const [resolved] = findTimelineElementsWithTracksByIds({
|
||||
tracks: activeScene.tracks,
|
||||
elementIds: [elementId],
|
||||
});
|
||||
if (!resolved) {
|
||||
return { error: `Timeline element not found: ${elementId}` };
|
||||
}
|
||||
|
||||
const { element } = resolved;
|
||||
const patch: Record<string, unknown> = {};
|
||||
const applied: Record<string, unknown> = {};
|
||||
|
||||
if (name !== undefined) {
|
||||
if (typeof name !== "string" || !name.trim()) {
|
||||
return { error: "name must be a non-empty string" };
|
||||
}
|
||||
patch.name = name;
|
||||
applied.name = name;
|
||||
}
|
||||
|
||||
if (muted !== undefined) {
|
||||
if (typeof muted !== "boolean") {
|
||||
return { error: "muted must be a boolean" };
|
||||
}
|
||||
if (element.type !== "video" && element.type !== "audio") {
|
||||
return {
|
||||
error: `Element type '${element.type}' does not support muted`,
|
||||
};
|
||||
}
|
||||
patch.muted = muted;
|
||||
applied.muted = muted;
|
||||
}
|
||||
|
||||
if (opacity !== undefined) {
|
||||
if (typeof opacity !== "number" || opacity < 0 || opacity > 100) {
|
||||
return { error: "opacity must be a number between 0 and 100" };
|
||||
}
|
||||
if (!hasProperty(element, "opacity")) {
|
||||
return {
|
||||
error: `Element type '${element.type}' does not support opacity`,
|
||||
};
|
||||
}
|
||||
patch.opacity = opacity;
|
||||
applied.opacity = opacity;
|
||||
}
|
||||
|
||||
if (hidden !== undefined) {
|
||||
if (typeof hidden !== "boolean") {
|
||||
return { error: "hidden must be a boolean" };
|
||||
}
|
||||
patch.hidden = hidden;
|
||||
applied.hidden = hidden;
|
||||
}
|
||||
|
||||
if (blendMode !== undefined) {
|
||||
if (typeof blendMode !== "string") {
|
||||
return { error: "blendMode must be a string" };
|
||||
}
|
||||
if (!hasProperty(element, "blendMode")) {
|
||||
return {
|
||||
error: `Element type '${element.type}' does not support blendMode`,
|
||||
};
|
||||
}
|
||||
patch.blendMode = blendMode as BlendMode;
|
||||
applied.blendMode = blendMode;
|
||||
}
|
||||
|
||||
if (volume !== undefined) {
|
||||
if (typeof volume !== "number" || volume < 0 || volume > 100) {
|
||||
return { error: "volume must be a number between 0 and 100" };
|
||||
}
|
||||
if (element.type !== "video" && element.type !== "audio") {
|
||||
return {
|
||||
error: `Element type '${element.type}' does not support volume`,
|
||||
};
|
||||
}
|
||||
patch.volume = volume;
|
||||
applied.volume = volume;
|
||||
}
|
||||
|
||||
if (
|
||||
positionX !== undefined ||
|
||||
positionY !== undefined ||
|
||||
rotation !== undefined ||
|
||||
scaleX !== undefined ||
|
||||
scaleY !== undefined
|
||||
) {
|
||||
if (!hasProperty(element, "transform")) {
|
||||
return {
|
||||
error: `Element type '${element.type}' does not support transform properties`,
|
||||
};
|
||||
}
|
||||
const currentTransform = (element as { transform: Transform }).transform;
|
||||
const nextTransform: Transform = {
|
||||
scaleX: scaleX ?? currentTransform.scaleX,
|
||||
scaleY: scaleY ?? currentTransform.scaleY,
|
||||
position: {
|
||||
x: positionX ?? currentTransform.position.x,
|
||||
y: positionY ?? currentTransform.position.y,
|
||||
},
|
||||
rotate: rotation ?? currentTransform.rotate,
|
||||
};
|
||||
patch.transform = nextTransform;
|
||||
applied.transform = nextTransform;
|
||||
}
|
||||
|
||||
if (mask !== undefined) {
|
||||
if (!isMaskableElement(element)) {
|
||||
return {
|
||||
error: `Element type '${element.type}' does not support masks. Only video, image, and graphic elements support masks.`,
|
||||
};
|
||||
}
|
||||
|
||||
const maskable = element as MaskableElement;
|
||||
const currentMasks = maskable.masks ?? [];
|
||||
|
||||
if (mask.action === "add") {
|
||||
if (!mask.maskType || typeof mask.maskType !== "string") {
|
||||
return { error: "mask.maskType is required when action is 'add'" };
|
||||
}
|
||||
if (!masksRegistry.has(mask.maskType as MaskType)) {
|
||||
return {
|
||||
error: `Unknown mask type: ${mask.maskType}. Available: split, cinematic-bars, rectangle, ellipse, heart, diamond, star`,
|
||||
};
|
||||
}
|
||||
|
||||
const newMask = buildDefaultMaskInstance({
|
||||
maskType: mask.maskType as MaskType,
|
||||
});
|
||||
if (mask.params) {
|
||||
newMask.params = { ...newMask.params, ...mask.params };
|
||||
}
|
||||
patch.masks = [...currentMasks, newMask];
|
||||
applied.mask = {
|
||||
id: newMask.id,
|
||||
type: newMask.type,
|
||||
params: newMask.params,
|
||||
};
|
||||
} else if (mask.action === "update") {
|
||||
if (currentMasks.length === 0) {
|
||||
return {
|
||||
error: `Element has no mask to update. Use action 'add' first.`,
|
||||
};
|
||||
}
|
||||
if (!mask.params || typeof mask.params !== "object") {
|
||||
return { error: "mask.params is required when action is 'update'" };
|
||||
}
|
||||
|
||||
const existingMask = currentMasks[0];
|
||||
const updatedMasks = currentMasks.map((m, i) =>
|
||||
i === 0 ? { ...m, params: { ...m.params, ...mask.params } } : m,
|
||||
);
|
||||
patch.masks = updatedMasks;
|
||||
applied.mask = {
|
||||
type: existingMask.type,
|
||||
params: updatedMasks[0].params,
|
||||
};
|
||||
} else if (mask.action === "remove") {
|
||||
if (currentMasks.length === 0) {
|
||||
return { error: "Element has no mask to remove" };
|
||||
}
|
||||
patch.masks = [];
|
||||
applied.mask = null;
|
||||
}
|
||||
}
|
||||
|
||||
if (Object.keys(patch).length === 0) {
|
||||
return {
|
||||
error: "No properties to update. Provide at least one property.",
|
||||
};
|
||||
}
|
||||
|
||||
core.timeline.updateElements({
|
||||
updates: [
|
||||
{
|
||||
trackId: resolved.track.id,
|
||||
elementId,
|
||||
patch: patch as Partial<TimelineElement>,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
return { success: true, elementId, applied };
|
||||
},
|
||||
};
|
||||
|
||||
function secondsToTicks(seconds: number): number {
|
||||
return Math.round(seconds * TICKS_PER_SECOND);
|
||||
}
|
||||
|
||||
function hasProperty(obj: unknown, prop: string): boolean {
|
||||
return typeof obj === "object" && obj !== null && prop in obj;
|
||||
}
|
||||
|
||||
function ticksToSeconds(ticks: number): number {
|
||||
return ticks / TICKS_PER_SECOND;
|
||||
}
|
||||
|
|
@ -1070,3 +1330,104 @@ function validateEffectParams(
|
|||
}
|
||||
|
||||
export { buildSystemPrompt } from "@/agent/system-prompt";
|
||||
|
||||
function serializeElement(
|
||||
element: TimelineElement,
|
||||
trackId: string,
|
||||
): Record<string, unknown> {
|
||||
const base = {
|
||||
elementId: element.id,
|
||||
trackId,
|
||||
type: element.type,
|
||||
name: element.name,
|
||||
start: ticksToSeconds(element.startTime),
|
||||
end: ticksToSeconds(element.startTime + element.duration),
|
||||
duration: ticksToSeconds(element.duration),
|
||||
trimStart: ticksToSeconds(element.trimStart),
|
||||
trimEnd: ticksToSeconds(element.trimEnd),
|
||||
};
|
||||
|
||||
switch (element.type) {
|
||||
case "video":
|
||||
return {
|
||||
...base,
|
||||
assetId: element.mediaId,
|
||||
transform: element.transform,
|
||||
opacity: element.opacity,
|
||||
blendMode: element.blendMode ?? null,
|
||||
hidden: element.hidden ?? false,
|
||||
volume: element.volume ?? 100,
|
||||
muted: element.muted ?? false,
|
||||
masks: element.masks ?? [],
|
||||
effects: element.effects ?? [],
|
||||
};
|
||||
case "image":
|
||||
return {
|
||||
...base,
|
||||
assetId: element.mediaId,
|
||||
transform: element.transform,
|
||||
opacity: element.opacity,
|
||||
blendMode: element.blendMode ?? null,
|
||||
hidden: element.hidden ?? false,
|
||||
masks: element.masks ?? [],
|
||||
effects: element.effects ?? [],
|
||||
};
|
||||
case "text":
|
||||
return {
|
||||
...base,
|
||||
content: element.content,
|
||||
fontSize: element.fontSize,
|
||||
fontFamily: element.fontFamily,
|
||||
color: element.color,
|
||||
fontWeight: element.fontWeight,
|
||||
fontStyle: element.fontStyle,
|
||||
textAlign: element.textAlign,
|
||||
letterSpacing: element.letterSpacing ?? null,
|
||||
lineHeight: element.lineHeight ?? null,
|
||||
background: element.background,
|
||||
transform: element.transform,
|
||||
opacity: element.opacity,
|
||||
blendMode: element.blendMode ?? null,
|
||||
hidden: element.hidden ?? false,
|
||||
effects: element.effects ?? [],
|
||||
};
|
||||
case "sticker":
|
||||
return {
|
||||
...base,
|
||||
stickerId: element.stickerId,
|
||||
transform: element.transform,
|
||||
opacity: element.opacity,
|
||||
blendMode: element.blendMode ?? null,
|
||||
hidden: element.hidden ?? false,
|
||||
effects: element.effects ?? [],
|
||||
};
|
||||
case "graphic":
|
||||
return {
|
||||
...base,
|
||||
definitionId: element.definitionId,
|
||||
params: element.params,
|
||||
transform: element.transform,
|
||||
opacity: element.opacity,
|
||||
blendMode: element.blendMode ?? null,
|
||||
hidden: element.hidden ?? false,
|
||||
masks: element.masks ?? [],
|
||||
effects: element.effects ?? [],
|
||||
};
|
||||
case "audio":
|
||||
return {
|
||||
...base,
|
||||
assetId: element.sourceType === "upload" ? element.mediaId : null,
|
||||
sourceType: element.sourceType,
|
||||
volume: element.volume,
|
||||
muted: element.muted ?? false,
|
||||
};
|
||||
case "effect":
|
||||
return {
|
||||
...base,
|
||||
effectType: element.effectType,
|
||||
params: element.params,
|
||||
};
|
||||
default:
|
||||
return base;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,22 @@
|
|||
import type { AgentContext, ToolDefinition } from "@/agent/types";
|
||||
import { toolRegistry } from "@/agent/tools/registry";
|
||||
import { getElementSchema } from "@/agent/tools/schemas";
|
||||
import { EditorContextAdapter } from "@/agent/context";
|
||||
|
||||
const getElementTool: ToolDefinition = {
|
||||
...getElementSchema,
|
||||
execute: async (
|
||||
args: Record<string, unknown>,
|
||||
_context: AgentContext,
|
||||
): Promise<Record<string, unknown> | { error: string }> => {
|
||||
const elementId = args.elementId;
|
||||
|
||||
if (typeof elementId !== "string" || !elementId.trim()) {
|
||||
return { error: "Invalid elementId" };
|
||||
}
|
||||
|
||||
return EditorContextAdapter.getElement({ elementId });
|
||||
},
|
||||
};
|
||||
|
||||
toolRegistry.register(getElementSchema.name, getElementTool);
|
||||
|
|
@ -13,6 +13,7 @@
|
|||
import "@/agent/tools/load-context.tool";
|
||||
import "@/agent/tools/list-project-assets.tool";
|
||||
import "@/agent/tools/list-timeline.tool";
|
||||
import "@/agent/tools/get-element.tool";
|
||||
import "@/agent/tools/split.tool";
|
||||
import "@/agent/tools/delete-timeline-elements.tool";
|
||||
import "@/agent/tools/move-timeline-elements.tool";
|
||||
|
|
@ -24,3 +25,4 @@ import "@/agent/tools/list-effects.tool";
|
|||
import "@/agent/tools/get-effect.tool";
|
||||
import "@/agent/tools/apply-effect.tool";
|
||||
import "@/agent/tools/update-effect.tool";
|
||||
import "@/agent/tools/update-clip.tool";
|
||||
|
|
|
|||
|
|
@ -171,6 +171,34 @@ export const updateEffectSchema: ToolSchema = {
|
|||
],
|
||||
};
|
||||
|
||||
export const getElementSchema: ToolSchema = {
|
||||
name: "get_element",
|
||||
description:
|
||||
"Returns full metadata for a single timeline element. Use list_timeline to discover elementIds, then get_element for deep inspection. Returns type-specific properties: video/image/graphic elements include transform, opacity, blendMode, masks, hidden, and applied effects. Text elements include content, font styles, background, transform. Audio elements include volume, muted. Effect elements include effectType and all parameter values.",
|
||||
parameters: [{ key: "elementId", type: "string", required: true }],
|
||||
};
|
||||
|
||||
export const updateClipSchema: ToolSchema = {
|
||||
name: "update_clip",
|
||||
description:
|
||||
"Updates properties of any timeline element (video, image, graphic, text, sticker, audio, effect). Use list_timeline to discover elementId, then get_element to inspect current values. Only provide the properties you want to change. mask: { action: 'add', maskType } to add, { action: 'update', params: {...} } to modify, { action: 'remove' } to delete. Mask types: rectangle, ellipse, heart, diamond, star, split, cinematic-bars. Only video/image/graphic support masks. name: rename the element. opacity: 0-100. positionX/positionY: position offset. rotation: degrees. scaleX/scaleY: scale factor. blendMode: normal, darken, multiply, screen, etc. hidden: boolean. volume: 0-100 (video/audio only). muted: boolean (video/audio only).",
|
||||
parameters: [
|
||||
{ key: "elementId", type: "string", required: true },
|
||||
{ key: "name", type: "string", required: false },
|
||||
{ key: "mask", type: "object", required: false },
|
||||
{ key: "opacity", type: "number", required: false },
|
||||
{ key: "positionX", type: "number", required: false },
|
||||
{ key: "positionY", type: "number", required: false },
|
||||
{ key: "rotation", type: "number", required: false },
|
||||
{ key: "scaleX", type: "number", required: false },
|
||||
{ key: "scaleY", type: "number", required: false },
|
||||
{ key: "blendMode", type: "string", required: false },
|
||||
{ key: "hidden", type: "boolean", required: false },
|
||||
{ key: "volume", type: "number", required: false },
|
||||
{ key: "muted", type: "boolean", required: false },
|
||||
],
|
||||
};
|
||||
|
||||
/**
|
||||
* The exact list of schemas exposed to the LLM.
|
||||
* Excludes internal-only tools (transcribe_video, mock).
|
||||
|
|
@ -179,6 +207,7 @@ export const providerToolSchemas: ToolSchema[] = [
|
|||
loadContextSchema,
|
||||
listProjectAssetsSchema,
|
||||
listTimelineSchema,
|
||||
getElementSchema,
|
||||
splitSchema,
|
||||
deleteTimelineElementsSchema,
|
||||
moveTimelineElementsSchema,
|
||||
|
|
@ -190,4 +219,5 @@ export const providerToolSchemas: ToolSchema[] = [
|
|||
getEffectSchema,
|
||||
applyEffectSchema,
|
||||
updateEffectSchema,
|
||||
updateClipSchema,
|
||||
];
|
||||
|
|
|
|||
|
|
@ -0,0 +1,88 @@
|
|||
import type { AgentContext, ToolDefinition } from "@/agent/types";
|
||||
import { toolRegistry } from "@/agent/tools/registry";
|
||||
import { updateClipSchema } from "@/agent/tools/schemas";
|
||||
import { EditorContextAdapter } from "@/agent/context";
|
||||
|
||||
const updateClipTool: ToolDefinition = {
|
||||
...updateClipSchema,
|
||||
execute: async (
|
||||
args: Record<string, unknown>,
|
||||
_context: AgentContext,
|
||||
): Promise<
|
||||
| { success: boolean; elementId: string; applied: Record<string, unknown> }
|
||||
| { error: string }
|
||||
> => {
|
||||
const elementId = args.elementId;
|
||||
const name = args.name as string | undefined;
|
||||
const mask = args.mask as
|
||||
| {
|
||||
action: "add" | "update" | "remove";
|
||||
maskType?: string;
|
||||
params?: Record<string, number | string | boolean>;
|
||||
}
|
||||
| undefined;
|
||||
const opacity = args.opacity as number | undefined;
|
||||
const positionX = args.positionX as number | undefined;
|
||||
const positionY = args.positionY as number | undefined;
|
||||
const rotation = args.rotation as number | undefined;
|
||||
const scaleX = args.scaleX as number | undefined;
|
||||
const scaleY = args.scaleY as number | undefined;
|
||||
const blendMode = args.blendMode as string | undefined;
|
||||
const hidden = args.hidden as boolean | undefined;
|
||||
const volume = args.volume as number | undefined;
|
||||
const muted = args.muted as boolean | undefined;
|
||||
|
||||
if (typeof elementId !== "string" || !elementId.trim()) {
|
||||
return { error: "Invalid elementId" };
|
||||
}
|
||||
|
||||
const hasUpdate =
|
||||
name !== undefined ||
|
||||
mask !== undefined ||
|
||||
opacity !== undefined ||
|
||||
positionX !== undefined ||
|
||||
positionY !== undefined ||
|
||||
rotation !== undefined ||
|
||||
scaleX !== undefined ||
|
||||
scaleY !== undefined ||
|
||||
blendMode !== undefined ||
|
||||
hidden !== undefined ||
|
||||
volume !== undefined ||
|
||||
muted !== undefined;
|
||||
|
||||
if (!hasUpdate) {
|
||||
return {
|
||||
error: "No properties to update. Provide at least one property.",
|
||||
};
|
||||
}
|
||||
|
||||
if (
|
||||
mask !== undefined &&
|
||||
(typeof mask !== "object" || Array.isArray(mask))
|
||||
) {
|
||||
return { error: "mask must be an object with an 'action' property" };
|
||||
}
|
||||
|
||||
if (mask && !["add", "update", "remove"].includes(mask.action)) {
|
||||
return { error: "mask.action must be 'add', 'update', or 'remove'" };
|
||||
}
|
||||
|
||||
return EditorContextAdapter.updateClip({
|
||||
elementId,
|
||||
name,
|
||||
mask,
|
||||
opacity,
|
||||
positionX,
|
||||
positionY,
|
||||
rotation,
|
||||
scaleX,
|
||||
scaleY,
|
||||
blendMode,
|
||||
hidden,
|
||||
volume,
|
||||
muted,
|
||||
});
|
||||
},
|
||||
};
|
||||
|
||||
toolRegistry.register(updateClipSchema.name, updateClipTool);
|
||||
Loading…
Reference in New Issue