feat: add transcribe video tool flow
This commit is contained in:
Luis Esteban Acevedo Ladino
parent
a93c45586b
commit
e2d1cc8f34
|
|
@ -0,0 +1,160 @@
|
|||
import { beforeEach, describe, expect, mock, test } from "bun:test";
|
||||
|
||||
// --- Module mocks (must be set up before importing the module under test) ---
|
||||
|
||||
// bun:test exposes jest.fn() for mock function creation at runtime
|
||||
// We type the factory explicitly to avoid referencing the `jest` global type
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
type MockFn = {
|
||||
(...args: any[]): any;
|
||||
mockReturnValue(value: any): void;
|
||||
mockClear(): void;
|
||||
mockImplementation(fn: (...args: any[]) => any): void;
|
||||
};
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
type MockFnFactory = <T extends (...args: any[]) => any>() => T & MockFn;
|
||||
|
||||
const bunTestModule = await import("bun:test");
|
||||
const jestObj = (bunTestModule as Record<string, unknown>).jest as Record<string, unknown>;
|
||||
const mockFn = jestObj.fn as unknown as MockFnFactory;
|
||||
|
||||
const mockGetAssets = mockFn<() => any[]>();
|
||||
|
||||
mock.module("@/core", () => ({
|
||||
EditorCore: {
|
||||
getInstance: () => ({
|
||||
media: { getAssets: mockGetAssets },
|
||||
}),
|
||||
},
|
||||
}));
|
||||
|
||||
mock.module("@/lib/wasm", () => ({
|
||||
TICKS_PER_SECOND: 100,
|
||||
}));
|
||||
|
||||
const { EditorContextAdapter } = await import("@/agent/context");
|
||||
|
||||
// --- Helpers ---
|
||||
|
||||
function fakeAsset(
|
||||
overrides: Partial<{
|
||||
id: string;
|
||||
name: string;
|
||||
type: string;
|
||||
duration: number;
|
||||
file: File;
|
||||
hasAudio: boolean;
|
||||
}> = {},
|
||||
) {
|
||||
return {
|
||||
id: "v1",
|
||||
name: "clip.mp4",
|
||||
type: "video",
|
||||
duration: 30,
|
||||
file: new File([], "clip.mp4", { type: "video/mp4" }),
|
||||
hasAudio: true,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
// --- Tests ---
|
||||
|
||||
describe("EditorContextAdapter.resolveAssetFile", () => {
|
||||
beforeEach(() => {
|
||||
mockGetAssets.mockClear();
|
||||
});
|
||||
|
||||
test("returns File when assetId matches an asset", () => {
|
||||
const videoFile = new File([], "intro.mp4", { type: "video/mp4" });
|
||||
mockGetAssets.mockReturnValue([
|
||||
fakeAsset({ id: "v1", name: "intro.mp4", file: videoFile }),
|
||||
]);
|
||||
|
||||
const result = EditorContextAdapter.resolveAssetFile("v1");
|
||||
expect(result).toBe(videoFile);
|
||||
});
|
||||
|
||||
test("returns null when assetId does not match any asset", () => {
|
||||
mockGetAssets.mockReturnValue([
|
||||
fakeAsset({ id: "v1", name: "intro.mp4" }),
|
||||
]);
|
||||
|
||||
const result = EditorContextAdapter.resolveAssetFile("nonexistent");
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
test("returns File for first video asset when no assetId provided", () => {
|
||||
const videoFile = new File([], "clip.mp4", { type: "video/mp4" });
|
||||
mockGetAssets.mockReturnValue([
|
||||
fakeAsset({ id: "img1", name: "photo.jpg", type: "image", file: new File([], "p.jpg") }),
|
||||
fakeAsset({ id: "v1", name: "clip.mp4", type: "video", file: videoFile }),
|
||||
]);
|
||||
|
||||
const result = EditorContextAdapter.resolveAssetFile();
|
||||
expect(result).toBe(videoFile);
|
||||
});
|
||||
|
||||
test("returns File for first audio asset when no assetId and no video assets", () => {
|
||||
const audioFile = new File([], "song.mp3", { type: "audio/mpeg" });
|
||||
mockGetAssets.mockReturnValue([
|
||||
fakeAsset({ id: "img1", name: "photo.jpg", type: "image", file: new File([], "p.jpg") }),
|
||||
fakeAsset({ id: "a1", name: "song.mp3", type: "audio", file: audioFile }),
|
||||
]);
|
||||
|
||||
const result = EditorContextAdapter.resolveAssetFile();
|
||||
expect(result).toBe(audioFile);
|
||||
});
|
||||
|
||||
test("returns null when assets array is empty", () => {
|
||||
mockGetAssets.mockReturnValue([]);
|
||||
|
||||
const result = EditorContextAdapter.resolveAssetFile();
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
test("returns null when no video or audio assets exist and no assetId", () => {
|
||||
mockGetAssets.mockReturnValue([
|
||||
fakeAsset({ id: "img1", name: "photo.jpg", type: "image" }),
|
||||
]);
|
||||
|
||||
const result = EditorContextAdapter.resolveAssetFile();
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
test("returns null when matching asset has no file property", () => {
|
||||
mockGetAssets.mockReturnValue([
|
||||
{ id: "v1", name: "clip.mp4", type: "video" },
|
||||
]);
|
||||
|
||||
const result = EditorContextAdapter.resolveAssetFile("v1");
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("EditorContextAdapter.getAssetHasAudio", () => {
|
||||
beforeEach(() => {
|
||||
mockGetAssets.mockClear();
|
||||
});
|
||||
|
||||
test("returns true when asset has audio", () => {
|
||||
mockGetAssets.mockReturnValue([
|
||||
fakeAsset({ id: "v1", hasAudio: true }),
|
||||
]);
|
||||
|
||||
expect(EditorContextAdapter.getAssetHasAudio("v1")).toBe(true);
|
||||
});
|
||||
|
||||
test("returns false when asset has no audio", () => {
|
||||
mockGetAssets.mockReturnValue([
|
||||
fakeAsset({ id: "v1", hasAudio: false }),
|
||||
]);
|
||||
|
||||
expect(EditorContextAdapter.getAssetHasAudio("v1")).toBe(false);
|
||||
});
|
||||
|
||||
test("returns undefined when asset is not found", () => {
|
||||
mockGetAssets.mockReturnValue([]);
|
||||
|
||||
expect(EditorContextAdapter.getAssetHasAudio("nonexistent")).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
|
@ -18,6 +18,38 @@ export const EditorContextAdapter = {
|
|||
ticksPerSecond: TICKS_PER_SECOND,
|
||||
});
|
||||
},
|
||||
|
||||
/**
|
||||
* Resolve a media asset's File reference.
|
||||
* If assetId is provided, finds the exact asset by ID.
|
||||
* If omitted, returns the File for the first video or audio asset.
|
||||
* Returns null when no matching asset exists.
|
||||
* This is the ONLY sanctioned path for tools to obtain a File from EditorCore.
|
||||
*/
|
||||
resolveAssetFile(assetId?: string): File | null {
|
||||
const core = EditorCore.getInstance();
|
||||
const assets = core.media.getAssets();
|
||||
|
||||
if (assetId !== undefined) {
|
||||
const asset = assets.find((a) => a.id === assetId);
|
||||
return asset?.file ?? null;
|
||||
}
|
||||
|
||||
const firstMediaAsset = assets.find(
|
||||
(a) => a.type === "video" || a.type === "audio",
|
||||
);
|
||||
return firstMediaAsset?.file ?? null;
|
||||
},
|
||||
|
||||
/**
|
||||
* Check whether a media asset has an audio track.
|
||||
* Returns undefined when the asset is not found.
|
||||
*/
|
||||
getAssetHasAudio(assetId: string): boolean | undefined {
|
||||
const core = EditorCore.getInstance();
|
||||
const asset = core.media.getAssets().find((a) => a.id === assetId);
|
||||
return asset?.hasAudio;
|
||||
},
|
||||
};
|
||||
|
||||
export { buildSystemPrompt } from "@/agent/system-prompt";
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import type {
|
|||
} from "@/agent/types";
|
||||
import { toolRegistry } from "@/agent/tools/registry";
|
||||
import "@/agent/tools/mock.tool";
|
||||
import "@/agent/tools/transcribe-video.tool";
|
||||
import { useChatStore } from "@/stores/chat-store";
|
||||
import { useAgentStore } from "@/stores/agent-store";
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,282 @@
|
|||
import { beforeEach, describe, expect, mock, test } from "bun:test";
|
||||
import type { AgentContext } from "@/agent/types";
|
||||
import { toolRegistry } from "@/agent/tools/registry";
|
||||
|
||||
// bun:test exposes jest.fn() for mock function creation at runtime
|
||||
// We type the factory explicitly to avoid referencing the `jest` global type
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
type MockFn = {
|
||||
(...args: any[]): any;
|
||||
mockReturnValue(value: any): void;
|
||||
mockResolvedValue(value: any): void;
|
||||
mockClear(): void;
|
||||
mockImplementation(fn: (...args: any[]) => any): void;
|
||||
};
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
type MockFnFactory = <T extends (...args: any[]) => any>() => T & MockFn;
|
||||
|
||||
const bunTestModule = await import("bun:test");
|
||||
const jestObj = (bunTestModule as Record<string, unknown>).jest as Record<string, unknown>;
|
||||
const mockFn = jestObj.fn as unknown as MockFnFactory;
|
||||
|
||||
// --- Module mocks for tool dependencies ---
|
||||
|
||||
const mockResolveAssetFile = mockFn<(assetId?: string) => File | null>();
|
||||
const mockGetAssetHasAudio = mockFn<(assetId: string) => boolean | undefined>();
|
||||
const mockDecodeAudio = mockFn();
|
||||
const mockTranscribe = mockFn();
|
||||
|
||||
mock.module("@/agent/context", () => ({
|
||||
EditorContextAdapter: {
|
||||
resolveAssetFile: mockResolveAssetFile,
|
||||
getAssetHasAudio: mockGetAssetHasAudio,
|
||||
},
|
||||
buildSystemPrompt: () => "",
|
||||
}));
|
||||
|
||||
mock.module("@/lib/media/audio", () => ({
|
||||
decodeAudioToFloat32: mockDecodeAudio,
|
||||
}));
|
||||
|
||||
mock.module("@/services/transcription/service", () => ({
|
||||
transcriptionService: {
|
||||
transcribe: mockTranscribe,
|
||||
},
|
||||
}));
|
||||
|
||||
// Import the tool module to trigger side-effect registration
|
||||
await import("@/agent/tools/transcribe-video.tool");
|
||||
|
||||
// --- Helpers ---
|
||||
|
||||
function makeContext(
|
||||
overrides: Partial<AgentContext> = {},
|
||||
): AgentContext {
|
||||
return {
|
||||
projectId: "proj-1",
|
||||
activeSceneId: "scene-A",
|
||||
mediaAssets: [],
|
||||
playbackTimeMs: 0,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function fakeVideoAsset(
|
||||
overrides: Partial<{ id: string; name: string; type: string; duration: number }> = {},
|
||||
) {
|
||||
return {
|
||||
id: "v1",
|
||||
name: "clip.mp4",
|
||||
type: "video",
|
||||
duration: 30,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function fakeTranscriptionResult() {
|
||||
return {
|
||||
text: "Hello world, this is a test.",
|
||||
segments: [
|
||||
{ text: "Hello world,", start: 0, end: 2.5 },
|
||||
{ text: "this is a test.", start: 2.5, end: 5.0 },
|
||||
],
|
||||
language: "en",
|
||||
};
|
||||
}
|
||||
|
||||
// --- Tests ---
|
||||
|
||||
describe("transcribe_video tool — registration (contract)", () => {
|
||||
test("is registered in the tool registry", () => {
|
||||
expect(toolRegistry.has("transcribe_video")).toBe(true);
|
||||
});
|
||||
|
||||
test("has correct name and description", () => {
|
||||
const tool = toolRegistry.get("transcribe_video");
|
||||
expect(tool.name).toBe("transcribe_video");
|
||||
expect(tool.description).toContain("transcri");
|
||||
});
|
||||
|
||||
test("declares assetId as optional parameter", () => {
|
||||
const tool = toolRegistry.get("transcribe_video");
|
||||
const assetIdParam = tool.parameters.find((p) => p.key === "assetId");
|
||||
expect(assetIdParam).toBeDefined();
|
||||
expect(assetIdParam!.required).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("transcribe_video tool — success path", () => {
|
||||
beforeEach(() => {
|
||||
mockResolveAssetFile.mockClear();
|
||||
mockGetAssetHasAudio.mockClear();
|
||||
mockDecodeAudio.mockClear();
|
||||
mockTranscribe.mockClear();
|
||||
});
|
||||
|
||||
test("transcribes a single video asset successfully", async () => {
|
||||
const fakeFile = new File([], "clip.mp4", { type: "video/mp4" });
|
||||
const context = makeContext({
|
||||
mediaAssets: [fakeVideoAsset()],
|
||||
});
|
||||
|
||||
mockResolveAssetFile.mockReturnValue(fakeFile);
|
||||
mockGetAssetHasAudio.mockReturnValue(true);
|
||||
mockDecodeAudio.mockResolvedValue({
|
||||
samples: new Float32Array(16000),
|
||||
sampleRate: 16000,
|
||||
});
|
||||
mockTranscribe.mockResolvedValue(fakeTranscriptionResult());
|
||||
|
||||
const tool = toolRegistry.get("transcribe_video");
|
||||
const result = await tool.execute({}, context);
|
||||
|
||||
// Verify result shape
|
||||
expect(result).toEqual({
|
||||
assetName: "clip.mp4",
|
||||
language: "en",
|
||||
fullText: "Hello world, this is a test.",
|
||||
segmentCount: 2,
|
||||
segments: fakeTranscriptionResult().segments,
|
||||
duration: 30,
|
||||
});
|
||||
|
||||
// Verify the adapter was called with the correct asset id
|
||||
expect(mockResolveAssetFile).toHaveBeenCalledWith("v1");
|
||||
expect(mockGetAssetHasAudio).toHaveBeenCalledWith("v1");
|
||||
});
|
||||
|
||||
test("transcribes with explicit assetId", async () => {
|
||||
const fakeFile = new File([], "other.mp4", { type: "video/mp4" });
|
||||
const context = makeContext({
|
||||
mediaAssets: [
|
||||
fakeVideoAsset({ id: "v1", name: "first.mp4" }),
|
||||
fakeVideoAsset({ id: "v2", name: "other.mp4" }),
|
||||
],
|
||||
});
|
||||
|
||||
mockResolveAssetFile.mockReturnValue(fakeFile);
|
||||
mockGetAssetHasAudio.mockReturnValue(true);
|
||||
mockDecodeAudio.mockResolvedValue({
|
||||
samples: new Float32Array(16000),
|
||||
sampleRate: 16000,
|
||||
});
|
||||
mockTranscribe.mockResolvedValue(fakeTranscriptionResult());
|
||||
|
||||
const tool = toolRegistry.get("transcribe_video");
|
||||
const result = await tool.execute({ assetId: "v2" }, context);
|
||||
|
||||
expect(result).toEqual(
|
||||
expect.objectContaining({ assetName: "other.mp4" }),
|
||||
);
|
||||
expect(mockResolveAssetFile).toHaveBeenCalledWith("v2");
|
||||
});
|
||||
});
|
||||
|
||||
describe("transcribe_video tool — error branches", () => {
|
||||
beforeEach(() => {
|
||||
mockResolveAssetFile.mockClear();
|
||||
mockGetAssetHasAudio.mockClear();
|
||||
mockDecodeAudio.mockClear();
|
||||
mockTranscribe.mockClear();
|
||||
});
|
||||
|
||||
test("returns error when no media assets in project", async () => {
|
||||
const context = makeContext({ mediaAssets: [] });
|
||||
const tool = toolRegistry.get("transcribe_video");
|
||||
|
||||
const result = await tool.execute({}, context);
|
||||
expect(result).toEqual({ error: "No active media asset" });
|
||||
});
|
||||
|
||||
test("returns error when only image assets found (no audio track)", async () => {
|
||||
const context = makeContext({
|
||||
mediaAssets: [
|
||||
{ id: "img1", name: "photo.jpg", type: "image", duration: 0 },
|
||||
],
|
||||
});
|
||||
const tool = toolRegistry.get("transcribe_video");
|
||||
|
||||
const result = await tool.execute({}, context);
|
||||
expect(result).toEqual({ error: "Asset has no audio track" });
|
||||
});
|
||||
|
||||
test("returns error when multiple candidates but no assetId", async () => {
|
||||
const context = makeContext({
|
||||
mediaAssets: [
|
||||
fakeVideoAsset({ id: "v1", name: "intro.mp4" }),
|
||||
fakeVideoAsset({ id: "v2", name: "outro.mp4" }),
|
||||
],
|
||||
});
|
||||
const tool = toolRegistry.get("transcribe_video");
|
||||
|
||||
const result = await tool.execute({}, context);
|
||||
expect(result).toEqual({
|
||||
error: "Multiple video/audio assets found: intro.mp4, outro.mp4. Specify which one with assetId.",
|
||||
});
|
||||
});
|
||||
|
||||
test("returns error when specified assetId not in candidates", async () => {
|
||||
const context = makeContext({
|
||||
mediaAssets: [
|
||||
fakeVideoAsset({ id: "v1", name: "intro.mp4" }),
|
||||
],
|
||||
});
|
||||
const tool = toolRegistry.get("transcribe_video");
|
||||
|
||||
const result = await tool.execute({ assetId: "v99" }, context);
|
||||
expect(result).toEqual({
|
||||
error: "Could not access file for asset v99",
|
||||
});
|
||||
});
|
||||
|
||||
test("returns error when resolveAssetFile returns null", async () => {
|
||||
const context = makeContext({
|
||||
mediaAssets: [fakeVideoAsset()],
|
||||
});
|
||||
|
||||
mockResolveAssetFile.mockReturnValue(null);
|
||||
|
||||
const tool = toolRegistry.get("transcribe_video");
|
||||
const result = await tool.execute({}, context);
|
||||
|
||||
expect(result).toEqual({ error: "Could not access file for asset v1" });
|
||||
});
|
||||
|
||||
test("returns error when asset has no audio track", async () => {
|
||||
const fakeFile = new File([], "silent.mp4", { type: "video/mp4" });
|
||||
const context = makeContext({
|
||||
mediaAssets: [fakeVideoAsset({ name: "silent.mp4" })],
|
||||
});
|
||||
|
||||
mockResolveAssetFile.mockReturnValue(fakeFile);
|
||||
mockGetAssetHasAudio.mockReturnValue(false);
|
||||
|
||||
const tool = toolRegistry.get("transcribe_video");
|
||||
const result = await tool.execute({}, context);
|
||||
|
||||
expect(result).toEqual({ error: "Asset has no audio track" });
|
||||
});
|
||||
|
||||
test("returns error when transcriptionService.transcribe rejects", async () => {
|
||||
const fakeFile = new File([], "clip.mp4", { type: "video/mp4" });
|
||||
const context = makeContext({
|
||||
mediaAssets: [fakeVideoAsset()],
|
||||
});
|
||||
|
||||
mockResolveAssetFile.mockReturnValue(fakeFile);
|
||||
mockGetAssetHasAudio.mockReturnValue(true);
|
||||
mockDecodeAudio.mockResolvedValue({
|
||||
samples: new Float32Array(16000),
|
||||
sampleRate: 16000,
|
||||
});
|
||||
mockTranscribe.mockImplementation(() =>
|
||||
Promise.reject(new Error("Worker not initialized")),
|
||||
);
|
||||
|
||||
const tool = toolRegistry.get("transcribe_video");
|
||||
const result = await tool.execute({}, context);
|
||||
|
||||
// Tool catches service errors and returns { error } per spec
|
||||
expect(result).toEqual({ error: "Worker not initialized" });
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,118 @@
|
|||
import type { AgentContext, ToolDefinition } from "@/agent/types";
|
||||
import type {
|
||||
TranscriptionLanguage,
|
||||
TranscriptionModelId,
|
||||
} from "@/lib/transcription/types";
|
||||
import { toolRegistry } from "@/agent/tools/registry";
|
||||
import { EditorContextAdapter } from "@/agent/context";
|
||||
import { decodeAudioToFloat32 } from "@/lib/media/audio";
|
||||
import { transcriptionService } from "@/services/transcription/service";
|
||||
|
||||
export type TranscriptionToolResult = {
|
||||
assetName: string;
|
||||
language: string;
|
||||
fullText: string;
|
||||
segmentCount: number;
|
||||
segments: Array<{ text: string; start: number; end: number }>;
|
||||
duration: number;
|
||||
};
|
||||
|
||||
export type TranscribeVideoArgs = {
|
||||
assetId?: string;
|
||||
language?: TranscriptionLanguage;
|
||||
modelId?: TranscriptionModelId;
|
||||
};
|
||||
|
||||
const transcribeVideoTool: ToolDefinition = {
|
||||
name: "transcribe_video",
|
||||
description:
|
||||
"Transcribes the audio of a video or audio asset using Whisper. Returns structured transcript with segments and timestamps.",
|
||||
parameters: [
|
||||
{ key: "assetId", type: "string", required: false },
|
||||
{ key: "language", type: "string", required: false },
|
||||
{ key: "modelId", type: "string", required: false },
|
||||
],
|
||||
execute: async (
|
||||
args: Record<string, unknown>,
|
||||
context: AgentContext,
|
||||
): Promise<TranscriptionToolResult | { error: string }> => {
|
||||
const typedArgs = args as TranscribeVideoArgs;
|
||||
|
||||
// Validate: must have media assets
|
||||
if (context.mediaAssets.length === 0) {
|
||||
return { error: "No active media asset" };
|
||||
}
|
||||
|
||||
// Find candidate assets (video or audio only)
|
||||
const mediaCandidates = context.mediaAssets.filter(
|
||||
(a) => a.type === "video" || a.type === "audio",
|
||||
);
|
||||
|
||||
if (mediaCandidates.length === 0) {
|
||||
// Assets exist but none are video/audio (e.g. images only)
|
||||
return { error: "Asset has no audio track" };
|
||||
}
|
||||
|
||||
// Resolve the target asset
|
||||
let targetAsset: (typeof mediaCandidates)[number] | undefined;
|
||||
|
||||
if (typedArgs.assetId) {
|
||||
targetAsset = mediaCandidates.find((a) => a.id === typedArgs.assetId);
|
||||
if (!targetAsset) {
|
||||
return {
|
||||
error: `Could not access file for asset ${typedArgs.assetId}`,
|
||||
};
|
||||
}
|
||||
} else {
|
||||
if (mediaCandidates.length > 1) {
|
||||
const names = mediaCandidates.map((a) => a.name).join(", ");
|
||||
return {
|
||||
error: `Multiple video/audio assets found: ${names}. Specify which one with assetId.`,
|
||||
};
|
||||
}
|
||||
targetAsset = mediaCandidates[0];
|
||||
}
|
||||
|
||||
// Resolve the actual File via the adapter
|
||||
const file = EditorContextAdapter.resolveAssetFile(targetAsset.id);
|
||||
if (!file) {
|
||||
return { error: `Could not access file for asset ${targetAsset.id}` };
|
||||
}
|
||||
|
||||
// Guard: asset must have an audio track
|
||||
const hasAudio = EditorContextAdapter.getAssetHasAudio(targetAsset.id);
|
||||
if (hasAudio === false) {
|
||||
return { error: "Asset has no audio track" };
|
||||
}
|
||||
|
||||
try {
|
||||
// Decode audio → Float32Array
|
||||
const { samples } = await decodeAudioToFloat32({
|
||||
audioBlob: file,
|
||||
sampleRate: 16000,
|
||||
});
|
||||
|
||||
// Run Whisper transcription
|
||||
const result = await transcriptionService.transcribe({
|
||||
audioData: samples,
|
||||
language: typedArgs.language ?? "auto",
|
||||
modelId: typedArgs.modelId ?? "whisper-small",
|
||||
});
|
||||
|
||||
return {
|
||||
assetName: targetAsset.name,
|
||||
language: result.language,
|
||||
fullText: result.text,
|
||||
segmentCount: result.segments.length,
|
||||
segments: result.segments,
|
||||
duration: targetAsset.duration,
|
||||
};
|
||||
} catch (err) {
|
||||
const message =
|
||||
err instanceof Error ? err.message : "Transcription failed";
|
||||
return { error: message };
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
toolRegistry.register("transcribe_video", transcribeVideoTool);
|
||||
|
|
@ -64,4 +64,55 @@ describe("POST /api/agent/chat", () => {
|
|||
const res = await POST(req);
|
||||
expect(res.status).toBe(400);
|
||||
});
|
||||
|
||||
test("returns transcribe_video tool call when message contains 'transcribe'", async () => {
|
||||
const req = makeRequest({
|
||||
messages: [{ role: "user", content: "Please transcribe this video" }],
|
||||
});
|
||||
|
||||
const res = await POST(req);
|
||||
expect(res.status).toBe(200);
|
||||
|
||||
const data = await res.json();
|
||||
expect(data.content).toBe("I'll transcribe your video now.");
|
||||
expect(data.toolCalls).toHaveLength(1);
|
||||
expect(data.toolCalls[0].name).toBe("transcribe_video");
|
||||
expect(data.toolCalls[0].id).toBe("tc_transcribe_1");
|
||||
});
|
||||
|
||||
test("returns transcribe_video tool call for case-insensitive 'TRANSCRIBE'", async () => {
|
||||
const req = makeRequest({
|
||||
messages: [{ role: "user", content: "CAN YOU TRANSCRIBE THE AUDIO?" }],
|
||||
});
|
||||
|
||||
const res = await POST(req);
|
||||
expect(res.status).toBe(200);
|
||||
|
||||
const data = await res.json();
|
||||
expect(data.toolCalls[0].name).toBe("transcribe_video");
|
||||
});
|
||||
|
||||
test("returns transcribe_video when 'transcription' keyword is present", async () => {
|
||||
const req = makeRequest({
|
||||
messages: [{ role: "user", content: "I need a transcription of this clip" }],
|
||||
});
|
||||
|
||||
const res = await POST(req);
|
||||
expect(res.status).toBe(200);
|
||||
|
||||
const data = await res.json();
|
||||
expect(data.toolCalls[0].name).toBe("transcribe_video");
|
||||
});
|
||||
|
||||
test("falls back to echo_context when message has no transcription intent", async () => {
|
||||
const req = makeRequest({
|
||||
messages: [{ role: "user", content: "What's the weather like?" }],
|
||||
});
|
||||
|
||||
const res = await POST(req);
|
||||
expect(res.status).toBe(200);
|
||||
|
||||
const data = await res.json();
|
||||
expect(data.toolCalls[0].name).toBe("echo_context");
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -22,8 +22,28 @@ export async function POST(request: NextRequest) {
|
|||
);
|
||||
}
|
||||
|
||||
// Phase 1: canned mock response with a tool call to validate the full pipeline
|
||||
const mockResponse = {
|
||||
// Mock intent detection: keyword match on last user message
|
||||
const lastUserMessage = [...result.data.messages]
|
||||
.reverse()
|
||||
.find((m) => m.role === "user");
|
||||
const isTranscriptionRequest =
|
||||
lastUserMessage?.content.toLowerCase().includes("transcri") ?? false;
|
||||
|
||||
if (isTranscriptionRequest) {
|
||||
return NextResponse.json({
|
||||
content: "I'll transcribe your video now.",
|
||||
toolCalls: [
|
||||
{
|
||||
id: "tc_transcribe_1",
|
||||
name: "transcribe_video",
|
||||
args: {},
|
||||
},
|
||||
],
|
||||
});
|
||||
}
|
||||
|
||||
// Default: echo_context fallback
|
||||
return NextResponse.json({
|
||||
content: "Let me check your editor context.",
|
||||
toolCalls: [
|
||||
{
|
||||
|
|
@ -32,7 +52,5 @@ export async function POST(request: NextRequest) {
|
|||
args: {},
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
return NextResponse.json(mockResponse);
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,134 @@
|
|||
import { describe, expect, test } from "bun:test";
|
||||
import {
|
||||
isTranscriptData,
|
||||
formatTimestamp,
|
||||
} from "../transcript-utils";
|
||||
|
||||
describe("isTranscriptData — transcript detection", () => {
|
||||
test("returns true for valid transcript data", () => {
|
||||
const data = {
|
||||
assetName: "clip.mp4",
|
||||
language: "en",
|
||||
fullText: "Hello world",
|
||||
segments: [{ text: "Hello world", start: 0, end: 2.5 }],
|
||||
duration: 30,
|
||||
};
|
||||
|
||||
expect(isTranscriptData(data)).toBe(true);
|
||||
});
|
||||
|
||||
test("returns true for transcript with empty segments array", () => {
|
||||
const data = {
|
||||
assetName: "clip.mp4",
|
||||
language: "en",
|
||||
fullText: "",
|
||||
segments: [],
|
||||
duration: 0,
|
||||
};
|
||||
|
||||
expect(isTranscriptData(data)).toBe(true);
|
||||
});
|
||||
|
||||
test("returns false when fullText is missing", () => {
|
||||
const data = {
|
||||
assetName: "clip.mp4",
|
||||
language: "en",
|
||||
segments: [{ text: "Hello", start: 0, end: 1 }],
|
||||
duration: 30,
|
||||
};
|
||||
|
||||
expect(isTranscriptData(data)).toBe(false);
|
||||
});
|
||||
|
||||
test("returns false when segments is missing", () => {
|
||||
const data = {
|
||||
assetName: "clip.mp4",
|
||||
language: "en",
|
||||
fullText: "Hello",
|
||||
duration: 30,
|
||||
};
|
||||
|
||||
expect(isTranscriptData(data)).toBe(false);
|
||||
});
|
||||
|
||||
test("returns false when segments is not an array", () => {
|
||||
const data = {
|
||||
assetName: "clip.mp4",
|
||||
fullText: "Hello",
|
||||
segments: "not an array",
|
||||
duration: 30,
|
||||
};
|
||||
|
||||
expect(isTranscriptData(data)).toBe(false);
|
||||
});
|
||||
|
||||
test("returns false for null", () => {
|
||||
expect(isTranscriptData(null)).toBe(false);
|
||||
});
|
||||
|
||||
test("returns false for non-transcript tool result", () => {
|
||||
const data = { projectId: "p1", mediaCount: 2 };
|
||||
|
||||
expect(isTranscriptData(data)).toBe(false);
|
||||
});
|
||||
|
||||
test("returns false for a plain string", () => {
|
||||
expect(isTranscriptData("some text")).toBe(false);
|
||||
});
|
||||
|
||||
test("detects transcript from JSON.parse of tool_result content", () => {
|
||||
const toolResultContent = JSON.stringify({
|
||||
assetName: "clip.mp4",
|
||||
language: "en",
|
||||
fullText: "Hello world, this is a test.",
|
||||
segments: [
|
||||
{ text: "Hello world,", start: 0, end: 2.5 },
|
||||
{ text: "this is a test.", start: 2.5, end: 5.0 },
|
||||
],
|
||||
duration: 30,
|
||||
});
|
||||
|
||||
const parsed = JSON.parse(toolResultContent);
|
||||
expect(isTranscriptData(parsed)).toBe(true);
|
||||
});
|
||||
|
||||
test("does not detect error results as transcript", () => {
|
||||
const toolResultContent = JSON.stringify({
|
||||
error: "No active media asset",
|
||||
});
|
||||
|
||||
const parsed = JSON.parse(toolResultContent);
|
||||
expect(isTranscriptData(parsed)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("formatTimestamp — timestamp formatting", () => {
|
||||
test("formats zero seconds as 00:00", () => {
|
||||
expect(formatTimestamp(0)).toBe("00:00");
|
||||
});
|
||||
|
||||
test("formats seconds under a minute", () => {
|
||||
expect(formatTimestamp(45)).toBe("00:45");
|
||||
});
|
||||
|
||||
test("formats exact minutes", () => {
|
||||
expect(formatTimestamp(120)).toBe("02:00");
|
||||
});
|
||||
|
||||
test("formats minutes and seconds", () => {
|
||||
expect(formatTimestamp(125)).toBe("02:05");
|
||||
});
|
||||
|
||||
test("pads single-digit seconds", () => {
|
||||
expect(formatTimestamp(61)).toBe("01:01");
|
||||
});
|
||||
|
||||
test("formats large durations", () => {
|
||||
expect(formatTimestamp(3661)).toBe("61:01");
|
||||
});
|
||||
|
||||
test("formats segment start time matching spec [MM:SS]", () => {
|
||||
expect(formatTimestamp(2.5)).toBe("00:02");
|
||||
expect(formatTimestamp(150.7)).toBe("02:30");
|
||||
});
|
||||
});
|
||||
|
|
@ -2,6 +2,11 @@
|
|||
|
||||
import { cn } from "@/utils/ui";
|
||||
import type { ChatMessage } from "@/agent/types";
|
||||
import {
|
||||
isTranscriptData,
|
||||
formatTimestamp,
|
||||
} from "./transcript-utils";
|
||||
import type { TranscriptData } from "./transcript-utils";
|
||||
|
||||
interface MessageBubbleProps {
|
||||
message: ChatMessage;
|
||||
|
|
@ -13,9 +18,55 @@ const ROLE_LABELS: Record<ChatMessage["role"], string> = {
|
|||
tool_result: "Tool",
|
||||
};
|
||||
|
||||
function TranscriptCard({ data }: { data: TranscriptData }) {
|
||||
return (
|
||||
<div className="flex flex-col gap-2 rounded-md border bg-card p-3">
|
||||
<div className="flex items-center justify-between">
|
||||
<span className="text-sm font-medium">{data.assetName}</span>
|
||||
<span className="text-muted-foreground text-xs">
|
||||
{data.language} · {formatTimestamp(data.duration)}
|
||||
</span>
|
||||
</div>
|
||||
<p className="text-sm">{data.fullText}</p>
|
||||
<details className="text-xs">
|
||||
<summary className="text-muted-foreground cursor-pointer">
|
||||
{data.segments.length} segment{data.segments.length !== 1 && "s"}
|
||||
</summary>
|
||||
<div className="mt-1 flex flex-col gap-0.5">
|
||||
{data.segments.map((seg, i) => (
|
||||
<div key={i} className="text-muted-foreground">
|
||||
<span className="font-mono">[{formatTimestamp(seg.start)}]</span>{" "}
|
||||
{seg.text}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</details>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export function MessageBubble({ message }: MessageBubbleProps) {
|
||||
const isUser = message.role === "user";
|
||||
|
||||
// For tool_result messages, try to parse as transcript
|
||||
if (message.role === "tool_result") {
|
||||
try {
|
||||
const parsed = JSON.parse(message.content);
|
||||
if (isTranscriptData(parsed)) {
|
||||
return (
|
||||
<div className="flex flex-col gap-1 rounded-md px-3 py-2">
|
||||
<span className="text-muted-foreground text-xs font-medium">
|
||||
{ROLE_LABELS.tool_result}
|
||||
</span>
|
||||
<TranscriptCard data={parsed} />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
} catch {
|
||||
// Not JSON — fall through to plain text
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<div
|
||||
className={cn(
|
||||
|
|
|
|||
|
|
@ -0,0 +1,28 @@
|
|||
/**
|
||||
* Pure utilities for transcript data detection and formatting.
|
||||
* Extracted from MessageBubble for testability without React DOM.
|
||||
*/
|
||||
|
||||
export interface TranscriptData {
|
||||
assetName: string;
|
||||
language: string;
|
||||
fullText: string;
|
||||
segments: Array<{ text: string; start: number; end: number }>;
|
||||
duration: number;
|
||||
}
|
||||
|
||||
export function isTranscriptData(data: unknown): data is TranscriptData {
|
||||
return (
|
||||
typeof data === "object" &&
|
||||
data !== null &&
|
||||
"fullText" in data &&
|
||||
"segments" in data &&
|
||||
Array.isArray((data as TranscriptData).segments)
|
||||
);
|
||||
}
|
||||
|
||||
export function formatTimestamp(seconds: number): string {
|
||||
const mins = Math.floor(seconds / 60);
|
||||
const secs = Math.floor(seconds % 60);
|
||||
return `${String(mins).padStart(2, "0")}:${String(secs).padStart(2, "0")}`;
|
||||
}
|
||||
|
|
@ -0,0 +1,113 @@
|
|||
# Design: `transcribe_video` Phase 1
|
||||
|
||||
## Technical Approach
|
||||
|
||||
Add a single `transcribe_video` tool that wraps the existing `TranscriptionService` + `decodeAudioToFloat32()` pipeline. The tool resolves the target asset through an extended `EditorContextAdapter`, extracts audio, runs Whisper, and returns a structured `TranscriptionToolResult`. The mock API route triggers it; the orchestrator stores the result in chat state as a `tool_result` message. No new services, no new stores, no timeline mutations.
|
||||
|
||||
## Architecture Decisions
|
||||
|
||||
| Decision | Choice | Rejected | Rationale |
|
||||
|----------|--------|----------|-----------|
|
||||
| Asset file access | Extend `EditorContextAdapter` with `getAssetFile()` | Import `EditorCore` directly in tool | Maintains single-import boundary — only `context.ts` touches `core/` |
|
||||
| Tool registration | Side-effect import in `orchestrator.ts` (same as `mock.tool.ts`) | Dynamic/lazy registration | Follows existing pattern; tool count is small |
|
||||
| Asset selection | `assetId` arg → single video/audio → error if ambiguous | Always transcribe first asset | Explicit is better; ambiguous state gets a clear error message |
|
||||
| Result type | New `TranscriptionToolResult` interface in tool file | Re-export `TranscriptionResult` from lib | Tool result includes `assetName`, `segmentCount`, `duration` — richer than raw `TranscriptionResult` |
|
||||
| Mock API trigger | Keyword match on "transcri" in user message | Always return `transcribe_video` call | Keeps `echo_context` working for other messages; simple keyword gate |
|
||||
| Chat rendering | `MessageBubble` detects `tool_result` with transcript shape and formats it | New TranscriptMessage component | Minimal change — one formatting branch inside existing component |
|
||||
|
||||
## Data Flow
|
||||
|
||||
```
|
||||
User types "transcribe my video"
|
||||
→ ChatPanel.handleSend()
|
||||
→ sendMessage() (chat-store: user msg, loading=true)
|
||||
→ EditorContextAdapter.getContext() → AgentContext
|
||||
→ orchestrator.run(messages, context)
|
||||
→ POST /api/agent/chat → mock API returns { content, toolCalls: [transcribe_video] }
|
||||
→ resolveToolCalls()
|
||||
→ toolRegistry.get("transcribe_video").execute(args, context)
|
||||
→ find asset (context.mediaAssets + EditorContextAdapter.getAssetFile)
|
||||
→ guard: hasAudio? asset found?
|
||||
→ decodeAudioToFloat32({ audioBlob: file, sampleRate: 16000 })
|
||||
→ transcriptionService.transcribe({ audioData, language })
|
||||
→ return TranscriptionToolResult
|
||||
→ chatStore.addMessage({ role: "assistant", content, toolCalls })
|
||||
→ chatStore.addMessage({ role: "tool_result", content: JSON.stringify(result) })
|
||||
→ agentStore.setStatus("idle"), chatStore.setLoading(false)
|
||||
```
|
||||
|
||||
## File Changes
|
||||
|
||||
| File | Action | Description |
|
||||
|------|--------|-------------|
|
||||
| `apps/web/src/agent/tools/transcribe-video.tool.ts` | Create | Real transcription tool — asset resolution, audio decode, Whisper call, structured result |
|
||||
| `apps/web/src/agent/context.ts` | Modify | Add `getAssetFile(assetId: string): File \| null` method |
|
||||
| `apps/web/src/agent/orchestrator.ts` | Modify | Add `import "@/agent/tools/transcribe-video.tool"` (side-effect registration) |
|
||||
| `apps/web/src/app/api/agent/chat/route.ts` | Modify | Return `transcribe_video` tool call when message contains "transcri", keep `echo_context` fallback |
|
||||
| `apps/web/src/components/editor/panels/chat/message-bubble.tsx` | Modify | Detect transcript-shaped tool_result and render with timestamps |
|
||||
|
||||
## Interfaces / Contracts
|
||||
|
||||
```typescript
|
||||
// apps/web/src/agent/tools/transcribe-video.tool.ts
|
||||
|
||||
interface TranscriptionToolResult {
|
||||
assetName: string;
|
||||
language: string;
|
||||
fullText: string;
|
||||
segmentCount: number;
|
||||
segments: Array<{ text: string; start: number; end: number }>;
|
||||
duration: number;
|
||||
}
|
||||
|
||||
// Tool parameters
|
||||
interface TranscribeVideoArgs {
|
||||
assetId?: string; // optional — resolve automatically if omitted
|
||||
language?: string; // optional — defaults to "auto"
|
||||
modelId?: string; // optional — defaults to whisper-small
|
||||
}
|
||||
```
|
||||
|
||||
```typescript
|
||||
// apps/web/src/agent/context.ts — addition
|
||||
|
||||
export const EditorContextAdapter = {
|
||||
getContext() { /* existing */ },
|
||||
getAssetFile(assetId: string): File | null {
|
||||
const core = EditorCore.getInstance();
|
||||
const asset = core.media.getAssets().find(a => a.id === assetId);
|
||||
return asset?.file ?? null;
|
||||
},
|
||||
};
|
||||
```
|
||||
|
||||
## Error States (no UX redesign)
|
||||
|
||||
| State | Detection | Response |
|
||||
|-------|-----------|----------|
|
||||
| No media assets | `context.mediaAssets.length === 0` | Tool returns `{ error: "No media assets in project" }` — rendered as tool_result error text |
|
||||
| No video/audio asset | No asset with `type !== "video" && type !== "audio"` | Tool returns `{ error: "No video or audio assets found" }` |
|
||||
| Asset has no audio | `hasAudio === false` on resolved asset | Tool returns `{ error: "{assetName} has no audio track" }` |
|
||||
| Multiple candidates, no `assetId` | >1 video/audio asset and no explicit `assetId` arg | Tool returns `{ error: "Multiple video/audio assets found. Specify which one." }` with list of names |
|
||||
| Transcription service failure | `transcriptionService.transcribe()` rejects | Caught by orchestrator's existing try/catch → `ToolResult.error` message |
|
||||
| Asset file not found | `getAssetFile()` returns `null` | Tool returns `{ error: "Could not access file for asset {assetId}" }` |
|
||||
|
||||
All errors flow through the existing `tool_result` path — no new UI components needed.
|
||||
|
||||
## Testing Strategy
|
||||
|
||||
| Layer | What to Test | Approach |
|
||||
|-------|-------------|----------|
|
||||
| Unit | Asset selection logic (single/multiple/none) | Pure function test — pass `AgentContext` + `TranscribeVideoArgs`, verify asset resolution |
|
||||
| Unit | `getAssetFile()` on EditorContextAdapter | Mock `EditorCore.media.getAssets()` return, verify correct file returned |
|
||||
| Unit | Tool error branches (no audio, no asset, ambiguous) | Mock `decodeAudioToFloat32` and `transcriptionService`, verify error shapes |
|
||||
| Integration | End-to-end tool execution | Mock only `transcriptionService.transcribe()`, verify full result shape with real `decodeAudioToFloat32` on a small WAV |
|
||||
| Contract | Tool registration in registry | Verify `toolRegistry.get("transcribe_video")` returns the definition |
|
||||
|
||||
## Migration / Rollback
|
||||
|
||||
No migration required. Rollback = remove tool import from orchestrator, revert API route to `echo_context` only, revert `MessageBubble` changes.
|
||||
|
||||
## Open Questions
|
||||
|
||||
- [ ] Should the tool validate `MediaAsset.type === "video" || type === "audio"` or also accept images with embedded audio? (Recommendation: video/audio only for Phase 1)
|
||||
|
|
@ -0,0 +1,229 @@
|
|||
# Exploration: `transcribe_video` Tool — Phase 1
|
||||
|
||||
## Current State
|
||||
|
||||
### Completed Agent Infrastructure (infra-habilitadora-fase-1)
|
||||
|
||||
The full agent pipeline is wired end-to-end with a mock tool. Key components:
|
||||
|
||||
- **`agent/types.ts`** — `AgentContext`, `ChatMessage`, `ToolCall`, `ToolResult`, `ToolDefinition`, `ExecutionState`
|
||||
- **`agent/orchestrator.ts`** — client-side single-pass orchestrator: POST → resolve tool calls → append messages
|
||||
- **`agent/tools/registry.ts`** — `DefinitionRegistry<string, ToolDefinition>` (generic, reusable)
|
||||
- **`agent/tools/mock.tool.ts`** — `echo_context` tool (returns context summary)
|
||||
- **`agent/context.ts`** + **`agent/context-mapper.ts`** — thin adapter reading EditorCore → `AgentContext` POJO
|
||||
- **`agent/system-prompt.ts`** — builds prompt from `AgentContext`
|
||||
- **`stores/chat-store.ts`** — messages, loading, error
|
||||
- **`stores/agent-store.ts`** — execution status, active tool, context snapshot
|
||||
- **`app/api/agent/chat/route.ts`** — stateless mock API returning canned response with tool call
|
||||
- **`components/editor/panels/chat/`** — ChatPanel, MessageBubble, ChatInput
|
||||
|
||||
The orchestrator is **client-side only**. The API route is a stateless LLM proxy (currently mocked). Tool execution happens in-browser where `EditorCore` is accessible.
|
||||
|
||||
### Existing Transcription Infrastructure
|
||||
|
||||
A full Whisper-based transcription pipeline already exists:
|
||||
|
||||
| File | Role |
|
||||
|------|------|
|
||||
| `lib/transcription/types.ts` | `TranscriptionSegment { text, start, end }`, `TranscriptionResult { text, segments, language }`, `TranscriptionStatus`, `TranscriptionModelId` |
|
||||
| `lib/transcription/models.ts` | 4 Whisper models (tiny → large-v3-turbo), default is whisper-small |
|
||||
| `lib/transcription/audio.ts` | Sample rate (16000), chunk/stride constants |
|
||||
| `services/transcription/service.ts` | `TranscriptionService` — Web Worker manager, exposes `transcribe({ audioData, language, modelId, onProgress })` → `TranscriptionResult` |
|
||||
| `services/transcription/worker.ts` | HuggingFace `@huggingface/transformers` Whisper pipeline in Web Worker |
|
||||
| `lib/transcription/caption.ts` | `buildCaptionChunks({ segments })` → `CaptionChunk[]` |
|
||||
| `lib/transcription/diagnostics.ts` | Checks if timeline has audio before enabling transcription |
|
||||
| `lib/subtitles/insert.ts` | `insertCaptionChunksAsTextTrack({ editor, captions })` — adds text track via `BatchCommand` |
|
||||
|
||||
### How the Existing Captions Panel Works
|
||||
|
||||
`components/editor/panels/assets/views/captions.tsx` does the full flow:
|
||||
1. `extractTimelineAudio({ tracks, mediaAssets, totalDuration })` → audio Blob (from **entire timeline**)
|
||||
2. `decodeAudioToFloat32({ audioBlob, sampleRate: 16000 })` → `{ samples: Float32Array }`
|
||||
3. `transcriptionService.transcribe({ audioData: samples, language, onProgress })` → `TranscriptionResult`
|
||||
4. `buildCaptionChunks({ segments })` → `CaptionChunk[]`
|
||||
5. `insertCaptionChunksAsTextTrack({ editor, captions })` → creates text track
|
||||
|
||||
**Critical gap**: The existing flow extracts audio from the **entire timeline** (all tracks mixed). The agent tool needs to transcribe a **specific video/audio asset** — either the one on the main track or one explicitly referenced.
|
||||
|
||||
### MediaAsset Structure
|
||||
|
||||
`MediaAsset` (`lib/media/types.ts`) extends `MediaAssetData` with `file: File` and `url?: string`. `MediaAssetData` has: `id`, `name`, `type` (image|video|audio), `duration`, `hasAudio`, etc. The `file` property gives direct access to the raw `File` blob for audio extraction.
|
||||
|
||||
### Audio Extraction Paths
|
||||
|
||||
Two paths exist to get Float32Array audio data:
|
||||
|
||||
1. **From the timeline**: `extractTimelineAudio()` → Blob → `decodeAudioToFloat32()` (existing, used by Captions panel)
|
||||
2. **From a single asset**: `MediaAsset.file` (a `File`/`Blob`) → `decodeAudioToFloat32()` (existing function, just needs the Blob)
|
||||
|
||||
Path 2 is the one the agent tool needs. `decodeAudioToFloat32` already takes a `Blob` and returns `Float32Array`. No new audio extraction code is needed.
|
||||
|
||||
---
|
||||
|
||||
## Affected Areas
|
||||
|
||||
- `apps/web/src/agent/tools/transcribe-video.tool.ts` — **new** tool definition
|
||||
- `apps/web/src/agent/orchestrator.ts` — minor: must import the new tool module (like `mock.tool.ts`)
|
||||
- `apps/web/src/agent/system-prompt.ts` — likely needs update to describe available tools
|
||||
- `apps/web/src/app/api/agent/chat/route.ts` — update mock response to return a `transcribe_video` tool call instead of `echo_context`
|
||||
- `apps/web/src/agent/types.ts` — may need `TranscriptionToolResult` type
|
||||
- `apps/web/src/lib/transcription/types.ts` — may read/re-export for tool result
|
||||
- `apps/web/src/services/transcription/service.ts` — consumed by the tool (no changes needed)
|
||||
- `apps/web/src/lib/media/audio.ts` — `decodeAudioToFloat32()` consumed by the tool (no changes needed)
|
||||
- `apps/web/src/core/managers/media-manager.ts` — tool reads assets to find the target video (no changes needed)
|
||||
|
||||
---
|
||||
|
||||
## Approaches
|
||||
|
||||
### 1. Thin Agent Tool — Reuse Existing Services, Return TranscriptionResult
|
||||
|
||||
Create a `transcribe_video` tool that: (a) reads the active video from `AgentContext.mediaAssets`, (b) extracts audio via `decodeAudioToFloat32`, (c) calls `transcriptionService.transcribe()`, (d) returns a `TranscriptionResult` as the tool result. No caption insertion — that's Phase 2.
|
||||
|
||||
- **Pros**: Minimal new code. Reuses battle-tested `TranscriptionService` and `decodeAudioToFloat32`. Clean separation: tool returns data, agent/LLM decides what to do with it.
|
||||
- **Cons**: Long-running tool (model loading + transcription = 30s-3min depending on model). The orchestrator is single-pass and blocks until the tool completes. No progress reporting to the chat UI during execution.
|
||||
- **Effort**: Low
|
||||
|
||||
### 2. Thin Tool + Progress Reporting
|
||||
|
||||
Same as Approach 1, but adds progress callbacks to the tool execution: the tool emits `loading-model`, `transcribing`, and `complete` progress states that the agent store exposes to the chat UI.
|
||||
|
||||
- **Pros**: Better UX — user sees "Loading Whisper model... 45%" and "Transcribing..." instead of a frozen "Thinking..." state.
|
||||
- **Cons**: Requires modifying `ToolDefinition.execute` to support progress reporting (either a callback parameter or a different signature). Also needs the chat panel to render progress states. More surface area.
|
||||
- **Effort**: Medium
|
||||
|
||||
### 3. Thin Tool + Caption Insertion (Bigger Scope)
|
||||
|
||||
Same as Approach 1, but the tool also inserts caption chunks into the timeline via `insertCaptionChunksAsTextTrack`.
|
||||
|
||||
- **Pros**: End-to-end value — user asks "transcribe my video" and gets captions on the timeline.
|
||||
- **Cons**: Tool now has a side effect (timeline mutation). Violates "tool returns data, orchestrator decides" principle. Couples the tool to `EditorCore` and `CommandManager`. Should be a separate tool or an explicit follow-up action.
|
||||
- **Effort**: Medium-High
|
||||
|
||||
---
|
||||
|
||||
## Recommendation
|
||||
|
||||
**Approach 1 — Thin Agent Tool returning TranscriptionResult.** Reasons:
|
||||
|
||||
1. **Follows the existing architecture**: Tools are pure data-in/data-out. The LLM decides what to do with the result.
|
||||
2. **Zero new infra**: The tool wraps `transcriptionService.transcribe()` and `decodeAudioToFloat32()` — both already exist and are tested.
|
||||
3. **Clear Phase 2 boundary**: Caption insertion, progress reporting, streaming, and multi-turn tool loops are all natural Phase 2 additions that build ON TOP of this.
|
||||
4. **Fastest vertical slice**: One new file (`transcribe-video.tool.ts`) + minor wiring changes.
|
||||
|
||||
### Recommended Tool Design
|
||||
|
||||
```typescript
|
||||
// transcribe-video.tool.ts
|
||||
|
||||
const transcribeVideoTool: ToolDefinition = {
|
||||
name: "transcribe_video",
|
||||
description: "Transcribes the active video's audio using Whisper. Returns text and timestamped segments.",
|
||||
parameters: [
|
||||
{ key: "assetId", type: "string", required: false },
|
||||
{ key: "language", type: "string", required: false },
|
||||
{ key: "modelId", type: "string", required: false },
|
||||
],
|
||||
execute: async (args, context) => {
|
||||
// 1. Find the target asset (by assetId or first video/audio in context)
|
||||
// 2. Get the File blob from MediaManager (not in AgentContext — need to extend or read directly)
|
||||
// 3. decodeAudioToFloat32({ audioBlob: asset.file, sampleRate: 16000 })
|
||||
// 4. transcriptionService.transcribe({ audioData, language, modelId })
|
||||
// 5. Return TranscriptionResult
|
||||
},
|
||||
};
|
||||
```
|
||||
|
||||
### Critical Decision: Asset Resolution
|
||||
|
||||
The `AgentContext.mediaAssets` array has `id`, `name`, `type`, `duration` but **NOT the `File` blob** (it's intentionally a summary). The tool needs the actual `File` to extract audio. Two options:
|
||||
|
||||
- **Option A**: The tool directly reads `EditorCore.media.getAssets()` to get the full `MediaAsset` with `file`. This breaks the "no EditorCore access from agent/" rule. However, the rule only applies to the `agent/` directory, and tools already receive `AgentContext` which could be extended.
|
||||
- **Option B** (recommended): **Extend `AgentContext`** to include `activeAssetFile: File | null` or use the existing `EditorContextAdapter` pattern — the adapter can resolve the file and pass it as part of the context or as a separate parameter. Since tools already get `AgentContext`, we add an `assetFiles: Map<string, File>` or similar.
|
||||
|
||||
Actually, the simplest approach respecting the architecture: **the `transcribe_video` tool's `execute()` function imports `MediaManager` through the same adapter pattern**. The tool lives in `agent/tools/` which is allowed to import from `agent/context.ts`. We extend the adapter to provide asset file access:
|
||||
|
||||
```typescript
|
||||
// Extend agent/context.ts with a resolver
|
||||
export const EditorContextAdapter = {
|
||||
getContext() { ... },
|
||||
getAssetFile(assetId: string): File | null {
|
||||
const core = EditorCore.getInstance();
|
||||
const asset = core.media.getAssets().find(a => a.id === assetId);
|
||||
return asset?.file ?? null;
|
||||
},
|
||||
};
|
||||
```
|
||||
|
||||
This keeps the single-import rule (only `context.ts` touches EditorCore) while giving tools access to asset blobs.
|
||||
|
||||
### Asset Selection Logic
|
||||
|
||||
When the user says "transcribe my video" without specifying which one:
|
||||
|
||||
1. If `args.assetId` is provided → use that asset
|
||||
2. If only one video/audio asset in `context.mediaAssets` → use that one
|
||||
3. If multiple → return an error asking the user to specify
|
||||
|
||||
### What the Tool Returns
|
||||
|
||||
```typescript
|
||||
{
|
||||
assetName: string;
|
||||
language: string;
|
||||
fullText: string;
|
||||
segmentCount: number;
|
||||
segments: Array<{ text: string; start: number; end: number }>;
|
||||
duration: number;
|
||||
}
|
||||
```
|
||||
|
||||
The full text + segments are returned as the `ToolResult`. The LLM can then summarize, answer questions about the content, or in Phase 2, trigger caption insertion.
|
||||
|
||||
### Where the Result Is Stored
|
||||
|
||||
- **Tool result** → stored in `chatStore.messages` as a `tool_result` message (existing flow)
|
||||
- **No persistent storage** in Phase 1 — the transcription is ephemeral, tied to the chat session
|
||||
- Phase 2 can add persistent transcription storage (IndexedDB, project metadata) and caption insertion
|
||||
|
||||
---
|
||||
|
||||
## Phase 2 Boundary (explicitly OUT of scope for Phase 1)
|
||||
|
||||
| Feature | Why it's Phase 2 |
|
||||
|---------|-------------------|
|
||||
| Caption/subtitle insertion into timeline | Side-effect tool, needs separate design |
|
||||
| Progress reporting during transcription | Requires `ToolDefinition.execute` signature change |
|
||||
| Streaming transcription results | Requires orchestrator multi-turn support |
|
||||
| Persistent transcription storage | Needs data model design |
|
||||
| Multi-language selection UI in chat | UI concern, not tool concern |
|
||||
| Transcription of timeline (all tracks) vs single asset | Different extraction path, separate tool |
|
||||
| Real LLM integration (replacing mock API) | Separate change, infrastructure concern |
|
||||
|
||||
---
|
||||
|
||||
## Risks
|
||||
|
||||
1. **Long-running tool execution**: Whisper model loading can take 10-60s, transcription 30s-5min depending on video length and model. The orchestrator is single-pass and the chat UI shows "Thinking..." the entire time. Users may think it's frozen. **Mitigation**: Document this known limitation. Phase 2 adds progress reporting.
|
||||
|
||||
2. **Asset file access architecture tension**: The tool needs `MediaAsset.file` (a `File` blob) which is NOT in `AgentContext`. Adding a `getAssetFile()` method to the adapter is clean but expands the adapter's contract. **Mitigation**: The adapter already exists for exactly this purpose — it's the sanctioned bridge.
|
||||
|
||||
3. **Web Worker lifecycle**: `transcriptionService` manages a Web Worker with model lifecycle (init → ready). If the tool is called multiple times, the service handles reuse. But if the user switches models, there's a re-init delay. **Mitigation**: No change needed — the existing `TranscriptionService` already handles this correctly.
|
||||
|
||||
4. **No real LLM yet**: The API route still returns a canned mock response. The `transcribe_video` tool call must be triggered by the mock, not by a real LLM reasoning about the user's intent. **Mitigation**: Update the mock to return a `transcribe_video` tool call for any message containing "transcribe". This is clearly temporary.
|
||||
|
||||
5. **Memory pressure**: Loading a large video's audio as `Float32Array` can consume significant memory. A 10-minute video at 16kHz mono = ~19MB of float data. **Mitigation**: Acceptable for Phase 1. Phase 2 can add chunked processing.
|
||||
|
||||
6. **`hasAudio` guard**: Not all video assets have audio. The tool must check `MediaAssetData.hasAudio` before attempting transcription. **Mitigation**: Check in `execute()` and return a clear error message.
|
||||
|
||||
---
|
||||
|
||||
## Ready for Proposal
|
||||
|
||||
**Yes.** The exploration identifies:
|
||||
- One new file to create (`transcribe-video.tool.ts`)
|
||||
- Minor wiring changes (orchestrator import, mock API update, adapter extension)
|
||||
- Clear reuse of existing transcription service and audio decoding
|
||||
- A clean Phase 2 boundary
|
||||
|
||||
**Next**: Run `sdd-propose` with this exploration as input.
|
||||
|
|
@ -0,0 +1,63 @@
|
|||
# Proposal: `transcribe_video` Phase 1
|
||||
|
||||
## Intent
|
||||
|
||||
Ship the first real agent feature: ask the editor chat to transcribe the selected/active media asset and return usable transcript text with timestamps. This proves the current agent shell can drive real product value without adding a new workflow surface.
|
||||
|
||||
## Scope
|
||||
|
||||
### In Scope
|
||||
- Add one real `transcribe_video` tool to the current orchestrator/chat flow.
|
||||
- Reuse existing audio decode + Whisper transcription services for a single media asset.
|
||||
- Store transcript output in the current chat session and render timestamped segments in chat.
|
||||
- Extend the context bridge just enough to resolve the selected/active asset file safely.
|
||||
|
||||
### Out of Scope
|
||||
- Subtitle insertion, filler-word cleanup, highlights, summarization, and multi-tool chains.
|
||||
- Timeline-wide transcription, streaming/progress UX, persistent project-level transcript storage, and real LLM routing.
|
||||
|
||||
## Capabilities
|
||||
|
||||
### New Capabilities
|
||||
- `transcribe-video`: End-to-end chat-triggered transcription of one media asset, returning full text plus timestamped segments.
|
||||
|
||||
### Modified Capabilities
|
||||
- `agent-context-bridge`: Expand the sanctioned adapter so tools can safely resolve the selected/active asset file without direct `EditorCore` access.
|
||||
- `agent-session-shell`: Replace the mock-only tool path with real `transcribe_video` execution and transcript result persistence in chat state.
|
||||
- `editor-chat-panel`: Render transcript-oriented assistant/tool output with readable timestamps inside the existing chat experience.
|
||||
|
||||
## Approach
|
||||
|
||||
Keep the slice thin: mock API triggers `transcribe_video`, the tool resolves the target asset through `EditorContextAdapter`, decodes `MediaAsset.file`, calls `transcriptionService.transcribe()`, and returns `{ assetName, language, fullText, segments, duration }`. The orchestrator stores that result in chat state; the chat UI renders the transcript inline.
|
||||
|
||||
## Affected Areas
|
||||
|
||||
| Area | Impact | Description |
|
||||
|------|--------|-------------|
|
||||
| `apps/web/src/agent/tools/transcribe-video.tool.ts` | New | Real transcription tool |
|
||||
| `apps/web/src/agent/context.ts` | Modified | Safe asset-file resolver |
|
||||
| `apps/web/src/agent/orchestrator.ts` | Modified | Execute/store transcript result |
|
||||
| `apps/web/src/app/api/agent/chat/route.ts` | Modified | Mock trigger for transcription |
|
||||
| `apps/web/src/components/editor/panels/chat/` | Modified | Timestamped transcript rendering |
|
||||
|
||||
## Risks
|
||||
|
||||
| Risk | Likelihood | Mitigation |
|
||||
|------|------------|------------|
|
||||
| Long-running Whisper execution feels frozen | Med | Keep scope explicit; defer progress UX |
|
||||
| Ambiguous or silent asset selection | Med | Require selected/active asset fallback + clear errors |
|
||||
| Large files increase memory use | Low | Limit Phase 1 to current in-browser path |
|
||||
|
||||
## Rollback Plan
|
||||
|
||||
Remove `transcribe_video` registration, restore mock-only route behavior, and keep the chat panel on generic message rendering.
|
||||
|
||||
## Dependencies
|
||||
|
||||
- Existing `TranscriptionService`, `decodeAudioToFloat32()`, and current asset/editor context.
|
||||
|
||||
## Success Criteria
|
||||
|
||||
- [ ] A chat request can trigger `transcribe_video` for the selected/active media asset.
|
||||
- [ ] The result appears in chat with transcript text and timestamped segments.
|
||||
- [ ] Asset access stays behind `EditorContextAdapter`, with no direct `core/` imports from agent modules.
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
# Delta for Agent Context Bridge
|
||||
|
||||
## ADDED Requirements
|
||||
|
||||
### Requirement: Active Asset File Resolver
|
||||
|
||||
The system SHALL provide `resolveAssetFile(assetId?: string)` on `EditorContextAdapter` that returns the `File` reference for the requested media asset. When `assetId` is provided, it MUST resolve that specific asset. When omitted, it MUST resolve the first video/audio asset from `mediaAssets`. The method SHALL return `null` when no matching asset exists. This method is the ONLY sanctioned path for tools to obtain a `File` reference — tools MUST NOT access `EditorCore.media` directly.
|
||||
|
||||
#### Scenario: Resolve by explicit asset ID
|
||||
|
||||
- GIVEN `EditorCore.media.getAssets()` contains an asset with `id: "v1"` and `name: "clip.mp4"`
|
||||
- WHEN `resolveAssetFile("v1")` is called
|
||||
- THEN it returns the `File` object for that asset
|
||||
|
||||
#### Scenario: Resolve active asset without ID
|
||||
|
||||
- GIVEN `EditorCore.media.getAssets()` contains two assets, the first being a video
|
||||
- WHEN `resolveAssetFile()` is called without arguments
|
||||
- THEN it returns the `File` for the first video or audio asset
|
||||
|
||||
#### Scenario: No matching asset
|
||||
|
||||
- GIVEN `EditorCore.media.getAssets()` is empty
|
||||
- WHEN `resolveAssetFile()` is called
|
||||
- THEN it returns `null`
|
||||
|
||||
#### Scenario: Asset ID not found
|
||||
|
||||
- GIVEN `EditorCore.media.getAssets()` contains assets with IDs `["v1", "v2"]`
|
||||
- WHEN `resolveAssetFile("v99")` is called
|
||||
- THEN it returns `null`
|
||||
|
|
@ -0,0 +1,45 @@
|
|||
# Delta for Agent Session Shell
|
||||
|
||||
## MODIFIED Requirements
|
||||
|
||||
### Requirement: Tool Registry Shell
|
||||
|
||||
The system SHALL provide a `ToolRegistry` extending `DefinitionRegistry<string, ToolDefinition>` where `ToolDefinition` contains `{ name, description, parameters, execute: (args, context) => Promise<unknown> }`. The registry MUST register `echo_context` and `transcribe_video`. The registry MUST support `register`, `get`, `has`, and `getAll`.
|
||||
|
||||
(Previously: Registry registered exactly one tool — `echo_context`.)
|
||||
|
||||
#### Scenario: Mock echo tool executes
|
||||
|
||||
- GIVEN `echo_context` is registered with `execute: (args, ctx) => ({ projectId: ctx.projectId, ... })`
|
||||
- WHEN `registry.get("echo_context").execute({}, context)` is called
|
||||
- THEN it returns a context summary with projectId and mediaCount
|
||||
|
||||
#### Scenario: Transcribe tool is registered
|
||||
|
||||
- GIVEN the `transcribe_video` tool module is imported
|
||||
- WHEN `registry.has("transcribe_video")` is called
|
||||
- THEN it returns `true`
|
||||
|
||||
### Requirement: API Proxy Route
|
||||
|
||||
The system SHALL expose `POST /api/agent/chat` that accepts `{ messages: ChatMessage[], context?: AgentContext }` and returns `{ content: string, toolCalls?: ToolCall[] }`. In this phase, the route MUST return a canned mock response — no real LLM call. When the last user message contains transcription-related intent (e.g., "transcribe"), the route MUST return a `transcribe_video` tool call. Otherwise, it MUST return the existing `echo_context` tool call. The route MUST validate the request body schema and return 400 on invalid input.
|
||||
|
||||
(Previously: Route always returned `echo_context` tool call regardless of user intent.)
|
||||
|
||||
#### Scenario: Valid request returns mock response
|
||||
|
||||
- GIVEN a POST request with valid `{ messages: [{ role: "user", content: "Hi" }] }`
|
||||
- WHEN the handler processes the request
|
||||
- THEN it returns 200 with `{ content: "...", toolCalls: [{ name: "echo_context", ... }] }`
|
||||
|
||||
#### Scenario: Transcription intent triggers transcribe tool
|
||||
|
||||
- GIVEN a POST request with `{ messages: [{ role: "user", content: "Transcribe this video" }] }`
|
||||
- WHEN the handler processes the request
|
||||
- THEN it returns 200 with `{ content: "...", toolCalls: [{ name: "transcribe_video", args: {} }] }`
|
||||
|
||||
#### Scenario: Invalid request returns 400
|
||||
|
||||
- GIVEN a POST request with `{}` (missing `messages`)
|
||||
- WHEN the handler processes the request
|
||||
- THEN it returns 400 with an error description
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
# Delta for Editor Chat Panel
|
||||
|
||||
## MODIFIED Requirements
|
||||
|
||||
### Requirement: Message Display
|
||||
|
||||
The system SHALL render a scrollable `MessageList` displaying all messages in `chatStore.messages` in chronological order. Each message MUST show its role (`user`, `assistant`, or `tool_result`) and content. When a `tool_result` message contains a JSON-parseable transcript result (has `fullText` and `segments` keys), the system MUST render it as a transcript card showing the asset name, language, duration, full text, and timestamped segments formatted as `[MM:SS] text`. For all other messages, content MUST be rendered as plain text. The list MUST auto-scroll to the latest message when a new message arrives.
|
||||
|
||||
(Previously: All messages rendered as plain text with role labels only.)
|
||||
|
||||
#### Scenario: Messages render in order
|
||||
|
||||
- GIVEN `chatStore.messages` contains three messages: user, assistant, user
|
||||
- WHEN the ChatPanel renders
|
||||
- THEN all three messages appear in chronological order with correct role labels
|
||||
- AND the list is scrolled to the bottom
|
||||
|
||||
#### Scenario: Empty state
|
||||
|
||||
- GIVEN `chatStore.messages` is empty
|
||||
- WHEN the ChatPanel renders
|
||||
- THEN a placeholder prompt is shown (e.g., "Ask something about your project")
|
||||
|
||||
#### Scenario: Transcript tool result renders as card
|
||||
|
||||
- GIVEN `chatStore.messages` contains a `tool_result` with content `JSON.stringify({ assetName: "clip.mp4", fullText: "Hello world", segments: [{ text: "Hello world", start: 0, end: 2.5 }], language: "en", duration: 30 })`
|
||||
- WHEN the ChatPanel renders
|
||||
- THEN the message renders as a transcript card showing "clip.mp4", "en", duration, full text, and `[00:00] Hello world`
|
||||
|
||||
#### Scenario: Non-transcript tool result renders as plain text
|
||||
|
||||
- GIVEN `chatStore.messages` contains a `tool_result` with content `JSON.stringify({ projectId: "p1", mediaCount: 2 })`
|
||||
- WHEN the ChatPanel renders
|
||||
- THEN the message renders as plain text (no transcript card)
|
||||
|
|
@ -0,0 +1,56 @@
|
|||
# Transcribe Video Specification
|
||||
|
||||
## Purpose
|
||||
|
||||
End-to-end chat-triggered transcription of one media asset. Decodes the asset's audio, runs Whisper, and returns full text plus timestamped segments.
|
||||
|
||||
## Requirements
|
||||
|
||||
### Requirement: Transcription Tool Execution
|
||||
|
||||
The system SHALL register a `transcribe_video` tool in the tool registry with parameters `{ assetId?: string }`. When executed, the tool MUST resolve the target asset via `EditorContextAdapter.resolveAssetFile(assetId)`, decode its audio to Float32Array using `decodeAudioToFloat32`, and call `transcriptionService.transcribe()` with the decoded audio. The tool MUST return `{ assetName, language, fullText, segments: Array<{ text, start, end }>, duration }`.
|
||||
|
||||
#### Scenario: Successful transcription of a video asset
|
||||
|
||||
- GIVEN the active/selected asset is a video with an audible audio track
|
||||
- WHEN `transcribe_video` executes with no explicit `assetId`
|
||||
- THEN it decodes the asset audio, calls `transcriptionService.transcribe()`, and returns `{ assetName, language, fullText, segments, duration }`
|
||||
- AND `fullText` is a non-empty string and `segments` contains at least one entry
|
||||
|
||||
#### Scenario: Transcription with explicit asset ID
|
||||
|
||||
- GIVEN the context has multiple media assets and `assetId` matches one
|
||||
- WHEN `transcribe_video` executes with `assetId: "v2"`
|
||||
- THEN it resolves and transcribes that specific asset regardless of active selection
|
||||
|
||||
### Requirement: Asset Audio Validation
|
||||
|
||||
The tool MUST return an error result when the target asset has no audio track. The tool MUST return an error result when no asset can be resolved (no active/selected asset and no explicit `assetId`).
|
||||
|
||||
#### Scenario: Asset with no audio track
|
||||
|
||||
- GIVEN the resolved asset is a video file with no audio track
|
||||
- WHEN `transcribe_video` executes
|
||||
- THEN the tool returns a `ToolResult` with `error: "Asset has no audio track"`
|
||||
|
||||
#### Scenario: Image asset selected
|
||||
|
||||
- GIVEN the resolved asset has `type: "image"`
|
||||
- WHEN `transcribe_video` executes
|
||||
- THEN the tool returns a `ToolResult` with `error: "Asset has no audio track"`
|
||||
|
||||
#### Scenario: No asset resolved
|
||||
|
||||
- GIVEN no asset is active/selected and no `assetId` is provided
|
||||
- WHEN `transcribe_video` executes
|
||||
- THEN the tool returns a `ToolResult` with `error: "No active media asset"`
|
||||
|
||||
### Requirement: Transcription Error Handling
|
||||
|
||||
If `transcriptionService.transcribe()` rejects, the tool MUST catch the error and return a `ToolResult` with the error message. The tool MUST NOT throw unhandled exceptions.
|
||||
|
||||
#### Scenario: Transcription service fails
|
||||
|
||||
- GIVEN the audio decode succeeds but `transcriptionService.transcribe()` throws "Worker not initialized"
|
||||
- WHEN `transcribe_video` executes
|
||||
- THEN the tool returns a `ToolResult` with `error` containing "Worker not initialized"
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
# Tasks: `transcribe_video` Phase 1
|
||||
|
||||
## Phase 1: Foundation
|
||||
|
||||
- [x] 1.1 Add `resolveAssetFile(assetId?: string): File | null` to `EditorContextAdapter` in `apps/web/src/agent/context.ts` — uses `EditorCore.media.getAssets()` to find by ID or first video/audio, returns `null` on no match
|
||||
- [x] 1.2 Add `TranscriptionToolResult` and `TranscribeVideoArgs` interfaces at top of new file `apps/web/src/agent/tools/transcribe-video.tool.ts`
|
||||
|
||||
## Phase 2: Core Tool
|
||||
|
||||
- [x] 2.1 Implement `transcribe_video` tool in `apps/web/src/agent/tools/transcribe-video.tool.ts` — validate `context.mediaAssets`, call `resolveAssetFile`, guard on `hasAudio`/type/ambiguity, then `decodeAudioToFloat32` → `transcriptionService.transcribe` → return `TranscriptionToolResult`
|
||||
- [x] 2.2 Register tool via side-effect `toolRegistry.register("transcribe_video", …)` at module level (same pattern as `mock.tool.ts`)
|
||||
|
||||
## Phase 3: Wiring
|
||||
|
||||
- [x] 3.1 Add `import "@/agent/tools/transcribe-video.tool"` to `apps/web/src/agent/orchestrator.ts` (side-effect registration)
|
||||
- [x] 3.2 Update `apps/web/src/app/api/agent/chat/route.ts` — if last user message contains "transcri" (case-insensitive), return `transcribe_video` tool call; else keep `echo_context` fallback
|
||||
- [x] 3.3 Update `apps/web/src/components/editor/panels/chat/message-bubble.tsx` — for `tool_result` role, try `JSON.parse(content)` and if it has `fullText` + `segments`, render transcript card with asset name, language, duration, and `[MM:SS] text` segments; else render plain text
|
||||
|
||||
## Phase 4: Testing
|
||||
|
||||
- [x] 4.1 Unit test `resolveAssetFile()` in `apps/web/src/agent/__tests__/context.test.ts` — mock `EditorCore.media.getAssets()` for: match by ID, fallback to first video, no assets → null
|
||||
- [x] 4.2 Unit test asset selection + error branches in `apps/web/src/agent/tools/__tests__/transcribe-video.test.ts` — mock `decodeAudioToFloat32` + `transcriptionService`; cover: success path, no audio, no asset, multiple candidates, service failure
|
||||
- [x] 4.3 Contract test `toolRegistry.has("transcribe_video")` in same test file — verifies registration side-effect
|
||||
- [x] 4.4 Unit test API route in `apps/web/src/app/api/agent/chat/__tests__/route.test.ts` — "transcribe this" → returns `transcribe_video` call; other message → returns `echo_context` call
|
||||
|
||||
## Phase 5: Post-Verify Fixes
|
||||
|
||||
- [x] 5.1 Fix TypeScript errors: `jest.fn` type references in test files replaced with explicit `MockFn`/`MockFnFactory` types; `TranscribeVideoArgs.language` typed as `TranscriptionLanguage` instead of `string`
|
||||
- [x] 5.2 Align tool error messages with spec: `"No active media asset"` for no-asset, `"Asset has no audio track"` for image-only and no-audio-track cases
|
||||
- [x] 5.3 Wrap decode+transcribe in try/catch — service failures now return `{ error: message }` instead of throwing
|
||||
- [x] 5.4 Extract `isTranscriptData` + `formatTimestamp` to `transcript-utils.ts`; add 17 behavioral tests proving transcript detection and timestamp formatting
|
||||
|
|
@ -0,0 +1,123 @@
|
|||
# Verification Report
|
||||
|
||||
**Change**: transcribe-video-fase-1
|
||||
**Mode**: Standard
|
||||
**TDD Resolution**: Strict TDD config exists in `openspec/config.yaml`, but this verification used Standard mode because the orchestrator explicitly disabled strict TDD for this workflow.
|
||||
|
||||
---
|
||||
|
||||
## Completeness
|
||||
|
||||
| Metric | Value |
|
||||
|--------|-------|
|
||||
| Tasks total | 14 |
|
||||
| Tasks complete | 14 |
|
||||
| Tasks incomplete | 0 |
|
||||
|
||||
All listed tasks in `openspec/changes/transcribe-video-fase-1/tasks.md` are marked complete, including the post-verify fix batch.
|
||||
|
||||
---
|
||||
|
||||
## Build & Tests Execution
|
||||
|
||||
**Type check**: ✅ Passed
|
||||
Command: `bunx tsc --noEmit` (workdir: `apps/web`)
|
||||
|
||||
**Targeted verification tests**: ✅ 63 passed / ❌ 0 failed / ⚠️ 0 skipped
|
||||
Command:
|
||||
|
||||
`bun test apps/web/src/agent/__tests__/context.test.ts apps/web/src/agent/tools/__tests__/mock-tool.test.ts apps/web/src/agent/tools/__tests__/transcribe-video.test.ts apps/web/src/app/api/agent/chat/__tests__/route.test.ts apps/web/src/components/editor/panels/chat/__tests__/transcript-utils.test.ts apps/web/src/stores/__tests__/chat-panel-behavior.test.ts`
|
||||
|
||||
Result: `63 pass, 0 fail`
|
||||
|
||||
**Environment warning suite**: ⚠️ Unrelated failure observed
|
||||
Command: `bun test apps/web/src/agent/__tests__/orchestrator.test.ts`
|
||||
|
||||
Observed failure:
|
||||
|
||||
`TypeError: wasm.__wbindgen_start is not a function` from `opencut-wasm/opencut_wasm.js`
|
||||
|
||||
This prevents orchestrator runtime verification in the current Bun/WASM environment, but it appears pre-existing and unrelated to this slice's direct implementation.
|
||||
|
||||
**Coverage**: ➖ Not available
|
||||
|
||||
---
|
||||
|
||||
## Spec Compliance Matrix
|
||||
|
||||
| Requirement | Scenario | Test | Result |
|
||||
|-------------|----------|------|--------|
|
||||
| Active Asset File Resolver | Resolve by explicit asset ID | `apps/web/src/agent/__tests__/context.test.ts > returns File when assetId matches an asset` | ✅ COMPLIANT |
|
||||
| Active Asset File Resolver | Resolve active asset without ID | `apps/web/src/agent/__tests__/context.test.ts > returns File for first video asset when no assetId provided` | ✅ COMPLIANT |
|
||||
| Active Asset File Resolver | No matching asset | `apps/web/src/agent/__tests__/context.test.ts > returns null when assets array is empty` | ✅ COMPLIANT |
|
||||
| Active Asset File Resolver | Asset ID not found | `apps/web/src/agent/__tests__/context.test.ts > returns null when assetId does not match any asset` | ✅ COMPLIANT |
|
||||
| Transcription Tool Execution | Successful transcription of a video asset | `apps/web/src/agent/tools/__tests__/transcribe-video.test.ts > transcribes a single video asset successfully` | ✅ COMPLIANT |
|
||||
| Transcription Tool Execution | Transcription with explicit asset ID | `apps/web/src/agent/tools/__tests__/transcribe-video.test.ts > transcribes with explicit assetId` | ✅ COMPLIANT |
|
||||
| Asset Audio Validation | Asset with no audio track | `apps/web/src/agent/tools/__tests__/transcribe-video.test.ts > returns error when asset has no audio track` | ✅ COMPLIANT |
|
||||
| Asset Audio Validation | Image asset selected | `apps/web/src/agent/tools/__tests__/transcribe-video.test.ts > returns error when only image assets found (no audio track)` | ✅ COMPLIANT |
|
||||
| Asset Audio Validation | No asset resolved | `apps/web/src/agent/tools/__tests__/transcribe-video.test.ts > returns error when no media assets in project` | ✅ COMPLIANT |
|
||||
| Transcription Error Handling | Transcription service fails | `apps/web/src/agent/tools/__tests__/transcribe-video.test.ts > returns error when transcriptionService.transcribe rejects` | ✅ COMPLIANT |
|
||||
| Tool Registry Shell | Mock echo tool executes | `apps/web/src/agent/tools/__tests__/mock-tool.test.ts > returns context summary with media assets` | ✅ COMPLIANT |
|
||||
| Tool Registry Shell | Transcribe tool is registered | `apps/web/src/agent/tools/__tests__/transcribe-video.test.ts > is registered in the tool registry` | ✅ COMPLIANT |
|
||||
| API Proxy Route | Valid request returns mock response | `apps/web/src/app/api/agent/chat/__tests__/route.test.ts > returns mock response for valid input` | ✅ COMPLIANT |
|
||||
| API Proxy Route | Transcription intent triggers transcribe tool | `apps/web/src/app/api/agent/chat/__tests__/route.test.ts > returns transcribe_video tool call when message contains 'transcribe'` | ✅ COMPLIANT |
|
||||
| API Proxy Route | Invalid request returns 400 | `apps/web/src/app/api/agent/chat/__tests__/route.test.ts > returns 400 for invalid input — missing messages` | ✅ COMPLIANT |
|
||||
| Message Display | Messages render in order | `apps/web/src/stores/__tests__/chat-panel-behavior.test.ts > messages render in order — chatStore preserves chronological order` | ⚠️ PARTIAL |
|
||||
| Message Display | Empty state | `apps/web/src/stores/__tests__/chat-panel-behavior.test.ts > empty state — messages array is empty by default` | ⚠️ PARTIAL |
|
||||
| Message Display | Transcript tool result renders as card | `apps/web/src/components/editor/panels/chat/__tests__/transcript-utils.test.ts > detects transcript from JSON.parse of tool_result content` | ⚠️ PARTIAL |
|
||||
| Message Display | Non-transcript tool result renders as plain text | `apps/web/src/components/editor/panels/chat/__tests__/transcript-utils.test.ts > does not detect error results as transcript` | ⚠️ PARTIAL |
|
||||
|
||||
**Compliance summary**: 15/19 scenarios compliant, 4/19 partial, 0 failing, 0 fully untested
|
||||
|
||||
---
|
||||
|
||||
## Correctness (Static — Structural Evidence)
|
||||
|
||||
| Requirement | Status | Notes |
|
||||
|------------|--------|-------|
|
||||
| Active Asset File Resolver | ✅ Implemented | `EditorContextAdapter.resolveAssetFile()` exists and remains the sanctioned file access path from agent code. |
|
||||
| Transcription Tool Execution | ⚠️ Partial | Tool is implemented, registered, typed, and uses `resolveAssetFile`, but default no-`assetId` selection errors on multiple candidates instead of following the delta spec's “first video/audio asset” behavior. |
|
||||
| Asset Audio Validation | ✅ Implemented | No-media, image-only, missing-file, and no-audio cases return structured errors. |
|
||||
| Transcription Error Handling | ✅ Implemented | Decode/transcribe are wrapped in `try/catch`; service rejection returns `{ error }`. |
|
||||
| Tool Registry Shell | ✅ Implemented | Registry contains `echo_context` and `transcribe_video`; side-effect registration is wired in orchestrator. |
|
||||
| API Proxy Route | ✅ Implemented | Zod validation, mock response, transcription intent routing, and fallback remain present. |
|
||||
| Message Display | ⚠️ Partial | `MessageBubble` contains transcript formatting branch and `ChatPanel` still handles order/empty state/auto-scroll, but there is no direct runtime UI test proving the rendered transcript card behavior. |
|
||||
|
||||
---
|
||||
|
||||
## Coherence (Design)
|
||||
|
||||
| Decision | Followed? | Notes |
|
||||
|----------|-----------|-------|
|
||||
| Asset file access through `EditorContextAdapter` | ✅ Yes | Tool imports `EditorContextAdapter`; no direct `core/` access from the tool. |
|
||||
| Tool registration via side-effect import | ✅ Yes | `orchestrator.ts` imports `@/agent/tools/transcribe-video.tool`. |
|
||||
| Asset selection errors when ambiguous | ✅ Yes | Implementation matches design, but this conflicts with the current delta spec text. |
|
||||
| Rich `TranscriptionToolResult` shape | ✅ Yes | Tool returns `assetName`, `language`, `fullText`, `segmentCount`, `segments`, `duration`. |
|
||||
| Mock API keyword gate on `transcri` | ✅ Yes | Route uses case-insensitive `includes("transcri")`. |
|
||||
| Chat rendering inside existing `MessageBubble` | ✅ Yes | Transcript detection and timestamp formatting stay inside the existing component flow. |
|
||||
|
||||
---
|
||||
|
||||
## Issues Found
|
||||
|
||||
### CRITICAL
|
||||
|
||||
1. **Spec/implementation mismatch on default asset selection** — the written delta spec says `resolveAssetFile()` without `assetId` MUST resolve the first video/audio asset, but `transcribe_video` currently returns an ambiguity error when multiple candidates exist. Design/tasks/tests were updated around ambiguity handling, but the spec was not. This blocks archive until spec and implementation are reconciled.
|
||||
|
||||
### WARNING
|
||||
|
||||
1. **Editor chat panel scenarios are only partially proven behaviorally** — current evidence is store-level and utility-level, not direct rendered component verification. Transcript card rendering, plain-text fallback rendering, placeholder copy, and auto-scroll are not proven by passing UI-focused tests.
|
||||
2. **Orchestrator verification is blocked by a pre-existing WASM environment issue** — `apps/web/src/agent/__tests__/orchestrator.test.ts` crashes during WASM module startup with `wasm.__wbindgen_start is not a function`. Per instruction, this is treated as a warning because it does not directly invalidate this slice.
|
||||
|
||||
### SUGGESTION
|
||||
|
||||
1. Add a focused `MessageBubble`/`ChatPanel` render test suite so the four modified UI scenarios move from PARTIAL to COMPLIANT.
|
||||
2. Reconcile the transcribe-tool selection rule in `openspec` artifacts so design, tasks, tests, and spec all describe the same behavior.
|
||||
|
||||
---
|
||||
|
||||
## Verdict
|
||||
|
||||
**FAIL**
|
||||
|
||||
The implementation is mostly in place and the targeted non-WASM verification suite passes, but the change is not archive-ready because the shipped behavior contradicts the current delta spec on default asset selection.
|
||||
Loading…
Reference in New Issue