diff --git a/apps/web/src/agent/__tests__/context.test.ts b/apps/web/src/agent/__tests__/context.test.ts new file mode 100644 index 00000000..93f1c40c --- /dev/null +++ b/apps/web/src/agent/__tests__/context.test.ts @@ -0,0 +1,160 @@ +import { beforeEach, describe, expect, mock, test } from "bun:test"; + +// --- Module mocks (must be set up before importing the module under test) --- + +// bun:test exposes jest.fn() for mock function creation at runtime +// We type the factory explicitly to avoid referencing the `jest` global type +// eslint-disable-next-line @typescript-eslint/no-explicit-any +type MockFn = { + (...args: any[]): any; + mockReturnValue(value: any): void; + mockClear(): void; + mockImplementation(fn: (...args: any[]) => any): void; +}; +// eslint-disable-next-line @typescript-eslint/no-explicit-any +type MockFnFactory = any>() => T & MockFn; + +const bunTestModule = await import("bun:test"); +const jestObj = (bunTestModule as Record).jest as Record; +const mockFn = jestObj.fn as unknown as MockFnFactory; + +const mockGetAssets = mockFn<() => any[]>(); + +mock.module("@/core", () => ({ + EditorCore: { + getInstance: () => ({ + media: { getAssets: mockGetAssets }, + }), + }, +})); + +mock.module("@/lib/wasm", () => ({ + TICKS_PER_SECOND: 100, +})); + +const { EditorContextAdapter } = await import("@/agent/context"); + +// --- Helpers --- + +function fakeAsset( + overrides: Partial<{ + id: string; + name: string; + type: string; + duration: number; + file: File; + hasAudio: boolean; + }> = {}, +) { + return { + id: "v1", + name: "clip.mp4", + type: "video", + duration: 30, + file: new File([], "clip.mp4", { type: "video/mp4" }), + hasAudio: true, + ...overrides, + }; +} + +// --- Tests --- + +describe("EditorContextAdapter.resolveAssetFile", () => { + beforeEach(() => { + mockGetAssets.mockClear(); + }); + + test("returns File when assetId matches an asset", () => { + const videoFile = new File([], "intro.mp4", { type: "video/mp4" }); + mockGetAssets.mockReturnValue([ + fakeAsset({ id: "v1", name: "intro.mp4", file: videoFile }), + ]); + + const result = EditorContextAdapter.resolveAssetFile("v1"); + expect(result).toBe(videoFile); + }); + + test("returns null when assetId does not match any asset", () => { + mockGetAssets.mockReturnValue([ + fakeAsset({ id: "v1", name: "intro.mp4" }), + ]); + + const result = EditorContextAdapter.resolveAssetFile("nonexistent"); + expect(result).toBeNull(); + }); + + test("returns File for first video asset when no assetId provided", () => { + const videoFile = new File([], "clip.mp4", { type: "video/mp4" }); + mockGetAssets.mockReturnValue([ + fakeAsset({ id: "img1", name: "photo.jpg", type: "image", file: new File([], "p.jpg") }), + fakeAsset({ id: "v1", name: "clip.mp4", type: "video", file: videoFile }), + ]); + + const result = EditorContextAdapter.resolveAssetFile(); + expect(result).toBe(videoFile); + }); + + test("returns File for first audio asset when no assetId and no video assets", () => { + const audioFile = new File([], "song.mp3", { type: "audio/mpeg" }); + mockGetAssets.mockReturnValue([ + fakeAsset({ id: "img1", name: "photo.jpg", type: "image", file: new File([], "p.jpg") }), + fakeAsset({ id: "a1", name: "song.mp3", type: "audio", file: audioFile }), + ]); + + const result = EditorContextAdapter.resolveAssetFile(); + expect(result).toBe(audioFile); + }); + + test("returns null when assets array is empty", () => { + mockGetAssets.mockReturnValue([]); + + const result = EditorContextAdapter.resolveAssetFile(); + expect(result).toBeNull(); + }); + + test("returns null when no video or audio assets exist and no assetId", () => { + mockGetAssets.mockReturnValue([ + fakeAsset({ id: "img1", name: "photo.jpg", type: "image" }), + ]); + + const result = EditorContextAdapter.resolveAssetFile(); + expect(result).toBeNull(); + }); + + test("returns null when matching asset has no file property", () => { + mockGetAssets.mockReturnValue([ + { id: "v1", name: "clip.mp4", type: "video" }, + ]); + + const result = EditorContextAdapter.resolveAssetFile("v1"); + expect(result).toBeNull(); + }); +}); + +describe("EditorContextAdapter.getAssetHasAudio", () => { + beforeEach(() => { + mockGetAssets.mockClear(); + }); + + test("returns true when asset has audio", () => { + mockGetAssets.mockReturnValue([ + fakeAsset({ id: "v1", hasAudio: true }), + ]); + + expect(EditorContextAdapter.getAssetHasAudio("v1")).toBe(true); + }); + + test("returns false when asset has no audio", () => { + mockGetAssets.mockReturnValue([ + fakeAsset({ id: "v1", hasAudio: false }), + ]); + + expect(EditorContextAdapter.getAssetHasAudio("v1")).toBe(false); + }); + + test("returns undefined when asset is not found", () => { + mockGetAssets.mockReturnValue([]); + + expect(EditorContextAdapter.getAssetHasAudio("nonexistent")).toBeUndefined(); + }); +}); diff --git a/apps/web/src/agent/context.ts b/apps/web/src/agent/context.ts index a7606947..d419f0cc 100644 --- a/apps/web/src/agent/context.ts +++ b/apps/web/src/agent/context.ts @@ -18,6 +18,38 @@ export const EditorContextAdapter = { ticksPerSecond: TICKS_PER_SECOND, }); }, + + /** + * Resolve a media asset's File reference. + * If assetId is provided, finds the exact asset by ID. + * If omitted, returns the File for the first video or audio asset. + * Returns null when no matching asset exists. + * This is the ONLY sanctioned path for tools to obtain a File from EditorCore. + */ + resolveAssetFile(assetId?: string): File | null { + const core = EditorCore.getInstance(); + const assets = core.media.getAssets(); + + if (assetId !== undefined) { + const asset = assets.find((a) => a.id === assetId); + return asset?.file ?? null; + } + + const firstMediaAsset = assets.find( + (a) => a.type === "video" || a.type === "audio", + ); + return firstMediaAsset?.file ?? null; + }, + + /** + * Check whether a media asset has an audio track. + * Returns undefined when the asset is not found. + */ + getAssetHasAudio(assetId: string): boolean | undefined { + const core = EditorCore.getInstance(); + const asset = core.media.getAssets().find((a) => a.id === assetId); + return asset?.hasAudio; + }, }; export { buildSystemPrompt } from "@/agent/system-prompt"; diff --git a/apps/web/src/agent/orchestrator.ts b/apps/web/src/agent/orchestrator.ts index 04298782..54baf782 100644 --- a/apps/web/src/agent/orchestrator.ts +++ b/apps/web/src/agent/orchestrator.ts @@ -6,6 +6,7 @@ import type { } from "@/agent/types"; import { toolRegistry } from "@/agent/tools/registry"; import "@/agent/tools/mock.tool"; +import "@/agent/tools/transcribe-video.tool"; import { useChatStore } from "@/stores/chat-store"; import { useAgentStore } from "@/stores/agent-store"; diff --git a/apps/web/src/agent/tools/__tests__/transcribe-video.test.ts b/apps/web/src/agent/tools/__tests__/transcribe-video.test.ts new file mode 100644 index 00000000..e3013947 --- /dev/null +++ b/apps/web/src/agent/tools/__tests__/transcribe-video.test.ts @@ -0,0 +1,282 @@ +import { beforeEach, describe, expect, mock, test } from "bun:test"; +import type { AgentContext } from "@/agent/types"; +import { toolRegistry } from "@/agent/tools/registry"; + +// bun:test exposes jest.fn() for mock function creation at runtime +// We type the factory explicitly to avoid referencing the `jest` global type +// eslint-disable-next-line @typescript-eslint/no-explicit-any +type MockFn = { + (...args: any[]): any; + mockReturnValue(value: any): void; + mockResolvedValue(value: any): void; + mockClear(): void; + mockImplementation(fn: (...args: any[]) => any): void; +}; +// eslint-disable-next-line @typescript-eslint/no-explicit-any +type MockFnFactory = any>() => T & MockFn; + +const bunTestModule = await import("bun:test"); +const jestObj = (bunTestModule as Record).jest as Record; +const mockFn = jestObj.fn as unknown as MockFnFactory; + +// --- Module mocks for tool dependencies --- + +const mockResolveAssetFile = mockFn<(assetId?: string) => File | null>(); +const mockGetAssetHasAudio = mockFn<(assetId: string) => boolean | undefined>(); +const mockDecodeAudio = mockFn(); +const mockTranscribe = mockFn(); + +mock.module("@/agent/context", () => ({ + EditorContextAdapter: { + resolveAssetFile: mockResolveAssetFile, + getAssetHasAudio: mockGetAssetHasAudio, + }, + buildSystemPrompt: () => "", +})); + +mock.module("@/lib/media/audio", () => ({ + decodeAudioToFloat32: mockDecodeAudio, +})); + +mock.module("@/services/transcription/service", () => ({ + transcriptionService: { + transcribe: mockTranscribe, + }, +})); + +// Import the tool module to trigger side-effect registration +await import("@/agent/tools/transcribe-video.tool"); + +// --- Helpers --- + +function makeContext( + overrides: Partial = {}, +): AgentContext { + return { + projectId: "proj-1", + activeSceneId: "scene-A", + mediaAssets: [], + playbackTimeMs: 0, + ...overrides, + }; +} + +function fakeVideoAsset( + overrides: Partial<{ id: string; name: string; type: string; duration: number }> = {}, +) { + return { + id: "v1", + name: "clip.mp4", + type: "video", + duration: 30, + ...overrides, + }; +} + +function fakeTranscriptionResult() { + return { + text: "Hello world, this is a test.", + segments: [ + { text: "Hello world,", start: 0, end: 2.5 }, + { text: "this is a test.", start: 2.5, end: 5.0 }, + ], + language: "en", + }; +} + +// --- Tests --- + +describe("transcribe_video tool — registration (contract)", () => { + test("is registered in the tool registry", () => { + expect(toolRegistry.has("transcribe_video")).toBe(true); + }); + + test("has correct name and description", () => { + const tool = toolRegistry.get("transcribe_video"); + expect(tool.name).toBe("transcribe_video"); + expect(tool.description).toContain("transcri"); + }); + + test("declares assetId as optional parameter", () => { + const tool = toolRegistry.get("transcribe_video"); + const assetIdParam = tool.parameters.find((p) => p.key === "assetId"); + expect(assetIdParam).toBeDefined(); + expect(assetIdParam!.required).toBe(false); + }); +}); + +describe("transcribe_video tool — success path", () => { + beforeEach(() => { + mockResolveAssetFile.mockClear(); + mockGetAssetHasAudio.mockClear(); + mockDecodeAudio.mockClear(); + mockTranscribe.mockClear(); + }); + + test("transcribes a single video asset successfully", async () => { + const fakeFile = new File([], "clip.mp4", { type: "video/mp4" }); + const context = makeContext({ + mediaAssets: [fakeVideoAsset()], + }); + + mockResolveAssetFile.mockReturnValue(fakeFile); + mockGetAssetHasAudio.mockReturnValue(true); + mockDecodeAudio.mockResolvedValue({ + samples: new Float32Array(16000), + sampleRate: 16000, + }); + mockTranscribe.mockResolvedValue(fakeTranscriptionResult()); + + const tool = toolRegistry.get("transcribe_video"); + const result = await tool.execute({}, context); + + // Verify result shape + expect(result).toEqual({ + assetName: "clip.mp4", + language: "en", + fullText: "Hello world, this is a test.", + segmentCount: 2, + segments: fakeTranscriptionResult().segments, + duration: 30, + }); + + // Verify the adapter was called with the correct asset id + expect(mockResolveAssetFile).toHaveBeenCalledWith("v1"); + expect(mockGetAssetHasAudio).toHaveBeenCalledWith("v1"); + }); + + test("transcribes with explicit assetId", async () => { + const fakeFile = new File([], "other.mp4", { type: "video/mp4" }); + const context = makeContext({ + mediaAssets: [ + fakeVideoAsset({ id: "v1", name: "first.mp4" }), + fakeVideoAsset({ id: "v2", name: "other.mp4" }), + ], + }); + + mockResolveAssetFile.mockReturnValue(fakeFile); + mockGetAssetHasAudio.mockReturnValue(true); + mockDecodeAudio.mockResolvedValue({ + samples: new Float32Array(16000), + sampleRate: 16000, + }); + mockTranscribe.mockResolvedValue(fakeTranscriptionResult()); + + const tool = toolRegistry.get("transcribe_video"); + const result = await tool.execute({ assetId: "v2" }, context); + + expect(result).toEqual( + expect.objectContaining({ assetName: "other.mp4" }), + ); + expect(mockResolveAssetFile).toHaveBeenCalledWith("v2"); + }); +}); + +describe("transcribe_video tool — error branches", () => { + beforeEach(() => { + mockResolveAssetFile.mockClear(); + mockGetAssetHasAudio.mockClear(); + mockDecodeAudio.mockClear(); + mockTranscribe.mockClear(); + }); + + test("returns error when no media assets in project", async () => { + const context = makeContext({ mediaAssets: [] }); + const tool = toolRegistry.get("transcribe_video"); + + const result = await tool.execute({}, context); + expect(result).toEqual({ error: "No active media asset" }); + }); + + test("returns error when only image assets found (no audio track)", async () => { + const context = makeContext({ + mediaAssets: [ + { id: "img1", name: "photo.jpg", type: "image", duration: 0 }, + ], + }); + const tool = toolRegistry.get("transcribe_video"); + + const result = await tool.execute({}, context); + expect(result).toEqual({ error: "Asset has no audio track" }); + }); + + test("returns error when multiple candidates but no assetId", async () => { + const context = makeContext({ + mediaAssets: [ + fakeVideoAsset({ id: "v1", name: "intro.mp4" }), + fakeVideoAsset({ id: "v2", name: "outro.mp4" }), + ], + }); + const tool = toolRegistry.get("transcribe_video"); + + const result = await tool.execute({}, context); + expect(result).toEqual({ + error: "Multiple video/audio assets found: intro.mp4, outro.mp4. Specify which one with assetId.", + }); + }); + + test("returns error when specified assetId not in candidates", async () => { + const context = makeContext({ + mediaAssets: [ + fakeVideoAsset({ id: "v1", name: "intro.mp4" }), + ], + }); + const tool = toolRegistry.get("transcribe_video"); + + const result = await tool.execute({ assetId: "v99" }, context); + expect(result).toEqual({ + error: "Could not access file for asset v99", + }); + }); + + test("returns error when resolveAssetFile returns null", async () => { + const context = makeContext({ + mediaAssets: [fakeVideoAsset()], + }); + + mockResolveAssetFile.mockReturnValue(null); + + const tool = toolRegistry.get("transcribe_video"); + const result = await tool.execute({}, context); + + expect(result).toEqual({ error: "Could not access file for asset v1" }); + }); + + test("returns error when asset has no audio track", async () => { + const fakeFile = new File([], "silent.mp4", { type: "video/mp4" }); + const context = makeContext({ + mediaAssets: [fakeVideoAsset({ name: "silent.mp4" })], + }); + + mockResolveAssetFile.mockReturnValue(fakeFile); + mockGetAssetHasAudio.mockReturnValue(false); + + const tool = toolRegistry.get("transcribe_video"); + const result = await tool.execute({}, context); + + expect(result).toEqual({ error: "Asset has no audio track" }); + }); + + test("returns error when transcriptionService.transcribe rejects", async () => { + const fakeFile = new File([], "clip.mp4", { type: "video/mp4" }); + const context = makeContext({ + mediaAssets: [fakeVideoAsset()], + }); + + mockResolveAssetFile.mockReturnValue(fakeFile); + mockGetAssetHasAudio.mockReturnValue(true); + mockDecodeAudio.mockResolvedValue({ + samples: new Float32Array(16000), + sampleRate: 16000, + }); + mockTranscribe.mockImplementation(() => + Promise.reject(new Error("Worker not initialized")), + ); + + const tool = toolRegistry.get("transcribe_video"); + const result = await tool.execute({}, context); + + // Tool catches service errors and returns { error } per spec + expect(result).toEqual({ error: "Worker not initialized" }); + }); +}); diff --git a/apps/web/src/agent/tools/transcribe-video.tool.ts b/apps/web/src/agent/tools/transcribe-video.tool.ts new file mode 100644 index 00000000..805cd8f0 --- /dev/null +++ b/apps/web/src/agent/tools/transcribe-video.tool.ts @@ -0,0 +1,118 @@ +import type { AgentContext, ToolDefinition } from "@/agent/types"; +import type { + TranscriptionLanguage, + TranscriptionModelId, +} from "@/lib/transcription/types"; +import { toolRegistry } from "@/agent/tools/registry"; +import { EditorContextAdapter } from "@/agent/context"; +import { decodeAudioToFloat32 } from "@/lib/media/audio"; +import { transcriptionService } from "@/services/transcription/service"; + +export type TranscriptionToolResult = { + assetName: string; + language: string; + fullText: string; + segmentCount: number; + segments: Array<{ text: string; start: number; end: number }>; + duration: number; +}; + +export type TranscribeVideoArgs = { + assetId?: string; + language?: TranscriptionLanguage; + modelId?: TranscriptionModelId; +}; + +const transcribeVideoTool: ToolDefinition = { + name: "transcribe_video", + description: + "Transcribes the audio of a video or audio asset using Whisper. Returns structured transcript with segments and timestamps.", + parameters: [ + { key: "assetId", type: "string", required: false }, + { key: "language", type: "string", required: false }, + { key: "modelId", type: "string", required: false }, + ], + execute: async ( + args: Record, + context: AgentContext, + ): Promise => { + const typedArgs = args as TranscribeVideoArgs; + + // Validate: must have media assets + if (context.mediaAssets.length === 0) { + return { error: "No active media asset" }; + } + + // Find candidate assets (video or audio only) + const mediaCandidates = context.mediaAssets.filter( + (a) => a.type === "video" || a.type === "audio", + ); + + if (mediaCandidates.length === 0) { + // Assets exist but none are video/audio (e.g. images only) + return { error: "Asset has no audio track" }; + } + + // Resolve the target asset + let targetAsset: (typeof mediaCandidates)[number] | undefined; + + if (typedArgs.assetId) { + targetAsset = mediaCandidates.find((a) => a.id === typedArgs.assetId); + if (!targetAsset) { + return { + error: `Could not access file for asset ${typedArgs.assetId}`, + }; + } + } else { + if (mediaCandidates.length > 1) { + const names = mediaCandidates.map((a) => a.name).join(", "); + return { + error: `Multiple video/audio assets found: ${names}. Specify which one with assetId.`, + }; + } + targetAsset = mediaCandidates[0]; + } + + // Resolve the actual File via the adapter + const file = EditorContextAdapter.resolveAssetFile(targetAsset.id); + if (!file) { + return { error: `Could not access file for asset ${targetAsset.id}` }; + } + + // Guard: asset must have an audio track + const hasAudio = EditorContextAdapter.getAssetHasAudio(targetAsset.id); + if (hasAudio === false) { + return { error: "Asset has no audio track" }; + } + + try { + // Decode audio → Float32Array + const { samples } = await decodeAudioToFloat32({ + audioBlob: file, + sampleRate: 16000, + }); + + // Run Whisper transcription + const result = await transcriptionService.transcribe({ + audioData: samples, + language: typedArgs.language ?? "auto", + modelId: typedArgs.modelId ?? "whisper-small", + }); + + return { + assetName: targetAsset.name, + language: result.language, + fullText: result.text, + segmentCount: result.segments.length, + segments: result.segments, + duration: targetAsset.duration, + }; + } catch (err) { + const message = + err instanceof Error ? err.message : "Transcription failed"; + return { error: message }; + } + }, +}; + +toolRegistry.register("transcribe_video", transcribeVideoTool); diff --git a/apps/web/src/app/api/agent/chat/__tests__/route.test.ts b/apps/web/src/app/api/agent/chat/__tests__/route.test.ts index edeffb83..cda65b1f 100644 --- a/apps/web/src/app/api/agent/chat/__tests__/route.test.ts +++ b/apps/web/src/app/api/agent/chat/__tests__/route.test.ts @@ -64,4 +64,55 @@ describe("POST /api/agent/chat", () => { const res = await POST(req); expect(res.status).toBe(400); }); + + test("returns transcribe_video tool call when message contains 'transcribe'", async () => { + const req = makeRequest({ + messages: [{ role: "user", content: "Please transcribe this video" }], + }); + + const res = await POST(req); + expect(res.status).toBe(200); + + const data = await res.json(); + expect(data.content).toBe("I'll transcribe your video now."); + expect(data.toolCalls).toHaveLength(1); + expect(data.toolCalls[0].name).toBe("transcribe_video"); + expect(data.toolCalls[0].id).toBe("tc_transcribe_1"); + }); + + test("returns transcribe_video tool call for case-insensitive 'TRANSCRIBE'", async () => { + const req = makeRequest({ + messages: [{ role: "user", content: "CAN YOU TRANSCRIBE THE AUDIO?" }], + }); + + const res = await POST(req); + expect(res.status).toBe(200); + + const data = await res.json(); + expect(data.toolCalls[0].name).toBe("transcribe_video"); + }); + + test("returns transcribe_video when 'transcription' keyword is present", async () => { + const req = makeRequest({ + messages: [{ role: "user", content: "I need a transcription of this clip" }], + }); + + const res = await POST(req); + expect(res.status).toBe(200); + + const data = await res.json(); + expect(data.toolCalls[0].name).toBe("transcribe_video"); + }); + + test("falls back to echo_context when message has no transcription intent", async () => { + const req = makeRequest({ + messages: [{ role: "user", content: "What's the weather like?" }], + }); + + const res = await POST(req); + expect(res.status).toBe(200); + + const data = await res.json(); + expect(data.toolCalls[0].name).toBe("echo_context"); + }); }); diff --git a/apps/web/src/app/api/agent/chat/route.ts b/apps/web/src/app/api/agent/chat/route.ts index dc87c2f2..30ea25d3 100644 --- a/apps/web/src/app/api/agent/chat/route.ts +++ b/apps/web/src/app/api/agent/chat/route.ts @@ -22,8 +22,28 @@ export async function POST(request: NextRequest) { ); } - // Phase 1: canned mock response with a tool call to validate the full pipeline - const mockResponse = { + // Mock intent detection: keyword match on last user message + const lastUserMessage = [...result.data.messages] + .reverse() + .find((m) => m.role === "user"); + const isTranscriptionRequest = + lastUserMessage?.content.toLowerCase().includes("transcri") ?? false; + + if (isTranscriptionRequest) { + return NextResponse.json({ + content: "I'll transcribe your video now.", + toolCalls: [ + { + id: "tc_transcribe_1", + name: "transcribe_video", + args: {}, + }, + ], + }); + } + + // Default: echo_context fallback + return NextResponse.json({ content: "Let me check your editor context.", toolCalls: [ { @@ -32,7 +52,5 @@ export async function POST(request: NextRequest) { args: {}, }, ], - }; - - return NextResponse.json(mockResponse); + }); } diff --git a/apps/web/src/components/editor/panels/chat/__tests__/transcript-utils.test.ts b/apps/web/src/components/editor/panels/chat/__tests__/transcript-utils.test.ts new file mode 100644 index 00000000..54a3885f --- /dev/null +++ b/apps/web/src/components/editor/panels/chat/__tests__/transcript-utils.test.ts @@ -0,0 +1,134 @@ +import { describe, expect, test } from "bun:test"; +import { + isTranscriptData, + formatTimestamp, +} from "../transcript-utils"; + +describe("isTranscriptData — transcript detection", () => { + test("returns true for valid transcript data", () => { + const data = { + assetName: "clip.mp4", + language: "en", + fullText: "Hello world", + segments: [{ text: "Hello world", start: 0, end: 2.5 }], + duration: 30, + }; + + expect(isTranscriptData(data)).toBe(true); + }); + + test("returns true for transcript with empty segments array", () => { + const data = { + assetName: "clip.mp4", + language: "en", + fullText: "", + segments: [], + duration: 0, + }; + + expect(isTranscriptData(data)).toBe(true); + }); + + test("returns false when fullText is missing", () => { + const data = { + assetName: "clip.mp4", + language: "en", + segments: [{ text: "Hello", start: 0, end: 1 }], + duration: 30, + }; + + expect(isTranscriptData(data)).toBe(false); + }); + + test("returns false when segments is missing", () => { + const data = { + assetName: "clip.mp4", + language: "en", + fullText: "Hello", + duration: 30, + }; + + expect(isTranscriptData(data)).toBe(false); + }); + + test("returns false when segments is not an array", () => { + const data = { + assetName: "clip.mp4", + fullText: "Hello", + segments: "not an array", + duration: 30, + }; + + expect(isTranscriptData(data)).toBe(false); + }); + + test("returns false for null", () => { + expect(isTranscriptData(null)).toBe(false); + }); + + test("returns false for non-transcript tool result", () => { + const data = { projectId: "p1", mediaCount: 2 }; + + expect(isTranscriptData(data)).toBe(false); + }); + + test("returns false for a plain string", () => { + expect(isTranscriptData("some text")).toBe(false); + }); + + test("detects transcript from JSON.parse of tool_result content", () => { + const toolResultContent = JSON.stringify({ + assetName: "clip.mp4", + language: "en", + fullText: "Hello world, this is a test.", + segments: [ + { text: "Hello world,", start: 0, end: 2.5 }, + { text: "this is a test.", start: 2.5, end: 5.0 }, + ], + duration: 30, + }); + + const parsed = JSON.parse(toolResultContent); + expect(isTranscriptData(parsed)).toBe(true); + }); + + test("does not detect error results as transcript", () => { + const toolResultContent = JSON.stringify({ + error: "No active media asset", + }); + + const parsed = JSON.parse(toolResultContent); + expect(isTranscriptData(parsed)).toBe(false); + }); +}); + +describe("formatTimestamp — timestamp formatting", () => { + test("formats zero seconds as 00:00", () => { + expect(formatTimestamp(0)).toBe("00:00"); + }); + + test("formats seconds under a minute", () => { + expect(formatTimestamp(45)).toBe("00:45"); + }); + + test("formats exact minutes", () => { + expect(formatTimestamp(120)).toBe("02:00"); + }); + + test("formats minutes and seconds", () => { + expect(formatTimestamp(125)).toBe("02:05"); + }); + + test("pads single-digit seconds", () => { + expect(formatTimestamp(61)).toBe("01:01"); + }); + + test("formats large durations", () => { + expect(formatTimestamp(3661)).toBe("61:01"); + }); + + test("formats segment start time matching spec [MM:SS]", () => { + expect(formatTimestamp(2.5)).toBe("00:02"); + expect(formatTimestamp(150.7)).toBe("02:30"); + }); +}); diff --git a/apps/web/src/components/editor/panels/chat/message-bubble.tsx b/apps/web/src/components/editor/panels/chat/message-bubble.tsx index e9b3ebc4..636401de 100644 --- a/apps/web/src/components/editor/panels/chat/message-bubble.tsx +++ b/apps/web/src/components/editor/panels/chat/message-bubble.tsx @@ -2,6 +2,11 @@ import { cn } from "@/utils/ui"; import type { ChatMessage } from "@/agent/types"; +import { + isTranscriptData, + formatTimestamp, +} from "./transcript-utils"; +import type { TranscriptData } from "./transcript-utils"; interface MessageBubbleProps { message: ChatMessage; @@ -13,9 +18,55 @@ const ROLE_LABELS: Record = { tool_result: "Tool", }; +function TranscriptCard({ data }: { data: TranscriptData }) { + return ( +
+
+ {data.assetName} + + {data.language} · {formatTimestamp(data.duration)} + +
+

{data.fullText}

+
+ + {data.segments.length} segment{data.segments.length !== 1 && "s"} + +
+ {data.segments.map((seg, i) => ( +
+ [{formatTimestamp(seg.start)}]{" "} + {seg.text} +
+ ))} +
+
+
+ ); +} + export function MessageBubble({ message }: MessageBubbleProps) { const isUser = message.role === "user"; + // For tool_result messages, try to parse as transcript + if (message.role === "tool_result") { + try { + const parsed = JSON.parse(message.content); + if (isTranscriptData(parsed)) { + return ( +
+ + {ROLE_LABELS.tool_result} + + +
+ ); + } + } catch { + // Not JSON — fall through to plain text + } + } + return (
; + duration: number; +} + +export function isTranscriptData(data: unknown): data is TranscriptData { + return ( + typeof data === "object" && + data !== null && + "fullText" in data && + "segments" in data && + Array.isArray((data as TranscriptData).segments) + ); +} + +export function formatTimestamp(seconds: number): string { + const mins = Math.floor(seconds / 60); + const secs = Math.floor(seconds % 60); + return `${String(mins).padStart(2, "0")}:${String(secs).padStart(2, "0")}`; +} diff --git a/openspec/changes/transcribe-video-fase-1/design.md b/openspec/changes/transcribe-video-fase-1/design.md new file mode 100644 index 00000000..b6ffa832 --- /dev/null +++ b/openspec/changes/transcribe-video-fase-1/design.md @@ -0,0 +1,113 @@ +# Design: `transcribe_video` Phase 1 + +## Technical Approach + +Add a single `transcribe_video` tool that wraps the existing `TranscriptionService` + `decodeAudioToFloat32()` pipeline. The tool resolves the target asset through an extended `EditorContextAdapter`, extracts audio, runs Whisper, and returns a structured `TranscriptionToolResult`. The mock API route triggers it; the orchestrator stores the result in chat state as a `tool_result` message. No new services, no new stores, no timeline mutations. + +## Architecture Decisions + +| Decision | Choice | Rejected | Rationale | +|----------|--------|----------|-----------| +| Asset file access | Extend `EditorContextAdapter` with `getAssetFile()` | Import `EditorCore` directly in tool | Maintains single-import boundary — only `context.ts` touches `core/` | +| Tool registration | Side-effect import in `orchestrator.ts` (same as `mock.tool.ts`) | Dynamic/lazy registration | Follows existing pattern; tool count is small | +| Asset selection | `assetId` arg → single video/audio → error if ambiguous | Always transcribe first asset | Explicit is better; ambiguous state gets a clear error message | +| Result type | New `TranscriptionToolResult` interface in tool file | Re-export `TranscriptionResult` from lib | Tool result includes `assetName`, `segmentCount`, `duration` — richer than raw `TranscriptionResult` | +| Mock API trigger | Keyword match on "transcri" in user message | Always return `transcribe_video` call | Keeps `echo_context` working for other messages; simple keyword gate | +| Chat rendering | `MessageBubble` detects `tool_result` with transcript shape and formats it | New TranscriptMessage component | Minimal change — one formatting branch inside existing component | + +## Data Flow + +``` +User types "transcribe my video" + → ChatPanel.handleSend() + → sendMessage() (chat-store: user msg, loading=true) + → EditorContextAdapter.getContext() → AgentContext + → orchestrator.run(messages, context) + → POST /api/agent/chat → mock API returns { content, toolCalls: [transcribe_video] } + → resolveToolCalls() + → toolRegistry.get("transcribe_video").execute(args, context) + → find asset (context.mediaAssets + EditorContextAdapter.getAssetFile) + → guard: hasAudio? asset found? + → decodeAudioToFloat32({ audioBlob: file, sampleRate: 16000 }) + → transcriptionService.transcribe({ audioData, language }) + → return TranscriptionToolResult + → chatStore.addMessage({ role: "assistant", content, toolCalls }) + → chatStore.addMessage({ role: "tool_result", content: JSON.stringify(result) }) + → agentStore.setStatus("idle"), chatStore.setLoading(false) +``` + +## File Changes + +| File | Action | Description | +|------|--------|-------------| +| `apps/web/src/agent/tools/transcribe-video.tool.ts` | Create | Real transcription tool — asset resolution, audio decode, Whisper call, structured result | +| `apps/web/src/agent/context.ts` | Modify | Add `getAssetFile(assetId: string): File \| null` method | +| `apps/web/src/agent/orchestrator.ts` | Modify | Add `import "@/agent/tools/transcribe-video.tool"` (side-effect registration) | +| `apps/web/src/app/api/agent/chat/route.ts` | Modify | Return `transcribe_video` tool call when message contains "transcri", keep `echo_context` fallback | +| `apps/web/src/components/editor/panels/chat/message-bubble.tsx` | Modify | Detect transcript-shaped tool_result and render with timestamps | + +## Interfaces / Contracts + +```typescript +// apps/web/src/agent/tools/transcribe-video.tool.ts + +interface TranscriptionToolResult { + assetName: string; + language: string; + fullText: string; + segmentCount: number; + segments: Array<{ text: string; start: number; end: number }>; + duration: number; +} + +// Tool parameters +interface TranscribeVideoArgs { + assetId?: string; // optional — resolve automatically if omitted + language?: string; // optional — defaults to "auto" + modelId?: string; // optional — defaults to whisper-small +} +``` + +```typescript +// apps/web/src/agent/context.ts — addition + +export const EditorContextAdapter = { + getContext() { /* existing */ }, + getAssetFile(assetId: string): File | null { + const core = EditorCore.getInstance(); + const asset = core.media.getAssets().find(a => a.id === assetId); + return asset?.file ?? null; + }, +}; +``` + +## Error States (no UX redesign) + +| State | Detection | Response | +|-------|-----------|----------| +| No media assets | `context.mediaAssets.length === 0` | Tool returns `{ error: "No media assets in project" }` — rendered as tool_result error text | +| No video/audio asset | No asset with `type !== "video" && type !== "audio"` | Tool returns `{ error: "No video or audio assets found" }` | +| Asset has no audio | `hasAudio === false` on resolved asset | Tool returns `{ error: "{assetName} has no audio track" }` | +| Multiple candidates, no `assetId` | >1 video/audio asset and no explicit `assetId` arg | Tool returns `{ error: "Multiple video/audio assets found. Specify which one." }` with list of names | +| Transcription service failure | `transcriptionService.transcribe()` rejects | Caught by orchestrator's existing try/catch → `ToolResult.error` message | +| Asset file not found | `getAssetFile()` returns `null` | Tool returns `{ error: "Could not access file for asset {assetId}" }` | + +All errors flow through the existing `tool_result` path — no new UI components needed. + +## Testing Strategy + +| Layer | What to Test | Approach | +|-------|-------------|----------| +| Unit | Asset selection logic (single/multiple/none) | Pure function test — pass `AgentContext` + `TranscribeVideoArgs`, verify asset resolution | +| Unit | `getAssetFile()` on EditorContextAdapter | Mock `EditorCore.media.getAssets()` return, verify correct file returned | +| Unit | Tool error branches (no audio, no asset, ambiguous) | Mock `decodeAudioToFloat32` and `transcriptionService`, verify error shapes | +| Integration | End-to-end tool execution | Mock only `transcriptionService.transcribe()`, verify full result shape with real `decodeAudioToFloat32` on a small WAV | +| Contract | Tool registration in registry | Verify `toolRegistry.get("transcribe_video")` returns the definition | + +## Migration / Rollback + +No migration required. Rollback = remove tool import from orchestrator, revert API route to `echo_context` only, revert `MessageBubble` changes. + +## Open Questions + +- [ ] Should the tool validate `MediaAsset.type === "video" || type === "audio"` or also accept images with embedded audio? (Recommendation: video/audio only for Phase 1) diff --git a/openspec/changes/transcribe-video-fase-1/exploration.md b/openspec/changes/transcribe-video-fase-1/exploration.md new file mode 100644 index 00000000..e51c883c --- /dev/null +++ b/openspec/changes/transcribe-video-fase-1/exploration.md @@ -0,0 +1,229 @@ +# Exploration: `transcribe_video` Tool — Phase 1 + +## Current State + +### Completed Agent Infrastructure (infra-habilitadora-fase-1) + +The full agent pipeline is wired end-to-end with a mock tool. Key components: + +- **`agent/types.ts`** — `AgentContext`, `ChatMessage`, `ToolCall`, `ToolResult`, `ToolDefinition`, `ExecutionState` +- **`agent/orchestrator.ts`** — client-side single-pass orchestrator: POST → resolve tool calls → append messages +- **`agent/tools/registry.ts`** — `DefinitionRegistry` (generic, reusable) +- **`agent/tools/mock.tool.ts`** — `echo_context` tool (returns context summary) +- **`agent/context.ts`** + **`agent/context-mapper.ts`** — thin adapter reading EditorCore → `AgentContext` POJO +- **`agent/system-prompt.ts`** — builds prompt from `AgentContext` +- **`stores/chat-store.ts`** — messages, loading, error +- **`stores/agent-store.ts`** — execution status, active tool, context snapshot +- **`app/api/agent/chat/route.ts`** — stateless mock API returning canned response with tool call +- **`components/editor/panels/chat/`** — ChatPanel, MessageBubble, ChatInput + +The orchestrator is **client-side only**. The API route is a stateless LLM proxy (currently mocked). Tool execution happens in-browser where `EditorCore` is accessible. + +### Existing Transcription Infrastructure + +A full Whisper-based transcription pipeline already exists: + +| File | Role | +|------|------| +| `lib/transcription/types.ts` | `TranscriptionSegment { text, start, end }`, `TranscriptionResult { text, segments, language }`, `TranscriptionStatus`, `TranscriptionModelId` | +| `lib/transcription/models.ts` | 4 Whisper models (tiny → large-v3-turbo), default is whisper-small | +| `lib/transcription/audio.ts` | Sample rate (16000), chunk/stride constants | +| `services/transcription/service.ts` | `TranscriptionService` — Web Worker manager, exposes `transcribe({ audioData, language, modelId, onProgress })` → `TranscriptionResult` | +| `services/transcription/worker.ts` | HuggingFace `@huggingface/transformers` Whisper pipeline in Web Worker | +| `lib/transcription/caption.ts` | `buildCaptionChunks({ segments })` → `CaptionChunk[]` | +| `lib/transcription/diagnostics.ts` | Checks if timeline has audio before enabling transcription | +| `lib/subtitles/insert.ts` | `insertCaptionChunksAsTextTrack({ editor, captions })` — adds text track via `BatchCommand` | + +### How the Existing Captions Panel Works + +`components/editor/panels/assets/views/captions.tsx` does the full flow: +1. `extractTimelineAudio({ tracks, mediaAssets, totalDuration })` → audio Blob (from **entire timeline**) +2. `decodeAudioToFloat32({ audioBlob, sampleRate: 16000 })` → `{ samples: Float32Array }` +3. `transcriptionService.transcribe({ audioData: samples, language, onProgress })` → `TranscriptionResult` +4. `buildCaptionChunks({ segments })` → `CaptionChunk[]` +5. `insertCaptionChunksAsTextTrack({ editor, captions })` → creates text track + +**Critical gap**: The existing flow extracts audio from the **entire timeline** (all tracks mixed). The agent tool needs to transcribe a **specific video/audio asset** — either the one on the main track or one explicitly referenced. + +### MediaAsset Structure + +`MediaAsset` (`lib/media/types.ts`) extends `MediaAssetData` with `file: File` and `url?: string`. `MediaAssetData` has: `id`, `name`, `type` (image|video|audio), `duration`, `hasAudio`, etc. The `file` property gives direct access to the raw `File` blob for audio extraction. + +### Audio Extraction Paths + +Two paths exist to get Float32Array audio data: + +1. **From the timeline**: `extractTimelineAudio()` → Blob → `decodeAudioToFloat32()` (existing, used by Captions panel) +2. **From a single asset**: `MediaAsset.file` (a `File`/`Blob`) → `decodeAudioToFloat32()` (existing function, just needs the Blob) + +Path 2 is the one the agent tool needs. `decodeAudioToFloat32` already takes a `Blob` and returns `Float32Array`. No new audio extraction code is needed. + +--- + +## Affected Areas + +- `apps/web/src/agent/tools/transcribe-video.tool.ts` — **new** tool definition +- `apps/web/src/agent/orchestrator.ts` — minor: must import the new tool module (like `mock.tool.ts`) +- `apps/web/src/agent/system-prompt.ts` — likely needs update to describe available tools +- `apps/web/src/app/api/agent/chat/route.ts` — update mock response to return a `transcribe_video` tool call instead of `echo_context` +- `apps/web/src/agent/types.ts` — may need `TranscriptionToolResult` type +- `apps/web/src/lib/transcription/types.ts` — may read/re-export for tool result +- `apps/web/src/services/transcription/service.ts` — consumed by the tool (no changes needed) +- `apps/web/src/lib/media/audio.ts` — `decodeAudioToFloat32()` consumed by the tool (no changes needed) +- `apps/web/src/core/managers/media-manager.ts` — tool reads assets to find the target video (no changes needed) + +--- + +## Approaches + +### 1. Thin Agent Tool — Reuse Existing Services, Return TranscriptionResult + +Create a `transcribe_video` tool that: (a) reads the active video from `AgentContext.mediaAssets`, (b) extracts audio via `decodeAudioToFloat32`, (c) calls `transcriptionService.transcribe()`, (d) returns a `TranscriptionResult` as the tool result. No caption insertion — that's Phase 2. + +- **Pros**: Minimal new code. Reuses battle-tested `TranscriptionService` and `decodeAudioToFloat32`. Clean separation: tool returns data, agent/LLM decides what to do with it. +- **Cons**: Long-running tool (model loading + transcription = 30s-3min depending on model). The orchestrator is single-pass and blocks until the tool completes. No progress reporting to the chat UI during execution. +- **Effort**: Low + +### 2. Thin Tool + Progress Reporting + +Same as Approach 1, but adds progress callbacks to the tool execution: the tool emits `loading-model`, `transcribing`, and `complete` progress states that the agent store exposes to the chat UI. + +- **Pros**: Better UX — user sees "Loading Whisper model... 45%" and "Transcribing..." instead of a frozen "Thinking..." state. +- **Cons**: Requires modifying `ToolDefinition.execute` to support progress reporting (either a callback parameter or a different signature). Also needs the chat panel to render progress states. More surface area. +- **Effort**: Medium + +### 3. Thin Tool + Caption Insertion (Bigger Scope) + +Same as Approach 1, but the tool also inserts caption chunks into the timeline via `insertCaptionChunksAsTextTrack`. + +- **Pros**: End-to-end value — user asks "transcribe my video" and gets captions on the timeline. +- **Cons**: Tool now has a side effect (timeline mutation). Violates "tool returns data, orchestrator decides" principle. Couples the tool to `EditorCore` and `CommandManager`. Should be a separate tool or an explicit follow-up action. +- **Effort**: Medium-High + +--- + +## Recommendation + +**Approach 1 — Thin Agent Tool returning TranscriptionResult.** Reasons: + +1. **Follows the existing architecture**: Tools are pure data-in/data-out. The LLM decides what to do with the result. +2. **Zero new infra**: The tool wraps `transcriptionService.transcribe()` and `decodeAudioToFloat32()` — both already exist and are tested. +3. **Clear Phase 2 boundary**: Caption insertion, progress reporting, streaming, and multi-turn tool loops are all natural Phase 2 additions that build ON TOP of this. +4. **Fastest vertical slice**: One new file (`transcribe-video.tool.ts`) + minor wiring changes. + +### Recommended Tool Design + +```typescript +// transcribe-video.tool.ts + +const transcribeVideoTool: ToolDefinition = { + name: "transcribe_video", + description: "Transcribes the active video's audio using Whisper. Returns text and timestamped segments.", + parameters: [ + { key: "assetId", type: "string", required: false }, + { key: "language", type: "string", required: false }, + { key: "modelId", type: "string", required: false }, + ], + execute: async (args, context) => { + // 1. Find the target asset (by assetId or first video/audio in context) + // 2. Get the File blob from MediaManager (not in AgentContext — need to extend or read directly) + // 3. decodeAudioToFloat32({ audioBlob: asset.file, sampleRate: 16000 }) + // 4. transcriptionService.transcribe({ audioData, language, modelId }) + // 5. Return TranscriptionResult + }, +}; +``` + +### Critical Decision: Asset Resolution + +The `AgentContext.mediaAssets` array has `id`, `name`, `type`, `duration` but **NOT the `File` blob** (it's intentionally a summary). The tool needs the actual `File` to extract audio. Two options: + +- **Option A**: The tool directly reads `EditorCore.media.getAssets()` to get the full `MediaAsset` with `file`. This breaks the "no EditorCore access from agent/" rule. However, the rule only applies to the `agent/` directory, and tools already receive `AgentContext` which could be extended. +- **Option B** (recommended): **Extend `AgentContext`** to include `activeAssetFile: File | null` or use the existing `EditorContextAdapter` pattern — the adapter can resolve the file and pass it as part of the context or as a separate parameter. Since tools already get `AgentContext`, we add an `assetFiles: Map` or similar. + +Actually, the simplest approach respecting the architecture: **the `transcribe_video` tool's `execute()` function imports `MediaManager` through the same adapter pattern**. The tool lives in `agent/tools/` which is allowed to import from `agent/context.ts`. We extend the adapter to provide asset file access: + +```typescript +// Extend agent/context.ts with a resolver +export const EditorContextAdapter = { + getContext() { ... }, + getAssetFile(assetId: string): File | null { + const core = EditorCore.getInstance(); + const asset = core.media.getAssets().find(a => a.id === assetId); + return asset?.file ?? null; + }, +}; +``` + +This keeps the single-import rule (only `context.ts` touches EditorCore) while giving tools access to asset blobs. + +### Asset Selection Logic + +When the user says "transcribe my video" without specifying which one: + +1. If `args.assetId` is provided → use that asset +2. If only one video/audio asset in `context.mediaAssets` → use that one +3. If multiple → return an error asking the user to specify + +### What the Tool Returns + +```typescript +{ + assetName: string; + language: string; + fullText: string; + segmentCount: number; + segments: Array<{ text: string; start: number; end: number }>; + duration: number; +} +``` + +The full text + segments are returned as the `ToolResult`. The LLM can then summarize, answer questions about the content, or in Phase 2, trigger caption insertion. + +### Where the Result Is Stored + +- **Tool result** → stored in `chatStore.messages` as a `tool_result` message (existing flow) +- **No persistent storage** in Phase 1 — the transcription is ephemeral, tied to the chat session +- Phase 2 can add persistent transcription storage (IndexedDB, project metadata) and caption insertion + +--- + +## Phase 2 Boundary (explicitly OUT of scope for Phase 1) + +| Feature | Why it's Phase 2 | +|---------|-------------------| +| Caption/subtitle insertion into timeline | Side-effect tool, needs separate design | +| Progress reporting during transcription | Requires `ToolDefinition.execute` signature change | +| Streaming transcription results | Requires orchestrator multi-turn support | +| Persistent transcription storage | Needs data model design | +| Multi-language selection UI in chat | UI concern, not tool concern | +| Transcription of timeline (all tracks) vs single asset | Different extraction path, separate tool | +| Real LLM integration (replacing mock API) | Separate change, infrastructure concern | + +--- + +## Risks + +1. **Long-running tool execution**: Whisper model loading can take 10-60s, transcription 30s-5min depending on video length and model. The orchestrator is single-pass and the chat UI shows "Thinking..." the entire time. Users may think it's frozen. **Mitigation**: Document this known limitation. Phase 2 adds progress reporting. + +2. **Asset file access architecture tension**: The tool needs `MediaAsset.file` (a `File` blob) which is NOT in `AgentContext`. Adding a `getAssetFile()` method to the adapter is clean but expands the adapter's contract. **Mitigation**: The adapter already exists for exactly this purpose — it's the sanctioned bridge. + +3. **Web Worker lifecycle**: `transcriptionService` manages a Web Worker with model lifecycle (init → ready). If the tool is called multiple times, the service handles reuse. But if the user switches models, there's a re-init delay. **Mitigation**: No change needed — the existing `TranscriptionService` already handles this correctly. + +4. **No real LLM yet**: The API route still returns a canned mock response. The `transcribe_video` tool call must be triggered by the mock, not by a real LLM reasoning about the user's intent. **Mitigation**: Update the mock to return a `transcribe_video` tool call for any message containing "transcribe". This is clearly temporary. + +5. **Memory pressure**: Loading a large video's audio as `Float32Array` can consume significant memory. A 10-minute video at 16kHz mono = ~19MB of float data. **Mitigation**: Acceptable for Phase 1. Phase 2 can add chunked processing. + +6. **`hasAudio` guard**: Not all video assets have audio. The tool must check `MediaAssetData.hasAudio` before attempting transcription. **Mitigation**: Check in `execute()` and return a clear error message. + +--- + +## Ready for Proposal + +**Yes.** The exploration identifies: +- One new file to create (`transcribe-video.tool.ts`) +- Minor wiring changes (orchestrator import, mock API update, adapter extension) +- Clear reuse of existing transcription service and audio decoding +- A clean Phase 2 boundary + +**Next**: Run `sdd-propose` with this exploration as input. diff --git a/openspec/changes/transcribe-video-fase-1/proposal.md b/openspec/changes/transcribe-video-fase-1/proposal.md new file mode 100644 index 00000000..429dcdf0 --- /dev/null +++ b/openspec/changes/transcribe-video-fase-1/proposal.md @@ -0,0 +1,63 @@ +# Proposal: `transcribe_video` Phase 1 + +## Intent + +Ship the first real agent feature: ask the editor chat to transcribe the selected/active media asset and return usable transcript text with timestamps. This proves the current agent shell can drive real product value without adding a new workflow surface. + +## Scope + +### In Scope +- Add one real `transcribe_video` tool to the current orchestrator/chat flow. +- Reuse existing audio decode + Whisper transcription services for a single media asset. +- Store transcript output in the current chat session and render timestamped segments in chat. +- Extend the context bridge just enough to resolve the selected/active asset file safely. + +### Out of Scope +- Subtitle insertion, filler-word cleanup, highlights, summarization, and multi-tool chains. +- Timeline-wide transcription, streaming/progress UX, persistent project-level transcript storage, and real LLM routing. + +## Capabilities + +### New Capabilities +- `transcribe-video`: End-to-end chat-triggered transcription of one media asset, returning full text plus timestamped segments. + +### Modified Capabilities +- `agent-context-bridge`: Expand the sanctioned adapter so tools can safely resolve the selected/active asset file without direct `EditorCore` access. +- `agent-session-shell`: Replace the mock-only tool path with real `transcribe_video` execution and transcript result persistence in chat state. +- `editor-chat-panel`: Render transcript-oriented assistant/tool output with readable timestamps inside the existing chat experience. + +## Approach + +Keep the slice thin: mock API triggers `transcribe_video`, the tool resolves the target asset through `EditorContextAdapter`, decodes `MediaAsset.file`, calls `transcriptionService.transcribe()`, and returns `{ assetName, language, fullText, segments, duration }`. The orchestrator stores that result in chat state; the chat UI renders the transcript inline. + +## Affected Areas + +| Area | Impact | Description | +|------|--------|-------------| +| `apps/web/src/agent/tools/transcribe-video.tool.ts` | New | Real transcription tool | +| `apps/web/src/agent/context.ts` | Modified | Safe asset-file resolver | +| `apps/web/src/agent/orchestrator.ts` | Modified | Execute/store transcript result | +| `apps/web/src/app/api/agent/chat/route.ts` | Modified | Mock trigger for transcription | +| `apps/web/src/components/editor/panels/chat/` | Modified | Timestamped transcript rendering | + +## Risks + +| Risk | Likelihood | Mitigation | +|------|------------|------------| +| Long-running Whisper execution feels frozen | Med | Keep scope explicit; defer progress UX | +| Ambiguous or silent asset selection | Med | Require selected/active asset fallback + clear errors | +| Large files increase memory use | Low | Limit Phase 1 to current in-browser path | + +## Rollback Plan + +Remove `transcribe_video` registration, restore mock-only route behavior, and keep the chat panel on generic message rendering. + +## Dependencies + +- Existing `TranscriptionService`, `decodeAudioToFloat32()`, and current asset/editor context. + +## Success Criteria + +- [ ] A chat request can trigger `transcribe_video` for the selected/active media asset. +- [ ] The result appears in chat with transcript text and timestamped segments. +- [ ] Asset access stays behind `EditorContextAdapter`, with no direct `core/` imports from agent modules. diff --git a/openspec/changes/transcribe-video-fase-1/specs/agent-context-bridge/spec.md b/openspec/changes/transcribe-video-fase-1/specs/agent-context-bridge/spec.md new file mode 100644 index 00000000..add4e2ce --- /dev/null +++ b/openspec/changes/transcribe-video-fase-1/specs/agent-context-bridge/spec.md @@ -0,0 +1,31 @@ +# Delta for Agent Context Bridge + +## ADDED Requirements + +### Requirement: Active Asset File Resolver + +The system SHALL provide `resolveAssetFile(assetId?: string)` on `EditorContextAdapter` that returns the `File` reference for the requested media asset. When `assetId` is provided, it MUST resolve that specific asset. When omitted, it MUST resolve the first video/audio asset from `mediaAssets`. The method SHALL return `null` when no matching asset exists. This method is the ONLY sanctioned path for tools to obtain a `File` reference — tools MUST NOT access `EditorCore.media` directly. + +#### Scenario: Resolve by explicit asset ID + +- GIVEN `EditorCore.media.getAssets()` contains an asset with `id: "v1"` and `name: "clip.mp4"` +- WHEN `resolveAssetFile("v1")` is called +- THEN it returns the `File` object for that asset + +#### Scenario: Resolve active asset without ID + +- GIVEN `EditorCore.media.getAssets()` contains two assets, the first being a video +- WHEN `resolveAssetFile()` is called without arguments +- THEN it returns the `File` for the first video or audio asset + +#### Scenario: No matching asset + +- GIVEN `EditorCore.media.getAssets()` is empty +- WHEN `resolveAssetFile()` is called +- THEN it returns `null` + +#### Scenario: Asset ID not found + +- GIVEN `EditorCore.media.getAssets()` contains assets with IDs `["v1", "v2"]` +- WHEN `resolveAssetFile("v99")` is called +- THEN it returns `null` diff --git a/openspec/changes/transcribe-video-fase-1/specs/agent-session-shell/spec.md b/openspec/changes/transcribe-video-fase-1/specs/agent-session-shell/spec.md new file mode 100644 index 00000000..fd680aee --- /dev/null +++ b/openspec/changes/transcribe-video-fase-1/specs/agent-session-shell/spec.md @@ -0,0 +1,45 @@ +# Delta for Agent Session Shell + +## MODIFIED Requirements + +### Requirement: Tool Registry Shell + +The system SHALL provide a `ToolRegistry` extending `DefinitionRegistry` where `ToolDefinition` contains `{ name, description, parameters, execute: (args, context) => Promise }`. The registry MUST register `echo_context` and `transcribe_video`. The registry MUST support `register`, `get`, `has`, and `getAll`. + +(Previously: Registry registered exactly one tool — `echo_context`.) + +#### Scenario: Mock echo tool executes + +- GIVEN `echo_context` is registered with `execute: (args, ctx) => ({ projectId: ctx.projectId, ... })` +- WHEN `registry.get("echo_context").execute({}, context)` is called +- THEN it returns a context summary with projectId and mediaCount + +#### Scenario: Transcribe tool is registered + +- GIVEN the `transcribe_video` tool module is imported +- WHEN `registry.has("transcribe_video")` is called +- THEN it returns `true` + +### Requirement: API Proxy Route + +The system SHALL expose `POST /api/agent/chat` that accepts `{ messages: ChatMessage[], context?: AgentContext }` and returns `{ content: string, toolCalls?: ToolCall[] }`. In this phase, the route MUST return a canned mock response — no real LLM call. When the last user message contains transcription-related intent (e.g., "transcribe"), the route MUST return a `transcribe_video` tool call. Otherwise, it MUST return the existing `echo_context` tool call. The route MUST validate the request body schema and return 400 on invalid input. + +(Previously: Route always returned `echo_context` tool call regardless of user intent.) + +#### Scenario: Valid request returns mock response + +- GIVEN a POST request with valid `{ messages: [{ role: "user", content: "Hi" }] }` +- WHEN the handler processes the request +- THEN it returns 200 with `{ content: "...", toolCalls: [{ name: "echo_context", ... }] }` + +#### Scenario: Transcription intent triggers transcribe tool + +- GIVEN a POST request with `{ messages: [{ role: "user", content: "Transcribe this video" }] }` +- WHEN the handler processes the request +- THEN it returns 200 with `{ content: "...", toolCalls: [{ name: "transcribe_video", args: {} }] }` + +#### Scenario: Invalid request returns 400 + +- GIVEN a POST request with `{}` (missing `messages`) +- WHEN the handler processes the request +- THEN it returns 400 with an error description diff --git a/openspec/changes/transcribe-video-fase-1/specs/editor-chat-panel/spec.md b/openspec/changes/transcribe-video-fase-1/specs/editor-chat-panel/spec.md new file mode 100644 index 00000000..1ebc2c5b --- /dev/null +++ b/openspec/changes/transcribe-video-fase-1/specs/editor-chat-panel/spec.md @@ -0,0 +1,34 @@ +# Delta for Editor Chat Panel + +## MODIFIED Requirements + +### Requirement: Message Display + +The system SHALL render a scrollable `MessageList` displaying all messages in `chatStore.messages` in chronological order. Each message MUST show its role (`user`, `assistant`, or `tool_result`) and content. When a `tool_result` message contains a JSON-parseable transcript result (has `fullText` and `segments` keys), the system MUST render it as a transcript card showing the asset name, language, duration, full text, and timestamped segments formatted as `[MM:SS] text`. For all other messages, content MUST be rendered as plain text. The list MUST auto-scroll to the latest message when a new message arrives. + +(Previously: All messages rendered as plain text with role labels only.) + +#### Scenario: Messages render in order + +- GIVEN `chatStore.messages` contains three messages: user, assistant, user +- WHEN the ChatPanel renders +- THEN all three messages appear in chronological order with correct role labels +- AND the list is scrolled to the bottom + +#### Scenario: Empty state + +- GIVEN `chatStore.messages` is empty +- WHEN the ChatPanel renders +- THEN a placeholder prompt is shown (e.g., "Ask something about your project") + +#### Scenario: Transcript tool result renders as card + +- GIVEN `chatStore.messages` contains a `tool_result` with content `JSON.stringify({ assetName: "clip.mp4", fullText: "Hello world", segments: [{ text: "Hello world", start: 0, end: 2.5 }], language: "en", duration: 30 })` +- WHEN the ChatPanel renders +- THEN the message renders as a transcript card showing "clip.mp4", "en", duration, full text, and `[00:00] Hello world` + +#### Scenario: Non-transcript tool result renders as plain text + +- GIVEN `chatStore.messages` contains a `tool_result` with content `JSON.stringify({ projectId: "p1", mediaCount: 2 })` +- WHEN the ChatPanel renders +- THEN the message renders as plain text (no transcript card) diff --git a/openspec/changes/transcribe-video-fase-1/specs/transcribe-video/spec.md b/openspec/changes/transcribe-video-fase-1/specs/transcribe-video/spec.md new file mode 100644 index 00000000..c3938586 --- /dev/null +++ b/openspec/changes/transcribe-video-fase-1/specs/transcribe-video/spec.md @@ -0,0 +1,56 @@ +# Transcribe Video Specification + +## Purpose + +End-to-end chat-triggered transcription of one media asset. Decodes the asset's audio, runs Whisper, and returns full text plus timestamped segments. + +## Requirements + +### Requirement: Transcription Tool Execution + +The system SHALL register a `transcribe_video` tool in the tool registry with parameters `{ assetId?: string }`. When executed, the tool MUST resolve the target asset via `EditorContextAdapter.resolveAssetFile(assetId)`, decode its audio to Float32Array using `decodeAudioToFloat32`, and call `transcriptionService.transcribe()` with the decoded audio. The tool MUST return `{ assetName, language, fullText, segments: Array<{ text, start, end }>, duration }`. + +#### Scenario: Successful transcription of a video asset + +- GIVEN the active/selected asset is a video with an audible audio track +- WHEN `transcribe_video` executes with no explicit `assetId` +- THEN it decodes the asset audio, calls `transcriptionService.transcribe()`, and returns `{ assetName, language, fullText, segments, duration }` +- AND `fullText` is a non-empty string and `segments` contains at least one entry + +#### Scenario: Transcription with explicit asset ID + +- GIVEN the context has multiple media assets and `assetId` matches one +- WHEN `transcribe_video` executes with `assetId: "v2"` +- THEN it resolves and transcribes that specific asset regardless of active selection + +### Requirement: Asset Audio Validation + +The tool MUST return an error result when the target asset has no audio track. The tool MUST return an error result when no asset can be resolved (no active/selected asset and no explicit `assetId`). + +#### Scenario: Asset with no audio track + +- GIVEN the resolved asset is a video file with no audio track +- WHEN `transcribe_video` executes +- THEN the tool returns a `ToolResult` with `error: "Asset has no audio track"` + +#### Scenario: Image asset selected + +- GIVEN the resolved asset has `type: "image"` +- WHEN `transcribe_video` executes +- THEN the tool returns a `ToolResult` with `error: "Asset has no audio track"` + +#### Scenario: No asset resolved + +- GIVEN no asset is active/selected and no `assetId` is provided +- WHEN `transcribe_video` executes +- THEN the tool returns a `ToolResult` with `error: "No active media asset"` + +### Requirement: Transcription Error Handling + +If `transcriptionService.transcribe()` rejects, the tool MUST catch the error and return a `ToolResult` with the error message. The tool MUST NOT throw unhandled exceptions. + +#### Scenario: Transcription service fails + +- GIVEN the audio decode succeeds but `transcriptionService.transcribe()` throws "Worker not initialized" +- WHEN `transcribe_video` executes +- THEN the tool returns a `ToolResult` with `error` containing "Worker not initialized" diff --git a/openspec/changes/transcribe-video-fase-1/tasks.md b/openspec/changes/transcribe-video-fase-1/tasks.md new file mode 100644 index 00000000..e53778ae --- /dev/null +++ b/openspec/changes/transcribe-video-fase-1/tasks.md @@ -0,0 +1,31 @@ +# Tasks: `transcribe_video` Phase 1 + +## Phase 1: Foundation + +- [x] 1.1 Add `resolveAssetFile(assetId?: string): File | null` to `EditorContextAdapter` in `apps/web/src/agent/context.ts` — uses `EditorCore.media.getAssets()` to find by ID or first video/audio, returns `null` on no match +- [x] 1.2 Add `TranscriptionToolResult` and `TranscribeVideoArgs` interfaces at top of new file `apps/web/src/agent/tools/transcribe-video.tool.ts` + +## Phase 2: Core Tool + +- [x] 2.1 Implement `transcribe_video` tool in `apps/web/src/agent/tools/transcribe-video.tool.ts` — validate `context.mediaAssets`, call `resolveAssetFile`, guard on `hasAudio`/type/ambiguity, then `decodeAudioToFloat32` → `transcriptionService.transcribe` → return `TranscriptionToolResult` +- [x] 2.2 Register tool via side-effect `toolRegistry.register("transcribe_video", …)` at module level (same pattern as `mock.tool.ts`) + +## Phase 3: Wiring + +- [x] 3.1 Add `import "@/agent/tools/transcribe-video.tool"` to `apps/web/src/agent/orchestrator.ts` (side-effect registration) +- [x] 3.2 Update `apps/web/src/app/api/agent/chat/route.ts` — if last user message contains "transcri" (case-insensitive), return `transcribe_video` tool call; else keep `echo_context` fallback +- [x] 3.3 Update `apps/web/src/components/editor/panels/chat/message-bubble.tsx` — for `tool_result` role, try `JSON.parse(content)` and if it has `fullText` + `segments`, render transcript card with asset name, language, duration, and `[MM:SS] text` segments; else render plain text + +## Phase 4: Testing + +- [x] 4.1 Unit test `resolveAssetFile()` in `apps/web/src/agent/__tests__/context.test.ts` — mock `EditorCore.media.getAssets()` for: match by ID, fallback to first video, no assets → null +- [x] 4.2 Unit test asset selection + error branches in `apps/web/src/agent/tools/__tests__/transcribe-video.test.ts` — mock `decodeAudioToFloat32` + `transcriptionService`; cover: success path, no audio, no asset, multiple candidates, service failure +- [x] 4.3 Contract test `toolRegistry.has("transcribe_video")` in same test file — verifies registration side-effect +- [x] 4.4 Unit test API route in `apps/web/src/app/api/agent/chat/__tests__/route.test.ts` — "transcribe this" → returns `transcribe_video` call; other message → returns `echo_context` call + +## Phase 5: Post-Verify Fixes + +- [x] 5.1 Fix TypeScript errors: `jest.fn` type references in test files replaced with explicit `MockFn`/`MockFnFactory` types; `TranscribeVideoArgs.language` typed as `TranscriptionLanguage` instead of `string` +- [x] 5.2 Align tool error messages with spec: `"No active media asset"` for no-asset, `"Asset has no audio track"` for image-only and no-audio-track cases +- [x] 5.3 Wrap decode+transcribe in try/catch — service failures now return `{ error: message }` instead of throwing +- [x] 5.4 Extract `isTranscriptData` + `formatTimestamp` to `transcript-utils.ts`; add 17 behavioral tests proving transcript detection and timestamp formatting diff --git a/openspec/changes/transcribe-video-fase-1/verify-report.md b/openspec/changes/transcribe-video-fase-1/verify-report.md new file mode 100644 index 00000000..572c99e8 --- /dev/null +++ b/openspec/changes/transcribe-video-fase-1/verify-report.md @@ -0,0 +1,123 @@ +# Verification Report + +**Change**: transcribe-video-fase-1 +**Mode**: Standard +**TDD Resolution**: Strict TDD config exists in `openspec/config.yaml`, but this verification used Standard mode because the orchestrator explicitly disabled strict TDD for this workflow. + +--- + +## Completeness + +| Metric | Value | +|--------|-------| +| Tasks total | 14 | +| Tasks complete | 14 | +| Tasks incomplete | 0 | + +All listed tasks in `openspec/changes/transcribe-video-fase-1/tasks.md` are marked complete, including the post-verify fix batch. + +--- + +## Build & Tests Execution + +**Type check**: ✅ Passed +Command: `bunx tsc --noEmit` (workdir: `apps/web`) + +**Targeted verification tests**: ✅ 63 passed / ❌ 0 failed / ⚠️ 0 skipped +Command: + +`bun test apps/web/src/agent/__tests__/context.test.ts apps/web/src/agent/tools/__tests__/mock-tool.test.ts apps/web/src/agent/tools/__tests__/transcribe-video.test.ts apps/web/src/app/api/agent/chat/__tests__/route.test.ts apps/web/src/components/editor/panels/chat/__tests__/transcript-utils.test.ts apps/web/src/stores/__tests__/chat-panel-behavior.test.ts` + +Result: `63 pass, 0 fail` + +**Environment warning suite**: ⚠️ Unrelated failure observed +Command: `bun test apps/web/src/agent/__tests__/orchestrator.test.ts` + +Observed failure: + +`TypeError: wasm.__wbindgen_start is not a function` from `opencut-wasm/opencut_wasm.js` + +This prevents orchestrator runtime verification in the current Bun/WASM environment, but it appears pre-existing and unrelated to this slice's direct implementation. + +**Coverage**: ➖ Not available + +--- + +## Spec Compliance Matrix + +| Requirement | Scenario | Test | Result | +|-------------|----------|------|--------| +| Active Asset File Resolver | Resolve by explicit asset ID | `apps/web/src/agent/__tests__/context.test.ts > returns File when assetId matches an asset` | ✅ COMPLIANT | +| Active Asset File Resolver | Resolve active asset without ID | `apps/web/src/agent/__tests__/context.test.ts > returns File for first video asset when no assetId provided` | ✅ COMPLIANT | +| Active Asset File Resolver | No matching asset | `apps/web/src/agent/__tests__/context.test.ts > returns null when assets array is empty` | ✅ COMPLIANT | +| Active Asset File Resolver | Asset ID not found | `apps/web/src/agent/__tests__/context.test.ts > returns null when assetId does not match any asset` | ✅ COMPLIANT | +| Transcription Tool Execution | Successful transcription of a video asset | `apps/web/src/agent/tools/__tests__/transcribe-video.test.ts > transcribes a single video asset successfully` | ✅ COMPLIANT | +| Transcription Tool Execution | Transcription with explicit asset ID | `apps/web/src/agent/tools/__tests__/transcribe-video.test.ts > transcribes with explicit assetId` | ✅ COMPLIANT | +| Asset Audio Validation | Asset with no audio track | `apps/web/src/agent/tools/__tests__/transcribe-video.test.ts > returns error when asset has no audio track` | ✅ COMPLIANT | +| Asset Audio Validation | Image asset selected | `apps/web/src/agent/tools/__tests__/transcribe-video.test.ts > returns error when only image assets found (no audio track)` | ✅ COMPLIANT | +| Asset Audio Validation | No asset resolved | `apps/web/src/agent/tools/__tests__/transcribe-video.test.ts > returns error when no media assets in project` | ✅ COMPLIANT | +| Transcription Error Handling | Transcription service fails | `apps/web/src/agent/tools/__tests__/transcribe-video.test.ts > returns error when transcriptionService.transcribe rejects` | ✅ COMPLIANT | +| Tool Registry Shell | Mock echo tool executes | `apps/web/src/agent/tools/__tests__/mock-tool.test.ts > returns context summary with media assets` | ✅ COMPLIANT | +| Tool Registry Shell | Transcribe tool is registered | `apps/web/src/agent/tools/__tests__/transcribe-video.test.ts > is registered in the tool registry` | ✅ COMPLIANT | +| API Proxy Route | Valid request returns mock response | `apps/web/src/app/api/agent/chat/__tests__/route.test.ts > returns mock response for valid input` | ✅ COMPLIANT | +| API Proxy Route | Transcription intent triggers transcribe tool | `apps/web/src/app/api/agent/chat/__tests__/route.test.ts > returns transcribe_video tool call when message contains 'transcribe'` | ✅ COMPLIANT | +| API Proxy Route | Invalid request returns 400 | `apps/web/src/app/api/agent/chat/__tests__/route.test.ts > returns 400 for invalid input — missing messages` | ✅ COMPLIANT | +| Message Display | Messages render in order | `apps/web/src/stores/__tests__/chat-panel-behavior.test.ts > messages render in order — chatStore preserves chronological order` | ⚠️ PARTIAL | +| Message Display | Empty state | `apps/web/src/stores/__tests__/chat-panel-behavior.test.ts > empty state — messages array is empty by default` | ⚠️ PARTIAL | +| Message Display | Transcript tool result renders as card | `apps/web/src/components/editor/panels/chat/__tests__/transcript-utils.test.ts > detects transcript from JSON.parse of tool_result content` | ⚠️ PARTIAL | +| Message Display | Non-transcript tool result renders as plain text | `apps/web/src/components/editor/panels/chat/__tests__/transcript-utils.test.ts > does not detect error results as transcript` | ⚠️ PARTIAL | + +**Compliance summary**: 15/19 scenarios compliant, 4/19 partial, 0 failing, 0 fully untested + +--- + +## Correctness (Static — Structural Evidence) + +| Requirement | Status | Notes | +|------------|--------|-------| +| Active Asset File Resolver | ✅ Implemented | `EditorContextAdapter.resolveAssetFile()` exists and remains the sanctioned file access path from agent code. | +| Transcription Tool Execution | ⚠️ Partial | Tool is implemented, registered, typed, and uses `resolveAssetFile`, but default no-`assetId` selection errors on multiple candidates instead of following the delta spec's “first video/audio asset” behavior. | +| Asset Audio Validation | ✅ Implemented | No-media, image-only, missing-file, and no-audio cases return structured errors. | +| Transcription Error Handling | ✅ Implemented | Decode/transcribe are wrapped in `try/catch`; service rejection returns `{ error }`. | +| Tool Registry Shell | ✅ Implemented | Registry contains `echo_context` and `transcribe_video`; side-effect registration is wired in orchestrator. | +| API Proxy Route | ✅ Implemented | Zod validation, mock response, transcription intent routing, and fallback remain present. | +| Message Display | ⚠️ Partial | `MessageBubble` contains transcript formatting branch and `ChatPanel` still handles order/empty state/auto-scroll, but there is no direct runtime UI test proving the rendered transcript card behavior. | + +--- + +## Coherence (Design) + +| Decision | Followed? | Notes | +|----------|-----------|-------| +| Asset file access through `EditorContextAdapter` | ✅ Yes | Tool imports `EditorContextAdapter`; no direct `core/` access from the tool. | +| Tool registration via side-effect import | ✅ Yes | `orchestrator.ts` imports `@/agent/tools/transcribe-video.tool`. | +| Asset selection errors when ambiguous | ✅ Yes | Implementation matches design, but this conflicts with the current delta spec text. | +| Rich `TranscriptionToolResult` shape | ✅ Yes | Tool returns `assetName`, `language`, `fullText`, `segmentCount`, `segments`, `duration`. | +| Mock API keyword gate on `transcri` | ✅ Yes | Route uses case-insensitive `includes("transcri")`. | +| Chat rendering inside existing `MessageBubble` | ✅ Yes | Transcript detection and timestamp formatting stay inside the existing component flow. | + +--- + +## Issues Found + +### CRITICAL + +1. **Spec/implementation mismatch on default asset selection** — the written delta spec says `resolveAssetFile()` without `assetId` MUST resolve the first video/audio asset, but `transcribe_video` currently returns an ambiguity error when multiple candidates exist. Design/tasks/tests were updated around ambiguity handling, but the spec was not. This blocks archive until spec and implementation are reconciled. + +### WARNING + +1. **Editor chat panel scenarios are only partially proven behaviorally** — current evidence is store-level and utility-level, not direct rendered component verification. Transcript card rendering, plain-text fallback rendering, placeholder copy, and auto-scroll are not proven by passing UI-focused tests. +2. **Orchestrator verification is blocked by a pre-existing WASM environment issue** — `apps/web/src/agent/__tests__/orchestrator.test.ts` crashes during WASM module startup with `wasm.__wbindgen_start is not a function`. Per instruction, this is treated as a warning because it does not directly invalidate this slice. + +### SUGGESTION + +1. Add a focused `MessageBubble`/`ChatPanel` render test suite so the four modified UI scenarios move from PARTIAL to COMPLIANT. +2. Reconcile the transcribe-tool selection rule in `openspec` artifacts so design, tasks, tests, and spec all describe the same behavior. + +--- + +## Verdict + +**FAIL** + +The implementation is mostly in place and the targeted non-WASM verification suite passes, but the change is not archive-ready because the shipped behavior contradicts the current delta spec on default asset selection.