ft: transcription tool exposed to the agent using deepgram servie

apps/web/src/agent/system-prompt.ts

@@ -58,7 +58,7 @@ Do NOT perform write tools for complex edits before approval. This is required e
 ## EXECUTION RULES:
 
 1. Follow the plan steps in order. Each step lists the tools to use.
-2. After completing a step, call update_plan_step to mark it done.
+2. After completing a step, call update_plan_step with the step number (1, 2, 3...) and status 'done' to mark it complete.
 3. If a step fails, note the error in update_plan_step and decide whether to continue or skip.
 4. If you discover the plan needs adjustment mid-execution, explain what changed and why.
 5. When all steps are done, provide a summary of what was accomplished.
@@ -102,6 +102,7 @@ export function buildSystemPrompt(
 			`Available tools:\n${toolList}`,
 			"For questions about what is visible or audible in a media asset, never answer that you cannot see or hear the media if load_context is available. First infer the asset from the active assets or timeline; if needed call list_project_assets or list_timeline, then call load_context with the discovered internal id or timeline element ids.",
 			"If load_context has loaded media and the conversation contains an attached fileData part, treat it as the actual video/audio/image content. You may answer visual and audio questions directly from that loaded media without calling extraction tools unless the user asks for a separate extraction workflow.",
+			"When exact speech, quotes, word timing, subtitle timing, or audio-driven edit points matter, call transcribe_audio. Use load_context for broad multimodal understanding and transcribe_audio for precise spoken-word timing.",
 			"Only claim edits that were actually performed by tool calls in this conversation. Do not say you added, removed, cleaned, or updated text/subtitles unless the relevant add_text, update_text, delete_timeline_elements, or update_timeline_element_timing tool call succeeded.",
 			"When the user asks to add titles, hooks, labels, captions, subtitles, or visible text, call add_text. Do not add text proactively for unrelated edit requests such as cutting silence unless the user asks for text.",
 			"When you need to perform an action matching one of these tools, call the appropriate tool. For all other requests, respond directly in plain text.",

apps/web/src/agent/tools/__tests__/transcribe-audio.test.ts

@@ -0,0 +1,189 @@
+import { beforeEach, describe, expect, jest, mock, test } from "bun:test";
+import type { AgentContext } from "@/agent/types";
+import { providerToolSchemas } from "@/agent/tools/schemas";
+import { toolRegistry } from "@/agent/tools/registry";
+
+const mockResolveAssetFile = jest.fn<(assetId?: string) => File | null>();
+const mockGetAssetHasAudio =
+	jest.fn<(assetId: string) => boolean | undefined>();
+const mockExtractAssetAudio =
+	jest.fn<(args: { file: File; assetType: string }) => Promise<Blob>>();
+const mockDecodeAudioToFloat32 =
+	jest.fn<
+		(args: {
+			audioBlob: Blob;
+		}) => Promise<{ samples: Float32Array; sampleRate: number }>
+	>();
+const mockFetch = jest.fn<typeof fetch>();
+
+mock.module("@/agent/context", () => ({
+	EditorContextAdapter: {
+		resolveAssetFile: mockResolveAssetFile,
+		getAssetHasAudio: mockGetAssetHasAudio,
+	},
+}));
+
+mock.module("@/lib/media/mediabunny", () => ({
+	extractAssetAudio: mockExtractAssetAudio,
+}));
+
+mock.module("@/lib/media/audio", () => ({
+	decodeAudioToFloat32: mockDecodeAudioToFloat32,
+}));
+
+await import("@/agent/tools/transcribe-audio.tool");
+
+function makeContext(overrides: Partial<AgentContext> = {}): AgentContext {
+	return {
+		projectId: "proj-1",
+		activeSceneId: "scene-A",
+		mediaAssets: [],
+		playbackTimeMs: 0,
+		...overrides,
+	};
+}
+
+function fakeVideoAsset(
+	overrides: Partial<{
+		id: string;
+		name: string;
+		type: string;
+		duration: number;
+	}> = {},
+) {
+	return {
+		id: "v1",
+		name: "clip.mp4",
+		type: "video",
+		duration: 30,
+		...overrides,
+	};
+}
+
+function fakeTranscriptResult() {
+	return {
+		transcriptId: "tr_deepgram_123",
+		assetId: "v1",
+		assetName: "clip.mp4",
+		provider: "deepgram",
+		language: "es",
+		model: "nova-3",
+		fullText: "Hola mundo.",
+		duration: 30,
+		confidence: 0.98,
+		timingGranularity: "word",
+		wordCount: 2,
+		utteranceCount: 1,
+		words: [
+			{ text: "Hola", start: 1, end: 1.3, confidence: 0.99 },
+			{ text: "mundo.", start: 1.3, end: 1.8, confidence: 0.97 },
+		],
+		utterances: [
+			{
+				text: "Hola mundo.",
+				start: 1,
+				end: 1.8,
+				confidence: 0.98,
+				words: [
+					{ text: "Hola", start: 1, end: 1.3, confidence: 0.99 },
+					{ text: "mundo.", start: 1.3, end: 1.8, confidence: 0.97 },
+				],
+			},
+		],
+		segments: [{ text: "Hola mundo.", start: 1, end: 1.8 }],
+	};
+}
+
+describe("transcribe_audio tool", () => {
+	beforeEach(() => {
+		mockResolveAssetFile.mockClear();
+		mockGetAssetHasAudio.mockClear();
+		mockExtractAssetAudio.mockClear();
+		mockDecodeAudioToFloat32.mockClear();
+		mockFetch.mockClear();
+		mockDecodeAudioToFloat32.mockResolvedValue({
+			samples: new Float32Array([0, 0.25, -0.5]),
+			sampleRate: 3,
+		});
+		globalThis.fetch = mockFetch;
+	});
+
+	test("is registered and exposed to the provider without provider/model args", () => {
+		expect(toolRegistry.has("transcribe_audio")).toBe(true);
+		const schema = providerToolSchemas.find(
+			(tool) => tool.name === "transcribe_audio",
+		);
+		expect(schema).toBeDefined();
+		expect(schema?.parameters.map((param) => param.key)).toEqual([
+			"assetId",
+			"language",
+		]);
+	});
+
+	test("transcribes a single media asset through the Deepgram route", async () => {
+		const file = new File([], "clip.mp4", { type: "video/mp4" });
+		const audioBlob = new Blob([new Uint8Array([1, 2, 3])], {
+			type: "audio/wav",
+		});
+		mockResolveAssetFile.mockReturnValue(file);
+		mockGetAssetHasAudio.mockReturnValue(true);
+		mockExtractAssetAudio.mockResolvedValue(audioBlob);
+		mockFetch.mockResolvedValue(
+			new Response(JSON.stringify(fakeTranscriptResult()), { status: 200 }),
+		);
+
+		const tool = toolRegistry.get("transcribe_audio");
+		const result = await tool.execute(
+			{},
+			makeContext({ mediaAssets: [fakeVideoAsset()] }),
+		);
+
+		expect(result).toEqual(fakeTranscriptResult());
+		expect(mockExtractAssetAudio).toHaveBeenCalledWith({
+			file,
+			assetType: "video",
+		});
+		expect(mockFetch).toHaveBeenCalledWith(
+			"/api/transcription/deepgram",
+			expect.objectContaining({ method: "POST" }),
+		);
+	});
+
+	test("returns Deepgram route errors without local fallback", async () => {
+		const file = new File([], "clip.mp4", { type: "video/mp4" });
+		mockResolveAssetFile.mockReturnValue(file);
+		mockGetAssetHasAudio.mockReturnValue(true);
+		mockExtractAssetAudio.mockResolvedValue(
+			new Blob([], { type: "audio/wav" }),
+		);
+		mockFetch.mockResolvedValue(
+			new Response(JSON.stringify({ error: "missing key" }), { status: 501 }),
+		);
+
+		const tool = toolRegistry.get("transcribe_audio");
+		const result = await tool.execute(
+			{},
+			makeContext({ mediaAssets: [fakeVideoAsset()] }),
+		);
+
+		expect(result).toEqual({ error: "missing key" });
+	});
+
+	test("returns extraction errors before calling Deepgram", async () => {
+		const file = new File([], "clip.mov", { type: "video/quicktime" });
+		mockResolveAssetFile.mockReturnValue(file);
+		mockGetAssetHasAudio.mockReturnValue(true);
+		mockExtractAssetAudio.mockRejectedValue(
+			new Error("Could not extract audio"),
+		);
+
+		const tool = toolRegistry.get("transcribe_audio");
+		const result = await tool.execute(
+			{},
+			makeContext({ mediaAssets: [fakeVideoAsset({ name: "clip.mov" })] }),
+		);
+
+		expect(result).toEqual({ error: "Could not extract audio" });
+		expect(mockFetch).not.toHaveBeenCalled();
+	});
+});

apps/web/src/agent/tools/index.ts

@@ -11,6 +11,7 @@
  */
 
 import "@/agent/tools/load-context.tool";
+import "@/agent/tools/transcribe-audio.tool";
 import "@/agent/tools/list-project-assets.tool";
 import "@/agent/tools/list-timeline.tool";
 import "@/agent/tools/get-element.tool";

apps/web/src/agent/tools/load-context.tool.ts

@@ -147,23 +147,28 @@ async function loadTimelineElementContext(
 	args: LoadContextArgs,
 	context: AgentContext,
 ): Promise<LoadContextResult | { error: string }> {
-	const trackId = args.trackId;
 	const elementId = args.elementId ?? args.id;
 
-	if (!trackId || !elementId) {
+	if (!elementId) {
-		return { error: "trackId and elementId are required" };
+		return { error: "elementId (or id) is required for timeline_element target" };
 	}
 
-	const element = findTimelineElement(
+	let resolved: ReturnType<typeof findTimelineElement> = null;
-		context.timelineTracks,
+
-		trackId,
+	if (args.trackId) {
-		elementId,
+		resolved = findTimelineElement(context.timelineTracks, args.trackId, elementId);
-	);
+	} else {
-	if (!element) {
+		resolved = findTimelineElementAcrossTracks(context.timelineTracks, elementId);
+	}
+
+	if (!resolved) {
 		return { error: "Timeline element not found" };
 	}
 
-	if (element.element.content !== undefined) {
+	const { track, element } = resolved;
+	const trackId = track.trackId;
+
+	if (element.content !== undefined) {
 		const cacheKey = `${trackId}:${elementId}`;
 		const cached = textContextCache.get(cacheKey);
 		if (cached) {
@@ -181,11 +186,11 @@ async function loadTimelineElementContext(
 			kind: "text",
 			trackId,
 			elementId,
-			type: element.element.type,
+			type: element.type,
-			...(element.element.name ? { name: element.element.name } : {}),
+			...(element.name ? { name: element.name } : {}),
-			content: element.element.content,
+			content: element.content,
-			start: element.element.start,
+			start: element.start,
-			end: element.element.end,
+			end: element.end,
 		};
 
 		textContextCache.set(cacheKey, textContext);
@@ -200,17 +205,17 @@ async function loadTimelineElementContext(
 		};
 	}
 
-	if (element.element.assetId) {
+	if (element.assetId) {
 		return loadAssetContext(
-			{ targetType: "asset", assetId: element.element.assetId },
+			{ targetType: "asset", assetId: element.assetId },
 			context,
 			{
-				assetId: element.element.assetId,
+				assetId: element.assetId,
 				trackId,
 				elementId,
-				type: element.element.type,
+				type: element.type,
-				start: element.element.start,
+				start: element.start,
-				end: element.element.end,
+				end: element.end,
 			},
 		);
 	}
@@ -367,6 +372,23 @@ function findTimelineElement(
 	return track && element ? { track, element } : null;
 }
 
+function findTimelineElementAcrossTracks(
+	tracks: AgentTimelineTrack[] | undefined,
+	elementId: string,
+): {
+	track: AgentTimelineTrack;
+	element: AgentTimelineTrack["elements"][number];
+} | null {
+	if (!tracks) return null;
+	for (const track of tracks) {
+		const element = track.elements.find(
+			(candidate) => candidate.elementId === elementId,
+		);
+		if (element) return { track, element };
+	}
+	return null;
+}
+
 function isSupportedMediaAssetType(type: string): boolean {
 	return type === "video" || type === "audio" || type === "image";
 }

apps/web/src/agent/tools/schemas.ts

@@ -14,7 +14,7 @@ import type { ToolSchema } from "@/agent/types";
 export const loadContextSchema: ToolSchema = {
 	name: "load_context",
 	description:
-		"Loads the actual Gemini multimodal context for a project asset or timeline element. Use this before answering questions about visible objects, colors, scenes, speech, silence, or timestamps in media. Use targetType='asset' with assetId/id for media, or targetType='timeline_element' with trackId and elementId/id for captions, text, or timeline media elements.",
+		"Loads Gemini multimodal context for a project asset or timeline element. For media: targetType='asset' with assetId. For timeline elements (text, clips): targetType='timeline_element' with elementId — trackId is optional, the element will be found across all tracks automatically.",
 	parameters: [
 		{ key: "targetType", type: "string", required: true },
 		{ key: "id", type: "string", required: false },
@@ -24,6 +24,16 @@ export const loadContextSchema: ToolSchema = {
 	],
 };
 
+export const transcribeAudioSchema: ToolSchema = {
+	name: "transcribe_audio",
+	description:
+		"Transcribes a video or audio asset with precise word-level timing. Returns full text, per-word start/end times, confidence scores, and utterance segments. Use this when you need exact spoken words, timestamps for subtitles, or silence detection. Pass assetId to target a specific asset, or omit to auto-select the first video/audio asset.",
+	parameters: [
+		{ key: "assetId", type: "string", required: false },
+		{ key: "language", type: "string", required: false },
+	],
+};
+
 export const listProjectAssetsSchema: ToolSchema = {
 	name: "list_project_assets",
 	description:
@@ -93,7 +103,7 @@ export const updateTimelineElementTimingSchema: ToolSchema = {
 export const addTextSchema: ToolSchema = {
 	name: "add_text",
 	description:
-		"Adds visual text to the active timeline. Use for titles, hooks, labels, and simple subtitle blocks. start and end are timeline seconds. position must be top, center, or bottom; style may be plain, subtitle, hook, or label. Override style defaults with color (hex), fontSize (scaled units), fontFamily, fontWeight (normal/bold), fontStyle (normal/italic), textAlign (left/center/right), letterSpacing, positionX/positionY (offset from -50 to 50), or background ({ enabled, color?, cornerRadius?, padding? }).",
+		"Adds visual text to the active timeline. start and end are seconds. position must be exactly one of: 'top', 'center', or 'bottom'. style: 'plain', 'subtitle' (white on black bg), 'hook' (large bold), or 'label'. Override defaults with color (hex), fontSize, fontFamily, fontWeight, fontStyle, textAlign, letterSpacing, positionX/positionY (-50 to 50), or background.",
 	parameters: [
 		{ key: "text", type: "string", required: true },
 		{ key: "start", type: "number", required: true },
@@ -116,7 +126,7 @@ export const addTextSchema: ToolSchema = {
 export const updateTextSchema: ToolSchema = {
 	name: "update_text",
 	description:
-		"Updates visual properties of existing text elements. Pass elementIds (from list_timeline) and any combination of text style overrides. Only text elements are updated; non-text elements are skipped. All listed text elements receive the same overrides — use for bulk styling subtitle blocks.",
+		"Updates visual properties of existing text elements. Pass elementIds (from list_timeline) and any properties to change. All listed elements receive the same overrides — use for bulk styling. Non-text elements in the list are skipped.",
 	parameters: [
 		// "array" (not "string[]") so the orchestrator validator stays
 		// tolerant of CSV-string fallback handled by resolveElementIds.
@@ -308,7 +318,7 @@ export const loadSkillSchema: ToolSchema = {
 export const submitPlanSchema: ToolSchema = {
 	name: "submit_plan",
 	description:
-		"Submits a structured editing plan with steps and waits for user approval through a plan card with Go edit and Keep planning actions. Use this before complex edits, even from execute mode. Each step should describe a specific action with the tools to use. If approved=true is returned, continue executing the plan. If approved=false is returned, keep refining the plan with the user.",
+		"Submits a structured editing plan with numbered steps for user approval. Returns step IDs — use update_plan_step with the step number (1, 2, 3...) to track progress. Each step needs a description and tools array.",
 	parameters: [
 		{ key: "summary", type: "string", required: true },
 		{ key: "steps", type: "array", required: true },
@@ -336,9 +346,10 @@ export const requestPlanApprovalSchema: ToolSchema = {
 export const updatePlanStepSchema: ToolSchema = {
 	name: "update_plan_step",
 	description:
-		"Updates the status of a plan step during execution. Call this after completing each step to track progress. The plan panel updates in real-time. Use status 'done' for successful completion, 'skipped' if the step was unnecessary, or 'in_progress' if starting.",
+		"Updates the status of a plan step during execution. Prefer using step (1-based number matching plan order) over stepId. The plan panel updates in real-time. Use status 'in_progress' when starting, 'done' for successful completion, or 'skipped' if unnecessary.",
 	parameters: [
-		{ key: "stepId", type: "string", required: true },
+		{ key: "step", type: "number", required: false },
+		{ key: "stepId", type: "string", required: false },
 		{ key: "status", type: "string", required: true },
 		{ key: "result", type: "string", required: false },
 	],
@@ -350,6 +361,7 @@ export const updatePlanStepSchema: ToolSchema = {
  */
 export const providerToolSchemas: ToolSchema[] = [
 	loadContextSchema,
+	transcribeAudioSchema,
 	listProjectAssetsSchema,
 	listTimelineSchema,
 	getElementSchema,

apps/web/src/agent/tools/transcribe-audio.tool.ts

@@ -0,0 +1,240 @@
+import type { AgentContext, ToolDefinition } from "@/agent/types";
+import { EditorContextAdapter } from "@/agent/context";
+import { toolRegistry } from "@/agent/tools/registry";
+import { transcribeAudioSchema } from "@/agent/tools/schemas";
+import { decodeAudioToFloat32 } from "@/lib/media/audio";
+import { extractAssetAudio } from "@/lib/media/mediabunny";
+import type { TranscriptionLanguage } from "@/lib/transcription/types";
+
+type TimingGranularity = "word";
+
+type TranscriptWord = {
+	text: string;
+	start: number;
+	end: number;
+	confidence?: number;
+};
+
+type TranscriptUtterance = {
+	text: string;
+	start: number;
+	end: number;
+	confidence?: number;
+	speaker?: number;
+	words: TranscriptWord[];
+};
+
+export type TranscribeAudioResult = {
+	transcriptId: string;
+	assetId: string;
+	assetName: string;
+	provider: "deepgram";
+	language: string;
+	model?: string;
+	fullText: string;
+	duration: number;
+	confidence?: number;
+	timingGranularity: TimingGranularity;
+	wordCount: number;
+	utteranceCount: number;
+	words: TranscriptWord[];
+	utterances: TranscriptUtterance[];
+	segments: Array<{ text: string; start: number; end: number }>;
+};
+
+type TranscribeAudioArgs = {
+	assetId?: string;
+	language?: TranscriptionLanguage;
+};
+
+const transcribeAudioTool: ToolDefinition = {
+	...transcribeAudioSchema,
+	execute: async (
+		args: Record<string, unknown>,
+		context: AgentContext,
+	): Promise<TranscribeAudioResult | { error: string }> => {
+		const typedArgs = args as TranscribeAudioArgs;
+		const target = resolveTargetAsset({
+			assetId: typedArgs.assetId,
+			context,
+		});
+
+		if ("error" in target) return target;
+
+		const file = EditorContextAdapter.resolveAssetFile(target.asset.id);
+		if (!file) {
+			return { error: `Could not access file for asset ${target.asset.id}` };
+		}
+
+		const hasAudio = EditorContextAdapter.getAssetHasAudio(target.asset.id);
+		if (hasAudio === false) {
+			return { error: "Asset has no audio track" };
+		}
+
+		try {
+			const audioBlob = await extractAssetAudio({
+				file,
+				assetType: target.asset.type,
+			});
+			const diagnostics = await analyzeAudioBlob({ audioBlob });
+
+			return transcribeWithDeepgram({
+				audioBlob,
+				diagnostics,
+				assetId: target.asset.id,
+				assetName: target.asset.name,
+				duration: target.asset.duration,
+				language: typedArgs.language,
+			});
+		} catch (error) {
+			return {
+				error:
+					error instanceof Error
+						? error.message
+						: "Could not extract audio from asset",
+			};
+		}
+	},
+};
+
+async function transcribeWithDeepgram({
+	audioBlob,
+	diagnostics,
+	assetId,
+	assetName,
+	duration,
+	language,
+}: {
+	audioBlob: Blob;
+	diagnostics: AudioDiagnostics;
+	assetId: string;
+	assetName: string;
+	duration: number;
+	language?: TranscriptionLanguage;
+}): Promise<TranscribeAudioResult | { error: string }> {
+	const formData = new FormData();
+	formData.set("file", audioBlob, `${assetName.replace(/\.[^.]+$/, "")}.wav`);
+	formData.set("assetId", assetId);
+	formData.set("assetName", assetName);
+	formData.set("duration", String(duration));
+	formData.set("audioSize", String(audioBlob.size));
+	formData.set("audioType", audioBlob.type || "unknown");
+	formData.set("audioPeak", String(diagnostics.peak));
+	formData.set("audioRms", String(diagnostics.rms));
+	formData.set("extractedDuration", String(diagnostics.duration));
+	if (language && language !== "auto") formData.set("language", language);
+
+	const response = await fetch("/api/transcription/deepgram", {
+		method: "POST",
+		body: formData,
+	});
+	const data = (await response.json()) as TranscribeAudioResult & {
+		error?: string;
+	};
+
+	if (!response.ok) {
+		return { error: data.error ?? "Deepgram transcription failed" };
+	}
+
+	return data;
+}
+
+type AudioDiagnostics = {
+	duration: number;
+	peak: number;
+	rms: number;
+};
+
+async function analyzeAudioBlob({
+	audioBlob,
+}: {
+	audioBlob: Blob;
+}): Promise<AudioDiagnostics> {
+	const { samples, sampleRate } = await decodeAudioToFloat32({ audioBlob });
+	let peak = 0;
+	let sumSquares = 0;
+
+	for (const sample of samples) {
+		const abs = Math.abs(sample);
+		if (abs > peak) peak = abs;
+		sumSquares += sample * sample;
+	}
+
+	return {
+		duration: samples.length / sampleRate,
+		peak,
+		rms: samples.length > 0 ? Math.sqrt(sumSquares / samples.length) : 0,
+	};
+}
+
+function resolveTargetAsset({
+	assetId,
+	context,
+}: {
+	assetId?: string;
+	context: AgentContext;
+}): { asset: AgentContext["mediaAssets"][number] } | { error: string } {
+	if (context.mediaAssets.length === 0) {
+		return { error: "No active media asset" };
+	}
+
+	const mediaCandidates = context.mediaAssets.filter(
+		(asset) => asset.type === "video" || asset.type === "audio",
+	);
+	if (mediaCandidates.length === 0) {
+		return { error: "Asset has no audio track" };
+	}
+
+	if (!assetId) {
+		if (mediaCandidates.length > 1) {
+			return {
+				error: `Multiple video/audio assets found: ${mediaCandidates.map((asset) => asset.name).join(", ")}. Specify which one with assetId.`,
+			};
+		}
+		return { asset: mediaCandidates[0] };
+	}
+
+	const target = resolveAssetByIdOrName({ assetId, mediaCandidates });
+	if ("error" in target) return target;
+	return { asset: target.asset };
+}
+
+function resolveAssetByIdOrName({
+	assetId,
+	mediaCandidates,
+}: {
+	assetId: string;
+	mediaCandidates: AgentContext["mediaAssets"];
+}): { asset: AgentContext["mediaAssets"][number] } | { error: string } {
+	const idMatch = mediaCandidates.find((asset) => asset.id === assetId);
+	if (idMatch) return { asset: idMatch };
+
+	const exactNameMatches = mediaCandidates.filter(
+		(asset) => asset.name === assetId,
+	);
+	if (exactNameMatches.length === 1) return { asset: exactNameMatches[0] };
+	if (exactNameMatches.length > 1) {
+		return {
+			error: `Ambiguous asset name "${assetId}" matches multiple assets (${exactNameMatches.map((asset) => asset.name).join(", ")}). Specify the internal id: ${exactNameMatches.map((asset) => asset.id).join(", ")}.`,
+		};
+	}
+
+	const lower = assetId.toLowerCase();
+	const caseInsensitiveMatches = mediaCandidates.filter(
+		(asset) => asset.name.toLowerCase() === lower,
+	);
+	if (caseInsensitiveMatches.length === 1) {
+		return { asset: caseInsensitiveMatches[0] };
+	}
+	if (caseInsensitiveMatches.length > 1) {
+		return {
+			error: `Ambiguous asset name "${assetId}" matches multiple assets (${caseInsensitiveMatches.map((asset) => asset.name).join(", ")}). Specify the internal id: ${caseInsensitiveMatches.map((asset) => asset.id).join(", ")}.`,
+		};
+	}
+
+	return {
+		error: `No asset found with id or name "${assetId}". Available ids: [${mediaCandidates.map((asset) => asset.id).join(", ")}]. Available names: [${mediaCandidates.map((asset) => asset.name).join(", ")}].`,
+	};
+}
+
+toolRegistry.register(transcribeAudioSchema.name, transcribeAudioTool);

apps/web/src/agent/tools/update-plan-step.tool.ts

@@ -16,18 +16,18 @@ const updatePlanStepTool: ToolDefinition = {
 	execute: async (
 		args: Record<string, unknown>,
 		_context: AgentContext,
-	):
+	): Promise<
-		| { success: boolean; stepId: string; status: string }
+		| { success: boolean; step: number; stepId: string; status: string }
-		| { error: string } => {
+		| { error: string }
+	> => {
 		const { stepId, status, result } = args as {
-			stepId: string;
+			step?: number;
+			stepId?: string;
 			status: string;
 			result?: string;
 		};
 
-		if (!stepId || typeof stepId !== "string") {
+		const stepNumber = args.step as number | undefined;
-			return { error: "stepId is required" };
-		}
 
 		if (!VALID_STATUSES.includes(status as PlanStepStatus)) {
 			return {
@@ -41,16 +41,37 @@ const updatePlanStepTool: ToolDefinition = {
 			return { error: "No active plan" };
 		}
 
-		const step = plan.steps.find((s) => s.id === stepId);
+		let resolvedStepId: string | undefined;
+
+		if (stepNumber !== undefined && stepNumber !== null) {
+			const index = Math.floor(stepNumber) - 1;
+			if (index < 0 || index >= plan.steps.length) {
+				return {
+					error: `Step ${stepNumber} out of range. Plan has ${plan.steps.length} steps (1-${plan.steps.length}).`,
+				};
+			}
+			resolvedStepId = plan.steps[index].id;
+		} else if (stepId && typeof stepId === "string") {
+			resolvedStepId = stepId;
+		} else {
+			return { error: "Pass step (1-based number) or stepId" };
+		}
+
+		const step = plan.steps.find((s) => s.id === resolvedStepId);
 		if (!step) {
 			return {
-				error: `Step not found: ${stepId}. Available: ${plan.steps.map((s) => s.id).join(", ")}`,
+				error: `Step not found. Use step number (1-${plan.steps.length}) instead.`,
 			};
 		}
 
-		planStore.updateStepStatus(stepId, status as PlanStepStatus, result);
+		planStore.updateStepStatus(resolvedStepId, status as PlanStepStatus, result);
 
-		return { success: true, stepId, status };
+		return {
+			success: true,
+			step: plan.steps.findIndex((s) => s.id === resolvedStepId) + 1,
+			stepId: resolvedStepId,
+			status,
+		};
 	},
 };
 

apps/web/src/app/api/transcription/deepgram/route.ts

@@ -0,0 +1,268 @@
+import { type NextRequest, NextResponse } from "next/server";
+
+type DeepgramWord = {
+	word?: string;
+	punctuated_word?: string;
+	start?: number;
+	end?: number;
+	confidence?: number;
+	speaker?: number;
+};
+
+type DeepgramUtterance = {
+	transcript?: string;
+	start?: number;
+	end?: number;
+	confidence?: number;
+	speaker?: number;
+	words?: DeepgramWord[];
+};
+
+type DeepgramResponse = {
+	metadata?: { request_id?: string };
+	results?: {
+		channels?: Array<{
+			alternatives?: Array<{
+				transcript?: string;
+				confidence?: number;
+				words?: DeepgramWord[];
+			}>;
+		}>;
+		utterances?: DeepgramUtterance[];
+	};
+};
+
+const DEEPGRAM_MODEL = "nova-3";
+
+export async function POST(request: NextRequest) {
+	const apiKey = process.env.DEEPGRAM_API_KEY;
+	if (!apiKey) {
+		return NextResponse.json(
+			{ error: "Server configuration error: missing DEEPGRAM_API_KEY" },
+			{ status: 501 },
+		);
+	}
+
+	const formData = await request.formData();
+	const file = formData.get("file");
+	const assetId = formData.get("assetId");
+	const assetName = formData.get("assetName");
+	const duration = formData.get("duration");
+	const language = formData.get("language");
+	const audioSize = formData.get("audioSize");
+	const audioType = formData.get("audioType");
+	const audioPeak = formData.get("audioPeak");
+	const audioRms = formData.get("audioRms");
+	const extractedDuration = formData.get("extractedDuration");
+
+	if (!(file instanceof File)) {
+		return NextResponse.json({ error: "File is required" }, { status: 400 });
+	}
+	if (typeof assetId !== "string" || typeof assetName !== "string") {
+		return NextResponse.json(
+			{ error: "assetId and assetName are required" },
+			{ status: 400 },
+		);
+	}
+
+	const url = new URL("https://api.deepgram.com/v1/listen");
+	url.searchParams.set("model", DEEPGRAM_MODEL);
+	url.searchParams.set("smart_format", "true");
+	url.searchParams.set("punctuate", "true");
+	url.searchParams.set("utterances", "true");
+	if (typeof language === "string" && language.length > 0) {
+		url.searchParams.set("language", language);
+	} else {
+		url.searchParams.set("detect_language", "true");
+	}
+
+	try {
+		const response = await fetch(url, {
+			method: "POST",
+			headers: {
+				Authorization: `Token ${apiKey}`,
+				"Content-Type": resolveContentType(file),
+			},
+			body: Buffer.from(await file.arrayBuffer()),
+		});
+		const data = (await response.json()) as DeepgramResponse & {
+			error?: string;
+		};
+
+		if (!response.ok) {
+			return NextResponse.json(
+				{ error: data.error ?? "Deepgram transcription failed" },
+				{ status: response.status },
+			);
+		}
+
+		const normalized = normalizeDeepgramResponse({
+			data,
+			assetId,
+			assetName,
+			duration: parseFiniteNumber(duration) ?? 0,
+			language: typeof language === "string" ? language : "auto",
+			model: DEEPGRAM_MODEL,
+		});
+
+		if (isEmptyTranscript(normalized)) {
+			const diagnostics = formatAudioDiagnostics({
+				audioSize,
+				audioType,
+				audioPeak,
+				audioRms,
+				extractedDuration,
+			});
+			return NextResponse.json(
+				{
+					error: `Deepgram returned no speech. ${diagnostics}`,
+				},
+				{ status: 422 },
+			);
+		}
+
+		return NextResponse.json(normalized);
+	} catch (error) {
+		console.error("[transcription/deepgram] Deepgram error:", error);
+		return NextResponse.json(
+			{ error: "Deepgram transcription failed" },
+			{ status: 502 },
+		);
+	}
+}
+
+function isEmptyTranscript(transcript: {
+	fullText: string;
+	wordCount: number;
+	utteranceCount: number;
+}): boolean {
+	return (
+		transcript.fullText.trim().length === 0 &&
+		transcript.wordCount === 0 &&
+		transcript.utteranceCount === 0
+	);
+}
+
+function formatAudioDiagnostics({
+	audioSize,
+	audioType,
+	audioPeak,
+	audioRms,
+	extractedDuration,
+}: {
+	audioSize: FormDataEntryValue | null;
+	audioType: FormDataEntryValue | null;
+	audioPeak: FormDataEntryValue | null;
+	audioRms: FormDataEntryValue | null;
+	extractedDuration: FormDataEntryValue | null;
+}): string {
+	const size = typeof audioSize === "string" ? audioSize : "unknown";
+	const type = typeof audioType === "string" ? audioType : "unknown";
+	const peak = typeof audioPeak === "string" ? audioPeak : "unknown";
+	const rms = typeof audioRms === "string" ? audioRms : "unknown";
+	const duration =
+		typeof extractedDuration === "string" ? extractedDuration : "unknown";
+	return `Extracted audio diagnostics: type=${type}, size=${size} bytes, duration=${duration}s, peak=${peak}, rms=${rms}.`;
+}
+
+function normalizeDeepgramResponse({
+	data,
+	assetId,
+	assetName,
+	duration,
+	language,
+	model,
+}: {
+	data: DeepgramResponse;
+	assetId: string;
+	assetName: string;
+	duration: number;
+	language: string;
+	model: string;
+}) {
+	const alternative = data.results?.channels?.[0]?.alternatives?.[0];
+	const fullText = alternative?.transcript ?? "";
+	const words = normalizeWords(alternative?.words ?? []);
+	const utterances = (data.results?.utterances ?? []).map((utterance) => ({
+		text: utterance.transcript ?? "",
+		start: utterance.start ?? utterance.words?.[0]?.start ?? 0,
+		end: utterance.end ?? utterance.words?.at(-1)?.end ?? 0,
+		confidence: utterance.confidence,
+		speaker: utterance.speaker,
+		words: normalizeWords(utterance.words ?? []),
+	}));
+	const segments =
+		utterances.length > 0
+			? utterances.map((utterance) => ({
+					text: utterance.text,
+					start: utterance.start,
+					end: utterance.end,
+				}))
+			: [
+					{
+						text: fullText,
+						start: words[0]?.start ?? 0,
+						end: words.at(-1)?.end ?? 0,
+					},
+				];
+
+	return {
+		transcriptId: `tr_deepgram_${data.metadata?.request_id ?? `${assetId}_${Date.now().toString(36)}`}`,
+		assetId,
+		assetName,
+		provider: "deepgram",
+		language,
+		model,
+		fullText,
+		duration,
+		confidence: alternative?.confidence,
+		timingGranularity: "word",
+		wordCount: words.length,
+		utteranceCount: utterances.length,
+		words,
+		utterances,
+		segments,
+	};
+}
+
+function normalizeWords(words: DeepgramWord[]) {
+	return words
+		.filter(
+			(word) => typeof word.start === "number" && typeof word.end === "number",
+		)
+		.map((word) => ({
+			text: word.punctuated_word ?? word.word ?? "",
+			start: word.start ?? 0,
+			end: word.end ?? 0,
+			confidence: word.confidence,
+			speaker: word.speaker,
+		}));
+}
+
+function resolveContentType(file: File): string {
+	if (file.type && file.type !== "application/octet-stream") return file.type;
+	return inferMimeTypeFromName(file.name) ?? "application/octet-stream";
+}
+
+function parseFiniteNumber(value: FormDataEntryValue | null): number | null {
+	if (typeof value !== "string") return null;
+	const parsed = Number(value);
+	return Number.isFinite(parsed) ? parsed : null;
+}
+
+function inferMimeTypeFromName(name: string): string | null {
+	const extension = name.split(".").pop()?.toLowerCase();
+	return extension ? (MIME_BY_EXTENSION[extension] ?? null) : null;
+}
+
+const MIME_BY_EXTENSION: Record<string, string> = {
+	flac: "audio/flac",
+	m4a: "audio/mp4",
+	mov: "video/quicktime",
+	mp3: "audio/mpeg",
+	mp4: "video/mp4",
+	mpeg: "video/mpeg",
+	mpg: "video/mpeg",
+	wav: "audio/wav",
+	webm: "video/webm",
+};

apps/web/src/components/editor/panels/chat/tool-formatters.ts

@@ -209,6 +209,13 @@ const TOOL_CALL_FORMATTERS: Record<
 			description: lang ? `language: ${lang}` : "Transcribing audio",
 		};
 	},
+	transcribe_audio: (args) => {
+		const lang = args.language ? String(args.language) : null;
+		return {
+			label: "Transcribe Audio",
+			description: lang ? `language: ${lang}` : "Getting precise audio context",
+		};
+	},
 };
 
 const TOOL_RESULT_FORMATTERS: Record<
@@ -357,6 +364,19 @@ const TOOL_RESULT_FORMATTERS: Record<
 			description: `${data.assetName ?? "audio"} · ${data.segmentCount ?? 0} segments`,
 		};
 	},
+	transcribe_audio: (parsed) => {
+		const data = parsed as {
+			assetName?: string;
+			wordCount?: number;
+			utteranceCount?: number;
+			timingGranularity?: string;
+		} | null;
+		if (!data) return null;
+		return {
+			label: "Audio Context",
+			description: `${data.assetName ?? "audio"} · ${data.wordCount ?? 0} words · ${data.timingGranularity ?? "timed"}`,
+		};
+	},
 	list_keyframes: (parsed) => {
 		const data = parsed as { keyframes?: unknown[] } | null;
 		if (!data) return null;

apps/web/src/lib/media/mediabunny.ts

@@ -1,4 +1,4 @@
-import { Input, ALL_FORMATS, BlobSource } from "mediabunny";
+import { Input, ALL_FORMATS, BlobSource, AudioBufferSink } from "mediabunny";
 import { createTimelineAudioBuffer } from "@/lib/media/audio";
 import type { SceneTracks } from "@/lib/timeline";
 import type { MediaAsset } from "@/lib/media/types";
@@ -45,6 +45,50 @@ const NUM_CHANNELS = 2;
 const EMPTY_TIMELINE_SILENT_DURATION_SECONDS = 0.1;
 const MIN_SILENT_DURATION_SECONDS = 0.001;
 
+export async function extractAssetAudio({
+	file,
+	assetType,
+}: {
+	file: File;
+	assetType: string;
+}): Promise<Blob> {
+	if (assetType === "audio") {
+		return file;
+	}
+
+	const input = new Input({
+		source: new BlobSource(file),
+		formats: ALL_FORMATS,
+	});
+
+	try {
+		const audioTrack = await input.getPrimaryAudioTrack();
+		if (!audioTrack) {
+			throw new Error("Asset has no audio track");
+		}
+
+		const sink = new AudioBufferSink(audioTrack);
+		const chunks: AudioBuffer[] = [];
+		let totalSamples = 0;
+
+		for await (const { buffer } of sink.buffers(0)) {
+			chunks.push(buffer);
+			totalSamples += buffer.length;
+		}
+
+		if (chunks.length === 0 || totalSamples === 0) {
+			throw new Error("Could not extract audio from asset");
+		}
+
+		return createWavBlob({
+			samples: interleaveAudioChunks({ chunks, totalSamples }),
+			sampleRate: chunks[0].sampleRate,
+		});
+	} finally {
+		input.dispose();
+	}
+}
+
 export const extractTimelineAudio = async ({
 	tracks,
 	mediaAssets,
@@ -113,7 +157,38 @@ function interleaveAudioBuffer({
 	return interleavedSamples;
 }
 
-function createWavBlob({ samples }: { samples: Float32Array }): Blob {
+function interleaveAudioChunks({
+	chunks,
+	totalSamples,
+}: {
+	chunks: AudioBuffer[];
+	totalSamples: number;
+}): Float32Array {
+	const interleavedSamples = new Float32Array(totalSamples * NUM_CHANNELS);
+	let outputSampleIndex = 0;
+
+	for (const chunk of chunks) {
+		const numChannels = Math.min(NUM_CHANNELS, chunk.numberOfChannels);
+		for (let sampleIndex = 0; sampleIndex < chunk.length; sampleIndex++) {
+			for (let channel = 0; channel < NUM_CHANNELS; channel++) {
+				const sourceChannel = Math.min(channel, Math.max(0, numChannels - 1));
+				interleavedSamples[outputSampleIndex * NUM_CHANNELS + channel] =
+					chunk.getChannelData(sourceChannel)[sampleIndex] ?? 0;
+			}
+			outputSampleIndex++;
+		}
+	}
+
+	return interleavedSamples;
+}
+
+function createWavBlob({
+	samples,
+	sampleRate = SAMPLE_RATE,
+}: {
+	samples: Float32Array;
+	sampleRate?: number;
+}): Blob {
 	const numChannels = NUM_CHANNELS;
 	const bitsPerSample = 16;
 	const bytesPerSample = bitsPerSample / 8;
@@ -132,8 +207,8 @@ function createWavBlob({ samples }: { samples: Float32Array }): Blob {
 	view.setUint32(16, 16, true);
 	view.setUint16(20, 1, true);
 	view.setUint16(22, numChannels, true);
-	view.setUint32(24, SAMPLE_RATE, true);
+	view.setUint32(24, sampleRate, true);
-	view.setUint32(28, SAMPLE_RATE * numChannels * bytesPerSample, true);
+	view.setUint32(28, sampleRate * numChannels * bytesPerSample, true);
 	view.setUint16(32, numChannels * bytesPerSample, true);
 	view.setUint16(34, bitsPerSample, true);