ft: transcription tool exposed to the agent using deepgram servie
This commit is contained in:
Luis Esteban Acevedo Ladino
parent
09c28cce72
commit
33fd348a78
|
|
@ -58,7 +58,7 @@ Do NOT perform write tools for complex edits before approval. This is required e
|
|||
## EXECUTION RULES:
|
||||
|
||||
1. Follow the plan steps in order. Each step lists the tools to use.
|
||||
2. After completing a step, call update_plan_step to mark it done.
|
||||
2. After completing a step, call update_plan_step with the step number (1, 2, 3...) and status 'done' to mark it complete.
|
||||
3. If a step fails, note the error in update_plan_step and decide whether to continue or skip.
|
||||
4. If you discover the plan needs adjustment mid-execution, explain what changed and why.
|
||||
5. When all steps are done, provide a summary of what was accomplished.
|
||||
|
|
@ -102,6 +102,7 @@ export function buildSystemPrompt(
|
|||
`Available tools:\n${toolList}`,
|
||||
"For questions about what is visible or audible in a media asset, never answer that you cannot see or hear the media if load_context is available. First infer the asset from the active assets or timeline; if needed call list_project_assets or list_timeline, then call load_context with the discovered internal id or timeline element ids.",
|
||||
"If load_context has loaded media and the conversation contains an attached fileData part, treat it as the actual video/audio/image content. You may answer visual and audio questions directly from that loaded media without calling extraction tools unless the user asks for a separate extraction workflow.",
|
||||
"When exact speech, quotes, word timing, subtitle timing, or audio-driven edit points matter, call transcribe_audio. Use load_context for broad multimodal understanding and transcribe_audio for precise spoken-word timing.",
|
||||
"Only claim edits that were actually performed by tool calls in this conversation. Do not say you added, removed, cleaned, or updated text/subtitles unless the relevant add_text, update_text, delete_timeline_elements, or update_timeline_element_timing tool call succeeded.",
|
||||
"When the user asks to add titles, hooks, labels, captions, subtitles, or visible text, call add_text. Do not add text proactively for unrelated edit requests such as cutting silence unless the user asks for text.",
|
||||
"When you need to perform an action matching one of these tools, call the appropriate tool. For all other requests, respond directly in plain text.",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,189 @@
|
|||
import { beforeEach, describe, expect, jest, mock, test } from "bun:test";
|
||||
import type { AgentContext } from "@/agent/types";
|
||||
import { providerToolSchemas } from "@/agent/tools/schemas";
|
||||
import { toolRegistry } from "@/agent/tools/registry";
|
||||
|
||||
const mockResolveAssetFile = jest.fn<(assetId?: string) => File | null>();
|
||||
const mockGetAssetHasAudio =
|
||||
jest.fn<(assetId: string) => boolean | undefined>();
|
||||
const mockExtractAssetAudio =
|
||||
jest.fn<(args: { file: File; assetType: string }) => Promise<Blob>>();
|
||||
const mockDecodeAudioToFloat32 =
|
||||
jest.fn<
|
||||
(args: {
|
||||
audioBlob: Blob;
|
||||
}) => Promise<{ samples: Float32Array; sampleRate: number }>
|
||||
>();
|
||||
const mockFetch = jest.fn<typeof fetch>();
|
||||
|
||||
mock.module("@/agent/context", () => ({
|
||||
EditorContextAdapter: {
|
||||
resolveAssetFile: mockResolveAssetFile,
|
||||
getAssetHasAudio: mockGetAssetHasAudio,
|
||||
},
|
||||
}));
|
||||
|
||||
mock.module("@/lib/media/mediabunny", () => ({
|
||||
extractAssetAudio: mockExtractAssetAudio,
|
||||
}));
|
||||
|
||||
mock.module("@/lib/media/audio", () => ({
|
||||
decodeAudioToFloat32: mockDecodeAudioToFloat32,
|
||||
}));
|
||||
|
||||
await import("@/agent/tools/transcribe-audio.tool");
|
||||
|
||||
function makeContext(overrides: Partial<AgentContext> = {}): AgentContext {
|
||||
return {
|
||||
projectId: "proj-1",
|
||||
activeSceneId: "scene-A",
|
||||
mediaAssets: [],
|
||||
playbackTimeMs: 0,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function fakeVideoAsset(
|
||||
overrides: Partial<{
|
||||
id: string;
|
||||
name: string;
|
||||
type: string;
|
||||
duration: number;
|
||||
}> = {},
|
||||
) {
|
||||
return {
|
||||
id: "v1",
|
||||
name: "clip.mp4",
|
||||
type: "video",
|
||||
duration: 30,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function fakeTranscriptResult() {
|
||||
return {
|
||||
transcriptId: "tr_deepgram_123",
|
||||
assetId: "v1",
|
||||
assetName: "clip.mp4",
|
||||
provider: "deepgram",
|
||||
language: "es",
|
||||
model: "nova-3",
|
||||
fullText: "Hola mundo.",
|
||||
duration: 30,
|
||||
confidence: 0.98,
|
||||
timingGranularity: "word",
|
||||
wordCount: 2,
|
||||
utteranceCount: 1,
|
||||
words: [
|
||||
{ text: "Hola", start: 1, end: 1.3, confidence: 0.99 },
|
||||
{ text: "mundo.", start: 1.3, end: 1.8, confidence: 0.97 },
|
||||
],
|
||||
utterances: [
|
||||
{
|
||||
text: "Hola mundo.",
|
||||
start: 1,
|
||||
end: 1.8,
|
||||
confidence: 0.98,
|
||||
words: [
|
||||
{ text: "Hola", start: 1, end: 1.3, confidence: 0.99 },
|
||||
{ text: "mundo.", start: 1.3, end: 1.8, confidence: 0.97 },
|
||||
],
|
||||
},
|
||||
],
|
||||
segments: [{ text: "Hola mundo.", start: 1, end: 1.8 }],
|
||||
};
|
||||
}
|
||||
|
||||
describe("transcribe_audio tool", () => {
|
||||
beforeEach(() => {
|
||||
mockResolveAssetFile.mockClear();
|
||||
mockGetAssetHasAudio.mockClear();
|
||||
mockExtractAssetAudio.mockClear();
|
||||
mockDecodeAudioToFloat32.mockClear();
|
||||
mockFetch.mockClear();
|
||||
mockDecodeAudioToFloat32.mockResolvedValue({
|
||||
samples: new Float32Array([0, 0.25, -0.5]),
|
||||
sampleRate: 3,
|
||||
});
|
||||
globalThis.fetch = mockFetch;
|
||||
});
|
||||
|
||||
test("is registered and exposed to the provider without provider/model args", () => {
|
||||
expect(toolRegistry.has("transcribe_audio")).toBe(true);
|
||||
const schema = providerToolSchemas.find(
|
||||
(tool) => tool.name === "transcribe_audio",
|
||||
);
|
||||
expect(schema).toBeDefined();
|
||||
expect(schema?.parameters.map((param) => param.key)).toEqual([
|
||||
"assetId",
|
||||
"language",
|
||||
]);
|
||||
});
|
||||
|
||||
test("transcribes a single media asset through the Deepgram route", async () => {
|
||||
const file = new File([], "clip.mp4", { type: "video/mp4" });
|
||||
const audioBlob = new Blob([new Uint8Array([1, 2, 3])], {
|
||||
type: "audio/wav",
|
||||
});
|
||||
mockResolveAssetFile.mockReturnValue(file);
|
||||
mockGetAssetHasAudio.mockReturnValue(true);
|
||||
mockExtractAssetAudio.mockResolvedValue(audioBlob);
|
||||
mockFetch.mockResolvedValue(
|
||||
new Response(JSON.stringify(fakeTranscriptResult()), { status: 200 }),
|
||||
);
|
||||
|
||||
const tool = toolRegistry.get("transcribe_audio");
|
||||
const result = await tool.execute(
|
||||
{},
|
||||
makeContext({ mediaAssets: [fakeVideoAsset()] }),
|
||||
);
|
||||
|
||||
expect(result).toEqual(fakeTranscriptResult());
|
||||
expect(mockExtractAssetAudio).toHaveBeenCalledWith({
|
||||
file,
|
||||
assetType: "video",
|
||||
});
|
||||
expect(mockFetch).toHaveBeenCalledWith(
|
||||
"/api/transcription/deepgram",
|
||||
expect.objectContaining({ method: "POST" }),
|
||||
);
|
||||
});
|
||||
|
||||
test("returns Deepgram route errors without local fallback", async () => {
|
||||
const file = new File([], "clip.mp4", { type: "video/mp4" });
|
||||
mockResolveAssetFile.mockReturnValue(file);
|
||||
mockGetAssetHasAudio.mockReturnValue(true);
|
||||
mockExtractAssetAudio.mockResolvedValue(
|
||||
new Blob([], { type: "audio/wav" }),
|
||||
);
|
||||
mockFetch.mockResolvedValue(
|
||||
new Response(JSON.stringify({ error: "missing key" }), { status: 501 }),
|
||||
);
|
||||
|
||||
const tool = toolRegistry.get("transcribe_audio");
|
||||
const result = await tool.execute(
|
||||
{},
|
||||
makeContext({ mediaAssets: [fakeVideoAsset()] }),
|
||||
);
|
||||
|
||||
expect(result).toEqual({ error: "missing key" });
|
||||
});
|
||||
|
||||
test("returns extraction errors before calling Deepgram", async () => {
|
||||
const file = new File([], "clip.mov", { type: "video/quicktime" });
|
||||
mockResolveAssetFile.mockReturnValue(file);
|
||||
mockGetAssetHasAudio.mockReturnValue(true);
|
||||
mockExtractAssetAudio.mockRejectedValue(
|
||||
new Error("Could not extract audio"),
|
||||
);
|
||||
|
||||
const tool = toolRegistry.get("transcribe_audio");
|
||||
const result = await tool.execute(
|
||||
{},
|
||||
makeContext({ mediaAssets: [fakeVideoAsset({ name: "clip.mov" })] }),
|
||||
);
|
||||
|
||||
expect(result).toEqual({ error: "Could not extract audio" });
|
||||
expect(mockFetch).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
|
@ -11,6 +11,7 @@
|
|||
*/
|
||||
|
||||
import "@/agent/tools/load-context.tool";
|
||||
import "@/agent/tools/transcribe-audio.tool";
|
||||
import "@/agent/tools/list-project-assets.tool";
|
||||
import "@/agent/tools/list-timeline.tool";
|
||||
import "@/agent/tools/get-element.tool";
|
||||
|
|
|
|||
|
|
@ -147,23 +147,28 @@ async function loadTimelineElementContext(
|
|||
args: LoadContextArgs,
|
||||
context: AgentContext,
|
||||
): Promise<LoadContextResult | { error: string }> {
|
||||
const trackId = args.trackId;
|
||||
const elementId = args.elementId ?? args.id;
|
||||
|
||||
if (!trackId || !elementId) {
|
||||
return { error: "trackId and elementId are required" };
|
||||
if (!elementId) {
|
||||
return { error: "elementId (or id) is required for timeline_element target" };
|
||||
}
|
||||
|
||||
const element = findTimelineElement(
|
||||
context.timelineTracks,
|
||||
trackId,
|
||||
elementId,
|
||||
);
|
||||
if (!element) {
|
||||
let resolved: ReturnType<typeof findTimelineElement> = null;
|
||||
|
||||
if (args.trackId) {
|
||||
resolved = findTimelineElement(context.timelineTracks, args.trackId, elementId);
|
||||
} else {
|
||||
resolved = findTimelineElementAcrossTracks(context.timelineTracks, elementId);
|
||||
}
|
||||
|
||||
if (!resolved) {
|
||||
return { error: "Timeline element not found" };
|
||||
}
|
||||
|
||||
if (element.element.content !== undefined) {
|
||||
const { track, element } = resolved;
|
||||
const trackId = track.trackId;
|
||||
|
||||
if (element.content !== undefined) {
|
||||
const cacheKey = `${trackId}:${elementId}`;
|
||||
const cached = textContextCache.get(cacheKey);
|
||||
if (cached) {
|
||||
|
|
@ -181,11 +186,11 @@ async function loadTimelineElementContext(
|
|||
kind: "text",
|
||||
trackId,
|
||||
elementId,
|
||||
type: element.element.type,
|
||||
...(element.element.name ? { name: element.element.name } : {}),
|
||||
content: element.element.content,
|
||||
start: element.element.start,
|
||||
end: element.element.end,
|
||||
type: element.type,
|
||||
...(element.name ? { name: element.name } : {}),
|
||||
content: element.content,
|
||||
start: element.start,
|
||||
end: element.end,
|
||||
};
|
||||
|
||||
textContextCache.set(cacheKey, textContext);
|
||||
|
|
@ -200,17 +205,17 @@ async function loadTimelineElementContext(
|
|||
};
|
||||
}
|
||||
|
||||
if (element.element.assetId) {
|
||||
if (element.assetId) {
|
||||
return loadAssetContext(
|
||||
{ targetType: "asset", assetId: element.element.assetId },
|
||||
{ targetType: "asset", assetId: element.assetId },
|
||||
context,
|
||||
{
|
||||
assetId: element.element.assetId,
|
||||
assetId: element.assetId,
|
||||
trackId,
|
||||
elementId,
|
||||
type: element.element.type,
|
||||
start: element.element.start,
|
||||
end: element.element.end,
|
||||
type: element.type,
|
||||
start: element.start,
|
||||
end: element.end,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
|
@ -367,6 +372,23 @@ function findTimelineElement(
|
|||
return track && element ? { track, element } : null;
|
||||
}
|
||||
|
||||
function findTimelineElementAcrossTracks(
|
||||
tracks: AgentTimelineTrack[] | undefined,
|
||||
elementId: string,
|
||||
): {
|
||||
track: AgentTimelineTrack;
|
||||
element: AgentTimelineTrack["elements"][number];
|
||||
} | null {
|
||||
if (!tracks) return null;
|
||||
for (const track of tracks) {
|
||||
const element = track.elements.find(
|
||||
(candidate) => candidate.elementId === elementId,
|
||||
);
|
||||
if (element) return { track, element };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function isSupportedMediaAssetType(type: string): boolean {
|
||||
return type === "video" || type === "audio" || type === "image";
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ import type { ToolSchema } from "@/agent/types";
|
|||
export const loadContextSchema: ToolSchema = {
|
||||
name: "load_context",
|
||||
description:
|
||||
"Loads the actual Gemini multimodal context for a project asset or timeline element. Use this before answering questions about visible objects, colors, scenes, speech, silence, or timestamps in media. Use targetType='asset' with assetId/id for media, or targetType='timeline_element' with trackId and elementId/id for captions, text, or timeline media elements.",
|
||||
"Loads Gemini multimodal context for a project asset or timeline element. For media: targetType='asset' with assetId. For timeline elements (text, clips): targetType='timeline_element' with elementId — trackId is optional, the element will be found across all tracks automatically.",
|
||||
parameters: [
|
||||
{ key: "targetType", type: "string", required: true },
|
||||
{ key: "id", type: "string", required: false },
|
||||
|
|
@ -24,6 +24,16 @@ export const loadContextSchema: ToolSchema = {
|
|||
],
|
||||
};
|
||||
|
||||
export const transcribeAudioSchema: ToolSchema = {
|
||||
name: "transcribe_audio",
|
||||
description:
|
||||
"Transcribes a video or audio asset with precise word-level timing. Returns full text, per-word start/end times, confidence scores, and utterance segments. Use this when you need exact spoken words, timestamps for subtitles, or silence detection. Pass assetId to target a specific asset, or omit to auto-select the first video/audio asset.",
|
||||
parameters: [
|
||||
{ key: "assetId", type: "string", required: false },
|
||||
{ key: "language", type: "string", required: false },
|
||||
],
|
||||
};
|
||||
|
||||
export const listProjectAssetsSchema: ToolSchema = {
|
||||
name: "list_project_assets",
|
||||
description:
|
||||
|
|
@ -93,7 +103,7 @@ export const updateTimelineElementTimingSchema: ToolSchema = {
|
|||
export const addTextSchema: ToolSchema = {
|
||||
name: "add_text",
|
||||
description:
|
||||
"Adds visual text to the active timeline. Use for titles, hooks, labels, and simple subtitle blocks. start and end are timeline seconds. position must be top, center, or bottom; style may be plain, subtitle, hook, or label. Override style defaults with color (hex), fontSize (scaled units), fontFamily, fontWeight (normal/bold), fontStyle (normal/italic), textAlign (left/center/right), letterSpacing, positionX/positionY (offset from -50 to 50), or background ({ enabled, color?, cornerRadius?, padding? }).",
|
||||
"Adds visual text to the active timeline. start and end are seconds. position must be exactly one of: 'top', 'center', or 'bottom'. style: 'plain', 'subtitle' (white on black bg), 'hook' (large bold), or 'label'. Override defaults with color (hex), fontSize, fontFamily, fontWeight, fontStyle, textAlign, letterSpacing, positionX/positionY (-50 to 50), or background.",
|
||||
parameters: [
|
||||
{ key: "text", type: "string", required: true },
|
||||
{ key: "start", type: "number", required: true },
|
||||
|
|
@ -116,7 +126,7 @@ export const addTextSchema: ToolSchema = {
|
|||
export const updateTextSchema: ToolSchema = {
|
||||
name: "update_text",
|
||||
description:
|
||||
"Updates visual properties of existing text elements. Pass elementIds (from list_timeline) and any combination of text style overrides. Only text elements are updated; non-text elements are skipped. All listed text elements receive the same overrides — use for bulk styling subtitle blocks.",
|
||||
"Updates visual properties of existing text elements. Pass elementIds (from list_timeline) and any properties to change. All listed elements receive the same overrides — use for bulk styling. Non-text elements in the list are skipped.",
|
||||
parameters: [
|
||||
// "array" (not "string[]") so the orchestrator validator stays
|
||||
// tolerant of CSV-string fallback handled by resolveElementIds.
|
||||
|
|
@ -308,7 +318,7 @@ export const loadSkillSchema: ToolSchema = {
|
|||
export const submitPlanSchema: ToolSchema = {
|
||||
name: "submit_plan",
|
||||
description:
|
||||
"Submits a structured editing plan with steps and waits for user approval through a plan card with Go edit and Keep planning actions. Use this before complex edits, even from execute mode. Each step should describe a specific action with the tools to use. If approved=true is returned, continue executing the plan. If approved=false is returned, keep refining the plan with the user.",
|
||||
"Submits a structured editing plan with numbered steps for user approval. Returns step IDs — use update_plan_step with the step number (1, 2, 3...) to track progress. Each step needs a description and tools array.",
|
||||
parameters: [
|
||||
{ key: "summary", type: "string", required: true },
|
||||
{ key: "steps", type: "array", required: true },
|
||||
|
|
@ -336,9 +346,10 @@ export const requestPlanApprovalSchema: ToolSchema = {
|
|||
export const updatePlanStepSchema: ToolSchema = {
|
||||
name: "update_plan_step",
|
||||
description:
|
||||
"Updates the status of a plan step during execution. Call this after completing each step to track progress. The plan panel updates in real-time. Use status 'done' for successful completion, 'skipped' if the step was unnecessary, or 'in_progress' if starting.",
|
||||
"Updates the status of a plan step during execution. Prefer using step (1-based number matching plan order) over stepId. The plan panel updates in real-time. Use status 'in_progress' when starting, 'done' for successful completion, or 'skipped' if unnecessary.",
|
||||
parameters: [
|
||||
{ key: "stepId", type: "string", required: true },
|
||||
{ key: "step", type: "number", required: false },
|
||||
{ key: "stepId", type: "string", required: false },
|
||||
{ key: "status", type: "string", required: true },
|
||||
{ key: "result", type: "string", required: false },
|
||||
],
|
||||
|
|
@ -350,6 +361,7 @@ export const updatePlanStepSchema: ToolSchema = {
|
|||
*/
|
||||
export const providerToolSchemas: ToolSchema[] = [
|
||||
loadContextSchema,
|
||||
transcribeAudioSchema,
|
||||
listProjectAssetsSchema,
|
||||
listTimelineSchema,
|
||||
getElementSchema,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,240 @@
|
|||
import type { AgentContext, ToolDefinition } from "@/agent/types";
|
||||
import { EditorContextAdapter } from "@/agent/context";
|
||||
import { toolRegistry } from "@/agent/tools/registry";
|
||||
import { transcribeAudioSchema } from "@/agent/tools/schemas";
|
||||
import { decodeAudioToFloat32 } from "@/lib/media/audio";
|
||||
import { extractAssetAudio } from "@/lib/media/mediabunny";
|
||||
import type { TranscriptionLanguage } from "@/lib/transcription/types";
|
||||
|
||||
type TimingGranularity = "word";
|
||||
|
||||
type TranscriptWord = {
|
||||
text: string;
|
||||
start: number;
|
||||
end: number;
|
||||
confidence?: number;
|
||||
};
|
||||
|
||||
type TranscriptUtterance = {
|
||||
text: string;
|
||||
start: number;
|
||||
end: number;
|
||||
confidence?: number;
|
||||
speaker?: number;
|
||||
words: TranscriptWord[];
|
||||
};
|
||||
|
||||
export type TranscribeAudioResult = {
|
||||
transcriptId: string;
|
||||
assetId: string;
|
||||
assetName: string;
|
||||
provider: "deepgram";
|
||||
language: string;
|
||||
model?: string;
|
||||
fullText: string;
|
||||
duration: number;
|
||||
confidence?: number;
|
||||
timingGranularity: TimingGranularity;
|
||||
wordCount: number;
|
||||
utteranceCount: number;
|
||||
words: TranscriptWord[];
|
||||
utterances: TranscriptUtterance[];
|
||||
segments: Array<{ text: string; start: number; end: number }>;
|
||||
};
|
||||
|
||||
type TranscribeAudioArgs = {
|
||||
assetId?: string;
|
||||
language?: TranscriptionLanguage;
|
||||
};
|
||||
|
||||
const transcribeAudioTool: ToolDefinition = {
|
||||
...transcribeAudioSchema,
|
||||
execute: async (
|
||||
args: Record<string, unknown>,
|
||||
context: AgentContext,
|
||||
): Promise<TranscribeAudioResult | { error: string }> => {
|
||||
const typedArgs = args as TranscribeAudioArgs;
|
||||
const target = resolveTargetAsset({
|
||||
assetId: typedArgs.assetId,
|
||||
context,
|
||||
});
|
||||
|
||||
if ("error" in target) return target;
|
||||
|
||||
const file = EditorContextAdapter.resolveAssetFile(target.asset.id);
|
||||
if (!file) {
|
||||
return { error: `Could not access file for asset ${target.asset.id}` };
|
||||
}
|
||||
|
||||
const hasAudio = EditorContextAdapter.getAssetHasAudio(target.asset.id);
|
||||
if (hasAudio === false) {
|
||||
return { error: "Asset has no audio track" };
|
||||
}
|
||||
|
||||
try {
|
||||
const audioBlob = await extractAssetAudio({
|
||||
file,
|
||||
assetType: target.asset.type,
|
||||
});
|
||||
const diagnostics = await analyzeAudioBlob({ audioBlob });
|
||||
|
||||
return transcribeWithDeepgram({
|
||||
audioBlob,
|
||||
diagnostics,
|
||||
assetId: target.asset.id,
|
||||
assetName: target.asset.name,
|
||||
duration: target.asset.duration,
|
||||
language: typedArgs.language,
|
||||
});
|
||||
} catch (error) {
|
||||
return {
|
||||
error:
|
||||
error instanceof Error
|
||||
? error.message
|
||||
: "Could not extract audio from asset",
|
||||
};
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
async function transcribeWithDeepgram({
|
||||
audioBlob,
|
||||
diagnostics,
|
||||
assetId,
|
||||
assetName,
|
||||
duration,
|
||||
language,
|
||||
}: {
|
||||
audioBlob: Blob;
|
||||
diagnostics: AudioDiagnostics;
|
||||
assetId: string;
|
||||
assetName: string;
|
||||
duration: number;
|
||||
language?: TranscriptionLanguage;
|
||||
}): Promise<TranscribeAudioResult | { error: string }> {
|
||||
const formData = new FormData();
|
||||
formData.set("file", audioBlob, `${assetName.replace(/\.[^.]+$/, "")}.wav`);
|
||||
formData.set("assetId", assetId);
|
||||
formData.set("assetName", assetName);
|
||||
formData.set("duration", String(duration));
|
||||
formData.set("audioSize", String(audioBlob.size));
|
||||
formData.set("audioType", audioBlob.type || "unknown");
|
||||
formData.set("audioPeak", String(diagnostics.peak));
|
||||
formData.set("audioRms", String(diagnostics.rms));
|
||||
formData.set("extractedDuration", String(diagnostics.duration));
|
||||
if (language && language !== "auto") formData.set("language", language);
|
||||
|
||||
const response = await fetch("/api/transcription/deepgram", {
|
||||
method: "POST",
|
||||
body: formData,
|
||||
});
|
||||
const data = (await response.json()) as TranscribeAudioResult & {
|
||||
error?: string;
|
||||
};
|
||||
|
||||
if (!response.ok) {
|
||||
return { error: data.error ?? "Deepgram transcription failed" };
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
type AudioDiagnostics = {
|
||||
duration: number;
|
||||
peak: number;
|
||||
rms: number;
|
||||
};
|
||||
|
||||
async function analyzeAudioBlob({
|
||||
audioBlob,
|
||||
}: {
|
||||
audioBlob: Blob;
|
||||
}): Promise<AudioDiagnostics> {
|
||||
const { samples, sampleRate } = await decodeAudioToFloat32({ audioBlob });
|
||||
let peak = 0;
|
||||
let sumSquares = 0;
|
||||
|
||||
for (const sample of samples) {
|
||||
const abs = Math.abs(sample);
|
||||
if (abs > peak) peak = abs;
|
||||
sumSquares += sample * sample;
|
||||
}
|
||||
|
||||
return {
|
||||
duration: samples.length / sampleRate,
|
||||
peak,
|
||||
rms: samples.length > 0 ? Math.sqrt(sumSquares / samples.length) : 0,
|
||||
};
|
||||
}
|
||||
|
||||
function resolveTargetAsset({
|
||||
assetId,
|
||||
context,
|
||||
}: {
|
||||
assetId?: string;
|
||||
context: AgentContext;
|
||||
}): { asset: AgentContext["mediaAssets"][number] } | { error: string } {
|
||||
if (context.mediaAssets.length === 0) {
|
||||
return { error: "No active media asset" };
|
||||
}
|
||||
|
||||
const mediaCandidates = context.mediaAssets.filter(
|
||||
(asset) => asset.type === "video" || asset.type === "audio",
|
||||
);
|
||||
if (mediaCandidates.length === 0) {
|
||||
return { error: "Asset has no audio track" };
|
||||
}
|
||||
|
||||
if (!assetId) {
|
||||
if (mediaCandidates.length > 1) {
|
||||
return {
|
||||
error: `Multiple video/audio assets found: ${mediaCandidates.map((asset) => asset.name).join(", ")}. Specify which one with assetId.`,
|
||||
};
|
||||
}
|
||||
return { asset: mediaCandidates[0] };
|
||||
}
|
||||
|
||||
const target = resolveAssetByIdOrName({ assetId, mediaCandidates });
|
||||
if ("error" in target) return target;
|
||||
return { asset: target.asset };
|
||||
}
|
||||
|
||||
function resolveAssetByIdOrName({
|
||||
assetId,
|
||||
mediaCandidates,
|
||||
}: {
|
||||
assetId: string;
|
||||
mediaCandidates: AgentContext["mediaAssets"];
|
||||
}): { asset: AgentContext["mediaAssets"][number] } | { error: string } {
|
||||
const idMatch = mediaCandidates.find((asset) => asset.id === assetId);
|
||||
if (idMatch) return { asset: idMatch };
|
||||
|
||||
const exactNameMatches = mediaCandidates.filter(
|
||||
(asset) => asset.name === assetId,
|
||||
);
|
||||
if (exactNameMatches.length === 1) return { asset: exactNameMatches[0] };
|
||||
if (exactNameMatches.length > 1) {
|
||||
return {
|
||||
error: `Ambiguous asset name "${assetId}" matches multiple assets (${exactNameMatches.map((asset) => asset.name).join(", ")}). Specify the internal id: ${exactNameMatches.map((asset) => asset.id).join(", ")}.`,
|
||||
};
|
||||
}
|
||||
|
||||
const lower = assetId.toLowerCase();
|
||||
const caseInsensitiveMatches = mediaCandidates.filter(
|
||||
(asset) => asset.name.toLowerCase() === lower,
|
||||
);
|
||||
if (caseInsensitiveMatches.length === 1) {
|
||||
return { asset: caseInsensitiveMatches[0] };
|
||||
}
|
||||
if (caseInsensitiveMatches.length > 1) {
|
||||
return {
|
||||
error: `Ambiguous asset name "${assetId}" matches multiple assets (${caseInsensitiveMatches.map((asset) => asset.name).join(", ")}). Specify the internal id: ${caseInsensitiveMatches.map((asset) => asset.id).join(", ")}.`,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
error: `No asset found with id or name "${assetId}". Available ids: [${mediaCandidates.map((asset) => asset.id).join(", ")}]. Available names: [${mediaCandidates.map((asset) => asset.name).join(", ")}].`,
|
||||
};
|
||||
}
|
||||
|
||||
toolRegistry.register(transcribeAudioSchema.name, transcribeAudioTool);
|
||||
|
|
@ -16,18 +16,18 @@ const updatePlanStepTool: ToolDefinition = {
|
|||
execute: async (
|
||||
args: Record<string, unknown>,
|
||||
_context: AgentContext,
|
||||
):
|
||||
| { success: boolean; stepId: string; status: string }
|
||||
| { error: string } => {
|
||||
): Promise<
|
||||
| { success: boolean; step: number; stepId: string; status: string }
|
||||
| { error: string }
|
||||
> => {
|
||||
const { stepId, status, result } = args as {
|
||||
stepId: string;
|
||||
step?: number;
|
||||
stepId?: string;
|
||||
status: string;
|
||||
result?: string;
|
||||
};
|
||||
|
||||
if (!stepId || typeof stepId !== "string") {
|
||||
return { error: "stepId is required" };
|
||||
}
|
||||
const stepNumber = args.step as number | undefined;
|
||||
|
||||
if (!VALID_STATUSES.includes(status as PlanStepStatus)) {
|
||||
return {
|
||||
|
|
@ -41,16 +41,37 @@ const updatePlanStepTool: ToolDefinition = {
|
|||
return { error: "No active plan" };
|
||||
}
|
||||
|
||||
const step = plan.steps.find((s) => s.id === stepId);
|
||||
let resolvedStepId: string | undefined;
|
||||
|
||||
if (stepNumber !== undefined && stepNumber !== null) {
|
||||
const index = Math.floor(stepNumber) - 1;
|
||||
if (index < 0 || index >= plan.steps.length) {
|
||||
return {
|
||||
error: `Step ${stepNumber} out of range. Plan has ${plan.steps.length} steps (1-${plan.steps.length}).`,
|
||||
};
|
||||
}
|
||||
resolvedStepId = plan.steps[index].id;
|
||||
} else if (stepId && typeof stepId === "string") {
|
||||
resolvedStepId = stepId;
|
||||
} else {
|
||||
return { error: "Pass step (1-based number) or stepId" };
|
||||
}
|
||||
|
||||
const step = plan.steps.find((s) => s.id === resolvedStepId);
|
||||
if (!step) {
|
||||
return {
|
||||
error: `Step not found: ${stepId}. Available: ${plan.steps.map((s) => s.id).join(", ")}`,
|
||||
error: `Step not found. Use step number (1-${plan.steps.length}) instead.`,
|
||||
};
|
||||
}
|
||||
|
||||
planStore.updateStepStatus(stepId, status as PlanStepStatus, result);
|
||||
planStore.updateStepStatus(resolvedStepId, status as PlanStepStatus, result);
|
||||
|
||||
return { success: true, stepId, status };
|
||||
return {
|
||||
success: true,
|
||||
step: plan.steps.findIndex((s) => s.id === resolvedStepId) + 1,
|
||||
stepId: resolvedStepId,
|
||||
status,
|
||||
};
|
||||
},
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,268 @@
|
|||
import { type NextRequest, NextResponse } from "next/server";
|
||||
|
||||
type DeepgramWord = {
|
||||
word?: string;
|
||||
punctuated_word?: string;
|
||||
start?: number;
|
||||
end?: number;
|
||||
confidence?: number;
|
||||
speaker?: number;
|
||||
};
|
||||
|
||||
type DeepgramUtterance = {
|
||||
transcript?: string;
|
||||
start?: number;
|
||||
end?: number;
|
||||
confidence?: number;
|
||||
speaker?: number;
|
||||
words?: DeepgramWord[];
|
||||
};
|
||||
|
||||
type DeepgramResponse = {
|
||||
metadata?: { request_id?: string };
|
||||
results?: {
|
||||
channels?: Array<{
|
||||
alternatives?: Array<{
|
||||
transcript?: string;
|
||||
confidence?: number;
|
||||
words?: DeepgramWord[];
|
||||
}>;
|
||||
}>;
|
||||
utterances?: DeepgramUtterance[];
|
||||
};
|
||||
};
|
||||
|
||||
const DEEPGRAM_MODEL = "nova-3";
|
||||
|
||||
export async function POST(request: NextRequest) {
|
||||
const apiKey = process.env.DEEPGRAM_API_KEY;
|
||||
if (!apiKey) {
|
||||
return NextResponse.json(
|
||||
{ error: "Server configuration error: missing DEEPGRAM_API_KEY" },
|
||||
{ status: 501 },
|
||||
);
|
||||
}
|
||||
|
||||
const formData = await request.formData();
|
||||
const file = formData.get("file");
|
||||
const assetId = formData.get("assetId");
|
||||
const assetName = formData.get("assetName");
|
||||
const duration = formData.get("duration");
|
||||
const language = formData.get("language");
|
||||
const audioSize = formData.get("audioSize");
|
||||
const audioType = formData.get("audioType");
|
||||
const audioPeak = formData.get("audioPeak");
|
||||
const audioRms = formData.get("audioRms");
|
||||
const extractedDuration = formData.get("extractedDuration");
|
||||
|
||||
if (!(file instanceof File)) {
|
||||
return NextResponse.json({ error: "File is required" }, { status: 400 });
|
||||
}
|
||||
if (typeof assetId !== "string" || typeof assetName !== "string") {
|
||||
return NextResponse.json(
|
||||
{ error: "assetId and assetName are required" },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
|
||||
const url = new URL("https://api.deepgram.com/v1/listen");
|
||||
url.searchParams.set("model", DEEPGRAM_MODEL);
|
||||
url.searchParams.set("smart_format", "true");
|
||||
url.searchParams.set("punctuate", "true");
|
||||
url.searchParams.set("utterances", "true");
|
||||
if (typeof language === "string" && language.length > 0) {
|
||||
url.searchParams.set("language", language);
|
||||
} else {
|
||||
url.searchParams.set("detect_language", "true");
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Token ${apiKey}`,
|
||||
"Content-Type": resolveContentType(file),
|
||||
},
|
||||
body: Buffer.from(await file.arrayBuffer()),
|
||||
});
|
||||
const data = (await response.json()) as DeepgramResponse & {
|
||||
error?: string;
|
||||
};
|
||||
|
||||
if (!response.ok) {
|
||||
return NextResponse.json(
|
||||
{ error: data.error ?? "Deepgram transcription failed" },
|
||||
{ status: response.status },
|
||||
);
|
||||
}
|
||||
|
||||
const normalized = normalizeDeepgramResponse({
|
||||
data,
|
||||
assetId,
|
||||
assetName,
|
||||
duration: parseFiniteNumber(duration) ?? 0,
|
||||
language: typeof language === "string" ? language : "auto",
|
||||
model: DEEPGRAM_MODEL,
|
||||
});
|
||||
|
||||
if (isEmptyTranscript(normalized)) {
|
||||
const diagnostics = formatAudioDiagnostics({
|
||||
audioSize,
|
||||
audioType,
|
||||
audioPeak,
|
||||
audioRms,
|
||||
extractedDuration,
|
||||
});
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: `Deepgram returned no speech. ${diagnostics}`,
|
||||
},
|
||||
{ status: 422 },
|
||||
);
|
||||
}
|
||||
|
||||
return NextResponse.json(normalized);
|
||||
} catch (error) {
|
||||
console.error("[transcription/deepgram] Deepgram error:", error);
|
||||
return NextResponse.json(
|
||||
{ error: "Deepgram transcription failed" },
|
||||
{ status: 502 },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function isEmptyTranscript(transcript: {
|
||||
fullText: string;
|
||||
wordCount: number;
|
||||
utteranceCount: number;
|
||||
}): boolean {
|
||||
return (
|
||||
transcript.fullText.trim().length === 0 &&
|
||||
transcript.wordCount === 0 &&
|
||||
transcript.utteranceCount === 0
|
||||
);
|
||||
}
|
||||
|
||||
function formatAudioDiagnostics({
|
||||
audioSize,
|
||||
audioType,
|
||||
audioPeak,
|
||||
audioRms,
|
||||
extractedDuration,
|
||||
}: {
|
||||
audioSize: FormDataEntryValue | null;
|
||||
audioType: FormDataEntryValue | null;
|
||||
audioPeak: FormDataEntryValue | null;
|
||||
audioRms: FormDataEntryValue | null;
|
||||
extractedDuration: FormDataEntryValue | null;
|
||||
}): string {
|
||||
const size = typeof audioSize === "string" ? audioSize : "unknown";
|
||||
const type = typeof audioType === "string" ? audioType : "unknown";
|
||||
const peak = typeof audioPeak === "string" ? audioPeak : "unknown";
|
||||
const rms = typeof audioRms === "string" ? audioRms : "unknown";
|
||||
const duration =
|
||||
typeof extractedDuration === "string" ? extractedDuration : "unknown";
|
||||
return `Extracted audio diagnostics: type=${type}, size=${size} bytes, duration=${duration}s, peak=${peak}, rms=${rms}.`;
|
||||
}
|
||||
|
||||
function normalizeDeepgramResponse({
|
||||
data,
|
||||
assetId,
|
||||
assetName,
|
||||
duration,
|
||||
language,
|
||||
model,
|
||||
}: {
|
||||
data: DeepgramResponse;
|
||||
assetId: string;
|
||||
assetName: string;
|
||||
duration: number;
|
||||
language: string;
|
||||
model: string;
|
||||
}) {
|
||||
const alternative = data.results?.channels?.[0]?.alternatives?.[0];
|
||||
const fullText = alternative?.transcript ?? "";
|
||||
const words = normalizeWords(alternative?.words ?? []);
|
||||
const utterances = (data.results?.utterances ?? []).map((utterance) => ({
|
||||
text: utterance.transcript ?? "",
|
||||
start: utterance.start ?? utterance.words?.[0]?.start ?? 0,
|
||||
end: utterance.end ?? utterance.words?.at(-1)?.end ?? 0,
|
||||
confidence: utterance.confidence,
|
||||
speaker: utterance.speaker,
|
||||
words: normalizeWords(utterance.words ?? []),
|
||||
}));
|
||||
const segments =
|
||||
utterances.length > 0
|
||||
? utterances.map((utterance) => ({
|
||||
text: utterance.text,
|
||||
start: utterance.start,
|
||||
end: utterance.end,
|
||||
}))
|
||||
: [
|
||||
{
|
||||
text: fullText,
|
||||
start: words[0]?.start ?? 0,
|
||||
end: words.at(-1)?.end ?? 0,
|
||||
},
|
||||
];
|
||||
|
||||
return {
|
||||
transcriptId: `tr_deepgram_${data.metadata?.request_id ?? `${assetId}_${Date.now().toString(36)}`}`,
|
||||
assetId,
|
||||
assetName,
|
||||
provider: "deepgram",
|
||||
language,
|
||||
model,
|
||||
fullText,
|
||||
duration,
|
||||
confidence: alternative?.confidence,
|
||||
timingGranularity: "word",
|
||||
wordCount: words.length,
|
||||
utteranceCount: utterances.length,
|
||||
words,
|
||||
utterances,
|
||||
segments,
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeWords(words: DeepgramWord[]) {
|
||||
return words
|
||||
.filter(
|
||||
(word) => typeof word.start === "number" && typeof word.end === "number",
|
||||
)
|
||||
.map((word) => ({
|
||||
text: word.punctuated_word ?? word.word ?? "",
|
||||
start: word.start ?? 0,
|
||||
end: word.end ?? 0,
|
||||
confidence: word.confidence,
|
||||
speaker: word.speaker,
|
||||
}));
|
||||
}
|
||||
|
||||
function resolveContentType(file: File): string {
|
||||
if (file.type && file.type !== "application/octet-stream") return file.type;
|
||||
return inferMimeTypeFromName(file.name) ?? "application/octet-stream";
|
||||
}
|
||||
|
||||
function parseFiniteNumber(value: FormDataEntryValue | null): number | null {
|
||||
if (typeof value !== "string") return null;
|
||||
const parsed = Number(value);
|
||||
return Number.isFinite(parsed) ? parsed : null;
|
||||
}
|
||||
|
||||
function inferMimeTypeFromName(name: string): string | null {
|
||||
const extension = name.split(".").pop()?.toLowerCase();
|
||||
return extension ? (MIME_BY_EXTENSION[extension] ?? null) : null;
|
||||
}
|
||||
|
||||
const MIME_BY_EXTENSION: Record<string, string> = {
|
||||
flac: "audio/flac",
|
||||
m4a: "audio/mp4",
|
||||
mov: "video/quicktime",
|
||||
mp3: "audio/mpeg",
|
||||
mp4: "video/mp4",
|
||||
mpeg: "video/mpeg",
|
||||
mpg: "video/mpeg",
|
||||
wav: "audio/wav",
|
||||
webm: "video/webm",
|
||||
};
|
||||
|
|
@ -209,6 +209,13 @@ const TOOL_CALL_FORMATTERS: Record<
|
|||
description: lang ? `language: ${lang}` : "Transcribing audio",
|
||||
};
|
||||
},
|
||||
transcribe_audio: (args) => {
|
||||
const lang = args.language ? String(args.language) : null;
|
||||
return {
|
||||
label: "Transcribe Audio",
|
||||
description: lang ? `language: ${lang}` : "Getting precise audio context",
|
||||
};
|
||||
},
|
||||
};
|
||||
|
||||
const TOOL_RESULT_FORMATTERS: Record<
|
||||
|
|
@ -357,6 +364,19 @@ const TOOL_RESULT_FORMATTERS: Record<
|
|||
description: `${data.assetName ?? "audio"} · ${data.segmentCount ?? 0} segments`,
|
||||
};
|
||||
},
|
||||
transcribe_audio: (parsed) => {
|
||||
const data = parsed as {
|
||||
assetName?: string;
|
||||
wordCount?: number;
|
||||
utteranceCount?: number;
|
||||
timingGranularity?: string;
|
||||
} | null;
|
||||
if (!data) return null;
|
||||
return {
|
||||
label: "Audio Context",
|
||||
description: `${data.assetName ?? "audio"} · ${data.wordCount ?? 0} words · ${data.timingGranularity ?? "timed"}`,
|
||||
};
|
||||
},
|
||||
list_keyframes: (parsed) => {
|
||||
const data = parsed as { keyframes?: unknown[] } | null;
|
||||
if (!data) return null;
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import { Input, ALL_FORMATS, BlobSource } from "mediabunny";
|
||||
import { Input, ALL_FORMATS, BlobSource, AudioBufferSink } from "mediabunny";
|
||||
import { createTimelineAudioBuffer } from "@/lib/media/audio";
|
||||
import type { SceneTracks } from "@/lib/timeline";
|
||||
import type { MediaAsset } from "@/lib/media/types";
|
||||
|
|
@ -45,6 +45,50 @@ const NUM_CHANNELS = 2;
|
|||
const EMPTY_TIMELINE_SILENT_DURATION_SECONDS = 0.1;
|
||||
const MIN_SILENT_DURATION_SECONDS = 0.001;
|
||||
|
||||
export async function extractAssetAudio({
|
||||
file,
|
||||
assetType,
|
||||
}: {
|
||||
file: File;
|
||||
assetType: string;
|
||||
}): Promise<Blob> {
|
||||
if (assetType === "audio") {
|
||||
return file;
|
||||
}
|
||||
|
||||
const input = new Input({
|
||||
source: new BlobSource(file),
|
||||
formats: ALL_FORMATS,
|
||||
});
|
||||
|
||||
try {
|
||||
const audioTrack = await input.getPrimaryAudioTrack();
|
||||
if (!audioTrack) {
|
||||
throw new Error("Asset has no audio track");
|
||||
}
|
||||
|
||||
const sink = new AudioBufferSink(audioTrack);
|
||||
const chunks: AudioBuffer[] = [];
|
||||
let totalSamples = 0;
|
||||
|
||||
for await (const { buffer } of sink.buffers(0)) {
|
||||
chunks.push(buffer);
|
||||
totalSamples += buffer.length;
|
||||
}
|
||||
|
||||
if (chunks.length === 0 || totalSamples === 0) {
|
||||
throw new Error("Could not extract audio from asset");
|
||||
}
|
||||
|
||||
return createWavBlob({
|
||||
samples: interleaveAudioChunks({ chunks, totalSamples }),
|
||||
sampleRate: chunks[0].sampleRate,
|
||||
});
|
||||
} finally {
|
||||
input.dispose();
|
||||
}
|
||||
}
|
||||
|
||||
export const extractTimelineAudio = async ({
|
||||
tracks,
|
||||
mediaAssets,
|
||||
|
|
@ -113,7 +157,38 @@ function interleaveAudioBuffer({
|
|||
return interleavedSamples;
|
||||
}
|
||||
|
||||
function createWavBlob({ samples }: { samples: Float32Array }): Blob {
|
||||
function interleaveAudioChunks({
|
||||
chunks,
|
||||
totalSamples,
|
||||
}: {
|
||||
chunks: AudioBuffer[];
|
||||
totalSamples: number;
|
||||
}): Float32Array {
|
||||
const interleavedSamples = new Float32Array(totalSamples * NUM_CHANNELS);
|
||||
let outputSampleIndex = 0;
|
||||
|
||||
for (const chunk of chunks) {
|
||||
const numChannels = Math.min(NUM_CHANNELS, chunk.numberOfChannels);
|
||||
for (let sampleIndex = 0; sampleIndex < chunk.length; sampleIndex++) {
|
||||
for (let channel = 0; channel < NUM_CHANNELS; channel++) {
|
||||
const sourceChannel = Math.min(channel, Math.max(0, numChannels - 1));
|
||||
interleavedSamples[outputSampleIndex * NUM_CHANNELS + channel] =
|
||||
chunk.getChannelData(sourceChannel)[sampleIndex] ?? 0;
|
||||
}
|
||||
outputSampleIndex++;
|
||||
}
|
||||
}
|
||||
|
||||
return interleavedSamples;
|
||||
}
|
||||
|
||||
function createWavBlob({
|
||||
samples,
|
||||
sampleRate = SAMPLE_RATE,
|
||||
}: {
|
||||
samples: Float32Array;
|
||||
sampleRate?: number;
|
||||
}): Blob {
|
||||
const numChannels = NUM_CHANNELS;
|
||||
const bitsPerSample = 16;
|
||||
const bytesPerSample = bitsPerSample / 8;
|
||||
|
|
@ -132,8 +207,8 @@ function createWavBlob({ samples }: { samples: Float32Array }): Blob {
|
|||
view.setUint32(16, 16, true);
|
||||
view.setUint16(20, 1, true);
|
||||
view.setUint16(22, numChannels, true);
|
||||
view.setUint32(24, SAMPLE_RATE, true);
|
||||
view.setUint32(28, SAMPLE_RATE * numChannels * bytesPerSample, true);
|
||||
view.setUint32(24, sampleRate, true);
|
||||
view.setUint32(28, sampleRate * numChannels * bytesPerSample, true);
|
||||
view.setUint16(32, numChannels * bytesPerSample, true);
|
||||
view.setUint16(34, bitsPerSample, true);
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue