;
- if (shapePreset) {
- element = buildGraphicElement({
- definitionId: shapePreset.definitionId,
- name: shapePreset.name,
- startTime: currentTime,
- params: shapePreset.params,
- });
- } else {
- const { width: intrinsicWidth, height: intrinsicHeight } =
- await resolveStickerIntrinsicSize({ stickerId: item.id });
- element = buildStickerElement({
- stickerId: item.id,
- name: item.name,
- startTime: currentTime,
- intrinsicWidth,
- intrinsicHeight,
- });
- }
-
- editor.timeline.insertElement({
- placement: { mode: "auto" },
- element,
- });
-
- addToRecentStickers({ stickerId: item.id });
- } catch (error) {
- console.error("Failed to add sticker:", error);
- toast.error("Failed to add sticker to timeline");
- } finally {
- setIsAdding(false);
- }
- };
-
- const preview = (
-
- {hasImageError ? (
-
- {displayName}
-
- ) : (
- setHasImageError(true)}
- loading="lazy"
- unoptimized
- />
- )}
-
- );
-
- const dragData: TimelineDragData = shapePreset
- ? {
- id: item.id,
- type: "graphic",
- name: displayName,
- definitionId: shapePreset.definitionId,
- params: shapePreset.params ?? {},
- }
- : {
- id: item.id,
- type: "sticker",
- name: displayName,
- stickerId: item.id,
- };
-
- return (
-
-
- {isAdding && (
-
-
-
- )}
-
- );
-}
+"use client";
+
+import Image from "next/image";
+import type { CSSProperties } from "react";
+import { useEffect, useState } from "react";
+import { toast } from "sonner";
+import { DraggableItem } from "@/components/editor/panels/assets/draggable-item";
+import { Button } from "@/components/ui/button";
+import { Input } from "@/components/ui/input";
+import { Spinner } from "@/components/ui/spinner";
+import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs";
+import { useEditor } from "@/hooks/use-editor";
+import { resolveStickerIntrinsicSize } from "@/lib/stickers";
+import {
+ buildGraphicElement,
+ buildStickerElement,
+} from "@/lib/timeline/element-utils";
+import { STICKER_CATEGORIES } from "@/lib/stickers/categories";
+import { getRegionLabel, resolveQueryToRegions } from "@/lib/stickers";
+import { parseShapeStickerId } from "@/lib/stickers/providers/shapes";
+import type { TimelineDragData } from "@/lib/timeline/drag";
+import type {
+ StickerBrowseSection,
+ StickerCategory,
+ StickerItem as StickerData,
+} from "@/lib/stickers";
+import { useStickersStore } from "@/stores/stickers-store";
+import { cn } from "@/utils/ui";
+import {
+ HappyIcon,
+} from "@hugeicons/core-free-icons";
+import { HugeiconsIcon } from "@hugeicons/react";
+
+export function StickersView() {
+ const {
+ browseContent,
+ browseStickers,
+ searchQuery,
+ searchStickers,
+ selectedCategory,
+ setSearchQuery,
+ setSelectedCategory,
+ viewMode,
+ } = useStickersStore();
+
+ useEffect(() => {
+ if (viewMode === "browse" && !browseContent) {
+ void browseStickers();
+ }
+ }, [browseContent, browseStickers, viewMode]);
+
+ return (
+
+
+ {
+ setSearchQuery({ query: e.target.value });
+ void searchStickers({ query: e.target.value });
+ }}
+ showClearIcon
+ onClear={() => {
+ setSearchQuery({ query: "" });
+ void searchStickers({ query: "" });
+ }}
+ className="w-full"
+ containerClassName="w-full"
+ />
+
+
+
{
+ setSelectedCategory({ category: value as StickerCategory });
+ }}
+ variant="underline"
+ className="mt-2 flex min-h-0 flex-1 flex-col"
+ >
+
+ {Object.entries(STICKER_CATEGORIES).map(([key, label]) => (
+
+ {label}
+
+ ))}
+
+
+
+
+
+
+ );
+}
+
+function StickerGrid({
+ items,
+ shouldCapSize = false,
+}: {
+ items: StickerData[];
+ shouldCapSize?: boolean;
+}) {
+ const gridStyle: CSSProperties & {
+ "--sticker-min": string;
+ "--sticker-max"?: string;
+ } = {
+ gridTemplateColumns: shouldCapSize
+ ? "repeat(auto-fill, minmax(var(--sticker-min, 80px), var(--sticker-max, 140px)))"
+ : "repeat(auto-fill, minmax(var(--sticker-min, 80px), 1fr))",
+ "--sticker-min": "80px",
+ ...(shouldCapSize ? { "--sticker-max": "140px" } : {}),
+ };
+
+ return (
+
+ {items.map((item) => (
+
+ ))}
+
+ );
+}
+
+function StickerRow({ items }: { items: StickerData[] }) {
+ return (
+
+ {items.map((item) => (
+
+
+
+ ))}
+
+ );
+}
+
+function EmptyView({ message }: { message: string }) {
+ return (
+
+
+
+
No stickers found
+
{message}
+
+
+ );
+}
+
+function RegionBanner({ region }: { region: string }) {
+ return (
+
+ );
+}
+
+function StickersContentView() {
+ const {
+ browseContent,
+ clearRecentStickers,
+ isBrowsing,
+ isSearching,
+ searchQuery,
+ searchResults,
+ selectedCategory,
+ setSelectedCategory,
+ viewMode,
+ } = useStickersStore();
+
+ if (viewMode === "search") {
+ if (isSearching) {
+ return (
+
+
+
+ );
+ }
+
+ if (searchResults?.items.length) {
+ const normalizedQuery = searchQuery.trim().toLowerCase();
+ const isRegionSearch =
+ selectedCategory === "flags" &&
+ resolveQueryToRegions({ query: normalizedQuery }) !== null;
+ const regionLabel = getRegionLabel({ query: normalizedQuery });
+
+ return (
+
+ {isRegionSearch &&
}
+
+
+ {searchResults.total} results
+
+
+
+
+ );
+ }
+
+ // "all" tab search — sections are in browseContent, fall through to section rendering below
+ if (selectedCategory !== "all" && searchQuery) {
+ return ;
+ }
+ }
+
+ if (isBrowsing && !browseContent) {
+ return (
+
+
+
+ );
+ }
+
+ if (!browseContent?.sections.length) {
+ const categoryLabel = STICKER_CATEGORIES[selectedCategory];
+ return (
+
+ );
+ }
+
+ return (
+
+ {browseContent.sections.map((section) => (
+ {
+ setSelectedCategory({ category });
+ }}
+ />
+ ))}
+
+ );
+}
+
+function StickerSection({
+ section,
+ onClearRecent,
+ onSeeAll,
+}: {
+ section: StickerBrowseSection;
+ onClearRecent: () => void;
+ onSeeAll: (category: StickerCategory) => void;
+}) {
+ const hasHeader =
+ Boolean(section.title) || section.id === "recent" || section.action;
+
+ return (
+
+ {hasHeader && (
+
+ {section.title ? (
+
{section.title}
+ ) : (
+
+ )}
+
+
+ {section.id === "recent" && (
+
+ )}
+
+ {section.action?.type === "see-all" && section.action.category && (
+
+ )}
+
+
+ )}
+
+ {section.layout === "row" ? (
+
+ ) : (
+
+ )}
+
+ );
+}
+
+interface StickerItemProps {
+ item: StickerData;
+ shouldCapSize?: boolean;
+ containerClassName?: string;
+}
+
+function StickerItem({
+ item,
+ shouldCapSize = false,
+ containerClassName,
+}: StickerItemProps) {
+ const editor = useEditor();
+ const { addToRecentStickers } = useStickersStore();
+ const [isAdding, setIsAdding] = useState(false);
+ const [hasImageError, setHasImageError] = useState(false);
+
+ useEffect(() => {
+ if (!item.id) {
+ return;
+ }
+
+ setHasImageError(false);
+ }, [item.id]);
+
+ const displayName = item.name;
+ const shapePreset =
+ item.provider === "shapes" ? parseShapeStickerId({ stickerId: item.id }) : null;
+
+ const handleAdd = async () => {
+ setIsAdding(true);
+ try {
+ const currentTime = editor.playback.getCurrentTime();
+
+ let element:
+ | ReturnType
+ | ReturnType;
+ if (shapePreset) {
+ element = buildGraphicElement({
+ definitionId: shapePreset.definitionId,
+ name: shapePreset.name,
+ startTime: currentTime,
+ params: shapePreset.params,
+ });
+ } else {
+ const { width: intrinsicWidth, height: intrinsicHeight } =
+ await resolveStickerIntrinsicSize({ stickerId: item.id });
+ element = buildStickerElement({
+ stickerId: item.id,
+ name: item.name,
+ startTime: currentTime,
+ intrinsicWidth,
+ intrinsicHeight,
+ });
+ }
+
+ editor.timeline.insertElement({
+ placement: { mode: "auto" },
+ element,
+ });
+
+ addToRecentStickers({ stickerId: item.id });
+ } catch (error) {
+ console.error("Failed to add sticker:", error);
+ toast.error("Failed to add sticker to timeline");
+ } finally {
+ setIsAdding(false);
+ }
+ };
+
+ const preview = (
+
+ {hasImageError ? (
+
+ {displayName}
+
+ ) : (
+ setHasImageError(true)}
+ loading="lazy"
+ unoptimized
+ />
+ )}
+
+ );
+
+ const dragData: TimelineDragData = shapePreset
+ ? {
+ id: item.id,
+ type: "graphic",
+ name: displayName,
+ definitionId: shapePreset.definitionId,
+ params: shapePreset.params ?? {},
+ }
+ : {
+ id: item.id,
+ type: "sticker",
+ name: displayName,
+ stickerId: item.id,
+ };
+
+ return (
+
+
+ {isAdding && (
+
+
+
+ )}
+
+ );
+}
diff --git a/apps/web/src/components/ui/button.tsx b/apps/web/src/components/ui/button.tsx
index 677d9330..21b2c434 100644
--- a/apps/web/src/components/ui/button.tsx
+++ b/apps/web/src/components/ui/button.tsx
@@ -14,6 +14,7 @@ const buttonVariants = cva(
"bg-destructive text-destructive-foreground hover:bg-destructive/80",
"destructive-foreground":
"border bg-background hover:bg-destructive/15 text-destructive",
+ caution: "text-caution hover:bg-caution/10",
outline: "border border-border bg-background hover:bg-accent",
secondary:
"bg-secondary text-secondary-foreground border border-secondary-border",
diff --git a/apps/web/src/core/index.ts b/apps/web/src/core/index.ts
index e93b937d..483a26c1 100644
--- a/apps/web/src/core/index.ts
+++ b/apps/web/src/core/index.ts
@@ -9,8 +9,10 @@ import { SaveManager } from "./managers/save-manager";
import { AudioManager } from "./managers/audio-manager";
import { SelectionManager } from "./managers/selection-manager";
import { ClipboardManager } from "./managers/clipboard-manager";
+import { DiagnosticsManager } from "./managers/diagnostics-manager";
import { registerDefaultEffects } from "@/lib/effects";
import { registerDefaultMasks } from "@/lib/masks";
+import { registerTranscriptionDiagnostics } from "@/lib/transcription/diagnostics";
export class EditorCore {
private static instance: EditorCore | null = null;
@@ -25,6 +27,7 @@ export class EditorCore {
public readonly audio: AudioManager;
public readonly selection: SelectionManager;
public readonly clipboard: ClipboardManager;
+ public readonly diagnostics: DiagnosticsManager;
private constructor() {
registerDefaultEffects();
@@ -40,6 +43,8 @@ export class EditorCore {
this.audio = new AudioManager(this);
this.selection = new SelectionManager(this);
this.clipboard = new ClipboardManager(this);
+ this.diagnostics = new DiagnosticsManager(this);
+ registerTranscriptionDiagnostics({ diagnostics: this.diagnostics });
this.playback.bindTimelineScope();
this.command.registerReactor(() => {
const activeScene = this.scenes.getActiveSceneOrNull();
diff --git a/apps/web/src/core/managers/diagnostics-manager.ts b/apps/web/src/core/managers/diagnostics-manager.ts
new file mode 100644
index 00000000..d022a350
--- /dev/null
+++ b/apps/web/src/core/managers/diagnostics-manager.ts
@@ -0,0 +1,38 @@
+import type { EditorCore } from "@/core";
+import type { DiagnosticDefinition } from "@/lib/diagnostics/types";
+
+interface DiagnosticRegistration extends DiagnosticDefinition {
+ check: (editor: EditorCore) => boolean;
+}
+
+export class DiagnosticsManager {
+ private readonly registrations: DiagnosticRegistration[] = [];
+ private readonly listeners = new Set<() => void>();
+
+ constructor(private editor: EditorCore) {}
+
+ register(registration: DiagnosticRegistration): void {
+ this.registrations.push(registration);
+ this.notify();
+ }
+
+ getActive(options?: { scope?: string }): ReadonlyArray {
+ const candidates =
+ options?.scope !== undefined
+ ? this.registrations.filter((r) => r.scope === options.scope)
+ : this.registrations;
+
+ return candidates.filter((r) => r.check(this.editor));
+ }
+
+ subscribe(listener: () => void): () => void {
+ this.listeners.add(listener);
+ return () => this.listeners.delete(listener);
+ }
+
+ notify(): void {
+ this.listeners.forEach((listener) => {
+ listener();
+ });
+ }
+}
diff --git a/apps/web/src/hooks/use-editor.ts b/apps/web/src/hooks/use-editor.ts
index 1a5ca8ae..ce48e036 100644
--- a/apps/web/src/hooks/use-editor.ts
+++ b/apps/web/src/hooks/use-editor.ts
@@ -32,6 +32,7 @@ export function useEditor(
editor.renderer.subscribe(onChange),
editor.selection.subscribe(onChange),
editor.clipboard.subscribe(onChange),
+ editor.diagnostics.subscribe(onChange),
];
return () => {
unsubscribers.forEach((unsubscribe) => {
diff --git a/apps/web/src/lib/diagnostics/types.ts b/apps/web/src/lib/diagnostics/types.ts
new file mode 100644
index 00000000..486f362c
--- /dev/null
+++ b/apps/web/src/lib/diagnostics/types.ts
@@ -0,0 +1,8 @@
+export type DiagnosticSeverity = "caution" | "error";
+
+export interface DiagnosticDefinition {
+ id: string;
+ scope: string;
+ severity: DiagnosticSeverity;
+ message: string;
+}
diff --git a/apps/web/src/lib/media/audio.ts b/apps/web/src/lib/media/audio.ts
index 5a45b6ea..04ea580f 100644
--- a/apps/web/src/lib/media/audio.ts
+++ b/apps/web/src/lib/media/audio.ts
@@ -1,5 +1,6 @@
import type {
AudioElement,
+ VideoElement,
LibraryAudioElement,
RetimeConfig,
SceneTracks,
@@ -12,9 +13,7 @@ import {
hasAnimatedVolume,
resolveEffectiveAudioGain,
} from "@/lib/timeline/audio-state";
-import {
- doesElementHaveEnabledAudio,
-} from "@/lib/timeline/audio-separation";
+import { doesElementHaveEnabledAudio } from "@/lib/timeline/audio-separation";
import { canElementHaveAudio, hasMediaId } from "@/lib/timeline/element-utils";
import { canTrackHaveAudio } from "@/lib/timeline";
import { mediaSupportsAudio } from "@/lib/media/media-utils";
@@ -83,22 +82,23 @@ export async function decodeAudioToFloat32({
return { samples, sampleRate: audioBuffer.sampleRate };
}
-export async function collectAudioElements({
+export interface AudibleElementCandidate {
+ element: AudioElement | VideoElement;
+ mediaAsset: MediaAsset | null;
+}
+
+export function collectAudibleCandidates({
tracks,
mediaAssets,
- audioContext,
}: {
tracks: SceneTracks;
mediaAssets: MediaAsset[];
- audioContext: AudioContext;
-}): Promise {
- const orderedTracks = [...tracks.overlay, tracks.main, ...tracks.audio];
- const mediaMap = new Map(
- mediaAssets.map((media) => [media.id, media]),
- );
- const pendingElements: Array> = [];
+}): AudibleElementCandidate[] {
+ const allTracks = [...tracks.overlay, tracks.main, ...tracks.audio];
+ const mediaMap = new Map(mediaAssets.map((a) => [a.id, a]));
+ const candidates: AudibleElementCandidate[] = [];
- for (const track of orderedTracks) {
+ for (const track of allTracks) {
if (canTrackHaveAudio(track) && track.muted) continue;
for (const element of track.elements) {
@@ -110,65 +110,95 @@ export async function collectAudioElements({
: null;
if (!doesElementHaveEnabledAudio({ element, mediaAsset })) continue;
- const isTrackMuted = canTrackHaveAudio(track) && track.muted;
+ candidates.push({ element, mediaAsset });
+ }
+ }
- if (element.type === "audio") {
- pendingElements.push(
- resolveAudioBufferForElement({
- element,
- mediaMap,
- audioContext,
- }).then((audioBuffer) => {
- if (!audioBuffer) return null;
- const muted = element.muted === true || isTrackMuted;
- return {
- timelineElement: element,
- buffer: audioBuffer,
- startTime: element.startTime / TICKS_PER_SECOND,
- duration: element.duration / TICKS_PER_SECOND,
- trimStart: element.trimStart / TICKS_PER_SECOND,
- trimEnd: element.trimEnd / TICKS_PER_SECOND,
- volume: resolveEffectiveAudioGain({
- element,
- trackMuted: isTrackMuted,
- localTime: 0,
- }),
- muted,
- retime: element.retime,
- };
- }),
- );
- continue;
- }
+ return candidates;
+}
- if (element.type === "video") {
- if (!mediaAsset || !mediaSupportsAudio({ media: mediaAsset })) continue;
+export function timelineHasAudio({
+ tracks,
+ mediaAssets,
+}: {
+ tracks: SceneTracks;
+ mediaAssets: MediaAsset[];
+}): boolean {
+ return collectAudibleCandidates({ tracks, mediaAssets }).some(
+ ({ element }) => element.muted !== true,
+ );
+}
- pendingElements.push(
- resolveAudioBufferForVideoElement({
- mediaAsset,
- audioContext,
- }).then((audioBuffer) => {
- if (!audioBuffer) return null;
- const muted = (element.muted ?? false) || isTrackMuted;
- return {
- timelineElement: element,
- buffer: audioBuffer,
- startTime: element.startTime / TICKS_PER_SECOND,
- duration: element.duration / TICKS_PER_SECOND,
- trimStart: element.trimStart / TICKS_PER_SECOND,
- trimEnd: element.trimEnd / TICKS_PER_SECOND,
- volume: resolveEffectiveAudioGain({
- element,
- trackMuted: isTrackMuted,
- localTime: 0,
- }),
- muted,
- retime: element.retime,
- };
- }),
- );
- }
+export async function collectAudioElements({
+ tracks,
+ mediaAssets,
+ audioContext,
+}: {
+ tracks: SceneTracks;
+ mediaAssets: MediaAsset[];
+ audioContext: AudioContext;
+}): Promise {
+ const candidates = collectAudibleCandidates({ tracks, mediaAssets });
+ const mediaMap = new Map(
+ mediaAssets.map((media) => [media.id, media]),
+ );
+ const pendingElements: Array> = [];
+
+ for (const { element, mediaAsset } of candidates) {
+ if (element.type === "audio") {
+ pendingElements.push(
+ resolveAudioBufferForElement({
+ element,
+ mediaMap,
+ audioContext,
+ }).then((audioBuffer) => {
+ if (!audioBuffer) return null;
+ return {
+ timelineElement: element,
+ buffer: audioBuffer,
+ startTime: element.startTime / TICKS_PER_SECOND,
+ duration: element.duration / TICKS_PER_SECOND,
+ trimStart: element.trimStart / TICKS_PER_SECOND,
+ trimEnd: element.trimEnd / TICKS_PER_SECOND,
+ volume: resolveEffectiveAudioGain({
+ element,
+ trackMuted: false,
+ localTime: 0,
+ }),
+ muted: element.muted === true,
+ retime: element.retime,
+ };
+ }),
+ );
+ continue;
+ }
+
+ if (element.type === "video") {
+ if (!mediaAsset || !mediaSupportsAudio({ media: mediaAsset })) continue;
+
+ pendingElements.push(
+ resolveAudioBufferForVideoElement({
+ mediaAsset,
+ audioContext,
+ }).then((audioBuffer) => {
+ if (!audioBuffer) return null;
+ return {
+ timelineElement: element,
+ buffer: audioBuffer,
+ startTime: element.startTime / TICKS_PER_SECOND,
+ duration: element.duration / TICKS_PER_SECOND,
+ trimStart: element.trimStart / TICKS_PER_SECOND,
+ trimEnd: element.trimEnd / TICKS_PER_SECOND,
+ volume: resolveEffectiveAudioGain({
+ element,
+ trackMuted: false,
+ localTime: 0,
+ }),
+ muted: element.muted ?? false,
+ retime: element.retime,
+ };
+ }),
+ );
}
}
@@ -339,16 +369,16 @@ async function fetchLibraryAudioSource({
type: "audio/mpeg",
});
- return {
- timelineElement: element,
- file,
- startTime: element.startTime / TICKS_PER_SECOND,
- duration: element.duration / TICKS_PER_SECOND,
- trimStart: element.trimStart / TICKS_PER_SECOND,
- trimEnd: element.trimEnd / TICKS_PER_SECOND,
- volume,
- retime: element.retime,
- };
+ return {
+ timelineElement: element,
+ file,
+ startTime: element.startTime / TICKS_PER_SECOND,
+ duration: element.duration / TICKS_PER_SECOND,
+ trimStart: element.trimStart / TICKS_PER_SECOND,
+ trimEnd: element.trimEnd / TICKS_PER_SECOND,
+ volume,
+ retime: element.retime,
+ };
} catch (error) {
console.warn("Failed to fetch library audio:", error);
return null;
diff --git a/apps/web/src/lib/media/mediabunny.ts b/apps/web/src/lib/media/mediabunny.ts
index a313c6f0..1e97fc42 100644
--- a/apps/web/src/lib/media/mediabunny.ts
+++ b/apps/web/src/lib/media/mediabunny.ts
@@ -1,158 +1,168 @@
-import { Input, ALL_FORMATS, BlobSource } from "mediabunny";
-import { createTimelineAudioBuffer } from "@/lib/media/audio";
-import type { SceneTracks } from "@/lib/timeline";
-import type { MediaAsset } from "@/lib/media/types";
-
-export async function getVideoInfo({
- videoFile,
-}: {
- videoFile: File;
-}): Promise<{
- duration: number;
- width: number;
- height: number;
- fps: number;
- hasAudio: boolean;
-}> {
- const input = new Input({
- source: new BlobSource(videoFile),
- formats: ALL_FORMATS,
- });
-
- const duration = await input.computeDuration();
- const videoTrack = await input.getPrimaryVideoTrack();
-
- if (!videoTrack) {
- throw new Error("No video track found in the file");
- }
-
- const packetStats = await videoTrack.computePacketStats(100);
- const fps = packetStats.averagePacketRate;
- const audioTrack = await input.getPrimaryAudioTrack();
-
- return {
- duration,
- width: videoTrack.displayWidth,
- height: videoTrack.displayHeight,
- fps,
- hasAudio: audioTrack !== null,
- };
-}
-
-const SAMPLE_RATE = 44100;
-const NUM_CHANNELS = 2;
-
-export const extractTimelineAudio = async ({
- tracks,
- mediaAssets,
- totalDuration,
- onProgress,
-}: {
- tracks: SceneTracks;
- mediaAssets: MediaAsset[];
- totalDuration: number;
- onProgress?: (progress: number) => void;
-}): Promise => {
- if (totalDuration === 0) {
- return createWavBlob({ samples: new Float32Array(SAMPLE_RATE * 0.1) });
- }
-
- onProgress?.(10);
-
- const audioBuffer = await createTimelineAudioBuffer({
- tracks,
- mediaAssets,
- duration: totalDuration,
- sampleRate: SAMPLE_RATE,
- });
-
- if (!audioBuffer) {
- const silentDuration = Math.max(1, totalDuration);
- const silentSamples = new Float32Array(
- Math.ceil(silentDuration * SAMPLE_RATE) * NUM_CHANNELS,
- );
- return createWavBlob({ samples: silentSamples });
- }
-
- onProgress?.(90);
-
- const interleavedSamples = interleaveAudioBuffer({ audioBuffer });
- onProgress?.(100);
-
- return createWavBlob({ samples: interleavedSamples });
-};
-
-function interleaveAudioBuffer({
- audioBuffer,
-}: {
- audioBuffer: AudioBuffer;
-}): Float32Array {
- const numChannels = Math.min(NUM_CHANNELS, audioBuffer.numberOfChannels);
- const interleavedSamples = new Float32Array(
- audioBuffer.length * NUM_CHANNELS,
- );
-
- for (let sampleIndex = 0; sampleIndex < audioBuffer.length; sampleIndex++) {
- for (let channel = 0; channel < NUM_CHANNELS; channel++) {
- const sourceChannel = Math.min(channel, Math.max(0, numChannels - 1));
- interleavedSamples[sampleIndex * NUM_CHANNELS + channel] =
- audioBuffer.getChannelData(sourceChannel)[sampleIndex] ?? 0;
- }
- }
-
- return interleavedSamples;
-}
-
-function createWavBlob({ samples }: { samples: Float32Array }): Blob {
- const numChannels = NUM_CHANNELS;
- const bitsPerSample = 16;
- const bytesPerSample = bitsPerSample / 8;
- const numSamples = samples.length / numChannels;
- const dataSize = numSamples * numChannels * bytesPerSample;
- const buffer = new ArrayBuffer(44 + dataSize);
- const view = new DataView(buffer);
-
- // riff header
- writeString({ view, offset: 0, str: "RIFF" });
- view.setUint32(4, 36 + dataSize, true);
- writeString({ view, offset: 8, str: "WAVE" });
-
- // fmt chunk
- writeString({ view, offset: 12, str: "fmt " });
- view.setUint32(16, 16, true);
- view.setUint16(20, 1, true);
- view.setUint16(22, numChannels, true);
- view.setUint32(24, SAMPLE_RATE, true);
- view.setUint32(28, SAMPLE_RATE * numChannels * bytesPerSample, true);
- view.setUint16(32, numChannels * bytesPerSample, true);
- view.setUint16(34, bitsPerSample, true);
-
- // data chunk
- writeString({ view, offset: 36, str: "data" });
- view.setUint32(40, dataSize, true);
-
- // convert float32 to int16 and write
- let offset = 44;
- for (let i = 0; i < samples.length; i++) {
- const sample = Math.max(-1, Math.min(1, samples[i]));
- const int16 = sample < 0 ? sample * 0x8000 : sample * 0x7fff;
- view.setInt16(offset, int16, true);
- offset += 2;
- }
-
- return new Blob([buffer], { type: "audio/wav" });
-}
-
-function writeString({
- view,
- offset,
- str,
-}: {
- view: DataView;
- offset: number;
- str: string;
-}): void {
- for (let i = 0; i < str.length; i++) {
- view.setUint8(offset + i, str.charCodeAt(i));
- }
-}
+import { Input, ALL_FORMATS, BlobSource } from "mediabunny";
+import { createTimelineAudioBuffer } from "@/lib/media/audio";
+import type { SceneTracks } from "@/lib/timeline";
+import type { MediaAsset } from "@/lib/media/types";
+import { TICKS_PER_SECOND } from "@/lib/wasm";
+
+export async function getVideoInfo({
+ videoFile,
+}: {
+ videoFile: File;
+}): Promise<{
+ duration: number;
+ width: number;
+ height: number;
+ fps: number;
+ hasAudio: boolean;
+}> {
+ const input = new Input({
+ source: new BlobSource(videoFile),
+ formats: ALL_FORMATS,
+ });
+
+ const duration = await input.computeDuration();
+ const videoTrack = await input.getPrimaryVideoTrack();
+
+ if (!videoTrack) {
+ throw new Error("No video track found in the file");
+ }
+
+ const packetStats = await videoTrack.computePacketStats(100);
+ const fps = packetStats.averagePacketRate;
+ const audioTrack = await input.getPrimaryAudioTrack();
+
+ return {
+ duration,
+ width: videoTrack.displayWidth,
+ height: videoTrack.displayHeight,
+ fps,
+ hasAudio: audioTrack !== null,
+ };
+}
+
+const SAMPLE_RATE = 44100;
+const NUM_CHANNELS = 2;
+const EMPTY_TIMELINE_SILENT_DURATION_SECONDS = 0.1;
+const MIN_SILENT_DURATION_SECONDS = 0.001;
+
+export const extractTimelineAudio = async ({
+ tracks,
+ mediaAssets,
+ totalDuration,
+ onProgress,
+}: {
+ tracks: SceneTracks;
+ mediaAssets: MediaAsset[];
+ totalDuration: number;
+ onProgress?: (progress: number) => void;
+}): Promise => {
+ if (totalDuration === 0) {
+ return createWavBlob({
+ samples: new Float32Array(
+ SAMPLE_RATE * EMPTY_TIMELINE_SILENT_DURATION_SECONDS,
+ ),
+ });
+ }
+
+ onProgress?.(10);
+
+ const audioBuffer = await createTimelineAudioBuffer({
+ tracks,
+ mediaAssets,
+ duration: totalDuration,
+ sampleRate: SAMPLE_RATE,
+ });
+
+ if (!audioBuffer) {
+ const silentDurationSeconds = Math.max(
+ MIN_SILENT_DURATION_SECONDS,
+ totalDuration / TICKS_PER_SECOND,
+ );
+ const silentSamples = new Float32Array(
+ Math.ceil(silentDurationSeconds * SAMPLE_RATE) * NUM_CHANNELS,
+ );
+ return createWavBlob({ samples: silentSamples });
+ }
+
+ onProgress?.(90);
+
+ const interleavedSamples = interleaveAudioBuffer({ audioBuffer });
+ onProgress?.(100);
+
+ return createWavBlob({ samples: interleavedSamples });
+};
+
+function interleaveAudioBuffer({
+ audioBuffer,
+}: {
+ audioBuffer: AudioBuffer;
+}): Float32Array {
+ const numChannels = Math.min(NUM_CHANNELS, audioBuffer.numberOfChannels);
+ const interleavedSamples = new Float32Array(
+ audioBuffer.length * NUM_CHANNELS,
+ );
+
+ for (let sampleIndex = 0; sampleIndex < audioBuffer.length; sampleIndex++) {
+ for (let channel = 0; channel < NUM_CHANNELS; channel++) {
+ const sourceChannel = Math.min(channel, Math.max(0, numChannels - 1));
+ interleavedSamples[sampleIndex * NUM_CHANNELS + channel] =
+ audioBuffer.getChannelData(sourceChannel)[sampleIndex] ?? 0;
+ }
+ }
+
+ return interleavedSamples;
+}
+
+function createWavBlob({ samples }: { samples: Float32Array }): Blob {
+ const numChannels = NUM_CHANNELS;
+ const bitsPerSample = 16;
+ const bytesPerSample = bitsPerSample / 8;
+ const numSamples = samples.length / numChannels;
+ const dataSize = numSamples * numChannels * bytesPerSample;
+ const buffer = new ArrayBuffer(44 + dataSize);
+ const view = new DataView(buffer);
+
+ // riff header
+ writeString({ view, offset: 0, str: "RIFF" });
+ view.setUint32(4, 36 + dataSize, true);
+ writeString({ view, offset: 8, str: "WAVE" });
+
+ // fmt chunk
+ writeString({ view, offset: 12, str: "fmt " });
+ view.setUint32(16, 16, true);
+ view.setUint16(20, 1, true);
+ view.setUint16(22, numChannels, true);
+ view.setUint32(24, SAMPLE_RATE, true);
+ view.setUint32(28, SAMPLE_RATE * numChannels * bytesPerSample, true);
+ view.setUint16(32, numChannels * bytesPerSample, true);
+ view.setUint16(34, bitsPerSample, true);
+
+ // data chunk
+ writeString({ view, offset: 36, str: "data" });
+ view.setUint32(40, dataSize, true);
+
+ // convert float32 to int16 and write
+ let offset = 44;
+ for (let i = 0; i < samples.length; i++) {
+ const sample = Math.max(-1, Math.min(1, samples[i]));
+ const int16 = sample < 0 ? sample * 0x8000 : sample * 0x7fff;
+ view.setInt16(offset, int16, true);
+ offset += 2;
+ }
+
+ return new Blob([buffer], { type: "audio/wav" });
+}
+
+function writeString({
+ view,
+ offset,
+ str,
+}: {
+ view: DataView;
+ offset: number;
+ str: string;
+}): void {
+ for (let i = 0; i < str.length; i++) {
+ view.setUint8(offset + i, str.charCodeAt(i));
+ }
+}
diff --git a/apps/web/src/lib/timeline/audio-separation/index.ts b/apps/web/src/lib/timeline/audio-separation/index.ts
index fe3c1ca4..849dbda3 100644
--- a/apps/web/src/lib/timeline/audio-separation/index.ts
+++ b/apps/web/src/lib/timeline/audio-separation/index.ts
@@ -49,7 +49,9 @@ export function canToggleSourceAudio(
element: TimelineElement,
mediaAsset: MediaAudioState | null | undefined,
): element is VideoElement {
- return canRecoverSourceAudio(element) || canExtractSourceAudio(element, mediaAsset);
+ return (
+ canRecoverSourceAudio(element) || canExtractSourceAudio(element, mediaAsset)
+ );
}
export function doesElementHaveEnabledAudio({
@@ -63,7 +65,11 @@ export function doesElementHaveEnabledAudio({
return true;
}
- return !!mediaAsset && mediaAsset.hasAudio !== false && isSourceAudioEnabled({ element });
+ return (
+ !!mediaAsset &&
+ mediaAsset.hasAudio !== false &&
+ isSourceAudioEnabled({ element })
+ );
}
export function buildSeparatedAudioElement({
@@ -100,7 +106,9 @@ export function getSourceAudioActionLabel({
}: {
element: VideoElement;
}): "Extract audio" | "Recover audio" {
- return isSourceAudioSeparated({ element }) ? "Recover audio" : "Extract audio";
+ return isSourceAudioSeparated({ element })
+ ? "Recover audio"
+ : "Extract audio";
}
function cloneVolumeAnimations({
diff --git a/apps/web/src/lib/transcription/diagnostics.ts b/apps/web/src/lib/transcription/diagnostics.ts
new file mode 100644
index 00000000..a1921699
--- /dev/null
+++ b/apps/web/src/lib/transcription/diagnostics.ts
@@ -0,0 +1,25 @@
+import type { DiagnosticsManager } from "@/core/managers/diagnostics-manager";
+import { timelineHasAudio } from "@/lib/media/audio";
+
+export const TRANSCRIPTION_DIAGNOSTICS_SCOPE = "transcription";
+
+export function registerTranscriptionDiagnostics({
+ diagnostics,
+}: {
+ diagnostics: DiagnosticsManager;
+}): void {
+ diagnostics.register({
+ id: "transcription.no_audio",
+ scope: TRANSCRIPTION_DIAGNOSTICS_SCOPE,
+ severity: "caution",
+ message: "No audio detected. Add a clip with audio to the timeline first.",
+ check: (editor) => {
+ const scene = editor.scenes.getActiveSceneOrNull();
+ if (!scene) return false;
+ return !timelineHasAudio({
+ tracks: scene.tracks,
+ mediaAssets: editor.media.getAssets(),
+ });
+ },
+ });
+}