From 70a9bfebb196d5c0bde35ce0a165b696bb9ae1d0 Mon Sep 17 00:00:00 2001
From: Maze Winther <mazewinther@gmail.com>
Date: Thu, 28 Aug 2025 21:52:03 +0200
Subject: [PATCH] feat: add audio support for export

---
 .../src/components/editor/export-button.tsx   |  22 +
 apps/web/src/lib/export.ts                    | 443 ++++++++++++------
 apps/web/src/types/export.ts                  |  35 +-
 3 files changed, 336 insertions(+), 164 deletions(-)

diff --git a/apps/web/src/components/editor/export-button.tsx b/apps/web/src/components/editor/export-button.tsx
index e24d20df..c9738e81 100644
--- a/apps/web/src/components/editor/export-button.tsx
+++ b/apps/web/src/components/editor/export-button.tsx
@@ -7,6 +7,7 @@ import { Button } from "../ui/button";
 import { Label } from "../ui/label";
 import { RadioGroup, RadioGroupItem } from "../ui/radio-group";
 import { Progress } from "../ui/progress";
+import { Checkbox } from "../ui/checkbox";
 import { cn } from "@/lib/utils";
 import {
   exportProject,
@@ -75,6 +76,9 @@ function ExportPopover({
   const [quality, setQuality] = useState<ExportQuality>(
     DEFAULT_EXPORT_OPTIONS.quality
   );
+  const [includeAudio, setIncludeAudio] = useState<boolean>(
+    DEFAULT_EXPORT_OPTIONS.includeAudio || true
+  );
   const [isExporting, setIsExporting] = useState(false);
   const [progress, setProgress] = useState(0);
   const [exportResult, setExportResult] = useState<ExportResult | null>(null);
@@ -90,6 +94,7 @@ function ExportPopover({
       format,
       quality,
       fps: activeProject.fps,
+      includeAudio,
       onProgress: setProgress,
       onCancel: () => false, // TODO: Add cancel functionality
     });
@@ -197,6 +202,23 @@ function ExportPopover({
                     </div>
                   </RadioGroup>
                 </PropertyGroup>
+
+                <PropertyGroup
+                  title="Audio"
+                  titleClassName="text-sm"
+                  defaultExpanded={false}
+                >
+                  <div className="flex items-center space-x-2">
+                    <Checkbox
+                      id="include-audio"
+                      checked={includeAudio}
+                      onCheckedChange={(checked) => setIncludeAudio(!!checked)}
+                    />
+                    <Label htmlFor="include-audio">
+                      Include audio in export
+                    </Label>
+                  </div>
+                </PropertyGroup>
               </div>
 
               <Button onClick={handleExport} className="w-full gap-2">
diff --git a/apps/web/src/lib/export.ts b/apps/web/src/lib/export.ts
index 69bc8d1d..f25f5f6b 100644
--- a/apps/web/src/lib/export.ts
+++ b/apps/web/src/lib/export.ts
@@ -1,147 +1,296 @@
-import {
-  Output,
-  Mp4OutputFormat,
-  WebMOutputFormat,
-  BufferTarget,
-  CanvasSource,
-  QUALITY_LOW,
-  QUALITY_MEDIUM,
-  QUALITY_HIGH,
-  QUALITY_VERY_HIGH,
-} from "mediabunny";
-import { renderTimelineFrame } from "./timeline-renderer";
-import { useTimelineStore } from "@/stores/timeline-store";
-import { useMediaStore } from "@/stores/media-store";
-import { useProjectStore } from "@/stores/project-store";
-import { DEFAULT_FPS } from "@/stores/project-store";
-import { ExportOptions, ExportResult } from "@/types/export";
-
-export const DEFAULT_EXPORT_OPTIONS: ExportOptions = {
-  format: "mp4",
-  quality: "high",
-};
-
-const qualityMap = {
-  low: QUALITY_LOW,
-  medium: QUALITY_MEDIUM,
-  high: QUALITY_HIGH,
-  very_high: QUALITY_VERY_HIGH,
-};
-
-export async function exportProject(
-  options: ExportOptions
-): Promise<ExportResult> {
-  const { format, quality, fps, onProgress, onCancel } = options;
-
-  try {
-    const timelineStore = useTimelineStore.getState();
-    const mediaStore = useMediaStore.getState();
-    const projectStore = useProjectStore.getState();
-
-    const { tracks, getTotalDuration } = timelineStore;
-    const { mediaFiles } = mediaStore;
-    const { activeProject } = projectStore;
-
-    if (!activeProject) {
-      return { success: false, error: "No active project" };
-    }
-
-    const duration = getTotalDuration();
-    if (duration === 0) {
-      return { success: false, error: "Project is empty" };
-    }
-
-    const exportFps = fps || activeProject.fps || DEFAULT_FPS;
-    const canvasSize = activeProject.canvasSize;
-
-    const outputFormat =
-      format === "webm" ? new WebMOutputFormat() : new Mp4OutputFormat();
-
-    // BufferTarget for smaller files, StreamTarget for larger ones
-    // TODO: Implement StreamTarget
-    const output = new Output({
-      format: outputFormat,
-      target: new BufferTarget(),
-    });
-
-    // Canvas for rendering
-    const canvas = document.createElement("canvas");
-    canvas.width = canvasSize.width;
-    canvas.height = canvasSize.height;
-    const ctx = canvas.getContext("2d");
-
-    if (!ctx) {
-      return { success: false, error: "Failed to create canvas context" };
-    }
-
-    const videoSource = new CanvasSource(canvas, {
-      codec: format === "webm" ? "vp9" : "avc", // VP9 for WebM, H.264 for MP4
-      bitrate: qualityMap[quality],
-    });
-
-    output.addVideoTrack(videoSource, { frameRate: exportFps });
-
-    // Start the output
-    await output.start();
-
-    const totalFrames = Math.ceil(duration * exportFps);
-    let cancelled = false;
-
-    // Render each frame
-    for (let frameIndex = 0; frameIndex < totalFrames; frameIndex++) {
-      // Check for cancellation
-      if (onCancel?.()) {
-        cancelled = true;
-        break;
-      }
-
-      const time = frameIndex / exportFps;
-
-      await renderTimelineFrame({
-        ctx,
-        time,
-        canvasWidth: canvas.width,
-        canvasHeight: canvas.height,
-        tracks,
-        mediaFiles,
-        backgroundColor:
-          activeProject.backgroundType === "blur"
-            ? "transparent"
-            : activeProject.backgroundColor || "#000000",
-        projectCanvasSize: canvasSize,
-      });
-
-      const frameDuration = 1 / exportFps;
-      await videoSource.add(time, frameDuration);
-
-      onProgress?.(frameIndex / totalFrames);
-    }
-
-    if (cancelled) {
-      await output.cancel();
-      return { success: false, cancelled: true };
-    }
-    videoSource.close();
-    await output.finalize();
-    onProgress?.(1);
-
-    return {
-      success: true,
-      buffer: output.target.buffer || undefined,
-    };
-  } catch (error) {
-    console.error("Export failed:", error);
-    return {
-      success: false,
-      error: error instanceof Error ? error.message : "Unknown export error",
-    };
-  }
-}
-
-export function getExportMimeType(format: "mp4" | "webm"): string {
-  return format === "webm" ? "video/webm" : "video/mp4";
-}
-
-export function getExportFileExtension(format: "mp4" | "webm"): string {
-  return `.${format}`;
-}
+import {
+  Output,
+  Mp4OutputFormat,
+  WebMOutputFormat,
+  BufferTarget,
+  CanvasSource,
+  AudioBufferSource,
+  QUALITY_LOW,
+  QUALITY_MEDIUM,
+  QUALITY_HIGH,
+  QUALITY_VERY_HIGH,
+} from "mediabunny";
+import { renderTimelineFrame } from "./timeline-renderer";
+import { useTimelineStore } from "@/stores/timeline-store";
+import { useMediaStore } from "@/stores/media-store";
+import { useProjectStore } from "@/stores/project-store";
+import { DEFAULT_FPS } from "@/stores/project-store";
+import { ExportOptions, ExportResult } from "@/types/export";
+
+export const DEFAULT_EXPORT_OPTIONS: ExportOptions = {
+  format: "mp4",
+  quality: "high",
+  includeAudio: true,
+};
+
+const qualityMap = {
+  low: QUALITY_LOW,
+  medium: QUALITY_MEDIUM,
+  high: QUALITY_HIGH,
+  very_high: QUALITY_VERY_HIGH,
+};
+
+interface AudioElement {
+  buffer: AudioBuffer;
+  startTime: number;
+  duration: number;
+  trimStart: number;
+  trimEnd: number;
+  muted: boolean;
+}
+
+async function createTimelineAudioBuffer(
+  tracks: any[],
+  mediaFiles: any[],
+  duration: number,
+  sampleRate: number = 44100
+): Promise<AudioBuffer | null> {
+  // Get Web Audio context
+  const audioContext = new (window.AudioContext ||
+    (window as any).webkitAudioContext)();
+
+  // Collect all audio elements from timeline
+  const audioElements: AudioElement[] = [];
+  const mediaMap = new Map(mediaFiles.map((m) => [m.id, m]));
+
+  for (const track of tracks) {
+    if (track.muted) continue;
+
+    for (const element of track.elements) {
+      if (element.type !== "media") continue;
+
+      const mediaItem = mediaMap.get(element.mediaId);
+      if (!mediaItem || mediaItem.type !== "audio") continue;
+
+      const visibleDuration =
+        element.duration - element.trimStart - element.trimEnd;
+      if (visibleDuration <= 0) continue;
+
+      try {
+        // Decode audio file
+        const arrayBuffer = await mediaItem.file.arrayBuffer();
+        const audioBuffer = await audioContext.decodeAudioData(
+          arrayBuffer.slice(0)
+        );
+
+        audioElements.push({
+          buffer: audioBuffer,
+          startTime: element.startTime,
+          duration: element.duration,
+          trimStart: element.trimStart,
+          trimEnd: element.trimEnd,
+          muted: element.muted || track.muted || false,
+        });
+      } catch (error) {
+        console.warn(`Failed to decode audio file ${mediaItem.name}:`, error);
+      }
+    }
+  }
+
+  if (audioElements.length === 0) {
+    return null; // No audio to mix
+  }
+
+  // Create output buffer
+  const outputChannels = 2; // Stereo
+  const outputLength = Math.ceil(duration * sampleRate);
+  const outputBuffer = audioContext.createBuffer(
+    outputChannels,
+    outputLength,
+    sampleRate
+  );
+
+  // Mix all audio elements
+  for (const element of audioElements) {
+    if (element.muted) continue;
+
+    const {
+      buffer,
+      startTime,
+      trimStart,
+      trimEnd,
+      duration: elementDuration,
+    } = element;
+
+    // Calculate timing
+    const sourceStartSample = Math.floor(trimStart * buffer.sampleRate);
+    const sourceDuration = elementDuration - trimStart - trimEnd;
+    const sourceLengthSamples = Math.floor(sourceDuration * buffer.sampleRate);
+    const outputStartSample = Math.floor(startTime * sampleRate);
+
+    // Resample if needed (simple approach)
+    const resampleRatio = sampleRate / buffer.sampleRate;
+    const resampledLength = Math.floor(sourceLengthSamples * resampleRatio);
+
+    // Mix each channel
+    for (let channel = 0; channel < outputChannels; channel++) {
+      const outputData = outputBuffer.getChannelData(channel);
+      const sourceChannel = Math.min(channel, buffer.numberOfChannels - 1);
+      const sourceData = buffer.getChannelData(sourceChannel);
+
+      for (let i = 0; i < resampledLength; i++) {
+        const outputIndex = outputStartSample + i;
+        if (outputIndex >= outputLength) break;
+
+        // Simple resampling (could be improved with proper interpolation)
+        const sourceIndex = sourceStartSample + Math.floor(i / resampleRatio);
+        if (sourceIndex >= sourceData.length) break;
+
+        outputData[outputIndex] += sourceData[sourceIndex];
+      }
+    }
+  }
+
+  return outputBuffer;
+}
+
+export async function exportProject(
+  options: ExportOptions
+): Promise<ExportResult> {
+  const { format, quality, fps, includeAudio, onProgress, onCancel } = options;
+
+  try {
+    const timelineStore = useTimelineStore.getState();
+    const mediaStore = useMediaStore.getState();
+    const projectStore = useProjectStore.getState();
+
+    const { tracks, getTotalDuration } = timelineStore;
+    const { mediaFiles } = mediaStore;
+    const { activeProject } = projectStore;
+
+    if (!activeProject) {
+      return { success: false, error: "No active project" };
+    }
+
+    const duration = getTotalDuration();
+    if (duration === 0) {
+      return { success: false, error: "Project is empty" };
+    }
+
+    const exportFps = fps || activeProject.fps || DEFAULT_FPS;
+    const canvasSize = activeProject.canvasSize;
+
+    const outputFormat =
+      format === "webm" ? new WebMOutputFormat() : new Mp4OutputFormat();
+
+    // BufferTarget for smaller files, StreamTarget for larger ones
+    // TODO: Implement StreamTarget
+    const output = new Output({
+      format: outputFormat,
+      target: new BufferTarget(),
+    });
+
+    // Canvas for rendering
+    const canvas = document.createElement("canvas");
+    canvas.width = canvasSize.width;
+    canvas.height = canvasSize.height;
+    const ctx = canvas.getContext("2d");
+
+    if (!ctx) {
+      return { success: false, error: "Failed to create canvas context" };
+    }
+
+    const videoSource = new CanvasSource(canvas, {
+      codec: format === "webm" ? "vp9" : "avc", // VP9 for WebM, H.264 for MP4
+      bitrate: qualityMap[quality],
+    });
+
+    output.addVideoTrack(videoSource, { frameRate: exportFps });
+
+    // Add audio track if requested (but don't add data yet)
+    let audioSource: AudioBufferSource | null = null;
+    let audioBuffer: AudioBuffer | null = null;
+
+    if (includeAudio) {
+      onProgress?.(0.05); // 5% for audio processing
+
+      audioBuffer = await createTimelineAudioBuffer(
+        tracks,
+        mediaFiles,
+        duration
+      );
+
+      if (audioBuffer) {
+        audioSource = new AudioBufferSource({
+          codec: format === "webm" ? "opus" : "aac", // Opus for WebM, AAC for MP4
+          bitrate: qualityMap[quality], // Use same quality for audio
+        });
+
+        output.addAudioTrack(audioSource);
+      }
+    }
+
+    // Start the output (after all tracks are added)
+    await output.start();
+
+    // Now add audio data after starting
+    if (audioSource && audioBuffer) {
+      await audioSource.add(audioBuffer);
+      audioSource.close();
+    }
+
+    const totalFrames = Math.ceil(duration * exportFps);
+    let cancelled = false;
+
+    // Render each frame
+    for (let frameIndex = 0; frameIndex < totalFrames; frameIndex++) {
+      // Check for cancellation
+      if (onCancel?.()) {
+        cancelled = true;
+        break;
+      }
+
+      const time = frameIndex / exportFps;
+
+      await renderTimelineFrame({
+        ctx,
+        time,
+        canvasWidth: canvas.width,
+        canvasHeight: canvas.height,
+        tracks,
+        mediaFiles,
+        backgroundColor:
+          activeProject.backgroundType === "blur"
+            ? "transparent"
+            : activeProject.backgroundColor || "#000000",
+        projectCanvasSize: canvasSize,
+      });
+
+      const frameDuration = 1 / exportFps;
+      await videoSource.add(time, frameDuration);
+
+      // Adjust progress to account for audio processing (5% at start)
+      const videoProgress = includeAudio
+        ? 0.05 + (frameIndex / totalFrames) * 0.95
+        : frameIndex / totalFrames;
+      onProgress?.(videoProgress);
+    }
+
+    if (cancelled) {
+      await output.cancel();
+      return { success: false, cancelled: true };
+    }
+    videoSource.close();
+    await output.finalize();
+    onProgress?.(1);
+
+    return {
+      success: true,
+      buffer: output.target.buffer || undefined,
+    };
+  } catch (error) {
+    console.error("Export failed:", error);
+    return {
+      success: false,
+      error: error instanceof Error ? error.message : "Unknown export error",
+    };
+  }
+}
+
+export function getExportMimeType(format: "mp4" | "webm"): string {
+  return format === "webm" ? "video/webm" : "video/mp4";
+}
+
+export function getExportFileExtension(format: "mp4" | "webm"): string {
+  return `.${format}`;
+}
diff --git a/apps/web/src/types/export.ts b/apps/web/src/types/export.ts
index 2fa10a50..093268ce 100644
--- a/apps/web/src/types/export.ts
+++ b/apps/web/src/types/export.ts
@@ -1,17 +1,18 @@
-export type ExportFormat = "mp4" | "webm";
-export type ExportQuality = "low" | "medium" | "high" | "very_high";
-
-export interface ExportOptions {
-  format: ExportFormat;
-  quality: ExportQuality;
-  fps?: number;
-  onProgress?: (progress: number) => void;
-  onCancel?: () => boolean;
-}
-
-export interface ExportResult {
-  success: boolean;
-  buffer?: ArrayBuffer;
-  error?: string;
-  cancelled?: boolean;
-}
+export type ExportFormat = "mp4" | "webm";
+export type ExportQuality = "low" | "medium" | "high" | "very_high";
+
+export interface ExportOptions {
+  format: ExportFormat;
+  quality: ExportQuality;
+  fps?: number;
+  includeAudio?: boolean;
+  onProgress?: (progress: number) => void;
+  onCancel?: () => boolean;
+}
+
+export interface ExportResult {
+  success: boolean;
+  buffer?: ArrayBuffer;
+  error?: string;
+  cancelled?: boolean;
+}