diff --git a/apps/web/src/components/editor/media-panel/views/captions.tsx b/apps/web/src/components/editor/media-panel/views/captions.tsx index 4fe2cbd7..6e18781c 100644 --- a/apps/web/src/components/editor/media-panel/views/captions.tsx +++ b/apps/web/src/components/editor/media-panel/views/captions.tsx @@ -3,7 +3,7 @@ import { PropertyGroup } from "../../properties-panel/property-item"; import { BaseView } from "./base-view"; import { Language, LanguageSelect } from "@/components/language-select"; import { useState, useRef, useEffect } from "react"; -import { extractTimelineAudio } from "@/lib/ffmpeg-utils"; +import { extractTimelineAudio } from "@/lib/mediabunny-utils"; import { encryptWithRandomKey, arrayBufferToBase64 } from "@/lib/zk-encryption"; import { useTimelineStore } from "@/stores/timeline-store"; import { Loader2, Shield, Trash2, Upload } from "lucide-react"; diff --git a/apps/web/src/components/editor/preview-panel.tsx b/apps/web/src/components/editor/preview-panel.tsx index 42c4becc..99c4edef 100644 --- a/apps/web/src/components/editor/preview-panel.tsx +++ b/apps/web/src/components/editor/preview-panel.tsx @@ -5,16 +5,19 @@ import { TimelineElement, TimelineTrack } from "@/types/timeline"; import { useMediaStore, type MediaItem } from "@/stores/media-store"; import { usePlaybackStore } from "@/stores/playback-store"; import { useEditorStore } from "@/stores/editor-store"; -import { VideoPlayer } from "@/components/ui/video-player"; -import { AudioPlayer } from "@/components/ui/audio-player"; import { Button } from "@/components/ui/button"; import { Play, Pause, Expand, SkipBack, SkipForward } from "lucide-react"; import { useState, useRef, useEffect, useCallback } from "react"; +import { VideoSampleSink } from "mediabunny"; +import { renderTimelineFrame } from "@/lib/timeline-renderer"; import { cn } from "@/lib/utils"; import { formatTimeCode } from "@/lib/time"; import { EditableTimecode } from "@/components/ui/editable-timecode"; -import { FONT_CLASS_MAP } from "@/lib/font-config"; -import { DEFAULT_CANVAS_SIZE, DEFAULT_FPS, useProjectStore } from "@/stores/project-store"; +import { + DEFAULT_CANVAS_SIZE, + DEFAULT_FPS, + useProjectStore, +} from "@/stores/project-store"; import { TextElementDragState } from "@/types/editor"; import { Popover, @@ -37,8 +40,20 @@ export function PreviewPanel() { const { tracks, getTotalDuration, updateTextElement } = useTimelineStore(); const { mediaItems } = useMediaStore(); const { currentTime, toggle, setCurrentTime } = usePlaybackStore(); + const { isPlaying, volume, muted } = usePlaybackStore(); const { activeProject } = useProjectStore(); const previewRef = useRef(null); + const canvasRef = useRef(null); + const lastFrameTimeRef = useRef(0); + const renderSeqRef = useRef(0); + const offscreenCanvasRef = useRef( + null + ); + + const audioContextRef = useRef(null); + const audioGainRef = useRef(null); + const audioBuffersRef = useRef>(new Map()); + const playingSourcesRef = useRef>(new Set()); const containerRef = useRef(null); const [previewDimensions, setPreviewDimensions] = useState({ width: 0, @@ -262,6 +277,296 @@ export function PreviewPanel() { const activeElements = getActiveElements(); + // Ensure first frame after mount/seek renders immediately + useEffect(() => { + const onSeek = () => { + lastFrameTimeRef.current = -Infinity; + renderSeqRef.current++; + }; + window.addEventListener("playback-seek", onSeek as EventListener); + lastFrameTimeRef.current = -Infinity; + return () => { + window.removeEventListener("playback-seek", onSeek as EventListener); + }; + }, []); + + // Web Audio: schedule only on play/pause/seek/volume/mute changes + useEffect(() => { + const stopAll = () => { + for (const src of playingSourcesRef.current) { + try { + src.stop(); + } catch {} + } + playingSourcesRef.current.clear(); + }; + + type WebAudioWindow = Window & { + AudioContext?: typeof AudioContext; + webkitAudioContext?: typeof AudioContext; + }; + const ensureAudioGraph = async () => { + if (!audioContextRef.current) { + const win = window as WebAudioWindow; + const Ctx = win.AudioContext ?? win.webkitAudioContext; + if (!Ctx) return; + audioContextRef.current = new Ctx(); + } + if (!audioGainRef.current) { + audioGainRef.current = audioContextRef.current!.createGain(); + audioGainRef.current.connect(audioContextRef.current!.destination); + } + if (audioContextRef.current!.state === "suspended") { + try { + await audioContextRef.current!.resume(); + } catch {} + } + const gainValue = muted ? 0 : Math.max(0, Math.min(1, volume)); + audioGainRef.current!.gain.setValueAtTime( + gainValue, + audioContextRef.current!.currentTime + ); + }; + + const scheduleNow = async () => { + await ensureAudioGraph(); + const audioCtx = audioContextRef.current!; + const gain = audioGainRef.current!; + + const tracksSnapshot = useTimelineStore.getState().tracks; + const mediaList = useMediaStore.getState().mediaItems; + const idToMedia = new Map(mediaList.map((m) => [m.id, m] as const)); + const playbackNow = usePlaybackStore.getState().currentTime; + + const audible: Array<{ + id: string; + elementStart: number; + trimStart: number; + trimEnd: number; + duration: number; + muted: boolean; + trackMuted: boolean; + }> = []; + const uniqueIds = new Set(); + for (const track of tracksSnapshot) { + for (const element of track.elements) { + if (element.type !== "media") continue; + const media = idToMedia.get(element.mediaId); + if (!media || media.type !== "audio") continue; + const visibleDuration = + element.duration - element.trimStart - element.trimEnd; + if (visibleDuration <= 0) continue; + const localTime = playbackNow - element.startTime + element.trimStart; + if (localTime < 0 || localTime >= visibleDuration) continue; + audible.push({ + id: media.id, + elementStart: element.startTime, + trimStart: element.trimStart, + trimEnd: element.trimEnd, + duration: element.duration, + muted: !!element.muted, + trackMuted: !!track.muted, + }); + uniqueIds.add(media.id); + } + } + + if (audible.length === 0) return; + + // Decode buffers as needed + const decodePromises: Array> = []; + for (const id of uniqueIds) { + if (!audioBuffersRef.current.has(id)) { + const mediaItem = idToMedia.get(id); + if (!mediaItem) continue; + const p = (async () => { + const arr = await mediaItem.file.arrayBuffer(); + const buf = await audioCtx.decodeAudioData(arr.slice(0)); + audioBuffersRef.current.set(id, buf); + })(); + decodePromises.push(p); + } + } + await Promise.all(decodePromises); + + const startAt = audioCtx.currentTime + 0.02; + for (const entry of audible) { + if (entry.muted || entry.trackMuted) continue; + const buffer = audioBuffersRef.current.get(entry.id); + if (!buffer) continue; + const visibleDuration = + entry.duration - entry.trimStart - entry.trimEnd; + const localTime = Math.max( + 0, + playbackNow - entry.elementStart + entry.trimStart + ); + const playDuration = Math.max(0, visibleDuration - localTime); + if (playDuration <= 0) continue; + const src = audioCtx.createBufferSource(); + src.buffer = buffer; + src.connect(gain); + try { + src.start(startAt, localTime, playDuration); + playingSourcesRef.current.add(src); + } catch {} + } + }; + + const onSeek = () => { + if (!isPlaying) return; + for (const src of playingSourcesRef.current) { + try { + src.stop(); + } catch {} + } + playingSourcesRef.current.clear(); + void scheduleNow(); + }; + + // Apply volume/mute changes immediately + void ensureAudioGraph(); + + // Start/stop on play state changes + for (const src of playingSourcesRef.current) { + try { + src.stop(); + } catch {} + } + playingSourcesRef.current.clear(); + if (isPlaying) { + void scheduleNow(); + } + + window.addEventListener("playback-seek", onSeek as EventListener); + return () => { + window.removeEventListener("playback-seek", onSeek as EventListener); + for (const src of playingSourcesRef.current) { + try { + src.stop(); + } catch {} + } + playingSourcesRef.current.clear(); + }; + }, [isPlaying, volume, muted, mediaItems]); + + // Canvas: draw current frame for visible elements using offscreen compositing + useEffect(() => { + const draw = async () => { + const canvas = canvasRef.current; + if (!canvas) return; + const mainCtx = canvas.getContext("2d"); + if (!mainCtx) return; + + // Set canvas internal resolution to avoid blurry scaling + const displayWidth = Math.max(1, Math.floor(previewDimensions.width)); + const displayHeight = Math.max(1, Math.floor(previewDimensions.height)); + if (canvas.width !== displayWidth || canvas.height !== displayHeight) { + canvas.width = displayWidth; + canvas.height = displayHeight; + } + + // Throttle rendering to project FPS during playback only + const fps = activeProject?.fps || DEFAULT_FPS; + const minDelta = 1 / fps; + if (isPlaying) { + if (currentTime - lastFrameTimeRef.current < minDelta) { + return; + } + lastFrameTimeRef.current = currentTime; + } + + // Invalidate older async renders when user scrubs rapidly + const mySeq = (renderSeqRef.current += 1); + + // Offscreen buffer to avoid flicker (reuse canvas) + if (!offscreenCanvasRef.current) { + const hasOffscreen = + typeof (globalThis as unknown as { OffscreenCanvas?: unknown }) + .OffscreenCanvas !== "undefined"; + if (hasOffscreen) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + offscreenCanvasRef.current = new (globalThis as any).OffscreenCanvas( + displayWidth, + displayHeight + ) as OffscreenCanvas; + } else { + const c = document.createElement("canvas"); + c.width = displayWidth; + c.height = displayHeight; + offscreenCanvasRef.current = c; + } + } + // Ensure size matches + if ( + offscreenCanvasRef.current && + (offscreenCanvasRef.current as HTMLCanvasElement).getContext + ) { + const c = offscreenCanvasRef.current as HTMLCanvasElement; + if (c.width !== displayWidth || c.height !== displayHeight) { + c.width = displayWidth; + c.height = displayHeight; + } + } else { + const c = offscreenCanvasRef.current as OffscreenCanvas; + // @ts-ignore width/height exist on OffscreenCanvas in modern browsers + if ( + (c as unknown as { width: number }).width !== displayWidth || + (c as unknown as { height: number }).height !== displayHeight + ) { + // @ts-ignore + (c as unknown as { width: number }).width = displayWidth; + // @ts-ignore + (c as unknown as { height: number }).height = displayHeight; + } + } + const offscreenCanvas = offscreenCanvasRef.current as + | HTMLCanvasElement + | OffscreenCanvas; + const offCtx = (offscreenCanvas as HTMLCanvasElement).getContext + ? (offscreenCanvas as HTMLCanvasElement).getContext("2d") + : (offscreenCanvas as OffscreenCanvas).getContext("2d"); + if (!offCtx) return; + + await renderTimelineFrame({ + ctx: offCtx as CanvasRenderingContext2D, + time: currentTime, + canvasWidth: displayWidth, + canvasHeight: displayHeight, + tracks, + mediaItems, + backgroundColor: + activeProject?.backgroundType === "blur" + ? "transparent" + : activeProject?.backgroundColor || "#000000", + useCache: false, + fps: activeProject?.fps || DEFAULT_FPS, + }); + + // Blit offscreen to visible canvas + mainCtx.clearRect(0, 0, displayWidth, displayHeight); + if ((offscreenCanvas as HTMLCanvasElement).getContext) { + mainCtx.drawImage(offscreenCanvas as HTMLCanvasElement, 0, 0); + } else { + mainCtx.drawImage( + offscreenCanvas as unknown as CanvasImageSource, + 0, + 0 + ); + } + }; + + void draw(); + }, [ + activeElements, + currentTime, + previewDimensions.width, + previewDimensions.height, + canvasSize.width, + canvasSize.height, + activeProject?.backgroundType, + activeProject?.backgroundColor, + ]); + // Get media elements for blur background (video/image only) const getBlurBackgroundElements = (): ActiveElement[] => { return activeElements.filter( @@ -275,213 +580,11 @@ export function PreviewPanel() { const blurBackgroundElements = getBlurBackgroundElements(); - // Render blur background layer - const renderBlurBackground = () => { - if ( - !activeProject?.backgroundType || - activeProject.backgroundType !== "blur" || - blurBackgroundElements.length === 0 - ) { - return null; - } + // Render blur background layer (handled by canvas now) + const renderBlurBackground = () => null; - // Use the first media element for background (could be enhanced to use primary/focused element) - const backgroundElement = blurBackgroundElements[0]; - const { element, mediaItem } = backgroundElement; - - if (!mediaItem) return null; - - const blurIntensity = activeProject.blurIntensity || 8; - - if (mediaItem.type === "video") { - return ( -
- -
- ); - } - - if (mediaItem.type === "image") { - return ( -
- {mediaItem.name} -
- ); - } - - return null; - }; - - // Render an element - const renderElement = (elementData: ActiveElement, index: number) => { - const { element, mediaItem } = elementData; - - // Text elements - if (element.type === "text") { - const fontClassName = - FONT_CLASS_MAP[element.fontFamily as keyof typeof FONT_CLASS_MAP] || ""; - - const scaleRatio = previewDimensions.width / canvasSize.width; - - return ( -
- handleTextMouseDown(e, element, elementData.track.id) - } - style={{ - left: `${ - 50 + - ((dragState.isDragging && dragState.elementId === element.id - ? dragState.currentX - : element.x) / - canvasSize.width) * - 100 - }%`, - top: `${ - 50 + - ((dragState.isDragging && dragState.elementId === element.id - ? dragState.currentY - : element.y) / - canvasSize.height) * - 100 - }%`, - transform: `translate(-50%, -50%) rotate(${element.rotation}deg)`, - opacity: element.opacity, - zIndex: 100 + index, // Text elements on top - }} - > -
- {element.content} -
-
- ); - } - - // Media elements - if (element.type === "media") { - // Test elements - if (!mediaItem || element.mediaId === "test") { - return ( -
-
-
🎬
-

{element.name}

-
-
- ); - } - - // Video elements - if (mediaItem.type === "video") { - return ( -
- -
- ); - } - - // Image elements - if (mediaItem.type === "image") { - return ( -
- {mediaItem.name} -
- ); - } - - // Audio elements (no visual representation) - if (mediaItem.type === "audio") { - return ( -
- -
- ); - } - } - - return null; - }; + // Render an element (canvas handles visuals now). Audio playback to be implemented via Web Audio. + const renderElement = (_elementData: ActiveElement) => null; return ( <> @@ -505,14 +608,23 @@ export function PreviewPanel() { }} > {renderBlurBackground()} + {activeElements.length === 0 ? (
No elements at current time
) : ( - activeElements.map((elementData, index) => - renderElement(elementData, index) - ) + activeElements.map((elementData) => renderElement(elementData)) )} diff --git a/apps/web/src/lib/media-processing.ts b/apps/web/src/lib/media-processing.ts index 353c3113..15c71626 100644 --- a/apps/web/src/lib/media-processing.ts +++ b/apps/web/src/lib/media-processing.ts @@ -6,7 +6,7 @@ import { getImageDimensions, type MediaItem, } from "@/stores/media-store"; -import { generateThumbnail, getVideoInfo } from "./ffmpeg-utils"; +import { generateThumbnail, getVideoInfo } from "./mediabunny-utils"; export interface ProcessedMediaItem extends Omit {} @@ -37,33 +37,23 @@ export async function processMediaFiles( try { if (fileType === "image") { - // Get image dimensions const dimensions = await getImageDimensions(file); width = dimensions.width; height = dimensions.height; } else if (fileType === "video") { try { - // Use FFmpeg for comprehensive video info extraction - const videoInfo = await getVideoInfo(file); + const videoInfo = await getVideoInfo({ videoFile: file }); duration = videoInfo.duration; width = videoInfo.width; height = videoInfo.height; fps = videoInfo.fps; - // Generate thumbnail using FFmpeg - thumbnailUrl = await generateThumbnail(file, 1); + thumbnailUrl = await generateThumbnail({ + videoFile: file, + timeInSeconds: 1, + }); } catch (error) { - console.warn( - "FFmpeg processing failed, falling back to basic processing:", - error - ); - // Fallback to basic processing - const videoResult = await generateVideoThumbnail(file); - thumbnailUrl = videoResult.thumbnailUrl; - width = videoResult.width; - height = videoResult.height; - duration = await getMediaDuration(file); - // FPS will remain undefined for fallback + console.warn("Video processing failed", error); } } else if (fileType === "audio") { // For audio, we don't set width/height/fps (they'll be undefined) @@ -82,7 +72,6 @@ export async function processMediaFiles( fps, }); - // Yield back to the event loop to keep the UI responsive await new Promise((resolve) => setTimeout(resolve, 0)); completed += 1; diff --git a/apps/web/src/lib/ffmpeg-utils.ts b/apps/web/src/lib/mediabunny-utils.ts similarity index 53% rename from apps/web/src/lib/ffmpeg-utils.ts rename to apps/web/src/lib/mediabunny-utils.ts index 0643ecb9..28e6b97f 100644 --- a/apps/web/src/lib/ffmpeg-utils.ts +++ b/apps/web/src/lib/mediabunny-utils.ts @@ -1,7 +1,7 @@ import { FFmpeg } from "@ffmpeg/ffmpeg"; -import { toBlobURL } from "@ffmpeg/util"; import { useTimelineStore } from "@/stores/timeline-store"; import { useMediaStore } from "@/stores/media-store"; +import { Input, ALL_FORMATS, BlobSource, VideoSampleSink } from "mediabunny"; let ffmpeg: FFmpeg | null = null; @@ -14,256 +14,99 @@ export const initFFmpeg = async (): Promise => { return ffmpeg; }; -export const generateThumbnail = async ( - videoFile: File, - timeInSeconds = 1 -): Promise => { - const ffmpeg = await initFFmpeg(); +export async function generateThumbnail({ + videoFile, + timeInSeconds, +}: { + videoFile: File; + timeInSeconds: number; +}): Promise { + const input = new Input({ + source: new BlobSource(videoFile), + formats: ALL_FORMATS, + }); - const inputName = "input.mp4"; - const outputName = "thumbnail.jpg"; - - // Write input file - await ffmpeg.writeFile( - inputName, - new Uint8Array(await videoFile.arrayBuffer()) - ); - - // Generate thumbnail at specific time - await ffmpeg.exec([ - "-i", - inputName, - "-ss", - timeInSeconds.toString(), - "-vframes", - "1", - "-vf", - "scale=320:240", - "-q:v", - "2", - outputName, - ]); - - // Read output file - const data = await ffmpeg.readFile(outputName); - const blob = new Blob([data], { type: "image/jpeg" }); - - // Cleanup - await ffmpeg.deleteFile(inputName); - await ffmpeg.deleteFile(outputName); - - return URL.createObjectURL(blob); -}; - -export const trimVideo = async ( - videoFile: File, - startTime: number, - endTime: number, - onProgress?: (progress: number) => void -): Promise => { - const ffmpeg = await initFFmpeg(); - - const inputName = "input.mp4"; - const outputName = "output.mp4"; - - // Set up progress callback - if (onProgress) { - ffmpeg.on("progress", ({ progress }) => { - onProgress(progress * 100); - }); + const videoTrack = await input.getPrimaryVideoTrack(); + if (!videoTrack) { + throw new Error("No video track found in the file"); } - // Write input file - await ffmpeg.writeFile( - inputName, - new Uint8Array(await videoFile.arrayBuffer()) - ); + // Check if we can decode this video + const canDecode = await videoTrack.canDecode(); + if (!canDecode) { + throw new Error("Video codec not supported for decoding"); + } - const duration = endTime - startTime; + const sink = new VideoSampleSink(videoTrack); - // Trim video - await ffmpeg.exec([ - "-i", - inputName, - "-ss", - startTime.toString(), - "-t", - duration.toString(), - "-c", - "copy", // Use stream copy for faster processing - outputName, - ]); + const frame = await sink.getSample(timeInSeconds); - // Read output file - const data = await ffmpeg.readFile(outputName); - const blob = new Blob([data], { type: "video/mp4" }); + if (!frame) { + throw new Error("Could not get frame at specified time"); + } - // Cleanup - await ffmpeg.deleteFile(inputName); - await ffmpeg.deleteFile(outputName); + const canvas = document.createElement("canvas"); + canvas.width = 320; + canvas.height = 240; + const ctx = canvas.getContext("2d"); - return blob; -}; + if (!ctx) { + throw new Error("Could not get canvas context"); + } -export const getVideoInfo = async ( - videoFile: File -): Promise<{ + frame.draw(ctx, 0, 0, 320, 240); + + return new Promise((resolve, reject) => { + canvas.toBlob( + (blob) => { + if (blob) { + resolve(URL.createObjectURL(blob)); + } else { + reject(new Error("Failed to create thumbnail blob")); + } + }, + "image/jpeg", + 0.8 + ); + }); +} + +export async function getVideoInfo({ + videoFile, +}: { + videoFile: File; +}): Promise<{ duration: number; width: number; height: number; fps: number; -}> => { - const ffmpeg = await initFFmpeg(); +}> { + const input = new Input({ + source: new BlobSource(videoFile), + formats: ALL_FORMATS, + }); - const inputName = "input.mp4"; + const duration = await input.computeDuration(); + const videoTrack = await input.getPrimaryVideoTrack(); - // Write input file - await ffmpeg.writeFile( - inputName, - new Uint8Array(await videoFile.arrayBuffer()) - ); - - // Capture FFmpeg stderr output with a one-time listener pattern - let ffmpegOutput = ""; - let listening = true; - const listener = (data: string) => { - if (listening) ffmpegOutput += data; - }; - ffmpeg.on("log", ({ message }) => listener(message)); - - // Run ffmpeg to get info (stderr will contain the info) - try { - await ffmpeg.exec(["-i", inputName, "-f", "null", "-"]); - } catch (error) { - listening = false; - await ffmpeg.deleteFile(inputName); - console.error("FFmpeg execution failed:", error); - throw new Error( - "Failed to extract video info. The file may be corrupted or in an unsupported format." - ); + if (!videoTrack) { + throw new Error("No video track found in the file"); } - // Disable listener after exec completes - listening = false; - - // Cleanup - await ffmpeg.deleteFile(inputName); - - // Parse output for duration, resolution, and fps - // Example: Duration: 00:00:10.00, start: 0.000000, bitrate: 1234 kb/s - // Example: Stream #0:0: Video: h264 (High), yuv420p(progressive), 1920x1080 [SAR 1:1 DAR 16:9], 30 fps, 30 tbr, 90k tbn, 60 tbc - - const durationMatch = ffmpegOutput.match(/Duration: (\d+):(\d+):([\d.]+)/); - let duration = 0; - if (durationMatch) { - const [, h, m, s] = durationMatch; - duration = parseInt(h) * 3600 + parseInt(m) * 60 + parseFloat(s); - } - - const videoStreamMatch = ffmpegOutput.match( - /Video:.* (\d+)x(\d+)[^,]*, ([\d.]+) fps/ - ); - let width = 0, - height = 0, - fps = 0; - if (videoStreamMatch) { - width = parseInt(videoStreamMatch[1]); - height = parseInt(videoStreamMatch[2]); - fps = parseFloat(videoStreamMatch[3]); - } + // Get frame rate from packet statistics + const packetStats = await videoTrack.computePacketStats(100); + const fps = packetStats.averagePacketRate; return { duration, - width, - height, + width: videoTrack.displayWidth, + height: videoTrack.displayHeight, fps, }; -}; - -export const convertToWebM = async ( - videoFile: File, - onProgress?: (progress: number) => void -): Promise => { - const ffmpeg = await initFFmpeg(); - - const inputName = "input.mp4"; - const outputName = "output.webm"; - - // Set up progress callback - if (onProgress) { - ffmpeg.on("progress", ({ progress }) => { - onProgress(progress * 100); - }); - } - - // Write input file - await ffmpeg.writeFile( - inputName, - new Uint8Array(await videoFile.arrayBuffer()) - ); - - // Convert to WebM - await ffmpeg.exec([ - "-i", - inputName, - "-c:v", - "libvpx-vp9", - "-crf", - "30", - "-b:v", - "0", - "-c:a", - "libopus", - outputName, - ]); - - // Read output file - const data = await ffmpeg.readFile(outputName); - const blob = new Blob([data], { type: "video/webm" }); - - // Cleanup - await ffmpeg.deleteFile(inputName); - await ffmpeg.deleteFile(outputName); - - return blob; -}; - -export const extractAudio = async ( - videoFile: File, - format: "mp3" | "wav" = "mp3" -): Promise => { - const ffmpeg = await initFFmpeg(); - - const inputName = "input.mp4"; - const outputName = `output.${format}`; - - // Write input file - await ffmpeg.writeFile( - inputName, - new Uint8Array(await videoFile.arrayBuffer()) - ); - - // Extract audio - await ffmpeg.exec([ - "-i", - inputName, - "-vn", // Disable video - "-acodec", - format === "mp3" ? "libmp3lame" : "pcm_s16le", - outputName, - ]); - - // Read output file - const data = await ffmpeg.readFile(outputName); - const blob = new Blob([data], { type: `audio/${format}` }); - - // Cleanup - await ffmpeg.deleteFile(inputName); - await ffmpeg.deleteFile(outputName); - - return blob; -}; +} +// Audio mixing for timeline - keeping FFmpeg for now due to complexity +// TODO: Replace with Mediabunny audio processing when implementing canvas preview export const extractTimelineAudio = async ( onProgress?: (progress: number) => void ): Promise => { diff --git a/apps/web/src/lib/timeline-renderer.ts b/apps/web/src/lib/timeline-renderer.ts new file mode 100644 index 00000000..1c1eafcd --- /dev/null +++ b/apps/web/src/lib/timeline-renderer.ts @@ -0,0 +1,179 @@ +import { Input, ALL_FORMATS, BlobSource, VideoSampleSink } from "mediabunny"; +import type { TimelineTrack } from "@/types/timeline"; +import type { MediaItem } from "@/stores/media-store"; + +export interface RenderContext { + ctx: CanvasRenderingContext2D; + time: number; + canvasWidth: number; + canvasHeight: number; + tracks: TimelineTrack[]; + mediaItems: MediaItem[]; + backgroundColor?: string; + useCache?: boolean; + cache?: Map< + string, + Map< + number, + { + draw: ( + c: CanvasRenderingContext2D, + x: number, + y: number, + w: number, + h: number + ) => void; + } + > + >; + fps: number; +} + +export async function renderTimelineFrame({ + ctx, + time, + canvasWidth, + canvasHeight, + tracks, + mediaItems, + backgroundColor, + useCache = true, + cache, + fps, +}: RenderContext): Promise { + // Background + ctx.clearRect(0, 0, canvasWidth, canvasHeight); + if (backgroundColor && backgroundColor !== "transparent") { + ctx.fillStyle = backgroundColor; + ctx.fillRect(0, 0, canvasWidth, canvasHeight); + } + + const scaleX = 1; + const scaleY = 1; + const idToMedia = new Map(mediaItems.map((m) => [m.id, m] as const)); + const active: Array<{ + track: TimelineTrack; + element: TimelineTrack["elements"][number]; + mediaItem: MediaItem | null; + }> = []; + + for (let t = tracks.length - 1; t >= 0; t -= 1) { + const track = tracks[t]; + for (const element of track.elements) { + if ((element as any).hidden) continue; + const elementStart = element.startTime; + const elementEnd = + element.startTime + + (element.duration - element.trimStart - element.trimEnd); + if (time >= elementStart && time < elementEnd) { + let mediaItem: MediaItem | null = null; + if (element.type === "media") { + mediaItem = + element.mediaId === "test" + ? null + : idToMedia.get(element.mediaId) || null; + } + active.push({ track, element, mediaItem }); + } + } + } + + for (const { element, mediaItem } of active) { + if (element.type === "media" && mediaItem) { + if (mediaItem.type === "video") { + const input = new Input({ + source: new BlobSource(mediaItem.file), + formats: ALL_FORMATS, + }); + const track = await input.getPrimaryVideoTrack(); + if (!track) continue; + const decodable = await track.canDecode(); + if (!decodable) continue; + const sink = new VideoSampleSink(track); + + const localTime = time - element.startTime + element.trimStart; + const sample = await sink.getSample(localTime); + if (!sample) continue; + + const mediaW = Math.max(1, mediaItem.width || canvasWidth); + const mediaH = Math.max(1, mediaItem.height || canvasHeight); + const containScale = Math.min( + canvasWidth / mediaW, + canvasHeight / mediaH + ); + const drawW = mediaW * containScale; + const drawH = mediaH * containScale; + const drawX = (canvasWidth - drawW) / 2; + const drawY = (canvasHeight - drawH) / 2; + sample.draw(ctx, drawX, drawY, drawW, drawH); + } + if (mediaItem.type === "image") { + const img = new Image(); + await new Promise((resolve, reject) => { + img.onload = () => resolve(); + img.onerror = () => reject(new Error("Image load failed")); + img.src = mediaItem.url || URL.createObjectURL(mediaItem.file); + }); + const mediaW = Math.max( + 1, + mediaItem.width || img.naturalWidth || canvasWidth + ); + const mediaH = Math.max( + 1, + mediaItem.height || img.naturalHeight || canvasHeight + ); + const containScale = Math.min( + canvasWidth / mediaW, + canvasHeight / mediaH + ); + const drawW = mediaW * containScale; + const drawH = mediaH * containScale; + const drawX = (canvasWidth - drawW) / 2; + const drawY = (canvasHeight - drawH) / 2; + ctx.drawImage(img, drawX, drawY, drawW, drawH); + } + } + if (element.type === "text") { + const posX = canvasWidth / 2 + (element as any).x * scaleX; + const posY = canvasHeight / 2 + (element as any).y * scaleY; + ctx.save(); + ctx.translate(posX, posY); + ctx.rotate(((element as any).rotation * Math.PI) / 180); + ctx.globalAlpha = Math.max(0, Math.min(1, (element as any).opacity)); + const px = (element as any).fontSize * scaleX; + const weight = (element as any).fontWeight === "bold" ? "bold " : ""; + const style = (element as any).fontStyle === "italic" ? "italic " : ""; + ctx.font = `${style}${weight}${px}px ${(element as any).fontFamily}`; + ctx.fillStyle = (element as any).color; + ctx.textAlign = (element as any).textAlign; + ctx.textBaseline = "middle"; + const metrics = ctx.measureText((element as any).content); + const ascent = + (metrics as unknown as { actualBoundingBoxAscent?: number }) + .actualBoundingBoxAscent ?? px * 0.8; + const descent = + (metrics as unknown as { actualBoundingBoxDescent?: number }) + .actualBoundingBoxDescent ?? px * 0.2; + const textW = metrics.width; + const textH = ascent + descent; + const padX = 8 * scaleX; + const padY = 4 * scaleX; + if ((element as any).backgroundColor) { + ctx.save(); + ctx.fillStyle = (element as any).backgroundColor; + let bgLeft = -textW / 2; + if (ctx.textAlign === "left") bgLeft = 0; + if (ctx.textAlign === "right") bgLeft = -textW; + ctx.fillRect( + bgLeft - padX, + -textH / 2 - padY, + textW + padX * 2, + textH + padY * 2 + ); + ctx.restore(); + } + ctx.fillText((element as any).content, 0, 0); + ctx.restore(); + } + } +} diff --git a/bun.lock b/bun.lock index ea7c8c9a..95f7c874 100644 --- a/bun.lock +++ b/bun.lock @@ -4,6 +4,7 @@ "": { "name": "opencut", "dependencies": { + "mediabunny": "^1.9.3", "next": "^15.3.4", "react-country-flag": "^3.1.0", "wavesurfer.js": "^7.9.8", @@ -558,6 +559,10 @@ "@types/deep-eql": ["@types/deep-eql@4.0.2", "", {}, "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw=="], + "@types/dom-mediacapture-transform": ["@types/dom-mediacapture-transform@0.1.11", "", { "dependencies": { "@types/dom-webcodecs": "*" } }, "sha512-Y2p+nGf1bF2XMttBnsVPHUWzRRZzqUoJAKmiP10b5umnO6DDrWI0BrGDJy1pOHoOULVmGSfFNkQrAlC5dcj6nQ=="], + + "@types/dom-webcodecs": ["@types/dom-webcodecs@0.1.13", "", {}, "sha512-O5hkiFIcjjszPIYyUSyvScyvrBoV3NOEEZx/pMlsu44TKzWNkLVBBxnxJz42in5n3QIolYOcBYFCPZZ0h8SkwQ=="], + "@types/estree": ["@types/estree@1.0.8", "", {}, "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w=="], "@types/estree-jsx": ["@types/estree-jsx@1.0.5", "", { "dependencies": { "@types/estree": "*" } }, "sha512-52CcUVNFyfb1A2ALocQw/Dd1BQFNmSdkuC3BkZ6iqhdMfQz7JWOFRuJFloOzjk+6WijU56m9oKXFAXc7o3Towg=="], @@ -896,6 +901,8 @@ "mdast-util-to-string": ["mdast-util-to-string@4.0.0", "", { "dependencies": { "@types/mdast": "^4.0.0" } }, "sha512-0H44vDimn51F0YwvxSJSm0eCDOJTRlmN0R1yBh4HLj9wiV1Dn0QoXGbvFAWj2hSItVTlCmBF1hqKlIyUBVFLPg=="], + "mediabunny": ["mediabunny@1.9.3", "", { "dependencies": { "@types/dom-mediacapture-transform": "^0.1.11", "@types/dom-webcodecs": "0.1.13" } }, "sha512-uVspdzrpwvW8NxtGspHPOJvdNSY0dNFEmOQngSheX9PYX9oXijZENBjfYWOgm9c9IgSSlN2bdJF9iyPyY3f+fQ=="], + "micromark": ["micromark@4.0.2", "", { "dependencies": { "@types/debug": "^4.0.0", "debug": "^4.0.0", "decode-named-character-reference": "^1.0.0", "devlop": "^1.0.0", "micromark-core-commonmark": "^2.0.0", "micromark-factory-space": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-chunked": "^2.0.0", "micromark-util-combine-extensions": "^2.0.0", "micromark-util-decode-numeric-character-reference": "^2.0.0", "micromark-util-encode": "^2.0.0", "micromark-util-normalize-identifier": "^2.0.0", "micromark-util-resolve-all": "^2.0.0", "micromark-util-sanitize-uri": "^2.0.0", "micromark-util-subtokenize": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-zpe98Q6kvavpCr1NPVSCMebCKfD7CA2NqZ+rykeNhONIJBpc1tFKt9hucLGwha3jNTNI8lHpctWJWoimVF4PfA=="], "micromark-core-commonmark": ["micromark-core-commonmark@2.0.3", "", { "dependencies": { "decode-named-character-reference": "^1.0.0", "devlop": "^1.0.0", "micromark-factory-destination": "^2.0.0", "micromark-factory-label": "^2.0.0", "micromark-factory-space": "^2.0.0", "micromark-factory-title": "^2.0.0", "micromark-factory-whitespace": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-chunked": "^2.0.0", "micromark-util-classify-character": "^2.0.0", "micromark-util-html-tag-name": "^2.0.0", "micromark-util-normalize-identifier": "^2.0.0", "micromark-util-resolve-all": "^2.0.0", "micromark-util-subtokenize": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-RDBrHEMSxVFLg6xvnXmb1Ayr2WzLAWjeSATAoxwKYJV94TeNavgoIdA0a9ytzDSVzBy2YKFK+emCPOEibLeCrg=="], @@ -1320,10 +1327,10 @@ "motion/framer-motion/motion-utils": ["motion-utils@12.23.6", "", {}, "sha512-eAWoPgr4eFEOFfg2WjIsMoqJTW6Z8MTUCgn/GZ3VRpClWBdnbjryiA3ZSNLyxCTmCQx4RmYX6jX1iWHbenUPNQ=="], - "opencut/@types/node/undici-types": ["undici-types@7.10.0", "", {}, "sha512-t5Fy/nfn+14LuOc2KNYg75vZqClpAiqscVvMygNnlsHBFpSXdJaYtXMcdNLpl/Qvc3P2cB3s6lOV51nqsFq4ag=="], - "next/postcss/nanoid": ["nanoid@3.3.11", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w=="], + "opencut/@types/node/undici-types": ["undici-types@7.10.0", "", {}, "sha512-t5Fy/nfn+14LuOc2KNYg75vZqClpAiqscVvMygNnlsHBFpSXdJaYtXMcdNLpl/Qvc3P2cB3s6lOV51nqsFq4ag=="], + "opencut/next/@next/env": ["@next/env@15.4.5", "", {}, "sha512-ruM+q2SCOVCepUiERoxOmZY9ZVoecR3gcXNwCYZRvQQWRjhOiPJGmQ2fAiLR6YKWXcSAh7G79KEFxN3rwhs4LQ=="], "opencut/next/@next/swc-darwin-arm64": ["@next/swc-darwin-arm64@15.4.5", "", { "os": "darwin", "cpu": "arm64" }, "sha512-84dAN4fkfdC7nX6udDLz9GzQlMUwEMKD7zsseXrl7FTeIItF8vpk1lhLEnsotiiDt+QFu3O1FVWnqwcRD2U3KA=="], diff --git a/package.json b/package.json index 45a98282..616c97f7 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "format": "npx ultracite@latest format" }, "dependencies": { + "mediabunny": "^1.9.3", "next": "^15.3.4", "react-country-flag": "^3.1.0", "wavesurfer.js": "^7.9.8"