diff --git a/apps/web/src/components/editor/canvas-renderer.tsx b/apps/web/src/components/editor/canvas-renderer.tsx new file mode 100644 index 00000000..92aecf0b --- /dev/null +++ b/apps/web/src/components/editor/canvas-renderer.tsx @@ -0,0 +1,165 @@ +"use client"; + +import { useCallback, useMemo, useRef } from "react"; +import { useRafLoop } from "@/hooks/use-raf-loop"; +import { renderTimelineFrame } from "@/lib/timeline-renderer"; +import { useTimelineStore } from "@/stores/timeline-store"; +import { useMediaStore } from "@/stores/media-store"; +import { usePlaybackStore } from "@/stores/playback-store"; +import { useProjectStore, DEFAULT_FPS } from "@/stores/project-store"; +import { useFrameCache } from "@/hooks/use-frame-cache"; + +interface CanvasRendererProps { + width: number; + height: number; + className?: string; +} + +export function CanvasRenderer({ + width, + height, + className, +}: CanvasRendererProps) { + const canvasRef = useRef(null); + const lastFrameRef = useRef(-1); + const renderingRef = useRef(false); + const pendingFrameRef = useRef(null); + + const { tracks } = useTimelineStore(); + const { mediaFiles } = useMediaStore(); + const { activeProject } = useProjectStore(); + + const { getCachedFrame, cacheFrame, preRenderNearbyFrames } = useFrameCache(); + + const fps = useMemo( + () => activeProject?.fps || DEFAULT_FPS, + [activeProject?.fps] + ); + + const renderFrame = useCallback( + async (time: number, frame: number) => { + const canvas = canvasRef.current; + if (!canvas || renderingRef.current) return; + + renderingRef.current = true; + pendingFrameRef.current = null; + + try { + const ctx = canvas.getContext("2d"); + if (!ctx) return; + + const cachedFrame = getCachedFrame( + time, + tracks, + mediaFiles, + activeProject + ); + if (cachedFrame) { + ctx.putImageData(cachedFrame, 0, 0); + preRenderNearbyFrames( + time, + tracks, + mediaFiles, + activeProject, + async (preRenderTime: number) => { + const tempCanvas = document.createElement("canvas"); + tempCanvas.width = width; + tempCanvas.height = height; + const tempCtx = tempCanvas.getContext("2d"); + if (!tempCtx) + throw new Error("Failed to create temp canvas context"); + + await renderTimelineFrame({ + ctx: tempCtx, + time: preRenderTime, + canvasWidth: width, + canvasHeight: height, + tracks, + mediaFiles, + backgroundColor: + activeProject?.backgroundType === "blur" + ? undefined + : activeProject?.backgroundColor || "#000000", + backgroundType: activeProject?.backgroundType, + blurIntensity: activeProject?.blurIntensity, + projectCanvasSize: activeProject?.canvasSize, + }); + + return tempCtx.getImageData(0, 0, width, height); + } + ); + return; + } + await renderTimelineFrame({ + ctx, + time, + canvasWidth: width, + canvasHeight: height, + tracks, + mediaFiles, + backgroundColor: + activeProject?.backgroundType === "blur" + ? undefined + : activeProject?.backgroundColor || "#000000", + backgroundType: activeProject?.backgroundType, + blurIntensity: activeProject?.blurIntensity, + projectCanvasSize: activeProject?.canvasSize, + }); + + const imageData = ctx.getImageData(0, 0, width, height); + cacheFrame(time, imageData, tracks, mediaFiles, activeProject); + } catch (error) { + console.error("Frame render failed:", error); + } finally { + renderingRef.current = false; + + if (pendingFrameRef.current !== null) { + const pendingFrame = pendingFrameRef.current; + const pendingTime = pendingFrame / fps; + pendingFrameRef.current = null; + void renderFrame(pendingTime, pendingFrame); + } + } + }, + [ + tracks, + mediaFiles, + activeProject, + width, + height, + fps, + getCachedFrame, + cacheFrame, + preRenderNearbyFrames, + ] + ); + + const render = useCallback(() => { + const canvas = canvasRef.current; + if (!canvas) return; + + const time = usePlaybackStore.getState().currentTime; + const frame = Math.floor(time * fps); + + if (frame === lastFrameRef.current) return; + lastFrameRef.current = frame; + + if (renderingRef.current) { + pendingFrameRef.current = frame; + return; + } + + void renderFrame(time, frame); + }, [renderFrame, fps]); + + useRafLoop(render); + + return ( + + ); +} diff --git a/apps/web/src/components/editor/preview-panel.tsx b/apps/web/src/components/editor/preview-panel.tsx index 36f6a2ba..d4acbdc1 100644 --- a/apps/web/src/components/editor/preview-panel.tsx +++ b/apps/web/src/components/editor/preview-panel.tsx @@ -1,150 +1,97 @@ "use client"; -import { useTimelineStore } from "@/stores/timeline-store"; -import { TimelineElement, TimelineTrack } from "@/types/timeline"; -import { useMediaStore } from "@/stores/media-store"; -import { MediaFile } from "@/types/media"; -import { usePlaybackStore } from "@/stores/playback-store"; -import { useEditorStore } from "@/stores/editor-store"; -import { Button } from "@/components/ui/button"; -import { Play, Pause, Expand, SkipBack, SkipForward } from "lucide-react"; import { useState, useRef, useEffect, useCallback } from "react"; -import { renderTimelineFrame } from "@/lib/timeline-renderer"; -import { cn } from "@/lib/utils"; -import { formatTimeCode } from "@/lib/time"; -import { EditableTimecode } from "@/components/ui/editable-timecode"; -import { useFrameCache } from "@/hooks/use-frame-cache"; -import { useSceneStore } from "@/stores/scene-store"; -import { - DEFAULT_CANVAS_SIZE, - DEFAULT_FPS, - useProjectStore, -} from "@/stores/project-store"; -import { TextElementDragState } from "@/types/editor"; -import { - Popover, - PopoverContent, - PopoverTrigger, -} from "@/components/ui/popover"; -import { Checkbox } from "@/components/ui/checkbox"; +import { Button } from "@/components/ui/button"; +import { Play, Pause, Expand } from "lucide-react"; +import { CanvasRenderer } from "./canvas-renderer"; import { LayoutGuideOverlay } from "./layout-guide-overlay"; -import { Label } from "../ui/label"; -import { SocialsIcon } from "../icons"; -import { PLATFORM_LAYOUTS, type PlatformLayout } from "@/stores/editor-store"; +import { useTimelineStore } from "@/stores/timeline-store"; +import { usePlaybackStore } from "@/stores/playback-store"; +import { useProjectStore, DEFAULT_CANVAS_SIZE } from "@/stores/project-store"; +import { useMediaStore } from "@/stores/media-store"; +import { usePreviewAudio } from "@/hooks/use-preview-audio"; -interface ActiveElement { - element: TimelineElement; - track: TimelineTrack; - mediaItem: MediaFile | null; +function usePreviewSize() { + const { activeProject } = useProjectStore(); + return { + width: activeProject?.canvasSize?.width || DEFAULT_CANVAS_SIZE.width, + height: activeProject?.canvasSize?.height || DEFAULT_CANVAS_SIZE.height, + }; } -export function PreviewPanel() { - const { tracks, getTotalDuration, updateTextElement } = useTimelineStore(); - const { mediaFiles } = useMediaStore(); - const { currentTime, toggle, setCurrentTime } = usePlaybackStore(); - const { isPlaying, volume, muted } = usePlaybackStore(); - const { activeProject } = useProjectStore(); - const { currentScene } = useSceneStore(); - const previewRef = useRef(null); - const canvasRef = useRef(null); - const { getCachedFrame, cacheFrame, invalidateCache, preRenderNearbyFrames } = - useFrameCache(); - const lastFrameTimeRef = useRef(0); - const renderSeqRef = useRef(0); - const offscreenCanvasRef = useRef( - null - ); - - const audioContextRef = useRef(null); - const audioGainRef = useRef(null); - const audioBuffersRef = useRef>(new Map()); - const playingSourcesRef = useRef>(new Set()); - const containerRef = useRef(null); - const [previewDimensions, setPreviewDimensions] = useState({ - width: 0, - height: 0, - }); - const [isExpanded, setIsExpanded] = useState(false); - - const canvasSize = activeProject?.canvasSize || DEFAULT_CANVAS_SIZE; - const [dragState, setDragState] = useState({ - isDragging: false, - elementId: null, - trackId: null, - startX: 0, - startY: 0, - initialElementX: 0, - initialElementY: 0, - currentX: 0, - currentY: 0, - elementWidth: 0, - elementHeight: 0, - }); +function usePreviewDimensions(containerRef: React.RefObject) { + const [dimensions, setDimensions] = useState({ width: 0, height: 0 }); + const { width: canvasWidth, height: canvasHeight } = usePreviewSize(); useEffect(() => { - const updatePreviewSize = () => { + const updateSize = () => { if (!containerRef.current) return; - let availableWidth, availableHeight; + const container = containerRef.current.getBoundingClientRect(); + const computedStyle = getComputedStyle(containerRef.current); + const paddingTop = parseFloat(computedStyle.paddingTop); + const paddingBottom = parseFloat(computedStyle.paddingBottom); + const paddingLeft = parseFloat(computedStyle.paddingLeft); + const paddingRight = parseFloat(computedStyle.paddingRight); + const gap = parseFloat(computedStyle.gap) || 16; - if (isExpanded) { - const controlsHeight = 80; - const marginSpace = 24; - availableWidth = window.innerWidth - marginSpace; - availableHeight = window.innerHeight - controlsHeight - marginSpace; - } else { - const container = containerRef.current.getBoundingClientRect(); - const computedStyle = getComputedStyle(containerRef.current); - const paddingTop = parseFloat(computedStyle.paddingTop); - const paddingBottom = parseFloat(computedStyle.paddingBottom); - const paddingLeft = parseFloat(computedStyle.paddingLeft); - const paddingRight = parseFloat(computedStyle.paddingRight); - const gap = parseFloat(computedStyle.gap) || 16; - const toolbar = containerRef.current.querySelector("[data-toolbar]"); - const toolbarHeight = toolbar - ? toolbar.getBoundingClientRect().height - : 0; + const toolbar = containerRef.current.querySelector("[data-toolbar]"); + const toolbarHeight = toolbar + ? toolbar.getBoundingClientRect().height + : 0; - availableWidth = container.width - paddingLeft - paddingRight; - availableHeight = - container.height - - paddingTop - - paddingBottom - - toolbarHeight - - (toolbarHeight > 0 ? gap : 0); - } + const availableWidth = container.width - paddingLeft - paddingRight; + const availableHeight = + container.height - + paddingTop - + paddingBottom - + toolbarHeight - + (toolbarHeight > 0 ? gap : 0); - const targetRatio = canvasSize.width / canvasSize.height; + const targetRatio = canvasWidth / canvasHeight; const containerRatio = availableWidth / availableHeight; - let width, height; + let width, height; if (containerRatio > targetRatio) { - height = availableHeight * (isExpanded ? 0.95 : 1); + height = availableHeight; width = height * targetRatio; } else { - width = availableWidth * (isExpanded ? 0.95 : 1); + width = availableWidth; height = width / targetRatio; } - setPreviewDimensions({ width, height }); + setDimensions({ width, height }); }; - updatePreviewSize(); - const resizeObserver = new ResizeObserver(updatePreviewSize); + updateSize(); + const resizeObserver = new ResizeObserver(updateSize); if (containerRef.current) { resizeObserver.observe(containerRef.current); } - if (isExpanded) { - window.addEventListener("resize", updatePreviewSize); - } - return () => { - resizeObserver.disconnect(); - if (isExpanded) { - window.removeEventListener("resize", updatePreviewSize); - } - }; - }, [canvasSize.width, canvasSize.height, isExpanded]); + return () => resizeObserver.disconnect(); + }, [canvasWidth, canvasHeight]); + + return dimensions; +} + +export function PreviewPanel() { + const { tracks } = useTimelineStore(); + const { isPlaying, toggle } = usePlaybackStore(); + const { activeProject } = useProjectStore(); + const { mediaFiles } = useMediaStore(); + const containerRef = useRef(null); + const [isExpanded, setIsExpanded] = useState(false); + + usePreviewAudio(mediaFiles); + + const previewDimensions = usePreviewDimensions(containerRef); + const hasAnyElements = tracks.some((track) => track.elements.length > 0); + const shouldRenderPreview = hasAnyElements || activeProject?.backgroundType; + + const toggleExpanded = useCallback(() => { + setIsExpanded((prev) => !prev); + }, []); useEffect(() => { const handleEscapeKey = (event: KeyboardEvent) => { @@ -166,968 +113,98 @@ export function PreviewPanel() { }; }, [isExpanded]); - useEffect(() => { - const handleMouseMove = (e: MouseEvent) => { - if (!dragState.isDragging) return; - - const deltaX = e.clientX - dragState.startX; - const deltaY = e.clientY - dragState.startY; - - const scaleRatio = previewDimensions.width / canvasSize.width; - const newX = dragState.initialElementX + deltaX / scaleRatio; - const newY = dragState.initialElementY + deltaY / scaleRatio; - - const halfWidth = dragState.elementWidth / scaleRatio / 2; - const halfHeight = dragState.elementHeight / scaleRatio / 2; - - const constrainedX = Math.max( - -canvasSize.width / 2 + halfWidth, - Math.min(canvasSize.width / 2 - halfWidth, newX) - ); - const constrainedY = Math.max( - -canvasSize.height / 2 + halfHeight, - Math.min(canvasSize.height / 2 - halfHeight, newY) - ); - - setDragState((prev) => ({ - ...prev, - currentX: constrainedX, - currentY: constrainedY, - })); - }; - - const handleMouseUp = () => { - if (dragState.isDragging && dragState.trackId && dragState.elementId) { - updateTextElement(dragState.trackId, dragState.elementId, { - x: dragState.currentX, - y: dragState.currentY, - }); - } - setDragState((prev) => ({ ...prev, isDragging: false })); - }; - - if (dragState.isDragging) { - document.addEventListener("mousemove", handleMouseMove); - document.addEventListener("mouseup", handleMouseUp); - document.body.style.cursor = "grabbing"; - document.body.style.userSelect = "none"; - } - - return () => { - document.removeEventListener("mousemove", handleMouseMove); - document.removeEventListener("mouseup", handleMouseUp); - document.body.style.cursor = ""; - document.body.style.userSelect = ""; - }; - }, [dragState, previewDimensions, canvasSize, updateTextElement]); - - // Clear the frame cache when background settings change since they affect rendering - useEffect(() => { - invalidateCache(); - }, [ - mediaFiles, - activeProject?.backgroundColor, - activeProject?.backgroundType, - invalidateCache, - ]); - - const handleTextMouseDown = ( - e: React.MouseEvent, - element: any, - trackId: string - ) => { - e.preventDefault(); - e.stopPropagation(); - - const rect = e.currentTarget.getBoundingClientRect(); - - setDragState({ - isDragging: true, - elementId: element.id, - trackId, - startX: e.clientX, - startY: e.clientY, - initialElementX: element.x, - initialElementY: element.y, - currentX: element.x, - currentY: element.y, - elementWidth: rect.width, - elementHeight: rect.height, - }); - }; - - const toggleExpanded = useCallback(() => { - setIsExpanded((prev) => !prev); - }, []); - - const hasAnyElements = tracks.some((track) => track.elements.length > 0); - const shouldRenderPreview = hasAnyElements || activeProject?.backgroundType; - const getActiveElements = (): ActiveElement[] => { - const activeElements: ActiveElement[] = []; - - // Iterate tracks from bottom to top so topmost track renders last (on top) - [...tracks].reverse().forEach((track) => { - track.elements.forEach((element) => { - if (element.hidden) return; - const elementStart = element.startTime; - const elementEnd = - element.startTime + - (element.duration - element.trimStart - element.trimEnd); - - if (currentTime >= elementStart && currentTime < elementEnd) { - let mediaItem = null; - if (element.type === "media") { - mediaItem = - element.mediaId === "test" - ? null - : mediaFiles.find((item) => item.id === element.mediaId) || - null; - } - activeElements.push({ element, track, mediaItem }); - } - }); - }); - - return activeElements; - }; - - const activeElements = getActiveElements(); - - // Ensure first frame after mount/seek renders immediately - useEffect(() => { - const onSeek = () => { - lastFrameTimeRef.current = -Infinity; - renderSeqRef.current++; - }; - window.addEventListener("playback-seek", onSeek as EventListener); - lastFrameTimeRef.current = -Infinity; - return () => { - window.removeEventListener("playback-seek", onSeek as EventListener); - }; - }, []); - - // Web Audio: schedule only on play/pause/seek/volume/mute changes - useEffect(() => { - const stopAll = () => { - for (const src of playingSourcesRef.current) { - try { - src.stop(); - } catch {} - } - playingSourcesRef.current.clear(); - }; - - type WebAudioWindow = Window & { - AudioContext?: typeof AudioContext; - webkitAudioContext?: typeof AudioContext; - }; - const ensureAudioGraph = async () => { - if (!audioContextRef.current) { - const win = window as WebAudioWindow; - const Ctx = win.AudioContext ?? win.webkitAudioContext; - if (!Ctx) return; - audioContextRef.current = new Ctx(); - } - if (!audioGainRef.current) { - audioGainRef.current = audioContextRef.current!.createGain(); - audioGainRef.current.connect(audioContextRef.current!.destination); - } - if (audioContextRef.current!.state === "suspended") { - try { - await audioContextRef.current!.resume(); - } catch {} - } - const gainValue = muted ? 0 : Math.max(0, Math.min(1, volume)); - audioGainRef.current!.gain.setValueAtTime( - gainValue, - audioContextRef.current!.currentTime - ); - }; - - const scheduleNow = async () => { - await ensureAudioGraph(); - const audioCtx = audioContextRef.current!; - const gain = audioGainRef.current!; - - const tracksSnapshot = useTimelineStore.getState().tracks; - const mediaList = mediaFiles; - const idToMedia = new Map(mediaList.map((m) => [m.id, m] as const)); - const playbackNow = usePlaybackStore.getState().currentTime; - - const audible: Array<{ - id: string; - elementStart: number; - trimStart: number; - trimEnd: number; - duration: number; - muted: boolean; - trackMuted: boolean; - }> = []; - const uniqueIds = new Set(); - for (const track of tracksSnapshot) { - for (const element of track.elements) { - if (element.type !== "media") continue; - const media = idToMedia.get(element.mediaId); - if (!media || media.type !== "audio") continue; - const visibleDuration = - element.duration - element.trimStart - element.trimEnd; - if (visibleDuration <= 0) continue; - const localTime = playbackNow - element.startTime + element.trimStart; - if (localTime < 0 || localTime >= visibleDuration) continue; - audible.push({ - id: media.id, - elementStart: element.startTime, - trimStart: element.trimStart, - trimEnd: element.trimEnd, - duration: element.duration, - muted: !!element.muted, - trackMuted: !!track.muted, - }); - uniqueIds.add(media.id); - } - } - - if (audible.length === 0) return; - - // Decode buffers as needed - const decodePromises: Array> = []; - for (const id of uniqueIds) { - if (!audioBuffersRef.current.has(id)) { - const mediaItem = idToMedia.get(id); - if (!mediaItem) continue; - const p = (async () => { - const arr = await mediaItem.file.arrayBuffer(); - const buf = await audioCtx.decodeAudioData(arr.slice(0)); - audioBuffersRef.current.set(id, buf); - })(); - decodePromises.push(p); - } - } - await Promise.all(decodePromises); - - const startAt = audioCtx.currentTime + 0.02; - for (const entry of audible) { - if (entry.muted || entry.trackMuted) continue; - const buffer = audioBuffersRef.current.get(entry.id); - if (!buffer) continue; - const visibleDuration = - entry.duration - entry.trimStart - entry.trimEnd; - const localTime = Math.max( - 0, - playbackNow - entry.elementStart + entry.trimStart - ); - const playDuration = Math.max(0, visibleDuration - localTime); - if (playDuration <= 0) continue; - const src = audioCtx.createBufferSource(); - src.buffer = buffer; - src.connect(gain); - try { - src.start(startAt, localTime, playDuration); - playingSourcesRef.current.add(src); - } catch {} - } - }; - - const onSeek = () => { - if (!isPlaying) return; - for (const src of playingSourcesRef.current) { - try { - src.stop(); - } catch {} - } - playingSourcesRef.current.clear(); - void scheduleNow(); - }; - - // Apply volume/mute changes immediately - void ensureAudioGraph(); - - // Start/stop on play state changes - for (const src of playingSourcesRef.current) { - try { - src.stop(); - } catch {} - } - playingSourcesRef.current.clear(); - if (isPlaying) { - void scheduleNow(); - } - - window.addEventListener("playback-seek", onSeek as EventListener); - return () => { - window.removeEventListener("playback-seek", onSeek as EventListener); - for (const src of playingSourcesRef.current) { - try { - src.stop(); - } catch {} - } - playingSourcesRef.current.clear(); - }; - }, [isPlaying, volume, muted, mediaFiles]); - - // Canvas: draw current frame with caching - useEffect(() => { - const draw = async () => { - const canvas = canvasRef.current; - if (!canvas) return; - const mainCtx = canvas.getContext("2d"); - if (!mainCtx) return; - - // Set canvas internal resolution to avoid blurry scaling - const displayWidth = Math.max(1, Math.floor(previewDimensions.width)); - const displayHeight = Math.max(1, Math.floor(previewDimensions.height)); - if (canvas.width !== displayWidth || canvas.height !== displayHeight) { - canvas.width = displayWidth; - canvas.height = displayHeight; - } - - // Throttle rendering to project FPS during playback only - const fps = activeProject?.fps || DEFAULT_FPS; - const minDelta = 1 / fps; - if (isPlaying) { - if (currentTime - lastFrameTimeRef.current < minDelta) { - return; - } - lastFrameTimeRef.current = currentTime; - } - - const cachedFrame = getCachedFrame( - currentTime, - tracks, - mediaFiles, - activeProject, - currentScene?.id - ); - if (cachedFrame) { - mainCtx.putImageData(cachedFrame, 0, 0); - - // Pre-render nearby frames in background - if (!isPlaying) { - // Only during scrubbing to avoid interfering with playback - preRenderNearbyFrames( - currentTime, - tracks, - mediaFiles, - activeProject, - async (time: number) => { - const tempCanvas = document.createElement("canvas"); - tempCanvas.width = displayWidth; - tempCanvas.height = displayHeight; - const tempCtx = tempCanvas.getContext("2d"); - if (!tempCtx) - throw new Error("Failed to create temp canvas context"); - - await renderTimelineFrame({ - ctx: tempCtx, - time, - canvasWidth: displayWidth, - canvasHeight: displayHeight, - tracks, - mediaFiles, - backgroundType: activeProject?.backgroundType, - blurIntensity: activeProject?.blurIntensity, - backgroundColor: - activeProject?.backgroundType === "blur" - ? undefined - : activeProject?.backgroundColor || "#000000", - projectCanvasSize: canvasSize, - }); - - return tempCtx.getImageData(0, 0, displayWidth, displayHeight); - }, - currentScene?.id, - 3 - ); - } else { - // Small lookahead while playing - preRenderNearbyFrames( - currentTime, - tracks, - mediaFiles, - activeProject, - async (time: number) => { - const tempCanvas = document.createElement("canvas"); - tempCanvas.width = displayWidth; - tempCanvas.height = displayHeight; - const tempCtx = tempCanvas.getContext("2d"); - if (!tempCtx) - throw new Error("Failed to create temp canvas context"); - - await renderTimelineFrame({ - ctx: tempCtx, - time, - canvasWidth: displayWidth, - canvasHeight: displayHeight, - tracks, - mediaFiles, - backgroundType: activeProject?.backgroundType, - blurIntensity: activeProject?.blurIntensity, - backgroundColor: - activeProject?.backgroundType === "blur" - ? undefined - : activeProject?.backgroundColor || "#000000", - projectCanvasSize: canvasSize, - }); - - return tempCtx.getImageData(0, 0, displayWidth, displayHeight); - }, - currentScene?.id, - 1 - ); - } - return; - } - - // Cache miss - render from scratch - if (!offscreenCanvasRef.current) { - const hasOffscreen = - typeof (globalThis as unknown as { OffscreenCanvas?: unknown }) - .OffscreenCanvas !== "undefined"; - if (hasOffscreen) { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - offscreenCanvasRef.current = new (globalThis as any).OffscreenCanvas( - displayWidth, - displayHeight - ) as OffscreenCanvas; - } else { - const c = document.createElement("canvas"); - c.width = displayWidth; - c.height = displayHeight; - offscreenCanvasRef.current = c; - } - } - // Ensure size matches - if ( - offscreenCanvasRef.current && - (offscreenCanvasRef.current as HTMLCanvasElement).getContext - ) { - const c = offscreenCanvasRef.current as HTMLCanvasElement; - if (c.width !== displayWidth || c.height !== displayHeight) { - c.width = displayWidth; - c.height = displayHeight; - } - } else { - const c = offscreenCanvasRef.current as OffscreenCanvas; - // @ts-ignore width/height exist on OffscreenCanvas in modern browsers - if ( - (c as unknown as { width: number }).width !== displayWidth || - (c as unknown as { height: number }).height !== displayHeight - ) { - // @ts-ignore - (c as unknown as { width: number }).width = displayWidth; - // @ts-ignore - (c as unknown as { height: number }).height = displayHeight; - } - } - const offscreenCanvas = offscreenCanvasRef.current as - | HTMLCanvasElement - | OffscreenCanvas; - const offCtx = (offscreenCanvas as HTMLCanvasElement).getContext - ? (offscreenCanvas as HTMLCanvasElement).getContext("2d") - : (offscreenCanvas as OffscreenCanvas).getContext("2d"); - if (!offCtx) return; - - await renderTimelineFrame({ - ctx: offCtx as CanvasRenderingContext2D, - time: currentTime, - canvasWidth: displayWidth, - canvasHeight: displayHeight, - tracks, - mediaFiles, - backgroundType: activeProject?.backgroundType, - blurIntensity: activeProject?.blurIntensity, - backgroundColor: - activeProject?.backgroundType === "blur" - ? undefined - : activeProject?.backgroundColor || "#000000", - projectCanvasSize: canvasSize, - }); - - const imageData = (offCtx as CanvasRenderingContext2D).getImageData( - 0, - 0, - displayWidth, - displayHeight - ); - cacheFrame( - currentTime, - imageData, - tracks, - mediaFiles, - activeProject, - currentScene?.id - ); - - // Blit offscreen to visible canvas - mainCtx.clearRect(0, 0, displayWidth, displayHeight); - if ((offscreenCanvas as HTMLCanvasElement).getContext) { - mainCtx.drawImage(offscreenCanvas as HTMLCanvasElement, 0, 0); - } else { - mainCtx.drawImage( - offscreenCanvas as unknown as CanvasImageSource, - 0, - 0 - ); - } - }; - - void draw(); - }, [ - activeElements, - currentTime, - previewDimensions.width, - previewDimensions.height, - canvasSize.width, - canvasSize.height, - activeProject?.backgroundType, - activeProject?.backgroundColor, - getCachedFrame, - cacheFrame, - preRenderNearbyFrames, - isPlaying, - ]); - - // Get media elements for blur background (video/image only) - const getBlurBackgroundElements = (): ActiveElement[] => { - return activeElements.filter( - ({ element, mediaItem }) => - element.type === "media" && - mediaItem && - (mediaItem.type === "video" || mediaItem.type === "image") && - element.mediaId !== "test" // Exclude test elements - ); - }; - - const blurBackgroundElements = getBlurBackgroundElements(); - - // Render blur background layer (handled by canvas now) - const renderBlurBackground = () => null; - - // Render an element (canvas handles visuals now). Audio playback to be implemented via Web Audio. - const renderElement = (_elementData: ActiveElement) => null; - - return ( - <> -
-
-
- {shouldRenderPreview ? ( -
- {renderBlurBackground()} - - {activeElements.length === 0 ? ( - <> - ) : ( - activeElements.map((elementData) => renderElement(elementData)) - )} - -
- ) : null} - -
- - -
-
- - {isExpanded && ( - - )} - - ); -} - -function FullscreenToolbar({ - hasAnyElements, - onToggleExpanded, - currentTime, - setCurrentTime, - toggle, - getTotalDuration, -}: { - hasAnyElements: boolean; - onToggleExpanded: () => void; - currentTime: number; - setCurrentTime: (time: number) => void; - toggle: () => void; - getTotalDuration: () => number; -}) { - const { isPlaying, seek } = usePlaybackStore(); - const { activeProject } = useProjectStore(); - const [isDragging, setIsDragging] = useState(false); - - const totalDuration = getTotalDuration(); - const progress = totalDuration > 0 ? (currentTime / totalDuration) * 100 : 0; - - const handleTimelineClick = (e: React.MouseEvent) => { - if (!hasAnyElements) return; - const rect = e.currentTarget.getBoundingClientRect(); - const clickX = e.clientX - rect.left; - const percentage = Math.max(0, Math.min(1, clickX / rect.width)); - const newTime = percentage * totalDuration; - setCurrentTime(Math.max(0, Math.min(newTime, totalDuration))); - }; - - const handleTimelineDrag = (e: React.MouseEvent) => { - if (!hasAnyElements) return; - e.preventDefault(); - e.stopPropagation(); - const rect = e.currentTarget.getBoundingClientRect(); - setIsDragging(true); - - const handleMouseMove = (moveEvent: MouseEvent) => { - moveEvent.preventDefault(); - const dragX = moveEvent.clientX - rect.left; - const percentage = Math.max(0, Math.min(1, dragX / rect.width)); - const newTime = percentage * totalDuration; - setCurrentTime(Math.max(0, Math.min(newTime, totalDuration))); - }; - - const handleMouseUp = () => { - setIsDragging(false); - document.removeEventListener("mousemove", handleMouseMove); - document.removeEventListener("mouseup", handleMouseUp); - document.body.style.userSelect = ""; - }; - - document.body.style.userSelect = "none"; - document.addEventListener("mousemove", handleMouseMove); - document.addEventListener("mouseup", handleMouseUp); - handleMouseMove(e.nativeEvent); - }; - - const skipBackward = () => { - const newTime = Math.max(0, currentTime - 1); - setCurrentTime(newTime); - }; - - const skipForward = () => { - const newTime = Math.min(totalDuration, currentTime + 1); - setCurrentTime(newTime); - }; - - return ( -
-
- - / - - {formatTimeCode( - totalDuration, - "HH:MM:SS:FF", - activeProject?.fps || DEFAULT_FPS - )} - -
- -
- - - -
- -
-
-
-
-
-
- - -
- ); -} - -function FullscreenPreview({ - previewDimensions, - activeProject, - renderBlurBackground, - activeElements, - renderElement, - blurBackgroundElements, - hasAnyElements, - toggleExpanded, - currentTime, - setCurrentTime, - toggle, - getTotalDuration, -}: { - previewDimensions: { width: number; height: number }; - activeProject: any; - renderBlurBackground: () => React.ReactNode; - activeElements: ActiveElement[]; - renderElement: (elementData: ActiveElement, index: number) => React.ReactNode; - blurBackgroundElements: ActiveElement[]; - hasAnyElements: boolean; - toggleExpanded: () => void; - currentTime: number; - setCurrentTime: (time: number) => void; - toggle: () => void; - getTotalDuration: () => number; -}) { - return ( -
-
-
- {renderBlurBackground()} - {activeElements.length === 0 ? ( -
- No elements at current time -
- ) : ( - activeElements.map((elementData, index) => - renderElement(elementData, index) - ) - )} - -
-
-
- -
-
- ); -} - -function PreviewToolbar({ - hasAnyElements, - onToggleExpanded, - isExpanded, - currentTime, - setCurrentTime, - toggle, - getTotalDuration, -}: { - hasAnyElements: boolean; - onToggleExpanded: () => void; - isExpanded: boolean; - currentTime: number; - setCurrentTime: (time: number) => void; - toggle: () => void; - getTotalDuration: () => number; -}) { - const { isPlaying } = usePlaybackStore(); - const { layoutGuide, toggleLayoutGuide } = useEditorStore(); - if (isExpanded) { return ( - +
+
+
+ + +
+
+
+ +
+
); } return ( -
-
- - - +
- - -
-
-

Layout guide

-

- Show platform-specific layout guides to help align your - content with interface elements like profile pictures, - usernames, and interaction buttons. -

-
-
-
- - toggleLayoutGuide(layoutGuide.platform || "tiktok") - } - /> - -
- {Object.entries(PLATFORM_LAYOUTS).map(([platform, label]) => ( -
- - toggleLayoutGuide(platform as PlatformLayout) - } - /> - -
- ))} -
-
- - - + +
+
); diff --git a/apps/web/src/hooks/use-preview-audio.ts b/apps/web/src/hooks/use-preview-audio.ts new file mode 100644 index 00000000..7b58fd15 --- /dev/null +++ b/apps/web/src/hooks/use-preview-audio.ts @@ -0,0 +1,172 @@ +import { useRef, useEffect } from "react"; +import { useTimelineStore } from "@/stores/timeline-store"; +import { usePlaybackStore } from "@/stores/playback-store"; +import { MediaFile } from "@/types/media"; + +export function usePreviewAudio(mediaFiles: MediaFile[]) { + const audioContextRef = useRef(null); + const audioGainRef = useRef(null); + const audioBuffersRef = useRef>(new Map()); + const playingSourcesRef = useRef>(new Set()); + + const { isPlaying, volume, muted } = usePlaybackStore(); + + useEffect(() => { + const stopAll = () => { + for (const src of playingSourcesRef.current) { + try { + src.stop(); + } catch {} + } + playingSourcesRef.current.clear(); + }; + + type WebAudioWindow = Window & { + AudioContext?: typeof AudioContext; + webkitAudioContext?: typeof AudioContext; + }; + + const ensureAudioGraph = async () => { + if (!audioContextRef.current) { + const win = window as WebAudioWindow; + const Ctx = win.AudioContext ?? win.webkitAudioContext; + if (!Ctx) return; + audioContextRef.current = new Ctx(); + } + if (!audioGainRef.current) { + audioGainRef.current = audioContextRef.current!.createGain(); + audioGainRef.current.connect(audioContextRef.current!.destination); + } + if (audioContextRef.current!.state === "suspended") { + try { + await audioContextRef.current!.resume(); + } catch {} + } + const gainValue = muted ? 0 : Math.max(0, Math.min(1, volume)); + audioGainRef.current!.gain.setValueAtTime( + gainValue, + audioContextRef.current!.currentTime + ); + }; + + const scheduleNow = async () => { + await ensureAudioGraph(); + const audioCtx = audioContextRef.current!; + const gain = audioGainRef.current!; + + const tracksSnapshot = useTimelineStore.getState().tracks; + const mediaList = mediaFiles; + const idToMedia = new Map(mediaList.map((m) => [m.id, m] as const)); + const playbackNow = usePlaybackStore.getState().currentTime; + + const audible: Array<{ + id: string; + elementStart: number; + trimStart: number; + trimEnd: number; + duration: number; + muted: boolean; + trackMuted: boolean; + }> = []; + const uniqueIds = new Set(); + + for (const track of tracksSnapshot) { + for (const element of track.elements) { + if (element.type !== "media") continue; + const media = idToMedia.get(element.mediaId); + if (!media || media.type !== "audio") continue; + const visibleDuration = + element.duration - element.trimStart - element.trimEnd; + if (visibleDuration <= 0) continue; + const localTime = playbackNow - element.startTime + element.trimStart; + if (localTime < 0 || localTime >= visibleDuration) continue; + + const mediaElement = element; + audible.push({ + id: media.id, + elementStart: element.startTime, + trimStart: element.trimStart, + trimEnd: element.trimEnd, + duration: element.duration, + muted: !!mediaElement.muted, + trackMuted: !!track.muted, + }); + uniqueIds.add(media.id); + } + } + + if (audible.length === 0) return; + + const decodePromises: Array> = []; + for (const id of uniqueIds) { + if (!audioBuffersRef.current.has(id)) { + const mediaItem = idToMedia.get(id); + if (!mediaItem) continue; + const p = (async () => { + const arr = await mediaItem.file.arrayBuffer(); + const buf = await audioCtx.decodeAudioData(arr.slice(0)); + audioBuffersRef.current.set(id, buf); + })(); + decodePromises.push(p); + } + } + await Promise.all(decodePromises); + + const startAt = audioCtx.currentTime + 0.02; + for (const entry of audible) { + if (entry.muted || entry.trackMuted) continue; + const buffer = audioBuffersRef.current.get(entry.id); + if (!buffer) continue; + const visibleDuration = + entry.duration - entry.trimStart - entry.trimEnd; + const localTime = Math.max( + 0, + playbackNow - entry.elementStart + entry.trimStart + ); + const playDuration = Math.max(0, visibleDuration - localTime); + if (playDuration <= 0) continue; + const src = audioCtx.createBufferSource(); + src.buffer = buffer; + src.connect(gain); + try { + src.start(startAt, localTime, playDuration); + playingSourcesRef.current.add(src); + } catch {} + } + }; + + const onSeek = () => { + if (!isPlaying) return; + for (const src of playingSourcesRef.current) { + try { + src.stop(); + } catch {} + } + playingSourcesRef.current.clear(); + void scheduleNow(); + }; + + void ensureAudioGraph(); + + for (const src of playingSourcesRef.current) { + try { + src.stop(); + } catch {} + } + playingSourcesRef.current.clear(); + if (isPlaying) { + void scheduleNow(); + } + + window.addEventListener("playback-seek", onSeek as EventListener); + return () => { + window.removeEventListener("playback-seek", onSeek as EventListener); + for (const src of playingSourcesRef.current) { + try { + src.stop(); + } catch {} + } + playingSourcesRef.current.clear(); + }; + }, [isPlaying, volume, muted, mediaFiles]); +} diff --git a/apps/web/src/hooks/use-raf-loop.ts b/apps/web/src/hooks/use-raf-loop.ts new file mode 100644 index 00000000..4f8b34ed --- /dev/null +++ b/apps/web/src/hooks/use-raf-loop.ts @@ -0,0 +1,28 @@ +import { useEffect, useRef } from "react"; + +export function useRafLoop(callback: () => void) { + const requestRef = useRef(0); + const callbackRef = useRef(callback); + + // Update callback ref when callback changes + useEffect(() => { + callbackRef.current = callback; + }, [callback]); + + useEffect(() => { + const loop = () => { + callbackRef.current(); + requestRef.current = requestAnimationFrame(loop); + }; + + requestRef.current = requestAnimationFrame(loop); + + return () => { + cancelAnimationFrame(requestRef.current); + }; + }, []); +} + + + +