diff --git a/apps/web/src/components/editor/panels/timeline/audio-volume-line.tsx b/apps/web/src/components/editor/panels/timeline/audio-volume-line.tsx new file mode 100644 index 00000000..208f3beb --- /dev/null +++ b/apps/web/src/components/editor/panels/timeline/audio-volume-line.tsx @@ -0,0 +1,275 @@ +"use client"; + +import { useCallback, useRef, useState } from "react"; +import { createPortal } from "react-dom"; +import { useEditor } from "@/hooks/use-editor"; +import { DEFAULTS } from "@/lib/timeline/defaults"; +import { + getDbFromLinePos, + getLinePosFromDb, +} from "@/lib/timeline/audio-display"; +import { VOLUME_DB_MAX, VOLUME_DB_MIN } from "@/lib/timeline/audio-constants"; +import { hasAnimatedVolume } from "@/lib/timeline/audio-state"; +import type { AudioElement } from "@/lib/timeline/types"; +import { + clamp, + formatNumberForDisplay, + getFractionDigitsForStep, + isNearlyEqual, + snapToStep, +} from "@/utils/math"; +import { cn } from "@/utils/ui"; + +const HIT_AREA_HEIGHT_PX = 14; +const TOOLTIP_OFFSET_PX = 10; +const VOLUME_STEP = 0.1; +const VOLUME_FRACTION_DIGITS = getFractionDigitsForStep({ step: VOLUME_STEP }); + +function clampVolume({ value }: { value: number }): number { + return clamp({ + value: snapToStep({ value, step: VOLUME_STEP }), + min: VOLUME_DB_MIN, + max: VOLUME_DB_MAX, + }); +} + +function getVolumeFromPointer({ + clientY, + rect, +}: { + clientY: number; + rect: DOMRect; +}): number { + const clampedOffset = clamp({ + value: clientY - rect.top, + min: 0, + max: rect.height, + }); + const progressPercent = + rect.height <= 0 ? 0 : (clampedOffset / rect.height) * 100; + return clampVolume({ value: getDbFromLinePos({ percent: progressPercent }) }); +} + +export function AudioVolumeLine({ + element, + trackId, +}: { + element: AudioElement; + trackId: string; +}) { + const editor = useEditor(); + const surfaceRef = useRef(null); + const activePointerIdRef = useRef(null); + const startVolumeRef = useRef(element.volume ?? DEFAULTS.element.volume); + const lastPreviewVolumeRef = useRef( + element.volume ?? DEFAULTS.element.volume, + ); + const hasChangedRef = useRef(false); + const [isDragging, setIsDragging] = useState(false); + const [tooltipClientPos, setTooltipClientPos] = useState<{ + x: number; + y: number; + } | null>(null); + + const hasAnimatedEnvelope = hasAnimatedVolume({ element }); + const currentVolume = element.volume ?? DEFAULTS.element.volume; + const lineTop = `${getLinePosFromDb({ db: currentVolume })}%`; + + const volumeLabel = `${formatNumberForDisplay({ + value: currentVolume, + fractionDigits: VOLUME_FRACTION_DIGITS, + })} dB`; + + const previewVolume = useCallback( + (nextVolume: number) => { + if ( + isNearlyEqual({ + leftValue: nextVolume, + rightValue: lastPreviewVolumeRef.current, + }) + ) { + return; + } + + editor.timeline.previewElements({ + updates: [ + { + trackId, + elementId: element.id, + updates: { volume: nextVolume }, + }, + ], + }); + lastPreviewVolumeRef.current = nextVolume; + hasChangedRef.current = !isNearlyEqual({ + leftValue: startVolumeRef.current, + rightValue: nextVolume, + }); + }, + [editor, element.id, trackId], + ); + + const finishDrag = useCallback( + ({ shouldCommit }: { shouldCommit: boolean }) => { + activePointerIdRef.current = null; + setIsDragging(false); + + if (shouldCommit && hasChangedRef.current) { + editor.timeline.commitPreview(); + } else { + editor.timeline.discardPreview(); + } + + hasChangedRef.current = false; + lastPreviewVolumeRef.current = startVolumeRef.current; + setTooltipClientPos(null); + }, + [editor], + ); + + const updateFromPointer = useCallback( + ({ clientX, clientY }: { clientX: number; clientY: number }) => { + const rect = surfaceRef.current?.getBoundingClientRect(); + if (!rect) { + return; + } + + setTooltipClientPos({ + x: clientX + TOOLTIP_OFFSET_PX, + y: clientY - TOOLTIP_OFFSET_PX, + }); + previewVolume(getVolumeFromPointer({ clientY, rect })); + }, + [previewVolume], + ); + + const handleClick = useCallback((event: React.MouseEvent) => { + event.preventDefault(); + event.stopPropagation(); + }, []); + + const handleMouseDown = useCallback((event: React.MouseEvent) => { + event.stopPropagation(); + }, []); + + const handlePointerDown = useCallback( + (event: React.PointerEvent) => { + if (event.button !== 0) { + return; + } + + event.preventDefault(); + event.stopPropagation(); + editor.selection.setSelectedElements({ + elements: [{ trackId, elementId: element.id }], + }); + activePointerIdRef.current = event.pointerId; + startVolumeRef.current = currentVolume; + lastPreviewVolumeRef.current = currentVolume; + hasChangedRef.current = false; + setIsDragging(true); + event.currentTarget.setPointerCapture(event.pointerId); + updateFromPointer({ + clientX: event.clientX, + clientY: event.clientY, + }); + }, + [currentVolume, editor.selection, element.id, trackId, updateFromPointer], + ); + + const handlePointerMove = useCallback( + (event: React.PointerEvent) => { + if (activePointerIdRef.current !== event.pointerId) { + return; + } + + event.preventDefault(); + updateFromPointer({ + clientX: event.clientX, + clientY: event.clientY, + }); + }, + [updateFromPointer], + ); + + const handlePointerUp = useCallback( + (event: React.PointerEvent) => { + if (activePointerIdRef.current !== event.pointerId) { + return; + } + + event.preventDefault(); + event.stopPropagation(); + finishDrag({ shouldCommit: true }); + }, + [finishDrag], + ); + + const handlePointerCancel = useCallback( + (event: React.PointerEvent) => { + if (activePointerIdRef.current !== event.pointerId) { + return; + } + + event.preventDefault(); + event.stopPropagation(); + finishDrag({ shouldCommit: false }); + }, + [finishDrag], + ); + + const handleLostPointerCapture = useCallback(() => { + if (activePointerIdRef.current === null) { + return; + } + + finishDrag({ shouldCommit: hasChangedRef.current }); + }, [finishDrag]); + + if (hasAnimatedEnvelope) { + return null; + } + + return ( +
+
+
+ {/* biome-ignore lint/a11y/noStaticElementInteractions: timeline volume line is a pointer-only editing surface */} + {/* biome-ignore lint/a11y/useKeyWithClickEvents: timeline volume line is a pointer-only editing surface */} +
+ {isDragging && + tooltipClientPos && + createPortal( +
+ {volumeLabel} +
, + document.body, + )} +
+
+ ); +} diff --git a/apps/web/src/components/editor/panels/timeline/audio-waveform.tsx b/apps/web/src/components/editor/panels/timeline/audio-waveform.tsx index 430575e1..d2442a84 100644 --- a/apps/web/src/components/editor/panels/timeline/audio-waveform.tsx +++ b/apps/web/src/components/editor/panels/timeline/audio-waveform.tsx @@ -1,73 +1,123 @@ "use client"; -import { useCallback, useEffect, useRef } from "react"; +import { useCallback, useEffect, useLayoutEffect, useRef } from "react"; import { useResizeObserver } from "@/hooks/use-resize-observer"; -import { computeGlobalMaxRms, extractRmsRange } from "@/lib/media/audio"; +import { + buildWaveformSampleBuckets, + sampleSourceWaveformSummary, + type SourceWaveformSummary, +} from "@/lib/media/waveform-summary"; +import type { RetimeConfig } from "@/lib/timeline"; +import { getBarFractionFromOutputAmplitude } from "@/lib/timeline/audio-display"; +import { waveformCache } from "@/services/waveform-cache/service"; import { findScrollParent } from "@/utils/browser"; import { cn } from "@/utils/ui"; -const BAR_WIDTH = 2; +const BAR_WIDTH = 1; const BAR_GAP = 1; const BAR_STEP = BAR_WIDTH + BAR_GAP; +const WAVEFORM_BURN_COLOR = "rgba(255, 110, 20, 0.9)"; export const WAVEFORM_GAIN_SAMPLE_COUNT = 200; -function sampleGain({ +function sampleGainAtClipTime({ samples, - startFraction, - endFraction, - barIndex, - barCount, + clipTimeSec, + clipDurationSec, }: { samples: number[]; - startFraction: number; - endFraction: number; - barIndex: number; - barCount: number; + clipTimeSec: number; + clipDurationSec: number; }): number { - if (samples.length === 0) return 1; - const progress = - startFraction + - ((barIndex + 0.5) / barCount) * (endFraction - startFraction); - const rawIndex = Math.max(0, Math.min(1, progress)) * (samples.length - 1); + if (samples.length === 0 || clipDurationSec <= 0) { + return 1; + } + + const progress = Math.max(0, Math.min(1, clipTimeSec / clipDurationSec)); + const rawIndex = progress * (samples.length - 1); const lo = Math.floor(rawIndex); const hi = Math.min(samples.length - 1, lo + 1); return samples[lo] + (samples[hi] - samples[lo]) * (rawIndex - lo); } + interface AudioWaveformProps { + sourceKey: string; + sourceFile?: File; audioUrl?: string; audioBuffer?: AudioBuffer; gainSamples?: number[]; + pixelsPerSecond: number; + clipDurationSec: number; + retime?: RetimeConfig; + sourceStartSec: number; color?: string; + burnColor?: string; className?: string; } export function AudioWaveform({ + sourceKey, + sourceFile, audioUrl, audioBuffer, gainSamples, - color = "rgba(255, 255, 255, 0.7)", + pixelsPerSecond, + clipDurationSec, + retime, + sourceStartSec, + color, + burnColor = WAVEFORM_BURN_COLOR, className = "", }: AudioWaveformProps) { const canvasRef = useRef(null); const containerRef = useRef(null); - const bufferRef = useRef(null); - const globalMaxRef = useRef(1); + const summaryRef = useRef(null); const gainSamplesRef = useRef(gainSamples); + const pixelsPerSecondRef = useRef(pixelsPerSecond); + const clipDurationSecRef = useRef(clipDurationSec); + const retimeRef = useRef(retime); + const sourceStartSecRef = useRef(sourceStartSec); const scrollParentRef = useRef(null); const heightRef = useRef(0); + const lastRenderSignatureRef = useRef(null); + + gainSamplesRef.current = gainSamples; + pixelsPerSecondRef.current = pixelsPerSecond; + clipDurationSecRef.current = clipDurationSec; + retimeRef.current = retime; + sourceStartSecRef.current = sourceStartSec; + + const clearCanvas = useCallback(() => { + const canvas = canvasRef.current; + if (!canvas) { + return; + } + + const ctx = canvas.getContext("2d"); + if (ctx) { + ctx.setTransform(1, 0, 0, 1, 0, 0); + ctx.clearRect(0, 0, canvas.width, canvas.height); + } + lastRenderSignatureRef.current = null; + }, []); const drawVisible = useCallback(() => { const container = containerRef.current; const canvas = canvasRef.current; - const buffer = bufferRef.current; + const summary = summaryRef.current; const height = heightRef.current; - if (!container || !canvas || !buffer || height <= 0) return; - - const elementWidth = container.offsetWidth; - if (elementWidth <= 0) return; + if (!container || !canvas || !summary || height <= 0) { + clearCanvas(); + return; + } const containerRect = container.getBoundingClientRect(); + const elementWidth = containerRect.width; + if (elementWidth <= 0) { + clearCanvas(); + return; + } + const scrollParent = scrollParentRef.current; let clipLeft: number; @@ -86,13 +136,43 @@ export function AudioWaveform({ } const visibleWidth = clipRight - clipLeft; - if (visibleWidth <= 0) return; + if (visibleWidth <= 0) { + clearCanvas(); + return; + } const dpr = window.devicePixelRatio || 1; - const canvasW = Math.round(visibleWidth * dpr); - const canvasH = Math.round(height * dpr); - - if (canvasW <= 0 || canvasH <= 0) return; + const canvasW = Math.max(1, Math.ceil(visibleWidth * dpr)); + const canvasH = Math.max(1, Math.round(height * dpr)); + const barCount = Math.max(1, Math.floor(visibleWidth / BAR_STEP)); + const pixelsPerSecondValue = pixelsPerSecondRef.current; + const clipDurationSecValue = clipDurationSecRef.current; + const samples = gainSamplesRef.current; + const renderSignature = JSON.stringify({ + elementWidth, + clipLeft, + clipRight, + visibleWidth, + canvasW, + canvasH, + barCount, + dpr, + clipDurationSec: clipDurationSecValue, + sourceStartSec: sourceStartSecRef.current, + pixelsPerSecond: pixelsPerSecondValue, + retime: retimeRef.current ?? null, + summarySourceKey: summary.sourceKey, + summarySampleRate: summary.sampleRate, + summaryTotalSamples: summary.totalSamples, + summaryBucketSize: summary.bucketSize, + gainSamples: samples ?? null, + color, + burnColor, + }); + if (lastRenderSignatureRef.current === renderSignature) { + return; + } + lastRenderSignatureRef.current = renderSignature; canvas.width = canvasW; canvas.height = canvasH; @@ -100,97 +180,150 @@ export function AudioWaveform({ canvas.style.height = `${height}px`; canvas.style.left = `${clipLeft}px`; - const barCount = Math.max(1, Math.floor(visibleWidth / BAR_STEP)); - const startFraction = clipLeft / elementWidth; - const endFraction = clipRight / elementWidth; - const startSample = Math.floor(startFraction * buffer.length); - const endSample = Math.min( - buffer.length, - Math.ceil(endFraction * buffer.length), - ); + const backingScaleX = dpr; + const backingScaleY = canvasH / height; - const peaks = extractRmsRange({ - buffer, - count: barCount, - startSample, - endSample, - globalMax: globalMaxRef.current, + const sampleBuckets = buildWaveformSampleBuckets({ + clipLeftPx: clipLeft, + clipRightPx: clipRight, + barCount, + pixelsPerSecond: pixelsPerSecondValue, + clipDurationSec: clipDurationSecValue, + sourceStartSec: sourceStartSecRef.current, + retime: retimeRef.current, + sampleRate: summary.sampleRate, + maxSampleExclusive: summary.totalSamples, + barStepPx: BAR_STEP, + }); + const amplitudes = sampleSourceWaveformSummary({ + summary, + buckets: sampleBuckets, }); const ctx = canvas.getContext("2d"); - if (!ctx) return; + if (!ctx) { + return; + } + ctx.setTransform(1, 0, 0, 1, 0, 0); ctx.clearRect(0, 0, canvasW, canvasH); - ctx.scale(dpr, dpr); - ctx.fillStyle = color; - const maxBarHeight = height * 0.7; + const clipBottom = canvasH; - const samples = gainSamplesRef.current; for (let i = 0; i < barCount; i++) { + const barCenterPx = clipLeft + i * BAR_STEP + BAR_WIDTH * 0.5; + const clipCenterSec = Math.max( + 0, + Math.min(clipDurationSecValue, barCenterPx / pixelsPerSecondValue), + ); const gain = samples != null - ? sampleGain({ + ? sampleGainAtClipTime({ samples, - startFraction, - endFraction, - barIndex: i, - barCount, + clipTimeSec: clipCenterSec, + clipDurationSec: clipDurationSecValue, }) : 1; - const scaledPeak = Math.min(1, Math.max(0, peaks[i] * gain)); - const scaled = Math.log1p(scaledPeak) / Math.log1p(1); - const barH = Math.max(1, scaled * maxBarHeight); - ctx.fillRect(i * BAR_STEP, height - barH, BAR_WIDTH, barH); - } - }, [color]); + const amplitude = Math.max(0, amplitudes[i] ?? 0); + const outputAmplitude = amplitude * Math.max(0, gain); + const fraction = getBarFractionFromOutputAmplitude({ outputAmplitude }); + const barH = fraction > 0 ? Math.max(1, fraction * height) : 0; + if (barH <= 0) { + continue; + } - useEffect(() => { - gainSamplesRef.current = gainSamples; - drawVisible(); - }, [gainSamples, drawVisible]); + const barLeft = i * BAR_STEP; + const barRight = barLeft + BAR_WIDTH; + const deviceLeft = Math.round(barLeft * backingScaleX); + const deviceRight = Math.max( + deviceLeft + 1, + Math.round(barRight * backingScaleX), + ); + const deviceTop = Math.round((height - barH) * backingScaleY); + const deviceHeight = Math.max(1, clipBottom - deviceTop); + + ctx.fillStyle = color; + ctx.fillRect( + deviceLeft, + deviceTop, + deviceRight - deviceLeft, + deviceHeight, + ); + + if (outputAmplitude > 1) { + const burnHeight = Math.max(1, Math.round(BAR_WIDTH * backingScaleY)); + ctx.fillStyle = burnColor; + ctx.fillRect( + deviceLeft, + deviceTop, + deviceRight - deviceLeft, + burnHeight, + ); + } + } + }, [burnColor, clearCanvas, color]); useEffect(() => { let isCancelled = false; + summaryRef.current = null; + clearCanvas(); - async function load() { - let buffer = audioBuffer ?? null; - - if (!buffer && audioUrl) { - try { - const resp = await fetch(audioUrl); - const arrayBuffer = await resp.arrayBuffer(); - const actx = new AudioContext(); - buffer = await actx.decodeAudioData(arrayBuffer); - actx.close(); - } catch { + void waveformCache + .getSourceSummary({ + sourceKey, + audioBuffer, + sourceFile, + audioUrl, + }) + .then((summary) => { + if (isCancelled) { return; } - } + summaryRef.current = summary; + drawVisible(); + }) + .catch(() => { + // Waveform loading failed (e.g. corrupt file, unsupported format). + // Fail silently — a missing waveform is preferable to an error state. + if (!isCancelled) { + clearCanvas(); + } + }); - if (!buffer || isCancelled) return; - - bufferRef.current = buffer; - globalMaxRef.current = computeGlobalMaxRms({ buffer }); - drawVisible(); - } - - load(); return () => { isCancelled = true; }; - }, [audioUrl, audioBuffer, drawVisible]); + }, [audioBuffer, audioUrl, clearCanvas, drawVisible, sourceFile, sourceKey]); + + // biome-ignore lint/correctness/useExhaustiveDependencies: these props are mirrored into refs during render, but the effect must still re-run to redraw when they change. + useLayoutEffect(() => { + drawVisible(); + }, [ + drawVisible, + gainSamples, + pixelsPerSecond, + clipDurationSec, + retime, + sourceStartSec, + ]); useEffect(() => { const container = containerRef.current; - if (!container) return; + if (!container) { + return; + } scrollParentRef.current = findScrollParent({ element: container }); const scrollParent = scrollParentRef.current; - if (!scrollParent) return; + if (!scrollParent) { + return; + } - scrollParent.addEventListener("scroll", drawVisible, { passive: true }); - return () => scrollParent.removeEventListener("scroll", drawVisible); + const handleScroll = () => { + drawVisible(); + }; + scrollParent.addEventListener("scroll", handleScroll, { passive: true }); + return () => scrollParent.removeEventListener("scroll", handleScroll); }, [drawVisible]); const onResize = useCallback( diff --git a/apps/web/src/components/editor/panels/timeline/theme.ts b/apps/web/src/components/editor/panels/timeline/theme.ts index e7fc1f11..aae82ca3 100644 --- a/apps/web/src/components/editor/panels/timeline/theme.ts +++ b/apps/web/src/components/editor/panels/timeline/theme.ts @@ -11,7 +11,7 @@ export const TIMELINE_TRACK_THEME: Record< text: { elementClassName: "bg-[#5DBAA0]" }, audio: { elementClassName: "bg-[#8F5DBA]", - waveformColor: "rgba(255, 255, 255, 0.5)", + waveformColor: "rgba(255, 255, 255, 0.7)", }, graphic: { elementClassName: "bg-[#BA5D7A]" }, effect: { elementClassName: "bg-[#5d93ba]" }, diff --git a/apps/web/src/components/editor/panels/timeline/timeline-element.tsx b/apps/web/src/components/editor/panels/timeline/timeline-element.tsx index 9283c651..75a2dfa8 100644 --- a/apps/web/src/components/editor/panels/timeline/timeline-element.tsx +++ b/apps/web/src/components/editor/panels/timeline/timeline-element.tsx @@ -1,8 +1,10 @@ "use client"; +import { createContext, useContext } from "react"; import { useEditor } from "@/hooks/use-editor"; import { useAssetsPanelStore } from "@/stores/assets-panel-store"; import { AudioWaveform, WAVEFORM_GAIN_SAMPLE_COUNT } from "./audio-waveform"; +import { AudioVolumeLine } from "./audio-volume-line"; import { useElementPreview } from "@/hooks/use-element-preview"; import { useKeyframeDrag, @@ -45,6 +47,9 @@ import { isSourceAudioSeparated, } from "@/lib/timeline/audio-separation"; import { buildWaveformGainSamples } from "@/lib/timeline/audio-state"; +import { getTimelinePixelsPerSecond } from "@/lib/timeline"; +import { buildWaveformSourceKey } from "@/lib/media/waveform-summary"; +import { TICKS_PER_SECOND } from "@/lib/wasm/ticks"; import { getActionDefinition, type TAction, @@ -89,6 +94,8 @@ import { const KEYFRAME_INDICATOR_MIN_WIDTH_PX = 40; const ELEMENT_RING_WIDTH_PX = 1.5; + +const PixelsPerSecondContext = createContext(null); const THUMBNAIL_ASPECT_RATIO = 16 / 9; interface KeyframeIndicator { @@ -264,6 +271,7 @@ export function TimelineElement({ time: displayedDuration, zoomLevel, }); + const timelinePixelsPerSecond = getTimelinePixelsPerSecond({ zoomLevel }); const elementLeft = timelineTimeToSnappedPixels({ time: displayedStartTime, zoomLevel, @@ -364,137 +372,141 @@ export function TimelineElement({ ) : null; return ( - - -
0 - ? `${baseTrackHeight + expansionHeight}px` - : "100%", - transform: - isBeingDragged && dragState.isDragging - ? `translate3d(0, ${dragOffsetY}px, 0)` - : undefined, - }} - > - 0} - baseTrackHeight={baseTrackHeight} - expandedContent={expandedContent} - onElementClick={onElementClick} - onElementMouseDown={onElementMouseDown} - onResizeStart={onResizeStart} - isDropTarget={isDropTarget} - /> - {isSelected && ( -
- -
- )} -
- - - } - > - Split - - - {selectedElements.length === 1 && ( + + + +
0 + ? `${baseTrackHeight + expansionHeight}px` + : "100%", + transform: + isBeingDragged && dragState.isDragging + ? `translate3d(0, ${dragOffsetY}px, 0)` + : undefined, + }} + > + 0} + baseTrackHeight={baseTrackHeight} + expandedContent={expandedContent} + onElementClick={onElementClick} + onElementMouseDown={onElementMouseDown} + onResizeStart={onResizeStart} + isDropTarget={isDropTarget} + /> + {isSelected && ( +
+ +
+ )} +
+ + } + action="split" + icon={} > - Duplicate + Split - )} - {canElementHaveAudio(element) && hasAudio && ( - 1} - isCurrentElementSelected={isCurrentElementSelected} - isMuted={isMuted} - /> - )} - {canToggleCurrentSourceAudio && ( - - } - onClick={(event: React.MouseEvent) => { - event.stopPropagation(); - invokeAction("toggle-source-audio"); - }} - > - {sourceAudioLabel} - - )} - {canElementBeHidden(element) && ( - 1} - isCurrentElementSelected={isCurrentElementSelected} - /> - )} - {hasKeyframes && ( - } - onClick={(event: React.MouseEvent) => { - event.stopPropagation(); - toggleElementExpanded(element.id); - }} - > - {isExpanded ? "Collapse keyframes" : "Expand keyframes"} - - )} - {selectedElements.length === 1 && hasMediaId(element) && ( - <> + + {selectedElements.length === 1 && ( + } + > + Duplicate + + )} + {canElementHaveAudio(element) && hasAudio && ( + 1} + isCurrentElementSelected={isCurrentElementSelected} + isMuted={isMuted} + /> + )} + {canToggleCurrentSourceAudio && ( } - onClick={(event: React.MouseEvent) => - handleRevealInMedia({ event }) + icon={ + } + onClick={(event: React.MouseEvent) => { + event.stopPropagation(); + invokeAction("toggle-source-audio"); + }} > - Reveal media + {sourceAudioLabel} + )} + {canElementBeHidden(element) && ( + 1} + isCurrentElementSelected={isCurrentElementSelected} + /> + )} + {hasKeyframes && ( } - disabled + icon={} + onClick={(event: React.MouseEvent) => { + event.stopPropagation(); + toggleElementExpanded(element.id); + }} > - Replace media + {isExpanded ? "Collapse keyframes" : "Expand keyframes"} - - )} - - 1} - isCurrentElementSelected={isCurrentElementSelected} - elementType={element.type} - selectedCount={selectedElements.length} - /> - - + )} + {selectedElements.length === 1 && hasMediaId(element) && ( + <> + } + onClick={(event: React.MouseEvent) => + handleRevealInMedia({ event }) + } + > + Reveal media + + } + disabled + > + Replace media + + + )} + + 1} + isCurrentElementSelected={isCurrentElementSelected} + elementType={element.type} + selectedCount={selectedElements.length} + /> + + + ); } @@ -973,7 +985,19 @@ function GraphicElementContent({ ); } -function AudioElementContent({ element }: { element: AudioElement }) { +function AudioElementContent({ + element, + trackId, +}: { + element: AudioElement; + trackId: string; +}) { + const pixelsPerSecond = useContext(PixelsPerSecondContext); + if (pixelsPerSecond === null) { + throw new Error( + "AudioElementContent must be rendered inside PixelsPerSecondContext.Provider", + ); + } const mediaAssets = useEditor((e) => e.media.getAssets()); const mediaAsset = element.sourceType === "upload" @@ -984,29 +1008,53 @@ function AudioElementContent({ element }: { element: AudioElement }) { element.sourceType === "library" ? element.buffer : undefined; const audioUrl = element.sourceType === "library" ? element.sourceUrl : mediaAsset?.url; + const sourceFile = + element.sourceType === "upload" ? mediaAsset?.file : undefined; + const sourceKey = + element.sourceType === "upload" + ? buildWaveformSourceKey({ kind: "media", id: element.mediaId }) + : buildWaveformSourceKey({ kind: "library", id: element.sourceUrl }); const mediaLabel = mediaAsset?.name ?? element.name; const gainSamples = useMemo( () => - buildWaveformGainSamples({ element, count: WAVEFORM_GAIN_SAMPLE_COUNT }), + buildWaveformGainSamples({ + element, + count: WAVEFORM_GAIN_SAMPLE_COUNT, + }), [element], ); - - if (audioBuffer || audioUrl) { + if (audioBuffer || audioUrl || sourceFile) { return ( -
- +
+
+ + +
); } return ( - {element.name} +
+
+ + {element.name} + +
+ +
); } @@ -1108,7 +1156,7 @@ function MediaElementHeader({ return (
@@ -1133,7 +1181,7 @@ function ElementContent({ element, track }: ElementContentProps) { case "graphic": return ; case "audio": - return ; + return ; case "video": case "image": return ; diff --git a/apps/web/src/core/managers/media-manager.ts b/apps/web/src/core/managers/media-manager.ts index 4add3c04..e43340a7 100644 --- a/apps/web/src/core/managers/media-manager.ts +++ b/apps/web/src/core/managers/media-manager.ts @@ -4,6 +4,7 @@ import type { MediaAsset } from "@/lib/media/types"; import { storageService } from "@/services/storage/service"; import { generateUUID } from "@/utils/id"; import { videoCache } from "@/services/video-cache/service"; +import { waveformCache } from "@/services/waveform-cache/service"; import { BatchCommand, RemoveMediaAssetCommand } from "@/lib/commands"; export class MediaManager { @@ -94,6 +95,8 @@ export class MediaManager { } async clearProjectMedia({ projectId }: { projectId: string }): Promise { + waveformCache.clearAll(); + this.assets.forEach((asset) => { if (asset.url) { URL.revokeObjectURL(asset.url); @@ -120,6 +123,7 @@ export class MediaManager { clearAllAssets(): void { videoCache.clearAll(); + waveformCache.clearAll(); this.assets.forEach((asset) => { if (asset.url) { diff --git a/apps/web/src/core/managers/timeline-manager.ts b/apps/web/src/core/managers/timeline-manager.ts index 898a7e89..c58128fb 100644 --- a/apps/web/src/core/managers/timeline-manager.ts +++ b/apps/web/src/core/managers/timeline-manager.ts @@ -712,18 +712,31 @@ export class TimelineManager { updates: Partial; }>; }): void { + let changedOverlayCount = 0; for (const { elementId, updates: elementUpdates } of updates) { const existingOverlay = this.previewOverlay.get(elementId); - const mergedOverlay = { - ...existingOverlay, - ...elementUpdates, - } as Partial; - this.previewOverlay.set(elementId, mergedOverlay); + const changed = Object.entries(elementUpdates).some(([key, value]) => { + return !Object.is( + existingOverlay?.[key as keyof TimelineElement], + value, + ); + }); + if (changed) { + changedOverlayCount += 1; + const mergedOverlay = { + ...existingOverlay, + ...elementUpdates, + } as Partial; + this.previewOverlay.set(elementId, mergedOverlay); + } } const committedTracks = this.editor.scenes.getActiveSceneOrNull()?.tracks; if (!committedTracks) { return; } + if (changedOverlayCount === 0) { + return; + } this.previewTracks = this.applyPreviewOverlay(committedTracks); this.notify(); } diff --git a/apps/web/src/hooks/timeline/use-timeline-drag-drop.ts b/apps/web/src/hooks/timeline/use-timeline-drag-drop.ts index 868c638d..25c24b8b 100644 --- a/apps/web/src/hooks/timeline/use-timeline-drag-drop.ts +++ b/apps/web/src/hooks/timeline/use-timeline-drag-drop.ts @@ -89,8 +89,8 @@ export function useTimelineDragDrop({ const mediaAssets = editor.media.getAssets(); const media = mediaAssets.find((m) => m.id === mediaId); return media?.duration != null - ? Math.round(media.duration * TICKS_PER_SECOND) - : DEFAULT_NEW_ELEMENT_DURATION; + ? Math.round(media.duration * TICKS_PER_SECOND) + : DEFAULT_NEW_ELEMENT_DURATION; } return DEFAULT_NEW_ELEMENT_DURATION; }, @@ -468,10 +468,10 @@ export function useTimelineDragDrop({ }); if (!createdAsset) continue; - const duration = - createdAsset.duration != null - ? Math.round(createdAsset.duration * TICKS_PER_SECOND) - : DEFAULT_NEW_ELEMENT_DURATION; + const duration = + createdAsset.duration != null + ? Math.round(createdAsset.duration * TICKS_PER_SECOND) + : DEFAULT_NEW_ELEMENT_DURATION; const sceneTracks = editor.scenes.getActiveScene().tracks; const currentTime = editor.playback.getCurrentTime(); const reuseMainTrackId = @@ -498,9 +498,22 @@ export function useTimelineDragDrop({ const trackType: TrackType = createdAsset.type === "audio" ? "audio" : "video"; - let trackId: string | undefined; + const startTime = dropTarget?.xPosition ?? currentTime; + const element = buildElementFromMedia({ + mediaId: createdAsset.id, + mediaType: createdAsset.type, + name: createdAsset.name, + duration, + startTime, + }); + if (reuseMainTrackId) { - trackId = reuseMainTrackId; + editor.command.execute({ + command: new InsertElementCommand({ + element, + placement: { mode: "explicit", trackId: reuseMainTrackId }, + }), + }); } else { if (!dropTarget) continue; if (dropTarget.isNewTrack) { @@ -508,36 +521,33 @@ export function useTimelineDragDrop({ trackType, dropTarget.trackIndex, ); - trackId = addTrackCmd.getTrackId(); - editor.command.execute({ command: addTrackCmd }); + editor.command.execute({ + command: new BatchCommand([ + addTrackCmd, + new InsertElementCommand({ + element, + placement: { + mode: "explicit", + trackId: addTrackCmd.getTrackId(), + }, + }), + ]), + }); } else { - trackId = [ + const trackId = [ ...sceneTracks.overlay, sceneTracks.main, ...sceneTracks.audio, ][dropTarget.trackIndex]?.id; + if (!trackId) continue; + editor.command.execute({ + command: new InsertElementCommand({ + element, + placement: { mode: "explicit", trackId }, + }), + }); } } - - if (!trackId) continue; - - const element = buildElementFromMedia({ - mediaId: createdAsset.id, - mediaType: createdAsset.type, - name: createdAsset.name, - duration, - startTime: dropTarget?.xPosition ?? currentTime, - buffer: - createdAsset.type === "audio" - ? new AudioBuffer({ length: 1, sampleRate: 44100 }) - : undefined, - }); - - const insertCmd = new InsertElementCommand({ - element, - placement: { mode: "explicit", trackId }, - }); - editor.command.execute({ command: insertCmd }); } return { diff --git a/apps/web/src/lib/changelog/entries/0.4.0.md b/apps/web/src/lib/changelog/entries/0.4.0.md index e20c2785..141df133 100644 --- a/apps/web/src/lib/changelog/entries/0.4.0.md +++ b/apps/web/src/lib/changelog/entries/0.4.0.md @@ -10,4 +10,10 @@ changes: - type: new text: "Two new mask types: text and custom (draw any shape with the pen tool using bezier curves). Both support feather, invert, and stroke." - type: fixed - text: "Preview panning now keeps working while the Masks tab is open." \ No newline at end of file + text: "Preview panning now keeps working while the Masks tab is open." +- type: new + text: "Audio clips have a horizontal volume line you can drag up or down to change volume. The line position reflects the current volume setting and is consistent across clips: same dB always means the same position." +- type: improved + text: "Audio waveforms now show the actual shape of the audio. Quiet parts are visible, loud parts are tall, and any bar that peaks above 0 dBFS gets a colored tip at the top." +- type: fixed + text: "Trimming or cutting an audio clip no longer causes the waveform to compress or show the wrong section of audio." \ No newline at end of file diff --git a/apps/web/src/lib/commands/media/remove-media-asset.ts b/apps/web/src/lib/commands/media/remove-media-asset.ts index 5601a7e3..85d187d9 100644 --- a/apps/web/src/lib/commands/media/remove-media-asset.ts +++ b/apps/web/src/lib/commands/media/remove-media-asset.ts @@ -1,8 +1,10 @@ import { Command, type CommandResult } from "@/lib/commands/base-command"; import { EditorCore } from "@/core"; import type { MediaAsset } from "@/lib/media/types"; +import { buildWaveformSourceKey } from "@/lib/media/waveform-summary"; import { storageService } from "@/services/storage/service"; import { videoCache } from "@/services/video-cache/service"; +import { waveformCache } from "@/services/waveform-cache/service"; import { hasMediaId } from "@/lib/timeline/element-utils"; import type { SceneTracks } from "@/lib/timeline"; @@ -41,6 +43,12 @@ export class RemoveMediaAssetCommand extends Command { } videoCache.clearVideo({ mediaId: this.assetId }); + waveformCache.clearSource({ + sourceKey: buildWaveformSourceKey({ + kind: "media", + id: this.assetId, + }), + }); editor.media.setAssets({ assets: assets.filter((media) => media.id !== this.assetId), diff --git a/apps/web/src/lib/commands/timeline/element/insert-element.ts b/apps/web/src/lib/commands/timeline/element/insert-element.ts index 0e806d6e..ed05b8c6 100644 --- a/apps/web/src/lib/commands/timeline/element/insert-element.ts +++ b/apps/web/src/lib/commands/timeline/element/insert-element.ts @@ -3,7 +3,6 @@ import { EditorCore } from "@/core"; import type { CreateTimelineElement, SceneTracks, - TimelineTrack, TimelineElement, TrackType, } from "@/lib/timeline"; diff --git a/apps/web/src/lib/commands/timeline/element/split-elements.ts b/apps/web/src/lib/commands/timeline/element/split-elements.ts index 9799f7e8..0f2495f3 100644 --- a/apps/web/src/lib/commands/timeline/element/split-elements.ts +++ b/apps/web/src/lib/commands/timeline/element/split-elements.ts @@ -37,7 +37,9 @@ export class SplitElementsCommand extends Command { this.savedState = editor.scenes.getActiveScene().tracks; this.rightSideElements = []; - const splitTrack = ( + const splitTrack = < + TTrack extends { id: string; elements: TimelineElement[] }, + >( track: TTrack, ): TTrack => { const elementsToSplit = this.elements.filter( @@ -48,7 +50,7 @@ export class SplitElementsCommand extends Command { return track; } - let elements = track.elements.flatMap((element) => { + const elements = track.elements.flatMap((element) => { const shouldSplit = elementsToSplit.some( (target) => target.elementId === element.id, ); @@ -87,67 +89,67 @@ export class SplitElementsCommand extends Command { splitTime: relativeTime, shouldIncludeSplitBoundary: true, }); + let splitResult: TimelineElement[]; if (this.retainSide === "left") { - return [ - { - ...element, - duration: leftVisibleDuration, - trimEnd: element.trimEnd + rightSourceSpan, - name: `${element.name} (left)`, - animations: leftAnimations, - ...(retimeRef !== undefined ? { retime: retimeRef } : {}), - }, - ]; - } + splitResult = [ + { + ...element, + duration: leftVisibleDuration, + trimEnd: element.trimEnd + rightSourceSpan, + name: `${element.name} (left)`, + animations: leftAnimations, + ...(retimeRef !== undefined ? { retime: retimeRef } : {}), + }, + ]; + } else if (this.retainSide === "right") { + const newId = generateUUID(); + this.rightSideElements.push({ + trackId: track.id, + elementId: newId, + }); + splitResult = [ + { + ...element, + id: newId, + startTime: this.splitTime, + duration: rightVisibleDuration, + trimStart: element.trimStart + leftSourceSpan, + name: `${element.name} (right)`, + animations: rightAnimations, + ...(retimeRef !== undefined ? { retime: retimeRef } : {}), + }, + ]; + } else { + // "both" - split into two pieces + const secondElementId = generateUUID(); + this.rightSideElements.push({ + trackId: track.id, + elementId: secondElementId, + }); + splitResult = [ + { + ...element, + duration: leftVisibleDuration, + trimEnd: element.trimEnd + rightSourceSpan, + name: `${element.name} (left)`, + animations: leftAnimations, + ...(retimeRef !== undefined ? { retime: retimeRef } : {}), + }, + { + ...element, + id: secondElementId, + startTime: this.splitTime, + duration: rightVisibleDuration, + trimStart: element.trimStart + leftSourceSpan, + name: `${element.name} (right)`, + animations: rightAnimations, + ...(retimeRef !== undefined ? { retime: retimeRef } : {}), + }, + ]; + } - if (this.retainSide === "right") { - const newId = generateUUID(); - this.rightSideElements.push({ - trackId: track.id, - elementId: newId, - }); - return [ - { - ...element, - id: newId, - startTime: this.splitTime, - duration: rightVisibleDuration, - trimStart: element.trimStart + leftSourceSpan, - name: `${element.name} (right)`, - animations: rightAnimations, - ...(retimeRef !== undefined ? { retime: retimeRef } : {}), - }, - ]; - } - - // "both" - split into two pieces - const secondElementId = generateUUID(); - this.rightSideElements.push({ - trackId: track.id, - elementId: secondElementId, - }); - - return [ - { - ...element, - duration: leftVisibleDuration, - trimEnd: element.trimEnd + rightSourceSpan, - name: `${element.name} (left)`, - animations: leftAnimations, - ...(retimeRef !== undefined ? { retime: retimeRef } : {}), - }, - { - ...element, - id: secondElementId, - startTime: this.splitTime, - duration: rightVisibleDuration, - trimStart: element.trimStart + leftSourceSpan, - name: `${element.name} (right)`, - animations: rightAnimations, - ...(retimeRef !== undefined ? { retime: retimeRef } : {}), - }, - ]; + return splitResult; }); return { ...track, elements } as TTrack; diff --git a/apps/web/src/lib/commands/timeline/element/toggle-source-audio-separation.ts b/apps/web/src/lib/commands/timeline/element/toggle-source-audio-separation.ts index 36b3b7a7..7300f540 100644 --- a/apps/web/src/lib/commands/timeline/element/toggle-source-audio-separation.ts +++ b/apps/web/src/lib/commands/timeline/element/toggle-source-audio-separation.ts @@ -5,10 +5,12 @@ import { canExtractSourceAudio, isSourceAudioSeparated, } from "@/lib/timeline/audio-separation"; -import { buildEmptyTrack } from "@/lib/timeline/placement"; +import { + applyPlacement, + resolveTrackPlacement, +} from "@/lib/timeline/placement"; import { updateElementInSceneTracks } from "@/lib/timeline/track-element-update"; import type { - AudioTrack, SceneTracks, TimelineElement, VideoElement, @@ -35,9 +37,7 @@ export class ToggleSourceAudioSeparationCommand extends Command { ...this.savedState.overlay, this.savedState.main, ...this.savedState.audio, - ].find( - (track) => track.id === this.params.trackId, - ); + ].find((track) => track.id === this.params.trackId); if (!sourceTrack) { return; } @@ -61,8 +61,7 @@ export class ToggleSourceAudioSeparationCommand extends Command { return; } - const mediaAsset = editor - .media + const mediaAsset = editor.media .getAssets() .find((asset) => asset.id === videoElement.mediaId); if (!canExtractSourceAudio(videoElement, mediaAsset)) { @@ -78,20 +77,32 @@ export class ToggleSourceAudioSeparationCommand extends Command { }), id: generateUUID(), }; - const newAudioTrack = { - ...buildEmptyTrack({ - id: generateUUID(), - type: "audio", - }), + const placementResult = resolveTrackPlacement({ + tracks: this.savedState, + trackType: "audio", + timeSpans: [ + { + startTime: separatedAudioElement.startTime, + duration: separatedAudioElement.duration, + }, + ], + strategy: { type: "firstAvailable" }, + }); + if (!placementResult) { + return; + } + const appliedPlacement = applyPlacement({ + tracks: this.savedState, + placementResult, elements: [separatedAudioElement], - } as AudioTrack; + }); + if (!appliedPlacement) { + return; + } editor.timeline.updateTracks( updateSourceAudioEnabled({ - tracks: { - ...this.savedState, - audio: [...this.savedState.audio, newAudioTrack], - }, + tracks: appliedPlacement.updatedTracks, trackId: this.params.trackId, elementId: this.params.elementId, isSourceAudioEnabled: false, @@ -124,7 +135,8 @@ function updateSourceAudioEnabled({ tracks, trackId, elementId, - elementPredicate: (element): element is VideoElement => element.type === "video", + elementPredicate: (element): element is VideoElement => + element.type === "video", update: (element) => ({ ...element, isSourceAudioEnabled, diff --git a/apps/web/src/lib/commands/timeline/track/add-track.ts b/apps/web/src/lib/commands/timeline/track/add-track.ts index 66b9c348..a577c693 100644 --- a/apps/web/src/lib/commands/timeline/track/add-track.ts +++ b/apps/web/src/lib/commands/timeline/track/add-track.ts @@ -69,10 +69,7 @@ function buildAudioTrackState({ insertIndex: number; trackId: string; }): SceneTracks { - const audioInsertIndex = Math.max( - 0, - insertIndex - tracks.overlay.length - 1, - ); + const audioInsertIndex = Math.max(0, insertIndex - tracks.overlay.length - 1); const newTrack = buildEmptyTrack({ id: trackId, type: "audio", diff --git a/apps/web/src/lib/media/audio.ts b/apps/web/src/lib/media/audio.ts index 04ea580f..675cc3a5 100644 --- a/apps/web/src/lib/media/audio.ts +++ b/apps/web/src/lib/media/audio.ts @@ -20,10 +20,13 @@ import { mediaSupportsAudio } from "@/lib/media/media-utils"; import { getSourceTimeAtClipTime, renderRetimedBuffer } from "@/lib/retime"; import { Input, ALL_FORMATS, BlobSource, AudioBufferSink } from "mediabunny"; import { TICKS_PER_SECOND } from "@/lib/wasm"; +import { + computeRmsBuckets, + type SampleBucket, +} from "@/lib/media/waveform-summary"; const MAX_AUDIO_CHANNELS = 2; const EXPORT_SAMPLE_RATE = 44100; -const COARSE_SAMPLE_COUNT = 2048; export interface CollectedAudioElement { timelineElement: AudioCapableElement; @@ -177,8 +180,8 @@ export async function collectAudioElements({ if (!mediaAsset || !mediaSupportsAudio({ media: mediaAsset })) continue; pendingElements.push( - resolveAudioBufferForVideoElement({ - mediaAsset, + resolveAudioBufferForAsset({ + asset: mediaAsset, audioContext, }).then((audioBuffer) => { if (!audioBuffer) return null; @@ -222,10 +225,8 @@ async function resolveAudioBufferForElement({ try { if (element.sourceType === "upload") { const asset = mediaMap.get(element.mediaId); - if (!asset || asset.type !== "audio") return null; - - const arrayBuffer = await asset.file.arrayBuffer(); - return await audioContext.decodeAudioData(arrayBuffer.slice(0)); + if (!asset) return null; + return await resolveAudioBufferForAsset({ asset, audioContext }); } if (element.buffer) return element.buffer; @@ -243,15 +244,25 @@ async function resolveAudioBufferForElement({ } } -async function resolveAudioBufferForVideoElement({ - mediaAsset, +async function resolveAudioBufferForAsset({ + asset, audioContext, }: { - mediaAsset: MediaAsset; + asset: MediaAsset; audioContext: AudioContext; }): Promise { + if (asset.type === "audio") { + try { + const arrayBuffer = await asset.file.arrayBuffer(); + return await audioContext.decodeAudioData(arrayBuffer.slice(0)); + } catch (error) { + console.warn("Failed to decode audio asset:", error); + return null; + } + } + const input = new Input({ - source: new BlobSource(mediaAsset.file), + source: new BlobSource(asset.file), formats: ALL_FORMATS, }); @@ -319,7 +330,7 @@ async function resolveAudioBufferForVideoElement({ return await offlineContext.startRendering(); } catch (error) { - console.warn("Failed to decode video audio:", error); + console.warn("Failed to decode asset audio:", error); return null; } finally { input.dispose(); @@ -675,51 +686,29 @@ export async function createTimelineAudioBuffer({ return await applyAudioMasteringToBuffer({ audioBuffer: outputBuffer }); } -export function computeGlobalMaxRms({ - buffer, -}: { - buffer: AudioBuffer; -}): number { - const channels = buffer.numberOfChannels; - const step = Math.max(1, Math.floor(buffer.length / COARSE_SAMPLE_COUNT)); - let globalMax = 0; - - for (let c = 0; c < channels; c++) { - const data = buffer.getChannelData(c); - for (let i = 0; i + step <= buffer.length; i += step) { - for (let j = i; j < i + step; j++) { - const abs = Math.abs(data[j]); - if (abs > globalMax) globalMax = abs; - } - } - } - - return globalMax || 1; -} - -export function extractRmsRange({ +function collectPeakRange({ buffer, count, startSample, endSample, - globalMax, }: { buffer: AudioBuffer; count: number; startSample: number; endSample: number; - globalMax: number; -}): number[] { +}): Float32Array { const channels = buffer.numberOfChannels; - const rangeLength = endSample - startSample; - const step = Math.max(1, Math.floor(rangeLength / count)); const peaks = new Float32Array(count); for (let c = 0; c < channels; c++) { const data = buffer.getChannelData(c); for (let i = 0; i < count; i++) { - const start = startSample + i * step; - const end = Math.min(start + step, endSample); + const { bucketStart: start, bucketEnd: end } = getSampleBucketRange({ + startSample, + endSample, + bucketIndex: i, + bucketCount: count, + }); for (let j = start; j < end; j++) { const abs = Math.abs(data[j]); if (abs > peaks[i]) peaks[i] = abs; @@ -727,11 +716,100 @@ export function extractRmsRange({ } } - const norm = 1 / globalMax; - const result = new Array(count); - for (let i = 0; i < count; i++) result[i] = Math.min(1, peaks[i] * norm); + return peaks; +} - return result; +export function extractPeakRange({ + buffer, + count, + startSample, + endSample, +}: { + buffer: AudioBuffer; + count: number; + startSample: number; + endSample: number; +}): number[] { + return Array.from( + collectPeakRange({ + buffer, + count, + startSample, + endSample, + }), + ); +} + +export function getSampleBucketRange({ + startSample, + endSample, + bucketIndex, + bucketCount, +}: { + startSample: number; + endSample: number; + bucketIndex: number; + bucketCount: number; +}): { + bucketStart: number; + bucketEnd: number; +} { + const rangeLength = Math.max(0, endSample - startSample); + const bucketStart = + startSample + Math.floor((bucketIndex * rangeLength) / bucketCount); + const bucketEnd = + startSample + Math.floor(((bucketIndex + 1) * rangeLength) / bucketCount); + return { + bucketStart, + bucketEnd: Math.max(bucketStart, bucketEnd), + }; +} + +export function extractRmsBuckets({ + buffer, + buckets, +}: { + buffer: AudioBuffer; + buckets: SampleBucket[]; +}): number[] { + return computeRmsBuckets({ buffer, buckets }); +} + +/** + * Computes per-bucket waveform amplitude using the maximum RMS over a short + * analysis window inside each bucket. + * + * A naive mean-RMS over a whole bucket averages silence together with nearby + * sound, which smears transitions (e.g. the onset of speech) across the + * bucket and makes the waveform respond late. Taking the max over fixed + * short windows (~20 ms) preserves the smooth, non-jittery RMS character + * while making transitions land where they actually happen in the audio. + * + * Channels are combined per-window before taking the max, so the measure + * reflects total energy regardless of stereo layout. + */ +export function extractRmsRange({ + buffer, + count, + startSample, + endSample, +}: { + buffer: AudioBuffer; + count: number; + startSample: number; + endSample: number; +}): number[] { + return extractRmsBuckets({ + buffer, + buckets: Array.from({ length: count }, (_, bucketIndex) => + getSampleBucketRange({ + startSample, + endSample, + bucketIndex, + bucketCount: count, + }), + ), + }); } function mixAudioChannels({ diff --git a/apps/web/src/lib/media/waveform-summary.ts b/apps/web/src/lib/media/waveform-summary.ts new file mode 100644 index 00000000..522d6e04 --- /dev/null +++ b/apps/web/src/lib/media/waveform-summary.ts @@ -0,0 +1,233 @@ +"use client"; + +import { getSourceTimeAtClipTime } from "@/lib/retime"; +import type { RetimeConfig } from "@/lib/timeline"; + +const RMS_ANALYSIS_WINDOW_SECONDS = 0.02; +const DEFAULT_SOURCE_WAVEFORM_BUCKET_SIZE = 128; + +function computePeakBuckets({ + buffer, + buckets, +}: { + buffer: AudioBuffer; + buckets: SampleBucket[]; +}): number[] { + const channels = buffer.numberOfChannels; + const channelData: Float32Array[] = Array.from({ length: channels }, (_, c) => + buffer.getChannelData(c), + ); + + return buckets.map(({ bucketStart, bucketEnd }) => { + let peak = 0; + for (let c = 0; c < channels; c++) { + const data = channelData[c]; + for (let j = bucketStart; j < bucketEnd; j++) { + const abs = Math.abs(data[j] ?? 0); + if (abs > peak) { + peak = abs; + } + } + } + return peak; + }); +} + +export interface SampleBucket { + bucketStart: number; + bucketEnd: number; +} + +export interface SourceWaveformSummary { + sourceKey: string; + sampleRate: number; + totalSamples: number; + bucketSize: number; + amplitudes: Float32Array; +} + +export function buildWaveformSourceKey({ + kind, + id, +}: { + kind: "media" | "library"; + id: string; +}): string { + return `${kind}:${id}`; +} + +export function buildSourceWaveformSummary({ + sourceKey, + buffer, + bucketSize = DEFAULT_SOURCE_WAVEFORM_BUCKET_SIZE, +}: { + sourceKey: string; + buffer: AudioBuffer; + bucketSize?: number; +}): SourceWaveformSummary { + const safeBucketSize = Math.max(1, Math.floor(bucketSize)); + const bucketCount = Math.max(1, Math.ceil(buffer.length / safeBucketSize)); + const amplitudes = computePeakBuckets({ + buffer, + buckets: Array.from({ length: bucketCount }, (_, bucketIndex) => { + const bucketStart = bucketIndex * safeBucketSize; + const bucketEnd = Math.min(buffer.length, bucketStart + safeBucketSize); + return { bucketStart, bucketEnd }; + }), + }); + + return { + sourceKey, + sampleRate: buffer.sampleRate, + totalSamples: buffer.length, + bucketSize: safeBucketSize, + amplitudes: Float32Array.from(amplitudes), + }; +} + +export function buildWaveformSampleBuckets({ + clipLeftPx, + clipRightPx, + barCount, + pixelsPerSecond, + clipDurationSec, + sourceStartSec, + retime, + sampleRate, + maxSampleExclusive, + barStepPx, +}: { + clipLeftPx: number; + clipRightPx: number; + barCount: number; + pixelsPerSecond: number; + clipDurationSec: number; + sourceStartSec: number; + retime?: RetimeConfig; + sampleRate: number; + maxSampleExclusive: number; + barStepPx: number; +}): SampleBucket[] { + return Array.from({ length: barCount }, (_, index) => { + const bucketLeftPx = clipLeftPx + index * barStepPx; + const bucketRightPx = Math.min(clipRightPx, bucketLeftPx + barStepPx); + const clipStartSec = Math.max( + 0, + Math.min(clipDurationSec, bucketLeftPx / pixelsPerSecond), + ); + const clipEndSec = Math.max( + clipStartSec, + Math.min(clipDurationSec, bucketRightPx / pixelsPerSecond), + ); + const sourceBucketStartSec = + sourceStartSec + + getSourceTimeAtClipTime({ + clipTime: clipStartSec, + retime, + }); + const sourceBucketEndSec = + sourceStartSec + + getSourceTimeAtClipTime({ + clipTime: clipEndSec, + retime, + }); + + return { + bucketStart: Math.max(0, Math.floor(sourceBucketStartSec * sampleRate)), + bucketEnd: Math.min( + maxSampleExclusive, + Math.max(0, Math.ceil(sourceBucketEndSec * sampleRate)), + ), + }; + }); +} + +export function sampleSourceWaveformSummary({ + summary, + buckets, +}: { + summary: SourceWaveformSummary; + buckets: SampleBucket[]; +}): number[] { + return buckets.map(({ bucketStart, bucketEnd }) => { + if (bucketEnd <= bucketStart) { + return 0; + } + + const startIndex = Math.max( + 0, + Math.floor(bucketStart / summary.bucketSize), + ); + const endIndex = Math.min( + summary.amplitudes.length, + Math.max(startIndex + 1, Math.ceil(bucketEnd / summary.bucketSize)), + ); + + let maxAmplitude = 0; + for (let i = startIndex; i < endIndex; i++) { + const amplitude = summary.amplitudes[i] ?? 0; + if (amplitude > maxAmplitude) { + maxAmplitude = amplitude; + } + } + + return maxAmplitude; + }); +} + +export function computeRmsBuckets({ + buffer, + buckets, +}: { + buffer: AudioBuffer; + buckets: SampleBucket[]; +}): number[] { + const channels = buffer.numberOfChannels; + const maxWindowLength = Math.max( + 1, + Math.floor(buffer.sampleRate * RMS_ANALYSIS_WINDOW_SECONDS), + ); + + const channelData: Float32Array[] = new Array(channels); + for (let c = 0; c < channels; c++) { + channelData[c] = buffer.getChannelData(c); + } + + const result = new Array(buckets.length); + + for (let i = 0; i < buckets.length; i++) { + const { bucketStart, bucketEnd } = buckets[i]; + const bucketLength = bucketEnd - bucketStart; + if (bucketLength <= 0) { + result[i] = 0; + continue; + } + + const windowLength = Math.max(1, Math.min(bucketLength, maxWindowLength)); + let maxMeanSquare = 0; + + for (let winStart = bucketStart; winStart < bucketEnd; ) { + const winEnd = Math.min(winStart + windowLength, bucketEnd); + const n = winEnd - winStart; + if (n > 0) { + let sum = 0; + for (let c = 0; c < channels; c++) { + const data = channelData[c]; + for (let j = winStart; j < winEnd; j++) { + const v = data[j]; + sum += v * v; + } + } + const meanSquare = sum / (n * channels); + if (meanSquare > maxMeanSquare) { + maxMeanSquare = meanSquare; + } + } + winStart = winEnd; + } + + result[i] = Math.sqrt(maxMeanSquare); + } + + return result; +} diff --git a/apps/web/src/lib/timeline/audio-display.ts b/apps/web/src/lib/timeline/audio-display.ts new file mode 100644 index 00000000..9958c472 --- /dev/null +++ b/apps/web/src/lib/timeline/audio-display.ts @@ -0,0 +1,64 @@ +import { clampDb } from "./audio-state"; +import { VOLUME_DB_MAX, VOLUME_DB_MIN } from "./audio-constants"; + +const SLIDER_CURVE_EXPONENT = 2; +const MIN_DISPLAY_DB = -40; +const WAVEFORM_BAR_EXPONENT = 1.5; +const MIN_LINEAR_GAIN = 10 ** (VOLUME_DB_MIN / 20); +const MAX_LINEAR_GAIN = 10 ** (VOLUME_DB_MAX / 20); +const LINEAR_GAIN_RANGE = MAX_LINEAR_GAIN - MIN_LINEAR_GAIN; + +function getNormalizedGainFromDb({ db }: { db: number }): number { + const clampedDb = clampDb(db); + const linearGain = 10 ** (clampedDb / 20); + return (linearGain - MIN_LINEAR_GAIN) / LINEAR_GAIN_RANGE; +} + +/** + * Maps the clip's volume setting to the line position. The curve is defined in + * linear gain space so dragging near 0 dB is precise while mute compresses into + * the bottom of the clip. + */ +export function getLinePosFromDb({ db }: { db: number }): number { + const normalizedGain = Math.max( + 0, + Math.min(1, getNormalizedGainFromDb({ db })), + ); + const progress = normalizedGain ** (1 / SLIDER_CURVE_EXPONENT); + return (1 - progress) * 100; +} + +/** + * Inverse of getLinePosFromDb. Converts a drag position back into the clip's + * volume setting without depending on the underlying audio content. + */ +export function getDbFromLinePos({ percent }: { percent: number }): number { + const clampedPercent = Math.max(0, Math.min(100, percent)); + const progress = 1 - clampedPercent / 100; + const normalizedGain = progress ** SLIDER_CURVE_EXPONENT; + const linearGain = MIN_LINEAR_GAIN + normalizedGain * LINEAR_GAIN_RANGE; + return clampDb(20 * Math.log10(linearGain)); +} + +/** + * Maps an output amplitude (raw sample amplitude × gain) to a visible waveform + * height fraction using a dB scale. Agnostic to whether the input is peak or + * RMS — the caller decides which measure feeds this function; the mapping + * curve is the same either way. + * + * The log scale keeps quiet content visible while reserving the top of the + * element for amplitudes that approach 0 dBFS. + */ +export function getBarFractionFromOutputAmplitude({ + outputAmplitude, +}: { + outputAmplitude: number; +}): number { + if (outputAmplitude <= 0) return 0; + const db = 20 * Math.log10(outputAmplitude); + if (db <= MIN_DISPLAY_DB) return 0; + return Math.min( + 1, + ((db - MIN_DISPLAY_DB) / -MIN_DISPLAY_DB) ** WAVEFORM_BAR_EXPONENT, + ); +} diff --git a/apps/web/src/lib/timeline/group-resize/compute-resize.ts b/apps/web/src/lib/timeline/group-resize/compute-resize.ts index 8d7d9b2d..5730b117 100644 --- a/apps/web/src/lib/timeline/group-resize/compute-resize.ts +++ b/apps/web/src/lib/timeline/group-resize/compute-resize.ts @@ -44,14 +44,34 @@ export function computeGroupResize({ ? minimumDeltaTime : Math.min(maximumDeltaTime, Math.max(minimumDeltaTime, deltaTime)); + // Snap the drag delta to a frame exactly once, then derive every patch + // field from that single snapped value. This keeps the invariant + // `trimStart + duration*rate + trimEnd == sourceDuration` exact: the same + // delta is added on one side of the element and removed from the other, + // so the rounding cancels by construction. Per-field rounding (the old + // approach) couldn't preserve this because the individual rounds don't + // compose when `sourceDuration` isn't frame-aligned. + const snappedDeltaTime = + roundToFrame({ time: clampedDeltaTime, rate: fps }) ?? clampedDeltaTime; + // Re-clamp after rounding. Bounds derived from other elements are + // frame-aligned, so this is normally a no-op; at the source-extent limit + // the bound may not be frame-aligned, and honouring the bound takes + // precedence over frame alignment (you can't extend past real content). + const finalDeltaTime = + minimumDeltaTime > maximumDeltaTime + ? minimumDeltaTime + : Math.min( + maximumDeltaTime, + Math.max(minimumDeltaTime, snappedDeltaTime), + ); + return { - deltaTime: Object.is(clampedDeltaTime, -0) ? 0 : clampedDeltaTime, + deltaTime: Object.is(finalDeltaTime, -0) ? 0 : finalDeltaTime, updates: members.map((member) => buildResizeUpdate({ member, side, - deltaTime: clampedDeltaTime, - fps, + deltaTime: finalDeltaTime, }), ), }; @@ -61,98 +81,37 @@ function buildResizeUpdate({ member, side, deltaTime, - fps, }: { member: GroupResizeMember; side: ResizeSide; deltaTime: number; - fps: ComputeGroupResizeArgs["fps"]; }): GroupResizeUpdate { - const totalSourceDuration = getSourceDuration({ member }); const sourceDelta = getSourceDeltaForClipDelta({ member, clipDelta: deltaTime, }); - const visibleSourceSpan = getVisibleSourceSpanForDuration({ - member, - duration: member.duration, - }); if (side === "left") { - if (deltaTime < 0 && member.sourceDuration == null) { - const rawStartTime = member.startTime + deltaTime; - const rawDuration = member.duration - deltaTime; - return { - trackId: member.trackId, - elementId: member.elementId, - patch: { - trimStart: Math.max(0, member.trimStart + sourceDelta), - trimEnd: member.trimEnd, - startTime: - roundToFrame({ time: rawStartTime, rate: fps }) ?? rawStartTime, - duration: - roundToFrame({ time: rawDuration, rate: fps }) ?? rawDuration, - }, - }; - } - - const nextTrimStart = Math.max(0, member.trimStart + sourceDelta); - const nextVisibleSourceSpan = Math.max( - 0, - totalSourceDuration - nextTrimStart - member.trimEnd, - ); - const rawDuration = getDurationForVisibleSourceSpan({ - member, - sourceSpan: nextVisibleSourceSpan, - }); - const nextDuration = - roundToFrame({ time: rawDuration, rate: fps }) ?? rawDuration; - const rawStartTime = member.startTime + (member.duration - nextDuration); return { trackId: member.trackId, elementId: member.elementId, patch: { - trimStart: - roundToFrame({ time: nextTrimStart, rate: fps }) ?? nextTrimStart, + trimStart: Math.max(0, member.trimStart + sourceDelta), trimEnd: member.trimEnd, - startTime: - roundToFrame({ time: rawStartTime, rate: fps }) ?? rawStartTime, - duration: nextDuration, + startTime: member.startTime + deltaTime, + duration: member.duration - deltaTime, }, }; } - const nextVisibleSourceSpan = Math.max(0, visibleSourceSpan + sourceDelta); - if (deltaTime > 0 && member.sourceDuration == null) { - const rawDuration = member.duration + deltaTime; - return { - trackId: member.trackId, - elementId: member.elementId, - patch: { - trimStart: member.trimStart, - trimEnd: Math.max(0, member.trimEnd - sourceDelta), - startTime: member.startTime, - duration: roundToFrame({ time: rawDuration, rate: fps }) ?? rawDuration, - }, - }; - } - - const nextTrimEnd = Math.max( - 0, - totalSourceDuration - member.trimStart - nextVisibleSourceSpan, - ); - const rawDuration = getDurationForVisibleSourceSpan({ - member, - sourceSpan: nextVisibleSourceSpan, - }); return { trackId: member.trackId, elementId: member.elementId, patch: { trimStart: member.trimStart, - trimEnd: roundToFrame({ time: nextTrimEnd, rate: fps }) ?? nextTrimEnd, + trimEnd: Math.max(0, member.trimEnd - sourceDelta), startTime: member.startTime, - duration: roundToFrame({ time: rawDuration, rate: fps }) ?? rawDuration, + duration: member.duration + deltaTime, }, }; } diff --git a/apps/web/src/services/waveform-cache/service.ts b/apps/web/src/services/waveform-cache/service.ts new file mode 100644 index 00000000..3b7d34f3 --- /dev/null +++ b/apps/web/src/services/waveform-cache/service.ts @@ -0,0 +1,87 @@ +"use client"; + +import { createAudioContext } from "@/lib/media/audio"; +import { + buildSourceWaveformSummary, + type SourceWaveformSummary, +} from "@/lib/media/waveform-summary"; + +interface GetSourceWaveformSummaryArgs { + sourceKey: string; + audioBuffer?: AudioBuffer; + sourceFile?: File; + audioUrl?: string; +} + +export class WaveformCache { + private summaries = new Map>(); + + getSourceSummary({ + sourceKey, + audioBuffer, + sourceFile, + audioUrl, + }: GetSourceWaveformSummaryArgs): Promise { + const existing = this.summaries.get(sourceKey); + if (existing) { + return existing; + } + + const promise = this.buildSummary({ + sourceKey, + audioBuffer, + sourceFile, + audioUrl, + }).catch((error) => { + this.summaries.delete(sourceKey); + throw error; + }); + + this.summaries.set(sourceKey, promise); + return promise; + } + + clearSource({ sourceKey }: { sourceKey: string }): void { + this.summaries.delete(sourceKey); + } + + clearAll(): void { + this.summaries.clear(); + } + + private async buildSummary({ + sourceKey, + audioBuffer, + sourceFile, + audioUrl, + }: GetSourceWaveformSummaryArgs): Promise { + if (audioBuffer) { + return buildSourceWaveformSummary({ sourceKey, buffer: audioBuffer }); + } + + let arrayBuffer: ArrayBuffer | null = null; + if (sourceFile) { + arrayBuffer = await sourceFile.arrayBuffer(); + } else if (audioUrl) { + const response = await fetch(audioUrl); + if (!response.ok) { + throw new Error(`Failed to fetch waveform source: ${response.status}`); + } + arrayBuffer = await response.arrayBuffer(); + } + + if (!arrayBuffer) { + throw new Error(`No waveform source available for ${sourceKey}`); + } + + const audioContext = createAudioContext(); + try { + const buffer = await audioContext.decodeAudioData(arrayBuffer.slice(0)); + return buildSourceWaveformSummary({ sourceKey, buffer }); + } finally { + void audioContext.close(); + } + } +} + +export const waveformCache = new WaveformCache(); diff --git a/apps/web/src/stores/sounds-store.ts b/apps/web/src/stores/sounds-store.ts index e7622240..b4b5772b 100644 --- a/apps/web/src/stores/sounds-store.ts +++ b/apps/web/src/stores/sounds-store.ts @@ -1,261 +1,251 @@ -import { create } from "zustand"; -import type { SoundEffect, SavedSound } from "@/lib/sounds/types"; -import { storageService } from "@/services/storage/service"; -import { toast } from "sonner"; -import { EditorCore } from "@/core"; -import { buildLibraryAudioElement } from "@/lib/timeline/element-utils"; - -interface SoundsStore { - topSoundEffects: SoundEffect[]; - isLoading: boolean; - error: string | null; - hasLoaded: boolean; - showCommercialOnly: boolean; - toggleCommercialFilter: () => void; - searchQuery: string; - searchResults: SoundEffect[]; - isSearching: boolean; - searchError: string | null; - lastSearchQuery: string; - scrollPosition: number; - currentPage: number; - hasNextPage: boolean; - totalCount: number; - isLoadingMore: boolean; - savedSounds: SavedSound[]; - isSavedSoundsLoaded: boolean; - isLoadingSavedSounds: boolean; - savedSoundsError: string | null; - - addSoundToTimeline: ({ sound }: { sound: SoundEffect }) => Promise; - setTopSoundEffects: ({ sounds }: { sounds: SoundEffect[] }) => void; - setLoading: ({ loading }: { loading: boolean }) => void; - setError: ({ error }: { error: string | null }) => void; - setHasLoaded: ({ loaded }: { loaded: boolean }) => void; - setSearchQuery: ({ query }: { query: string }) => void; - setSearchResults: ({ results }: { results: SoundEffect[] }) => void; - setSearching: ({ searching }: { searching: boolean }) => void; - setSearchError: ({ error }: { error: string | null }) => void; - setLastSearchQuery: ({ query }: { query: string }) => void; - setScrollPosition: ({ position }: { position: number }) => void; - setCurrentPage: ({ page }: { page: number }) => void; - setHasNextPage: ({ hasNext }: { hasNext: boolean }) => void; - setTotalCount: ({ count }: { count: number }) => void; - setLoadingMore: ({ loading }: { loading: boolean }) => void; - appendSearchResults: ({ results }: { results: SoundEffect[] }) => void; - appendTopSounds: ({ results }: { results: SoundEffect[] }) => void; - resetPagination: () => void; - loadSavedSounds: () => Promise; - saveSoundEffect: ({ - soundEffect, - }: { - soundEffect: SoundEffect; - }) => Promise; - removeSavedSound: ({ soundId }: { soundId: number }) => Promise; - isSoundSaved: ({ soundId }: { soundId: number }) => boolean; - toggleSavedSound: ({ - soundEffect, - }: { - soundEffect: SoundEffect; - }) => Promise; - clearSavedSounds: () => Promise; -} - -export const useSoundsStore = create((set, get) => ({ - topSoundEffects: [], - isLoading: false, - error: null, - hasLoaded: false, - showCommercialOnly: true, - - toggleCommercialFilter: () => { - set((state) => ({ showCommercialOnly: !state.showCommercialOnly })); - }, - - searchQuery: "", - searchResults: [], - isSearching: false, - searchError: null, - lastSearchQuery: "", - scrollPosition: 0, - currentPage: 1, - hasNextPage: false, - totalCount: 0, - isLoadingMore: false, - savedSounds: [], - isSavedSoundsLoaded: false, - isLoadingSavedSounds: false, - savedSoundsError: null, - - setTopSoundEffects: ({ sounds }) => set({ topSoundEffects: sounds }), - setLoading: ({ loading }) => set({ isLoading: loading }), - setError: ({ error }) => set({ error }), - setHasLoaded: ({ loaded }) => set({ hasLoaded: loaded }), - setSearchQuery: ({ query }) => set({ searchQuery: query }), - setSearchResults: ({ results }) => - set({ searchResults: results, currentPage: 1 }), - setSearching: ({ searching }) => set({ isSearching: searching }), - setSearchError: ({ error }) => set({ searchError: error }), - setLastSearchQuery: ({ query }) => set({ lastSearchQuery: query }), - setScrollPosition: ({ position }) => set({ scrollPosition: position }), - setCurrentPage: ({ page }) => set({ currentPage: page }), - setHasNextPage: ({ hasNext }) => set({ hasNextPage: hasNext }), - setTotalCount: ({ count }) => set({ totalCount: count }), - setLoadingMore: ({ loading }) => set({ isLoadingMore: loading }), - - appendSearchResults: ({ results }) => - set((state) => ({ - searchResults: [...state.searchResults, ...results], - })), - - appendTopSounds: ({ results }) => - set((state) => ({ - topSoundEffects: [...state.topSoundEffects, ...results], - })), - - resetPagination: () => - set({ - currentPage: 1, - hasNextPage: false, - totalCount: 0, - isLoadingMore: false, - }), - - loadSavedSounds: async () => { - if (get().isSavedSoundsLoaded) return; - - try { - set({ isLoadingSavedSounds: true, savedSoundsError: null }); - const savedSoundsData = await storageService.loadSavedSounds(); - set({ - savedSounds: savedSoundsData.sounds, - isSavedSoundsLoaded: true, - isLoadingSavedSounds: false, - }); - } catch (error) { - const errorMessage = - error instanceof Error ? error.message : "Failed to load saved sounds"; - set({ - savedSoundsError: errorMessage, - isLoadingSavedSounds: false, - }); - console.error("Failed to load saved sounds:", error); - } - }, - - saveSoundEffect: async ({ soundEffect }) => { - try { - await storageService.saveSoundEffect({ soundEffect }); - - const savedSoundsData = await storageService.loadSavedSounds(); - set({ savedSounds: savedSoundsData.sounds }); - } catch (error) { - const errorMessage = - error instanceof Error ? error.message : "Failed to save sound"; - set({ savedSoundsError: errorMessage }); - toast.error("Failed to save sound"); - console.error("Failed to save sound:", error); - } - }, - - removeSavedSound: async ({ soundId }) => { - try { - await storageService.removeSavedSound({ soundId }); - - set((state) => ({ - savedSounds: state.savedSounds.filter((sound) => sound.id !== soundId), - })); - } catch (error) { - const errorMessage = - error instanceof Error ? error.message : "Failed to remove sound"; - set({ savedSoundsError: errorMessage }); - toast.error("Failed to remove sound"); - console.error("Failed to remove sound:", error); - } - }, - - isSoundSaved: ({ soundId }) => { - const { savedSounds } = get(); - return savedSounds.some((sound) => sound.id === soundId); - }, - - toggleSavedSound: async ({ soundEffect }) => { - const { isSoundSaved, saveSoundEffect, removeSavedSound } = get(); - - if (isSoundSaved({ soundId: soundEffect.id })) { - await removeSavedSound({ soundId: soundEffect.id }); - } else { - await saveSoundEffect({ soundEffect }); - } - }, - - clearSavedSounds: async () => { - try { - await storageService.clearSavedSounds(); - set({ - savedSounds: [], - savedSoundsError: null, - }); - } catch (error) { - const errorMessage = - error instanceof Error ? error.message : "Failed to clear saved sounds"; - set({ savedSoundsError: errorMessage }); - toast.error("Failed to clear saved sounds"); - console.error("Failed to clear saved sounds:", error); - } - }, - - addSoundToTimeline: async ({ sound }) => { - const audioUrl = sound.previewUrl; - if (!audioUrl) { - toast.error("Sound file not available"); - return false; - } - - try { - const editor = EditorCore.getInstance(); - const currentTime = editor.playback.getCurrentTime(); - const tracks = editor.scenes.getActiveScene().tracks; - - const response = await fetch(audioUrl); - if (!response.ok) - throw new Error(`Failed to download audio: ${response.statusText}`); - - const arrayBuffer = await response.arrayBuffer(); - const audioContext = new AudioContext(); - const buffer = await audioContext.decodeAudioData(arrayBuffer); - - const audioTrack = tracks.audio[0]; - let trackId: string; - - if (audioTrack) { - trackId = audioTrack.id; - } else { - trackId = editor.timeline.addTrack({ type: "audio" }); - } - - const element = buildLibraryAudioElement({ - sourceUrl: audioUrl, - name: sound.name, - duration: sound.duration, - startTime: currentTime, - buffer, - }); - - editor.timeline.insertElement({ - placement: { mode: "explicit", trackId }, - element, - }); - return true; - } catch (error) { - console.error("Failed to add sound to timeline:", error); - toast.error( - error instanceof Error - ? error.message - : "Failed to add sound to timeline", - { id: `sound-${sound.id}` }, - ); - return false; - } - }, -})); +import { create } from "zustand"; +import type { SoundEffect, SavedSound } from "@/lib/sounds/types"; +import { storageService } from "@/services/storage/service"; +import { toast } from "sonner"; +import { EditorCore } from "@/core"; +import { buildLibraryAudioElement } from "@/lib/timeline/element-utils"; + +interface SoundsStore { + topSoundEffects: SoundEffect[]; + isLoading: boolean; + error: string | null; + hasLoaded: boolean; + showCommercialOnly: boolean; + toggleCommercialFilter: () => void; + searchQuery: string; + searchResults: SoundEffect[]; + isSearching: boolean; + searchError: string | null; + lastSearchQuery: string; + scrollPosition: number; + currentPage: number; + hasNextPage: boolean; + totalCount: number; + isLoadingMore: boolean; + savedSounds: SavedSound[]; + isSavedSoundsLoaded: boolean; + isLoadingSavedSounds: boolean; + savedSoundsError: string | null; + + addSoundToTimeline: ({ sound }: { sound: SoundEffect }) => Promise; + setTopSoundEffects: ({ sounds }: { sounds: SoundEffect[] }) => void; + setLoading: ({ loading }: { loading: boolean }) => void; + setError: ({ error }: { error: string | null }) => void; + setHasLoaded: ({ loaded }: { loaded: boolean }) => void; + setSearchQuery: ({ query }: { query: string }) => void; + setSearchResults: ({ results }: { results: SoundEffect[] }) => void; + setSearching: ({ searching }: { searching: boolean }) => void; + setSearchError: ({ error }: { error: string | null }) => void; + setLastSearchQuery: ({ query }: { query: string }) => void; + setScrollPosition: ({ position }: { position: number }) => void; + setCurrentPage: ({ page }: { page: number }) => void; + setHasNextPage: ({ hasNext }: { hasNext: boolean }) => void; + setTotalCount: ({ count }: { count: number }) => void; + setLoadingMore: ({ loading }: { loading: boolean }) => void; + appendSearchResults: ({ results }: { results: SoundEffect[] }) => void; + appendTopSounds: ({ results }: { results: SoundEffect[] }) => void; + resetPagination: () => void; + loadSavedSounds: () => Promise; + saveSoundEffect: ({ + soundEffect, + }: { + soundEffect: SoundEffect; + }) => Promise; + removeSavedSound: ({ soundId }: { soundId: number }) => Promise; + isSoundSaved: ({ soundId }: { soundId: number }) => boolean; + toggleSavedSound: ({ + soundEffect, + }: { + soundEffect: SoundEffect; + }) => Promise; + clearSavedSounds: () => Promise; +} + +export const useSoundsStore = create((set, get) => ({ + topSoundEffects: [], + isLoading: false, + error: null, + hasLoaded: false, + showCommercialOnly: true, + + toggleCommercialFilter: () => { + set((state) => ({ showCommercialOnly: !state.showCommercialOnly })); + }, + + searchQuery: "", + searchResults: [], + isSearching: false, + searchError: null, + lastSearchQuery: "", + scrollPosition: 0, + currentPage: 1, + hasNextPage: false, + totalCount: 0, + isLoadingMore: false, + savedSounds: [], + isSavedSoundsLoaded: false, + isLoadingSavedSounds: false, + savedSoundsError: null, + + setTopSoundEffects: ({ sounds }) => set({ topSoundEffects: sounds }), + setLoading: ({ loading }) => set({ isLoading: loading }), + setError: ({ error }) => set({ error }), + setHasLoaded: ({ loaded }) => set({ hasLoaded: loaded }), + setSearchQuery: ({ query }) => set({ searchQuery: query }), + setSearchResults: ({ results }) => + set({ searchResults: results, currentPage: 1 }), + setSearching: ({ searching }) => set({ isSearching: searching }), + setSearchError: ({ error }) => set({ searchError: error }), + setLastSearchQuery: ({ query }) => set({ lastSearchQuery: query }), + setScrollPosition: ({ position }) => set({ scrollPosition: position }), + setCurrentPage: ({ page }) => set({ currentPage: page }), + setHasNextPage: ({ hasNext }) => set({ hasNextPage: hasNext }), + setTotalCount: ({ count }) => set({ totalCount: count }), + setLoadingMore: ({ loading }) => set({ isLoadingMore: loading }), + + appendSearchResults: ({ results }) => + set((state) => ({ + searchResults: [...state.searchResults, ...results], + })), + + appendTopSounds: ({ results }) => + set((state) => ({ + topSoundEffects: [...state.topSoundEffects, ...results], + })), + + resetPagination: () => + set({ + currentPage: 1, + hasNextPage: false, + totalCount: 0, + isLoadingMore: false, + }), + + loadSavedSounds: async () => { + if (get().isSavedSoundsLoaded) return; + + try { + set({ isLoadingSavedSounds: true, savedSoundsError: null }); + const savedSoundsData = await storageService.loadSavedSounds(); + set({ + savedSounds: savedSoundsData.sounds, + isSavedSoundsLoaded: true, + isLoadingSavedSounds: false, + }); + } catch (error) { + const errorMessage = + error instanceof Error ? error.message : "Failed to load saved sounds"; + set({ + savedSoundsError: errorMessage, + isLoadingSavedSounds: false, + }); + console.error("Failed to load saved sounds:", error); + } + }, + + saveSoundEffect: async ({ soundEffect }) => { + try { + await storageService.saveSoundEffect({ soundEffect }); + + const savedSoundsData = await storageService.loadSavedSounds(); + set({ savedSounds: savedSoundsData.sounds }); + } catch (error) { + const errorMessage = + error instanceof Error ? error.message : "Failed to save sound"; + set({ savedSoundsError: errorMessage }); + toast.error("Failed to save sound"); + console.error("Failed to save sound:", error); + } + }, + + removeSavedSound: async ({ soundId }) => { + try { + await storageService.removeSavedSound({ soundId }); + + set((state) => ({ + savedSounds: state.savedSounds.filter((sound) => sound.id !== soundId), + })); + } catch (error) { + const errorMessage = + error instanceof Error ? error.message : "Failed to remove sound"; + set({ savedSoundsError: errorMessage }); + toast.error("Failed to remove sound"); + console.error("Failed to remove sound:", error); + } + }, + + isSoundSaved: ({ soundId }) => { + const { savedSounds } = get(); + return savedSounds.some((sound) => sound.id === soundId); + }, + + toggleSavedSound: async ({ soundEffect }) => { + const { isSoundSaved, saveSoundEffect, removeSavedSound } = get(); + + if (isSoundSaved({ soundId: soundEffect.id })) { + await removeSavedSound({ soundId: soundEffect.id }); + } else { + await saveSoundEffect({ soundEffect }); + } + }, + + clearSavedSounds: async () => { + try { + await storageService.clearSavedSounds(); + set({ + savedSounds: [], + savedSoundsError: null, + }); + } catch (error) { + const errorMessage = + error instanceof Error ? error.message : "Failed to clear saved sounds"; + set({ savedSoundsError: errorMessage }); + toast.error("Failed to clear saved sounds"); + console.error("Failed to clear saved sounds:", error); + } + }, + + addSoundToTimeline: async ({ sound }) => { + const audioUrl = sound.previewUrl; + if (!audioUrl) { + toast.error("Sound file not available"); + return false; + } + + try { + const editor = EditorCore.getInstance(); + const currentTime = editor.playback.getCurrentTime(); + + const response = await fetch(audioUrl); + if (!response.ok) + throw new Error(`Failed to download audio: ${response.statusText}`); + + const arrayBuffer = await response.arrayBuffer(); + const audioContext = new AudioContext(); + const buffer = await audioContext.decodeAudioData(arrayBuffer); + + const element = buildLibraryAudioElement({ + sourceUrl: audioUrl, + name: sound.name, + duration: sound.duration, + startTime: currentTime, + buffer, + }); + + editor.timeline.insertElement({ + placement: { mode: "auto", trackType: "audio" }, + element, + }); + return true; + } catch (error) { + console.error("Failed to add sound to timeline:", error); + toast.error( + error instanceof Error + ? error.message + : "Failed to add sound to timeline", + { id: `sound-${sound.id}` }, + ); + return false; + } + }, +}));