RomanNum3ral
/
OpenCut
mirror of https://github.com/OpenCut-app/OpenCut.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat: clip volume line and timeline waveform improvements

This commit is contained in:
Maze Winther 2026-04-18 01:40:18 +02:00
parent c46d28891f
commit 011f51051c
19 changed files with 1639 additions and 721 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

275
apps/web/src/components/editor/panels/timeline/audio-volume-line.tsx Normal file
View File

@ -0,0 +1,275 @@
"use client";
import { useCallback, useRef, useState } from "react";
import { createPortal } from "react-dom";
import { useEditor } from "@/hooks/use-editor";
import { DEFAULTS } from "@/lib/timeline/defaults";
import {
getDbFromLinePos,
getLinePosFromDb,
} from "@/lib/timeline/audio-display";
import { VOLUME_DB_MAX, VOLUME_DB_MIN } from "@/lib/timeline/audio-constants";
import { hasAnimatedVolume } from "@/lib/timeline/audio-state";
import type { AudioElement } from "@/lib/timeline/types";
import {
clamp,
formatNumberForDisplay,
getFractionDigitsForStep,
isNearlyEqual,
snapToStep,
} from "@/utils/math";
import { cn } from "@/utils/ui";
const HIT_AREA_HEIGHT_PX = 14;
const TOOLTIP_OFFSET_PX = 10;
const VOLUME_STEP = 0.1;
const VOLUME_FRACTION_DIGITS = getFractionDigitsForStep({ step: VOLUME_STEP });
function clampVolume({ value }: { value: number }): number {
return clamp({
value: snapToStep({ value, step: VOLUME_STEP }),
min: VOLUME_DB_MIN,
max: VOLUME_DB_MAX,
});
}
function getVolumeFromPointer({
clientY,
rect,
}: {
clientY: number;
rect: DOMRect;
}): number {
const clampedOffset = clamp({
value: clientY - rect.top,
min: 0,
max: rect.height,
});
const progressPercent =
rect.height <= 0 ? 0 : (clampedOffset / rect.height) * 100;
return clampVolume({ value: getDbFromLinePos({ percent: progressPercent }) });
}
export function AudioVolumeLine({
element,
trackId,
}: {
element: AudioElement;
trackId: string;
}) {
const editor = useEditor();
const surfaceRef = useRef<HTMLDivElement>(null);
const activePointerIdRef = useRef<number | null>(null);
const startVolumeRef = useRef(element.volume ?? DEFAULTS.element.volume);
const lastPreviewVolumeRef = useRef(
element.volume ?? DEFAULTS.element.volume,
);
const hasChangedRef = useRef(false);
const [isDragging, setIsDragging] = useState(false);
const [tooltipClientPos, setTooltipClientPos] = useState<{
x: number;
y: number;
} | null>(null);
const hasAnimatedEnvelope = hasAnimatedVolume({ element });
const currentVolume = element.volume ?? DEFAULTS.element.volume;
const lineTop = `${getLinePosFromDb({ db: currentVolume })}%`;
const volumeLabel = `${formatNumberForDisplay({
value: currentVolume,
fractionDigits: VOLUME_FRACTION_DIGITS,
})} dB`;
const previewVolume = useCallback(
(nextVolume: number) => {
if (
isNearlyEqual({
leftValue: nextVolume,
rightValue: lastPreviewVolumeRef.current,
})
) {
return;
}
editor.timeline.previewElements({
updates: [
{
trackId,
elementId: element.id,
updates: { volume: nextVolume },
},
],
});
lastPreviewVolumeRef.current = nextVolume;
hasChangedRef.current = !isNearlyEqual({
leftValue: startVolumeRef.current,
rightValue: nextVolume,
});
},
[editor, element.id, trackId],
);
const finishDrag = useCallback(
({ shouldCommit }: { shouldCommit: boolean }) => {
activePointerIdRef.current = null;
setIsDragging(false);
if (shouldCommit && hasChangedRef.current) {
editor.timeline.commitPreview();
} else {
editor.timeline.discardPreview();
}
hasChangedRef.current = false;
lastPreviewVolumeRef.current = startVolumeRef.current;
setTooltipClientPos(null);
},
[editor],
);
const updateFromPointer = useCallback(
({ clientX, clientY }: { clientX: number; clientY: number }) => {
const rect = surfaceRef.current?.getBoundingClientRect();
if (!rect) {
return;
}
setTooltipClientPos({
x: clientX + TOOLTIP_OFFSET_PX,
y: clientY - TOOLTIP_OFFSET_PX,
});
previewVolume(getVolumeFromPointer({ clientY, rect }));
},
[previewVolume],
);
const handleClick = useCallback((event: React.MouseEvent) => {
event.preventDefault();
event.stopPropagation();
}, []);
const handleMouseDown = useCallback((event: React.MouseEvent) => {
event.stopPropagation();
}, []);
const handlePointerDown = useCallback(
(event: React.PointerEvent<HTMLDivElement>) => {
if (event.button !== 0) {
return;
}
event.preventDefault();
event.stopPropagation();
editor.selection.setSelectedElements({
elements: [{ trackId, elementId: element.id }],
});
activePointerIdRef.current = event.pointerId;
startVolumeRef.current = currentVolume;
lastPreviewVolumeRef.current = currentVolume;
hasChangedRef.current = false;
setIsDragging(true);
event.currentTarget.setPointerCapture(event.pointerId);
updateFromPointer({
clientX: event.clientX,
clientY: event.clientY,
});
},
[currentVolume, editor.selection, element.id, trackId, updateFromPointer],
);
const handlePointerMove = useCallback(
(event: React.PointerEvent) => {
if (activePointerIdRef.current !== event.pointerId) {
return;
}
event.preventDefault();
updateFromPointer({
clientX: event.clientX,
clientY: event.clientY,
});
},
[updateFromPointer],
);
const handlePointerUp = useCallback(
(event: React.PointerEvent) => {
if (activePointerIdRef.current !== event.pointerId) {
return;
}
event.preventDefault();
event.stopPropagation();
finishDrag({ shouldCommit: true });
},
[finishDrag],
);
const handlePointerCancel = useCallback(
(event: React.PointerEvent) => {
if (activePointerIdRef.current !== event.pointerId) {
return;
}
event.preventDefault();
event.stopPropagation();
finishDrag({ shouldCommit: false });
},
[finishDrag],
);
const handleLostPointerCapture = useCallback(() => {
if (activePointerIdRef.current === null) {
return;
}
finishDrag({ shouldCommit: hasChangedRef.current });
}, [finishDrag]);
if (hasAnimatedEnvelope) {
return null;
}
return (
<div className="pointer-events-none absolute inset-0">
<div ref={surfaceRef} className="absolute inset-0">
<div
className={cn(
"pointer-events-none absolute inset-x-0 -translate-y-1/2 border-t transition-colors",
isDragging
? "border-white"
: "border-white/50 group-hover/audio:border-white/80",
)}
style={{ top: lineTop }}
/>
{/* biome-ignore lint/a11y/noStaticElementInteractions: timeline volume line is a pointer-only editing surface */}
{/* biome-ignore lint/a11y/useKeyWithClickEvents: timeline volume line is a pointer-only editing surface */}
<div
className="absolute inset-x-0 -translate-y-1/2 touch-none cursor-ns-resize pointer-events-auto"
style={{ top: lineTop, height: `${HIT_AREA_HEIGHT_PX}px` }}
onClick={handleClick}
onMouseDown={handleMouseDown}
onPointerDown={handlePointerDown}
onPointerMove={handlePointerMove}
onPointerUp={handlePointerUp}
onPointerCancel={handlePointerCancel}
onLostPointerCapture={handleLostPointerCapture}
title="Drag to adjust clip volume"
/>
{isDragging &&
tooltipClientPos &&
createPortal(
<div
className="pointer-events-none fixed left-0 top-0 z-50 -translate-y-full rounded bg-black/75 px-1.5 py-0.5 text-[10px] font-medium text-white whitespace-nowrap"
style={{
transform: `translate(${tooltipClientPos.x}px, ${tooltipClientPos.y}px)`,
}}
>
{volumeLabel}
</div>,
document.body,
)}
</div>
</div>
);
}

311
apps/web/src/components/editor/panels/timeline/audio-waveform.tsx
View File

@ -1,73 +1,123 @@
"use client";
import { useCallback, useEffect, useRef } from "react";
import { useCallback, useEffect, useLayoutEffect, useRef } from "react";
import { useResizeObserver } from "@/hooks/use-resize-observer";
import { computeGlobalMaxRms, extractRmsRange } from "@/lib/media/audio";
import {
buildWaveformSampleBuckets,
sampleSourceWaveformSummary,
type SourceWaveformSummary,
} from "@/lib/media/waveform-summary";
import type { RetimeConfig } from "@/lib/timeline";
import { getBarFractionFromOutputAmplitude } from "@/lib/timeline/audio-display";
import { waveformCache } from "@/services/waveform-cache/service";
import { findScrollParent } from "@/utils/browser";
import { cn } from "@/utils/ui";
const BAR_WIDTH = 2;
const BAR_WIDTH = 1;
const BAR_GAP = 1;
const BAR_STEP = BAR_WIDTH + BAR_GAP;
const WAVEFORM_BURN_COLOR = "rgba(255, 110, 20, 0.9)";
export const WAVEFORM_GAIN_SAMPLE_COUNT = 200;
function sampleGain({
function sampleGainAtClipTime({
samples,
startFraction,
endFraction,
barIndex,
barCount,
clipTimeSec,
clipDurationSec,
}: {
samples: number[];
startFraction: number;
endFraction: number;
barIndex: number;
barCount: number;
clipTimeSec: number;
clipDurationSec: number;
}): number {
if (samples.length === 0) return 1;
const progress =
startFraction +
((barIndex + 0.5) / barCount) * (endFraction - startFraction);
const rawIndex = Math.max(0, Math.min(1, progress)) * (samples.length - 1);
if (samples.length === 0 || clipDurationSec <= 0) {
return 1;
}
const progress = Math.max(0, Math.min(1, clipTimeSec / clipDurationSec));
const rawIndex = progress * (samples.length - 1);
const lo = Math.floor(rawIndex);
const hi = Math.min(samples.length - 1, lo + 1);
return samples[lo] + (samples[hi] - samples[lo]) * (rawIndex - lo);
}
interface AudioWaveformProps {
sourceKey: string;
sourceFile?: File;
audioUrl?: string;
audioBuffer?: AudioBuffer;
gainSamples?: number[];
pixelsPerSecond: number;
clipDurationSec: number;
retime?: RetimeConfig;
sourceStartSec: number;
color?: string;
burnColor?: string;
className?: string;
}
export function AudioWaveform({
sourceKey,
sourceFile,
audioUrl,
audioBuffer,
gainSamples,
color = "rgba(255, 255, 255, 0.7)",
pixelsPerSecond,
clipDurationSec,
retime,
sourceStartSec,
color,
burnColor = WAVEFORM_BURN_COLOR,
className = "",
}: AudioWaveformProps) {
const canvasRef = useRef<HTMLCanvasElement>(null);
const containerRef = useRef<HTMLDivElement>(null);
const bufferRef = useRef<AudioBuffer | null>(null);
const globalMaxRef = useRef<number>(1);
const summaryRef = useRef<SourceWaveformSummary | null>(null);
const gainSamplesRef = useRef<number[] | undefined>(gainSamples);
const pixelsPerSecondRef = useRef<number>(pixelsPerSecond);
const clipDurationSecRef = useRef<number>(clipDurationSec);
const retimeRef = useRef<RetimeConfig | undefined>(retime);
const sourceStartSecRef = useRef<number>(sourceStartSec);
const scrollParentRef = useRef<HTMLElement | null>(null);
const heightRef = useRef<number>(0);
const lastRenderSignatureRef = useRef<string | null>(null);
gainSamplesRef.current = gainSamples;
pixelsPerSecondRef.current = pixelsPerSecond;
clipDurationSecRef.current = clipDurationSec;
retimeRef.current = retime;
sourceStartSecRef.current = sourceStartSec;
const clearCanvas = useCallback(() => {
const canvas = canvasRef.current;
if (!canvas) {
return;
}
const ctx = canvas.getContext("2d");
if (ctx) {
ctx.setTransform(1, 0, 0, 1, 0, 0);
ctx.clearRect(0, 0, canvas.width, canvas.height);
}
lastRenderSignatureRef.current = null;
}, []);
const drawVisible = useCallback(() => {
const container = containerRef.current;
const canvas = canvasRef.current;
const buffer = bufferRef.current;
const summary = summaryRef.current;
const height = heightRef.current;
if (!container || !canvas || !buffer || height <= 0) return;
const elementWidth = container.offsetWidth;
if (elementWidth <= 0) return;
if (!container || !canvas || !summary || height <= 0) {
clearCanvas();
return;
}
const containerRect = container.getBoundingClientRect();
const elementWidth = containerRect.width;
if (elementWidth <= 0) {
clearCanvas();
return;
}
const scrollParent = scrollParentRef.current;
let clipLeft: number;
@ -86,13 +136,43 @@ export function AudioWaveform({
}
const visibleWidth = clipRight - clipLeft;
if (visibleWidth <= 0) return;
if (visibleWidth <= 0) {
clearCanvas();
return;
}
const dpr = window.devicePixelRatio || 1;
const canvasW = Math.round(visibleWidth * dpr);
const canvasH = Math.round(height * dpr);
if (canvasW <= 0 || canvasH <= 0) return;
const canvasW = Math.max(1, Math.ceil(visibleWidth * dpr));
const canvasH = Math.max(1, Math.round(height * dpr));
const barCount = Math.max(1, Math.floor(visibleWidth / BAR_STEP));
const pixelsPerSecondValue = pixelsPerSecondRef.current;
const clipDurationSecValue = clipDurationSecRef.current;
const samples = gainSamplesRef.current;
const renderSignature = JSON.stringify({
elementWidth,
clipLeft,
clipRight,
visibleWidth,
canvasW,
canvasH,
barCount,
dpr,
clipDurationSec: clipDurationSecValue,
sourceStartSec: sourceStartSecRef.current,
pixelsPerSecond: pixelsPerSecondValue,
retime: retimeRef.current ?? null,
summarySourceKey: summary.sourceKey,
summarySampleRate: summary.sampleRate,
summaryTotalSamples: summary.totalSamples,
summaryBucketSize: summary.bucketSize,
gainSamples: samples ?? null,
color,
burnColor,
});
if (lastRenderSignatureRef.current === renderSignature) {
return;
}
lastRenderSignatureRef.current = renderSignature;
canvas.width = canvasW;
canvas.height = canvasH;
@ -100,97 +180,150 @@ export function AudioWaveform({
canvas.style.height = `${height}px`;
canvas.style.left = `${clipLeft}px`;
const barCount = Math.max(1, Math.floor(visibleWidth / BAR_STEP));
const startFraction = clipLeft / elementWidth;
const endFraction = clipRight / elementWidth;
const startSample = Math.floor(startFraction * buffer.length);
const endSample = Math.min(
buffer.length,
Math.ceil(endFraction * buffer.length),
);
const backingScaleX = dpr;
const backingScaleY = canvasH / height;
const peaks = extractRmsRange({
buffer,
count: barCount,
startSample,
endSample,
globalMax: globalMaxRef.current,
const sampleBuckets = buildWaveformSampleBuckets({
clipLeftPx: clipLeft,
clipRightPx: clipRight,
barCount,
pixelsPerSecond: pixelsPerSecondValue,
clipDurationSec: clipDurationSecValue,
sourceStartSec: sourceStartSecRef.current,
retime: retimeRef.current,
sampleRate: summary.sampleRate,
maxSampleExclusive: summary.totalSamples,
barStepPx: BAR_STEP,
});
const amplitudes = sampleSourceWaveformSummary({
summary,
buckets: sampleBuckets,
});
const ctx = canvas.getContext("2d");
if (!ctx) return;
if (!ctx) {
return;
}
ctx.setTransform(1, 0, 0, 1, 0, 0);
ctx.clearRect(0, 0, canvasW, canvasH);
ctx.scale(dpr, dpr);
ctx.fillStyle = color;
const maxBarHeight = height * 0.7;
const clipBottom = canvasH;
const samples = gainSamplesRef.current;
for (let i = 0; i < barCount; i++) {
const barCenterPx = clipLeft + i * BAR_STEP + BAR_WIDTH * 0.5;
const clipCenterSec = Math.max(
0,
Math.min(clipDurationSecValue, barCenterPx / pixelsPerSecondValue),
);
const gain =
samples != null
? sampleGain({
? sampleGainAtClipTime({
samples,
startFraction,
endFraction,
barIndex: i,
barCount,
clipTimeSec: clipCenterSec,
clipDurationSec: clipDurationSecValue,
})
: 1;
const scaledPeak = Math.min(1, Math.max(0, peaks[i] * gain));
const scaled = Math.log1p(scaledPeak) / Math.log1p(1);
const barH = Math.max(1, scaled * maxBarHeight);
ctx.fillRect(i * BAR_STEP, height - barH, BAR_WIDTH, barH);
}
}, [color]);
const amplitude = Math.max(0, amplitudes[i] ?? 0);
const outputAmplitude = amplitude * Math.max(0, gain);
const fraction = getBarFractionFromOutputAmplitude({ outputAmplitude });
const barH = fraction > 0 ? Math.max(1, fraction * height) : 0;
if (barH <= 0) {
continue;
}
useEffect(() => {
gainSamplesRef.current = gainSamples;
drawVisible();
}, [gainSamples, drawVisible]);
const barLeft = i * BAR_STEP;
const barRight = barLeft + BAR_WIDTH;
const deviceLeft = Math.round(barLeft * backingScaleX);
const deviceRight = Math.max(
deviceLeft + 1,
Math.round(barRight * backingScaleX),
);
const deviceTop = Math.round((height - barH) * backingScaleY);
const deviceHeight = Math.max(1, clipBottom - deviceTop);
ctx.fillStyle = color;
ctx.fillRect(
deviceLeft,
deviceTop,
deviceRight - deviceLeft,
deviceHeight,
);
if (outputAmplitude > 1) {
const burnHeight = Math.max(1, Math.round(BAR_WIDTH * backingScaleY));
ctx.fillStyle = burnColor;
ctx.fillRect(
deviceLeft,
deviceTop,
deviceRight - deviceLeft,
burnHeight,
);
}
}
}, [burnColor, clearCanvas, color]);
useEffect(() => {
let isCancelled = false;
summaryRef.current = null;
clearCanvas();
async function load() {
let buffer = audioBuffer ?? null;
if (!buffer && audioUrl) {
try {
const resp = await fetch(audioUrl);
const arrayBuffer = await resp.arrayBuffer();
const actx = new AudioContext();
buffer = await actx.decodeAudioData(arrayBuffer);
actx.close();
} catch {
void waveformCache
.getSourceSummary({
sourceKey,
audioBuffer,
sourceFile,
audioUrl,
})
.then((summary) => {
if (isCancelled) {
return;
}
}
summaryRef.current = summary;
drawVisible();
})
.catch(() => {
// Waveform loading failed (e.g. corrupt file, unsupported format).
// Fail silently — a missing waveform is preferable to an error state.
if (!isCancelled) {
clearCanvas();
}
});
if (!buffer || isCancelled) return;
bufferRef.current = buffer;
globalMaxRef.current = computeGlobalMaxRms({ buffer });
drawVisible();
}
load();
return () => {
isCancelled = true;
};
}, [audioUrl, audioBuffer, drawVisible]);
}, [audioBuffer, audioUrl, clearCanvas, drawVisible, sourceFile, sourceKey]);
// biome-ignore lint/correctness/useExhaustiveDependencies: these props are mirrored into refs during render, but the effect must still re-run to redraw when they change.
useLayoutEffect(() => {
drawVisible();
}, [
drawVisible,
gainSamples,
pixelsPerSecond,
clipDurationSec,
retime,
sourceStartSec,
]);
useEffect(() => {
const container = containerRef.current;
if (!container) return;
if (!container) {
return;
}
scrollParentRef.current = findScrollParent({ element: container });
const scrollParent = scrollParentRef.current;
if (!scrollParent) return;
if (!scrollParent) {
return;
}
scrollParent.addEventListener("scroll", drawVisible, { passive: true });
return () => scrollParent.removeEventListener("scroll", drawVisible);
const handleScroll = () => {
drawVisible();
};
scrollParent.addEventListener("scroll", handleScroll, { passive: true });
return () => scrollParent.removeEventListener("scroll", handleScroll);
}, [drawVisible]);
const onResize = useCallback(

2
apps/web/src/components/editor/panels/timeline/theme.ts
View File

@ -11,7 +11,7 @@ export const TIMELINE_TRACK_THEME: Record<
text: { elementClassName: "bg-[#5DBAA0]" },
audio: {
elementClassName: "bg-[#8F5DBA]",
waveformColor: "rgba(255, 255, 255, 0.5)",
waveformColor: "rgba(255, 255, 255, 0.7)",
},
graphic: { elementClassName: "bg-[#BA5D7A]" },
effect: { elementClassName: "bg-[#5d93ba]" },

318
apps/web/src/components/editor/panels/timeline/timeline-element.tsx
View File

@ -1,8 +1,10 @@
"use client";
import { createContext, useContext } from "react";
import { useEditor } from "@/hooks/use-editor";
import { useAssetsPanelStore } from "@/stores/assets-panel-store";
import { AudioWaveform, WAVEFORM_GAIN_SAMPLE_COUNT } from "./audio-waveform";
import { AudioVolumeLine } from "./audio-volume-line";
import { useElementPreview } from "@/hooks/use-element-preview";
import {
useKeyframeDrag,
@ -45,6 +47,9 @@ import {
isSourceAudioSeparated,
} from "@/lib/timeline/audio-separation";
import { buildWaveformGainSamples } from "@/lib/timeline/audio-state";
import { getTimelinePixelsPerSecond } from "@/lib/timeline";
import { buildWaveformSourceKey } from "@/lib/media/waveform-summary";
import { TICKS_PER_SECOND } from "@/lib/wasm/ticks";
import {
getActionDefinition,
type TAction,
@ -89,6 +94,8 @@ import {
const KEYFRAME_INDICATOR_MIN_WIDTH_PX = 40;
const ELEMENT_RING_WIDTH_PX = 1.5;
const PixelsPerSecondContext = createContext<number | null>(null);
const THUMBNAIL_ASPECT_RATIO = 16 / 9;
interface KeyframeIndicator {
@ -264,6 +271,7 @@ export function TimelineElement({
time: displayedDuration,
zoomLevel,
});
const timelinePixelsPerSecond = getTimelinePixelsPerSecond({ zoomLevel });
const elementLeft = timelineTimeToSnappedPixels({
time: displayedStartTime,
zoomLevel,
@ -364,137 +372,141 @@ export function TimelineElement({
) : null;
return (
<ContextMenu>
<ContextMenuTrigger asChild>
<div
className="absolute top-0 select-none"
style={{
left: `${elementLeft}px`,
width: `${elementWidth}px`,
height:
expandedRows.length > 0
? `${baseTrackHeight + expansionHeight}px`
: "100%",
transform:
isBeingDragged && dragState.isDragging
? `translate3d(0, ${dragOffsetY}px, 0)`
: undefined,
}}
>
<ElementInner
element={element}
displayElement={renderElement}
track={track}
isSelected={isSelected}
isExpanded={expandedRows.length > 0}
baseTrackHeight={baseTrackHeight}
expandedContent={expandedContent}
onElementClick={onElementClick}
onElementMouseDown={onElementMouseDown}
onResizeStart={onResizeStart}
isDropTarget={isDropTarget}
/>
{isSelected && (
<div
className="pointer-events-none absolute inset-x-0 top-0 overflow-hidden"
style={{ height: `${baseTrackHeight}px` }}
>
<KeyframeIndicators
indicators={keyframeIndicators}
dragState={keyframeDragState}
displayedStartTime={displayedStartTime}
elementLeft={elementLeft}
onKeyframeMouseDown={handleKeyframeMouseDown}
onKeyframeClick={handleKeyframeClick}
getVisualOffsetPx={getVisualOffsetPx}
/>
</div>
)}
</div>
</ContextMenuTrigger>
<ContextMenuContent className="w-64">
<ActionMenuItem
action="split"
icon={<HugeiconsIcon icon={ScissorIcon} />}
>
Split
</ActionMenuItem>
<CopyMenuItem />
{selectedElements.length === 1 && (
<PixelsPerSecondContext.Provider value={timelinePixelsPerSecond}>
<ContextMenu>
<ContextMenuTrigger asChild>
<div
className="absolute top-0 select-none"
style={{
left: `${elementLeft}px`,
width: `${elementWidth}px`,
height:
expandedRows.length > 0
? `${baseTrackHeight + expansionHeight}px`
: "100%",
transform:
isBeingDragged && dragState.isDragging
? `translate3d(0, ${dragOffsetY}px, 0)`
: undefined,
}}
>
<ElementInner
element={element}
displayElement={renderElement}
track={track}
isSelected={isSelected}
isExpanded={expandedRows.length > 0}
baseTrackHeight={baseTrackHeight}
expandedContent={expandedContent}
onElementClick={onElementClick}
onElementMouseDown={onElementMouseDown}
onResizeStart={onResizeStart}
isDropTarget={isDropTarget}
/>
{isSelected && (
<div
className="pointer-events-none absolute inset-x-0 top-0 overflow-hidden"
style={{ height: `${baseTrackHeight}px` }}
>
<KeyframeIndicators
indicators={keyframeIndicators}
dragState={keyframeDragState}
displayedStartTime={displayedStartTime}
elementLeft={elementLeft}
onKeyframeMouseDown={handleKeyframeMouseDown}
onKeyframeClick={handleKeyframeClick}
getVisualOffsetPx={getVisualOffsetPx}
/>
</div>
)}
</div>
</ContextMenuTrigger>
<ContextMenuContent className="w-64">
<ActionMenuItem
action="duplicate-selected"
icon={<HugeiconsIcon icon={Copy01Icon} />}
action="split"
icon={<HugeiconsIcon icon={ScissorIcon} />}
>
Duplicate
Split
</ActionMenuItem>
)}
{canElementHaveAudio(element) && hasAudio && (
<MuteMenuItem
isMultipleSelected={selectedElements.length > 1}
isCurrentElementSelected={isCurrentElementSelected}
isMuted={isMuted}
/>
)}
{canToggleCurrentSourceAudio && (
<ContextMenuItem
icon={
<HugeiconsIcon
icon={isElementSourceAudioSeparated ? ScissorIcon : ScissorIcon}
/>
}
onClick={(event: React.MouseEvent) => {
event.stopPropagation();
invokeAction("toggle-source-audio");
}}
>
{sourceAudioLabel}
</ContextMenuItem>
)}
{canElementBeHidden(element) && (
<VisibilityMenuItem
element={element}
isMultipleSelected={selectedElements.length > 1}
isCurrentElementSelected={isCurrentElementSelected}
/>
)}
{hasKeyframes && (
<ContextMenuItem
icon={<HugeiconsIcon icon={KeyframeIcon} />}
onClick={(event: React.MouseEvent) => {
event.stopPropagation();
toggleElementExpanded(element.id);
}}
>
{isExpanded ? "Collapse keyframes" : "Expand keyframes"}
</ContextMenuItem>
)}
{selectedElements.length === 1 && hasMediaId(element) && (
<>
<CopyMenuItem />
{selectedElements.length === 1 && (
<ActionMenuItem
action="duplicate-selected"
icon={<HugeiconsIcon icon={Copy01Icon} />}
>
Duplicate
</ActionMenuItem>
)}
{canElementHaveAudio(element) && hasAudio && (
<MuteMenuItem
isMultipleSelected={selectedElements.length > 1}
isCurrentElementSelected={isCurrentElementSelected}
isMuted={isMuted}
/>
)}
{canToggleCurrentSourceAudio && (
<ContextMenuItem
icon={<HugeiconsIcon icon={Search01Icon} />}
onClick={(event: React.MouseEvent) =>
handleRevealInMedia({ event })
icon={
<HugeiconsIcon
icon={
isElementSourceAudioSeparated ? ScissorIcon : ScissorIcon
}
/>
}
onClick={(event: React.MouseEvent) => {
event.stopPropagation();
invokeAction("toggle-source-audio");
}}
>
Reveal media
{sourceAudioLabel}
</ContextMenuItem>
)}
{canElementBeHidden(element) && (
<VisibilityMenuItem
element={element}
isMultipleSelected={selectedElements.length > 1}
isCurrentElementSelected={isCurrentElementSelected}
/>
)}
{hasKeyframes && (
<ContextMenuItem
icon={<HugeiconsIcon icon={Exchange01Icon} />}
disabled
icon={<HugeiconsIcon icon={KeyframeIcon} />}
onClick={(event: React.MouseEvent) => {
event.stopPropagation();
toggleElementExpanded(element.id);
}}
>
Replace media
{isExpanded ? "Collapse keyframes" : "Expand keyframes"}
</ContextMenuItem>
</>
)}
<ContextMenuSeparator />
<DeleteMenuItem
isMultipleSelected={selectedElements.length > 1}
isCurrentElementSelected={isCurrentElementSelected}
elementType={element.type}
selectedCount={selectedElements.length}
/>
</ContextMenuContent>
</ContextMenu>
)}
{selectedElements.length === 1 && hasMediaId(element) && (
<>
<ContextMenuItem
icon={<HugeiconsIcon icon={Search01Icon} />}
onClick={(event: React.MouseEvent) =>
handleRevealInMedia({ event })
}
>
Reveal media
</ContextMenuItem>
<ContextMenuItem
icon={<HugeiconsIcon icon={Exchange01Icon} />}
disabled
>
Replace media
</ContextMenuItem>
</>
)}
<ContextMenuSeparator />
<DeleteMenuItem
isMultipleSelected={selectedElements.length > 1}
isCurrentElementSelected={isCurrentElementSelected}
elementType={element.type}
selectedCount={selectedElements.length}
/>
</ContextMenuContent>
</ContextMenu>
</PixelsPerSecondContext.Provider>
);
}
@ -973,7 +985,19 @@ function GraphicElementContent({
);
}
function AudioElementContent({ element }: { element: AudioElement }) {
function AudioElementContent({
element,
trackId,
}: {
element: AudioElement;
trackId: string;
}) {
const pixelsPerSecond = useContext(PixelsPerSecondContext);
if (pixelsPerSecond === null) {
throw new Error(
"AudioElementContent must be rendered inside PixelsPerSecondContext.Provider",
);
}
const mediaAssets = useEditor((e) => e.media.getAssets());
const mediaAsset =
element.sourceType === "upload"
@ -984,29 +1008,53 @@ function AudioElementContent({ element }: { element: AudioElement }) {
element.sourceType === "library" ? element.buffer : undefined;
const audioUrl =
element.sourceType === "library" ? element.sourceUrl : mediaAsset?.url;
const sourceFile =
element.sourceType === "upload" ? mediaAsset?.file : undefined;
const sourceKey =
element.sourceType === "upload"
? buildWaveformSourceKey({ kind: "media", id: element.mediaId })
: buildWaveformSourceKey({ kind: "library", id: element.sourceUrl });
const mediaLabel = mediaAsset?.name ?? element.name;
const gainSamples = useMemo(
() =>
buildWaveformGainSamples({ element, count: WAVEFORM_GAIN_SAMPLE_COUNT }),
buildWaveformGainSamples({
element,
count: WAVEFORM_GAIN_SAMPLE_COUNT,
}),
[element],
);
if (audioBuffer || audioUrl) {
if (audioBuffer || audioUrl || sourceFile) {
return (
<div className="relative size-full">
<AudioWaveform
audioBuffer={audioBuffer}
audioUrl={audioUrl}
gainSamples={gainSamples}
color={TIMELINE_TRACK_THEME.audio.waveformColor}
/>
<div className="group/audio relative size-full">
<MediaElementHeader name={mediaLabel} hasFade={false} />
<div className="absolute inset-x-0 top-5 bottom-0 overflow-hidden">
<AudioWaveform
sourceKey={sourceKey}
sourceFile={sourceFile}
audioBuffer={audioBuffer}
audioUrl={audioUrl}
gainSamples={gainSamples}
pixelsPerSecond={pixelsPerSecond}
clipDurationSec={element.duration / TICKS_PER_SECOND}
retime={element.retime}
sourceStartSec={element.trimStart / TICKS_PER_SECOND}
color={TIMELINE_TRACK_THEME.audio.waveformColor}
/>
<AudioVolumeLine element={element} trackId={trackId} />
</div>
</div>
);
}
return (
<span className="text-foreground/80 truncate text-xs">{element.name}</span>
<div className="group/audio relative size-full">
<div className="flex size-full items-center pl-2">
<span className="text-foreground/80 truncate text-xs">
{element.name}
</span>
</div>
<AudioVolumeLine element={element} trackId={trackId} />
</div>
);
}
@ -1108,7 +1156,7 @@ function MediaElementHeader({
return (
<div
className={cn(
"absolute top-0 left-0 flex h-7 w-full bg-linear-to-b pt-1",
"absolute top-0 left-0 flex h-5 w-full bg-linear-to-b pt-1",
hasFade && "from-black/30 to-transparent",
)}
>
@ -1133,7 +1181,7 @@ function ElementContent({ element, track }: ElementContentProps) {
case "graphic":
return <GraphicElementContent element={element} />;
case "audio":
return <AudioElementContent element={element} />;
return <AudioElementContent element={element} trackId={track.id} />;
case "video":
case "image":
return <TiledMediaContent element={element} track={track} />;

4
apps/web/src/core/managers/media-manager.ts
View File

@ -4,6 +4,7 @@ import type { MediaAsset } from "@/lib/media/types";
import { storageService } from "@/services/storage/service";
import { generateUUID } from "@/utils/id";
import { videoCache } from "@/services/video-cache/service";
import { waveformCache } from "@/services/waveform-cache/service";
import { BatchCommand, RemoveMediaAssetCommand } from "@/lib/commands";
export class MediaManager {
@ -94,6 +95,8 @@ export class MediaManager {
}
async clearProjectMedia({ projectId }: { projectId: string }): Promise<void> {
waveformCache.clearAll();
this.assets.forEach((asset) => {
if (asset.url) {
URL.revokeObjectURL(asset.url);
@ -120,6 +123,7 @@ export class MediaManager {
clearAllAssets(): void {
videoCache.clearAll();
waveformCache.clearAll();
this.assets.forEach((asset) => {
if (asset.url) {

23
apps/web/src/core/managers/timeline-manager.ts
View File

@ -712,18 +712,31 @@ export class TimelineManager {
updates: Partial<TimelineElement>;
}>;
}): void {
let changedOverlayCount = 0;
for (const { elementId, updates: elementUpdates } of updates) {
const existingOverlay = this.previewOverlay.get(elementId);
const mergedOverlay = {
...existingOverlay,
...elementUpdates,
} as Partial<TimelineElement>;
this.previewOverlay.set(elementId, mergedOverlay);
const changed = Object.entries(elementUpdates).some(([key, value]) => {
return !Object.is(
existingOverlay?.[key as keyof TimelineElement],
value,
);
});
if (changed) {
changedOverlayCount += 1;
const mergedOverlay = {
...existingOverlay,
...elementUpdates,
} as Partial<TimelineElement>;
this.previewOverlay.set(elementId, mergedOverlay);
}
}
const committedTracks = this.editor.scenes.getActiveSceneOrNull()?.tracks;
if (!committedTracks) {
return;
}
if (changedOverlayCount === 0) {
return;
}
this.previewTracks = this.applyPreviewOverlay(committedTracks);
this.notify();
}

72
apps/web/src/hooks/timeline/use-timeline-drag-drop.ts
View File

@ -89,8 +89,8 @@ export function useTimelineDragDrop({
const mediaAssets = editor.media.getAssets();
const media = mediaAssets.find((m) => m.id === mediaId);
return media?.duration != null
? Math.round(media.duration * TICKS_PER_SECOND)
: DEFAULT_NEW_ELEMENT_DURATION;
? Math.round(media.duration * TICKS_PER_SECOND)
: DEFAULT_NEW_ELEMENT_DURATION;
}
return DEFAULT_NEW_ELEMENT_DURATION;
},
@ -468,10 +468,10 @@ export function useTimelineDragDrop({
});
if (!createdAsset) continue;
const duration =
createdAsset.duration != null
? Math.round(createdAsset.duration * TICKS_PER_SECOND)
: DEFAULT_NEW_ELEMENT_DURATION;
const duration =
createdAsset.duration != null
? Math.round(createdAsset.duration * TICKS_PER_SECOND)
: DEFAULT_NEW_ELEMENT_DURATION;
const sceneTracks = editor.scenes.getActiveScene().tracks;
const currentTime = editor.playback.getCurrentTime();
const reuseMainTrackId =
@ -498,9 +498,22 @@ export function useTimelineDragDrop({
const trackType: TrackType =
createdAsset.type === "audio" ? "audio" : "video";
let trackId: string | undefined;
const startTime = dropTarget?.xPosition ?? currentTime;
const element = buildElementFromMedia({
mediaId: createdAsset.id,
mediaType: createdAsset.type,
name: createdAsset.name,
duration,
startTime,
});
if (reuseMainTrackId) {
trackId = reuseMainTrackId;
editor.command.execute({
command: new InsertElementCommand({
element,
placement: { mode: "explicit", trackId: reuseMainTrackId },
}),
});
} else {
if (!dropTarget) continue;
if (dropTarget.isNewTrack) {
@ -508,36 +521,33 @@ export function useTimelineDragDrop({
trackType,
dropTarget.trackIndex,
);
trackId = addTrackCmd.getTrackId();
editor.command.execute({ command: addTrackCmd });
editor.command.execute({
command: new BatchCommand([
addTrackCmd,
new InsertElementCommand({
element,
placement: {
mode: "explicit",
trackId: addTrackCmd.getTrackId(),
},
}),
]),
});
} else {
trackId = [
const trackId = [
...sceneTracks.overlay,
sceneTracks.main,
...sceneTracks.audio,
][dropTarget.trackIndex]?.id;
if (!trackId) continue;
editor.command.execute({
command: new InsertElementCommand({
element,
placement: { mode: "explicit", trackId },
}),
});
}
}
if (!trackId) continue;
const element = buildElementFromMedia({
mediaId: createdAsset.id,
mediaType: createdAsset.type,
name: createdAsset.name,
duration,
startTime: dropTarget?.xPosition ?? currentTime,
buffer:
createdAsset.type === "audio"
? new AudioBuffer({ length: 1, sampleRate: 44100 })
: undefined,
});
const insertCmd = new InsertElementCommand({
element,
placement: { mode: "explicit", trackId },
});
editor.command.execute({ command: insertCmd });
}
return {

8
apps/web/src/lib/changelog/entries/0.4.0.md
View File

@ -10,4 +10,10 @@ changes:
- type: new
text: "Two new mask types: text and custom (draw any shape with the pen tool using bezier curves). Both support feather, invert, and stroke."
- type: fixed
text: "Preview panning now keeps working while the Masks tab is open."
text: "Preview panning now keeps working while the Masks tab is open."
- type: new
text: "Audio clips have a horizontal volume line you can drag up or down to change volume. The line position reflects the current volume setting and is consistent across clips: same dB always means the same position."
- type: improved
text: "Audio waveforms now show the actual shape of the audio. Quiet parts are visible, loud parts are tall, and any bar that peaks above 0 dBFS gets a colored tip at the top."
- type: fixed
text: "Trimming or cutting an audio clip no longer causes the waveform to compress or show the wrong section of audio."

8
apps/web/src/lib/commands/media/remove-media-asset.ts
View File

@ -1,8 +1,10 @@
import { Command, type CommandResult } from "@/lib/commands/base-command";
import { EditorCore } from "@/core";
import type { MediaAsset } from "@/lib/media/types";
import { buildWaveformSourceKey } from "@/lib/media/waveform-summary";
import { storageService } from "@/services/storage/service";
import { videoCache } from "@/services/video-cache/service";
import { waveformCache } from "@/services/waveform-cache/service";
import { hasMediaId } from "@/lib/timeline/element-utils";
import type { SceneTracks } from "@/lib/timeline";
@ -41,6 +43,12 @@ export class RemoveMediaAssetCommand extends Command {
}
videoCache.clearVideo({ mediaId: this.assetId });
waveformCache.clearSource({
sourceKey: buildWaveformSourceKey({
kind: "media",
id: this.assetId,
}),
});
editor.media.setAssets({
assets: assets.filter((media) => media.id !== this.assetId),

1
apps/web/src/lib/commands/timeline/element/insert-element.ts
View File

@ -3,7 +3,6 @@ import { EditorCore } from "@/core";
import type {
CreateTimelineElement,
SceneTracks,
TimelineTrack,
TimelineElement,
TrackType,
} from "@/lib/timeline";

122
apps/web/src/lib/commands/timeline/element/split-elements.ts
View File

@ -37,7 +37,9 @@ export class SplitElementsCommand extends Command {
this.savedState = editor.scenes.getActiveScene().tracks;
this.rightSideElements = [];
const splitTrack = <TTrack extends { id: string; elements: TimelineElement[] }>(
const splitTrack = <
TTrack extends { id: string; elements: TimelineElement[] },
>(
track: TTrack,
): TTrack => {
const elementsToSplit = this.elements.filter(
@ -48,7 +50,7 @@ export class SplitElementsCommand extends Command {
return track;
}
let elements = track.elements.flatMap((element) => {
const elements = track.elements.flatMap((element) => {
const shouldSplit = elementsToSplit.some(
(target) => target.elementId === element.id,
);
@ -87,67 +89,67 @@ export class SplitElementsCommand extends Command {
splitTime: relativeTime,
shouldIncludeSplitBoundary: true,
});
let splitResult: TimelineElement[];
if (this.retainSide === "left") {
return [
{
...element,
duration: leftVisibleDuration,
trimEnd: element.trimEnd + rightSourceSpan,
name: `${element.name} (left)`,
animations: leftAnimations,
...(retimeRef !== undefined ? { retime: retimeRef } : {}),
},
];
}
splitResult = [
{
...element,
duration: leftVisibleDuration,
trimEnd: element.trimEnd + rightSourceSpan,
name: `${element.name} (left)`,
animations: leftAnimations,
...(retimeRef !== undefined ? { retime: retimeRef } : {}),
},
];
} else if (this.retainSide === "right") {
const newId = generateUUID();
this.rightSideElements.push({
trackId: track.id,
elementId: newId,
});
splitResult = [
{
...element,
id: newId,
startTime: this.splitTime,
duration: rightVisibleDuration,
trimStart: element.trimStart + leftSourceSpan,
name: `${element.name} (right)`,
animations: rightAnimations,
...(retimeRef !== undefined ? { retime: retimeRef } : {}),
},
];
} else {
// "both" - split into two pieces
const secondElementId = generateUUID();
this.rightSideElements.push({
trackId: track.id,
elementId: secondElementId,
});
splitResult = [
{
...element,
duration: leftVisibleDuration,
trimEnd: element.trimEnd + rightSourceSpan,
name: `${element.name} (left)`,
animations: leftAnimations,
...(retimeRef !== undefined ? { retime: retimeRef } : {}),
},
{
...element,
id: secondElementId,
startTime: this.splitTime,
duration: rightVisibleDuration,
trimStart: element.trimStart + leftSourceSpan,
name: `${element.name} (right)`,
animations: rightAnimations,
...(retimeRef !== undefined ? { retime: retimeRef } : {}),
},
];
}
if (this.retainSide === "right") {
const newId = generateUUID();
this.rightSideElements.push({
trackId: track.id,
elementId: newId,
});
return [
{
...element,
id: newId,
startTime: this.splitTime,
duration: rightVisibleDuration,
trimStart: element.trimStart + leftSourceSpan,
name: `${element.name} (right)`,
animations: rightAnimations,
...(retimeRef !== undefined ? { retime: retimeRef } : {}),
},
];
}
// "both" - split into two pieces
const secondElementId = generateUUID();
this.rightSideElements.push({
trackId: track.id,
elementId: secondElementId,
});
return [
{
...element,
duration: leftVisibleDuration,
trimEnd: element.trimEnd + rightSourceSpan,
name: `${element.name} (left)`,
animations: leftAnimations,
...(retimeRef !== undefined ? { retime: retimeRef } : {}),
},
{
...element,
id: secondElementId,
startTime: this.splitTime,
duration: rightVisibleDuration,
trimStart: element.trimStart + leftSourceSpan,
name: `${element.name} (right)`,
animations: rightAnimations,
...(retimeRef !== undefined ? { retime: retimeRef } : {}),
},
];
return splitResult;
});
return { ...track, elements } as TTrack;

48
apps/web/src/lib/commands/timeline/element/toggle-source-audio-separation.ts
View File

@ -5,10 +5,12 @@ import {
canExtractSourceAudio,
isSourceAudioSeparated,
} from "@/lib/timeline/audio-separation";
import { buildEmptyTrack } from "@/lib/timeline/placement";
import {
applyPlacement,
resolveTrackPlacement,
} from "@/lib/timeline/placement";
import { updateElementInSceneTracks } from "@/lib/timeline/track-element-update";
import type {
AudioTrack,
SceneTracks,
TimelineElement,
VideoElement,
@ -35,9 +37,7 @@ export class ToggleSourceAudioSeparationCommand extends Command {
...this.savedState.overlay,
this.savedState.main,
...this.savedState.audio,
].find(
(track) => track.id === this.params.trackId,
);
].find((track) => track.id === this.params.trackId);
if (!sourceTrack) {
return;
}
@ -61,8 +61,7 @@ export class ToggleSourceAudioSeparationCommand extends Command {
return;
}
const mediaAsset = editor
.media
const mediaAsset = editor.media
.getAssets()
.find((asset) => asset.id === videoElement.mediaId);
if (!canExtractSourceAudio(videoElement, mediaAsset)) {
@ -78,20 +77,32 @@ export class ToggleSourceAudioSeparationCommand extends Command {
}),
id: generateUUID(),
};
const newAudioTrack = {
...buildEmptyTrack({
id: generateUUID(),
type: "audio",
}),
const placementResult = resolveTrackPlacement({
tracks: this.savedState,
trackType: "audio",
timeSpans: [
{
startTime: separatedAudioElement.startTime,
duration: separatedAudioElement.duration,
},
],
strategy: { type: "firstAvailable" },
});
if (!placementResult) {
return;
}
const appliedPlacement = applyPlacement({
tracks: this.savedState,
placementResult,
elements: [separatedAudioElement],
} as AudioTrack;
});
if (!appliedPlacement) {
return;
}
editor.timeline.updateTracks(
updateSourceAudioEnabled({
tracks: {
...this.savedState,
audio: [...this.savedState.audio, newAudioTrack],
},
tracks: appliedPlacement.updatedTracks,
trackId: this.params.trackId,
elementId: this.params.elementId,
isSourceAudioEnabled: false,
@ -124,7 +135,8 @@ function updateSourceAudioEnabled({
tracks,
trackId,
elementId,
elementPredicate: (element): element is VideoElement => element.type === "video",
elementPredicate: (element): element is VideoElement =>
element.type === "video",
update: (element) => ({
...element,
isSourceAudioEnabled,

5
apps/web/src/lib/commands/timeline/track/add-track.ts
View File

@ -69,10 +69,7 @@ function buildAudioTrackState({
insertIndex: number;
trackId: string;
}): SceneTracks {
const audioInsertIndex = Math.max(
0,
insertIndex - tracks.overlay.length - 1,
);
const audioInsertIndex = Math.max(0, insertIndex - tracks.overlay.length - 1);
const newTrack = buildEmptyTrack({
id: trackId,
type: "audio",

170
apps/web/src/lib/media/audio.ts
View File

@ -20,10 +20,13 @@ import { mediaSupportsAudio } from "@/lib/media/media-utils";
import { getSourceTimeAtClipTime, renderRetimedBuffer } from "@/lib/retime";
import { Input, ALL_FORMATS, BlobSource, AudioBufferSink } from "mediabunny";
import { TICKS_PER_SECOND } from "@/lib/wasm";
import {
computeRmsBuckets,
type SampleBucket,
} from "@/lib/media/waveform-summary";
const MAX_AUDIO_CHANNELS = 2;
const EXPORT_SAMPLE_RATE = 44100;
const COARSE_SAMPLE_COUNT = 2048;
export interface CollectedAudioElement {
timelineElement: AudioCapableElement;
@ -177,8 +180,8 @@ export async function collectAudioElements({
if (!mediaAsset || !mediaSupportsAudio({ media: mediaAsset })) continue;
pendingElements.push(
resolveAudioBufferForVideoElement({
mediaAsset,
resolveAudioBufferForAsset({
asset: mediaAsset,
audioContext,
}).then((audioBuffer) => {
if (!audioBuffer) return null;
@ -222,10 +225,8 @@ async function resolveAudioBufferForElement({
try {
if (element.sourceType === "upload") {
const asset = mediaMap.get(element.mediaId);
if (!asset || asset.type !== "audio") return null;
const arrayBuffer = await asset.file.arrayBuffer();
return await audioContext.decodeAudioData(arrayBuffer.slice(0));
if (!asset) return null;
return await resolveAudioBufferForAsset({ asset, audioContext });
}
if (element.buffer) return element.buffer;
@ -243,15 +244,25 @@ async function resolveAudioBufferForElement({
}
}
async function resolveAudioBufferForVideoElement({
mediaAsset,
async function resolveAudioBufferForAsset({
asset,
audioContext,
}: {
mediaAsset: MediaAsset;
asset: MediaAsset;
audioContext: AudioContext;
}): Promise<AudioBuffer | null> {
if (asset.type === "audio") {
try {
const arrayBuffer = await asset.file.arrayBuffer();
return await audioContext.decodeAudioData(arrayBuffer.slice(0));
} catch (error) {
console.warn("Failed to decode audio asset:", error);
return null;
}
}
const input = new Input({
source: new BlobSource(mediaAsset.file),
source: new BlobSource(asset.file),
formats: ALL_FORMATS,
});
@ -319,7 +330,7 @@ async function resolveAudioBufferForVideoElement({
return await offlineContext.startRendering();
} catch (error) {
console.warn("Failed to decode video audio:", error);
console.warn("Failed to decode asset audio:", error);
return null;
} finally {
input.dispose();
@ -675,51 +686,29 @@ export async function createTimelineAudioBuffer({
return await applyAudioMasteringToBuffer({ audioBuffer: outputBuffer });
}
export function computeGlobalMaxRms({
buffer,
}: {
buffer: AudioBuffer;
}): number {
const channels = buffer.numberOfChannels;
const step = Math.max(1, Math.floor(buffer.length / COARSE_SAMPLE_COUNT));
let globalMax = 0;
for (let c = 0; c < channels; c++) {
const data = buffer.getChannelData(c);
for (let i = 0; i + step <= buffer.length; i += step) {
for (let j = i; j < i + step; j++) {
const abs = Math.abs(data[j]);
if (abs > globalMax) globalMax = abs;
}
}
}
return globalMax || 1;
}
export function extractRmsRange({
function collectPeakRange({
buffer,
count,
startSample,
endSample,
globalMax,
}: {
buffer: AudioBuffer;
count: number;
startSample: number;
endSample: number;
globalMax: number;
}): number[] {
}): Float32Array {
const channels = buffer.numberOfChannels;
const rangeLength = endSample - startSample;
const step = Math.max(1, Math.floor(rangeLength / count));
const peaks = new Float32Array(count);
for (let c = 0; c < channels; c++) {
const data = buffer.getChannelData(c);
for (let i = 0; i < count; i++) {
const start = startSample + i * step;
const end = Math.min(start + step, endSample);
const { bucketStart: start, bucketEnd: end } = getSampleBucketRange({
startSample,
endSample,
bucketIndex: i,
bucketCount: count,
});
for (let j = start; j < end; j++) {
const abs = Math.abs(data[j]);
if (abs > peaks[i]) peaks[i] = abs;
@ -727,11 +716,100 @@ export function extractRmsRange({
}
}
const norm = 1 / globalMax;
const result = new Array<number>(count);
for (let i = 0; i < count; i++) result[i] = Math.min(1, peaks[i] * norm);
return peaks;
}
return result;
export function extractPeakRange({
buffer,
count,
startSample,
endSample,
}: {
buffer: AudioBuffer;
count: number;
startSample: number;
endSample: number;
}): number[] {
return Array.from(
collectPeakRange({
buffer,
count,
startSample,
endSample,
}),
);
}
export function getSampleBucketRange({
startSample,
endSample,
bucketIndex,
bucketCount,
}: {
startSample: number;
endSample: number;
bucketIndex: number;
bucketCount: number;
}): {
bucketStart: number;
bucketEnd: number;
} {
const rangeLength = Math.max(0, endSample - startSample);
const bucketStart =
startSample + Math.floor((bucketIndex * rangeLength) / bucketCount);
const bucketEnd =
startSample + Math.floor(((bucketIndex + 1) * rangeLength) / bucketCount);
return {
bucketStart,
bucketEnd: Math.max(bucketStart, bucketEnd),
};
}
export function extractRmsBuckets({
buffer,
buckets,
}: {
buffer: AudioBuffer;
buckets: SampleBucket[];
}): number[] {
return computeRmsBuckets({ buffer, buckets });
}
/**
* Computes per-bucket waveform amplitude using the maximum RMS over a short
* analysis window inside each bucket.
*
* A naive mean-RMS over a whole bucket averages silence together with nearby
* sound, which smears transitions (e.g. the onset of speech) across the
* bucket and makes the waveform respond late. Taking the max over fixed
* short windows (~20 ms) preserves the smooth, non-jittery RMS character
* while making transitions land where they actually happen in the audio.
*
* Channels are combined per-window before taking the max, so the measure
* reflects total energy regardless of stereo layout.
*/
export function extractRmsRange({
buffer,
count,
startSample,
endSample,
}: {
buffer: AudioBuffer;
count: number;
startSample: number;
endSample: number;
}): number[] {
return extractRmsBuckets({
buffer,
buckets: Array.from({ length: count }, (_, bucketIndex) =>
getSampleBucketRange({
startSample,
endSample,
bucketIndex,
bucketCount: count,
}),
),
});
}
function mixAudioChannels({

233
apps/web/src/lib/media/waveform-summary.ts Normal file
View File

@ -0,0 +1,233 @@
"use client";
import { getSourceTimeAtClipTime } from "@/lib/retime";
import type { RetimeConfig } from "@/lib/timeline";
const RMS_ANALYSIS_WINDOW_SECONDS = 0.02;
const DEFAULT_SOURCE_WAVEFORM_BUCKET_SIZE = 128;
function computePeakBuckets({
buffer,
buckets,
}: {
buffer: AudioBuffer;
buckets: SampleBucket[];
}): number[] {
const channels = buffer.numberOfChannels;
const channelData: Float32Array[] = Array.from({ length: channels }, (_, c) =>
buffer.getChannelData(c),
);
return buckets.map(({ bucketStart, bucketEnd }) => {
let peak = 0;
for (let c = 0; c < channels; c++) {
const data = channelData[c];
for (let j = bucketStart; j < bucketEnd; j++) {
const abs = Math.abs(data[j] ?? 0);
if (abs > peak) {
peak = abs;
}
}
}
return peak;
});
}
export interface SampleBucket {
bucketStart: number;
bucketEnd: number;
}
export interface SourceWaveformSummary {
sourceKey: string;
sampleRate: number;
totalSamples: number;
bucketSize: number;
amplitudes: Float32Array;
}
export function buildWaveformSourceKey({
kind,
id,
}: {
kind: "media" | "library";
id: string;
}): string {
return `${kind}:${id}`;
}
export function buildSourceWaveformSummary({
sourceKey,
buffer,
bucketSize = DEFAULT_SOURCE_WAVEFORM_BUCKET_SIZE,
}: {
sourceKey: string;
buffer: AudioBuffer;
bucketSize?: number;
}): SourceWaveformSummary {
const safeBucketSize = Math.max(1, Math.floor(bucketSize));
const bucketCount = Math.max(1, Math.ceil(buffer.length / safeBucketSize));
const amplitudes = computePeakBuckets({
buffer,
buckets: Array.from({ length: bucketCount }, (_, bucketIndex) => {
const bucketStart = bucketIndex * safeBucketSize;
const bucketEnd = Math.min(buffer.length, bucketStart + safeBucketSize);
return { bucketStart, bucketEnd };
}),
});
return {
sourceKey,
sampleRate: buffer.sampleRate,
totalSamples: buffer.length,
bucketSize: safeBucketSize,
amplitudes: Float32Array.from(amplitudes),
};
}
export function buildWaveformSampleBuckets({
clipLeftPx,
clipRightPx,
barCount,
pixelsPerSecond,
clipDurationSec,
sourceStartSec,
retime,
sampleRate,
maxSampleExclusive,
barStepPx,
}: {
clipLeftPx: number;
clipRightPx: number;
barCount: number;
pixelsPerSecond: number;
clipDurationSec: number;
sourceStartSec: number;
retime?: RetimeConfig;
sampleRate: number;
maxSampleExclusive: number;
barStepPx: number;
}): SampleBucket[] {
return Array.from({ length: barCount }, (_, index) => {
const bucketLeftPx = clipLeftPx + index * barStepPx;
const bucketRightPx = Math.min(clipRightPx, bucketLeftPx + barStepPx);
const clipStartSec = Math.max(
0,
Math.min(clipDurationSec, bucketLeftPx / pixelsPerSecond),
);
const clipEndSec = Math.max(
clipStartSec,
Math.min(clipDurationSec, bucketRightPx / pixelsPerSecond),
);
const sourceBucketStartSec =
sourceStartSec +
getSourceTimeAtClipTime({
clipTime: clipStartSec,
retime,
});
const sourceBucketEndSec =
sourceStartSec +
getSourceTimeAtClipTime({
clipTime: clipEndSec,
retime,
});
return {
bucketStart: Math.max(0, Math.floor(sourceBucketStartSec * sampleRate)),
bucketEnd: Math.min(
maxSampleExclusive,
Math.max(0, Math.ceil(sourceBucketEndSec * sampleRate)),
),
};
});
}
export function sampleSourceWaveformSummary({
summary,
buckets,
}: {
summary: SourceWaveformSummary;
buckets: SampleBucket[];
}): number[] {
return buckets.map(({ bucketStart, bucketEnd }) => {
if (bucketEnd <= bucketStart) {
return 0;
}
const startIndex = Math.max(
0,
Math.floor(bucketStart / summary.bucketSize),
);
const endIndex = Math.min(
summary.amplitudes.length,
Math.max(startIndex + 1, Math.ceil(bucketEnd / summary.bucketSize)),
);
let maxAmplitude = 0;
for (let i = startIndex; i < endIndex; i++) {
const amplitude = summary.amplitudes[i] ?? 0;
if (amplitude > maxAmplitude) {
maxAmplitude = amplitude;
}
}
return maxAmplitude;
});
}
export function computeRmsBuckets({
buffer,
buckets,
}: {
buffer: AudioBuffer;
buckets: SampleBucket[];
}): number[] {
const channels = buffer.numberOfChannels;
const maxWindowLength = Math.max(
1,
Math.floor(buffer.sampleRate * RMS_ANALYSIS_WINDOW_SECONDS),
);
const channelData: Float32Array[] = new Array(channels);
for (let c = 0; c < channels; c++) {
channelData[c] = buffer.getChannelData(c);
}
const result = new Array<number>(buckets.length);
for (let i = 0; i < buckets.length; i++) {
const { bucketStart, bucketEnd } = buckets[i];
const bucketLength = bucketEnd - bucketStart;
if (bucketLength <= 0) {
result[i] = 0;
continue;
}
const windowLength = Math.max(1, Math.min(bucketLength, maxWindowLength));
let maxMeanSquare = 0;
for (let winStart = bucketStart; winStart < bucketEnd; ) {
const winEnd = Math.min(winStart + windowLength, bucketEnd);
const n = winEnd - winStart;
if (n > 0) {
let sum = 0;
for (let c = 0; c < channels; c++) {
const data = channelData[c];
for (let j = winStart; j < winEnd; j++) {
const v = data[j];
sum += v * v;
}
}
const meanSquare = sum / (n * channels);
if (meanSquare > maxMeanSquare) {
maxMeanSquare = meanSquare;
}
}
winStart = winEnd;
}
result[i] = Math.sqrt(maxMeanSquare);
}
return result;
}

64
apps/web/src/lib/timeline/audio-display.ts Normal file
View File

@ -0,0 +1,64 @@
import { clampDb } from "./audio-state";
import { VOLUME_DB_MAX, VOLUME_DB_MIN } from "./audio-constants";
const SLIDER_CURVE_EXPONENT = 2;
const MIN_DISPLAY_DB = -40;
const WAVEFORM_BAR_EXPONENT = 1.5;
const MIN_LINEAR_GAIN = 10 ** (VOLUME_DB_MIN / 20);
const MAX_LINEAR_GAIN = 10 ** (VOLUME_DB_MAX / 20);
const LINEAR_GAIN_RANGE = MAX_LINEAR_GAIN - MIN_LINEAR_GAIN;
function getNormalizedGainFromDb({ db }: { db: number }): number {
const clampedDb = clampDb(db);
const linearGain = 10 ** (clampedDb / 20);
return (linearGain - MIN_LINEAR_GAIN) / LINEAR_GAIN_RANGE;
}
/**
* Maps the clip's volume setting to the line position. The curve is defined in
* linear gain space so dragging near 0 dB is precise while mute compresses into
* the bottom of the clip.
*/
export function getLinePosFromDb({ db }: { db: number }): number {
const normalizedGain = Math.max(
0,
Math.min(1, getNormalizedGainFromDb({ db })),
);
const progress = normalizedGain ** (1 / SLIDER_CURVE_EXPONENT);
return (1 - progress) * 100;
}
/**
* Inverse of getLinePosFromDb. Converts a drag position back into the clip's
* volume setting without depending on the underlying audio content.
*/
export function getDbFromLinePos({ percent }: { percent: number }): number {
const clampedPercent = Math.max(0, Math.min(100, percent));
const progress = 1 - clampedPercent / 100;
const normalizedGain = progress ** SLIDER_CURVE_EXPONENT;
const linearGain = MIN_LINEAR_GAIN + normalizedGain * LINEAR_GAIN_RANGE;
return clampDb(20 * Math.log10(linearGain));
}
/**
* Maps an output amplitude (raw sample amplitude × gain) to a visible waveform
* height fraction using a dB scale. Agnostic to whether the input is peak or
* RMS — the caller decides which measure feeds this function; the mapping
* curve is the same either way.
*
* The log scale keeps quiet content visible while reserving the top of the
* element for amplitudes that approach 0 dBFS.
*/
export function getBarFractionFromOutputAmplitude({
outputAmplitude,
}: {
outputAmplitude: number;
}): number {
if (outputAmplitude <= 0) return 0;
const db = 20 * Math.log10(outputAmplitude);
if (db <= MIN_DISPLAY_DB) return 0;
return Math.min(
1,
((db - MIN_DISPLAY_DB) / -MIN_DISPLAY_DB) ** WAVEFORM_BAR_EXPONENT,
);
}

97
apps/web/src/lib/timeline/group-resize/compute-resize.ts
View File

@ -44,14 +44,34 @@ export function computeGroupResize({
? minimumDeltaTime
: Math.min(maximumDeltaTime, Math.max(minimumDeltaTime, deltaTime));
// Snap the drag delta to a frame exactly once, then derive every patch
// field from that single snapped value. This keeps the invariant
// `trimStart + duration*rate + trimEnd == sourceDuration` exact: the same
// delta is added on one side of the element and removed from the other,
// so the rounding cancels by construction. Per-field rounding (the old
// approach) couldn't preserve this because the individual rounds don't
// compose when `sourceDuration` isn't frame-aligned.
const snappedDeltaTime =
roundToFrame({ time: clampedDeltaTime, rate: fps }) ?? clampedDeltaTime;
// Re-clamp after rounding. Bounds derived from other elements are
// frame-aligned, so this is normally a no-op; at the source-extent limit
// the bound may not be frame-aligned, and honouring the bound takes
// precedence over frame alignment (you can't extend past real content).
const finalDeltaTime =
minimumDeltaTime > maximumDeltaTime
? minimumDeltaTime
: Math.min(
maximumDeltaTime,
Math.max(minimumDeltaTime, snappedDeltaTime),
);
return {
deltaTime: Object.is(clampedDeltaTime, -0) ? 0 : clampedDeltaTime,
deltaTime: Object.is(finalDeltaTime, -0) ? 0 : finalDeltaTime,
updates: members.map((member) =>
buildResizeUpdate({
member,
side,
deltaTime: clampedDeltaTime,
fps,
deltaTime: finalDeltaTime,
}),
),
};
@ -61,98 +81,37 @@ function buildResizeUpdate({
member,
side,
deltaTime,
fps,
}: {
member: GroupResizeMember;
side: ResizeSide;
deltaTime: number;
fps: ComputeGroupResizeArgs["fps"];
}): GroupResizeUpdate {
const totalSourceDuration = getSourceDuration({ member });
const sourceDelta = getSourceDeltaForClipDelta({
member,
clipDelta: deltaTime,
});
const visibleSourceSpan = getVisibleSourceSpanForDuration({
member,
duration: member.duration,
});
if (side === "left") {
if (deltaTime < 0 && member.sourceDuration == null) {
const rawStartTime = member.startTime + deltaTime;
const rawDuration = member.duration - deltaTime;
return {
trackId: member.trackId,
elementId: member.elementId,
patch: {
trimStart: Math.max(0, member.trimStart + sourceDelta),
trimEnd: member.trimEnd,
startTime:
roundToFrame({ time: rawStartTime, rate: fps }) ?? rawStartTime,
duration:
roundToFrame({ time: rawDuration, rate: fps }) ?? rawDuration,
},
};
}
const nextTrimStart = Math.max(0, member.trimStart + sourceDelta);
const nextVisibleSourceSpan = Math.max(
0,
totalSourceDuration - nextTrimStart - member.trimEnd,
);
const rawDuration = getDurationForVisibleSourceSpan({
member,
sourceSpan: nextVisibleSourceSpan,
});
const nextDuration =
roundToFrame({ time: rawDuration, rate: fps }) ?? rawDuration;
const rawStartTime = member.startTime + (member.duration - nextDuration);
return {
trackId: member.trackId,
elementId: member.elementId,
patch: {
trimStart:
roundToFrame({ time: nextTrimStart, rate: fps }) ?? nextTrimStart,
trimStart: Math.max(0, member.trimStart + sourceDelta),
trimEnd: member.trimEnd,
startTime:
roundToFrame({ time: rawStartTime, rate: fps }) ?? rawStartTime,
duration: nextDuration,
startTime: member.startTime + deltaTime,
duration: member.duration - deltaTime,
},
};
}
const nextVisibleSourceSpan = Math.max(0, visibleSourceSpan + sourceDelta);
if (deltaTime > 0 && member.sourceDuration == null) {
const rawDuration = member.duration + deltaTime;
return {
trackId: member.trackId,
elementId: member.elementId,
patch: {
trimStart: member.trimStart,
trimEnd: Math.max(0, member.trimEnd - sourceDelta),
startTime: member.startTime,
duration: roundToFrame({ time: rawDuration, rate: fps }) ?? rawDuration,
},
};
}
const nextTrimEnd = Math.max(
0,
totalSourceDuration - member.trimStart - nextVisibleSourceSpan,
);
const rawDuration = getDurationForVisibleSourceSpan({
member,
sourceSpan: nextVisibleSourceSpan,
});
return {
trackId: member.trackId,
elementId: member.elementId,
patch: {
trimStart: member.trimStart,
trimEnd: roundToFrame({ time: nextTrimEnd, rate: fps }) ?? nextTrimEnd,
trimEnd: Math.max(0, member.trimEnd - sourceDelta),
startTime: member.startTime,
duration: roundToFrame({ time: rawDuration, rate: fps }) ?? rawDuration,
duration: member.duration + deltaTime,
},
};
}

87
apps/web/src/services/waveform-cache/service.ts Normal file
View File

@ -0,0 +1,87 @@
"use client";
import { createAudioContext } from "@/lib/media/audio";
import {
buildSourceWaveformSummary,
type SourceWaveformSummary,
} from "@/lib/media/waveform-summary";
interface GetSourceWaveformSummaryArgs {
sourceKey: string;
audioBuffer?: AudioBuffer;
sourceFile?: File;
audioUrl?: string;
}
export class WaveformCache {
private summaries = new Map<string, Promise<SourceWaveformSummary>>();
getSourceSummary({
sourceKey,
audioBuffer,
sourceFile,
audioUrl,
}: GetSourceWaveformSummaryArgs): Promise<SourceWaveformSummary> {
const existing = this.summaries.get(sourceKey);
if (existing) {
return existing;
}
const promise = this.buildSummary({
sourceKey,
audioBuffer,
sourceFile,
audioUrl,
}).catch((error) => {
this.summaries.delete(sourceKey);
throw error;
});
this.summaries.set(sourceKey, promise);
return promise;
}
clearSource({ sourceKey }: { sourceKey: string }): void {
this.summaries.delete(sourceKey);
}
clearAll(): void {
this.summaries.clear();
}
private async buildSummary({
sourceKey,
audioBuffer,
sourceFile,
audioUrl,
}: GetSourceWaveformSummaryArgs): Promise<SourceWaveformSummary> {
if (audioBuffer) {
return buildSourceWaveformSummary({ sourceKey, buffer: audioBuffer });
}
let arrayBuffer: ArrayBuffer | null = null;
if (sourceFile) {
arrayBuffer = await sourceFile.arrayBuffer();
} else if (audioUrl) {
const response = await fetch(audioUrl);
if (!response.ok) {
throw new Error(`Failed to fetch waveform source: ${response.status}`);
}
arrayBuffer = await response.arrayBuffer();
}
if (!arrayBuffer) {
throw new Error(`No waveform source available for ${sourceKey}`);
}
const audioContext = createAudioContext();
try {
const buffer = await audioContext.decodeAudioData(arrayBuffer.slice(0));
return buildSourceWaveformSummary({ sourceKey, buffer });
} finally {
void audioContext.close();
}
}
}
export const waveformCache = new WaveformCache();

512
apps/web/src/stores/sounds-store.ts
View File

@ -1,261 +1,251 @@
import { create } from "zustand";
import type { SoundEffect, SavedSound } from "@/lib/sounds/types";
import { storageService } from "@/services/storage/service";
import { toast } from "sonner";
import { EditorCore } from "@/core";
import { buildLibraryAudioElement } from "@/lib/timeline/element-utils";
interface SoundsStore {
topSoundEffects: SoundEffect[];
isLoading: boolean;
error: string | null;
hasLoaded: boolean;
showCommercialOnly: boolean;
toggleCommercialFilter: () => void;
searchQuery: string;
searchResults: SoundEffect[];
isSearching: boolean;
searchError: string | null;
lastSearchQuery: string;
scrollPosition: number;
currentPage: number;
hasNextPage: boolean;
totalCount: number;
isLoadingMore: boolean;
savedSounds: SavedSound[];
isSavedSoundsLoaded: boolean;
isLoadingSavedSounds: boolean;
savedSoundsError: string | null;
addSoundToTimeline: ({ sound }: { sound: SoundEffect }) => Promise<boolean>;
setTopSoundEffects: ({ sounds }: { sounds: SoundEffect[] }) => void;
setLoading: ({ loading }: { loading: boolean }) => void;
setError: ({ error }: { error: string | null }) => void;
setHasLoaded: ({ loaded }: { loaded: boolean }) => void;
setSearchQuery: ({ query }: { query: string }) => void;
setSearchResults: ({ results }: { results: SoundEffect[] }) => void;
setSearching: ({ searching }: { searching: boolean }) => void;
setSearchError: ({ error }: { error: string | null }) => void;
setLastSearchQuery: ({ query }: { query: string }) => void;
setScrollPosition: ({ position }: { position: number }) => void;
setCurrentPage: ({ page }: { page: number }) => void;
setHasNextPage: ({ hasNext }: { hasNext: boolean }) => void;
setTotalCount: ({ count }: { count: number }) => void;
setLoadingMore: ({ loading }: { loading: boolean }) => void;
appendSearchResults: ({ results }: { results: SoundEffect[] }) => void;
appendTopSounds: ({ results }: { results: SoundEffect[] }) => void;
resetPagination: () => void;
loadSavedSounds: () => Promise<void>;
saveSoundEffect: ({
soundEffect,
}: {
soundEffect: SoundEffect;
}) => Promise<void>;
removeSavedSound: ({ soundId }: { soundId: number }) => Promise<void>;
isSoundSaved: ({ soundId }: { soundId: number }) => boolean;
toggleSavedSound: ({
soundEffect,
}: {
soundEffect: SoundEffect;
}) => Promise<void>;
clearSavedSounds: () => Promise<void>;
}
export const useSoundsStore = create<SoundsStore>((set, get) => ({
topSoundEffects: [],
isLoading: false,
error: null,
hasLoaded: false,
showCommercialOnly: true,
toggleCommercialFilter: () => {
set((state) => ({ showCommercialOnly: !state.showCommercialOnly }));
},
searchQuery: "",
searchResults: [],
isSearching: false,
searchError: null,
lastSearchQuery: "",
scrollPosition: 0,
currentPage: 1,
hasNextPage: false,
totalCount: 0,
isLoadingMore: false,
savedSounds: [],
isSavedSoundsLoaded: false,
isLoadingSavedSounds: false,
savedSoundsError: null,
setTopSoundEffects: ({ sounds }) => set({ topSoundEffects: sounds }),
setLoading: ({ loading }) => set({ isLoading: loading }),
setError: ({ error }) => set({ error }),
setHasLoaded: ({ loaded }) => set({ hasLoaded: loaded }),
setSearchQuery: ({ query }) => set({ searchQuery: query }),
setSearchResults: ({ results }) =>
set({ searchResults: results, currentPage: 1 }),
setSearching: ({ searching }) => set({ isSearching: searching }),
setSearchError: ({ error }) => set({ searchError: error }),
setLastSearchQuery: ({ query }) => set({ lastSearchQuery: query }),
setScrollPosition: ({ position }) => set({ scrollPosition: position }),
setCurrentPage: ({ page }) => set({ currentPage: page }),
setHasNextPage: ({ hasNext }) => set({ hasNextPage: hasNext }),
setTotalCount: ({ count }) => set({ totalCount: count }),
setLoadingMore: ({ loading }) => set({ isLoadingMore: loading }),
appendSearchResults: ({ results }) =>
set((state) => ({
searchResults: [...state.searchResults, ...results],
})),
appendTopSounds: ({ results }) =>
set((state) => ({
topSoundEffects: [...state.topSoundEffects, ...results],
})),
resetPagination: () =>
set({
currentPage: 1,
hasNextPage: false,
totalCount: 0,
isLoadingMore: false,
}),
loadSavedSounds: async () => {
if (get().isSavedSoundsLoaded) return;
try {
set({ isLoadingSavedSounds: true, savedSoundsError: null });
const savedSoundsData = await storageService.loadSavedSounds();
set({
savedSounds: savedSoundsData.sounds,
isSavedSoundsLoaded: true,
isLoadingSavedSounds: false,
});
} catch (error) {
const errorMessage =
error instanceof Error ? error.message : "Failed to load saved sounds";
set({
savedSoundsError: errorMessage,
isLoadingSavedSounds: false,
});
console.error("Failed to load saved sounds:", error);
}
},
saveSoundEffect: async ({ soundEffect }) => {
try {
await storageService.saveSoundEffect({ soundEffect });
const savedSoundsData = await storageService.loadSavedSounds();
set({ savedSounds: savedSoundsData.sounds });
} catch (error) {
const errorMessage =
error instanceof Error ? error.message : "Failed to save sound";
set({ savedSoundsError: errorMessage });
toast.error("Failed to save sound");
console.error("Failed to save sound:", error);
}
},
removeSavedSound: async ({ soundId }) => {
try {
await storageService.removeSavedSound({ soundId });
set((state) => ({
savedSounds: state.savedSounds.filter((sound) => sound.id !== soundId),
}));
} catch (error) {
const errorMessage =
error instanceof Error ? error.message : "Failed to remove sound";
set({ savedSoundsError: errorMessage });
toast.error("Failed to remove sound");
console.error("Failed to remove sound:", error);
}
},
isSoundSaved: ({ soundId }) => {
const { savedSounds } = get();
return savedSounds.some((sound) => sound.id === soundId);
},
toggleSavedSound: async ({ soundEffect }) => {
const { isSoundSaved, saveSoundEffect, removeSavedSound } = get();
if (isSoundSaved({ soundId: soundEffect.id })) {
await removeSavedSound({ soundId: soundEffect.id });
} else {
await saveSoundEffect({ soundEffect });
}
},
clearSavedSounds: async () => {
try {
await storageService.clearSavedSounds();
set({
savedSounds: [],
savedSoundsError: null,
});
} catch (error) {
const errorMessage =
error instanceof Error ? error.message : "Failed to clear saved sounds";
set({ savedSoundsError: errorMessage });
toast.error("Failed to clear saved sounds");
console.error("Failed to clear saved sounds:", error);
}
},
addSoundToTimeline: async ({ sound }) => {
const audioUrl = sound.previewUrl;
if (!audioUrl) {
toast.error("Sound file not available");
return false;
}
try {
const editor = EditorCore.getInstance();
const currentTime = editor.playback.getCurrentTime();
const tracks = editor.scenes.getActiveScene().tracks;
const response = await fetch(audioUrl);
if (!response.ok)
throw new Error(`Failed to download audio: ${response.statusText}`);
const arrayBuffer = await response.arrayBuffer();
const audioContext = new AudioContext();
const buffer = await audioContext.decodeAudioData(arrayBuffer);
const audioTrack = tracks.audio[0];
let trackId: string;
if (audioTrack) {
trackId = audioTrack.id;
} else {
trackId = editor.timeline.addTrack({ type: "audio" });
}
const element = buildLibraryAudioElement({
sourceUrl: audioUrl,
name: sound.name,
duration: sound.duration,
startTime: currentTime,
buffer,
});
editor.timeline.insertElement({
placement: { mode: "explicit", trackId },
element,
});
return true;
} catch (error) {
console.error("Failed to add sound to timeline:", error);
toast.error(
error instanceof Error
? error.message
: "Failed to add sound to timeline",
{ id: `sound-${sound.id}` },
);
return false;
}
},
}));
import { create } from "zustand";
import type { SoundEffect, SavedSound } from "@/lib/sounds/types";
import { storageService } from "@/services/storage/service";
import { toast } from "sonner";
import { EditorCore } from "@/core";
import { buildLibraryAudioElement } from "@/lib/timeline/element-utils";
interface SoundsStore {
topSoundEffects: SoundEffect[];
isLoading: boolean;
error: string | null;
hasLoaded: boolean;
showCommercialOnly: boolean;
toggleCommercialFilter: () => void;
searchQuery: string;
searchResults: SoundEffect[];
isSearching: boolean;
searchError: string | null;
lastSearchQuery: string;
scrollPosition: number;
currentPage: number;
hasNextPage: boolean;
totalCount: number;
isLoadingMore: boolean;
savedSounds: SavedSound[];
isSavedSoundsLoaded: boolean;
isLoadingSavedSounds: boolean;
savedSoundsError: string | null;
addSoundToTimeline: ({ sound }: { sound: SoundEffect }) => Promise<boolean>;
setTopSoundEffects: ({ sounds }: { sounds: SoundEffect[] }) => void;
setLoading: ({ loading }: { loading: boolean }) => void;
setError: ({ error }: { error: string | null }) => void;
setHasLoaded: ({ loaded }: { loaded: boolean }) => void;
setSearchQuery: ({ query }: { query: string }) => void;
setSearchResults: ({ results }: { results: SoundEffect[] }) => void;
setSearching: ({ searching }: { searching: boolean }) => void;
setSearchError: ({ error }: { error: string | null }) => void;
setLastSearchQuery: ({ query }: { query: string }) => void;
setScrollPosition: ({ position }: { position: number }) => void;
setCurrentPage: ({ page }: { page: number }) => void;
setHasNextPage: ({ hasNext }: { hasNext: boolean }) => void;
setTotalCount: ({ count }: { count: number }) => void;
setLoadingMore: ({ loading }: { loading: boolean }) => void;
appendSearchResults: ({ results }: { results: SoundEffect[] }) => void;
appendTopSounds: ({ results }: { results: SoundEffect[] }) => void;
resetPagination: () => void;
loadSavedSounds: () => Promise<void>;
saveSoundEffect: ({
soundEffect,
}: {
soundEffect: SoundEffect;
}) => Promise<void>;
removeSavedSound: ({ soundId }: { soundId: number }) => Promise<void>;
isSoundSaved: ({ soundId }: { soundId: number }) => boolean;
toggleSavedSound: ({
soundEffect,
}: {
soundEffect: SoundEffect;
}) => Promise<void>;
clearSavedSounds: () => Promise<void>;
}
export const useSoundsStore = create<SoundsStore>((set, get) => ({
topSoundEffects: [],
isLoading: false,
error: null,
hasLoaded: false,
showCommercialOnly: true,
toggleCommercialFilter: () => {
set((state) => ({ showCommercialOnly: !state.showCommercialOnly }));
},
searchQuery: "",
searchResults: [],
isSearching: false,
searchError: null,
lastSearchQuery: "",
scrollPosition: 0,
currentPage: 1,
hasNextPage: false,
totalCount: 0,
isLoadingMore: false,
savedSounds: [],
isSavedSoundsLoaded: false,
isLoadingSavedSounds: false,
savedSoundsError: null,
setTopSoundEffects: ({ sounds }) => set({ topSoundEffects: sounds }),
setLoading: ({ loading }) => set({ isLoading: loading }),
setError: ({ error }) => set({ error }),
setHasLoaded: ({ loaded }) => set({ hasLoaded: loaded }),
setSearchQuery: ({ query }) => set({ searchQuery: query }),
setSearchResults: ({ results }) =>
set({ searchResults: results, currentPage: 1 }),
setSearching: ({ searching }) => set({ isSearching: searching }),
setSearchError: ({ error }) => set({ searchError: error }),
setLastSearchQuery: ({ query }) => set({ lastSearchQuery: query }),
setScrollPosition: ({ position }) => set({ scrollPosition: position }),
setCurrentPage: ({ page }) => set({ currentPage: page }),
setHasNextPage: ({ hasNext }) => set({ hasNextPage: hasNext }),
setTotalCount: ({ count }) => set({ totalCount: count }),
setLoadingMore: ({ loading }) => set({ isLoadingMore: loading }),
appendSearchResults: ({ results }) =>
set((state) => ({
searchResults: [...state.searchResults, ...results],
})),
appendTopSounds: ({ results }) =>
set((state) => ({
topSoundEffects: [...state.topSoundEffects, ...results],
})),
resetPagination: () =>
set({
currentPage: 1,
hasNextPage: false,
totalCount: 0,
isLoadingMore: false,
}),
loadSavedSounds: async () => {
if (get().isSavedSoundsLoaded) return;
try {
set({ isLoadingSavedSounds: true, savedSoundsError: null });
const savedSoundsData = await storageService.loadSavedSounds();
set({
savedSounds: savedSoundsData.sounds,
isSavedSoundsLoaded: true,
isLoadingSavedSounds: false,
});
} catch (error) {
const errorMessage =
error instanceof Error ? error.message : "Failed to load saved sounds";
set({
savedSoundsError: errorMessage,
isLoadingSavedSounds: false,
});
console.error("Failed to load saved sounds:", error);
}
},
saveSoundEffect: async ({ soundEffect }) => {
try {
await storageService.saveSoundEffect({ soundEffect });
const savedSoundsData = await storageService.loadSavedSounds();
set({ savedSounds: savedSoundsData.sounds });
} catch (error) {
const errorMessage =
error instanceof Error ? error.message : "Failed to save sound";
set({ savedSoundsError: errorMessage });
toast.error("Failed to save sound");
console.error("Failed to save sound:", error);
}
},
removeSavedSound: async ({ soundId }) => {
try {
await storageService.removeSavedSound({ soundId });
set((state) => ({
savedSounds: state.savedSounds.filter((sound) => sound.id !== soundId),
}));
} catch (error) {
const errorMessage =
error instanceof Error ? error.message : "Failed to remove sound";
set({ savedSoundsError: errorMessage });
toast.error("Failed to remove sound");
console.error("Failed to remove sound:", error);
}
},
isSoundSaved: ({ soundId }) => {
const { savedSounds } = get();
return savedSounds.some((sound) => sound.id === soundId);
},
toggleSavedSound: async ({ soundEffect }) => {
const { isSoundSaved, saveSoundEffect, removeSavedSound } = get();
if (isSoundSaved({ soundId: soundEffect.id })) {
await removeSavedSound({ soundId: soundEffect.id });
} else {
await saveSoundEffect({ soundEffect });
}
},
clearSavedSounds: async () => {
try {
await storageService.clearSavedSounds();
set({
savedSounds: [],
savedSoundsError: null,
});
} catch (error) {
const errorMessage =
error instanceof Error ? error.message : "Failed to clear saved sounds";
set({ savedSoundsError: errorMessage });
toast.error("Failed to clear saved sounds");
console.error("Failed to clear saved sounds:", error);
}
},
addSoundToTimeline: async ({ sound }) => {
const audioUrl = sound.previewUrl;
if (!audioUrl) {
toast.error("Sound file not available");
return false;
}
try {
const editor = EditorCore.getInstance();
const currentTime = editor.playback.getCurrentTime();
const response = await fetch(audioUrl);
if (!response.ok)
throw new Error(`Failed to download audio: ${response.statusText}`);
const arrayBuffer = await response.arrayBuffer();
const audioContext = new AudioContext();
const buffer = await audioContext.decodeAudioData(arrayBuffer);
const element = buildLibraryAudioElement({
sourceUrl: audioUrl,
name: sound.name,
duration: sound.duration,
startTime: currentTime,
buffer,
});
editor.timeline.insertElement({
placement: { mode: "auto", trackType: "audio" },
element,
});
return true;
} catch (error) {
console.error("Failed to add sound to timeline:", error);
toast.error(
error instanceof Error
? error.message
: "Failed to add sound to timeline",
{ id: `sound-${sound.id}` },
);
return false;
}
},
}));