From f1d5932c54178c4f8eac61d3d3e0e4a30101780f Mon Sep 17 00:00:00 2001 From: Maze Winther Date: Tue, 31 Mar 2026 23:23:36 +0200 Subject: [PATCH] feat: separate audio from elements --- .../panels/properties/tabs/audio-tab.tsx | 98 +++++++----- .../panels/timeline/timeline-element.tsx | 35 +++++ .../panels/timeline/timeline-toolbar.tsx | 63 +++++++- .../web/src/core/managers/timeline-manager.ts | 15 ++ .../src/hooks/actions/use-editor-actions.ts | 48 +++++- apps/web/src/lib/actions/definitions.ts | 4 + .../lib/commands/timeline/element/index.ts | 1 + .../element/toggle-source-audio-separation.ts | 143 +++++++++++++++++ apps/web/src/lib/media/audio.ts | 30 ++-- .../audio-separation/__tests__/index.test.ts | 147 ++++++++++++++++++ .../lib/timeline/audio-separation/index.ts | 133 ++++++++++++++++ apps/web/src/lib/timeline/element-utils.ts | 1 + apps/web/src/lib/timeline/index.ts | 1 + apps/web/src/lib/timeline/types.ts | 1 + .../migrations/__tests__/v19-to-v20.test.ts | 141 +++++++++++++++++ .../src/services/storage/migrations/index.ts | 4 +- .../migrations/transformers/v19-to-v20.ts | 87 +++++++++++ .../services/storage/migrations/v19-to-v20.ts | 16 ++ 18 files changed, 915 insertions(+), 53 deletions(-) create mode 100644 apps/web/src/lib/commands/timeline/element/toggle-source-audio-separation.ts create mode 100644 apps/web/src/lib/timeline/audio-separation/__tests__/index.test.ts create mode 100644 apps/web/src/lib/timeline/audio-separation/index.ts create mode 100644 apps/web/src/services/storage/migrations/__tests__/v19-to-v20.test.ts create mode 100644 apps/web/src/services/storage/migrations/transformers/v19-to-v20.ts create mode 100644 apps/web/src/services/storage/migrations/v19-to-v20.ts diff --git a/apps/web/src/components/editor/panels/properties/tabs/audio-tab.tsx b/apps/web/src/components/editor/panels/properties/tabs/audio-tab.tsx index 4dbc1a17..5ac0fbb7 100644 --- a/apps/web/src/components/editor/panels/properties/tabs/audio-tab.tsx +++ b/apps/web/src/components/editor/panels/properties/tabs/audio-tab.tsx @@ -1,5 +1,7 @@ +import { Button } from "@/components/ui/button"; import { NumberField } from "@/components/ui/number-field"; import { VOLUME_DB_MAX, VOLUME_DB_MIN } from "@/lib/timeline/audio-constants"; +import { isSourceAudioSeparated } from "@/lib/timeline/audio-separation"; import { DEFAULTS } from "@/lib/timeline/defaults"; import { clamp, @@ -10,6 +12,7 @@ import { } from "@/utils/math"; import type { AudioElement, VideoElement } from "@/lib/timeline"; import { resolveNumberAtTime } from "@/lib/animation"; +import { useEditor } from "@/hooks/use-editor"; import { useElementPlayhead } from "../hooks/use-element-playhead"; import { useKeyframedNumberProperty } from "../hooks/use-keyframed-number-property"; import { KeyframeToggle } from "../components/keyframe-toggle"; @@ -34,6 +37,7 @@ export function AudioTab({ element: AudioElement | VideoElement; trackId: string; }) { + const editor = useEditor(); const { localTime, isPlayheadWithinElementRange } = useElementPlayhead({ startTime: element.startTime, duration: element.duration, @@ -80,46 +84,68 @@ export function AudioTab({ rightValue: DEFAULTS.element.volume, }) : (element.volume ?? DEFAULTS.element.volume) === DEFAULTS.element.volume; + const isSeparated = + element.type === "video" && isSourceAudioSeparated({ element }); return ( -
- - Audio - - - - + <> + {isSeparated && ( +
+

Audio has been separated.

+ +
+ )} +
+ + Audio + + + + } - isDefault={isDefault} - suffix="dB" - /> - - - -
+ > + } + value={volume.displayValue} + onFocus={volume.onFocus} + onChange={volume.onChange} + onBlur={volume.onBlur} + dragSensitivity="slow" + scrubClamp={{ min: VOLUME_DB_MIN, max: VOLUME_DB_MAX }} + onScrub={volume.scrubTo} + onScrubEnd={volume.commitScrub} + onReset={() => + volume.commitValue({ + value: DEFAULTS.element.volume, + }) + } + isDefault={isDefault} + suffix="dB" + /> + + + +
+ ); } diff --git a/apps/web/src/components/editor/panels/timeline/timeline-element.tsx b/apps/web/src/components/editor/panels/timeline/timeline-element.tsx index f196da44..bb30b0b0 100644 --- a/apps/web/src/components/editor/panels/timeline/timeline-element.tsx +++ b/apps/web/src/components/editor/panels/timeline/timeline-element.tsx @@ -39,6 +39,11 @@ import type { } from "@/lib/timeline"; import type { MediaAsset } from "@/lib/media/types"; import { mediaSupportsAudio } from "@/lib/media/media-utils"; +import { + canToggleSourceAudio, + getSourceAudioActionLabel, + isSourceAudioSeparated, +} from "@/lib/timeline/audio-separation"; import { getActionDefinition, type TAction, @@ -61,7 +66,9 @@ import { Search01Icon, Exchange01Icon, KeyframeIcon, + Link02Icon, MagicWand05Icon, + Unlink02Icon, } from "@hugeicons/core-free-icons"; import { HugeiconsIcon } from "@hugeicons/react"; import { uppercase } from "@/utils/string"; @@ -280,6 +287,19 @@ export function TimelineElement({ }; const isMuted = canElementHaveAudio(element) && element.muted === true; + const canToggleCurrentSourceAudio = + selectedElements.length === 1 && + isCurrentElementSelected && + canToggleSourceAudio({ + element, + mediaAsset, + }); + const sourceAudioLabel = + element.type === "video" + ? getSourceAudioActionLabel({ element }) + : "Extract audio"; + const isElementSourceAudioSeparated = + element.type === "video" && isSourceAudioSeparated({ element }); return ( @@ -334,6 +354,21 @@ export function TimelineElement({ isMuted={isMuted} /> )} + {canToggleCurrentSourceAudio && ( + + } + onClick={(event: React.MouseEvent) => { + event.stopPropagation(); + invokeAction("toggle-source-audio"); + }} + > + {sourceAudioLabel} + + )} {canElementBeHidden(element) && ( currentEditor.media.getAssets()); + const { selectedElements } = useElementSelection(); const isCurrentlyBookmarked = useEditor((e) => e.scenes.isBookmarked({ time: e.playback.getCurrentTime() }), ); + const selectedElement = + selectedElements.length === 1 + ? (editor.timeline.getElementsWithTracks({ + elements: selectedElements, + })[0] ?? null) + : null; + const selectedMediaAsset = (() => { + if (!selectedElement) { + return null; + } + + const { element } = selectedElement; + if (!hasMediaId(element)) { + return null; + } + + return mediaAssets.find((asset) => asset.id === element.mediaId) ?? null; + })(); + const canToggleSelectedSourceAudio = + !!selectedElement && + canToggleSourceAudio({ + element: selectedElement.element, + mediaAsset: selectedMediaAsset, + }); + const sourceAudioLabel = + selectedElement?.element.type === "video" + ? getSourceAudioActionLabel({ + element: selectedElement.element, + }) + : "Extract audio"; + const isSelectedSourceAudioSeparated = + selectedElement?.element.type === "video" && + isSourceAudioSeparated({ + element: selectedElement.element, + }); const handleAction = ({ action, @@ -117,10 +163,16 @@ function ToolbarLeftSection() { /> } - tooltip="Separate audio (coming soon)" - disabled={true} - onClick={({ event: _event }) => {}} + icon={ + + } + tooltip={sourceAudioLabel} + disabled={!canToggleSelectedSourceAudio} + onClick={({ event }) => + handleAction({ action: "toggle-source-audio", event }) + } /> onClick({ event })} className={cn( "rounded-sm", diff --git a/apps/web/src/core/managers/timeline-manager.ts b/apps/web/src/core/managers/timeline-manager.ts index 263de41f..db8c684f 100644 --- a/apps/web/src/core/managers/timeline-manager.ts +++ b/apps/web/src/core/managers/timeline-manager.ts @@ -45,6 +45,7 @@ import { ToggleMaskInvertedCommand, UpsertEffectParamKeyframeCommand, RemoveEffectParamKeyframeCommand, + ToggleSourceAudioSeparationCommand, } from "@/lib/commands/timeline"; import { BatchCommand } from "@/lib/commands"; import type { InsertElementParams } from "@/lib/commands/timeline/element/insert-element"; @@ -281,6 +282,20 @@ export class TimelineManager { this.editor.command.execute({ command }); } + toggleSourceAudioSeparation({ + trackId, + elementId, + }: { + trackId: string; + elementId: string; + }): void { + const command = new ToggleSourceAudioSeparationCommand({ + trackId, + elementId, + }); + this.editor.command.execute({ command }); + } + updateElements({ updates, pushHistory = true, diff --git a/apps/web/src/hooks/actions/use-editor-actions.ts b/apps/web/src/hooks/actions/use-editor-actions.ts index bfac931b..087ea687 100644 --- a/apps/web/src/hooks/actions/use-editor-actions.ts +++ b/apps/web/src/hooks/actions/use-editor-actions.ts @@ -6,9 +6,13 @@ import { useActionHandler } from "@/hooks/actions/use-action-handler"; import { useEditor } from "../use-editor"; import { useElementSelection } from "../timeline/element/use-element-selection"; import { useKeyframeSelection } from "../timeline/element/use-keyframe-selection"; -import { getElementsAtTime } from "@/lib/timeline"; +import { + getElementsAtTime, + hasMediaId, +} from "@/lib/timeline"; import { cancelInteraction } from "@/lib/cancel-interaction"; import { invokeAction } from "@/lib/actions"; +import { canToggleSourceAudio } from "@/lib/timeline/audio-separation"; import { activateScope, clearActiveScope, @@ -266,6 +270,48 @@ export function useEditorActions() { undefined, ); + useActionHandler( + "toggle-source-audio", + () => { + if (selectedElements.length !== 1) { + return; + } + + const selectedElement = editor.timeline.getElementsWithTracks({ + elements: selectedElements, + })[0]; + if (!selectedElement) { + return; + } + + const mediaAsset = (() => { + const { element } = selectedElement; + if (!hasMediaId(element)) { + return null; + } + + return ( + editor.media.getAssets().find((asset) => asset.id === element.mediaId) ?? + null + ); + })(); + if ( + !canToggleSourceAudio({ + element: selectedElement.element, + mediaAsset, + }) + ) { + return; + } + + editor.timeline.toggleSourceAudioSeparation({ + trackId: selectedElement.track.id, + elementId: selectedElement.element.id, + }); + }, + undefined, + ); + useActionHandler( "select-all", () => { diff --git a/apps/web/src/lib/actions/definitions.ts b/apps/web/src/lib/actions/definitions.ts index e48126d0..3059a2e3 100644 --- a/apps/web/src/lib/actions/definitions.ts +++ b/apps/web/src/lib/actions/definitions.ts @@ -101,6 +101,10 @@ export const ACTIONS = { description: "Toggle ripple editing", category: "editing", }, + "toggle-source-audio": { + description: "Extract or recover source audio", + category: "editing", + }, "select-all": { description: "Select all elements", category: "selection", diff --git a/apps/web/src/lib/commands/timeline/element/index.ts b/apps/web/src/lib/commands/timeline/element/index.ts index a7e513fe..5ce48d55 100644 --- a/apps/web/src/lib/commands/timeline/element/index.ts +++ b/apps/web/src/lib/commands/timeline/element/index.ts @@ -8,6 +8,7 @@ export { SplitElementsCommand } from "./split-elements"; export { UpdateElementCommand } from "./update-element"; export { ToggleElementsVisibilityCommand } from "./toggle-elements-visibility"; export { ToggleElementsMutedCommand } from "./toggle-elements-muted"; +export { ToggleSourceAudioSeparationCommand } from "./toggle-source-audio-separation"; export { MoveElementCommand } from "./move-elements"; export * from "./keyframes"; export * from "./effects"; diff --git a/apps/web/src/lib/commands/timeline/element/toggle-source-audio-separation.ts b/apps/web/src/lib/commands/timeline/element/toggle-source-audio-separation.ts new file mode 100644 index 00000000..d0907c5b --- /dev/null +++ b/apps/web/src/lib/commands/timeline/element/toggle-source-audio-separation.ts @@ -0,0 +1,143 @@ +import { EditorCore } from "@/core"; +import { Command } from "@/lib/commands/base-command"; +import { + buildSeparatedAudioElement, + canExtractSourceAudio, + canRecoverSourceAudio, +} from "@/lib/timeline/audio-separation"; +import { + applyPlacement, + resolveTrackPlacement, +} from "@/lib/timeline/placement"; +import { updateElementInTracks } from "@/lib/timeline/track-element-update"; +import type { TimelineTrack, VideoElement } from "@/lib/timeline/types"; +import { generateUUID } from "@/utils/id"; + +export class ToggleSourceAudioSeparationCommand extends Command { + private savedState: TimelineTrack[] | null = null; + + constructor( + private readonly params: { + trackId: string; + elementId: string; + }, + ) { + super(); + } + + execute(): void { + const editor = EditorCore.getInstance(); + this.savedState = editor.timeline.getTracks(); + + const trackIndex = this.savedState.findIndex( + (track) => track.id === this.params.trackId, + ); + if (trackIndex < 0) { + return; + } + + const sourceTrack = this.savedState[trackIndex]; + const sourceElement = sourceTrack.elements.find( + (element) => element.id === this.params.elementId, + ); + if (!sourceElement) { + return; + } + + if (canRecoverSourceAudio({ element: sourceElement })) { + editor.timeline.updateTracks( + updateSourceAudioEnabled({ + tracks: this.savedState, + trackId: this.params.trackId, + elementId: this.params.elementId, + isSourceAudioEnabled: true, + }), + ); + return; + } + + const mediaAsset = editor + .media + .getAssets() + .find((asset) => + sourceElement.type === "video" ? asset.id === sourceElement.mediaId : false, + ); + if (!canExtractSourceAudio({ element: sourceElement, mediaAsset })) { + return; + } + if (sourceElement.duration <= 0) { + return; + } + + const separatedAudioElement = { + ...buildSeparatedAudioElement({ + sourceElement, + }), + id: generateUUID(), + }; + const placementResult = resolveTrackPlacement({ + tracks: this.savedState, + trackType: "audio", + timeSpans: [ + { + startTime: separatedAudioElement.startTime, + duration: separatedAudioElement.duration, + }, + ], + strategy: { type: "aboveSource", sourceTrackIndex: trackIndex }, + }); + if (!placementResult) { + return; + } + + const appliedPlacement = applyPlacement({ + tracks: this.savedState, + placementResult, + elements: [separatedAudioElement], + }); + if (!appliedPlacement) { + return; + } + + editor.timeline.updateTracks( + updateSourceAudioEnabled({ + tracks: appliedPlacement.updatedTracks, + trackId: this.params.trackId, + elementId: this.params.elementId, + isSourceAudioEnabled: false, + }), + ); + } + + undo(): void { + if (!this.savedState) { + return; + } + + const editor = EditorCore.getInstance(); + editor.timeline.updateTracks(this.savedState); + } +} + +function updateSourceAudioEnabled({ + tracks, + trackId, + elementId, + isSourceAudioEnabled, +}: { + tracks: TimelineTrack[]; + trackId: string; + elementId: string; + isSourceAudioEnabled: boolean; +}): TimelineTrack[] { + return updateElementInTracks({ + tracks, + trackId, + elementId, + elementPredicate: (element): element is VideoElement => element.type === "video", + update: (element) => ({ + ...element, + isSourceAudioEnabled, + }), + }); +} diff --git a/apps/web/src/lib/media/audio.ts b/apps/web/src/lib/media/audio.ts index 41d3b38e..0afc42fb 100644 --- a/apps/web/src/lib/media/audio.ts +++ b/apps/web/src/lib/media/audio.ts @@ -12,7 +12,10 @@ import { hasAnimatedVolume, resolveEffectiveAudioGain, } from "@/lib/timeline/audio-state"; -import { canElementHaveAudio } from "@/lib/timeline/element-utils"; +import { + doesElementHaveEnabledAudio, +} from "@/lib/timeline/audio-separation"; +import { canElementHaveAudio, hasMediaId } from "@/lib/timeline/element-utils"; import { canTracktHaveAudio } from "@/lib/timeline"; import { mediaSupportsAudio } from "@/lib/media/media-utils"; import { getSourceTimeAtClipTime, renderRetimedBuffer } from "@/lib/retime"; @@ -100,6 +103,11 @@ export async function collectAudioElements({ if (!canElementHaveAudio(element)) continue; if (element.duration <= 0) continue; + const mediaAsset = hasMediaId(element) + ? (mediaMap.get(element.mediaId) ?? null) + : null; + if (!doesElementHaveEnabledAudio({ element, mediaAsset })) continue; + const isTrackMuted = canTracktHaveAudio(track) && track.muted; if (element.type === "audio") { @@ -132,7 +140,6 @@ export async function collectAudioElements({ } if (element.type === "video") { - const mediaAsset = mediaMap.get(element.mediaId); if (!mediaAsset || !mediaSupportsAudio({ media: mediaAsset })) continue; pendingElements.push( @@ -451,6 +458,10 @@ export async function collectAudioMixSources({ for (const element of track.elements) { if (!canElementHaveAudio(element)) continue; if (element.muted === true) continue; + const mediaAsset = hasMediaId(element) + ? (mediaMap.get(element.mediaId) ?? null) + : null; + if (!doesElementHaveEnabledAudio({ element, mediaAsset })) continue; const volume = resolveEffectiveAudioGain({ element, localTime: 0, @@ -473,10 +484,7 @@ export async function collectAudioMixSources({ } if (element.type === "video") { - const mediaAsset = mediaMap.get(element.mediaId); - if (!mediaAsset) continue; - - if (mediaSupportsAudio({ media: mediaAsset })) { + if (mediaAsset && mediaSupportsAudio({ media: mediaAsset })) { audioMixSources.push( collectMediaAudioSource({ element, mediaAsset, volume }), ); @@ -512,6 +520,11 @@ export async function collectAudioClips({ for (const element of track.elements) { if (!canElementHaveAudio(element)) continue; + const mediaAsset = hasMediaId(element) + ? (mediaMap.get(element.mediaId) ?? null) + : null; + if (!doesElementHaveEnabledAudio({ element, mediaAsset })) continue; + const isElementMuted = "muted" in element ? (element.muted ?? false) : false; const muted = isTrackMuted || isElementMuted; @@ -543,10 +556,7 @@ export async function collectAudioClips({ } if (element.type === "video") { - const mediaAsset = mediaMap.get(element.mediaId); - if (!mediaAsset) continue; - - if (mediaSupportsAudio({ media: mediaAsset })) { + if (mediaAsset && mediaSupportsAudio({ media: mediaAsset })) { clips.push( collectMediaAudioClip({ element, diff --git a/apps/web/src/lib/timeline/audio-separation/__tests__/index.test.ts b/apps/web/src/lib/timeline/audio-separation/__tests__/index.test.ts new file mode 100644 index 00000000..173b6402 --- /dev/null +++ b/apps/web/src/lib/timeline/audio-separation/__tests__/index.test.ts @@ -0,0 +1,147 @@ +import { describe, expect, test } from "bun:test"; +import type { AudioElement, VideoElement } from "@/lib/timeline"; +import { + buildSeparatedAudioElement, + doesElementHaveEnabledAudio, + isSourceAudioEnabled, + isSourceAudioSeparated, +} from ".."; + +describe("audio separation", () => { + test("treats missing source audio state as enabled", () => { + const element = buildVideoElement({}); + + expect(isSourceAudioEnabled({ element })).toBe(true); + expect(isSourceAudioSeparated({ element })).toBe(false); + }); + + test("builds a detached audio element from the source clip audio state", () => { + const element = buildVideoElement({ + duration: 8, + startTime: 3, + trimStart: 1.5, + trimEnd: 0.5, + sourceDuration: 12, + volume: -6, + muted: true, + retime: { rate: 1.25, maintainPitch: true }, + animations: { + channels: { + volume: { + valueKind: "number", + keyframes: [ + { + id: "volume-keyframe", + time: 2, + value: -12, + interpolation: "linear", + }, + ], + }, + opacity: { + valueKind: "number", + keyframes: [ + { + id: "opacity-keyframe", + time: 1, + value: 0.5, + interpolation: "linear", + }, + ], + }, + }, + }, + }); + + const separatedAudioElement = buildSeparatedAudioElement({ + sourceElement: element, + }); + + expect(separatedAudioElement).toMatchObject({ + type: "audio", + sourceType: "upload", + mediaId: element.mediaId, + name: element.name, + duration: 8, + startTime: 3, + trimStart: 1.5, + trimEnd: 0.5, + sourceDuration: 12, + volume: -6, + muted: true, + retime: { rate: 1.25, maintainPitch: true }, + }); + expect(Object.keys(separatedAudioElement.animations?.channels ?? {})).toEqual([ + "volume", + ]); + expect( + separatedAudioElement.animations?.channels.volume?.keyframes[0]?.id, + ).not.toBe("volume-keyframe"); + }); + + test("skips source audio collection when the source clip is separated", () => { + const mediaAsset = { + id: "media-1", + type: "video", + name: "Clip", + size: 1, + lastModified: 1, + file: new File(["video"], "clip.mp4", { type: "video/mp4" }), + url: "blob:clip", + hasAudio: true, + }; + const videoElement = buildVideoElement({ + isSourceAudioEnabled: false, + }); + const audioElement = { + id: "audio-1", + type: "audio", + sourceType: "upload", + mediaId: "audio-media-1", + name: "Detached audio", + duration: 5, + startTime: 0, + trimStart: 0, + trimEnd: 0, + volume: 0, + } as AudioElement; + + expect( + doesElementHaveEnabledAudio({ + element: videoElement, + mediaAsset, + }), + ).toBe(false); + expect( + doesElementHaveEnabledAudio({ + element: audioElement, + }), + ).toBe(true); + }); +}); + +function buildVideoElement( + overrides: Partial, +): VideoElement { + return { + id: "video-1", + type: "video", + mediaId: "media-1", + name: "Clip", + duration: 5, + startTime: 0, + trimStart: 0, + trimEnd: 0, + volume: 0, + muted: false, + isSourceAudioEnabled: true, + transform: { + scaleX: 1, + scaleY: 1, + position: { x: 0, y: 0 }, + rotate: 0, + }, + opacity: 1, + ...overrides, + }; +} diff --git a/apps/web/src/lib/timeline/audio-separation/index.ts b/apps/web/src/lib/timeline/audio-separation/index.ts new file mode 100644 index 00000000..b92cb79b --- /dev/null +++ b/apps/web/src/lib/timeline/audio-separation/index.ts @@ -0,0 +1,133 @@ +import { cloneAnimations, getChannel } from "@/lib/animation"; +import type { ElementAnimations } from "@/lib/animation/types"; +import type { MediaAsset } from "@/lib/media/types"; +import { DEFAULTS } from "@/lib/timeline/defaults"; +import type { + CreateUploadAudioElement, + TimelineElement, + AudioElement, + VideoElement, +} from "../types"; + +export function isSourceAudioEnabled({ + element, +}: { + element: VideoElement; +}): boolean { + return element.isSourceAudioEnabled !== false; +} + +export function isSourceAudioSeparated({ + element, +}: { + element: VideoElement; +}): boolean { + return !isSourceAudioEnabled({ element }); +} + +export function canExtractSourceAudio({ + element, + mediaAsset, +}: { + element: TimelineElement; + mediaAsset: MediaAsset | null | undefined; +}): element is VideoElement { + return ( + element.type === "video" && + isSourceAudioEnabled({ element }) && + !!mediaAsset && + mediaAsset.hasAudio !== false + ); +} + +export function canRecoverSourceAudio({ + element, +}: { + element: TimelineElement; +}): element is VideoElement { + return element.type === "video" && isSourceAudioSeparated({ element }); +} + +export function canToggleSourceAudio({ + element, + mediaAsset, +}: { + element: TimelineElement; + mediaAsset: MediaAsset | null | undefined; +}): element is VideoElement { + return ( + canRecoverSourceAudio({ element }) || + canExtractSourceAudio({ element, mediaAsset }) + ); +} + +export function doesElementHaveEnabledAudio({ + element, + mediaAsset, +}: { + element: AudioElement | VideoElement; + mediaAsset?: MediaAsset | null; +}): boolean { + if (element.type === "audio") { + return true; + } + + return !!mediaAsset && mediaAsset.hasAudio !== false && isSourceAudioEnabled({ element }); +} + +export function buildSeparatedAudioElement({ + sourceElement, +}: { + sourceElement: VideoElement; +}): CreateUploadAudioElement { + return { + type: "audio", + sourceType: "upload", + mediaId: sourceElement.mediaId, + name: sourceElement.name, + duration: sourceElement.duration, + startTime: sourceElement.startTime, + trimStart: sourceElement.trimStart, + trimEnd: sourceElement.trimEnd, + sourceDuration: sourceElement.sourceDuration, + volume: sourceElement.volume ?? DEFAULTS.element.volume, + muted: sourceElement.muted ?? false, + retime: sourceElement.retime + ? { + rate: sourceElement.retime.rate, + maintainPitch: sourceElement.retime.maintainPitch, + } + : undefined, + animations: cloneVolumeAnimations({ + animations: sourceElement.animations, + }), + }; +} + +export function getSourceAudioActionLabel({ + element, +}: { + element: VideoElement; +}): "Extract audio" | "Recover audio" { + return isSourceAudioSeparated({ element }) ? "Recover audio" : "Extract audio"; +} + +function cloneVolumeAnimations({ + animations, +}: { + animations: ElementAnimations | undefined; +}): ElementAnimations | undefined { + const volumeChannel = getChannel({ animations, propertyPath: "volume" }); + if (!volumeChannel) { + return undefined; + } + + return cloneAnimations({ + animations: { + channels: { + volume: volumeChannel, + }, + }, + shouldRegenerateKeyframeIds: true, + }); +} diff --git a/apps/web/src/lib/timeline/element-utils.ts b/apps/web/src/lib/timeline/element-utils.ts index ed7bb174..5820b96d 100644 --- a/apps/web/src/lib/timeline/element-utils.ts +++ b/apps/web/src/lib/timeline/element-utils.ts @@ -242,6 +242,7 @@ function buildVideoElement({ trimEnd: 0, sourceDuration: duration, muted: false, + isSourceAudioEnabled: true, hidden: false, transform: { ...DEFAULTS.element.transform, diff --git a/apps/web/src/lib/timeline/index.ts b/apps/web/src/lib/timeline/index.ts index d4337833..83a6a0f0 100644 --- a/apps/web/src/lib/timeline/index.ts +++ b/apps/web/src/lib/timeline/index.ts @@ -5,6 +5,7 @@ export * from "./drag"; export * from "./track-utils"; export * from "./track-element-update"; export * from "./element-utils"; +export * from "./audio-separation"; export * from "./zoom-utils"; export * from "./ruler-utils"; export * from "./ripple-utils"; diff --git a/apps/web/src/lib/timeline/types.ts b/apps/web/src/lib/timeline/types.ts index 69d363c7..ebdc4c23 100644 --- a/apps/web/src/lib/timeline/types.ts +++ b/apps/web/src/lib/timeline/types.ts @@ -113,6 +113,7 @@ export interface VideoElement extends BaseTimelineElement { mediaId: string; volume?: number; muted?: boolean; + isSourceAudioEnabled?: boolean; hidden?: boolean; retime?: RetimeConfig; transform: Transform; diff --git a/apps/web/src/services/storage/migrations/__tests__/v19-to-v20.test.ts b/apps/web/src/services/storage/migrations/__tests__/v19-to-v20.test.ts new file mode 100644 index 00000000..0e0cfbbe --- /dev/null +++ b/apps/web/src/services/storage/migrations/__tests__/v19-to-v20.test.ts @@ -0,0 +1,141 @@ +import { describe, expect, test } from "bun:test"; +import { transformProjectV19ToV20 } from "../transformers/v19-to-v20"; + +describe("V19 to V20 Migration", () => { + test("backfills source audio enabled state on video elements", () => { + const result = transformProjectV19ToV20({ + project: { + id: "project-v19-source-audio", + version: 19, + metadata: { + id: "project-v19-source-audio", + name: "Project", + createdAt: "2024-01-01T00:00:00.000Z", + updatedAt: "2024-01-01T00:00:00.000Z", + }, + settings: { + fps: 30, + canvasSize: { width: 1920, height: 1080 }, + background: { type: "color", color: "#000000" }, + }, + currentSceneId: "scene-main", + scenes: [ + { + id: "scene-main", + name: "Main", + isMain: true, + bookmarks: [], + createdAt: "2024-01-01T00:00:00.000Z", + updatedAt: "2024-01-01T00:00:00.000Z", + tracks: [ + { + id: "track-video", + type: "video", + name: "Video", + isMain: true, + muted: false, + hidden: false, + elements: [ + { + id: "video-1", + type: "video", + name: "Clip", + mediaId: "media-1", + duration: 5, + startTime: 0, + trimStart: 0, + trimEnd: 0, + transform: { + position: { x: 0, y: 0 }, + scale: { x: 1, y: 1 }, + rotation: 0, + }, + opacity: 1, + }, + ], + }, + { + id: "track-audio", + type: "audio", + name: "Audio", + muted: false, + elements: [ + { + id: "audio-1", + type: "audio", + sourceType: "upload", + mediaId: "media-audio-1", + name: "Audio", + duration: 5, + startTime: 0, + trimStart: 0, + trimEnd: 0, + volume: 0, + }, + ], + }, + ], + }, + ], + }, + }); + + expect(result.skipped).toBe(false); + expect(result.project.version).toBe(20); + expect( + ((result.project.scenes as Array>)[0] + .tracks as Array>)[0].elements, + ).toEqual([ + expect.objectContaining({ + id: "video-1", + isSourceAudioEnabled: true, + }), + ]); + expect( + (((result.project.scenes as Array>)[0] + .tracks as Array>)[1] + .elements as Array>)[0].isSourceAudioEnabled, + ).toBeUndefined(); + }); + + test("preserves existing explicit source audio state", () => { + const result = transformProjectV19ToV20({ + project: { + id: "project-v19-existing-state", + version: 19, + scenes: [ + { + tracks: [ + { + elements: [ + { + type: "video", + isSourceAudioEnabled: false, + }, + ], + }, + ], + }, + ], + }, + }); + + expect( + (((result.project.scenes as Array>)[0] + .tracks as Array>)[0] + .elements as Array>)[0].isSourceAudioEnabled, + ).toBe(false); + }); + + test("skips projects already on v20", () => { + const result = transformProjectV19ToV20({ + project: { + id: "project-v20", + version: 20, + }, + }); + + expect(result.skipped).toBe(true); + expect(result.reason).toBe("already v20"); + }); +}); diff --git a/apps/web/src/services/storage/migrations/index.ts b/apps/web/src/services/storage/migrations/index.ts index e87d1cb9..d278263b 100644 --- a/apps/web/src/services/storage/migrations/index.ts +++ b/apps/web/src/services/storage/migrations/index.ts @@ -18,10 +18,11 @@ import { V15toV16Migration } from "./v15-to-v16"; import { V16toV17Migration } from "./v16-to-v17"; import { V17toV18Migration } from "./v17-to-v18"; import { V18toV19Migration } from "./v18-to-v19"; +import { V19toV20Migration } from "./v19-to-v20"; export { runStorageMigrations } from "./runner"; export type { MigrationProgress } from "./runner"; -export const CURRENT_PROJECT_VERSION = 19; +export const CURRENT_PROJECT_VERSION = 20; export const migrations = [ new V0toV1Migration(), @@ -43,4 +44,5 @@ export const migrations = [ new V16toV17Migration(), new V17toV18Migration(), new V18toV19Migration(), + new V19toV20Migration(), ]; diff --git a/apps/web/src/services/storage/migrations/transformers/v19-to-v20.ts b/apps/web/src/services/storage/migrations/transformers/v19-to-v20.ts new file mode 100644 index 00000000..f72812a4 --- /dev/null +++ b/apps/web/src/services/storage/migrations/transformers/v19-to-v20.ts @@ -0,0 +1,87 @@ +import type { MigrationResult, ProjectRecord } from "./types"; +import { getProjectId, isRecord } from "./utils"; + +export function transformProjectV19ToV20({ + project, +}: { + project: ProjectRecord; +}): MigrationResult { + if (!getProjectId({ project })) { + return { project, skipped: true, reason: "no project id" }; + } + + if (typeof project.version === "number" && project.version >= 20) { + return { project, skipped: true, reason: "already v20" }; + } + + return { + project: { + ...migrateVideoSourceAudioState({ project }), + version: 20, + }, + skipped: false, + }; +} + +function migrateVideoSourceAudioState({ + project, +}: { + project: ProjectRecord; +}): ProjectRecord { + if (!Array.isArray(project.scenes)) { + return project; + } + + return { + ...project, + scenes: project.scenes.map((scene) => migrateScene({ scene })), + }; +} + +function migrateScene({ + scene, +}: { + scene: unknown; +}): unknown { + if (!isRecord(scene) || !Array.isArray(scene.tracks)) { + return scene; + } + + return { + ...scene, + tracks: scene.tracks.map((track) => migrateTrack({ track })), + }; +} + +function migrateTrack({ + track, +}: { + track: unknown; +}): unknown { + if (!isRecord(track) || !Array.isArray(track.elements)) { + return track; + } + + return { + ...track, + elements: track.elements.map((element) => migrateElement({ element })), + }; +} + +function migrateElement({ + element, +}: { + element: unknown; +}): unknown { + if (!isRecord(element) || element.type !== "video") { + return element; + } + + return { + ...element, + isSourceAudioEnabled: + typeof element.isSourceAudioEnabled === "boolean" + ? element.isSourceAudioEnabled + : true, + }; +} diff --git a/apps/web/src/services/storage/migrations/v19-to-v20.ts b/apps/web/src/services/storage/migrations/v19-to-v20.ts new file mode 100644 index 00000000..c6809067 --- /dev/null +++ b/apps/web/src/services/storage/migrations/v19-to-v20.ts @@ -0,0 +1,16 @@ +import { StorageMigration } from "./base"; +import type { ProjectRecord } from "./transformers/types"; +import { transformProjectV19ToV20 } from "./transformers/v19-to-v20"; + +export class V19toV20Migration extends StorageMigration { + from = 19; + to = 20; + + async transform(project: ProjectRecord): Promise<{ + project: ProjectRecord; + skipped: boolean; + reason?: string; + }> { + return transformProjectV19ToV20({ project }); + } +}