From e4b67094e757917588fb624a30aa046667403add Mon Sep 17 00:00:00 2001 From: Maze Winther Date: Tue, 7 Apr 2026 01:09:13 +0200 Subject: [PATCH] refactor: split effects and masks into dedicated rust crates, introduce MediaTime and FrameRate --- Cargo.lock | 26 +- Cargo.toml | 5 +- apps/web/content/changelog/0.3.0.md | 2 + apps/web/package.json | 2 +- apps/web/src/app/projects/page.tsx | 7 +- apps/web/src/components/editable-timecode.tsx | 15 +- .../editor/dialogs/project-info-dialog.tsx | 5 +- .../editor/panels/assets/views/assets.tsx | 4 +- .../panels/assets/views/settings/index.tsx | 13 +- .../editor/panels/preview/index.tsx | 11 +- .../editor/panels/preview/toolbar.tsx | 4 +- .../properties/hooks/use-element-playhead.ts | 45 +- .../panels/properties/tabs/masks-tab.tsx | 3 +- .../editor/panels/timeline/bookmarks.tsx | 9 +- .../panels/timeline/selection-hit-testing.ts | 7 +- .../panels/timeline/timeline-playhead.tsx | 6 +- .../editor/panels/timeline/timeline-ruler.tsx | 31 +- .../editor/panels/timeline/timeline-tick.tsx | 81 +- apps/web/src/constants/animation-constants.ts | 3 +- apps/web/src/core/managers/audio-manager.ts | 1417 +++++++++-------- .../web/src/core/managers/playback-manager.ts | 29 +- apps/web/src/core/managers/project-manager.ts | 2 +- .../web/src/core/managers/renderer-manager.ts | 12 +- .../web/src/core/managers/timeline-manager.ts | 6 +- .../src/hooks/actions/use-editor-actions.ts | 7 +- .../element/use-element-interaction.ts | 30 +- .../timeline/element/use-element-resize.ts | 42 +- .../timeline/element/use-keyframe-drag.ts | 11 +- .../src/hooks/timeline/use-bookmark-drag.ts | 10 +- .../hooks/timeline/use-timeline-drag-drop.ts | 16 +- .../hooks/timeline/use-timeline-playhead.ts | 42 +- .../src/hooks/timeline/use-timeline-seek.ts | 27 +- .../src/hooks/timeline/use-timeline-zoom.ts | 8 +- apps/web/src/hooks/use-paste-media.ts | 228 +-- apps/web/src/lib/animation/curve-bridge.ts | 5 +- apps/web/src/lib/animation/interpolation.ts | 26 +- apps/web/src/lib/animation/keyframe-query.ts | 8 +- apps/web/src/lib/animation/keyframes.ts | 7 +- .../src/lib/commands/media/add-media-asset.ts | 12 +- .../timeline/element/insert-element.ts | 529 +++--- apps/web/src/lib/export.ts | 3 +- apps/web/src/lib/fps/__tests__/fps.test.ts | 8 +- apps/web/src/lib/fps/constants.ts | 4 +- apps/web/src/lib/fps/utils.ts | 64 +- apps/web/src/lib/media/audio.ts | 57 +- apps/web/src/lib/project/types.ts | 107 +- apps/web/src/lib/ripple/diff.ts | 20 +- apps/web/src/lib/timeline/audio-state.ts | 202 +-- apps/web/src/lib/timeline/bookmarks.ts | 14 +- apps/web/src/lib/timeline/creation.ts | 4 +- apps/web/src/lib/timeline/defaults.ts | 154 +- apps/web/src/lib/timeline/drag-utils.ts | 40 +- apps/web/src/lib/timeline/element-utils.ts | 836 +++++----- apps/web/src/lib/timeline/pixel-utils.ts | 159 +- apps/web/src/lib/timeline/ruler-utils.ts | 17 +- apps/web/src/lib/timeline/snap-utils.ts | 344 ++-- apps/web/src/lib/timeline/update-pipeline.ts | 2 +- apps/web/src/lib/timeline/zoom-utils.ts | 5 +- apps/web/src/lib/wasm/constants.ts | 3 + apps/web/src/lib/wasm/index.ts | 1 + .../src/services/renderer/canvas-renderer.ts | 6 +- .../renderer/nodes/blur-background-node.ts | 9 +- .../src/services/renderer/nodes/video-node.ts | 76 +- .../services/renderer/nodes/visual-node.ts | 3 +- .../src/services/renderer/scene-exporter.ts | 333 ++-- .../migrations/__tests__/v1-to-v2.test.ts | 2 +- .../migrations/__tests__/v22-to-v23.test.ts | 192 +++ .../src/services/storage/migrations/index.ts | 4 +- .../migrations/transformers/v1-to-v2.ts | 2 +- .../migrations/transformers/v22-to-v23.ts | 335 ++++ .../services/storage/migrations/v22-to-v23.ts | 16 + bun.lock | 4 +- rust/crates/bridge/src/bridge.rs | 50 +- rust/crates/effects/Cargo.toml | 14 + rust/crates/effects/src/effects.rs | 5 + rust/crates/effects/src/pipeline.rs | 306 ++++ .../src/shaders/gaussian_blur.wgsl | 0 rust/crates/effects/src/types.rs | 13 + rust/crates/gpu/Cargo.toml | 4 + rust/crates/gpu/src/context.rs | 163 +- rust/crates/gpu/src/effect_pipeline.rs | 194 --- rust/crates/gpu/src/lib.rs | 23 +- rust/crates/gpu/src/shader_registry.rs | 183 --- rust/crates/masks/Cargo.toml | 13 + .../mask_feather.rs => masks/src/feather.rs} | 50 +- rust/crates/masks/src/masks.rs | 5 + .../src/sdf_pipeline.rs => masks/src/sdf.rs} | 33 +- .../src/shaders/jfa_distance.wgsl | 0 .../{gpu => masks}/src/shaders/jfa_init.wgsl | 0 .../{gpu => masks}/src/shaders/jfa_step.wgsl | 0 rust/crates/time/Cargo.toml | 5 +- rust/crates/time/src/frame_rate.rs | 121 ++ rust/crates/time/src/media_time.rs | 428 +++++ rust/crates/time/src/time.rs | 439 +---- rust/crates/time/src/timecode.rs | 287 ++++ rust/wasm/Cargo.toml | 12 +- rust/wasm/README.md | 5 +- rust/wasm/src/effects.rs | 101 ++ rust/wasm/src/gpu.rs | 123 ++ rust/wasm/src/gpu_bridge.rs | 256 --- rust/wasm/src/masks.rs | 60 + rust/wasm/src/wasm.rs | 12 +- 102 files changed, 4977 insertions(+), 3707 deletions(-) create mode 100644 apps/web/src/lib/wasm/constants.ts create mode 100644 apps/web/src/lib/wasm/index.ts create mode 100644 apps/web/src/services/storage/migrations/__tests__/v22-to-v23.test.ts create mode 100644 apps/web/src/services/storage/migrations/transformers/v22-to-v23.ts create mode 100644 apps/web/src/services/storage/migrations/v22-to-v23.ts create mode 100644 rust/crates/effects/Cargo.toml create mode 100644 rust/crates/effects/src/effects.rs create mode 100644 rust/crates/effects/src/pipeline.rs rename rust/crates/{gpu => effects}/src/shaders/gaussian_blur.wgsl (100%) create mode 100644 rust/crates/effects/src/types.rs delete mode 100644 rust/crates/gpu/src/effect_pipeline.rs delete mode 100644 rust/crates/gpu/src/shader_registry.rs create mode 100644 rust/crates/masks/Cargo.toml rename rust/crates/{gpu/src/mask_feather.rs => masks/src/feather.rs} (88%) create mode 100644 rust/crates/masks/src/masks.rs rename rust/crates/{gpu/src/sdf_pipeline.rs => masks/src/sdf.rs} (93%) rename rust/crates/{gpu => masks}/src/shaders/jfa_distance.wgsl (100%) rename rust/crates/{gpu => masks}/src/shaders/jfa_init.wgsl (100%) rename rust/crates/{gpu => masks}/src/shaders/jfa_step.wgsl (100%) create mode 100644 rust/crates/time/src/frame_rate.rs create mode 100644 rust/crates/time/src/media_time.rs create mode 100644 rust/crates/time/src/timecode.rs create mode 100644 rust/wasm/src/effects.rs create mode 100644 rust/wasm/src/gpu.rs delete mode 100644 rust/wasm/src/gpu_bridge.rs create mode 100644 rust/wasm/src/masks.rs diff --git a/Cargo.lock b/Cargo.lock index d2a3fd4c..b673fb3e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1427,6 +1427,16 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +[[package]] +name = "effects" +version = "0.1.0" +dependencies = [ + "bytemuck", + "gpu", + "thiserror 2.0.18", + "wgpu", +] + [[package]] name = "either" version = "1.15.0" @@ -3217,6 +3227,15 @@ dependencies = [ "libc", ] +[[package]] +name = "masks" +version = "0.1.0" +dependencies = [ + "bytemuck", + "gpu", + "wgpu", +] + [[package]] name = "maybe-rayon" version = "0.1.1" @@ -3777,10 +3796,14 @@ dependencies = [ [[package]] name = "opencut-wasm" -version = "0.1.3" +version = "0.2.3" dependencies = [ + "bridge", + "effects", "gpu", "js-sys", + "masks", + "num-traits", "serde", "serde-wasm-bindgen", "time", @@ -5581,6 +5604,7 @@ name = "time" version = "0.1.0" dependencies = [ "bridge", + "num-traits", "serde", "tsify-next", "wasm-bindgen", diff --git a/Cargo.toml b/Cargo.toml index 2b63476b..6c80c54f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,5 +4,8 @@ members = [ "apps/desktop", "rust/crates/time", "rust/crates/bridge", - "rust/wasm", "rust/crates/gpu", + "rust/crates/effects", + "rust/crates/gpu", + "rust/crates/masks", + "rust/wasm", ] diff --git a/apps/web/content/changelog/0.3.0.md b/apps/web/content/changelog/0.3.0.md index 42c7c9a7..92675321 100644 --- a/apps/web/content/changelog/0.3.0.md +++ b/apps/web/content/changelog/0.3.0.md @@ -79,5 +79,7 @@ changes: text: "Auto-generated captions were sometimes inaccurate. The underlying issue has been fixed." - type: new text: "You can now import transcript files to generate captions instead of running auto-transcription." + - type: new + text: "Graph editor for keyframe curves. Shape the easing between keyframes by dragging bezier handles." --- diff --git a/apps/web/package.json b/apps/web/package.json index 0317d3b9..7ee5fe1a 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -52,7 +52,7 @@ "nanoid": "^5.1.5", "next": "16.1.3", "next-themes": "^0.4.4", - "opencut-wasm": "^0.1.3", + "opencut-wasm": "^0.2.3", "pg": "^8.16.2", "postgres": "^3.4.5", "radix-ui": "^1.4.3", diff --git a/apps/web/src/app/projects/page.tsx b/apps/web/src/app/projects/page.tsx index 8ee9d504..93d6e189 100644 --- a/apps/web/src/app/projects/page.tsx +++ b/apps/web/src/app/projects/page.tsx @@ -21,7 +21,7 @@ import type { TProjectSortKey, TProjectSortOption, } from "@/lib/project/types"; -import { formatTimeCode } from "opencut-wasm"; +import { formatTimecode, mediaTimeToSeconds } from "opencut-wasm"; import { formatDate } from "@/utils/date"; import { HugeiconsIcon } from "@hugeicons/react"; import { @@ -76,8 +76,9 @@ const formatProjectDuration = ({ return null; } - const format = duration >= 3600 ? "HH:MM:SS" : "MM:SS"; - return formatTimeCode({ timeInSeconds: duration, format }) ?? ""; + const durationSeconds = mediaTimeToSeconds({ time: duration }); + const format = durationSeconds >= 3600 ? "HH:MM:SS" : "MM:SS"; + return formatTimecode({ time: duration, format }) ?? ""; }; const VIEW_MODE_OPTIONS = [ diff --git a/apps/web/src/components/editable-timecode.tsx b/apps/web/src/components/editable-timecode.tsx index 6a323a1c..276cb490 100644 --- a/apps/web/src/components/editable-timecode.tsx +++ b/apps/web/src/components/editable-timecode.tsx @@ -1,14 +1,14 @@ "use client"; import { useEffect, useRef, useState } from "react"; -import { formatTimeCode, parseTimeCode, type TimeCodeFormat } from "opencut-wasm"; +import { formatTimecode, parseTimecode, snappedSeekTime, type FrameRate, type TimeCodeFormat } from "opencut-wasm"; import { cn } from "@/utils/ui"; interface EditableTimecodeProps { time: number; duration: number; format?: TimeCodeFormat; - fps: number; + fps: FrameRate; onTimeChange?: ({ time }: { time: number }) => void; className?: string; disabled?: boolean; @@ -28,7 +28,7 @@ export function EditableTimecode({ const [hasError, setHasError] = useState(false); const inputRef = useRef(null); const enterPressedRef = useRef(false); - const formattedTime = formatTimeCode({ timeInSeconds: time, format, fps }) ?? ""; + const formattedTime = formatTimecode({ time, format, rate: fps }) ?? ""; const startEditing = () => { if (disabled) return; @@ -46,17 +46,16 @@ export function EditableTimecode({ }; const applyEdit = () => { - const parsedTime = parseTimeCode({ timeCode: inputValue, format, fps }); + const parsedTime = parseTimecode({ timeCode: inputValue, format, rate: fps }); if (parsedTime == null) { setHasError(true); return; } - const clampedTime = Math.max( - 0, - duration ? Math.min(duration, parsedTime) : parsedTime, - ); + const clampedTime = duration + ? (snappedSeekTime({ time: parsedTime, duration, rate: fps }) ?? parsedTime) + : parsedTime; onTimeChange?.({ time: clampedTime }); setIsEditing(false); diff --git a/apps/web/src/components/editor/dialogs/project-info-dialog.tsx b/apps/web/src/components/editor/dialogs/project-info-dialog.tsx index c31c8e96..ed4eb990 100644 --- a/apps/web/src/components/editor/dialogs/project-info-dialog.tsx +++ b/apps/web/src/components/editor/dialogs/project-info-dialog.tsx @@ -8,7 +8,7 @@ import { } from "@/components/ui/dialog"; import type { TProjectMetadata } from "@/lib/project/types"; import { formatDate } from "@/utils/date"; -import { formatTimeCode } from "opencut-wasm"; +import { formatTimecode, mediaTimeToSeconds } from "opencut-wasm"; import { Button } from "@/components/ui/button"; function InfoRow({ @@ -35,9 +35,10 @@ export function ProjectInfoDialog({ onOpenChange: (open: boolean) => void; project: TProjectMetadata; }) { + const durationSeconds = mediaTimeToSeconds({ time: project.duration }); const durationFormatted = project.duration > 0 - ? (formatTimeCode({ timeInSeconds: project.duration, format: project.duration >= 3600 ? "HH:MM:SS" : "MM:SS" }) ?? "") + ? (formatTimecode({ time: project.duration, format: durationSeconds >= 3600 ? "HH:MM:SS" : "MM:SS" }) ?? "") : "0:00"; return ( diff --git a/apps/web/src/components/editor/panels/assets/views/assets.tsx b/apps/web/src/components/editor/panels/assets/views/assets.tsx index 031d2e6a..85888dce 100644 --- a/apps/web/src/components/editor/panels/assets/views/assets.tsx +++ b/apps/web/src/components/editor/panels/assets/views/assets.tsx @@ -25,7 +25,7 @@ import { TooltipProvider, TooltipTrigger, } from "@/components/ui/tooltip"; -import { DEFAULT_NEW_ELEMENT_DURATION_SECONDS } from "@/lib/timeline/creation"; +import { DEFAULT_NEW_ELEMENT_DURATION } from "@/lib/timeline/creation"; import { useEditor } from "@/hooks/use-editor"; import { useFileUpload } from "@/hooks/use-file-upload"; import { invokeAction } from "@/lib/actions"; @@ -257,7 +257,7 @@ function MediaAssetDraggable({ startTime: number; }) => { const duration = - asset.duration ?? DEFAULT_NEW_ELEMENT_DURATION_SECONDS; + asset.duration ?? DEFAULT_NEW_ELEMENT_DURATION; const element = buildElementFromMedia({ mediaId: asset.id, mediaType: asset.type, diff --git a/apps/web/src/components/editor/panels/assets/views/settings/index.tsx b/apps/web/src/components/editor/panels/assets/views/settings/index.tsx index 6c819a36..16faad3d 100644 --- a/apps/web/src/components/editor/panels/assets/views/settings/index.tsx +++ b/apps/web/src/components/editor/panels/assets/views/settings/index.tsx @@ -10,6 +10,7 @@ import { SelectValue, } from "@/components/ui/select"; import { FPS_PRESETS } from "@/lib/fps/constants"; +import { floatToFrameRate, frameRateToFloat } from "@/lib/fps/utils"; import { useEditor } from "@/hooks/use-editor"; import { Section, @@ -232,12 +233,12 @@ export function SettingsView() {
Frame rate - { + const fps = floatToFrameRate(parseFloat(value)); + editor.project.updateSettings({ settings: { fps } }); + }} > diff --git a/apps/web/src/components/editor/panels/preview/index.tsx b/apps/web/src/components/editor/panels/preview/index.tsx index 8cfe7f49..786d9a46 100644 --- a/apps/web/src/components/editor/panels/preview/index.tsx +++ b/apps/web/src/components/editor/panels/preview/index.tsx @@ -7,6 +7,7 @@ import { useRafLoop } from "@/hooks/use-raf-loop"; import { useContainerSize } from "@/hooks/use-container-size"; import { useFullscreen } from "@/hooks/use-fullscreen"; import { CanvasRenderer } from "@/services/renderer/canvas-renderer"; +import { TICKS_PER_SECOND } from "@/lib/wasm"; import type { RootNode } from "@/services/renderer/nodes/root-node"; import { buildScene } from "@/services/renderer/scene-builder"; import { PreviewInteractionOverlay } from "./preview-interaction-overlay"; @@ -129,12 +130,16 @@ function PreviewCanvas({ height: nativeHeight, fps: activeProject.settings.fps, }); - }, [nativeWidth, nativeHeight, activeProject.settings.fps]); + }, [nativeWidth, nativeHeight, activeProject.settings.fps.numerator, activeProject.settings.fps.denominator]); const render = useCallback(() => { if (canvasRef.current && renderTree && !renderingRef.current) { - const renderTime = editor.playback.getCurrentTime(); - const frame = Math.floor(renderTime * renderer.fps); + const renderTime = Math.min( + editor.playback.getCurrentTime(), + editor.timeline.getLastFrameTime(), + ); + const ticksPerFrame = Math.round(TICKS_PER_SECOND * renderer.fps.denominator / renderer.fps.numerator); + const frame = Math.floor(renderTime / ticksPerFrame); if ( frame !== lastFrameRef.current || diff --git a/apps/web/src/components/editor/panels/preview/toolbar.tsx b/apps/web/src/components/editor/panels/preview/toolbar.tsx index 0be1ec46..555cde17 100644 --- a/apps/web/src/components/editor/panels/preview/toolbar.tsx +++ b/apps/web/src/components/editor/panels/preview/toolbar.tsx @@ -2,7 +2,7 @@ import { useState, useEffect } from "react"; import { useEditor } from "@/hooks/use-editor"; -import { formatTimeCode } from "opencut-wasm"; +import { formatTimecode } from "opencut-wasm"; import { invokeAction } from "@/lib/actions"; import { EditableTimecode } from "@/components/editable-timecode"; import { Button } from "@/components/ui/button"; @@ -93,7 +93,7 @@ function TimecodeDisplay() { /> / - {formatTimeCode({ timeInSeconds: totalDuration, format: "HH:MM:SS:FF", fps })} + {formatTimecode({ time: totalDuration, format: "HH:MM:SS:FF", rate: fps })} ); diff --git a/apps/web/src/components/editor/panels/properties/hooks/use-element-playhead.ts b/apps/web/src/components/editor/panels/properties/hooks/use-element-playhead.ts index a900cb13..27300784 100644 --- a/apps/web/src/components/editor/panels/properties/hooks/use-element-playhead.ts +++ b/apps/web/src/components/editor/panels/properties/hooks/use-element-playhead.ts @@ -1,23 +1,22 @@ -import { useEditor } from "@/hooks/use-editor"; -import { getElementLocalTime } from "@/lib/animation"; -import { TIME_EPSILON_SECONDS } from "@/constants/animation-constants"; - -export function useElementPlayhead({ - startTime, - duration, -}: { - startTime: number; - duration: number; -}) { - const playheadTime = useEditor((editor) => editor.playback.getCurrentTime()); - const localTime = getElementLocalTime({ - timelineTime: playheadTime, - elementStartTime: startTime, - elementDuration: duration, - }); - const isPlayheadWithinElementRange = - playheadTime >= startTime - TIME_EPSILON_SECONDS && - playheadTime <= startTime + duration + TIME_EPSILON_SECONDS; - - return { localTime, isPlayheadWithinElementRange }; -} +import { useEditor } from "@/hooks/use-editor"; +import { getElementLocalTime } from "@/lib/animation"; + +export function useElementPlayhead({ + startTime, + duration, +}: { + startTime: number; + duration: number; +}) { + const playheadTime = useEditor((editor) => editor.playback.getCurrentTime()); + const localTime = getElementLocalTime({ + timelineTime: playheadTime, + elementStartTime: startTime, + elementDuration: duration, + }); + const isPlayheadWithinElementRange = + playheadTime >= startTime && + playheadTime <= startTime + duration; + + return { localTime, isPlayheadWithinElementRange }; +} diff --git a/apps/web/src/components/editor/panels/properties/tabs/masks-tab.tsx b/apps/web/src/components/editor/panels/properties/tabs/masks-tab.tsx index f48a50cb..9b0ffc88 100644 --- a/apps/web/src/components/editor/panels/properties/tabs/masks-tab.tsx +++ b/apps/web/src/components/editor/panels/properties/tabs/masks-tab.tsx @@ -15,7 +15,6 @@ import { PlusSignIcon, RotateClockwiseIcon, } from "@hugeicons/core-free-icons"; -import { TIME_EPSILON_SECONDS } from "@/constants/animation-constants"; import { useMemo, useState } from "react"; import { Button } from "@/components/ui/button"; import { ColorPicker } from "@/components/ui/color-picker"; @@ -102,7 +101,7 @@ export function MasksTab({ element, trackId }: MasksTabProps) { const elementBounds = useMemo(() => { const clampedTime = Math.min( Math.max(currentTime, element.startTime), - element.startTime + element.duration - TIME_EPSILON_SECONDS, + element.startTime + element.duration - 1, ); return ( diff --git a/apps/web/src/components/editor/panels/timeline/bookmarks.tsx b/apps/web/src/components/editor/panels/timeline/bookmarks.tsx index d4b1657e..9342c30e 100644 --- a/apps/web/src/components/editor/panels/timeline/bookmarks.tsx +++ b/apps/web/src/components/editor/panels/timeline/bookmarks.tsx @@ -4,13 +4,12 @@ import { useEffect, useState } from "react"; import type { EditorCore } from "@/core"; import { useEditor } from "@/hooks/use-editor"; import type { BookmarkDragState } from "@/hooks/timeline/use-bookmark-drag"; -import { BOOKMARK_TIME_EPSILON } from "@/lib/timeline/bookmarks"; import { DEFAULT_TIMELINE_BOOKMARK_COLOR, } from "./theme"; import { TIMELINE_BOOKMARK_ROW_HEIGHT_PX } from "./layout"; import { DEFAULT_FPS } from "@/lib/fps/constants"; -import { getSnappedSeekTime } from "opencut-wasm"; +import { snappedSeekTime } from "opencut-wasm"; import { ArrowTurnBackwardIcon, Delete02Icon, @@ -49,8 +48,8 @@ function seekToBookmarkTime({ }) { const activeProject = editor.project.getActive(); const duration = editor.timeline.getTotalDuration(); - const fps = activeProject?.settings.fps ?? DEFAULT_FPS; - const snappedTime = getSnappedSeekTime({ rawTime: time, duration, fps }); + const rate = activeProject?.settings.fps ?? DEFAULT_FPS; + const snappedTime = snappedSeekTime({ time, duration, rate }) ?? time; editor.playback.seek({ time: snappedTime }); } @@ -139,7 +138,7 @@ function TimelineBookmark({ const isDragging = dragState.isDragging && dragState.bookmarkTime !== null && - Math.abs(dragState.bookmarkTime - bookmark.time) < BOOKMARK_TIME_EPSILON; + dragState.bookmarkTime === bookmark.time; const displayTime = isDragging ? dragState.currentTime : bookmark.time; const time = bookmark.time; diff --git a/apps/web/src/components/editor/panels/timeline/selection-hit-testing.ts b/apps/web/src/components/editor/panels/timeline/selection-hit-testing.ts index 6488e29a..551393e9 100644 --- a/apps/web/src/components/editor/panels/timeline/selection-hit-testing.ts +++ b/apps/web/src/components/editor/panels/timeline/selection-hit-testing.ts @@ -1,5 +1,5 @@ import type { TimelineTrack } from "@/lib/timeline"; -import { getTimelinePixelsPerSecond } from "@/lib/timeline/pixel-utils"; +import { timelineTimeToPixels } from "@/lib/timeline/pixel-utils"; import { TIMELINE_CONTENT_TOP_PADDING_PX, } from "./layout"; @@ -96,7 +96,6 @@ export function resolveTimelineElementIntersections({ startPos, endPos: currentPos, }); - const pixelsPerSecond = getTimelinePixelsPerSecond({ zoomLevel }); const selectedElements: TimelineElementRef[] = []; for (const [trackIndex, track] of tracks.entries()) { @@ -109,8 +108,8 @@ export function resolveTimelineElementIntersections({ const elementBottom = elementTop + trackHeight; for (const element of track.elements) { - const elementLeft = element.startTime * pixelsPerSecond; - const elementRight = elementLeft + element.duration * pixelsPerSecond; + const elementLeft = timelineTimeToPixels({ time: element.startTime, zoomLevel }); + const elementRight = timelineTimeToPixels({ time: element.startTime + element.duration, zoomLevel }); const elementRectangle = { left: elementLeft, top: elementTop, diff --git a/apps/web/src/components/editor/panels/timeline/timeline-playhead.tsx b/apps/web/src/components/editor/panels/timeline/timeline-playhead.tsx index 361f1398..d9218371 100644 --- a/apps/web/src/components/editor/panels/timeline/timeline-playhead.tsx +++ b/apps/web/src/components/editor/panels/timeline/timeline-playhead.tsx @@ -7,6 +7,7 @@ import { timelineTimeToSnappedPixels, } from "@/lib/timeline"; import { useTimelinePlayhead } from "@/hooks/timeline/use-timeline-playhead"; +import { TICKS_PER_SECOND } from "@/lib/wasm"; import { useEditor } from "@/hooks/use-editor"; import { TIMELINE_SCROLLBAR_SIZE_PX, @@ -72,10 +73,11 @@ export function TimelinePlayhead({ if (event.key !== "ArrowLeft" && event.key !== "ArrowRight") return; event.preventDefault(); - const step = 1 / Math.max(1, editor.project.getActive().settings.fps); + const fps = editor.project.getActive().settings.fps; + const ticksPerFrame = Math.round(TICKS_PER_SECOND * fps.denominator / fps.numerator); const direction = event.key === "ArrowRight" ? 1 : -1; const now = editor.playback.getCurrentTime(); - const nextTime = Math.max(0, Math.min(duration, now + direction * step)); + const nextTime = Math.max(0, Math.min(duration, now + direction * ticksPerFrame)); editor.playback.seek({ time: nextTime }); }; diff --git a/apps/web/src/components/editor/panels/timeline/timeline-ruler.tsx b/apps/web/src/components/editor/panels/timeline/timeline-ruler.tsx index 9e49bf6c..8b8dab32 100644 --- a/apps/web/src/components/editor/panels/timeline/timeline-ruler.tsx +++ b/apps/web/src/components/editor/panels/timeline/timeline-ruler.tsx @@ -2,6 +2,8 @@ import { type JSX, useLayoutEffect, useRef } from "react"; import { BASE_TIMELINE_PIXELS_PER_SECOND, } from "@/lib/timeline/scale"; +import { mediaTimeToSeconds } from "opencut-wasm"; +import { TICKS_PER_SECOND } from "@/lib/wasm"; import { TIMELINE_RULER_HEIGHT_PX } from "./layout"; import { DEFAULT_FPS } from "@/lib/fps/constants"; import { useEditor } from "@/hooks/use-editor"; @@ -30,17 +32,18 @@ export function TimelineRuler({ handleRulerTrackingMouseDown, handleRulerMouseDown, }: TimelineRulerProps) { - const duration = useEditor((e) => e.timeline.getTotalDuration()); + const durationTicks = useEditor((e) => e.timeline.getTotalDuration()); + const durationSeconds = mediaTimeToSeconds({ time: durationTicks }); const pixelsPerSecond = BASE_TIMELINE_PIXELS_PER_SECOND * zoomLevel; - const visibleDuration = dynamicTimelineWidth / pixelsPerSecond; - const effectiveDuration = Math.max(duration, visibleDuration); + const visibleDurationSeconds = dynamicTimelineWidth / pixelsPerSecond; + const effectiveDurationSeconds = Math.max(durationSeconds, visibleDurationSeconds); const fps = useEditor((e) => e.project.getActiveOrNull()?.settings.fps) ?? DEFAULT_FPS; const { labelIntervalSeconds, tickIntervalSeconds } = getRulerConfig({ zoomLevel, fps, }); - const tickCount = Math.ceil(effectiveDuration / tickIntervalSeconds) + 1; + const tickCount = Math.ceil(effectiveDurationSeconds / tickIntervalSeconds) + 1; const { scrollLeft, viewportWidth } = useScrollPosition({ scrollRef: tracksScrollRef, @@ -61,20 +64,20 @@ export function TimelineRuler({ prevZoomRef.current = zoomLevel; }, [zoomLevel]); - const visibleStartTime = Math.max( + const visibleStartTimeSeconds = Math.max( 0, (scrollLeft - bufferPx) / pixelsPerSecond, ); - const visibleEndTime = + const visibleEndTimeSeconds = (scrollLeft + viewportWidth + bufferPx) / pixelsPerSecond; const startTickIndex = Math.max( 0, - Math.floor(visibleStartTime / tickIntervalSeconds), + Math.floor(visibleStartTimeSeconds / tickIntervalSeconds), ); const endTickIndex = Math.min( tickCount - 1, - Math.ceil(visibleEndTime / tickIntervalSeconds), + Math.ceil(visibleEndTimeSeconds / tickIntervalSeconds), ); const timelineTicks: Array = []; @@ -83,14 +86,16 @@ export function TimelineRuler({ tickIndex <= endTickIndex; tickIndex += 1 ) { - const time = tickIndex * tickIntervalSeconds; - if (time > effectiveDuration) break; + const timeSeconds = tickIndex * tickIntervalSeconds; + if (timeSeconds > effectiveDurationSeconds) break; - const showLabel = shouldShowLabel({ time, labelIntervalSeconds }); + const timeTicks = Math.round(timeSeconds * TICKS_PER_SECOND); + const showLabel = shouldShowLabel({ time: timeSeconds, labelIntervalSeconds }); timelineTicks.push( - {label} - - ); - } - - return ( -
- ); -} +"use client"; + +import type { FrameRate } from "opencut-wasm"; +import { timelineTimeToSnappedPixels } from "@/lib/timeline"; +import { formatRulerLabel } from "@/lib/timeline/ruler-utils"; + +interface TimelineTickProps { + time: number; + timeInSeconds: number; + zoomLevel: number; + fps: FrameRate; + showLabel: boolean; +} + +export function TimelineTick({ + time, + timeInSeconds, + zoomLevel, + fps, + showLabel, +}: TimelineTickProps) { + const leftPosition = timelineTimeToSnappedPixels({ time, zoomLevel }); + + if (showLabel) { + const label = formatRulerLabel({ timeInSeconds, fps }); + return ( + + {label} + + ); + } + + return ( +
+ ); +} diff --git a/apps/web/src/constants/animation-constants.ts b/apps/web/src/constants/animation-constants.ts index 17056114..ab932ea2 100644 --- a/apps/web/src/constants/animation-constants.ts +++ b/apps/web/src/constants/animation-constants.ts @@ -1,2 +1 @@ -export const TIME_EPSILON_SECONDS = 1 / 1000; -export const MIN_TRANSFORM_SCALE = 0.01; +export const MIN_TRANSFORM_SCALE = 0.01; diff --git a/apps/web/src/core/managers/audio-manager.ts b/apps/web/src/core/managers/audio-manager.ts index 0f642b1f..aaeb7ade 100644 --- a/apps/web/src/core/managers/audio-manager.ts +++ b/apps/web/src/core/managers/audio-manager.ts @@ -1,708 +1,709 @@ -import type { EditorCore } from "@/core"; -import { - clampRetimeRate, - shouldMaintainPitch, -} from "@/lib/retime/rate"; -import type { AudioClipSource } from "@/lib/media/audio"; -import { createAudioContext, collectAudioClips } from "@/lib/media/audio"; -import { - buildAudioGainAutomation, - hasAnimatedVolume, -} from "@/lib/timeline/audio-state"; -import { createAudioMasteringChain } from "@/lib/media/audio-mastering"; -import { - getClipTimeAtSourceTime, - getSourceTimeAtClipTime, - renderRetimedBuffer, -} from "@/lib/retime"; -import { - ALL_FORMATS, - AudioBufferSink, - BlobSource, - Input, - type WrappedAudioBuffer, -} from "mediabunny"; - -export class AudioManager { - private audioContext: AudioContext | null = null; - private masterGain: GainNode | null = null; - private playbackStartTime = 0; - private playbackStartContextTime = 0; - private scheduleTimer: number | null = null; - private lookaheadSeconds = 2; - private scheduleIntervalMs = 500; - private clips: AudioClipSource[] = []; - private sinks = new Map(); - private inputs = new Map(); - private activeClipIds = new Set(); - private clipIterators = new Map< - string, - AsyncGenerator - >(); - private queuedSources = new Set(); - private preparedClipBuffers = new Map>(); - private decodedBuffers = new Map>(); - private playbackSessionId = 0; - private lastIsPlaying = false; - private lastVolume = 1; - private playbackLatencyCompensationSeconds = 0; - private unsubscribers: Array<() => void> = []; - - constructor(private editor: EditorCore) { - this.lastVolume = this.editor.playback.getVolume(); - - this.unsubscribers.push( - this.editor.playback.subscribe(this.handlePlaybackChange), - this.editor.timeline.subscribe(this.handleTimelineChange), - this.editor.media.subscribe(this.handleTimelineChange), - ); - if (typeof window !== "undefined") { - window.addEventListener("playback-seek", this.handleSeek); - } - } - - dispose(): void { - this.stopPlayback(); - for (const unsub of this.unsubscribers) { - unsub(); - } - this.unsubscribers = []; - if (typeof window !== "undefined") { - window.removeEventListener("playback-seek", this.handleSeek); - } - this.disposeSinks(); - this.preparedClipBuffers.clear(); - this.decodedBuffers.clear(); - if (this.audioContext) { - void this.audioContext.close(); - this.audioContext = null; - this.masterGain = null; - } - } - - private handlePlaybackChange = (): void => { - const isPlaying = this.editor.playback.getIsPlaying(); - const volume = this.editor.playback.getVolume(); - - if (volume !== this.lastVolume) { - this.lastVolume = volume; - this.updateGain(); - } - - if (isPlaying !== this.lastIsPlaying) { - this.lastIsPlaying = isPlaying; - if (isPlaying) { - void this.startPlayback({ - time: this.editor.playback.getCurrentTime(), - }); - } else { - this.stopPlayback(); - } - } - }; - - private handleSeek = (event: Event): void => { - const detail = (event as CustomEvent<{ time: number }>).detail; - if (!detail) return; - - if (this.editor.playback.getIsScrubbing()) { - this.stopPlayback(); - return; - } - - if (this.editor.playback.getIsPlaying()) { - void this.startPlayback({ time: detail.time }); - return; - } - - this.stopPlayback(); - }; - - private handleTimelineChange = (): void => { - this.disposeSinks(); - this.preparedClipBuffers.clear(); - this.decodedBuffers.clear(); - - if (!this.editor.playback.getIsPlaying()) return; - - void this.startPlayback({ time: this.editor.playback.getCurrentTime() }); - }; - - private ensureAudioContext(): AudioContext | null { - if (this.audioContext) return this.audioContext; - if (typeof window === "undefined") return null; - - this.audioContext = createAudioContext(); - const { input } = createAudioMasteringChain({ - audioContext: this.audioContext, - destination: this.audioContext.destination, - }); - this.masterGain = input; - this.masterGain.gain.value = this.lastVolume; - return this.audioContext; - } - - private updateGain(): void { - if (!this.masterGain) return; - this.masterGain.gain.value = this.lastVolume; - } - - private getPlaybackTime(): number { - if (!this.audioContext) return this.playbackStartTime; - const elapsed = - this.audioContext.currentTime - this.playbackStartContextTime; - return this.playbackStartTime + elapsed; - } - - private async startPlayback({ time }: { time: number }): Promise { - const audioContext = this.ensureAudioContext(); - if (!audioContext) return; - - this.stopPlayback(); - this.playbackSessionId++; - this.playbackLatencyCompensationSeconds = 0; - - const tracks = this.editor.timeline.getTracks(); - const mediaAssets = this.editor.media.getAssets(); - const duration = this.editor.timeline.getTotalDuration(); - - if (duration <= 0) return; - - if (audioContext.state === "suspended") { - await audioContext.resume(); - } - - this.clips = await collectAudioClips({ tracks, mediaAssets }); - if (!this.editor.playback.getIsPlaying()) return; - - this.playbackStartTime = time; - this.playbackStartContextTime = audioContext.currentTime; - - this.scheduleUpcomingClips(); - - if (typeof window !== "undefined") { - this.scheduleTimer = window.setInterval(() => { - this.scheduleUpcomingClips(); - }, this.scheduleIntervalMs); - } - } - - private scheduleUpcomingClips(): void { - if (!this.editor.playback.getIsPlaying()) return; - - const currentTime = this.getPlaybackTime(); - const windowEnd = currentTime + this.lookaheadSeconds; - - for (const clip of this.clips) { - if (clip.muted) continue; - if (this.activeClipIds.has(clip.id)) continue; - - const clipEnd = clip.startTime + clip.duration; - if (clipEnd <= currentTime) continue; - if (clip.startTime > windowEnd) continue; - - this.activeClipIds.add(clip.id); - if (this.shouldUsePreparedClipBuffer({ clip })) { - void this.schedulePreparedClip({ - clip, - startTime: currentTime, - sessionId: this.playbackSessionId, - }); - } else { - void this.runClipIterator({ - clip, - startTime: currentTime, - sessionId: this.playbackSessionId, - }); - } - } - } - - private stopPlayback(): void { - if (this.scheduleTimer && typeof window !== "undefined") { - window.clearInterval(this.scheduleTimer); - } - this.scheduleTimer = null; - - for (const iterator of this.clipIterators.values()) { - void iterator.return(); - } - this.clipIterators.clear(); - this.activeClipIds.clear(); - - for (const source of this.queuedSources) { - try { - source.stop(); - } catch {} - source.disconnect(); - } - this.queuedSources.clear(); - } - - private async runClipIterator({ - clip, - startTime, - sessionId, - }: { - clip: AudioClipSource; - startTime: number; - sessionId: number; - }): Promise { - const audioContext = this.ensureAudioContext(); - if (!audioContext) return; - - const sink = await this.getAudioSink({ clip }); - if (!sink || !this.editor.playback.getIsPlaying()) return; - if (sessionId !== this.playbackSessionId) return; - - const clipStart = clip.startTime; - const clipEnd = clip.startTime + clip.duration; - const playbackTimeAfterSinkReady = this.getPlaybackTime(); - const iteratorStartTime = Math.max( - startTime, - clipStart, - playbackTimeAfterSinkReady, - ); - if (iteratorStartTime >= clipEnd) { - return; - } - const sourceStartTime = - clip.trimStart + - getSourceTimeAtClipTime({ - clipTime: iteratorStartTime - clip.startTime, - retime: clip.retime, - }); - - const iterator = sink.buffers(sourceStartTime); - this.clipIterators.set(clip.id, iterator); - let consecutiveDroppedBufferCount = 0; - - for await (const { buffer, timestamp } of iterator) { - if (!this.editor.playback.getIsPlaying()) return; - if (sessionId !== this.playbackSessionId) return; - - const timelineTime = - clip.startTime + - getClipTimeAtSourceTime({ - sourceTime: timestamp - clip.trimStart, - retime: clip.retime, - }); - if (timelineTime >= clipEnd) break; - - const node = audioContext.createBufferSource(); - node.buffer = buffer; - if (clip.retime) { - node.playbackRate.value = clampRetimeRate({ rate: clip.retime.rate }); - } - const clipGain = audioContext.createGain(); - clipGain.gain.value = clip.volume; - node.connect(clipGain); - clipGain.connect(this.masterGain ?? audioContext.destination); - - const startTimestamp = - this.playbackStartContextTime + - this.playbackLatencyCompensationSeconds + - (timelineTime - this.playbackStartTime); - - if (startTimestamp >= audioContext.currentTime) { - node.start(startTimestamp); - consecutiveDroppedBufferCount = 0; - } else { - const offset = audioContext.currentTime - startTimestamp; - if (offset < buffer.duration) { - node.start(audioContext.currentTime, offset); - consecutiveDroppedBufferCount = 0; - } else { - consecutiveDroppedBufferCount += 1; - if (consecutiveDroppedBufferCount >= 5) { - const nextCompensationSeconds = Math.max( - this.playbackLatencyCompensationSeconds, - Math.min(0.25, offset + 0.01), - ); - if ( - nextCompensationSeconds > - this.playbackLatencyCompensationSeconds + 0.001 - ) { - this.playbackLatencyCompensationSeconds = nextCompensationSeconds; - } - const resyncStartTime = this.getPlaybackTime(); - this.clipIterators.delete(clip.id); - void this.runClipIterator({ - clip, - startTime: resyncStartTime, - sessionId, - }); - return; - } - continue; - } - } - - this.queuedSources.add(node); - node.addEventListener("ended", () => { - node.disconnect(); - clipGain.disconnect(); - this.queuedSources.delete(node); - }); - - const aheadTime = timelineTime - this.getPlaybackTime(); - if (aheadTime >= 1) { - await this.waitUntilCaughtUp({ timelineTime, targetAhead: 1 }); - if (sessionId !== this.playbackSessionId) return; - } - } - - this.clipIterators.delete(clip.id); - // don't remove from activeClipIds - prevents scheduler from restarting this clip - // the set is cleared on stopPlayback anyway - } - - private async schedulePreparedClip({ - clip, - startTime, - sessionId, - }: { - clip: AudioClipSource; - startTime: number; - sessionId: number; - }): Promise { - const audioContext = this.ensureAudioContext(); - if (!audioContext) return; - - const buffer = await this.getPreparedClipBuffer({ clip }); - if (!buffer || !this.editor.playback.getIsPlaying()) return; - if (sessionId !== this.playbackSessionId) return; - - const clipStart = clip.startTime; - const clipEnd = clip.startTime + clip.duration; - const playbackTimeAfterReady = this.getPlaybackTime(); - const effectiveStartTime = Math.max( - startTime, - clipStart, - playbackTimeAfterReady, - ); - if (effectiveStartTime >= clipEnd) { - return; - } - - const node = audioContext.createBufferSource(); - node.buffer = buffer; - const clipGain = audioContext.createGain(); - node.connect(clipGain); - clipGain.connect(this.masterGain ?? audioContext.destination); - - const startTimestamp = - this.playbackStartContextTime + - this.playbackLatencyCompensationSeconds + - (effectiveStartTime - this.playbackStartTime); - const clipOffset = effectiveStartTime - clipStart; - let actualStartTimestamp = startTimestamp; - let actualClipOffset = clipOffset; - - if (startTimestamp >= audioContext.currentTime) { - node.start(startTimestamp, clipOffset); - } else { - const lateOffset = audioContext.currentTime - startTimestamp; - actualStartTimestamp = audioContext.currentTime; - actualClipOffset = clipOffset + lateOffset; - node.start(actualStartTimestamp, actualClipOffset); - } - - this.scheduleClipGainAutomation({ - audioContext, - clip, - clipGain, - startTimestamp: actualStartTimestamp, - startLocalTime: actualClipOffset, - }); - - this.queuedSources.add(node); - node.addEventListener("ended", () => { - node.disconnect(); - clipGain.disconnect(); - this.queuedSources.delete(node); - }); - } - - private waitUntilCaughtUp({ - timelineTime, - targetAhead, - }: { - timelineTime: number; - targetAhead: number; - }): Promise { - return new Promise((resolve) => { - const checkInterval = setInterval(() => { - if (!this.editor.playback.getIsPlaying()) { - clearInterval(checkInterval); - resolve(); - return; - } - - const playbackTime = this.getPlaybackTime(); - if (timelineTime - playbackTime < targetAhead) { - clearInterval(checkInterval); - resolve(); - } - }, 100); - }); - } - - private disposeSinks(): void { - for (const iterator of this.clipIterators.values()) { - void iterator.return(); - } - this.clipIterators.clear(); - this.activeClipIds.clear(); - - for (const input of this.inputs.values()) { - input.dispose(); - } - this.inputs.clear(); - this.sinks.clear(); - } - - private shouldUsePreparedClipBuffer({ - clip, - }: { - clip: AudioClipSource; - }): boolean { - return ( - this.hasCurveRetime({ clip }) || - hasAnimatedVolume({ element: clip.timelineElement }) || - shouldMaintainPitch({ - rate: clip.retime?.rate ?? 1, - maintainPitch: clip.retime?.maintainPitch, - }) - ); - } - - private hasCurveRetime({ clip }: { clip: AudioClipSource }): boolean { - const mode = (clip.retime as { mode?: unknown } | undefined)?.mode; - return mode === "curve"; - } - - private scheduleClipGainAutomation({ - audioContext, - clip, - clipGain, - startTimestamp, - startLocalTime, - }: { - audioContext: AudioContext; - clip: AudioClipSource; - clipGain: GainNode; - startTimestamp: number; - startLocalTime: number; - }): void { - clipGain.gain.cancelScheduledValues(startTimestamp); - clipGain.gain.setValueAtTime(clip.volume, startTimestamp); - - if (!hasAnimatedVolume({ element: clip.timelineElement })) { - return; - } - - const points = buildAudioGainAutomation({ - element: clip.timelineElement, - fromLocalTime: startLocalTime, - toLocalTime: clip.duration, - }); - - if (points.length === 0) { - return; - } - - clipGain.gain.setValueAtTime(points[0].gain, startTimestamp); - for (let index = 1; index < points.length; index++) { - const point = points[index]; - const pointTimestamp = - startTimestamp + (point.localTime - startLocalTime); - if (pointTimestamp < audioContext.currentTime) { - continue; - } - - clipGain.gain.linearRampToValueAtTime(point.gain, pointTimestamp); - } - } - - private buildPreparedClipCacheKey({ - clip, - }: { - clip: AudioClipSource; - }): string { - return JSON.stringify({ - id: clip.id, - sourceKey: clip.sourceKey, - startTime: clip.startTime, - duration: clip.duration, - trimStart: clip.trimStart, - trimEnd: clip.trimEnd, - retime: clip.retime ?? null, - }); - } - - private async getPreparedClipBuffer({ - clip, - }: { - clip: AudioClipSource; - }): Promise { - const cacheKey = this.buildPreparedClipCacheKey({ clip }); - const existing = this.preparedClipBuffers.get(cacheKey); - if (existing) { - return existing; - } - - const promise = (async () => { - const audioContext = this.ensureAudioContext(); - if (!audioContext) { - return null; - } - - const decodedBuffer = await this.getDecodedBuffer({ clip }); - if (!decodedBuffer) { - return null; - } - - return await renderRetimedBuffer({ - audioContext, - sourceBuffer: decodedBuffer, - trimStart: clip.trimStart, - clipDuration: clip.duration, - retime: clip.retime, - maintainPitch: clip.retime?.maintainPitch === true, - }); - })(); - - this.preparedClipBuffers.set(cacheKey, promise); - return promise; - } - - private async getDecodedBuffer({ - clip, - }: { - clip: AudioClipSource; - }): Promise { - const existing = this.decodedBuffers.get(clip.sourceKey); - if (existing) { - return existing; - } - - const promise = this.decodeClipBuffer({ clip }); - this.decodedBuffers.set(clip.sourceKey, promise); - return promise; - } - - private async decodeClipBuffer({ - clip, - }: { - clip: AudioClipSource; - }): Promise { - const audioContext = this.ensureAudioContext(); - if (!audioContext) { - return null; - } - - const input = new Input({ - source: new BlobSource(clip.file), - formats: ALL_FORMATS, - }); - - try { - const audioTrack = await input.getPrimaryAudioTrack(); - if (!audioTrack) { - return null; - } - - const sink = new AudioBufferSink(audioTrack); - const chunks: AudioBuffer[] = []; - let totalSamples = 0; - - for await (const { buffer } of sink.buffers(0)) { - chunks.push(buffer); - totalSamples += buffer.length; - } - - if (chunks.length === 0) { - return null; - } - - const targetSampleRate = audioContext.sampleRate; - const nativeSampleRate = chunks[0].sampleRate; - const numChannels = Math.min(2, chunks[0].numberOfChannels); - const nativeChannels = Array.from( - { length: numChannels }, - () => new Float32Array(totalSamples), - ); - - let offset = 0; - for (const chunk of chunks) { - for (let channel = 0; channel < numChannels; channel++) { - nativeChannels[channel].set( - chunk.getChannelData(Math.min(channel, chunk.numberOfChannels - 1)), - offset, - ); - } - offset += chunk.length; - } - - const outputSamples = Math.ceil( - totalSamples * (targetSampleRate / nativeSampleRate), - ); - const offlineContext = new OfflineAudioContext( - numChannels, - outputSamples, - targetSampleRate, - ); - const nativeBuffer = audioContext.createBuffer( - numChannels, - totalSamples, - nativeSampleRate, - ); - - for (let channel = 0; channel < numChannels; channel++) { - nativeBuffer.copyToChannel(nativeChannels[channel], channel); - } - - const sourceNode = offlineContext.createBufferSource(); - sourceNode.buffer = nativeBuffer; - sourceNode.connect(offlineContext.destination); - sourceNode.start(0); - - return await offlineContext.startRendering(); - } catch (error) { - console.warn("Failed to decode clip audio:", error); - return null; - } finally { - input.dispose(); - } - } - - private async getAudioSink({ - clip, - }: { - clip: AudioClipSource; - }): Promise { - const existingSink = this.sinks.get(clip.sourceKey); - if (existingSink) return existingSink; - - try { - const input = new Input({ - source: new BlobSource(clip.file), - formats: ALL_FORMATS, - }); - const audioTrack = await input.getPrimaryAudioTrack(); - if (!audioTrack) { - input.dispose(); - return null; - } - - const sink = new AudioBufferSink(audioTrack); - this.inputs.set(clip.sourceKey, input); - this.sinks.set(clip.sourceKey, sink); - return sink; - } catch (error) { - console.warn("Failed to initialize audio sink:", error); - return null; - } - } -} +import type { EditorCore } from "@/core"; +import { TICKS_PER_SECOND } from "@/lib/wasm"; +import { + clampRetimeRate, + shouldMaintainPitch, +} from "@/lib/retime/rate"; +import type { AudioClipSource } from "@/lib/media/audio"; +import { createAudioContext, collectAudioClips } from "@/lib/media/audio"; +import { + buildAudioGainAutomation, + hasAnimatedVolume, +} from "@/lib/timeline/audio-state"; +import { createAudioMasteringChain } from "@/lib/media/audio-mastering"; +import { + getClipTimeAtSourceTime, + getSourceTimeAtClipTime, + renderRetimedBuffer, +} from "@/lib/retime"; +import { + ALL_FORMATS, + AudioBufferSink, + BlobSource, + Input, + type WrappedAudioBuffer, +} from "mediabunny"; + +export class AudioManager { + private audioContext: AudioContext | null = null; + private masterGain: GainNode | null = null; + private playbackStartTime = 0; + private playbackStartContextTime = 0; + private scheduleTimer: number | null = null; + private lookaheadSeconds = 2; + private scheduleIntervalMs = 500; + private clips: AudioClipSource[] = []; + private sinks = new Map(); + private inputs = new Map(); + private activeClipIds = new Set(); + private clipIterators = new Map< + string, + AsyncGenerator + >(); + private queuedSources = new Set(); + private preparedClipBuffers = new Map>(); + private decodedBuffers = new Map>(); + private playbackSessionId = 0; + private lastIsPlaying = false; + private lastVolume = 1; + private playbackLatencyCompensationSeconds = 0; + private unsubscribers: Array<() => void> = []; + + constructor(private editor: EditorCore) { + this.lastVolume = this.editor.playback.getVolume(); + + this.unsubscribers.push( + this.editor.playback.subscribe(this.handlePlaybackChange), + this.editor.timeline.subscribe(this.handleTimelineChange), + this.editor.media.subscribe(this.handleTimelineChange), + ); + if (typeof window !== "undefined") { + window.addEventListener("playback-seek", this.handleSeek); + } + } + + dispose(): void { + this.stopPlayback(); + for (const unsub of this.unsubscribers) { + unsub(); + } + this.unsubscribers = []; + if (typeof window !== "undefined") { + window.removeEventListener("playback-seek", this.handleSeek); + } + this.disposeSinks(); + this.preparedClipBuffers.clear(); + this.decodedBuffers.clear(); + if (this.audioContext) { + void this.audioContext.close(); + this.audioContext = null; + this.masterGain = null; + } + } + + private handlePlaybackChange = (): void => { + const isPlaying = this.editor.playback.getIsPlaying(); + const volume = this.editor.playback.getVolume(); + + if (volume !== this.lastVolume) { + this.lastVolume = volume; + this.updateGain(); + } + + if (isPlaying !== this.lastIsPlaying) { + this.lastIsPlaying = isPlaying; + if (isPlaying) { + void this.startPlayback({ + time: this.editor.playback.getCurrentTime() / TICKS_PER_SECOND, + }); + } else { + this.stopPlayback(); + } + } + }; + + private handleSeek = (event: Event): void => { + const detail = (event as CustomEvent<{ time: number }>).detail; + if (!detail) return; + + if (this.editor.playback.getIsScrubbing()) { + this.stopPlayback(); + return; + } + + if (this.editor.playback.getIsPlaying()) { + void this.startPlayback({ time: detail.time / TICKS_PER_SECOND }); + return; + } + + this.stopPlayback(); + }; + + private handleTimelineChange = (): void => { + this.disposeSinks(); + this.preparedClipBuffers.clear(); + this.decodedBuffers.clear(); + + if (!this.editor.playback.getIsPlaying()) return; + + void this.startPlayback({ time: this.editor.playback.getCurrentTime() / TICKS_PER_SECOND }); + }; + + private ensureAudioContext(): AudioContext | null { + if (this.audioContext) return this.audioContext; + if (typeof window === "undefined") return null; + + this.audioContext = createAudioContext(); + const { input } = createAudioMasteringChain({ + audioContext: this.audioContext, + destination: this.audioContext.destination, + }); + this.masterGain = input; + this.masterGain.gain.value = this.lastVolume; + return this.audioContext; + } + + private updateGain(): void { + if (!this.masterGain) return; + this.masterGain.gain.value = this.lastVolume; + } + + private getPlaybackTime(): number { + if (!this.audioContext) return this.playbackStartTime; + const elapsed = + this.audioContext.currentTime - this.playbackStartContextTime; + return this.playbackStartTime + elapsed; + } + + private async startPlayback({ time }: { time: number }): Promise { + const audioContext = this.ensureAudioContext(); + if (!audioContext) return; + + this.stopPlayback(); + this.playbackSessionId++; + this.playbackLatencyCompensationSeconds = 0; + + const tracks = this.editor.timeline.getTracks(); + const mediaAssets = this.editor.media.getAssets(); + const duration = this.editor.timeline.getTotalDuration(); + + if (duration <= 0) return; + + if (audioContext.state === "suspended") { + await audioContext.resume(); + } + + this.clips = await collectAudioClips({ tracks, mediaAssets }); + if (!this.editor.playback.getIsPlaying()) return; + + this.playbackStartTime = time; + this.playbackStartContextTime = audioContext.currentTime; + + this.scheduleUpcomingClips(); + + if (typeof window !== "undefined") { + this.scheduleTimer = window.setInterval(() => { + this.scheduleUpcomingClips(); + }, this.scheduleIntervalMs); + } + } + + private scheduleUpcomingClips(): void { + if (!this.editor.playback.getIsPlaying()) return; + + const currentTime = this.getPlaybackTime(); + const windowEnd = currentTime + this.lookaheadSeconds; + + for (const clip of this.clips) { + if (clip.muted) continue; + if (this.activeClipIds.has(clip.id)) continue; + + const clipEnd = clip.startTime + clip.duration; + if (clipEnd <= currentTime) continue; + if (clip.startTime > windowEnd) continue; + + this.activeClipIds.add(clip.id); + if (this.shouldUsePreparedClipBuffer({ clip })) { + void this.schedulePreparedClip({ + clip, + startTime: currentTime, + sessionId: this.playbackSessionId, + }); + } else { + void this.runClipIterator({ + clip, + startTime: currentTime, + sessionId: this.playbackSessionId, + }); + } + } + } + + private stopPlayback(): void { + if (this.scheduleTimer && typeof window !== "undefined") { + window.clearInterval(this.scheduleTimer); + } + this.scheduleTimer = null; + + for (const iterator of this.clipIterators.values()) { + void iterator.return(); + } + this.clipIterators.clear(); + this.activeClipIds.clear(); + + for (const source of this.queuedSources) { + try { + source.stop(); + } catch {} + source.disconnect(); + } + this.queuedSources.clear(); + } + + private async runClipIterator({ + clip, + startTime, + sessionId, + }: { + clip: AudioClipSource; + startTime: number; + sessionId: number; + }): Promise { + const audioContext = this.ensureAudioContext(); + if (!audioContext) return; + + const sink = await this.getAudioSink({ clip }); + if (!sink || !this.editor.playback.getIsPlaying()) return; + if (sessionId !== this.playbackSessionId) return; + + const clipStart = clip.startTime; + const clipEnd = clip.startTime + clip.duration; + const playbackTimeAfterSinkReady = this.getPlaybackTime(); + const iteratorStartTime = Math.max( + startTime, + clipStart, + playbackTimeAfterSinkReady, + ); + if (iteratorStartTime >= clipEnd) { + return; + } + const sourceStartTime = + clip.trimStart + + getSourceTimeAtClipTime({ + clipTime: iteratorStartTime - clip.startTime, + retime: clip.retime, + }); + + const iterator = sink.buffers(sourceStartTime); + this.clipIterators.set(clip.id, iterator); + let consecutiveDroppedBufferCount = 0; + + for await (const { buffer, timestamp } of iterator) { + if (!this.editor.playback.getIsPlaying()) return; + if (sessionId !== this.playbackSessionId) return; + + const timelineTime = + clip.startTime + + getClipTimeAtSourceTime({ + sourceTime: timestamp - clip.trimStart, + retime: clip.retime, + }); + if (timelineTime >= clipEnd) break; + + const node = audioContext.createBufferSource(); + node.buffer = buffer; + if (clip.retime) { + node.playbackRate.value = clampRetimeRate({ rate: clip.retime.rate }); + } + const clipGain = audioContext.createGain(); + clipGain.gain.value = clip.volume; + node.connect(clipGain); + clipGain.connect(this.masterGain ?? audioContext.destination); + + const startTimestamp = + this.playbackStartContextTime + + this.playbackLatencyCompensationSeconds + + (timelineTime - this.playbackStartTime); + + if (startTimestamp >= audioContext.currentTime) { + node.start(startTimestamp); + consecutiveDroppedBufferCount = 0; + } else { + const offset = audioContext.currentTime - startTimestamp; + if (offset < buffer.duration) { + node.start(audioContext.currentTime, offset); + consecutiveDroppedBufferCount = 0; + } else { + consecutiveDroppedBufferCount += 1; + if (consecutiveDroppedBufferCount >= 5) { + const nextCompensationSeconds = Math.max( + this.playbackLatencyCompensationSeconds, + Math.min(0.25, offset + 0.01), + ); + if ( + nextCompensationSeconds > + this.playbackLatencyCompensationSeconds + 0.001 + ) { + this.playbackLatencyCompensationSeconds = nextCompensationSeconds; + } + const resyncStartTime = this.getPlaybackTime(); + this.clipIterators.delete(clip.id); + void this.runClipIterator({ + clip, + startTime: resyncStartTime, + sessionId, + }); + return; + } + continue; + } + } + + this.queuedSources.add(node); + node.addEventListener("ended", () => { + node.disconnect(); + clipGain.disconnect(); + this.queuedSources.delete(node); + }); + + const aheadTime = timelineTime - this.getPlaybackTime(); + if (aheadTime >= 1) { + await this.waitUntilCaughtUp({ timelineTime, targetAhead: 1 }); + if (sessionId !== this.playbackSessionId) return; + } + } + + this.clipIterators.delete(clip.id); + // don't remove from activeClipIds - prevents scheduler from restarting this clip + // the set is cleared on stopPlayback anyway + } + + private async schedulePreparedClip({ + clip, + startTime, + sessionId, + }: { + clip: AudioClipSource; + startTime: number; + sessionId: number; + }): Promise { + const audioContext = this.ensureAudioContext(); + if (!audioContext) return; + + const buffer = await this.getPreparedClipBuffer({ clip }); + if (!buffer || !this.editor.playback.getIsPlaying()) return; + if (sessionId !== this.playbackSessionId) return; + + const clipStart = clip.startTime; + const clipEnd = clip.startTime + clip.duration; + const playbackTimeAfterReady = this.getPlaybackTime(); + const effectiveStartTime = Math.max( + startTime, + clipStart, + playbackTimeAfterReady, + ); + if (effectiveStartTime >= clipEnd) { + return; + } + + const node = audioContext.createBufferSource(); + node.buffer = buffer; + const clipGain = audioContext.createGain(); + node.connect(clipGain); + clipGain.connect(this.masterGain ?? audioContext.destination); + + const startTimestamp = + this.playbackStartContextTime + + this.playbackLatencyCompensationSeconds + + (effectiveStartTime - this.playbackStartTime); + const clipOffset = effectiveStartTime - clipStart; + let actualStartTimestamp = startTimestamp; + let actualClipOffset = clipOffset; + + if (startTimestamp >= audioContext.currentTime) { + node.start(startTimestamp, clipOffset); + } else { + const lateOffset = audioContext.currentTime - startTimestamp; + actualStartTimestamp = audioContext.currentTime; + actualClipOffset = clipOffset + lateOffset; + node.start(actualStartTimestamp, actualClipOffset); + } + + this.scheduleClipGainAutomation({ + audioContext, + clip, + clipGain, + startTimestamp: actualStartTimestamp, + startLocalTime: actualClipOffset, + }); + + this.queuedSources.add(node); + node.addEventListener("ended", () => { + node.disconnect(); + clipGain.disconnect(); + this.queuedSources.delete(node); + }); + } + + private waitUntilCaughtUp({ + timelineTime, + targetAhead, + }: { + timelineTime: number; + targetAhead: number; + }): Promise { + return new Promise((resolve) => { + const checkInterval = setInterval(() => { + if (!this.editor.playback.getIsPlaying()) { + clearInterval(checkInterval); + resolve(); + return; + } + + const playbackTime = this.getPlaybackTime(); + if (timelineTime - playbackTime < targetAhead) { + clearInterval(checkInterval); + resolve(); + } + }, 100); + }); + } + + private disposeSinks(): void { + for (const iterator of this.clipIterators.values()) { + void iterator.return(); + } + this.clipIterators.clear(); + this.activeClipIds.clear(); + + for (const input of this.inputs.values()) { + input.dispose(); + } + this.inputs.clear(); + this.sinks.clear(); + } + + private shouldUsePreparedClipBuffer({ + clip, + }: { + clip: AudioClipSource; + }): boolean { + return ( + this.hasCurveRetime({ clip }) || + hasAnimatedVolume({ element: clip.timelineElement }) || + shouldMaintainPitch({ + rate: clip.retime?.rate ?? 1, + maintainPitch: clip.retime?.maintainPitch, + }) + ); + } + + private hasCurveRetime({ clip }: { clip: AudioClipSource }): boolean { + const mode = (clip.retime as { mode?: unknown } | undefined)?.mode; + return mode === "curve"; + } + + private scheduleClipGainAutomation({ + audioContext, + clip, + clipGain, + startTimestamp, + startLocalTime, + }: { + audioContext: AudioContext; + clip: AudioClipSource; + clipGain: GainNode; + startTimestamp: number; + startLocalTime: number; + }): void { + clipGain.gain.cancelScheduledValues(startTimestamp); + clipGain.gain.setValueAtTime(clip.volume, startTimestamp); + + if (!hasAnimatedVolume({ element: clip.timelineElement })) { + return; + } + + const points = buildAudioGainAutomation({ + element: clip.timelineElement, + fromLocalTime: startLocalTime, + toLocalTime: clip.duration, + }); + + if (points.length === 0) { + return; + } + + clipGain.gain.setValueAtTime(points[0].gain, startTimestamp); + for (let index = 1; index < points.length; index++) { + const point = points[index]; + const pointTimestamp = + startTimestamp + (point.localTime - startLocalTime); + if (pointTimestamp < audioContext.currentTime) { + continue; + } + + clipGain.gain.linearRampToValueAtTime(point.gain, pointTimestamp); + } + } + + private buildPreparedClipCacheKey({ + clip, + }: { + clip: AudioClipSource; + }): string { + return JSON.stringify({ + id: clip.id, + sourceKey: clip.sourceKey, + startTime: clip.startTime, + duration: clip.duration, + trimStart: clip.trimStart, + trimEnd: clip.trimEnd, + retime: clip.retime ?? null, + }); + } + + private async getPreparedClipBuffer({ + clip, + }: { + clip: AudioClipSource; + }): Promise { + const cacheKey = this.buildPreparedClipCacheKey({ clip }); + const existing = this.preparedClipBuffers.get(cacheKey); + if (existing) { + return existing; + } + + const promise = (async () => { + const audioContext = this.ensureAudioContext(); + if (!audioContext) { + return null; + } + + const decodedBuffer = await this.getDecodedBuffer({ clip }); + if (!decodedBuffer) { + return null; + } + + return await renderRetimedBuffer({ + audioContext, + sourceBuffer: decodedBuffer, + trimStart: clip.trimStart, + clipDuration: clip.duration, + retime: clip.retime, + maintainPitch: clip.retime?.maintainPitch === true, + }); + })(); + + this.preparedClipBuffers.set(cacheKey, promise); + return promise; + } + + private async getDecodedBuffer({ + clip, + }: { + clip: AudioClipSource; + }): Promise { + const existing = this.decodedBuffers.get(clip.sourceKey); + if (existing) { + return existing; + } + + const promise = this.decodeClipBuffer({ clip }); + this.decodedBuffers.set(clip.sourceKey, promise); + return promise; + } + + private async decodeClipBuffer({ + clip, + }: { + clip: AudioClipSource; + }): Promise { + const audioContext = this.ensureAudioContext(); + if (!audioContext) { + return null; + } + + const input = new Input({ + source: new BlobSource(clip.file), + formats: ALL_FORMATS, + }); + + try { + const audioTrack = await input.getPrimaryAudioTrack(); + if (!audioTrack) { + return null; + } + + const sink = new AudioBufferSink(audioTrack); + const chunks: AudioBuffer[] = []; + let totalSamples = 0; + + for await (const { buffer } of sink.buffers(0)) { + chunks.push(buffer); + totalSamples += buffer.length; + } + + if (chunks.length === 0) { + return null; + } + + const targetSampleRate = audioContext.sampleRate; + const nativeSampleRate = chunks[0].sampleRate; + const numChannels = Math.min(2, chunks[0].numberOfChannels); + const nativeChannels = Array.from( + { length: numChannels }, + () => new Float32Array(totalSamples), + ); + + let offset = 0; + for (const chunk of chunks) { + for (let channel = 0; channel < numChannels; channel++) { + nativeChannels[channel].set( + chunk.getChannelData(Math.min(channel, chunk.numberOfChannels - 1)), + offset, + ); + } + offset += chunk.length; + } + + const outputSamples = Math.ceil( + totalSamples * (targetSampleRate / nativeSampleRate), + ); + const offlineContext = new OfflineAudioContext( + numChannels, + outputSamples, + targetSampleRate, + ); + const nativeBuffer = audioContext.createBuffer( + numChannels, + totalSamples, + nativeSampleRate, + ); + + for (let channel = 0; channel < numChannels; channel++) { + nativeBuffer.copyToChannel(nativeChannels[channel], channel); + } + + const sourceNode = offlineContext.createBufferSource(); + sourceNode.buffer = nativeBuffer; + sourceNode.connect(offlineContext.destination); + sourceNode.start(0); + + return await offlineContext.startRendering(); + } catch (error) { + console.warn("Failed to decode clip audio:", error); + return null; + } finally { + input.dispose(); + } + } + + private async getAudioSink({ + clip, + }: { + clip: AudioClipSource; + }): Promise { + const existingSink = this.sinks.get(clip.sourceKey); + if (existingSink) return existingSink; + + try { + const input = new Input({ + source: new BlobSource(clip.file), + formats: ALL_FORMATS, + }); + const audioTrack = await input.getPrimaryAudioTrack(); + if (!audioTrack) { + input.dispose(); + return null; + } + + const sink = new AudioBufferSink(audioTrack); + this.inputs.set(clip.sourceKey, input); + this.sinks.set(clip.sourceKey, sink); + return sink; + } catch (error) { + console.warn("Failed to initialize audio sink:", error); + return null; + } + } +} diff --git a/apps/web/src/core/managers/playback-manager.ts b/apps/web/src/core/managers/playback-manager.ts index 6943f187..47909bfe 100644 --- a/apps/web/src/core/managers/playback-manager.ts +++ b/apps/web/src/core/managers/playback-manager.ts @@ -1,4 +1,6 @@ import type { EditorCore } from "@/core"; +import { TICKS_PER_SECOND } from "@/lib/wasm"; +import { roundToFrame } from "opencut-wasm"; export class PlaybackManager { private isPlaying = false; @@ -9,11 +11,12 @@ export class PlaybackManager { private isScrubbing = false; private listeners = new Set<() => void>(); private playbackTimer: number | null = null; - private lastUpdate = 0; + private playbackStartWallTime = 0; + private playbackStartTime = 0; constructor(private editor: EditorCore) { this.editor.timeline.subscribe(() => { - const maxTime = this.editor.timeline.getLastSeekableTime(); + const maxTime = this.editor.timeline.getTotalDuration(); if (this.currentTime > maxTime && maxTime > 0) { this.currentTime = maxTime; this.notify(); @@ -22,7 +25,7 @@ export class PlaybackManager { } play(): void { - const maxTime = this.editor.timeline.getLastSeekableTime(); + const maxTime = this.editor.timeline.getTotalDuration(); if (maxTime > 0) { if (this.currentTime >= maxTime) { @@ -50,8 +53,12 @@ export class PlaybackManager { } seek({ time }: { time: number }): void { - const maxTime = this.editor.timeline.getLastSeekableTime(); + const maxTime = this.editor.timeline.getTotalDuration(); this.currentTime = Math.max(0, Math.min(maxTime, time)); + if (this.isPlaying) { + this.playbackStartWallTime = performance.now(); + this.playbackStartTime = this.currentTime; + } this.notify(); window.dispatchEvent( @@ -135,7 +142,8 @@ export class PlaybackManager { cancelAnimationFrame(this.playbackTimer); } - this.lastUpdate = performance.now(); + this.playbackStartWallTime = performance.now(); + this.playbackStartTime = this.currentTime; this.updateTime(); } @@ -149,12 +157,11 @@ export class PlaybackManager { private updateTime = (): void => { if (!this.isPlaying) return; - const now = performance.now(); - const delta = (now - this.lastUpdate) / 1000; - this.lastUpdate = now; - - const newTime = this.currentTime + delta; - const maxTime = this.editor.timeline.getLastSeekableTime(); + const fps = this.editor.project.getActive()?.settings.fps; + const elapsedSeconds = (performance.now() - this.playbackStartWallTime) / 1000; + const rawTime = this.playbackStartTime + Math.round(elapsedSeconds * TICKS_PER_SECOND); + const newTime = fps ? (roundToFrame({ time: rawTime, rate: fps }) ?? rawTime) : rawTime; + const maxTime = this.editor.timeline.getTotalDuration(); if (maxTime > 0 && newTime >= maxTime) { this.pause(); diff --git a/apps/web/src/core/managers/project-manager.ts b/apps/web/src/core/managers/project-manager.ts index 10a9229d..513eab94 100644 --- a/apps/web/src/core/managers/project-manager.ts +++ b/apps/web/src/core/managers/project-manager.ts @@ -496,7 +496,7 @@ export class ProjectManager { importedAssets, }: { importedAssets: Array>; - }): number | null { + }): import("opencut-wasm").FrameRate | null { if (!this.active) return null; const nextFps = getRaisedProjectFpsForImportedMedia({ diff --git a/apps/web/src/core/managers/renderer-manager.ts b/apps/web/src/core/managers/renderer-manager.ts index aa34f496..e0d5b895 100644 --- a/apps/web/src/core/managers/renderer-manager.ts +++ b/apps/web/src/core/managers/renderer-manager.ts @@ -5,7 +5,8 @@ import { CanvasRenderer } from "@/services/renderer/canvas-renderer"; import { SceneExporter } from "@/services/renderer/scene-exporter"; import { buildScene } from "@/services/renderer/scene-builder"; import { createTimelineAudioBuffer } from "@/lib/media/audio"; -import { formatTimeCode } from "opencut-wasm"; +import { formatTimecode } from "opencut-wasm"; +import { frameRateToFloat } from "@/lib/fps/utils"; import { downloadBlob } from "@/utils/browser"; type SnapshotResult = @@ -81,7 +82,10 @@ export class RendererManager { } const { canvasSize, fps } = activeProject.settings; - const renderTime = this.editor.playback.getCurrentTime(); + const renderTime = Math.min( + this.editor.playback.getCurrentTime(), + this.editor.timeline.getLastFrameTime(), + ); const renderer = new CanvasRenderer({ width: canvasSize.width, @@ -107,7 +111,7 @@ export class RendererManager { return { success: false, error: "Failed to create image" }; } - const timecode = formatTimeCode({ timeInSeconds: renderTime, fps })!.replace(/:/g, "-"); + const timecode = formatTimecode({ time: renderTime, rate: fps })!.replace(/:/g, "-"); const safeName = activeProject.metadata.name.replace(/[<>:"/\\|?*]/g, "-").trim() || "snapshot"; @@ -148,7 +152,7 @@ export class RendererManager { return { success: false, error: "Project is empty" }; } - const exportFps = fps || activeProject.settings.fps; + const exportFps = fps ?? activeProject.settings.fps; const canvasSize = activeProject.settings.canvasSize; let audioBuffer: AudioBuffer | null = null; diff --git a/apps/web/src/core/managers/timeline-manager.ts b/apps/web/src/core/managers/timeline-manager.ts index 3abde47b..a5cb428e 100644 --- a/apps/web/src/core/managers/timeline-manager.ts +++ b/apps/web/src/core/managers/timeline-manager.ts @@ -18,7 +18,7 @@ import type { AnimationValue, ScalarCurveKeyframePatch, } from "@/lib/animation/types"; -import { getLastFrameTime } from "opencut-wasm"; +import { lastFrameTime } from "opencut-wasm"; import { BatchCommand } from "@/lib/commands"; import { AddTrackCommand, @@ -196,11 +196,11 @@ export class TimelineManager { return calculateTotalDuration({ tracks: this.getTracks() }); } - getLastSeekableTime(): number { + getLastFrameTime(): number { const duration = this.getTotalDuration(); const fps = this.editor.project.getActive()?.settings.fps; if (!fps || duration <= 0) return duration; - return getLastFrameTime({ duration, fps }); + return lastFrameTime({ duration, rate: fps }) ?? duration; } getTrackById({ trackId }: { trackId: string }): TimelineTrack | null { diff --git a/apps/web/src/hooks/actions/use-editor-actions.ts b/apps/web/src/hooks/actions/use-editor-actions.ts index 76f705e9..599b4260 100644 --- a/apps/web/src/hooks/actions/use-editor-actions.ts +++ b/apps/web/src/hooks/actions/use-editor-actions.ts @@ -5,6 +5,7 @@ import { useTimelineStore } from "@/stores/timeline-store"; import { useActionHandler } from "@/hooks/actions/use-action-handler"; import { useEditor } from "../use-editor"; import { useElementSelection } from "../timeline/element/use-element-selection"; +import { TICKS_PER_SECOND } from "@/lib/wasm"; import { useKeyframeSelection } from "../timeline/element/use-keyframe-selection"; import { getElementsAtTime, @@ -110,10 +111,11 @@ export function useEditorActions() { "frame-step-forward", () => { const fps = editor.project.getActive().settings.fps; + const ticksPerFrame = Math.round(TICKS_PER_SECOND * fps.denominator / fps.numerator); editor.playback.seek({ time: Math.min( editor.timeline.getTotalDuration(), - editor.playback.getCurrentTime() + 1 / fps, + editor.playback.getCurrentTime() + ticksPerFrame, ), }); }, @@ -124,8 +126,9 @@ export function useEditorActions() { "frame-step-backward", () => { const fps = editor.project.getActive().settings.fps; + const ticksPerFrame = Math.round(TICKS_PER_SECOND * fps.denominator / fps.numerator); editor.playback.seek({ - time: Math.max(0, editor.playback.getCurrentTime() - 1 / fps), + time: Math.max(0, editor.playback.getCurrentTime() - ticksPerFrame), }); }, undefined, diff --git a/apps/web/src/hooks/timeline/element/use-element-interaction.ts b/apps/web/src/hooks/timeline/element/use-element-interaction.ts index 71ae4704..6fcd4168 100644 --- a/apps/web/src/hooks/timeline/element/use-element-interaction.ts +++ b/apps/web/src/hooks/timeline/element/use-element-interaction.ts @@ -10,8 +10,9 @@ import { useEditor } from "@/hooks/use-editor"; import { useShiftKey } from "@/hooks/use-shift-key"; import { useElementSelection } from "@/hooks/timeline/element/use-element-selection"; import { BASE_TIMELINE_PIXELS_PER_SECOND } from "@/lib/timeline/scale"; +import { TICKS_PER_SECOND } from "@/lib/wasm"; import { TIMELINE_DRAG_THRESHOLD_PX } from "@/components/editor/panels/timeline/interaction"; -import { snapTimeToFrame } from "opencut-wasm"; +import { roundToFrame } from "opencut-wasm"; import { computeDropTarget } from "@/components/editor/panels/timeline/drop-target"; import { getMouseTimeFromClientX } from "@/lib/timeline/drag-utils"; import { generateUUID } from "@/utils/id"; @@ -67,7 +68,8 @@ function getClickOffsetTime({ zoomLevel: number; }): number { const clickOffsetX = clientX - elementRect.left; - return clickOffsetX / (BASE_TIMELINE_PIXELS_PER_SECOND * zoomLevel); + const seconds = clickOffsetX / (BASE_TIMELINE_PIXELS_PER_SECOND * zoomLevel); + return Math.round(seconds * TICKS_PER_SECOND); } function getVerticalDragDirection({ @@ -297,15 +299,17 @@ export function useElementInteraction({ zoomLevel, scrollLeft, }); - const adjustedTime = - mouseTime - pendingDragRef.current.clickOffsetTime; - const snappedTime = snapTimeToFrame({ - time: adjustedTime, - fps: activeProject.settings.fps, - }); - startDrag({ - ...pendingDragRef.current, - initialCurrentTime: snappedTime, + const adjustedTime = Math.max( + 0, + mouseTime - pendingDragRef.current.clickOffsetTime, + ); + const snappedTime = roundToFrame({ + time: adjustedTime, + rate: activeProject.settings.fps, + }) ?? adjustedTime; + startDrag({ + ...pendingDragRef.current, + initialCurrentTime: snappedTime, initialCurrentMouseY: clientY, }); startedDragThisEvent = true; @@ -343,9 +347,9 @@ export function useElementInteraction({ zoomLevel, scrollLeft, }); - const adjustedTime = mouseTime - dragState.clickOffsetTime; + const adjustedTime = Math.max(0, mouseTime - dragState.clickOffsetTime); const fps = activeProject.settings.fps; - const frameSnappedTime = snapTimeToFrame({ time: adjustedTime, fps }); + const frameSnappedTime = roundToFrame({ time: adjustedTime, rate: fps }) ?? adjustedTime; const sourceTrack = tracks.find(({ id }) => id === dragState.trackId); const movingElement = sourceTrack?.elements.find( diff --git a/apps/web/src/hooks/timeline/element/use-element-resize.ts b/apps/web/src/hooks/timeline/element/use-element-resize.ts index 3dd98bea..dec87eef 100644 --- a/apps/web/src/hooks/timeline/element/use-element-resize.ts +++ b/apps/web/src/hooks/timeline/element/use-element-resize.ts @@ -1,6 +1,7 @@ import { useState, useEffect, useRef, useCallback } from "react"; import { BASE_TIMELINE_PIXELS_PER_SECOND } from "@/lib/timeline/scale"; -import { snapTimeToFrame } from "opencut-wasm"; +import { TICKS_PER_SECOND } from "@/lib/wasm"; +import { roundToFrame } from "opencut-wasm"; import type { TimelineElement, TimelineTrack } from "@/lib/timeline"; import { useEditor } from "@/hooks/use-editor"; import { useShiftKey } from "@/hooks/use-shift-key"; @@ -185,13 +186,14 @@ export function useTimelineElementResize({ ({ clientX }: { clientX: number }) => { if (!resizing) return; - const deltaX = clientX - resizing.startX; - let deltaTime = - deltaX / (BASE_TIMELINE_PIXELS_PER_SECOND * zoomLevel); - let resizeSnapPoint: SnapPoint | null = null; + const deltaX = clientX - resizing.startX; + let deltaTime = Math.round( + (deltaX / (BASE_TIMELINE_PIXELS_PER_SECOND * zoomLevel)) * TICKS_PER_SECOND, + ); + let resizeSnapPoint: SnapPoint | null = null; - const projectFps = editor.project.getActive().settings.fps; - const minDurationSeconds = 1 / projectFps; + const projectFps = editor.project.getActive().settings.fps; + const minDuration = Math.round(TICKS_PER_SECOND * projectFps.denominator / projectFps.numerator); const shouldSnap = snappingEnabled && !isShiftHeldRef.current; if (shouldSnap) { const tracks = editor.timeline.getTracks(); @@ -277,19 +279,19 @@ export function useTimelineElementResize({ const maxAllowed = sourceDuration - resizing.initialTrimEnd - - getVisibleSourceSpanForDuration(minDurationSeconds); + getVisibleSourceSpanForDuration(minDuration); const calculated = resizing.initialTrimStart + getSourceDeltaForClipDelta(deltaTime); if (calculated >= 0 && calculated <= maxAllowed) { - const newTrimStart = snapTimeToFrame({ time: Math.min(maxAllowed, Math.max(minTrimStartForNeighbor, calculated)), fps: projectFps }); + const newTrimStart = roundToFrame({ time: Math.min(maxAllowed, Math.max(minTrimStartForNeighbor, calculated)), rate: projectFps }) ?? Math.min(maxAllowed, Math.max(minTrimStartForNeighbor, calculated)); const visibleSourceSpan = Math.max( 0, sourceDuration - newTrimStart - resizing.initialTrimEnd, ); - const newDuration = snapTimeToFrame({ time: getDurationForVisibleSourceSpan(visibleSourceSpan), fps: projectFps }); + const newDuration = roundToFrame({ time: getDurationForVisibleSourceSpan(visibleSourceSpan), rate: projectFps }) ?? getDurationForVisibleSourceSpan(visibleSourceSpan); const trimDelta = resizing.initialDuration - newDuration; - const newStartTime = snapTimeToFrame({ time: resizing.initialStartTime + trimDelta, fps: projectFps }); + const newStartTime = roundToFrame({ time: resizing.initialStartTime + trimDelta, rate: projectFps }) ?? resizing.initialStartTime + trimDelta; setCurrentTrimStart(newTrimStart); setCurrentStartTime(newStartTime); @@ -311,8 +313,8 @@ export function useTimelineElementResize({ ) : Math.min(extensionAmount, maxExtension), ); - const newStartTime = snapTimeToFrame({ time: resizing.initialStartTime - actualExtension, fps: projectFps }); - const newDuration = snapTimeToFrame({ time: resizing.initialDuration + actualExtension, fps: projectFps }); + const newStartTime = roundToFrame({ time: resizing.initialStartTime - actualExtension, rate: projectFps }) ?? resizing.initialStartTime - actualExtension; + const newDuration = roundToFrame({ time: resizing.initialDuration + actualExtension, rate: projectFps }) ?? resizing.initialDuration + actualExtension; setCurrentTrimStart(0); setCurrentStartTime(newStartTime); @@ -338,8 +340,8 @@ export function useTimelineElementResize({ 0, sourceDuration - newTrimStart - resizing.initialTrimEnd, ); - const newDuration = snapTimeToFrame({ time: getDurationForVisibleSourceSpan(visibleSourceSpan), fps: projectFps }); - const newStartTime = snapTimeToFrame({ time: resizing.initialStartTime + (resizing.initialDuration - newDuration), fps: projectFps }); + const newDuration = roundToFrame({ time: getDurationForVisibleSourceSpan(visibleSourceSpan), rate: projectFps }) ?? getDurationForVisibleSourceSpan(visibleSourceSpan); + const newStartTime = roundToFrame({ time: resizing.initialStartTime + (resizing.initialDuration - newDuration), rate: projectFps }) ?? resizing.initialStartTime + (resizing.initialDuration - newDuration); setCurrentTrimStart(newTrimStart); setCurrentStartTime(newStartTime); @@ -366,7 +368,7 @@ export function useTimelineElementResize({ const extensionNeeded = Math.abs(newTrimEnd); const baseDuration = resizing.initialDuration + resizing.initialTrimEnd; - const newDuration = snapTimeToFrame({ time: Math.min(baseDuration + extensionNeeded, maxAllowedDuration), fps: projectFps }); + const newDuration = roundToFrame({ time: Math.min(baseDuration + extensionNeeded, maxAllowedDuration), rate: projectFps }) ?? Math.min(baseDuration + extensionNeeded, maxAllowedDuration); setCurrentDuration(newDuration); setCurrentTrimEnd(0); @@ -376,7 +378,7 @@ export function useTimelineElementResize({ const unclampedDuration = getDurationForVisibleSourceSpan( Math.max(0, sourceDuration - resizing.initialTrimStart), ); - const newDuration = snapTimeToFrame({ time: Math.min(unclampedDuration, maxAllowedDuration), fps: projectFps }); + const newDuration = roundToFrame({ time: Math.min(unclampedDuration, maxAllowedDuration), rate: projectFps }) ?? Math.min(unclampedDuration, maxAllowedDuration); setCurrentDuration(newDuration); setCurrentTrimEnd(0); @@ -395,17 +397,17 @@ export function useTimelineElementResize({ const maxTrimEnd = sourceDuration - resizing.initialTrimStart - - getVisibleSourceSpanForDuration(minDurationSeconds); + getVisibleSourceSpanForDuration(minDuration); const clampedTrimEnd = Math.min( maxTrimEnd, Math.max(minTrimEndForNeighbor, newTrimEnd), ); - const finalTrimEnd = snapTimeToFrame({ time: clampedTrimEnd, fps: projectFps }); + const finalTrimEnd = roundToFrame({ time: clampedTrimEnd, rate: projectFps }) ?? clampedTrimEnd; const visibleSourceSpan = Math.max( 0, sourceDuration - resizing.initialTrimStart - finalTrimEnd, ); - const newDuration = snapTimeToFrame({ time: getDurationForVisibleSourceSpan(visibleSourceSpan), fps: projectFps }); + const newDuration = roundToFrame({ time: getDurationForVisibleSourceSpan(visibleSourceSpan), rate: projectFps }) ?? getDurationForVisibleSourceSpan(visibleSourceSpan); setCurrentTrimEnd(finalTrimEnd); setCurrentDuration(newDuration); diff --git a/apps/web/src/hooks/timeline/element/use-keyframe-drag.ts b/apps/web/src/hooks/timeline/element/use-keyframe-drag.ts index b66bbea8..24bff26a 100644 --- a/apps/web/src/hooks/timeline/element/use-keyframe-drag.ts +++ b/apps/web/src/hooks/timeline/element/use-keyframe-drag.ts @@ -8,9 +8,10 @@ import { import { useEditor } from "@/hooks/use-editor"; import { getKeyframeById } from "@/lib/animation"; import { useKeyframeSelection } from "./use-keyframe-selection"; -import { snapTimeToFrame, getSnappedSeekTime } from "opencut-wasm"; +import { roundToFrame, snappedSeekTime } from "opencut-wasm"; import { timelineTimeToSnappedPixels } from "@/lib/timeline"; import { BASE_TIMELINE_PIXELS_PER_SECOND } from "@/lib/timeline/scale"; +import { TICKS_PER_SECOND } from "@/lib/wasm"; import { TIMELINE_DRAG_THRESHOLD_PX } from "@/components/editor/panels/timeline/interaction"; import { RetimeKeyframeCommand } from "@/lib/commands/timeline/element/keyframes/retime-keyframe"; import { BatchCommand } from "@/lib/commands"; @@ -144,9 +145,9 @@ export function useKeyframeDrag({ if (!dragState.isDragging) return; - const startX = mouseDownXRef.current ?? clientX; - const rawDelta = (clientX - startX) / pixelsPerSecond; - const snappedDelta = snapTimeToFrame({ time: rawDelta, fps }); + const startX = mouseDownXRef.current ?? clientX; + const rawDelta = Math.round(((clientX - startX) / pixelsPerSecond) * TICKS_PER_SECOND); + const snappedDelta = roundToFrame({ time: rawDelta, rate: fps }) ?? rawDelta; setDragState((previous) => ({ ...previous, deltaTime: snappedDelta })); }; @@ -255,7 +256,7 @@ export function useKeyframeDrag({ if (wasDrag) return; const duration = editor.timeline.getTotalDuration(); - const seekTime = getSnappedSeekTime({ rawTime: displayedStartTime + indicatorTime, duration, fps }); + const seekTime = snappedSeekTime({ time: displayedStartTime + indicatorTime, duration, rate: fps }) ?? displayedStartTime + indicatorTime; editor.playback.seek({ time: seekTime }); if (event.shiftKey) { diff --git a/apps/web/src/hooks/timeline/use-bookmark-drag.ts b/apps/web/src/hooks/timeline/use-bookmark-drag.ts index 14d58d9a..90edbede 100644 --- a/apps/web/src/hooks/timeline/use-bookmark-drag.ts +++ b/apps/web/src/hooks/timeline/use-bookmark-drag.ts @@ -8,7 +8,7 @@ import { import { useEditor } from "@/hooks/use-editor"; import { useShiftKey } from "@/hooks/use-shift-key"; import { TIMELINE_DRAG_THRESHOLD_PX } from "@/components/editor/panels/timeline/interaction"; -import { snapTimeToFrame } from "opencut-wasm"; +import { roundToFrame } from "opencut-wasm"; import { getMouseTimeFromClientX } from "@/lib/timeline/drag-utils"; import { findSnapPoints, @@ -148,7 +148,7 @@ export function useBookmarkDrag({ zoomLevel, scrollLeft, }); - const frameSnappedTime = snapTimeToFrame({ time: Math.max(0, Math.min(mouseTime, duration)), fps: activeProject.settings.fps }); + const frameSnappedTime = roundToFrame({ time: Math.max(0, Math.min(mouseTime, duration)), rate: activeProject.settings.fps }) ?? Math.max(0, Math.min(mouseTime, duration)); const { snappedTime: initialTime } = getSnapResult({ rawTime: frameSnappedTime, excludeBookmarkTime: bookmarkTime, @@ -176,9 +176,9 @@ export function useBookmarkDrag({ scrollLeft, }); const clampedTime = Math.max(0, Math.min(mouseTime, duration)); - const frameSnappedTime = snapTimeToFrame({ time: clampedTime, fps: activeProject.settings.fps }); - const snapResult = getSnapResult({ - rawTime: frameSnappedTime, + const frameSnappedTime = roundToFrame({ time: clampedTime, rate: activeProject.settings.fps }) ?? clampedTime; + const snapResult = getSnapResult({ + rawTime: frameSnappedTime, excludeBookmarkTime: dragState.bookmarkTime, }); diff --git a/apps/web/src/hooks/timeline/use-timeline-drag-drop.ts b/apps/web/src/hooks/timeline/use-timeline-drag-drop.ts index 8e87510c..7b9ee2ee 100644 --- a/apps/web/src/hooks/timeline/use-timeline-drag-drop.ts +++ b/apps/web/src/hooks/timeline/use-timeline-drag-drop.ts @@ -3,9 +3,9 @@ import { useEditor } from "@/hooks/use-editor"; import { processMediaAssets } from "@/lib/media/processing"; import { toast } from "sonner"; import { showMediaUploadToast } from "@/lib/media/upload-toast"; -import { DEFAULT_NEW_ELEMENT_DURATION_SECONDS } from "@/lib/timeline/creation"; +import { DEFAULT_NEW_ELEMENT_DURATION } from "@/lib/timeline/creation"; import { BASE_TIMELINE_PIXELS_PER_SECOND } from "@/lib/timeline/scale"; -import { snapTimeToFrame } from "opencut-wasm"; +import { roundToFrame } from "opencut-wasm"; import { buildTextElement, buildGraphicElement, @@ -47,7 +47,7 @@ export function useTimelineDragDrop({ const getSnappedTime = useCallback( ({ time }: { time: number }) => { const projectFps = editor.project.getActive().settings.fps; - return snapTimeToFrame({ time, fps: projectFps }); + return roundToFrame({ time, rate: projectFps }) ?? time; }, [editor], ); @@ -83,14 +83,14 @@ export function useTimelineDragDrop({ elementType === "sticker" || elementType === "effect" ) { - return DEFAULT_NEW_ELEMENT_DURATION_SECONDS; + return DEFAULT_NEW_ELEMENT_DURATION; } if (mediaId) { const mediaAssets = editor.media.getAssets(); const media = mediaAssets.find((m) => m.id === mediaId); - return media?.duration ?? DEFAULT_NEW_ELEMENT_DURATION_SECONDS; + return media?.duration ?? DEFAULT_NEW_ELEMENT_DURATION; } - return DEFAULT_NEW_ELEMENT_DURATION_SECONDS; + return DEFAULT_NEW_ELEMENT_DURATION; }, [editor], ); @@ -331,7 +331,7 @@ export function useTimelineDragDrop({ dragData.mediaType === "audio" ? "audio" : "video"; const duration = - mediaAsset.duration ?? DEFAULT_NEW_ELEMENT_DURATION_SECONDS; + mediaAsset.duration ?? DEFAULT_NEW_ELEMENT_DURATION; const element = buildElementFromMedia({ mediaId: mediaAsset.id, mediaType: mediaAsset.type, @@ -446,7 +446,7 @@ export function useTimelineDragDrop({ const duration = createdAsset.duration ?? - DEFAULT_NEW_ELEMENT_DURATION_SECONDS; + DEFAULT_NEW_ELEMENT_DURATION; const currentTracks = editor.timeline.getTracks(); const currentTime = editor.playback.getCurrentTime(); const onlyTrack = currentTracks[0]; diff --git a/apps/web/src/hooks/timeline/use-timeline-playhead.ts b/apps/web/src/hooks/timeline/use-timeline-playhead.ts index fce84135..70364e90 100644 --- a/apps/web/src/hooks/timeline/use-timeline-playhead.ts +++ b/apps/web/src/hooks/timeline/use-timeline-playhead.ts @@ -1,4 +1,5 @@ -import { getSnappedSeekTime } from "opencut-wasm"; +import { snappedSeekTime } from "opencut-wasm"; +import { TICKS_PER_SECOND } from "@/lib/wasm"; import { useEffect, useCallback, useRef } from "react"; import { useEdgeAutoScroll } from "@/hooks/timeline/use-edge-auto-scroll"; import { useEditor } from "../use-editor"; @@ -6,6 +7,7 @@ import { useShiftKey } from "@/hooks/use-shift-key"; import { findSnapPoints, snapToNearestPoint } from "@/lib/timeline/snap-utils"; import { getCenteredLineLeft, + timelineTimeToPixels, timelineTimeToSnappedPixels, } from "@/lib/timeline"; import { BASE_TIMELINE_PIXELS_PER_SECOND } from "@/lib/timeline/scale"; @@ -66,24 +68,24 @@ export function useTimelinePlayhead({ const rulerRect = ruler.getBoundingClientRect(); const relativeMouseX = event.clientX - rulerRect.left; - const timelineContentWidth = - duration * BASE_TIMELINE_PIXELS_PER_SECOND * zoomLevel; + const timelineContentWidth = timelineTimeToPixels({ time: duration, zoomLevel }); - const clampedMouseX = Math.max( - 0, - Math.min(timelineContentWidth, relativeMouseX), - ); + const clampedMouseX = Math.max( + 0, + Math.min(timelineContentWidth, relativeMouseX), + ); - const rawTime = Math.max( - 0, - Math.min( - duration, - clampedMouseX / (BASE_TIMELINE_PIXELS_PER_SECOND * zoomLevel), - ), - ); + const rawTimeSeconds = Math.max( + 0, + Math.min( + duration / TICKS_PER_SECOND, + clampedMouseX / (BASE_TIMELINE_PIXELS_PER_SECOND * zoomLevel), + ), + ); + const rawTime = Math.round(rawTimeSeconds * TICKS_PER_SECOND); - const framesPerSecond = activeProject.settings.fps; - const frameTime = getSnappedSeekTime({ rawTime, duration, fps: framesPerSecond }); + const rate = activeProject.settings.fps; + const frameTime = snappedSeekTime({ time: rawTime, duration, rate }) ?? rawTime; const shouldSnap = snappingEnabled && !isShiftHeldRef.current; const time = (() => { @@ -161,7 +163,7 @@ export function useTimelinePlayhead({ getMouseClientX: () => lastMouseXRef.current, rulerScrollRef, tracksScrollRef, - contentWidth: duration * BASE_TIMELINE_PIXELS_PER_SECOND * zoomLevel, + contentWidth: timelineTimeToPixels({ time: duration, zoomLevel }), }); useEffect(() => { @@ -247,8 +249,10 @@ export function useTimelinePlayhead({ const tracksViewport = tracksScrollRef.current; if (!rulerViewport || !tracksViewport) return; - const playheadPixels = - time * BASE_TIMELINE_PIXELS_PER_SECOND * zoomLevelRef.current; + const playheadPixels = timelineTimeToPixels({ + time, + zoomLevel: zoomLevelRef.current, + }); const viewportWidth = rulerViewport.clientWidth; const scrollMinimum = 0; const scrollMaximum = rulerViewport.scrollWidth - viewportWidth; diff --git a/apps/web/src/hooks/timeline/use-timeline-seek.ts b/apps/web/src/hooks/timeline/use-timeline-seek.ts index 1690153e..756ec7ee 100644 --- a/apps/web/src/hooks/timeline/use-timeline-seek.ts +++ b/apps/web/src/hooks/timeline/use-timeline-seek.ts @@ -1,7 +1,8 @@ import { useCallback, useRef } from "react"; import type { MutableRefObject, RefObject } from "react"; import { BASE_TIMELINE_PIXELS_PER_SECOND } from "@/lib/timeline/scale"; -import { getSnappedSeekTime } from "opencut-wasm"; +import { snappedSeekTime } from "opencut-wasm"; +import { TICKS_PER_SECOND } from "@/lib/wasm"; import { useEditor } from "../use-editor"; interface UseTimelineSeekProps { @@ -127,17 +128,18 @@ export function useTimelineSeek({ const mouseX = event.clientX - rect.left; const scrollLeft = scrollContainer.scrollLeft; - const rawTime = Math.max( - 0, - Math.min( - duration, - (mouseX + scrollLeft) / - (BASE_TIMELINE_PIXELS_PER_SECOND * zoomLevel), - ), - ); + const rawTimeSeconds = Math.max( + 0, + Math.min( + duration / TICKS_PER_SECOND, + (mouseX + scrollLeft) / + (BASE_TIMELINE_PIXELS_PER_SECOND * zoomLevel), + ), + ); + const rawTime = Math.round(rawTimeSeconds * TICKS_PER_SECOND); - const projectFps = activeProject?.settings.fps || 30; - const time = getSnappedSeekTime({ rawTime, duration, fps: projectFps }); + const rate = activeProject?.settings.fps; + const time = rate ? (snappedSeekTime({ time: rawTime, duration, rate }) ?? rawTime) : rawTime; seek(time); editor.project.setTimelineViewState({ viewState: { @@ -154,7 +156,8 @@ export function useTimelineSeek({ tracksScrollRef, seek, editor, - activeProject?.settings.fps, + activeProject?.settings.fps.numerator, + activeProject?.settings.fps.denominator, ], ); diff --git a/apps/web/src/hooks/timeline/use-timeline-zoom.ts b/apps/web/src/hooks/timeline/use-timeline-zoom.ts index a1740384..1f79a5b2 100644 --- a/apps/web/src/hooks/timeline/use-timeline-zoom.ts +++ b/apps/web/src/hooks/timeline/use-timeline-zoom.ts @@ -13,8 +13,8 @@ import { import { TIMELINE_ZOOM_MAX, TIMELINE_ZOOM_MIN, - BASE_TIMELINE_PIXELS_PER_SECOND, } from "@/lib/timeline/scale"; +import { timelineTimeToPixels } from "@/lib/timeline/pixel-utils"; import { useEditor } from "@/hooks/use-editor"; import { zoomToSlider } from "@/lib/timeline/zoom-utils"; @@ -186,10 +186,8 @@ export function useTimelineZoom({ } if (sliderPercent >= TIMELINE_ZOOM_ANCHOR_PLAYHEAD_THRESHOLD) { - const playheadPixelsBefore = - playheadTime * BASE_TIMELINE_PIXELS_PER_SECOND * previousZoom; - const playheadPixelsAfter = - playheadTime * BASE_TIMELINE_PIXELS_PER_SECOND * zoomLevel; + const playheadPixelsBefore = timelineTimeToPixels({ time: playheadTime, zoomLevel: previousZoom }); + const playheadPixelsAfter = timelineTimeToPixels({ time: playheadTime, zoomLevel }); const viewportOffset = playheadPixelsBefore - currentScrollLeft; const newScrollLeft = playheadPixelsAfter - viewportOffset; diff --git a/apps/web/src/hooks/use-paste-media.ts b/apps/web/src/hooks/use-paste-media.ts index 1cba34c0..36d70ebe 100644 --- a/apps/web/src/hooks/use-paste-media.ts +++ b/apps/web/src/hooks/use-paste-media.ts @@ -1,114 +1,114 @@ -import { useEffect } from "react"; -import { useEditor } from "@/hooks/use-editor"; -import { processMediaAssets } from "@/lib/media/processing"; -import { showMediaUploadToast } from "@/lib/media/upload-toast"; -import { invokeAction } from "@/lib/actions"; -import { buildElementFromMedia } from "@/lib/timeline/element-utils"; -import { AddMediaAssetCommand } from "@/lib/commands/media"; -import { InsertElementCommand } from "@/lib/commands/timeline"; -import { BatchCommand } from "@/lib/commands"; -import { DEFAULT_NEW_ELEMENT_DURATION_SECONDS } from "@/lib/timeline/creation"; -import { isTypableDOMElement } from "@/utils/browser"; -import type { MediaType } from "@/lib/media/types"; - -const MEDIA_MIME_PREFIXES: MediaType[] = ["image", "video", "audio"]; - -function isMediaMimeType({ type }: { type: string }): boolean { - return MEDIA_MIME_PREFIXES.some((prefix) => type.startsWith(`${prefix}/`)); -} - -function extractMediaFilesFromClipboard({ - clipboardData, -}: { - clipboardData: DataTransfer | null; -}): File[] { - if (!clipboardData?.items) return []; - - const files: File[] = []; - for (const item of clipboardData.items) { - if (item.kind !== "file") continue; - if (!isMediaMimeType({ type: item.type })) continue; - - const file = item.getAsFile(); - if (file) files.push(file); - } - return files; -} - -export function usePasteMedia() { - const editor = useEditor(); - - useEffect(() => { - const handlePaste = async (event: ClipboardEvent) => { - const activeElement = document.activeElement as HTMLElement; - - if (activeElement && isTypableDOMElement({ element: activeElement })) { - return; - } - - const files = extractMediaFilesFromClipboard({ - clipboardData: event.clipboardData, - }); - if (files.length === 0) { - event.preventDefault(); - invokeAction("paste-copied"); - return; - } - - event.preventDefault(); - - const activeProject = editor.project.getActive(); - if (!activeProject) return; - - try { - await showMediaUploadToast({ - filesCount: files.length, - promise: async () => { - const processedAssets = await processMediaAssets({ files }); - const startTime = editor.playback.getCurrentTime(); - - for (const asset of processedAssets) { - const addMediaCmd = new AddMediaAssetCommand( - activeProject.metadata.id, - asset, - ); - const assetId = addMediaCmd.getAssetId(); - const duration = - asset.duration ?? DEFAULT_NEW_ELEMENT_DURATION_SECONDS; - const trackType = asset.type === "audio" ? "audio" : "video"; - - const element = buildElementFromMedia({ - mediaId: assetId, - mediaType: asset.type, - name: asset.name, - duration, - startTime, - buffer: - asset.type === "audio" - ? new AudioBuffer({ length: 1, sampleRate: 44100 }) - : undefined, - }); - - const insertCmd = new InsertElementCommand({ - element, - placement: { mode: "auto", trackType }, - }); - const batchCmd = new BatchCommand([addMediaCmd, insertCmd]); - editor.command.execute({ command: batchCmd }); - } - - return { - uploadedCount: processedAssets.length, - assetNames: processedAssets.map((asset) => asset.name), - }; - }, - }); - } catch (error) { - console.error("Failed to paste media:", error); - } - }; - - window.addEventListener("paste", handlePaste); - return () => window.removeEventListener("paste", handlePaste); - }, [editor]); -} +import { useEffect } from "react"; +import { useEditor } from "@/hooks/use-editor"; +import { processMediaAssets } from "@/lib/media/processing"; +import { showMediaUploadToast } from "@/lib/media/upload-toast"; +import { invokeAction } from "@/lib/actions"; +import { buildElementFromMedia } from "@/lib/timeline/element-utils"; +import { AddMediaAssetCommand } from "@/lib/commands/media"; +import { InsertElementCommand } from "@/lib/commands/timeline"; +import { BatchCommand } from "@/lib/commands"; +import { DEFAULT_NEW_ELEMENT_DURATION } from "@/lib/timeline/creation"; +import { isTypableDOMElement } from "@/utils/browser"; +import type { MediaType } from "@/lib/media/types"; + +const MEDIA_MIME_PREFIXES: MediaType[] = ["image", "video", "audio"]; + +function isMediaMimeType({ type }: { type: string }): boolean { + return MEDIA_MIME_PREFIXES.some((prefix) => type.startsWith(`${prefix}/`)); +} + +function extractMediaFilesFromClipboard({ + clipboardData, +}: { + clipboardData: DataTransfer | null; +}): File[] { + if (!clipboardData?.items) return []; + + const files: File[] = []; + for (const item of clipboardData.items) { + if (item.kind !== "file") continue; + if (!isMediaMimeType({ type: item.type })) continue; + + const file = item.getAsFile(); + if (file) files.push(file); + } + return files; +} + +export function usePasteMedia() { + const editor = useEditor(); + + useEffect(() => { + const handlePaste = async (event: ClipboardEvent) => { + const activeElement = document.activeElement as HTMLElement; + + if (activeElement && isTypableDOMElement({ element: activeElement })) { + return; + } + + const files = extractMediaFilesFromClipboard({ + clipboardData: event.clipboardData, + }); + if (files.length === 0) { + event.preventDefault(); + invokeAction("paste-copied"); + return; + } + + event.preventDefault(); + + const activeProject = editor.project.getActive(); + if (!activeProject) return; + + try { + await showMediaUploadToast({ + filesCount: files.length, + promise: async () => { + const processedAssets = await processMediaAssets({ files }); + const startTime = editor.playback.getCurrentTime(); + + for (const asset of processedAssets) { + const addMediaCmd = new AddMediaAssetCommand( + activeProject.metadata.id, + asset, + ); + const assetId = addMediaCmd.getAssetId(); + const duration = + asset.duration ?? DEFAULT_NEW_ELEMENT_DURATION; + const trackType = asset.type === "audio" ? "audio" : "video"; + + const element = buildElementFromMedia({ + mediaId: assetId, + mediaType: asset.type, + name: asset.name, + duration, + startTime, + buffer: + asset.type === "audio" + ? new AudioBuffer({ length: 1, sampleRate: 44100 }) + : undefined, + }); + + const insertCmd = new InsertElementCommand({ + element, + placement: { mode: "auto", trackType }, + }); + const batchCmd = new BatchCommand([addMediaCmd, insertCmd]); + editor.command.execute({ command: batchCmd }); + } + + return { + uploadedCount: processedAssets.length, + assetNames: processedAssets.map((asset) => asset.name), + }; + }, + }); + } catch (error) { + console.error("Failed to paste media:", error); + } + }; + + window.addEventListener("paste", handlePaste); + return () => window.removeEventListener("paste", handlePaste); + }, [editor]); +} diff --git a/apps/web/src/lib/animation/curve-bridge.ts b/apps/web/src/lib/animation/curve-bridge.ts index 55f96933..12103c3f 100644 --- a/apps/web/src/lib/animation/curve-bridge.ts +++ b/apps/web/src/lib/animation/curve-bridge.ts @@ -1,4 +1,3 @@ -import { TIME_EPSILON_SECONDS } from "@/constants/animation-constants"; import { getDefaultLeftHandle, getDefaultRightHandle, @@ -25,7 +24,7 @@ export function getNormalizedCubicBezierForScalarSegment({ const spanTime = rightKey.time - leftKey.time; const spanValue = rightKey.value - leftKey.value; if ( - Math.abs(spanTime) <= TIME_EPSILON_SECONDS || + spanTime === 0 || Math.abs(spanValue) <= VALUE_EPSILON ) { return null; @@ -59,7 +58,7 @@ export function getCurveHandlesForNormalizedCubicBezier({ const spanTime = rightKey.time - leftKey.time; const spanValue = rightKey.value - leftKey.value; if ( - Math.abs(spanTime) <= TIME_EPSILON_SECONDS || + spanTime === 0 || Math.abs(spanValue) <= VALUE_EPSILON ) { return null; diff --git a/apps/web/src/lib/animation/interpolation.ts b/apps/web/src/lib/animation/interpolation.ts index 67456aa0..1a3c0fed 100644 --- a/apps/web/src/lib/animation/interpolation.ts +++ b/apps/web/src/lib/animation/interpolation.ts @@ -8,7 +8,6 @@ import type { ScalarAnimationKey, ScalarSegmentType, } from "@/lib/animation/types"; -import { TIME_EPSILON_SECONDS } from "@/constants/animation-constants"; import { clamp } from "@/utils/math"; import { getBezierPoint, @@ -36,10 +35,7 @@ function isWithinTimePair({ leftTime: number; rightTime: number; }): boolean { - return ( - time >= leftTime - TIME_EPSILON_SECONDS && - time <= rightTime + TIME_EPSILON_SECONDS - ); + return time >= leftTime && time <= rightTime; } function lerpNumber({ @@ -67,7 +63,7 @@ function normalizeRightHandle({ return undefined; } - const span = Math.max(TIME_EPSILON_SECONDS, rightKey.time - leftKey.time); + const span = Math.max(1, rightKey.time - leftKey.time); return { dt: Math.min(span, Math.max(0, handle.dt)), dv: handle.dv, @@ -87,7 +83,7 @@ function normalizeLeftHandle({ return undefined; } - const span = Math.max(TIME_EPSILON_SECONDS, rightKey.time - leftKey.time); + const span = Math.max(1, rightKey.time - leftKey.time); return { dt: Math.max(-span, Math.min(0, handle.dt)), dv: handle.dv, @@ -187,7 +183,7 @@ function extrapolateScalarEdge({ } const span = neighborKey.time - edgeKey.time; - if (Math.abs(span) <= TIME_EPSILON_SECONDS) { + if (span === 0) { return edgeKey.value; } @@ -228,8 +224,8 @@ export function getScalarChannelValueAtTime({ return fallbackValue; } - if (time <= firstKey.time + TIME_EPSILON_SECONDS) { - if (time < firstKey.time - TIME_EPSILON_SECONDS) { + if (time <= firstKey.time) { + if (time < firstKey.time) { return extrapolateScalarEdge({ mode: normalizedChannel.extrapolation?.before ?? "hold", edgeKey: firstKey, @@ -241,8 +237,8 @@ export function getScalarChannelValueAtTime({ return firstKey.value; } - if (time >= lastKey.time - TIME_EPSILON_SECONDS) { - if (time > lastKey.time + TIME_EPSILON_SECONDS) { + if (time >= lastKey.time) { + if (time > lastKey.time) { return extrapolateScalarEdge({ mode: normalizedChannel.extrapolation?.after ?? "hold", edgeKey: lastKey, @@ -261,7 +257,7 @@ export function getScalarChannelValueAtTime({ ) { const leftKey = normalizedChannel.keys[keyIndex]; const rightKey = normalizedChannel.keys[keyIndex + 1]; - if (Math.abs(time - rightKey.time) <= TIME_EPSILON_SECONDS) { + if (time === rightKey.time) { return rightKey.value; } @@ -280,7 +276,7 @@ export function getScalarChannelValueAtTime({ } const span = rightKey.time - leftKey.time; - if (Math.abs(span) <= TIME_EPSILON_SECONDS) { + if (span === 0) { return rightKey.value; } @@ -336,7 +332,7 @@ export function getDiscreteChannelValueAtTime({ }); let currentValue = fallbackValue; for (const key of normalizedChannel.keys) { - if (time + TIME_EPSILON_SECONDS < key.time) { + if (time < key.time) { break; } currentValue = key.value; diff --git a/apps/web/src/lib/animation/keyframe-query.ts b/apps/web/src/lib/animation/keyframe-query.ts index 304a6698..dbcd041c 100644 --- a/apps/web/src/lib/animation/keyframe-query.ts +++ b/apps/web/src/lib/animation/keyframe-query.ts @@ -1,4 +1,3 @@ -import { TIME_EPSILON_SECONDS } from "@/constants/animation-constants"; import type { AnimationBindingInstance, AnimationChannel, @@ -76,8 +75,7 @@ function getUniqueBindingKeyframeMatches({ const previousMatch = uniqueMatches[uniqueMatches.length - 1]; if ( !previousMatch || - Math.abs(previousMatch.keyframe.time - match.keyframe.time) > - TIME_EPSILON_SECONDS + previousMatch.keyframe.time !== match.keyframe.time ) { uniqueMatches.push(match); continue; @@ -249,9 +247,7 @@ export function getKeyframeAtTime({ matches: getBindingKeyframeMatches({ animations, binding, - }).filter(({ keyframe }) => - Math.abs(keyframe.time - time) <= TIME_EPSILON_SECONDS, - ), + }).filter(({ keyframe }) => keyframe.time === time), }); if (!keyframeMatch) { return null; diff --git a/apps/web/src/lib/animation/keyframes.ts b/apps/web/src/lib/animation/keyframes.ts index d72f9e16..05290c1d 100644 --- a/apps/web/src/lib/animation/keyframes.ts +++ b/apps/web/src/lib/animation/keyframes.ts @@ -14,7 +14,6 @@ import type { ScalarCurveKeyframePatch, ScalarSegmentType, } from "@/lib/animation/types"; -import { TIME_EPSILON_SECONDS } from "@/constants/animation-constants"; import { generateUUID } from "@/utils/id"; import { cloneAnimationBinding, @@ -44,7 +43,7 @@ function isNearlySameTime({ leftTime: number; rightTime: number; }): boolean { - return Math.abs(leftTime - rightTime) <= TIME_EPSILON_SECONDS; + return leftTime === rightTime; } function hasChannelKeys({ @@ -1032,8 +1031,8 @@ function splitScalarChannelAtTime({ const rightKey = normalizedChannel.keys[keyIndex + 1]; if ( !( - splitTime > leftKey.time + TIME_EPSILON_SECONDS && - splitTime < rightKey.time - TIME_EPSILON_SECONDS + splitTime > leftKey.time && + splitTime < rightKey.time ) ) { continue; diff --git a/apps/web/src/lib/commands/media/add-media-asset.ts b/apps/web/src/lib/commands/media/add-media-asset.ts index c12d7ea1..8b3067a1 100644 --- a/apps/web/src/lib/commands/media/add-media-asset.ts +++ b/apps/web/src/lib/commands/media/add-media-asset.ts @@ -4,16 +4,17 @@ import { toast } from "sonner"; import type { MediaAsset } from "@/lib/media/types"; import { generateUUID } from "@/utils/id"; import { storageService } from "@/services/storage/service"; +import type { FrameRate } from "opencut-wasm"; import { hasMediaId } from "@/lib/timeline/element-utils"; -import { getHighestImportedVideoFps } from "@/lib/fps/utils"; +import { frameRatesEqual, getHighestImportedVideoFps } from "@/lib/fps/utils"; import { UpdateProjectSettingsCommand } from "@/lib/commands/project"; export class AddMediaAssetCommand extends Command { private assetId: string; private savedAssets: MediaAsset[] | null = null; private createdAsset: MediaAsset | null = null; - private previousProjectFps: number | null = null; - private appliedProjectFps: number | null = null; + private previousProjectFps: FrameRate | null = null; + private appliedProjectFps: FrameRate | null = null; constructor( private projectId: string, @@ -112,14 +113,15 @@ export class AddMediaAssetCommand extends Command { const activeProject = editor.project.getActiveOrNull(); if (!activeProject) return; - if (activeProject.settings.fps !== this.appliedProjectFps) return; + if (!this.appliedProjectFps || !frameRatesEqual(activeProject.settings.fps, this.appliedProjectFps)) return; const highestRemainingVideoFps = getHighestImportedVideoFps({ mediaAssets: editor.media.getAssets(), }); + const appliedFpsFloat = this.appliedProjectFps.numerator / this.appliedProjectFps.denominator; if ( highestRemainingVideoFps !== null && - highestRemainingVideoFps >= this.appliedProjectFps + highestRemainingVideoFps >= appliedFpsFloat ) { return; } diff --git a/apps/web/src/lib/commands/timeline/element/insert-element.ts b/apps/web/src/lib/commands/timeline/element/insert-element.ts index fbef1613..9f0bf941 100644 --- a/apps/web/src/lib/commands/timeline/element/insert-element.ts +++ b/apps/web/src/lib/commands/timeline/element/insert-element.ts @@ -1,264 +1,265 @@ -import { Command, type CommandResult } from "@/lib/commands/base-command"; -import { EditorCore } from "@/core"; -import type { - CreateTimelineElement, - TimelineTrack, - TimelineElement, - TrackType, -} from "@/lib/timeline"; -import { generateUUID } from "@/utils/id"; -import { requiresMediaId } from "@/lib/timeline/element-utils"; -import type { MediaAsset } from "@/lib/media/types"; -import { DEFAULT_NEW_ELEMENT_DURATION_SECONDS } from "@/lib/timeline/creation"; -import { graphicsRegistry, registerDefaultGraphics } from "@/lib/graphics"; -import { - applyPlacement, - canElementGoOnTrack, - resolveTrackPlacement, - validateElementTrackCompatibility, -} from "@/lib/timeline/placement"; - -type InsertElementPlacement = - | { mode: "explicit"; trackId: string } - | { mode: "auto"; trackType?: TrackType; insertIndex?: number }; - -export interface InsertElementParams { - element: CreateTimelineElement; - placement: InsertElementPlacement; -} - -export class InsertElementCommand extends Command { - private elementId: string; - private savedState: TimelineTrack[] | null = null; - private targetTrackId: string | null = null; - - constructor({ element, placement }: InsertElementParams) { - super(); - this.elementId = generateUUID(); - this.element = element; - this.placement = placement; - } - - private element: CreateTimelineElement; - private placement: InsertElementPlacement; - - execute(): CommandResult | undefined { - const editor = EditorCore.getInstance(); - this.savedState = editor.timeline.getTracks(); - - if (!this.savedState) { - console.error("Tracks not available"); - return; - } - - if (!this.validateElementBasics({ element: this.element })) { - return; - } - - const totalElementsInTimeline = this.savedState.reduce( - (total, t) => total + t.elements.length, - 0, - ); - const isFirstElement = totalElementsInTimeline === 0; - - const newElement = this.buildElement({ element: this.element }); - const updateResult = this.applyPlacementResult({ - tracks: this.savedState, - element: newElement, - }); - - if (!updateResult) { - return; - } - - const { updatedTracks, targetTrackId } = updateResult; - this.targetTrackId = targetTrackId; - - const isVisualMedia = - newElement.type === "video" || newElement.type === "image"; - - if (isFirstElement && isVisualMedia) { - const mediaAssets = editor.media.getAssets(); - const activeProject = editor.project.getActive(); - const asset = mediaAssets.find( - (item: MediaAsset) => item.id === newElement.mediaId, - ); - - if (asset?.width && asset?.height) { - const nextCanvasSize = { width: asset.width, height: asset.height }; - const shouldSetOriginalCanvasSize = - !activeProject?.settings.originalCanvasSize; - editor.project.updateSettings({ - settings: { - canvasSize: nextCanvasSize, - ...(shouldSetOriginalCanvasSize - ? { originalCanvasSize: nextCanvasSize } - : {}), - }, - pushHistory: false, - }); - } - - if (asset?.type === "video" && asset?.fps) { - editor.project.updateSettings({ - settings: { fps: asset.fps }, - pushHistory: false, - }); - } - } - - editor.timeline.updateTracks(updatedTracks); - } - - undo(): void { - if (this.savedState) { - const editor = EditorCore.getInstance(); - editor.timeline.updateTracks(this.savedState); - } - } - - getElementId(): string { - return this.elementId; - } - - getTrackId(): string | null { - return this.targetTrackId; - } - - private buildElement({ - element, - }: { - element: CreateTimelineElement; - }): TimelineElement { - return { - ...element, - id: this.elementId, - startTime: element.startTime, - trimStart: element.trimStart ?? 0, - trimEnd: element.trimEnd ?? 0, - duration: element.duration ?? DEFAULT_NEW_ELEMENT_DURATION_SECONDS, - } as TimelineElement; - } - - private validateElementBasics({ - element, - }: { - element: CreateTimelineElement; - }): boolean { - if (requiresMediaId({ element }) && !("mediaId" in element)) { - console.error("Element requires mediaId"); - return false; - } - - if ( - element.type === "audio" && - element.sourceType === "library" && - !element.sourceUrl - ) { - console.error("Library audio element must have sourceUrl"); - return false; - } - - if (element.type === "sticker" && !element.stickerId) { - console.error("Sticker element must have stickerId"); - return false; - } - - if (element.type === "graphic") { - registerDefaultGraphics(); - if (!element.definitionId || !graphicsRegistry.has(element.definitionId)) { - console.error("Graphic element must have a valid definitionId"); - return false; - } - } - - if (element.type === "text" && !element.content) { - console.error("Text element must have content"); - return false; - } - - if (element.type === "effect" && !element.effectType) { - console.error("Effect element must have effectType"); - return false; - } - - return true; - } - - private applyPlacementResult({ - tracks, - element, - }: { - tracks: TimelineTrack[]; - element: TimelineElement; - }): { updatedTracks: TimelineTrack[]; targetTrackId: string } | null { - const placement = this.placement; - - if ( - placement.mode === "auto" && - placement.trackType && - !canElementGoOnTrack({ - elementType: element.type, - trackType: placement.trackType, - }) - ) { - console.error( - `${element.type} elements cannot be placed on ${placement.trackType} tracks`, - ); - return null; - } - - const placementResult = resolveTrackPlacement({ - tracks, - ...(placement.mode === "auto" && placement.trackType - ? { trackType: placement.trackType } - : { elementType: element.type }), - timeSpans: [ - { - startTime: element.startTime, - duration: element.duration, - }, - ], - strategy: - placement.mode === "explicit" - ? { type: "explicit", trackId: placement.trackId } - : { type: "firstAvailable" }, - }); - if (!placementResult) { - if (placement.mode === "explicit") { - const targetTrack = tracks.find((track) => track.id === placement.trackId); - if (!targetTrack) { - console.error("Track not found:", placement.trackId); - return null; - } - - const validation = validateElementTrackCompatibility({ - element, - track: targetTrack, - }); - console.error(validation.errorMessage); - } - - return null; - } - - const elementToPlace = - placementResult.kind === "existingTrack" - ? { - ...element, - startTime: - placementResult.adjustedStartTime ?? element.startTime, - } - : element; - - return applyPlacement({ - tracks, - placementResult, - elements: [elementToPlace], - newTrackInsertIndexOverride: - placement.mode === "auto" && typeof placement.insertIndex === "number" - ? placement.insertIndex - : undefined, - }); - } -} +import { Command, type CommandResult } from "@/lib/commands/base-command"; +import { EditorCore } from "@/core"; +import type { + CreateTimelineElement, + TimelineTrack, + TimelineElement, + TrackType, +} from "@/lib/timeline"; +import { generateUUID } from "@/utils/id"; +import { requiresMediaId } from "@/lib/timeline/element-utils"; +import type { MediaAsset } from "@/lib/media/types"; +import { DEFAULT_NEW_ELEMENT_DURATION } from "@/lib/timeline/creation"; +import { floatToFrameRate } from "@/lib/fps/utils"; +import { graphicsRegistry, registerDefaultGraphics } from "@/lib/graphics"; +import { + applyPlacement, + canElementGoOnTrack, + resolveTrackPlacement, + validateElementTrackCompatibility, +} from "@/lib/timeline/placement"; + +type InsertElementPlacement = + | { mode: "explicit"; trackId: string } + | { mode: "auto"; trackType?: TrackType; insertIndex?: number }; + +export interface InsertElementParams { + element: CreateTimelineElement; + placement: InsertElementPlacement; +} + +export class InsertElementCommand extends Command { + private elementId: string; + private savedState: TimelineTrack[] | null = null; + private targetTrackId: string | null = null; + + constructor({ element, placement }: InsertElementParams) { + super(); + this.elementId = generateUUID(); + this.element = element; + this.placement = placement; + } + + private element: CreateTimelineElement; + private placement: InsertElementPlacement; + + execute(): CommandResult | undefined { + const editor = EditorCore.getInstance(); + this.savedState = editor.timeline.getTracks(); + + if (!this.savedState) { + console.error("Tracks not available"); + return; + } + + if (!this.validateElementBasics({ element: this.element })) { + return; + } + + const totalElementsInTimeline = this.savedState.reduce( + (total, t) => total + t.elements.length, + 0, + ); + const isFirstElement = totalElementsInTimeline === 0; + + const newElement = this.buildElement({ element: this.element }); + const updateResult = this.applyPlacementResult({ + tracks: this.savedState, + element: newElement, + }); + + if (!updateResult) { + return; + } + + const { updatedTracks, targetTrackId } = updateResult; + this.targetTrackId = targetTrackId; + + const isVisualMedia = + newElement.type === "video" || newElement.type === "image"; + + if (isFirstElement && isVisualMedia) { + const mediaAssets = editor.media.getAssets(); + const activeProject = editor.project.getActive(); + const asset = mediaAssets.find( + (item: MediaAsset) => item.id === newElement.mediaId, + ); + + if (asset?.width && asset?.height) { + const nextCanvasSize = { width: asset.width, height: asset.height }; + const shouldSetOriginalCanvasSize = + !activeProject?.settings.originalCanvasSize; + editor.project.updateSettings({ + settings: { + canvasSize: nextCanvasSize, + ...(shouldSetOriginalCanvasSize + ? { originalCanvasSize: nextCanvasSize } + : {}), + }, + pushHistory: false, + }); + } + + if (asset?.type === "video" && asset?.fps) { + editor.project.updateSettings({ + settings: { fps: floatToFrameRate(asset.fps) }, + pushHistory: false, + }); + } + } + + editor.timeline.updateTracks(updatedTracks); + } + + undo(): void { + if (this.savedState) { + const editor = EditorCore.getInstance(); + editor.timeline.updateTracks(this.savedState); + } + } + + getElementId(): string { + return this.elementId; + } + + getTrackId(): string | null { + return this.targetTrackId; + } + + private buildElement({ + element, + }: { + element: CreateTimelineElement; + }): TimelineElement { + return { + ...element, + id: this.elementId, + startTime: element.startTime, + trimStart: element.trimStart ?? 0, + trimEnd: element.trimEnd ?? 0, + duration: element.duration ?? DEFAULT_NEW_ELEMENT_DURATION, + } as TimelineElement; + } + + private validateElementBasics({ + element, + }: { + element: CreateTimelineElement; + }): boolean { + if (requiresMediaId({ element }) && !("mediaId" in element)) { + console.error("Element requires mediaId"); + return false; + } + + if ( + element.type === "audio" && + element.sourceType === "library" && + !element.sourceUrl + ) { + console.error("Library audio element must have sourceUrl"); + return false; + } + + if (element.type === "sticker" && !element.stickerId) { + console.error("Sticker element must have stickerId"); + return false; + } + + if (element.type === "graphic") { + registerDefaultGraphics(); + if (!element.definitionId || !graphicsRegistry.has(element.definitionId)) { + console.error("Graphic element must have a valid definitionId"); + return false; + } + } + + if (element.type === "text" && !element.content) { + console.error("Text element must have content"); + return false; + } + + if (element.type === "effect" && !element.effectType) { + console.error("Effect element must have effectType"); + return false; + } + + return true; + } + + private applyPlacementResult({ + tracks, + element, + }: { + tracks: TimelineTrack[]; + element: TimelineElement; + }): { updatedTracks: TimelineTrack[]; targetTrackId: string } | null { + const placement = this.placement; + + if ( + placement.mode === "auto" && + placement.trackType && + !canElementGoOnTrack({ + elementType: element.type, + trackType: placement.trackType, + }) + ) { + console.error( + `${element.type} elements cannot be placed on ${placement.trackType} tracks`, + ); + return null; + } + + const placementResult = resolveTrackPlacement({ + tracks, + ...(placement.mode === "auto" && placement.trackType + ? { trackType: placement.trackType } + : { elementType: element.type }), + timeSpans: [ + { + startTime: element.startTime, + duration: element.duration, + }, + ], + strategy: + placement.mode === "explicit" + ? { type: "explicit", trackId: placement.trackId } + : { type: "firstAvailable" }, + }); + if (!placementResult) { + if (placement.mode === "explicit") { + const targetTrack = tracks.find((track) => track.id === placement.trackId); + if (!targetTrack) { + console.error("Track not found:", placement.trackId); + return null; + } + + const validation = validateElementTrackCompatibility({ + element, + track: targetTrack, + }); + console.error(validation.errorMessage); + } + + return null; + } + + const elementToPlace = + placementResult.kind === "existingTrack" + ? { + ...element, + startTime: + placementResult.adjustedStartTime ?? element.startTime, + } + : element; + + return applyPlacement({ + tracks, + placementResult, + elements: [elementToPlace], + newTrackInsertIndexOverride: + placement.mode === "auto" && typeof placement.insertIndex === "number" + ? placement.insertIndex + : undefined, + }); + } +} diff --git a/apps/web/src/lib/export.ts b/apps/web/src/lib/export.ts index 68d2bc7f..f397ac4b 100644 --- a/apps/web/src/lib/export.ts +++ b/apps/web/src/lib/export.ts @@ -1,3 +1,4 @@ +import type { FrameRate } from "opencut-wasm"; import { EXPORT_MIME_TYPES } from "@/constants/export-constants"; export const EXPORT_QUALITY_VALUES = [ @@ -15,7 +16,7 @@ export type ExportQuality = (typeof EXPORT_QUALITY_VALUES)[number]; export interface ExportOptions { format: ExportFormat; quality: ExportQuality; - fps?: number; + fps?: FrameRate; includeAudio?: boolean; } diff --git a/apps/web/src/lib/fps/__tests__/fps.test.ts b/apps/web/src/lib/fps/__tests__/fps.test.ts index fd082bb2..10eed8d9 100644 --- a/apps/web/src/lib/fps/__tests__/fps.test.ts +++ b/apps/web/src/lib/fps/__tests__/fps.test.ts @@ -36,16 +36,16 @@ describe("getRaisedProjectFpsForImportedMedia", () => { test("raises the project fps to match a higher-fps import", () => { expect( getRaisedProjectFpsForImportedMedia({ - currentFps: 30, + currentFps: { numerator: 30, denominator: 1 }, importedAssets: [{ type: "video", fps: 60 }], }), - ).toBe(60); + ).toEqual({ numerator: 60, denominator: 1 }); }); test("does not lower the project fps for lower-fps imports", () => { expect( getRaisedProjectFpsForImportedMedia({ - currentFps: 60, + currentFps: { numerator: 60, denominator: 1 }, importedAssets: [{ type: "video", fps: 10 }], }), ).toBeNull(); @@ -54,7 +54,7 @@ describe("getRaisedProjectFpsForImportedMedia", () => { test("ignores non-video imports", () => { expect( getRaisedProjectFpsForImportedMedia({ - currentFps: 30, + currentFps: { numerator: 30, denominator: 1 }, importedAssets: [ { type: "image", fps: 60 }, { type: "audio", fps: 120 }, diff --git a/apps/web/src/lib/fps/constants.ts b/apps/web/src/lib/fps/constants.ts index 100c043a..30c3d2bc 100644 --- a/apps/web/src/lib/fps/constants.ts +++ b/apps/web/src/lib/fps/constants.ts @@ -1,3 +1,5 @@ +import type { FrameRate } from "opencut-wasm"; + export const FPS_PRESETS = [ { value: "24", label: "24 fps" }, { value: "25", label: "25 fps" }, @@ -6,4 +8,4 @@ export const FPS_PRESETS = [ { value: "120", label: "120 fps" }, ] as const; -export const DEFAULT_FPS = 30; +export const DEFAULT_FPS: FrameRate = { numerator: 30, denominator: 1 }; diff --git a/apps/web/src/lib/fps/utils.ts b/apps/web/src/lib/fps/utils.ts index 8e3221a1..932788c4 100644 --- a/apps/web/src/lib/fps/utils.ts +++ b/apps/web/src/lib/fps/utils.ts @@ -1,7 +1,61 @@ +import type { FrameRate } from "opencut-wasm"; import type { MediaAsset } from "@/lib/media/types"; type MediaAssetFpsInput = Pick; +const STANDARD_FRAME_RATES: Array<{ value: number; rate: FrameRate }> = [ + { value: 24_000 / 1_001, rate: { numerator: 24_000, denominator: 1_001 } }, + { value: 24, rate: { numerator: 24, denominator: 1 } }, + { value: 25, rate: { numerator: 25, denominator: 1 } }, + { value: 30_000 / 1_001, rate: { numerator: 30_000, denominator: 1_001 } }, + { value: 30, rate: { numerator: 30, denominator: 1 } }, + { value: 48, rate: { numerator: 48, denominator: 1 } }, + { value: 50, rate: { numerator: 50, denominator: 1 } }, + { value: 60_000 / 1_001, rate: { numerator: 60_000, denominator: 1_001 } }, + { value: 60, rate: { numerator: 60, denominator: 1 } }, + { value: 120, rate: { numerator: 120, denominator: 1 } }, +]; + +const STANDARD_FRAME_RATE_TOLERANCE = 0.01; + +export function frameRateToFloat(rate: FrameRate): number { + return rate.numerator / rate.denominator; +} + +export function frameRatesEqual(a: FrameRate, b: FrameRate): boolean { + return a.numerator === b.numerator && a.denominator === b.denominator; +} + +export function floatToFrameRate(fps: number): FrameRate { + const standard = STANDARD_FRAME_RATES.find( + (candidate) => Math.abs(fps - candidate.value) <= STANDARD_FRAME_RATE_TOLERANCE, + ); + if (standard) return standard.rate; + + if (Number.isInteger(fps)) { + return { numerator: fps, denominator: 1 }; + } + + const ARBITRARY_DENOMINATOR = 1_000_000; + const scaledNumerator = Math.round(fps * ARBITRARY_DENOMINATOR); + const divisor = gcd(scaledNumerator, ARBITRARY_DENOMINATOR); + return { + numerator: scaledNumerator / divisor, + denominator: ARBITRARY_DENOMINATOR / divisor, + }; +} + +function gcd(left: number, right: number): number { + let a = Math.abs(left); + let b = Math.abs(right); + while (b !== 0) { + const remainder = a % b; + a = b; + b = remainder; + } + return a || 1; +} + export function getHighestImportedVideoFps({ mediaAssets, }: { @@ -24,16 +78,18 @@ export function getRaisedProjectFpsForImportedMedia({ currentFps, importedAssets, }: { - currentFps: number; + currentFps: FrameRate; importedAssets: MediaAssetFpsInput[]; -}): number | null { +}): FrameRate | null { const highestImportedVideoFps = getHighestImportedVideoFps({ mediaAssets: importedAssets, }); - if (highestImportedVideoFps === null || highestImportedVideoFps <= currentFps) { + const currentFpsFloat = frameRateToFloat(currentFps); + + if (highestImportedVideoFps === null || highestImportedVideoFps <= currentFpsFloat) { return null; } - return highestImportedVideoFps; + return floatToFrameRate(highestImportedVideoFps); } diff --git a/apps/web/src/lib/media/audio.ts b/apps/web/src/lib/media/audio.ts index d1025aa9..2fbea32e 100644 --- a/apps/web/src/lib/media/audio.ts +++ b/apps/web/src/lib/media/audio.ts @@ -21,6 +21,8 @@ import { mediaSupportsAudio } from "@/lib/media/media-utils"; import { getSourceTimeAtClipTime, renderRetimedBuffer } from "@/lib/retime"; import { Input, ALL_FORMATS, BlobSource, AudioBufferSink } from "mediabunny"; +import { TICKS_PER_SECOND } from "@/lib/wasm"; + const MAX_AUDIO_CHANNELS = 2; const EXPORT_SAMPLE_RATE = 44100; const COARSE_SAMPLE_COUNT = 2048; @@ -122,10 +124,10 @@ export async function collectAudioElements({ return { timelineElement: element, buffer: audioBuffer, - startTime: element.startTime, - duration: element.duration, - trimStart: element.trimStart, - trimEnd: element.trimEnd, + startTime: element.startTime / TICKS_PER_SECOND, + duration: element.duration / TICKS_PER_SECOND, + trimStart: element.trimStart / TICKS_PER_SECOND, + trimEnd: element.trimEnd / TICKS_PER_SECOND, volume: resolveEffectiveAudioGain({ element, trackMuted: isTrackMuted, @@ -152,10 +154,10 @@ export async function collectAudioElements({ return { timelineElement: element, buffer: audioBuffer, - startTime: element.startTime, - duration: element.duration, - trimStart: element.trimStart, - trimEnd: element.trimEnd, + startTime: element.startTime / TICKS_PER_SECOND, + duration: element.duration / TICKS_PER_SECOND, + trimStart: element.trimStart / TICKS_PER_SECOND, + trimEnd: element.trimEnd / TICKS_PER_SECOND, volume: resolveEffectiveAudioGain({ element, trackMuted: isTrackMuted, @@ -337,16 +339,16 @@ async function fetchLibraryAudioSource({ type: "audio/mpeg", }); - return { - timelineElement: element, - file, - startTime: element.startTime, - duration: element.duration, - trimStart: element.trimStart, - trimEnd: element.trimEnd, - volume, - retime: element.retime, - }; + return { + timelineElement: element, + file, + startTime: element.startTime / TICKS_PER_SECOND, + duration: element.duration / TICKS_PER_SECOND, + trimStart: element.trimStart / TICKS_PER_SECOND, + trimEnd: element.trimEnd / TICKS_PER_SECOND, + volume, + retime: element.retime, + }; } catch (error) { console.warn("Failed to fetch library audio:", error); return null; @@ -404,10 +406,10 @@ function collectMediaAudioSource({ return { timelineElement: element, file: mediaAsset.file, - startTime: element.startTime, - duration: element.duration, - trimStart: element.trimStart, - trimEnd: element.trimEnd, + startTime: element.startTime / TICKS_PER_SECOND, + duration: element.duration / TICKS_PER_SECOND, + trimStart: element.trimStart / TICKS_PER_SECOND, + trimEnd: element.trimEnd / TICKS_PER_SECOND, volume, retime: element.retime, }; @@ -429,10 +431,10 @@ function collectMediaAudioClip({ id: element.id, sourceKey: mediaAsset.id, file: mediaAsset.file, - startTime: element.startTime, - duration: element.duration, - trimStart: element.trimStart, - trimEnd: element.trimEnd, + startTime: element.startTime / TICKS_PER_SECOND, + duration: element.duration / TICKS_PER_SECOND, + trimStart: element.trimStart / TICKS_PER_SECOND, + trimEnd: element.trimEnd / TICKS_PER_SECOND, volume, muted, retime: element.retime, @@ -602,7 +604,8 @@ export async function createTimelineAudioBuffer({ if (audioElements.length === 0) return null; const outputChannels = 2; - const outputLength = Math.ceil(duration * sampleRate); + const durationSeconds = duration / TICKS_PER_SECOND; + const outputLength = Math.ceil(durationSeconds * sampleRate); const outputBuffer = context.createBuffer( outputChannels, outputLength, diff --git a/apps/web/src/lib/project/types.ts b/apps/web/src/lib/project/types.ts index 8f45db9e..0a454e07 100644 --- a/apps/web/src/lib/project/types.ts +++ b/apps/web/src/lib/project/types.ts @@ -1,53 +1,54 @@ -import type { TScene } from "@/lib/timeline/types"; - -export type TBackground = - | { - type: "color"; - color: string; - } - | { - type: "blur"; - blurIntensity: number; - }; - -export interface TCanvasSize { - width: number; - height: number; -} - -export interface TProjectMetadata { - id: string; - name: string; - thumbnail?: string; - duration: number; - createdAt: Date; - updatedAt: Date; -} - -export interface TProjectSettings { - fps: number; - canvasSize: TCanvasSize; - canvasSizeMode?: "preset" | "custom"; - lastCustomCanvasSize?: TCanvasSize | null; - originalCanvasSize?: TCanvasSize | null; - background: TBackground; -} - -export interface TTimelineViewState { - zoomLevel: number; - scrollLeft: number; - playheadTime: number; -} - -export interface TProject { - metadata: TProjectMetadata; - scenes: TScene[]; - currentSceneId: string; - settings: TProjectSettings; - version: number; - timelineViewState?: TTimelineViewState; -} - -export type TProjectSortKey = "createdAt" | "updatedAt" | "name" | "duration"; -export type TSortOrder = "asc" | "desc"; -export type TProjectSortOption = `${TProjectSortKey}-${TSortOrder}`; +import type { FrameRate } from "opencut-wasm"; +import type { TScene } from "@/lib/timeline/types"; + +export type TBackground = + | { + type: "color"; + color: string; + } + | { + type: "blur"; + blurIntensity: number; + }; + +export interface TCanvasSize { + width: number; + height: number; +} + +export interface TProjectMetadata { + id: string; + name: string; + thumbnail?: string; + duration: number; + createdAt: Date; + updatedAt: Date; +} + +export interface TProjectSettings { + fps: FrameRate; + canvasSize: TCanvasSize; + canvasSizeMode?: "preset" | "custom"; + lastCustomCanvasSize?: TCanvasSize | null; + originalCanvasSize?: TCanvasSize | null; + background: TBackground; +} + +export interface TTimelineViewState { + zoomLevel: number; + scrollLeft: number; + playheadTime: number; +} + +export interface TProject { + metadata: TProjectMetadata; + scenes: TScene[]; + currentSceneId: string; + settings: TProjectSettings; + version: number; + timelineViewState?: TTimelineViewState; +} + +export type TProjectSortKey = "createdAt" | "updatedAt" | "name" | "duration"; +export type TSortOrder = "asc" | "desc"; +export type TProjectSortOption = `${TProjectSortKey}-${TSortOrder}`; diff --git a/apps/web/src/lib/ripple/diff.ts b/apps/web/src/lib/ripple/diff.ts index 8a9730e1..22b8d996 100644 --- a/apps/web/src/lib/ripple/diff.ts +++ b/apps/web/src/lib/ripple/diff.ts @@ -1,4 +1,4 @@ -import { TIME_EPSILON_SECONDS } from "@/constants/animation-constants"; + import type { TimelineElement, TimelineTrack } from "@/lib/timeline/types"; import type { RippleAdjustment } from "./apply"; @@ -105,7 +105,7 @@ function collectTrackIntervals({ continue; } - if (beforeElement.endTime > afterElement.endTime + TIME_EPSILON_SECONDS) { + if (beforeElement.endTime > afterElement.endTime) { pushInterval({ intervals: vacatedIntervals, startTime: afterElement.endTime, @@ -141,7 +141,7 @@ function buildAdjustments({ }): RippleAdjustment[] { return intervals.flatMap((interval): RippleAdjustment[] => { const shiftAmount = interval.endTime - interval.startTime; - if (shiftAmount <= TIME_EPSILON_SECONDS) { + if (shiftAmount <= 0) { return []; } @@ -203,7 +203,7 @@ function normalizeIntervals({ const mergedIntervals: Interval[] = [{ ...sortedIntervals[0] }]; for (const interval of sortedIntervals.slice(1)) { const previousInterval = mergedIntervals[mergedIntervals.length - 1]; - if (interval.startTime <= previousInterval.endTime + TIME_EPSILON_SECONDS) { + if (interval.startTime <= previousInterval.endTime) { previousInterval.endTime = Math.max( previousInterval.endTime, interval.endTime, @@ -228,12 +228,10 @@ function subtractSingleInterval({ for (const overlappingInterval of overlappingIntervals) { remainingIntervals = remainingIntervals.flatMap((remainingInterval) => { - if ( - overlappingInterval.endTime <= - remainingInterval.startTime + TIME_EPSILON_SECONDS || - overlappingInterval.startTime >= - remainingInterval.endTime - TIME_EPSILON_SECONDS - ) { + if ( + overlappingInterval.endTime <= remainingInterval.startTime || + overlappingInterval.startTime >= remainingInterval.endTime + ) { return [remainingInterval]; } @@ -264,7 +262,7 @@ function pushInterval({ startTime, endTime, }: { intervals: Interval[]; startTime: number; endTime: number }): void { - if (endTime - startTime <= TIME_EPSILON_SECONDS) { + if (endTime <= startTime) { return; } diff --git a/apps/web/src/lib/timeline/audio-state.ts b/apps/web/src/lib/timeline/audio-state.ts index 28e381c2..9625c482 100644 --- a/apps/web/src/lib/timeline/audio-state.ts +++ b/apps/web/src/lib/timeline/audio-state.ts @@ -1,100 +1,102 @@ -import { hasKeyframesForPath } from "@/lib/animation/keyframe-query"; -import { resolveNumberAtTime } from "@/lib/animation/resolve"; -import { VOLUME_DB_MAX, VOLUME_DB_MIN } from "./audio-constants"; -import type { TimelineElement } from "./types"; -const DEFAULT_STEP_SECONDS = 1 / 60; - -export type AudioCapableElement = Extract< - TimelineElement, - { type: "audio" | "video" } ->; - -export function clampDb(value: number): number { - if (!Number.isFinite(value)) { - return 0; - } - - return Math.min(VOLUME_DB_MAX, Math.max(VOLUME_DB_MIN, value)); -} - -export function dBToLinear(db: number): number { - return 10 ** (clampDb(db) / 20); -} - -export function hasAnimatedVolume({ - element, -}: { - element: AudioCapableElement; -}): boolean { - return hasKeyframesForPath({ - animations: element.animations, - propertyPath: "volume", - }); -} - -export function resolveEffectiveAudioGain({ - element, - trackMuted = false, - localTime, -}: { - element: AudioCapableElement; - trackMuted?: boolean; - localTime: number; -}): number { - if (trackMuted || element.muted === true) { - return 0; - } - - const resolvedDb = resolveNumberAtTime({ - baseValue: element.volume ?? 0, - animations: element.animations, - propertyPath: "volume", - localTime, - }); - - return dBToLinear(resolvedDb); -} - -export function buildAudioGainAutomation({ - element, - trackMuted = false, - fromLocalTime, - toLocalTime, - stepSeconds = DEFAULT_STEP_SECONDS, -}: { - element: AudioCapableElement; - trackMuted?: boolean; - fromLocalTime: number; - toLocalTime: number; - stepSeconds?: number; -}): Array<{ localTime: number; gain: number }> { - const startTime = Math.max(0, fromLocalTime); - const endTime = Math.max(startTime, toLocalTime); - const safeStep = - Number.isFinite(stepSeconds) && stepSeconds > 0 - ? stepSeconds - : DEFAULT_STEP_SECONDS; - const points: Array<{ localTime: number; gain: number }> = []; - - for (let localTime = startTime; localTime < endTime; localTime += safeStep) { - points.push({ - localTime, - gain: resolveEffectiveAudioGain({ - element, - trackMuted, - localTime, - }), - }); - } - - points.push({ - localTime: endTime, - gain: resolveEffectiveAudioGain({ - element, - trackMuted, - localTime: endTime, - }), - }); - - return points; -} +import { hasKeyframesForPath } from "@/lib/animation/keyframe-query"; +import { resolveNumberAtTime } from "@/lib/animation/resolve"; +import { VOLUME_DB_MAX, VOLUME_DB_MIN } from "./audio-constants"; +import type { TimelineElement } from "./types"; +const DEFAULT_STEP_SECONDS = 1 / 60; + +export type AudioCapableElement = Extract< + TimelineElement, + { type: "audio" | "video" } +>; + +export function clampDb(value: number): number { + if (!Number.isFinite(value)) { + return 0; + } + + return Math.min(VOLUME_DB_MAX, Math.max(VOLUME_DB_MIN, value)); +} + +export function dBToLinear(db: number): number { + return 10 ** (clampDb(db) / 20); +} + +export function hasAnimatedVolume({ + element, +}: { + element: AudioCapableElement; +}): boolean { + return hasKeyframesForPath({ + animations: element.animations, + propertyPath: "volume", + }); +} + +import { TICKS_PER_SECOND } from "@/lib/wasm"; + +export function resolveEffectiveAudioGain({ + element, + trackMuted = false, + localTime, +}: { + element: AudioCapableElement; + trackMuted?: boolean; + localTime: number; +}): number { + if (trackMuted || element.muted === true) { + return 0; + } + + const resolvedDb = resolveNumberAtTime({ + baseValue: element.volume ?? 0, + animations: element.animations, + propertyPath: "volume", + localTime: Math.round(localTime * TICKS_PER_SECOND), + }); + + return dBToLinear(resolvedDb); +} + +export function buildAudioGainAutomation({ + element, + trackMuted = false, + fromLocalTime, + toLocalTime, + stepSeconds = DEFAULT_STEP_SECONDS, +}: { + element: AudioCapableElement; + trackMuted?: boolean; + fromLocalTime: number; + toLocalTime: number; + stepSeconds?: number; +}): Array<{ localTime: number; gain: number }> { + const startTime = Math.max(0, fromLocalTime); + const endTime = Math.max(startTime, toLocalTime); + const safeStep = + Number.isFinite(stepSeconds) && stepSeconds > 0 + ? stepSeconds + : DEFAULT_STEP_SECONDS; + const points: Array<{ localTime: number; gain: number }> = []; + + for (let localTime = startTime; localTime < endTime; localTime += safeStep) { + points.push({ + localTime, + gain: resolveEffectiveAudioGain({ + element, + trackMuted, + localTime, + }), + }); + } + + points.push({ + localTime: endTime, + gain: resolveEffectiveAudioGain({ + element, + trackMuted, + localTime: endTime, + }), + }); + + return points; +} diff --git a/apps/web/src/lib/timeline/bookmarks.ts b/apps/web/src/lib/timeline/bookmarks.ts index 70d1b5ee..1fbf17bf 100644 --- a/apps/web/src/lib/timeline/bookmarks.ts +++ b/apps/web/src/lib/timeline/bookmarks.ts @@ -1,8 +1,7 @@ import type { Bookmark } from "@/lib/timeline"; +import type { FrameRate } from "opencut-wasm"; import { roundToFrame } from "opencut-wasm"; -export const BOOKMARK_TIME_EPSILON = 0.001; - function bookmarkTimeEqual({ bookmarkTime, frameTime, @@ -10,7 +9,7 @@ function bookmarkTimeEqual({ bookmarkTime: number; frameTime: number; }): boolean { - return Math.abs(bookmarkTime - frameTime) < BOOKMARK_TIME_EPSILON; + return bookmarkTime === frameTime; } export function findBookmarkIndex({ @@ -112,9 +111,9 @@ export function getFrameTime({ fps, }: { time: number; - fps: number; + fps: FrameRate; }): number { - return roundToFrame({ time, fps }); + return roundToFrame({ time, rate: fps }) ?? time; } export function getBookmarkAtTime({ @@ -141,9 +140,6 @@ export function getBookmarksActiveAtTime({ bookmark.duration != null && bookmark.duration > 0 ? start + bookmark.duration : start; - return ( - time >= start - BOOKMARK_TIME_EPSILON && - time <= end + BOOKMARK_TIME_EPSILON - ); + return time >= start && time <= end; }); } diff --git a/apps/web/src/lib/timeline/creation.ts b/apps/web/src/lib/timeline/creation.ts index 335ab912..a339d806 100644 --- a/apps/web/src/lib/timeline/creation.ts +++ b/apps/web/src/lib/timeline/creation.ts @@ -1 +1,3 @@ -export const DEFAULT_NEW_ELEMENT_DURATION_SECONDS = 5; +import { TICKS_PER_SECOND } from "@/lib/wasm"; + +export const DEFAULT_NEW_ELEMENT_DURATION = 5 * TICKS_PER_SECOND; diff --git a/apps/web/src/lib/timeline/defaults.ts b/apps/web/src/lib/timeline/defaults.ts index bc968a3c..cf452800 100644 --- a/apps/web/src/lib/timeline/defaults.ts +++ b/apps/web/src/lib/timeline/defaults.ts @@ -1,77 +1,77 @@ -import { DEFAULT_NEW_ELEMENT_DURATION_SECONDS } from "@/lib/timeline/creation"; -import type { TTimelineViewState } from "@/lib/project/types"; -import type { BlendMode, Transform } from "@/lib/rendering"; -import type { TextElement } from "./types"; - -const defaultTransform: Transform = { - scaleX: 1, - scaleY: 1, - position: { x: 0, y: 0 }, - rotate: 0, -}; - -const defaultOpacity = 1; -const defaultBlendMode: BlendMode = "normal"; -const defaultVolume = 0; - -const defaultTextLetterSpacing = 0; -const defaultTextLineHeight = 1.2; - -const defaultTextBackground = { - enabled: false, - color: "#000000", - cornerRadius: 0, - paddingX: 30, - paddingY: 42, - offsetX: 0, - offsetY: 0, -}; - -const defaultTextElement: Omit = { - type: "text", - name: "Text", - content: "Default text", - fontSize: 15, - fontFamily: "Arial", - color: "#ffffff", - background: { ...defaultTextBackground }, - textAlign: "center", - fontWeight: "normal", - fontStyle: "normal", - textDecoration: "none", - letterSpacing: defaultTextLetterSpacing, - lineHeight: defaultTextLineHeight, - duration: DEFAULT_NEW_ELEMENT_DURATION_SECONDS, - startTime: 0, - trimStart: 0, - trimEnd: 0, - transform: { - ...defaultTransform, - position: { ...defaultTransform.position }, - }, - opacity: defaultOpacity, -}; - -const defaultTimelineViewState: TTimelineViewState = { - zoomLevel: 1, - scrollLeft: 0, - playheadTime: 0, -}; - -export const DEFAULTS = { - element: { - transform: defaultTransform, - opacity: defaultOpacity, - blendMode: defaultBlendMode, - volume: defaultVolume, - }, - text: { - letterSpacing: defaultTextLetterSpacing, - lineHeight: defaultTextLineHeight, - background: defaultTextBackground, - element: defaultTextElement, - }, - timeline: { - viewState: defaultTimelineViewState, - }, -}; +import { DEFAULT_NEW_ELEMENT_DURATION } from "@/lib/timeline/creation"; +import type { TTimelineViewState } from "@/lib/project/types"; +import type { BlendMode, Transform } from "@/lib/rendering"; +import type { TextElement } from "./types"; + +const defaultTransform: Transform = { + scaleX: 1, + scaleY: 1, + position: { x: 0, y: 0 }, + rotate: 0, +}; + +const defaultOpacity = 1; +const defaultBlendMode: BlendMode = "normal"; +const defaultVolume = 0; + +const defaultTextLetterSpacing = 0; +const defaultTextLineHeight = 1.2; + +const defaultTextBackground = { + enabled: false, + color: "#000000", + cornerRadius: 0, + paddingX: 30, + paddingY: 42, + offsetX: 0, + offsetY: 0, +}; + +const defaultTextElement: Omit = { + type: "text", + name: "Text", + content: "Default text", + fontSize: 15, + fontFamily: "Arial", + color: "#ffffff", + background: { ...defaultTextBackground }, + textAlign: "center", + fontWeight: "normal", + fontStyle: "normal", + textDecoration: "none", + letterSpacing: defaultTextLetterSpacing, + lineHeight: defaultTextLineHeight, + duration: DEFAULT_NEW_ELEMENT_DURATION, + startTime: 0, + trimStart: 0, + trimEnd: 0, + transform: { + ...defaultTransform, + position: { ...defaultTransform.position }, + }, + opacity: defaultOpacity, +}; + +const defaultTimelineViewState: TTimelineViewState = { + zoomLevel: 1, + scrollLeft: 0, + playheadTime: 0, +}; + +export const DEFAULTS = { + element: { + transform: defaultTransform, + opacity: defaultOpacity, + blendMode: defaultBlendMode, + volume: defaultVolume, + }, + text: { + letterSpacing: defaultTextLetterSpacing, + lineHeight: defaultTextLineHeight, + background: defaultTextBackground, + element: defaultTextElement, + }, + timeline: { + viewState: defaultTimelineViewState, + }, +}; diff --git a/apps/web/src/lib/timeline/drag-utils.ts b/apps/web/src/lib/timeline/drag-utils.ts index 8efeab79..35d00613 100644 --- a/apps/web/src/lib/timeline/drag-utils.ts +++ b/apps/web/src/lib/timeline/drag-utils.ts @@ -1,19 +1,21 @@ -import { BASE_TIMELINE_PIXELS_PER_SECOND } from "@/lib/timeline/scale"; - -export function getMouseTimeFromClientX({ - clientX, - containerRect, - zoomLevel, - scrollLeft, -}: { - clientX: number; - containerRect: DOMRect; - zoomLevel: number; - scrollLeft: number; -}): number { - const mouseX = clientX - containerRect.left + scrollLeft; - return Math.max( - 0, - mouseX / (BASE_TIMELINE_PIXELS_PER_SECOND * zoomLevel), - ); -} +import { BASE_TIMELINE_PIXELS_PER_SECOND } from "@/lib/timeline/scale"; +import { TICKS_PER_SECOND } from "@/lib/wasm"; + +export function getMouseTimeFromClientX({ + clientX, + containerRect, + zoomLevel, + scrollLeft, +}: { + clientX: number; + containerRect: DOMRect; + zoomLevel: number; + scrollLeft: number; +}): number { + const mouseX = clientX - containerRect.left + scrollLeft; + const seconds = Math.max( + 0, + mouseX / (BASE_TIMELINE_PIXELS_PER_SECOND * zoomLevel), + ); + return Math.round(seconds * TICKS_PER_SECOND); +} diff --git a/apps/web/src/lib/timeline/element-utils.ts b/apps/web/src/lib/timeline/element-utils.ts index 96588903..6f9e9dbc 100644 --- a/apps/web/src/lib/timeline/element-utils.ts +++ b/apps/web/src/lib/timeline/element-utils.ts @@ -1,418 +1,418 @@ -import { DEFAULT_NEW_ELEMENT_DURATION_SECONDS } from "@/lib/timeline/creation"; -import { - MASKABLE_ELEMENT_TYPES, - RETIMABLE_ELEMENT_TYPES, - VISUAL_ELEMENT_TYPES, - type CreateEffectElement, - type CreateGraphicElement, - type CreateTimelineElement, - type CreateVideoElement, - type CreateImageElement, - type CreateStickerElement, - type CreateUploadAudioElement, - type CreateLibraryAudioElement, - type TextBackground, - type TextElement, - type TimelineElement, - type TimelineTrack, - type AudioElement, - type VideoElement, - type ImageElement, - type MaskableElement, - type RetimableElement, - type VisualElement, - type UploadAudioElement, -} from "@/lib/timeline"; -import { DEFAULTS } from "@/lib/timeline/defaults"; -import type { MediaType } from "@/lib/media/types"; -import { buildDefaultEffectInstance } from "@/lib/effects"; -import { buildDefaultGraphicInstance } from "@/lib/graphics"; -import type { ParamValues } from "@/lib/params"; -import { capitalizeFirstLetter } from "@/utils/string"; - -export function canElementHaveAudio( - element: TimelineElement, -): element is AudioElement | VideoElement { - return element.type === "audio" || element.type === "video"; -} - -export function isVisualElement( - element: TimelineElement, -): element is VisualElement { - return (VISUAL_ELEMENT_TYPES as readonly string[]).includes(element.type); -} - -export function isMaskableElement( - element: TimelineElement, -): element is MaskableElement { - return (MASKABLE_ELEMENT_TYPES as readonly string[]).includes(element.type); -} - -export function isRetimableElement( - element: TimelineElement, -): element is RetimableElement { - return (RETIMABLE_ELEMENT_TYPES as readonly string[]).includes(element.type); -} - -export function canElementBeHidden( - element: TimelineElement, -): element is VisualElement { - return isVisualElement(element); -} - -export function hasElementEffects({ - element, -}: { - element: TimelineElement; -}): boolean { - return isVisualElement(element) && (element.effects?.length ?? 0) > 0; -} - -export function hasMediaId( - element: TimelineElement, -): element is UploadAudioElement | VideoElement | ImageElement { - return "mediaId" in element; -} - -export function requiresMediaId({ - element, -}: { - element: CreateTimelineElement; -}): boolean { - return ( - element.type === "video" || - element.type === "image" || - (element.type === "audio" && element.sourceType === "upload") - ); -} - -function buildTextBackground( - raw: Partial | undefined, -): TextBackground { - const color = raw?.color ?? DEFAULTS.text.element.background.color; - const enabled = raw?.enabled ?? color !== "transparent"; - return { - enabled, - color, - cornerRadius: raw?.cornerRadius, - paddingX: raw?.paddingX, - paddingY: raw?.paddingY, - offsetX: raw?.offsetX, - offsetY: raw?.offsetY, - }; -} - -export function buildTextElement({ - raw, - startTime, -}: { - raw: Partial>; - startTime: number; -}): CreateTimelineElement { - const t = raw as Partial; - - return { - type: "text", - name: t.name ?? DEFAULTS.text.element.name, - content: t.content ?? DEFAULTS.text.element.content, - duration: t.duration ?? DEFAULT_NEW_ELEMENT_DURATION_SECONDS, - startTime, - trimStart: 0, - trimEnd: 0, - fontSize: t.fontSize ?? DEFAULTS.text.element.fontSize, - fontFamily: t.fontFamily ?? DEFAULTS.text.element.fontFamily, - color: t.color ?? DEFAULTS.text.element.color, - background: buildTextBackground(t.background), - textAlign: t.textAlign ?? DEFAULTS.text.element.textAlign, - fontWeight: t.fontWeight ?? DEFAULTS.text.element.fontWeight, - fontStyle: t.fontStyle ?? DEFAULTS.text.element.fontStyle, - textDecoration: t.textDecoration ?? DEFAULTS.text.element.textDecoration, - letterSpacing: t.letterSpacing ?? DEFAULTS.text.element.letterSpacing, - lineHeight: t.lineHeight ?? DEFAULTS.text.element.lineHeight, - transform: t.transform ?? DEFAULTS.text.element.transform, - opacity: t.opacity ?? DEFAULTS.text.element.opacity, - blendMode: t.blendMode ?? DEFAULTS.element.blendMode, - }; -} - -export function buildEffectElement({ - effectType, - startTime, - duration, -}: { - effectType: string; - startTime: number; - duration?: number; -}): CreateEffectElement { - const instance = buildDefaultEffectInstance({ effectType }); - return { - type: "effect", - name: capitalizeFirstLetter({ string: instance.type }), - effectType, - params: instance.params, - duration: duration ?? DEFAULT_NEW_ELEMENT_DURATION_SECONDS, - startTime, - trimStart: 0, - trimEnd: 0, - }; -} - -export function buildStickerElement({ - stickerId, - name, - startTime, - intrinsicWidth, - intrinsicHeight, -}: { - stickerId: string; - name?: string; - startTime: number; - intrinsicWidth?: number; - intrinsicHeight?: number; -}): CreateStickerElement { - const stickerNameFromId = - stickerId.split(":").slice(1).pop()?.replaceAll("-", " ") ?? stickerId; - return { - type: "sticker", - name: name ?? stickerNameFromId, - stickerId, - intrinsicWidth, - intrinsicHeight, - duration: DEFAULT_NEW_ELEMENT_DURATION_SECONDS, - startTime, - trimStart: 0, - trimEnd: 0, - transform: { - ...DEFAULTS.element.transform, - position: { ...DEFAULTS.element.transform.position }, - }, - opacity: DEFAULTS.element.opacity, - blendMode: DEFAULTS.element.blendMode, - }; -} - -export function buildGraphicElement({ - definitionId, - name, - startTime, - params, -}: { - definitionId: string; - name?: string; - startTime: number; - params?: Partial; -}): CreateGraphicElement { - const instance = buildDefaultGraphicInstance({ definitionId }); - return { - type: "graphic", - name: name ?? capitalizeFirstLetter({ string: instance.definitionId }), - definitionId: instance.definitionId, - params: { ...instance.params, ...(params ?? {}) } as ParamValues, - duration: DEFAULT_NEW_ELEMENT_DURATION_SECONDS, - startTime, - trimStart: 0, - trimEnd: 0, - transform: { - ...DEFAULTS.element.transform, - position: { ...DEFAULTS.element.transform.position }, - }, - opacity: DEFAULTS.element.opacity, - blendMode: DEFAULTS.element.blendMode, - }; -} - -function buildVideoElement({ - mediaId, - name, - duration, - startTime, -}: { - mediaId: string; - name: string; - duration: number; - startTime: number; -}): CreateVideoElement { - return { - type: "video", - mediaId, - name, - duration, - startTime, - trimStart: 0, - trimEnd: 0, - sourceDuration: duration, - muted: false, - isSourceAudioEnabled: true, - hidden: false, - transform: { - ...DEFAULTS.element.transform, - position: { ...DEFAULTS.element.transform.position }, - }, - opacity: DEFAULTS.element.opacity, - blendMode: DEFAULTS.element.blendMode, - volume: DEFAULTS.element.volume, - }; -} - -function buildImageElement({ - mediaId, - name, - duration, - startTime, -}: { - mediaId: string; - name: string; - duration: number; - startTime: number; -}): CreateImageElement { - return { - type: "image", - mediaId, - name, - duration, - startTime, - trimStart: 0, - trimEnd: 0, - hidden: false, - transform: { - ...DEFAULTS.element.transform, - position: { ...DEFAULTS.element.transform.position }, - }, - opacity: DEFAULTS.element.opacity, - blendMode: DEFAULTS.element.blendMode, - }; -} - -function buildUploadAudioElement({ - mediaId, - name, - duration, - startTime, - buffer, -}: { - mediaId: string; - name: string; - duration: number; - startTime: number; - buffer?: AudioBuffer; -}): CreateUploadAudioElement { - const element: CreateUploadAudioElement = { - type: "audio", - sourceType: "upload", - mediaId, - name, - duration, - startTime, - trimStart: 0, - trimEnd: 0, - sourceDuration: duration, - volume: DEFAULTS.element.volume, - muted: false, - }; - if (buffer) { - element.buffer = buffer; - } - return element; -} - -export function buildElementFromMedia({ - mediaId, - mediaType, - name, - duration, - startTime, - buffer, -}: { - mediaId: string; - mediaType: MediaType; - name: string; - duration: number; - startTime: number; - buffer?: AudioBuffer; -}): CreateTimelineElement { - switch (mediaType) { - case "audio": - return buildUploadAudioElement({ - mediaId, - name, - duration, - startTime, - buffer, - }); - case "video": - return buildVideoElement({ mediaId, name, duration, startTime }); - case "image": - return buildImageElement({ mediaId, name, duration, startTime }); - } -} - -export function buildLibraryAudioElement({ - sourceUrl, - name, - duration, - startTime, - buffer, -}: { - sourceUrl: string; - name: string; - duration: number; - startTime: number; - buffer?: AudioBuffer; -}): CreateLibraryAudioElement { - const element: CreateLibraryAudioElement = { - type: "audio", - sourceType: "library", - sourceUrl, - name, - duration, - startTime, - trimStart: 0, - trimEnd: 0, - sourceDuration: duration, - volume: DEFAULTS.element.volume, - muted: false, - }; - if (buffer) { - element.buffer = buffer; - } - return element; -} - -export function getElementsAtTime({ - tracks, - time, -}: { - tracks: TimelineTrack[]; - time: number; -}): { trackId: string; elementId: string }[] { - const result: { trackId: string; elementId: string }[] = []; - - for (const track of tracks) { - for (const element of track.elements) { - const elementStart = element.startTime; - const elementEnd = element.startTime + element.duration; - - if (time > elementStart && time < elementEnd) { - result.push({ trackId: track.id, elementId: element.id }); - } - } - } - - return result; -} - -export function getElementFontFamilies({ - tracks, -}: { - tracks: TimelineTrack[]; -}): string[] { - const families = new Set(); - for (const track of tracks) { - for (const element of track.elements) { - if (element.type === "text" && element.fontFamily) { - families.add(element.fontFamily); - } - } - } - return [...families]; -} +import { DEFAULT_NEW_ELEMENT_DURATION } from "@/lib/timeline/creation"; +import { + MASKABLE_ELEMENT_TYPES, + RETIMABLE_ELEMENT_TYPES, + VISUAL_ELEMENT_TYPES, + type CreateEffectElement, + type CreateGraphicElement, + type CreateTimelineElement, + type CreateVideoElement, + type CreateImageElement, + type CreateStickerElement, + type CreateUploadAudioElement, + type CreateLibraryAudioElement, + type TextBackground, + type TextElement, + type TimelineElement, + type TimelineTrack, + type AudioElement, + type VideoElement, + type ImageElement, + type MaskableElement, + type RetimableElement, + type VisualElement, + type UploadAudioElement, +} from "@/lib/timeline"; +import { DEFAULTS } from "@/lib/timeline/defaults"; +import type { MediaType } from "@/lib/media/types"; +import { buildDefaultEffectInstance } from "@/lib/effects"; +import { buildDefaultGraphicInstance } from "@/lib/graphics"; +import type { ParamValues } from "@/lib/params"; +import { capitalizeFirstLetter } from "@/utils/string"; + +export function canElementHaveAudio( + element: TimelineElement, +): element is AudioElement | VideoElement { + return element.type === "audio" || element.type === "video"; +} + +export function isVisualElement( + element: TimelineElement, +): element is VisualElement { + return (VISUAL_ELEMENT_TYPES as readonly string[]).includes(element.type); +} + +export function isMaskableElement( + element: TimelineElement, +): element is MaskableElement { + return (MASKABLE_ELEMENT_TYPES as readonly string[]).includes(element.type); +} + +export function isRetimableElement( + element: TimelineElement, +): element is RetimableElement { + return (RETIMABLE_ELEMENT_TYPES as readonly string[]).includes(element.type); +} + +export function canElementBeHidden( + element: TimelineElement, +): element is VisualElement { + return isVisualElement(element); +} + +export function hasElementEffects({ + element, +}: { + element: TimelineElement; +}): boolean { + return isVisualElement(element) && (element.effects?.length ?? 0) > 0; +} + +export function hasMediaId( + element: TimelineElement, +): element is UploadAudioElement | VideoElement | ImageElement { + return "mediaId" in element; +} + +export function requiresMediaId({ + element, +}: { + element: CreateTimelineElement; +}): boolean { + return ( + element.type === "video" || + element.type === "image" || + (element.type === "audio" && element.sourceType === "upload") + ); +} + +function buildTextBackground( + raw: Partial | undefined, +): TextBackground { + const color = raw?.color ?? DEFAULTS.text.element.background.color; + const enabled = raw?.enabled ?? color !== "transparent"; + return { + enabled, + color, + cornerRadius: raw?.cornerRadius, + paddingX: raw?.paddingX, + paddingY: raw?.paddingY, + offsetX: raw?.offsetX, + offsetY: raw?.offsetY, + }; +} + +export function buildTextElement({ + raw, + startTime, +}: { + raw: Partial>; + startTime: number; +}): CreateTimelineElement { + const t = raw as Partial; + + return { + type: "text", + name: t.name ?? DEFAULTS.text.element.name, + content: t.content ?? DEFAULTS.text.element.content, + duration: t.duration ?? DEFAULT_NEW_ELEMENT_DURATION, + startTime, + trimStart: 0, + trimEnd: 0, + fontSize: t.fontSize ?? DEFAULTS.text.element.fontSize, + fontFamily: t.fontFamily ?? DEFAULTS.text.element.fontFamily, + color: t.color ?? DEFAULTS.text.element.color, + background: buildTextBackground(t.background), + textAlign: t.textAlign ?? DEFAULTS.text.element.textAlign, + fontWeight: t.fontWeight ?? DEFAULTS.text.element.fontWeight, + fontStyle: t.fontStyle ?? DEFAULTS.text.element.fontStyle, + textDecoration: t.textDecoration ?? DEFAULTS.text.element.textDecoration, + letterSpacing: t.letterSpacing ?? DEFAULTS.text.element.letterSpacing, + lineHeight: t.lineHeight ?? DEFAULTS.text.element.lineHeight, + transform: t.transform ?? DEFAULTS.text.element.transform, + opacity: t.opacity ?? DEFAULTS.text.element.opacity, + blendMode: t.blendMode ?? DEFAULTS.element.blendMode, + }; +} + +export function buildEffectElement({ + effectType, + startTime, + duration, +}: { + effectType: string; + startTime: number; + duration?: number; +}): CreateEffectElement { + const instance = buildDefaultEffectInstance({ effectType }); + return { + type: "effect", + name: capitalizeFirstLetter({ string: instance.type }), + effectType, + params: instance.params, + duration: duration ?? DEFAULT_NEW_ELEMENT_DURATION, + startTime, + trimStart: 0, + trimEnd: 0, + }; +} + +export function buildStickerElement({ + stickerId, + name, + startTime, + intrinsicWidth, + intrinsicHeight, +}: { + stickerId: string; + name?: string; + startTime: number; + intrinsicWidth?: number; + intrinsicHeight?: number; +}): CreateStickerElement { + const stickerNameFromId = + stickerId.split(":").slice(1).pop()?.replaceAll("-", " ") ?? stickerId; + return { + type: "sticker", + name: name ?? stickerNameFromId, + stickerId, + intrinsicWidth, + intrinsicHeight, + duration: DEFAULT_NEW_ELEMENT_DURATION, + startTime, + trimStart: 0, + trimEnd: 0, + transform: { + ...DEFAULTS.element.transform, + position: { ...DEFAULTS.element.transform.position }, + }, + opacity: DEFAULTS.element.opacity, + blendMode: DEFAULTS.element.blendMode, + }; +} + +export function buildGraphicElement({ + definitionId, + name, + startTime, + params, +}: { + definitionId: string; + name?: string; + startTime: number; + params?: Partial; +}): CreateGraphicElement { + const instance = buildDefaultGraphicInstance({ definitionId }); + return { + type: "graphic", + name: name ?? capitalizeFirstLetter({ string: instance.definitionId }), + definitionId: instance.definitionId, + params: { ...instance.params, ...(params ?? {}) } as ParamValues, + duration: DEFAULT_NEW_ELEMENT_DURATION, + startTime, + trimStart: 0, + trimEnd: 0, + transform: { + ...DEFAULTS.element.transform, + position: { ...DEFAULTS.element.transform.position }, + }, + opacity: DEFAULTS.element.opacity, + blendMode: DEFAULTS.element.blendMode, + }; +} + +function buildVideoElement({ + mediaId, + name, + duration, + startTime, +}: { + mediaId: string; + name: string; + duration: number; + startTime: number; +}): CreateVideoElement { + return { + type: "video", + mediaId, + name, + duration, + startTime, + trimStart: 0, + trimEnd: 0, + sourceDuration: duration, + muted: false, + isSourceAudioEnabled: true, + hidden: false, + transform: { + ...DEFAULTS.element.transform, + position: { ...DEFAULTS.element.transform.position }, + }, + opacity: DEFAULTS.element.opacity, + blendMode: DEFAULTS.element.blendMode, + volume: DEFAULTS.element.volume, + }; +} + +function buildImageElement({ + mediaId, + name, + duration, + startTime, +}: { + mediaId: string; + name: string; + duration: number; + startTime: number; +}): CreateImageElement { + return { + type: "image", + mediaId, + name, + duration, + startTime, + trimStart: 0, + trimEnd: 0, + hidden: false, + transform: { + ...DEFAULTS.element.transform, + position: { ...DEFAULTS.element.transform.position }, + }, + opacity: DEFAULTS.element.opacity, + blendMode: DEFAULTS.element.blendMode, + }; +} + +function buildUploadAudioElement({ + mediaId, + name, + duration, + startTime, + buffer, +}: { + mediaId: string; + name: string; + duration: number; + startTime: number; + buffer?: AudioBuffer; +}): CreateUploadAudioElement { + const element: CreateUploadAudioElement = { + type: "audio", + sourceType: "upload", + mediaId, + name, + duration, + startTime, + trimStart: 0, + trimEnd: 0, + sourceDuration: duration, + volume: DEFAULTS.element.volume, + muted: false, + }; + if (buffer) { + element.buffer = buffer; + } + return element; +} + +export function buildElementFromMedia({ + mediaId, + mediaType, + name, + duration, + startTime, + buffer, +}: { + mediaId: string; + mediaType: MediaType; + name: string; + duration: number; + startTime: number; + buffer?: AudioBuffer; +}): CreateTimelineElement { + switch (mediaType) { + case "audio": + return buildUploadAudioElement({ + mediaId, + name, + duration, + startTime, + buffer, + }); + case "video": + return buildVideoElement({ mediaId, name, duration, startTime }); + case "image": + return buildImageElement({ mediaId, name, duration, startTime }); + } +} + +export function buildLibraryAudioElement({ + sourceUrl, + name, + duration, + startTime, + buffer, +}: { + sourceUrl: string; + name: string; + duration: number; + startTime: number; + buffer?: AudioBuffer; +}): CreateLibraryAudioElement { + const element: CreateLibraryAudioElement = { + type: "audio", + sourceType: "library", + sourceUrl, + name, + duration, + startTime, + trimStart: 0, + trimEnd: 0, + sourceDuration: duration, + volume: DEFAULTS.element.volume, + muted: false, + }; + if (buffer) { + element.buffer = buffer; + } + return element; +} + +export function getElementsAtTime({ + tracks, + time, +}: { + tracks: TimelineTrack[]; + time: number; +}): { trackId: string; elementId: string }[] { + const result: { trackId: string; elementId: string }[] = []; + + for (const track of tracks) { + for (const element of track.elements) { + const elementStart = element.startTime; + const elementEnd = element.startTime + element.duration; + + if (time > elementStart && time < elementEnd) { + result.push({ trackId: track.id, elementId: element.id }); + } + } + } + + return result; +} + +export function getElementFontFamilies({ + tracks, +}: { + tracks: TimelineTrack[]; +}): string[] { + const families = new Set(); + for (const track of tracks) { + for (const element of track.elements) { + if (element.type === "text" && element.fontFamily) { + families.add(element.fontFamily); + } + } + } + return [...families]; +} diff --git a/apps/web/src/lib/timeline/pixel-utils.ts b/apps/web/src/lib/timeline/pixel-utils.ts index e93b5ab5..10f258aa 100644 --- a/apps/web/src/lib/timeline/pixel-utils.ts +++ b/apps/web/src/lib/timeline/pixel-utils.ts @@ -1,79 +1,80 @@ -import { BASE_TIMELINE_PIXELS_PER_SECOND } from "@/lib/timeline/scale"; - -export const TIMELINE_INDICATOR_LINE_WIDTH_PX = 2; - -function getDevicePixelRatio({ - devicePixelRatio, -}: { - devicePixelRatio?: number; -}): number { - if ( - typeof devicePixelRatio === "number" && - Number.isFinite(devicePixelRatio) && - devicePixelRatio > 0 - ) { - return devicePixelRatio; - } - - if (typeof window === "undefined") { - return 1; - } - - if (Number.isFinite(window.devicePixelRatio) && window.devicePixelRatio > 0) { - return window.devicePixelRatio; - } - - return 1; -} - -export function getTimelinePixelsPerSecond({ - zoomLevel, -}: { - zoomLevel: number; -}): number { - return BASE_TIMELINE_PIXELS_PER_SECOND * zoomLevel; -} - -export function timelineTimeToPixels({ - time, - zoomLevel, -}: { - time: number; - zoomLevel: number; -}): number { - return time * getTimelinePixelsPerSecond({ zoomLevel }); -} - -export function snapPixelToDeviceGrid({ - pixel, - devicePixelRatio, -}: { - pixel: number; - devicePixelRatio?: number; -}): number { - const safeDevicePixelRatio = getDevicePixelRatio({ devicePixelRatio }); - return Math.round(pixel * safeDevicePixelRatio) / safeDevicePixelRatio; -} - -export function timelineTimeToSnappedPixels({ - time, - zoomLevel, - devicePixelRatio, -}: { - time: number; - zoomLevel: number; - devicePixelRatio?: number; -}): number { - const rawPixel = timelineTimeToPixels({ time, zoomLevel }); - return snapPixelToDeviceGrid({ pixel: rawPixel, devicePixelRatio }); -} - -export function getCenteredLineLeft({ - centerPixel, - lineWidthPx = TIMELINE_INDICATOR_LINE_WIDTH_PX, -}: { - centerPixel: number; - lineWidthPx?: number; -}): number { - return centerPixel - lineWidthPx / 2; -} +import { BASE_TIMELINE_PIXELS_PER_SECOND } from "@/lib/timeline/scale"; +import { TICKS_PER_SECOND } from "@/lib/wasm"; + +export const TIMELINE_INDICATOR_LINE_WIDTH_PX = 2; + +function getDevicePixelRatio({ + devicePixelRatio, +}: { + devicePixelRatio?: number; +}): number { + if ( + typeof devicePixelRatio === "number" && + Number.isFinite(devicePixelRatio) && + devicePixelRatio > 0 + ) { + return devicePixelRatio; + } + + if (typeof window === "undefined") { + return 1; + } + + if (Number.isFinite(window.devicePixelRatio) && window.devicePixelRatio > 0) { + return window.devicePixelRatio; + } + + return 1; +} + +export function getTimelinePixelsPerSecond({ + zoomLevel, +}: { + zoomLevel: number; +}): number { + return BASE_TIMELINE_PIXELS_PER_SECOND * zoomLevel; +} + +export function timelineTimeToPixels({ + time, + zoomLevel, +}: { + time: number; + zoomLevel: number; +}): number { + return (time / TICKS_PER_SECOND) * getTimelinePixelsPerSecond({ zoomLevel }); +} + +export function snapPixelToDeviceGrid({ + pixel, + devicePixelRatio, +}: { + pixel: number; + devicePixelRatio?: number; +}): number { + const safeDevicePixelRatio = getDevicePixelRatio({ devicePixelRatio }); + return Math.round(pixel * safeDevicePixelRatio) / safeDevicePixelRatio; +} + +export function timelineTimeToSnappedPixels({ + time, + zoomLevel, + devicePixelRatio, +}: { + time: number; + zoomLevel: number; + devicePixelRatio?: number; +}): number { + const rawPixel = timelineTimeToPixels({ time, zoomLevel }); + return snapPixelToDeviceGrid({ pixel: rawPixel, devicePixelRatio }); +} + +export function getCenteredLineLeft({ + centerPixel, + lineWidthPx = TIMELINE_INDICATOR_LINE_WIDTH_PX, +}: { + centerPixel: number; + lineWidthPx?: number; +}): number { + return centerPixel - lineWidthPx / 2; +} diff --git a/apps/web/src/lib/timeline/ruler-utils.ts b/apps/web/src/lib/timeline/ruler-utils.ts index 732d9ea4..ea2982aa 100644 --- a/apps/web/src/lib/timeline/ruler-utils.ts +++ b/apps/web/src/lib/timeline/ruler-utils.ts @@ -1,4 +1,6 @@ +import type { FrameRate } from "opencut-wasm"; import { BASE_TIMELINE_PIXELS_PER_SECOND } from "@/lib/timeline/scale"; +import { frameRateToFloat } from "@/lib/fps/utils"; /** * frame intervals for labels - starts at 2 so there's always at least @@ -54,15 +56,16 @@ export function getRulerConfig({ fps, }: { zoomLevel: number; - fps: number; + fps: FrameRate; }): RulerConfig { + const fpsFloat = frameRateToFloat(fps); const pixelsPerSecond = BASE_TIMELINE_PIXELS_PER_SECOND * zoomLevel; - const pixelsPerFrame = pixelsPerSecond / fps; + const pixelsPerFrame = pixelsPerSecond / fpsFloat; const labelIntervalSeconds = findOptimalInterval({ pixelsPerFrame, pixelsPerSecond, - fps, + fps: fpsFloat, minSpacingPx: MIN_LABEL_SPACING_PX, frameIntervals: LABEL_FRAME_INTERVALS, }); @@ -70,7 +73,7 @@ export function getRulerConfig({ const rawTickIntervalSeconds = findOptimalInterval({ pixelsPerFrame, pixelsPerSecond, - fps, + fps: fpsFloat, minSpacingPx: MIN_TICK_SPACING_PX, frameIntervals: TICK_FRAME_INTERVALS, }); @@ -81,7 +84,7 @@ export function getRulerConfig({ labelIntervalSeconds, pixelsPerFrame, pixelsPerSecond, - fps, + fps: fpsFloat, }); return { labelIntervalSeconds, tickIntervalSeconds }; @@ -197,13 +200,13 @@ export function formatRulerLabel({ fps, }: { timeInSeconds: number; - fps: number; + fps: FrameRate; }): string { if (isSecondBoundary({ timeInSeconds })) { return formatTimestamp({ timeInSeconds }); } - const frameWithinSecond = getFrameWithinSecond({ timeInSeconds, fps }); + const frameWithinSecond = getFrameWithinSecond({ timeInSeconds, fps: frameRateToFloat(fps) }); return `${frameWithinSecond}f`; } diff --git a/apps/web/src/lib/timeline/snap-utils.ts b/apps/web/src/lib/timeline/snap-utils.ts index ef5d6e11..2e7bdcec 100644 --- a/apps/web/src/lib/timeline/snap-utils.ts +++ b/apps/web/src/lib/timeline/snap-utils.ts @@ -1,172 +1,172 @@ -import type { Bookmark, TimelineTrack } from "@/lib/timeline"; -import { BASE_TIMELINE_PIXELS_PER_SECOND } from "@/lib/timeline/scale"; -import { BOOKMARK_TIME_EPSILON } from "@/lib/timeline/bookmarks"; -import { getElementKeyframes } from "@/lib/animation"; - -export interface SnapPoint { - time: number; - type: "element-start" | "element-end" | "playhead" | "bookmark" | "keyframe"; - elementId?: string; - trackId?: string; -} - -export interface SnapResult { - snappedTime: number; - snapPoint: SnapPoint | null; - snapDistance: number; -} - -const DEFAULT_SNAP_THRESHOLD_PX = 10; - -export function findSnapPoints({ - tracks, - playheadTime, - excludeElementId, - bookmarks = [], - excludeBookmarkTime, - enableElementSnapping = true, - enablePlayheadSnapping = true, - enableBookmarkSnapping = true, - enableKeyframeSnapping = true, -}: { - tracks: Array; - playheadTime: number; - excludeElementId?: string; - bookmarks?: Array; - excludeBookmarkTime?: number; - enableElementSnapping?: boolean; - enablePlayheadSnapping?: boolean; - enableBookmarkSnapping?: boolean; - enableKeyframeSnapping?: boolean; -}): SnapPoint[] { - const snapPoints: SnapPoint[] = []; - - for (const track of tracks) { - for (const element of track.elements) { - if (element.id === excludeElementId) continue; - - if (enableElementSnapping) { - snapPoints.push( - { - time: element.startTime, - type: "element-start", - elementId: element.id, - trackId: track.id, - }, - { - time: element.startTime + element.duration, - type: "element-end", - elementId: element.id, - trackId: track.id, - }, - ); - } - - if (enableKeyframeSnapping) { - for (const keyframe of getElementKeyframes({ - animations: element.animations, - })) { - snapPoints.push({ - time: element.startTime + keyframe.time, - type: "keyframe", - elementId: element.id, - trackId: track.id, - }); - } - } - } - } - - if (enablePlayheadSnapping) { - snapPoints.push({ time: playheadTime, type: "playhead" }); - } - - if (enableBookmarkSnapping) { - for (const bookmark of bookmarks) { - if ( - excludeBookmarkTime != null && - Math.abs(bookmark.time - excludeBookmarkTime) < BOOKMARK_TIME_EPSILON - ) { - continue; - } - snapPoints.push({ time: bookmark.time, type: "bookmark" }); - } - } - - return snapPoints; -} - -export function snapToNearestPoint({ - targetTime, - snapPoints, - zoomLevel, - snapThreshold = DEFAULT_SNAP_THRESHOLD_PX, -}: { - targetTime: number; - snapPoints: Array; - zoomLevel: number; - snapThreshold?: number; -}): SnapResult { - const pixelsPerSecond = BASE_TIMELINE_PIXELS_PER_SECOND * zoomLevel; - const thresholdInSeconds = snapThreshold / pixelsPerSecond; - - let closestSnapPoint: SnapPoint | null = null; - let closestDistance = Infinity; - - for (const snapPoint of snapPoints) { - const distance = Math.abs(targetTime - snapPoint.time); - if (distance < thresholdInSeconds && distance < closestDistance) { - closestDistance = distance; - closestSnapPoint = snapPoint; - } - } - - return { - snappedTime: closestSnapPoint ? closestSnapPoint.time : targetTime, - snapPoint: closestSnapPoint, - snapDistance: closestDistance, - }; -} - -export function snapElementEdge({ - targetTime, - elementDuration, - tracks, - playheadTime, - zoomLevel, - excludeElementId, - snapToStart = true, - bookmarks = [], -}: { - targetTime: number; - elementDuration: number; - tracks: Array; - playheadTime: number; - zoomLevel: number; - excludeElementId?: string; - snapToStart?: boolean; - bookmarks?: Array; -}): SnapResult { - const snapPoints = findSnapPoints({ - tracks, - playheadTime, - excludeElementId, - bookmarks, - }); - - const effectiveTargetTime = snapToStart - ? targetTime - : targetTime + elementDuration; - - const snapResult = snapToNearestPoint({ - targetTime: effectiveTargetTime, - snapPoints, - zoomLevel, - }); - - if (!snapToStart && snapResult.snapPoint) { - snapResult.snappedTime = snapResult.snappedTime - elementDuration; - } - - return snapResult; -} +import type { Bookmark, TimelineTrack } from "@/lib/timeline"; +import { BASE_TIMELINE_PIXELS_PER_SECOND } from "@/lib/timeline/scale"; +import { getElementKeyframes } from "@/lib/animation"; +import { TICKS_PER_SECOND } from "@/lib/wasm"; + +export interface SnapPoint { + time: number; + type: "element-start" | "element-end" | "playhead" | "bookmark" | "keyframe"; + elementId?: string; + trackId?: string; +} + +export interface SnapResult { + snappedTime: number; + snapPoint: SnapPoint | null; + snapDistance: number; +} + +const DEFAULT_SNAP_THRESHOLD_PX = 10; + +export function findSnapPoints({ + tracks, + playheadTime, + excludeElementId, + bookmarks = [], + excludeBookmarkTime, + enableElementSnapping = true, + enablePlayheadSnapping = true, + enableBookmarkSnapping = true, + enableKeyframeSnapping = true, +}: { + tracks: Array; + playheadTime: number; + excludeElementId?: string; + bookmarks?: Array; + excludeBookmarkTime?: number; + enableElementSnapping?: boolean; + enablePlayheadSnapping?: boolean; + enableBookmarkSnapping?: boolean; + enableKeyframeSnapping?: boolean; +}): SnapPoint[] { + const snapPoints: SnapPoint[] = []; + + for (const track of tracks) { + for (const element of track.elements) { + if (element.id === excludeElementId) continue; + + if (enableElementSnapping) { + snapPoints.push( + { + time: element.startTime, + type: "element-start", + elementId: element.id, + trackId: track.id, + }, + { + time: element.startTime + element.duration, + type: "element-end", + elementId: element.id, + trackId: track.id, + }, + ); + } + + if (enableKeyframeSnapping) { + for (const keyframe of getElementKeyframes({ + animations: element.animations, + })) { + snapPoints.push({ + time: element.startTime + keyframe.time, + type: "keyframe", + elementId: element.id, + trackId: track.id, + }); + } + } + } + } + + if (enablePlayheadSnapping) { + snapPoints.push({ time: playheadTime, type: "playhead" }); + } + + if (enableBookmarkSnapping) { + for (const bookmark of bookmarks) { + if ( + excludeBookmarkTime != null && + bookmark.time === excludeBookmarkTime + ) { + continue; + } + snapPoints.push({ time: bookmark.time, type: "bookmark" }); + } + } + + return snapPoints; +} + +export function snapToNearestPoint({ + targetTime, + snapPoints, + zoomLevel, + snapThreshold = DEFAULT_SNAP_THRESHOLD_PX, +}: { + targetTime: number; + snapPoints: Array; + zoomLevel: number; + snapThreshold?: number; +}): SnapResult { + const pixelsPerSecond = BASE_TIMELINE_PIXELS_PER_SECOND * zoomLevel; + const thresholdInTicks = (snapThreshold / pixelsPerSecond) * TICKS_PER_SECOND; + + let closestSnapPoint: SnapPoint | null = null; + let closestDistance = Infinity; + + for (const snapPoint of snapPoints) { + const distance = Math.abs(targetTime - snapPoint.time); + if (distance < thresholdInTicks && distance < closestDistance) { + closestDistance = distance; + closestSnapPoint = snapPoint; + } + } + + return { + snappedTime: closestSnapPoint ? closestSnapPoint.time : targetTime, + snapPoint: closestSnapPoint, + snapDistance: closestDistance, + }; +} + +export function snapElementEdge({ + targetTime, + elementDuration, + tracks, + playheadTime, + zoomLevel, + excludeElementId, + snapToStart = true, + bookmarks = [], +}: { + targetTime: number; + elementDuration: number; + tracks: Array; + playheadTime: number; + zoomLevel: number; + excludeElementId?: string; + snapToStart?: boolean; + bookmarks?: Array; +}): SnapResult { + const snapPoints = findSnapPoints({ + tracks, + playheadTime, + excludeElementId, + bookmarks, + }); + + const effectiveTargetTime = snapToStart + ? targetTime + : targetTime + elementDuration; + + const snapResult = snapToNearestPoint({ + targetTime: effectiveTargetTime, + snapPoints, + zoomLevel, + }); + + if (!snapToStart && snapResult.snapPoint) { + snapResult.snappedTime = snapResult.snappedTime - elementDuration; + } + + return snapResult; +} diff --git a/apps/web/src/lib/timeline/update-pipeline.ts b/apps/web/src/lib/timeline/update-pipeline.ts index cef99abc..61d09a9d 100644 --- a/apps/web/src/lib/timeline/update-pipeline.ts +++ b/apps/web/src/lib/timeline/update-pipeline.ts @@ -8,7 +8,7 @@ import { enforceMainTrackStart } from "@/lib/timeline/placement"; import type { RetimeConfig, TimelineElement, TimelineTrack } from "@/lib/timeline"; import { isRetimableElement } from "@/lib/timeline"; -type ElementUpdateField = keyof TimelineElement; +type ElementUpdateField = keyof TimelineElement | string; export interface ElementUpdateContext { tracks: TimelineTrack[]; diff --git a/apps/web/src/lib/timeline/zoom-utils.ts b/apps/web/src/lib/timeline/zoom-utils.ts index c06957d0..680f8d0d 100644 --- a/apps/web/src/lib/timeline/zoom-utils.ts +++ b/apps/web/src/lib/timeline/zoom-utils.ts @@ -2,6 +2,7 @@ import { BASE_TIMELINE_PIXELS_PER_SECOND, TIMELINE_ZOOM_MAX, } from "@/lib/timeline/scale"; +import { TICKS_PER_SECOND } from "@/lib/wasm"; const PADDING_MAX_RATIO = 0.75; const PADDING_MIN_RATIO = 0.15; @@ -14,12 +15,12 @@ export function getTimelineZoomMin({ duration: number; containerWidth: number | null | undefined; }): number { - const safeDuration = Math.max(duration, 1); + const safeDurationSeconds = Math.max(duration / TICKS_PER_SECOND, 1); const safeContainerWidth = containerWidth ?? 1000; const contentRatioAtMinZoom = 1 - PADDING_MAX_RATIO; const availableWidth = safeContainerWidth * contentRatioAtMinZoom; const zoomToFit = - availableWidth / (safeDuration * BASE_TIMELINE_PIXELS_PER_SECOND); + availableWidth / (safeDurationSeconds * BASE_TIMELINE_PIXELS_PER_SECOND); return Math.min(TIMELINE_ZOOM_MAX, zoomToFit); } diff --git a/apps/web/src/lib/wasm/constants.ts b/apps/web/src/lib/wasm/constants.ts new file mode 100644 index 00000000..ababe6b3 --- /dev/null +++ b/apps/web/src/lib/wasm/constants.ts @@ -0,0 +1,3 @@ +import { TICKS_PER_SECOND as _TICKS_PER_SECOND } from "opencut-wasm"; + +export const TICKS_PER_SECOND = _TICKS_PER_SECOND(); diff --git a/apps/web/src/lib/wasm/index.ts b/apps/web/src/lib/wasm/index.ts new file mode 100644 index 00000000..b04bfcf7 --- /dev/null +++ b/apps/web/src/lib/wasm/index.ts @@ -0,0 +1 @@ +export * from "./constants"; diff --git a/apps/web/src/services/renderer/canvas-renderer.ts b/apps/web/src/services/renderer/canvas-renderer.ts index 840d8b52..b142d97a 100644 --- a/apps/web/src/services/renderer/canvas-renderer.ts +++ b/apps/web/src/services/renderer/canvas-renderer.ts @@ -1,9 +1,11 @@ +import type { FrameRate } from "opencut-wasm"; +import { frameRateToFloat } from "@/lib/fps/utils"; import type { BaseNode } from "./nodes/base-node"; export type CanvasRendererParams = { width: number; height: number; - fps: number; + fps: FrameRate; }; export class CanvasRenderer { @@ -11,7 +13,7 @@ export class CanvasRenderer { context: OffscreenCanvasRenderingContext2D | CanvasRenderingContext2D; width: number; height: number; - fps: number; + fps: FrameRate; constructor({ width, height, fps }: CanvasRendererParams) { this.width = width; diff --git a/apps/web/src/services/renderer/nodes/blur-background-node.ts b/apps/web/src/services/renderer/nodes/blur-background-node.ts index c67a5113..4cb45b33 100644 --- a/apps/web/src/services/renderer/nodes/blur-background-node.ts +++ b/apps/web/src/services/renderer/nodes/blur-background-node.ts @@ -1,5 +1,5 @@ -import { TIME_EPSILON_SECONDS } from "@/constants/animation-constants"; import { buildGaussianBlurPasses, intensityToSigma } from "@/lib/effects/definitions/blur"; +import { mediaTimeToSeconds } from "opencut-wasm"; import { getSourceTimeAtClipTime } from "@/lib/retime"; import { videoCache } from "@/services/video-cache/service"; import type { RetimeConfig } from "@/lib/timeline"; @@ -39,9 +39,7 @@ export class BlurBackgroundNode extends BaseNode { private isInRange({ time }: { time: number }): boolean { const localTime = time - this.params.timeOffset; - return ( - localTime >= -TIME_EPSILON_SECONDS && localTime < this.params.duration - ); + return localTime >= 0 && localTime < this.params.duration; } private getSourceLocalTime({ time }: { time: number }): number { @@ -61,10 +59,11 @@ export class BlurBackgroundNode extends BaseNode { time: number; }): Promise { if (this.params.mediaType === "video") { + const sourceTimeTicks = this.getSourceLocalTime({ time }); const frame = await videoCache.getFrameAt({ mediaId: this.params.mediaId, file: this.params.file, - time: this.getSourceLocalTime({ time }), + time: mediaTimeToSeconds({ time: sourceTimeTicks }), }); if (!frame) { diff --git a/apps/web/src/services/renderer/nodes/video-node.ts b/apps/web/src/services/renderer/nodes/video-node.ts index 3bb836ec..4f225567 100644 --- a/apps/web/src/services/renderer/nodes/video-node.ts +++ b/apps/web/src/services/renderer/nodes/video-node.ts @@ -1,37 +1,39 @@ -import type { CanvasRenderer } from "../canvas-renderer"; -import { VisualNode, type VisualNodeParams } from "./visual-node"; -import { videoCache } from "@/services/video-cache/service"; - -export interface VideoNodeParams extends VisualNodeParams { - url: string; - file: File; - mediaId: string; -} - -export class VideoNode extends VisualNode { - async render({ renderer, time }: { renderer: CanvasRenderer; time: number }) { - await super.render({ renderer, time }); - - if (!this.isInRange({ time })) { - return; - } - - const videoTime = this.getSourceLocalTime({ time }); - - const frame = await videoCache.getFrameAt({ - mediaId: this.params.mediaId, - file: this.params.file, - time: videoTime, - }); - - if (frame) { - this.renderVisual({ - renderer, - source: frame.canvas, - sourceWidth: frame.canvas.width, - sourceHeight: frame.canvas.height, - timelineTime: time, - }); - } - } -} +import type { CanvasRenderer } from "../canvas-renderer"; +import { mediaTimeToSeconds } from "opencut-wasm"; +import { VisualNode, type VisualNodeParams } from "./visual-node"; +import { videoCache } from "@/services/video-cache/service"; + +export interface VideoNodeParams extends VisualNodeParams { + url: string; + file: File; + mediaId: string; +} + +export class VideoNode extends VisualNode { + async render({ renderer, time }: { renderer: CanvasRenderer; time: number }) { + await super.render({ renderer, time }); + + if (!this.isInRange({ time })) { + return; + } + + const videoTimeTicks = this.getSourceLocalTime({ time }); + const videoTimeSeconds = mediaTimeToSeconds({ time: videoTimeTicks }); + + const frame = await videoCache.getFrameAt({ + mediaId: this.params.mediaId, + file: this.params.file, + time: videoTimeSeconds, + }); + + if (frame) { + this.renderVisual({ + renderer, + source: frame.canvas, + sourceWidth: frame.canvas.width, + sourceHeight: frame.canvas.height, + timelineTime: time, + }); + } + } +} diff --git a/apps/web/src/services/renderer/nodes/visual-node.ts b/apps/web/src/services/renderer/nodes/visual-node.ts index 82093da4..4a5fcdc6 100644 --- a/apps/web/src/services/renderer/nodes/visual-node.ts +++ b/apps/web/src/services/renderer/nodes/visual-node.ts @@ -12,7 +12,6 @@ import { resolveTransformAtTime, } from "@/lib/animation"; import { resolveEffectParamsAtTime } from "@/lib/animation/effect-param-channel"; -import { TIME_EPSILON_SECONDS } from "@/constants/animation-constants"; import { effectsRegistry, resolveEffectPasses } from "@/lib/effects"; import { masksRegistry } from "@/lib/masks"; import { getSourceTimeAtClipTime } from "@/lib/retime"; @@ -58,7 +57,7 @@ export abstract class VisualNode< protected isInRange({ time }: { time: number }): boolean { const localTime = time - this.params.timeOffset; return ( - localTime >= -TIME_EPSILON_SECONDS && + localTime >= 0 && localTime < this.params.duration ); } diff --git a/apps/web/src/services/renderer/scene-exporter.ts b/apps/web/src/services/renderer/scene-exporter.ts index 2dbb7cdd..0c2ee17f 100644 --- a/apps/web/src/services/renderer/scene-exporter.ts +++ b/apps/web/src/services/renderer/scene-exporter.ts @@ -1,163 +1,170 @@ -import EventEmitter from "eventemitter3"; - -import { - Output, - Mp4OutputFormat, - WebMOutputFormat, - BufferTarget, - CanvasSource, - AudioBufferSource, - QUALITY_LOW, - QUALITY_MEDIUM, - QUALITY_HIGH, - QUALITY_VERY_HIGH, -} from "mediabunny"; -import type { RootNode } from "./nodes/root-node"; -import type { ExportFormat, ExportQuality } from "@/lib/export"; -import { CanvasRenderer } from "./canvas-renderer"; - -type ExportParams = { - width: number; - height: number; - fps: number; - format: ExportFormat; - quality: ExportQuality; - shouldIncludeAudio?: boolean; - audioBuffer?: AudioBuffer; -}; - -const qualityMap = { - low: QUALITY_LOW, - medium: QUALITY_MEDIUM, - high: QUALITY_HIGH, - very_high: QUALITY_VERY_HIGH, -}; - -export type SceneExporterEvents = { - progress: [progress: number]; - complete: [buffer: ArrayBuffer]; - error: [error: Error]; - cancelled: []; -}; - -export class SceneExporter extends EventEmitter { - private renderer: CanvasRenderer; - private format: ExportFormat; - private quality: ExportQuality; - private shouldIncludeAudio: boolean; - private audioBuffer?: AudioBuffer; - - private isCancelled = false; - - constructor({ - width, - height, - fps, - format, - quality, - shouldIncludeAudio, - audioBuffer, - }: ExportParams) { - super(); - this.renderer = new CanvasRenderer({ - width, - height, - fps, - }); - - this.format = format; - this.quality = quality; - this.shouldIncludeAudio = shouldIncludeAudio ?? false; - this.audioBuffer = audioBuffer; - } - - cancel(): void { - this.isCancelled = true; - } - - async export({ - rootNode, - }: { - rootNode: RootNode; - }): Promise { - const { fps } = this.renderer; - const frameCount = Math.ceil(rootNode.duration * fps); - - const outputFormat = - this.format === "webm" ? new WebMOutputFormat() : new Mp4OutputFormat(); - - const output = new Output({ - format: outputFormat, - target: new BufferTarget(), - }); - - const videoSource = new CanvasSource(this.renderer.canvas, { - codec: this.format === "webm" ? "vp9" : "avc", - bitrate: qualityMap[this.quality], - }); - - output.addVideoTrack(videoSource, { frameRate: fps }); - - let audioSource: AudioBufferSource | null = null; - if (this.shouldIncludeAudio && this.audioBuffer) { - let audioCodec: "aac" | "opus" = - this.format === "webm" ? "opus" : "aac"; - - if (audioCodec === "aac" && typeof AudioEncoder !== "undefined") { - const { supported } = await AudioEncoder.isConfigSupported({ - codec: "mp4a.40.2", - sampleRate: this.audioBuffer.sampleRate, - numberOfChannels: this.audioBuffer.numberOfChannels, - bitrate: 192000, - }); - if (!supported) audioCodec = "opus"; - } - - audioSource = new AudioBufferSource({ - codec: audioCodec, - bitrate: qualityMap[this.quality], - }); - output.addAudioTrack(audioSource); - } - - await output.start(); - - if (audioSource && this.audioBuffer) { - await audioSource.add(this.audioBuffer); - audioSource.close(); - } - - for (let i = 0; i < frameCount; i++) { - if (this.isCancelled) { - await output.cancel(); - this.emit("cancelled"); - return null; - } - - const time = i / fps; - await this.renderer.render({ node: rootNode, time }); - await videoSource.add(time, 1 / fps); - - this.emit("progress", i / frameCount); - } - - if (this.isCancelled) { - await output.cancel(); - this.emit("cancelled"); - return null; - } - - videoSource.close(); - await output.finalize(); - this.emit("progress", 1); - - const buffer = output.target.buffer; - if (!buffer) { - this.emit("error", new Error("Failed to export video")); - return null; - } - - this.emit("complete", buffer); - return buffer; - } -} +import EventEmitter from "eventemitter3"; + +import { + Output, + Mp4OutputFormat, + WebMOutputFormat, + BufferTarget, + CanvasSource, + AudioBufferSource, + QUALITY_LOW, + QUALITY_MEDIUM, + QUALITY_HIGH, + QUALITY_VERY_HIGH, +} from "mediabunny"; +import type { FrameRate } from "opencut-wasm"; +import { mediaTimeToSeconds } from "opencut-wasm"; +import { TICKS_PER_SECOND } from "@/lib/wasm"; +import { frameRateToFloat } from "@/lib/fps/utils"; +import type { RootNode } from "./nodes/root-node"; +import type { ExportFormat, ExportQuality } from "@/lib/export"; +import { CanvasRenderer } from "./canvas-renderer"; + +type ExportParams = { + width: number; + height: number; + fps: FrameRate; + format: ExportFormat; + quality: ExportQuality; + shouldIncludeAudio?: boolean; + audioBuffer?: AudioBuffer; +}; + +const qualityMap = { + low: QUALITY_LOW, + medium: QUALITY_MEDIUM, + high: QUALITY_HIGH, + very_high: QUALITY_VERY_HIGH, +}; + +export type SceneExporterEvents = { + progress: [progress: number]; + complete: [buffer: ArrayBuffer]; + error: [error: Error]; + cancelled: []; +}; + +export class SceneExporter extends EventEmitter { + private renderer: CanvasRenderer; + private format: ExportFormat; + private quality: ExportQuality; + private shouldIncludeAudio: boolean; + private audioBuffer?: AudioBuffer; + + private isCancelled = false; + + constructor({ + width, + height, + fps, + format, + quality, + shouldIncludeAudio, + audioBuffer, + }: ExportParams) { + super(); + this.renderer = new CanvasRenderer({ + width, + height, + fps, + }); + + this.format = format; + this.quality = quality; + this.shouldIncludeAudio = shouldIncludeAudio ?? false; + this.audioBuffer = audioBuffer; + } + + cancel(): void { + this.isCancelled = true; + } + + async export({ + rootNode, + }: { + rootNode: RootNode; + }): Promise { + const fps = this.renderer.fps; + const fpsFloat = frameRateToFloat(fps); + const ticksPerFrame = Math.round(TICKS_PER_SECOND * fps.denominator / fps.numerator); + const frameCount = Math.floor(rootNode.duration / ticksPerFrame); + + const outputFormat = + this.format === "webm" ? new WebMOutputFormat() : new Mp4OutputFormat(); + + const output = new Output({ + format: outputFormat, + target: new BufferTarget(), + }); + + const videoSource = new CanvasSource(this.renderer.canvas, { + codec: this.format === "webm" ? "vp9" : "avc", + bitrate: qualityMap[this.quality], + }); + + output.addVideoTrack(videoSource, { frameRate: fpsFloat }); + + let audioSource: AudioBufferSource | null = null; + if (this.shouldIncludeAudio && this.audioBuffer) { + let audioCodec: "aac" | "opus" = + this.format === "webm" ? "opus" : "aac"; + + if (audioCodec === "aac" && typeof AudioEncoder !== "undefined") { + const { supported } = await AudioEncoder.isConfigSupported({ + codec: "mp4a.40.2", + sampleRate: this.audioBuffer.sampleRate, + numberOfChannels: this.audioBuffer.numberOfChannels, + bitrate: 192000, + }); + if (!supported) audioCodec = "opus"; + } + + audioSource = new AudioBufferSource({ + codec: audioCodec, + bitrate: qualityMap[this.quality], + }); + output.addAudioTrack(audioSource); + } + + await output.start(); + + if (audioSource && this.audioBuffer) { + await audioSource.add(this.audioBuffer); + audioSource.close(); + } + + for (let i = 0; i < frameCount; i++) { + if (this.isCancelled) { + await output.cancel(); + this.emit("cancelled"); + return null; + } + + const timeTicks = i * ticksPerFrame; + const timeSeconds = mediaTimeToSeconds({ time: timeTicks }); + await this.renderer.render({ node: rootNode, time: timeTicks }); + await videoSource.add(timeSeconds, 1 / fpsFloat); + + this.emit("progress", i / frameCount); + } + + if (this.isCancelled) { + await output.cancel(); + this.emit("cancelled"); + return null; + } + + videoSource.close(); + await output.finalize(); + this.emit("progress", 1); + + const buffer = output.target.buffer; + if (!buffer) { + this.emit("error", new Error("Failed to export video")); + return null; + } + + this.emit("complete", buffer); + return buffer; + } +} diff --git a/apps/web/src/services/storage/migrations/__tests__/v1-to-v2.test.ts b/apps/web/src/services/storage/migrations/__tests__/v1-to-v2.test.ts index 7a4dd641..7a0617a7 100644 --- a/apps/web/src/services/storage/migrations/__tests__/v1-to-v2.test.ts +++ b/apps/web/src/services/storage/migrations/__tests__/v1-to-v2.test.ts @@ -4,7 +4,7 @@ import { DEFAULT_BACKGROUND_COLOR, } from "@/lib/background/constants"; import { DEFAULT_CANVAS_SIZE } from "@/lib/canvas/constants"; -import { DEFAULT_FPS } from "@/lib/fps/constants"; +const DEFAULT_FPS = 30; import type { MediaAssetData } from "@/services/storage/types"; import { getProjectId, transformProjectV1ToV2 } from "../transformers/v1-to-v2"; import { diff --git a/apps/web/src/services/storage/migrations/__tests__/v22-to-v23.test.ts b/apps/web/src/services/storage/migrations/__tests__/v22-to-v23.test.ts new file mode 100644 index 00000000..5b764b48 --- /dev/null +++ b/apps/web/src/services/storage/migrations/__tests__/v22-to-v23.test.ts @@ -0,0 +1,192 @@ +import { describe, expect, test } from "bun:test"; +import { transformProjectV22ToV23 } from "../transformers/v22-to-v23"; + +describe("V22 to V23 Migration", () => { + test("converts project time values from seconds to ticks and fps to a frame-rate object", () => { + const result = transformProjectV22ToV23({ + project: { + id: "project-v22-time", + version: 22, + metadata: { + id: "project-v22-time", + name: "Project", + duration: 15.5, + createdAt: "2026-01-01T00:00:00.000Z", + updatedAt: "2026-01-01T00:00:00.000Z", + }, + settings: { + fps: 29.97, + canvasSize: { width: 1920, height: 1080 }, + background: { type: "color", color: "#000000" }, + }, + timelineViewState: { + zoomLevel: 1, + scrollLeft: 120, + playheadTime: 1.25, + }, + scenes: [ + { + id: "scene-1", + bookmarks: [ + { time: 2.5, duration: 0.75, note: "Marker", color: "#ff0000" }, + { time: 4.5 }, + ], + tracks: [ + { + id: "track-1", + type: "video", + elements: [ + { + id: "element-1", + type: "video", + startTime: 1.25, + duration: 5.5, + trimStart: 0.25, + trimEnd: 0.5, + sourceDuration: 6.25, + animations: { + bindings: { + opacity: { + path: "opacity", + kind: "number", + components: [ + { + key: "value", + channelId: "opacity:value", + }, + ], + }, + }, + channels: { + "opacity:value": { + kind: "scalar", + keys: [ + { + id: "key-1", + time: 0.5, + value: 1, + segmentToNext: "bezier", + tangentMode: "flat", + rightHandle: { + dt: 0.25, + dv: 0.2, + }, + }, + { + id: "key-2", + time: 1.0, + value: 0.4, + segmentToNext: "linear", + tangentMode: "flat", + leftHandle: { + dt: -0.125, + dv: -0.1, + }, + }, + ], + }, + }, + }, + }, + ], + }, + ], + createdAt: "2026-01-01T00:00:00.000Z", + updatedAt: "2026-01-01T00:00:00.000Z", + }, + ], + }, + }); + + expect(result.skipped).toBe(false); + expect(result.project.version).toBe(23); + + const metadata = result.project.metadata as Record; + expect(metadata.duration).toBe(1_860_000); + + const settings = result.project.settings as Record; + expect(settings.fps).toEqual({ numerator: 30_000, denominator: 1_001 }); + + const timelineViewState = result.project.timelineViewState as Record< + string, + unknown + >; + expect(timelineViewState.playheadTime).toBe(150_000); + expect(timelineViewState.scrollLeft).toBe(120); + + const scenes = result.project.scenes as Array>; + const scene = scenes[0]; + expect(scene.bookmarks).toEqual([ + { + time: 300_000, + duration: 90_000, + note: "Marker", + color: "#ff0000", + }, + { time: 540_000 }, + ]); + + const tracks = scene.tracks as Array>; + const elements = tracks[0].elements as Array>; + const element = elements[0]; + expect(element.startTime).toBe(150_000); + expect(element.duration).toBe(660_000); + expect(element.trimStart).toBe(30_000); + expect(element.trimEnd).toBe(60_000); + expect(element.sourceDuration).toBe(750_000); + + const animations = element.animations as Record; + const channels = animations.channels as Record>; + expect(channels["opacity:value"]).toEqual({ + kind: "scalar", + keys: [ + { + id: "key-1", + time: 60_000, + value: 1, + segmentToNext: "bezier", + tangentMode: "flat", + rightHandle: { + dt: 30_000, + dv: 0.2, + }, + }, + { + id: "key-2", + time: 120_000, + value: 0.4, + segmentToNext: "linear", + tangentMode: "flat", + leftHandle: { + dt: -15_000, + dv: -0.1, + }, + }, + ], + }); + }); + + test("skips projects already on v23", () => { + const result = transformProjectV22ToV23({ + project: { + id: "project-v23", + version: 23, + }, + }); + + expect(result.skipped).toBe(true); + expect(result.reason).toBe("already v23"); + }); + + test("skips projects not on v22", () => { + const result = transformProjectV22ToV23({ + project: { + id: "project-v21", + version: 21, + }, + }); + + expect(result.skipped).toBe(true); + expect(result.reason).toBe("not v22"); + }); +}); diff --git a/apps/web/src/services/storage/migrations/index.ts b/apps/web/src/services/storage/migrations/index.ts index ef41673e..9f0f3e0d 100644 --- a/apps/web/src/services/storage/migrations/index.ts +++ b/apps/web/src/services/storage/migrations/index.ts @@ -21,10 +21,11 @@ import { V18toV19Migration } from "./v18-to-v19"; import { V19toV20Migration } from "./v19-to-v20"; import { V20toV21Migration } from "./v20-to-v21"; import { V21toV22Migration } from "./v21-to-v22"; +import { V22toV23Migration } from "./v22-to-v23"; export { runStorageMigrations } from "./runner"; export type { MigrationProgress } from "./runner"; -export const CURRENT_PROJECT_VERSION = 22; +export const CURRENT_PROJECT_VERSION = 23; export const migrations = [ new V0toV1Migration(), @@ -49,4 +50,5 @@ export const migrations = [ new V19toV20Migration(), new V20toV21Migration(), new V21toV22Migration(), + new V22toV23Migration(), ]; diff --git a/apps/web/src/services/storage/migrations/transformers/v1-to-v2.ts b/apps/web/src/services/storage/migrations/transformers/v1-to-v2.ts index a2c72478..6313c311 100644 --- a/apps/web/src/services/storage/migrations/transformers/v1-to-v2.ts +++ b/apps/web/src/services/storage/migrations/transformers/v1-to-v2.ts @@ -3,7 +3,7 @@ import { DEFAULT_BACKGROUND_COLOR, } from "@/lib/background/constants"; import { DEFAULT_CANVAS_SIZE } from "@/lib/canvas/constants"; -import { DEFAULT_FPS } from "@/lib/fps/constants"; +const DEFAULT_FPS = 30; import { IndexedDBAdapter } from "@/services/storage/indexeddb-adapter"; import type { MediaAssetData } from "@/services/storage/types"; import type { MigrationResult, ProjectRecord } from "./types"; diff --git a/apps/web/src/services/storage/migrations/transformers/v22-to-v23.ts b/apps/web/src/services/storage/migrations/transformers/v22-to-v23.ts new file mode 100644 index 00000000..cde1294e --- /dev/null +++ b/apps/web/src/services/storage/migrations/transformers/v22-to-v23.ts @@ -0,0 +1,335 @@ +import type { MigrationResult, ProjectRecord } from "./types"; +import { getProjectId, isRecord } from "./utils"; + +import { TICKS_PER_SECOND } from "@/lib/wasm"; +const ARBITRARY_FPS_DENOMINATOR = 1_000_000; +const STANDARD_FRAME_RATES = [ + { value: 24_000 / 1_001, numerator: 24_000, denominator: 1_001 }, + { value: 24, numerator: 24, denominator: 1 }, + { value: 25, numerator: 25, denominator: 1 }, + { value: 30_000 / 1_001, numerator: 30_000, denominator: 1_001 }, + { value: 30, numerator: 30, denominator: 1 }, + { value: 48, numerator: 48, denominator: 1 }, + { value: 50, numerator: 50, denominator: 1 }, + { value: 60_000 / 1_001, numerator: 60_000, denominator: 1_001 }, + { value: 60, numerator: 60, denominator: 1 }, + { value: 120, numerator: 120, denominator: 1 }, +] as const; +const STANDARD_FRAME_RATE_TOLERANCE = 0.01; + +export function transformProjectV22ToV23({ + project, +}: { + project: ProjectRecord; +}): MigrationResult { + if (!getProjectId({ project })) { + return { project, skipped: true, reason: "no project id" }; + } + + const version = project.version; + if (typeof version !== "number") { + return { project, skipped: true, reason: "invalid version" }; + } + if (version >= 23) { + return { project, skipped: true, reason: "already v23" }; + } + if (version !== 22) { + return { project, skipped: true, reason: "not v22" }; + } + + return { + project: { + ...migrateProject({ project }), + version: 23, + }, + skipped: false, + }; +} + +function migrateProject({ + project, +}: { + project: ProjectRecord; +}): ProjectRecord { + const nextProject = { ...project }; + + if (isRecord(project.metadata)) { + nextProject.metadata = migrateMetadata({ metadata: project.metadata }); + } + + if (isRecord(project.settings)) { + nextProject.settings = migrateSettings({ settings: project.settings }); + } + + if (isRecord(project.timelineViewState)) { + nextProject.timelineViewState = migrateTimelineViewState({ + timelineViewState: project.timelineViewState, + }); + } + + if (Array.isArray(project.scenes)) { + nextProject.scenes = project.scenes.map((scene) => migrateScene({ scene })); + } + + return nextProject; +} + +function migrateMetadata({ + metadata, +}: { + metadata: ProjectRecord; +}): ProjectRecord { + return migrateTimeFields({ + record: metadata, + keys: ["duration"], + }); +} + +function migrateSettings({ + settings, +}: { + settings: ProjectRecord; +}): ProjectRecord { + const nextSettings = { ...settings }; + if ("fps" in settings) { + nextSettings.fps = migrateFrameRate({ fps: settings.fps }); + } + return nextSettings; +} + +function migrateTimelineViewState({ + timelineViewState, +}: { + timelineViewState: ProjectRecord; +}): ProjectRecord { + return migrateTimeFields({ + record: timelineViewState, + keys: ["playheadTime"], + }); +} + +function migrateScene({ scene }: { scene: unknown }): unknown { + if (!isRecord(scene)) { + return scene; + } + + const nextScene = { ...scene }; + + if (Array.isArray(scene.bookmarks)) { + nextScene.bookmarks = scene.bookmarks.map((bookmark) => + migrateBookmark({ bookmark }), + ); + } + + if (Array.isArray(scene.tracks)) { + nextScene.tracks = scene.tracks.map((track) => migrateTrack({ track })); + } + + return nextScene; +} + +function migrateTrack({ track }: { track: unknown }): unknown { + if (!isRecord(track)) { + return track; + } + + if (!Array.isArray(track.elements)) { + return track; + } + + return { + ...track, + elements: track.elements.map((element) => migrateElement({ element })), + }; +} + +function migrateElement({ element }: { element: unknown }): unknown { + if (!isRecord(element)) { + return element; + } + + const nextElement = migrateTimeFields({ + record: element, + keys: ["duration", "startTime", "trimStart", "trimEnd", "sourceDuration"], + }); + + if (isRecord(element.animations)) { + nextElement.animations = migrateAnimations({ + animations: element.animations, + }); + } + + return nextElement; +} + +function migrateAnimations({ + animations, +}: { + animations: ProjectRecord; +}): ProjectRecord { + if (!isRecord(animations.channels)) { + return animations; + } + + return { + ...animations, + channels: Object.fromEntries( + Object.entries(animations.channels).map(([channelId, channel]) => [ + channelId, + migrateAnimationChannel({ channel }), + ]), + ), + }; +} + +function migrateAnimationChannel({ channel }: { channel: unknown }): unknown { + if (!isRecord(channel)) { + return channel; + } + + if (!Array.isArray(channel.keys)) { + return channel; + } + + return { + ...channel, + keys: channel.keys.map((keyframe) => migrateAnimationKeyframe({ keyframe })), + }; +} + +function migrateAnimationKeyframe({ + keyframe, +}: { + keyframe: unknown; +}): unknown { + if (!isRecord(keyframe)) { + return keyframe; + } + + const nextKeyframe = migrateTimeFields({ + record: keyframe, + keys: ["time"], + }); + + if (isRecord(keyframe.leftHandle)) { + nextKeyframe.leftHandle = migrateCurveHandle({ + handle: keyframe.leftHandle, + }); + } + + if (isRecord(keyframe.rightHandle)) { + nextKeyframe.rightHandle = migrateCurveHandle({ + handle: keyframe.rightHandle, + }); + } + + return nextKeyframe; +} + +function migrateCurveHandle({ + handle, +}: { + handle: ProjectRecord; +}): ProjectRecord { + return migrateTimeFields({ + record: handle, + keys: ["dt"], + }); +} + +function migrateBookmark({ bookmark }: { bookmark: unknown }): unknown { + if (!isRecord(bookmark)) { + return bookmark; + } + + return migrateTimeFields({ + record: bookmark, + keys: ["time", "duration"], + }); +} + +function migrateTimeFields({ + record, + keys, +}: { + record: ProjectRecord; + keys: string[]; +}): ProjectRecord { + const nextRecord = { ...record }; + + for (const key of keys) { + if (!(key in record)) { + continue; + } + + nextRecord[key] = migrateTimeValue({ value: record[key] }); + } + + return nextRecord; +} + +function migrateTimeValue({ value }: { value: unknown }): unknown { + if (typeof value !== "number" || !Number.isFinite(value)) { + return value; + } + + return secondsToTicks({ value }); +} + +function secondsToTicks({ value }: { value: number }): number { + return Math.round(value * TICKS_PER_SECOND); +} + +function migrateFrameRate({ fps }: { fps: unknown }): unknown { + if (isRecord(fps)) { + return fps; + } + + if (typeof fps !== "number" || !Number.isFinite(fps) || fps <= 0) { + return fps; + } + + const standardFrameRate = STANDARD_FRAME_RATES.find( + (candidate) => Math.abs(fps - candidate.value) <= STANDARD_FRAME_RATE_TOLERANCE, + ); + if (standardFrameRate) { + return { + numerator: standardFrameRate.numerator, + denominator: standardFrameRate.denominator, + }; + } + + if (Number.isInteger(fps)) { + return { numerator: fps, denominator: 1 }; + } + + const scaledNumerator = Math.round(fps * ARBITRARY_FPS_DENOMINATOR); + const divisor = greatestCommonDivisor({ + left: scaledNumerator, + right: ARBITRARY_FPS_DENOMINATOR, + }); + + return { + numerator: scaledNumerator / divisor, + denominator: ARBITRARY_FPS_DENOMINATOR / divisor, + }; +} + +function greatestCommonDivisor({ + left, + right, +}: { + left: number; + right: number; +}): number { + let nextLeft = Math.abs(left); + let nextRight = Math.abs(right); + + while (nextRight !== 0) { + const remainder = nextLeft % nextRight; + nextLeft = nextRight; + nextRight = remainder; + } + + return nextLeft || 1; +} diff --git a/apps/web/src/services/storage/migrations/v22-to-v23.ts b/apps/web/src/services/storage/migrations/v22-to-v23.ts new file mode 100644 index 00000000..cd352e45 --- /dev/null +++ b/apps/web/src/services/storage/migrations/v22-to-v23.ts @@ -0,0 +1,16 @@ +import { StorageMigration } from "./base"; +import type { ProjectRecord } from "./transformers/types"; +import { transformProjectV22ToV23 } from "./transformers/v22-to-v23"; + +export class V22toV23Migration extends StorageMigration { + from = 22; + to = 23; + + async transform(project: ProjectRecord): Promise<{ + project: ProjectRecord; + skipped: boolean; + reason?: string; + }> { + return transformProjectV22ToV23({ project }); + } +} diff --git a/bun.lock b/bun.lock index d8b0e0ff..c8eceb53 100644 --- a/bun.lock +++ b/bun.lock @@ -54,7 +54,7 @@ "nanoid": "^5.1.5", "next": "16.1.3", "next-themes": "^0.4.4", - "opencut-wasm": "^0.1.3", + "opencut-wasm": "^0.2.3", "pg": "^8.16.2", "postgres": "^3.4.5", "radix-ui": "^1.4.3", @@ -1359,7 +1359,7 @@ "onnxruntime-web": ["onnxruntime-web@1.22.0-dev.20250409-89f8206ba4", "", { "dependencies": { "flatbuffers": "^25.1.24", "guid-typescript": "^1.0.9", "long": "^5.2.3", "onnxruntime-common": "1.22.0-dev.20250409-89f8206ba4", "platform": "^1.3.6", "protobufjs": "^7.2.4" } }, "sha512-0uS76OPgH0hWCPrFKlL8kYVV7ckM7t/36HfbgoFw6Nd0CZVVbQC4PkrR8mBX8LtNUFZO25IQBqV2Hx2ho3FlbQ=="], - "opencut-wasm": ["opencut-wasm@0.1.3", "", {}, "sha512-3MlWL8J8NCRBfm/6LrdvW08rgIPqyhk5dCG3j/zokKDOf5mwULox9Vpqi2jgZ+dSKIWReHuUCZq2b0cGCFuVHQ=="], + "opencut-wasm": ["opencut-wasm@0.2.3", "", {}, "sha512-SaLe2fgvLK+EcW7qqggmcfXaQwo6SOh83/h3dtEEbdqEka7gn1PGJECKnTZq4OWhkNEgf8SKJy/HiJOQvyZC+Q=="], "p-limit": ["p-limit@6.2.0", "", { "dependencies": { "yocto-queue": "^1.1.1" } }, "sha512-kuUqqHNUqoIWp/c467RI4X6mmyuojY5jGutNU0wVTmEOOfcuwLqyMVoAi9MKi2Ak+5i9+nhmrK4ufZE8069kHA=="], diff --git a/rust/crates/bridge/src/bridge.rs b/rust/crates/bridge/src/bridge.rs index 52dee731..99b00615 100644 --- a/rust/crates/bridge/src/bridge.rs +++ b/rust/crates/bridge/src/bridge.rs @@ -1,10 +1,36 @@ use proc_macro::TokenStream; use quote::quote; -use syn::{ItemFn, parse_macro_input}; +use syn::{FnArg, Item, ItemConst, ItemFn, parse_macro_input}; #[proc_macro_attribute] pub fn export(_attr: TokenStream, item: TokenStream) -> TokenStream { - let function = parse_macro_input!(item as ItemFn); + match parse_macro_input!(item as Item) { + Item::Fn(function) => export_fn(function), + Item::Const(constant) => export_const(constant), + other => syn::Error::new_spanned(other, "#[export] only supports fn and const items") + .to_compile_error() + .into(), + } +} + +fn export_fn(function: ItemFn) -> TokenStream { + let param_count = function + .sig + .inputs + .iter() + .filter(|arg| matches!(arg, FnArg::Typed(_))) + .count(); + + if param_count > 1 { + return syn::Error::new_spanned( + &function.sig.inputs, + "#[export] functions must accept a single options struct, not positional arguments. \ + Wrap parameters in a struct: `fn foo(FooOptions { a, b }: FooOptions)`", + ) + .to_compile_error() + .into(); + } + let js_name = snake_to_camel(&function.sig.ident.to_string()); quote! { @@ -14,6 +40,26 @@ pub fn export(_attr: TokenStream, item: TokenStream) -> TokenStream { .into() } +fn export_const(constant: ItemConst) -> TokenStream { + let js_name = constant.ident.to_string(); + let const_ident = &constant.ident; + let getter_ident = syn::Ident::new( + &format!("__const_{}", constant.ident.to_string().to_lowercase()), + constant.ident.span(), + ); + + quote! { + #constant + + #[cfg(feature = "wasm")] + #[::wasm_bindgen::prelude::wasm_bindgen(js_name = #js_name)] + pub fn #getter_ident() -> f64 { + #const_ident as f64 + } + } + .into() +} + fn snake_to_camel(name: &str) -> String { let mut camel = String::with_capacity(name.len()); let mut should_uppercase_next = false; diff --git a/rust/crates/effects/Cargo.toml b/rust/crates/effects/Cargo.toml new file mode 100644 index 00000000..5b6e2a1e --- /dev/null +++ b/rust/crates/effects/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "effects" +version = "0.1.0" +edition = "2024" + +[lib] +path = "src/effects.rs" +crate-type = ["rlib"] + +[dependencies] +bytemuck = { version = "1.25.0", features = ["derive"] } +gpu = { version = "0.1.0", path = "../gpu" } +thiserror = "2.0.18" +wgpu = "29.0.1" diff --git a/rust/crates/effects/src/effects.rs b/rust/crates/effects/src/effects.rs new file mode 100644 index 00000000..c6d750bb --- /dev/null +++ b/rust/crates/effects/src/effects.rs @@ -0,0 +1,5 @@ +mod pipeline; +mod types; + +pub use pipeline::{ApplyEffectsOptions, EffectPipeline, EffectsError}; +pub use types::{EffectPass, UniformValue}; diff --git a/rust/crates/effects/src/pipeline.rs b/rust/crates/effects/src/pipeline.rs new file mode 100644 index 00000000..58692bde --- /dev/null +++ b/rust/crates/effects/src/pipeline.rs @@ -0,0 +1,306 @@ +use std::collections::HashMap; + +use bytemuck::{Pod, Zeroable}; +use gpu::{FULLSCREEN_SHADER_SOURCE, GPU_TEXTURE_FORMAT, GpuContext}; +use thiserror::Error; +use wgpu::util::DeviceExt; + +use crate::{EffectPass, UniformValue}; + +const GAUSSIAN_BLUR_SHADER_ID: &str = "gaussian-blur"; +const GAUSSIAN_BLUR_SHADER_SOURCE: &str = include_str!("shaders/gaussian_blur.wgsl"); + +pub struct ApplyEffectsOptions<'a> { + pub source: &'a wgpu::Texture, + pub width: u32, + pub height: u32, + pub passes: &'a [EffectPass], +} + +pub struct EffectPipeline { + uniform_bind_group_layout: wgpu::BindGroupLayout, + pipelines: HashMap, +} + +#[derive(Debug, Error)] +pub enum EffectsError { + #[error("At least one effect pass is required")] + MissingEffectPasses, + #[error("Unknown effect shader '{shader}'")] + UnknownEffectShader { shader: String }, + #[error("Missing uniform '{uniform}' for shader '{shader}'")] + MissingUniform { shader: String, uniform: String }, + #[error("Uniform '{uniform}' for shader '{shader}' must be a number")] + InvalidNumberUniform { shader: String, uniform: String }, + #[error( + "Uniform '{uniform}' for shader '{shader}' must be a vector of length {expected_length}" + )] + InvalidVectorUniform { + shader: String, + uniform: String, + expected_length: usize, + }, + #[error("Shader '{shader}' does not support uniform '{uniform}'")] + UnsupportedUniform { shader: String, uniform: String }, +} + +#[repr(C)] +#[derive(Clone, Copy, Pod, Zeroable)] +struct EffectUniformBuffer { + resolution: [f32; 2], + direction: [f32; 2], + scalars: [f32; 4], +} + +impl EffectPipeline { + pub fn new(context: &GpuContext) -> Self { + let uniform_bind_group_layout = + context + .device() + .create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("effects-uniform-bind-group-layout"), + entries: &[wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Uniform, + has_dynamic_offset: false, + min_binding_size: None, + }, + count: None, + }], + }); + let vertex_shader_module = + context + .device() + .create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("effects-fullscreen-shader"), + source: wgpu::ShaderSource::Wgsl(FULLSCREEN_SHADER_SOURCE.into()), + }); + let gaussian_blur_shader_module = + context + .device() + .create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("effects-gaussian-blur-shader"), + source: wgpu::ShaderSource::Wgsl(GAUSSIAN_BLUR_SHADER_SOURCE.into()), + }); + let pipeline_layout = + context + .device() + .create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("effects-pipeline-layout"), + bind_group_layouts: &[ + Some(context.texture_sampler_bind_group_layout()), + Some(&uniform_bind_group_layout), + ], + immediate_size: 0, + }); + let gaussian_blur_pipeline = + context + .device() + .create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some("effects-gaussian-blur-pipeline"), + layout: Some(&pipeline_layout), + vertex: wgpu::VertexState { + module: &vertex_shader_module, + entry_point: Some("vertex_main"), + buffers: &[wgpu::VertexBufferLayout { + array_stride: std::mem::size_of::<[f32; 2]>() as u64, + step_mode: wgpu::VertexStepMode::Vertex, + attributes: &[wgpu::VertexAttribute { + format: wgpu::VertexFormat::Float32x2, + offset: 0, + shader_location: 0, + }], + }], + compilation_options: wgpu::PipelineCompilationOptions::default(), + }, + fragment: Some(wgpu::FragmentState { + module: &gaussian_blur_shader_module, + entry_point: Some("fragment_main"), + targets: &[Some(wgpu::ColorTargetState { + format: GPU_TEXTURE_FORMAT, + blend: None, + write_mask: wgpu::ColorWrites::ALL, + })], + compilation_options: wgpu::PipelineCompilationOptions::default(), + }), + primitive: wgpu::PrimitiveState::default(), + depth_stencil: None, + multisample: wgpu::MultisampleState::default(), + multiview_mask: None, + cache: None, + }); + let pipelines = + HashMap::from([(GAUSSIAN_BLUR_SHADER_ID.to_string(), gaussian_blur_pipeline)]); + + Self { + uniform_bind_group_layout, + pipelines, + } + } + + pub fn apply( + &self, + context: &GpuContext, + ApplyEffectsOptions { + source, + width, + height, + passes, + }: ApplyEffectsOptions<'_>, + ) -> Result { + let mut current_texture: Option = None; + + for pass in passes { + let input_texture = current_texture.as_ref().unwrap_or(source); + let output_texture = + context.create_render_texture(width, height, "effects-pass-output"); + let input_view = input_texture.create_view(&wgpu::TextureViewDescriptor::default()); + let output_view = output_texture.create_view(&wgpu::TextureViewDescriptor::default()); + let texture_bind_group = + context + .device() + .create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("effects-texture-bind-group"), + layout: context.texture_sampler_bind_group_layout(), + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: wgpu::BindingResource::TextureView(&input_view), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: wgpu::BindingResource::Sampler(context.linear_sampler()), + }, + ], + }); + let uniform_buffer = + context + .device() + .create_buffer_init(&wgpu::util::BufferInitDescriptor { + label: Some("effects-uniform-buffer"), + contents: bytemuck::bytes_of(&pack_effect_uniforms(pass, width, height)?), + usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, + }); + let uniform_bind_group = + context + .device() + .create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("effects-uniform-bind-group"), + layout: &self.uniform_bind_group_layout, + entries: &[wgpu::BindGroupEntry { + binding: 0, + resource: uniform_buffer.as_entire_binding(), + }], + }); + let pipeline = self.pipelines.get(&pass.shader).ok_or_else(|| { + EffectsError::UnknownEffectShader { + shader: pass.shader.clone(), + } + })?; + let mut encoder = + context + .device() + .create_command_encoder(&wgpu::CommandEncoderDescriptor { + label: Some("effects-command-encoder"), + }); + + { + let mut render_pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("effects-render-pass"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: &output_view, + resolve_target: None, + depth_slice: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Clear(wgpu::Color::TRANSPARENT), + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + occlusion_query_set: None, + timestamp_writes: None, + multiview_mask: None, + }); + render_pass.set_pipeline(pipeline); + render_pass.set_vertex_buffer(0, context.fullscreen_quad().slice(..)); + render_pass.set_bind_group(0, &texture_bind_group, &[]); + render_pass.set_bind_group(1, &uniform_bind_group, &[]); + render_pass.draw(0..6, 0..1); + } + + context.queue().submit([encoder.finish()]); + current_texture = Some(output_texture); + } + + current_texture.ok_or(EffectsError::MissingEffectPasses) + } +} + +fn pack_effect_uniforms( + pass: &EffectPass, + width: u32, + height: u32, +) -> Result { + let shader = pass.shader.as_str(); + let sigma = read_number_uniform(pass, "u_sigma")?; + let step = read_number_uniform(pass, "u_step")?; + let direction = read_vec2_uniform(pass, "u_direction")?; + + for uniform in pass.uniforms.keys() { + if uniform == "u_sigma" || uniform == "u_step" || uniform == "u_direction" { + continue; + } + return Err(EffectsError::UnsupportedUniform { + shader: shader.to_string(), + uniform: uniform.clone(), + }); + } + + Ok(EffectUniformBuffer { + resolution: [width as f32, height as f32], + direction, + scalars: [sigma, step, 0.0, 0.0], + }) +} + +fn read_number_uniform(pass: &EffectPass, uniform: &str) -> Result { + let Some(value) = pass.uniforms.get(uniform) else { + return Err(EffectsError::MissingUniform { + shader: pass.shader.clone(), + uniform: uniform.to_string(), + }); + }; + match value { + UniformValue::Number(value) => Ok(*value), + UniformValue::Vector(_) => Err(EffectsError::InvalidNumberUniform { + shader: pass.shader.clone(), + uniform: uniform.to_string(), + }), + } +} + +fn read_vec2_uniform(pass: &EffectPass, uniform: &str) -> Result<[f32; 2], EffectsError> { + let Some(value) = pass.uniforms.get(uniform) else { + return Err(EffectsError::MissingUniform { + shader: pass.shader.clone(), + uniform: uniform.to_string(), + }); + }; + let UniformValue::Vector(values) = value else { + return Err(EffectsError::InvalidVectorUniform { + shader: pass.shader.clone(), + uniform: uniform.to_string(), + expected_length: 2, + }); + }; + if values.len() != 2 { + return Err(EffectsError::InvalidVectorUniform { + shader: pass.shader.clone(), + uniform: uniform.to_string(), + expected_length: 2, + }); + } + Ok([values[0], values[1]]) +} diff --git a/rust/crates/gpu/src/shaders/gaussian_blur.wgsl b/rust/crates/effects/src/shaders/gaussian_blur.wgsl similarity index 100% rename from rust/crates/gpu/src/shaders/gaussian_blur.wgsl rename to rust/crates/effects/src/shaders/gaussian_blur.wgsl diff --git a/rust/crates/effects/src/types.rs b/rust/crates/effects/src/types.rs new file mode 100644 index 00000000..bd013195 --- /dev/null +++ b/rust/crates/effects/src/types.rs @@ -0,0 +1,13 @@ +use std::collections::HashMap; + +#[derive(Clone, Debug)] +pub struct EffectPass { + pub shader: String, + pub uniforms: HashMap, +} + +#[derive(Clone, Debug)] +pub enum UniformValue { + Number(f32), + Vector(Vec), +} diff --git a/rust/crates/gpu/Cargo.toml b/rust/crates/gpu/Cargo.toml index d68e8a65..4be1e8d6 100644 --- a/rust/crates/gpu/Cargo.toml +++ b/rust/crates/gpu/Cargo.toml @@ -7,3 +7,7 @@ edition = "2024" bytemuck = { version = "1.25.0", features = ["derive"] } thiserror = "2.0.18" wgpu = "29.0.1" + +[features] +default = [] +wasm = [] diff --git a/rust/crates/gpu/src/context.rs b/rust/crates/gpu/src/context.rs index 702f65ba..1526a67d 100644 --- a/rust/crates/gpu/src/context.rs +++ b/rust/crates/gpu/src/context.rs @@ -1,12 +1,8 @@ use wgpu::util::DeviceExt; -use crate::{ - GPU_TEXTURE_FORMAT, GpuError, - effect_pipeline::{ApplyEffectsOptions, apply_effects}, - mask_feather::{ApplyMaskFeatherOptions, MaskFeatherPipeline}, - sdf_pipeline::SdfPipeline, - shader_registry::ShaderRegistry, -}; +use crate::{FULLSCREEN_SHADER_SOURCE, GPU_TEXTURE_FORMAT, GpuError}; + +const BLIT_SHADER_SOURCE: &str = include_str!("shaders/blit.wgsl"); const FULLSCREEN_QUAD_POSITIONS: [[f32; 2]; 6] = [ [-1.0, -1.0], @@ -25,9 +21,8 @@ pub struct GpuContext { fullscreen_quad: wgpu::Buffer, linear_sampler: wgpu::Sampler, nearest_sampler: wgpu::Sampler, - shader_registry: ShaderRegistry, - sdf_pipeline: SdfPipeline, - mask_feather_pipeline: MaskFeatherPipeline, + texture_sampler_bind_group_layout: wgpu::BindGroupLayout, + blit_pipeline: wgpu::RenderPipeline, } impl GpuContext { @@ -77,9 +72,74 @@ impl GpuContext { mipmap_filter: wgpu::MipmapFilterMode::Nearest, ..Default::default() }); - let shader_registry = ShaderRegistry::new(&device); - let sdf_pipeline = SdfPipeline::new(&device); - let mask_feather_pipeline = MaskFeatherPipeline::new(&device); + let texture_sampler_bind_group_layout = + device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("gpu-texture-sampler-bind-group-layout"), + entries: &[ + wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Texture { + multisampled: false, + view_dimension: wgpu::TextureViewDimension::D2, + sample_type: wgpu::TextureSampleType::Float { filterable: true }, + }, + count: None, + }, + wgpu::BindGroupLayoutEntry { + binding: 1, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering), + count: None, + }, + ], + }); + let vertex_shader_module = device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("gpu-fullscreen-shader"), + source: wgpu::ShaderSource::Wgsl(FULLSCREEN_SHADER_SOURCE.into()), + }); + let blit_shader_module = device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("gpu-blit-shader"), + source: wgpu::ShaderSource::Wgsl(BLIT_SHADER_SOURCE.into()), + }); + let blit_pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("gpu-blit-pipeline-layout"), + bind_group_layouts: &[Some(&texture_sampler_bind_group_layout)], + immediate_size: 0, + }); + let blit_pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some("gpu-blit-pipeline"), + layout: Some(&blit_pipeline_layout), + vertex: wgpu::VertexState { + module: &vertex_shader_module, + entry_point: Some("vertex_main"), + buffers: &[wgpu::VertexBufferLayout { + array_stride: std::mem::size_of::<[f32; 2]>() as u64, + step_mode: wgpu::VertexStepMode::Vertex, + attributes: &[wgpu::VertexAttribute { + format: wgpu::VertexFormat::Float32x2, + offset: 0, + shader_location: 0, + }], + }], + compilation_options: wgpu::PipelineCompilationOptions::default(), + }, + fragment: Some(wgpu::FragmentState { + module: &blit_shader_module, + entry_point: Some("fragment_main"), + targets: &[Some(wgpu::ColorTargetState { + format: GPU_TEXTURE_FORMAT, + blend: None, + write_mask: wgpu::ColorWrites::ALL, + })], + compilation_options: wgpu::PipelineCompilationOptions::default(), + }), + primitive: wgpu::PrimitiveState::default(), + depth_stencil: None, + multisample: wgpu::MultisampleState::default(), + multiview_mask: None, + cache: None, + }); Ok(Self { instance, @@ -89,26 +149,11 @@ impl GpuContext { fullscreen_quad, linear_sampler, nearest_sampler, - shader_registry, - sdf_pipeline, - mask_feather_pipeline, + texture_sampler_bind_group_layout, + blit_pipeline, }) } - pub fn apply_effects( - &self, - options: ApplyEffectsOptions<'_>, - ) -> Result { - apply_effects(self, options) - } - - pub fn apply_mask_feather( - &self, - options: ApplyMaskFeatherOptions<'_>, - ) -> wgpu::Texture { - self.mask_feather_pipeline.apply_mask_feather(self, options) - } - pub fn create_render_texture( &self, width: u32, @@ -162,12 +207,8 @@ impl GpuContext { &self.nearest_sampler } - pub fn shader_registry(&self) -> &ShaderRegistry { - &self.shader_registry - } - - pub fn sdf_pipeline(&self) -> &SdfPipeline { - &self.sdf_pipeline + pub fn texture_sampler_bind_group_layout(&self) -> &wgpu::BindGroupLayout { + &self.texture_sampler_bind_group_layout } pub fn render_texture_to_surface( @@ -202,7 +243,7 @@ impl GpuContext { .create_view(&wgpu::TextureViewDescriptor::default()); let bind_group = self.device.create_bind_group(&wgpu::BindGroupDescriptor { label: Some("gpu-blit-bind-group"), - layout: self.shader_registry.effect_texture_bind_group_layout(), + layout: &self.texture_sampler_bind_group_layout, entries: &[ wgpu::BindGroupEntry { binding: 0, @@ -237,7 +278,7 @@ impl GpuContext { timestamp_writes: None, multiview_mask: None, }); - render_pass.set_pipeline(self.shader_registry.blit_pipeline()); + render_pass.set_pipeline(&self.blit_pipeline); render_pass.set_vertex_buffer(0, self.fullscreen_quad.slice(..)); render_pass.set_bind_group(0, &bind_group, &[]); render_pass.draw(0..6, 0..1); @@ -247,4 +288,50 @@ impl GpuContext { surface_texture.present(); Ok(()) } + + #[cfg(all(feature = "wasm", target_arch = "wasm32"))] + pub fn import_offscreen_canvas_texture( + &self, + canvas: &wgpu::web_sys::OffscreenCanvas, + width: u32, + height: u32, + label: &'static str, + ) -> wgpu::Texture { + let texture = self.create_render_texture(width, height, label); + self.queue.copy_external_image_to_texture( + &wgpu::CopyExternalImageSourceInfo { + source: wgpu::ExternalImageSource::OffscreenCanvas(canvas.clone()), + origin: wgpu::Origin2d::ZERO, + flip_y: true, + }, + wgpu::CopyExternalImageDestInfo { + texture: &texture, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + color_space: wgpu::PredefinedColorSpace::Srgb, + premultiplied_alpha: false, + }, + wgpu::Extent3d { + width, + height, + depth_or_array_layers: 1, + }, + ); + texture + } + + #[cfg(all(feature = "wasm", target_arch = "wasm32"))] + pub fn render_texture_to_offscreen_canvas( + &self, + texture: &wgpu::Texture, + canvas: &wgpu::web_sys::OffscreenCanvas, + width: u32, + height: u32, + ) -> Result<(), GpuError> { + let surface = self + .instance + .create_surface(wgpu::SurfaceTarget::OffscreenCanvas(canvas.clone()))?; + self.render_texture_to_surface(texture, &surface, width, height) + } } diff --git a/rust/crates/gpu/src/effect_pipeline.rs b/rust/crates/gpu/src/effect_pipeline.rs deleted file mode 100644 index c600551a..00000000 --- a/rust/crates/gpu/src/effect_pipeline.rs +++ /dev/null @@ -1,194 +0,0 @@ -use std::collections::HashMap; - -use bytemuck::{Pod, Zeroable}; -use wgpu::util::DeviceExt; - -use crate::{GpuError, context::GpuContext}; - -pub struct ApplyEffectsOptions<'a> { - pub source: &'a wgpu::Texture, - pub width: u32, - pub height: u32, - pub passes: &'a [EffectPass], -} - -#[derive(Clone, Debug)] -pub struct EffectPass { - pub shader: String, - pub uniforms: HashMap, -} - -#[derive(Clone, Debug)] -pub enum UniformValue { - Number(f32), - Vector(Vec), -} - -#[repr(C)] -#[derive(Clone, Copy, Pod, Zeroable)] -struct EffectUniformBuffer { - resolution: [f32; 2], - direction: [f32; 2], - scalars: [f32; 4], -} - -pub fn apply_effects( - context: &GpuContext, - ApplyEffectsOptions { - source, - width, - height, - passes, - }: ApplyEffectsOptions<'_>, -) -> Result { - let mut current_texture: Option = None; - - for pass in passes { - let input_texture = current_texture.as_ref().unwrap_or(source); - let output_texture = - context.create_render_texture(width, height, "gpu-effect-pass-output"); - let input_view = input_texture.create_view(&wgpu::TextureViewDescriptor::default()); - let output_view = output_texture.create_view(&wgpu::TextureViewDescriptor::default()); - let texture_bind_group = context - .device() - .create_bind_group(&wgpu::BindGroupDescriptor { - label: Some("gpu-effect-texture-bind-group"), - layout: context - .shader_registry() - .effect_texture_bind_group_layout(), - entries: &[ - wgpu::BindGroupEntry { - binding: 0, - resource: wgpu::BindingResource::TextureView(&input_view), - }, - wgpu::BindGroupEntry { - binding: 1, - resource: wgpu::BindingResource::Sampler(context.linear_sampler()), - }, - ], - }); - let uniform_buffer = context - .device() - .create_buffer_init(&wgpu::util::BufferInitDescriptor { - label: Some("gpu-effect-uniform-buffer"), - contents: bytemuck::bytes_of(&pack_effect_uniforms(pass, width, height)?), - usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, - }); - let uniform_bind_group = context - .device() - .create_bind_group(&wgpu::BindGroupDescriptor { - label: Some("gpu-effect-uniform-bind-group"), - layout: context - .shader_registry() - .effect_uniform_bind_group_layout(), - entries: &[wgpu::BindGroupEntry { - binding: 0, - resource: uniform_buffer.as_entire_binding(), - }], - }); - let pipeline = context.shader_registry().get_effect_pipeline(&pass.shader)?; - let mut encoder = context - .device() - .create_command_encoder(&wgpu::CommandEncoderDescriptor { - label: Some("gpu-effect-command-encoder"), - }); - - { - let mut render_pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { - label: Some("gpu-effect-render-pass"), - color_attachments: &[Some(wgpu::RenderPassColorAttachment { - view: &output_view, - resolve_target: None, - depth_slice: None, - ops: wgpu::Operations { - load: wgpu::LoadOp::Clear(wgpu::Color::TRANSPARENT), - store: wgpu::StoreOp::Store, - }, - })], - depth_stencil_attachment: None, - occlusion_query_set: None, - timestamp_writes: None, - multiview_mask: None, - }); - render_pass.set_pipeline(pipeline); - render_pass.set_vertex_buffer(0, context.fullscreen_quad().slice(..)); - render_pass.set_bind_group(0, &texture_bind_group, &[]); - render_pass.set_bind_group(1, &uniform_bind_group, &[]); - render_pass.draw(0..6, 0..1); - } - - context.queue().submit([encoder.finish()]); - current_texture = Some(output_texture); - } - - current_texture.ok_or_else(|| GpuError::UnknownEffectShader { - shader: "missing-effect-pass".to_string(), - }) -} - -fn pack_effect_uniforms( - pass: &EffectPass, - width: u32, - height: u32, -) -> Result { - let shader = pass.shader.as_str(); - let sigma = read_number_uniform(pass, "u_sigma")?; - let step = read_number_uniform(pass, "u_step")?; - let direction = read_vec2_uniform(pass, "u_direction")?; - - for uniform in pass.uniforms.keys() { - if uniform == "u_sigma" || uniform == "u_step" || uniform == "u_direction" { - continue; - } - return Err(GpuError::UnsupportedUniform { - shader: shader.to_string(), - uniform: uniform.clone(), - }); - } - - Ok(EffectUniformBuffer { - resolution: [width as f32, height as f32], - direction, - scalars: [sigma, step, 0.0, 0.0], - }) -} - -fn read_number_uniform(pass: &EffectPass, uniform: &str) -> Result { - let Some(value) = pass.uniforms.get(uniform) else { - return Err(GpuError::MissingUniform { - shader: pass.shader.clone(), - uniform: uniform.to_string(), - }); - }; - match value { - UniformValue::Number(value) => Ok(*value), - UniformValue::Vector(_) => Err(GpuError::InvalidNumberUniform { - shader: pass.shader.clone(), - uniform: uniform.to_string(), - }), - } -} - -fn read_vec2_uniform(pass: &EffectPass, uniform: &str) -> Result<[f32; 2], GpuError> { - let Some(value) = pass.uniforms.get(uniform) else { - return Err(GpuError::MissingUniform { - shader: pass.shader.clone(), - uniform: uniform.to_string(), - }); - }; - let UniformValue::Vector(values) = value else { - return Err(GpuError::InvalidVectorUniform { - shader: pass.shader.clone(), - uniform: uniform.to_string(), - expected_length: 2, - }); - }; - if values.len() != 2 { - return Err(GpuError::InvalidVectorUniform { - shader: pass.shader.clone(), - uniform: uniform.to_string(), - expected_length: 2, - }); - } - Ok([values[0], values[1]]) -} diff --git a/rust/crates/gpu/src/lib.rs b/rust/crates/gpu/src/lib.rs index c53e63ab..b6ca6c1c 100644 --- a/rust/crates/gpu/src/lib.rs +++ b/rust/crates/gpu/src/lib.rs @@ -1,17 +1,12 @@ mod context; -mod effect_pipeline; -mod mask_feather; -mod sdf_pipeline; -mod shader_registry; use thiserror::Error; -pub use wgpu; pub use context::GpuContext; -pub use effect_pipeline::{ApplyEffectsOptions, EffectPass, UniformValue}; -pub use mask_feather::ApplyMaskFeatherOptions; +pub use wgpu; pub const GPU_TEXTURE_FORMAT: wgpu::TextureFormat = wgpu::TextureFormat::Bgra8Unorm; +pub const FULLSCREEN_SHADER_SOURCE: &str = include_str!("shaders/fullscreen.wgsl"); #[derive(Debug, Error)] pub enum GpuError { @@ -23,18 +18,4 @@ pub enum GpuError { CreateSurface(#[from] wgpu::CreateSurfaceError), #[error("The output surface does not support the required texture format")] UnsupportedSurfaceFormat, - #[error("Unknown effect shader '{shader}'")] - UnknownEffectShader { shader: String }, - #[error("Missing uniform '{uniform}' for shader '{shader}'")] - MissingUniform { shader: String, uniform: String }, - #[error("Uniform '{uniform}' for shader '{shader}' must be a number")] - InvalidNumberUniform { shader: String, uniform: String }, - #[error("Uniform '{uniform}' for shader '{shader}' must be a vector of length {expected_length}")] - InvalidVectorUniform { - shader: String, - uniform: String, - expected_length: usize, - }, - #[error("Shader '{shader}' does not support uniform '{uniform}'")] - UnsupportedUniform { shader: String, uniform: String }, } diff --git a/rust/crates/gpu/src/shader_registry.rs b/rust/crates/gpu/src/shader_registry.rs deleted file mode 100644 index b6af173e..00000000 --- a/rust/crates/gpu/src/shader_registry.rs +++ /dev/null @@ -1,183 +0,0 @@ -use std::collections::HashMap; - -use crate::{GPU_TEXTURE_FORMAT, GpuError}; - -const FULLSCREEN_SHADER_SOURCE: &str = include_str!("shaders/fullscreen.wgsl"); -const GAUSSIAN_BLUR_SHADER_SOURCE: &str = include_str!("shaders/gaussian_blur.wgsl"); -const BLIT_SHADER_SOURCE: &str = include_str!("shaders/blit.wgsl"); -pub const GAUSSIAN_BLUR_SHADER_ID: &str = "gaussian-blur"; - -pub struct ShaderRegistry { - effect_texture_bind_group_layout: wgpu::BindGroupLayout, - effect_uniform_bind_group_layout: wgpu::BindGroupLayout, - effect_pipelines: HashMap, - blit_pipeline: wgpu::RenderPipeline, -} - -impl ShaderRegistry { - pub fn new(device: &wgpu::Device) -> Self { - let effect_texture_bind_group_layout = - device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { - label: Some("gpu-effect-texture-bind-group-layout"), - entries: &[ - wgpu::BindGroupLayoutEntry { - binding: 0, - visibility: wgpu::ShaderStages::FRAGMENT, - ty: wgpu::BindingType::Texture { - multisampled: false, - view_dimension: wgpu::TextureViewDimension::D2, - sample_type: wgpu::TextureSampleType::Float { filterable: true }, - }, - count: None, - }, - wgpu::BindGroupLayoutEntry { - binding: 1, - visibility: wgpu::ShaderStages::FRAGMENT, - ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering), - count: None, - }, - ], - }); - let effect_uniform_bind_group_layout = - device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { - label: Some("gpu-effect-uniform-bind-group-layout"), - entries: &[wgpu::BindGroupLayoutEntry { - binding: 0, - visibility: wgpu::ShaderStages::FRAGMENT, - ty: wgpu::BindingType::Buffer { - ty: wgpu::BufferBindingType::Uniform, - has_dynamic_offset: false, - min_binding_size: None, - }, - count: None, - }], - }); - - let vertex_shader_module = device.create_shader_module(wgpu::ShaderModuleDescriptor { - label: Some("gpu-fullscreen-shader"), - source: wgpu::ShaderSource::Wgsl(FULLSCREEN_SHADER_SOURCE.into()), - }); - let gaussian_blur_shader_module = - device.create_shader_module(wgpu::ShaderModuleDescriptor { - label: Some("gpu-gaussian-blur-shader"), - source: wgpu::ShaderSource::Wgsl(GAUSSIAN_BLUR_SHADER_SOURCE.into()), - }); - let blit_shader_module = device.create_shader_module(wgpu::ShaderModuleDescriptor { - label: Some("gpu-blit-shader"), - source: wgpu::ShaderSource::Wgsl(BLIT_SHADER_SOURCE.into()), - }); - let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { - label: Some("gpu-effect-pipeline-layout"), - bind_group_layouts: &[ - Some(&effect_texture_bind_group_layout), - Some(&effect_uniform_bind_group_layout), - ], - immediate_size: 0, - }); - let blit_pipeline_layout = - device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { - label: Some("gpu-blit-pipeline-layout"), - bind_group_layouts: &[Some(&effect_texture_bind_group_layout)], - immediate_size: 0, - }); - let gaussian_blur_pipeline = - device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { - label: Some("gpu-gaussian-blur-pipeline"), - layout: Some(&pipeline_layout), - vertex: wgpu::VertexState { - module: &vertex_shader_module, - entry_point: Some("vertex_main"), - buffers: &[wgpu::VertexBufferLayout { - array_stride: std::mem::size_of::<[f32; 2]>() as u64, - step_mode: wgpu::VertexStepMode::Vertex, - attributes: &[wgpu::VertexAttribute { - format: wgpu::VertexFormat::Float32x2, - offset: 0, - shader_location: 0, - }], - }], - compilation_options: wgpu::PipelineCompilationOptions::default(), - }, - fragment: Some(wgpu::FragmentState { - module: &gaussian_blur_shader_module, - entry_point: Some("fragment_main"), - targets: &[Some(wgpu::ColorTargetState { - format: GPU_TEXTURE_FORMAT, - blend: None, - write_mask: wgpu::ColorWrites::ALL, - })], - compilation_options: wgpu::PipelineCompilationOptions::default(), - }), - primitive: wgpu::PrimitiveState::default(), - depth_stencil: None, - multisample: wgpu::MultisampleState::default(), - multiview_mask: None, - cache: None, - }); - let blit_pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { - label: Some("gpu-blit-pipeline"), - layout: Some(&blit_pipeline_layout), - vertex: wgpu::VertexState { - module: &vertex_shader_module, - entry_point: Some("vertex_main"), - buffers: &[wgpu::VertexBufferLayout { - array_stride: std::mem::size_of::<[f32; 2]>() as u64, - step_mode: wgpu::VertexStepMode::Vertex, - attributes: &[wgpu::VertexAttribute { - format: wgpu::VertexFormat::Float32x2, - offset: 0, - shader_location: 0, - }], - }], - compilation_options: wgpu::PipelineCompilationOptions::default(), - }, - fragment: Some(wgpu::FragmentState { - module: &blit_shader_module, - entry_point: Some("fragment_main"), - targets: &[Some(wgpu::ColorTargetState { - format: GPU_TEXTURE_FORMAT, - blend: None, - write_mask: wgpu::ColorWrites::ALL, - })], - compilation_options: wgpu::PipelineCompilationOptions::default(), - }), - primitive: wgpu::PrimitiveState::default(), - depth_stencil: None, - multisample: wgpu::MultisampleState::default(), - multiview_mask: None, - cache: None, - }); - - let effect_pipelines = HashMap::from([( - GAUSSIAN_BLUR_SHADER_ID.to_string(), - gaussian_blur_pipeline, - )]); - - Self { - effect_texture_bind_group_layout, - effect_uniform_bind_group_layout, - effect_pipelines, - blit_pipeline, - } - } - - pub fn get_effect_pipeline(&self, shader: &str) -> Result<&wgpu::RenderPipeline, GpuError> { - self.effect_pipelines - .get(shader) - .ok_or_else(|| GpuError::UnknownEffectShader { - shader: shader.to_string(), - }) - } - - pub fn effect_texture_bind_group_layout(&self) -> &wgpu::BindGroupLayout { - &self.effect_texture_bind_group_layout - } - - pub fn effect_uniform_bind_group_layout(&self) -> &wgpu::BindGroupLayout { - &self.effect_uniform_bind_group_layout - } - - pub fn blit_pipeline(&self) -> &wgpu::RenderPipeline { - &self.blit_pipeline - } -} diff --git a/rust/crates/masks/Cargo.toml b/rust/crates/masks/Cargo.toml new file mode 100644 index 00000000..58e10712 --- /dev/null +++ b/rust/crates/masks/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "masks" +version = "0.1.0" +edition = "2024" + +[lib] +path = "src/masks.rs" +crate-type = ["rlib"] + +[dependencies] +bytemuck = { version = "1.25.0", features = ["derive"] } +gpu = { version = "0.1.0", path = "../gpu" } +wgpu = "29.0.1" diff --git a/rust/crates/gpu/src/mask_feather.rs b/rust/crates/masks/src/feather.rs similarity index 88% rename from rust/crates/gpu/src/mask_feather.rs rename to rust/crates/masks/src/feather.rs index 029b59ed..e58572f4 100644 --- a/rust/crates/gpu/src/mask_feather.rs +++ b/rust/crates/masks/src/feather.rs @@ -1,9 +1,9 @@ use bytemuck::{Pod, Zeroable}; +use gpu::{FULLSCREEN_SHADER_SOURCE, GPU_TEXTURE_FORMAT, GpuContext}; use wgpu::util::DeviceExt; -use crate::{GPU_TEXTURE_FORMAT, context::GpuContext}; +use crate::SdfPipeline; -const FULLSCREEN_SHADER_SOURCE: &str = include_str!("shaders/fullscreen.wgsl"); const JFA_DISTANCE_SHADER_SOURCE: &str = include_str!("shaders/jfa_distance.wgsl"); pub struct ApplyMaskFeatherOptions<'a> { @@ -14,6 +14,7 @@ pub struct ApplyMaskFeatherOptions<'a> { } pub struct MaskFeatherPipeline { + sdf_pipeline: SdfPipeline, inside_texture_bind_group_layout: wgpu::BindGroupLayout, outside_texture_bind_group_layout: wgpu::BindGroupLayout, uniform_bind_group_layout: wgpu::BindGroupLayout, @@ -29,7 +30,8 @@ struct DistanceUniformBuffer { } impl MaskFeatherPipeline { - pub fn new(device: &wgpu::Device) -> Self { + pub fn new(context: &GpuContext) -> Self { + let device = context.device(); let inside_texture_bind_group_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { label: Some("gpu-mask-distance-inside-layout"), @@ -140,6 +142,7 @@ impl MaskFeatherPipeline { }); Self { + sdf_pipeline: SdfPipeline::new(context), inside_texture_bind_group_layout, outside_texture_bind_group_layout, uniform_bind_group_layout, @@ -157,11 +160,10 @@ impl MaskFeatherPipeline { feather, }: ApplyMaskFeatherOptions<'_>, ) -> wgpu::Texture { - let sdf = context - .sdf_pipeline() + let sdf = self + .sdf_pipeline .compute_signed_distance_field(context, mask, width, height); - let output_texture = - context.create_render_texture(width, height, "gpu-mask-feather-output"); + let output_texture = context.create_render_texture(width, height, "masks-feather-output"); let inside_view = sdf .inside_texture .create_view(&wgpu::TextureViewDescriptor::default()); @@ -201,17 +203,18 @@ impl MaskFeatherPipeline { }, ], }); - let uniform_buffer = context - .device() - .create_buffer_init(&wgpu::util::BufferInitDescriptor { - label: Some("gpu-mask-distance-uniform-buffer"), - contents: bytemuck::bytes_of(&DistanceUniformBuffer { - resolution: [width as f32, height as f32], - feather_half: feather / 2.0, - _padding: 0.0, - }), - usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, - }); + let uniform_buffer = + context + .device() + .create_buffer_init(&wgpu::util::BufferInitDescriptor { + label: Some("gpu-mask-distance-uniform-buffer"), + contents: bytemuck::bytes_of(&DistanceUniformBuffer { + resolution: [width as f32, height as f32], + feather_half: feather / 2.0, + _padding: 0.0, + }), + usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, + }); let uniform_bind_group = context .device() .create_bind_group(&wgpu::BindGroupDescriptor { @@ -222,11 +225,12 @@ impl MaskFeatherPipeline { resource: uniform_buffer.as_entire_binding(), }], }); - let mut encoder = context - .device() - .create_command_encoder(&wgpu::CommandEncoderDescriptor { - label: Some("gpu-mask-distance-command-encoder"), - }); + let mut encoder = + context + .device() + .create_command_encoder(&wgpu::CommandEncoderDescriptor { + label: Some("gpu-mask-distance-command-encoder"), + }); { let mut render_pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { diff --git a/rust/crates/masks/src/masks.rs b/rust/crates/masks/src/masks.rs new file mode 100644 index 00000000..2bf33342 --- /dev/null +++ b/rust/crates/masks/src/masks.rs @@ -0,0 +1,5 @@ +mod feather; +mod sdf; + +pub use feather::{ApplyMaskFeatherOptions, MaskFeatherPipeline}; +pub use sdf::{SdfPipeline, SignedDistanceFieldTextures}; diff --git a/rust/crates/gpu/src/sdf_pipeline.rs b/rust/crates/masks/src/sdf.rs similarity index 93% rename from rust/crates/gpu/src/sdf_pipeline.rs rename to rust/crates/masks/src/sdf.rs index d2612fe1..0e7ec01a 100644 --- a/rust/crates/gpu/src/sdf_pipeline.rs +++ b/rust/crates/masks/src/sdf.rs @@ -1,9 +1,7 @@ use bytemuck::{Pod, Zeroable}; +use gpu::{FULLSCREEN_SHADER_SOURCE, GPU_TEXTURE_FORMAT, GpuContext}; use wgpu::util::DeviceExt; -use crate::{GPU_TEXTURE_FORMAT, context::GpuContext}; - -const FULLSCREEN_SHADER_SOURCE: &str = include_str!("shaders/fullscreen.wgsl"); const JFA_INIT_SHADER_SOURCE: &str = include_str!("shaders/jfa_init.wgsl"); const JFA_STEP_SHADER_SOURCE: &str = include_str!("shaders/jfa_step.wgsl"); @@ -36,7 +34,8 @@ struct JfaStepUniformBuffer { } impl SdfPipeline { - pub fn new(device: &wgpu::Device) -> Self { + pub fn new(context: &GpuContext) -> Self { + let device = context.device(); let texture_bind_group_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { label: Some("gpu-sdf-texture-bind-group-layout"), @@ -213,13 +212,14 @@ impl SdfPipeline { }, ], }); - let uniform_buffer = context - .device() - .create_buffer_init(&wgpu::util::BufferInitDescriptor { - label: Some("gpu-sdf-uniform-buffer"), - contents: uniform_buffer_bytes, - usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, - }); + let uniform_buffer = + context + .device() + .create_buffer_init(&wgpu::util::BufferInitDescriptor { + label: Some("gpu-sdf-uniform-buffer"), + contents: uniform_buffer_bytes, + usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, + }); let uniform_bind_group = context .device() .create_bind_group(&wgpu::BindGroupDescriptor { @@ -230,11 +230,12 @@ impl SdfPipeline { resource: uniform_buffer.as_entire_binding(), }], }); - let mut encoder = context - .device() - .create_command_encoder(&wgpu::CommandEncoderDescriptor { - label: Some("gpu-sdf-command-encoder"), - }); + let mut encoder = + context + .device() + .create_command_encoder(&wgpu::CommandEncoderDescriptor { + label: Some("gpu-sdf-command-encoder"), + }); { let mut render_pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { diff --git a/rust/crates/gpu/src/shaders/jfa_distance.wgsl b/rust/crates/masks/src/shaders/jfa_distance.wgsl similarity index 100% rename from rust/crates/gpu/src/shaders/jfa_distance.wgsl rename to rust/crates/masks/src/shaders/jfa_distance.wgsl diff --git a/rust/crates/gpu/src/shaders/jfa_init.wgsl b/rust/crates/masks/src/shaders/jfa_init.wgsl similarity index 100% rename from rust/crates/gpu/src/shaders/jfa_init.wgsl rename to rust/crates/masks/src/shaders/jfa_init.wgsl diff --git a/rust/crates/gpu/src/shaders/jfa_step.wgsl b/rust/crates/masks/src/shaders/jfa_step.wgsl similarity index 100% rename from rust/crates/gpu/src/shaders/jfa_step.wgsl rename to rust/crates/masks/src/shaders/jfa_step.wgsl diff --git a/rust/crates/time/Cargo.toml b/rust/crates/time/Cargo.toml index bf66e4e2..070695dc 100644 --- a/rust/crates/time/Cargo.toml +++ b/rust/crates/time/Cargo.toml @@ -5,13 +5,14 @@ edition = "2024" [lib] path = "src/time.rs" -crate-type = ["rlib", "cdylib"] +crate-type = ["rlib"] [dependencies] bridge = { version = "0.1.0", path = "../bridge" } +num-traits = "0.2.19" serde = { version = "1", features = ["derive"] } tsify-next = { version = "0.5", optional = true } wasm-bindgen = { version = "0.2.115", optional = true } [features] -wasm = ["dep:wasm-bindgen", "dep:tsify-next", "tsify-next/js"] \ No newline at end of file +wasm = ["dep:wasm-bindgen", "dep:tsify-next", "tsify-next/js"] diff --git a/rust/crates/time/src/frame_rate.rs b/rust/crates/time/src/frame_rate.rs new file mode 100644 index 00000000..7af10649 --- /dev/null +++ b/rust/crates/time/src/frame_rate.rs @@ -0,0 +1,121 @@ +use serde::{Deserialize, Serialize}; + +use crate::media_time::TICKS_PER_SECOND; + +#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] +#[cfg_attr(feature = "wasm", tsify(from_wasm_abi, into_wasm_abi))] +#[derive(Serialize, Deserialize, Clone, Copy, Debug, Eq, PartialEq)] +pub struct FrameRate { + pub numerator: u32, + pub denominator: u32, +} + +impl FrameRate { + pub const FPS_23_976: Self = Self { + numerator: 24_000, + denominator: 1_001, + }; + pub const FPS_24: Self = Self { + numerator: 24, + denominator: 1, + }; + pub const FPS_25: Self = Self { + numerator: 25, + denominator: 1, + }; + pub const FPS_29_97: Self = Self { + numerator: 30_000, + denominator: 1_001, + }; + pub const FPS_30: Self = Self { + numerator: 30, + denominator: 1, + }; + pub const FPS_48: Self = Self { + numerator: 48, + denominator: 1, + }; + pub const FPS_50: Self = Self { + numerator: 50, + denominator: 1, + }; + pub const FPS_59_94: Self = Self { + numerator: 60_000, + denominator: 1_001, + }; + pub const FPS_60: Self = Self { + numerator: 60, + denominator: 1, + }; + pub const FPS_120: Self = Self { + numerator: 120, + denominator: 1, + }; + + pub const fn new(numerator: u32, denominator: u32) -> Self { + Self { + numerator, + denominator, + } + } + + pub const fn is_valid(self) -> bool { + self.numerator > 0 && self.denominator > 0 + } + + pub fn as_f64(self) -> Option { + if !self.is_valid() { + return None; + } + + Some(f64::from(self.numerator) / f64::from(self.denominator)) + } + + pub fn frame_number_upper_bound(self) -> Option { + if !self.is_valid() { + return None; + } + + Some(self.numerator.div_ceil(self.denominator)) + } + + pub fn ticks_per_frame(self) -> Option { + if !self.is_valid() { + return None; + } + + let tick_numerator = TICKS_PER_SECOND.checked_mul(i64::from(self.denominator))?; + let tick_denominator = i64::from(self.numerator); + if tick_numerator % tick_denominator != 0 { + return None; + } + + Some(tick_numerator / tick_denominator) + } +} + +#[cfg(test)] +mod tests { + use super::FrameRate; + + #[test] + fn resolves_ticks_per_standard_frame_rate() { + assert_eq!(FrameRate::FPS_23_976.ticks_per_frame(), Some(5_005)); + assert_eq!(FrameRate::FPS_24.ticks_per_frame(), Some(5_000)); + assert_eq!(FrameRate::FPS_25.ticks_per_frame(), Some(4_800)); + assert_eq!(FrameRate::FPS_29_97.ticks_per_frame(), Some(4_004)); + assert_eq!(FrameRate::FPS_30.ticks_per_frame(), Some(4_000)); + assert_eq!(FrameRate::FPS_48.ticks_per_frame(), Some(2_500)); + assert_eq!(FrameRate::FPS_50.ticks_per_frame(), Some(2_400)); + assert_eq!(FrameRate::FPS_59_94.ticks_per_frame(), Some(2_002)); + assert_eq!(FrameRate::FPS_60.ticks_per_frame(), Some(2_000)); + assert_eq!(FrameRate::FPS_120.ticks_per_frame(), Some(1_000)); + } + + #[test] + fn rejects_invalid_or_unsupported_rates() { + assert_eq!(FrameRate::new(0, 1).ticks_per_frame(), None); + assert_eq!(FrameRate::new(1, 0).ticks_per_frame(), None); + assert_eq!(FrameRate::new(7, 3).ticks_per_frame(), None); + } +} diff --git a/rust/crates/time/src/media_time.rs b/rust/crates/time/src/media_time.rs new file mode 100644 index 00000000..11d268e5 --- /dev/null +++ b/rust/crates/time/src/media_time.rs @@ -0,0 +1,428 @@ +use std::ops::{Add, Div, Mul, Neg, Sub}; + +use bridge::export; +use num_traits::ToPrimitive; +use serde::{Deserialize, Serialize}; + +use crate::frame_rate::FrameRate; + +#[export] +pub const TICKS_PER_SECOND: i64 = 120_000; +const TICKS_PER_SECOND_F64: f64 = TICKS_PER_SECOND as f64; + +#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] +#[cfg_attr(feature = "wasm", tsify(from_wasm_abi, into_wasm_abi))] +#[derive(Serialize, Deserialize, Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub struct MediaTime(i64); + +impl MediaTime { + pub const ZERO: Self = Self(0); + pub const ONE_TICK: Self = Self(1); + + pub const fn from_ticks(ticks: i64) -> Self { + Self(ticks) + } + + pub const fn as_ticks(self) -> i64 { + self.0 + } + + pub fn from_seconds_f64(seconds: f64) -> Option { + if !seconds.is_finite() { + return None; + } + + let ticks = (seconds * TICKS_PER_SECOND_F64).round().to_i64()?; + Some(Self(ticks)) + } + + pub fn to_seconds_f64(self) -> f64 { + self.0.to_f64().unwrap_or(0.0) / TICKS_PER_SECOND_F64 + } + + pub fn from_frame(frame: i64, rate: FrameRate) -> Option { + let ticks_per_frame = rate.ticks_per_frame()?; + Some(Self(frame.checked_mul(ticks_per_frame)?)) + } + + pub fn to_frame_round(self, rate: FrameRate) -> Option { + let ticks_per_frame = rate.ticks_per_frame()?; + let remainder = self.0.rem_euclid(ticks_per_frame); + let floor = self.0.div_euclid(ticks_per_frame); + if remainder * 2 >= ticks_per_frame { + Some(floor + 1) + } else { + Some(floor) + } + } + + pub fn to_frame_floor(self, rate: FrameRate) -> Option { + let ticks_per_frame = rate.ticks_per_frame()?; + Some(self.0.div_euclid(ticks_per_frame)) + } + + pub fn round_to_frame(self, rate: FrameRate) -> Option { + Self::from_frame(self.to_frame_round(rate)?, rate) + } + + pub fn floor_to_frame(self, rate: FrameRate) -> Option { + let ticks_per_frame = rate.ticks_per_frame()?; + Some(Self(self.0.div_euclid(ticks_per_frame) * ticks_per_frame)) + } + + pub fn is_frame_aligned(self, rate: FrameRate) -> Option { + let ticks_per_frame = rate.ticks_per_frame()?; + Some(self.0.rem_euclid(ticks_per_frame) == 0) + } + + pub fn last_frame_time(self, rate: FrameRate) -> Option { + if self <= Self::ZERO { + return Some(Self::ZERO); + } + + let last_inclusive_tick = self.0.checked_sub(1).unwrap_or(0); + Self::from_ticks(last_inclusive_tick).floor_to_frame(rate) + } + + pub fn snapped_seek_time(self, duration: Self, rate: FrameRate) -> Option { + let snapped = self.round_to_frame(rate)?; + Some(snapped.clamp(Self::ZERO, duration)) + } + + pub fn clamp(self, min: Self, max: Self) -> Self { + Self(self.0.clamp(min.0, max.0)) + } + + pub fn min(self, other: Self) -> Self { + Self(self.0.min(other.0)) + } + + pub fn max(self, other: Self) -> Self { + Self(self.0.max(other.0)) + } +} + +impl Add for MediaTime { + type Output = Self; + + fn add(self, rhs: Self) -> Self::Output { + Self(self.0 + rhs.0) + } +} + +impl Sub for MediaTime { + type Output = Self; + + fn sub(self, rhs: Self) -> Self::Output { + Self(self.0 - rhs.0) + } +} + +impl Neg for MediaTime { + type Output = Self; + + fn neg(self) -> Self::Output { + Self(-self.0) + } +} + +impl Mul for MediaTime { + type Output = Self; + + fn mul(self, rhs: i64) -> Self::Output { + Self(self.0 * rhs) + } +} + +impl Div for MediaTime { + type Output = Self; + + fn div(self, rhs: i64) -> Self::Output { + Self(self.0 / rhs) + } +} + +#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] +#[cfg_attr(feature = "wasm", tsify(from_wasm_abi))] +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MediaTimeFromSecondsOptions { + pub seconds: f64, +} + +#[export] +pub fn media_time_from_seconds( + MediaTimeFromSecondsOptions { seconds }: MediaTimeFromSecondsOptions, +) -> Option { + MediaTime::from_seconds_f64(seconds) +} + +#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] +#[cfg_attr(feature = "wasm", tsify(from_wasm_abi))] +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MediaTimeToSecondsOptions { + pub time: MediaTime, +} + +#[export] +pub fn media_time_to_seconds(MediaTimeToSecondsOptions { time }: MediaTimeToSecondsOptions) -> f64 { + time.to_seconds_f64() +} + +#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] +#[cfg_attr(feature = "wasm", tsify(from_wasm_abi))] +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MediaTimeFromFrameOptions { + pub frame: i64, + pub rate: FrameRate, +} + +#[export] +pub fn media_time_from_frame( + MediaTimeFromFrameOptions { frame, rate }: MediaTimeFromFrameOptions, +) -> Option { + MediaTime::from_frame(frame, rate) +} + +#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] +#[cfg_attr(feature = "wasm", tsify(from_wasm_abi))] +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MediaTimeToFrameOptions { + pub time: MediaTime, + pub rate: FrameRate, +} + +#[export] +pub fn media_time_to_frame( + MediaTimeToFrameOptions { time, rate }: MediaTimeToFrameOptions, +) -> Option { + time.to_frame_round(rate) +} + +#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] +#[cfg_attr(feature = "wasm", tsify(from_wasm_abi))] +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RoundToFrameOptions { + pub time: MediaTime, + pub rate: FrameRate, +} + +#[export] +pub fn round_to_frame( + RoundToFrameOptions { time, rate }: RoundToFrameOptions, +) -> Option { + time.round_to_frame(rate) +} + +#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] +#[cfg_attr(feature = "wasm", tsify(from_wasm_abi))] +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct FloorToFrameOptions { + pub time: MediaTime, + pub rate: FrameRate, +} + +#[export] +pub fn floor_to_frame( + FloorToFrameOptions { time, rate }: FloorToFrameOptions, +) -> Option { + time.floor_to_frame(rate) +} + +#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] +#[cfg_attr(feature = "wasm", tsify(from_wasm_abi))] +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct IsFrameAlignedOptions { + pub time: MediaTime, + pub rate: FrameRate, +} + +#[export] +pub fn is_frame_aligned( + IsFrameAlignedOptions { time, rate }: IsFrameAlignedOptions, +) -> Option { + time.is_frame_aligned(rate) +} + +#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] +#[cfg_attr(feature = "wasm", tsify(from_wasm_abi))] +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LastFrameTimeOptions { + pub duration: MediaTime, + pub rate: FrameRate, +} + +#[export] +pub fn last_frame_time( + LastFrameTimeOptions { duration, rate }: LastFrameTimeOptions, +) -> Option { + duration.last_frame_time(rate) +} + +#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] +#[cfg_attr(feature = "wasm", tsify(from_wasm_abi))] +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SnappedSeekTimeOptions { + pub time: MediaTime, + pub duration: MediaTime, + pub rate: FrameRate, +} + +#[export] +pub fn snapped_seek_time( + SnappedSeekTimeOptions { + time, + duration, + rate, + }: SnappedSeekTimeOptions, +) -> Option { + time.snapped_seek_time(duration, rate) +} + +#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] +#[cfg_attr(feature = "wasm", tsify(from_wasm_abi))] +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MediaTimeAddOptions { + pub lhs: MediaTime, + pub rhs: MediaTime, +} + +#[export] +pub fn media_time_add(MediaTimeAddOptions { lhs, rhs }: MediaTimeAddOptions) -> MediaTime { + lhs + rhs +} + +#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] +#[cfg_attr(feature = "wasm", tsify(from_wasm_abi))] +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MediaTimeSubOptions { + pub lhs: MediaTime, + pub rhs: MediaTime, +} + +#[export] +pub fn media_time_sub(MediaTimeSubOptions { lhs, rhs }: MediaTimeSubOptions) -> MediaTime { + lhs - rhs +} + +#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] +#[cfg_attr(feature = "wasm", tsify(from_wasm_abi))] +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MediaTimeMinOptions { + pub lhs: MediaTime, + pub rhs: MediaTime, +} + +#[export] +pub fn media_time_min(MediaTimeMinOptions { lhs, rhs }: MediaTimeMinOptions) -> MediaTime { + lhs.min(rhs) +} + +#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] +#[cfg_attr(feature = "wasm", tsify(from_wasm_abi))] +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MediaTimeMaxOptions { + pub lhs: MediaTime, + pub rhs: MediaTime, +} + +#[export] +pub fn media_time_max(MediaTimeMaxOptions { lhs, rhs }: MediaTimeMaxOptions) -> MediaTime { + lhs.max(rhs) +} + +#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] +#[cfg_attr(feature = "wasm", tsify(from_wasm_abi))] +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MediaTimeClampOptions { + pub time: MediaTime, + pub min: MediaTime, + pub max: MediaTime, +} + +#[export] +pub fn media_time_clamp( + MediaTimeClampOptions { time, min, max }: MediaTimeClampOptions, +) -> MediaTime { + time.clamp(min, max) +} + +#[cfg(test)] +mod tests { + use crate::frame_rate::FrameRate; + + use super::{MediaTime, TICKS_PER_SECOND}; + + #[test] + fn converts_between_seconds_and_ticks() { + assert_eq!( + MediaTime::from_seconds_f64(1.5), + Some(MediaTime::from_ticks(180_000)) + ); + assert_eq!(MediaTime::from_ticks(180_000).to_seconds_f64(), 1.5); + assert_eq!(TICKS_PER_SECOND, 120_000); + } + + #[test] + fn rejects_non_finite_seconds() { + assert_eq!(MediaTime::from_seconds_f64(f64::NAN), None); + assert_eq!(MediaTime::from_seconds_f64(f64::INFINITY), None); + assert_eq!(MediaTime::from_seconds_f64(f64::NEG_INFINITY), None); + } + + #[test] + fn snaps_to_the_nearest_frame() { + let rate = FrameRate::FPS_30; + let time = MediaTime::from_seconds_f64(1.26).unwrap(); + + assert_eq!(time.to_frame_round(rate), Some(38)); + assert_eq!( + time.round_to_frame(rate), + Some(MediaTime::from_ticks(152_000)) + ); + } + + #[test] + fn floors_to_frame() { + let rate = FrameRate::FPS_30; + let ticks_per_frame = 4_000; + let time = MediaTime::from_ticks(ticks_per_frame * 5 + 1); + + assert_eq!(time.to_frame_floor(rate), Some(5)); + assert_eq!(time.to_frame_round(rate), Some(5)); + + let almost_next = MediaTime::from_ticks(ticks_per_frame * 5 + ticks_per_frame / 2); + assert_eq!(almost_next.to_frame_floor(rate), Some(5)); + assert_eq!(almost_next.to_frame_round(rate), Some(6)); + } + + #[test] + fn computes_last_frame_time_and_snapped_seek_time() { + let rate = FrameRate::new(5, 1); + let duration = MediaTime::from_seconds_f64(10.0).unwrap(); + + assert_eq!( + duration.last_frame_time(rate), + Some(MediaTime::from_seconds_f64(9.8).unwrap()), + ); + assert_eq!( + MediaTime::from_seconds_f64(10.0) + .unwrap() + .snapped_seek_time(duration, rate), + Some(MediaTime::from_seconds_f64(10.0).unwrap()), + ); + } +} diff --git a/rust/crates/time/src/time.rs b/rust/crates/time/src/time.rs index 84b69327..3d45c47f 100644 --- a/rust/crates/time/src/time.rs +++ b/rust/crates/time/src/time.rs @@ -1,422 +1,19 @@ -use bridge::export; -use serde::{Deserialize, Serialize}; +mod frame_rate; +mod media_time; +mod timecode; -const SECONDS_PER_HOUR: f64 = 3600.0; -const SECONDS_PER_MINUTE: f64 = 60.0; -const CENTISECONDS_PER_SECOND: f64 = 100.0; - -#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] -#[cfg_attr(feature = "wasm", tsify(from_wasm_abi, into_wasm_abi))] -#[derive(Serialize, Deserialize, Clone, Copy, Debug, Eq, PartialEq)] -pub enum TimeCodeFormat { - #[serde(rename = "MM:SS")] - MmSs, - #[serde(rename = "HH:MM:SS")] - HhMmSs, - #[serde(rename = "HH:MM:SS:CS")] - HhMmSsCs, - #[serde(rename = "HH:MM:SS:FF")] - HhMmSsFf, -} - -#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] -#[cfg_attr(feature = "wasm", tsify(from_wasm_abi))] -#[derive(Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct RoundToFrameOptions { - pub time: f64, - pub fps: f64, -} - -#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] -#[cfg_attr(feature = "wasm", tsify(from_wasm_abi))] -#[derive(Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct FormatTimeCodeOptions { - pub time_in_seconds: f64, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub format: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub fps: Option, -} - -#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] -#[cfg_attr(feature = "wasm", tsify(from_wasm_abi))] -#[derive(Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ParseTimeCodeOptions { - pub time_code: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub format: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub fps: Option, -} - -#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] -#[cfg_attr(feature = "wasm", tsify(from_wasm_abi))] -#[derive(Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct GuessTimeCodeFormatOptions { - pub time_code: String, -} - -#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] -#[cfg_attr(feature = "wasm", tsify(from_wasm_abi))] -#[derive(Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct TimeToFrameOptions { - pub time: f64, - pub fps: f64, -} - -#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] -#[cfg_attr(feature = "wasm", tsify(from_wasm_abi))] -#[derive(Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct FrameToTimeOptions { - pub frame: f64, - pub fps: f64, -} - -#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] -#[cfg_attr(feature = "wasm", tsify(from_wasm_abi))] -#[derive(Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct SnapTimeToFrameOptions { - pub time: f64, - pub fps: f64, -} - -#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] -#[cfg_attr(feature = "wasm", tsify(from_wasm_abi))] -#[derive(Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct GetSnappedSeekTimeOptions { - pub raw_time: f64, - pub duration: f64, - pub fps: f64, -} - -#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] -#[cfg_attr(feature = "wasm", tsify(from_wasm_abi))] -#[derive(Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct GetLastFrameTimeOptions { - pub duration: f64, - pub fps: f64, -} - -#[export] -pub fn round_to_frame(RoundToFrameOptions { time, fps }: RoundToFrameOptions) -> f64 { - (time * fps).round() / fps -} - -#[export] -pub fn format_time_code( - FormatTimeCodeOptions { - time_in_seconds, - format, - fps, - }: FormatTimeCodeOptions, -) -> Option { - let format = format.unwrap_or(TimeCodeFormat::HhMmSsCs); - let hours = (time_in_seconds / SECONDS_PER_HOUR).floor() as u64; - let minutes = ((time_in_seconds % SECONDS_PER_HOUR) / SECONDS_PER_MINUTE).floor() as u64; - let seconds = (time_in_seconds % SECONDS_PER_MINUTE).floor() as u64; - let centiseconds = ((time_in_seconds % 1.0) * CENTISECONDS_PER_SECOND).floor() as u64; - - match format { - TimeCodeFormat::MmSs => Some(format!("{minutes:02}:{seconds:02}")), - TimeCodeFormat::HhMmSs => Some(format!("{hours:02}:{minutes:02}:{seconds:02}")), - TimeCodeFormat::HhMmSsCs => Some(format!( - "{hours:02}:{minutes:02}:{seconds:02}:{centiseconds:02}", - )), - TimeCodeFormat::HhMmSsFf => { - let fps = fps?; - if fps <= 0.0 { - return None; - } - - let frames = ((time_in_seconds % 1.0) * fps).floor() as u64; - Some(format!( - "{hours:02}:{minutes:02}:{seconds:02}:{frames:02}", - )) - } - } -} - -#[export] -pub fn parse_time_code( - ParseTimeCodeOptions { - time_code, - format, - fps, - }: ParseTimeCodeOptions, -) -> Option { - if time_code.trim().is_empty() { - return None; - } - - let format = format.unwrap_or(TimeCodeFormat::HhMmSsCs); - let parts = time_code - .trim() - .split(':') - .map(|part| part.parse::().ok()) - .collect::>>()?; - - match format { - TimeCodeFormat::MmSs => { - let [minutes, seconds] = parts.as_slice() else { - return None; - }; - if *seconds >= SECONDS_PER_MINUTE as u32 { - return None; - } - - Some((*minutes as f64 * SECONDS_PER_MINUTE) + *seconds as f64) - } - TimeCodeFormat::HhMmSs => { - let [hours, minutes, seconds] = parts.as_slice() else { - return None; - }; - if *minutes >= SECONDS_PER_MINUTE as u32 || *seconds >= SECONDS_PER_MINUTE as u32 { - return None; - } - - Some( - (*hours as f64 * SECONDS_PER_HOUR) - + (*minutes as f64 * SECONDS_PER_MINUTE) - + *seconds as f64, - ) - } - TimeCodeFormat::HhMmSsCs => { - let [hours, minutes, seconds, centiseconds] = parts.as_slice() else { - return None; - }; - if *minutes >= SECONDS_PER_MINUTE as u32 - || *seconds >= SECONDS_PER_MINUTE as u32 - || *centiseconds >= CENTISECONDS_PER_SECOND as u32 - { - return None; - } - - Some( - (*hours as f64 * SECONDS_PER_HOUR) - + (*minutes as f64 * SECONDS_PER_MINUTE) - + *seconds as f64 - + (*centiseconds as f64 / CENTISECONDS_PER_SECOND), - ) - } - TimeCodeFormat::HhMmSsFf => { - let fps = fps?; - if fps <= 0.0 { - return None; - } - - let [hours, minutes, seconds, frames] = parts.as_slice() else { - return None; - }; - if *minutes >= SECONDS_PER_MINUTE as u32 - || *seconds >= SECONDS_PER_MINUTE as u32 - || *frames as f64 >= fps - { - return None; - } - - Some( - (*hours as f64 * SECONDS_PER_HOUR) - + (*minutes as f64 * SECONDS_PER_MINUTE) - + *seconds as f64 - + (*frames as f64 / fps), - ) - } - } -} - -#[export] -pub fn guess_time_code_format( - GuessTimeCodeFormatOptions { time_code }: GuessTimeCodeFormatOptions, -) -> Option { - if time_code.trim().is_empty() { - return None; - } - - let part_count = time_code - .split(':') - .try_fold(0usize, |count, part| { - part.parse::().ok().map(|_| count + 1) - })?; - - match part_count { - 2 => Some(TimeCodeFormat::MmSs), - 3 => Some(TimeCodeFormat::HhMmSs), - 4 => Some(TimeCodeFormat::HhMmSsFf), - _ => None, - } -} - -#[export] -pub fn time_to_frame(TimeToFrameOptions { time, fps }: TimeToFrameOptions) -> f64 { - (time * fps).round() -} - -#[export] -pub fn frame_to_time(FrameToTimeOptions { frame, fps }: FrameToTimeOptions) -> f64 { - frame / fps -} - -#[export] -pub fn snap_time_to_frame(SnapTimeToFrameOptions { time, fps }: SnapTimeToFrameOptions) -> f64 { - if fps <= 0.0 { - return time; - } - - frame_to_time(FrameToTimeOptions { - frame: time_to_frame(TimeToFrameOptions { time, fps }), - fps, - }) -} - -#[export] -pub fn get_snapped_seek_time( - GetSnappedSeekTimeOptions { - raw_time, - duration, - fps, - }: GetSnappedSeekTimeOptions, -) -> f64 { - let snapped_time = snap_time_to_frame(SnapTimeToFrameOptions { time: raw_time, fps }); - let last_frame = get_last_frame_time(GetLastFrameTimeOptions { duration, fps }); - snapped_time.clamp(0.0, last_frame) -} - -#[export] -pub fn get_last_frame_time( - GetLastFrameTimeOptions { duration, fps }: GetLastFrameTimeOptions, -) -> f64 { - if duration <= 0.0 { - return 0.0; - } - - if fps <= 0.0 { - return duration; - } - - let frame_offset = 1.0 / fps; - (duration - frame_offset).max(0.0) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn rounds_to_the_nearest_frame() { - assert_eq!(round_to_frame(RoundToFrameOptions { time: 1.24, fps: 10.0 }), 1.2); - assert_eq!(round_to_frame(RoundToFrameOptions { time: 1.26, fps: 10.0 }), 1.3); - } - - #[test] - fn formats_default_time_codes() { - assert_eq!( - format_time_code(FormatTimeCodeOptions { - time_in_seconds: 3723.45, - format: None, - fps: None, - }), - Some("01:02:03:44".to_string()), - ); - assert_eq!( - format_time_code(FormatTimeCodeOptions { - time_in_seconds: 65.0, - format: Some(TimeCodeFormat::MmSs), - fps: None, - }), - Some("01:05".to_string()), - ); - } - - #[test] - fn formats_frame_based_time_codes() { - assert_eq!( - format_time_code(FormatTimeCodeOptions { - time_in_seconds: 1.5, - format: Some(TimeCodeFormat::HhMmSsFf), - fps: Some(30.0), - }), - Some("00:00:01:15".to_string()), - ); - assert_eq!( - format_time_code(FormatTimeCodeOptions { - time_in_seconds: 1.5, - format: Some(TimeCodeFormat::HhMmSsFf), - fps: None, - }), - None, - ); - } - - #[test] - fn parses_time_codes() { - assert_eq!( - parse_time_code(ParseTimeCodeOptions { - time_code: "01:05".to_string(), - format: Some(TimeCodeFormat::MmSs), - fps: None, - }), - Some(65.0), - ); - assert_eq!( - parse_time_code(ParseTimeCodeOptions { - time_code: "00:00:01:15".to_string(), - format: Some(TimeCodeFormat::HhMmSsFf), - fps: Some(30.0), - }), - Some(1.5), - ); - assert_eq!( - parse_time_code(ParseTimeCodeOptions { - time_code: "00:00:01:30".to_string(), - format: Some(TimeCodeFormat::HhMmSsFf), - fps: Some(30.0), - }), - None, - ); - } - - #[test] - fn guesses_time_code_formats() { - assert_eq!( - guess_time_code_format(GuessTimeCodeFormatOptions { time_code: "01:05".to_string() }), - Some(TimeCodeFormat::MmSs), - ); - assert_eq!( - guess_time_code_format(GuessTimeCodeFormatOptions { - time_code: "00:00:01".to_string(), - }), - Some(TimeCodeFormat::HhMmSs), - ); - assert_eq!( - guess_time_code_format(GuessTimeCodeFormatOptions { - time_code: "00:00:01:15".to_string(), - }), - Some(TimeCodeFormat::HhMmSsFf), - ); - } - - #[test] - fn snaps_and_clamps_seek_time() { - assert_eq!(time_to_frame(TimeToFrameOptions { time: 1.26, fps: 10.0 }), 13.0); - assert_eq!(frame_to_time(FrameToTimeOptions { frame: 13.0, fps: 10.0 }), 1.3); - assert_eq!(snap_time_to_frame(SnapTimeToFrameOptions { time: 1.26, fps: 10.0 }), 1.3); - assert_eq!(get_last_frame_time(GetLastFrameTimeOptions { duration: 10.0, fps: 5.0 }), 9.8); - assert_eq!( - get_snapped_seek_time(GetSnappedSeekTimeOptions { - raw_time: 10.0, - duration: 10.0, - fps: 5.0, - }), - 9.8, - ); - } -} +pub use frame_rate::FrameRate; +pub use media_time::{ + FloorToFrameOptions, IsFrameAlignedOptions, LastFrameTimeOptions, MediaTime, + MediaTimeAddOptions, MediaTimeClampOptions, MediaTimeFromFrameOptions, + MediaTimeFromSecondsOptions, MediaTimeMaxOptions, MediaTimeMinOptions, MediaTimeSubOptions, + MediaTimeToFrameOptions, MediaTimeToSecondsOptions, RoundToFrameOptions, + SnappedSeekTimeOptions, TICKS_PER_SECOND, floor_to_frame, is_frame_aligned, last_frame_time, + media_time_add, media_time_clamp, media_time_from_frame, media_time_from_seconds, + media_time_max, media_time_min, media_time_sub, media_time_to_frame, media_time_to_seconds, + round_to_frame, snapped_seek_time, +}; +pub use timecode::{ + FormatTimecodeOptions, GuessTimecodeFormatOptions, ParseTimecodeOptions, TimeCodeFormat, + format_timecode, guess_timecode_format, parse_timecode, +}; diff --git a/rust/crates/time/src/timecode.rs b/rust/crates/time/src/timecode.rs new file mode 100644 index 00000000..0eb484d3 --- /dev/null +++ b/rust/crates/time/src/timecode.rs @@ -0,0 +1,287 @@ +use bridge::export; +use serde::{Deserialize, Serialize}; + +use crate::{ + frame_rate::FrameRate, + media_time::{MediaTime, TICKS_PER_SECOND}, +}; + +const SECONDS_PER_HOUR: i64 = 3_600; +const SECONDS_PER_MINUTE: i64 = 60; +const CENTISECONDS_PER_SECOND: i64 = 100; +const TICKS_PER_CENTISECOND: i64 = TICKS_PER_SECOND / CENTISECONDS_PER_SECOND; + +#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] +#[cfg_attr(feature = "wasm", tsify(from_wasm_abi, into_wasm_abi))] +#[derive(Serialize, Deserialize, Clone, Copy, Debug, Eq, PartialEq)] +pub enum TimeCodeFormat { + #[serde(rename = "MM:SS")] + MmSs, + #[serde(rename = "HH:MM:SS")] + HhMmSs, + #[serde(rename = "HH:MM:SS:CS")] + HhMmSsCs, + #[serde(rename = "HH:MM:SS:FF")] + HhMmSsFf, +} + +#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] +#[cfg_attr(feature = "wasm", tsify(from_wasm_abi))] +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct FormatTimecodeOptions { + pub time: MediaTime, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub format: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub rate: Option, +} + +#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] +#[cfg_attr(feature = "wasm", tsify(from_wasm_abi))] +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ParseTimecodeOptions { + pub time_code: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub format: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub rate: Option, +} + +#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] +#[cfg_attr(feature = "wasm", tsify(from_wasm_abi))] +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct GuessTimecodeFormatOptions { + pub time_code: String, +} + +#[export] +pub fn guess_timecode_format( + GuessTimecodeFormatOptions { time_code }: GuessTimecodeFormatOptions, +) -> Option { + if time_code.trim().is_empty() { + return None; + } + + let part_count = time_code + .trim() + .split(':') + .try_fold(0usize, |count, part| { + part.parse::().ok().map(|_| count + 1) + })?; + + match part_count { + 2 => Some(TimeCodeFormat::MmSs), + 3 => Some(TimeCodeFormat::HhMmSs), + 4 => Some(TimeCodeFormat::HhMmSsFf), + _ => None, + } +} + +#[export] +pub fn format_timecode( + FormatTimecodeOptions { time, format, rate }: FormatTimecodeOptions, +) -> Option { + let format = format.unwrap_or(TimeCodeFormat::HhMmSsCs); + let total_ticks = u64::try_from(time.as_ticks().max(0)).ok()?; + let ticks_per_second = u64::try_from(TICKS_PER_SECOND).ok()?; + let total_seconds = total_ticks / ticks_per_second; + let hour_ticks = u64::try_from(SECONDS_PER_HOUR).ok()? * ticks_per_second; + let minute_ticks = u64::try_from(SECONDS_PER_MINUTE).ok()? * ticks_per_second; + let seconds_per_minute = u64::try_from(SECONDS_PER_MINUTE).ok()?; + let ticks_per_centisecond = u64::try_from(TICKS_PER_CENTISECOND).ok()?; + + let hours = total_ticks / hour_ticks; + let minutes = (total_ticks % hour_ticks) / minute_ticks; + let seconds = total_seconds % seconds_per_minute; + let second_ticks = total_ticks % ticks_per_second; + let centiseconds = second_ticks / ticks_per_centisecond; + + match format { + TimeCodeFormat::MmSs => Some(format!("{minutes:02}:{seconds:02}")), + TimeCodeFormat::HhMmSs => Some(format!("{hours:02}:{minutes:02}:{seconds:02}")), + TimeCodeFormat::HhMmSsCs => Some(format!( + "{hours:02}:{minutes:02}:{seconds:02}:{centiseconds:02}" + )), + TimeCodeFormat::HhMmSsFf => { + let rate = rate?; + let ticks_per_frame = rate.ticks_per_frame()?; + let frames = second_ticks / u64::try_from(ticks_per_frame).ok()?; + Some(format!("{hours:02}:{minutes:02}:{seconds:02}:{frames:02}")) + } + } +} + +#[export] +pub fn parse_timecode( + ParseTimecodeOptions { + time_code, + format, + rate, + }: ParseTimecodeOptions, +) -> Option { + if time_code.trim().is_empty() { + return None; + } + + let format = format.unwrap_or(TimeCodeFormat::HhMmSsCs); + let parts = time_code + .trim() + .split(':') + .map(|part| part.parse::().ok()) + .collect::>>()?; + + match format { + TimeCodeFormat::MmSs => { + let [minutes, seconds] = parts.as_slice() else { + return None; + }; + if i64::from(*seconds) >= SECONDS_PER_MINUTE { + return None; + } + + Some(MediaTime::from_ticks( + (i64::from(*minutes) * SECONDS_PER_MINUTE + i64::from(*seconds)) * TICKS_PER_SECOND, + )) + } + TimeCodeFormat::HhMmSs => { + let [hours, minutes, seconds] = parts.as_slice() else { + return None; + }; + if i64::from(*minutes) >= SECONDS_PER_MINUTE + || i64::from(*seconds) >= SECONDS_PER_MINUTE + { + return None; + } + + Some(MediaTime::from_ticks( + (i64::from(*hours) * SECONDS_PER_HOUR + + i64::from(*minutes) * SECONDS_PER_MINUTE + + i64::from(*seconds)) + * TICKS_PER_SECOND, + )) + } + TimeCodeFormat::HhMmSsCs => { + let [hours, minutes, seconds, centiseconds] = parts.as_slice() else { + return None; + }; + if i64::from(*minutes) >= SECONDS_PER_MINUTE + || i64::from(*seconds) >= SECONDS_PER_MINUTE + || i64::from(*centiseconds) >= CENTISECONDS_PER_SECOND + { + return None; + } + + Some(MediaTime::from_ticks( + (i64::from(*hours) * SECONDS_PER_HOUR + + i64::from(*minutes) * SECONDS_PER_MINUTE + + i64::from(*seconds)) + * TICKS_PER_SECOND + + i64::from(*centiseconds) * TICKS_PER_CENTISECOND, + )) + } + TimeCodeFormat::HhMmSsFf => { + let rate = rate?; + let frame_upper_bound = rate.frame_number_upper_bound()?; + let [hours, minutes, seconds, frames] = parts.as_slice() else { + return None; + }; + if i64::from(*minutes) >= SECONDS_PER_MINUTE + || i64::from(*seconds) >= SECONDS_PER_MINUTE + || *frames >= frame_upper_bound + { + return None; + } + + Some( + MediaTime::from_ticks( + (i64::from(*hours) * SECONDS_PER_HOUR + + i64::from(*minutes) * SECONDS_PER_MINUTE + + i64::from(*seconds)) + * TICKS_PER_SECOND, + ) + MediaTime::from_frame(i64::from(*frames), rate)?, + ) + } + } +} + +#[cfg(test)] +mod tests { + use crate::frame_rate::FrameRate; + use crate::media_time::MediaTime; + + use super::{FormatTimecodeOptions, GuessTimecodeFormatOptions, ParseTimecodeOptions}; + use super::{TimeCodeFormat, format_timecode, guess_timecode_format, parse_timecode}; + + #[test] + fn formats_default_and_frame_timecodes() { + assert_eq!( + format_timecode(FormatTimecodeOptions { + time: MediaTime::from_seconds_f64(3723.45).unwrap(), + format: None, + rate: None, + }), + Some("01:02:03:45".to_string()), + ); + assert_eq!( + format_timecode(FormatTimecodeOptions { + time: MediaTime::from_seconds_f64(1.5).unwrap(), + format: Some(TimeCodeFormat::HhMmSsFf), + rate: Some(FrameRate::FPS_30), + }), + Some("00:00:01:15".to_string()), + ); + } + + #[test] + fn parses_timecodes() { + assert_eq!( + parse_timecode(ParseTimecodeOptions { + time_code: "01:05".to_string(), + format: Some(TimeCodeFormat::MmSs), + rate: None, + }), + Some(MediaTime::from_seconds_f64(65.0).unwrap()), + ); + assert_eq!( + parse_timecode(ParseTimecodeOptions { + time_code: "00:00:01:15".to_string(), + format: Some(TimeCodeFormat::HhMmSsFf), + rate: Some(FrameRate::FPS_30), + }), + Some(MediaTime::from_seconds_f64(1.5).unwrap()), + ); + assert_eq!( + parse_timecode(ParseTimecodeOptions { + time_code: "00:00:01:30".to_string(), + format: Some(TimeCodeFormat::HhMmSsFf), + rate: Some(FrameRate::FPS_30), + }), + None, + ); + } + + #[test] + fn guesses_timecode_formats() { + assert_eq!( + guess_timecode_format(GuessTimecodeFormatOptions { + time_code: "01:05".to_string(), + }), + Some(TimeCodeFormat::MmSs), + ); + assert_eq!( + guess_timecode_format(GuessTimecodeFormatOptions { + time_code: "00:00:01".to_string(), + }), + Some(TimeCodeFormat::HhMmSs), + ); + assert_eq!( + guess_timecode_format(GuessTimecodeFormatOptions { + time_code: "00:00:01:15".to_string(), + }), + Some(TimeCodeFormat::HhMmSsFf), + ); + } +} diff --git a/rust/wasm/Cargo.toml b/rust/wasm/Cargo.toml index a5b0d9e0..88f92a5c 100644 --- a/rust/wasm/Cargo.toml +++ b/rust/wasm/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "opencut-wasm" -version = "0.1.3" +version = "0.2.3" edition = "2024" description = "Shared video editor logic compiled to WebAssembly" repository = "https://github.com/opencut/opencut" @@ -11,11 +11,19 @@ path = "src/wasm.rs" crate-type = ["cdylib", "rlib"] [dependencies] -gpu = { version = "0.1.0", path = "../crates/gpu" } +bridge = { version = "0.1.0", path = "../crates/bridge" } +effects = { version = "0.1.0", path = "../crates/effects" } +gpu = { version = "0.1.0", path = "../crates/gpu", features = ["wasm"] } js-sys = "0.3.93" +masks = { version = "0.1.0", path = "../crates/masks" } +num-traits = "0.2.19" serde = { version = "1.0.228", features = ["derive"] } serde-wasm-bindgen = "0.6.5" time = { version = "0.1.0", path = "../crates/time", features = ["wasm"] } wasm-bindgen = "0.2.116" wasm-bindgen-futures = "0.4.66" web-sys = { version = "0.3.93", features = ["OffscreenCanvas", "HtmlCanvasElement", "CanvasRenderingContext2d", "Document", "Window"] } + +[features] +default = ["wasm"] +wasm = [] diff --git a/rust/wasm/README.md b/rust/wasm/README.md index e90c58c2..8147bb08 100644 --- a/rust/wasm/README.md +++ b/rust/wasm/README.md @@ -11,7 +11,10 @@ npm install opencut-wasm ## Usage ```ts -import { formatTimeCode } from "opencut-wasm"; +import { formatTimecode, mediaTimeFromSeconds } from "opencut-wasm"; + +const ticks = mediaTimeFromSeconds(1.5); +const label = formatTimecode({ ticks }); ``` All exports are documented in the [TypeScript definitions](./opencut_wasm.d.ts). diff --git a/rust/wasm/src/effects.rs b/rust/wasm/src/effects.rs new file mode 100644 index 00000000..1f5c2947 --- /dev/null +++ b/rust/wasm/src/effects.rs @@ -0,0 +1,101 @@ +#![cfg(target_arch = "wasm32")] + +use effects::{ApplyEffectsOptions, EffectPass, UniformValue}; +use gpu::wgpu; +use js_sys::Object; +use serde::Deserialize; +use wasm_bindgen::{JsCast, JsValue, prelude::wasm_bindgen}; + +use crate::gpu::{ + import_canvas_texture, read_offscreen_canvas_property, read_serde_property, read_u32_property, + render_texture_to_canvas, with_gpu_runtime, +}; + +struct ApplyEffectPassesOptions { + source: wgpu::web_sys::OffscreenCanvas, + width: u32, + height: u32, + passes: Vec, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct EffectPassInput { + shader: String, + uniforms: Vec, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct EffectUniformInput { + name: String, + value: Vec, +} + +#[wasm_bindgen(js_name = applyEffectPasses)] +pub fn apply_effect_passes(options: JsValue) -> Result { + let ApplyEffectPassesOptions { + source, + width, + height, + passes, + } = parse_apply_effect_passes_options(options)?; + + with_gpu_runtime(|runtime| { + let source_texture = import_canvas_texture( + &runtime.context, + &source, + width, + height, + "effects-input-texture", + ); + let effect_passes = map_effect_passes(passes); + let result_texture = runtime + .effects + .apply( + &runtime.context, + ApplyEffectsOptions { + source: &source_texture, + width, + height, + passes: &effect_passes, + }, + ) + .map_err(|error| JsValue::from_str(&error.to_string()))?; + render_texture_to_canvas(&runtime.context, &result_texture, width, height) + }) +} + +fn map_effect_passes(effect_passes: Vec) -> Vec { + effect_passes + .into_iter() + .map(|pass| EffectPass { + shader: pass.shader, + uniforms: pass + .uniforms + .into_iter() + .map(|uniform| { + let value = if uniform.value.len() == 1 { + UniformValue::Number(uniform.value[0]) + } else { + UniformValue::Vector(uniform.value) + }; + (uniform.name, value) + }) + .collect(), + }) + .collect() +} + +fn parse_apply_effect_passes_options(value: JsValue) -> Result { + let object: Object = value + .dyn_into() + .map_err(|_| JsValue::from_str("applyEffectPasses expects an options object"))?; + + Ok(ApplyEffectPassesOptions { + source: read_offscreen_canvas_property(&object, "source")?, + width: read_u32_property(&object, "width")?, + height: read_u32_property(&object, "height")?, + passes: read_serde_property(&object, "passes")?, + }) +} diff --git a/rust/wasm/src/gpu.rs b/rust/wasm/src/gpu.rs new file mode 100644 index 00000000..70f3a241 --- /dev/null +++ b/rust/wasm/src/gpu.rs @@ -0,0 +1,123 @@ +#![cfg(target_arch = "wasm32")] + +use std::cell::RefCell; + +use effects::EffectPipeline; +use gpu::{GpuContext, wgpu}; +use js_sys::{Object, Reflect}; +use masks::MaskFeatherPipeline; +use serde::Deserialize; +use wasm_bindgen::{JsCast, JsValue, prelude::wasm_bindgen}; + +pub(crate) struct GpuRuntime { + pub(crate) context: GpuContext, + pub(crate) effects: EffectPipeline, + pub(crate) masks: MaskFeatherPipeline, +} + +thread_local! { + static GPU_RUNTIME: RefCell> = const { RefCell::new(None) }; +} + +#[wasm_bindgen(js_name = initializeGpu)] +pub async fn initialize_gpu() -> Result<(), JsValue> { + if GPU_RUNTIME.with(|runtime| runtime.borrow().is_some()) { + return Ok(()); + } + + let context = GpuContext::new() + .await + .map_err(|error| JsValue::from_str(&error.to_string()))?; + let effects = EffectPipeline::new(&context); + let masks = MaskFeatherPipeline::new(&context); + + GPU_RUNTIME.with(|runtime| { + runtime.replace(Some(GpuRuntime { + context, + effects, + masks, + })); + }); + + Ok(()) +} + +pub(crate) fn with_gpu_runtime( + action: impl FnOnce(&GpuRuntime) -> Result, +) -> Result { + GPU_RUNTIME.with(|runtime| { + let borrow = runtime.borrow(); + let Some(gpu_runtime) = borrow.as_ref() else { + return Err(JsValue::from_str( + "GPU context not initialized. Call initializeGpu() first.", + )); + }; + action(gpu_runtime) + }) +} + +pub(crate) fn import_canvas_texture( + context: &GpuContext, + canvas: &wgpu::web_sys::OffscreenCanvas, + width: u32, + height: u32, + label: &'static str, +) -> wgpu::Texture { + context.import_offscreen_canvas_texture(canvas, width, height, label) +} + +pub(crate) fn render_texture_to_canvas( + context: &GpuContext, + texture: &wgpu::Texture, + width: u32, + height: u32, +) -> Result { + let canvas = wgpu::web_sys::OffscreenCanvas::new(width, height)?; + context + .render_texture_to_offscreen_canvas(texture, &canvas, width, height) + .map_err(|error| JsValue::from_str(&error.to_string()))?; + Ok(canvas) +} + +pub(crate) fn read_property(object: &Object, name: &str) -> Result { + Reflect::get(object, &JsValue::from_str(name)) + .map_err(|_| JsValue::from_str(&format!("Missing property '{name}'"))) +} + +pub(crate) fn read_offscreen_canvas_property( + object: &Object, + name: &str, +) -> Result { + read_property(object, name)? + .dyn_into::() + .map_err(|_| JsValue::from_str(&format!("Property '{name}' must be an OffscreenCanvas"))) +} + +pub(crate) fn read_u32_property(object: &Object, name: &str) -> Result { + let value = read_property(object, name)?; + let Some(number) = value.as_f64() else { + return Err(JsValue::from_str(&format!( + "Property '{name}' must be a number" + ))); + }; + Ok(number as u32) +} + +pub(crate) fn read_f32_property(object: &Object, name: &str) -> Result { + let value = read_property(object, name)?; + let Some(number) = value.as_f64() else { + return Err(JsValue::from_str(&format!( + "Property '{name}' must be a number" + ))); + }; + Ok(number as f32) +} + +pub(crate) fn read_serde_property(object: &Object, name: &str) -> Result +where + T: for<'de> Deserialize<'de>, +{ + let value = read_property(object, name)?; + serde_wasm_bindgen::from_value(value) + .map_err(|error| JsValue::from_str(&format!("Invalid property '{name}': {error}"))) +} diff --git a/rust/wasm/src/gpu_bridge.rs b/rust/wasm/src/gpu_bridge.rs deleted file mode 100644 index 85db0c8b..00000000 --- a/rust/wasm/src/gpu_bridge.rs +++ /dev/null @@ -1,256 +0,0 @@ -#![cfg(target_arch = "wasm32")] - -use std::cell::RefCell; - -use gpu::{EffectPass, GpuContext, UniformValue, wgpu}; -use js_sys::{Object, Reflect}; -use serde::Deserialize; -use wasm_bindgen::{JsCast, JsValue, prelude::wasm_bindgen}; - -thread_local! { - static GPU_CONTEXT: RefCell> = const { RefCell::new(None) }; -} - -struct ApplyEffectPassesOptions { - source: wgpu::web_sys::OffscreenCanvas, - width: u32, - height: u32, - passes: Vec, -} - -#[derive(Deserialize)] -#[serde(rename_all = "camelCase")] -struct EffectPassInput { - shader: String, - uniforms: Vec, -} - -#[derive(Deserialize)] -#[serde(rename_all = "camelCase")] -struct EffectUniformInput { - name: String, - value: Vec, -} - -struct ApplyMaskFeatherOptions { - mask: wgpu::web_sys::OffscreenCanvas, - width: u32, - height: u32, - feather: f32, -} - -#[wasm_bindgen(js_name = initializeGpu)] -pub async fn initialize_gpu() -> Result<(), JsValue> { - if GPU_CONTEXT.with(|context| context.borrow().is_some()) { - return Ok(()); - } - - let context = GpuContext::new() - .await - .map_err(|error| JsValue::from_str(&error.to_string()))?; - GPU_CONTEXT.with(|gpu_context| { - gpu_context.replace(Some(context)); - }); - Ok(()) -} - -#[wasm_bindgen(js_name = applyEffectPasses)] -pub fn apply_effect_passes( - options: JsValue, -) -> Result { - let ApplyEffectPassesOptions { - source, - width, - height, - passes, - } = parse_apply_effect_passes_options(options)?; - - with_gpu_context(|context| { - let source_texture = import_canvas_texture(context, &source, width, height)?; - let effect_passes = map_effect_passes(passes); - let result_texture = context - .apply_effects(gpu::ApplyEffectsOptions { - source: &source_texture, - width, - height, - passes: &effect_passes, - }) - .map_err(|error| JsValue::from_str(&error.to_string()))?; - let output_canvas = wgpu::web_sys::OffscreenCanvas::new(width, height)?; - let surface = context - .instance() - .create_surface(wgpu::SurfaceTarget::OffscreenCanvas(output_canvas.clone())) - .map_err(|error| JsValue::from_str(&error.to_string()))?; - - context - .render_texture_to_surface(&result_texture, &surface, width, height) - .map_err(|error| JsValue::from_str(&error.to_string()))?; - Ok(output_canvas) - }) -} - -#[wasm_bindgen(js_name = applyMaskFeather)] -pub fn apply_mask_feather( - options: JsValue, -) -> Result { - let ApplyMaskFeatherOptions { - mask, - width, - height, - feather, - } = parse_apply_mask_feather_options(options)?; - - with_gpu_context(|context| { - let mask_texture = import_canvas_texture(context, &mask, width, height)?; - let result_texture = context.apply_mask_feather(gpu::ApplyMaskFeatherOptions { - mask: &mask_texture, - width, - height, - feather, - }); - let output_canvas = wgpu::web_sys::OffscreenCanvas::new(width, height)?; - let surface = context - .instance() - .create_surface(wgpu::SurfaceTarget::OffscreenCanvas(output_canvas.clone())) - .map_err(|error| JsValue::from_str(&error.to_string()))?; - - context - .render_texture_to_surface(&result_texture, &surface, width, height) - .map_err(|error| JsValue::from_str(&error.to_string()))?; - Ok(output_canvas) - }) -} - -fn with_gpu_context( - action: impl FnOnce(&GpuContext) -> Result, -) -> Result { - GPU_CONTEXT.with(|context| { - let borrow = context.borrow(); - let Some(gpu_context) = borrow.as_ref() else { - return Err(JsValue::from_str( - "GPU context not initialized. Call initializeGpu() first.", - )); - }; - action(gpu_context) - }) -} - -fn import_canvas_texture( - context: &GpuContext, - canvas: &wgpu::web_sys::OffscreenCanvas, - width: u32, - height: u32, -) -> Result { - let texture = context.create_render_texture(width, height, "gpu-bridge-input-texture"); - context.queue().copy_external_image_to_texture( - &wgpu::CopyExternalImageSourceInfo { - source: wgpu::ExternalImageSource::OffscreenCanvas(canvas.clone()), - origin: wgpu::Origin2d::ZERO, - flip_y: true, - }, - wgpu::CopyExternalImageDestInfo { - texture: &texture, - mip_level: 0, - origin: wgpu::Origin3d::ZERO, - aspect: wgpu::TextureAspect::All, - color_space: wgpu::PredefinedColorSpace::Srgb, - premultiplied_alpha: false, - }, - wgpu::Extent3d { - width, - height, - depth_or_array_layers: 1, - }, - ); - Ok(texture) -} - -fn map_effect_passes(effect_passes: Vec) -> Vec { - effect_passes - .into_iter() - .map(|pass| EffectPass { - shader: pass.shader, - uniforms: pass - .uniforms - .into_iter() - .map(|uniform| { - let value = if uniform.value.len() == 1 { - UniformValue::Number(uniform.value[0]) - } else { - UniformValue::Vector(uniform.value) - }; - (uniform.name, value) - }) - .collect(), - }) - .collect() -} - -fn parse_apply_effect_passes_options(value: JsValue) -> Result { - let object: Object = value - .dyn_into() - .map_err(|_| JsValue::from_str("applyEffectPasses expects an options object"))?; - - Ok(ApplyEffectPassesOptions { - source: read_offscreen_canvas_property(&object, "source")?, - width: read_u32_property(&object, "width")?, - height: read_u32_property(&object, "height")?, - passes: read_serde_property(&object, "passes")?, - }) -} - -fn parse_apply_mask_feather_options(value: JsValue) -> Result { - let object: Object = value - .dyn_into() - .map_err(|_| JsValue::from_str("applyMaskFeather expects an options object"))?; - - Ok(ApplyMaskFeatherOptions { - mask: read_offscreen_canvas_property(&object, "mask")?, - width: read_u32_property(&object, "width")?, - height: read_u32_property(&object, "height")?, - feather: read_f32_property(&object, "feather")?, - }) -} - -fn read_property(object: &Object, name: &str) -> Result { - Reflect::get(object, &JsValue::from_str(name)) - .map_err(|_| JsValue::from_str(&format!("Missing property '{name}'"))) -} - -fn read_offscreen_canvas_property( - object: &Object, - name: &str, -) -> Result { - read_property(object, name)? - .dyn_into::() - .map_err(|_| JsValue::from_str(&format!("Property '{name}' must be an OffscreenCanvas"))) -} - -fn read_u32_property(object: &Object, name: &str) -> Result { - let value = read_property(object, name)?; - let Some(number) = value.as_f64() else { - return Err(JsValue::from_str(&format!( - "Property '{name}' must be a number" - ))); - }; - Ok(number as u32) -} - -fn read_f32_property(object: &Object, name: &str) -> Result { - let value = read_property(object, name)?; - let Some(number) = value.as_f64() else { - return Err(JsValue::from_str(&format!( - "Property '{name}' must be a number" - ))); - }; - Ok(number as f32) -} - -fn read_serde_property(object: &Object, name: &str) -> Result -where - T: for<'de> Deserialize<'de>, -{ - let value = read_property(object, name)?; - serde_wasm_bindgen::from_value(value) - .map_err(|error| JsValue::from_str(&format!("Invalid property '{name}': {error}"))) -} diff --git a/rust/wasm/src/masks.rs b/rust/wasm/src/masks.rs new file mode 100644 index 00000000..e4f30e21 --- /dev/null +++ b/rust/wasm/src/masks.rs @@ -0,0 +1,60 @@ +#![cfg(target_arch = "wasm32")] + +use gpu::wgpu; +use js_sys::Object; +use wasm_bindgen::{JsCast, JsValue, prelude::wasm_bindgen}; + +use crate::gpu::{ + import_canvas_texture, read_f32_property, read_offscreen_canvas_property, read_u32_property, + render_texture_to_canvas, with_gpu_runtime, +}; + +struct ApplyMaskFeatherOptions { + mask: wgpu::web_sys::OffscreenCanvas, + width: u32, + height: u32, + feather: f32, +} + +#[wasm_bindgen(js_name = applyMaskFeather)] +pub fn apply_mask_feather(options: JsValue) -> Result { + let ApplyMaskFeatherOptions { + mask, + width, + height, + feather, + } = parse_apply_mask_feather_options(options)?; + + with_gpu_runtime(|runtime| { + let mask_texture = import_canvas_texture( + &runtime.context, + &mask, + width, + height, + "masks-input-texture", + ); + let result_texture = runtime.masks.apply_mask_feather( + &runtime.context, + masks::ApplyMaskFeatherOptions { + mask: &mask_texture, + width, + height, + feather, + }, + ); + render_texture_to_canvas(&runtime.context, &result_texture, width, height) + }) +} + +fn parse_apply_mask_feather_options(value: JsValue) -> Result { + let object: Object = value + .dyn_into() + .map_err(|_| JsValue::from_str("applyMaskFeather expects an options object"))?; + + Ok(ApplyMaskFeatherOptions { + mask: read_offscreen_canvas_property(&object, "mask")?, + width: read_u32_property(&object, "width")?, + height: read_u32_property(&object, "height")?, + feather: read_f32_property(&object, "feather")?, + }) +} diff --git a/rust/wasm/src/wasm.rs b/rust/wasm/src/wasm.rs index 0a78a314..27ece766 100644 --- a/rust/wasm/src/wasm.rs +++ b/rust/wasm/src/wasm.rs @@ -1,6 +1,14 @@ #[cfg(target_arch = "wasm32")] -mod gpu_bridge; +mod effects; +#[cfg(target_arch = "wasm32")] +mod gpu; +#[cfg(target_arch = "wasm32")] +mod masks; #[cfg(target_arch = "wasm32")] -pub use gpu_bridge::*; +pub use effects::*; +#[cfg(target_arch = "wasm32")] +pub use gpu::*; +#[cfg(target_arch = "wasm32")] +pub use masks::*; pub use time::*;