diff --git a/Cargo.lock b/Cargo.lock index f3bd3c00..04c00a6c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3820,7 +3820,7 @@ dependencies = [ [[package]] name = "opencut-wasm" -version = "0.2.9" +version = "0.2.10" dependencies = [ "bridge", "compositor", diff --git a/README.md b/README.md index 3cb67abf..5fe7c25f 100644 --- a/README.md +++ b/README.md @@ -1,167 +1,144 @@ - - - - - -
- OpenCut Logo - -

OpenCut

-

A free, open-source video editor for web, desktop, and mobile.

-
- -## Sponsors - -Thanks to [Vercel](https://vercel.com?utm_source=github-opencut&utm_campaign=oss) and [fal.ai](https://fal.ai?utm_source=github-opencut&utm_campaign=oss) for their support of open-source software. - - - Vercel OSS Program - - - - Powered by fal.ai - - -## Why? - -- **Privacy**: Your videos stay on your device -- **Free features**: Most basic CapCut features are now paywalled -- **Simple**: People want editors that are easy to use - CapCut proved that - -## Project Structure - -- `apps/web/`: Next.js web application -- `apps/desktop/`: Native desktop app built with GPUI (in progress) -- `rust/`: Platform-agnostic core: GPU compositor, effects, masks, and WASM bindings. We're actively migrating business logic here from TypeScript. -- `docs/`: Architecture and subsystem documentation - -## Getting Started - -### Prerequisites - -- [Bun](https://bun.sh/docs/installation) -- [Docker](https://docs.docker.com/get-docker/) and [Docker Compose](https://docs.docker.com/compose/install/) - -> **Note:** Docker is optional but recommended for running the local database and Redis. If you only want to work on frontend features, you can skip it. - -### Setup - -1. Fork and clone the repository - -2. Copy the environment file: - - ```bash - # Unix/Linux/Mac - cp apps/web/.env.example apps/web/.env.local - - # Windows PowerShell - Copy-Item apps/web/.env.example apps/web/.env.local - ``` - -3. Start the database and Redis: - - ```bash - docker compose up -d db redis serverless-redis-http - ``` - -4. Install dependencies and start the dev server: - - ```bash - bun install - bun dev:web - ``` - -The application will be available at [http://localhost:3000](http://localhost:3000). - -The `.env.example` has sensible defaults that match the Docker Compose config — it should work out of the box. - -### Desktop setup - -Desktop is opt-in. If you're only working on the web app, skip this entirely. - -If you want to get ready for `apps/desktop`, see [`apps/desktop/README.md`](apps/desktop/README.md). It's a two-step setup: Rust toolchain first, then desktop native dependencies. - -### Local WASM development - -Only needed if you're editing `rust/wasm` and want the web app to use your local build instead of the published package. - -**Prerequisites** — install these once before anything else: - -```bash -# Rust toolchain -curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh - -# build the WASM package -cargo install wasm-pack - -# reruns the build on file changes, used by bun dev:wasm -cargo install cargo-watch -``` - -1. Build the package once from the repo root: - - ```bash - bun run build:wasm - ``` - -2. Register the generated package for linking: - - ```bash - cd rust/wasm/pkg - bun link - ``` - -3. Link `apps/web` to the local package: - - ```bash - cd apps/web - bun link opencut-wasm - ``` - -4. Rebuild on changes while you work: - - ```bash - bun dev:wasm - ``` - -To switch `apps/web` back to the published package, run: - -```bash -cd apps/web -bun add opencut-wasm -``` - -### Self-Hosting with Docker - -To run everything (including a production build of the app) in Docker: - -```bash -docker compose up -d -``` - -The app will be available at [http://localhost:3100](http://localhost:3100). - -## Contributing - -We welcome contributions! While we're actively developing and refactoring certain areas, there are plenty of opportunities to contribute effectively. - -**🎯 Focus areas:** Timeline functionality, project management, performance, bug fixes, and UI improvements outside the preview panel. - -**⚠️ Avoid for now:** Preview panel enhancements (fonts, stickers, effects) and export functionality - we're refactoring these with a new binary rendering approach. - -See our [Contributing Guide](.github/CONTRIBUTING.md) for detailed setup instructions, development guidelines, and complete focus area guidance. - -**Quick start for contributors:** - -- Fork the repo and clone locally -- Follow the setup instructions in CONTRIBUTING.md -- Working on `apps/desktop`? See [`apps/desktop/README.md`](apps/desktop/README.md) for setup -- Create a feature branch and submit a PR - -## License - -[MIT LICENSE](LICENSE) - ---- - -![Star History Chart](https://api.star-history.com/svg?repos=opencut-app/opencut&type=Date) + +| | | +| --- | ---------------------------------------------------------------------- | +| | OpenCutA free, open-source video editor for web, desktop, and mobile. | + + +## Sponsors + +Thanks to [Vercel](https://vercel.com?utm_source=github-opencut&utm_campaign=oss) and [fal.ai](https://fal.ai?utm_source=github-opencut&utm_campaign=oss) for their support of open-source software. + + + + + +## Why? + +- **Privacy**: Your videos stay on your device +- **Free features**: Most basic CapCut features are now paywalled +- **Simple**: People want editors that are easy to use - CapCut proved that + +## Project Structure + +- `apps/web/`: Next.js web application +- `apps/desktop/`: Native desktop app built with GPUI (in progress) +- `rust/`: Platform-agnostic core: GPU compositor, effects, masks, and WASM bindings. We're actively migrating business logic here from TypeScript. +- `docs/`: Architecture and subsystem documentation + +## Getting Started + +### Prerequisites + +- [Bun](https://bun.sh/docs/installation) +- [Docker](https://docs.docker.com/get-docker/) and [Docker Compose](https://docs.docker.com/compose/install/) + +> **Note:** Docker is optional but recommended for running the local database and Redis. If you only want to work on frontend features, you can skip it. + +### Setup + +1. Fork and clone the repository +2. Copy the environment file: + ```bash + # Unix/Linux/Mac + cp apps/web/.env.example apps/web/.env.local + + # Windows PowerShell + Copy-Item apps/web/.env.example apps/web/.env.local + ``` +3. Start the database and Redis: + ```bash + docker compose up -d db redis serverless-redis-http + ``` +4. Install dependencies and start the dev server: + ```bash + bun install + bun dev:web + ``` + +The application will be available at [http://localhost:3000](http://localhost:3000). + +The `.env.example` has sensible defaults that match the Docker Compose config — it should work out of the box. + +### Desktop setup + +Desktop is opt-in. If you're only working on the web app, skip this entirely. + +If you want to get ready for `apps/desktop`, see `[apps/desktop/README.md](apps/desktop/README.md)`. It's a two-step setup: Rust toolchain first, then desktop native dependencies. + +### Local WASM development + +Only needed if you're editing `rust/wasm` and want the web app to use your local build instead of the published package. + +**Prerequisites** — install these once before anything else: + +```bash +# Rust toolchain +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh + +# build the WASM package +cargo install wasm-pack + +# reruns the build on file changes, used by bun dev:wasm +cargo install cargo-watch +``` + +1. Build the package once from the repo root: + ```bash + bun run build:wasm + ``` +2. Register the generated package for linking: + ```bash + cd rust/wasm/pkg + bun link + ``` +3. Link `apps/web` to the local package: + ```bash + cd apps/web + bun link opencut-wasm + ``` +4. Rebuild on changes while you work: + ```bash + bun dev:wasm + ``` + +To switch `apps/web` back to the published package, run: + +```bash +cd apps/web +bun add opencut-wasm +``` + +### Self-Hosting with Docker + +To run everything (including a production build of the app) in Docker: + +```bash +docker compose up -d +``` + +The app will be available at [http://localhost:3100](http://localhost:3100). + +## Contributing + +We welcome contributions! While we're actively developing and refactoring certain areas, there are plenty of opportunities to contribute effectively. + +**🎯 Focus areas:** Timeline functionality, project management, performance, bug fixes, and UI improvements outside the preview panel. + +**⚠️ Avoid for now:** Preview panel enhancements (fonts, stickers, effects) and export functionality - we're refactoring these with a new binary rendering approach. + +See our [Contributing Guide](.github/CONTRIBUTING.md) for detailed setup instructions, development guidelines, and complete focus area guidance. + +**Quick start for contributors:** + +- Fork the repo and clone locally +- Follow the setup instructions in CONTRIBUTING.md +- Working on `apps/desktop`? See `[apps/desktop/README.md](apps/desktop/README.md)` for setup +- Create a feature branch and submit a PR + +## License + +[MIT LICENSE](LICENSE) + +--- + +Star History Chart \ No newline at end of file diff --git a/apps/web/package.json b/apps/web/package.json index 0034ebad..bbc3ad0a 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -52,7 +52,7 @@ "nanoid": "^5.1.5", "next": "16.1.3", "next-themes": "^0.4.4", - "opencut-wasm": "^0.2.9", + "opencut-wasm": "^0.2.10", "pg": "^8.16.2", "postgres": "^3.4.5", "radix-ui": "^1.4.3", diff --git a/apps/web/src/animation/__tests__/animated-params.test.ts b/apps/web/src/animation/__tests__/animated-params.test.ts new file mode 100644 index 00000000..7b9d312d --- /dev/null +++ b/apps/web/src/animation/__tests__/animated-params.test.ts @@ -0,0 +1,241 @@ +import { describe, expect, test } from "bun:test"; +import { + coerceAnimationParamValue, + getAnimationParamDefaultInterpolation, + getAnimationParamNumericRange, + getAnimationParamValueKind, +} from "@/animation/animated-params"; + +describe("animated params", () => { + test("snaps and clamps number params", () => { + expect( + coerceAnimationParamValue({ + param: { + key: "intensity", + label: "Intensity", + type: "number", + default: 0, + min: 0, + max: 1, + step: 0.25, + }, + value: 0.62, + }), + ).toBe(0.5); + + expect( + coerceAnimationParamValue({ + param: { + key: "intensity", + label: "Intensity", + type: "number", + default: 0, + min: 0, + max: 1, + step: 0.25, + }, + value: 1.2, + }), + ).toBe(1); + }); + + test("rejects NaN and non-number values for number params", () => { + const param = { + key: "intensity", + label: "Intensity", + type: "number" as const, + default: 0, + min: 0, + max: 1, + step: 0.25, + }; + expect(coerceAnimationParamValue({ param, value: Number.NaN })).toBeNull(); + expect(coerceAnimationParamValue({ param, value: "0.5" })).toBeNull(); + expect(coerceAnimationParamValue({ param, value: true })).toBeNull(); + }); + + test("passthrough with step <= 0 guard", () => { + expect( + coerceAnimationParamValue({ + param: { + key: "x", + label: "X", + type: "number", + default: 0, + min: 0, + step: 0, + }, + value: 0.123, + }), + ).toBe(0.123); + }); + + test("accepts valid select values", () => { + const param = { + key: "blend", + label: "Blend", + type: "select" as const, + default: "normal", + options: [ + { value: "normal", label: "Normal" }, + { value: "multiply", label: "Multiply" }, + ], + }; + expect(coerceAnimationParamValue({ param, value: "normal" })).toBe("normal"); + expect(coerceAnimationParamValue({ param, value: "multiply" })).toBe("multiply"); + }); + + test("rejects select values outside the allowed options", () => { + expect( + coerceAnimationParamValue({ + param: { + key: "blend", + label: "Blend", + type: "select", + default: "normal", + options: [ + { value: "normal", label: "Normal" }, + { value: "multiply", label: "Multiply" }, + ], + }, + value: "screen", + }), + ).toBeNull(); + }); + + test("rejects non-string select values", () => { + const param = { + key: "blend", + label: "Blend", + type: "select" as const, + default: "normal", + options: [{ value: "normal", label: "Normal" }], + }; + expect(coerceAnimationParamValue({ param, value: 42 })).toBeNull(); + expect(coerceAnimationParamValue({ param, value: null })).toBeNull(); + expect(coerceAnimationParamValue({ param, value: undefined })).toBeNull(); + }); + + test("boolean params accept booleans and reject other types", () => { + const param = { + key: "visible", + label: "Visible", + type: "boolean" as const, + default: true, + }; + expect(coerceAnimationParamValue({ param, value: true })).toBe(true); + expect(coerceAnimationParamValue({ param, value: false })).toBe(false); + expect(coerceAnimationParamValue({ param, value: 1 })).toBeNull(); + expect(coerceAnimationParamValue({ param, value: "true" })).toBeNull(); + }); + + test("color params accept strings and reject other types", () => { + const param = { + key: "fill", + label: "Fill", + type: "color" as const, + default: "#ffffff", + }; + expect(coerceAnimationParamValue({ param, value: "#ff0000" })).toBe("#ff0000"); + expect(coerceAnimationParamValue({ param, value: 0xff0000 })).toBeNull(); + expect(coerceAnimationParamValue({ param, value: null })).toBeNull(); + }); + + test("getAnimationParamValueKind maps param type to binding kind", () => { + expect( + getAnimationParamValueKind({ + param: { + key: "n", + label: "N", + type: "number", + default: 0, + min: 0, + step: 1, + }, + }), + ).toBe("number"); + expect( + getAnimationParamValueKind({ + param: { key: "c", label: "C", type: "color", default: "#fff" }, + }), + ).toBe("color"); + expect( + getAnimationParamValueKind({ + param: { key: "b", label: "B", type: "boolean", default: false }, + }), + ).toBe("discrete"); + expect( + getAnimationParamValueKind({ + param: { + key: "s", + label: "S", + type: "select", + default: "a", + options: [{ value: "a", label: "A" }], + }, + }), + ).toBe("discrete"); + }); + + test("getAnimationParamDefaultInterpolation is linear for continuous, hold for discrete", () => { + expect( + getAnimationParamDefaultInterpolation({ + param: { + key: "n", + label: "N", + type: "number", + default: 0, + min: 0, + step: 1, + }, + }), + ).toBe("linear"); + expect( + getAnimationParamDefaultInterpolation({ + param: { key: "c", label: "C", type: "color", default: "#fff" }, + }), + ).toBe("linear"); + expect( + getAnimationParamDefaultInterpolation({ + param: { key: "b", label: "B", type: "boolean", default: false }, + }), + ).toBe("hold"); + expect( + getAnimationParamDefaultInterpolation({ + param: { + key: "s", + label: "S", + type: "select", + default: "a", + options: [{ value: "a", label: "A" }], + }, + }), + ).toBe("hold"); + }); + + test("getAnimationParamNumericRange returns spec for number params, undefined otherwise", () => { + expect( + getAnimationParamNumericRange({ + param: { + key: "intensity", + label: "Intensity", + type: "number", + default: 0.5, + min: 0, + max: 1, + step: 0.1, + }, + }), + ).toEqual({ min: 0, max: 1, step: 0.1 }); + expect( + getAnimationParamNumericRange({ + param: { key: "c", label: "C", type: "color", default: "#fff" }, + }), + ).toBeUndefined(); + expect( + getAnimationParamNumericRange({ + param: { key: "b", label: "B", type: "boolean", default: false }, + }), + ).toBeUndefined(); + }); +}); diff --git a/apps/web/src/animation/animated-params.ts b/apps/web/src/animation/animated-params.ts new file mode 100644 index 00000000..bc27b15d --- /dev/null +++ b/apps/web/src/animation/animated-params.ts @@ -0,0 +1,83 @@ +import { snapToStep } from "@/utils/math"; +import type { ParamDefinition } from "@/params"; +import type { DynamicAnimationPathValue, NumericSpec } from "./types"; + +export function getAnimationParamValueKind({ + param, +}: { + param: ParamDefinition; +}): "number" | "color" | "discrete" { + if (param.type === "number") { + return "number"; + } + + if (param.type === "color") { + return "color"; + } + + return "discrete"; +} + +export function getAnimationParamDefaultInterpolation({ + param, +}: { + param: ParamDefinition; +}): "linear" | "hold" { + return param.type === "number" || param.type === "color" ? "linear" : "hold"; +} + +export function getAnimationParamNumericRange({ + param, +}: { + param: ParamDefinition; +}): NumericSpec | undefined { + if (param.type !== "number") { + return undefined; + } + + return { + min: param.min, + max: param.max, + step: param.step, + }; +} + +/** + * `coerceAnimationParamValue` accepts `unknown` rather than a narrow + * `DynamicAnimationPathValue` because it doubles as a runtime gate for + * untrusted inputs (persisted state, paste payloads, programmatic updates). + * The caller does not need to pre-validate; null means "this value cannot live + * on this param". + */ +export function coerceAnimationParamValue({ + param, + value, +}: { + param: ParamDefinition; + value: unknown; +}): DynamicAnimationPathValue | null { + if (param.type === "number") { + if (typeof value !== "number" || Number.isNaN(value)) { + return null; + } + + const steppedValue = snapToStep({ value, step: param.step }); + const minValue = param.min; + const maxValue = param.max ?? Number.POSITIVE_INFINITY; + return Math.min(maxValue, Math.max(minValue, steppedValue)); + } + + if (param.type === "color") { + return typeof value === "string" ? value : null; + } + + if (param.type === "boolean") { + return typeof value === "boolean" ? value : null; + } + + if (typeof value !== "string") { + return null; + } + + return param.options.some((option) => option.value === value) ? value : null; +} diff --git a/apps/web/src/animation/effect-param-channel.ts b/apps/web/src/animation/effect-param-channel.ts index c635bb82..e1c442f7 100644 --- a/apps/web/src/animation/effect-param-channel.ts +++ b/apps/web/src/animation/effect-param-channel.ts @@ -1,9 +1,8 @@ -import type { ParamValues } from "@/params"; -import type { Effect } from "@/effects/types"; import type { ElementAnimations, EffectParamPath, } from "@/animation/types"; +import type { ParamValues } from "@/params"; import { removeElementKeyframe } from "./keyframes"; import { resolveAnimationPathValueAtTime } from "./resolve"; @@ -56,23 +55,26 @@ export function parseEffectParamPath({ } export function resolveEffectParamsAtTime({ - effect, + effectId, + params, animations, localTime, }: { - effect: Effect; + effectId: string; + params: ParamValues; animations: ElementAnimations | undefined; localTime: number; }): ParamValues { + const safeLocalTime = Math.max(0, localTime); const resolved: ParamValues = {}; - for (const [paramKey, staticValue] of Object.entries(effect.params)) { - const path = buildEffectParamPath({ effectId: effect.id, paramKey }); + for (const [paramKey, staticValue] of Object.entries(params)) { + const path = buildEffectParamPath({ effectId, paramKey }); resolved[paramKey] = animations?.bindings[path] ? resolveAnimationPathValueAtTime({ animations, propertyPath: path, - localTime, + localTime: safeLocalTime, fallbackValue: staticValue, }) : staticValue; diff --git a/apps/web/src/animation/graphic-param-channel.ts b/apps/web/src/animation/graphic-param-channel.ts index ebf0f940..d0bfdded 100644 --- a/apps/web/src/animation/graphic-param-channel.ts +++ b/apps/web/src/animation/graphic-param-channel.ts @@ -2,11 +2,7 @@ import type { ElementAnimations, GraphicParamPath, } from "@/animation/types"; -import type { ParamValues } from "@/params"; -import { - getGraphicDefinition, - resolveGraphicParams, -} from "@/graphics"; +import type { ParamDefinition, ParamValues } from "@/params"; import { resolveAnimationPathValueAtTime } from "./resolve"; export const GRAPHIC_PARAM_PATH_PREFIX = "params."; @@ -39,33 +35,29 @@ export function parseGraphicParamPath({ } export function resolveGraphicParamsAtTime({ - element, + params, + definitions, + animations, localTime, }: { - element: { - definitionId: string; - params: ParamValues; - animations?: ElementAnimations; - }; + params: ParamValues; + definitions: ParamDefinition[]; + animations?: ElementAnimations; localTime: number; }): ParamValues { - const definition = getGraphicDefinition({ - definitionId: element.definitionId, - }); - const baseParams = resolveGraphicParams(definition, element.params); - const resolved: ParamValues = { ...baseParams }; + const resolved: ParamValues = { ...params }; - for (const param of definition.params) { + for (const param of definitions) { const path = buildGraphicParamPath({ paramKey: param.key }); - if (!element.animations?.bindings[path]) { + if (!animations?.bindings[path]) { continue; } resolved[param.key] = resolveAnimationPathValueAtTime({ - animations: element.animations, + animations, propertyPath: path, localTime: Math.max(0, localTime), - fallbackValue: baseParams[param.key] ?? param.default, + fallbackValue: params[param.key] ?? param.default, }); } diff --git a/apps/web/src/animation/index.ts b/apps/web/src/animation/index.ts index 384f3d17..33e637c3 100644 --- a/apps/web/src/animation/index.ts +++ b/apps/web/src/animation/index.ts @@ -23,24 +23,8 @@ export { export { getElementLocalTime, resolveAnimationPathValueAtTime, - resolveColorAtTime, - resolveNumberAtTime, - resolveOpacityAtTime, - resolveTransformAtTime, } from "./resolve"; -export { - coerceAnimationValueForProperty, - getAnimationPropertyDefinition, - getDefaultInterpolationForProperty, - getElementBaseValueForProperty, - isAnimationPropertyPath, - supportsAnimationProperty, - type AnimationPropertyDefinition, - type NumericSpec, - withElementBaseValueForProperty, -} from "./property-registry"; - export { getElementKeyframes, getKeyframeById, @@ -75,15 +59,6 @@ export { resolveEffectParamsAtTime, } from "./effect-param-channel"; -export { - isAnimationPath, - coerceAnimationValueForParam, - resolveAnimationTarget, - getParamValueKind, - getParamDefaultInterpolation, - type AnimationPathDescriptor, -} from "./target-resolver"; - export { getGroupKeyframesAtTime, hasGroupKeyframeAtTime, @@ -94,3 +69,19 @@ export { type EasingMode, getEasingModeForKind, } from "./binding-values"; + +export { + isAnimationPath, + isAnimationPropertyPath, +} from "./path"; + +export { + coerceAnimationValueForProperty, + getAnimationPropertyDefinition, + getDefaultInterpolationForProperty, + getElementBaseValueForProperty, + supportsAnimationProperty, + type AnimationPropertyDefinition, + type NumericSpec, + withElementBaseValueForProperty, +} from "./property-registry"; diff --git a/apps/web/src/animation/interpolation.ts b/apps/web/src/animation/interpolation.ts index 2169024a..5d5212a6 100644 --- a/apps/web/src/animation/interpolation.ts +++ b/apps/web/src/animation/interpolation.ts @@ -8,7 +8,6 @@ import type { ScalarAnimationKey, ScalarSegmentType, } from "@/animation/types"; -import { clamp } from "@/utils/math"; import { mediaTime } from "@/wasm"; import { getBezierPoint, @@ -16,6 +15,7 @@ import { getDefaultRightHandle, solveBezierProgressForTime, } from "./bezier"; +import { clamp } from "@/utils/math"; function byTimeAscending({ leftTime, diff --git a/apps/web/src/animation/keyframe-query.ts b/apps/web/src/animation/keyframe-query.ts index 231aa5f2..e65db881 100644 --- a/apps/web/src/animation/keyframe-query.ts +++ b/apps/web/src/animation/keyframe-query.ts @@ -13,7 +13,7 @@ import { getChannelValueAtTime, getScalarSegmentInterpolation, } from "./interpolation"; -import { isAnimationPath } from "./target-resolver"; +import { isAnimationPath } from "./path"; function getBindingFallbackValue({ channel, diff --git a/apps/web/src/animation/keyframes.ts b/apps/web/src/animation/keyframes.ts index bdfe7c4c..70f9b5db 100644 --- a/apps/web/src/animation/keyframes.ts +++ b/apps/web/src/animation/keyframes.ts @@ -14,7 +14,6 @@ import type { ScalarCurveKeyframePatch, ScalarSegmentType, } from "@/animation/types"; -import { generateUUID } from "@/utils/id"; import { cloneAnimationBinding, createAnimationBinding, @@ -41,6 +40,7 @@ import { subMediaTime, ZERO_MEDIA_TIME, } from "@/wasm"; +import { generateUUID } from "@/utils/id"; function isNearlySameTime({ leftTime, diff --git a/apps/web/src/animation/path.ts b/apps/web/src/animation/path.ts new file mode 100644 index 00000000..69e5e228 --- /dev/null +++ b/apps/web/src/animation/path.ts @@ -0,0 +1,22 @@ +import type { AnimationPath, AnimationPropertyPath } from "@/animation/types"; +import { ANIMATION_PROPERTY_PATHS } from "./types"; +import { isEffectParamPath } from "./effect-param-channel"; +import { isGraphicParamPath } from "./graphic-param-channel"; + +const ANIMATION_PROPERTY_PATH_SET = new Set(ANIMATION_PROPERTY_PATHS); + +export function isAnimationPropertyPath( + propertyPath: string, +): propertyPath is AnimationPropertyPath { + return ANIMATION_PROPERTY_PATH_SET.has(propertyPath); +} + +export function isAnimationPath( + propertyPath: string, +): propertyPath is AnimationPath { + return ( + isAnimationPropertyPath(propertyPath) || + isGraphicParamPath(propertyPath) || + isEffectParamPath(propertyPath) + ); +} diff --git a/apps/web/src/animation/resolve.ts b/apps/web/src/animation/resolve.ts index 1b51c126..19685cea 100644 --- a/apps/web/src/animation/resolve.ts +++ b/apps/web/src/animation/resolve.ts @@ -1,12 +1,8 @@ import type { - AnimationColorPropertyPath, - AnimationNumericPropertyPath, AnimationPath, - AnimationPropertyPath, AnimationValueForPath, ElementAnimations, } from "@/animation/types"; -import type { Transform } from "@/rendering"; import { type AnimationComponentValue, composeAnimationValue, @@ -37,107 +33,6 @@ export function getElementLocalTime({ return localTime; } -export function resolveTransformAtTime({ - baseTransform, - animations, - localTime, -}: { - baseTransform: Transform; - animations: ElementAnimations | undefined; - localTime: number; -}): Transform { - const safeLocalTime = Math.max(0, localTime); - return { - position: { - x: resolveAnimationPathValueAtTime({ - animations, - propertyPath: "transform.positionX", - localTime: safeLocalTime, - fallbackValue: baseTransform.position.x, - }), - y: resolveAnimationPathValueAtTime({ - animations, - propertyPath: "transform.positionY", - localTime: safeLocalTime, - fallbackValue: baseTransform.position.y, - }), - }, - scaleX: resolveAnimationPathValueAtTime({ - animations, - propertyPath: "transform.scaleX", - localTime: safeLocalTime, - fallbackValue: baseTransform.scaleX, - }), - scaleY: resolveAnimationPathValueAtTime({ - animations, - propertyPath: "transform.scaleY", - localTime: safeLocalTime, - fallbackValue: baseTransform.scaleY, - }), - rotate: resolveAnimationPathValueAtTime({ - animations, - propertyPath: "transform.rotate", - localTime: safeLocalTime, - fallbackValue: baseTransform.rotate, - }), - }; -} - -export function resolveOpacityAtTime({ - baseOpacity, - animations, - localTime, -}: { - baseOpacity: number; - animations: ElementAnimations | undefined; - localTime: number; -}): number { - return resolveAnimationPathValueAtTime({ - animations, - propertyPath: "opacity", - localTime: Math.max(0, localTime), - fallbackValue: baseOpacity, - }); -} - -export function resolveNumberAtTime({ - baseValue, - animations, - propertyPath, - localTime, -}: { - baseValue: number; - animations: ElementAnimations | undefined; - propertyPath: AnimationNumericPropertyPath; - localTime: number; -}): number { - return resolveAnimationPathValueAtTime({ - animations, - propertyPath, - localTime: Math.max(0, localTime), - fallbackValue: baseValue, - }); -} - -export function resolveColorAtTime({ - baseColor, - animations, - propertyPath, - localTime, -}: { - baseColor: string; - animations: ElementAnimations | undefined; - propertyPath: AnimationColorPropertyPath; - localTime: number; -}): string { - return resolveAnimationPathValueAtTime({ - animations, - propertyPath, - localTime: Math.max(0, localTime), - fallbackValue: baseColor, - }); -} - export function resolveAnimationPathValueAtTime({ animations, propertyPath, diff --git a/apps/web/src/animation/types.ts b/apps/web/src/animation/types.ts index e02c8ba8..cfecfada 100644 --- a/apps/web/src/animation/types.ts +++ b/apps/web/src/animation/types.ts @@ -51,7 +51,13 @@ export interface AnimationPropertyValueMap { "background.offsetY": number; "background.cornerRadius": number; } -export type DynamicAnimationPathValue = ParamValues[string]; +export type DynamicAnimationPathValue = number | string | boolean; + +export interface NumericSpec { + min?: number; + max?: number; + step?: number; +} export type AnimationValueForPath = TPath extends AnimationPropertyPath ? AnimationPropertyValueMap[TPath] diff --git a/apps/web/src/animation/values.ts b/apps/web/src/animation/values.ts new file mode 100644 index 00000000..7733ec4c --- /dev/null +++ b/apps/web/src/animation/values.ts @@ -0,0 +1,61 @@ +import type { + AnimationColorPropertyPath, + AnimationNumericPropertyPath, + ElementAnimations, +} from "./types"; +import { resolveAnimationPathValueAtTime } from "./resolve"; + +export function resolveOpacityAtTime({ + baseOpacity, + animations, + localTime, +}: { + baseOpacity: number; + animations: ElementAnimations | undefined; + localTime: number; +}): number { + return resolveAnimationPathValueAtTime({ + animations, + propertyPath: "opacity", + localTime: Math.max(0, localTime), + fallbackValue: baseOpacity, + }); +} + +export function resolveNumberAtTime({ + baseValue, + animations, + propertyPath, + localTime, +}: { + baseValue: number; + animations: ElementAnimations | undefined; + propertyPath: AnimationNumericPropertyPath; + localTime: number; +}): number { + return resolveAnimationPathValueAtTime({ + animations, + propertyPath, + localTime: Math.max(0, localTime), + fallbackValue: baseValue, + }); +} + +export function resolveColorAtTime({ + baseColor, + animations, + propertyPath, + localTime, +}: { + baseColor: string; + animations: ElementAnimations | undefined; + propertyPath: AnimationColorPropertyPath; + localTime: number; +}): string { + return resolveAnimationPathValueAtTime({ + animations, + propertyPath, + localTime: Math.max(0, localTime), + fallbackValue: baseColor, + }); +} diff --git a/apps/web/src/commands/timeline/clipboard/paste-keyframes.ts b/apps/web/src/commands/timeline/clipboard/paste-keyframes.ts index c1127921..8d970f4d 100644 --- a/apps/web/src/commands/timeline/clipboard/paste-keyframes.ts +++ b/apps/web/src/commands/timeline/clipboard/paste-keyframes.ts @@ -1,7 +1,6 @@ import { EditorCore } from "@/core"; import { getKeyframeAtTime, - resolveAnimationTarget, updateScalarKeyframeCurve, upsertPathKeyframe, } from "@/animation"; @@ -9,6 +8,7 @@ import { Command, type CommandResult } from "@/commands/base-command"; import type { KeyframeClipboardItem } from "@/clipboard"; import type { SceneTracks, TimelineElement } from "@/timeline"; import { updateElementInSceneTracks } from "@/timeline"; +import { resolveAnimationTarget } from "@/timeline/animation-targets"; import { generateUUID } from "@/utils/id"; import { addMediaTime, diff --git a/apps/web/src/commands/timeline/element/keyframes/remove-keyframe.ts b/apps/web/src/commands/timeline/element/keyframes/remove-keyframe.ts index ed917406..07abe2be 100644 --- a/apps/web/src/commands/timeline/element/keyframes/remove-keyframe.ts +++ b/apps/web/src/commands/timeline/element/keyframes/remove-keyframe.ts @@ -2,12 +2,12 @@ import { EditorCore } from "@/core"; import { hasKeyframesForPath, removeElementKeyframe, - resolveAnimationTarget, } from "@/animation"; import { Command, type CommandResult } from "@/commands/base-command"; import { updateElementInSceneTracks } from "@/timeline"; import type { AnimationPath, AnimationValue } from "@/animation/types"; import type { SceneTracks, TimelineElement } from "@/timeline"; +import { resolveAnimationTarget } from "@/timeline/animation-targets"; function removeKeyframeAndPersist({ element, diff --git a/apps/web/src/commands/timeline/element/keyframes/retime-keyframe.ts b/apps/web/src/commands/timeline/element/keyframes/retime-keyframe.ts index 9337e02d..94054372 100644 --- a/apps/web/src/commands/timeline/element/keyframes/retime-keyframe.ts +++ b/apps/web/src/commands/timeline/element/keyframes/retime-keyframe.ts @@ -1,5 +1,5 @@ import { EditorCore } from "@/core"; -import { resolveAnimationTarget, retimeElementKeyframe } from "@/animation"; +import { retimeElementKeyframe } from "@/animation"; import { Command, type CommandResult } from "@/commands/base-command"; import { updateElementInSceneTracks } from "@/timeline"; import type { AnimationPath } from "@/animation/types"; @@ -10,6 +10,7 @@ import { minMediaTime, ZERO_MEDIA_TIME, } from "@/wasm"; +import { resolveAnimationTarget } from "@/timeline/animation-targets"; export class RetimeKeyframeCommand extends Command { private savedState: SceneTracks | null = null; diff --git a/apps/web/src/commands/timeline/element/keyframes/update-scalar-keyframe-curve.ts b/apps/web/src/commands/timeline/element/keyframes/update-scalar-keyframe-curve.ts index 4a3db39d..51f10ee9 100644 --- a/apps/web/src/commands/timeline/element/keyframes/update-scalar-keyframe-curve.ts +++ b/apps/web/src/commands/timeline/element/keyframes/update-scalar-keyframe-curve.ts @@ -1,10 +1,10 @@ import { EditorCore } from "@/core"; import { - resolveAnimationTarget, updateScalarKeyframeCurve, } from "@/animation"; import { Command, type CommandResult } from "@/commands/base-command"; import { updateElementInSceneTracks } from "@/timeline"; +import { resolveAnimationTarget } from "@/timeline/animation-targets"; import type { AnimationPath, ScalarCurveKeyframePatch, diff --git a/apps/web/src/commands/timeline/element/keyframes/upsert-effect-param-keyframe.ts b/apps/web/src/commands/timeline/element/keyframes/upsert-effect-param-keyframe.ts index 91ddb23d..6eae0847 100644 --- a/apps/web/src/commands/timeline/element/keyframes/upsert-effect-param-keyframe.ts +++ b/apps/web/src/commands/timeline/element/keyframes/upsert-effect-param-keyframe.ts @@ -2,11 +2,11 @@ import { EditorCore } from "@/core"; import { Command, type CommandResult } from "@/commands/base-command"; import { buildEffectParamPath, - resolveAnimationTarget, upsertPathKeyframe, } from "@/animation"; import { updateElementInSceneTracks } from "@/timeline"; import { isVisualElement } from "@/timeline/element-utils"; +import { resolveAnimationTarget } from "@/timeline/animation-targets"; import type { AnimationInterpolation } from "@/animation/types"; import type { SceneTracks } from "@/timeline"; import { diff --git a/apps/web/src/commands/timeline/element/keyframes/upsert-keyframe.ts b/apps/web/src/commands/timeline/element/keyframes/upsert-keyframe.ts index 66f0c434..e1b78f33 100644 --- a/apps/web/src/commands/timeline/element/keyframes/upsert-keyframe.ts +++ b/apps/web/src/commands/timeline/element/keyframes/upsert-keyframe.ts @@ -1,8 +1,9 @@ import { EditorCore } from "@/core"; import { Command, type CommandResult } from "@/commands/base-command"; -import { resolveAnimationTarget, upsertPathKeyframe } from "@/animation"; +import { upsertPathKeyframe } from "@/animation"; import { updateElementInSceneTracks } from "@/timeline"; import type { SceneTracks } from "@/timeline"; +import { resolveAnimationTarget } from "@/timeline/animation-targets"; import type { AnimationPath, AnimationInterpolation, diff --git a/apps/web/src/components/editor/panels/properties/hooks/use-keyframed-param-property.ts b/apps/web/src/components/editor/panels/properties/hooks/use-keyframed-param-property.ts index e679f5c3..4f4b9cf8 100644 --- a/apps/web/src/components/editor/panels/properties/hooks/use-keyframed-param-property.ts +++ b/apps/web/src/components/editor/panels/properties/hooks/use-keyframed-param-property.ts @@ -3,16 +3,18 @@ import { useEditor } from "@/editor/use-editor"; import { buildGraphicParamPath, - coerceAnimationValueForParam, getKeyframeAtTime, - getParamDefaultInterpolation, - getParamValueKind, hasKeyframesForPath, upsertPathKeyframe, } from "@/animation"; import type { ElementAnimations, } from "@/animation/types"; +import { + coerceAnimationParamValue, + getAnimationParamDefaultInterpolation, + getAnimationParamValueKind, +} from "@/animation/animated-params"; import type { ParamDefinition } from "@/params"; import type { TimelineElement } from "@/timeline"; import type { MediaTime } from "@/wasm"; @@ -80,12 +82,12 @@ export function useKeyframedParamProperty({ propertyPath, time: localTime, value, - kind: getParamValueKind({ param }), - defaultInterpolation: getParamDefaultInterpolation({ + kind: getAnimationParamValueKind({ param }), + defaultInterpolation: getAnimationParamDefaultInterpolation({ param, }), coerceValue: ({ value: nextValue }) => - coerceAnimationValueForParam({ + coerceAnimationParamValue({ param, value: nextValue, }), diff --git a/apps/web/src/core/managers/playback-manager.ts b/apps/web/src/core/managers/playback-manager.ts index 8275344f..9b9d26bc 100644 --- a/apps/web/src/core/managers/playback-manager.ts +++ b/apps/web/src/core/managers/playback-manager.ts @@ -75,7 +75,7 @@ export class PlaybackManager { this.playbackStartTime = this.currentTime; } this.notify(); - this.dispatchSeekEvent(this.currentTime); + this.notifySeek(this.currentTime); } setVolume({ volume }: { volume: number }): void { @@ -227,9 +227,9 @@ export class PlaybackManager { this.pause(); this.currentTime = maxTime; this.notify(); - this.notifySeek(maxTime); - this.dispatchSeekEvent(maxTime); - return; + this.notifySeek(maxTime); + this.dispatchSeekEvent(maxTime); + return; } this.currentTime = newTime; @@ -247,23 +247,11 @@ export class PlaybackManager { if (typeof window === "undefined") { return; } - - window.dispatchEvent( - new CustomEvent("playback-seek", { - detail: { time }, - }), - ); } private dispatchUpdateEvent(time: MediaTime): void { if (typeof window === "undefined") { return; } - - window.dispatchEvent( - new CustomEvent("playback-update", { - detail: { time }, - }), - ); } } diff --git a/apps/web/src/core/managers/timeline-manager.ts b/apps/web/src/core/managers/timeline-manager.ts index 3c6b7599..8fbfdb08 100644 --- a/apps/web/src/core/managers/timeline-manager.ts +++ b/apps/web/src/core/managers/timeline-manager.ts @@ -25,9 +25,9 @@ import type { } from "@/animation/types"; import { getElementLocalTime, - resolveAnimationTarget, resolveAnimationPathValueAtTime, } from "@/animation"; +import { resolveAnimationTarget } from "@/timeline/animation-targets"; import { BatchCommand } from "@/commands"; import { AddTrackCommand, diff --git a/apps/web/src/diagnostics/render-perf.ts b/apps/web/src/diagnostics/render-perf.ts new file mode 100644 index 00000000..7b0b7fd9 --- /dev/null +++ b/apps/web/src/diagnostics/render-perf.ts @@ -0,0 +1,182 @@ +/** + * Lightweight rolling perf instrumentation for the render pipeline. + * + * Toggle at runtime from the devtools console: + * window.__renderPerf = true + * + * Every FLUSH_EVERY frames the aggregator dumps: + * - per-span timing summary (count / mean / p50 / p95 / max, in ms) + * - per-counter totals (uploads, canvas allocations by kind, etc.) + * + * Zero overhead when disabled: `isRenderPerfEnabled()` short-circuits before + * any recording happens, so call sites only pay for a global read. + */ + +type SpanSample = number; + +type SpanStats = { + samples: SpanSample[]; +}; + +type CounterStats = { + total: number; + frames: number; +}; + +const FLUSH_EVERY = 60; + +const spans = new Map(); +const counters = new Map(); +const pendingCountersThisFrame = new Map(); + +let framesSinceFlush = 0; + +declare global { + interface Window { + __renderPerf?: boolean; + } +} + +export function isRenderPerfEnabled(): boolean { + return typeof window !== "undefined" && window.__renderPerf === true; +} + +export function recordSpan({ + name, + durationMs, +}: { + name: string; + durationMs: number; +}): void { + if (!isRenderPerfEnabled()) return; + let stats = spans.get(name); + if (!stats) { + stats = { samples: [] }; + spans.set(name, stats); + } + stats.samples.push(durationMs); +} + +export async function measureSpanAsync({ + name, + fn, +}: { + name: string; + fn: () => Promise; +}): Promise { + if (!isRenderPerfEnabled()) return fn(); + const start = performance.now(); + try { + return await fn(); + } finally { + recordSpan({ name, durationMs: performance.now() - start }); + } +} + +export function measureSpanSync({ + name, + fn, +}: { + name: string; + fn: () => T; +}): T { + if (!isRenderPerfEnabled()) return fn(); + const start = performance.now(); + try { + return fn(); + } finally { + recordSpan({ name, durationMs: performance.now() - start }); + } +} + +export function incrementCounter({ + name, + by = 1, +}: { + name: string; + by?: number; +}): void { + if (!isRenderPerfEnabled()) return; + pendingCountersThisFrame.set( + name, + (pendingCountersThisFrame.get(name) ?? 0) + by, + ); +} + +/** + * Pulls sub-span timings recorded inside the wasm `renderFrame` call and + * feeds them into the aggregator as ordinary spans. + */ +export function recordWasmFrameProfile( + entries: Array<{ name: string; durationMs: number }>, +): void { + if (!isRenderPerfEnabled()) return; + for (const entry of entries) { + recordSpan({ name: entry.name, durationMs: entry.durationMs }); + } +} + +/** + * Called once per frame by the top of the render pipeline. Rolls the + * pending-frame counters into the aggregate and triggers a flush on cadence. + */ +export function onRenderPerfFrameComplete(): void { + if (!isRenderPerfEnabled()) return; + for (const [name, count] of pendingCountersThisFrame) { + let stats = counters.get(name); + if (!stats) { + stats = { total: 0, frames: 0 }; + counters.set(name, stats); + } + stats.total += count; + stats.frames += 1; + } + pendingCountersThisFrame.clear(); + + framesSinceFlush += 1; + if (framesSinceFlush >= FLUSH_EVERY) { + flush(); + } +} + +function flush(): void { + const spanRows: Array> = []; + for (const [name, stats] of spans) { + if (stats.samples.length === 0) continue; + const sorted = [...stats.samples].sort((a, b) => a - b); + const p = (q: number) => + sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * q))]; + const sum = sorted.reduce((acc, v) => acc + v, 0); + spanRows.push({ + span: name, + count: sorted.length, + meanMs: +(sum / sorted.length).toFixed(2), + p50Ms: +p(0.5).toFixed(2), + p95Ms: +p(0.95).toFixed(2), + maxMs: +sorted[sorted.length - 1].toFixed(2), + }); + } + spanRows.sort((a, b) => Number(b.meanMs) - Number(a.meanMs)); + + const counterRows: Array> = []; + for (const [name, stats] of counters) { + counterRows.push({ + counter: name, + perFrame: +(stats.total / Math.max(1, stats.frames)).toFixed(2), + total: stats.total, + frames: stats.frames, + }); + } + counterRows.sort((a, b) => Number(b.perFrame) - Number(a.perFrame)); + + console.groupCollapsed( + `[render-perf] summary over ${framesSinceFlush} frames`, + ); + if (spanRows.length > 0) console.table(spanRows); + if (counterRows.length > 0) console.table(counterRows); + console.groupEnd(); + + spans.clear(); + counters.clear(); + framesSinceFlush = 0; +} diff --git a/apps/web/src/graphics/components/graphic-tab.tsx b/apps/web/src/graphics/components/graphic-tab.tsx index 549ef4b6..13cd7b84 100644 --- a/apps/web/src/graphics/components/graphic-tab.tsx +++ b/apps/web/src/graphics/components/graphic-tab.tsx @@ -6,10 +6,9 @@ import { useKeyframedParamProperty, type KeyframedParamPropertyResult, } from "@/components/editor/panels/properties/hooks/use-keyframed-param-property"; -import { resolveGraphicParamsAtTime } from "@/animation"; import type { ParamDefinition, ParamValues } from "@/params"; import type { GraphicElement } from "@/timeline"; -import { graphicsRegistry, registerDefaultGraphics } from "@/graphics"; +import { graphicsRegistry, registerDefaultGraphics, resolveGraphicElementParamsAtTime } from "@/graphics"; import { useElementPreview } from "@/timeline/hooks/use-element-preview"; import { useEditor } from "@/editor/use-editor"; import { @@ -49,7 +48,7 @@ export function GraphicTab({ }); const liveElement = renderElement as GraphicElement; - const resolvedParams = resolveGraphicParamsAtTime({ + const resolvedParams = resolveGraphicElementParamsAtTime({ element: liveElement, localTime, }); @@ -104,7 +103,7 @@ function StrokeSection({ }); const liveElement = renderElement as GraphicElement; - const resolvedParams = resolveGraphicParamsAtTime({ + const resolvedParams = resolveGraphicElementParamsAtTime({ element: liveElement, localTime, }); diff --git a/apps/web/src/graphics/index.ts b/apps/web/src/graphics/index.ts index c69a4ab0..3f277d40 100644 --- a/apps/web/src/graphics/index.ts +++ b/apps/web/src/graphics/index.ts @@ -1,3 +1,5 @@ +import { resolveGraphicParamsAtTime } from "@/animation"; +import type { ElementAnimations } from "@/animation/types"; import { buildDefaultParamValues } from "@/params/registry"; import type { ParamValues } from "@/params"; import { graphicsRegistry } from "./registry"; @@ -68,6 +70,28 @@ export function resolveGraphicParams( }; } +export function resolveGraphicElementParamsAtTime({ + element, + localTime, +}: { + element: { + definitionId: string; + params: ParamValues; + animations?: ElementAnimations; + }; + localTime: number; +}): ParamValues { + const definition = getGraphicDefinition({ + definitionId: element.definitionId, + }); + return resolveGraphicParamsAtTime({ + params: resolveGraphicParams(definition, element.params), + definitions: definition.params, + animations: element.animations, + localTime, + }); +} + export function buildGraphicPreviewUrl({ definitionId, params, diff --git a/apps/web/src/media/mediabunny.ts b/apps/web/src/media/mediabunny.ts index 47ffaeb1..3c949b35 100644 --- a/apps/web/src/media/mediabunny.ts +++ b/apps/web/src/media/mediabunny.ts @@ -1,43 +1,81 @@ -import { Input, ALL_FORMATS, BlobSource } from "mediabunny"; +import { + Input, + ALL_FORMATS, + BlobSource, + VideoSampleSink, + type VideoCodec, +} from "mediabunny"; import { createTimelineAudioBuffer } from "@/media/audio"; import type { SceneTracks } from "@/timeline"; import type { MediaAsset } from "@/media/types"; import { TICKS_PER_SECOND } from "@/wasm"; +import { renderThumbnailDataUrl } from "./thumbnail"; -export async function getVideoInfo({ - videoFile, -}: { - videoFile: File; -}): Promise<{ +export type VideoFileData = { duration: number; width: number; height: number; fps: number; hasAudio: boolean; -}> { + codec: VideoCodec | null; + canDecode: boolean; + thumbnailUrl: string | null; +}; + +export async function readVideoFile({ + file, +}: { + file: File; +}): Promise { const input = new Input({ - source: new BlobSource(videoFile), + source: new BlobSource(file), formats: ALL_FORMATS, }); - const duration = await input.computeDuration(); - const videoTrack = await input.getPrimaryVideoTrack(); + try { + const duration = await input.computeDuration(); + const videoTrack = await input.getPrimaryVideoTrack(); - if (!videoTrack) { - throw new Error("No video track found in the file"); + if (!videoTrack) { + throw new Error("No video track found in the file"); + } + + const canDecode = await videoTrack.canDecode(); + const packetStats = await videoTrack.computePacketStats(100); + const audioTrack = await input.getPrimaryAudioTrack(); + + let thumbnailUrl: string | null = null; + if (canDecode) { + const sink = new VideoSampleSink(videoTrack); + const frame = await sink.getSample(1); + if (frame) { + try { + thumbnailUrl = renderThumbnailDataUrl({ + width: videoTrack.displayWidth, + height: videoTrack.displayHeight, + draw: ({ context, width, height }) => { + frame.draw(context, 0, 0, width, height); + }, + }); + } finally { + frame.close(); + } + } + } + + return { + duration, + width: videoTrack.displayWidth, + height: videoTrack.displayHeight, + fps: packetStats.averagePacketRate, + hasAudio: audioTrack !== null, + codec: videoTrack.codec, + canDecode, + thumbnailUrl, + }; + } finally { + input.dispose(); } - - const packetStats = await videoTrack.computePacketStats(100); - const fps = packetStats.averagePacketRate; - const audioTrack = await input.getPrimaryAudioTrack(); - - return { - duration, - width: videoTrack.displayWidth, - height: videoTrack.displayHeight, - fps, - hasAudio: audioTrack !== null, - }; } const SAMPLE_RATE = 44100; diff --git a/apps/web/src/media/processing.ts b/apps/web/src/media/processing.ts index d8d483f8..34ccadbc 100644 --- a/apps/web/src/media/processing.ts +++ b/apps/web/src/media/processing.ts @@ -1,15 +1,25 @@ -import { Input, ALL_FORMATS, BlobSource, VideoSampleSink } from "mediabunny"; import { toast } from "sonner"; import { getMediaTypeFromFile } from "@/media/media-utils"; import { formatStorageBytes } from "@/services/storage/quota"; import { storageService } from "@/services/storage/service"; import type { MediaAsset } from "@/media/types"; -import { getVideoInfo } from "./mediabunny"; +import { readVideoFile } from "./mediabunny"; +import type { VideoFileData } from "./mediabunny"; +import { renderThumbnailDataUrl } from "./thumbnail"; export interface ProcessedMediaAsset extends Omit {} -const THUMBNAIL_MAX_WIDTH = 1280; -const THUMBNAIL_MAX_HEIGHT = 720; +const getUnsupportedVideoDescription = ({ + codec, +}: { + codec: VideoFileData["codec"]; +}): string => { + const codecLabel = codec ? codec.toUpperCase() : "this video codec"; + + return codec === "hevc" + ? `${codecLabel} cannot be decoded in this browser, so this clip may not preview correctly. Convert it to H.264 MP4 or try importing it in Safari.` + : `${codecLabel} cannot be decoded in this browser, so this clip may not preview correctly. Convert it to H.264 MP4 and reimport it.`; +}; const getStorageLimitDescription = ({ fileSize, @@ -29,103 +39,6 @@ const getStorageLimitDescription = ({ })} is safely available in browser storage.`; }; -const getThumbnailSize = ({ - width, - height, -}: { - width: number; - height: number; -}): { width: number; height: number } => { - const aspectRatio = width / height; - let targetWidth = width; - let targetHeight = height; - - if (targetWidth > THUMBNAIL_MAX_WIDTH) { - targetWidth = THUMBNAIL_MAX_WIDTH; - targetHeight = Math.round(targetWidth / aspectRatio); - } - if (targetHeight > THUMBNAIL_MAX_HEIGHT) { - targetHeight = THUMBNAIL_MAX_HEIGHT; - targetWidth = Math.round(targetHeight * aspectRatio); - } - - return { width: targetWidth, height: targetHeight }; -}; - -const renderToThumbnailDataUrl = ({ - width, - height, - draw, -}: { - width: number; - height: number; - draw: ({ - context, - width, - height, - }: { - context: CanvasRenderingContext2D; - width: number; - height: number; - }) => void; -}): string => { - const size = getThumbnailSize({ width, height }); - const canvas = document.createElement("canvas"); - canvas.width = size.width; - canvas.height = size.height; - const context = canvas.getContext("2d"); - - if (!context) { - throw new Error("Could not get canvas context"); - } - - draw({ context, width: size.width, height: size.height }); - return canvas.toDataURL("image/jpeg", 0.8); -}; - -async function generateThumbnail({ - videoFile, - timeInSeconds, -}: { - videoFile: File; - timeInSeconds: number; -}): Promise { - const input = new Input({ - source: new BlobSource(videoFile), - formats: ALL_FORMATS, - }); - - const videoTrack = await input.getPrimaryVideoTrack(); - if (!videoTrack) { - throw new Error("No video track found in the file"); - } - - const canDecode = await videoTrack.canDecode(); - if (!canDecode) { - throw new Error("Video codec not supported for decoding"); - } - - const sink = new VideoSampleSink(videoTrack); - - const frame = await sink.getSample(timeInSeconds); - - if (!frame) { - throw new Error("Could not get frame at specified time"); - } - - try { - return renderToThumbnailDataUrl({ - width: videoTrack.displayWidth, - height: videoTrack.displayHeight, - draw: ({ context, width, height }) => { - frame.draw(context, 0, 0, width, height); - }, - }); - } finally { - frame.close(); - } -} - async function generateImageThumbnail({ imageFile, }: { @@ -137,7 +50,7 @@ async function generateImageThumbnail({ image.addEventListener("load", () => { try { - const thumbnailUrl = renderToThumbnailDataUrl({ + const thumbnailUrl = renderThumbnailDataUrl({ width: image.naturalWidth, height: image.naturalHeight, draw: ({ context, width, height }) => { @@ -220,24 +133,34 @@ export async function processMediaAssets({ height = result.height; } else if (fileType === "video") { try { - const videoInfo = await getVideoInfo({ videoFile: file }); - duration = videoInfo.duration; - width = videoInfo.width; - height = videoInfo.height; - fps = Number.isFinite(videoInfo.fps) - ? Math.round(videoInfo.fps) + const videoData = await readVideoFile({ file }); + duration = videoData.duration; + width = videoData.width; + height = videoData.height; + fps = Number.isFinite(videoData.fps) + ? Math.round(videoData.fps) : undefined; - hasAudio = videoInfo.hasAudio; + hasAudio = videoData.hasAudio; + thumbnailUrl = videoData.thumbnailUrl ?? undefined; - thumbnailUrl = await generateThumbnail({ - videoFile: file, - timeInSeconds: 1, - }); + if (!videoData.canDecode) { + toast.error(`Can't preview ${file.name}`, { + description: getUnsupportedVideoDescription({ + codec: videoData.codec, + }), + }); + } } catch (error) { - console.warn("Video processing failed", error); + const message = + error instanceof Error + ? error.message + : "Could not process video"; + + toast.error(`Couldn't process ${file.name}`, { + description: message, + }); } } else if (fileType === "audio") { - // For audio, we don't set width/height/fps (they'll be undefined) duration = await getMediaDuration({ file }); } @@ -264,7 +187,7 @@ export async function processMediaAssets({ } catch (error) { console.error("Error processing file:", file.name, error); toast.error(`Failed to process ${file.name}`); - URL.revokeObjectURL(url); // Clean up on error + URL.revokeObjectURL(url); } } diff --git a/apps/web/src/media/thumbnail.ts b/apps/web/src/media/thumbnail.ts new file mode 100644 index 00000000..e71077d9 --- /dev/null +++ b/apps/web/src/media/thumbnail.ts @@ -0,0 +1,56 @@ +const THUMBNAIL_MAX_WIDTH = 1280; +const THUMBNAIL_MAX_HEIGHT = 720; + +export function thumbnailSize({ + width, + height, +}: { + width: number; + height: number; +}): { width: number; height: number } { + const aspectRatio = width / height; + let targetWidth = width; + let targetHeight = height; + + if (targetWidth > THUMBNAIL_MAX_WIDTH) { + targetWidth = THUMBNAIL_MAX_WIDTH; + targetHeight = Math.round(targetWidth / aspectRatio); + } + if (targetHeight > THUMBNAIL_MAX_HEIGHT) { + targetHeight = THUMBNAIL_MAX_HEIGHT; + targetWidth = Math.round(targetHeight * aspectRatio); + } + + return { width: targetWidth, height: targetHeight }; +} + +export function renderThumbnailDataUrl({ + width, + height, + draw, +}: { + width: number; + height: number; + draw: ({ + context, + width, + height, + }: { + context: CanvasRenderingContext2D; + width: number; + height: number; + }) => void; +}): string { + const size = thumbnailSize({ width, height }); + const canvas = document.createElement("canvas"); + canvas.width = size.width; + canvas.height = size.height; + const context = canvas.getContext("2d"); + + if (!context) { + throw new Error("Could not get canvas context"); + } + + draw({ context, width: size.width, height: size.height }); + return canvas.toDataURL("image/jpeg", 0.8); +} diff --git a/apps/web/src/preview/components/index.tsx b/apps/web/src/preview/components/index.tsx index d3cb2a49..e8f5c572 100644 --- a/apps/web/src/preview/components/index.tsx +++ b/apps/web/src/preview/components/index.tsx @@ -132,7 +132,7 @@ function PreviewCanvas({ isVisible: boolean; }) => void; }) { - const canvasRef = useRef(null); + const canvasMountRef = useRef(null); const viewportRef = useRef(null); const lastFrameRef = useRef(-1); const lastSceneRef = useRef(null); @@ -158,35 +158,51 @@ function PreviewCanvas({ }); }, [nativeWidth, nativeHeight, activeProject.settings.fps]); - const render = useCallback(() => { - if (canvasRef.current && renderTree && !renderingRef.current) { - const renderTime = Math.min( - editor.playback.getCurrentTime(), - editor.timeline.getLastFrameTime(), - ); - const ticksPerFrame = Math.round( - (TICKS_PER_SECOND * renderer.fps.denominator) / renderer.fps.numerator, - ); - const frame = Math.floor(renderTime / ticksPerFrame); - - if ( - frame !== lastFrameRef.current || - renderTree !== lastSceneRef.current - ) { - renderingRef.current = true; - lastSceneRef.current = renderTree; - lastFrameRef.current = frame; - renderer - .renderToCanvas({ - node: renderTree, - time: renderTime, - targetCanvas: canvasRef.current, - }) - .then(() => { - renderingRef.current = false; - }); + // Mount the compositor's output canvas directly into the preview. wgpu + // renders straight into this element, so there is no intermediate copy — + // the container div owns positioning/styling, the canvas itself fills it. + useEffect(() => { + const mount = canvasMountRef.current; + if (!mount) return; + const outputCanvas = renderer.getOutputCanvas(); + outputCanvas.style.display = "block"; + outputCanvas.style.width = "100%"; + outputCanvas.style.height = "100%"; + mount.appendChild(outputCanvas); + return () => { + if (outputCanvas.parentElement === mount) { + mount.removeChild(outputCanvas); } + }; + }, [renderer]); + + const render = useCallback(() => { + if (!renderTree || renderingRef.current) return; + + const renderTime = Math.min( + editor.playback.getCurrentTime(), + editor.timeline.getLastFrameTime(), + ); + const ticksPerFrame = Math.round( + (TICKS_PER_SECOND * renderer.fps.denominator) / renderer.fps.numerator, + ); + const frame = Math.floor(renderTime / ticksPerFrame); + + if ( + frame === lastFrameRef.current && + renderTree === lastSceneRef.current + ) { + return; } + + renderingRef.current = true; + lastSceneRef.current = renderTree; + lastFrameRef.current = frame; + renderer + .render({ node: renderTree, time: renderTime }) + .then(() => { + renderingRef.current = false; + }); }, [renderer, renderTree, editor.playback, editor.timeline]); useRafLoop(render); @@ -286,22 +302,20 @@ function PreviewCanvas({ ref={viewportRef} className="relative flex size-full min-h-0 min-w-0 items-center justify-center overflow-hidden" > - +
void; }) { - const activeGuide = usePreviewStore((state) => state.activeGuide); - const activeGuideDefinition = getGuideById(activeGuide); - return (
@@ -93,7 +88,11 @@ function TimecodeDisplay() { /> / - {formatTimecode({ time: totalDuration, format: "HH:MM:SS:FF", rate: fps })} + {formatTimecode({ + time: totalDuration, + format: "HH:MM:SS:FF", + rate: fps, + })}
); diff --git a/apps/web/src/preview/controllers/transform-handle-controller.ts b/apps/web/src/preview/controllers/transform-handle-controller.ts index e9d251e3..67f677f6 100644 --- a/apps/web/src/preview/controllers/transform-handle-controller.ts +++ b/apps/web/src/preview/controllers/transform-handle-controller.ts @@ -20,11 +20,11 @@ import { isVisualElement } from "@/timeline/element-utils"; import { getElementLocalTime, hasKeyframesForPath, - resolveTransformAtTime, setChannel, } from "@/animation"; import type { ElementAnimations } from "@/animation/types"; import type { Transform } from "@/rendering"; +import { resolveTransformAtTime } from "@/rendering/animation-values"; import type { ElementRef, SceneTracks, diff --git a/apps/web/src/preview/element-bounds.ts b/apps/web/src/preview/element-bounds.ts index ac556181..0afc6e4e 100644 --- a/apps/web/src/preview/element-bounds.ts +++ b/apps/web/src/preview/element-bounds.ts @@ -1,308 +1,308 @@ -import type { SceneTracks, TimelineElement } from "@/timeline"; -import type { MediaAsset } from "@/media/types"; -import { STICKER_INTRINSIC_SIZE_FALLBACK } from "@/stickers/intrinsic-size"; -import { DEFAULT_GRAPHIC_SOURCE_SIZE } from "@/graphics"; -import { measureTextElement } from "@/text/measure-element"; -import { - getElementLocalTime, - resolveTransformAtTime, -} from "@/animation"; - -export interface ElementBounds { - cx: number; - cy: number; - width: number; - height: number; - rotation: number; -} - -export interface ElementWithBounds { - trackId: string; - elementId: string; - element: TimelineElement; - bounds: ElementBounds; -} - -function getVisualElementBounds({ - canvasWidth, - canvasHeight, - sourceWidth, - sourceHeight, - transform, -}: { - canvasWidth: number; - canvasHeight: number; - sourceWidth: number; - sourceHeight: number; - transform: { - scaleX: number; - scaleY: number; - position: { x: number; y: number }; - rotate: number; - }; -}): ElementBounds { - const containScale = Math.min( - canvasWidth / sourceWidth, - canvasHeight / sourceHeight, - ); - const scaledWidth = sourceWidth * containScale * transform.scaleX; - const scaledHeight = sourceHeight * containScale * transform.scaleY; - const cx = canvasWidth / 2 + transform.position.x; - const cy = canvasHeight / 2 + transform.position.y; - - return { - cx, - cy, - width: scaledWidth, - height: scaledHeight, - rotation: transform.rotate, - }; -} - -function getTransformedRectBounds({ - canvasWidth, - canvasHeight, - rect, - transform, -}: { - canvasWidth: number; - canvasHeight: number; - rect: { left: number; top: number; width: number; height: number }; - transform: { - scaleX: number; - scaleY: number; - position: { x: number; y: number }; - rotate: number; - }; -}): ElementBounds { - const localCenterX = rect.left + rect.width / 2; - const localCenterY = rect.top + rect.height / 2; - const scaledCenterX = localCenterX * transform.scaleX; - const scaledCenterY = localCenterY * transform.scaleY; - const rotationRad = (transform.rotate * Math.PI) / 180; - const cos = Math.cos(rotationRad); - const sin = Math.sin(rotationRad); - return { - cx: - canvasWidth / 2 + - transform.position.x + - scaledCenterX * cos - - scaledCenterY * sin, - cy: - canvasHeight / 2 + - transform.position.y + - scaledCenterX * sin + - scaledCenterY * cos, - width: rect.width * transform.scaleX, - height: rect.height * transform.scaleY, - rotation: transform.rotate, - }; -} - -/** - * Bounds policy: bounds reflect base content geometry (text glyphs + background, - * sticker/image/video content area) and base transform. Post-effect spill (blur, - * glow) and mask-clipped regions are intentionally excluded — handles manipulate - * the canonical element geometry, not visual effect output. - */ -function getElementBounds({ - element, - canvasSize, - mediaAsset, - localTime, -}: { - element: TimelineElement; - canvasSize: { width: number; height: number }; - mediaAsset?: MediaAsset | null; - localTime: number; -}): ElementBounds | null { - if (element.type === "audio" || element.type === "effect") return null; - if ("hidden" in element && element.hidden) return null; - - const { width: canvasWidth, height: canvasHeight } = canvasSize; - - if (element.type === "video" || element.type === "image") { - const transform = resolveTransformAtTime({ - baseTransform: element.transform, - animations: element.animations, - localTime, - }); - const sourceWidth = mediaAsset?.width ?? canvasWidth; - const sourceHeight = mediaAsset?.height ?? canvasHeight; - return getVisualElementBounds({ - canvasWidth, - canvasHeight, - sourceWidth, - sourceHeight, - transform, - }); - } - - if (element.type === "sticker") { - const transform = resolveTransformAtTime({ - baseTransform: element.transform, - animations: element.animations, - localTime, - }); - return getVisualElementBounds({ - canvasWidth, - canvasHeight, - sourceWidth: element.intrinsicWidth ?? STICKER_INTRINSIC_SIZE_FALLBACK, - sourceHeight: element.intrinsicHeight ?? STICKER_INTRINSIC_SIZE_FALLBACK, - transform, - }); - } - - if (element.type === "graphic") { - const transform = resolveTransformAtTime({ - baseTransform: element.transform, - animations: element.animations, - localTime, - }); - return getVisualElementBounds({ - canvasWidth, - canvasHeight, - sourceWidth: DEFAULT_GRAPHIC_SOURCE_SIZE, - sourceHeight: DEFAULT_GRAPHIC_SOURCE_SIZE, - transform, - }); - } - - if (element.type === "text") { - const transform = resolveTransformAtTime({ - baseTransform: element.transform, - animations: element.animations, - localTime, - }); - - const canvas = document.createElement("canvas"); - const ctx = canvas.getContext("2d"); - if (!ctx) return null; - - const measured = measureTextElement({ - element, - canvasHeight, - localTime, - ctx, - }); - - return getTransformedRectBounds({ - canvasWidth, - canvasHeight, - rect: measured.visualRect, - transform, - }); - } - - return null; -} - -export const ROTATION_HANDLE_OFFSET = 24; - -export type Corner = "top-left" | "top-right" | "bottom-left" | "bottom-right"; -export type Edge = "right" | "left" | "bottom"; - -export function getCornerPosition({ - bounds, - corner, -}: { - bounds: ElementBounds; - corner: Corner; -}): { x: number; y: number } { - const halfW = bounds.width / 2; - const halfH = bounds.height / 2; - const angleRad = (bounds.rotation * Math.PI) / 180; - const cos = Math.cos(angleRad); - const sin = Math.sin(angleRad); - const localX = - corner === "top-left" || corner === "bottom-left" ? -halfW : halfW; - const localY = - corner === "top-left" || corner === "top-right" ? -halfH : halfH; - return { - x: bounds.cx + (localX * cos - localY * sin), - y: bounds.cy + (localX * sin + localY * cos), - }; -} - -export function getEdgeHandlePosition({ - bounds, - edge, -}: { - bounds: ElementBounds; - edge: Edge; -}): { x: number; y: number } { - const halfWidth = bounds.width / 2; - const halfHeight = bounds.height / 2; - const angleRad = (bounds.rotation * Math.PI) / 180; - const cos = Math.cos(angleRad); - const sin = Math.sin(angleRad); - const localX = edge === "right" ? halfWidth : edge === "left" ? -halfWidth : 0; - const localY = edge === "bottom" ? halfHeight : 0; - return { - x: bounds.cx + (localX * cos - localY * sin), - y: bounds.cy + (localX * sin + localY * cos), - }; -} - -export function getVisibleElementsWithBounds({ - tracks, - currentTime, - canvasSize, - mediaAssets, -}: { - tracks: SceneTracks; - currentTime: number; - canvasSize: { width: number; height: number }; - mediaAssets: MediaAsset[]; -}): ElementWithBounds[] { - const mediaMap = new Map(mediaAssets.map((m) => [m.id, m])); - const orderedTracks = [ - ...tracks.overlay.filter((track) => !("hidden" in track && track.hidden)), - ...(!tracks.main.hidden ? [tracks.main] : []), - ].reverse(); - - const result: ElementWithBounds[] = []; - - for (const track of orderedTracks) { - const elements = track.elements - .filter((element) => !("hidden" in element && element.hidden)) - .filter( - (element) => - currentTime >= element.startTime && - currentTime < element.startTime + element.duration, - ) - .slice() - .sort((a, b) => { - if (a.startTime !== b.startTime) return a.startTime - b.startTime; - return a.id.localeCompare(b.id); - }); - - for (const element of elements) { - const localTime = getElementLocalTime({ - timelineTime: currentTime, - elementStartTime: element.startTime, - elementDuration: element.duration, - }); - const mediaAsset = - element.type === "video" || element.type === "image" - ? mediaMap.get(element.mediaId) - : undefined; - const bounds = getElementBounds({ - element, - canvasSize, - mediaAsset, - localTime, - }); - if (bounds) { - result.push({ - trackId: track.id, - elementId: element.id, - element, - bounds, - }); - } - } - } - - return result; -} +import type { SceneTracks, TimelineElement } from "@/timeline"; +import type { MediaAsset } from "@/media/types"; +import { STICKER_INTRINSIC_SIZE_FALLBACK } from "@/stickers/intrinsic-size"; +import { DEFAULT_GRAPHIC_SOURCE_SIZE } from "@/graphics"; +import { measureTextElement } from "@/text/measure-element"; +import { + getElementLocalTime, +} from "@/animation"; +import { resolveTransformAtTime } from "@/rendering/animation-values"; + +export interface ElementBounds { + cx: number; + cy: number; + width: number; + height: number; + rotation: number; +} + +export interface ElementWithBounds { + trackId: string; + elementId: string; + element: TimelineElement; + bounds: ElementBounds; +} + +function getVisualElementBounds({ + canvasWidth, + canvasHeight, + sourceWidth, + sourceHeight, + transform, +}: { + canvasWidth: number; + canvasHeight: number; + sourceWidth: number; + sourceHeight: number; + transform: { + scaleX: number; + scaleY: number; + position: { x: number; y: number }; + rotate: number; + }; +}): ElementBounds { + const containScale = Math.min( + canvasWidth / sourceWidth, + canvasHeight / sourceHeight, + ); + const scaledWidth = sourceWidth * containScale * transform.scaleX; + const scaledHeight = sourceHeight * containScale * transform.scaleY; + const cx = canvasWidth / 2 + transform.position.x; + const cy = canvasHeight / 2 + transform.position.y; + + return { + cx, + cy, + width: scaledWidth, + height: scaledHeight, + rotation: transform.rotate, + }; +} + +function getTransformedRectBounds({ + canvasWidth, + canvasHeight, + rect, + transform, +}: { + canvasWidth: number; + canvasHeight: number; + rect: { left: number; top: number; width: number; height: number }; + transform: { + scaleX: number; + scaleY: number; + position: { x: number; y: number }; + rotate: number; + }; +}): ElementBounds { + const localCenterX = rect.left + rect.width / 2; + const localCenterY = rect.top + rect.height / 2; + const scaledCenterX = localCenterX * transform.scaleX; + const scaledCenterY = localCenterY * transform.scaleY; + const rotationRad = (transform.rotate * Math.PI) / 180; + const cos = Math.cos(rotationRad); + const sin = Math.sin(rotationRad); + return { + cx: + canvasWidth / 2 + + transform.position.x + + scaledCenterX * cos - + scaledCenterY * sin, + cy: + canvasHeight / 2 + + transform.position.y + + scaledCenterX * sin + + scaledCenterY * cos, + width: rect.width * transform.scaleX, + height: rect.height * transform.scaleY, + rotation: transform.rotate, + }; +} + +/** + * Bounds policy: bounds reflect base content geometry (text glyphs + background, + * sticker/image/video content area) and base transform. Post-effect spill (blur, + * glow) and mask-clipped regions are intentionally excluded — handles manipulate + * the canonical element geometry, not visual effect output. + */ +function getElementBounds({ + element, + canvasSize, + mediaAsset, + localTime, +}: { + element: TimelineElement; + canvasSize: { width: number; height: number }; + mediaAsset?: MediaAsset | null; + localTime: number; +}): ElementBounds | null { + if (element.type === "audio" || element.type === "effect") return null; + if ("hidden" in element && element.hidden) return null; + + const { width: canvasWidth, height: canvasHeight } = canvasSize; + + if (element.type === "video" || element.type === "image") { + const transform = resolveTransformAtTime({ + baseTransform: element.transform, + animations: element.animations, + localTime, + }); + const sourceWidth = mediaAsset?.width ?? canvasWidth; + const sourceHeight = mediaAsset?.height ?? canvasHeight; + return getVisualElementBounds({ + canvasWidth, + canvasHeight, + sourceWidth, + sourceHeight, + transform, + }); + } + + if (element.type === "sticker") { + const transform = resolveTransformAtTime({ + baseTransform: element.transform, + animations: element.animations, + localTime, + }); + return getVisualElementBounds({ + canvasWidth, + canvasHeight, + sourceWidth: element.intrinsicWidth ?? STICKER_INTRINSIC_SIZE_FALLBACK, + sourceHeight: element.intrinsicHeight ?? STICKER_INTRINSIC_SIZE_FALLBACK, + transform, + }); + } + + if (element.type === "graphic") { + const transform = resolveTransformAtTime({ + baseTransform: element.transform, + animations: element.animations, + localTime, + }); + return getVisualElementBounds({ + canvasWidth, + canvasHeight, + sourceWidth: DEFAULT_GRAPHIC_SOURCE_SIZE, + sourceHeight: DEFAULT_GRAPHIC_SOURCE_SIZE, + transform, + }); + } + + if (element.type === "text") { + const transform = resolveTransformAtTime({ + baseTransform: element.transform, + animations: element.animations, + localTime, + }); + + const canvas = document.createElement("canvas"); + const ctx = canvas.getContext("2d"); + if (!ctx) return null; + + const measured = measureTextElement({ + element, + canvasHeight, + localTime, + ctx, + }); + + return getTransformedRectBounds({ + canvasWidth, + canvasHeight, + rect: measured.visualRect, + transform, + }); + } + + return null; +} + +export const ROTATION_HANDLE_OFFSET = 24; + +export type Corner = "top-left" | "top-right" | "bottom-left" | "bottom-right"; +export type Edge = "right" | "left" | "bottom"; + +export function getCornerPosition({ + bounds, + corner, +}: { + bounds: ElementBounds; + corner: Corner; +}): { x: number; y: number } { + const halfW = bounds.width / 2; + const halfH = bounds.height / 2; + const angleRad = (bounds.rotation * Math.PI) / 180; + const cos = Math.cos(angleRad); + const sin = Math.sin(angleRad); + const localX = + corner === "top-left" || corner === "bottom-left" ? -halfW : halfW; + const localY = + corner === "top-left" || corner === "top-right" ? -halfH : halfH; + return { + x: bounds.cx + (localX * cos - localY * sin), + y: bounds.cy + (localX * sin + localY * cos), + }; +} + +export function getEdgeHandlePosition({ + bounds, + edge, +}: { + bounds: ElementBounds; + edge: Edge; +}): { x: number; y: number } { + const halfWidth = bounds.width / 2; + const halfHeight = bounds.height / 2; + const angleRad = (bounds.rotation * Math.PI) / 180; + const cos = Math.cos(angleRad); + const sin = Math.sin(angleRad); + const localX = edge === "right" ? halfWidth : edge === "left" ? -halfWidth : 0; + const localY = edge === "bottom" ? halfHeight : 0; + return { + x: bounds.cx + (localX * cos - localY * sin), + y: bounds.cy + (localX * sin + localY * cos), + }; +} + +export function getVisibleElementsWithBounds({ + tracks, + currentTime, + canvasSize, + mediaAssets, +}: { + tracks: SceneTracks; + currentTime: number; + canvasSize: { width: number; height: number }; + mediaAssets: MediaAsset[]; +}): ElementWithBounds[] { + const mediaMap = new Map(mediaAssets.map((m) => [m.id, m])); + const orderedTracks = [ + ...tracks.overlay.filter((track) => !("hidden" in track && track.hidden)), + ...(!tracks.main.hidden ? [tracks.main] : []), + ].reverse(); + + const result: ElementWithBounds[] = []; + + for (const track of orderedTracks) { + const elements = track.elements + .filter((element) => !("hidden" in element && element.hidden)) + .filter( + (element) => + currentTime >= element.startTime && + currentTime < element.startTime + element.duration, + ) + .slice() + .sort((a, b) => { + if (a.startTime !== b.startTime) return a.startTime - b.startTime; + return a.id.localeCompare(b.id); + }); + + for (const element of elements) { + const localTime = getElementLocalTime({ + timelineTime: currentTime, + elementStartTime: element.startTime, + elementDuration: element.duration, + }); + const mediaAsset = + element.type === "video" || element.type === "image" + ? mediaMap.get(element.mediaId) + : undefined; + const bounds = getElementBounds({ + element, + canvasSize, + mediaAsset, + localTime, + }); + if (bounds) { + result.push({ + trackId: track.id, + elementId: element.id, + element, + bounds, + }); + } + } + } + + return result; +} diff --git a/apps/web/src/rendering/NOTES.md b/apps/web/src/rendering/NOTES.md new file mode 100644 index 00000000..38055c14 --- /dev/null +++ b/apps/web/src/rendering/NOTES.md @@ -0,0 +1,27 @@ +# Notes + +## `Transform` is misplaced + +`Transform` is currently defined in `apps/web/src/rendering/index.ts`. It's +exported from the rendering domain because rendering happens to be one of its +consumers — but every other domain that touches scene geometry consumes it too +(`@/timeline`, `@/preview`, `@/animation/values`, `@/text`, etc.). It's not +"of" rendering any more than `Vector2` would be "of" math. + +`Transform` is closer to a primitive than a domain concept. It's infrastructure: +a small value type with no behavior, no dependencies, no domain-specific +invariants. Anything that wants to position something on a 2D canvas needs it. + +The codebase should be refactored so it's defined lower-level, rather than +sitting in a domain. Candidate homes: + +- `apps/web/src/primitives/transform.ts` (new dir for these value types) +- `apps/web/src/geometry/transform.ts` if `Vector2` and friends end up there +- `apps/web/src/rendering/transform.ts` is acceptable only if `@/rendering` +becomes a primitives folder rather than a domain (it's borderline today — +it also exports `BlendMode`, which has the same problem). + +Side effect of fixing this: `@/rendering/animation-values.ts` (which exists +only because `Transform` lives next to it) can move into `@/animation/values.ts` +alongside the other resolve-at-time helpers, since `Transform` would no longer +pull in a domain dependency. \ No newline at end of file diff --git a/apps/web/src/rendering/animation-values.ts b/apps/web/src/rendering/animation-values.ts new file mode 100644 index 00000000..8e8b4528 --- /dev/null +++ b/apps/web/src/rendering/animation-values.ts @@ -0,0 +1,49 @@ +import type { ElementAnimations } from "@/animation/types"; +import { resolveAnimationPathValueAtTime } from "@/animation"; +import type { Transform } from "./index"; + +export function resolveTransformAtTime({ + baseTransform, + animations, + localTime, +}: { + baseTransform: Transform; + animations: ElementAnimations | undefined; + localTime: number; +}): Transform { + const safeLocalTime = Math.max(0, localTime); + return { + position: { + x: resolveAnimationPathValueAtTime({ + animations, + propertyPath: "transform.positionX", + localTime: safeLocalTime, + fallbackValue: baseTransform.position.x, + }), + y: resolveAnimationPathValueAtTime({ + animations, + propertyPath: "transform.positionY", + localTime: safeLocalTime, + fallbackValue: baseTransform.position.y, + }), + }, + scaleX: resolveAnimationPathValueAtTime({ + animations, + propertyPath: "transform.scaleX", + localTime: safeLocalTime, + fallbackValue: baseTransform.scaleX, + }), + scaleY: resolveAnimationPathValueAtTime({ + animations, + propertyPath: "transform.scaleY", + localTime: safeLocalTime, + fallbackValue: baseTransform.scaleY, + }), + rotate: resolveAnimationPathValueAtTime({ + animations, + propertyPath: "transform.rotate", + localTime: safeLocalTime, + fallbackValue: baseTransform.rotate, + }), + }; +} diff --git a/apps/web/src/rendering/components/blending-tab.tsx b/apps/web/src/rendering/components/blending-tab.tsx index 870390bf..52d2f051 100644 --- a/apps/web/src/rendering/components/blending-tab.tsx +++ b/apps/web/src/rendering/components/blending-tab.tsx @@ -27,7 +27,7 @@ import { RainDropIcon } from "@hugeicons/core-free-icons"; import { KeyframeToggle } from "@/components/editor/panels/properties/components/keyframe-toggle"; import { useKeyframedNumberProperty } from "@/components/editor/panels/properties/hooks/use-keyframed-number-property"; import { useElementPlayhead } from "@/components/editor/panels/properties/hooks/use-element-playhead"; -import { resolveOpacityAtTime } from "@/animation"; +import { resolveOpacityAtTime } from "@/animation/values"; import { DEFAULTS } from "@/timeline/defaults"; import { isPropertyAtDefault } from "./transform-tab"; import type { MediaTime } from "@/wasm"; diff --git a/apps/web/src/rendering/components/transform-tab.tsx b/apps/web/src/rendering/components/transform-tab.tsx index 60d4c7df..f4d5a183 100644 --- a/apps/web/src/rendering/components/transform-tab.tsx +++ b/apps/web/src/rendering/components/transform-tab.tsx @@ -20,8 +20,8 @@ import { import { getGroupKeyframesAtTime, hasGroupKeyframeAtTime, - resolveTransformAtTime, } from "@/animation"; +import { resolveTransformAtTime } from "@/rendering/animation-values"; import { DEFAULTS } from "@/timeline/defaults"; import { useElementPlayhead } from "@/components/editor/panels/properties/hooks/use-element-playhead"; import { KeyframeToggle } from "@/components/editor/panels/properties/components/keyframe-toggle"; diff --git a/apps/web/src/services/renderer/canvas-renderer.ts b/apps/web/src/services/renderer/canvas-renderer.ts index 50f7fd12..4a002675 100644 --- a/apps/web/src/services/renderer/canvas-renderer.ts +++ b/apps/web/src/services/renderer/canvas-renderer.ts @@ -3,6 +3,11 @@ import type { AnyBaseNode } from "./nodes/base-node"; import { buildFrameDescriptor } from "./compositor/frame-descriptor"; import { wasmCompositor } from "./compositor/wasm-compositor"; import { resolveRenderTree } from "./resolve"; +import { + measureSpanAsync, + measureSpanSync, + onRenderPerfFrameComplete, +} from "@/diagnostics/render-perf"; export type CanvasRendererParams = { width: number; @@ -69,17 +74,26 @@ export class CanvasRenderer { } async render({ node, time }: { node: AnyBaseNode; time: number }) { - await resolveRenderTree({ node, renderer: this, time }); - const { frame, textures } = await buildFrameDescriptor({ - node, - renderer: this, + await measureSpanAsync({ + name: "resolve", + fn: () => resolveRenderTree({ node, renderer: this, time }), + }); + const { frame, textures } = await measureSpanAsync({ + name: "buildFrame", + fn: () => buildFrameDescriptor({ node, renderer: this }), }); wasmCompositor.ensureInitialized({ width: this.width, height: this.height, }); - wasmCompositor.syncTextures(textures); - wasmCompositor.render(frame); + measureSpanSync({ + name: "syncTextures", + fn: () => wasmCompositor.syncTextures(textures), + }); + measureSpanSync({ + name: "renderFrame", + fn: () => wasmCompositor.render(frame), + }); } async renderToCanvas({ @@ -98,12 +112,17 @@ export class CanvasRenderer { throw new Error("Failed to get target canvas context"); } - ctx.drawImage( - wasmCompositor.getCanvas(), - 0, - 0, - targetCanvas.width, - targetCanvas.height, - ); + measureSpanSync({ + name: "drawImage", + fn: () => + ctx.drawImage( + wasmCompositor.getCanvas(), + 0, + 0, + targetCanvas.width, + targetCanvas.height, + ), + }); + onRenderPerfFrameComplete(); } } diff --git a/apps/web/src/services/renderer/compositor/frame-descriptor.ts b/apps/web/src/services/renderer/compositor/frame-descriptor.ts index 666bea03..190a0237 100644 --- a/apps/web/src/services/renderer/compositor/frame-descriptor.ts +++ b/apps/web/src/services/renderer/compositor/frame-descriptor.ts @@ -1,5 +1,6 @@ import { drawCssBackground } from "@/gradients"; import { masksRegistry } from "@/masks"; +import { incrementCounter } from "@/diagnostics/render-perf"; import type { AnyBaseNode } from "../nodes/base-node"; import type { CanvasRenderer } from "../canvas-renderer"; import { createOffscreenCanvas } from "../canvas-utils"; @@ -21,16 +22,11 @@ import type { FrameItemDescriptor, LayerMaskDescriptor, QuadTransformDescriptor, + TextureCanvasDrawFn, + TextureUploadDescriptor, } from "./types"; import { DEFAULT_GRAPHIC_SOURCE_SIZE } from "@/graphics"; -export type TextureUploadDescriptor = { - id: string; - source: CanvasImageSource; - width: number; - height: number; -}; - export async function buildFrameDescriptor({ node, renderer, @@ -52,6 +48,9 @@ export async function buildFrameDescriptor({ textures, }); + incrementCounter({ name: "frameItems", by: items.length }); + incrementCounter({ name: "frameTextures", by: textures.size }); + return { frame: { width: renderer.width, @@ -93,31 +92,21 @@ async function collectNode({ if (node instanceof ColorNode) { const textureId = `${path}:color`; - const canvas = createOffscreenCanvas({ - width: renderer.width, - height: renderer.height, - }); - const ctx = canvas.getContext("2d") as - | CanvasRenderingContext2D - | OffscreenCanvasRenderingContext2D - | null; - if (!ctx) return; - if (/gradient\(/i.test(node.params.color)) { - drawCssBackground({ - ctx, - width: renderer.width, - height: renderer.height, - css: node.params.color, - }); - } else { - ctx.fillStyle = node.params.color; - ctx.fillRect(0, 0, renderer.width, renderer.height); - } + const { width, height } = renderer; textures.set(textureId, { + kind: "rendered", id: textureId, - source: canvas, - width: renderer.width, - height: renderer.height, + contentHash: `color:${node.params.color}:${width}x${height}`, + width, + height, + draw: (ctx) => { + if (/gradient\(/i.test(node.params.color)) { + drawCssBackground({ ctx, width, height, css: node.params.color }); + } else { + ctx.fillStyle = node.params.color; + ctx.fillRect(0, 0, width, height); + } + }, }); items.push({ type: "layer", @@ -147,36 +136,35 @@ async function collectNode({ return; } const textureId = `${path}:blur-background`; - const backdropCanvas = createOffscreenCanvas({ - width: renderer.width, - height: renderer.height, - }); - const backdropCtx = backdropCanvas.getContext("2d") as - | CanvasRenderingContext2D - | OffscreenCanvasRenderingContext2D - | null; - if (!backdropCtx) return; + const { width, height } = renderer; const { backdropSource, passes } = node.resolved; - const coverScale = Math.max( - renderer.width / backdropSource.width, - renderer.height / backdropSource.height, - ); - const scaledWidth = backdropSource.width * coverScale; - const scaledHeight = backdropSource.height * coverScale; - const offsetX = (renderer.width - scaledWidth) / 2; - const offsetY = (renderer.height - scaledHeight) / 2; - backdropCtx.drawImage( - backdropSource.source, - offsetX, - offsetY, - scaledWidth, - scaledHeight, - ); + // Backdrop pixels come from a decoded video/image frame whose identity + // already changes when it changes. Hashing the source reference is + // enough to let us skip redraws on frozen frames. + const contentHash = `blur:${identityKey(backdropSource.source)}:${backdropSource.width}x${backdropSource.height}:${width}x${height}`; textures.set(textureId, { + kind: "rendered", id: textureId, - source: backdropCanvas, - width: renderer.width, - height: renderer.height, + contentHash, + width, + height, + draw: (ctx) => { + const coverScale = Math.max( + width / backdropSource.width, + height / backdropSource.height, + ); + const scaledWidth = backdropSource.width * coverScale; + const scaledHeight = backdropSource.height * coverScale; + const offsetX = (width - scaledWidth) / 2; + const offsetY = (height - scaledHeight) / 2; + ctx.drawImage( + backdropSource.source, + offsetX, + offsetY, + scaledWidth, + scaledHeight, + ); + }, }); items.push({ type: "layer", @@ -253,6 +241,7 @@ async function collectVisualSourceNode({ const textureId = `${path}:source`; textures.set(textureId, { + kind: "external", id: textureId, source, width: sourceWidth, @@ -305,28 +294,24 @@ function collectTextNode({ } const textureId = `${path}:text`; - const canvas = createOffscreenCanvas({ - width: renderer.width, - height: renderer.height, - }); - const ctx = canvas.getContext("2d") as - | CanvasRenderingContext2D - | OffscreenCanvasRenderingContext2D - | null; - if (!ctx) { - return; - } - - renderTextToContext({ - node, - ctx, - }); - + const { width, height } = renderer; + // Text output is fully determined by node.params + node.resolved. Both are + // plain data we can stringify cheaply; the resolved measured layout is the + // expensive part of text setup, so stringifying it here is orders of + // magnitude cheaper than re-rasterizing when nothing changed. + const contentHash = `text:${width}x${height}:${JSON.stringify({ + params: node.params, + resolved: node.resolved, + })}`; textures.set(textureId, { + kind: "rendered", id: textureId, - source: canvas, - width: renderer.width, - height: renderer.height, + contentHash, + width, + height, + draw: (ctx) => { + renderTextToContext({ node, ctx }); + }, }); items.push({ type: "layer", @@ -411,20 +396,6 @@ function buildMaskArtifacts({ return { mask: null, strokeLayer: null }; } - const elementMaskCanvas = createOffscreenCanvas({ - width: Math.round(transform.width), - height: Math.round(transform.height), - }); - const elementMaskCtx = elementMaskCanvas.getContext("2d") as - | CanvasRenderingContext2D - | OffscreenCanvasRenderingContext2D - | null; - if (!elementMaskCtx) { - return { mask: null, strokeLayer: null }; - } - elementMaskCtx.clearRect(0, 0, transform.width, transform.height); - - let strokePath: Path2D | null = null; let feather = mask.params.feather; const canRenderMaskDirectly = Boolean(definition.renderer.renderMask); const shouldRenderMaskDirectly = @@ -432,81 +403,78 @@ function buildMaskArtifacts({ (!definition.renderer.buildPath || (mask.params.feather > 0 && definition.renderer.renderMaskHandlesFeather)); - if (shouldRenderMaskDirectly && definition.renderer.renderMask) { - definition.renderer.renderMask({ - resolvedParams: mask.params, - ctx: elementMaskCtx, - width: Math.round(transform.width), - height: Math.round(transform.height), - feather: mask.params.feather, - }); - if (definition.renderer.renderMaskHandlesFeather) { - feather = 0; - } - strokePath = - definition.renderer.buildStrokePath?.({ - resolvedParams: mask.params, - width: transform.width, - height: transform.height, - }) ?? null; - } else { - if (!definition.renderer.buildPath) { - return { mask: null, strokeLayer: null }; - } - const path2d = definition.renderer.buildPath({ - resolvedParams: mask.params, - width: transform.width, - height: transform.height, - }); - elementMaskCtx.fillStyle = "white"; - elementMaskCtx.fill(path2d); - strokePath = - definition.renderer.buildStrokePath?.({ - resolvedParams: mask.params, - width: transform.width, - height: transform.height, - }) ?? path2d; + if ( + shouldRenderMaskDirectly && + definition.renderer.renderMaskHandlesFeather + ) { + feather = 0; } - const fullMaskCanvas = createOffscreenCanvas({ - width: renderer.width, - height: renderer.height, - }); - const fullMaskCtx = fullMaskCanvas.getContext("2d") as - | CanvasRenderingContext2D - | OffscreenCanvasRenderingContext2D - | null; - if (!fullMaskCtx) { - return { mask: null, strokeLayer: null }; - } - drawTransformedCanvas({ - ctx: fullMaskCtx, - source: elementMaskCanvas, - transform, - }); - const maskTextureId = `${path}:mask`; - textures.set(maskTextureId, { - id: maskTextureId, - source: fullMaskCanvas, - width: renderer.width, - height: renderer.height, - }); - - let strokeLayer: FrameItemDescriptor | null = null; - if ( - mask.params.strokeWidth > 0 && - (strokePath || definition.renderer.renderStroke) - ) { - const strokeCanvas = createOffscreenCanvas({ + const { width: canvasWidth, height: canvasHeight } = renderer; + const maskContentHash = `mask:${mask.type}:${JSON.stringify(mask.params)}:${transformHash(transform)}:${canvasWidth}x${canvasHeight}:direct=${shouldRenderMaskDirectly}`; + const drawMask: TextureCanvasDrawFn = (ctx) => { + const elementMaskCanvas = createOffscreenCanvas({ width: Math.round(transform.width), height: Math.round(transform.height), }); - const strokeCtx = strokeCanvas.getContext("2d") as + const elementMaskCtx = elementMaskCanvas.getContext("2d") as | CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D | null; - if (strokeCtx) { + if (!elementMaskCtx) return; + + if (shouldRenderMaskDirectly && definition.renderer.renderMask) { + definition.renderer.renderMask({ + resolvedParams: mask.params, + ctx: elementMaskCtx, + width: Math.round(transform.width), + height: Math.round(transform.height), + feather: mask.params.feather, + }); + } else if (definition.renderer.buildPath) { + const path2d = definition.renderer.buildPath({ + resolvedParams: mask.params, + width: transform.width, + height: transform.height, + }); + elementMaskCtx.fillStyle = "white"; + elementMaskCtx.fill(path2d); + } else { + return; + } + + drawTransformedCanvas({ ctx, source: elementMaskCanvas, transform }); + }; + textures.set(maskTextureId, { + kind: "rendered", + id: maskTextureId, + contentHash: maskContentHash, + width: canvasWidth, + height: canvasHeight, + draw: drawMask, + }); + + const hasStroke = + mask.params.strokeWidth > 0 && + (definition.renderer.renderStroke || + definition.renderer.buildStrokePath || + definition.renderer.buildPath); + let strokeLayer: FrameItemDescriptor | null = null; + if (hasStroke) { + const strokeTextureId = `${path}:mask-stroke`; + const strokeContentHash = `stroke:${mask.type}:${JSON.stringify(mask.params)}:${transformHash(transform)}:${canvasWidth}x${canvasHeight}`; + const drawStroke: TextureCanvasDrawFn = (ctx) => { + const strokeCanvas = createOffscreenCanvas({ + width: Math.round(transform.width), + height: Math.round(transform.height), + }); + const strokeCtx = strokeCanvas.getContext("2d") as + | CanvasRenderingContext2D + | OffscreenCanvasRenderingContext2D + | null; + if (!strokeCtx) return; + if (definition.renderer.renderStroke) { definition.renderer.renderStroke({ resolvedParams: mask.params, @@ -514,44 +482,44 @@ function buildMaskArtifacts({ width: transform.width, height: transform.height, }); - } else if (strokePath) { + } else { + const strokePath = + definition.renderer.buildStrokePath?.({ + resolvedParams: mask.params, + width: transform.width, + height: transform.height, + }) ?? + definition.renderer.buildPath?.({ + resolvedParams: mask.params, + width: transform.width, + height: transform.height, + }) ?? + null; + if (!strokePath) return; strokeCtx.strokeStyle = mask.params.strokeColor; strokeCtx.lineWidth = mask.params.strokeWidth; strokeCtx.stroke(strokePath); } - const fullStrokeCanvas = createOffscreenCanvas({ - width: renderer.width, - height: renderer.height, - }); - const fullStrokeCtx = fullStrokeCanvas.getContext("2d") as - | CanvasRenderingContext2D - | OffscreenCanvasRenderingContext2D - | null; - if (fullStrokeCtx) { - drawTransformedCanvas({ - ctx: fullStrokeCtx, - source: strokeCanvas, - transform, - }); - const strokeTextureId = `${path}:mask-stroke`; - textures.set(strokeTextureId, { - id: strokeTextureId, - source: fullStrokeCanvas, - width: renderer.width, - height: renderer.height, - }); - strokeLayer = { - type: "layer", - textureId: strokeTextureId, - transform: fullCanvasTransform(renderer), - opacity: 1, - blendMode: "normal", - effectPassGroups: [], - mask: null, - }; - } - } + drawTransformedCanvas({ ctx, source: strokeCanvas, transform }); + }; + textures.set(strokeTextureId, { + kind: "rendered", + id: strokeTextureId, + contentHash: strokeContentHash, + width: canvasWidth, + height: canvasHeight, + draw: drawStroke, + }); + strokeLayer = { + type: "layer", + textureId: strokeTextureId, + transform: fullCanvasTransform(renderer), + opacity: 1, + blendMode: "normal", + effectPassGroups: [], + mask: null, + }; } return { @@ -590,3 +558,23 @@ function drawTransformedCanvas({ ctx.drawImage(source, x, y, transform.width, transform.height); ctx.restore(); } + +function transformHash(transform: QuadTransformDescriptor): string { + return `${transform.centerX}:${transform.centerY}:${transform.width}:${transform.height}:${transform.rotationDegrees}:${transform.flipX ? 1 : 0}:${transform.flipY ? 1 : 0}`; +} + +// Stable identity key for CanvasImageSource. Using a WeakMap → counter keeps +// hash string length bounded and avoids holding sources alive. +const identityKeys = new WeakMap(); +let nextIdentity = 1; +function identityKey(source: CanvasImageSource): string { + if (typeof source === "object" && source !== null) { + let key = identityKeys.get(source); + if (key === undefined) { + key = nextIdentity++; + identityKeys.set(source, key); + } + return `@${key}`; + } + return "@?"; +} diff --git a/apps/web/src/services/renderer/compositor/types.ts b/apps/web/src/services/renderer/compositor/types.ts index af5a58c2..1a509909 100644 --- a/apps/web/src/services/renderer/compositor/types.ts +++ b/apps/web/src/services/renderer/compositor/types.ts @@ -40,3 +40,39 @@ export type LayerMaskDescriptor = { feather: number; inverted: boolean; }; + +export type TextureCanvasDrawFn = ( + ctx: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D, +) => void; + +/** + * A layer texture whose pixels come from somewhere outside the renderer — + * typically a decoded video/image frame or a sticker. Cached by reference + * identity of the source object. + */ +export type ExternalTextureDescriptor = { + kind: "external"; + id: string; + source: CanvasImageSource; + width: number; + height: number; +}; + +/** + * A layer texture that the renderer rasterizes from scene state (color fill, + * text layout, mask shape, blur backdrop). Cached by `contentHash`: when it + * matches the previous frame's hash for this id, the upload is skipped + * entirely and the persistent canvas is not even cleared. + */ +export type RenderedTextureDescriptor = { + kind: "rendered"; + id: string; + contentHash: string; + width: number; + height: number; + draw: TextureCanvasDrawFn; +}; + +export type TextureUploadDescriptor = + | ExternalTextureDescriptor + | RenderedTextureDescriptor; diff --git a/apps/web/src/services/renderer/compositor/wasm-compositor.ts b/apps/web/src/services/renderer/compositor/wasm-compositor.ts index 16b03853..10684f11 100644 --- a/apps/web/src/services/renderer/compositor/wasm-compositor.ts +++ b/apps/web/src/services/renderer/compositor/wasm-compositor.ts @@ -1,12 +1,201 @@ import { getCompositorCanvas, + getLastFrameProfile, initCompositor, releaseTexture, renderFrame, resizeCompositor, uploadTexture, } from "opencut-wasm"; -import type { FrameDescriptor } from "./types"; +import { + incrementCounter, + isRenderPerfEnabled, + recordWasmFrameProfile, +} from "@/diagnostics/render-perf"; +import type { + ExternalTextureDescriptor, + FrameDescriptor, + RenderedTextureDescriptor, + TextureUploadDescriptor, +} from "./types"; + +/** + * One slot in the derived-texture cache. The OffscreenCanvas is persistent — + * we reuse and redraw into it across frames, which is the change that takes + * the WebGL upload path off the per-frame critical path for static content. + */ +type RenderedCacheEntry = { + kind: "rendered"; + canvas: OffscreenCanvas; + contentHash: string; + width: number; + height: number; +}; + +type ExternalCacheEntry = { + kind: "external"; + source: CanvasImageSource; + width: number; + height: number; +}; + +class WasmCompositor { + private canvas: HTMLCanvasElement | null = null; + private initializedSize: { width: number; height: number } | null = null; + private cache = new Map(); + + ensureInitialized({ width, height }: { width: number; height: number }) { + if (!this.canvas) { + initCompositor(width, height); + this.canvas = getCompositorCanvas(); + this.initializedSize = { width, height }; + return; + } + + if ( + !this.initializedSize || + this.initializedSize.width !== width || + this.initializedSize.height !== height + ) { + resizeCompositor(width, height); + this.initializedSize = { width, height }; + } + } + + getCanvas(): HTMLCanvasElement { + if (!this.canvas) { + throw new Error("Compositor is not initialized"); + } + return this.canvas; + } + + syncTextures(textures: TextureUploadDescriptor[]) { + const nextIds = new Set(textures.map((texture) => texture.id)); + for (const previousId of this.cache.keys()) { + if (!nextIds.has(previousId)) { + releaseTexture(previousId); + this.cache.delete(previousId); + } + } + + for (const texture of textures) { + if (texture.kind === "external") { + this.syncExternalTexture(texture); + } else { + this.syncRenderedTexture(texture); + } + } + } + + render(frame: FrameDescriptor) { + renderFrame(frame); + if (isRenderPerfEnabled()) { + recordWasmFrameProfile( + getLastFrameProfile() as Array<{ name: string; durationMs: number }>, + ); + } + } + + private syncExternalTexture(texture: ExternalTextureDescriptor) { + const previous = this.cache.get(texture.id); + if ( + previous?.kind === "external" && + previous.source === texture.source && + previous.width === texture.width && + previous.height === texture.height + ) { + incrementCounter({ name: "textureCacheHit" }); + return; + } + + incrementCounter({ name: "textureUpload" }); + incrementCounter({ + name: "textureUploadPixels", + by: texture.width * texture.height, + }); + uploadTexture({ + id: texture.id, + source: ensureOffscreenCanvas({ + source: texture.source, + width: texture.width, + height: texture.height, + label: `texture upload ${texture.id}`, + }), + width: texture.width, + height: texture.height, + }); + this.cache.set(texture.id, { + kind: "external", + source: texture.source, + width: texture.width, + height: texture.height, + }); + } + + private syncRenderedTexture(texture: RenderedTextureDescriptor) { + const previous = this.cache.get(texture.id); + if ( + previous?.kind === "rendered" && + previous.contentHash === texture.contentHash && + previous.width === texture.width && + previous.height === texture.height + ) { + incrementCounter({ name: "textureCacheHit" }); + return; + } + + const canvas = + previous?.kind === "rendered" && + previous.width === texture.width && + previous.height === texture.height + ? previous.canvas + : createBackingCanvas({ + width: texture.width, + height: texture.height, + }); + + const ctx = canvas.getContext("2d"); + if (!ctx) { + throw new Error(`Failed to get 2d context for texture ${texture.id}`); + } + ctx.clearRect(0, 0, texture.width, texture.height); + texture.draw(ctx); + + incrementCounter({ name: "textureUpload" }); + incrementCounter({ + name: "textureUploadPixels", + by: texture.width * texture.height, + }); + uploadTexture({ + id: texture.id, + source: canvas, + width: texture.width, + height: texture.height, + }); + this.cache.set(texture.id, { + kind: "rendered", + canvas, + contentHash: texture.contentHash, + width: texture.width, + height: texture.height, + }); + } +} + +export const wasmCompositor = new WasmCompositor(); + +function createBackingCanvas({ + width, + height, +}: { + width: number; + height: number; +}): OffscreenCanvas { + if (typeof OffscreenCanvas === "undefined") { + throw new Error("OffscreenCanvas is not supported in this environment"); + } + return new OffscreenCanvas(width, height); +} function ensureOffscreenCanvas({ source, @@ -36,92 +225,3 @@ function ensureOffscreenCanvas({ context.drawImage(source, 0, 0, width, height); return canvas; } - -export type TextureUploadDescriptor = { - id: string; - source: CanvasImageSource; - width: number; - height: number; -}; - -class WasmCompositor { - private canvas: HTMLCanvasElement | null = null; - private initializedSize: { width: number; height: number } | null = null; - private retainedTextureIds = new Set(); - private uploadedTextures = new Map< - string, - { source: CanvasImageSource; width: number; height: number } - >(); - - ensureInitialized({ width, height }: { width: number; height: number }) { - if (!this.canvas) { - initCompositor(width, height); - this.canvas = getCompositorCanvas(); - this.initializedSize = { width, height }; - return; - } - - if ( - !this.initializedSize || - this.initializedSize.width !== width || - this.initializedSize.height !== height - ) { - resizeCompositor(width, height); - this.initializedSize = { width, height }; - } - } - - getCanvas(): HTMLCanvasElement { - if (!this.canvas) { - throw new Error("Compositor is not initialized"); - } - return this.canvas; - } - - syncTextures(textures: TextureUploadDescriptor[]) { - const nextIds = new Set(textures.map((texture) => texture.id)); - for (const previousId of this.retainedTextureIds) { - if (!nextIds.has(previousId)) { - releaseTexture(previousId); - this.uploadedTextures.delete(previousId); - } - } - - for (const texture of textures) { - const previousTexture = this.uploadedTextures.get(texture.id); - if ( - previousTexture?.source === texture.source && - previousTexture.width === texture.width && - previousTexture.height === texture.height - ) { - continue; - } - - const sourceCanvas = ensureOffscreenCanvas({ - source: texture.source, - width: texture.width, - height: texture.height, - label: `texture upload ${texture.id}`, - }); - uploadTexture({ - id: texture.id, - source: sourceCanvas, - width: texture.width, - height: texture.height, - }); - this.uploadedTextures.set(texture.id, { - source: texture.source, - width: texture.width, - height: texture.height, - }); - } - - this.retainedTextureIds = nextIds; - } - - render(frame: FrameDescriptor) { - renderFrame(frame); - } -} - -export const wasmCompositor = new WasmCompositor(); diff --git a/apps/web/src/services/renderer/resolve.ts b/apps/web/src/services/renderer/resolve.ts index 0623b399..36a6ecb5 100644 --- a/apps/web/src/services/renderer/resolve.ts +++ b/apps/web/src/services/renderer/resolve.ts @@ -1,11 +1,5 @@ import { mediaTimeToSeconds, roundMediaTime } from "@/wasm"; -import { - getElementLocalTime, - resolveColorAtTime, - resolveGraphicParamsAtTime, - resolveOpacityAtTime, - resolveTransformAtTime, -} from "@/animation"; +import { getElementLocalTime } from "@/animation"; import { resolveEffectParamsAtTime } from "@/animation/effect-param-channel"; import { buildGaussianBlurPasses, @@ -14,11 +8,16 @@ import { import { effectsRegistry, resolveEffectPasses } from "@/effects"; import type { Effect, EffectPass } from "@/effects/types"; import { getSourceTimeAtClipTime } from "@/retime"; -import { DEFAULT_GRAPHIC_SOURCE_SIZE } from "@/graphics"; +import { + DEFAULT_GRAPHIC_SOURCE_SIZE, + resolveGraphicElementParamsAtTime, +} from "@/graphics"; import { getTextMeasurementContext, measureTextElement, } from "@/text/measure-element"; +import { resolveColorAtTime, resolveOpacityAtTime } from "@/animation/values"; +import { resolveTransformAtTime } from "@/rendering/animation-values"; import { videoCache } from "@/services/video-cache/service"; import type { CanvasRenderer } from "./canvas-renderer"; import type { AnyBaseNode } from "./nodes/base-node"; @@ -113,7 +112,8 @@ function resolveEffectPassGroups({ .filter((effect) => effect.enabled) .map((effect) => { const resolvedParams = resolveEffectParamsAtTime({ - effect, + effectId: effect.id, + params: effect.params, animations, localTime, }); @@ -304,7 +304,7 @@ function resolveGraphicNode({ return { ...visualState, - resolvedParams: resolveGraphicParamsAtTime({ + resolvedParams: resolveGraphicElementParamsAtTime({ element: node.params, localTime: visualState.localTime, }), diff --git a/apps/web/src/text/components/text-tab.tsx b/apps/web/src/text/components/text-tab.tsx index bf14efba..0f0b274a 100644 --- a/apps/web/src/text/components/text-tab.tsx +++ b/apps/web/src/text/components/text-tab.tsx @@ -30,7 +30,10 @@ import { useKeyframedNumberProperty } from "@/components/editor/panels/propertie import { useElementPlayhead } from "@/components/editor/panels/properties/hooks/use-element-playhead"; import { KeyframeToggle } from "@/components/editor/panels/properties/components/keyframe-toggle"; import { isPropertyAtDefault } from "@/rendering/components/transform-tab"; -import { resolveColorAtTime, resolveNumberAtTime } from "@/animation"; +import { + resolveColorAtTime, + resolveNumberAtTime, +} from "@/animation/values"; import { HugeiconsIcon } from "@hugeicons/react"; import { MinusSignIcon, diff --git a/apps/web/src/text/measure-element.ts b/apps/web/src/text/measure-element.ts index 3a49696f..ec14421b 100644 --- a/apps/web/src/text/measure-element.ts +++ b/apps/web/src/text/measure-element.ts @@ -1,7 +1,7 @@ import { CORNER_RADIUS_MIN } from "@/text/background"; -import { resolveNumberAtTime } from "@/animation"; import { DEFAULTS } from "@/timeline/defaults"; import type { TextBackground, TextElement } from "@/timeline"; +import { resolveNumberAtTime } from "@/animation/values"; import { getTextVisualRect, } from "./layout"; diff --git a/apps/web/src/timeline/animation-properties.ts b/apps/web/src/timeline/animation-properties.ts new file mode 100644 index 00000000..308f6cf0 --- /dev/null +++ b/apps/web/src/timeline/animation-properties.ts @@ -0,0 +1,426 @@ +import type { + AnimationBindingKind, + AnimationInterpolation, + AnimationPropertyPath, + AnimationValue, + ElementAnimations, + NumericSpec, +} from "@/animation/types"; +import { upsertPathKeyframe } from "@/animation"; +import { parseColorToLinearRgba } from "@/animation/binding-values"; +import type { MediaTime } from "@/wasm"; +import { isAnimationPropertyPath } from "@/animation/path"; +import { MIN_TRANSFORM_SCALE } from "@/animation/transform"; +import { snapToStep } from "@/utils/math"; +import type { TimelineElement } from "@/timeline"; +import { + CORNER_RADIUS_MAX, + CORNER_RADIUS_MIN, +} from "@/text/background"; +import { + canElementHaveAudio, + isVisualElement, +} from "@/timeline/element-utils"; +import { VOLUME_DB_MAX, VOLUME_DB_MIN } from "@/timeline/audio-constants"; +import { DEFAULTS } from "@/timeline/defaults"; + +export interface AnimationPropertyDefinition { + kind: AnimationBindingKind; + defaultInterpolation: AnimationInterpolation; + numericRanges?: Partial>; + supportsElement: ({ element }: { element: TimelineElement }) => boolean; + getValue: ({ element }: { element: TimelineElement }) => AnimationValue | null; + coerceValue: ({ value }: { value: AnimationValue }) => AnimationValue | null; + setValue: ({ + element, + value, + }: { + element: TimelineElement; + value: AnimationValue; + }) => TimelineElement; +} + +function applyNumericSpec({ + value, + numericRange, +}: { + value: number; + numericRange: NumericSpec | undefined; +}): number { + if (!numericRange) { + return value; + } + + const steppedValue = + numericRange.step != null + ? snapToStep({ value, step: numericRange.step }) + : value; + const minValue = numericRange.min ?? Number.NEGATIVE_INFINITY; + const maxValue = numericRange.max ?? Number.POSITIVE_INFINITY; + return Math.min(maxValue, Math.max(minValue, steppedValue)); +} + +function coerceNumberValue({ + value, + numericRange, +}: { + value: AnimationValue; + numericRange?: NumericSpec; +}): number | null { + if (typeof value !== "number" || Number.isNaN(value)) { + return null; + } + + return applyNumericSpec({ value, numericRange }); +} + +function coerceColorValue({ + value, +}: { + value: AnimationValue; +}): string | null { + return typeof value === "string" && parseColorToLinearRgba({ color: value }) + ? value + : null; +} + +function createNumberPropertyDefinition({ + numericRange, + supportsElement, + getValue, + setValue, +}: { + numericRange?: NumericSpec; + supportsElement: AnimationPropertyDefinition["supportsElement"]; + getValue: AnimationPropertyDefinition["getValue"]; + setValue: AnimationPropertyDefinition["setValue"]; +}): AnimationPropertyDefinition { + return { + kind: "number", + defaultInterpolation: "linear", + numericRanges: numericRange ? { value: numericRange } : undefined, + supportsElement, + getValue, + coerceValue: ({ value }) => + coerceNumberValue({ + value, + numericRange, + }), + setValue, + }; +} + +const ANIMATION_PROPERTY_REGISTRY: Record< + AnimationPropertyPath, + AnimationPropertyDefinition +> = { + "transform.positionX": createNumberPropertyDefinition({ + numericRange: { step: 1 }, + supportsElement: ({ element }) => isVisualElement(element), + getValue: ({ element }) => + isVisualElement(element) ? element.transform.position.x : null, + setValue: ({ element, value }) => + isVisualElement(element) + ? { + ...element, + transform: { + ...element.transform, + position: { ...element.transform.position, x: value as number }, + }, + } + : element, + }), + "transform.positionY": createNumberPropertyDefinition({ + numericRange: { step: 1 }, + supportsElement: ({ element }) => isVisualElement(element), + getValue: ({ element }) => + isVisualElement(element) ? element.transform.position.y : null, + setValue: ({ element, value }) => + isVisualElement(element) + ? { + ...element, + transform: { + ...element.transform, + position: { ...element.transform.position, y: value as number }, + }, + } + : element, + }), + "transform.scaleX": createNumberPropertyDefinition({ + numericRange: { min: MIN_TRANSFORM_SCALE, step: 0.01 }, + supportsElement: ({ element }) => isVisualElement(element), + getValue: ({ element }) => + isVisualElement(element) ? element.transform.scaleX : null, + setValue: ({ element, value }) => + isVisualElement(element) + ? { + ...element, + transform: { ...element.transform, scaleX: value as number }, + } + : element, + }), + "transform.scaleY": createNumberPropertyDefinition({ + numericRange: { min: MIN_TRANSFORM_SCALE, step: 0.01 }, + supportsElement: ({ element }) => isVisualElement(element), + getValue: ({ element }) => + isVisualElement(element) ? element.transform.scaleY : null, + setValue: ({ element, value }) => + isVisualElement(element) + ? { + ...element, + transform: { ...element.transform, scaleY: value as number }, + } + : element, + }), + "transform.rotate": createNumberPropertyDefinition({ + numericRange: { min: -360, max: 360, step: 1 }, + supportsElement: ({ element }) => isVisualElement(element), + getValue: ({ element }) => + isVisualElement(element) ? element.transform.rotate : null, + setValue: ({ element, value }) => + isVisualElement(element) + ? { + ...element, + transform: { ...element.transform, rotate: value as number }, + } + : element, + }), + opacity: createNumberPropertyDefinition({ + numericRange: { min: 0, max: 1, step: 0.01 }, + supportsElement: ({ element }) => isVisualElement(element), + getValue: ({ element }) => + isVisualElement(element) ? element.opacity : null, + setValue: ({ element, value }) => + isVisualElement(element) + ? { ...element, opacity: value as number } + : element, + }), + volume: createNumberPropertyDefinition({ + numericRange: { min: VOLUME_DB_MIN, max: VOLUME_DB_MAX, step: 0.01 }, + supportsElement: ({ element }) => canElementHaveAudio(element), + getValue: ({ element }) => + canElementHaveAudio(element) ? element.volume ?? 0 : null, + setValue: ({ element, value }) => + canElementHaveAudio(element) + ? { ...element, volume: value as number } + : element, + }), + color: { + kind: "color", + defaultInterpolation: "linear", + supportsElement: ({ element }) => element.type === "text", + getValue: ({ element }) => (element.type === "text" ? element.color : null), + coerceValue: ({ value }) => coerceColorValue({ value }), + setValue: ({ element, value }) => + element.type === "text" + ? { ...element, color: value as string } + : element, + }, + "background.color": { + kind: "color", + defaultInterpolation: "linear", + supportsElement: ({ element }) => element.type === "text", + getValue: ({ element }) => + element.type === "text" ? element.background.color : null, + coerceValue: ({ value }) => coerceColorValue({ value }), + setValue: ({ element, value }) => + element.type === "text" + ? { + ...element, + background: { ...element.background, color: value as string }, + } + : element, + }, + "background.paddingX": createNumberPropertyDefinition({ + numericRange: { min: 0, step: 1 }, + supportsElement: ({ element }) => element.type === "text", + getValue: ({ element }) => + element.type === "text" + ? (element.background.paddingX ?? DEFAULTS.text.background.paddingX) + : null, + setValue: ({ element, value }) => + element.type === "text" + ? { + ...element, + background: { ...element.background, paddingX: value as number }, + } + : element, + }), + "background.paddingY": createNumberPropertyDefinition({ + numericRange: { min: 0, step: 1 }, + supportsElement: ({ element }) => element.type === "text", + getValue: ({ element }) => + element.type === "text" + ? (element.background.paddingY ?? DEFAULTS.text.background.paddingY) + : null, + setValue: ({ element, value }) => + element.type === "text" + ? { + ...element, + background: { ...element.background, paddingY: value as number }, + } + : element, + }), + "background.offsetX": createNumberPropertyDefinition({ + numericRange: { step: 1 }, + supportsElement: ({ element }) => element.type === "text", + getValue: ({ element }) => + element.type === "text" + ? (element.background.offsetX ?? DEFAULTS.text.background.offsetX) + : null, + setValue: ({ element, value }) => + element.type === "text" + ? { + ...element, + background: { ...element.background, offsetX: value as number }, + } + : element, + }), + "background.offsetY": createNumberPropertyDefinition({ + numericRange: { step: 1 }, + supportsElement: ({ element }) => element.type === "text", + getValue: ({ element }) => + element.type === "text" + ? (element.background.offsetY ?? DEFAULTS.text.background.offsetY) + : null, + setValue: ({ element, value }) => + element.type === "text" + ? { + ...element, + background: { ...element.background, offsetY: value as number }, + } + : element, + }), + "background.cornerRadius": createNumberPropertyDefinition({ + numericRange: { + min: CORNER_RADIUS_MIN, + max: CORNER_RADIUS_MAX, + step: 1, + }, + supportsElement: ({ element }) => element.type === "text", + getValue: ({ element }) => + element.type === "text" + ? (element.background.cornerRadius ?? CORNER_RADIUS_MIN) + : null, + setValue: ({ element, value }) => + element.type === "text" + ? { + ...element, + background: { ...element.background, cornerRadius: value as number }, + } + : element, + }), +}; + +export function getAnimationPropertyDefinition({ + propertyPath, +}: { + propertyPath: AnimationPropertyPath; +}): AnimationPropertyDefinition { + return ANIMATION_PROPERTY_REGISTRY[propertyPath]; +} + +export function supportsAnimationProperty({ + element, + propertyPath, +}: { + element: TimelineElement; + propertyPath: AnimationPropertyPath; +}): boolean { + const propertyDefinition = getAnimationPropertyDefinition({ propertyPath }); + return propertyDefinition.supportsElement({ element }); +} + +export function getElementBaseValueForProperty({ + element, + propertyPath, +}: { + element: TimelineElement; + propertyPath: AnimationPropertyPath; +}): AnimationValue | null { + const definition = getAnimationPropertyDefinition({ propertyPath }); + if (!definition.supportsElement({ element })) { + return null; + } + + return definition.getValue({ element }); +} + +export function withElementBaseValueForProperty({ + element, + propertyPath, + value, +}: { + element: TimelineElement; + propertyPath: AnimationPropertyPath; + value: AnimationValue; +}): TimelineElement { + const definition = getAnimationPropertyDefinition({ propertyPath }); + const coercedValue = definition.coerceValue({ value }); + if (coercedValue === null || !definition.supportsElement({ element })) { + return element; + } + + return definition.setValue({ element, value: coercedValue }); +} + +export function getDefaultInterpolationForProperty({ + propertyPath, +}: { + propertyPath: AnimationPropertyPath; +}): AnimationInterpolation { + const propertyDefinition = getAnimationPropertyDefinition({ propertyPath }); + return propertyDefinition.defaultInterpolation; +} + +export function coerceAnimationValueForProperty({ + propertyPath, + value, +}: { + propertyPath: AnimationPropertyPath; + value: AnimationValue; +}): AnimationValue | null { + const propertyDefinition = getAnimationPropertyDefinition({ propertyPath }); + return propertyDefinition.coerceValue({ value }); +} + +export function upsertElementKeyframe({ + animations, + propertyPath, + time, + value, + interpolation, + keyframeId, +}: { + animations: ElementAnimations | undefined; + propertyPath: AnimationPropertyPath; + time: MediaTime; + value: AnimationValue; + interpolation?: AnimationInterpolation; + keyframeId?: string; +}): ElementAnimations | undefined { + const coercedValue = coerceAnimationValueForProperty({ + propertyPath, + value, + }); + if (coercedValue === null) { + return animations; + } + + const propertyDefinition = getAnimationPropertyDefinition({ propertyPath }); + return upsertPathKeyframe({ + animations, + propertyPath, + time, + value: coercedValue, + interpolation, + keyframeId, + kind: propertyDefinition.kind, + defaultInterpolation: propertyDefinition.defaultInterpolation, + coerceValue: ({ value: nextValue }) => + coerceAnimationValueForProperty({ + propertyPath, + value: nextValue, + }), + }); +} diff --git a/apps/web/src/animation/timeline-snap-points.ts b/apps/web/src/timeline/animation-snap-points.ts similarity index 100% rename from apps/web/src/animation/timeline-snap-points.ts rename to apps/web/src/timeline/animation-snap-points.ts diff --git a/apps/web/src/animation/target-resolver.ts b/apps/web/src/timeline/animation-targets.ts similarity index 64% rename from apps/web/src/animation/target-resolver.ts rename to apps/web/src/timeline/animation-targets.ts index 32e7917c..f9d53fcc 100644 --- a/apps/web/src/animation/target-resolver.ts +++ b/apps/web/src/timeline/animation-targets.ts @@ -3,30 +3,32 @@ import type { AnimationInterpolation, AnimationPath, AnimationValue, + NumericSpec, } from "@/animation/types"; +import { + coerceAnimationParamValue, + getAnimationParamDefaultInterpolation, + getAnimationParamNumericRange, + getAnimationParamValueKind, +} from "@/animation/animated-params"; import { parseEffectParamPath, - isEffectParamPath, } from "@/animation/effect-param-channel"; import { - isGraphicParamPath, parseGraphicParamPath, } from "@/animation/graphic-param-channel"; -import type { ParamDefinition } from "@/params"; import { effectsRegistry, registerDefaultEffects } from "@/effects"; import { getGraphicDefinition } from "@/graphics"; +import type { ParamDefinition } from "@/params"; import type { TimelineElement } from "@/timeline"; import { isVisualElement } from "@/timeline/element-utils"; -import { snapToStep } from "@/utils/math"; +import { isAnimationPropertyPath } from "@/animation/path"; import { coerceAnimationValueForProperty, getAnimationPropertyDefinition, getElementBaseValueForProperty, - isAnimationPropertyPath, - type NumericSpec, withElementBaseValueForProperty, -} from "./property-registry"; -import { parseColorToLinearRgba } from "./binding-values"; +} from "./animation-properties"; export interface AnimationPathDescriptor { kind: AnimationBindingKind; @@ -37,79 +39,16 @@ export interface AnimationPathDescriptor { setBaseValue: ({ value }: { value: AnimationValue }) => TimelineElement; } -export function getParamValueKind({ - param, -}: { - param: ParamDefinition; -}): AnimationBindingKind { - if (param.type === "number") { - return "number"; - } - - if (param.type === "color") { - return "color"; - } - - return "discrete"; -} - -export function getParamDefaultInterpolation({ - param, -}: { - param: ParamDefinition; -}): AnimationInterpolation { - return param.type === "number" || param.type === "color" ? "linear" : "hold"; -} - -function getParamNumericRange({ +// Number/discrete bindings expose a single component named "value" +// (see binding-values.ts). Multi-component kinds (vector2, color) don't carry +// numeric ranges yet — revisit when one does. +function paramNumericRanges({ param, }: { param: ParamDefinition; }): Partial> | undefined { - if (param.type !== "number") { - return undefined; - } - - return { - value: { - min: param.min, - max: param.max, - step: param.step, - }, - }; -} - -export function coerceAnimationValueForParam({ - param, - value, -}: { - param: ParamDefinition; - value: AnimationValue; -}): number | string | boolean | null { - if (param.type === "number") { - if (typeof value !== "number" || Number.isNaN(value)) { - return null; - } - - const steppedValue = snapToStep({ value, step: param.step }); - const minValue = param.min; - const maxValue = param.max ?? Number.POSITIVE_INFINITY; - return Math.min(maxValue, Math.max(minValue, steppedValue)); - } - - if (param.type === "color") { - return typeof value === "string" ? value : null; - } - - if (param.type === "boolean") { - return typeof value === "boolean" ? value : null; - } - - if (typeof value !== "string") { - return null; - } - - return param.options.some((option) => option.value === value) ? value : null; + const range = getAnimationParamNumericRange({ param }); + return range ? { value: range } : undefined; } function buildGraphicParamDescriptor({ @@ -132,13 +71,13 @@ function buildGraphicParamDescriptor({ } return { - kind: getParamValueKind({ param }), - defaultInterpolation: getParamDefaultInterpolation({ param }), - numericRanges: getParamNumericRange({ param }), - coerceValue: ({ value }) => coerceAnimationValueForParam({ param, value }), + kind: getAnimationParamValueKind({ param }), + defaultInterpolation: getAnimationParamDefaultInterpolation({ param }), + numericRanges: paramNumericRanges({ param }), + coerceValue: ({ value }) => coerceAnimationParamValue({ param, value }), getBaseValue: () => element.params[param.key] ?? param.default, setBaseValue: ({ value }) => { - const coercedValue = coerceAnimationValueForParam({ param, value }); + const coercedValue = coerceAnimationParamValue({ param, value }); if (coercedValue === null) { return element; } @@ -180,13 +119,13 @@ function buildEffectParamDescriptor({ } return { - kind: getParamValueKind({ param }), - defaultInterpolation: getParamDefaultInterpolation({ param }), - numericRanges: getParamNumericRange({ param }), - coerceValue: ({ value }) => coerceAnimationValueForParam({ param, value }), + kind: getAnimationParamValueKind({ param }), + defaultInterpolation: getAnimationParamDefaultInterpolation({ param }), + numericRanges: paramNumericRanges({ param }), + coerceValue: ({ value }) => coerceAnimationParamValue({ param, value }), getBaseValue: () => effect.params[param.key] ?? param.default, setBaseValue: ({ value }) => { - const coercedValue = coerceAnimationValueForParam({ param, value }); + const coercedValue = coerceAnimationParamValue({ param, value }); if (coercedValue === null) { return element; } @@ -210,16 +149,6 @@ function buildEffectParamDescriptor({ }; } -export function isAnimationPath( - propertyPath: string, -): propertyPath is AnimationPath { - return ( - isAnimationPropertyPath(propertyPath) || - isGraphicParamPath(propertyPath) || - isEffectParamPath(propertyPath) - ); -} - export function resolveAnimationTarget({ element, path, diff --git a/apps/web/src/timeline/audio-state.ts b/apps/web/src/timeline/audio-state.ts index 4ef559ea..44df5aaf 100644 --- a/apps/web/src/timeline/audio-state.ts +++ b/apps/web/src/timeline/audio-state.ts @@ -1,5 +1,5 @@ import { hasKeyframesForPath } from "@/animation/keyframe-query"; -import { resolveNumberAtTime } from "@/animation/resolve"; +import { resolveNumberAtTime } from "@/animation/values"; import { VOLUME_DB_MAX, VOLUME_DB_MIN } from "./audio-constants"; import type { TimelineElement } from "./types"; const DEFAULT_STEP_SECONDS = 1 / 60; diff --git a/apps/web/src/timeline/bookmarks/hooks/use-bookmark-drag.ts b/apps/web/src/timeline/bookmarks/hooks/use-bookmark-drag.ts index 343fda80..4c03ef8c 100644 --- a/apps/web/src/timeline/bookmarks/hooks/use-bookmark-drag.ts +++ b/apps/web/src/timeline/bookmarks/hooks/use-bookmark-drag.ts @@ -18,7 +18,7 @@ import { import { getBookmarkSnapPoints } from "../snap-source"; import { getElementEdgeSnapPoints } from "@/timeline/element-snap-source"; import { getPlayheadSnapPoints } from "@/timeline/playhead-snap-source"; -import { getAnimationKeyframeSnapPointsForTimeline } from "@/animation/timeline-snap-points"; +import { getAnimationKeyframeSnapPointsForTimeline } from "@/timeline/animation-snap-points"; import type { Bookmark } from "@/timeline"; import { roundFrameTime, type MediaTime, ZERO_MEDIA_TIME } from "@/wasm"; diff --git a/apps/web/src/timeline/components/audio-tab.tsx b/apps/web/src/timeline/components/audio-tab.tsx index 1dbb3d1a..1e47dcc8 100644 --- a/apps/web/src/timeline/components/audio-tab.tsx +++ b/apps/web/src/timeline/components/audio-tab.tsx @@ -1,151 +1,151 @@ -import { Button } from "@/components/ui/button"; -import { NumberField } from "@/components/ui/number-field"; -import { VOLUME_DB_MAX, VOLUME_DB_MIN } from "@/timeline/audio-constants"; -import { isSourceAudioSeparated } from "@/timeline/audio-separation"; -import { DEFAULTS } from "@/timeline/defaults"; -import { - clamp, - formatNumberForDisplay, - getFractionDigitsForStep, - isNearlyEqual, - snapToStep, -} from "@/utils/math"; -import type { AudioElement, VideoElement } from "@/timeline"; -import { resolveNumberAtTime } from "@/animation"; -import { useEditor } from "@/editor/use-editor"; -import { useElementPlayhead } from "@/components/editor/panels/properties/hooks/use-element-playhead"; -import { useKeyframedNumberProperty } from "@/components/editor/panels/properties/hooks/use-keyframed-number-property"; -import { KeyframeToggle } from "@/components/editor/panels/properties/components/keyframe-toggle"; -import { HugeiconsIcon } from "@hugeicons/react"; -import { VolumeHighIcon } from "@hugeicons/core-free-icons"; -import { - Section, - SectionContent, - SectionField, - SectionFields, - SectionHeader, - SectionTitle, -} from "@/components/section"; - -const VOLUME_STEP = 0.1; -const VOLUME_FRACTION_DIGITS = getFractionDigitsForStep({ step: VOLUME_STEP }); - -export function AudioTab({ - element, - trackId, -}: { - element: AudioElement | VideoElement; - trackId: string; -}) { - const editor = useEditor(); - const { localTime, isPlayheadWithinElementRange } = useElementPlayhead({ - startTime: element.startTime, - duration: element.duration, - }); - const resolvedVolume = resolveNumberAtTime({ - baseValue: element.volume ?? DEFAULTS.element.volume, - animations: element.animations, - propertyPath: "volume", - localTime, - }); - const volume = useKeyframedNumberProperty({ - trackId, - elementId: element.id, - animations: element.animations, - propertyPath: "volume", - localTime, - isPlayheadWithinElementRange, - displayValue: formatNumberForDisplay({ - value: resolvedVolume, - fractionDigits: VOLUME_FRACTION_DIGITS, - }), - parse: (input) => { - const parsed = parseFloat(input); - if (Number.isNaN(parsed)) { - return null; - } - - return clamp({ - value: snapToStep({ value: parsed, step: VOLUME_STEP }), - min: VOLUME_DB_MIN, - max: VOLUME_DB_MAX, - }); - }, - valueAtPlayhead: resolvedVolume, - step: VOLUME_STEP, - buildBaseUpdates: ({ value }) => ({ - volume: value, - }), - }); - const isDefault = - volume.hasAnimatedKeyframes && isPlayheadWithinElementRange - ? isNearlyEqual({ - leftValue: resolvedVolume, - rightValue: DEFAULTS.element.volume, - }) - : (element.volume ?? DEFAULTS.element.volume) === DEFAULTS.element.volume; - const isSeparated = - element.type === "video" && isSourceAudioSeparated({ element }); - - return ( - <> - {isSeparated && ( -
-

Audio has been separated.

- -
- )} -
- - Audio - - - - - } - > - } - value={volume.displayValue} - onFocus={volume.onFocus} - onChange={volume.onChange} - onBlur={volume.onBlur} - dragSensitivity="slow" - scrubClamp={{ min: VOLUME_DB_MIN, max: VOLUME_DB_MAX }} - onScrub={volume.scrubTo} - onScrubEnd={volume.commitScrub} - onReset={() => - volume.commitValue({ - value: DEFAULTS.element.volume, - }) - } - isDefault={isDefault} - suffix="dB" - /> - - - -
- - ); -} +import { Button } from "@/components/ui/button"; +import { NumberField } from "@/components/ui/number-field"; +import { VOLUME_DB_MAX, VOLUME_DB_MIN } from "@/timeline/audio-constants"; +import { isSourceAudioSeparated } from "@/timeline/audio-separation"; +import { DEFAULTS } from "@/timeline/defaults"; +import { + clamp, + formatNumberForDisplay, + getFractionDigitsForStep, + isNearlyEqual, + snapToStep, +} from "@/utils/math"; +import type { AudioElement, VideoElement } from "@/timeline"; +import { resolveNumberAtTime } from "@/animation/values"; +import { useEditor } from "@/editor/use-editor"; +import { useElementPlayhead } from "@/components/editor/panels/properties/hooks/use-element-playhead"; +import { useKeyframedNumberProperty } from "@/components/editor/panels/properties/hooks/use-keyframed-number-property"; +import { KeyframeToggle } from "@/components/editor/panels/properties/components/keyframe-toggle"; +import { HugeiconsIcon } from "@hugeicons/react"; +import { VolumeHighIcon } from "@hugeicons/core-free-icons"; +import { + Section, + SectionContent, + SectionField, + SectionFields, + SectionHeader, + SectionTitle, +} from "@/components/section"; + +const VOLUME_STEP = 0.1; +const VOLUME_FRACTION_DIGITS = getFractionDigitsForStep({ step: VOLUME_STEP }); + +export function AudioTab({ + element, + trackId, +}: { + element: AudioElement | VideoElement; + trackId: string; +}) { + const editor = useEditor(); + const { localTime, isPlayheadWithinElementRange } = useElementPlayhead({ + startTime: element.startTime, + duration: element.duration, + }); + const resolvedVolume = resolveNumberAtTime({ + baseValue: element.volume ?? DEFAULTS.element.volume, + animations: element.animations, + propertyPath: "volume", + localTime, + }); + const volume = useKeyframedNumberProperty({ + trackId, + elementId: element.id, + animations: element.animations, + propertyPath: "volume", + localTime, + isPlayheadWithinElementRange, + displayValue: formatNumberForDisplay({ + value: resolvedVolume, + fractionDigits: VOLUME_FRACTION_DIGITS, + }), + parse: (input) => { + const parsed = parseFloat(input); + if (Number.isNaN(parsed)) { + return null; + } + + return clamp({ + value: snapToStep({ value: parsed, step: VOLUME_STEP }), + min: VOLUME_DB_MIN, + max: VOLUME_DB_MAX, + }); + }, + valueAtPlayhead: resolvedVolume, + step: VOLUME_STEP, + buildBaseUpdates: ({ value }) => ({ + volume: value, + }), + }); + const isDefault = + volume.hasAnimatedKeyframes && isPlayheadWithinElementRange + ? isNearlyEqual({ + leftValue: resolvedVolume, + rightValue: DEFAULTS.element.volume, + }) + : (element.volume ?? DEFAULTS.element.volume) === DEFAULTS.element.volume; + const isSeparated = + element.type === "video" && isSourceAudioSeparated({ element }); + + return ( + <> + {isSeparated && ( +
+

Audio has been separated.

+ +
+ )} +
+ + Audio + + + + + } + > + } + value={volume.displayValue} + onFocus={volume.onFocus} + onChange={volume.onChange} + onBlur={volume.onBlur} + dragSensitivity="slow" + scrubClamp={{ min: VOLUME_DB_MIN, max: VOLUME_DB_MAX }} + onScrub={volume.scrubTo} + onScrubEnd={volume.commitScrub} + onReset={() => + volume.commitValue({ + value: DEFAULTS.element.volume, + }) + } + isDefault={isDefault} + suffix="dB" + /> + + + +
+ + ); +} diff --git a/apps/web/src/timeline/components/index.tsx b/apps/web/src/timeline/components/index.tsx index 68f3f37f..490ad1cc 100644 --- a/apps/web/src/timeline/components/index.tsx +++ b/apps/web/src/timeline/components/index.tsx @@ -778,8 +778,8 @@ function TimelineTrackRows({ const draggingElementIds = useMemo( () => dragView.kind === "dragging" - ? dragView.memberTimeOffsets - : (null as ReadonlyMap | null), + ? dragView.memberTimeOffsets + : (null as ReadonlyMap | null), [dragView], ); const sortedTracks = useMemo(() => { diff --git a/apps/web/src/timeline/controllers/keyframe-drag-controller.ts b/apps/web/src/timeline/controllers/keyframe-drag-controller.ts index 29dfc5dd..e3a9cd96 100644 --- a/apps/web/src/timeline/controllers/keyframe-drag-controller.ts +++ b/apps/web/src/timeline/controllers/keyframe-drag-controller.ts @@ -282,14 +282,14 @@ export class KeyframeDragController { elementId: ref.elementId, propertyPath: ref.propertyPath, keyframeId: ref.keyframeId, - nextTime: clampMediaTime({ - time: addMediaTime({ - a: keyframe.time, - b: mediaTime({ ticks: deltaTicks }), - }), - min: ZERO_MEDIA_TIME, - max: element.duration, + nextTime: clampMediaTime({ + time: addMediaTime({ + a: keyframe.time, + b: mediaTime({ ticks: deltaTicks }), }), + min: ZERO_MEDIA_TIME, + max: element.duration, + }), }), ]; }); @@ -329,8 +329,7 @@ export class KeyframeDragController { ((clientX - this.session.startMouseX) / pixelsPerSecond) * TICKS_PER_SECOND, ); - this.session.deltaTicks = - roundFrameTicks({ ticks: rawDeltaTicks, fps }); + this.session.deltaTicks = roundFrameTicks({ ticks: rawDeltaTicks, fps }); this.notify(); } diff --git a/apps/web/src/timeline/controllers/playhead-controller.ts b/apps/web/src/timeline/controllers/playhead-controller.ts index b96b868b..e914ff21 100644 --- a/apps/web/src/timeline/controllers/playhead-controller.ts +++ b/apps/web/src/timeline/controllers/playhead-controller.ts @@ -13,7 +13,7 @@ import { } from "@/timeline/snapping"; import { getBookmarkSnapPoints } from "@/timeline/bookmarks/index"; import { getElementEdgeSnapPoints } from "@/timeline/element-snap-source"; -import { getAnimationKeyframeSnapPointsForTimeline } from "@/animation/timeline-snap-points"; +import { getAnimationKeyframeSnapPointsForTimeline } from "@/timeline/animation-snap-points"; import { getCenteredLineLeft, timelineTimeToPixels, diff --git a/apps/web/src/timeline/controllers/resize-controller.ts b/apps/web/src/timeline/controllers/resize-controller.ts index 8b0119a0..e8cbaa80 100644 --- a/apps/web/src/timeline/controllers/resize-controller.ts +++ b/apps/web/src/timeline/controllers/resize-controller.ts @@ -24,7 +24,7 @@ import { } from "@/timeline/snapping"; import { getElementEdgeSnapPoints } from "@/timeline/element-snap-source"; import { getPlayheadSnapPoints } from "@/timeline/playhead-snap-source"; -import { getAnimationKeyframeSnapPointsForTimeline } from "@/animation/timeline-snap-points"; +import { getAnimationKeyframeSnapPointsForTimeline } from "@/timeline/animation-snap-points"; import { isRetimableElement, type SceneTracks, diff --git a/apps/web/src/timeline/group-move/snap.ts b/apps/web/src/timeline/group-move/snap.ts index 8b6d196b..7aaf4423 100644 --- a/apps/web/src/timeline/group-move/snap.ts +++ b/apps/web/src/timeline/group-move/snap.ts @@ -7,7 +7,7 @@ import { } from "@/timeline/snapping"; import { getElementEdgeSnapPoints } from "@/timeline/element-snap-source"; import { getPlayheadSnapPoints } from "@/timeline/playhead-snap-source"; -import { getAnimationKeyframeSnapPointsForTimeline } from "@/animation/timeline-snap-points"; +import { getAnimationKeyframeSnapPointsForTimeline } from "@/timeline/animation-snap-points"; import type { MoveGroup } from "./types"; import { addMediaTime, type MediaTime, subMediaTime } from "@/wasm"; diff --git a/apps/web/src/timeline/group-resize/compute-resize.ts b/apps/web/src/timeline/group-resize/compute-resize.ts index 465531bf..ed335997 100644 --- a/apps/web/src/timeline/group-resize/compute-resize.ts +++ b/apps/web/src/timeline/group-resize/compute-resize.ts @@ -131,15 +131,15 @@ function buildResizeUpdate({ return { trackId: member.trackId, elementId: member.elementId, - patch: { - trimStart: maxMediaTime({ - a: ZERO_MEDIA_TIME, - b: addMediaTime({ a: member.trimStart, b: sourceDelta }), - }), - trimEnd: member.trimEnd, - startTime: addMediaTime({ a: member.startTime, b: deltaTime }), - duration: subMediaTime({ a: member.duration, b: deltaTime }), - }, + patch: { + trimStart: maxMediaTime({ + a: ZERO_MEDIA_TIME, + b: addMediaTime({ a: member.trimStart, b: sourceDelta }), + }), + trimEnd: member.trimEnd, + startTime: addMediaTime({ a: member.startTime, b: deltaTime }), + duration: subMediaTime({ a: member.duration, b: deltaTime }), + }, }; } diff --git a/bun.lock b/bun.lock index e342cd5f..e2198a3e 100644 --- a/bun.lock +++ b/bun.lock @@ -10,6 +10,7 @@ "@types/react-dom": "^19.2.3", "better-auth": "^1.4.15", "next": "^16.1.3", + "opencut-wasm": "^0.2.10", }, "devDependencies": { "turbo": "^2.8.20", @@ -54,7 +55,7 @@ "nanoid": "^5.1.5", "next": "16.1.3", "next-themes": "^0.4.4", - "opencut-wasm": "^0.2.9", + "opencut-wasm": "^0.2.10", "pg": "^8.16.2", "postgres": "^3.4.5", "radix-ui": "^1.4.3", @@ -1359,7 +1360,7 @@ "onnxruntime-web": ["onnxruntime-web@1.22.0-dev.20250409-89f8206ba4", "", { "dependencies": { "flatbuffers": "^25.1.24", "guid-typescript": "^1.0.9", "long": "^5.2.3", "onnxruntime-common": "1.22.0-dev.20250409-89f8206ba4", "platform": "^1.3.6", "protobufjs": "^7.2.4" } }, "sha512-0uS76OPgH0hWCPrFKlL8kYVV7ckM7t/36HfbgoFw6Nd0CZVVbQC4PkrR8mBX8LtNUFZO25IQBqV2Hx2ho3FlbQ=="], - "opencut-wasm": ["opencut-wasm@0.2.9", "", {}, "sha512-yZilhRRgNA02XY9Bq2yt6FidbnDQS1OYsvtNczxF6jL+nvl3vRboG0owwpQY0SPIbeJoJjJBuOVy0i1Pp6JS/w=="], + "opencut-wasm": ["opencut-wasm@0.2.10", "", {}, "sha512-dy+Z9SWwpjLjgmTAMQoMMIUmdbUk9OXoWXLoacl9xT9TCrekIGSeMM0F7bJ1H3VJwwUxbGasaVibvn5AfmeZrg=="], "p-limit": ["p-limit@6.2.0", "", { "dependencies": { "yocto-queue": "^1.1.1" } }, "sha512-kuUqqHNUqoIWp/c467RI4X6mmyuojY5jGutNU0wVTmEOOfcuwLqyMVoAi9MKi2Ak+5i9+nhmrK4ufZE8069kHA=="], diff --git a/package.json b/package.json index 22504650..941dd0ad 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,8 @@ "@types/react": "^19.2.10", "@types/react-dom": "^19.2.3", "better-auth": "^1.4.15", - "next": "^16.1.3" + "next": "^16.1.3", + "opencut-wasm": "^0.2.10" }, "devDependencies": { "turbo": "^2.8.20", diff --git a/random_gibberish.txt b/random_gibberish.txt new file mode 100644 index 00000000..6a8c5579 --- /dev/null +++ b/random_gibberish.txt @@ -0,0 +1,9 @@ +bananas are not real +the moon was invented in 1987 by a consortium of cheese manufacturers +every time you blink, a squirrel forgets how to count +this file contains exactly 0 useful information +potatoes dream in hexadecimal +the ocean is just a very large puddle that got too confident +chairs were banned in 1423 but nobody noticed +purple smells like thursday +EOF diff --git a/rust/crates/compositor/src/compositor.rs b/rust/crates/compositor/src/compositor.rs index 84bcceae..99f0e7e9 100644 --- a/rust/crates/compositor/src/compositor.rs +++ b/rust/crates/compositor/src/compositor.rs @@ -348,7 +348,6 @@ impl Compositor { ) -> Result<(), CompositorError> { let frame = options.frame; self.texture_pool.recycle_frame(); - context.configure_surface(options.surface, frame.width, frame.height)?; let surface_texture = context.acquire_surface_texture(options.surface)?; let surface_view = surface_texture .texture diff --git a/rust/crates/gpu/src/context.rs b/rust/crates/gpu/src/context.rs index e38eff8c..da835280 100644 --- a/rust/crates/gpu/src/context.rs +++ b/rust/crates/gpu/src/context.rs @@ -1,5 +1,8 @@ use wgpu::util::DeviceExt; +#[cfg(all(feature = "wasm", target_arch = "wasm32"))] +use std::cell::RefCell; + #[cfg(all(feature = "wasm", target_arch = "wasm32"))] use wasm_bindgen::JsCast; @@ -17,6 +20,12 @@ impl wgpu::rwh::HasDisplayHandle for WebDisplay { } } +#[cfg(all(feature = "wasm", target_arch = "wasm32"))] +struct CachedCanvasSurface { + surface: wgpu::Surface<'static>, + size: (u32, u32), +} + const BLIT_SHADER_SOURCE: &str = include_str!("shaders/blit.wgsl"); const FULLSCREEN_QUAD_POSITIONS: [[f32; 2]; 6] = [ @@ -44,6 +53,8 @@ pub struct GpuContext { /// fallback path. Used by render_texture_via_gl_canvas to output frames on WebGL. #[cfg(all(feature = "wasm", target_arch = "wasm32"))] gl_canvas: Option, + #[cfg(all(feature = "wasm", target_arch = "wasm32"))] + gl_surface: RefCell>, } impl GpuContext { @@ -170,6 +181,8 @@ impl GpuContext { supports_external_texture_copies, #[cfg(all(feature = "wasm", target_arch = "wasm32"))] gl_canvas, + #[cfg(all(feature = "wasm", target_arch = "wasm32"))] + gl_surface: RefCell::new(None), }) } @@ -184,17 +197,15 @@ impl GpuContext { ), GpuError, > { - // Temporary fix: force the wasm renderer onto the WebGL backend even when - // WebGPU is available while a WebGPU bug is being investigated. - // let instance = wgpu::util::new_instance_with_webgpu_detection( - // wgpu::InstanceDescriptor::new_without_display_handle(), - // ) - // .await; + let instance = wgpu::util::new_instance_with_webgpu_detection( + wgpu::InstanceDescriptor::new_without_display_handle(), + ) + .await; - // match Self::try_request_device(&instance, None).await { - // Ok((adapter, device, queue)) => return Ok((instance, adapter, device, queue, None)), - // Err(_) => {} - // } + match Self::try_request_device(&instance, None).await { + Ok((adapter, device, queue)) => return Ok((instance, adapter, device, queue, None)), + Err(_) => {} + } let (gl_instance, adapter, device, queue, canvas) = Self::try_gl_fallback().await?; Ok((gl_instance, adapter, device, queue, Some(canvas))) } @@ -353,6 +364,14 @@ impl GpuContext { self.supports_external_texture_copies } + /// The HTML canvas that owns the backing WebGL context, if running on the + /// WebGL fallback. Callers on that path can mount this canvas directly + /// instead of copying pixels out of it every frame. + #[cfg(all(feature = "wasm", target_arch = "wasm32"))] + pub fn gl_canvas(&self) -> Option<&web_sys::HtmlCanvasElement> { + self.gl_canvas.as_ref() + } + pub fn render_texture_to_surface( &self, texture: &wgpu::Texture, @@ -361,6 +380,14 @@ impl GpuContext { height: u32, ) -> Result<(), GpuError> { self.configure_surface(surface, width, height)?; + self.present_texture_to_surface(texture, surface) + } + + pub fn present_texture_to_surface( + &self, + texture: &wgpu::Texture, + surface: &wgpu::Surface<'_>, + ) -> Result<(), GpuError> { let surface_texture = self.acquire_surface_texture(surface)?; let target_view = surface_texture .texture @@ -382,16 +409,38 @@ impl GpuContext { width: u32, height: u32, ) -> Result<(), GpuError> { - let Some(config) = surface.get_default_config(&self.adapter, width, height) else { - return Err(GpuError::UnsupportedSurfaceFormat); - }; - if config.format != self.texture_format { - return Err(GpuError::UnsupportedSurfaceFormat); - } + let config = self.build_surface_configuration(surface, width, height)?; surface.configure(&self.device, &config); Ok(()) } + fn build_surface_configuration( + &self, + surface: &wgpu::Surface<'_>, + width: u32, + height: u32, + ) -> Result { + let caps = surface.get_capabilities(&self.adapter); + if !caps.formats.contains(&self.texture_format) { + return Err(GpuError::UnsupportedSurfaceFormat); + } + + Ok(wgpu::SurfaceConfiguration { + usage: wgpu::TextureUsages::RENDER_ATTACHMENT, + format: self.texture_format, + width, + height, + present_mode: wgpu::PresentMode::Fifo, + alpha_mode: caps + .alpha_modes + .first() + .copied() + .unwrap_or(wgpu::CompositeAlphaMode::Auto), + view_formats: vec![], + desired_maximum_frame_latency: 2, + }) + } + pub fn acquire_surface_texture( &self, surface: &wgpu::Surface<'_>, @@ -571,7 +620,7 @@ impl GpuContext { } #[cfg(all(feature = "wasm", target_arch = "wasm32"))] - fn render_texture_to_gl_canvas_surface( + pub fn render_texture_to_gl_canvas_surface( &self, texture: &wgpu::Texture, width: u32, @@ -585,57 +634,29 @@ impl GpuContext { gl_canvas.set_width(width); gl_canvas.set_height(height); - let surface = self - .instance - .create_surface(wgpu::SurfaceTarget::Canvas(gl_canvas.clone()))?; - - let caps = surface.get_capabilities(&self.adapter); - let surface_format = if caps.formats.contains(&self.texture_format) { - self.texture_format - } else if !caps.formats.is_empty() { - caps.formats[0] - } else { - return Err(GpuError::UnsupportedSurfaceFormat); + let mut cached_surface = self.gl_surface.borrow_mut(); + let cached_surface = match cached_surface.as_mut() { + Some(cached_surface) => cached_surface, + None => { + let surface = self + .instance + .create_surface(wgpu::SurfaceTarget::Canvas(gl_canvas.clone()))?; + cached_surface.replace(CachedCanvasSurface { + surface, + size: (0, 0), + }); + cached_surface + .as_mut() + .expect("gl_surface cache should exist after initialization") + } }; - if surface_format != self.texture_format { - return Err(GpuError::UnsupportedSurfaceFormat); + if cached_surface.size != (width, height) { + self.configure_surface(&cached_surface.surface, width, height)?; + cached_surface.size = (width, height); } - let config = wgpu::SurfaceConfiguration { - usage: wgpu::TextureUsages::RENDER_ATTACHMENT, - format: surface_format, - width, - height, - present_mode: wgpu::PresentMode::Fifo, - alpha_mode: caps - .alpha_modes - .first() - .copied() - .unwrap_or(wgpu::CompositeAlphaMode::Auto), - view_formats: vec![], - desired_maximum_frame_latency: 2, - }; - surface.configure(&self.device, &config); - - let surface_texture = self.acquire_surface_texture(&surface)?; - let surface_view = surface_texture - .texture - .create_view(&wgpu::TextureViewDescriptor::default()); - - let mut encoder = self - .device - .create_command_encoder(&wgpu::CommandEncoderDescriptor { - label: Some("gpu-gl-canvas-blit-encoder"), - }); - self.encode_texture_blit_to_view( - &mut encoder, - texture, - &surface_view, - "gpu-gl-canvas-blit", - ); - self.queue.submit([encoder.finish()]); - surface_texture.present(); + self.present_texture_to_surface(texture, &cached_surface.surface)?; Ok(gl_canvas) } diff --git a/rust/wasm/Cargo.toml b/rust/wasm/Cargo.toml index f12a8cbc..0f2bfc56 100644 --- a/rust/wasm/Cargo.toml +++ b/rust/wasm/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "opencut-wasm" -version = "0.2.9" +version = "0.2.10" edition = "2024" description = "Shared video editor logic compiled to WebAssembly" repository = "https://github.com/opencut/opencut" @@ -24,7 +24,7 @@ serde-wasm-bindgen = "0.6.5" time = { version = "0.1.0", path = "../crates/time", features = ["wasm"] } wasm-bindgen = "0.2.116" wasm-bindgen-futures = "0.4.66" -web-sys = { version = "0.3.93", features = ["OffscreenCanvas", "HtmlCanvasElement", "CanvasRenderingContext2d", "Document", "Window"] } +web-sys = { version = "0.3.93", features = ["OffscreenCanvas", "HtmlCanvasElement", "CanvasRenderingContext2d", "Document", "Window", "Performance"] } [features] default = ["wasm"] diff --git a/rust/wasm/src/compositor.rs b/rust/wasm/src/compositor.rs index 126c7468..7652b3f3 100644 --- a/rust/wasm/src/compositor.rs +++ b/rust/wasm/src/compositor.rs @@ -11,10 +11,13 @@ use crate::gpu::{ import_canvas_texture, read_offscreen_canvas_property, read_serde_property, read_u32_property, with_gpu_runtime, }; +use crate::perf; struct CompositorRuntime { canvas: web_sys::HtmlCanvasElement, compositor: Compositor, + surface: wgpu::Surface<'static>, + surface_size: (u32, u32), } thread_local! { @@ -24,20 +27,42 @@ thread_local! { #[wasm_bindgen(js_name = initCompositor)] pub fn init_compositor(width: u32, height: u32) -> Result<(), JsValue> { with_gpu_runtime(|gpu_runtime| { - let document = web_sys::window() - .and_then(|window| window.document()) - .ok_or_else(|| JsValue::from_str("Document is not available"))?; - let canvas: web_sys::HtmlCanvasElement = document - .create_element("canvas")? - .dyn_into() - .map_err(|_| JsValue::from_str("Failed to create compositor canvas"))?; + // On WebGL, wgpu is bound to a specific canvas; reuse it so the UI + // can mount the output directly instead of copying pixels through + // an intermediate 2D canvas every frame. On WebGPU, surface rendering + // works against any canvas so we create a fresh one. + let canvas = if let Some(gl_canvas) = gpu_runtime.context.gl_canvas() { + gl_canvas.clone() + } else { + let document = web_sys::window() + .and_then(|window| window.document()) + .ok_or_else(|| JsValue::from_str("Document is not available"))?; + document + .create_element("canvas")? + .dyn_into::() + .map_err(|_| JsValue::from_str("Failed to create compositor canvas"))? + }; canvas.set_width(width); canvas.set_height(height); let compositor = Compositor::new(&gpu_runtime.context); + let surface = gpu_runtime + .context + .instance() + .create_surface(wgpu::SurfaceTarget::Canvas(canvas.clone())) + .map_err(|error| JsValue::from_str(&error.to_string()))?; + gpu_runtime + .context + .configure_surface(&surface, width, height) + .map_err(|error| JsValue::from_str(&error.to_string()))?; COMPOSITOR_RUNTIME.with(|runtime| { - runtime.replace(Some(CompositorRuntime { canvas, compositor })); + runtime.replace(Some(CompositorRuntime { + canvas, + compositor, + surface, + surface_size: (width, height), + })); }); Ok(()) @@ -46,16 +71,25 @@ pub fn init_compositor(width: u32, height: u32) -> Result<(), JsValue> { #[wasm_bindgen(js_name = resizeCompositor)] pub fn resize_compositor(width: u32, height: u32) -> Result<(), JsValue> { - COMPOSITOR_RUNTIME.with(|runtime| { - let mut borrow = runtime.borrow_mut(); - let Some(runtime) = borrow.as_mut() else { - return Err(JsValue::from_str( - "Compositor is not initialized. Call initCompositor() first.", - )); - }; - runtime.canvas.set_width(width); - runtime.canvas.set_height(height); - Ok(()) + with_gpu_runtime(|gpu_runtime| { + COMPOSITOR_RUNTIME.with(|runtime| { + let mut borrow = runtime.borrow_mut(); + let Some(runtime) = borrow.as_mut() else { + return Err(JsValue::from_str( + "Compositor is not initialized. Call initCompositor() first.", + )); + }; + runtime.canvas.set_width(width); + runtime.canvas.set_height(height); + if runtime.surface_size != (width, height) { + gpu_runtime + .context + .configure_surface(&runtime.surface, width, height) + .map_err(|error| JsValue::from_str(&error.to_string()))?; + runtime.surface_size = (width, height); + } + Ok(()) + }) }) } @@ -119,8 +153,12 @@ pub fn release_texture(id: String) -> Result<(), JsValue> { #[wasm_bindgen(js_name = renderFrame)] pub fn render_frame(options: JsValue) -> Result<(), JsValue> { + perf::reset(); + + let t_deserialize = perf::now_ms(); let frame: FrameDescriptor = serde_wasm_bindgen::from_value(options) .map_err(|error| JsValue::from_str(&format!("Invalid frame descriptor: {error}")))?; + perf::record("wasm.deserialize", perf::now_ms() - t_deserialize); with_gpu_runtime(|gpu_runtime| { COMPOSITOR_RUNTIME.with(|runtime| { @@ -131,40 +169,50 @@ pub fn render_frame(options: JsValue) -> Result<(), JsValue> { )); }; - if gpu_runtime.context.supports_surface_rendering() { - let surface = gpu_runtime + if runtime.surface_size != (frame.width, frame.height) { + runtime.canvas.set_width(frame.width); + runtime.canvas.set_height(frame.height); + let t_surface = perf::now_ms(); + gpu_runtime .context - .instance() - .create_surface(wgpu::SurfaceTarget::Canvas(runtime.canvas.clone())) + .configure_surface(&runtime.surface, frame.width, frame.height) .map_err(|error| JsValue::from_str(&error.to_string()))?; + perf::record("wasm.surfaceConfigure", perf::now_ms() - t_surface); + runtime.surface_size = (frame.width, frame.height); + } - runtime + if gpu_runtime.context.supports_surface_rendering() { + let t_render = perf::now_ms(); + let result = runtime .compositor .render_frame( &gpu_runtime.context, RenderFrameOptions { frame: &frame, - surface: &surface, + surface: &runtime.surface, }, ) - .map_err(|error| JsValue::from_str(&error.to_string())) + .map_err(|error| JsValue::from_str(&error.to_string())); + perf::record("wasm.renderFrameToSurface", perf::now_ms() - t_render); + result } else { - // WebGL cannot surface-render to an arbitrary canvas element. - // Composite to a texture, then blit to the canvas via the 2D context. + // WebGL still needs a separate composition pass, but the output + // surface is now persistent just like the WebGPU path. + let t_composite = perf::now_ms(); let texture = runtime .compositor .render_frame_to_texture(&gpu_runtime.context, &frame) .map_err(|error| JsValue::from_str(&error.to_string()))?; + perf::record("wasm.compositeToTexture", perf::now_ms() - t_composite); + let t_present = perf::now_ms(); gpu_runtime .context - .render_texture_via_gl_canvas( - &texture, - &runtime.canvas, - frame.width, - frame.height, - ) - .map_err(|error| JsValue::from_str(&error.to_string())) + .present_texture_to_surface(&texture, &runtime.surface) + .map_err(|error| JsValue::from_str(&error.to_string()))?; + perf::record("wasm.presentToSurface", perf::now_ms() - t_present); + + Ok(()) } }) }) diff --git a/rust/wasm/src/perf.rs b/rust/wasm/src/perf.rs new file mode 100644 index 00000000..0c346a4d --- /dev/null +++ b/rust/wasm/src/perf.rs @@ -0,0 +1,51 @@ +#![cfg(target_arch = "wasm32")] + +//! Per-frame profile buffer for the render pipeline. +//! +//! Sub-span timings are recorded into a thread-local during `renderFrame` +//! and drained by JS via `getLastFrameProfile()`. + +use std::cell::RefCell; + +use js_sys::{Array, Object, Reflect}; +use wasm_bindgen::{JsValue, prelude::wasm_bindgen}; + +thread_local! { + static LAST_FRAME_PROFILE: RefCell> = const { RefCell::new(Vec::new()) }; +} + +pub(crate) fn now_ms() -> f64 { + web_sys::window() + .and_then(|window| window.performance()) + .map(|performance| performance.now()) + .unwrap_or(0.0) +} + +pub(crate) fn reset() { + LAST_FRAME_PROFILE.with(|cell| cell.borrow_mut().clear()); +} + +pub(crate) fn record(name: &'static str, duration_ms: f64) { + LAST_FRAME_PROFILE.with(|cell| cell.borrow_mut().push((name, duration_ms))); +} + +#[wasm_bindgen(js_name = getLastFrameProfile)] +pub fn get_last_frame_profile() -> Array { + LAST_FRAME_PROFILE.with(|cell| { + let entries = cell.borrow(); + let array = Array::new_with_length(entries.len() as u32); + for (index, (name, duration_ms)) in entries.iter().enumerate() { + let entry = Object::new(); + Reflect::set(&entry, &JsValue::from_str("name"), &JsValue::from_str(name)) + .expect("set name"); + Reflect::set( + &entry, + &JsValue::from_str("durationMs"), + &JsValue::from_f64(*duration_ms), + ) + .expect("set durationMs"); + array.set(index as u32, entry.into()); + } + array + }) +} diff --git a/rust/wasm/src/wasm.rs b/rust/wasm/src/wasm.rs index 1a33092a..0687ff4b 100644 --- a/rust/wasm/src/wasm.rs +++ b/rust/wasm/src/wasm.rs @@ -6,6 +6,8 @@ mod effects; mod gpu; #[cfg(target_arch = "wasm32")] mod masks; +#[cfg(target_arch = "wasm32")] +mod perf; #[cfg(target_arch = "wasm32")] pub use compositor::*; @@ -15,4 +17,6 @@ pub use effects::*; pub use gpu::*; #[cfg(target_arch = "wasm32")] pub use masks::*; +#[cfg(target_arch = "wasm32")] +pub use perf::*; pub use time::*;