From cad88ee4d1c9e06e7529b88ef5e5d3fdf3f89149 Mon Sep 17 00:00:00 2001 From: Maze Winther Date: Mon, 13 Apr 2026 04:41:17 +0200 Subject: [PATCH] refactor: rendering pipeline Made-with: Cursor --- apps/web/src/lib/preview/element-bounds.ts | 15 - .../src/services/renderer/canvas-renderer.ts | 44 +- .../renderer/compositor/frame-descriptor.ts | 559 ++++++++++++++++++ .../src/services/renderer/compositor/types.ts | 42 ++ .../renderer/compositor/wasm-compositor.ts | 142 +++++ .../web/src/services/renderer/gpu-renderer.ts | 6 + .../src/services/renderer/nodes/base-node.ts | 27 +- .../renderer/nodes/blur-background-node.ts | 136 +---- .../src/services/renderer/nodes/color-node.ts | 26 +- .../renderer/nodes/effect-layer-node.ts | 78 +-- .../services/renderer/nodes/graphic-node.ts | 166 +++--- .../src/services/renderer/nodes/image-node.ts | 37 +- .../services/renderer/nodes/sticker-node.ts | 49 +- .../src/services/renderer/nodes/text-node.ts | 380 ++++-------- .../src/services/renderer/nodes/video-node.ts | 41 +- .../services/renderer/nodes/visual-node.ts | 294 +-------- apps/web/src/services/renderer/resolve.ts | 468 +++++++++++++++ .../src/services/renderer/scene-builder.ts | 530 ++++++++--------- .../src/services/renderer/scene-exporter.ts | 2 +- 19 files changed, 1778 insertions(+), 1264 deletions(-) create mode 100644 apps/web/src/services/renderer/compositor/frame-descriptor.ts create mode 100644 apps/web/src/services/renderer/compositor/types.ts create mode 100644 apps/web/src/services/renderer/compositor/wasm-compositor.ts create mode 100644 apps/web/src/services/renderer/resolve.ts diff --git a/apps/web/src/lib/preview/element-bounds.ts b/apps/web/src/lib/preview/element-bounds.ts index 40145d02..0dc2d0d5 100644 --- a/apps/web/src/lib/preview/element-bounds.ts +++ b/apps/web/src/lib/preview/element-bounds.ts @@ -244,21 +244,6 @@ export function getEdgeHandlePosition({ }; } -export function getRotationHandlePosition({ - bounds, -}: { - bounds: ElementBounds; -}): { x: number; y: number } { - const angleRad = (bounds.rotation * Math.PI) / 180; - const cos = Math.cos(angleRad); - const sin = Math.sin(angleRad); - const localY = -bounds.height / 2 - ROTATION_HANDLE_OFFSET; - return { - x: bounds.cx - localY * sin, - y: bounds.cy + localY * cos, - }; -} - export function getVisibleElementsWithBounds({ tracks, currentTime, diff --git a/apps/web/src/services/renderer/canvas-renderer.ts b/apps/web/src/services/renderer/canvas-renderer.ts index b142d97a..50f7fd12 100644 --- a/apps/web/src/services/renderer/canvas-renderer.ts +++ b/apps/web/src/services/renderer/canvas-renderer.ts @@ -1,6 +1,8 @@ import type { FrameRate } from "opencut-wasm"; -import { frameRateToFloat } from "@/lib/fps/utils"; -import type { BaseNode } from "./nodes/base-node"; +import type { AnyBaseNode } from "./nodes/base-node"; +import { buildFrameDescriptor } from "./compositor/frame-descriptor"; +import { wasmCompositor } from "./compositor/wasm-compositor"; +import { resolveRenderTree } from "./resolve"; export type CanvasRendererParams = { width: number; @@ -38,6 +40,14 @@ export class CanvasRenderer { | CanvasRenderingContext2D; } + getOutputCanvas(): HTMLCanvasElement { + wasmCompositor.ensureInitialized({ + width: this.width, + height: this.height, + }); + return wasmCompositor.getCanvas(); + } + setSize({ width, height }: { width: number; height: number }) { this.width = width; this.height = height; @@ -58,14 +68,18 @@ export class CanvasRenderer { | CanvasRenderingContext2D; } - private clear() { - this.context.fillStyle = "black"; - this.context.fillRect(0, 0, this.canvas.width, this.canvas.height); - } - - async render({ node, time }: { node: BaseNode; time: number }) { - this.clear(); - await node.render({ renderer: this, time }); + async render({ node, time }: { node: AnyBaseNode; time: number }) { + await resolveRenderTree({ node, renderer: this, time }); + const { frame, textures } = await buildFrameDescriptor({ + node, + renderer: this, + }); + wasmCompositor.ensureInitialized({ + width: this.width, + height: this.height, + }); + wasmCompositor.syncTextures(textures); + wasmCompositor.render(frame); } async renderToCanvas({ @@ -73,7 +87,7 @@ export class CanvasRenderer { time, targetCanvas, }: { - node: BaseNode; + node: AnyBaseNode; time: number; targetCanvas: HTMLCanvasElement; }) { @@ -84,6 +98,12 @@ export class CanvasRenderer { throw new Error("Failed to get target canvas context"); } - ctx.drawImage(this.canvas, 0, 0, targetCanvas.width, targetCanvas.height); + ctx.drawImage( + wasmCompositor.getCanvas(), + 0, + 0, + targetCanvas.width, + targetCanvas.height, + ); } } diff --git a/apps/web/src/services/renderer/compositor/frame-descriptor.ts b/apps/web/src/services/renderer/compositor/frame-descriptor.ts new file mode 100644 index 00000000..8b8201d5 --- /dev/null +++ b/apps/web/src/services/renderer/compositor/frame-descriptor.ts @@ -0,0 +1,559 @@ +import { drawCssBackground } from "@/lib/gradients"; +import { masksRegistry } from "@/lib/masks"; +import type { AnyBaseNode } from "../nodes/base-node"; +import type { CanvasRenderer } from "../canvas-renderer"; +import { createOffscreenCanvas } from "../canvas-utils"; +import { BlurBackgroundNode } from "../nodes/blur-background-node"; +import { ColorNode } from "../nodes/color-node"; +import { EffectLayerNode } from "../nodes/effect-layer-node"; +import { GraphicNode, type ResolvedGraphicNodeState } from "../nodes/graphic-node"; +import { ImageNode } from "../nodes/image-node"; +import { RootNode } from "../nodes/root-node"; +import { StickerNode } from "../nodes/sticker-node"; +import { renderTextToContext, TextNode } from "../nodes/text-node"; +import { VideoNode } from "../nodes/video-node"; +import type { ResolvedVisualSourceNodeState } from "../nodes/visual-node"; +import type { + FrameDescriptor, + FrameItemDescriptor, + LayerMaskDescriptor, + QuadTransformDescriptor, +} from "./types"; +import { DEFAULT_GRAPHIC_SOURCE_SIZE } from "@/lib/graphics"; + +export type TextureUploadDescriptor = { + id: string; + source: CanvasImageSource; + width: number; + height: number; +}; + +export async function buildFrameDescriptor({ + node, + renderer, +}: { + node: AnyBaseNode; + renderer: CanvasRenderer; +}): Promise<{ + frame: FrameDescriptor; + textures: TextureUploadDescriptor[]; +}> { + const items: FrameItemDescriptor[] = []; + const textures = new Map(); + + await collectNode({ + node, + renderer, + path: "root", + items, + textures, + }); + + return { + frame: { + width: renderer.width, + height: renderer.height, + clear: { + color: [0, 0, 0, 1], + }, + items, + }, + textures: [...textures.values()], + }; +} + +async function collectNode({ + node, + renderer, + path, + items, + textures, +}: { + node: AnyBaseNode; + renderer: CanvasRenderer; + path: string; + items: FrameItemDescriptor[]; + textures: Map; +}): Promise { + if (node instanceof RootNode) { + for (let index = 0; index < node.children.length; index++) { + await collectNode({ + node: node.children[index], + renderer, + path: `${path}:${index}`, + items, + textures, + }); + } + return; + } + + if (node instanceof ColorNode) { + const textureId = `${path}:color`; + const canvas = createOffscreenCanvas({ + width: renderer.width, + height: renderer.height, + }); + const ctx = canvas.getContext("2d") as + | CanvasRenderingContext2D + | OffscreenCanvasRenderingContext2D + | null; + if (!ctx) return; + if (/gradient\(/i.test(node.params.color)) { + drawCssBackground({ + ctx, + width: renderer.width, + height: renderer.height, + css: node.params.color, + }); + } else { + ctx.fillStyle = node.params.color; + ctx.fillRect(0, 0, renderer.width, renderer.height); + } + textures.set(textureId, { + id: textureId, + source: canvas, + width: renderer.width, + height: renderer.height, + }); + items.push({ + type: "layer", + textureId, + transform: fullCanvasTransform(renderer), + opacity: 1, + blendMode: "normal", + effectPassGroups: [], + mask: null, + }); + return; + } + + if (node instanceof EffectLayerNode) { + if (!node.resolved || node.resolved.passes.length === 0) { + return; + } + items.push({ + type: "sceneEffect", + effectPassGroups: [node.resolved.passes], + }); + return; + } + + if (node instanceof BlurBackgroundNode) { + if (!node.resolved) { + return; + } + const textureId = `${path}:blur-background`; + const backdropCanvas = createOffscreenCanvas({ + width: renderer.width, + height: renderer.height, + }); + const backdropCtx = backdropCanvas.getContext("2d") as + | CanvasRenderingContext2D + | OffscreenCanvasRenderingContext2D + | null; + if (!backdropCtx) return; + const { backdropSource, passes } = node.resolved; + const coverScale = Math.max( + renderer.width / backdropSource.width, + renderer.height / backdropSource.height, + ); + const scaledWidth = backdropSource.width * coverScale; + const scaledHeight = backdropSource.height * coverScale; + const offsetX = (renderer.width - scaledWidth) / 2; + const offsetY = (renderer.height - scaledHeight) / 2; + backdropCtx.drawImage( + backdropSource.source, + offsetX, + offsetY, + scaledWidth, + scaledHeight, + ); + textures.set(textureId, { + id: textureId, + source: backdropCanvas, + width: renderer.width, + height: renderer.height, + }); + items.push({ + type: "layer", + textureId, + transform: fullCanvasTransform(renderer), + opacity: 1, + blendMode: "normal", + effectPassGroups: [passes], + mask: null, + }); + return; + } + + if ( + node instanceof VideoNode || + node instanceof ImageNode || + node instanceof StickerNode || + node instanceof GraphicNode + ) { + await collectVisualSourceNode({ + node, + renderer, + path, + items, + textures, + }); + return; + } + + if (node instanceof TextNode) { + collectTextNode({ + node, + renderer, + path, + items, + textures, + }); + } +} + +async function collectVisualSourceNode({ + node, + renderer, + path, + items, + textures, +}: { + node: VideoNode | ImageNode | StickerNode | GraphicNode; + renderer: CanvasRenderer; + path: string; + items: FrameItemDescriptor[]; + textures: Map; +}) { + if (!node.resolved) { + return; + } + + const source = + node instanceof GraphicNode + ? node.getSource({ resolvedParams: node.resolved.resolvedParams }) + : node.resolved.source; + if (!source) { + return; + } + + const sourceWidth = + node instanceof GraphicNode + ? DEFAULT_GRAPHIC_SOURCE_SIZE + : (node.resolved as ResolvedVisualSourceNodeState).sourceWidth; + const sourceHeight = + node instanceof GraphicNode + ? DEFAULT_GRAPHIC_SOURCE_SIZE + : (node.resolved as ResolvedVisualSourceNodeState).sourceHeight; + + const textureId = `${path}:source`; + textures.set(textureId, { + id: textureId, + source, + width: sourceWidth, + height: sourceHeight, + }); + + const transform = computeVisualTransform({ + renderer, + resolved: node.resolved, + sourceWidth, + sourceHeight, + }); + const { mask, strokeLayer } = buildMaskArtifacts({ + node, + renderer, + path, + transform, + textures, + }); + + items.push({ + type: "layer", + textureId, + transform, + opacity: node.resolved.opacity, + blendMode: node.params.blendMode ?? "normal", + effectPassGroups: node.resolved.effectPasses, + mask, + }); + if (strokeLayer) { + items.push(strokeLayer); + } +} + +function collectTextNode({ + node, + renderer, + path, + items, + textures, +}: { + node: TextNode; + renderer: CanvasRenderer; + path: string; + items: FrameItemDescriptor[]; + textures: Map; +}) { + if (!node.resolved) { + return; + } + + const textureId = `${path}:text`; + const canvas = createOffscreenCanvas({ + width: renderer.width, + height: renderer.height, + }); + const ctx = canvas.getContext("2d") as + | CanvasRenderingContext2D + | OffscreenCanvasRenderingContext2D + | null; + if (!ctx) { + return; + } + + renderTextToContext({ + node, + ctx, + }); + + textures.set(textureId, { + id: textureId, + source: canvas, + width: renderer.width, + height: renderer.height, + }); + items.push({ + type: "layer", + textureId, + transform: fullCanvasTransform(renderer), + opacity: node.resolved.opacity, + blendMode: node.params.blendMode ?? "normal", + effectPassGroups: node.resolved.effectPasses, + mask: null, + }); +} + +function computeVisualTransform({ + renderer, + resolved, + sourceWidth, + sourceHeight, +}: { + renderer: CanvasRenderer; + resolved: ResolvedVisualSourceNodeState | ResolvedGraphicNodeState; + sourceWidth: number; + sourceHeight: number; +}): QuadTransformDescriptor { + const containScale = Math.min( + renderer.width / sourceWidth, + renderer.height / sourceHeight, + ); + const scaledWidth = sourceWidth * containScale * resolved.transform.scaleX; + const scaledHeight = sourceHeight * containScale * resolved.transform.scaleY; + const absWidth = Math.abs(scaledWidth); + const absHeight = Math.abs(scaledHeight); + + return { + centerX: renderer.width / 2 + resolved.transform.position.x, + centerY: renderer.height / 2 + resolved.transform.position.y, + width: absWidth, + height: absHeight, + rotationDegrees: resolved.transform.rotate, + flipX: scaledWidth < 0, + flipY: scaledHeight < 0, + }; +} + +function fullCanvasTransform(renderer: CanvasRenderer): QuadTransformDescriptor { + return { + centerX: renderer.width / 2, + centerY: renderer.height / 2, + width: renderer.width, + height: renderer.height, + rotationDegrees: 0, + flipX: false, + flipY: false, + }; +} + +function buildMaskArtifacts({ + node, + renderer, + path, + transform, + textures, +}: { + node: VideoNode | ImageNode | StickerNode | GraphicNode; + renderer: CanvasRenderer; + path: string; + transform: QuadTransformDescriptor; + textures: Map; +}): { + mask: LayerMaskDescriptor | null; + strokeLayer: FrameItemDescriptor | null; +} { + const mask = node.params.masks?.[0]; + if (!mask) { + return { mask: null, strokeLayer: null }; + } + + const definition = masksRegistry.get(mask.type); + const elementMaskCanvas = createOffscreenCanvas({ + width: Math.round(transform.width), + height: Math.round(transform.height), + }); + const elementMaskCtx = elementMaskCanvas.getContext("2d") as + | CanvasRenderingContext2D + | OffscreenCanvasRenderingContext2D + | null; + if (!elementMaskCtx) { + return { mask: null, strokeLayer: null }; + } + elementMaskCtx.clearRect(0, 0, transform.width, transform.height); + + let strokePath: Path2D | null = null; + let feather = mask.params.feather; + if (mask.params.feather > 0 && definition.renderer.renderMask) { + definition.renderer.renderMask({ + resolvedParams: mask.params, + ctx: elementMaskCtx, + width: Math.round(transform.width), + height: Math.round(transform.height), + feather: mask.params.feather, + }); + feather = 0; + strokePath = definition.renderer.buildStrokePath?.({ + resolvedParams: mask.params, + width: transform.width, + height: transform.height, + }) ?? null; + } else { + const path2d = definition.renderer.buildPath({ + resolvedParams: mask.params, + width: transform.width, + height: transform.height, + }); + elementMaskCtx.fillStyle = "white"; + elementMaskCtx.fill(path2d); + strokePath = + definition.renderer.buildStrokePath?.({ + resolvedParams: mask.params, + width: transform.width, + height: transform.height, + }) ?? path2d; + } + + const fullMaskCanvas = createOffscreenCanvas({ + width: renderer.width, + height: renderer.height, + }); + const fullMaskCtx = fullMaskCanvas.getContext("2d") as + | CanvasRenderingContext2D + | OffscreenCanvasRenderingContext2D + | null; + if (!fullMaskCtx) { + return { mask: null, strokeLayer: null }; + } + drawTransformedCanvas({ + ctx: fullMaskCtx, + source: elementMaskCanvas, + transform, + }); + + const maskTextureId = `${path}:mask`; + textures.set(maskTextureId, { + id: maskTextureId, + source: fullMaskCanvas, + width: renderer.width, + height: renderer.height, + }); + + let strokeLayer: FrameItemDescriptor | null = null; + if (mask.params.strokeWidth > 0 && strokePath) { + const strokeCanvas = createOffscreenCanvas({ + width: Math.round(transform.width), + height: Math.round(transform.height), + }); + const strokeCtx = strokeCanvas.getContext("2d") as + | CanvasRenderingContext2D + | OffscreenCanvasRenderingContext2D + | null; + if (strokeCtx) { + strokeCtx.strokeStyle = mask.params.strokeColor; + strokeCtx.lineWidth = mask.params.strokeWidth; + strokeCtx.stroke(strokePath); + + const fullStrokeCanvas = createOffscreenCanvas({ + width: renderer.width, + height: renderer.height, + }); + const fullStrokeCtx = fullStrokeCanvas.getContext("2d") as + | CanvasRenderingContext2D + | OffscreenCanvasRenderingContext2D + | null; + if (fullStrokeCtx) { + drawTransformedCanvas({ + ctx: fullStrokeCtx, + source: strokeCanvas, + transform, + }); + const strokeTextureId = `${path}:mask-stroke`; + textures.set(strokeTextureId, { + id: strokeTextureId, + source: fullStrokeCanvas, + width: renderer.width, + height: renderer.height, + }); + strokeLayer = { + type: "layer", + textureId: strokeTextureId, + transform: fullCanvasTransform(renderer), + opacity: 1, + blendMode: "normal", + effectPassGroups: [], + mask: null, + }; + } + } + } + + return { + mask: { + textureId: maskTextureId, + feather, + inverted: mask.params.inverted, + }, + strokeLayer, + }; +} + +function drawTransformedCanvas({ + ctx, + source, + transform, +}: { + ctx: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D; + source: CanvasImageSource; + transform: QuadTransformDescriptor; +}) { + const x = transform.centerX - transform.width / 2; + const y = transform.centerY - transform.height / 2; + const flipX = transform.flipX ? -1 : 1; + const flipY = transform.flipY ? -1 : 1; + const requiresTransform = + transform.rotationDegrees !== 0 || flipX !== 1 || flipY !== 1; + + ctx.save(); + if (requiresTransform) { + ctx.translate(transform.centerX, transform.centerY); + ctx.rotate((transform.rotationDegrees * Math.PI) / 180); + ctx.scale(flipX, flipY); + ctx.translate(-transform.centerX, -transform.centerY); + } + ctx.drawImage(source, x, y, transform.width, transform.height); + ctx.restore(); +} + diff --git a/apps/web/src/services/renderer/compositor/types.ts b/apps/web/src/services/renderer/compositor/types.ts new file mode 100644 index 00000000..5d9322e8 --- /dev/null +++ b/apps/web/src/services/renderer/compositor/types.ts @@ -0,0 +1,42 @@ +import type { BlendMode } from "@/lib/rendering"; +import type { EffectPass } from "@/lib/effects/types"; + +export type FrameDescriptor = { + width: number; + height: number; + clear: { + color: [number, number, number, number]; + }; + items: FrameItemDescriptor[]; +}; + +export type FrameItemDescriptor = + | { + type: "layer"; + textureId: string; + transform: QuadTransformDescriptor; + opacity: number; + blendMode: BlendMode; + effectPassGroups: EffectPass[][]; + mask: LayerMaskDescriptor | null; + } + | { + type: "sceneEffect"; + effectPassGroups: EffectPass[][]; + }; + +export type QuadTransformDescriptor = { + centerX: number; + centerY: number; + width: number; + height: number; + rotationDegrees: number; + flipX: boolean; + flipY: boolean; +}; + +export type LayerMaskDescriptor = { + textureId: string; + feather: number; + inverted: boolean; +}; diff --git a/apps/web/src/services/renderer/compositor/wasm-compositor.ts b/apps/web/src/services/renderer/compositor/wasm-compositor.ts new file mode 100644 index 00000000..1faecc32 --- /dev/null +++ b/apps/web/src/services/renderer/compositor/wasm-compositor.ts @@ -0,0 +1,142 @@ +import { + getCompositorCanvas, + initCompositor, + releaseTexture, + renderFrame, + resizeCompositor, + uploadTexture, +} from "opencut-wasm"; +import type { FrameDescriptor } from "./types"; + +function ensureOffscreenCanvas({ + source, + width, + height, + label, +}: { + source: CanvasImageSource; + width: number; + height: number; + label: string; +}): OffscreenCanvas { + if (source instanceof OffscreenCanvas) { + return source; + } + + if (typeof OffscreenCanvas === "undefined") { + throw new Error(`OffscreenCanvas is required for ${label}`); + } + + const canvas = new OffscreenCanvas(width, height); + const context = canvas.getContext("2d"); + if (!context) { + throw new Error(`Failed to get 2d context for ${label}`); + } + context.clearRect(0, 0, width, height); + context.drawImage(source, 0, 0, width, height); + return canvas; +} + +export type TextureUploadDescriptor = { + id: string; + source: CanvasImageSource; + width: number; + height: number; +}; + +class WasmCompositor { + private canvas: HTMLCanvasElement | null = null; + private initializedSize: { width: number; height: number } | null = null; + private retainedTextureIds = new Set(); + private uploadedTextures = new Map< + string, + { source: CanvasImageSource; width: number; height: number } + >(); + private _debugLogged = false; + + ensureInitialized({ width, height }: { width: number; height: number }) { + if (!this.canvas) { + initCompositor(width, height); + this.canvas = getCompositorCanvas(); + this.initializedSize = { width, height }; + return; + } + + if ( + !this.initializedSize || + this.initializedSize.width !== width || + this.initializedSize.height !== height + ) { + resizeCompositor(width, height); + this.initializedSize = { width, height }; + } + } + + getCanvas(): HTMLCanvasElement { + if (!this.canvas) { + throw new Error("Compositor is not initialized"); + } + return this.canvas; + } + + syncTextures(textures: TextureUploadDescriptor[]) { + const nextIds = new Set(textures.map((texture) => texture.id)); + for (const previousId of this.retainedTextureIds) { + if (!nextIds.has(previousId)) { + releaseTexture(previousId); + this.uploadedTextures.delete(previousId); + } + } + + for (const texture of textures) { + const previousTexture = this.uploadedTextures.get(texture.id); + if ( + previousTexture?.source === texture.source && + previousTexture.width === texture.width && + previousTexture.height === texture.height + ) { + continue; + } + + const sourceCanvas = ensureOffscreenCanvas({ + source: texture.source, + width: texture.width, + height: texture.height, + label: `texture upload ${texture.id}`, + }); + uploadTexture({ + id: texture.id, + source: sourceCanvas, + width: texture.width, + height: texture.height, + }); + this.uploadedTextures.set(texture.id, { + source: texture.source, + width: texture.width, + height: texture.height, + }); + } + + this.retainedTextureIds = nextIds; + } + + render(frame: FrameDescriptor) { + if (!this._debugLogged) { + this._debugLogged = true; + const firstLayer = frame.items.find((item) => item.type === "layer"); + console.log( + "[compositor] first frame — canvas size:", + JSON.stringify({ width: frame.width, height: frame.height }), + "| first layer transform:", + firstLayer && firstLayer.type === "layer" + ? JSON.stringify(firstLayer.transform) + : "none", + "| webgpu available:", + typeof navigator !== "undefined" && "gpu" in navigator, + ); + } + renderFrame(frame); + } +} + +export const wasmCompositor = new WasmCompositor(); diff --git a/apps/web/src/services/renderer/gpu-renderer.ts b/apps/web/src/services/renderer/gpu-renderer.ts index b2049ef5..b11aa8e2 100644 --- a/apps/web/src/services/renderer/gpu-renderer.ts +++ b/apps/web/src/services/renderer/gpu-renderer.ts @@ -13,12 +13,18 @@ export function initializeGpuRenderer(): Promise { initPromise = initializeGpu() .then(() => { gpuAvailable = true; + // #region agent log + fetch('http://127.0.0.1:7408/ingest/669b22f8-172b-4e65-aa3f-1c702ede83f7',{method:'POST',headers:{'Content-Type':'application/json','X-Debug-Session-Id':'b140a6'},body:JSON.stringify({sessionId:'b140a6',location:'gpu-renderer.ts',message:'GPU init SUCCESS',data:{userAgent:navigator.userAgent},timestamp:Date.now()})}).catch(()=>{}); + // #endregion }) .catch((error: unknown) => { gpuAvailable = false; const message = error instanceof Error ? error.message : String(error); console.warn(`GPU renderer unavailable: ${message}`); + // #region agent log + fetch('http://127.0.0.1:7408/ingest/669b22f8-172b-4e65-aa3f-1c702ede83f7',{method:'POST',headers:{'Content-Type':'application/json','X-Debug-Session-Id':'b140a6'},body:JSON.stringify({sessionId:'b140a6',location:'gpu-renderer.ts',message:'GPU init FAILED',data:{error:message,userAgent:navigator.userAgent,hasGpu:!!navigator.gpu},timestamp:Date.now()})}).catch(()=>{}); + // #endregion }); } return initPromise; diff --git a/apps/web/src/services/renderer/nodes/base-node.ts b/apps/web/src/services/renderer/nodes/base-node.ts index 55bd055a..1d6fc0e4 100644 --- a/apps/web/src/services/renderer/nodes/base-node.ts +++ b/apps/web/src/services/renderer/nodes/base-node.ts @@ -1,35 +1,26 @@ -import type { CanvasRenderer } from "../canvas-renderer"; - export type BaseNodeParams = object | undefined; +export type AnyBaseNode = BaseNode; -export class BaseNode { +export class BaseNode< + Params extends BaseNodeParams = BaseNodeParams, + Resolved = unknown, +> { params: Params; + resolved: Resolved | null = null; constructor(params?: Params) { this.params = params ?? ({} as Params); } - children: BaseNode[] = []; + children: AnyBaseNode[] = []; - add(child: BaseNode) { + add(child: AnyBaseNode) { this.children.push(child); return this; } - remove(child: BaseNode) { + remove(child: AnyBaseNode) { this.children = this.children.filter((c) => c !== child); return this; } - - async render({ - renderer, - time, - }: { - renderer: CanvasRenderer; - time: number; - }): Promise { - for (const child of this.children) { - await child.render({ renderer, time }); - } - } } diff --git a/apps/web/src/services/renderer/nodes/blur-background-node.ts b/apps/web/src/services/renderer/nodes/blur-background-node.ts index 4cb45b33..d20e2533 100644 --- a/apps/web/src/services/renderer/nodes/blur-background-node.ts +++ b/apps/web/src/services/renderer/nodes/blur-background-node.ts @@ -1,13 +1,6 @@ -import { buildGaussianBlurPasses, intensityToSigma } from "@/lib/effects/definitions/blur"; -import { mediaTimeToSeconds } from "opencut-wasm"; -import { getSourceTimeAtClipTime } from "@/lib/retime"; -import { videoCache } from "@/services/video-cache/service"; +import type { EffectPass } from "@/lib/effects/types"; import type { RetimeConfig } from "@/lib/timeline"; -import type { CanvasRenderer } from "../canvas-renderer"; -import { createOffscreenCanvas } from "../canvas-utils"; -import { gpuRenderer } from "../gpu-renderer"; import { BaseNode } from "./base-node"; -import { loadImageSource, type CachedImageSource } from "./image-node"; export type BlurBackgroundNodeParams = { mediaId: string; @@ -22,127 +15,18 @@ export type BlurBackgroundNodeParams = { blurIntensity: number; }; -type BackdropSource = { +export type BackdropSource = { source: CanvasImageSource; width: number; height: number; }; -export class BlurBackgroundNode extends BaseNode { - private cachedImageSource: Promise | null; - - constructor(params: BlurBackgroundNodeParams) { - super(params); - this.cachedImageSource = - params.mediaType === "image" ? loadImageSource(params.url) : null; - } - - private isInRange({ time }: { time: number }): boolean { - const localTime = time - this.params.timeOffset; - return localTime >= 0 && localTime < this.params.duration; - } - - private getSourceLocalTime({ time }: { time: number }): number { - const clipTime = time - this.params.timeOffset; - return ( - this.params.trimStart + - getSourceTimeAtClipTime({ - clipTime, - retime: this.params.retime, - }) - ); - } - - private async getBackdropSource({ - time, - }: { - time: number; - }): Promise { - if (this.params.mediaType === "video") { - const sourceTimeTicks = this.getSourceLocalTime({ time }); - const frame = await videoCache.getFrameAt({ - mediaId: this.params.mediaId, - file: this.params.file, - time: mediaTimeToSeconds({ time: sourceTimeTicks }), - }); - - if (!frame) { - return null; - } - - return { - source: frame.canvas, - width: frame.canvas.width, - height: frame.canvas.height, - }; - } - - if (!this.cachedImageSource) { - return null; - } - - const { source, width, height } = await this.cachedImageSource; - return { source, width, height }; - } - - async render({ renderer, time }: { renderer: CanvasRenderer; time: number }) { - await super.render({ renderer, time }); - - if (!this.isInRange({ time })) { - return; - } - - const backdropSource = await this.getBackdropSource({ time }); - if (!backdropSource) { - return; - } - - const offscreen = createOffscreenCanvas({ - width: renderer.width, - height: renderer.height, - }); - const offscreenCtx = offscreen.getContext("2d") as - | CanvasRenderingContext2D - | OffscreenCanvasRenderingContext2D - | null; - if (!offscreenCtx) { - return; - } - - const coverScale = Math.max( - renderer.width / backdropSource.width, - renderer.height / backdropSource.height, - ); - const scaledWidth = backdropSource.width * coverScale; - const scaledHeight = backdropSource.height * coverScale; - const offsetX = (renderer.width - scaledWidth) / 2; - const offsetY = (renderer.height - scaledHeight) / 2; - - offscreenCtx.drawImage( - backdropSource.source, - offsetX, - offsetY, - scaledWidth, - scaledHeight, - ); - - const passes = buildGaussianBlurPasses({ - sigmaX: intensityToSigma({ intensity: this.params.blurIntensity, resolution: renderer.width, reference: 1920 }), - sigmaY: intensityToSigma({ intensity: this.params.blurIntensity, resolution: renderer.height, reference: 1080 }), - }); - const effectResult = gpuRenderer.applyEffect({ - source: offscreen as CanvasImageSource, - width: renderer.width, - height: renderer.height, - passes, - }); - - renderer.context.drawImage( - effectResult, - 0, - 0, - renderer.width, - renderer.height, - ); - } +export interface ResolvedBlurBackgroundNodeState { + backdropSource: BackdropSource; + passes: EffectPass[]; } + +export class BlurBackgroundNode extends BaseNode< + BlurBackgroundNodeParams, + ResolvedBlurBackgroundNodeState +> {} diff --git a/apps/web/src/services/renderer/nodes/color-node.ts b/apps/web/src/services/renderer/nodes/color-node.ts index d3afbf9b..493fef64 100644 --- a/apps/web/src/services/renderer/nodes/color-node.ts +++ b/apps/web/src/services/renderer/nodes/color-node.ts @@ -1,31 +1,7 @@ -import { drawCssBackground } from "@/lib/gradients"; -import type { CanvasRenderer } from "../canvas-renderer"; import { BaseNode } from "./base-node"; export type ColorNodeParams = { color: string; }; -export class ColorNode extends BaseNode { - private color: string; - - constructor(params: ColorNodeParams) { - super(params); - this.color = params.color; - } - - async render({ renderer }: { renderer: CanvasRenderer }) { - if (/gradient\(/i.test(this.color)) { - drawCssBackground({ - ctx: renderer.context, - width: renderer.width, - height: renderer.height, - css: this.color, - }); - return; - } - - renderer.context.fillStyle = this.color; - renderer.context.fillRect(0, 0, renderer.width, renderer.height); - } -} +export class ColorNode extends BaseNode {} diff --git a/apps/web/src/services/renderer/nodes/effect-layer-node.ts b/apps/web/src/services/renderer/nodes/effect-layer-node.ts index d8389fb5..d6e70fcf 100644 --- a/apps/web/src/services/renderer/nodes/effect-layer-node.ts +++ b/apps/web/src/services/renderer/nodes/effect-layer-node.ts @@ -1,10 +1,6 @@ -import type { CanvasRenderer } from "../canvas-renderer"; -import { effectsRegistry, resolveEffectPasses } from "@/lib/effects"; +import type { EffectPass } from "@/lib/effects/types"; import type { ParamValues } from "@/lib/params"; import { BaseNode } from "./base-node"; -import { gpuRenderer } from "../gpu-renderer"; - -const TIME_EPSILON = 1e-6; export type EffectLayerNodeParams = { effectType: string; @@ -13,69 +9,11 @@ export type EffectLayerNodeParams = { duration: number; }; -function isInRange({ - time, - timeOffset, - duration, -}: { - time: number; - timeOffset: number; - duration: number; -}): boolean { - return ( - time >= timeOffset - TIME_EPSILON && - time < timeOffset + duration + TIME_EPSILON - ); -} +export type ResolvedEffectLayerNodeState = { + passes: EffectPass[]; +}; -// snapshots whatever is currently on the canvas, applies the effect, draws it back -export class EffectLayerNode extends BaseNode { - async render({ - renderer, - time, - }: { - renderer: CanvasRenderer; - time: number; - }): Promise { - if ( - !isInRange({ - time, - timeOffset: this.params.timeOffset, - duration: this.params.duration, - }) - ) { - return; - } - - const source = renderer.context.canvas as CanvasImageSource; - - const effectDefinition = effectsRegistry.get(this.params.effectType); - - const passes = resolveEffectPasses({ - definition: effectDefinition, - effectParams: this.params.effectParams, - width: renderer.width, - height: renderer.height, - }); - if (passes.length === 0) { - return; - } - const effectResult = gpuRenderer.applyEffect({ - source, - width: renderer.width, - height: renderer.height, - passes, - }); - - renderer.context.save(); - renderer.context.clearRect(0, 0, renderer.width, renderer.height); - renderer.context.drawImage( - effectResult, - 0, - 0, - renderer.width, - renderer.height, - ); - renderer.context.restore(); - } -} +export class EffectLayerNode extends BaseNode< + EffectLayerNodeParams, + ResolvedEffectLayerNodeState +> {} diff --git a/apps/web/src/services/renderer/nodes/graphic-node.ts b/apps/web/src/services/renderer/nodes/graphic-node.ts index 0a3dc408..494d2179 100644 --- a/apps/web/src/services/renderer/nodes/graphic-node.ts +++ b/apps/web/src/services/renderer/nodes/graphic-node.ts @@ -1,92 +1,74 @@ -import type { CanvasRenderer } from "../canvas-renderer"; -import { createOffscreenCanvas } from "../canvas-utils"; -import { - DEFAULT_GRAPHIC_SOURCE_SIZE, - getGraphicDefinition, - registerDefaultGraphics, -} from "@/lib/graphics"; -import { resolveGraphicParamsAtTime } from "@/lib/animation"; -import type { ParamValues } from "@/lib/params"; -import { VisualNode, type VisualNodeParams } from "./visual-node"; - -export interface GraphicNodeParams extends VisualNodeParams { - definitionId: string; - params: ParamValues; -} - -export class GraphicNode extends VisualNode { - private cachedKey: string | null = null; - private cachedSource: OffscreenCanvas | HTMLCanvasElement | null = null; - - constructor(params: GraphicNodeParams) { - super(params); - registerDefaultGraphics(); - } - - private getSource({ - localTime, - }: { - localTime: number; - }): OffscreenCanvas | HTMLCanvasElement | null { - const definition = getGraphicDefinition({ - definitionId: this.params.definitionId, - }); - const resolvedParams = resolveGraphicParamsAtTime({ - element: this.params, - localTime, - }); - const cacheKey = JSON.stringify({ - definitionId: this.params.definitionId, - params: resolvedParams, - }); - if (this.cachedSource && this.cachedKey === cacheKey) { - return this.cachedSource; - } - - const canvas = createOffscreenCanvas({ - width: DEFAULT_GRAPHIC_SOURCE_SIZE, - height: DEFAULT_GRAPHIC_SOURCE_SIZE, - }); - const ctx = canvas.getContext("2d") as - | CanvasRenderingContext2D - | OffscreenCanvasRenderingContext2D - | null; - if (!ctx) { - return null; - } - - definition.render({ - ctx, - params: resolvedParams, - width: DEFAULT_GRAPHIC_SOURCE_SIZE, - height: DEFAULT_GRAPHIC_SOURCE_SIZE, - }); - - this.cachedKey = cacheKey; - this.cachedSource = canvas; - return canvas; - } - - async render({ renderer, time }: { renderer: CanvasRenderer; time: number }) { - await super.render({ renderer, time }); - - if (!this.isInRange({ time })) { - return; - } - - const source = this.getSource({ - localTime: this.getAnimationLocalTime({ time }), - }); - if (!source) { - return; - } - - this.renderVisual({ - renderer, - source, - sourceWidth: DEFAULT_GRAPHIC_SOURCE_SIZE, - sourceHeight: DEFAULT_GRAPHIC_SOURCE_SIZE, - timelineTime: time, - }); - } -} +import { createOffscreenCanvas } from "../canvas-utils"; +import { + DEFAULT_GRAPHIC_SOURCE_SIZE, + getGraphicDefinition, + registerDefaultGraphics, +} from "@/lib/graphics"; +import type { ParamValues } from "@/lib/params"; +import { + VisualNode, + type ResolvedVisualNodeState, + type VisualNodeParams, +} from "./visual-node"; + +export interface GraphicNodeParams extends VisualNodeParams { + definitionId: string; + params: ParamValues; +} + +export interface ResolvedGraphicNodeState extends ResolvedVisualNodeState { + resolvedParams: ParamValues; +} + +export class GraphicNode extends VisualNode< + GraphicNodeParams, + ResolvedGraphicNodeState +> { + private cachedKey: string | null = null; + private cachedSource: OffscreenCanvas | HTMLCanvasElement | null = null; + + constructor(params: GraphicNodeParams) { + super(params); + registerDefaultGraphics(); + } + + getSource({ + resolvedParams, + }: { + resolvedParams: ParamValues; + }): OffscreenCanvas | HTMLCanvasElement | null { + const definition = getGraphicDefinition({ + definitionId: this.params.definitionId, + }); + const cacheKey = JSON.stringify({ + definitionId: this.params.definitionId, + params: resolvedParams, + }); + if (this.cachedSource && this.cachedKey === cacheKey) { + return this.cachedSource; + } + + const canvas = createOffscreenCanvas({ + width: DEFAULT_GRAPHIC_SOURCE_SIZE, + height: DEFAULT_GRAPHIC_SOURCE_SIZE, + }); + const ctx = canvas.getContext("2d") as + | CanvasRenderingContext2D + | OffscreenCanvasRenderingContext2D + | null; + if (!ctx) { + return null; + } + + definition.render({ + ctx, + params: resolvedParams, + width: DEFAULT_GRAPHIC_SOURCE_SIZE, + height: DEFAULT_GRAPHIC_SOURCE_SIZE, + }); + + this.cachedKey = cacheKey; + this.cachedSource = canvas; + return canvas; + } +} diff --git a/apps/web/src/services/renderer/nodes/image-node.ts b/apps/web/src/services/renderer/nodes/image-node.ts index 7d74f511..d8c56803 100644 --- a/apps/web/src/services/renderer/nodes/image-node.ts +++ b/apps/web/src/services/renderer/nodes/image-node.ts @@ -1,5 +1,8 @@ -import type { CanvasRenderer } from "../canvas-renderer"; -import { VisualNode, type VisualNodeParams } from "./visual-node"; +import { + VisualNode, + type ResolvedVisualSourceNodeState, + type VisualNodeParams, +} from "./visual-node"; export interface ImageNodeParams extends VisualNodeParams { url: string; @@ -62,29 +65,7 @@ export function loadImageSource( return promise; } -export class ImageNode extends VisualNode { - private cachedSource: Promise; - - constructor(params: ImageNodeParams) { - super(params); - this.cachedSource = loadImageSource(params.url, params.maxSourceSize); - } - - async render({ renderer, time }: { renderer: CanvasRenderer; time: number }) { - await super.render({ renderer, time }); - - if (!this.isInRange({ time })) { - return; - } - - const { source, width, height } = await this.cachedSource; - - this.renderVisual({ - renderer, - source, - sourceWidth: width || renderer.width, - sourceHeight: height || renderer.height, - timelineTime: time, - }); - } -} +export class ImageNode extends VisualNode< + ImageNodeParams, + ResolvedVisualSourceNodeState +> {} diff --git a/apps/web/src/services/renderer/nodes/sticker-node.ts b/apps/web/src/services/renderer/nodes/sticker-node.ts index 71cb37b5..3c155cb6 100644 --- a/apps/web/src/services/renderer/nodes/sticker-node.ts +++ b/apps/web/src/services/renderer/nodes/sticker-node.ts @@ -1,6 +1,9 @@ -import type { CanvasRenderer } from "../canvas-renderer"; import { resolveStickerId } from "@/lib/stickers"; -import { VisualNode, type VisualNodeParams } from "./visual-node"; +import { + VisualNode, + type ResolvedVisualSourceNodeState, + type VisualNodeParams, +} from "./visual-node"; export interface StickerNodeParams extends VisualNodeParams { stickerId: string; @@ -16,7 +19,11 @@ interface CachedStickerSource { const stickerSourceCache = new Map>(); -function loadStickerSource({ stickerId }: { stickerId: string }): Promise { +export function loadStickerSource({ + stickerId, +}: { + stickerId: string; +}): Promise { const cached = stickerSourceCache.get(stickerId); if (cached) return cached; @@ -42,35 +49,7 @@ function loadStickerSource({ stickerId }: { stickerId: string }): Promise { - private cachedSource: Promise; - - constructor(params: StickerNodeParams) { - super(params); - this.cachedSource = loadStickerSource({ stickerId: params.stickerId }); - } - - async render({ renderer, time }: { renderer: CanvasRenderer; time: number }) { - await super.render({ renderer, time }); - - if (!this.isInRange({ time })) { - return; - } - - const { source, width: loadedWidth, height: loadedHeight } = - await this.cachedSource; - - // Prefer element-stored intrinsic dimensions as the geometry authority. - // The loaded image is only the drawable source. - const sourceWidth = this.params.intrinsicWidth ?? loadedWidth; - const sourceHeight = this.params.intrinsicHeight ?? loadedHeight; - - this.renderVisual({ - renderer, - source, - sourceWidth, - sourceHeight, - timelineTime: time, - }); - } -} +export class StickerNode extends VisualNode< + StickerNodeParams, + ResolvedVisualSourceNodeState +> {} diff --git a/apps/web/src/services/renderer/nodes/text-node.ts b/apps/web/src/services/renderer/nodes/text-node.ts index 643e538a..fc503991 100644 --- a/apps/web/src/services/renderer/nodes/text-node.ts +++ b/apps/web/src/services/renderer/nodes/text-node.ts @@ -1,289 +1,129 @@ -import type { CanvasRenderer } from "../canvas-renderer"; -import { createOffscreenCanvas } from "../canvas-utils"; import { BaseNode } from "./base-node"; import type { TextElement } from "@/lib/timeline"; +import type { EffectPass } from "@/lib/effects/types"; +import type { Transform } from "@/lib/rendering"; import { CORNER_RADIUS_MAX, CORNER_RADIUS_MIN, } from "@/constants/text-constants"; import { - getMetricAscent, - getMetricDescent, + drawTextDecoration, getTextBackgroundRect, setCanvasLetterSpacing, } from "@/lib/text/layout"; -import { measureTextElement } from "@/lib/text/measure-element"; -import { - getElementLocalTime, - resolveColorAtTime, - resolveOpacityAtTime, - resolveTransformAtTime, -} from "@/lib/animation"; -import { resolveEffectParamsAtTime } from "@/lib/animation/effect-param-channel"; -import { effectsRegistry, resolveEffectPasses } from "@/lib/effects"; -import { gpuRenderer } from "../gpu-renderer"; +import type { MeasuredTextElement } from "@/lib/text/measure-element"; import { clamp } from "@/utils/math"; -const TEXT_DECORATION_THICKNESS_RATIO = 0.07; -const STRIKETHROUGH_VERTICAL_RATIO = 0.35; - -function drawTextDecoration({ - ctx, - textDecoration, - lineWidth, - lineY, - metrics, - scaledFontSize, - textAlign, -}: { - ctx: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D; - textDecoration: string; - lineWidth: number; - lineY: number; - metrics: TextMetrics; - scaledFontSize: number; - textAlign: CanvasTextAlign; -}): void { - if (textDecoration === "none" || !textDecoration) return; - - const thickness = Math.max( - 1, - scaledFontSize * TEXT_DECORATION_THICKNESS_RATIO, - ); - const ascent = getMetricAscent({ metrics, fallbackFontSize: scaledFontSize }); - const descent = getMetricDescent({ - metrics, - fallbackFontSize: scaledFontSize, - }); - - let xStart = -lineWidth / 2; - if (textAlign === "left") xStart = 0; - if (textAlign === "right") xStart = -lineWidth; - - if (textDecoration === "underline") { - const underlineY = lineY + descent + thickness; - ctx.fillRect(xStart, underlineY, lineWidth, thickness); - } - - if (textDecoration === "line-through") { - const strikeY = lineY - (ascent - descent) * STRIKETHROUGH_VERTICAL_RATIO; - ctx.fillRect(xStart, strikeY, lineWidth, thickness); - } -} - export type TextNodeParams = TextElement & { canvasCenter: { x: number; y: number }; canvasHeight: number; textBaseline?: CanvasTextBaseline; }; -export class TextNode extends BaseNode { - isInRange({ time }: { time: number }) { - return ( - time >= this.params.startTime && - time < this.params.startTime + this.params.duration - ); - } - - async render({ renderer, time }: { renderer: CanvasRenderer; time: number }) { - if (!this.isInRange({ time })) { - return; - } - - const localTime = getElementLocalTime({ - timelineTime: time, - elementStartTime: this.params.startTime, - elementDuration: this.params.duration, - }); - const transform = resolveTransformAtTime({ - baseTransform: this.params.transform, - animations: this.params.animations, - localTime, - }); - const opacity = resolveOpacityAtTime({ - baseOpacity: this.params.opacity, - animations: this.params.animations, - localTime, - }); - - const x = transform.position.x + this.params.canvasCenter.x; - const y = transform.position.y + this.params.canvasCenter.y; - - const baseline = this.params.textBaseline ?? "middle"; - const blendMode = ( - this.params.blendMode && this.params.blendMode !== "normal" - ? this.params.blendMode - : "source-over" - ) as GlobalCompositeOperation; - - const { - scaledFontSize, - fontString, - letterSpacing, - lineHeightPx, - lines, - lineMetrics, - block, - fontSizeRatio, - resolvedBackground, - } = measureTextElement({ - element: this.params, - canvasHeight: this.params.canvasHeight, - localTime, - ctx: renderer.context, - }); - - const lineCount = lines.length; - - const textColor = resolveColorAtTime({ - baseColor: this.params.color, - animations: this.params.animations, - propertyPath: "color", - localTime, - }); - const resolvedBackgroundWithColor = { - ...resolvedBackground, - color: resolveColorAtTime({ - baseColor: this.params.background.color, - animations: this.params.animations, - propertyPath: "background.color", - localTime, - }), - }; - - const drawContent = ( - ctx: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D, - ) => { - ctx.font = fontString; - ctx.textAlign = this.params.textAlign; - ctx.textBaseline = baseline; - ctx.fillStyle = textColor; - setCanvasLetterSpacing({ ctx, letterSpacingPx: letterSpacing }); - - if ( - this.params.background.enabled && - this.params.background.color && - this.params.background.color !== "transparent" && - lineCount > 0 - ) { - const backgroundRect = getTextBackgroundRect({ - textAlign: this.params.textAlign, - block, - background: resolvedBackgroundWithColor, - fontSizeRatio, - }); - if (backgroundRect) { - const p = - clamp({ - value: resolvedBackgroundWithColor.cornerRadius, - min: CORNER_RADIUS_MIN, - max: CORNER_RADIUS_MAX, - }) / 100; - const radius = - (Math.min(backgroundRect.width, backgroundRect.height) / 2) * p; - ctx.fillStyle = resolvedBackgroundWithColor.color; - ctx.beginPath(); - ctx.roundRect( - backgroundRect.left, - backgroundRect.top, - backgroundRect.width, - backgroundRect.height, - radius, - ); - ctx.fill(); - ctx.fillStyle = textColor; - } - } - - for (let i = 0; i < lineCount; i++) { - const lineY = i * lineHeightPx - block.visualCenterOffset; - ctx.fillText(lines[i], 0, lineY); - drawTextDecoration({ - ctx, - textDecoration: this.params.textDecoration ?? "none", - lineWidth: lineMetrics[i].width, - lineY, - metrics: lineMetrics[i], - scaledFontSize, - textAlign: this.params.textAlign, - }); - } - }; - - const applyTransform = ( - ctx: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D, - ) => { - ctx.translate(x, y); - ctx.scale(transform.scaleX, transform.scaleY); - if (transform.rotate) { - ctx.rotate((transform.rotate * Math.PI) / 180); - } - }; - - const enabledEffects = - this.params.effects?.filter((effect) => effect.enabled) ?? []; - - if (enabledEffects.length === 0) { - renderer.context.save(); - applyTransform(renderer.context); - renderer.context.globalCompositeOperation = blendMode; - renderer.context.globalAlpha = opacity; - drawContent(renderer.context); - renderer.context.restore(); - return; - } - - // Effects path: render text to a same-size offscreen canvas so the blur - // can spread into the surrounding transparent area without hard clipping. - const offscreen = createOffscreenCanvas({ - width: renderer.width, - height: renderer.height, - }); - const offscreenCtx = offscreen.getContext( - "2d", - ) as OffscreenCanvasRenderingContext2D | null; - - if (!offscreenCtx) { - renderer.context.save(); - applyTransform(renderer.context); - renderer.context.globalCompositeOperation = blendMode; - renderer.context.globalAlpha = opacity; - drawContent(renderer.context); - renderer.context.restore(); - return; - } - - offscreenCtx.save(); - applyTransform(offscreenCtx); - drawContent(offscreenCtx); - offscreenCtx.restore(); - - let currentSource: CanvasImageSource = offscreen; - for (const effect of enabledEffects) { - const resolvedParams = resolveEffectParamsAtTime({ - effect, - animations: this.params.animations, - localTime, - }); - const definition = effectsRegistry.get(effect.type); - const passes = resolveEffectPasses({ - definition, - effectParams: resolvedParams, - width: renderer.width, - height: renderer.height, - }); - currentSource = gpuRenderer.applyEffect({ - source: currentSource, - width: renderer.width, - height: renderer.height, - passes, - }); - } - - renderer.context.save(); - renderer.context.globalCompositeOperation = blendMode; - renderer.context.globalAlpha = opacity; - renderer.context.drawImage(currentSource, 0, 0); - renderer.context.restore(); - } +export interface ResolvedTextNodeState { + transform: Transform; + opacity: number; + textColor: string; + backgroundColor: string; + effectPasses: EffectPass[][]; + measuredText: MeasuredTextElement; +} + +export class TextNode extends BaseNode {} + +export function renderTextToContext({ + node, + ctx, +}: { + node: TextNode; + ctx: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D; +}): void { + const resolved = node.resolved; + if (!resolved) { + return; + } + + const x = resolved.transform.position.x + node.params.canvasCenter.x; + const y = resolved.transform.position.y + node.params.canvasCenter.y; + const baseline = node.params.textBaseline ?? "middle"; + const { + scaledFontSize, + fontString, + letterSpacing, + lineHeightPx, + lines, + lineMetrics, + block, + fontSizeRatio, + resolvedBackground, + } = resolved.measuredText; + const lineCount = lines.length; + const resolvedBackgroundWithColor = { + ...resolvedBackground, + color: resolved.backgroundColor, + }; + + ctx.save(); + ctx.translate(x, y); + ctx.scale(resolved.transform.scaleX, resolved.transform.scaleY); + if (resolved.transform.rotate) { + ctx.rotate((resolved.transform.rotate * Math.PI) / 180); + } + + ctx.font = fontString; + ctx.textAlign = node.params.textAlign; + ctx.textBaseline = baseline; + ctx.fillStyle = resolved.textColor; + setCanvasLetterSpacing({ ctx, letterSpacingPx: letterSpacing }); + + if ( + node.params.background.enabled && + node.params.background.color && + node.params.background.color !== "transparent" && + lineCount > 0 + ) { + const backgroundRect = getTextBackgroundRect({ + textAlign: node.params.textAlign, + block, + background: resolvedBackgroundWithColor, + fontSizeRatio, + }); + if (backgroundRect) { + const p = + clamp({ + value: resolvedBackgroundWithColor.cornerRadius, + min: CORNER_RADIUS_MIN, + max: CORNER_RADIUS_MAX, + }) / 100; + const radius = + (Math.min(backgroundRect.width, backgroundRect.height) / 2) * p; + ctx.fillStyle = resolvedBackgroundWithColor.color; + ctx.beginPath(); + ctx.roundRect( + backgroundRect.left, + backgroundRect.top, + backgroundRect.width, + backgroundRect.height, + radius, + ); + ctx.fill(); + ctx.fillStyle = resolved.textColor; + } + } + + for (let index = 0; index < lineCount; index++) { + const lineY = index * lineHeightPx - block.visualCenterOffset; + ctx.fillText(lines[index], 0, lineY); + drawTextDecoration({ + ctx, + textDecoration: node.params.textDecoration ?? "none", + lineWidth: lineMetrics[index].width, + lineY, + metrics: lineMetrics[index], + scaledFontSize, + textAlign: node.params.textAlign, + }); + } + + ctx.restore(); } diff --git a/apps/web/src/services/renderer/nodes/video-node.ts b/apps/web/src/services/renderer/nodes/video-node.ts index 4f225567..a903e2b2 100644 --- a/apps/web/src/services/renderer/nodes/video-node.ts +++ b/apps/web/src/services/renderer/nodes/video-node.ts @@ -1,7 +1,8 @@ -import type { CanvasRenderer } from "../canvas-renderer"; -import { mediaTimeToSeconds } from "opencut-wasm"; -import { VisualNode, type VisualNodeParams } from "./visual-node"; -import { videoCache } from "@/services/video-cache/service"; +import { + VisualNode, + type ResolvedVisualSourceNodeState, + type VisualNodeParams, +} from "./visual-node"; export interface VideoNodeParams extends VisualNodeParams { url: string; @@ -9,31 +10,7 @@ export interface VideoNodeParams extends VisualNodeParams { mediaId: string; } -export class VideoNode extends VisualNode { - async render({ renderer, time }: { renderer: CanvasRenderer; time: number }) { - await super.render({ renderer, time }); - - if (!this.isInRange({ time })) { - return; - } - - const videoTimeTicks = this.getSourceLocalTime({ time }); - const videoTimeSeconds = mediaTimeToSeconds({ time: videoTimeTicks }); - - const frame = await videoCache.getFrameAt({ - mediaId: this.params.mediaId, - file: this.params.file, - time: videoTimeSeconds, - }); - - if (frame) { - this.renderVisual({ - renderer, - source: frame.canvas, - sourceWidth: frame.canvas.width, - sourceHeight: frame.canvas.height, - timelineTime: time, - }); - } - } -} +export class VideoNode extends VisualNode< + VideoNodeParams, + ResolvedVisualSourceNodeState +> {} diff --git a/apps/web/src/services/renderer/nodes/visual-node.ts b/apps/web/src/services/renderer/nodes/visual-node.ts index 4a5fcdc6..010f146e 100644 --- a/apps/web/src/services/renderer/nodes/visual-node.ts +++ b/apps/web/src/services/renderer/nodes/visual-node.ts @@ -1,22 +1,8 @@ -import type { CanvasRenderer } from "../canvas-renderer"; -import { createOffscreenCanvas } from "../canvas-utils"; import { BaseNode } from "./base-node"; -import type { Effect } from "@/lib/effects/types"; +import type { Effect, EffectPass } from "@/lib/effects/types"; import type { Mask } from "@/lib/masks/types"; import type { BlendMode, Transform } from "@/lib/rendering"; -import type { ElementAnimations } from "@/lib/animation/types"; -import type { RetimeConfig } from "@/lib/timeline"; -import { - getElementLocalTime, - resolveOpacityAtTime, - resolveTransformAtTime, -} from "@/lib/animation"; -import { resolveEffectParamsAtTime } from "@/lib/animation/effect-param-channel"; -import { effectsRegistry, resolveEffectPasses } from "@/lib/effects"; -import { masksRegistry } from "@/lib/masks"; -import { getSourceTimeAtClipTime } from "@/lib/retime"; -import { gpuRenderer } from "../gpu-renderer"; -import { applyMaskFeather } from "../mask-feather"; +import type { RetimeConfig, VisualElement } from "@/lib/timeline"; export interface VisualNodeParams { duration: number; @@ -25,270 +11,28 @@ export interface VisualNodeParams { trimEnd: number; retime?: RetimeConfig; transform: Transform; - animations?: ElementAnimations; + animations?: VisualElement["animations"]; opacity: number; blendMode?: BlendMode; effects?: Effect[]; masks?: Mask[]; } +export interface ResolvedVisualNodeState { + localTime: number; + transform: Transform; + opacity: number; + effectPasses: EffectPass[][]; +} + +export interface ResolvedVisualSourceNodeState + extends ResolvedVisualNodeState { + source: CanvasImageSource; + sourceWidth: number; + sourceHeight: number; +} + export abstract class VisualNode< Params extends VisualNodeParams = VisualNodeParams, -> extends BaseNode { - protected getSourceLocalTime({ time }: { time: number }): number { - const clipTime = time - this.params.timeOffset; - return ( - this.params.trimStart + - getSourceTimeAtClipTime({ - clipTime, - retime: this.params.retime, - }) - ); - } - - protected getAnimationLocalTime({ time }: { time: number }): number { - return getElementLocalTime({ - timelineTime: time, - elementStartTime: this.params.timeOffset, - elementDuration: this.params.duration, - }); - } - - protected isInRange({ time }: { time: number }): boolean { - const localTime = time - this.params.timeOffset; - return ( - localTime >= 0 && - localTime < this.params.duration - ); - } - - protected renderVisual({ - renderer, - source, - sourceWidth, - sourceHeight, - timelineTime, - }: { - renderer: CanvasRenderer; - source: CanvasImageSource; - sourceWidth: number; - sourceHeight: number; - timelineTime: number; - }): void { - renderer.context.save(); - - const animationLocalTime = this.getAnimationLocalTime({ - time: timelineTime, - }); - const transform = resolveTransformAtTime({ - baseTransform: this.params.transform, - animations: this.params.animations, - localTime: animationLocalTime, - }); - const opacity = resolveOpacityAtTime({ - baseOpacity: this.params.opacity, - animations: this.params.animations, - localTime: animationLocalTime, - }); - const containScale = Math.min( - renderer.width / sourceWidth, - renderer.height / sourceHeight, - ); - const scaledWidth = sourceWidth * containScale * transform.scaleX; - const scaledHeight = sourceHeight * containScale * transform.scaleY; - const absWidth = Math.abs(scaledWidth); - const absHeight = Math.abs(scaledHeight); - const x = renderer.width / 2 + transform.position.x - absWidth / 2; - const y = renderer.height / 2 + transform.position.y - absHeight / 2; - - renderer.context.globalCompositeOperation = ( - this.params.blendMode && this.params.blendMode !== "normal" - ? this.params.blendMode - : "source-over" - ) as GlobalCompositeOperation; - renderer.context.globalAlpha = opacity; - - const flipX = scaledWidth < 0 ? -1 : 1; - const flipY = scaledHeight < 0 ? -1 : 1; - const needsTransform = transform.rotate !== 0 || flipX !== 1 || flipY !== 1; - - if (needsTransform) { - const centerX = x + absWidth / 2; - const centerY = y + absHeight / 2; - renderer.context.translate(centerX, centerY); - renderer.context.rotate((transform.rotate * Math.PI) / 180); - renderer.context.scale(flipX, flipY); - renderer.context.translate(-centerX, -centerY); - } - - const enabledEffects = - this.params.effects?.filter((effect) => effect.enabled) ?? []; - const activeMasks = this.params.masks ?? []; - - if (activeMasks.length === 0 && enabledEffects.length === 0) { - renderer.context.drawImage(source, x, y, absWidth, absHeight); - renderer.context.restore(); - return; - } - - const currentResult = - enabledEffects.length > 0 - ? this.applyEffects({ - source, - effects: enabledEffects, - width: absWidth, - height: absHeight, - animationLocalTime, - }) - : source; - - if (activeMasks.length === 0) { - renderer.context.drawImage(currentResult, x, y, absWidth, absHeight); - renderer.context.restore(); - return; - } - - const elementCanvas = createOffscreenCanvas({ - width: Math.round(absWidth), - height: Math.round(absHeight), - }); - const elementCtx = elementCanvas.getContext("2d") as - | CanvasRenderingContext2D - | OffscreenCanvasRenderingContext2D - | null; - if (!elementCtx) { - renderer.context.drawImage(currentResult, x, y, absWidth, absHeight); - renderer.context.restore(); - return; - } - - elementCtx.drawImage(currentResult, 0, 0, absWidth, absHeight); - - for (const mask of activeMasks) { - this.applyMask({ - mask, - elementCtx, - scaledWidth: absWidth, - scaledHeight: absHeight, - }); - } - - renderer.context.drawImage(elementCanvas, x, y, absWidth, absHeight); - renderer.context.restore(); - } - - private applyEffects({ - source, - effects, - width, - height, - animationLocalTime, - }: { - source: CanvasImageSource; - effects: Effect[]; - width: number; - height: number; - animationLocalTime: number; - }): CanvasImageSource { - let current: CanvasImageSource = source; - for (const effect of effects) { - const resolvedParams = resolveEffectParamsAtTime({ - effect, - animations: this.params.animations, - localTime: animationLocalTime, - }); - const definition = effectsRegistry.get(effect.type); - const passes = resolveEffectPasses({ - definition, - effectParams: resolvedParams, - width, - height, - }); - current = gpuRenderer.applyEffect({ - source: current, - width: Math.round(width), - height: Math.round(height), - passes, - }); - } - return current; - } - - private applyMask({ - mask, - elementCtx, - scaledWidth, - scaledHeight, - }: { - mask: Mask; - elementCtx: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D; - scaledWidth: number; - scaledHeight: number; - }): void { - const definition = masksRegistry.get(mask.type); - const { feather, inverted } = mask.params; - - const maskCanvas = createOffscreenCanvas({ - width: Math.round(scaledWidth), - height: Math.round(scaledHeight), - }); - const maskCtx = maskCanvas.getContext("2d") as - | CanvasRenderingContext2D - | OffscreenCanvasRenderingContext2D - | null; - if (!maskCtx) return; - - maskCtx.clearRect(0, 0, scaledWidth, scaledHeight); - - let maskResult: CanvasImageSource = maskCanvas; - let path: Path2D | null = null; - - if (feather > 0 && definition.renderer.renderMask) { - // Bypasses JFA — avoids the two-sided distance artifact where strips - // near the canvas edge appear semi-transparent. - definition.renderer.renderMask({ - resolvedParams: mask.params, - ctx: maskCtx, - width: Math.round(scaledWidth), - height: Math.round(scaledHeight), - feather, - }); - } else { - path = definition.renderer.buildPath({ - resolvedParams: mask.params, - width: scaledWidth, - height: scaledHeight, - }); - maskCtx.fillStyle = "white"; - maskCtx.fill(path); - - if (feather > 0) { - maskResult = applyMaskFeather({ - maskCanvas, - width: Math.round(scaledWidth), - height: Math.round(scaledHeight), - feather, - }); - } - } - - elementCtx.globalCompositeOperation = inverted - ? "destination-out" - : "destination-in"; - elementCtx.drawImage(maskResult, 0, 0, scaledWidth, scaledHeight); - elementCtx.globalCompositeOperation = "source-over"; - - const strokePath = - definition.renderer.buildStrokePath?.({ - resolvedParams: mask.params, - width: scaledWidth, - height: scaledHeight, - }) ?? path; - - if (mask.params.strokeWidth > 0 && strokePath) { - elementCtx.strokeStyle = mask.params.strokeColor; - elementCtx.lineWidth = mask.params.strokeWidth; - elementCtx.stroke(strokePath); - } - } -} + Resolved extends ResolvedVisualNodeState = ResolvedVisualNodeState, +> extends BaseNode {} diff --git a/apps/web/src/services/renderer/resolve.ts b/apps/web/src/services/renderer/resolve.ts new file mode 100644 index 00000000..3f30f0d5 --- /dev/null +++ b/apps/web/src/services/renderer/resolve.ts @@ -0,0 +1,468 @@ +import { mediaTimeToSeconds } from "opencut-wasm"; +import { + getElementLocalTime, + resolveColorAtTime, + resolveGraphicParamsAtTime, + resolveOpacityAtTime, + resolveTransformAtTime, +} from "@/lib/animation"; +import { resolveEffectParamsAtTime } from "@/lib/animation/effect-param-channel"; +import { + buildGaussianBlurPasses, + intensityToSigma, +} from "@/lib/effects/definitions/blur"; +import { effectsRegistry, resolveEffectPasses } from "@/lib/effects"; +import type { Effect, EffectPass } from "@/lib/effects/types"; +import { getSourceTimeAtClipTime } from "@/lib/retime"; +import { DEFAULT_GRAPHIC_SOURCE_SIZE } from "@/lib/graphics"; +import { + getTextMeasurementContext, + measureTextElement, +} from "@/lib/text/measure-element"; +import { videoCache } from "@/services/video-cache/service"; +import type { CanvasRenderer } from "./canvas-renderer"; +import type { AnyBaseNode } from "./nodes/base-node"; +import { + BlurBackgroundNode, + type BackdropSource, + type ResolvedBlurBackgroundNodeState, +} from "./nodes/blur-background-node"; +import { EffectLayerNode, type ResolvedEffectLayerNodeState } from "./nodes/effect-layer-node"; +import { + GraphicNode, + type ResolvedGraphicNodeState, +} from "./nodes/graphic-node"; +import { ImageNode, loadImageSource } from "./nodes/image-node"; +import { StickerNode, loadStickerSource } from "./nodes/sticker-node"; +import { TextNode, type ResolvedTextNodeState } from "./nodes/text-node"; +import { VideoNode } from "./nodes/video-node"; +import type { + ResolvedVisualNodeState, + ResolvedVisualSourceNodeState, + VisualNodeParams, +} from "./nodes/visual-node"; + +type ResolveContext = { + renderer: CanvasRenderer; + time: number; +}; + +export async function resolveRenderTree({ + node, + renderer, + time, +}: { + node: AnyBaseNode; + renderer: CanvasRenderer; + time: number; +}): Promise { + await resolveNode({ + node, + context: { + renderer, + time, + }, + }); +} + +async function resolveNode({ + node, + context, +}: { + node: AnyBaseNode; + context: ResolveContext; +}): Promise { + if (node instanceof VideoNode) { + node.resolved = await resolveVideoNode({ node, context }); + } else if (node instanceof ImageNode) { + node.resolved = await resolveImageNode({ node, context }); + } else if (node instanceof StickerNode) { + node.resolved = await resolveStickerNode({ node, context }); + } else if (node instanceof GraphicNode) { + node.resolved = resolveGraphicNode({ node, context }); + } else if (node instanceof TextNode) { + node.resolved = resolveTextNode({ node, context }); + } else if (node instanceof BlurBackgroundNode) { + node.resolved = await resolveBlurBackgroundNode({ node, context }); + } else if (node instanceof EffectLayerNode) { + node.resolved = resolveEffectLayerNode({ node, context }); + } + + await Promise.all( + node.children.map((child) => resolveNode({ node: child, context })), + ); +} + +function resolveEffectPassGroups({ + effects, + animations, + localTime, + width, + height, +}: { + effects: Effect[] | undefined; + animations: VisualNodeParams["animations"]; + localTime: number; + width: number; + height: number; +}): EffectPass[][] { + return (effects ?? []) + .filter((effect) => effect.enabled) + .map((effect) => { + const resolvedParams = resolveEffectParamsAtTime({ + effect, + animations, + localTime, + }); + const definition = effectsRegistry.get(effect.type); + return resolveEffectPasses({ + definition, + effectParams: resolvedParams, + width, + height, + }); + }); +} + +function resolveVisualState({ + params, + context, + sourceWidth, + sourceHeight, +}: { + params: VisualNodeParams; + context: ResolveContext; + sourceWidth: number; + sourceHeight: number; +}): ResolvedVisualNodeState | null { + const clipTime = context.time - params.timeOffset; + if (clipTime < 0 || clipTime >= params.duration) { + return null; + } + + const localTime = getElementLocalTime({ + timelineTime: context.time, + elementStartTime: params.timeOffset, + elementDuration: params.duration, + }); + const transform = resolveTransformAtTime({ + baseTransform: params.transform, + animations: params.animations, + localTime, + }); + const opacity = resolveOpacityAtTime({ + baseOpacity: params.opacity, + animations: params.animations, + localTime, + }); + const containScale = Math.min( + context.renderer.width / sourceWidth, + context.renderer.height / sourceHeight, + ); + const effectWidth = Math.round( + Math.abs(sourceWidth * containScale * transform.scaleX), + ); + const effectHeight = Math.round( + Math.abs(sourceHeight * containScale * transform.scaleY), + ); + + return { + localTime, + transform, + opacity, + effectPasses: resolveEffectPassGroups({ + effects: params.effects, + animations: params.animations, + localTime, + width: effectWidth, + height: effectHeight, + }), + }; +} + +async function resolveVideoNode({ + node, + context, +}: { + node: VideoNode; + context: ResolveContext; +}): Promise { + const clipTime = context.time - node.params.timeOffset; + if (clipTime < 0 || clipTime >= node.params.duration) { + return null; + } + + const sourceTimeTicks = + node.params.trimStart + + getSourceTimeAtClipTime({ + clipTime, + retime: node.params.retime, + }); + const frame = await videoCache.getFrameAt({ + mediaId: node.params.mediaId, + file: node.params.file, + time: mediaTimeToSeconds({ time: sourceTimeTicks }), + }); + if (!frame) { + return null; + } + + const visualState = resolveVisualState({ + params: node.params, + context, + sourceWidth: frame.canvas.width, + sourceHeight: frame.canvas.height, + }); + if (!visualState) { + return null; + } + + return { + ...visualState, + source: frame.canvas, + sourceWidth: frame.canvas.width, + sourceHeight: frame.canvas.height, + }; +} + +async function resolveImageNode({ + node, + context, +}: { + node: ImageNode; + context: ResolveContext; +}): Promise { + const source = await loadImageSource(node.params.url, node.params.maxSourceSize); + const visualState = resolveVisualState({ + params: node.params, + context, + sourceWidth: source.width, + sourceHeight: source.height, + }); + if (!visualState) { + return null; + } + + return { + ...visualState, + source: source.source, + sourceWidth: source.width, + sourceHeight: source.height, + }; +} + +async function resolveStickerNode({ + node, + context, +}: { + node: StickerNode; + context: ResolveContext; +}): Promise { + const source = await loadStickerSource({ stickerId: node.params.stickerId }); + const sourceWidth = node.params.intrinsicWidth ?? source.width; + const sourceHeight = node.params.intrinsicHeight ?? source.height; + const visualState = resolveVisualState({ + params: node.params, + context, + sourceWidth, + sourceHeight, + }); + if (!visualState) { + return null; + } + + return { + ...visualState, + source: source.source, + sourceWidth, + sourceHeight, + }; +} + +function resolveGraphicNode({ + node, + context, +}: { + node: GraphicNode; + context: ResolveContext; +}): ResolvedGraphicNodeState | null { + const visualState = resolveVisualState({ + params: node.params, + context, + sourceWidth: DEFAULT_GRAPHIC_SOURCE_SIZE, + sourceHeight: DEFAULT_GRAPHIC_SOURCE_SIZE, + }); + if (!visualState) { + return null; + } + + return { + ...visualState, + resolvedParams: resolveGraphicParamsAtTime({ + element: node.params, + localTime: visualState.localTime, + }), + }; +} + +function resolveTextNode({ + node, + context, +}: { + node: TextNode; + context: ResolveContext; +}): ResolvedTextNodeState | null { + if ( + context.time < node.params.startTime || + context.time >= node.params.startTime + node.params.duration + ) { + return null; + } + + const localTime = getElementLocalTime({ + timelineTime: context.time, + elementStartTime: node.params.startTime, + elementDuration: node.params.duration, + }); + + return { + transform: resolveTransformAtTime({ + baseTransform: node.params.transform, + animations: node.params.animations, + localTime, + }), + opacity: resolveOpacityAtTime({ + baseOpacity: node.params.opacity, + animations: node.params.animations, + localTime, + }), + textColor: resolveColorAtTime({ + baseColor: node.params.color, + animations: node.params.animations, + propertyPath: "color", + localTime, + }), + backgroundColor: resolveColorAtTime({ + baseColor: node.params.background.color, + animations: node.params.animations, + propertyPath: "background.color", + localTime, + }), + effectPasses: resolveEffectPassGroups({ + effects: node.params.effects, + animations: node.params.animations, + localTime, + width: context.renderer.width, + height: context.renderer.height, + }), + measuredText: measureTextElement({ + element: node.params, + canvasHeight: node.params.canvasHeight, + localTime, + ctx: getTextMeasurementContext(), + }), + }; +} + +async function resolveBlurBackgroundNode({ + node, + context, +}: { + node: BlurBackgroundNode; + context: ResolveContext; +}): Promise { + const clipTime = context.time - node.params.timeOffset; + if (clipTime < 0 || clipTime >= node.params.duration) { + return null; + } + + const backdropSource = await resolveBackdropSource({ node, clipTime }); + if (!backdropSource) { + return null; + } + + return { + backdropSource, + passes: buildGaussianBlurPasses({ + sigmaX: intensityToSigma({ + intensity: node.params.blurIntensity, + resolution: context.renderer.width, + reference: 1920, + }), + sigmaY: intensityToSigma({ + intensity: node.params.blurIntensity, + resolution: context.renderer.height, + reference: 1080, + }), + }), + }; +} + +async function resolveBackdropSource({ + node, + clipTime, +}: { + node: BlurBackgroundNode; + clipTime: number; +}): Promise { + if (node.params.mediaType === "video") { + const sourceTimeTicks = + node.params.trimStart + + getSourceTimeAtClipTime({ + clipTime, + retime: node.params.retime, + }); + const frame = await videoCache.getFrameAt({ + mediaId: node.params.mediaId, + file: node.params.file, + time: mediaTimeToSeconds({ time: sourceTimeTicks }), + }); + if (!frame) { + return null; + } + + return { + source: frame.canvas, + width: frame.canvas.width, + height: frame.canvas.height, + }; + } + + const source = await loadImageSource(node.params.url); + return { + source: source.source, + width: source.width, + height: source.height, + }; +} + +function resolveEffectLayerNode({ + node, + context, +}: { + node: EffectLayerNode; + context: ResolveContext; +}): ResolvedEffectLayerNodeState | null { + const time = context.time; + if ( + time < node.params.timeOffset - 1e-6 || + time >= node.params.timeOffset + node.params.duration + 1e-6 + ) { + return null; + } + + const definition = effectsRegistry.get(node.params.effectType); + const passes = resolveEffectPasses({ + definition, + effectParams: node.params.effectParams, + width: context.renderer.width, + height: context.renderer.height, + }); + if (passes.length === 0) { + return null; + } + + return { + passes, + }; +} diff --git a/apps/web/src/services/renderer/scene-builder.ts b/apps/web/src/services/renderer/scene-builder.ts index 22d941be..68ba956b 100644 --- a/apps/web/src/services/renderer/scene-builder.ts +++ b/apps/web/src/services/renderer/scene-builder.ts @@ -1,265 +1,265 @@ -import type { SceneTracks, TimelineTrack } from "@/lib/timeline"; -import type { MediaAsset } from "@/lib/media/types"; -import { RootNode } from "./nodes/root-node"; -import { VideoNode } from "./nodes/video-node"; -import { ImageNode } from "./nodes/image-node"; -import { TextNode } from "./nodes/text-node"; -import { StickerNode } from "./nodes/sticker-node"; -import { GraphicNode } from "./nodes/graphic-node"; -import { ColorNode } from "./nodes/color-node"; -import { BlurBackgroundNode } from "./nodes/blur-background-node"; -import { EffectLayerNode } from "./nodes/effect-layer-node"; -import type { BaseNode } from "./nodes/base-node"; -import type { TBackground, TCanvasSize } from "@/lib/project/types"; -import { DEFAULT_BACKGROUND_BLUR_INTENSITY } from "@/lib/background/constants"; - -const PREVIEW_MAX_IMAGE_SIZE = 2048; - -function getVisibleSortedElements({ track }: { track: TimelineTrack }) { - return track.elements - .filter((element) => !("hidden" in element && element.hidden)) - .slice() - .sort((a, b) => { - if (a.startTime !== b.startTime) return a.startTime - b.startTime; - return a.id.localeCompare(b.id); - }); -} - -function buildTrackNodes({ - tracks, - mediaMap, - canvasSize, - isPreview, -}: { - tracks: TimelineTrack[]; - mediaMap: Map; - canvasSize: TCanvasSize; - isPreview?: boolean; -}): BaseNode[] { - const nodes: BaseNode[] = []; - - for (const track of tracks) { - const elements = getVisibleSortedElements({ track }); - - for (const element of elements) { - if (element.type === "effect") { - nodes.push( - new EffectLayerNode({ - effectType: element.effectType, - effectParams: element.params, - timeOffset: element.startTime, - duration: element.duration, - }), - ); - continue; - } - - if (element.type === "video" || element.type === "image") { - const mediaAsset = mediaMap.get(element.mediaId); - if (!mediaAsset?.file || !mediaAsset?.url) { - continue; - } - - if (element.type === "video" && mediaAsset.type === "video") { - nodes.push( - new VideoNode({ - mediaId: mediaAsset.id, - url: mediaAsset.url, - file: mediaAsset.file, - duration: element.duration, - timeOffset: element.startTime, - trimStart: element.trimStart, - trimEnd: element.trimEnd, - retime: element.retime, - transform: element.transform, - animations: element.animations, - opacity: element.opacity, - blendMode: element.blendMode, - effects: element.effects, - masks: element.masks, - }), - ); - } - if (element.type === "image" && mediaAsset.type === "image") { - nodes.push( - new ImageNode({ - url: mediaAsset.url, - duration: element.duration, - timeOffset: element.startTime, - trimStart: element.trimStart, - trimEnd: element.trimEnd, - transform: element.transform, - animations: element.animations, - opacity: element.opacity, - blendMode: element.blendMode, - effects: element.effects, - masks: element.masks, - ...(isPreview && { - maxSourceSize: PREVIEW_MAX_IMAGE_SIZE, - }), - }), - ); - } - } - - if (element.type === "text") { - nodes.push( - new TextNode({ - ...element, - canvasCenter: { x: canvasSize.width / 2, y: canvasSize.height / 2 }, - canvasHeight: canvasSize.height, - textBaseline: "middle", - effects: element.effects, - }), - ); - } - - if (element.type === "sticker") { - nodes.push( - new StickerNode({ - stickerId: element.stickerId, - intrinsicWidth: element.intrinsicWidth, - intrinsicHeight: element.intrinsicHeight, - duration: element.duration, - timeOffset: element.startTime, - trimStart: element.trimStart, - trimEnd: element.trimEnd, - transform: element.transform, - animations: element.animations, - opacity: element.opacity, - blendMode: element.blendMode, - effects: element.effects, - }), - ); - } - - if (element.type === "graphic") { - nodes.push( - new GraphicNode({ - definitionId: element.definitionId, - params: element.params, - duration: element.duration, - timeOffset: element.startTime, - trimStart: element.trimStart, - trimEnd: element.trimEnd, - transform: element.transform, - animations: element.animations, - opacity: element.opacity, - blendMode: element.blendMode, - effects: element.effects, - masks: element.masks, - }), - ); - } - } - } - - return nodes; -} - -function buildBlurBackgroundNodes({ - track, - mediaMap, - blurIntensity, -}: { - track: TimelineTrack | undefined; - mediaMap: Map; - blurIntensity: number; -}): BaseNode[] { - if (!track) { - return []; - } - - const nodes: BaseNode[] = []; - const elements = getVisibleSortedElements({ track }); - - for (const element of elements) { - if (element.type !== "video" && element.type !== "image") { - continue; - } - - const mediaAsset = mediaMap.get(element.mediaId); - if ( - !mediaAsset?.file || - !mediaAsset?.url || - (mediaAsset.type !== "video" && mediaAsset.type !== "image") - ) { - continue; - } - - nodes.push( - new BlurBackgroundNode({ - mediaId: mediaAsset.id, - url: mediaAsset.url, - file: mediaAsset.file, - mediaType: mediaAsset.type, - duration: element.duration, - timeOffset: element.startTime, - trimStart: element.trimStart, - trimEnd: element.trimEnd, - retime: element.type === "video" ? element.retime : undefined, - blurIntensity, - }), - ); - } - - return nodes; -} - -export type BuildSceneParams = { - canvasSize: TCanvasSize; - tracks: SceneTracks; - mediaAssets: MediaAsset[]; - duration: number; - background: TBackground; - isPreview?: boolean; -}; - -export function buildScene({ - canvasSize, - tracks, - mediaAssets, - duration, - background, - isPreview, -}: BuildSceneParams) { - const rootNode = new RootNode({ duration }); - const mediaMap = new Map(mediaAssets.map((m) => [m.id, m])); - - const visibleTracks = [ - ...tracks.overlay.filter((track) => !("hidden" in track && track.hidden)), - ...(!tracks.main.hidden ? [tracks.main] : []), - ]; - const orderedTracksBottomToTop = visibleTracks.slice().reverse(); - const mainTrack = tracks.main.hidden ? undefined : tracks.main; - - const allNodes = buildTrackNodes({ - tracks: orderedTracksBottomToTop, - mediaMap, - canvasSize, - isPreview, - }); - - if (background.type === "blur") { - const blurNodes = buildBlurBackgroundNodes({ - track: mainTrack, - mediaMap, - blurIntensity: - background.blurIntensity ?? DEFAULT_BACKGROUND_BLUR_INTENSITY, - }); - for (const node of blurNodes) { - rootNode.add(node); - } - } else if ( - background.type === "color" && - background.color !== "transparent" - ) { - rootNode.add(new ColorNode({ color: background.color })); - } - - for (const node of allNodes) { - rootNode.add(node); - } - - return rootNode; -} +import type { SceneTracks, TimelineTrack } from "@/lib/timeline"; +import type { MediaAsset } from "@/lib/media/types"; +import { RootNode } from "./nodes/root-node"; +import { VideoNode } from "./nodes/video-node"; +import { ImageNode } from "./nodes/image-node"; +import { TextNode } from "./nodes/text-node"; +import { StickerNode } from "./nodes/sticker-node"; +import { GraphicNode } from "./nodes/graphic-node"; +import { ColorNode } from "./nodes/color-node"; +import { BlurBackgroundNode } from "./nodes/blur-background-node"; +import { EffectLayerNode } from "./nodes/effect-layer-node"; +import type { AnyBaseNode } from "./nodes/base-node"; +import type { TBackground, TCanvasSize } from "@/lib/project/types"; +import { DEFAULT_BACKGROUND_BLUR_INTENSITY } from "@/lib/background/constants"; + +const PREVIEW_MAX_IMAGE_SIZE = 2048; + +function getVisibleSortedElements({ track }: { track: TimelineTrack }) { + return track.elements + .filter((element) => !("hidden" in element && element.hidden)) + .slice() + .sort((a, b) => { + if (a.startTime !== b.startTime) return a.startTime - b.startTime; + return a.id.localeCompare(b.id); + }); +} + +function buildTrackNodes({ + tracks, + mediaMap, + canvasSize, + isPreview, +}: { + tracks: TimelineTrack[]; + mediaMap: Map; + canvasSize: TCanvasSize; + isPreview?: boolean; +}): AnyBaseNode[] { + const nodes: AnyBaseNode[] = []; + + for (const track of tracks) { + const elements = getVisibleSortedElements({ track }); + + for (const element of elements) { + if (element.type === "effect") { + nodes.push( + new EffectLayerNode({ + effectType: element.effectType, + effectParams: element.params, + timeOffset: element.startTime, + duration: element.duration, + }), + ); + continue; + } + + if (element.type === "video" || element.type === "image") { + const mediaAsset = mediaMap.get(element.mediaId); + if (!mediaAsset?.file || !mediaAsset?.url) { + continue; + } + + if (element.type === "video" && mediaAsset.type === "video") { + nodes.push( + new VideoNode({ + mediaId: mediaAsset.id, + url: mediaAsset.url, + file: mediaAsset.file, + duration: element.duration, + timeOffset: element.startTime, + trimStart: element.trimStart, + trimEnd: element.trimEnd, + retime: element.retime, + transform: element.transform, + animations: element.animations, + opacity: element.opacity, + blendMode: element.blendMode, + effects: element.effects ?? [], + masks: element.masks ?? [], + }), + ); + } + if (element.type === "image" && mediaAsset.type === "image") { + nodes.push( + new ImageNode({ + url: mediaAsset.url, + duration: element.duration, + timeOffset: element.startTime, + trimStart: element.trimStart, + trimEnd: element.trimEnd, + transform: element.transform, + animations: element.animations, + opacity: element.opacity, + blendMode: element.blendMode, + effects: element.effects ?? [], + masks: element.masks ?? [], + ...(isPreview && { + maxSourceSize: PREVIEW_MAX_IMAGE_SIZE, + }), + }), + ); + } + } + + if (element.type === "text") { + nodes.push( + new TextNode({ + ...element, + canvasCenter: { x: canvasSize.width / 2, y: canvasSize.height / 2 }, + canvasHeight: canvasSize.height, + textBaseline: "middle", + effects: element.effects ?? [], + }), + ); + } + + if (element.type === "sticker") { + nodes.push( + new StickerNode({ + stickerId: element.stickerId, + intrinsicWidth: element.intrinsicWidth, + intrinsicHeight: element.intrinsicHeight, + duration: element.duration, + timeOffset: element.startTime, + trimStart: element.trimStart, + trimEnd: element.trimEnd, + transform: element.transform, + animations: element.animations, + opacity: element.opacity, + blendMode: element.blendMode, + effects: element.effects ?? [], + }), + ); + } + + if (element.type === "graphic") { + nodes.push( + new GraphicNode({ + definitionId: element.definitionId, + params: element.params, + duration: element.duration, + timeOffset: element.startTime, + trimStart: element.trimStart, + trimEnd: element.trimEnd, + transform: element.transform, + animations: element.animations, + opacity: element.opacity, + blendMode: element.blendMode, + effects: element.effects ?? [], + masks: element.masks ?? [], + }), + ); + } + } + } + + return nodes; +} + +function buildBlurBackgroundNodes({ + track, + mediaMap, + blurIntensity, +}: { + track: TimelineTrack | undefined; + mediaMap: Map; + blurIntensity: number; +}): AnyBaseNode[] { + if (!track) { + return []; + } + + const nodes: AnyBaseNode[] = []; + const elements = getVisibleSortedElements({ track }); + + for (const element of elements) { + if (element.type !== "video" && element.type !== "image") { + continue; + } + + const mediaAsset = mediaMap.get(element.mediaId); + if ( + !mediaAsset?.file || + !mediaAsset?.url || + (mediaAsset.type !== "video" && mediaAsset.type !== "image") + ) { + continue; + } + + nodes.push( + new BlurBackgroundNode({ + mediaId: mediaAsset.id, + url: mediaAsset.url, + file: mediaAsset.file, + mediaType: mediaAsset.type, + duration: element.duration, + timeOffset: element.startTime, + trimStart: element.trimStart, + trimEnd: element.trimEnd, + retime: element.type === "video" ? element.retime : undefined, + blurIntensity, + }), + ); + } + + return nodes; +} + +export type BuildSceneParams = { + canvasSize: TCanvasSize; + tracks: SceneTracks; + mediaAssets: MediaAsset[]; + duration: number; + background: TBackground; + isPreview?: boolean; +}; + +export function buildScene({ + canvasSize, + tracks, + mediaAssets, + duration, + background, + isPreview, +}: BuildSceneParams) { + const rootNode = new RootNode({ duration }); + const mediaMap = new Map(mediaAssets.map((m) => [m.id, m])); + + const visibleTracks = [ + ...tracks.overlay.filter((track) => !("hidden" in track && track.hidden)), + ...(!tracks.main.hidden ? [tracks.main] : []), + ]; + const orderedTracksBottomToTop = visibleTracks.slice().reverse(); + const mainTrack = tracks.main.hidden ? undefined : tracks.main; + + const allNodes = buildTrackNodes({ + tracks: orderedTracksBottomToTop, + mediaMap, + canvasSize, + isPreview, + }); + + if (background.type === "blur") { + const blurNodes = buildBlurBackgroundNodes({ + track: mainTrack, + mediaMap, + blurIntensity: + background.blurIntensity ?? DEFAULT_BACKGROUND_BLUR_INTENSITY, + }); + for (const node of blurNodes) { + rootNode.add(node); + } + } else if ( + background.type === "color" && + background.color !== "transparent" + ) { + rootNode.add(new ColorNode({ color: background.color })); + } + + for (const node of allNodes) { + rootNode.add(node); + } + + return rootNode; +} diff --git a/apps/web/src/services/renderer/scene-exporter.ts b/apps/web/src/services/renderer/scene-exporter.ts index 0c2ee17f..b19005cb 100644 --- a/apps/web/src/services/renderer/scene-exporter.ts +++ b/apps/web/src/services/renderer/scene-exporter.ts @@ -97,7 +97,7 @@ export class SceneExporter extends EventEmitter { target: new BufferTarget(), }); - const videoSource = new CanvasSource(this.renderer.canvas, { + const videoSource = new CanvasSource(this.renderer.getOutputCanvas(), { codec: this.format === "webm" ? "vp9" : "avc", bitrate: qualityMap[this.quality], });