RomanNum3ral
/
OpenCut
mirror of https://github.com/OpenCut-app/OpenCut.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

refactor: rendering pipeline 

Made-with: Cursor
This commit is contained in:
Maze Winther 2026-04-13 04:41:17 +02:00
parent 36efdf66e7
commit cad88ee4d1
19 changed files with 1778 additions and 1264 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
apps/web/src/lib/preview/element-bounds.ts
View File

@ -244,21 +244,6 @@ export function getEdgeHandlePosition({
};
}
export function getRotationHandlePosition({
bounds,
}: {
bounds: ElementBounds;
}): { x: number; y: number } {
const angleRad = (bounds.rotation * Math.PI) / 180;
const cos = Math.cos(angleRad);
const sin = Math.sin(angleRad);
const localY = -bounds.height / 2 - ROTATION_HANDLE_OFFSET;
return {
x: bounds.cx - localY * sin,
y: bounds.cy + localY * cos,
};
}
export function getVisibleElementsWithBounds({
tracks,
currentTime,

44
apps/web/src/services/renderer/canvas-renderer.ts
View File

@ -1,6 +1,8 @@
import type { FrameRate } from "opencut-wasm";
import { frameRateToFloat } from "@/lib/fps/utils";
import type { BaseNode } from "./nodes/base-node";
import type { AnyBaseNode } from "./nodes/base-node";
import { buildFrameDescriptor } from "./compositor/frame-descriptor";
import { wasmCompositor } from "./compositor/wasm-compositor";
import { resolveRenderTree } from "./resolve";
export type CanvasRendererParams = {
width: number;
@ -38,6 +40,14 @@ export class CanvasRenderer {
| CanvasRenderingContext2D;
}
getOutputCanvas(): HTMLCanvasElement {
wasmCompositor.ensureInitialized({
width: this.width,
height: this.height,
});
return wasmCompositor.getCanvas();
}
setSize({ width, height }: { width: number; height: number }) {
this.width = width;
this.height = height;
@ -58,14 +68,18 @@ export class CanvasRenderer {
| CanvasRenderingContext2D;
}
private clear() {
this.context.fillStyle = "black";
this.context.fillRect(0, 0, this.canvas.width, this.canvas.height);
}
async render({ node, time }: { node: BaseNode; time: number }) {
this.clear();
await node.render({ renderer: this, time });
async render({ node, time }: { node: AnyBaseNode; time: number }) {
await resolveRenderTree({ node, renderer: this, time });
const { frame, textures } = await buildFrameDescriptor({
node,
renderer: this,
});
wasmCompositor.ensureInitialized({
width: this.width,
height: this.height,
});
wasmCompositor.syncTextures(textures);
wasmCompositor.render(frame);
}
async renderToCanvas({
@ -73,7 +87,7 @@ export class CanvasRenderer {
time,
targetCanvas,
}: {
node: BaseNode;
node: AnyBaseNode;
time: number;
targetCanvas: HTMLCanvasElement;
}) {
@ -84,6 +98,12 @@ export class CanvasRenderer {
throw new Error("Failed to get target canvas context");
}
ctx.drawImage(this.canvas, 0, 0, targetCanvas.width, targetCanvas.height);
ctx.drawImage(
wasmCompositor.getCanvas(),
0,
0,
targetCanvas.width,
targetCanvas.height,
);
}
}

559
apps/web/src/services/renderer/compositor/frame-descriptor.ts Normal file
View File

@ -0,0 +1,559 @@
import { drawCssBackground } from "@/lib/gradients";
import { masksRegistry } from "@/lib/masks";
import type { AnyBaseNode } from "../nodes/base-node";
import type { CanvasRenderer } from "../canvas-renderer";
import { createOffscreenCanvas } from "../canvas-utils";
import { BlurBackgroundNode } from "../nodes/blur-background-node";
import { ColorNode } from "../nodes/color-node";
import { EffectLayerNode } from "../nodes/effect-layer-node";
import { GraphicNode, type ResolvedGraphicNodeState } from "../nodes/graphic-node";
import { ImageNode } from "../nodes/image-node";
import { RootNode } from "../nodes/root-node";
import { StickerNode } from "../nodes/sticker-node";
import { renderTextToContext, TextNode } from "../nodes/text-node";
import { VideoNode } from "../nodes/video-node";
import type { ResolvedVisualSourceNodeState } from "../nodes/visual-node";
import type {
FrameDescriptor,
FrameItemDescriptor,
LayerMaskDescriptor,
QuadTransformDescriptor,
} from "./types";
import { DEFAULT_GRAPHIC_SOURCE_SIZE } from "@/lib/graphics";
export type TextureUploadDescriptor = {
id: string;
source: CanvasImageSource;
width: number;
height: number;
};
export async function buildFrameDescriptor({
node,
renderer,
}: {
node: AnyBaseNode;
renderer: CanvasRenderer;
}): Promise<{
frame: FrameDescriptor;
textures: TextureUploadDescriptor[];
}> {
const items: FrameItemDescriptor[] = [];
const textures = new Map<string, TextureUploadDescriptor>();
await collectNode({
node,
renderer,
path: "root",
items,
textures,
});
return {
frame: {
width: renderer.width,
height: renderer.height,
clear: {
color: [0, 0, 0, 1],
},
items,
},
textures: [...textures.values()],
};
}
async function collectNode({
node,
renderer,
path,
items,
textures,
}: {
node: AnyBaseNode;
renderer: CanvasRenderer;
path: string;
items: FrameItemDescriptor[];
textures: Map<string, TextureUploadDescriptor>;
}): Promise<void> {
if (node instanceof RootNode) {
for (let index = 0; index < node.children.length; index++) {
await collectNode({
node: node.children[index],
renderer,
path: `${path}:${index}`,
items,
textures,
});
}
return;
}
if (node instanceof ColorNode) {
const textureId = `${path}:color`;
const canvas = createOffscreenCanvas({
width: renderer.width,
height: renderer.height,
});
const ctx = canvas.getContext("2d") as
| CanvasRenderingContext2D
| OffscreenCanvasRenderingContext2D
| null;
if (!ctx) return;
if (/gradient\(/i.test(node.params.color)) {
drawCssBackground({
ctx,
width: renderer.width,
height: renderer.height,
css: node.params.color,
});
} else {
ctx.fillStyle = node.params.color;
ctx.fillRect(0, 0, renderer.width, renderer.height);
}
textures.set(textureId, {
id: textureId,
source: canvas,
width: renderer.width,
height: renderer.height,
});
items.push({
type: "layer",
textureId,
transform: fullCanvasTransform(renderer),
opacity: 1,
blendMode: "normal",
effectPassGroups: [],
mask: null,
});
return;
}
if (node instanceof EffectLayerNode) {
if (!node.resolved || node.resolved.passes.length === 0) {
return;
}
items.push({
type: "sceneEffect",
effectPassGroups: [node.resolved.passes],
});
return;
}
if (node instanceof BlurBackgroundNode) {
if (!node.resolved) {
return;
}
const textureId = `${path}:blur-background`;
const backdropCanvas = createOffscreenCanvas({
width: renderer.width,
height: renderer.height,
});
const backdropCtx = backdropCanvas.getContext("2d") as
| CanvasRenderingContext2D
| OffscreenCanvasRenderingContext2D
| null;
if (!backdropCtx) return;
const { backdropSource, passes } = node.resolved;
const coverScale = Math.max(
renderer.width / backdropSource.width,
renderer.height / backdropSource.height,
);
const scaledWidth = backdropSource.width * coverScale;
const scaledHeight = backdropSource.height * coverScale;
const offsetX = (renderer.width - scaledWidth) / 2;
const offsetY = (renderer.height - scaledHeight) / 2;
backdropCtx.drawImage(
backdropSource.source,
offsetX,
offsetY,
scaledWidth,
scaledHeight,
);
textures.set(textureId, {
id: textureId,
source: backdropCanvas,
width: renderer.width,
height: renderer.height,
});
items.push({
type: "layer",
textureId,
transform: fullCanvasTransform(renderer),
opacity: 1,
blendMode: "normal",
effectPassGroups: [passes],
mask: null,
});
return;
}
if (
node instanceof VideoNode ||
node instanceof ImageNode ||
node instanceof StickerNode ||
node instanceof GraphicNode
) {
await collectVisualSourceNode({
node,
renderer,
path,
items,
textures,
});
return;
}
if (node instanceof TextNode) {
collectTextNode({
node,
renderer,
path,
items,
textures,
});
}
}
async function collectVisualSourceNode({
node,
renderer,
path,
items,
textures,
}: {
node: VideoNode | ImageNode | StickerNode | GraphicNode;
renderer: CanvasRenderer;
path: string;
items: FrameItemDescriptor[];
textures: Map<string, TextureUploadDescriptor>;
}) {
if (!node.resolved) {
return;
}
const source =
node instanceof GraphicNode
? node.getSource({ resolvedParams: node.resolved.resolvedParams })
: node.resolved.source;
if (!source) {
return;
}
const sourceWidth =
node instanceof GraphicNode
? DEFAULT_GRAPHIC_SOURCE_SIZE
: (node.resolved as ResolvedVisualSourceNodeState).sourceWidth;
const sourceHeight =
node instanceof GraphicNode
? DEFAULT_GRAPHIC_SOURCE_SIZE
: (node.resolved as ResolvedVisualSourceNodeState).sourceHeight;
const textureId = `${path}:source`;
textures.set(textureId, {
id: textureId,
source,
width: sourceWidth,
height: sourceHeight,
});
const transform = computeVisualTransform({
renderer,
resolved: node.resolved,
sourceWidth,
sourceHeight,
});
const { mask, strokeLayer } = buildMaskArtifacts({
node,
renderer,
path,
transform,
textures,
});
items.push({
type: "layer",
textureId,
transform,
opacity: node.resolved.opacity,
blendMode: node.params.blendMode ?? "normal",
effectPassGroups: node.resolved.effectPasses,
mask,
});
if (strokeLayer) {
items.push(strokeLayer);
}
}
function collectTextNode({
node,
renderer,
path,
items,
textures,
}: {
node: TextNode;
renderer: CanvasRenderer;
path: string;
items: FrameItemDescriptor[];
textures: Map<string, TextureUploadDescriptor>;
}) {
if (!node.resolved) {
return;
}
const textureId = `${path}:text`;
const canvas = createOffscreenCanvas({
width: renderer.width,
height: renderer.height,
});
const ctx = canvas.getContext("2d") as
| CanvasRenderingContext2D
| OffscreenCanvasRenderingContext2D
| null;
if (!ctx) {
return;
}
renderTextToContext({
node,
ctx,
});
textures.set(textureId, {
id: textureId,
source: canvas,
width: renderer.width,
height: renderer.height,
});
items.push({
type: "layer",
textureId,
transform: fullCanvasTransform(renderer),
opacity: node.resolved.opacity,
blendMode: node.params.blendMode ?? "normal",
effectPassGroups: node.resolved.effectPasses,
mask: null,
});
}
function computeVisualTransform({
renderer,
resolved,
sourceWidth,
sourceHeight,
}: {
renderer: CanvasRenderer;
resolved: ResolvedVisualSourceNodeState | ResolvedGraphicNodeState;
sourceWidth: number;
sourceHeight: number;
}): QuadTransformDescriptor {
const containScale = Math.min(
renderer.width / sourceWidth,
renderer.height / sourceHeight,
);
const scaledWidth = sourceWidth * containScale * resolved.transform.scaleX;
const scaledHeight = sourceHeight * containScale * resolved.transform.scaleY;
const absWidth = Math.abs(scaledWidth);
const absHeight = Math.abs(scaledHeight);
return {
centerX: renderer.width / 2 + resolved.transform.position.x,
centerY: renderer.height / 2 + resolved.transform.position.y,
width: absWidth,
height: absHeight,
rotationDegrees: resolved.transform.rotate,
flipX: scaledWidth < 0,
flipY: scaledHeight < 0,
};
}
function fullCanvasTransform(renderer: CanvasRenderer): QuadTransformDescriptor {
return {
centerX: renderer.width / 2,
centerY: renderer.height / 2,
width: renderer.width,
height: renderer.height,
rotationDegrees: 0,
flipX: false,
flipY: false,
};
}
function buildMaskArtifacts({
node,
renderer,
path,
transform,
textures,
}: {
node: VideoNode | ImageNode | StickerNode | GraphicNode;
renderer: CanvasRenderer;
path: string;
transform: QuadTransformDescriptor;
textures: Map<string, TextureUploadDescriptor>;
}): {
mask: LayerMaskDescriptor | null;
strokeLayer: FrameItemDescriptor | null;
} {
const mask = node.params.masks?.[0];
if (!mask) {
return { mask: null, strokeLayer: null };
}
const definition = masksRegistry.get(mask.type);
const elementMaskCanvas = createOffscreenCanvas({
width: Math.round(transform.width),
height: Math.round(transform.height),
});
const elementMaskCtx = elementMaskCanvas.getContext("2d") as
| CanvasRenderingContext2D
| OffscreenCanvasRenderingContext2D
| null;
if (!elementMaskCtx) {
return { mask: null, strokeLayer: null };
}
elementMaskCtx.clearRect(0, 0, transform.width, transform.height);
let strokePath: Path2D | null = null;
let feather = mask.params.feather;
if (mask.params.feather > 0 && definition.renderer.renderMask) {
definition.renderer.renderMask({
resolvedParams: mask.params,
ctx: elementMaskCtx,
width: Math.round(transform.width),
height: Math.round(transform.height),
feather: mask.params.feather,
});
feather = 0;
strokePath = definition.renderer.buildStrokePath?.({
resolvedParams: mask.params,
width: transform.width,
height: transform.height,
}) ?? null;
} else {
const path2d = definition.renderer.buildPath({
resolvedParams: mask.params,
width: transform.width,
height: transform.height,
});
elementMaskCtx.fillStyle = "white";
elementMaskCtx.fill(path2d);
strokePath =
definition.renderer.buildStrokePath?.({
resolvedParams: mask.params,
width: transform.width,
height: transform.height,
}) ?? path2d;
}
const fullMaskCanvas = createOffscreenCanvas({
width: renderer.width,
height: renderer.height,
});
const fullMaskCtx = fullMaskCanvas.getContext("2d") as
| CanvasRenderingContext2D
| OffscreenCanvasRenderingContext2D
| null;
if (!fullMaskCtx) {
return { mask: null, strokeLayer: null };
}
drawTransformedCanvas({
ctx: fullMaskCtx,
source: elementMaskCanvas,
transform,
});
const maskTextureId = `${path}:mask`;
textures.set(maskTextureId, {
id: maskTextureId,
source: fullMaskCanvas,
width: renderer.width,
height: renderer.height,
});
let strokeLayer: FrameItemDescriptor | null = null;
if (mask.params.strokeWidth > 0 && strokePath) {
const strokeCanvas = createOffscreenCanvas({
width: Math.round(transform.width),
height: Math.round(transform.height),
});
const strokeCtx = strokeCanvas.getContext("2d") as
| CanvasRenderingContext2D
| OffscreenCanvasRenderingContext2D
| null;
if (strokeCtx) {
strokeCtx.strokeStyle = mask.params.strokeColor;
strokeCtx.lineWidth = mask.params.strokeWidth;
strokeCtx.stroke(strokePath);
const fullStrokeCanvas = createOffscreenCanvas({
width: renderer.width,
height: renderer.height,
});
const fullStrokeCtx = fullStrokeCanvas.getContext("2d") as
| CanvasRenderingContext2D
| OffscreenCanvasRenderingContext2D
| null;
if (fullStrokeCtx) {
drawTransformedCanvas({
ctx: fullStrokeCtx,
source: strokeCanvas,
transform,
});
const strokeTextureId = `${path}:mask-stroke`;
textures.set(strokeTextureId, {
id: strokeTextureId,
source: fullStrokeCanvas,
width: renderer.width,
height: renderer.height,
});
strokeLayer = {
type: "layer",
textureId: strokeTextureId,
transform: fullCanvasTransform(renderer),
opacity: 1,
blendMode: "normal",
effectPassGroups: [],
mask: null,
};
}
}
}
return {
mask: {
textureId: maskTextureId,
feather,
inverted: mask.params.inverted,
},
strokeLayer,
};
}
function drawTransformedCanvas({
ctx,
source,
transform,
}: {
ctx: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;
source: CanvasImageSource;
transform: QuadTransformDescriptor;
}) {
const x = transform.centerX - transform.width / 2;
const y = transform.centerY - transform.height / 2;
const flipX = transform.flipX ? -1 : 1;
const flipY = transform.flipY ? -1 : 1;
const requiresTransform =
transform.rotationDegrees !== 0 || flipX !== 1 || flipY !== 1;
ctx.save();
if (requiresTransform) {
ctx.translate(transform.centerX, transform.centerY);
ctx.rotate((transform.rotationDegrees * Math.PI) / 180);
ctx.scale(flipX, flipY);
ctx.translate(-transform.centerX, -transform.centerY);
}
ctx.drawImage(source, x, y, transform.width, transform.height);
ctx.restore();
}

42
apps/web/src/services/renderer/compositor/types.ts Normal file
View File

@ -0,0 +1,42 @@
import type { BlendMode } from "@/lib/rendering";
import type { EffectPass } from "@/lib/effects/types";
export type FrameDescriptor = {
width: number;
height: number;
clear: {
color: [number, number, number, number];
};
items: FrameItemDescriptor[];
};
export type FrameItemDescriptor =
| {
type: "layer";
textureId: string;
transform: QuadTransformDescriptor;
opacity: number;
blendMode: BlendMode;
effectPassGroups: EffectPass[][];
mask: LayerMaskDescriptor | null;
}
| {
type: "sceneEffect";
effectPassGroups: EffectPass[][];
};
export type QuadTransformDescriptor = {
centerX: number;
centerY: number;
width: number;
height: number;
rotationDegrees: number;
flipX: boolean;
flipY: boolean;
};
export type LayerMaskDescriptor = {
textureId: string;
feather: number;
inverted: boolean;
};

142
apps/web/src/services/renderer/compositor/wasm-compositor.ts Normal file
View File

@ -0,0 +1,142 @@
import {
getCompositorCanvas,
initCompositor,
releaseTexture,
renderFrame,
resizeCompositor,
uploadTexture,
} from "opencut-wasm";
import type { FrameDescriptor } from "./types";
function ensureOffscreenCanvas({
source,
width,
height,
label,
}: {
source: CanvasImageSource;
width: number;
height: number;
label: string;
}): OffscreenCanvas {
if (source instanceof OffscreenCanvas) {
return source;
}
if (typeof OffscreenCanvas === "undefined") {
throw new Error(`OffscreenCanvas is required for ${label}`);
}
const canvas = new OffscreenCanvas(width, height);
const context = canvas.getContext("2d");
if (!context) {
throw new Error(`Failed to get 2d context for ${label}`);
}
context.clearRect(0, 0, width, height);
context.drawImage(source, 0, 0, width, height);
return canvas;
}
export type TextureUploadDescriptor = {
id: string;
source: CanvasImageSource;
width: number;
height: number;
};
class WasmCompositor {
private canvas: HTMLCanvasElement | null = null;
private initializedSize: { width: number; height: number } | null = null;
private retainedTextureIds = new Set<string>();
private uploadedTextures = new Map<
string,
{ source: CanvasImageSource; width: number; height: number }
>();
private _debugLogged = false;
ensureInitialized({ width, height }: { width: number; height: number }) {
if (!this.canvas) {
initCompositor(width, height);
this.canvas = getCompositorCanvas();
this.initializedSize = { width, height };
return;
}
if (
!this.initializedSize ||
this.initializedSize.width !== width ||
this.initializedSize.height !== height
) {
resizeCompositor(width, height);
this.initializedSize = { width, height };
}
}
getCanvas(): HTMLCanvasElement {
if (!this.canvas) {
throw new Error("Compositor is not initialized");
}
return this.canvas;
}
syncTextures(textures: TextureUploadDescriptor[]) {
const nextIds = new Set(textures.map((texture) => texture.id));
for (const previousId of this.retainedTextureIds) {
if (!nextIds.has(previousId)) {
releaseTexture(previousId);
this.uploadedTextures.delete(previousId);
}
}
for (const texture of textures) {
const previousTexture = this.uploadedTextures.get(texture.id);
if (
previousTexture?.source === texture.source &&
previousTexture.width === texture.width &&
previousTexture.height === texture.height
) {
continue;
}
const sourceCanvas = ensureOffscreenCanvas({
source: texture.source,
width: texture.width,
height: texture.height,
label: `texture upload ${texture.id}`,
});
uploadTexture({
id: texture.id,
source: sourceCanvas,
width: texture.width,
height: texture.height,
});
this.uploadedTextures.set(texture.id, {
source: texture.source,
width: texture.width,
height: texture.height,
});
}
this.retainedTextureIds = nextIds;
}
render(frame: FrameDescriptor) {
if (!this._debugLogged) {
this._debugLogged = true;
const firstLayer = frame.items.find((item) => item.type === "layer");
console.log(
"[compositor] first frame — canvas size:",
JSON.stringify({ width: frame.width, height: frame.height }),
"| first layer transform:",
firstLayer && firstLayer.type === "layer"
? JSON.stringify(firstLayer.transform)
: "none",
"| webgpu available:",
typeof navigator !== "undefined" && "gpu" in navigator,
);
}
renderFrame(frame);
}
}
export const wasmCompositor = new WasmCompositor();

6
apps/web/src/services/renderer/gpu-renderer.ts
View File

@ -13,12 +13,18 @@ export function initializeGpuRenderer(): Promise<void> {
initPromise = initializeGpu()
.then(() => {
gpuAvailable = true;
// #region agent log
fetch('http://127.0.0.1:7408/ingest/669b22f8-172b-4e65-aa3f-1c702ede83f7',{method:'POST',headers:{'Content-Type':'application/json','X-Debug-Session-Id':'b140a6'},body:JSON.stringify({sessionId:'b140a6',location:'gpu-renderer.ts',message:'GPU init SUCCESS',data:{userAgent:navigator.userAgent},timestamp:Date.now()})}).catch(()=>{});
// #endregion
})
.catch((error: unknown) => {
gpuAvailable = false;
const message =
error instanceof Error ? error.message : String(error);
console.warn(`GPU renderer unavailable: ${message}`);
// #region agent log
fetch('http://127.0.0.1:7408/ingest/669b22f8-172b-4e65-aa3f-1c702ede83f7',{method:'POST',headers:{'Content-Type':'application/json','X-Debug-Session-Id':'b140a6'},body:JSON.stringify({sessionId:'b140a6',location:'gpu-renderer.ts',message:'GPU init FAILED',data:{error:message,userAgent:navigator.userAgent,hasGpu:!!navigator.gpu},timestamp:Date.now()})}).catch(()=>{});
// #endregion
});
}
return initPromise;

27
apps/web/src/services/renderer/nodes/base-node.ts
View File

@ -1,35 +1,26 @@
import type { CanvasRenderer } from "../canvas-renderer";
export type BaseNodeParams = object | undefined;
export type AnyBaseNode = BaseNode<BaseNodeParams, unknown>;
export class BaseNode<Params extends BaseNodeParams = BaseNodeParams> {
export class BaseNode<
Params extends BaseNodeParams = BaseNodeParams,
Resolved = unknown,
> {
params: Params;
resolved: Resolved | null = null;
constructor(params?: Params) {
this.params = params ?? ({} as Params);
}
children: BaseNode[] = [];
children: AnyBaseNode[] = [];
add(child: BaseNode) {
add(child: AnyBaseNode) {
this.children.push(child);
return this;
}
remove(child: BaseNode) {
remove(child: AnyBaseNode) {
this.children = this.children.filter((c) => c !== child);
return this;
}
async render({
renderer,
time,
}: {
renderer: CanvasRenderer;
time: number;
}): Promise<void> {
for (const child of this.children) {
await child.render({ renderer, time });
}
}
}

136
apps/web/src/services/renderer/nodes/blur-background-node.ts
View File

@ -1,13 +1,6 @@
import { buildGaussianBlurPasses, intensityToSigma } from "@/lib/effects/definitions/blur";
import { mediaTimeToSeconds } from "opencut-wasm";
import { getSourceTimeAtClipTime } from "@/lib/retime";
import { videoCache } from "@/services/video-cache/service";
import type { EffectPass } from "@/lib/effects/types";
import type { RetimeConfig } from "@/lib/timeline";
import type { CanvasRenderer } from "../canvas-renderer";
import { createOffscreenCanvas } from "../canvas-utils";
import { gpuRenderer } from "../gpu-renderer";
import { BaseNode } from "./base-node";
import { loadImageSource, type CachedImageSource } from "./image-node";
export type BlurBackgroundNodeParams = {
mediaId: string;
@ -22,127 +15,18 @@ export type BlurBackgroundNodeParams = {
blurIntensity: number;
};
type BackdropSource = {
export type BackdropSource = {
source: CanvasImageSource;
width: number;
height: number;
};
export class BlurBackgroundNode extends BaseNode<BlurBackgroundNodeParams> {
private cachedImageSource: Promise<CachedImageSource> | null;
constructor(params: BlurBackgroundNodeParams) {
super(params);
this.cachedImageSource =
params.mediaType === "image" ? loadImageSource(params.url) : null;
}
private isInRange({ time }: { time: number }): boolean {
const localTime = time - this.params.timeOffset;
return localTime >= 0 && localTime < this.params.duration;
}
private getSourceLocalTime({ time }: { time: number }): number {
const clipTime = time - this.params.timeOffset;
return (
this.params.trimStart +
getSourceTimeAtClipTime({
clipTime,
retime: this.params.retime,
})
);
}
private async getBackdropSource({
time,
}: {
time: number;
}): Promise<BackdropSource | null> {
if (this.params.mediaType === "video") {
const sourceTimeTicks = this.getSourceLocalTime({ time });
const frame = await videoCache.getFrameAt({
mediaId: this.params.mediaId,
file: this.params.file,
time: mediaTimeToSeconds({ time: sourceTimeTicks }),
});
if (!frame) {
return null;
}
return {
source: frame.canvas,
width: frame.canvas.width,
height: frame.canvas.height,
};
}
if (!this.cachedImageSource) {
return null;
}
const { source, width, height } = await this.cachedImageSource;
return { source, width, height };
}
async render({ renderer, time }: { renderer: CanvasRenderer; time: number }) {
await super.render({ renderer, time });
if (!this.isInRange({ time })) {
return;
}
const backdropSource = await this.getBackdropSource({ time });
if (!backdropSource) {
return;
}
const offscreen = createOffscreenCanvas({
width: renderer.width,
height: renderer.height,
});
const offscreenCtx = offscreen.getContext("2d") as
| CanvasRenderingContext2D
| OffscreenCanvasRenderingContext2D
| null;
if (!offscreenCtx) {
return;
}
const coverScale = Math.max(
renderer.width / backdropSource.width,
renderer.height / backdropSource.height,
);
const scaledWidth = backdropSource.width * coverScale;
const scaledHeight = backdropSource.height * coverScale;
const offsetX = (renderer.width - scaledWidth) / 2;
const offsetY = (renderer.height - scaledHeight) / 2;
offscreenCtx.drawImage(
backdropSource.source,
offsetX,
offsetY,
scaledWidth,
scaledHeight,
);
const passes = buildGaussianBlurPasses({
sigmaX: intensityToSigma({ intensity: this.params.blurIntensity, resolution: renderer.width, reference: 1920 }),
sigmaY: intensityToSigma({ intensity: this.params.blurIntensity, resolution: renderer.height, reference: 1080 }),
});
const effectResult = gpuRenderer.applyEffect({
source: offscreen as CanvasImageSource,
width: renderer.width,
height: renderer.height,
passes,
});
renderer.context.drawImage(
effectResult,
0,
0,
renderer.width,
renderer.height,
);
}
export interface ResolvedBlurBackgroundNodeState {
backdropSource: BackdropSource;
passes: EffectPass[];
}
export class BlurBackgroundNode extends BaseNode<
BlurBackgroundNodeParams,
ResolvedBlurBackgroundNodeState
> {}

26
apps/web/src/services/renderer/nodes/color-node.ts
View File

@ -1,31 +1,7 @@
import { drawCssBackground } from "@/lib/gradients";
import type { CanvasRenderer } from "../canvas-renderer";
import { BaseNode } from "./base-node";
export type ColorNodeParams = {
color: string;
};
export class ColorNode extends BaseNode<ColorNodeParams> {
private color: string;
constructor(params: ColorNodeParams) {
super(params);
this.color = params.color;
}
async render({ renderer }: { renderer: CanvasRenderer }) {
if (/gradient\(/i.test(this.color)) {
drawCssBackground({
ctx: renderer.context,
width: renderer.width,
height: renderer.height,
css: this.color,
});
return;
}
renderer.context.fillStyle = this.color;
renderer.context.fillRect(0, 0, renderer.width, renderer.height);
}
}
export class ColorNode extends BaseNode<ColorNodeParams> {}

78
apps/web/src/services/renderer/nodes/effect-layer-node.ts
View File

@ -1,10 +1,6 @@
import type { CanvasRenderer } from "../canvas-renderer";
import { effectsRegistry, resolveEffectPasses } from "@/lib/effects";
import type { EffectPass } from "@/lib/effects/types";
import type { ParamValues } from "@/lib/params";
import { BaseNode } from "./base-node";
import { gpuRenderer } from "../gpu-renderer";
const TIME_EPSILON = 1e-6;
export type EffectLayerNodeParams = {
effectType: string;
@ -13,69 +9,11 @@ export type EffectLayerNodeParams = {
duration: number;
};
function isInRange({
time,
timeOffset,
duration,
}: {
time: number;
timeOffset: number;
duration: number;
}): boolean {
return (
time >= timeOffset - TIME_EPSILON &&
time < timeOffset + duration + TIME_EPSILON
);
}
export type ResolvedEffectLayerNodeState = {
passes: EffectPass[];
};
// snapshots whatever is currently on the canvas, applies the effect, draws it back
export class EffectLayerNode extends BaseNode<EffectLayerNodeParams> {
async render({
renderer,
time,
}: {
renderer: CanvasRenderer;
time: number;
}): Promise<void> {
if (
!isInRange({
time,
timeOffset: this.params.timeOffset,
duration: this.params.duration,
})
) {
return;
}
const source = renderer.context.canvas as CanvasImageSource;
const effectDefinition = effectsRegistry.get(this.params.effectType);
const passes = resolveEffectPasses({
definition: effectDefinition,
effectParams: this.params.effectParams,
width: renderer.width,
height: renderer.height,
});
if (passes.length === 0) {
return;
}
const effectResult = gpuRenderer.applyEffect({
source,
width: renderer.width,
height: renderer.height,
passes,
});
renderer.context.save();
renderer.context.clearRect(0, 0, renderer.width, renderer.height);
renderer.context.drawImage(
effectResult,
0,
0,
renderer.width,
renderer.height,
);
renderer.context.restore();
}
}
export class EffectLayerNode extends BaseNode<
EffectLayerNodeParams,
ResolvedEffectLayerNodeState
> {}

166
apps/web/src/services/renderer/nodes/graphic-node.ts
View File

@ -1,92 +1,74 @@
import type { CanvasRenderer } from "../canvas-renderer";
import { createOffscreenCanvas } from "../canvas-utils";
import {
DEFAULT_GRAPHIC_SOURCE_SIZE,
getGraphicDefinition,
registerDefaultGraphics,
} from "@/lib/graphics";
import { resolveGraphicParamsAtTime } from "@/lib/animation";
import type { ParamValues } from "@/lib/params";
import { VisualNode, type VisualNodeParams } from "./visual-node";
export interface GraphicNodeParams extends VisualNodeParams {
definitionId: string;
params: ParamValues;
}
export class GraphicNode extends VisualNode<GraphicNodeParams> {
private cachedKey: string | null = null;
private cachedSource: OffscreenCanvas | HTMLCanvasElement | null = null;
constructor(params: GraphicNodeParams) {
super(params);
registerDefaultGraphics();
}
private getSource({
localTime,
}: {
localTime: number;
}): OffscreenCanvas | HTMLCanvasElement | null {
const definition = getGraphicDefinition({
definitionId: this.params.definitionId,
});
const resolvedParams = resolveGraphicParamsAtTime({
element: this.params,
localTime,
});
const cacheKey = JSON.stringify({
definitionId: this.params.definitionId,
params: resolvedParams,
});
if (this.cachedSource && this.cachedKey === cacheKey) {
return this.cachedSource;
}
const canvas = createOffscreenCanvas({
width: DEFAULT_GRAPHIC_SOURCE_SIZE,
height: DEFAULT_GRAPHIC_SOURCE_SIZE,
});
const ctx = canvas.getContext("2d") as
| CanvasRenderingContext2D
| OffscreenCanvasRenderingContext2D
| null;
if (!ctx) {
return null;
}
definition.render({
ctx,
params: resolvedParams,
width: DEFAULT_GRAPHIC_SOURCE_SIZE,
height: DEFAULT_GRAPHIC_SOURCE_SIZE,
});
this.cachedKey = cacheKey;
this.cachedSource = canvas;
return canvas;
}
async render({ renderer, time }: { renderer: CanvasRenderer; time: number }) {
await super.render({ renderer, time });
if (!this.isInRange({ time })) {
return;
}
const source = this.getSource({
localTime: this.getAnimationLocalTime({ time }),
});
if (!source) {
return;
}
this.renderVisual({
renderer,
source,
sourceWidth: DEFAULT_GRAPHIC_SOURCE_SIZE,
sourceHeight: DEFAULT_GRAPHIC_SOURCE_SIZE,
timelineTime: time,
});
}
}
import { createOffscreenCanvas } from "../canvas-utils";
import {
DEFAULT_GRAPHIC_SOURCE_SIZE,
getGraphicDefinition,
registerDefaultGraphics,
} from "@/lib/graphics";
import type { ParamValues } from "@/lib/params";
import {
VisualNode,
type ResolvedVisualNodeState,
type VisualNodeParams,
} from "./visual-node";
export interface GraphicNodeParams extends VisualNodeParams {
definitionId: string;
params: ParamValues;
}
export interface ResolvedGraphicNodeState extends ResolvedVisualNodeState {
resolvedParams: ParamValues;
}
export class GraphicNode extends VisualNode<
GraphicNodeParams,
ResolvedGraphicNodeState
> {
private cachedKey: string | null = null;
private cachedSource: OffscreenCanvas | HTMLCanvasElement | null = null;
constructor(params: GraphicNodeParams) {
super(params);
registerDefaultGraphics();
}
getSource({
resolvedParams,
}: {
resolvedParams: ParamValues;
}): OffscreenCanvas | HTMLCanvasElement | null {
const definition = getGraphicDefinition({
definitionId: this.params.definitionId,
});
const cacheKey = JSON.stringify({
definitionId: this.params.definitionId,
params: resolvedParams,
});
if (this.cachedSource && this.cachedKey === cacheKey) {
return this.cachedSource;
}
const canvas = createOffscreenCanvas({
width: DEFAULT_GRAPHIC_SOURCE_SIZE,
height: DEFAULT_GRAPHIC_SOURCE_SIZE,
});
const ctx = canvas.getContext("2d") as
| CanvasRenderingContext2D
| OffscreenCanvasRenderingContext2D
| null;
if (!ctx) {
return null;
}
definition.render({
ctx,
params: resolvedParams,
width: DEFAULT_GRAPHIC_SOURCE_SIZE,
height: DEFAULT_GRAPHIC_SOURCE_SIZE,
});
this.cachedKey = cacheKey;
this.cachedSource = canvas;
return canvas;
}
}

37
apps/web/src/services/renderer/nodes/image-node.ts
View File

@ -1,5 +1,8 @@
import type { CanvasRenderer } from "../canvas-renderer";
import { VisualNode, type VisualNodeParams } from "./visual-node";
import {
VisualNode,
type ResolvedVisualSourceNodeState,
type VisualNodeParams,
} from "./visual-node";
export interface ImageNodeParams extends VisualNodeParams {
url: string;
@ -62,29 +65,7 @@ export function loadImageSource(
return promise;
}
export class ImageNode extends VisualNode<ImageNodeParams> {
private cachedSource: Promise<CachedImageSource>;
constructor(params: ImageNodeParams) {
super(params);
this.cachedSource = loadImageSource(params.url, params.maxSourceSize);
}
async render({ renderer, time }: { renderer: CanvasRenderer; time: number }) {
await super.render({ renderer, time });
if (!this.isInRange({ time })) {
return;
}
const { source, width, height } = await this.cachedSource;
this.renderVisual({
renderer,
source,
sourceWidth: width || renderer.width,
sourceHeight: height || renderer.height,
timelineTime: time,
});
}
}
export class ImageNode extends VisualNode<
ImageNodeParams,
ResolvedVisualSourceNodeState
> {}

49
apps/web/src/services/renderer/nodes/sticker-node.ts
View File

@ -1,6 +1,9 @@
import type { CanvasRenderer } from "../canvas-renderer";
import { resolveStickerId } from "@/lib/stickers";
import { VisualNode, type VisualNodeParams } from "./visual-node";
import {
VisualNode,
type ResolvedVisualSourceNodeState,
type VisualNodeParams,
} from "./visual-node";
export interface StickerNodeParams extends VisualNodeParams {
stickerId: string;
@ -16,7 +19,11 @@ interface CachedStickerSource {
const stickerSourceCache = new Map<string, Promise<CachedStickerSource>>();
function loadStickerSource({ stickerId }: { stickerId: string }): Promise<CachedStickerSource> {
export function loadStickerSource({
stickerId,
}: {
stickerId: string;
}): Promise<CachedStickerSource> {
const cached = stickerSourceCache.get(stickerId);
if (cached) return cached;
@ -42,35 +49,7 @@ function loadStickerSource({ stickerId }: { stickerId: string }): Promise<Cached
return promise;
}
export class StickerNode extends VisualNode<StickerNodeParams> {
private cachedSource: Promise<CachedStickerSource>;
constructor(params: StickerNodeParams) {
super(params);
this.cachedSource = loadStickerSource({ stickerId: params.stickerId });
}
async render({ renderer, time }: { renderer: CanvasRenderer; time: number }) {
await super.render({ renderer, time });
if (!this.isInRange({ time })) {
return;
}
const { source, width: loadedWidth, height: loadedHeight } =
await this.cachedSource;
// Prefer element-stored intrinsic dimensions as the geometry authority.
// The loaded image is only the drawable source.
const sourceWidth = this.params.intrinsicWidth ?? loadedWidth;
const sourceHeight = this.params.intrinsicHeight ?? loadedHeight;
this.renderVisual({
renderer,
source,
sourceWidth,
sourceHeight,
timelineTime: time,
});
}
}
export class StickerNode extends VisualNode<
StickerNodeParams,
ResolvedVisualSourceNodeState
> {}

380
apps/web/src/services/renderer/nodes/text-node.ts
View File

@ -1,289 +1,129 @@
import type { CanvasRenderer } from "../canvas-renderer";
import { createOffscreenCanvas } from "../canvas-utils";
import { BaseNode } from "./base-node";
import type { TextElement } from "@/lib/timeline";
import type { EffectPass } from "@/lib/effects/types";
import type { Transform } from "@/lib/rendering";
import {
CORNER_RADIUS_MAX,
CORNER_RADIUS_MIN,
} from "@/constants/text-constants";
import {
getMetricAscent,
getMetricDescent,
drawTextDecoration,
getTextBackgroundRect,
setCanvasLetterSpacing,
} from "@/lib/text/layout";
import { measureTextElement } from "@/lib/text/measure-element";
import {
getElementLocalTime,
resolveColorAtTime,
resolveOpacityAtTime,
resolveTransformAtTime,
} from "@/lib/animation";
import { resolveEffectParamsAtTime } from "@/lib/animation/effect-param-channel";
import { effectsRegistry, resolveEffectPasses } from "@/lib/effects";
import { gpuRenderer } from "../gpu-renderer";
import type { MeasuredTextElement } from "@/lib/text/measure-element";
import { clamp } from "@/utils/math";
const TEXT_DECORATION_THICKNESS_RATIO = 0.07;
const STRIKETHROUGH_VERTICAL_RATIO = 0.35;
function drawTextDecoration({
ctx,
textDecoration,
lineWidth,
lineY,
metrics,
scaledFontSize,
textAlign,
}: {
ctx: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;
textDecoration: string;
lineWidth: number;
lineY: number;
metrics: TextMetrics;
scaledFontSize: number;
textAlign: CanvasTextAlign;
}): void {
if (textDecoration === "none" || !textDecoration) return;
const thickness = Math.max(
1,
scaledFontSize * TEXT_DECORATION_THICKNESS_RATIO,
);
const ascent = getMetricAscent({ metrics, fallbackFontSize: scaledFontSize });
const descent = getMetricDescent({
metrics,
fallbackFontSize: scaledFontSize,
});
let xStart = -lineWidth / 2;
if (textAlign === "left") xStart = 0;
if (textAlign === "right") xStart = -lineWidth;
if (textDecoration === "underline") {
const underlineY = lineY + descent + thickness;
ctx.fillRect(xStart, underlineY, lineWidth, thickness);
}
if (textDecoration === "line-through") {
const strikeY = lineY - (ascent - descent) * STRIKETHROUGH_VERTICAL_RATIO;
ctx.fillRect(xStart, strikeY, lineWidth, thickness);
}
}
export type TextNodeParams = TextElement & {
canvasCenter: { x: number; y: number };
canvasHeight: number;
textBaseline?: CanvasTextBaseline;
};
export class TextNode extends BaseNode<TextNodeParams> {
isInRange({ time }: { time: number }) {
return (
time >= this.params.startTime &&
time < this.params.startTime + this.params.duration
);
}
async render({ renderer, time }: { renderer: CanvasRenderer; time: number }) {
if (!this.isInRange({ time })) {
return;
}
const localTime = getElementLocalTime({
timelineTime: time,
elementStartTime: this.params.startTime,
elementDuration: this.params.duration,
});
const transform = resolveTransformAtTime({
baseTransform: this.params.transform,
animations: this.params.animations,
localTime,
});
const opacity = resolveOpacityAtTime({
baseOpacity: this.params.opacity,
animations: this.params.animations,
localTime,
});
const x = transform.position.x + this.params.canvasCenter.x;
const y = transform.position.y + this.params.canvasCenter.y;
const baseline = this.params.textBaseline ?? "middle";
const blendMode = (
this.params.blendMode && this.params.blendMode !== "normal"
? this.params.blendMode
: "source-over"
) as GlobalCompositeOperation;
const {
scaledFontSize,
fontString,
letterSpacing,
lineHeightPx,
lines,
lineMetrics,
block,
fontSizeRatio,
resolvedBackground,
} = measureTextElement({
element: this.params,
canvasHeight: this.params.canvasHeight,
localTime,
ctx: renderer.context,
});
const lineCount = lines.length;
const textColor = resolveColorAtTime({
baseColor: this.params.color,
animations: this.params.animations,
propertyPath: "color",
localTime,
});
const resolvedBackgroundWithColor = {
...resolvedBackground,
color: resolveColorAtTime({
baseColor: this.params.background.color,
animations: this.params.animations,
propertyPath: "background.color",
localTime,
}),
};
const drawContent = (
ctx: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D,
) => {
ctx.font = fontString;
ctx.textAlign = this.params.textAlign;
ctx.textBaseline = baseline;
ctx.fillStyle = textColor;
setCanvasLetterSpacing({ ctx, letterSpacingPx: letterSpacing });
if (
this.params.background.enabled &&
this.params.background.color &&
this.params.background.color !== "transparent" &&
lineCount > 0
) {
const backgroundRect = getTextBackgroundRect({
textAlign: this.params.textAlign,
block,
background: resolvedBackgroundWithColor,
fontSizeRatio,
});
if (backgroundRect) {
const p =
clamp({
value: resolvedBackgroundWithColor.cornerRadius,
min: CORNER_RADIUS_MIN,
max: CORNER_RADIUS_MAX,
}) / 100;
const radius =
(Math.min(backgroundRect.width, backgroundRect.height) / 2) * p;
ctx.fillStyle = resolvedBackgroundWithColor.color;
ctx.beginPath();
ctx.roundRect(
backgroundRect.left,
backgroundRect.top,
backgroundRect.width,
backgroundRect.height,
radius,
);
ctx.fill();
ctx.fillStyle = textColor;
}
}
for (let i = 0; i < lineCount; i++) {
const lineY = i * lineHeightPx - block.visualCenterOffset;
ctx.fillText(lines[i], 0, lineY);
drawTextDecoration({
ctx,
textDecoration: this.params.textDecoration ?? "none",
lineWidth: lineMetrics[i].width,
lineY,
metrics: lineMetrics[i],
scaledFontSize,
textAlign: this.params.textAlign,
});
}
};
const applyTransform = (
ctx: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D,
) => {
ctx.translate(x, y);
ctx.scale(transform.scaleX, transform.scaleY);
if (transform.rotate) {
ctx.rotate((transform.rotate * Math.PI) / 180);
}
};
const enabledEffects =
this.params.effects?.filter((effect) => effect.enabled) ?? [];
if (enabledEffects.length === 0) {
renderer.context.save();
applyTransform(renderer.context);
renderer.context.globalCompositeOperation = blendMode;
renderer.context.globalAlpha = opacity;
drawContent(renderer.context);
renderer.context.restore();
return;
}
// Effects path: render text to a same-size offscreen canvas so the blur
// can spread into the surrounding transparent area without hard clipping.
const offscreen = createOffscreenCanvas({
width: renderer.width,
height: renderer.height,
});
const offscreenCtx = offscreen.getContext(
"2d",
) as OffscreenCanvasRenderingContext2D | null;
if (!offscreenCtx) {
renderer.context.save();
applyTransform(renderer.context);
renderer.context.globalCompositeOperation = blendMode;
renderer.context.globalAlpha = opacity;
drawContent(renderer.context);
renderer.context.restore();
return;
}
offscreenCtx.save();
applyTransform(offscreenCtx);
drawContent(offscreenCtx);
offscreenCtx.restore();
let currentSource: CanvasImageSource = offscreen;
for (const effect of enabledEffects) {
const resolvedParams = resolveEffectParamsAtTime({
effect,
animations: this.params.animations,
localTime,
});
const definition = effectsRegistry.get(effect.type);
const passes = resolveEffectPasses({
definition,
effectParams: resolvedParams,
width: renderer.width,
height: renderer.height,
});
currentSource = gpuRenderer.applyEffect({
source: currentSource,
width: renderer.width,
height: renderer.height,
passes,
});
}
renderer.context.save();
renderer.context.globalCompositeOperation = blendMode;
renderer.context.globalAlpha = opacity;
renderer.context.drawImage(currentSource, 0, 0);
renderer.context.restore();
}
export interface ResolvedTextNodeState {
transform: Transform;
opacity: number;
textColor: string;
backgroundColor: string;
effectPasses: EffectPass[][];
measuredText: MeasuredTextElement;
}
export class TextNode extends BaseNode<TextNodeParams, ResolvedTextNodeState> {}
export function renderTextToContext({
node,
ctx,
}: {
node: TextNode;
ctx: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;
}): void {
const resolved = node.resolved;
if (!resolved) {
return;
}
const x = resolved.transform.position.x + node.params.canvasCenter.x;
const y = resolved.transform.position.y + node.params.canvasCenter.y;
const baseline = node.params.textBaseline ?? "middle";
const {
scaledFontSize,
fontString,
letterSpacing,
lineHeightPx,
lines,
lineMetrics,
block,
fontSizeRatio,
resolvedBackground,
} = resolved.measuredText;
const lineCount = lines.length;
const resolvedBackgroundWithColor = {
...resolvedBackground,
color: resolved.backgroundColor,
};
ctx.save();
ctx.translate(x, y);
ctx.scale(resolved.transform.scaleX, resolved.transform.scaleY);
if (resolved.transform.rotate) {
ctx.rotate((resolved.transform.rotate * Math.PI) / 180);
}
ctx.font = fontString;
ctx.textAlign = node.params.textAlign;
ctx.textBaseline = baseline;
ctx.fillStyle = resolved.textColor;
setCanvasLetterSpacing({ ctx, letterSpacingPx: letterSpacing });
if (
node.params.background.enabled &&
node.params.background.color &&
node.params.background.color !== "transparent" &&
lineCount > 0
) {
const backgroundRect = getTextBackgroundRect({
textAlign: node.params.textAlign,
block,
background: resolvedBackgroundWithColor,
fontSizeRatio,
});
if (backgroundRect) {
const p =
clamp({
value: resolvedBackgroundWithColor.cornerRadius,
min: CORNER_RADIUS_MIN,
max: CORNER_RADIUS_MAX,
}) / 100;
const radius =
(Math.min(backgroundRect.width, backgroundRect.height) / 2) * p;
ctx.fillStyle = resolvedBackgroundWithColor.color;
ctx.beginPath();
ctx.roundRect(
backgroundRect.left,
backgroundRect.top,
backgroundRect.width,
backgroundRect.height,
radius,
);
ctx.fill();
ctx.fillStyle = resolved.textColor;
}
}
for (let index = 0; index < lineCount; index++) {
const lineY = index * lineHeightPx - block.visualCenterOffset;
ctx.fillText(lines[index], 0, lineY);
drawTextDecoration({
ctx,
textDecoration: node.params.textDecoration ?? "none",
lineWidth: lineMetrics[index].width,
lineY,
metrics: lineMetrics[index],
scaledFontSize,
textAlign: node.params.textAlign,
});
}
ctx.restore();
}

41
apps/web/src/services/renderer/nodes/video-node.ts
View File

@ -1,7 +1,8 @@
import type { CanvasRenderer } from "../canvas-renderer";
import { mediaTimeToSeconds } from "opencut-wasm";
import { VisualNode, type VisualNodeParams } from "./visual-node";
import { videoCache } from "@/services/video-cache/service";
import {
VisualNode,
type ResolvedVisualSourceNodeState,
type VisualNodeParams,
} from "./visual-node";
export interface VideoNodeParams extends VisualNodeParams {
url: string;
@ -9,31 +10,7 @@ export interface VideoNodeParams extends VisualNodeParams {
mediaId: string;
}
export class VideoNode extends VisualNode<VideoNodeParams> {
async render({ renderer, time }: { renderer: CanvasRenderer; time: number }) {
await super.render({ renderer, time });
if (!this.isInRange({ time })) {
return;
}
const videoTimeTicks = this.getSourceLocalTime({ time });
const videoTimeSeconds = mediaTimeToSeconds({ time: videoTimeTicks });
const frame = await videoCache.getFrameAt({
mediaId: this.params.mediaId,
file: this.params.file,
time: videoTimeSeconds,
});
if (frame) {
this.renderVisual({
renderer,
source: frame.canvas,
sourceWidth: frame.canvas.width,
sourceHeight: frame.canvas.height,
timelineTime: time,
});
}
}
}
export class VideoNode extends VisualNode<
VideoNodeParams,
ResolvedVisualSourceNodeState
> {}

294
apps/web/src/services/renderer/nodes/visual-node.ts
View File

@ -1,22 +1,8 @@
import type { CanvasRenderer } from "../canvas-renderer";
import { createOffscreenCanvas } from "../canvas-utils";
import { BaseNode } from "./base-node";
import type { Effect } from "@/lib/effects/types";
import type { Effect, EffectPass } from "@/lib/effects/types";
import type { Mask } from "@/lib/masks/types";
import type { BlendMode, Transform } from "@/lib/rendering";
import type { ElementAnimations } from "@/lib/animation/types";
import type { RetimeConfig } from "@/lib/timeline";
import {
getElementLocalTime,
resolveOpacityAtTime,
resolveTransformAtTime,
} from "@/lib/animation";
import { resolveEffectParamsAtTime } from "@/lib/animation/effect-param-channel";
import { effectsRegistry, resolveEffectPasses } from "@/lib/effects";
import { masksRegistry } from "@/lib/masks";
import { getSourceTimeAtClipTime } from "@/lib/retime";
import { gpuRenderer } from "../gpu-renderer";
import { applyMaskFeather } from "../mask-feather";
import type { RetimeConfig, VisualElement } from "@/lib/timeline";
export interface VisualNodeParams {
duration: number;
@ -25,270 +11,28 @@ export interface VisualNodeParams {
trimEnd: number;
retime?: RetimeConfig;
transform: Transform;
animations?: ElementAnimations;
animations?: VisualElement["animations"];
opacity: number;
blendMode?: BlendMode;
effects?: Effect[];
masks?: Mask[];
}
export interface ResolvedVisualNodeState {
localTime: number;
transform: Transform;
opacity: number;
effectPasses: EffectPass[][];
}
export interface ResolvedVisualSourceNodeState
extends ResolvedVisualNodeState {
source: CanvasImageSource;
sourceWidth: number;
sourceHeight: number;
}
export abstract class VisualNode<
Params extends VisualNodeParams = VisualNodeParams,
> extends BaseNode<Params> {
protected getSourceLocalTime({ time }: { time: number }): number {
const clipTime = time - this.params.timeOffset;
return (
this.params.trimStart +
getSourceTimeAtClipTime({
clipTime,
retime: this.params.retime,
})
);
}
protected getAnimationLocalTime({ time }: { time: number }): number {
return getElementLocalTime({
timelineTime: time,
elementStartTime: this.params.timeOffset,
elementDuration: this.params.duration,
});
}
protected isInRange({ time }: { time: number }): boolean {
const localTime = time - this.params.timeOffset;
return (
localTime >= 0 &&
localTime < this.params.duration
);
}
protected renderVisual({
renderer,
source,
sourceWidth,
sourceHeight,
timelineTime,
}: {
renderer: CanvasRenderer;
source: CanvasImageSource;
sourceWidth: number;
sourceHeight: number;
timelineTime: number;
}): void {
renderer.context.save();
const animationLocalTime = this.getAnimationLocalTime({
time: timelineTime,
});
const transform = resolveTransformAtTime({
baseTransform: this.params.transform,
animations: this.params.animations,
localTime: animationLocalTime,
});
const opacity = resolveOpacityAtTime({
baseOpacity: this.params.opacity,
animations: this.params.animations,
localTime: animationLocalTime,
});
const containScale = Math.min(
renderer.width / sourceWidth,
renderer.height / sourceHeight,
);
const scaledWidth = sourceWidth * containScale * transform.scaleX;
const scaledHeight = sourceHeight * containScale * transform.scaleY;
const absWidth = Math.abs(scaledWidth);
const absHeight = Math.abs(scaledHeight);
const x = renderer.width / 2 + transform.position.x - absWidth / 2;
const y = renderer.height / 2 + transform.position.y - absHeight / 2;
renderer.context.globalCompositeOperation = (
this.params.blendMode && this.params.blendMode !== "normal"
? this.params.blendMode
: "source-over"
) as GlobalCompositeOperation;
renderer.context.globalAlpha = opacity;
const flipX = scaledWidth < 0 ? -1 : 1;
const flipY = scaledHeight < 0 ? -1 : 1;
const needsTransform = transform.rotate !== 0 || flipX !== 1 || flipY !== 1;
if (needsTransform) {
const centerX = x + absWidth / 2;
const centerY = y + absHeight / 2;
renderer.context.translate(centerX, centerY);
renderer.context.rotate((transform.rotate * Math.PI) / 180);
renderer.context.scale(flipX, flipY);
renderer.context.translate(-centerX, -centerY);
}
const enabledEffects =
this.params.effects?.filter((effect) => effect.enabled) ?? [];
const activeMasks = this.params.masks ?? [];
if (activeMasks.length === 0 && enabledEffects.length === 0) {
renderer.context.drawImage(source, x, y, absWidth, absHeight);
renderer.context.restore();
return;
}
const currentResult =
enabledEffects.length > 0
? this.applyEffects({
source,
effects: enabledEffects,
width: absWidth,
height: absHeight,
animationLocalTime,
})
: source;
if (activeMasks.length === 0) {
renderer.context.drawImage(currentResult, x, y, absWidth, absHeight);
renderer.context.restore();
return;
}
const elementCanvas = createOffscreenCanvas({
width: Math.round(absWidth),
height: Math.round(absHeight),
});
const elementCtx = elementCanvas.getContext("2d") as
| CanvasRenderingContext2D
| OffscreenCanvasRenderingContext2D
| null;
if (!elementCtx) {
renderer.context.drawImage(currentResult, x, y, absWidth, absHeight);
renderer.context.restore();
return;
}
elementCtx.drawImage(currentResult, 0, 0, absWidth, absHeight);
for (const mask of activeMasks) {
this.applyMask({
mask,
elementCtx,
scaledWidth: absWidth,
scaledHeight: absHeight,
});
}
renderer.context.drawImage(elementCanvas, x, y, absWidth, absHeight);
renderer.context.restore();
}
private applyEffects({
source,
effects,
width,
height,
animationLocalTime,
}: {
source: CanvasImageSource;
effects: Effect[];
width: number;
height: number;
animationLocalTime: number;
}): CanvasImageSource {
let current: CanvasImageSource = source;
for (const effect of effects) {
const resolvedParams = resolveEffectParamsAtTime({
effect,
animations: this.params.animations,
localTime: animationLocalTime,
});
const definition = effectsRegistry.get(effect.type);
const passes = resolveEffectPasses({
definition,
effectParams: resolvedParams,
width,
height,
});
current = gpuRenderer.applyEffect({
source: current,
width: Math.round(width),
height: Math.round(height),
passes,
});
}
return current;
}
private applyMask({
mask,
elementCtx,
scaledWidth,
scaledHeight,
}: {
mask: Mask;
elementCtx: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;
scaledWidth: number;
scaledHeight: number;
}): void {
const definition = masksRegistry.get(mask.type);
const { feather, inverted } = mask.params;
const maskCanvas = createOffscreenCanvas({
width: Math.round(scaledWidth),
height: Math.round(scaledHeight),
});
const maskCtx = maskCanvas.getContext("2d") as
| CanvasRenderingContext2D
| OffscreenCanvasRenderingContext2D
| null;
if (!maskCtx) return;
maskCtx.clearRect(0, 0, scaledWidth, scaledHeight);
let maskResult: CanvasImageSource = maskCanvas;
let path: Path2D | null = null;
if (feather > 0 && definition.renderer.renderMask) {
// Bypasses JFA — avoids the two-sided distance artifact where strips
// near the canvas edge appear semi-transparent.
definition.renderer.renderMask({
resolvedParams: mask.params,
ctx: maskCtx,
width: Math.round(scaledWidth),
height: Math.round(scaledHeight),
feather,
});
} else {
path = definition.renderer.buildPath({
resolvedParams: mask.params,
width: scaledWidth,
height: scaledHeight,
});
maskCtx.fillStyle = "white";
maskCtx.fill(path);
if (feather > 0) {
maskResult = applyMaskFeather({
maskCanvas,
width: Math.round(scaledWidth),
height: Math.round(scaledHeight),
feather,
});
}
}
elementCtx.globalCompositeOperation = inverted
? "destination-out"
: "destination-in";
elementCtx.drawImage(maskResult, 0, 0, scaledWidth, scaledHeight);
elementCtx.globalCompositeOperation = "source-over";
const strokePath =
definition.renderer.buildStrokePath?.({
resolvedParams: mask.params,
width: scaledWidth,
height: scaledHeight,
}) ?? path;
if (mask.params.strokeWidth > 0 && strokePath) {
elementCtx.strokeStyle = mask.params.strokeColor;
elementCtx.lineWidth = mask.params.strokeWidth;
elementCtx.stroke(strokePath);
}
}
}
Resolved extends ResolvedVisualNodeState = ResolvedVisualNodeState,
> extends BaseNode<Params, Resolved> {}

468
apps/web/src/services/renderer/resolve.ts Normal file
View File

@ -0,0 +1,468 @@
import { mediaTimeToSeconds } from "opencut-wasm";
import {
getElementLocalTime,
resolveColorAtTime,
resolveGraphicParamsAtTime,
resolveOpacityAtTime,
resolveTransformAtTime,
} from "@/lib/animation";
import { resolveEffectParamsAtTime } from "@/lib/animation/effect-param-channel";
import {
buildGaussianBlurPasses,
intensityToSigma,
} from "@/lib/effects/definitions/blur";
import { effectsRegistry, resolveEffectPasses } from "@/lib/effects";
import type { Effect, EffectPass } from "@/lib/effects/types";
import { getSourceTimeAtClipTime } from "@/lib/retime";
import { DEFAULT_GRAPHIC_SOURCE_SIZE } from "@/lib/graphics";
import {
getTextMeasurementContext,
measureTextElement,
} from "@/lib/text/measure-element";
import { videoCache } from "@/services/video-cache/service";
import type { CanvasRenderer } from "./canvas-renderer";
import type { AnyBaseNode } from "./nodes/base-node";
import {
BlurBackgroundNode,
type BackdropSource,
type ResolvedBlurBackgroundNodeState,
} from "./nodes/blur-background-node";
import { EffectLayerNode, type ResolvedEffectLayerNodeState } from "./nodes/effect-layer-node";
import {
GraphicNode,
type ResolvedGraphicNodeState,
} from "./nodes/graphic-node";
import { ImageNode, loadImageSource } from "./nodes/image-node";
import { StickerNode, loadStickerSource } from "./nodes/sticker-node";
import { TextNode, type ResolvedTextNodeState } from "./nodes/text-node";
import { VideoNode } from "./nodes/video-node";
import type {
ResolvedVisualNodeState,
ResolvedVisualSourceNodeState,
VisualNodeParams,
} from "./nodes/visual-node";
type ResolveContext = {
renderer: CanvasRenderer;
time: number;
};
export async function resolveRenderTree({
node,
renderer,
time,
}: {
node: AnyBaseNode;
renderer: CanvasRenderer;
time: number;
}): Promise<void> {
await resolveNode({
node,
context: {
renderer,
time,
},
});
}
async function resolveNode({
node,
context,
}: {
node: AnyBaseNode;
context: ResolveContext;
}): Promise<void> {
if (node instanceof VideoNode) {
node.resolved = await resolveVideoNode({ node, context });
} else if (node instanceof ImageNode) {
node.resolved = await resolveImageNode({ node, context });
} else if (node instanceof StickerNode) {
node.resolved = await resolveStickerNode({ node, context });
} else if (node instanceof GraphicNode) {
node.resolved = resolveGraphicNode({ node, context });
} else if (node instanceof TextNode) {
node.resolved = resolveTextNode({ node, context });
} else if (node instanceof BlurBackgroundNode) {
node.resolved = await resolveBlurBackgroundNode({ node, context });
} else if (node instanceof EffectLayerNode) {
node.resolved = resolveEffectLayerNode({ node, context });
}
await Promise.all(
node.children.map((child) => resolveNode({ node: child, context })),
);
}
function resolveEffectPassGroups({
effects,
animations,
localTime,
width,
height,
}: {
effects: Effect[] | undefined;
animations: VisualNodeParams["animations"];
localTime: number;
width: number;
height: number;
}): EffectPass[][] {
return (effects ?? [])
.filter((effect) => effect.enabled)
.map((effect) => {
const resolvedParams = resolveEffectParamsAtTime({
effect,
animations,
localTime,
});
const definition = effectsRegistry.get(effect.type);
return resolveEffectPasses({
definition,
effectParams: resolvedParams,
width,
height,
});
});
}
function resolveVisualState({
params,
context,
sourceWidth,
sourceHeight,
}: {
params: VisualNodeParams;
context: ResolveContext;
sourceWidth: number;
sourceHeight: number;
}): ResolvedVisualNodeState | null {
const clipTime = context.time - params.timeOffset;
if (clipTime < 0 || clipTime >= params.duration) {
return null;
}
const localTime = getElementLocalTime({
timelineTime: context.time,
elementStartTime: params.timeOffset,
elementDuration: params.duration,
});
const transform = resolveTransformAtTime({
baseTransform: params.transform,
animations: params.animations,
localTime,
});
const opacity = resolveOpacityAtTime({
baseOpacity: params.opacity,
animations: params.animations,
localTime,
});
const containScale = Math.min(
context.renderer.width / sourceWidth,
context.renderer.height / sourceHeight,
);
const effectWidth = Math.round(
Math.abs(sourceWidth * containScale * transform.scaleX),
);
const effectHeight = Math.round(
Math.abs(sourceHeight * containScale * transform.scaleY),
);
return {
localTime,
transform,
opacity,
effectPasses: resolveEffectPassGroups({
effects: params.effects,
animations: params.animations,
localTime,
width: effectWidth,
height: effectHeight,
}),
};
}
async function resolveVideoNode({
node,
context,
}: {
node: VideoNode;
context: ResolveContext;
}): Promise<ResolvedVisualSourceNodeState | null> {
const clipTime = context.time - node.params.timeOffset;
if (clipTime < 0 || clipTime >= node.params.duration) {
return null;
}
const sourceTimeTicks =
node.params.trimStart +
getSourceTimeAtClipTime({
clipTime,
retime: node.params.retime,
});
const frame = await videoCache.getFrameAt({
mediaId: node.params.mediaId,
file: node.params.file,
time: mediaTimeToSeconds({ time: sourceTimeTicks }),
});
if (!frame) {
return null;
}
const visualState = resolveVisualState({
params: node.params,
context,
sourceWidth: frame.canvas.width,
sourceHeight: frame.canvas.height,
});
if (!visualState) {
return null;
}
return {
...visualState,
source: frame.canvas,
sourceWidth: frame.canvas.width,
sourceHeight: frame.canvas.height,
};
}
async function resolveImageNode({
node,
context,
}: {
node: ImageNode;
context: ResolveContext;
}): Promise<ResolvedVisualSourceNodeState | null> {
const source = await loadImageSource(node.params.url, node.params.maxSourceSize);
const visualState = resolveVisualState({
params: node.params,
context,
sourceWidth: source.width,
sourceHeight: source.height,
});
if (!visualState) {
return null;
}
return {
...visualState,
source: source.source,
sourceWidth: source.width,
sourceHeight: source.height,
};
}
async function resolveStickerNode({
node,
context,
}: {
node: StickerNode;
context: ResolveContext;
}): Promise<ResolvedVisualSourceNodeState | null> {
const source = await loadStickerSource({ stickerId: node.params.stickerId });
const sourceWidth = node.params.intrinsicWidth ?? source.width;
const sourceHeight = node.params.intrinsicHeight ?? source.height;
const visualState = resolveVisualState({
params: node.params,
context,
sourceWidth,
sourceHeight,
});
if (!visualState) {
return null;
}
return {
...visualState,
source: source.source,
sourceWidth,
sourceHeight,
};
}
function resolveGraphicNode({
node,
context,
}: {
node: GraphicNode;
context: ResolveContext;
}): ResolvedGraphicNodeState | null {
const visualState = resolveVisualState({
params: node.params,
context,
sourceWidth: DEFAULT_GRAPHIC_SOURCE_SIZE,
sourceHeight: DEFAULT_GRAPHIC_SOURCE_SIZE,
});
if (!visualState) {
return null;
}
return {
...visualState,
resolvedParams: resolveGraphicParamsAtTime({
element: node.params,
localTime: visualState.localTime,
}),
};
}
function resolveTextNode({
node,
context,
}: {
node: TextNode;
context: ResolveContext;
}): ResolvedTextNodeState | null {
if (
context.time < node.params.startTime ||
context.time >= node.params.startTime + node.params.duration
) {
return null;
}
const localTime = getElementLocalTime({
timelineTime: context.time,
elementStartTime: node.params.startTime,
elementDuration: node.params.duration,
});
return {
transform: resolveTransformAtTime({
baseTransform: node.params.transform,
animations: node.params.animations,
localTime,
}),
opacity: resolveOpacityAtTime({
baseOpacity: node.params.opacity,
animations: node.params.animations,
localTime,
}),
textColor: resolveColorAtTime({
baseColor: node.params.color,
animations: node.params.animations,
propertyPath: "color",
localTime,
}),
backgroundColor: resolveColorAtTime({
baseColor: node.params.background.color,
animations: node.params.animations,
propertyPath: "background.color",
localTime,
}),
effectPasses: resolveEffectPassGroups({
effects: node.params.effects,
animations: node.params.animations,
localTime,
width: context.renderer.width,
height: context.renderer.height,
}),
measuredText: measureTextElement({
element: node.params,
canvasHeight: node.params.canvasHeight,
localTime,
ctx: getTextMeasurementContext(),
}),
};
}
async function resolveBlurBackgroundNode({
node,
context,
}: {
node: BlurBackgroundNode;
context: ResolveContext;
}): Promise<ResolvedBlurBackgroundNodeState | null> {
const clipTime = context.time - node.params.timeOffset;
if (clipTime < 0 || clipTime >= node.params.duration) {
return null;
}
const backdropSource = await resolveBackdropSource({ node, clipTime });
if (!backdropSource) {
return null;
}
return {
backdropSource,
passes: buildGaussianBlurPasses({
sigmaX: intensityToSigma({
intensity: node.params.blurIntensity,
resolution: context.renderer.width,
reference: 1920,
}),
sigmaY: intensityToSigma({
intensity: node.params.blurIntensity,
resolution: context.renderer.height,
reference: 1080,
}),
}),
};
}
async function resolveBackdropSource({
node,
clipTime,
}: {
node: BlurBackgroundNode;
clipTime: number;
}): Promise<BackdropSource | null> {
if (node.params.mediaType === "video") {
const sourceTimeTicks =
node.params.trimStart +
getSourceTimeAtClipTime({
clipTime,
retime: node.params.retime,
});
const frame = await videoCache.getFrameAt({
mediaId: node.params.mediaId,
file: node.params.file,
time: mediaTimeToSeconds({ time: sourceTimeTicks }),
});
if (!frame) {
return null;
}
return {
source: frame.canvas,
width: frame.canvas.width,
height: frame.canvas.height,
};
}
const source = await loadImageSource(node.params.url);
return {
source: source.source,
width: source.width,
height: source.height,
};
}
function resolveEffectLayerNode({
node,
context,
}: {
node: EffectLayerNode;
context: ResolveContext;
}): ResolvedEffectLayerNodeState | null {
const time = context.time;
if (
time < node.params.timeOffset - 1e-6 ||
time >= node.params.timeOffset + node.params.duration + 1e-6
) {
return null;
}
const definition = effectsRegistry.get(node.params.effectType);
const passes = resolveEffectPasses({
definition,
effectParams: node.params.effectParams,
width: context.renderer.width,
height: context.renderer.height,
});
if (passes.length === 0) {
return null;
}
return {
passes,
};
}

530
apps/web/src/services/renderer/scene-builder.ts
View File

@ -1,265 +1,265 @@
import type { SceneTracks, TimelineTrack } from "@/lib/timeline";
import type { MediaAsset } from "@/lib/media/types";
import { RootNode } from "./nodes/root-node";
import { VideoNode } from "./nodes/video-node";
import { ImageNode } from "./nodes/image-node";
import { TextNode } from "./nodes/text-node";
import { StickerNode } from "./nodes/sticker-node";
import { GraphicNode } from "./nodes/graphic-node";
import { ColorNode } from "./nodes/color-node";
import { BlurBackgroundNode } from "./nodes/blur-background-node";
import { EffectLayerNode } from "./nodes/effect-layer-node";
import type { BaseNode } from "./nodes/base-node";
import type { TBackground, TCanvasSize } from "@/lib/project/types";
import { DEFAULT_BACKGROUND_BLUR_INTENSITY } from "@/lib/background/constants";
const PREVIEW_MAX_IMAGE_SIZE = 2048;
function getVisibleSortedElements({ track }: { track: TimelineTrack }) {
return track.elements
.filter((element) => !("hidden" in element && element.hidden))
.slice()
.sort((a, b) => {
if (a.startTime !== b.startTime) return a.startTime - b.startTime;
return a.id.localeCompare(b.id);
});
}
function buildTrackNodes({
tracks,
mediaMap,
canvasSize,
isPreview,
}: {
tracks: TimelineTrack[];
mediaMap: Map<string, MediaAsset>;
canvasSize: TCanvasSize;
isPreview?: boolean;
}): BaseNode[] {
const nodes: BaseNode[] = [];
for (const track of tracks) {
const elements = getVisibleSortedElements({ track });
for (const element of elements) {
if (element.type === "effect") {
nodes.push(
new EffectLayerNode({
effectType: element.effectType,
effectParams: element.params,
timeOffset: element.startTime,
duration: element.duration,
}),
);
continue;
}
if (element.type === "video" || element.type === "image") {
const mediaAsset = mediaMap.get(element.mediaId);
if (!mediaAsset?.file || !mediaAsset?.url) {
continue;
}
if (element.type === "video" && mediaAsset.type === "video") {
nodes.push(
new VideoNode({
mediaId: mediaAsset.id,
url: mediaAsset.url,
file: mediaAsset.file,
duration: element.duration,
timeOffset: element.startTime,
trimStart: element.trimStart,
trimEnd: element.trimEnd,
retime: element.retime,
transform: element.transform,
animations: element.animations,
opacity: element.opacity,
blendMode: element.blendMode,
effects: element.effects,
masks: element.masks,
}),
);
}
if (element.type === "image" && mediaAsset.type === "image") {
nodes.push(
new ImageNode({
url: mediaAsset.url,
duration: element.duration,
timeOffset: element.startTime,
trimStart: element.trimStart,
trimEnd: element.trimEnd,
transform: element.transform,
animations: element.animations,
opacity: element.opacity,
blendMode: element.blendMode,
effects: element.effects,
masks: element.masks,
...(isPreview && {
maxSourceSize: PREVIEW_MAX_IMAGE_SIZE,
}),
}),
);
}
}
if (element.type === "text") {
nodes.push(
new TextNode({
...element,
canvasCenter: { x: canvasSize.width / 2, y: canvasSize.height / 2 },
canvasHeight: canvasSize.height,
textBaseline: "middle",
effects: element.effects,
}),
);
}
if (element.type === "sticker") {
nodes.push(
new StickerNode({
stickerId: element.stickerId,
intrinsicWidth: element.intrinsicWidth,
intrinsicHeight: element.intrinsicHeight,
duration: element.duration,
timeOffset: element.startTime,
trimStart: element.trimStart,
trimEnd: element.trimEnd,
transform: element.transform,
animations: element.animations,
opacity: element.opacity,
blendMode: element.blendMode,
effects: element.effects,
}),
);
}
if (element.type === "graphic") {
nodes.push(
new GraphicNode({
definitionId: element.definitionId,
params: element.params,
duration: element.duration,
timeOffset: element.startTime,
trimStart: element.trimStart,
trimEnd: element.trimEnd,
transform: element.transform,
animations: element.animations,
opacity: element.opacity,
blendMode: element.blendMode,
effects: element.effects,
masks: element.masks,
}),
);
}
}
}
return nodes;
}
function buildBlurBackgroundNodes({
track,
mediaMap,
blurIntensity,
}: {
track: TimelineTrack | undefined;
mediaMap: Map<string, MediaAsset>;
blurIntensity: number;
}): BaseNode[] {
if (!track) {
return [];
}
const nodes: BaseNode[] = [];
const elements = getVisibleSortedElements({ track });
for (const element of elements) {
if (element.type !== "video" && element.type !== "image") {
continue;
}
const mediaAsset = mediaMap.get(element.mediaId);
if (
!mediaAsset?.file ||
!mediaAsset?.url ||
(mediaAsset.type !== "video" && mediaAsset.type !== "image")
) {
continue;
}
nodes.push(
new BlurBackgroundNode({
mediaId: mediaAsset.id,
url: mediaAsset.url,
file: mediaAsset.file,
mediaType: mediaAsset.type,
duration: element.duration,
timeOffset: element.startTime,
trimStart: element.trimStart,
trimEnd: element.trimEnd,
retime: element.type === "video" ? element.retime : undefined,
blurIntensity,
}),
);
}
return nodes;
}
export type BuildSceneParams = {
canvasSize: TCanvasSize;
tracks: SceneTracks;
mediaAssets: MediaAsset[];
duration: number;
background: TBackground;
isPreview?: boolean;
};
export function buildScene({
canvasSize,
tracks,
mediaAssets,
duration,
background,
isPreview,
}: BuildSceneParams) {
const rootNode = new RootNode({ duration });
const mediaMap = new Map(mediaAssets.map((m) => [m.id, m]));
const visibleTracks = [
...tracks.overlay.filter((track) => !("hidden" in track && track.hidden)),
...(!tracks.main.hidden ? [tracks.main] : []),
];
const orderedTracksBottomToTop = visibleTracks.slice().reverse();
const mainTrack = tracks.main.hidden ? undefined : tracks.main;
const allNodes = buildTrackNodes({
tracks: orderedTracksBottomToTop,
mediaMap,
canvasSize,
isPreview,
});
if (background.type === "blur") {
const blurNodes = buildBlurBackgroundNodes({
track: mainTrack,
mediaMap,
blurIntensity:
background.blurIntensity ?? DEFAULT_BACKGROUND_BLUR_INTENSITY,
});
for (const node of blurNodes) {
rootNode.add(node);
}
} else if (
background.type === "color" &&
background.color !== "transparent"
) {
rootNode.add(new ColorNode({ color: background.color }));
}
for (const node of allNodes) {
rootNode.add(node);
}
return rootNode;
}
import type { SceneTracks, TimelineTrack } from "@/lib/timeline";
import type { MediaAsset } from "@/lib/media/types";
import { RootNode } from "./nodes/root-node";
import { VideoNode } from "./nodes/video-node";
import { ImageNode } from "./nodes/image-node";
import { TextNode } from "./nodes/text-node";
import { StickerNode } from "./nodes/sticker-node";
import { GraphicNode } from "./nodes/graphic-node";
import { ColorNode } from "./nodes/color-node";
import { BlurBackgroundNode } from "./nodes/blur-background-node";
import { EffectLayerNode } from "./nodes/effect-layer-node";
import type { AnyBaseNode } from "./nodes/base-node";
import type { TBackground, TCanvasSize } from "@/lib/project/types";
import { DEFAULT_BACKGROUND_BLUR_INTENSITY } from "@/lib/background/constants";
const PREVIEW_MAX_IMAGE_SIZE = 2048;
function getVisibleSortedElements({ track }: { track: TimelineTrack }) {
return track.elements
.filter((element) => !("hidden" in element && element.hidden))
.slice()
.sort((a, b) => {
if (a.startTime !== b.startTime) return a.startTime - b.startTime;
return a.id.localeCompare(b.id);
});
}
function buildTrackNodes({
tracks,
mediaMap,
canvasSize,
isPreview,
}: {
tracks: TimelineTrack[];
mediaMap: Map<string, MediaAsset>;
canvasSize: TCanvasSize;
isPreview?: boolean;
}): AnyBaseNode[] {
const nodes: AnyBaseNode[] = [];
for (const track of tracks) {
const elements = getVisibleSortedElements({ track });
for (const element of elements) {
if (element.type === "effect") {
nodes.push(
new EffectLayerNode({
effectType: element.effectType,
effectParams: element.params,
timeOffset: element.startTime,
duration: element.duration,
}),
);
continue;
}
if (element.type === "video" || element.type === "image") {
const mediaAsset = mediaMap.get(element.mediaId);
if (!mediaAsset?.file || !mediaAsset?.url) {
continue;
}
if (element.type === "video" && mediaAsset.type === "video") {
nodes.push(
new VideoNode({
mediaId: mediaAsset.id,
url: mediaAsset.url,
file: mediaAsset.file,
duration: element.duration,
timeOffset: element.startTime,
trimStart: element.trimStart,
trimEnd: element.trimEnd,
retime: element.retime,
transform: element.transform,
animations: element.animations,
opacity: element.opacity,
blendMode: element.blendMode,
effects: element.effects ?? [],
masks: element.masks ?? [],
}),
);
}
if (element.type === "image" && mediaAsset.type === "image") {
nodes.push(
new ImageNode({
url: mediaAsset.url,
duration: element.duration,
timeOffset: element.startTime,
trimStart: element.trimStart,
trimEnd: element.trimEnd,
transform: element.transform,
animations: element.animations,
opacity: element.opacity,
blendMode: element.blendMode,
effects: element.effects ?? [],
masks: element.masks ?? [],
...(isPreview && {
maxSourceSize: PREVIEW_MAX_IMAGE_SIZE,
}),
}),
);
}
}
if (element.type === "text") {
nodes.push(
new TextNode({
...element,
canvasCenter: { x: canvasSize.width / 2, y: canvasSize.height / 2 },
canvasHeight: canvasSize.height,
textBaseline: "middle",
effects: element.effects ?? [],
}),
);
}
if (element.type === "sticker") {
nodes.push(
new StickerNode({
stickerId: element.stickerId,
intrinsicWidth: element.intrinsicWidth,
intrinsicHeight: element.intrinsicHeight,
duration: element.duration,
timeOffset: element.startTime,
trimStart: element.trimStart,
trimEnd: element.trimEnd,
transform: element.transform,
animations: element.animations,
opacity: element.opacity,
blendMode: element.blendMode,
effects: element.effects ?? [],
}),
);
}
if (element.type === "graphic") {
nodes.push(
new GraphicNode({
definitionId: element.definitionId,
params: element.params,
duration: element.duration,
timeOffset: element.startTime,
trimStart: element.trimStart,
trimEnd: element.trimEnd,
transform: element.transform,
animations: element.animations,
opacity: element.opacity,
blendMode: element.blendMode,
effects: element.effects ?? [],
masks: element.masks ?? [],
}),
);
}
}
}
return nodes;
}
function buildBlurBackgroundNodes({
track,
mediaMap,
blurIntensity,
}: {
track: TimelineTrack | undefined;
mediaMap: Map<string, MediaAsset>;
blurIntensity: number;
}): AnyBaseNode[] {
if (!track) {
return [];
}
const nodes: AnyBaseNode[] = [];
const elements = getVisibleSortedElements({ track });
for (const element of elements) {
if (element.type !== "video" && element.type !== "image") {
continue;
}
const mediaAsset = mediaMap.get(element.mediaId);
if (
!mediaAsset?.file ||
!mediaAsset?.url ||
(mediaAsset.type !== "video" && mediaAsset.type !== "image")
) {
continue;
}
nodes.push(
new BlurBackgroundNode({
mediaId: mediaAsset.id,
url: mediaAsset.url,
file: mediaAsset.file,
mediaType: mediaAsset.type,
duration: element.duration,
timeOffset: element.startTime,
trimStart: element.trimStart,
trimEnd: element.trimEnd,
retime: element.type === "video" ? element.retime : undefined,
blurIntensity,
}),
);
}
return nodes;
}
export type BuildSceneParams = {
canvasSize: TCanvasSize;
tracks: SceneTracks;
mediaAssets: MediaAsset[];
duration: number;
background: TBackground;
isPreview?: boolean;
};
export function buildScene({
canvasSize,
tracks,
mediaAssets,
duration,
background,
isPreview,
}: BuildSceneParams) {
const rootNode = new RootNode({ duration });
const mediaMap = new Map(mediaAssets.map((m) => [m.id, m]));
const visibleTracks = [
...tracks.overlay.filter((track) => !("hidden" in track && track.hidden)),
...(!tracks.main.hidden ? [tracks.main] : []),
];
const orderedTracksBottomToTop = visibleTracks.slice().reverse();
const mainTrack = tracks.main.hidden ? undefined : tracks.main;
const allNodes = buildTrackNodes({
tracks: orderedTracksBottomToTop,
mediaMap,
canvasSize,
isPreview,
});
if (background.type === "blur") {
const blurNodes = buildBlurBackgroundNodes({
track: mainTrack,
mediaMap,
blurIntensity:
background.blurIntensity ?? DEFAULT_BACKGROUND_BLUR_INTENSITY,
});
for (const node of blurNodes) {
rootNode.add(node);
}
} else if (
background.type === "color" &&
background.color !== "transparent"
) {
rootNode.add(new ColorNode({ color: background.color }));
}
for (const node of allNodes) {
rootNode.add(node);
}
return rootNode;
}

2
apps/web/src/services/renderer/scene-exporter.ts
View File

@ -97,7 +97,7 @@ export class SceneExporter extends EventEmitter<SceneExporterEvents> {
target: new BufferTarget(),
});
const videoSource = new CanvasSource(this.renderer.canvas, {
const videoSource = new CanvasSource(this.renderer.getOutputCanvas(), {
codec: this.format === "webm" ? "vp9" : "avc",
bitrate: qualityMap[this.quality],
});