import { drawCssBackground } from "@/gradients"; import { getMaskDefinition } from "@/masks"; import { incrementCounter } from "@/diagnostics/render-perf"; import type { AnyBaseNode } from "../nodes/base-node"; import type { CanvasRenderer } from "../canvas-renderer"; import { createCanvasSurface } from "../canvas-utils"; import { BlurBackgroundNode } from "../nodes/blur-background-node"; import { ColorNode } from "../nodes/color-node"; import { EffectLayerNode } from "../nodes/effect-layer-node"; import { GraphicNode, type ResolvedGraphicNodeState, } from "../nodes/graphic-node"; import { ImageNode } from "../nodes/image-node"; import { RootNode } from "../nodes/root-node"; import { StickerNode } from "../nodes/sticker-node"; import { renderTextToContext, TextNode } from "../nodes/text-node"; import { VideoNode } from "../nodes/video-node"; import type { ResolvedVisualSourceNodeState } from "../nodes/visual-node"; import type { FrameDescriptor, FrameItemDescriptor, LayerMaskDescriptor, QuadTransformDescriptor, TextureCanvasDrawFn, TextureUploadDescriptor, } from "./types"; import { DEFAULT_GRAPHIC_SOURCE_SIZE } from "@/graphics"; export async function buildFrameDescriptor({ node, renderer, }: { node: AnyBaseNode; renderer: CanvasRenderer; }): Promise<{ frame: FrameDescriptor; textures: TextureUploadDescriptor[]; }> { const items: FrameItemDescriptor[] = []; const textures = new Map(); await collectNode({ node, renderer, path: "root", items, textures, }); incrementCounter({ name: "frameItems", by: items.length }); incrementCounter({ name: "frameTextures", by: textures.size }); return { frame: { width: renderer.width, height: renderer.height, clear: { color: [0, 0, 0, 1], }, items, }, textures: [...textures.values()], }; } async function collectNode({ node, renderer, path, items, textures, }: { node: AnyBaseNode; renderer: CanvasRenderer; path: string; items: FrameItemDescriptor[]; textures: Map; }): Promise { if (node instanceof RootNode) { for (let index = 0; index < node.children.length; index++) { await collectNode({ node: node.children[index], renderer, path: `${path}:${index}`, items, textures, }); } return; } if (node instanceof ColorNode) { const textureId = `${path}:color`; const { width, height } = renderer; textures.set(textureId, { kind: "rendered", id: textureId, contentHash: `color:${node.params.color}:${width}x${height}`, width, height, draw: (ctx) => { if (/gradient\(/i.test(node.params.color)) { drawCssBackground({ ctx, width, height, css: node.params.color }); } else { ctx.fillStyle = node.params.color; ctx.fillRect(0, 0, width, height); } }, }); items.push({ type: "layer", textureId, transform: fullCanvasTransform(renderer), opacity: 1, blendMode: "normal", effectPassGroups: [], mask: null, }); return; } if (node instanceof EffectLayerNode) { if (!node.resolved || node.resolved.passes.length === 0) { return; } items.push({ type: "sceneEffect", effectPassGroups: [node.resolved.passes], }); return; } if (node instanceof BlurBackgroundNode) { if (!node.resolved) { return; } const textureId = `${path}:blur-background`; const { width, height } = renderer; const { backdropSource, passes } = node.resolved; // Backdrop pixels come from a decoded video/image frame whose identity // already changes when it changes. Hashing the source reference is // enough to let us skip redraws on frozen frames. const contentHash = `blur:${identityKey(backdropSource.source)}:${backdropSource.width}x${backdropSource.height}:${width}x${height}`; textures.set(textureId, { kind: "rendered", id: textureId, contentHash, width, height, draw: (ctx) => { const coverScale = Math.max( width / backdropSource.width, height / backdropSource.height, ); const scaledWidth = backdropSource.width * coverScale; const scaledHeight = backdropSource.height * coverScale; const offsetX = (width - scaledWidth) / 2; const offsetY = (height - scaledHeight) / 2; ctx.drawImage( backdropSource.source, offsetX, offsetY, scaledWidth, scaledHeight, ); }, }); items.push({ type: "layer", textureId, transform: fullCanvasTransform(renderer), opacity: 1, blendMode: "normal", effectPassGroups: [passes], mask: null, }); return; } if ( node instanceof VideoNode || node instanceof ImageNode || node instanceof StickerNode || node instanceof GraphicNode ) { await collectVisualSourceNode({ node, renderer, path, items, textures, }); return; } if (node instanceof TextNode) { collectTextNode({ node, renderer, path, items, textures, }); } } async function collectVisualSourceNode({ node, renderer, path, items, textures, }: { node: VideoNode | ImageNode | StickerNode | GraphicNode; renderer: CanvasRenderer; path: string; items: FrameItemDescriptor[]; textures: Map; }) { if (!node.resolved) { return; } const source = node instanceof GraphicNode ? node.getSource({ resolvedParams: node.resolved.resolvedParams }) : node.resolved.source; if (!source) { return; } const sourceWidth = node instanceof GraphicNode ? DEFAULT_GRAPHIC_SOURCE_SIZE : (node.resolved as ResolvedVisualSourceNodeState).sourceWidth; const sourceHeight = node instanceof GraphicNode ? DEFAULT_GRAPHIC_SOURCE_SIZE : (node.resolved as ResolvedVisualSourceNodeState).sourceHeight; const textureId = `${path}:source`; textures.set(textureId, { kind: "external", id: textureId, source, width: sourceWidth, height: sourceHeight, }); const transform = computeVisualTransform({ renderer, resolved: node.resolved, sourceWidth, sourceHeight, }); const { mask, strokeLayer } = buildMaskArtifacts({ node, renderer, path, transform, textures, }); items.push({ type: "layer", textureId, transform, opacity: node.resolved.opacity, blendMode: node.params.blendMode ?? "normal", effectPassGroups: node.resolved.effectPasses, mask, }); if (strokeLayer) { items.push(strokeLayer); } } function collectTextNode({ node, renderer, path, items, textures, }: { node: TextNode; renderer: CanvasRenderer; path: string; items: FrameItemDescriptor[]; textures: Map; }) { if (!node.resolved) { return; } const textureId = `${path}:text`; const { width, height } = renderer; // Text output is fully determined by node.params + node.resolved. Both are // plain data we can stringify cheaply; the resolved measured layout is the // expensive part of text setup, so stringifying it here is orders of // magnitude cheaper than re-rasterizing when nothing changed. const contentHash = `text:${width}x${height}:${JSON.stringify({ params: node.params, resolved: node.resolved, })}`; textures.set(textureId, { kind: "rendered", id: textureId, contentHash, width, height, draw: (ctx) => { renderTextToContext({ node, ctx }); }, }); items.push({ type: "layer", textureId, transform: fullCanvasTransform(renderer), opacity: node.resolved.opacity, blendMode: node.params.blendMode ?? "normal", effectPassGroups: node.resolved.effectPasses, mask: null, }); } function computeVisualTransform({ renderer, resolved, sourceWidth, sourceHeight, }: { renderer: CanvasRenderer; resolved: ResolvedVisualSourceNodeState | ResolvedGraphicNodeState; sourceWidth: number; sourceHeight: number; }): QuadTransformDescriptor { const containScale = Math.min( renderer.width / sourceWidth, renderer.height / sourceHeight, ); const scaledWidth = sourceWidth * containScale * resolved.transform.scaleX; const scaledHeight = sourceHeight * containScale * resolved.transform.scaleY; const absWidth = Math.abs(scaledWidth); const absHeight = Math.abs(scaledHeight); return { centerX: renderer.width / 2 + resolved.transform.position.x, centerY: renderer.height / 2 + resolved.transform.position.y, width: absWidth, height: absHeight, rotationDegrees: resolved.transform.rotate, flipX: scaledWidth < 0, flipY: scaledHeight < 0, }; } function fullCanvasTransform( renderer: CanvasRenderer, ): QuadTransformDescriptor { return { centerX: renderer.width / 2, centerY: renderer.height / 2, width: renderer.width, height: renderer.height, rotationDegrees: 0, flipX: false, flipY: false, }; } function buildMaskArtifacts({ node, renderer, path, transform, textures, }: { node: VideoNode | ImageNode | StickerNode | GraphicNode; renderer: CanvasRenderer; path: string; transform: QuadTransformDescriptor; textures: Map; }): { mask: LayerMaskDescriptor | null; strokeLayer: FrameItemDescriptor | null; } { const mask = node.params.masks?.[0]; if (!mask) { return { mask: null, strokeLayer: null }; } const definition = getMaskDefinition(mask.type); if (definition.isActive?.(mask.params) === false) { return { mask: null, strokeLayer: null }; } const { body } = definition.renderer; const usesOpaqueFastPath = body.kind === "drawWithFeather" && mask.params.feather === 0 && Boolean(body.opaqueFastPath); // drawWithFeather renderers encode feathering analytically in their canvas output // (e.g. split mask uses a linear gradient instead of JFA). The descriptor feather is // zeroed so the GPU compositor copies the mask texture as-is and does not run a second // JFA feather pass on top of an already-soft texture. const feather = body.kind === "drawWithFeather" ? 0 : mask.params.feather; const maskTextureId = `${path}:mask`; const { width: canvasWidth, height: canvasHeight } = renderer; const maskContentHash = `mask:${mask.type}:${JSON.stringify(mask.params)}:${transformHash(transform)}:${canvasWidth}x${canvasHeight}:body=${body.kind}:fastPath=${usesOpaqueFastPath}`; const drawMask: TextureCanvasDrawFn = (ctx) => { const { canvas: elementMaskCanvas, context: elementMaskCtx } = createCanvasSurface({ width: Math.round(transform.width), height: Math.round(transform.height), }); switch (body.kind) { case "fillPath": { const path2d = body.buildPath({ resolvedParams: mask.params, width: transform.width, height: transform.height, }); elementMaskCtx.fillStyle = "white"; elementMaskCtx.fill(path2d); break; } case "drawOpaque": body.drawOpaque({ resolvedParams: mask.params, ctx: elementMaskCtx, width: Math.round(transform.width), height: Math.round(transform.height), }); break; case "drawWithFeather": if (usesOpaqueFastPath && body.opaqueFastPath) { const path2d = body.opaqueFastPath.buildPath({ resolvedParams: mask.params, width: transform.width, height: transform.height, }); elementMaskCtx.fillStyle = "white"; elementMaskCtx.fill(path2d); } else { body.drawWithFeather({ resolvedParams: mask.params, ctx: elementMaskCtx, width: Math.round(transform.width), height: Math.round(transform.height), feather: mask.params.feather, }); } break; } drawTransformedCanvas({ ctx, source: elementMaskCanvas, transform }); }; textures.set(maskTextureId, { kind: "rendered", id: maskTextureId, contentHash: maskContentHash, width: canvasWidth, height: canvasHeight, draw: drawMask, }); const stroke = definition.renderer.stroke; const hasStroke = mask.params.strokeWidth > 0 && Boolean(stroke); let strokeLayer: FrameItemDescriptor | null = null; if (hasStroke && stroke) { const strokeTextureId = `${path}:mask-stroke`; const strokeContentHash = `stroke:${mask.type}:${JSON.stringify(mask.params)}:${transformHash(transform)}:${canvasWidth}x${canvasHeight}:stroke=${stroke.kind}`; const drawStroke: TextureCanvasDrawFn = (ctx) => { const { canvas: strokeCanvas, context: strokeCtx } = createCanvasSurface({ width: Math.round(transform.width), height: Math.round(transform.height), }); switch (stroke.kind) { case "renderStroke": stroke.renderStroke({ resolvedParams: mask.params, ctx: strokeCtx, width: transform.width, height: transform.height, }); break; case "strokeFromPath": { const strokePath = stroke.buildStrokePath({ resolvedParams: mask.params, width: transform.width, height: transform.height, }); strokeCtx.strokeStyle = mask.params.strokeColor; strokeCtx.lineWidth = mask.params.strokeWidth; strokeCtx.stroke(strokePath); break; } } drawTransformedCanvas({ ctx, source: strokeCanvas, transform }); }; textures.set(strokeTextureId, { kind: "rendered", id: strokeTextureId, contentHash: strokeContentHash, width: canvasWidth, height: canvasHeight, draw: drawStroke, }); strokeLayer = { type: "layer", textureId: strokeTextureId, transform: fullCanvasTransform(renderer), opacity: 1, blendMode: "normal", effectPassGroups: [], mask: null, }; } return { mask: { textureId: maskTextureId, feather, inverted: mask.params.inverted, }, strokeLayer, }; } function drawTransformedCanvas({ ctx, source, transform, }: { ctx: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D; source: CanvasImageSource; transform: QuadTransformDescriptor; }) { const x = transform.centerX - transform.width / 2; const y = transform.centerY - transform.height / 2; const flipX = transform.flipX ? -1 : 1; const flipY = transform.flipY ? -1 : 1; const requiresTransform = transform.rotationDegrees !== 0 || flipX !== 1 || flipY !== 1; ctx.save(); if (requiresTransform) { ctx.translate(transform.centerX, transform.centerY); ctx.rotate((transform.rotationDegrees * Math.PI) / 180); ctx.scale(flipX, flipY); ctx.translate(-transform.centerX, -transform.centerY); } ctx.drawImage(source, x, y, transform.width, transform.height); ctx.restore(); } function transformHash(transform: QuadTransformDescriptor): string { return `${transform.centerX}:${transform.centerY}:${transform.width}:${transform.height}:${transform.rotationDegrees}:${transform.flipX ? 1 : 0}:${transform.flipY ? 1 : 0}`; } // Stable identity key for CanvasImageSource. Using a WeakMap → counter keeps // hash string length bounded and avoids holding sources alive. const identityKeys = new WeakMap(); let nextIdentity = 1; function identityKey(source: CanvasImageSource): string { if (typeof source === "object" && source !== null) { let key = identityKeys.get(source); if (key === undefined) { key = nextIdentity++; identityKeys.set(source, key); } return `@${key}`; } return "@?"; }