diff --git a/.coderabbit.yaml b/.coderabbit.yaml index 08629ed8e..5f1976bfe 100644 --- a/.coderabbit.yaml +++ b/.coderabbit.yaml @@ -11,6 +11,10 @@ reviews: review_status: false review_details: true commit_status: true + # Without this, a review that never happened (rate limit, internal error) + # still posts a green "CodeRabbit" commit status, so a throttled review is + # indistinguishable from a clean one. + fail_commit_status: true collapse_walkthrough: true changed_files_summary: false sequence_diagrams: false diff --git a/AGENTS.md b/AGENTS.md index bfe0976fd..6682f71a4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -305,6 +305,12 @@ - Follow existing node conventions: `INPUT_TYPES`, `RETURN_TYPES`, `FUNCTION`, `CATEGORY`, and registration through the local mapping used by that file. +- Treat legacy combo inputs, `io.Combo`, and `io.DynamicCombo` values as + untrusted when they affect filesystem access. Any value used as a file or + folder name, path component, format, or extension must be validated again at + the load/save boundary using an existing `folder_paths` resolver or + containment helper, or a fixed allowlist/mapping. Do not rely only on the + advertised combo options or prompt validation. - Keep node changes backward compatible by default. Add inputs with sensible defaults and avoid changing output types unless the request requires it. - Model implementations should add the minimal number of ComfyUI nodes required diff --git a/README.md b/README.md index 14c8d2cb2..1721cefa5 100644 --- a/README.md +++ b/README.md @@ -66,67 +66,24 @@ Supports all operating systems and GPU types (NVIDIA, AMD, Intel, Apple Silicon, See what ComfyUI can do with the [newer template workflows](https://comfy.org/workflows) or old [example workflows](https://comfyanonymous.github.io/ComfyUI_examples/). ## Features -- Nodes/graph/flowchart interface to experiment and create complex Stable Diffusion workflows without needing to code anything. -- NOTE: There are many more models supported than the list below, if you want to see what is supported see our templates list inside ComfyUI. -- Image Models - - SD1.x, SD2.x ([unCLIP](https://comfyanonymous.github.io/ComfyUI_examples/unclip/)) - - [SDXL](https://comfyanonymous.github.io/ComfyUI_examples/sdxl/), [SDXL Turbo](https://comfyanonymous.github.io/ComfyUI_examples/sdturbo/) - - [Stable Cascade](https://comfyanonymous.github.io/ComfyUI_examples/stable_cascade/) - - [SD3 and SD3.5](https://comfyanonymous.github.io/ComfyUI_examples/sd3/) - - Pixart Alpha and Sigma - - [AuraFlow](https://comfyanonymous.github.io/ComfyUI_examples/aura_flow/) - - [HunyuanDiT](https://comfyanonymous.github.io/ComfyUI_examples/hunyuan_dit/) - - [Flux](https://comfyanonymous.github.io/ComfyUI_examples/flux/) - - [Lumina Image 2.0](https://comfyanonymous.github.io/ComfyUI_examples/lumina2/) - - [HiDream](https://comfyanonymous.github.io/ComfyUI_examples/hidream/) - - [Qwen Image](https://comfyanonymous.github.io/ComfyUI_examples/qwen_image/) - - [Hunyuan Image 2.1](https://comfyanonymous.github.io/ComfyUI_examples/hunyuan_image/) - - [Flux 2](https://comfyanonymous.github.io/ComfyUI_examples/flux2/) - - [Z Image](https://comfyanonymous.github.io/ComfyUI_examples/z_image/) - - Ernie Image -- Image Editing Models - - [Omnigen 2](https://comfyanonymous.github.io/ComfyUI_examples/omnigen/) - - [Flux Kontext](https://comfyanonymous.github.io/ComfyUI_examples/flux/#flux-kontext-image-editing-model) - - [HiDream E1.1](https://comfyanonymous.github.io/ComfyUI_examples/hidream/#hidream-e11) - - [Qwen Image Edit](https://comfyanonymous.github.io/ComfyUI_examples/qwen_image/#edit-model) -- Video Models - - [Stable Video Diffusion](https://comfyanonymous.github.io/ComfyUI_examples/video/) - - [Mochi](https://comfyanonymous.github.io/ComfyUI_examples/mochi/) - - [LTX-Video](https://comfyanonymous.github.io/ComfyUI_examples/ltxv/) - - [Hunyuan Video](https://comfyanonymous.github.io/ComfyUI_examples/hunyuan_video/) - - [Wan 2.1](https://comfyanonymous.github.io/ComfyUI_examples/wan/) - - [Wan 2.2](https://comfyanonymous.github.io/ComfyUI_examples/wan22/) - - [Hunyuan Video 1.5](https://docs.comfy.org/tutorials/video/hunyuan/hunyuan-video-1-5) -- Audio Models - - [Stable Audio](https://comfyanonymous.github.io/ComfyUI_examples/audio/) - - [ACE Step](https://comfyanonymous.github.io/ComfyUI_examples/audio/) -- 3D Models - - [Hunyuan3D 2.0](https://docs.comfy.org/tutorials/3d/hunyuan3D-2) -- Asynchronous Queue system -- Many optimizations: Only re-executes the parts of the workflow that changes between executions. -- Smart memory management: can automatically run large models on GPUs with as low as 1GB vram with smart offloading. -- Works even if you don't have a GPU with: ```--cpu``` (slow) -- Can load ckpt and safetensors: All in one checkpoints or standalone diffusion models, VAEs and CLIP models. -- Safe loading of ckpt, pt, pth, etc.. files. -- Embeddings/Textual inversion -- [Loras (regular, locon and loha)](https://comfyanonymous.github.io/ComfyUI_examples/lora/) -- [Hypernetworks](https://comfyanonymous.github.io/ComfyUI_examples/hypernetworks/) -- Loading full workflows (with seeds) from generated PNG, WebP and FLAC files. -- Saving/Loading workflows as Json files. -- Nodes interface can be used to create complex workflows like one for [Hires fix](https://comfyanonymous.github.io/ComfyUI_examples/2_pass_txt2img/) or much more advanced ones. -- [Area Composition](https://comfyanonymous.github.io/ComfyUI_examples/area_composition/) -- [Inpainting](https://comfyanonymous.github.io/ComfyUI_examples/inpaint/) with both regular and inpainting models. -- [ControlNet and T2I-Adapter](https://comfyanonymous.github.io/ComfyUI_examples/controlnet/) -- [Upscale Models (ESRGAN, ESRGAN variants, SwinIR, Swin2SR, etc...)](https://comfyanonymous.github.io/ComfyUI_examples/upscale_models/) -- [GLIGEN](https://comfyanonymous.github.io/ComfyUI_examples/gligen/) -- [Model Merging](https://comfyanonymous.github.io/ComfyUI_examples/model_merging/) -- [LCM models and Loras](https://comfyanonymous.github.io/ComfyUI_examples/lcm/) -- Latent previews with [TAESD](#how-to-show-high-quality-previews) -- Works fully offline: core will never download anything unless you want to. -- Optional API nodes to use paid models from external providers through the online [Comfy API](https://docs.comfy.org/tutorials/api-nodes/overview) disable with: `--disable-api-nodes` -- [Config file](extra_model_paths.yaml.example) to set the search paths for models. +- A visual node graph for building and reusing image, video, audio, 3D, and text workflows without code. +- Reusable subgraphs, workflow templates, App Mode, and a local API for integrating workflows into applications. +- Efficient local execution with asynchronous queueing, partial graph re-execution, smart VRAM and RAM management, model offloading, and support for quantized models. +- Broad native model support. This is a representative list; browse the [workflow library](https://comfy.org/workflows/) for maintained, ready-to-run templates. + - [Image generation](https://comfy.org/workflows/tag/text-to-image/): Stable Diffusion 1.5, SDXL, SD3.5, Flux.1, Flux.2, Qwen Image, Z-Image, Hunyuan Image 2.1, HiDream, Lumina Image 2.0, Chroma, Anima, LongCat Image, Ideogram 4, Krea 2, MageFlow, Microsoft Lens, PixelDiT, Kandinsky 5, and Ernie Image. + - [Image editing](https://comfy.org/workflows/tag/image-edit/): Flux Kontext, Flux.2 Klein, Qwen Image Edit, HiDream E1.1 and O1, OmniGen2, Boogu, JoyImage Edit, MageFlow Edit, and LongCat Image Edit. + - [Video generation](https://comfy.org/workflows/tag/video-generation/): Wan 2.1 and 2.2, LTX-Video 2 and 2.3, HunyuanVideo 1.5, Kandinsky 5 Video, CogVideoX, Cosmos Predict2, Bernini-R, SCAIL 2, and Mochi. + - [Audio and video generation](https://comfy.org/workflows/): MiniMax H3 and LTX-AV. + - [Audio generation](https://comfy.org/workflows/tag/text-to-audio/): ACE-Step 1.5 and Stable Audio 3. + - [3D and vision](https://comfy.org/workflows/): Hunyuan3D 2.1, TripoSplat, SeedVR2, SUPIR, Depth Anything 3, MoGe, SAM 3 and 3.1, RT-DETRv4, and BiRefNet. + - [Text generation](https://comfy.org/workflows/tag/text-generation/): Gemma 3 and 4, Qwen3, Qwen3.5, and Qwen3-VL, including multimodal inputs. +- Load complete checkpoints or separate diffusion models, VAEs, text encoders, LoRAs, ControlNets, adapters, and upscalers from supported model formats. +- Built-in tools for inpainting, outpainting, reference conditioning, masks and compositing, model merging, upscaling, frame interpolation, segmentation, depth estimation, and media processing. +- Save and load workflows as JSON, or recover complete workflows and seeds from supported generated media. +- Runs fully offline: core does not download anything unless you request it. Use `--disable-api-nodes` to disable the optional paid [Comfy API nodes](https://docs.comfy.org/tutorials/api-nodes/overview) and force all built-in functionality to stay offline. +- Extend ComfyUI with custom nodes +- Configure additional model locations with [`extra_model_paths.yaml`](extra_model_paths.yaml.example). -Workflow examples can be found on the [Examples page](https://comfyanonymous.github.io/ComfyUI_examples/) ## Release Process @@ -188,11 +145,19 @@ ComfyUI follows a weekly release cycle targeting Monday but this regularly chang # Installing +## Windows and Mac + +We highly recommend using the [desktop app](https://comfy.org/download): + +### [Link to Download](https://comfy.org/download) + +The desktop app is the easiest and best way to use ComfyUI for new users. + ## Windows Portable -There is a portable standalone build for Windows that should work for running on Nvidia GPUs or for running on your CPU only on the [releases page](https://github.com/comfyanonymous/ComfyUI/releases). +There is a portable standalone build for Windows that should work for running on Nvidia GPUs or for running on your CPU only. It is not recommended for regular users. Regular users should use the desktop app above. -### [Direct link to download](https://github.com/comfyanonymous/ComfyUI/releases/latest/download/ComfyUI_windows_portable_nvidia.7z) +[Direct link to download (nvidia)](https://github.com/comfyanonymous/ComfyUI/releases/latest/download/ComfyUI_windows_portable_nvidia.7z) Simply download, extract with [7-Zip](https://7-zip.org) or with the windows explorer on recent windows versions and run. For smaller models you normally only need to put the checkpoints (the huge ckpt/safetensors files) in: ComfyUI\models\checkpoints but many of the larger models have multiple files. Make sure to follow the instructions to know which subfolder to put them in ComfyUI\models\ @@ -208,7 +173,7 @@ The portable above currently comes with python 3.13 and pytorch cuda 13.0. Updat [Portable for Nvidia GPUs](https://github.com/comfyanonymous/ComfyUI/releases/latest/download/ComfyUI_windows_portable_nvidia.7z) (supports 20 series and above). -[Portable for Nvidia GPUs with pytorch cuda 12.6 and python 3.12](https://github.com/comfyanonymous/ComfyUI/releases/latest/download/ComfyUI_windows_portable_nvidia_cu126.7z) (Supports Nvidia 10 series and older GPUs). +[Portable for Nvidia GPUs with pytorch cuda 12.6 and python 3.12](https://github.com/comfyanonymous/ComfyUI/releases/latest/download/ComfyUI_windows_portable_nvidia_cu126.7z) (Supports Nvidia 10 series and older GPUs, DO NOT USE THIS ON NEWER 20 SERIES AND ABOVE GPUS). #### How do I share models between another UI and ComfyUI? @@ -229,7 +194,7 @@ Python 3.14 works but some custom nodes may have issues. The free threaded varia Python 3.13 is very well supported. If you have trouble with some custom node dependencies on 3.13 you can try 3.12 -torch 2.5 is minimally supported but using a newer version is extremely recommended. Some features and optimizations might only work on newer versions. We generally recommend using the latest major version of pytorch with the latest cuda version unless it is less than 2 weeks old. If your pytorch is more than 6 months old, please update it. +torch 2.7 is minimally supported but using a newer version is extremely recommended. Using a cu130 or above version of pytorch is required on Nvidia 20 series and above. Some features and optimizations might only work on newer versions. We generally recommend using the latest major version of pytorch with the latest cuda version unless it is less than 2 weeks old. If your pytorch is more than 6 months old, please update it. ### Instructions: diff --git a/app/logger.py b/app/logger.py index 2e6116813..fe82c40c9 100644 --- a/app/logger.py +++ b/app/logger.py @@ -6,7 +6,7 @@ import os import sys import threading -import comfy.logging +import comfy.internal_logging ANSI_NAMED_COLORS = { 'black': '\033[30m', @@ -91,7 +91,7 @@ def on_flush(callback): def get_log_level(level): - return comfy.logging.DETAIL if level == "DETAIL" else logging.getLevelName(level) + return comfy.internal_logging.DETAIL if level == "DETAIL" else logging.getLevelName(level) def setup_logger(log_level: str = 'INFO', file_outputs=None, capacity: int = 300, use_stdout: bool = False): diff --git a/comfy/cli_args.py b/comfy/cli_args.py index 792148f0a..ee9e1ce9f 100644 --- a/comfy/cli_args.py +++ b/comfy/cli_args.py @@ -172,6 +172,7 @@ vram_group.add_argument("--cpu", action="store_true", help="To use the CPU for e parser.add_argument("--reserve-vram", type=float, default=None, help="Set the amount of vram in GB you want to reserve for use by your OS/other software. By default some amount is reserved depending on your OS.") parser.add_argument("--vram-headroom", type=float, default=0, help="Set the amount of vram in GB for DynamicVRAM to maintain as extra headroom above default. ComfyUI will try and keep this much VRAM completely free and unused, even counting VRAM from other apps.") +parser.add_argument("--disable-nvml-pressure", action="store_true", help="Use CUDA instead of NVML for DynamicVRAM memory pressure.") parser.add_argument("--async-offload", nargs='?', const=2, type=int, default=None, metavar="NUM_STREAMS", help="Use async weight offloading. An optional argument controls the amount of offload streams. Default is 2. Enabled by default on Nvidia.") parser.add_argument("--disable-async-offload", action="store_true", help="Disable async weight offloading.") diff --git a/comfy/logging.py b/comfy/internal_logging.py similarity index 100% rename from comfy/logging.py rename to comfy/internal_logging.py diff --git a/comfy/k_diffusion/sampling.py b/comfy/k_diffusion/sampling.py index 11db46d94..11b2479e0 100644 --- a/comfy/k_diffusion/sampling.py +++ b/comfy/k_diffusion/sampling.py @@ -1445,7 +1445,7 @@ def res_multistep(model, x, sigmas, extra_args=None, callback=None, disable=None x = sigma_fn(h) * x + h * (b1 * denoised + b2 * old_denoised) # Noise addition - if sigmas[i + 1] > 0: + if sigma_up > 0: x = x + noise_sampler(sigmas[i], sigmas[i + 1]) * s_noise * sigma_up if cfg_pp: diff --git a/comfy/latent_formats.py b/comfy/latent_formats.py index 8a16cfe55..c4270022b 100644 --- a/comfy/latent_formats.py +++ b/comfy/latent_formats.py @@ -434,8 +434,177 @@ class LTXV(LatentFormat): class LTXAV(LTXV): def __init__(self): - self.latent_rgb_factors = None - self.latent_rgb_factors_bias = None + # video-stream preview factors for the packed AV latent (audio stream is not previewed) + self.latent_rgb_factors = [ + [ 0.001135, -0.010555, -0.004925], + [-0.008019, -0.006231, -0.005564], + [ 0.012637, 0.005605, 0.012713], + [ 0.023454, 0.020771, 0.017844], + [-0.011940, -0.000932, 0.009292], + [ 0.018602, 0.011018, 0.013969], + [-0.036369, -0.046631, -0.057898], + [-0.031919, 0.000131, 0.015214], + [ 0.014519, 0.021041, 0.015325], + [ 0.018889, 0.016149, -0.002836], + [-0.003784, -0.006057, -0.008195], + [ 0.013262, 0.030259, 0.029775], + [ 0.050465, 0.050366, 0.025255], + [ 0.018628, 0.007691, 0.002893], + [-0.015698, -0.008451, -0.000676], + [-0.013600, -0.012587, -0.004437], + [ 0.012482, 0.021469, 0.027913], + [-0.018241, -0.013488, -0.010975], + [ 0.013828, 0.012568, 0.021984], + [ 0.017911, 0.006552, 0.005567], + [ 0.026769, 0.006803, -0.009360], + [-0.006794, -0.008447, -0.013921], + [ 0.029708, 0.018671, 0.022811], + [-0.014732, -0.019169, 0.000903], + [ 0.019607, 0.032595, 0.053409], + [-0.003721, 0.003976, 0.010364], + [-0.020193, -0.026076, -0.036068], + [-0.002328, 0.006527, 0.013052], + [ 0.017171, 0.009224, 0.006548], + [ 0.001104, -0.000591, 0.000147], + [-0.000217, 0.011834, 0.017945], + [-0.015329, -0.012463, -0.006178], + [-0.009478, -0.008680, -0.004107], + [-0.005565, -0.006006, -0.001493], + [ 0.009451, 0.008794, 0.013207], + [-0.009989, -0.008027, -0.009568], + [-0.001505, -0.008805, -0.006828], + [ 0.001105, 0.008999, 0.009079], + [ 0.025935, 0.016426, 0.008036], + [ 0.006313, 0.000694, -0.006039], + [-0.001893, -0.006951, -0.009560], + [-0.007082, -0.002566, -0.007152], + [-0.005231, 0.004829, 0.008220], + [-0.004333, 0.001251, -0.004852], + [-0.017024, -0.012730, -0.007457], + [ 0.024988, 0.032963, 0.036556], + [ 0.013697, 0.012278, 0.009979], + [-0.013751, -0.008369, -0.015446], + [-0.009348, -0.001047, 0.007622], + [-0.003135, -0.003350, -0.003766], + [ 0.007436, 0.004957, 0.010480], + [ 0.018315, 0.022066, 0.021104], + [-0.005621, -0.006770, -0.008219], + [-0.007427, 0.001911, -0.001231], + [-0.007413, 0.000486, -0.006039], + [-0.014698, -0.007160, 0.006509], + [ 0.013775, 0.014185, 0.008203], + [ 0.060246, 0.069787, 0.072833], + [ 0.009861, 0.004870, 0.001194], + [-0.003660, 0.003251, 0.008015], + [ 0.003696, -0.003680, -0.008851], + [ 0.014924, 0.006196, 0.005282], + [-0.006740, -0.004319, -0.006729], + [ 0.020635, 0.015163, 0.012385], + [-0.032623, -0.006105, 0.010436], + [-0.058988, -0.030162, -0.037961], + [-0.035614, -0.021929, -0.011062], + [-0.023412, -0.011305, -0.005054], + [-0.002716, -0.005184, -0.004084], + [ 0.014591, 0.015294, 0.014045], + [ 0.008310, 0.002466, -0.003225], + [ 0.005176, 0.001119, 0.000695], + [-0.021569, -0.030886, -0.044732], + [ 0.007517, 0.003891, 0.000551], + [-0.006793, 0.004059, 0.010184], + [-0.086481, -0.082033, -0.083414], + [ 0.004192, 0.000762, -0.008658], + [ 0.010970, 0.009002, 0.007384], + [ 0.004042, -0.006732, -0.011031], + [ 0.012164, 0.006401, 0.007483], + [ 0.029252, 0.013990, 0.011128], + [ 0.048452, 0.034648, 0.016269], + [ 0.024104, 0.012647, 0.011754], + [-0.013216, -0.020192, -0.019752], + [-0.010799, -0.008535, -0.005467], + [ 0.005823, 0.001403, 0.001890], + [ 0.052393, 0.044771, 0.032777], + [ 0.007576, -0.008080, -0.012453], + [ 0.009830, 0.004244, 0.001213], + [-0.025867, -0.013169, -0.010636], + [ 0.008494, 0.003135, 0.000790], + [ 0.003969, -0.002625, -0.010204], + [ 0.006509, 0.008272, 0.020819], + [-0.004943, -0.013424, -0.015351], + [ 0.005541, 0.009136, -0.003666], + [-0.014300, -0.015864, -0.016853], + [ 0.002650, 0.028393, 0.014125], + [-0.027661, -0.045422, -0.064995], + [ 0.009220, 0.015522, 0.010574], + [-0.002236, 0.002915, 0.004557], + [-0.020269, -0.008212, -0.000532], + [ 0.019294, 0.003655, -0.002809], + [ 0.007116, -0.002784, 0.000017], + [ 0.057277, 0.073270, 0.074401], + [-0.002616, -0.001696, -0.000498], + [ 0.007248, 0.009793, 0.022829], + [-0.002590, -0.005601, -0.000436], + [-0.007681, 0.003893, -0.004119], + [-0.057392, -0.045545, -0.025290], + [ 0.045188, 0.047985, 0.054059], + [ 0.000937, -0.008861, -0.038406], + [-0.010192, -0.008036, -0.005385], + [-0.030222, -0.027498, -0.030765], + [-0.008359, 0.013247, 0.010918], + [ 0.004102, 0.002093, 0.006934], + [ 0.039461, 0.027339, 0.008284], + [-0.075747, -0.076340, -0.071625], + [ 0.002692, 0.005096, -0.002247], + [-0.002453, -0.002785, -0.010483], + [ 0.012265, 0.005481, 0.001729], + [ 0.017755, 0.008655, 0.003532], + [ 0.055560, 0.049128, 0.044137], + [-0.025861, -0.023798, -0.018815], + [-0.014876, -0.010770, -0.010713], + [-0.017315, -0.012599, -0.008661], + [-0.008461, -0.006210, -0.007744], + [-0.040175, -0.042255, -0.048119], + [-0.019355, -0.021055, -0.021919], + ] + self.latent_rgb_factors_bias = [-0.347892, -0.363814, -0.370287] + +class MiniMaxH3Video(LatentFormat): + latent_channels = 24 + latent_dimensions = 3 + spacial_downscale_ratio = 16 + temporal_downscale_ratio = 4 + scale_factor = 1.0 + + latent_rgb_factors = [ + [-0.018555, 0.024344, -0.017536], + [ 0.150164, 0.137244, 0.129221], + [ 0.027367, -0.050369, -0.208606], + [-0.000793, -0.164622, -0.323161], + [-0.048556, 0.013970, -0.074286], + [ 0.011740, 0.014172, -0.006906], + [ 0.061517, 0.061212, 0.110025], + [ 0.035321, 0.086879, 0.110059], + [-0.017426, 0.002997, 0.035356], + [ 0.531539, 0.548819, 0.624404], + [-0.024968, -0.040234, -0.034302], + [-0.032549, -0.029096, -0.017221], + [ 0.022609, 0.020286, 0.050661], + [-0.084001, -0.038131, -0.020805], + [-0.018830, 0.010412, 0.061120], + [ 0.020777, 0.011196, -0.030994], + [-0.008390, -0.012201, -0.025687], + [-0.013281, -0.002924, 0.006331], + [ 0.000260, 0.001833, -0.011038], + [ 0.105471, 0.100482, 0.132106], + [ 0.016529, 0.015213, 0.009999], + [-0.014015, -0.017438, -0.019134], + [-0.033787, -0.009984, -0.019725], + [ 0.004224, 0.017284, 0.027196], + ] + latent_rgb_factors_bias = [ 0.057426, -0.022078, -0.071449] + +class MiniMaxH3AV(MiniMaxH3Video): + # max channels across the two streams (video 24, audio 32) so per-stream slices keep both streams whole + latent_channels = 32 class HunyuanVideo(LatentFormat): latent_channels = 16 diff --git a/comfy/ldm/lightricks/av_model.py b/comfy/ldm/lightricks/av_model.py index ef9938465..8e360f6a8 100644 --- a/comfy/ldm/lightricks/av_model.py +++ b/comfy/ldm/lightricks/av_model.py @@ -16,7 +16,9 @@ from comfy.ldm.lightricks.model import ( from comfy.ldm.lightricks.symmetric_patchifier import AudioPatchifier from comfy.ldm.lightricks.embeddings_connector import Embeddings1DConnector import comfy.ldm.common_dit +import comfy.model_management import comfy.model_prefetch +import comfy.quant_ops class CompressedTimestep: """Store video timestep embeddings in compressed form using per-frame indexing.""" @@ -271,7 +273,10 @@ class BasicAVTransformerBlock(nn.Module): if run_vx: # video self-attention vshift_msa, vscale_msa = (self.get_ada_values(self.scale_shift_table, vx.shape[0], v_timestep, slice(0, 2))) - norm_vx = comfy.ldm.common_dit.rms_norm(vx) * (1 + vscale_msa) + vshift_msa + if comfy.model_management.in_training: + norm_vx = comfy.ldm.common_dit.rms_norm(vx) * (1 + vscale_msa) + vshift_msa + else: + norm_vx = comfy.quant_ops.ck.rms_adaln(vx, vscale_msa, vshift_msa) del vshift_msa, vscale_msa attn1_out = self.attn1(norm_vx, pe=v_pe, mask=self_attention_mask, transformer_options=transformer_options) del norm_vx @@ -305,7 +310,6 @@ class BasicAVTransformerBlock(nn.Module): # video - audio cross attention. if run_a2v or run_v2a: - vx_norm3 = comfy.ldm.common_dit.rms_norm(vx) ax_norm3 = comfy.ldm.common_dit.rms_norm(ax) # audio to video cross attention @@ -315,7 +319,10 @@ class BasicAVTransformerBlock(nn.Module): scale_ca_video_hidden_states_a2v_v, shift_ca_video_hidden_states_a2v_v = self.get_ada_values( self.scale_shift_table_a2v_ca_video[:4, :], vx.shape[0], v_cross_scale_shift_timestep)[:2] - vx_scaled = vx_norm3 * (1 + scale_ca_video_hidden_states_a2v_v) + shift_ca_video_hidden_states_a2v_v + if comfy.model_management.in_training: + vx_scaled = comfy.ldm.common_dit.rms_norm(vx) * (1 + scale_ca_video_hidden_states_a2v_v) + shift_ca_video_hidden_states_a2v_v + else: + vx_scaled = comfy.quant_ops.ck.rms_adaln(vx, scale_ca_video_hidden_states_a2v_v, shift_ca_video_hidden_states_a2v_v) ax_scaled = ax_norm3 * (1 + scale_ca_audio_hidden_states_a2v) + shift_ca_audio_hidden_states_a2v del scale_ca_video_hidden_states_a2v_v, shift_ca_video_hidden_states_a2v_v, scale_ca_audio_hidden_states_a2v, shift_ca_audio_hidden_states_a2v @@ -334,7 +341,10 @@ class BasicAVTransformerBlock(nn.Module): self.scale_shift_table_a2v_ca_video[:4, :], vx.shape[0], v_cross_scale_shift_timestep)[2:4] ax_scaled = ax_norm3 * (1 + scale_ca_audio_hidden_states_v2a) + shift_ca_audio_hidden_states_v2a - vx_scaled = vx_norm3 * (1 + scale_ca_video_hidden_states_v2a) + shift_ca_video_hidden_states_v2a + if comfy.model_management.in_training: + vx_scaled = comfy.ldm.common_dit.rms_norm(vx) * (1 + scale_ca_video_hidden_states_v2a) + shift_ca_video_hidden_states_v2a + else: + vx_scaled = comfy.quant_ops.ck.rms_adaln(vx, scale_ca_video_hidden_states_v2a, shift_ca_video_hidden_states_v2a) del scale_ca_video_hidden_states_v2a, shift_ca_video_hidden_states_v2a, scale_ca_audio_hidden_states_v2a, shift_ca_audio_hidden_states_v2a v2a_out = self.video_to_audio_attn(ax_scaled, context=vx_scaled, pe=a_cross_pe, k_pe=v_cross_pe, transformer_options=transformer_options) @@ -344,12 +354,14 @@ class BasicAVTransformerBlock(nn.Module): ax.addcmul_(v2a_out, gate_out_v2a) del gate_out_v2a, v2a_out - del vx_norm3, ax_norm3 # video feedforward if run_vx: vshift_mlp, vscale_mlp = self.get_ada_values(self.scale_shift_table, vx.shape[0], v_timestep, slice(3, 5)) - vx_scaled = comfy.ldm.common_dit.rms_norm(vx) * (1 + vscale_mlp) + vshift_mlp + if comfy.model_management.in_training: + vx_scaled = comfy.ldm.common_dit.rms_norm(vx) * (1 + vscale_mlp) + vshift_mlp + else: + vx_scaled = comfy.quant_ops.ck.rms_adaln(vx, vscale_mlp, vshift_mlp) del vshift_mlp, vscale_mlp ff_out = self.ff(vx_scaled) diff --git a/comfy/ldm/lightricks/model.py b/comfy/ldm/lightricks/model.py index f9de3a38e..f80bffba7 100644 --- a/comfy/ldm/lightricks/model.py +++ b/comfy/ldm/lightricks/model.py @@ -13,6 +13,7 @@ import comfy.patcher_extension import comfy.ldm.modules.attention import comfy.ldm.common_dit import comfy.model_management +import comfy.ops import comfy.quant_ops from .symmetric_patchifier import SymmetricPatchifier, latent_to_pixel_coords @@ -321,7 +322,11 @@ class FeedForward(nn.Module): ) def forward(self, x): - return self.net(x) + # net = [GELU_approx(proj), Dropout, Linear]; the fused path skips the + # Dropout, so leave it to the stock path whenever it could be active. + if comfy.model_management.in_training: + return self.net(x) + return comfy.ops.linear_input_act(self.net[2], self.net[0].proj(x), "gelu_tanh") def apply_rotary_emb(input_tensor, freqs_cis): rotation_matrix, split_pe = freqs_cis @@ -535,7 +540,12 @@ class BasicTransformerBlock(nn.Module): def forward(self, x, context=None, attention_mask=None, timestep=None, pe=None, transformer_options={}, self_attention_mask=None, prompt_timestep=None): shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = (self.scale_shift_table[None, None, :6].to(device=x.device, dtype=x.dtype) + timestep.reshape(x.shape[0], timestep.shape[1], self.scale_shift_table.shape[0], -1)[:, :, :6, :]).unbind(dim=2) - x += self.attn1(comfy.ldm.common_dit.rms_norm(x) * (1 + scale_msa) + shift_msa, pe=pe, mask=self_attention_mask, transformer_options=transformer_options) * gate_msa + if comfy.model_management.in_training: + norm_x = comfy.ldm.common_dit.rms_norm(x) * (1 + scale_msa) + shift_msa + else: + norm_x = comfy.quant_ops.ck.rms_adaln(x, scale_msa, shift_msa) + + x += self.attn1(norm_x, pe=pe, mask=self_attention_mask, transformer_options=transformer_options) * gate_msa if self.cross_attention_adaln: shift_q_mca, scale_q_mca, gate_mca = (self.scale_shift_table[None, None, 6:9].to(device=x.device, dtype=x.dtype) + timestep.reshape(x.shape[0], timestep.shape[1], self.scale_shift_table.shape[0], -1)[:, :, 6:9, :]).unbind(dim=2) @@ -589,7 +599,10 @@ def apply_cross_attention_adaln( prompt_scale_shift_table[None, None].to(device=x.device, dtype=x.dtype) + prompt_timestep.reshape(batch_size, prompt_timestep.shape[1], 2, -1) ).unbind(dim=2) - attn_input = comfy.ldm.common_dit.rms_norm(x) * (1 + q_scale) + q_shift + if comfy.model_management.in_training: + attn_input = comfy.ldm.common_dit.rms_norm(x) * (1 + q_scale) + q_shift + else: + attn_input = comfy.quant_ops.ck.rms_adaln(x, q_scale, q_shift) encoder_hidden_states = context * (1 + scale_kv) + shift_kv return attn(attn_input, context=encoder_hidden_states, mask=attention_mask, transformer_options=transformer_options) * q_gate diff --git a/comfy/ldm/minimax/audio_vae.py b/comfy/ldm/minimax/audio_vae.py new file mode 100644 index 000000000..0b9f2b6fc --- /dev/null +++ b/comfy/ldm/minimax/audio_vae.py @@ -0,0 +1,443 @@ +# MiniMax H3 audio VAE: DAC-lineage waveform encoder + BigVGAN decoder. +# Weight-norm parametrizations are folded into plain conv weights, so this +# module uses ordinary ops.Conv1d / ops.ConvTranspose1d and loads the converted +# checkpoint (plain "*.weight" tensors) with strict=True. +# +# Lineage / licenses of the reference implementation: +# DAC encoder: descript-audio-codec (MIT) +# BigVGAN decoder: NVIDIA BigVGAN (MIT), adapted from hifi-gan (MIT) +# Alias-free ops: junjun3518/alias-free-torch (Apache-2.0), julius (MIT) + +import math + +import torch +import torch.nn as nn +import torch.nn.functional as F + +import comfy.ops + +ops = comfy.ops.disable_weight_init + + +# Snake activations + +def snake(x, alpha, beta): + # x + 1/beta * sin^2(alpha * x) + t = torch.sin(alpha * x) + return t.mul_(t).mul_((beta + 1e-9).reciprocal()).add_(x) + + +class Snake1d(nn.Module): + """Snake activation with per-channel alpha (encoder side).""" + + def __init__(self, channels): + super().__init__() + self.alpha = nn.Parameter(torch.empty(1, channels, 1)) + + def forward(self, x): + alpha = comfy.ops.cast_to_input(self.alpha, x) + return snake(x, alpha, alpha) + + +class SnakeBeta(nn.Module): + """SnakeBeta := x + 1/beta * sin^2(alpha * x); alpha/beta stored in log scale.""" + + def __init__(self, in_features): + super().__init__() + self.alpha = nn.Parameter(torch.empty(in_features)) + self.beta = nn.Parameter(torch.empty(in_features)) + + def forward(self, x): + alpha = torch.exp(comfy.ops.cast_to_input(self.alpha, x)).view(1, -1, 1) + beta = torch.exp(comfy.ops.cast_to_input(self.beta, x)).view(1, -1, 1) + return snake(x, alpha, beta) + + +# Alias-free (anti-aliased) activation: kaiser-windowed sinc resampling + +def kaiser_sinc_filter1d(cutoff, half_width, kernel_size): + # returns filter [1, 1, kernel_size] + even = kernel_size % 2 == 0 + half_size = kernel_size // 2 + + # kaiser window design + delta_f = 4 * half_width + A = 2.285 * (half_size - 1) * math.pi * delta_f + 7.95 + if A > 50.0: + beta = 0.1102 * (A - 8.7) + elif A >= 21.0: + beta = 0.5842 * (A - 21) ** 0.4 + 0.07886 * (A - 21.0) + else: + beta = 0.0 + window = torch.kaiser_window(kernel_size, beta=beta, periodic=False) + + if even: + time = torch.arange(-half_size, half_size) + 0.5 + else: + time = torch.arange(kernel_size) - half_size + + filter_ = 2 * cutoff * window * torch.sinc(2 * cutoff * time) + # Normalize filter to have sum = 1, otherwise there is a small leakage of + # the constant component in the input signal. + filter_ /= filter_.sum() + return filter_.view(1, 1, kernel_size) + + +class UpSample1d(nn.Module): + def __init__(self, ratio=2, kernel_size=12): + super().__init__() + self.ratio = ratio + self.stride = ratio + self.pad = kernel_size // ratio - 1 + self.pad_left = self.pad * ratio + (kernel_size - ratio) // 2 + self.pad_right = self.pad * ratio + (kernel_size - ratio + 1) // 2 + self.register_buffer( + "filter", + kaiser_sinc_filter1d(cutoff=0.5 / ratio, half_width=0.6 / ratio, kernel_size=kernel_size), + ) + + def forward(self, x): + _, C, _ = x.shape + x = F.pad(x, (self.pad, self.pad), mode="replicate") + x = F.conv_transpose1d(x, comfy.ops.cast_to_input(self.filter.expand(C, -1, -1), x), stride=self.stride, groups=C).mul_(self.ratio) + x = x[..., self.pad_left:-self.pad_right] + return x + + +class LowPassFilter1d(nn.Module): + def __init__(self, cutoff=0.5, half_width=0.6, stride=1, kernel_size=12): + super().__init__() + self.pad_left = kernel_size // 2 - int(kernel_size % 2 == 0) + self.pad_right = kernel_size // 2 + self.stride = stride + self.register_buffer("filter", kaiser_sinc_filter1d(cutoff, half_width, kernel_size)) + + def forward(self, x): + _, C, _ = x.shape + x = F.pad(x, (self.pad_left, self.pad_right), mode="replicate") + return F.conv1d(x, comfy.ops.cast_to_input(self.filter.expand(C, -1, -1), x), stride=self.stride, groups=C) + + +class DownSample1d(nn.Module): + def __init__(self, ratio=2, kernel_size=12): + super().__init__() + self.ratio = ratio + self.kernel_size = kernel_size + self.lowpass = LowPassFilter1d( + cutoff=0.5 / ratio, + half_width=0.6 / ratio, + stride=ratio, + kernel_size=self.kernel_size, + ) + + def forward(self, x): + return self.lowpass(x) + + +class Activation1d(nn.Module): + """upsample x2 -> pointwise activation -> downsample x2 (anti-aliased).""" + + def __init__(self, activation, up_ratio=2, down_ratio=2, up_kernel_size=12, down_kernel_size=12): + super().__init__() + self.act = activation + self.upsample = UpSample1d(up_ratio, up_kernel_size) + self.downsample = DownSample1d(down_ratio, down_kernel_size) + + def forward(self, x): + x = self.upsample(x) + x = self.act(x) + x = self.downsample(x) + return x + + +# DAC encoder + +class ResidualUnit(nn.Module): + def __init__(self, dim=16, dilation=1): + super().__init__() + pad = ((7 - 1) * dilation) // 2 + self.block = nn.Sequential( + Snake1d(dim), + ops.Conv1d(dim, dim, kernel_size=7, dilation=dilation, padding=pad), + Snake1d(dim), + ops.Conv1d(dim, dim, kernel_size=1), + ) + + def forward(self, x): + y = self.block(x) + pad = (x.shape[-1] - y.shape[-1]) // 2 + if pad > 0: + x = x[..., pad:-pad] + return y.add_(x) + + +class EncoderBlock(nn.Module): + def __init__(self, dim=16, stride=1): + super().__init__() + self.block = nn.Sequential( + ResidualUnit(dim // 2, dilation=1), + ResidualUnit(dim // 2, dilation=3), + ResidualUnit(dim // 2, dilation=9), + Snake1d(dim // 2), + ops.Conv1d( + dim // 2, + dim, + kernel_size=2 * stride, + stride=stride, + padding=math.ceil(stride / 2), + ), + ) + + def forward(self, x): + return self.block(x) + + +class Encoder(nn.Module): + def __init__(self, d_model=64, strides=(2, 4, 4, 5, 5), d_latent=2048): + super().__init__() + block = [ops.Conv1d(1, d_model, kernel_size=7, padding=3)] + for stride in strides: + d_model *= 2 + block += [EncoderBlock(d_model, stride=stride)] + block += [ + Snake1d(d_model), + ops.Conv1d(d_model, d_latent, kernel_size=3, padding=1), + ] + self.block = nn.Sequential(*block) + + def forward(self, x): + return self.block(x) + + +# Attention projection (encoder posterior head) + +class GeGluMlp(nn.Module): + def __init__(self, in_features, hidden_features): + super().__init__() + self.norm = ops.LayerNorm(in_features) + self.act = nn.GELU(approximate="tanh") + self.w0 = ops.Linear(in_features, hidden_features) + self.w1 = ops.Linear(in_features, hidden_features) + self.w2 = ops.Linear(hidden_features, in_features) + + def forward(self, x): + x = self.norm(x) + return self.w2(self.act(self.w0(x)).mul_(self.w1(x))) + + +class CausalAttention(nn.Module): + def __init__(self, in_dim, out_dim, num_heads): + super().__init__() + self.head_dim = in_dim // num_heads + self.num_heads = num_heads + self.out_dim = out_dim + self.qkv = ops.Linear(in_dim, in_dim * 3, bias=False) + self.q_bias = nn.Parameter(torch.empty(in_dim)) + self.v_bias = nn.Parameter(torch.empty(in_dim)) + self.register_buffer("zero_k_bias", torch.empty(in_dim)) + self.proj = ops.Linear(out_dim, out_dim) + + def forward(self, x): + B, N, C = x.shape + weight, _, offload_stream = comfy.ops.cast_bias_weight(self.qkv, x, offloadable=True) + qkv = F.linear(x, weight=weight, bias=comfy.ops.cast_to_input(torch.cat((self.q_bias, self.zero_k_bias, self.v_bias)), x)) + comfy.ops.uncast_bias_weight(self.qkv, weight, None, offload_stream) + q, k, v = qkv.reshape(B, N, 3, self.num_heads, self.head_dim).permute(2, 0, 3, 1, 4).unbind(0) + + # mean over heads then pool down to the latent width (in_dim >> out_dim) + x = comfy.ops.scaled_dot_product_attention(q, k, v, is_causal=True) + x = F.adaptive_avg_pool1d(torch.mean(x, dim=1), self.out_dim) + return self.proj(x) + + +class AttnProjection(nn.Module): + def __init__(self, in_dim, out_dim, num_heads, mlp_ratio=2): + super().__init__() + self.norm1 = ops.LayerNorm(in_dim) + self.attn = CausalAttention(in_dim, out_dim, num_heads) + self.proj = ops.Linear(in_dim, out_dim) + self.norm3 = ops.LayerNorm(in_dim) + + self.norm2 = ops.LayerNorm(out_dim) + hidden_dim = int(out_dim * mlp_ratio) + self.mlp = GeGluMlp(in_features=out_dim, hidden_features=hidden_dim) + + def forward(self, x): + # x: [B, T, in_dim] + x = self.proj(self.norm3(x)).add_(self.attn(self.norm1(x))) + return x.add_(self.mlp(self.norm2(x))) + + +# BigVGAN decoder + +def get_padding(kernel_size, dilation=1): + return int((kernel_size * dilation - dilation) / 2) + + +class AMPBlock1(nn.Module): + def __init__(self, channels, kernel_size=3, dilation=(1, 3, 5)): + super().__init__() + self.convs1 = nn.ModuleList( + [ + ops.Conv1d(channels, channels, kernel_size, stride=1, dilation=d, padding=get_padding(kernel_size, d)) + for d in dilation + ] + ) + self.convs2 = nn.ModuleList( + [ + ops.Conv1d(channels, channels, kernel_size, stride=1, dilation=1, padding=get_padding(kernel_size, 1)) + for _ in range(len(dilation)) + ] + ) + self.num_layers = len(self.convs1) + len(self.convs2) + self.activations = nn.ModuleList( + [Activation1d(activation=SnakeBeta(channels)) for _ in range(self.num_layers)] + ) + + def forward(self, x): + acts1, acts2 = self.activations[::2], self.activations[1::2] + for c1, c2, a1, a2 in zip(self.convs1, self.convs2, acts1, acts2): + xt = a1(x) + xt = c1(xt) + xt = a2(xt) + xt = c2(xt) + x = xt.add_(x) + return x + + +class BigVGAN(nn.Module): + """BigVGAN vocoder (MiniMax H3 32 kHz configuration). + + use_bias_at_final=False, use_tanh_at_final=False (output clamped to [-1, 1]). + """ + + def __init__( + self, + num_mels=2048, + upsample_initial_channel=1024, + upsample_rates=(5, 5, 2, 2, 2, 2, 2), + upsample_kernel_sizes=(9, 9, 4, 4, 4, 4, 4), + resblock_kernel_sizes=(3, 7, 11), + resblock_dilation_sizes=((1, 3, 5), (1, 3, 5), (1, 3, 5)), + ): + super().__init__() + self.num_kernels = len(resblock_kernel_sizes) + self.num_upsamples = len(upsample_rates) + + self.conv_pre = ops.Conv1d(num_mels, upsample_initial_channel, 7, 1, padding=3) + + self.ups = nn.ModuleList() + for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)): + self.ups.append( + nn.ModuleList( + [ + ops.ConvTranspose1d( + upsample_initial_channel // (2 ** i), + upsample_initial_channel // (2 ** (i + 1)), + k, + u, + padding=(k - u) // 2, + ) + ] + ) + ) + + self.resblocks = nn.ModuleList() + for i in range(len(self.ups)): + ch = upsample_initial_channel // (2 ** (i + 1)) + for k, d in zip(resblock_kernel_sizes, resblock_dilation_sizes): + self.resblocks.append(AMPBlock1(ch, k, d)) + + self.activation_post = Activation1d(activation=SnakeBeta(ch)) + self.conv_post = ops.Conv1d(ch, 1, 7, 1, padding=3, bias=False) + + def forward(self, x): + x = self.conv_pre(x) + + for i in range(self.num_upsamples): + for i_up in range(len(self.ups[i])): + x = self.ups[i][i_up](x) + xs = None + for j in range(self.num_kernels): + if xs is None: + xs = self.resblocks[i * self.num_kernels + j](x) + else: + xs += self.resblocks[i * self.num_kernels + j](x) + x = xs.div_(self.num_kernels) + + x = self.activation_post(x) + return self.conv_post(x).clamp_(-1.0, 1.0) + + +# Top-level VAE + +class MiniMaxH3AudioVAE(nn.Module): + """MiniMax H3 stereo audio VAE at 32 kHz. + + Latents are [B, 32, 2, T]: 32 channels, 2 stereo channels, T frames at + 40 latent frames per second (800 audio samples per latent frame). The + stereo channels are processed independently by the mono encoder/decoder. + Latents are normalized with the stored per-channel latents_mean/std. + """ + + def __init__( + self, + encoder_dim=64, + encoder_rates=(2, 4, 4, 5, 5), + latent_dim=2048, + decoder_dim=1024, + vae_latent_channels=32, + ): + super().__init__() + self.sample_rate = 32000 + + self.hop_length = 1 + for r in encoder_rates: + self.hop_length *= r + self.samples_per_latent = self.hop_length # 800 + self.latents_per_second = self.sample_rate // self.hop_length # 40 + self.output_sample_rate = self.sample_rate # read by LTXVAudioVAEDecode + + self.encoder = Encoder(encoder_dim, encoder_rates, latent_dim) + + self.pre_block = AttnProjection(latent_dim, vae_latent_channels, num_heads=8) + + self.mean_proj = ops.Conv1d(vae_latent_channels, vae_latent_channels, 1) + # logs_proj exists in the checkpoint but is unused at inference + # (encode returns the posterior mean, no sampling). + self.logs_proj = ops.Conv1d(vae_latent_channels, vae_latent_channels, 1) + + self.dec_in_proj = ops.Conv1d(vae_latent_channels, latent_dim, 1) + self.decoder = BigVGAN(num_mels=latent_dim, upsample_initial_channel=decoder_dim) + + self.register_buffer("latents_mean", torch.empty(vae_latent_channels)) + self.register_buffer("latents_std", torch.empty(vae_latent_channels)) + + def decode(self, z): + """Decode normalized latents [B, 32, 2, T] to stereo waveforms [B, 2, L] at 32 kHz.""" + b, c, s, t = z.shape + z = z.permute(0, 2, 1, 3).reshape(b * s, c, t) + mean = self.latents_mean.view(1, -1, 1).to(device=z.device, dtype=z.dtype) + std = self.latents_std.view(1, -1, 1).to(device=z.device, dtype=z.dtype) + z = z * std + mean + x = self.dec_in_proj(z) + x = self.decoder(x) # [b * s, 1, L], already clamped to [-1, 1] + return x.reshape(b, s, -1) + + def encode(self, waveform): + """Encode stereo waveforms [B, 2, L] at 32 kHz (in [-1, 1]) to normalized latents [B, 32, 2, T]. + + L is right-padded with zeros to a multiple of 800 samples; the returned + posterior mean is used directly (no sampling). + """ + b, s, length = waveform.shape + right_pad = math.ceil(length / self.hop_length) * self.hop_length - length + waveform = F.pad(waveform, (0, right_pad)) + x = waveform.reshape(b * s, 1, -1) + x = self.encoder(x) # [b * s, latent_dim, T] + x = self.pre_block(x.transpose(1, 2)).transpose(1, 2) # [b * s, 32, T] + z = self.mean_proj(x) + mean = self.latents_mean.view(1, -1, 1).to(device=z.device, dtype=z.dtype) + std = self.latents_std.view(1, -1, 1).to(device=z.device, dtype=z.dtype) + z = (z - mean) / std + return z.reshape(b, s, z.shape[1], z.shape[2]).permute(0, 2, 1, 3) diff --git a/comfy/ldm/minimax/model.py b/comfy/ldm/minimax/model.py new file mode 100644 index 000000000..bc06288ab --- /dev/null +++ b/comfy/ldm/minimax/model.py @@ -0,0 +1,650 @@ +"""MiniMax H3 audio-video DiT. + +Single-stream packed-token transformer denoising video (24ch, patch 1x2x2) and +stereo audio (32ch, 40 Hz) latents jointly, conditioned on Qwen3-VL layer-50 hidden states. +The packed sequence is: +[text | cond rows | audio | video] for t2va/fl2va +[text | reference blocks | audio | video] for ref2va + +Timestep domain: the model receives the *video* sigma from the sampler and +derives per-token timesteps t = 1 - sigma internally; the audio stream runs on +its own shifted schedule (sigma_shift video 12.0 / audio 3.0), mapped from the +video sigma in closed form. The sampler carries the audio latent scaled onto the +video schedule (ModelSamplingAV); forward() undoes that scale and converts the +velocity back, so _forward only ever sees the stream's own latent. +""" + +import math + +import torch +import torch.nn as nn + +import comfy.ldm.common_dit +import comfy.model_management +import comfy.model_prefetch +import comfy.ops +import comfy.patcher_extension +import comfy.quant_ops +from comfy.ldm.modules.attention import optimized_attention + +FRAME_PER_TOKEN = (1, 4, 4, 4, 4) +FRAME_RESCALE = 5.0 / 3.0 +VISUAL_COND_TIMESTEP = 0.999 +AUDIO_COND_TIMESTEP = 1.0 + + +def time_shift_sigma(sigma, from_shift, to_shift): + # invert sigma = s*b/(1+(s-1)*b) to the base grid, re-apply the other shift + base = sigma / (from_shift + sigma * (1.0 - from_shift)) + return to_shift * base / (1.0 + (to_shift - 1.0) * base) + + +def patchify_video(latent, patch_size=(1, 2, 2)): + # [B, C, T, H, W] -> [B*t*h*w, C*pt*ph*pw] + b, c, t_full, h_full, w_full = latent.shape + pt, ph, pw = patch_size + t, h, w = t_full // pt, h_full // ph, w_full // pw + x = latent.reshape(b, c, t, pt, h, ph, w, pw) + x = torch.einsum("nctrhpwq->nthwcrpq", x) + return x.reshape(b * t * h * w, c * pt * ph * pw) + + +def unpatchify_video(rows, t, h, w, c=24, patch_size=(1, 2, 2)): + pt, ph, pw = patch_size + x = rows.reshape(-1, t, h, w, c, pt, ph, pw) + x = torch.einsum("nthwcrpq->nctrhpwq", x) + return x.reshape(-1, c, t * pt, h * ph, w * pw) + + +def pack_audio(latent): + # [B, C=32, ch=2, T] -> [ch*T, 32] channel-major (ch0 t0..T-1, ch1 t0..T-1) + b, c, ch, t = latent.shape + return latent[0].permute(1, 2, 0).reshape(ch * t, c) + + +def unpack_audio(rows, ch=2): + t = rows.shape[0] // ch + return rows.reshape(ch, t, rows.shape[-1]).permute(2, 0, 1).unsqueeze(0) + + +def _axis_from_sqrt_area(dim, patch, sqrt_area): + # linspace((1 - ratio) / 2, (1 + ratio) / 2, dim // patch, endpoint=False) * 32 + ratio = dim / sqrt_area + n = dim // patch + return (torch.arange(n, dtype=torch.float64) * (ratio / n) + (1.0 - ratio) / 2.0) * 32.0 + + +def _frame_grid(h, w): + # area-normalized (h, w) coordinates of one latent frame's 2x2-patch rows + area = math.sqrt(h * w) + hh, ww = torch.meshgrid(_axis_from_sqrt_area(h, 2, area), _axis_from_sqrt_area(w, 2, area), indexing="ij") + return torch.stack([hh.reshape(-1), ww.reshape(-1)], dim=-1), _axis_from_sqrt_area(w, 2, area) + + +def _video_t_spans(n): + return [FRAME_RESCALE * FRAME_PER_TOKEN[k % 5] for k in range(n)] + + +def _video_t_grid(n, origin): + # origin + exclusive cumsum + spans = torch.tensor(_video_t_spans(n), dtype=torch.float64) + return float(origin) + torch.cat([torch.zeros(1, dtype=torch.float64), spans[:-1].cumsum(0)]) + + +def _audio_grid(cursor, t, w_low, w_high): + # channel-major stereo rows: t advances per latent frame, w pinned to the grid extremes per stereo channel, h stays 0 + g = torch.zeros(t * 2, 3, dtype=torch.float64) + g[:, 0] = (cursor + torch.arange(t, dtype=torch.float64)).repeat(2) + g[:t, 2] = w_low + g[t:, 2] = w_high + return g + + +def _video_grid(vt, frame, cursor): + g = torch.empty(vt, frame.shape[0], 3, dtype=torch.float64) + g[:, :, 0] = _video_t_grid(vt, cursor)[:, None] + g[:, :, 1:] = frame[None] + return g.reshape(-1, 3) + + +class TimeEmbedder(nn.Module): + def __init__(self, freq_dim, hidden, out, dtype=None, device=None, operations=None): + super().__init__() + self.freq_dim = freq_dim + self.proj_in = operations.Linear(freq_dim, hidden, bias=True, dtype=dtype, device=device) + self.proj_out = operations.Linear(hidden, out, bias=True, dtype=dtype, device=device) + + def forward(self, t): + # t: [M] in [0, 1]; fp32 throughout, cos before sin + half = self.freq_dim // 2 + freqs = torch.exp(-math.log(10000.0) * torch.arange(half, dtype=torch.float32, device=t.device) / half) + args = t.to(torch.float32)[:, None] * freqs[None] + emb = torch.cat([torch.cos(args), torch.sin(args)], dim=-1) + return self.proj_out(nn.functional.silu(self.proj_in(emb))) + + +def rope_rotation_table(angles, dtype): + """[S, rot_dim] pair angles -> [1, S, 1, rot_dim/2, 2, 2] rotation matrices.""" + half = angles.shape[-1] // 2 + ang = angles[:, :half] # duplicated halves: [:, :half] == [:, half:] + c, s = torch.cos(ang), torch.sin(ang) + table = torch.stack([c, -s, s, c], dim=-1).reshape(1, angles.shape[0], 1, half, 2, 2) + return table.to(dtype) + + +class Attention(nn.Module): + def __init__(self, hidden, heads, head_dim, eps, dtype=None, device=None, operations=None): + super().__init__() + self.heads = heads + self.head_dim = head_dim + inner = heads * head_dim + self.qkv_proj = operations.Linear(hidden, inner * 3, bias=False, dtype=dtype, device=device) + self.q_norm = operations.RMSNorm(head_dim, eps=eps, dtype=dtype, device=device) + self.k_norm = operations.RMSNorm(head_dim, eps=eps, dtype=dtype, device=device) + self.out_proj = operations.Linear(inner, hidden, bias=False, dtype=dtype, device=device) + + def forward(self, x, rope_freqs=None, transformer_options={}): + s = x.shape[0] + q, k, v = self.qkv_proj(x).split(self.heads * self.head_dim, dim=-1) + v = v.view(s, self.heads, self.head_dim) + if rope_freqs is not None: + # fused per-head RMSNorm + partial split-half rope, in place on the qkv buffer + q = q.view(1, s, self.heads, self.head_dim) + k = k.view(1, s, self.heads, self.head_dim) + qw = comfy.model_management.cast_to(self.q_norm.weight, device=x.device) + kw = comfy.model_management.cast_to(self.k_norm.weight, device=x.device) + rot = rope_freqs.shape[-3] * 2 + if comfy.model_management.in_training: + q, k = comfy.quant_ops.ck.rms_rope_split_half( + q, k, rope_freqs, qw, kw, epsilon=self.q_norm.eps, rot_dim=rot) + else: + comfy.quant_ops.ck.rms_rope_split_half_( + q, k, rope_freqs, qw, kw, epsilon=self.q_norm.eps, rot_dim=rot) + q = q[0] + k = k[0] + else: + q = self.q_norm(q.view(s, self.heads, self.head_dim)) + k = self.k_norm(k.view(s, self.heads, self.head_dim)) + q = q.transpose(0, 1).unsqueeze(0) + k = k.transpose(0, 1).unsqueeze(0) + v = v.transpose(0, 1).unsqueeze(0) + out = optimized_attention(q, k, v, self.heads, mask=None, skip_reshape=True, transformer_options=transformer_options) + return self.out_proj(out.squeeze(0)) + + +class MLP(nn.Module): + def __init__(self, hidden, ffn, dtype=None, device=None, operations=None): + super().__init__() + self.fc1 = operations.Linear(hidden, ffn * 2, bias=False, dtype=dtype, device=device) + self.fc2 = operations.Linear(ffn, hidden, bias=False, dtype=dtype, device=device) + + def forward(self, x): + return comfy.ops.linear_input_act(self.fc2, self.fc1(x), "swiglu") + + +class AdalnProj(nn.Module): + def __init__(self, t_dim, hidden, expand, modalities, apply_silu=True, + dtype=None, device=None, operations=None): + super().__init__() + self.expand = expand + self.modalities = modalities + self.hidden = hidden + self.apply_silu = apply_silu + self.linear = operations.Linear(t_dim, expand * hidden * modalities, bias=True, dtype=dtype, device=device) + + def forward(self, t_emb): + # [M, t_dim] -> expand tensors of [M*modalities, hidden] + x = self.linear(nn.functional.silu(t_emb) if self.apply_silu else t_emb) + x = x.view(x.shape[0] * self.modalities, self.expand * self.hidden) + return x.chunk(self.expand, dim=-1) + + +def _mod_scale_shift(h, shift, scale, segments): + # segments: [(start, stop, mod_row)] covering h contiguously. + for a, b, row in segments: + h[a:b].mul_(1.0 + scale[row].to(h.dtype)).add_(shift[row].to(h.dtype)) + return h + + +def _mod_gate(x, gate, other, segments): + # other is the fresh attn/mlp output: accumulate the gated residual into the stream in place, one fused kernel per segment + for a, b, row in segments: + x[a:b].addcmul_(other[a:b], gate[row].to(x.dtype)) + return x + + +class RefinerBlock(nn.Module): + def __init__(self, hidden, heads, head_dim, ffn, eps, qk_eps, dtype=None, device=None, operations=None): + super().__init__() + self.norm1 = operations.RMSNorm(hidden, eps=eps, dtype=dtype, device=device) + self.norm2 = operations.RMSNorm(hidden, eps=eps, dtype=dtype, device=device) + self.attn = Attention(hidden, heads, head_dim, qk_eps, dtype=dtype, device=device, operations=operations) + self.mlp = MLP(hidden, ffn, dtype=dtype, device=device, operations=operations) + + def forward(self, x, transformer_options={}): + # attn/mlp outputs are fresh: accumulate residuals in place + x = self.attn(self.norm1(x), transformer_options=transformer_options).add_(x) + return self.mlp(self.norm2(x)).add_(x) + + +class TokenRefiner(nn.Module): + def __init__(self, num_layers, hidden, heads, head_dim, ffn, eps, qk_eps, final_eps, + dtype=None, device=None, operations=None): + super().__init__() + self.blocks = nn.ModuleList([ + RefinerBlock(hidden, heads, head_dim, ffn, eps, qk_eps, dtype=dtype, device=device, operations=operations) + for _ in range(num_layers)]) + self.final_norm = operations.RMSNorm(hidden, eps=final_eps, dtype=dtype, device=device) + + def forward(self, x, transformer_options={}): + for block in self.blocks: + x = block(x, transformer_options=transformer_options) + return self.final_norm(x) + + +class DiTBlock(nn.Module): + def __init__(self, hidden, heads, head_dim, ffn, t_dim, eps, qk_eps, + apply_silu=True, adaln_dtype=None, dtype=None, device=None, operations=None): + super().__init__() + self.norm1 = operations.RMSNorm(hidden, eps=eps, dtype=dtype, device=device) + self.norm2 = operations.RMSNorm(hidden, eps=eps, dtype=dtype, device=device) + self.attn = Attention(hidden, heads, head_dim, qk_eps, dtype=dtype, device=device, operations=operations) + self.mlp = MLP(hidden, ffn, dtype=dtype, device=device, operations=operations) + self.adaln_proj = AdalnProj(t_dim, hidden, 6, 3, apply_silu=apply_silu, + dtype=adaln_dtype if adaln_dtype is not None else dtype, + device=device, operations=operations) + + def forward(self, x, t_emb, mod_segments, rope_freqs, transformer_options={}): + shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.adaln_proj(t_emb) + h = _mod_scale_shift(self.norm1(x), shift_msa, scale_msa, mod_segments) + x = _mod_gate(x, gate_msa, self.attn(h, rope_freqs=rope_freqs, transformer_options=transformer_options), mod_segments) + h = _mod_scale_shift(self.norm2(x), shift_mlp, scale_mlp, mod_segments) + return _mod_gate(x, gate_mlp, self.mlp(h), mod_segments) + + +class FinalLayer(nn.Module): + def __init__(self, hidden, t_dim, video_dim, audio_dim, eps, apply_silu=True, adaln_dtype=None, + dtype=None, device=None, operations=None): + super().__init__() + self.norm = operations.RMSNorm(hidden, eps=eps, dtype=dtype, device=device) + self.adaln_proj = AdalnProj(t_dim, hidden, 2, 1, apply_silu=apply_silu, + dtype=adaln_dtype if adaln_dtype is not None else dtype, + device=device, operations=operations) + # output heads are the checkpoint's fp32 island; norm/adaln are stored at model dtype + self.video_out = operations.Linear(hidden, video_dim, bias=True, dtype=torch.float32, device=device) + self.audio_out = operations.Linear(hidden, audio_dim, bias=True, dtype=torch.float32, device=device) + + def forward(self, x, t_emb, video_seg, audio_seg): + # video_seg / audio_seg: (start, stop, timestep_row) of the target streams + shift, scale = self.adaln_proj(t_emb) + va, vb, vrow = video_seg + aa, ab, arow = audio_seg + hv = (self.norm(x[va:vb]) * (1.0 + scale[vrow]) + shift[vrow]).to(torch.float32) + ha = (self.norm(x[aa:ab]) * (1.0 + scale[arow]) + shift[arow]).to(torch.float32) + return self.video_out(hv), self.audio_out(ha) + + +class PackedLayout: + """Static packed-sequence structure for one shape/conditioning signature.""" + + def __init__(self, text_len, latent_t, latent_h, latent_w, audio_t, keyframes=None, refs=None, frame_count=None): + frame, w_grid = _frame_grid(latent_h, latent_w) + frame_rows = frame.shape[0] + + segments = [("text", text_len)] # (kind, n_rows) + g = torch.zeros(text_len, 3, dtype=torch.float64) + g[:, 0] = torch.arange(text_len, dtype=torch.float64) + pos = [g] # per segment: [n, 3] float64 (t, h, w) + + img_pos, img_update = [], [] + audio_pos, audio_update = [], [] + cursor = text_len + row = text_len + + if keyframes: + # fl2va: keyframe cond rows right after text, sharing the target spatial grid + for kf in keyframes: + pixel_index = kf["resolved_frame_index"] + if pixel_index == 0: + cond_t = float(text_len) + elif frame_count is not None and pixel_index == frame_count - 1: + cond_t = float(text_len) + sum(_video_t_spans(latent_t)) - FRAME_RESCALE + else: + raise ValueError("only first/last keyframe anchors are supported") + g = torch.empty(frame_rows, 3, dtype=torch.float64) + g[:, 0] = cond_t + g[:, 1:] = frame + segments.append(("cond", frame_rows)) + pos.append(g) + img_pos.append(torch.arange(row, row + frame_rows)) + img_update.append(torch.zeros(frame_rows, dtype=torch.bool)) + row += frame_rows + + target_audio_w = (float(w_grid[0]), float(w_grid[-1])) + if refs: + cursor = float(text_len) + for blk in refs: + kind = blk["kind"] + if kind == "image": + r_frame, _ = _frame_grid(blk["latent_h"], blk["latent_w"]) + n = r_frame.shape[0] + g = torch.empty(n, 3, dtype=torch.float64) + g[:, 0] = cursor + g[:, 1:] = r_frame + segments.append(("ref_img", n)) + pos.append(g) + img_pos.append(torch.arange(row, row + n)) + img_update.append(torch.zeros(n, dtype=torch.bool)) + row += n + cursor += 1.0 + elif kind == "audio": + rt = blk["ref_audio_t"] + if rt > 0: + segments.append(("ref_audio", rt * 2)) + pos.append(_audio_grid(cursor, rt, *target_audio_w)) + audio_pos.append(torch.arange(row, row + rt * 2)) + audio_update.append(torch.zeros(rt * 2, dtype=torch.bool)) + row += rt * 2 + cursor += float(rt) + elif kind in ("video", "video_audio"): + # the block's audio rows pack immediately before its video + # rows, both sharing the cursor origin + rt = blk["ref_audio_t"] + vt = blk["latent_t"] + r_frame, r_w_grid = _frame_grid(blk["latent_h"], blk["latent_w"]) + if rt > 0: + segments.append(("ref_audio", rt * 2)) + pos.append(_audio_grid(cursor, rt, float(r_w_grid[0]), float(r_w_grid[-1]))) + audio_pos.append(torch.arange(row, row + rt * 2)) + audio_update.append(torch.zeros(rt * 2, dtype=torch.bool)) + row += rt * 2 + n = vt * r_frame.shape[0] + segments.append(("ref_img", n)) + pos.append(_video_grid(vt, r_frame, cursor)) + img_pos.append(torch.arange(row, row + n)) + img_update.append(torch.zeros(n, dtype=torch.bool)) + row += n + cursor += max(float(rt), sum(_video_t_spans(vt))) + + # target audio then target video, always the last two segments + segments.append(("audio", audio_t * 2)) + pos.append(_audio_grid(cursor, audio_t, *target_audio_w)) + audio_pos.append(torch.arange(row, row + audio_t * 2)) + audio_update.append(torch.ones(audio_t * 2, dtype=torch.bool)) + row += audio_t * 2 + + n_video = latent_t * frame_rows + segments.append(("video", n_video)) + pos.append(_video_grid(latent_t, frame, cursor)) + img_pos.append(torch.arange(row, row + n_video)) + img_update.append(torch.ones(n_video, dtype=torch.bool)) + row += n_video + + self.seq_len = row + self.position_ids = torch.cat(pos) # [S, 3] float64 + self.img_pos = torch.cat(img_pos) + self.img_update = torch.cat(img_update) + self.audio_pos = torch.cat(audio_pos) + self.audio_update = torch.cat(audio_update) + self.signature = (text_len, latent_t, latent_h, latent_w, audio_t) + # contiguous segment table (start, stop, kind) + # kinds: text / cond / ref_img / ref_audio / audio / video + # the packed sequence is uniform per segment in (modality tag, timestep class), + # except the text span (tag runs resolved at forward time from the presentation tags) + seg_abs = [] + off = 0 + for kind, n in segments: + seg_abs.append((off, off + n, kind)) + off += n + self.segments = seg_abs + + +class MiniMaxH3Model(nn.Module): + def __init__(self, hidden_size=5376, num_layers=50, token_refiner_num_layers=2, + num_attention_heads=56, attention_head_dim=128, ffn_hidden_size=14336, + latents_dim=24, audio_latents_dim=32, patch_size=(1, 2, 2), text_dim=5120, + timestep_input_dim=256, time_embed_hidden_size=5376, time_embed_dim=2688, + rope_inv_freq_len=16, norm_eps=1e-5, qk_norm_eps=1e-5, final_norm_eps=1e-5, + sigma_shift_video=12.0, sigma_shift_audio=3.0, + adaln_curve_grid=None, + image_model=None, dtype=None, device=None, operations=None, **kwargs): + super().__init__() + self.dtype = dtype + self.hidden_size = hidden_size + self.patch_size = tuple(patch_size) + self.latents_dim = latents_dim + self.audio_latents_dim = audio_latents_dim + self.sigma_shift_video = sigma_shift_video + self.sigma_shift_audio = sigma_shift_audio + self.use_adaln_curves = adaln_curve_grid is not None + # curve-form checkpoints replace the time embedder and full-width adaln weights with a small shared basis of the time-embedding curve + curve = {"apply_silu": not self.use_adaln_curves, + "adaln_dtype": torch.float32 if self.use_adaln_curves else dtype} + video_patch_dim = latents_dim * self.patch_size[0] * self.patch_size[1] * self.patch_size[2] + + self.video_patch_proj = operations.Linear(video_patch_dim, hidden_size, bias=True, dtype=torch.float32, device=device) + self.audio_patch_proj = operations.Linear(audio_latents_dim, hidden_size, bias=True, dtype=torch.float32, device=device) + self.condition_proj = operations.Linear(text_dim, hidden_size, bias=True, dtype=dtype, device=device) + if self.use_adaln_curves: + self.register_buffer("adaln_t_table", torch.empty(adaln_curve_grid, time_embed_dim, dtype=torch.float32)) + else: + self.time_embedder = TimeEmbedder(timestep_input_dim, time_embed_hidden_size, time_embed_dim, + dtype=torch.float32, device=device, operations=operations) + self.rope = nn.Module() + self.rope.register_buffer("inv_freq", torch.empty(rope_inv_freq_len, dtype=torch.float32)) + self.token_refiner = TokenRefiner(token_refiner_num_layers, hidden_size, num_attention_heads, + attention_head_dim, ffn_hidden_size, norm_eps, qk_norm_eps, + final_norm_eps, dtype=dtype, device=device, operations=operations) + self.blocks = nn.ModuleList([ + DiTBlock(hidden_size, num_attention_heads, attention_head_dim, ffn_hidden_size, + time_embed_dim, norm_eps, qk_norm_eps, **curve, dtype=dtype, device=device, operations=operations) + for _ in range(num_layers)]) + self.final_layer = FinalLayer(hidden_size, time_embed_dim, video_patch_dim, audio_latents_dim, + final_norm_eps, **curve, dtype=dtype, device=device, operations=operations) + + def preprocess_text_embeds(self, text_states): + """[B, L, text_dim] Qwen states -> [B, L, hidden] refined text embeds.""" + if text_states.shape[-1] == self.hidden_size: + return text_states + return self.token_refiner(self.condition_proj(text_states[0])).unsqueeze(0) + + def rope_freqs(self, position_ids, device): + # [S, 3] float64 -> [S, 96] fp32 + pos = position_ids.to(torch.float32).to(device) + inv = comfy.model_management.cast_to(self.rope.inv_freq, device=device) + per_axis = pos.unsqueeze(-1) * inv.view(1, 1, -1) # [S, 3, 16] + t_f, h_f, w_f = per_axis.unbind(dim=1) + half = torch.cat((t_f, h_f, w_f), dim=-1) # [S, 48] + return torch.cat((half, half), dim=-1) # [S, 96] + + def _cond_video_rows(self, payload, device): + """Concatenated visual condition rows (normalized latents -> patchified), with condition noise augmentation.""" + rows = [] + aug = payload.get("visual_cond_noise_aug", VISUAL_COND_TIMESTEP) + seed = int(payload.get("seed", 0)) + # every condition intentionally restarts the same RNG stream + for z in payload.get("cond_video_latents", []): + r = patchify_video(z.to(torch.float32), self.patch_size) + if aug < 1.0: + gen = torch.Generator("cpu").manual_seed(seed) + noise = torch.randn(r.shape, generator=gen, dtype=torch.float32) + r = aug * r + (1.0 - aug) * noise.to(r.device) + rows.append(r.to(device)) + return torch.cat(rows, dim=0) if rows else None + + def _cond_audio_rows(self, payload, device): + rows = [] + aug = payload.get("audio_cond_noise_aug", AUDIO_COND_TIMESTEP) + seed = int(payload.get("seed", 0)) + 1 + for z in payload.get("cond_audio_latents", []): + r = pack_audio(z.to(torch.float32)) + if aug < 1.0: + gen = torch.Generator("cpu").manual_seed(seed) + noise = torch.randn(r.shape, generator=gen, dtype=torch.float32) + r = aug * r + (1.0 - aug) * noise.to(r.device) + rows.append(r.to(device)) + return torch.cat(rows, dim=0) if rows else None + + def forward(self, x, timestep, context, transformer_options={}, minimax_payload=None, **kwargs): + # the sampler carries the audio as (sigma_v / sigma_a) * x_audio; undo it outside + # the wrappers so they and the network see the stream's own latent and velocity + scale = float((minimax_payload or {}).get("audio_scale", 1.0)) + audio_src = x[1] + if scale != 1.0: + shift_v = float(transformer_options.get("minimax_h3_sigma_shift_video", self.sigma_shift_video)) + shift_a = float(transformer_options.get("minimax_h3_sigma_shift_audio", self.sigma_shift_audio)) + sigma_v = (timestep.flatten()[0] / 1000.0).float().clamp(min=1e-6) + sigma_a = time_shift_sigma(sigma_v, shift_v, shift_a) + carry = (sigma_a / sigma_v).to(audio_src.dtype) + x = [x[0], audio_src * carry] + + out = comfy.patcher_extension.WrapperExecutor.new_class_executor( + self._forward, + self, + comfy.patcher_extension.get_all_wrappers(comfy.patcher_extension.WrappersMP.DIFFUSION_MODEL, transformer_options) + ).execute(x, timestep, context, transformer_options, minimax_payload=minimax_payload, **kwargs) + + if scale != 1.0: + # d/d(sigma_v) of the carried variable + out[1] = ((1.0 - scale) * (audio_src * carry) + + (1.0 + (scale - 1.0) * sigma_a).to(out[1].dtype) * out[1]) + return out + + def _forward(self, x, timestep, context, transformer_options={}, minimax_payload=None, **kwargs): + video_x, audio_x = x[0], x[1] + orig_t, orig_h, orig_w = video_x.shape[2], video_x.shape[3], video_x.shape[4] + video_x = comfy.ldm.common_dit.pad_to_patch_size(video_x, self.patch_size) + if video_x.shape[0] != 1: + raise ValueError("MiniMax H3 supports batch size 1") + payload = minimax_payload or {} + device = video_x.device + dtype = context.dtype # compute dtype + + latent_t, lat_h, lat_w = video_x.shape[2], video_x.shape[3], video_x.shape[4] + audio_t = audio_x.shape[-1] + text_len = context.shape[1] + # extra_conds prebuilds the layout once per sampling run + layout = payload.get("layout") + if layout is None or layout.signature != (text_len, latent_t, lat_h, lat_w, audio_t): + layout = PackedLayout(text_len, latent_t, lat_h, lat_w, audio_t, + keyframes=payload.get("keyframes"), + refs=payload.get("refs"), + frame_count=payload.get("frame_count")) + + # model_base passes model_sampling.timestep(sigma) = sigma * 1000 + shift_v = float(transformer_options.get("minimax_h3_sigma_shift_video", self.sigma_shift_video)) + shift_a = float(transformer_options.get("minimax_h3_sigma_shift_audio", self.sigma_shift_audio)) + sigma_v = (timestep.flatten()[0] / 1000.0).float().clamp(min=1e-6) + t_v = float(1.0 - sigma_v) + t_a = float(1.0 - time_shift_sigma(sigma_v, shift_v, shift_a)) + + # distinct timesteps are known analytically: text/pad follow video, cond rows pin near 1 + vis_aug = float(payload.get("visual_cond_noise_aug", VISUAL_COND_TIMESTEP)) + aud_aug = float(payload.get("audio_cond_noise_aug", AUDIO_COND_TIMESTEP)) + has_vis_cond = any(k in ("cond", "ref_img") for _, _, k in layout.segments) + has_aud_cond = any(k == "ref_audio" for _, _, k in layout.segments) + seg_t = {"text": t_v, "video": t_v, "audio": t_a, + "cond": max(t_v, vis_aug), "ref_img": max(t_v, vis_aug), + "ref_audio": max(t_a, aud_aug)} + unique_t = sorted({t_v, t_a} | ({seg_t["cond"]} if has_vis_cond else set()) + | ({seg_t["ref_audio"]} if has_aud_cond else set())) + t_row = {t: i for i, t in enumerate(unique_t)} + seg_tag = {"text": 1, "video": 0, "audio": 2, "cond": 0, "ref_img": 0, "ref_audio": 2} + + text_tags = payload.get("text_token_tags") + mod_segments = [] + for a, b, kind in layout.segments: + row_base = t_row[seg_t[kind]] * 3 + if kind == "text" and text_tags is not None: + # the presentation text span mixes tags (vision pads carry the video modality) split into tag runs + tags = text_tags.view(-1).tolist() + run_start = 0 + for i in range(1, b - a + 1): + if i == b - a or tags[i] != tags[run_start]: + mod_segments.append((a + run_start, a + i, row_base + int(tags[run_start]))) + run_start = i + else: + mod_segments.append((a, b, row_base + seg_tag[kind])) + + # embed + img_update = layout.img_update.to(device) + audio_update = layout.audio_update.to(device) + video_rows = patchify_video(video_x.to(torch.float32), self.patch_size) + audio_rows = pack_audio(audio_x.to(torch.float32)) + cond_video_rows = self._cond_video_rows(payload, device) + cond_audio_rows = self._cond_audio_rows(payload, device) + + all_video_rows = video_rows + if cond_video_rows is not None: + all_video_rows = torch.empty(img_update.shape[0], video_rows.shape[1], dtype=torch.float32, device=device) + all_video_rows[~img_update] = cond_video_rows + all_video_rows[img_update] = video_rows + all_audio_rows = audio_rows + if cond_audio_rows is not None: + all_audio_rows = torch.empty(audio_update.shape[0], audio_rows.shape[1], dtype=torch.float32, device=device) + all_audio_rows[~audio_update] = cond_audio_rows + all_audio_rows[audio_update] = audio_rows + + video_embed = self.video_patch_proj(all_video_rows).to(dtype) + audio_embed = self.audio_patch_proj(all_audio_rows).to(dtype) + text_states = context[0] + if text_states.shape[-1] != self.hidden_size: + text_states = self.token_refiner(self.condition_proj(text_states), + transformer_options=transformer_options) + + # segments are contiguous: assemble by slices, embed rows follow segment order + h = torch.empty(layout.seq_len, self.hidden_size, dtype=dtype, device=device) + voff = aoff = 0 + for a, b, kind in layout.segments: + n = b - a + if kind == "text": + h[a:b] = text_states + elif kind in ("cond", "ref_img", "video"): + h[a:b] = video_embed[voff:voff + n] + voff += n + else: # ref_audio / audio + h[a:b] = audio_embed[aoff:aoff + n] + aoff += n + + t_vals = torch.tensor(unique_t, dtype=torch.float32, device=device) + if self.use_adaln_curves: + # adaln projections consume interpolated coordinates of the time-embedding curve + table = comfy.model_management.cast_to(self.adaln_t_table, device=device) + pos = t_vals.clamp(0.0, 1.0) * (table.shape[0] - 1) # t in [0,1] -> fractional grid index, out-of-range t clamps to the curve ends + i0 = pos.floor().long().clamp(max=table.shape[0] - 2) # lower grid row, max-clamp keeps t=1.0 on the last interval instead of reading past the table + t_emb = torch.lerp(table[i0], table[i0 + 1], (pos - i0).unsqueeze(1)) # blend the two rows by the fractional part + else: + t_emb = self.time_embedder(t_vals).to(dtype) + + # rotation table computed once per forward, consumed by the kitchen split-half rope + rope_freqs = rope_rotation_table(self.rope_freqs(layout.position_ids, device), dtype) + + # blocks + patches_replace = transformer_options.get("patches_replace", {}) + blocks_replace = patches_replace.get("dit", {}) + prefetch_queue = comfy.model_prefetch.make_prefetch_queue(list(self.blocks), device, transformer_options) + for i, block in enumerate(self.blocks): + comfy.model_prefetch.prefetch_queue_pop(prefetch_queue, device, block) + if ("double_block", i) in blocks_replace: + def block_wrap(args): + return {"img": block(args["img"], args["t_emb"], args["mod_segments"], args["rope_freqs"], + transformer_options=args["transformer_options"])} + h = blocks_replace[("double_block", i)]( + {"img": h, "t_emb": t_emb, "mod_segments": mod_segments, "rope_freqs": rope_freqs, + "transformer_options": transformer_options}, + {"original_block": block_wrap})["img"] + else: + h = block(h, t_emb, mod_segments, rope_freqs, transformer_options=transformer_options) + if prefetch_queue is not None: + comfy.model_prefetch.prefetch_queue_pop(prefetch_queue, device, None) + + # target streams are single contiguous segments (audio then video, last two) + video_seg = next((a, b, t_row[seg_t["video"]]) for a, b, k in layout.segments if k == "video") + audio_seg = next((a, b, t_row[seg_t["audio"]]) for a, b, k in layout.segments if k == "audio") + v, a = self.final_layer(h, t_emb, video_seg, audio_seg) + + video_out = unpatchify_video(v, latent_t, lat_h // 2, lat_w // 2, self.latents_dim, self.patch_size) + video_out = video_out[:, :, :orig_t, :orig_h, :orig_w] + audio_out = unpack_audio(a) + + return [-video_out.to(video_x.dtype), -audio_out.to(audio_x.dtype)] diff --git a/comfy/ldm/minimax/vae.py b/comfy/ldm/minimax/vae.py new file mode 100644 index 000000000..65d06f3e9 --- /dev/null +++ b/comfy/ldm/minimax/vae.py @@ -0,0 +1,696 @@ +# MiniMax H3 video VAE: 3D causal CNN encoder + ViT3D decoder. + +import math + +import torch +import torch.nn as nn +import torch.nn.functional as F + +import comfy.ops +import comfy.quant_ops +import comfy.rmsnorm +from comfy.ldm.modules.attention import optimized_attention + +ops = comfy.ops.disable_weight_init + +IMAGENET_MEAN = (0.485, 0.456, 0.406) +IMAGENET_STD = (0.229, 0.224, 0.225) + +LATENTS_MEAN = [ + 0.858090341091156, -0.9606591463088989, 1.0661640167236328, -0.5090325474739075, + -0.2727581858634949, -1.3675414323806763, -0.2553254961967468, -0.26907554268836975, + -0.5376840829849243, -0.0464097298681736, 0.6657370328903198, 0.19690127670764923, + -0.5460608005523682, -0.4035342037677765, -0.23683024942874908, 0.25928452610969543, + -0.30133944749832153, 0.211341992020607, -1.1206848621368408, 0.3581933379173279, + -0.04225143790245056, 0.2604829967021942, 0.22864092886447906, 0.7056031823158264, +] + +LATENTS_STD = [ + 1.2223774194717407, 1.2767263650894165, 1.68317747116088865, 1.7549455165863037, + 1.5636216402053833, 2.194143533706665, 0.96531379222869875, 1.05698859691619875, + 0.841948926448822, 0.7729952931404114, 1.8955937623977661, 0.946841835975647, + 0.7996809482574463, 0.44988900423049925, 0.7197399735450745, 0.69362932443618775, + 2.961095094680786, 2.7694199085235595, 3.0496184825897215, 2.1088054180145265, + 3.276226282119751, 3.1627357006073, 2.28168129920959475, 2.6127843856811525, +] + + +# 3D causal CNN encoder + +class CausalConv3d(ops.Conv3d): + # Reflect spatial padding, causal (zeros, front-only) temporal padding. + def __init__(self, in_channels, out_channels, kernel_size, stride=1, padding=0): + super().__init__(in_channels, out_channels, kernel_size=kernel_size, stride=stride) + self.causal_padding = (padding,) * 3 if isinstance(padding, int) else tuple(padding) + + def forward(self, x): + if sum(self.causal_padding) == 0: + return super().forward(x) + + x = F.pad(x, (self.causal_padding[2], self.causal_padding[2], self.causal_padding[1], self.causal_padding[1], 0, 0), mode="reflect") + if x.shape[2] == 1: + # single frame: the causal front padding is all zeros truncate the temporal taps instead of convolving zero frames + return super().forward(x, autopad="causal_zero") + x = F.pad(x, (0, 0, 0, 0, self.causal_padding[0] * 2, 0), mode="constant") + return super().forward(x) + + +class TemporalIsolatedGroupNorm(ops.GroupNorm): + # GroupNorm with statistics computed per frame (time merged into batch). + def forward(self, x): + if x.dim() == 5: + b, c, t, h, w = x.shape + x = x.permute(0, 2, 1, 3, 4).contiguous().view(b * t, c, 1, h, w) + x = super().forward(x) + return x.view(b, t, c, h, w).permute(0, 2, 1, 3, 4).contiguous() + return super().forward(x) + + +def group_norm_3d(num_channels): + return TemporalIsolatedGroupNorm(num_groups=32, num_channels=num_channels, eps=1e-6, affine=True) + + +class Downsample3D(nn.Module): + def __init__(self, in_channels, out_channels, time_stride=1, space_stride=2): + super().__init__() + self.space_stride = space_stride + self.conv = CausalConv3d( + in_channels, + out_channels, + kernel_size=3, + padding=(1, 0, 0), + stride=(time_stride, space_stride, space_stride), + ) + + def forward(self, x): + if self.space_stride == 2: + x = F.pad(x, (0, 1, 0, 1, 0, 0), mode="reflect") + return self.conv(x) + + +class ResnetBlock3D(nn.Module): + def __init__(self, in_channels, out_channels=None): + super().__init__() + self.in_channels = in_channels + out_channels = in_channels if out_channels is None else out_channels + self.out_channels = out_channels + + self.norm1 = group_norm_3d(in_channels) + self.norm2 = group_norm_3d(out_channels) + self.conv1 = CausalConv3d(in_channels, out_channels, kernel_size=3, padding=1) + self.conv2 = CausalConv3d(out_channels, out_channels, kernel_size=3, padding=1) + if in_channels != out_channels: + self.nin_shortcut = CausalConv3d(in_channels, out_channels, kernel_size=1) + + def forward(self, x): + h = self.conv1(F.silu(self.norm1(x), inplace=True)) + h = self.conv2(F.silu(self.norm2(h), inplace=True)) + if self.in_channels != self.out_channels: + x = self.nin_shortcut(x) + return h.add_(x) + + +class EncoderFCN3D(nn.Module): + def __init__(self, ch, ch_mult, space_down, time_down, num_res_blocks, in_channels, z_channels, double_z=True): + super().__init__() + self.num_levels = len(ch_mult) + if isinstance(num_res_blocks, int): + num_res_blocks = [num_res_blocks] * self.num_levels + self.num_res_blocks = num_res_blocks + + block_mid = [ch * ch_mult[i] for i in range(self.num_levels)] + block_in = [block_mid[0]] + block_mid[:-1] + block_out = block_mid + + self.conv_in = CausalConv3d(in_channels, block_in[0], kernel_size=3, padding=1) + + self.down = nn.ModuleList() + for i_level in range(self.num_levels): + down = nn.Module() + down.block = nn.ModuleList() + for i in range(self.num_res_blocks[i_level]): + down.block.append( + ResnetBlock3D( + in_channels=block_in[i_level] if i == 0 else block_mid[i_level], + out_channels=block_mid[i_level], + ) + ) + if space_down[i_level] * time_down[i_level] > 1: + down.downsample = Downsample3D( + block_mid[i_level], + block_out[i_level], + time_stride=time_down[i_level], + space_stride=space_down[i_level], + ) + self.down.append(down) + + self.norm_out = group_norm_3d(block_out[-1]) + self.conv_out = CausalConv3d( + block_out[-1], + 2 * z_channels if double_z else z_channels, + kernel_size=3, + padding=1, + ) + + def forward(self, x): + h = self.conv_in(x) + for i_level in range(self.num_levels): + for i_block in range(self.num_res_blocks[i_level]): + h = self.down[i_level].block[i_block](h) + if hasattr(self.down[i_level], "downsample"): + h = self.down[i_level].downsample(h) + h = F.silu(self.norm_out(h)) + return self.conv_out(h) + + +# ViT3D decoder + +def create_token_ids(patch_dims, device, dtype): + coords_list = [] + for dim_size in patch_dims: + coords = torch.arange(0.5, dim_size, dtype=dtype, device=device) + coords = coords / dim_size + coords = 2.0 * coords - 1.0 + coords_list.append(coords) + coords = torch.stack(torch.meshgrid(*coords_list, indexing="ij"), dim=-1) + return coords.flatten(0, len(patch_dims) - 1).unsqueeze(0) + + +class RotaryEmbeddingND(nn.Module): + def __init__(self, dim, rotary_base=100.0, n_dim=3): + super().__init__() + self.n_dim = n_dim + self.angle_scale = 2.0 * math.pi + inv_freq = 1 / rotary_base ** torch.arange(0, 1, 2 * n_dim / dim, dtype=torch.float32) + self.register_buffer("inv_freq", inv_freq, persistent=False) + + def forward(self, img_ids): + # [B, S, n_dim] -> [B, S, 1, pairs, 2, 2] rotation table for the kitchen split-half rope + angles = ( + self.angle_scale + * img_ids[:, :, :, None].float() + * self.inv_freq.to(img_ids.device)[None, None, None, :] + ) + angles = angles.flatten(2, 3) + c, s = torch.cos(angles), torch.sin(angles) + table = torch.stack([c, -s, s, c], dim=-1).reshape(*angles.shape[:2], 1, angles.shape[-1], 2, 2) + return table.to(img_ids.dtype) + + +class FeedForward(nn.Module): + # Gated SiLU FFN. + def __init__(self, dim, mult=4, bias=True, operations=ops): + super().__init__() + inner_dim = dim * mult + self.w1 = operations.Linear(dim, inner_dim * 2, bias=bias) + self.w2 = operations.Linear(inner_dim, dim, bias=bias) + + def forward(self, x): + gate, x = self.w1(x).chunk(2, dim=-1) + return self.w2(F.silu(gate).mul_(x)) + + +class Attention(nn.Module): + def __init__(self, heads, dim_head, bias=True, eps=1e-5, operations=ops): + super().__init__() + self.dim_head = dim_head + self.heads = heads + inner_dim = dim_head * heads + self.norm_q = ops.RMSNorm(dim_head, eps=eps, elementwise_affine=False) + self.norm_k = ops.RMSNorm(dim_head, eps=eps, elementwise_affine=False) + self.to_qkv = operations.Linear(inner_dim, inner_dim * 3, bias=bias) + self.to_out = operations.Linear(inner_dim, inner_dim, bias=bias) + + def forward(self, x, rotary_pos_emb=None): + batch_size, seq_len, _ = x.shape + + qkv = self.to_qkv(x) + qkv = qkv.view(batch_size, seq_len, -1, 3 * self.dim_head) + query, key, value = torch.chunk(qkv, 3, dim=-1) + + query = comfy.rmsnorm.rms_norm(query, self.norm_q.weight, self.norm_q.eps) + key = comfy.rmsnorm.rms_norm(key, self.norm_k.weight, self.norm_k.eps) + + if rotary_pos_emb is not None: + rot = rotary_pos_emb.shape[-3] * 2 + query[..., :rot], key[..., :rot] = comfy.quant_ops.ck.apply_rope_split_half( + query[..., :rot], key[..., :rot], rotary_pos_emb) + + out = optimized_attention(query.transpose(1, 2), key.transpose(1, 2), value.transpose(1, 2), + self.heads, skip_reshape=True).nan_to_num_(0.0) + return self.to_out(out) + + +class TransformerBlock(nn.Module): + def __init__(self, heads, dim_head, bias=True, eps=1e-5, operations=ops): + super().__init__() + dim = heads * dim_head + self.norm1 = ops.RMSNorm(dim, elementwise_affine=True, eps=eps) + self.attn = Attention(heads=heads, dim_head=dim_head, bias=bias, eps=eps, operations=operations) + self.scale1 = nn.Parameter(torch.empty(dim)) + self.norm2 = ops.RMSNorm(dim, elementwise_affine=True, eps=eps) + self.ff = FeedForward(dim=dim, bias=bias, operations=operations) + self.scale2 = nn.Parameter(torch.empty(dim)) + + def forward(self, x, rotary_pos_emb=None): + x = x.addcmul_(self.attn(comfy.rmsnorm.rms_norm(x, self.norm1.weight, self.norm1.eps), rotary_pos_emb), comfy.ops.cast_to_input(self.scale1, x)) + return x.addcmul_(self.ff(comfy.rmsnorm.rms_norm(x, self.norm2.weight, self.norm2.eps)), comfy.ops.cast_to_input(self.scale2, x)) + + +class ViT3DDecoder(nn.Module): + def __init__(self, patch_size=16, patch_size_t=4, in_channels=24, out_channels=3, num_layers=36, heads=32, dim_head=64, rope_theta=100.0, + rope_dim_ratio=0.75, bias=True, eps=1e-5, num_register_tokens=4, operations=ops): + super().__init__() + dim = heads * dim_head + self.patch_size = patch_size + self.patch_size_t = patch_size_t + self.out_channels = out_channels + self.num_register_tokens = num_register_tokens + + self.pos_embed = RotaryEmbeddingND(int(dim_head * rope_dim_ratio), rope_theta, n_dim=3) + self.x_embedder = ops.Linear(in_channels, dim) + self.register_tokens = nn.Parameter(torch.empty(1, num_register_tokens, dim)) + # unused at inference; kept so the checkpoint loads without leftover keys + self.register_buffer("mask_token", torch.empty(1, 1, dim)) + + self.transformer_blocks = nn.ModuleList( + [TransformerBlock(heads=heads, dim_head=dim_head, bias=bias, eps=eps, operations=operations) + for _ in range(num_layers)] + ) + + self.norm_out = ops.LayerNorm(dim, elementwise_affine=True, eps=eps) + self.proj_out = ops.Linear(dim, out_channels * patch_size_t * patch_size * patch_size) + + def forward(self, x): + B, C, latent_T, latent_H, latent_W = x.shape + + h = self.x_embedder(x.flatten(2).transpose(1, 2)) # [B, T*H*W, C] + + num_patches = h.shape[1] + num_suffix = 1 + self.num_register_tokens + + h = torch.cat([h, comfy.ops.cast_to_input(self.register_tokens, h).expand(B, -1, -1), torch.zeros_like(h[:, 0:1, :])], dim=1) + + img_ids = create_token_ids((latent_T, latent_H, latent_W), x.device, x.dtype).expand(B, -1, -1) + suffix_ids = torch.zeros((B, num_suffix, 3), device=x.device, dtype=img_ids.dtype) + img_ids = torch.cat([img_ids, suffix_ids], dim=1) + + rotary_pos_emb = self.pos_embed(img_ids) + + for block in self.transformer_blocks: + h = block(h, rotary_pos_emb) + + output = self.proj_out(self.norm_out(h)) + + output = output[:, :num_patches, :] + + output = output.view( + B, latent_T, latent_H, latent_W, + self.out_channels, self.patch_size_t, self.patch_size, self.patch_size, + ) + output = output.permute(0, 4, 1, 5, 2, 6, 3, 7).contiguous() + output = output.reshape( + B, self.out_channels, + latent_T * self.patch_size_t, + latent_H * self.patch_size, + latent_W * self.patch_size, + ) + return output + + +# Full VAE + +class MiniMaxH3VideoVAE(nn.Module): + def __init__( + self, + in_channels=3, + out_ch=3, + ch=128, + embed_dim=24, + z_channels=24, + ch_mult=(1, 2, 2, 4, 4, 8), + num_res_blocks=2, + space_down=(2, 2, 2, 2, 1, 1), + time_down=(1, 2, 2, 1, 1, 1), + clip_length=17, + token_drop=3, + tile_size=256, + tile_overlap_min=64, + tiling=True, + operations=ops, + ): + super().__init__() + self.vae_ratio = int(math.prod(space_down)) + self.vae_ratio_t = int(math.prod(time_down)) + + # temporal chunking parameters + self.clip_length = clip_length + self.token_drop = token_drop + self.frame_pre_padding = (-clip_length) % self.vae_ratio_t + self.tokens_chunk_size = math.ceil(clip_length / self.vae_ratio_t) + self.token_overlap = (-token_drop) % self.tokens_chunk_size + self.frame_overlap = max(self.token_overlap * self.vae_ratio_t - self.frame_pre_padding, 0) + + # spatial tiling parameters + self.tiling = tiling + self.tile_size = tile_size + self.tile_overlap_min = tile_overlap_min + + self.encoder = EncoderFCN3D( + ch=ch, + ch_mult=list(ch_mult), + space_down=list(space_down), + time_down=list(time_down), + num_res_blocks=num_res_blocks, + in_channels=in_channels, + z_channels=z_channels, + double_z=True, + ) + self.quant_conv = ops.Conv3d(z_channels * 2, 2 * embed_dim, 1) + self.post_quant_conv = ops.Conv3d(embed_dim, z_channels, 1) + self.decoder = ViT3DDecoder( + patch_size=self.vae_ratio, + patch_size_t=self.vae_ratio_t, + in_channels=z_channels, + out_channels=out_ch, + operations=operations, + ) + + self.register_buffer("latents_mean", torch.tensor(LATENTS_MEAN)) + self.register_buffer("latents_std", torch.tensor(LATENTS_STD)) + self.register_buffer("pixel_mean", torch.tensor(IMAGENET_MEAN).view(1, 3, 1, 1, 1), persistent=False) + self.register_buffer("pixel_std", torch.tensor(IMAGENET_STD).view(1, 3, 1, 1, 1), persistent=False) + + # single-shot forward + + def _encode_moments(self, x): + return self.quant_conv(self.encoder(x)) + + def _decode_pixels(self, z): + return self.decoder(self.post_quant_conv(z)) + + def _adaptive_encode(self, x): + if self.tiling: + return self.tiled_encode(x) + return self._encode_moments(x) + + def _adaptive_decode(self, z): + if self.tiling: + return self.tiled_decode(z) + return self._decode_pixels(z) + + # spatial tiling + + def split_tiles(self, input_len): + tile_size = self.tile_size + if tile_size >= input_len: + return [0], [input_len], [] + + N = math.ceil(input_len / tile_size) + while True: + overlaps = [self.tile_overlap_min] * (N - 1) + remaining = tile_size * N - sum(overlaps) - input_len + if remaining < 0: + N += 1 + else: + break + + remaining_units = remaining // self.vae_ratio + for i in range(remaining_units): + overlaps[i % (N - 1)] += self.vae_ratio + + tile_start_idx = [0] + for i in range(N - 1): + tile_start_idx.append(tile_start_idx[-1] + tile_size - overlaps[i]) + + return tile_start_idx, [tile_size] * N, overlaps + + def blend(self, a, b, blend_extent, dim): + blend_extent = min(a.shape[dim], b.shape[dim], blend_extent) + + positions = torch.arange(blend_extent, device=b.device, dtype=b.dtype) + weight_a = 1 - positions / blend_extent + weight_b = positions / blend_extent + + shape = [1] * a.ndim + shape[dim] = blend_extent + weight_a = weight_a.view(shape) + weight_b = weight_b.view(shape) + + slice_a = [slice(None)] * a.ndim + slice_a[dim] = slice(-blend_extent, None) + slice_b = [slice(None)] * b.ndim + slice_b[dim] = slice(0, blend_extent) + + blended = a[tuple(slice_a)] * weight_a + b[tuple(slice_b)] * weight_b + + if blend_extent < b.shape[dim]: + slice_b_rest = [slice(None)] * b.ndim + slice_b_rest[dim] = slice(blend_extent, None) + return torch.cat([blended, b[tuple(slice_b_rest)]], dim=dim) + return blended + + def tiled_encode(self, x): + height, width = x.shape[-2], x.shape[-1] + y_idx, y_len, y_overlap = self.split_tiles(height) + x_idx, x_len, x_overlap = self.split_tiles(width) + + rows = [] + for i_pos, i_len in zip(y_idx, y_len): + row = [] + for j_pos, j_len in zip(x_idx, x_len): + tile = x[..., i_pos:i_pos + i_len, j_pos:j_pos + j_len] + row.append(self._encode_moments(tile)) + rows.append(row) + + latent_y_overlap = [o // self.vae_ratio for o in y_overlap] + latent_x_overlap = [o // self.vae_ratio for o in x_overlap] + + result_rows = [] + for i, row in enumerate(rows): + result_row = [] + for j, tile in enumerate(row): + if i > 0: + tile = self.blend(rows[i - 1][j], tile, latent_y_overlap[i - 1], dim=-2) + if j > 0: + tile = self.blend(row[j - 1], tile, latent_x_overlap[j - 1], dim=-1) + if i < len(rows) - 1: + tile = tile[..., :-latent_y_overlap[i], :] + if j < len(row) - 1: + tile = tile[..., :, :-latent_x_overlap[j]] + result_row.append(tile) + result_rows.append(torch.cat(result_row, dim=-1)) + return torch.cat(result_rows, dim=-2) + + def tiled_decode(self, z): + height, width = z.shape[-2] * self.vae_ratio, z.shape[-1] * self.vae_ratio + y_idx, y_len, y_overlap = self.split_tiles(height) + x_idx, x_len, x_overlap = self.split_tiles(width) + + # Blended tiles are written straight into a pre-allocated canvas. + canvas = None + row_tails = [] + out_y = 0 + for i, (i_pos, i_len) in enumerate(zip(y_idx, y_len)): + zi, zl = i_pos // self.vae_ratio, i_len // self.vae_ratio + new_tails = [] + left_tail = None + out_x = 0 + for j, (j_pos, j_len) in enumerate(zip(x_idx, x_len)): + zj, zw = j_pos // self.vae_ratio, j_len // self.vae_ratio + tile = self._decode_pixels(z[..., zi:zi + zl, zj:zj + zw]) + if i < len(y_idx) - 1: + new_tails.append(tile[..., -y_overlap[i]:, :].clone()) + next_left_tail = tile[..., :, -x_overlap[j]:].clone() if j < len(x_idx) - 1 else None + if i > 0: + tile = self.blend(row_tails[j], tile, y_overlap[i - 1], dim=-2) + if j > 0: + tile = self.blend(left_tail, tile, x_overlap[j - 1], dim=-1) + left_tail = next_left_tail + if i < len(y_idx) - 1: + tile = tile[..., :-y_overlap[i], :] + if j < len(x_idx) - 1: + tile = tile[..., :, :-x_overlap[j]] + if canvas is None: + canvas = torch.empty(*tile.shape[:-2], height, width, dtype=tile.dtype, device=tile.device) + canvas[..., out_y:out_y + tile.shape[-2], out_x:out_x + tile.shape[-1]].copy_(tile) + out_x += tile.shape[-1] + row_tails = new_tails + out_y += tile.shape[-2] + return canvas + + # temporal chunking + + def encode_temporal(self, x): + if x.shape[2] % self.clip_length != 0: + pad_size = (-x.shape[2]) % self.clip_length + pad_frames = x[:, :, -1:].repeat(1, 1, pad_size, 1, 1) + x = torch.cat([x, pad_frames], dim=2) + + num_chunks = x.shape[2] // self.clip_length + + z_list = [] + for i in range(num_chunks): + clip_x = x[:, :, i * self.clip_length:(i + 1) * self.clip_length, :, :] + z_list.append(self._adaptive_encode(clip_x)) + + z = torch.cat(z_list, dim=2) + if self.token_drop > 0: + z = z[:, :, :-self.token_drop] + return z + + def _decode_temporal_pad_frames(self, z_len, pad_tokens): + if pad_tokens <= 0: + return 0 + intra_tail = self.clip_length % self.vae_ratio_t + if intra_tail == 0: + return pad_tokens * self.vae_ratio_t + + z_len_before_pad = z_len - pad_tokens + return sum( + (intra_tail if (z_len_before_pad + k) % self.tokens_chunk_size == 0 + else self.vae_ratio_t) + for k in range(pad_tokens) + ) + + def _decode_temporal_frame_plan(self, z_len, num_chunks, pad_tokens): + chunk_dec = self.tokens_chunk_size * self.vae_ratio_t + split_count = int(self.token_drop > 0) + 1 + total_frames = 0 + final_overlap_frames = 0 + + for i in range(num_chunks): + t_start_idx = i * self.tokens_chunk_size + t_end_idx = t_start_idx + self.tokens_chunk_size + self.token_overlap + clip_token_len = max(0, min(t_end_idx, z_len) - min(t_start_idx, z_len)) + clip_frame_len = clip_token_len * self.vae_ratio_t + + for j in range(split_count): + f_start_idx = j * chunk_dec + f_end_idx = min(f_start_idx + chunk_dec, clip_frame_len) + chunk_frames = max(0, f_end_idx - f_start_idx - self.frame_pre_padding) + if j == 0: + total_frames += chunk_frames + else: + final_overlap_frames = chunk_frames + + total_frames += final_overlap_frames + return total_frames - self._decode_temporal_pad_frames(z_len, pad_tokens) + + def decode_temporal(self, z): + chunk_dec = self.tokens_chunk_size * self.vae_ratio_t + split_count = int(self.token_drop > 0) + 1 + + pseudo_total_tokens = z.shape[2] + self.token_drop + + pad_tokens = 0 + remainder = pseudo_total_tokens % self.tokens_chunk_size + if remainder != 0: + pad_tokens = self.tokens_chunk_size - remainder + pseudo_total_tokens += pad_tokens + + num_chunks = pseudo_total_tokens // self.tokens_chunk_size - int(self.token_drop > 0) + if num_chunks < 1: + # too few tokens for one chunk (e.g. T_lat == 2): pad one extra chunk + pad_tokens += self.tokens_chunk_size + num_chunks += 1 + + if pad_tokens > 0: + pad_z = z[:, :, -1:, :, :].repeat(1, 1, pad_tokens, 1, 1) + z = torch.cat([z, pad_z], dim=2) + + output_frames = self._decode_temporal_frame_plan(z.shape[2], num_chunks, pad_tokens) + + dec = None + dec_overlap = None + write_pos = 0 + + def write_part(part): + nonlocal dec, write_pos + part_frames = part.shape[2] + if part_frames <= 0: + return + if dec is None: + out_shape = list(part.shape) + out_shape[2] = output_frames + dec = torch.empty(out_shape, dtype=part.dtype, device=part.device) + copy_frames = min(part_frames, max(0, dec.shape[2] - write_pos)) + if copy_frames > 0: + dec[:, :, write_pos:write_pos + copy_frames, :, :].copy_( + part[:, :, :copy_frames, :, :] + ) + write_pos += copy_frames + + for i in range(num_chunks): + t_start_idx = i * self.tokens_chunk_size + t_end_idx = t_start_idx + self.tokens_chunk_size + self.token_overlap + clip_z = z[:, :, t_start_idx:t_end_idx, :, :] + + clip_dec = self._adaptive_decode(clip_z) + + for j in range(split_count): + f_start_idx = j * chunk_dec + f_end_idx = min(f_start_idx + chunk_dec, clip_dec.shape[2]) + clip_dec_chunk = clip_dec[:, :, f_start_idx:f_end_idx, :, :] + clip_dec_chunk = clip_dec_chunk[:, :, self.frame_pre_padding:, :, :] + + if j == 0: + if dec_overlap is not None: + clip_dec_chunk = self.blend( + dec_overlap, clip_dec_chunk, self.frame_overlap, dim=-3 + ) + dec_overlap = None + write_part(clip_dec_chunk) + else: + dec_overlap = clip_dec_chunk.contiguous() + + if i == num_chunks - 1 and dec_overlap is not None: + write_part(dec_overlap) + dec_overlap = None + + del clip_dec, clip_z + + return dec + + + def encode(self, x): + # x: [B, 3, T, H, W] in [-1, 1] -> normalized latents [B, 24, T_lat, H/16, W/16] + if x.ndim == 4: + x = x.unsqueeze(2) + + x = x.add(1.0).mul_(0.5).sub_(self.pixel_mean.to(x)).div_(self.pixel_std.to(x)) + + if x.shape[2] == 1: + moments = self._adaptive_encode(x) + moments = moments[:, :, -1:, :, :] + else: + moments = self.encode_temporal(x) + + mean = torch.chunk(moments.float(), 2, dim=1)[0] + + latents_mean = self.latents_mean.view(1, -1, 1, 1, 1).to(mean) + latents_std = self.latents_std.view(1, -1, 1, 1, 1).to(mean) + return (mean - latents_mean) / latents_std + + def encode_tiled(self, x, **kwargs): + # tiling is always on internally with the reference's semantic tile sizes, ignore tiling fallbacks + return self.encode(x) + + def decode_tiled(self, z, **kwargs): + return self.decode(z) + + def decode(self, z): + # z: [B, 24, T_lat, H_lat, W_lat] normalized latents -> pixels [B, 3, T, H, W] in [-1, 1] + latents_mean = self.latents_mean.view(1, -1, 1, 1, 1).to(z) + latents_std = self.latents_std.view(1, -1, 1, 1, 1).to(z) + z = z * latents_std + latents_mean + + if z.shape[2] == 1: + dec = self._adaptive_decode(z) + dec = dec[:, :, -1:, :, :] + else: + dec = self.decode_temporal(z) + + dec = dec.float() + dec.mul_(self.pixel_std.to(dec)).add_(self.pixel_mean.to(dec)).clamp_(0.0, 1.0).mul_(2.0).sub_(1.0) + return dec diff --git a/comfy/ldm/modules/attention.py b/comfy/ldm/modules/attention.py index e6500cff4..2c549e095 100644 --- a/comfy/ldm/modules/attention.py +++ b/comfy/ldm/modules/attention.py @@ -90,22 +90,6 @@ def default(val, d): return val return d -def _gqa_repeat_factor(query_heads, key_heads, value_heads): - if key_heads != value_heads: - raise ValueError(f"Key/value head count mismatch for GQA: {key_heads} != {value_heads}") - if query_heads == key_heads: - return 1 - if query_heads % key_heads != 0: - raise ValueError(f"Query heads must be divisible by key/value heads for GQA: {query_heads} vs {key_heads}") - return query_heads // key_heads - -def _repeat_kv_for_gqa(k, v, query_heads, head_dim): - n_rep = _gqa_repeat_factor(query_heads, k.shape[head_dim], v.shape[head_dim]) - if n_rep > 1: - k = k.repeat_interleave(n_rep, dim=head_dim) - v = v.repeat_interleave(n_rep, dim=head_dim) - return k, v - def _heads_from_dim(tensor, dim_head, name): inner_dim = tensor.shape[-1] if inner_dim % dim_head != 0: @@ -122,10 +106,8 @@ def _reshape_qkv_to_heads(q, k, v, b, heads, dim_head, enable_gqa=False, expand_ value_heads = heads k = k.unsqueeze(3).reshape(b, -1, key_heads, dim_head) v = v.unsqueeze(3).reshape(b, -1, value_heads, dim_head) - if enable_gqa: - _gqa_repeat_factor(heads, key_heads, value_heads) - if expand_kv: - k, v = _repeat_kv_for_gqa(k, v, heads, -2) + if enable_gqa and expand_kv: + k, v = comfy.ops.repeat_kv_for_gqa(k, v, heads, -2) return q, k, v @@ -196,7 +178,7 @@ def attention_basic(q, k, v, heads, mask=None, attn_precision=None, skip_reshape h = heads if skip_reshape: if kwargs.get("enable_gqa", False): - k, v = _repeat_kv_for_gqa(k, v, q.shape[-3], -3) + k, v = comfy.ops.repeat_kv_for_gqa(k, v, q.shape[-3], -3) q, k, v = map( lambda t: t.reshape(b * heads, -1, dim_head), (q, k, v), @@ -262,7 +244,7 @@ def attention_sub_quad(query, key, value, heads, mask=None, attn_precision=None, if skip_reshape: if kwargs.get("enable_gqa", False): - key, value = _repeat_kv_for_gqa(key, value, query.shape[-3], -3) + key, value = comfy.ops.repeat_kv_for_gqa(key, value, query.shape[-3], -3) query = query.reshape(b * heads, -1, dim_head) value = value.reshape(b * heads, -1, dim_head) key = key.reshape(b * heads, -1, dim_head).movedim(1, 2) @@ -338,7 +320,7 @@ def attention_split(q, k, v, heads, mask=None, attn_precision=None, skip_reshape if skip_reshape: if kwargs.get("enable_gqa", False): - k, v = _repeat_kv_for_gqa(k, v, q.shape[-3], -3) + k, v = comfy.ops.repeat_kv_for_gqa(k, v, q.shape[-3], -3) q, k, v = map( lambda t: t.reshape(b * heads, -1, dim_head), (q, k, v), @@ -476,7 +458,7 @@ def attention_xformers(q, k, v, heads, mask=None, attn_precision=None, skip_resh (q, k, v), ) if kwargs.get("enable_gqa", False): - k, v = _repeat_kv_for_gqa(k, v, q.shape[-2], -2) + k, v = comfy.ops.repeat_kv_for_gqa(k, v, q.shape[-2], -2) # actually do the reshaping else: dim_head //= heads @@ -573,7 +555,7 @@ def attention_sage(q, k, v, heads, mask=None, attn_precision=None, skip_reshape= b, _, _, dim_head = q.shape tensor_layout = "HND" if kwargs.get("enable_gqa", False): - k, v = _repeat_kv_for_gqa(k, v, q.shape[-3], -3) + k, v = comfy.ops.repeat_kv_for_gqa(k, v, q.shape[-3], -3) else: b, _, dim_head = q.shape dim_head //= heads @@ -671,7 +653,7 @@ def attention3_sage(q, k, v, heads, mask=None, attn_precision=None, skip_reshape if skip_reshape: q_s = q if kwargs.get("enable_gqa", False): - k_s, v_s = _repeat_kv_for_gqa(k, v, H, -3) + k_s, v_s = comfy.ops.repeat_kv_for_gqa(k, v, H, -3) else: k_s, v_s = k, v else: diff --git a/comfy/ldm/wan/model.py b/comfy/ldm/wan/model.py index c042e93c4..ad207c3ef 100644 --- a/comfy/ldm/wan/model.py +++ b/comfy/ldm/wan/model.py @@ -11,6 +11,7 @@ from comfy.ldm.flux.layers import EmbedND from comfy.ldm.flux.math import apply_rope1, rope import comfy.ldm.common_dit import comfy.model_management +import comfy.ops import comfy.patcher_extension @@ -146,7 +147,8 @@ class WanI2VCrossAttention(WanSelfAttention): v = self.v(context) k_img = self.norm_k_img(self.k_img(context_img)) v_img = self.v_img(context_img) - img_x = optimized_attention(q, k_img, v_img, heads=self.num_heads, transformer_options=transformer_options) + # Sageattn can cause Nans here, don't allow it as there is no speed difference anyway as img attention is tiny. + img_x = optimized_attention(q, k_img, v_img, heads=self.num_heads, transformer_options=transformer_options, low_precision_attention=False) # compute attention x = optimized_attention(q, k, v, heads=self.num_heads, transformer_options=transformer_options) @@ -174,6 +176,13 @@ def repeat_e(e, x): return torch.repeat_interleave(e, repeats + 1, dim=1)[:, :x.size(1)] +class WanFeedForward(nn.Sequential): + """[Linear, GELU(tanh), Linear], with the GELU folded into the down-projection.""" + + def forward(self, x): + return comfy.ops.linear_input_act(self[2], self[0](x), "gelu_tanh") + + class WanAttentionBlock(nn.Module): def __init__(self, @@ -207,7 +216,7 @@ class WanAttentionBlock(nn.Module): qk_norm, eps, operation_settings=operation_settings) self.norm2 = operation_settings.get("operations").LayerNorm(dim, eps, elementwise_affine=False, device=operation_settings.get("device"), dtype=operation_settings.get("dtype")) - self.ffn = nn.Sequential( + self.ffn = WanFeedForward( operation_settings.get("operations").Linear(dim, ffn_dim, device=operation_settings.get("device"), dtype=operation_settings.get("dtype")), nn.GELU(approximate='tanh'), operation_settings.get("operations").Linear(ffn_dim, dim, device=operation_settings.get("device"), dtype=operation_settings.get("dtype"))) diff --git a/comfy/ldm/wan/model_animate2.py b/comfy/ldm/wan/model_animate2.py new file mode 100644 index 000000000..dc34cad88 --- /dev/null +++ b/comfy/ldm/wan/model_animate2.py @@ -0,0 +1,387 @@ +# Wan-Animate-2: https://github.com/Wan-Video/Wan-Animate-2 +"""Wan2.1-I2V-14B weights, driven by a video instead of a motion extractor. + +A pose branch over the pose video's latents runs in lockstep with the generation +branch, feeding it K/V per block. The reference image is one extra latent frame at the +front of the generation branch, trimmed off by the caller. Upstream calls the pose video +the driving video and its branch forward_ref, not to be confused with the reference image. +""" + +import torch + +import comfy.ldm.common_dit +import comfy.model_management +import comfy.quant_ops +import comfy.utils +from comfy.ldm.flux.math import apply_rope1 +from comfy.ldm.modules.attention import optimized_attention + +from .model import WanAttentionBlock, WanModel, WanSelfAttention, repeat_e, sinusoidal_embedding_1d + + +class WanAnimate2SelfAttention(WanSelfAttention): + + def qkv(self, x, freqs): + b, s, n, d = *x.shape[:2], self.num_heads, self.head_dim + q = apply_rope1(self.norm_q(self.q(x)).view(b, s, n, d), freqs) + k = apply_rope1(self.norm_k(self.k(x)).view(b, s, n, d), freqs) + return q, k, self.v(x).view(b, s, n, d) + + def _attn1_patch(self, x, q, k, transformer_options): + for p in transformer_options.get("patches", {}).get("attn1_patch", []): + x = p({"x": x, "q": q, "k": k, "transformer_options": transformer_options}) + return x + + def kv(self, x, freqs): + b, s, n, d = *x.shape[:2], self.num_heads, self.head_dim + return apply_rope1(self.norm_k(self.k(x)).view(b, s, n, d), freqs), self.v(x).view(b, s, n, d) + + def forward_pose(self, x, freqs, transformer_options={}): + b, s, n, d = *x.shape[:2], self.num_heads, self.head_dim + q, k, v = self.qkv(x, freqs) + out = optimized_attention(q.reshape(b, s, n * d), k.reshape(b, s, n * d), v.reshape(b, s, n * d), heads=self.num_heads, transformer_options=transformer_options) + return self.o(self._attn1_patch(out, q, k, transformer_options)), k, v + + def forward_gen(self, x, freqs, k_pose, v_pose, f_gen, hw, buffers, ref_strength=1.0, transformer_options={}): + # frame j attends every gen token plus pose frame j-1 (frame 0 is the reference slot and has none) + b, s, n, d = *x.shape[:2], self.num_heads, self.head_dim + q, k, v = self.qkv(x, freqs) + if ref_strength != 1.0: + v[:, :hw] *= ref_strength # frame 0 is the reference image's slot + + if k_pose is None: # pose influence windowed out: plain self-attention, no per-frame loop + out = optimized_attention(q.reshape(b, s, n * d), k.reshape(b, s, n * d), v.reshape(b, s, n * d), heads=self.num_heads, transformer_options=transformer_options) + return self.o(self._attn1_patch(out, q, k, transformer_options)) + + # gen half is the same every frame; only the hw-token pose tail is rewritten + kbuf, vbuf, out = buffers + kbuf[:, :s] = k + vbuf[:, :s] = v + + for j in range(f_gen): + q_j = q[:, j * hw:(j + 1) * hw].reshape(b, hw, n * d) + if j == 0: + kk, vv = k, v + else: + kbuf[:, s:] = k_pose[:, (j - 1) * hw:j * hw] + vbuf[:, s:] = v_pose[:, (j - 1) * hw:j * hw] + kk, vv = kbuf, vbuf + out[:, j * hw:(j + 1) * hw] = optimized_attention(q_j, kk.reshape(b, kk.shape[1], n * d), vv.reshape(b, kk.shape[1], n * d), heads=self.num_heads, transformer_options=transformer_options) + return self.o(self._attn1_patch(out, q, k, transformer_options)) + + +class WanAnimate2Block(WanAttentionBlock): + + def __init__(self, cross_attn_type, dim, ffn_dim, num_heads, window_size=(-1, -1), qk_norm=True, cross_attn_norm=False, eps=1e-6, operation_settings={}): + super().__init__(cross_attn_type, dim, ffn_dim, num_heads, window_size, qk_norm, cross_attn_norm, eps, operation_settings=operation_settings) + self.self_attn = WanAnimate2SelfAttention(dim, num_heads, window_size, qk_norm, eps, operation_settings=operation_settings) + + def _modulation(self, e, x): + if e.ndim < 4: + return (comfy.model_management.cast_to(self.modulation, dtype=x.dtype, device=x.device) + e).chunk(6, dim=1) + return (comfy.model_management.cast_to(self.modulation, dtype=x.dtype, device=x.device).unsqueeze(0) + e).unbind(2) + + def _cross_attn_ffn(self, x, e, context, context_img_len, transformer_options): + x = x + self.cross_attn(self.norm3(x), context, context_img_len=context_img_len, transformer_options=transformer_options) + for p in transformer_options.get("patches", {}).get("attn2_patch", []): + x = p({"x": x, "transformer_options": transformer_options}) + y = self.ffn(torch.addcmul(repeat_e(e[3], x), self.norm2(x), 1 + repeat_e(e[4], x))) + return torch.addcmul(x, y, repeat_e(e[5], x)) + + def forward_pose(self, x, e, freqs, context, context_img_len=257, transformer_options={}): + e = self._modulation(e, x) + x = x.contiguous() + y, k, v = self.self_attn.forward_pose(torch.addcmul(repeat_e(e[0], x), self.norm1(x), 1 + repeat_e(e[1], x)), freqs, transformer_options=transformer_options) + x = torch.addcmul(x, y, repeat_e(e[2], x)) + del y + return self._cross_attn_ffn(x, e, context, context_img_len, transformer_options), k, v + + def kv_from_input(self, x_pose, e, freqs, transformer_options={}): + e = self._modulation(e, x_pose) + x_pose = x_pose.contiguous() + return self.self_attn.kv(torch.addcmul(repeat_e(e[0], x_pose), self.norm1(x_pose), 1 + repeat_e(e[1], x_pose)), freqs) + + def forward_gen(self, x, e, freqs, context, k_pose, v_pose, f_gen, hw, buffers, ref_strength=1.0, context_img_len=257, transformer_options={}): + e = self._modulation(e, x) + x = x.contiguous() + y = self.self_attn.forward_gen(torch.addcmul(repeat_e(e[0], x), self.norm1(x), 1 + repeat_e(e[1], x)), freqs, k_pose, v_pose, f_gen, hw, buffers, ref_strength=ref_strength, transformer_options=transformer_options) + x = torch.addcmul(x, y, repeat_e(e[2], x)) + del y + return self._cross_attn_ffn(x, e, context, context_img_len, transformer_options) + + +class PoseBranchCache: + """Pose-branch block inputs, reused across the sampling steps of one execution. + + Caching the block input rather than its K/V halves the memory; reprojecting K/V on read + costs ~4% of re-running the block. One slot per distinct pose sequence, so under + context windows each window keeps its own; least recently used slots are evicted when + the store device runs low on memory. Created and freed by WanAnimate2Cache. + """ + + CONVROT_GROUPSIZE = 256 + + def __init__(self, store_device=None, dtype="default"): + self.store_device = torch.device(store_device) if store_device is not None else torch.device("cpu") + self.dtype = dtype + self.slots = [] # most recently used last + self.slot = None + self._pending = {} + self._staging = {} + + def select(self, pose_latents): + # select runs at a forward boundary: an interrupted forward can leave copies in flight that a different slot's forward would then mistake for its own + if self._pending: + for t, stream in self._pending.values(): + if stream is not None: + stream.synchronize() + self._pending = {} + # keyed on batch element 0, so a cond batch size change mid-run stays valid + k = pose_latents[:1] + for s in self.slots: + if s["key"].shape == k.shape and torch.equal(s["key"], k.to(s["key"].device)): + self.slots.remove(s) + self.slots.append(s) + self.slot = s + return + # cache what fits: a filled slot is the size estimate for the next one, and least recently used slots make room when the store device runs low + est = max((self._slot_bytes(s) for s in self.slots), default=0) * 1.5 + while self.slots and comfy.model_management.get_free_memory(self.store_device) < est: + self._free_slot(self.slots.pop(0)) + self.slot = {"key": k.clone().to(self.store_device), "blocks": {}, "params": {}, "shape": None, "pinned": []} + self.slots.append(self.slot) + + def _free_slot(self, s): + for t, stream in self._pending.values(): + if stream is not None: + stream.synchronize() # an aborted forward can leave a copy in flight, still reading memory we are about to unpin + self._pending = {} + for t in s["pinned"]: + comfy.model_management.unpin_memory(t) + + def free(self): + for s in self.slots: + self._free_slot(s) + self.slots = [] + self.slot = None + self._staging = {} + + def filled(self, num_blocks): + return self.slot is not None and len(self.slot["blocks"]) == num_blocks + + def put(self, i, x_pose): + t = x_pose[:1] + params = None + if self.dtype in ("int8", "int4"): + # convrot is what lets low-bit survive the ~125x per-channel outliers here, and over a [tokens, dim] view per-row scale means per-token. The kernels want 2D and a power-of-4 group that divides dim. + self.slot["shape"] = t.shape + g = self.CONVROT_GROUPSIZE + while g > 4 and t.shape[-1] % g: + g //= 4 + if self.dtype == "int4": + t, params = comfy.quant_ops.TensorCoreConvRotW4A4Layout.quantize(t.reshape(-1, t.shape[-1]), convrot_groupsize=g) + else: + t, params = comfy.quant_ops.TensorWiseINT8Layout.quantize(t.reshape(-1, t.shape[-1]), is_weight=True, per_channel=True, convrot=True, convrot_groupsize=g) + + t = t.to(self.store_device, copy=True) + if comfy.model_management.pin_memory(t): + self.slot["pinned"].append(t) + self.slot["blocks"][i] = t + # the scales follow the blocks off the GPU: per-window slots would otherwise pile them up in VRAM (~200 MB per window at 480p int4) + self.slot["params"][i] = params if params is None else params.to_device(self.store_device) + + def prefetch(self, i, device, dtype): + # call before the compute this should overlap, so the stream waits only on work already enqueued + if i not in self.slot["blocks"] or i in self._pending: + return + t = self.slot["blocks"][i] + cast_dtype = None if self.slot["params"][i] is not None else dtype # int8 entries move in their stored dtype and widen in take() + stream = None + r = None + if t.device != device: + stream = comfy.model_management.get_offload_stream(device) + cs = comfy.model_management.current_stream(device) + if stream is not None and cs is not None: + # the handed-out stream last waited on the main stream a full rotation ago, which does not cover the previous consumer's reads of this slot; wait now so the copy cannot overwrite a slot still being read + stream.wait_stream(cs) + # two persistent staging buffers per tensor shape instead of a fresh allocation per block (~29 GB of churn per pass at 720p); windows of different lengths get their own pair + buf_key = (tuple(t.shape), cast_dtype if cast_dtype is not None else t.dtype) + if buf_key not in self._staging: + self._staging[buf_key] = [torch.empty(t.shape, dtype=buf_key[1], device=device) for _ in range(2)] + r = self._staging[buf_key][i % 2] + self._pending[i] = (comfy.model_management.cast_to(t, cast_dtype, device, non_blocking=True, stream=stream, r=r), stream) + + def take(self, i, device, dtype, batch_size): + if i not in self._pending: + self.prefetch(i, device, dtype) + t, stream = self._pending.pop(i) + comfy.model_management.sync_stream(device, stream) + params = self.slot["params"][i] + if params is not None: + layout = comfy.quant_ops.TensorCoreConvRotW4A4Layout if self.dtype == "int4" else comfy.quant_ops.TensorWiseINT8Layout + t = layout.dequantize(t, params.to_device(t.device)).reshape(self.slot["shape"]).to(dtype) + return comfy.utils.repeat_to_batch_size(t, batch_size) + + def _slot_bytes(self, s): + return sum(t.numel() * t.element_size() for t in s["blocks"].values()) + + def memory_bytes(self): + return sum(self._slot_bytes(s) for s in self.slots) + + +class WanAnimate2Model(WanModel): + + def __init__(self, + model_type='animate2', + patch_size=(1, 2, 2), + text_len=512, + in_dim=36, + dim=5120, + ffn_dim=13824, + freq_dim=256, + text_dim=4096, + out_dim=16, + num_heads=40, + num_layers=40, + window_size=(-1, -1), + qk_norm=True, + cross_attn_norm=True, + eps=1e-6, + flf_pos_embed_token_number=None, + in_dim_ref_conv=None, + image_model=None, + device=None, dtype=None, operations=None, + ): + # model_type is 'animate2' in unet_config, but the checkpoint is i2v-shaped + super().__init__(model_type='i2v', patch_size=patch_size, text_len=text_len, in_dim=in_dim, dim=dim, ffn_dim=ffn_dim, freq_dim=freq_dim, + text_dim=text_dim, out_dim=out_dim, num_heads=num_heads, num_layers=num_layers, window_size=window_size, qk_norm=qk_norm, + cross_attn_norm=cross_attn_norm, eps=eps, flf_pos_embed_token_number=flf_pos_embed_token_number, in_dim_ref_conv=in_dim_ref_conv, + wan_attn_block_class=WanAnimate2Block, image_model=image_model, device=device, dtype=dtype, operations=operations) + + def rope_encode_pose(self, t, h, w, w_patches, device=None, dtype=None): + # t_start=1 lines pose frame j up with gen frame j+1, past the reference slot; shift_x parks it in its own strip of rope space. + # The caller's rope_options are a user scaling knob and deliberately not forwarded. + return super().rope_encode(t, h, w, t_start=1, device=device, dtype=dtype, transformer_options={"rope_options": {"shift_x": float(w_patches)}}) + + def _forward(self, x, timestep, context, clip_fea=None, time_dim_concat=None, transformer_options={}, pose_latents=None, clip_fea_pose=None, context_pose=None, **kwargs): + bs, c, t, h, w = x.shape + x = comfy.ldm.common_dit.pad_to_patch_size(x, self.patch_size) + + # h/w pre-pad: rope_encode's rounding reproduces the post-pad grid + freqs = self.rope_encode(t, h, w, device=x.device, dtype=x.dtype, transformer_options=transformer_options) + freqs_pose = None + if pose_latents is not None: # absent when the node's timestep window excludes this step + pose_latents = comfy.ldm.common_dit.pad_to_patch_size(pose_latents.to(x.dtype), self.patch_size) + w_patches = (w + (self.patch_size[2] // 2)) // self.patch_size[2] + freqs_pose = self.rope_encode_pose(pose_latents.shape[2], h, w, w_patches, device=x.device, dtype=x.dtype) + + return self.forward_orig(x, timestep, context, clip_fea=clip_fea, freqs=freqs, freqs_pose=freqs_pose, pose_latents=pose_latents, + clip_fea_pose=clip_fea_pose, context_pose=context_pose, transformer_options=transformer_options, **kwargs)[:, :, :t, :h, :w] + + def forward_orig(self, x, t, context, clip_fea=None, freqs=None, freqs_pose=None, pose_latents=None, clip_fea_pose=None, context_pose=None, pose_strength=1.0, reference_strength=1.0, transformer_options={}, **kwargs): + x_input = x[:, :, 1:] # video-only: frame 0 is the reference slot, offset past it below + x = self.patch_embedding(x.float()).to(x.dtype) + grid_sizes = x.shape[2:] + transformer_options["grid_sizes"] = grid_sizes + f_gen, gh, gw = grid_sizes + hw = gh * gw + x = x.flatten(2).transpose(1, 2) + + # the node windows the pose influence via cond timestep ranges: outside the window the cond carries no pose latents, and the branch, its cache traffic and the per-frame attention loop are all skipped + apply_pose = pose_latents is not None + if apply_pose and pose_latents.shape[2] != f_gen - 1: # before cache.select, which would otherwise keep an empty slot keyed to the rejected latents + raise ValueError("pose branch has {} latent frames, expected {} (generation frames minus the reference-image slot)".format(pose_latents.shape[2], f_gen - 1)) + + cache = transformer_options.get("animate2_cache", None) if apply_pose else None + if cache is not None: + cache.select(pose_latents) + cached = cache is not None and cache.filled(len(self.blocks)) + + x_pose = None + if not cached and apply_pose: + # 36ch = [latents(16) | mask(4) | latents(16)]; latents twice, and the mask is all ones since every pose frame is known + x_pose = self.patch_embedding(torch.cat([pose_latents, torch.ones_like(pose_latents[:, :4]), pose_latents], dim=1).float()).to(x.dtype) + x_pose = x_pose.flatten(2).transpose(1, 2) + + # time embeddings + e = self.time_embedding(sinusoidal_embedding_1d(self.freq_dim, t.flatten()).to(dtype=x.dtype)) + e = e.reshape(t.shape[0], -1, e.shape[-1]) + e0 = self.time_projection(e).unflatten(2, (6, self.dim)) + + e0_pose = None + if apply_pose: + t_pose = torch.ones_like(t.flatten()) + e_pose = self.time_embedding(sinusoidal_embedding_1d(self.freq_dim, t_pose).to(dtype=x.dtype)) + e_pose = e_pose.reshape(t.shape[0], -1, e_pose.shape[-1]) + e0_pose = self.time_projection(e_pose).unflatten(2, (6, self.dim)) + + context_gen = self.text_embedding(context) + + context_img_len = None + if clip_fea is not None: + if self.img_emb is not None: + context_gen = torch.cat([self.img_emb(clip_fea), context_gen], dim=1) + context_img_len = clip_fea.shape[-2] + + context_img_len_pose = None + if not cached and apply_pose: + context_pose = self.text_embedding(context if context_pose is None else context_pose) + clip_fea_pose = clip_fea if clip_fea_pose is None else clip_fea_pose + if clip_fea_pose is not None: + if self.img_emb is not None: + context_pose = torch.cat([self.img_emb(clip_fea_pose), context_pose], dim=1) + context_img_len_pose = clip_fea_pose.shape[-2] + + patches_replace = transformer_options.get("patches_replace", {}) + patches = transformer_options.get("patches", {}) + blocks_replace = patches_replace.get("dit", {}) + transformer_options["total_blocks"] = len(self.blocks) + transformer_options["block_type"] = "double" + + if cache is not None and not cached and apply_pose and "context_window" in transformer_options: + # pose-only prepass, to avoid inflating dynamic VRAM calibration when using multiple context windows + for i, block in enumerate(self.blocks): + transformer_options["block_index"] = i + cache.put(i, x_pose) + x_pose = block.forward_pose(x_pose, e0_pose, freqs_pose, context_pose, context_img_len=context_img_len_pose, transformer_options=transformer_options)[0] + x_pose = None + cached = True + + buffers = None + if apply_pose: + # allocated once and reused by every block + n, d = self.num_heads, self.dim // self.num_heads + buffers = (x.new_empty(x.shape[0], x.shape[1] + hw, n, d), x.new_empty(x.shape[0], x.shape[1] + hw, n, d), x.new_empty(x.shape[0], x.shape[1], self.dim)) + + for i, block in enumerate(self.blocks): + transformer_options["block_index"] = i + + if not apply_pose: + k_pose = v_pose = None + elif cached: + x_pose_in = cache.take(i, x.device, x.dtype, x.shape[0]) + cache.prefetch(i + 1, x.device, x.dtype) # queue the next block before the gen compute it should overlap + k_pose, v_pose = block.kv_from_input(x_pose_in, e0_pose, freqs_pose, transformer_options=transformer_options) + del x_pose_in + else: + if cache is not None: + cache.put(i, x_pose) + # runs even under a block replace: its state has to reach block i+1 + x_pose, k_pose, v_pose = block.forward_pose(x_pose, e0_pose, freqs_pose, context_pose, context_img_len=context_img_len_pose, transformer_options=transformer_options) + if v_pose is not None and pose_strength != 1.0: + v_pose = v_pose * pose_strength + + if ("double_block", i) in blocks_replace: + def block_wrap(args, block=block, k_pose=k_pose, v_pose=v_pose): + return {"img": block.forward_gen(args["img"], args["vec"], args["pe"], args["txt"], k_pose, v_pose, f_gen, hw, buffers, ref_strength=reference_strength, context_img_len=context_img_len, transformer_options=args["transformer_options"])} + out = blocks_replace[("double_block", i)]({"img": x, "txt": context_gen, "vec": e0, "pe": freqs, "transformer_options": transformer_options}, {"original_block": block_wrap}) + x = out["img"] + else: + x = block.forward_gen(x, e0, freqs, context_gen, k_pose, v_pose, f_gen, hw, buffers, ref_strength=reference_strength, context_img_len=context_img_len, transformer_options=transformer_options) + + if "double_block" in patches: + for p in patches["double_block"]: + out = p({"img": x, "x": x_input, "vec": e, "block_index": i, "img_offset": hw, "transformer_options": transformer_options}) + x = out["img"] + + return self.unpatchify(self.head(x, e), grid_sizes) diff --git a/comfy/ldm/wan/uni3c.py b/comfy/ldm/wan/uni3c.py index 827ad2339..f4bf68200 100644 --- a/comfy/ldm/wan/uni3c.py +++ b/comfy/ldm/wan/uni3c.py @@ -4,7 +4,7 @@ import torch import torch.nn as nn from comfy.ldm.flux.layers import EmbedND -from .model import WanSelfAttention +from .model import WanFeedForward, WanSelfAttention class Uni3CLayerNormZero(nn.Module): @@ -41,7 +41,7 @@ class Uni3CAttentionBlock(nn.Module): self.norm1 = Uni3CLayerNormZero(time_embed_dim, dim, device=device, dtype=dtype, operations=operations) self.self_attn = WanSelfAttention(dim, num_heads, qk_norm=True, eps=eps, operation_settings=operation_settings) self.norm2 = Uni3CLayerNormZero(time_embed_dim, dim, device=device, dtype=dtype, operations=operations) - self.ffn = nn.Sequential( + self.ffn = WanFeedForward( operations.Linear(dim, ffn_dim, device=device, dtype=dtype), nn.GELU(approximate='tanh'), operations.Linear(ffn_dim, dim, device=device, dtype=dtype)) diff --git a/comfy/model_base.py b/comfy/model_base.py index ee6dc57a2..469d301ea 100644 --- a/comfy/model_base.py +++ b/comfy/model_base.py @@ -21,6 +21,8 @@ import comfy.ldm.hunyuan3dv2_1.hunyuandit import torch import logging import comfy.ldm.lightricks.av_model +import comfy.ldm.minimax.model +import comfy.nested_tensor import comfy.ldm.lightricks.symmetric_patchifier import comfy.context_windows from comfy.ldm.modules.diffusionmodules.openaimodel import UNetModel, Timestep @@ -44,6 +46,7 @@ import comfy.ldm.cosmos.predict2 import comfy.ldm.lumina.model import comfy.ldm.wan.model import comfy.ldm.wan.model_animate +import comfy.ldm.wan.model_animate2 import comfy.ldm.wan.ar_model import comfy.ldm.wan.model_wandancer import comfy.ldm.hunyuan3d.model @@ -99,6 +102,7 @@ class ModelType(Enum): FLOW_COSMOS = 10 IMG_TO_IMG_FLOW = 11 V_PREDICTION_DDPM = 12 + FLOW_AV = 13 def model_sampling(model_config, model_type): @@ -135,6 +139,9 @@ def model_sampling(model_config, model_type): c = comfy.model_sampling.IMG_TO_IMG_FLOW elif model_type == ModelType.V_PREDICTION_DDPM: c = comfy.model_sampling.V_PREDICTION_DDPM + elif model_type == ModelType.FLOW_AV: + c = comfy.model_sampling.CONST + s = comfy.model_sampling.ModelSamplingAV class ModelSampling(s, c): pass @@ -169,6 +176,7 @@ class BaseModel(torch.nn.Module): else: operations = model_config.custom_operations self.diffusion_model = unet_model(**unet_config, device=device, operations=operations) + self.diffusion_model.requires_grad_(False) self.diffusion_model.eval() if comfy.model_management.force_channels_last(): self.diffusion_model.to(memory_format=torch.channels_last) @@ -178,6 +186,7 @@ class BaseModel(torch.nn.Module): self.model_type = model_type self.model_sampling = model_sampling(model_config, model_type) + self.latent_shapes = None # set by the sampler for models that pack several streams into one latent self.adm_channels = unet_config.get("adm_in_channels", None) if self.adm_channels is None: @@ -1805,6 +1814,41 @@ class WAN22_Animate(WAN21): return comfy.context_windows.slice_cond(cond_value, window, x_in, device, temporal_dim=2, temporal_offset=1) return super().resize_cond_for_context_window(cond_key, cond_value, window, x_in, device, retain_index_list=retain_index_list) +class WAN_Animate2(WAN21): + def __init__(self, model_config, model_type=ModelType.FLOW, device=None): + super(WAN21, self).__init__(model_config, model_type, device=device, unet_model=comfy.ldm.wan.model_animate2.WanAnimate2Model) + self.image_to_video = True + + def extra_conds(self, **kwargs): + out = super().extra_conds(**kwargs) + + pose_video_latent = kwargs.get("pose_video_latent", None) + if pose_video_latent is not None: + out['pose_latents'] = comfy.conds.CONDRegular(self.process_latent_in(pose_video_latent)) + + clip_vision_output_pose = kwargs.get("clip_vision_output_pose", None) + if clip_vision_output_pose is not None: + out['clip_fea_pose'] = comfy.conds.CONDRegular(clip_vision_output_pose.penultimate_hidden_states) + + cross_attn_pose = kwargs.get("cross_attn_pose", None) + if cross_attn_pose is not None: + out['context_pose'] = comfy.conds.CONDRegular(cross_attn_pose) + + pose_strength = kwargs.get("pose_strength", 1.0) + if pose_strength != 1.0: + out['pose_strength'] = comfy.conds.CONDConstant(pose_strength) + + reference_strength = kwargs.get("reference_strength", 1.0) + if reference_strength != 1.0: + out['reference_strength'] = comfy.conds.CONDConstant(reference_strength) + + return out + + def resize_cond_for_context_window(self, cond_key, cond_value, window, x_in, device, retain_index_list=[]): + if cond_key == "pose_latents": + return comfy.context_windows.slice_cond(cond_value, window, x_in, device, temporal_dim=2, temporal_offset=1) + return super().resize_cond_for_context_window(cond_key, cond_value, window, x_in, device, retain_index_list=retain_index_list) + class WAN22_S2V(WAN21): def __init__(self, model_config, model_type=ModelType.FLOW, device=None): super(WAN21, self).__init__(model_config, model_type, device=device, unet_model=comfy.ldm.wan.model.WanModel_S2V) @@ -2062,6 +2106,80 @@ class Hunyuan3Dv2_1(BaseModel): out['guidance'] = comfy.conds.CONDRegular(torch.FloatTensor([guidance])) return out +class MiniMaxH3(BaseModel): + def __init__(self, model_config, model_type=ModelType.FLOW_AV, device=None): + super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.minimax.model.MiniMaxH3Model) + + def audio_scale(self): + """Scale the sampler carries the audio stream at, 1.0 when not sampling the packed latent.""" + if self.latent_shapes is None or len(self.latent_shapes) < 2: + return 1.0 + return self.model_sampling.audio_scale + + def _scale_audio_slice(self, latent, scale): + # the sampler carries the audio stream scaled onto the video schedule + if scale == 1.0: + return latent + if latent.is_nested: # the x0 output hands back the unpacked view + streams = latent.unbind() + return comfy.nested_tensor.NestedTensor([streams[0], streams[1] * scale] + list(streams[2:])) + n = math.prod(self.latent_shapes[0][1:]) + latent = latent.clone() + latent[..., n:] *= scale + return latent + + def process_latent_in(self, latent): + return self._scale_audio_slice(super().process_latent_in(latent), self.audio_scale()) + + def process_latent_out(self, latent): + return super().process_latent_out(self._scale_audio_slice(latent, 1.0 / self.audio_scale())) + + def extra_conds(self, **kwargs): + out = super().extra_conds(**kwargs) + cross_attn = kwargs.get("cross_attn", None) + if cross_attn is not None: + # run condition_proj + token refiner once per sampling instead of per step + cross_attn = self.diffusion_model.preprocess_text_embeds( + cross_attn.to(device=kwargs["device"], dtype=self.get_dtype_inference())) + out['c_crossattn'] = comfy.conds.CONDRegular(cross_attn) + + latent_shapes = kwargs.get("latent_shapes", None) + if latent_shapes is not None: + out['latent_shapes'] = comfy.conds.CONDConstant(latent_shapes) + + # Everything H3-specific rides in one dict so _apply_model's dtype cast + # (which would flatten fp32 cond latents and long tags to bf16) skips it. + payload = {} + tags = kwargs.get("minimax_token_tags", None) + if tags is not None: + payload["text_token_tags"] = tags + keyframes = kwargs.get("minimax_keyframes", None) + if keyframes is not None: + payload["keyframes"] = keyframes + payload["frame_count"] = kwargs.get("minimax_frame_count", None) + payload["cond_video_latents"] = [kf["latent"] for kf in keyframes] + refs = kwargs.get("minimax_refs", None) + if refs is not None: + payload["refs"] = refs + payload["cond_video_latents"] = [r["latent"] for r in refs if "latent" in r] + payload["cond_audio_latents"] = [r["audio_latent"] for r in refs if r.get("audio_latent") is not None] + if kwargs.get("minimax_visual_cond_noise_aug", None) is not None: + payload["visual_cond_noise_aug"] = kwargs["minimax_visual_cond_noise_aug"] + if kwargs.get("minimax_audio_cond_noise_aug", None) is not None: + payload["audio_cond_noise_aug"] = kwargs["minimax_audio_cond_noise_aug"] + payload["seed"] = kwargs.get("seed", 0) + # same value process_latent_in/out used, so the model never undoes a scale that was not applied + payload["audio_scale"] = self.audio_scale() + if cross_attn is not None and latent_shapes is not None and len(latent_shapes) > 1: + # packed layout built once per sampling run, h/w rounded up to the DiT's 2x2 patch + vs = latent_shapes[0] + payload["layout"] = comfy.ldm.minimax.model.PackedLayout( + cross_attn.shape[1], vs[2], (vs[3] + 1) // 2 * 2, (vs[4] + 1) // 2 * 2, + latent_shapes[1][-1], keyframes=payload.get("keyframes"), + refs=payload.get("refs"), frame_count=payload.get("frame_count")) + out['minimax_payload'] = comfy.conds.CONDConstant(payload) + return out + class TripoSplat(BaseModel): def __init__(self, model_config, model_type=ModelType.FLOW, device=None): super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.triposplat.model.LatentSeqMMFlowModel) diff --git a/comfy/model_detection.py b/comfy/model_detection.py index 39e973d36..103680fd1 100644 --- a/comfy/model_detection.py +++ b/comfy/model_detection.py @@ -359,6 +359,35 @@ def detect_unet_config(state_dict, key_prefix, metadata=None): # PixArt diffusers return None + if '{}video_patch_proj.weight'.format(key_prefix) in state_dict_keys and '{}audio_patch_proj.weight'.format(key_prefix) in state_dict_keys: # MiniMax H3 + dit_config = {} + dit_config["image_model"] = "minimax_h3" + dit_config["num_layers"] = count_blocks(state_dict_keys, '{}blocks.'.format(key_prefix) + '{}.') + dit_config["token_refiner_num_layers"] = count_blocks(state_dict_keys, '{}token_refiner.blocks.'.format(key_prefix) + '{}.') + dit_config["hidden_size"] = state_dict['{}video_patch_proj.weight'.format(key_prefix)].shape[0] + dit_config["latents_dim"] = state_dict['{}final_layer.video_out.weight'.format(key_prefix)].shape[0] // 4 # patch 1x2x2 + dit_config["audio_latents_dim"] = state_dict['{}final_layer.audio_out.weight'.format(key_prefix)].shape[0] + dit_config["attention_head_dim"] = state_dict['{}blocks.0.attn.q_norm.weight'.format(key_prefix)].shape[0] + qkv = state_dict['{}blocks.0.attn.qkv_proj.weight'.format(key_prefix)] + dit_config["num_attention_heads"] = qkv.shape[0] // (3 * dit_config["attention_head_dim"]) + dit_config["ffn_hidden_size"] = state_dict['{}blocks.0.mlp.fc1.weight'.format(key_prefix)].shape[0] // 2 + dit_config["text_dim"] = state_dict['{}condition_proj.weight'.format(key_prefix)].shape[1] + table_key = '{}adaln_t_table'.format(key_prefix) + if table_key in state_dict_keys: + # adaln shipped over a precomputed curve basis: the adaln linears span a small shared basis of the time-embedding curve (no time embedder) + table = state_dict[table_key].shape # [grid, k] + dit_config["adaln_curve_grid"] = table[0] + dit_config["time_embed_dim"] = table[1] + else: + te = state_dict['{}time_embedder.proj_in.weight'.format(key_prefix)] + dit_config["timestep_input_dim"] = te.shape[1] + dit_config["time_embed_hidden_size"] = te.shape[0] + dit_config["time_embed_dim"] = state_dict['{}time_embedder.proj_out.weight'.format(key_prefix)].shape[0] + dit_config["rope_inv_freq_len"] = state_dict['{}rope.inv_freq'.format(key_prefix)].shape[0] + if metadata is not None and "config" in metadata: + dit_config.update(json.loads(metadata["config"]).get("transformer", {})) + return dit_config + if '{}adaln_single.emb.timestep_embedder.linear_1.bias'.format(key_prefix) in state_dict_keys: #Lightricks ltxv dit_config = {} dit_config["image_model"] = "ltxav" if f'{key_prefix}audio_adaln_single.linear.weight' in state_dict_keys else "ltxv" diff --git a/comfy/model_management.py b/comfy/model_management.py index f7351224d..9f8e7f07b 100644 --- a/comfy/model_management.py +++ b/comfy/model_management.py @@ -34,7 +34,7 @@ import comfy.utils import comfy.quant_ops import comfy_aimdo.host_buffer import comfy_aimdo.vram_buffer -from comfy.logging import detail +from comfy.internal_logging import detail from typing import TYPE_CHECKING if TYPE_CHECKING: @@ -671,6 +671,19 @@ def pin_eviction_tiers(loaded, evict_active): tiers.append((PIN_SUBSETS, True, True)) return tiers +def registration_eviction_tiers(evict_active): + subsets = PIN_SUBSETS + LOADED_PIN_SUBSETS + tiers = [ + (subsets, False, False), + (subsets, True, False), + ] + if evict_active: + tiers.extend([ + (subsets, False, True), + (subsets, True, True), + ]) + return tiers + def free_pins(size, evict_active=False, loaded=False): freed = 0 for subsets, current_prompt, active in pin_eviction_tiers(loaded, evict_active): @@ -684,7 +697,11 @@ def should_free_pins_for_ram_pressure(shortfall): return True if psutil.virtual_memory().available < WINDOWS_PIN_EVICTION_EMERGENCY_AVAILABLE: return True - return psutil.swap_memory().percent >= WINDOWS_PIN_EVICTION_SWAP_PERCENT + try: + return psutil.swap_memory().percent >= WINDOWS_PIN_EVICTION_SWAP_PERCENT + except RuntimeError as err: + logging.warning("Could not read Windows swap usage; falling back to RAM-pressure pin eviction: %s", err) + return True def ensure_pin_budget(size, evict_active=False, loaded=False): if args.high_ram: @@ -699,19 +716,19 @@ def ensure_pin_budget(size, evict_active=False, loaded=False): to_free = shortfall + PIN_PRESSURE_HYSTERESIS return free_pins(to_free, evict_active=evict_active, loaded=loaded) >= shortfall -def free_registrations(shortfall, evict_active=True, loaded=False): +def free_registrations(shortfall, evict_active=True): if MAX_PINNED_MEMORY <= 0: return False if shortfall <= 0: return True shortfall += REGISTERABLE_PIN_HYSTERESIS - for subsets, current_prompt, active in pin_eviction_tiers(loaded, evict_active): + for subsets, current_prompt, active in registration_eviction_tiers(evict_active): shortfall -= free_model_pins(shortfall, subsets, current_prompt, active, registrations=True) return shortfall <= REGISTERABLE_PIN_HYSTERESIS -def ensure_pin_registerable(size, evict_active=True, loaded=False): - return free_registrations(TOTAL_PINNED_MEMORY + size - MAX_PINNED_MEMORY, evict_active=evict_active, loaded=loaded) +def ensure_pin_registerable(size, evict_active=True): + return free_registrations(TOTAL_PINNED_MEMORY + size - MAX_PINNED_MEMORY, evict_active=evict_active) class LoadedModel: def __init__(self, model: ModelPatcher): @@ -1526,13 +1543,31 @@ def cast_to_device(tensor, device, dtype, copy=False): PINNED_MEMORY = {} TOTAL_PINNED_MEMORY = 0 MAX_PINNED_MEMORY = -1 + +def get_disk_swap_total(): + if not os.path.exists("/proc/swaps"): + return 0 + + total = 0 + try: + with open("/proc/swaps", encoding="utf-8") as swaps: + next(swaps, None) + for line in swaps: + filename, _, size, _, _ = line.rsplit(maxsplit=4) + if os.path.basename(os.path.realpath(filename)).startswith("zram"): + continue + total += int(size) * 1024 + except: + logging.warning("Could not get amount of swap memory on system.") + return total + if not args.disable_pinned_memory: if is_nvidia() or is_amd(): ram = get_total_memory(torch.device("cpu")) if WINDOWS: MAX_PINNED_MEMORY = ram * 0.40 # Windows limit is apparently 50% else: - MAX_PINNED_MEMORY = ram * 0.90 + MAX_PINNED_MEMORY = max(ram * 0.40, min(ram * 0.90, ram - 4 * 1024 ** 3, ram + get_disk_swap_total() - 16 * 1024 ** 3)) logging.info("Enabled pinned memory {}".format(MAX_PINNED_MEMORY // (1024 * 1024))) PINNING_ALLOWED_TYPES = set(["Tensor", "Parameter", "QuantizedTensor"]) diff --git a/comfy/model_patcher.py b/comfy/model_patcher.py index e44322e72..ae3f0191d 100644 --- a/comfy/model_patcher.py +++ b/comfy/model_patcher.py @@ -38,7 +38,7 @@ import comfy.patcher_extension import comfy.utils import comfy_aimdo.host_buffer from comfy.comfy_types import UnetWrapperFunction -from comfy.logging import detail +from comfy.internal_logging import detail from comfy.quant_ops import QuantizedTensor from comfy.patcher_extension import CallbacksMP, PatcherInjection, WrappersMP @@ -558,12 +558,9 @@ class ModelPatcher: new_multigpu_models = [] for mm in multigpu_models: # clone main model, but bring over relevant props from existing multigpu clone - n = self.clone() + n = self.clone(model_override=mm.get_clone_model_override()) n.load_device = mm.load_device - n.backup = mm.backup - n.object_patches_backup = mm.object_patches_backup n.hook_backup = mm.hook_backup - n.model = mm.model n.is_multigpu_base_clone = mm.is_multigpu_base_clone n.remove_additional_models("multigpu") orig_additional_models: dict[str, list[ModelPatcher]] = comfy.patcher_extension.copy_nested_dicts(n.additional_models) @@ -1758,6 +1755,9 @@ class ModelPatcherDynamic(ModelPatcher): self.register_load_device(self.load_device) self.non_dynamic_delegate_model = None assert load_device is not None + if not hasattr(self.model, "dynamic_patchers"): + self.model.dynamic_patchers = set() + self.model.dynamic_patchers.add(id(self)) def register_load_device(self, device): """Ensure dynamic_pins has an entry for *device*. @@ -1813,6 +1813,18 @@ class ModelPatcherDynamic(ModelPatcher): def unpin_all_weights(self): self.partially_unload_ram(1e32) + def __del__(self): + model = getattr(self, "model", None) + dynamic_patchers = getattr(model, "dynamic_patchers", None) + if dynamic_patchers is None or id(self) not in dynamic_patchers: + return + dynamic_patchers.discard(id(self)) + try: + if not dynamic_patchers: + self.unpin_all_weights() + finally: + self.detach(unpatch_all=False) + def memory_required(self, input_shape): #Pad this significantly. We are trying to get away from precise estimates. This #estimate is only used when using the ModelPatcherDynamic after ModelPatcher. If you diff --git a/comfy/model_sampling.py b/comfy/model_sampling.py index 5af336e76..7b00d6704 100644 --- a/comfy/model_sampling.py +++ b/comfy/model_sampling.py @@ -325,6 +325,27 @@ class ModelSamplingDiscreteFlow(torch.nn.Module): return 0.0 return time_snr_shift(self.shift, 1.0 - percent) +class ModelSamplingAV(ModelSamplingDiscreteFlow): + """Flow sampling for packed audio-video latents whose audio stream has its own flow shift. + + Carrying the audio latent scaled onto the video schedule makes the pack an ordinary + single-schedule flow latent whose audio target is scaled by audio_scale. + """ + def __init__(self, model_config=None): + super().__init__(model_config) + sampling_settings = model_config.sampling_settings if model_config is not None else {} + self.audio_shift = sampling_settings.get("audio_shift", None) + + def set_parameters(self, shift=1.0, audio_shift=None, timesteps=1000, multiplier=1000): + self.audio_shift = audio_shift + super().set_parameters(shift=shift, timesteps=timesteps, multiplier=multiplier) + + @property + def audio_scale(self): + if self.audio_shift is None: + return 1.0 + return self.shift / self.audio_shift + class StableCascadeSampling(ModelSamplingDiscrete): def __init__(self, model_config=None): super().__init__() diff --git a/comfy/nested_tensor.py b/comfy/nested_tensor.py index b700816fa..43835b15f 100644 --- a/comfy/nested_tensor.py +++ b/comfy/nested_tensor.py @@ -51,6 +51,9 @@ class NestedTensor: def float(self): return self.to(dtype=torch.float) + def cpu(self): + return self.to(device="cpu") + def chunk(self, *args, **kwargs): return self.apply_operation(None, lambda x, y: x.chunk(*args, **kwargs)) @@ -80,6 +83,9 @@ class NestedTensor: def layout(self): return self.tensors[0].layout + def __repr__(self): + return f"{type(self).__name__}({self.tensors!r})" + def cat_nested(tensors, *args, **kwargs): cated_tensors = [] diff --git a/comfy/ops.py b/comfy/ops.py index 9d692dcc7..14599997b 100644 --- a/comfy/ops.py +++ b/comfy/ops.py @@ -19,6 +19,7 @@ import torch import logging import contextlib +import inspect import comfy.model_management from comfy.cli_args import args, PerformanceFeature import comfy.float @@ -36,30 +37,61 @@ def run_every_op(): comfy.model_management.throw_exception_if_processing_interrupted() +def gqa_repeat_factor(query_heads, key_heads, value_heads): + if key_heads != value_heads: + raise ValueError(f"Key/value head count mismatch for GQA: {key_heads} != {value_heads}") + if query_heads == key_heads: + return 1 + if query_heads % key_heads != 0: + raise ValueError(f"Query heads must be divisible by key/value heads for GQA: {query_heads} vs {key_heads}") + return query_heads // key_heads + +def repeat_kv_for_gqa(k, v, query_heads, head_dim): + n_rep = gqa_repeat_factor(query_heads, k.shape[head_dim], v.shape[head_dim]) + if n_rep > 1: + k = k.repeat_interleave(n_rep, dim=head_dim) + v = v.repeat_interleave(n_rep, dim=head_dim) + return k, v + def scaled_dot_product_attention(q, k, v, *args, **kwargs): + attn_mask = args[0] if len(args) > 0 else kwargs.get("attn_mask") + if kwargs.get("enable_gqa", False) and attn_mask is not None: + k, v = repeat_kv_for_gqa(k, v, q.shape[-3], -3) + kwargs["enable_gqa"] = False return torch.nn.functional.scaled_dot_product_attention(q, k, v, *args, **kwargs) try: if torch.cuda.is_available(): from torch.nn.attention import SDPBackend, sdpa_kernel - import inspect if "set_priority" in inspect.signature(sdpa_kernel).parameters: SDPA_BACKEND_PRIORITY = [ SDPBackend.FLASH_ATTENTION, + SDPBackend.CUDNN_ATTENTION, SDPBackend.EFFICIENT_ATTENTION, SDPBackend.MATH, ] - if comfy.model_management.WINDOWS: - SDPA_BACKEND_PRIORITY.insert(0, SDPBackend.CUDNN_ATTENTION) - else: - SDPA_BACKEND_PRIORITY.insert(1, SDPBackend.CUDNN_ATTENTION) - def scaled_dot_product_attention(q, k, v, *args, **kwargs): if q.nelement() < 1024 * 128: # arbitrary number, for small inputs cudnn attention seems slower return torch.nn.functional.scaled_dot_product_attention(q, k, v, *args, **kwargs) + attn_mask = args[0] if len(args) > 0 else kwargs.get("attn_mask") + if kwargs.get("enable_gqa", False) and attn_mask is not None and not comfy.model_management.is_nvidia(): + k, v = repeat_kv_for_gqa(k, v, q.shape[-3], -3) + kwargs["enable_gqa"] = False with sdpa_kernel(SDPA_BACKEND_PRIORITY, set_priority=True): + if kwargs.get("enable_gqa", False) and attn_mask is not None and q.shape[-3] != k.shape[-3]: + dropout_p = args[1] if len(args) > 1 else kwargs.get("dropout_p", 0.0) + is_causal = args[2] if len(args) > 2 else kwargs.get("is_causal", False) + params = torch.backends.cuda.SDPAParams(q, k, v, attn_mask, dropout_p, is_causal, True) + supports_native_gqa = ( + torch.backends.cuda.can_use_flash_attention(params) + or torch.backends.cuda.can_use_cudnn_attention(params) + or torch.backends.cuda.can_use_efficient_attention(params) + ) + if not supports_native_gqa: + k, v = repeat_kv_for_gqa(k, v, q.shape[-3], -3) + kwargs["enable_gqa"] = False return torch.nn.functional.scaled_dot_product_attention(q, k, v, *args, **kwargs) else: logging.warning("Torch version too old to set sdpa backend priority.") @@ -464,8 +496,7 @@ class disable_weight_init: def __init__(self, in_features, out_features, bias=True, device=None, dtype=None): # don't trust subclasses that BYO state dict loader to call us. - if (not comfy.model_management.WINDOWS - or not comfy.memory_management.aimdo_enabled + if (not comfy.memory_management.aimdo_enabled or type(self)._load_from_state_dict is not disable_weight_init.Linear._load_from_state_dict): super().__init__(in_features, out_features, bias, device, dtype) return @@ -487,8 +518,7 @@ class disable_weight_init: def _load_from_state_dict(self, state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs): - if (not comfy.model_management.WINDOWS - or not comfy.memory_management.aimdo_enabled + if (not comfy.memory_management.aimdo_enabled or type(self)._load_from_state_dict is not disable_weight_init.Linear._load_from_state_dict): return super()._load_from_state_dict(state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs) @@ -716,8 +746,7 @@ class disable_weight_init: norm_type=2.0, scale_grad_by_freq=False, sparse=False, _weight=None, _freeze=False, device=None, dtype=None): # don't trust subclasses that BYO state dict loader to call us. - if (not comfy.model_management.WINDOWS - or not comfy.memory_management.aimdo_enabled + if (not comfy.memory_management.aimdo_enabled or type(self)._load_from_state_dict is not disable_weight_init.Embedding._load_from_state_dict): super().__init__(num_embeddings, embedding_dim, padding_idx, max_norm, norm_type, scale_grad_by_freq, sparse, _weight, @@ -744,8 +773,7 @@ class disable_weight_init: def _load_from_state_dict(self, state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs): - if (not comfy.model_management.WINDOWS - or not comfy.memory_management.aimdo_enabled + if (not comfy.memory_management.aimdo_enabled or type(self)._load_from_state_dict is not disable_weight_init.Embedding._load_from_state_dict): return super()._load_from_state_dict(state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs) @@ -917,13 +945,61 @@ if CUBLAS_IS_AVAILABLE: # ============================================================================== # Mixed Precision Operations # ============================================================================== +from . import quant_ops from .quant_ops import ( QuantizedTensor, QUANT_ALGOS, TensorCoreFP8Layout, + TensorWiseINT8Layout, get_layout_class, ) +def _swiglu_eager(x): + gate, up = x.chunk(2, dim=-1) + return torch.nn.functional.silu(gate).mul_(up) + + +INPUT_ACT_EAGER = { + "gelu_tanh": lambda x: torch.nn.functional.gelu(x, approximate="tanh"), + "swiglu": _swiglu_eager, +} + + +def linear_input_act(linear, x, input_act): + """``linear(act(x))``, with ``act`` folded into an INT8 activation quantizer. + + An INT8 linear quantizes its input anyway, so an elementwise activation can + ride along inside that kernel instead of writing a full-size intermediate to + HBM and reading it straight back. Worth it for an MLP's down-projection, + where the intermediate is several times the hidden size. + + """ + weight = linear.weight + if (comfy.model_management.in_training + or not isinstance(weight, QuantizedTensor) + or weight._layout_cls != "TensorWiseINT8Layout" + or getattr(weight._params, "transposed", False)): + return linear(INPUT_ACT_EAGER[input_act](x)) + + # want_requant keeps a vbar-streamed layer on the INT8 path when a LoRA is + # patched in on the fly; without it the cast hands back a dequantized weight. + weight, bias, offload_stream = cast_bias_weight( + linear, x, offloadable=True, compute_dtype=x.dtype, want_requant=True) + try: + if not isinstance(weight, QuantizedTensor): + # A LoRA weight_function, or activations whose dtype differs from the + # weight's, make the cast hand back a dequantized tensor. + return torch.nn.functional.linear(INPUT_ACT_EAGER[input_act](x), weight, bias) + qdata, scale = TensorWiseINT8Layout.get_plain_tensors(weight) + return quant_ops.ck.int8_linear( + x, qdata, scale, bias, x.dtype, + convrot=getattr(weight._params, "convrot", False), + convrot_groupsize=getattr(weight._params, "convrot_groupsize", 256), + input_act=input_act, + ) + finally: + uncast_bias_weight(linear, weight, bias, offload_stream) + class QuantLinearFunc(torch.autograd.Function): """Custom autograd function for quantized linear: quantized forward, optionally FP8 backward. @@ -1135,6 +1211,26 @@ def _load_quantized_module(module, super_load, state_dict, prefix, local_metadat "quant_group_size": 64, "linear_dtype": layer_conf.get("linear_dtype", params_conf.get("linear_dtype", "int4")), } + elif module.quant_format == "asym_w4a8_int8": + # int4 weight (packed int8 [N,K/2]) + fp8 per-group scale (weight_s_rel), + # fp32 per-channel scale (weight_s_channel) + optional Lloyd-Max codebook. + scale = pop_scale("weight_s_rel") + if scale is None: + raise ValueError(f"Missing W4A8 group scale (weight_s_rel) for layer {layer_name}") + if scale.dtype == torch.uint8: + scale = scale.view(torch.float8_e4m3fn) + params_conf = layer_conf.get("params", {}) + if not isinstance(params_conf, dict): + params_conf = {} + scales = { + "scale": scale, + "s_channel": pop_scale("weight_s_channel"), + "codebook": pop_scale("weight_codebook"), + "group_size": int(layer_conf.get("group_size", params_conf.get("group_size", 16))), + "convrot_groupsize": int( + layer_conf.get("convrot_groupsize", params_conf.get("convrot_groupsize", 256)) + ), + } else: raise ValueError(f"Unsupported quantization format: {module.quant_format}") @@ -1186,6 +1282,9 @@ def _quantized_weight_state_dict(module, sd, prefix, extra_quant_conf=None, extr linear_dtype = getattr(params, "linear_dtype", "int4") if linear_dtype != "int4": quant_conf["linear_dtype"] = linear_dtype + elif module.quant_format == "asym_w4a8_int8": + quant_conf["group_size"] = getattr(params, "group_size", 16) + quant_conf["convrot_groupsize"] = getattr(params, "convrot_groupsize", 256) if extra_quant_conf: quant_conf.update(extra_quant_conf) sd[f"{prefix}comfy_quant"] = torch.tensor(list(json.dumps(quant_conf).encode("utf-8")), dtype=torch.uint8) @@ -1233,7 +1332,7 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_prec def state_dict(self, *args, destination=None, prefix="", **kwargs): sd = destination if destination is not None else {} - return _quantized_weight_state_dict(self, sd, prefix, extra_quant_params=("input_scale",)) + return _quantized_weight_state_dict(self, sd, prefix, extra_quant_params=("input_scale", "pre_quant_scale")) def _forward(self, input, weight, bias): return torch.nn.functional.linear(input, weight, bias) @@ -1272,6 +1371,11 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_prec def forward(self, input, *args, **kwargs): run_every_op() + # ModelOpt AWQ-style smoothing + pre_quant_scale = getattr(self, 'pre_quant_scale', None) + if pre_quant_scale is not None: + input = input * comfy.model_management.cast_to_device(pre_quant_scale, input.device, input.dtype) + input_shape = input.shape reshaped_nd = False #If cast needs to apply lora, it should be done in the compute dtype diff --git a/comfy/pinned_memory.py b/comfy/pinned_memory.py index d78ab3c76..e9a9a70e2 100644 --- a/comfy/pinned_memory.py +++ b/comfy/pinned_memory.py @@ -56,7 +56,7 @@ def get_pin(module, subset="weights"): _, _, stack_split, pinned_size, *_ = module._pin_state[subset] size = pin.nbytes - comfy.model_management.ensure_pin_registerable(size, loaded=subset.endswith("-loaded")) + comfy.model_management.ensure_pin_registerable(size) if torch.cuda.cudart().cudaHostRegister(pin.data_ptr(), size, 1) != 0: comfy.model_management.discard_cuda_async_error() @@ -93,7 +93,7 @@ def pin_memory(module, subset="weights", size=None): comfy.memory_management.extra_ram_release(comfy.memory_management.RAM_CACHE_HEADROOM) if (not comfy.model_management.ensure_pin_budget(size, loaded=loaded) or - not comfy.model_management.ensure_pin_registerable(registerable_size, loaded=loaded)): + not comfy.model_management.ensure_pin_registerable(registerable_size)): return _steal_pin(module, stack, buckets, size, priority, subset) offset = hostbuf.size @@ -105,7 +105,7 @@ def pin_memory(module, subset="weights", size=None): pin.untyped_storage()._comfy_hostbuf = hostbuf if torch.cuda.cudart().cudaHostRegister(pin.data_ptr(), size, 1) != 0: comfy.model_management.discard_cuda_async_error() - comfy.model_management.free_registrations(size, loaded=loaded) + comfy.model_management.free_registrations(size) if torch.cuda.cudart().cudaHostRegister(pin.data_ptr(), size, 1) != 0: comfy.model_management.discard_cuda_async_error() del pin diff --git a/comfy/quant_ops.py b/comfy/quant_ops.py index 15f9b1fdb..6d9112dbb 100644 --- a/comfy/quant_ops.py +++ b/comfy/quant_ops.py @@ -28,6 +28,7 @@ try: TensorCoreNVFP4Layout as _CKNvfp4Layout, TensorCoreConvRotW4A4Layout as _CKTensorCoreConvRotW4A4Layout, TensorWiseINT8Layout as _CKTensorWiseINT8Layout, + AsymW4A8Int8Layout as _CKAsymW4A8Int8Layout, register_layout_op, register_layout_class, get_layout_class, @@ -83,6 +84,9 @@ except ImportError as e: class _CKTensorCoreConvRotW4A4Layout: pass + class _CKAsymW4A8Int8Layout: + pass + def register_layout_class(name, cls): pass @@ -212,7 +216,7 @@ class TensorCoreFP8E5M2Layout(_TensorCoreFP8LayoutBase): TensorCoreFP8Layout = TensorCoreFP8E4M3Layout TensorWiseINT8Layout = _CKTensorWiseINT8Layout TensorCoreConvRotW4A4Layout = _CKTensorCoreConvRotW4A4Layout - +AsymW4A8Int8Layout = _CKAsymW4A8Int8Layout # ============================================================================== # Registry @@ -226,6 +230,7 @@ register_layout_class("TensorWiseINT8Layout", _CKTensorWiseINT8Layout) register_layout_class("TensorCoreConvRotW4A4Layout", _CKTensorCoreConvRotW4A4Layout) if _CK_MXFP8_AVAILABLE: register_layout_class("TensorCoreMXFP8Layout", TensorCoreMXFP8Layout) +register_layout_class("AsymW4A8Int8Layout", _CKAsymW4A8Int8Layout) QUANT_ALGOS = { "float8_e4m3fn": { @@ -240,7 +245,7 @@ QUANT_ALGOS = { }, "nvfp4": { "storage_t": torch.uint8, - "parameters": {"weight_scale", "weight_scale_2", "input_scale"}, + "parameters": {"weight_scale", "weight_scale_2", "input_scale", "pre_quant_scale"}, "comfy_tensor_layout": "TensorCoreNVFP4Layout", "group_size": 16, }, @@ -268,6 +273,13 @@ QUANT_ALGOS["convrot_w4a4"] = { "quantize_input": False, } +QUANT_ALGOS["asym_w4a8_int8"] = { + "storage_t": torch.int8, + "parameters": {"weight_scale"}, + "comfy_tensor_layout": "AsymW4A8Int8Layout", + "quantize_input": False, +} + # ============================================================================== # Re-exports for backward compatibility @@ -282,6 +294,7 @@ __all__ = [ "TensorCoreNVFP4Layout", "TensorCoreConvRotW4A4Layout", "TensorWiseINT8Layout", + "AsymW4A8Int8Layout", "QUANT_ALGOS", "register_layout_op", ] diff --git a/comfy/samplers.py b/comfy/samplers.py index 9f571ece9..1d6a4e104 100755 --- a/comfy/samplers.py +++ b/comfy/samplers.py @@ -20,7 +20,7 @@ import comfy.hooks import comfy.context_windows import comfy.multigpu import comfy.utils -from comfy.logging import detail +from comfy.internal_logging import detail import scipy.stats import numpy @@ -997,7 +997,7 @@ class KSAMPLER(Sampler): def k_callback(x): nonlocal first_step if first_step: - detail("First sampler step: model=%s sampler=%s step=%s total_steps=%s cfg=%s seed=%s sigma=%s sigma_hat=%s latent_shape=%s denoised_shape=%s", model_wrap.model_patcher.model.__class__.__name__, self.sampler_function.__name__, x["i"], total_steps, model_wrap.cfg, extra_args.get("seed"), x.get("sigma"), x.get("sigma_hat"), tuple(x["x"].shape), tuple(x["denoised"].shape)) + detail("First sampler step: model=%s sampler=%s step=%s total_steps=%s cfg=%s seed=%s sigma=%s sigma_hat=%s latent_shape=%s denoised_shape=%s", model_wrap.model_patcher.model.__class__.__name__, getattr(self.sampler_function, "__name__", "unknown"), x["i"], total_steps, model_wrap.cfg, extra_args.get("seed"), x.get("sigma"), x.get("sigma_hat"), tuple(x["x"].shape), tuple(x["denoised"].shape)) first_step = False if callback is not None: callback(x["i"], x["denoised"], x["x"], total_steps) @@ -1218,6 +1218,8 @@ class CFGGuider: return sampling_function(self.inner_model, x, timestep, self.conds.get("negative", None), self.conds.get("positive", None), self.cfg, model_options=model_options, seed=seed) def inner_sample(self, noise, latent_image, device, sampler, sigmas, denoise_mask, callback, disable_pbar, seed, latent_shapes=None): + self.inner_model.latent_shapes = latent_shapes + if latent_image is not None and torch.count_nonzero(latent_image) > 0: #Don't shift the empty latent image. latent_image = self.inner_model.process_latent_in(latent_image) @@ -1284,6 +1286,14 @@ class CFGGuider: sampler_shapes = [tuple(latent_image.shape)] detail("Sampler: model=%s latent_shapes=%s", self.model_patcher.model.__class__.__name__, sampler_shapes) + if len(latent_shapes) > 1 and callback is not None: + # samplers run on the flat pack, hand callbacks (previews, x0 output) the nested view + packed_callback = callback + def callback(step, x0, x, total_steps): + x0 = comfy.nested_tensor.NestedTensor(comfy.utils.unpack_latents(x0, latent_shapes)) + x = comfy.nested_tensor.NestedTensor(comfy.utils.unpack_latents(x, latent_shapes)) + return packed_callback(step, x0, x, total_steps) + if denoise_mask is not None: if denoise_mask.is_nested: denoise_masks = denoise_mask.unbind() diff --git a/comfy/sd.py b/comfy/sd.py index caf78222d..9ccd561bc 100644 --- a/comfy/sd.py +++ b/comfy/sd.py @@ -72,6 +72,9 @@ import comfy.text_encoders.ace15 import comfy.text_encoders.longcat_image import comfy.text_encoders.qwen35 import comfy.text_encoders.qwen3vl +import comfy.text_encoders.minimax +import comfy.ldm.minimax.vae +import comfy.ldm.minimax.audio_vae import comfy.text_encoders.boogu import comfy.text_encoders.ernie import comfy.text_encoders.gemma4 @@ -936,6 +939,54 @@ class VAE: #Force cast it for --disable-dynamic-vram users until there is a true core fix. if not comfy.memory_management.aimdo_enabled: self.disable_offload = True + elif "decoder.transformer_blocks.0.scale1" in sd and "encoder.down.5.block.0.conv1.weight" in sd: # MiniMax H3 video VAE + minimax_ops = comfy.ops.disable_weight_init + minimax_quant = comfy.utils.detect_layer_quantization(sd, "") + if minimax_quant is not None: # int8+convrot quantized decoder + minimax_ops = comfy.ops.mixed_precision_ops(minimax_quant, dtype if dtype is not None else torch.float16) + self.first_stage_model = comfy.ldm.minimax.vae.MiniMaxH3VideoVAE(operations=minimax_ops) + self.latent_channels = 24 + self.latent_dim = 3 + # frames 17k+5 <-> latents 5k+2, 16x spatial + self.upscale_ratio = (lambda a: max(1, (a - 2) // 5 * 17 + 5), 16, 16) + self.upscale_index_formula = (4, 16, 16) + self.downscale_ratio = (lambda a: max(1, (a - 5) // 17 * 5 + 2) if a > 1 else 1, 16, 16) + self.downscale_index_formula = (4, 16, 16) + self.working_dtypes = [torch.float16, torch.float32] + # the model tiles internally (256px spatial, 17-frame temporal chunks) + self.handles_tiling = True + def estimate_encode_memory(frames, height, width, dtype): + fixed = 110_000_000 if frames == 1 else 1_300_000_000 + elements_per_pixel = 7 if frames == 1 else 9.5 + return (elements_per_pixel * frames * height * width + fixed) * model_management.dtype_size(dtype) * 1.03 + + def estimate_decode_memory(frames, height, width, dtype): + fixed = 110_000_000 if frames <= 22 else 270_000_000 + return (9.5 * frames * height * width + fixed) * model_management.dtype_size(dtype) * 1.03 + + self.memory_used_encode = lambda shape, dtype: estimate_encode_memory(shape[2], shape[3], shape[4], dtype) + self.memory_used_decode = lambda shape, dtype: estimate_decode_memory(self.upscale_ratio[0](shape[2]), shape[3] * self.upscale_ratio[1], shape[4] * self.upscale_ratio[2], dtype) + elif "pre_block.attn.zero_k_bias" in sd: # MiniMax H3 audio VAE (DAC encoder + BigVGAN decoder) + self.first_stage_model = comfy.ldm.minimax.audio_vae.MiniMaxH3AudioVAE() + self.latent_channels = 32 + self.output_channels = 2 + self.pad_channel_value = "replicate" + self.audio_sample_rate = 32000 + self.upscale_ratio = 800 + self.downscale_ratio = 800 + self.latent_dim = 2 # [B, 32, stereo 2, T] + self.process_output = lambda audio: audio + self.process_input = lambda audio: audio + self.working_dtypes = [torch.float32] + # encode gets the waveform shape [B, 2, samples], decode the latent shape [B, 32, 2, T] + def estimate_encode_memory(samples, dtype): + return (900 * samples + 105_000_000) * model_management.dtype_size(dtype) * 1.03 + + def estimate_decode_memory(samples, dtype): + return max(42_000_000, 220 * samples + 20_000_000) * model_management.dtype_size(dtype) * 1.03 + + self.memory_used_encode = lambda shape, dtype: estimate_encode_memory(shape[2], dtype) + self.memory_used_decode = lambda shape, dtype: estimate_decode_memory(shape[-1] * self.upscale_ratio, dtype) elif "gs.base_offset_scale" in sd and "octree.out_proj.weight" in sd: # TripoSplat octree gaussian decoder self.first_stage_model = comfy.ldm.triposplat.vae.OctreeGaussianDecoder() self.latent_channels = 16 @@ -1393,6 +1444,7 @@ class CLIPType(Enum): KREA2 = 32 JOYIMAGE = 33 MAGE = 34 + MINIMAX = 35 @@ -1449,6 +1501,7 @@ class TEModel(Enum): QWEN3VL_4B = 34 QWEN3VL_8B = 35 GEMMA_4_12B = 36 + QWEN3VL_32B = 37 def detect_te_model(sd): @@ -1515,6 +1568,9 @@ def detect_te_model(sd): return TEModel.QWEN35_2B if "model.visual.deepstack_merger_list.0.norm.weight" in sd: # DeepStack is unique to Qwen3-VL return TEModel.QWEN3VL_4B if sd["model.visual.merger.linear_fc2.weight"].shape[0] == 2560 else TEModel.QWEN3VL_8B + if "visual.deepstack_merger_list.0.norm.weight" in sd and "model.layers.49.self_attn.q_proj.weight" in sd: + # MiniMax H3 conditioning encoder: Qwen3-VL-32B, truncated to 50 layers + return TEModel.QWEN3VL_32B if "model.layers.0.post_attention_layernorm.weight" in sd: weight = sd['model.layers.0.post_attention_layernorm.weight'] if 'model.layers.0.self_attn.q_norm.weight' in sd: @@ -1744,6 +1800,9 @@ def load_text_encoder_state_dicts(state_dicts=[], embedding_directory=None, clip qwen3vl_type = {TEModel.QWEN3VL_4B: "qwen3vl_4b", TEModel.QWEN3VL_8B: "qwen3vl_8b"}[te_model] clip_target.clip = comfy.text_encoders.qwen3vl.te(**llama_detect(clip_data), model_type=qwen3vl_type) clip_target.tokenizer = comfy.text_encoders.qwen3vl.tokenizer(model_type=qwen3vl_type) + elif te_model == TEModel.QWEN3VL_32B: + clip_target.clip = comfy.text_encoders.minimax.te(**llama_detect(clip_data)) + clip_target.tokenizer = comfy.text_encoders.minimax.MiniMaxH3Tokenizer elif te_model == TEModel.QWEN3_06B: clip_target.clip = comfy.text_encoders.anima.te(**llama_detect(clip_data)) clip_target.tokenizer = comfy.text_encoders.anima.AnimaTokenizer diff --git a/comfy/supported_models.py b/comfy/supported_models.py index ca89850a5..b9952db55 100644 --- a/comfy/supported_models.py +++ b/comfy/supported_models.py @@ -15,6 +15,7 @@ import comfy.text_encoders.flux import comfy.text_encoders.genmo import comfy.text_encoders.lt import comfy.text_encoders.hunyuan_video +import comfy.text_encoders.minimax import comfy.text_encoders.cosmos import comfy.text_encoders.lumina2 import comfy.text_encoders.wan @@ -955,6 +956,34 @@ class LTXAV(LTXV): out = model_base.LTXAV(self, device=device) return out +class MiniMaxH3(supported_models_base.BASE): + unet_config = { + "image_model": "minimax_h3", + } + + sampling_settings = { + "shift": 12.0, + "audio_shift": 3.0, + } + + unet_extra_config = {} + latent_format = latent_formats.MiniMaxH3AV + + memory_usage_factor = 0.114 + + supported_inference_dtypes = [torch.bfloat16, torch.float32] + + vae_key_prefix = ["vae."] + text_encoder_key_prefix = ["text_encoders."] + + def get_model(self, state_dict, prefix="", device=None): + return model_base.MiniMaxH3(self, device=device) + + def clip_target(self, state_dict={}, prefix=""): + pref = self.text_encoder_key_prefix[0] + detect = comfy.text_encoders.hunyuan_video.llama_detect(state_dict, "{}qwen3vl_32b.transformer.".format(pref)) + return supported_models_base.ClipTarget(comfy.text_encoders.minimax.MiniMaxH3Tokenizer, comfy.text_encoders.minimax.te(**detect)) + class HunyuanVideo(supported_models_base.BASE): unet_config = { "image_model": "hunyuan_video", @@ -1421,6 +1450,20 @@ class WAN22_Animate(WAN21_T2V): out = model_base.WAN22_Animate(self, device=device) return out +class WAN_Animate2(WAN21_T2V): + unet_config = { + "image_model": "wan2.1", + "model_type": "animate2", + } + + sampling_settings = { + "shift": 5.0, + } + + def get_model(self, state_dict, prefix="", device=None): + out = model_base.WAN_Animate2(self, device=device) + return out + class WAN22_T2V(WAN21_T2V): unet_config = { "image_model": "wan2.1", @@ -2407,6 +2450,7 @@ models = [ GenmoMochi, LTXV, LTXAV, + MiniMaxH3, HunyuanVideo15_SR_Distilled, HunyuanVideo15, HunyuanImage21Refiner, @@ -2434,6 +2478,7 @@ models = [ WAN22_S2V, WAN21_HuMo, WAN22_Animate, + WAN_Animate2, WAN21_FlowRVS, WAN21_SCAIL, WAN21_SCAIL2, diff --git a/comfy/text_encoders/llama.py b/comfy/text_encoders/llama.py index 40d04007e..f5c5597ef 100644 --- a/comfy/text_encoders/llama.py +++ b/comfy/text_encoders/llama.py @@ -264,6 +264,17 @@ class Qwen3VL_4BConfig(Qwen3VL_8BConfig): intermediate_size: int = 9728 lm_head: bool = False # 4B ties word embeddings +@dataclass +class Qwen3VL_32BConfig(Qwen3VL_8BConfig): + # MiniMax H3 conditioning checkpoint: truncated to the first 50 of 64 layers, + # consumed as the unnormalized hidden state after layer 50 (no final norm, no lm_head) + hidden_size: int = 5120 + intermediate_size: int = 25600 + num_hidden_layers: int = 50 + num_attention_heads: int = 64 + lm_head: bool = False + final_norm: bool = False + @dataclass class Ovis25_2BConfig: vocab_size: int = 151936 diff --git a/comfy/text_encoders/minimax.py b/comfy/text_encoders/minimax.py new file mode 100644 index 000000000..c2dc47f7f --- /dev/null +++ b/comfy/text_encoders/minimax.py @@ -0,0 +1,201 @@ +"""MiniMax H3 text/vision conditioning: Qwen3-VL-32B (truncated to 50 layers). + +The H3 presentation is NOT chat-templated: token ids are raw prompt/label text +(no special tokens) with explicit vision blocks spliced in: + + t2va: + fl2va: ": " [": " ] + ref2va: per condition in request order (1-based ordinals per type): + image -> ": " + audio -> "