feat: Support MiniMax-H3 (CORE-375) (#15224)

comfy/latent_formats.py

@@ -567,6 +567,45 @@ class LTXAV(LTXV):
         ]
         self.latent_rgb_factors_bias = [-0.347892, -0.363814, -0.370287]
 
+class MiniMaxH3Video(LatentFormat):
+    latent_channels = 24
+    latent_dimensions = 3
+    spacial_downscale_ratio = 16
+    temporal_downscale_ratio = 4
+    scale_factor = 1.0
+
+    latent_rgb_factors = [
+        [-0.018555,  0.024344, -0.017536],
+        [ 0.150164,  0.137244,  0.129221],
+        [ 0.027367, -0.050369, -0.208606],
+        [-0.000793, -0.164622, -0.323161],
+        [-0.048556,  0.013970, -0.074286],
+        [ 0.011740,  0.014172, -0.006906],
+        [ 0.061517,  0.061212,  0.110025],
+        [ 0.035321,  0.086879,  0.110059],
+        [-0.017426,  0.002997,  0.035356],
+        [ 0.531539,  0.548819,  0.624404],
+        [-0.024968, -0.040234, -0.034302],
+        [-0.032549, -0.029096, -0.017221],
+        [ 0.022609,  0.020286,  0.050661],
+        [-0.084001, -0.038131, -0.020805],
+        [-0.018830,  0.010412,  0.061120],
+        [ 0.020777,  0.011196, -0.030994],
+        [-0.008390, -0.012201, -0.025687],
+        [-0.013281, -0.002924,  0.006331],
+        [ 0.000260,  0.001833, -0.011038],
+        [ 0.105471,  0.100482,  0.132106],
+        [ 0.016529,  0.015213,  0.009999],
+        [-0.014015, -0.017438, -0.019134],
+        [-0.033787, -0.009984, -0.019725],
+        [ 0.004224,  0.017284,  0.027196],
+    ]
+    latent_rgb_factors_bias = [ 0.057426, -0.022078, -0.071449]
+
+class MiniMaxH3AV(MiniMaxH3Video):
+    # max channels across the two streams (video 24, audio 32) so per-stream slices keep both streams whole
+    latent_channels = 32
+
 class HunyuanVideo(LatentFormat):
     latent_channels = 16
     latent_dimensions = 3

comfy/ldm/minimax/audio_vae.py

@@ -0,0 +1,442 @@
+# MiniMax H3 audio VAE: DAC-lineage waveform encoder + BigVGAN decoder.
+# Weight-norm parametrizations are folded into plain conv weights, so this
+# module uses ordinary ops.Conv1d / ops.ConvTranspose1d and loads the converted
+# checkpoint (plain "*.weight" tensors) with strict=True.
+#
+# Lineage / licenses of the reference implementation:
+#   DAC encoder:      descript-audio-codec (MIT)
+#   BigVGAN decoder:  NVIDIA BigVGAN (MIT), adapted from hifi-gan (MIT)
+#   Alias-free ops:   junjun3518/alias-free-torch (Apache-2.0), julius (MIT)
+
+import math
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+import comfy.ops
+
+ops = comfy.ops.disable_weight_init
+
+
+# Snake activations
+
+def snake(x, alpha, beta):
+    # x + 1/beta * sin^2(alpha * x)
+    t = torch.sin(alpha * x)
+    return t.mul_(t).mul_((beta + 1e-9).reciprocal()).add_(x)
+
+
+class Snake1d(nn.Module):
+    """Snake activation with per-channel alpha (encoder side)."""
+
+    def __init__(self, channels):
+        super().__init__()
+        self.alpha = nn.Parameter(torch.empty(1, channels, 1))
+
+    def forward(self, x):
+        return snake(x, self.alpha, self.alpha)
+
+
+class SnakeBeta(nn.Module):
+    """SnakeBeta := x + 1/beta * sin^2(alpha * x); alpha/beta stored in log scale."""
+
+    def __init__(self, in_features):
+        super().__init__()
+        self.alpha = nn.Parameter(torch.empty(in_features))
+        self.beta = nn.Parameter(torch.empty(in_features))
+
+    def forward(self, x):
+        alpha = torch.exp(self.alpha).view(1, -1, 1)
+        beta = torch.exp(self.beta).view(1, -1, 1)
+        return snake(x, alpha, beta)
+
+
+# Alias-free (anti-aliased) activation: kaiser-windowed sinc resampling
+
+def kaiser_sinc_filter1d(cutoff, half_width, kernel_size):
+    # returns filter [1, 1, kernel_size]
+    even = kernel_size % 2 == 0
+    half_size = kernel_size // 2
+
+    # kaiser window design
+    delta_f = 4 * half_width
+    A = 2.285 * (half_size - 1) * math.pi * delta_f + 7.95
+    if A > 50.0:
+        beta = 0.1102 * (A - 8.7)
+    elif A >= 21.0:
+        beta = 0.5842 * (A - 21) ** 0.4 + 0.07886 * (A - 21.0)
+    else:
+        beta = 0.0
+    window = torch.kaiser_window(kernel_size, beta=beta, periodic=False)
+
+    if even:
+        time = torch.arange(-half_size, half_size) + 0.5
+    else:
+        time = torch.arange(kernel_size) - half_size
+
+    filter_ = 2 * cutoff * window * torch.sinc(2 * cutoff * time)
+    # Normalize filter to have sum = 1, otherwise there is a small leakage of
+    # the constant component in the input signal.
+    filter_ /= filter_.sum()
+    return filter_.view(1, 1, kernel_size)
+
+
+class UpSample1d(nn.Module):
+    def __init__(self, ratio=2, kernel_size=12):
+        super().__init__()
+        self.ratio = ratio
+        self.stride = ratio
+        self.pad = kernel_size // ratio - 1
+        self.pad_left = self.pad * ratio + (kernel_size - ratio) // 2
+        self.pad_right = self.pad * ratio + (kernel_size - ratio + 1) // 2
+        self.register_buffer(
+            "filter",
+            kaiser_sinc_filter1d(cutoff=0.5 / ratio, half_width=0.6 / ratio, kernel_size=kernel_size),
+        )
+
+    def forward(self, x):
+        _, C, _ = x.shape
+        x = F.pad(x, (self.pad, self.pad), mode="replicate")
+        x = F.conv_transpose1d(x, self.filter.expand(C, -1, -1).to(x.dtype), stride=self.stride, groups=C).mul_(self.ratio)
+        x = x[..., self.pad_left:-self.pad_right]
+        return x
+
+
+class LowPassFilter1d(nn.Module):
+    def __init__(self, cutoff=0.5, half_width=0.6, stride=1, kernel_size=12):
+        super().__init__()
+        self.pad_left = kernel_size // 2 - int(kernel_size % 2 == 0)
+        self.pad_right = kernel_size // 2
+        self.stride = stride
+        self.register_buffer("filter", kaiser_sinc_filter1d(cutoff, half_width, kernel_size))
+
+    def forward(self, x):
+        _, C, _ = x.shape
+        x = F.pad(x, (self.pad_left, self.pad_right), mode="replicate")
+        return F.conv1d(x, self.filter.expand(C, -1, -1).to(x.dtype), stride=self.stride, groups=C)
+
+
+class DownSample1d(nn.Module):
+    def __init__(self, ratio=2, kernel_size=12):
+        super().__init__()
+        self.ratio = ratio
+        self.kernel_size = kernel_size
+        self.lowpass = LowPassFilter1d(
+            cutoff=0.5 / ratio,
+            half_width=0.6 / ratio,
+            stride=ratio,
+            kernel_size=self.kernel_size,
+        )
+
+    def forward(self, x):
+        return self.lowpass(x)
+
+
+class Activation1d(nn.Module):
+    """upsample x2 -> pointwise activation -> downsample x2 (anti-aliased)."""
+
+    def __init__(self, activation, up_ratio=2, down_ratio=2, up_kernel_size=12, down_kernel_size=12):
+        super().__init__()
+        self.act = activation
+        self.upsample = UpSample1d(up_ratio, up_kernel_size)
+        self.downsample = DownSample1d(down_ratio, down_kernel_size)
+
+    def forward(self, x):
+        x = self.upsample(x)
+        x = self.act(x)
+        x = self.downsample(x)
+        return x
+
+
+# DAC encoder
+
+class ResidualUnit(nn.Module):
+    def __init__(self, dim=16, dilation=1):
+        super().__init__()
+        pad = ((7 - 1) * dilation) // 2
+        self.block = nn.Sequential(
+            Snake1d(dim),
+            ops.Conv1d(dim, dim, kernel_size=7, dilation=dilation, padding=pad),
+            Snake1d(dim),
+            ops.Conv1d(dim, dim, kernel_size=1),
+        )
+
+    def forward(self, x):
+        y = self.block(x)
+        pad = (x.shape[-1] - y.shape[-1]) // 2
+        if pad > 0:
+            x = x[..., pad:-pad]
+        return y.add_(x)
+
+
+class EncoderBlock(nn.Module):
+    def __init__(self, dim=16, stride=1):
+        super().__init__()
+        self.block = nn.Sequential(
+            ResidualUnit(dim // 2, dilation=1),
+            ResidualUnit(dim // 2, dilation=3),
+            ResidualUnit(dim // 2, dilation=9),
+            Snake1d(dim // 2),
+            ops.Conv1d(
+                dim // 2,
+                dim,
+                kernel_size=2 * stride,
+                stride=stride,
+                padding=math.ceil(stride / 2),
+            ),
+        )
+
+    def forward(self, x):
+        return self.block(x)
+
+
+class Encoder(nn.Module):
+    def __init__(self, d_model=64, strides=(2, 4, 4, 5, 5), d_latent=2048):
+        super().__init__()
+        block = [ops.Conv1d(1, d_model, kernel_size=7, padding=3)]
+        for stride in strides:
+            d_model *= 2
+            block += [EncoderBlock(d_model, stride=stride)]
+        block += [
+            Snake1d(d_model),
+            ops.Conv1d(d_model, d_latent, kernel_size=3, padding=1),
+        ]
+        self.block = nn.Sequential(*block)
+
+    def forward(self, x):
+        return self.block(x)
+
+
+# Attention projection (encoder posterior head)
+
+class GeGluMlp(nn.Module):
+    def __init__(self, in_features, hidden_features):
+        super().__init__()
+        self.norm = ops.LayerNorm(in_features)
+        self.act = nn.GELU(approximate="tanh")
+        self.w0 = ops.Linear(in_features, hidden_features)
+        self.w1 = ops.Linear(in_features, hidden_features)
+        self.w2 = ops.Linear(hidden_features, in_features)
+
+    def forward(self, x):
+        x = self.norm(x)
+        return self.w2(self.act(self.w0(x)).mul_(self.w1(x)))
+
+
+class CausalAttention(nn.Module):
+    def __init__(self, in_dim, out_dim, num_heads):
+        super().__init__()
+        self.head_dim = in_dim // num_heads
+        self.num_heads = num_heads
+        self.out_dim = out_dim
+        self.qkv = ops.Linear(in_dim, in_dim * 3, bias=False)
+        self.q_bias = nn.Parameter(torch.empty(in_dim))
+        self.v_bias = nn.Parameter(torch.empty(in_dim))
+        self.register_buffer("zero_k_bias", torch.empty(in_dim))
+        self.proj = ops.Linear(out_dim, out_dim)
+
+    def forward(self, x):
+        B, N, C = x.shape
+        weight, _, offload_stream = comfy.ops.cast_bias_weight(self.qkv, x, offloadable=True)
+        qkv = F.linear(x, weight=weight, bias=torch.cat((self.q_bias, self.zero_k_bias, self.v_bias)))
+        comfy.ops.uncast_bias_weight(self.qkv, weight, None, offload_stream)
+        q, k, v = qkv.reshape(B, N, 3, self.num_heads, self.head_dim).permute(2, 0, 3, 1, 4).unbind(0)
+
+        # mean over heads then pool down to the latent width (in_dim >> out_dim)
+        x = comfy.ops.scaled_dot_product_attention(q, k, v, is_causal=True)
+        x = F.adaptive_avg_pool1d(torch.mean(x, dim=1), self.out_dim)
+        return self.proj(x)
+
+
+class AttnProjection(nn.Module):
+    def __init__(self, in_dim, out_dim, num_heads, mlp_ratio=2):
+        super().__init__()
+        self.norm1 = ops.LayerNorm(in_dim)
+        self.attn = CausalAttention(in_dim, out_dim, num_heads)
+        self.proj = ops.Linear(in_dim, out_dim)
+        self.norm3 = ops.LayerNorm(in_dim)
+
+        self.norm2 = ops.LayerNorm(out_dim)
+        hidden_dim = int(out_dim * mlp_ratio)
+        self.mlp = GeGluMlp(in_features=out_dim, hidden_features=hidden_dim)
+
+    def forward(self, x):
+        # x: [B, T, in_dim]
+        x = self.proj(self.norm3(x)).add_(self.attn(self.norm1(x)))
+        return x.add_(self.mlp(self.norm2(x)))
+
+
+# BigVGAN decoder
+
+def get_padding(kernel_size, dilation=1):
+    return int((kernel_size * dilation - dilation) / 2)
+
+
+class AMPBlock1(nn.Module):
+    def __init__(self, channels, kernel_size=3, dilation=(1, 3, 5)):
+        super().__init__()
+        self.convs1 = nn.ModuleList(
+            [
+                ops.Conv1d(channels, channels, kernel_size, stride=1, dilation=d, padding=get_padding(kernel_size, d))
+                for d in dilation
+            ]
+        )
+        self.convs2 = nn.ModuleList(
+            [
+                ops.Conv1d(channels, channels, kernel_size, stride=1, dilation=1, padding=get_padding(kernel_size, 1))
+                for _ in range(len(dilation))
+            ]
+        )
+        self.num_layers = len(self.convs1) + len(self.convs2)
+        self.activations = nn.ModuleList(
+            [Activation1d(activation=SnakeBeta(channels)) for _ in range(self.num_layers)]
+        )
+
+    def forward(self, x):
+        acts1, acts2 = self.activations[::2], self.activations[1::2]
+        for c1, c2, a1, a2 in zip(self.convs1, self.convs2, acts1, acts2):
+            xt = a1(x)
+            xt = c1(xt)
+            xt = a2(xt)
+            xt = c2(xt)
+            x = xt.add_(x)
+        return x
+
+
+class BigVGAN(nn.Module):
+    """BigVGAN vocoder (MiniMax H3 32 kHz configuration).
+
+    use_bias_at_final=False, use_tanh_at_final=False (output clamped to [-1, 1]).
+    """
+
+    def __init__(
+        self,
+        num_mels=2048,
+        upsample_initial_channel=1024,
+        upsample_rates=(5, 5, 2, 2, 2, 2, 2),
+        upsample_kernel_sizes=(9, 9, 4, 4, 4, 4, 4),
+        resblock_kernel_sizes=(3, 7, 11),
+        resblock_dilation_sizes=((1, 3, 5), (1, 3, 5), (1, 3, 5)),
+    ):
+        super().__init__()
+        self.num_kernels = len(resblock_kernel_sizes)
+        self.num_upsamples = len(upsample_rates)
+
+        self.conv_pre = ops.Conv1d(num_mels, upsample_initial_channel, 7, 1, padding=3)
+
+        self.ups = nn.ModuleList()
+        for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)):
+            self.ups.append(
+                nn.ModuleList(
+                    [
+                        ops.ConvTranspose1d(
+                            upsample_initial_channel // (2 ** i),
+                            upsample_initial_channel // (2 ** (i + 1)),
+                            k,
+                            u,
+                            padding=(k - u) // 2,
+                        )
+                    ]
+                )
+            )
+
+        self.resblocks = nn.ModuleList()
+        for i in range(len(self.ups)):
+            ch = upsample_initial_channel // (2 ** (i + 1))
+            for k, d in zip(resblock_kernel_sizes, resblock_dilation_sizes):
+                self.resblocks.append(AMPBlock1(ch, k, d))
+
+        self.activation_post = Activation1d(activation=SnakeBeta(ch))
+        self.conv_post = ops.Conv1d(ch, 1, 7, 1, padding=3, bias=False)
+
+    def forward(self, x):
+        x = self.conv_pre(x)
+
+        for i in range(self.num_upsamples):
+            for i_up in range(len(self.ups[i])):
+                x = self.ups[i][i_up](x)
+            xs = None
+            for j in range(self.num_kernels):
+                if xs is None:
+                    xs = self.resblocks[i * self.num_kernels + j](x)
+                else:
+                    xs += self.resblocks[i * self.num_kernels + j](x)
+            x = xs.div_(self.num_kernels)
+
+        x = self.activation_post(x)
+        return self.conv_post(x).clamp_(-1.0, 1.0)
+
+
+# Top-level VAE
+
+class MiniMaxH3AudioVAE(nn.Module):
+    """MiniMax H3 stereo audio VAE at 32 kHz.
+
+    Latents are [B, 32, 2, T]: 32 channels, 2 stereo channels, T frames at
+    40 latent frames per second (800 audio samples per latent frame). The
+    stereo channels are processed independently by the mono encoder/decoder.
+    Latents are normalized with the stored per-channel latents_mean/std.
+    """
+
+    def __init__(
+        self,
+        encoder_dim=64,
+        encoder_rates=(2, 4, 4, 5, 5),
+        latent_dim=2048,
+        decoder_dim=1024,
+        vae_latent_channels=32,
+    ):
+        super().__init__()
+        self.sample_rate = 32000
+
+        self.hop_length = 1
+        for r in encoder_rates:
+            self.hop_length *= r
+        self.samples_per_latent = self.hop_length  # 800
+        self.latents_per_second = self.sample_rate // self.hop_length  # 40
+        self.output_sample_rate = self.sample_rate  # read by LTXVAudioVAEDecode
+
+        self.encoder = Encoder(encoder_dim, encoder_rates, latent_dim)
+
+        self.pre_block = AttnProjection(latent_dim, vae_latent_channels, num_heads=8)
+
+        self.mean_proj = ops.Conv1d(vae_latent_channels, vae_latent_channels, 1)
+        # logs_proj exists in the checkpoint but is unused at inference
+        # (encode returns the posterior mean, no sampling).
+        self.logs_proj = ops.Conv1d(vae_latent_channels, vae_latent_channels, 1)
+
+        self.dec_in_proj = ops.Conv1d(vae_latent_channels, latent_dim, 1)
+        self.decoder = BigVGAN(num_mels=latent_dim, upsample_initial_channel=decoder_dim)
+
+        self.register_buffer("latents_mean", torch.empty(vae_latent_channels))
+        self.register_buffer("latents_std", torch.empty(vae_latent_channels))
+
+    def decode(self, z):
+        """Decode normalized latents [B, 32, 2, T] to stereo waveforms [B, 2, L] at 32 kHz."""
+        b, c, s, t = z.shape
+        z = z.permute(0, 2, 1, 3).reshape(b * s, c, t)
+        mean = self.latents_mean.view(1, -1, 1).to(device=z.device, dtype=z.dtype)
+        std = self.latents_std.view(1, -1, 1).to(device=z.device, dtype=z.dtype)
+        z = z * std + mean
+        x = self.dec_in_proj(z)
+        x = self.decoder(x)  # [b * s, 1, L], already clamped to [-1, 1]
+        return x.reshape(b, s, -1)
+
+    def encode(self, waveform):
+        """Encode stereo waveforms [B, 2, L] at 32 kHz (in [-1, 1]) to normalized latents [B, 32, 2, T].
+
+        L is right-padded with zeros to a multiple of 800 samples; the returned
+        posterior mean is used directly (no sampling).
+        """
+        b, s, length = waveform.shape
+        right_pad = math.ceil(length / self.hop_length) * self.hop_length - length
+        waveform = F.pad(waveform, (0, right_pad))
+        x = waveform.reshape(b * s, 1, -1)
+        x = self.encoder(x)  # [b * s, latent_dim, T]
+        x = self.pre_block(x.transpose(1, 2)).transpose(1, 2)  # [b * s, 32, T]
+        z = self.mean_proj(x)
+        mean = self.latents_mean.view(1, -1, 1).to(device=z.device, dtype=z.dtype)
+        std = self.latents_std.view(1, -1, 1).to(device=z.device, dtype=z.dtype)
+        z = (z - mean) / std
+        return z.reshape(b, s, z.shape[1], z.shape[2]).permute(0, 2, 1, 3)

comfy/ldm/minimax/model.py

@@ -0,0 +1,646 @@
+"""MiniMax H3 audio-video DiT.
+
+Single-stream packed-token transformer denoising video (24ch, patch 1x2x2) and
+stereo audio (32ch, 40 Hz) latents jointly, conditioned on Qwen3-VL layer-50 hidden states.
+The packed sequence is:
+[text | cond rows | audio | video] for t2va/fl2va
+[text | reference blocks | audio | video] for ref2va
+
+Timestep domain: the model receives the *video* sigma from the sampler and
+derives per-token timesteps t = 1 - sigma internally; the audio stream runs on
+its own shifted schedule (sigma_shift video 12.0 / audio 3.0), mapped from the
+video sigma in closed form. The audio velocity is returned scaled by the
+schedule map's derivative d(sigma_a)/d(sigma_v).
+"""
+
+import math
+
+import torch
+import torch.nn as nn
+
+import comfy.ldm.common_dit
+import comfy.model_management
+import comfy.model_prefetch
+import comfy.ops
+import comfy.patcher_extension
+import comfy.quant_ops
+from comfy.ldm.modules.attention import optimized_attention
+
+FRAME_PER_TOKEN = (1, 4, 4, 4, 4)
+FRAME_RESCALE = 5.0 / 3.0
+VISUAL_COND_TIMESTEP = 0.999
+AUDIO_COND_TIMESTEP = 1.0
+
+
+def time_shift_sigma(sigma, from_shift, to_shift):
+    # invert sigma = s*b/(1+(s-1)*b) to the base grid, re-apply the other shift
+    base = sigma / (from_shift + sigma * (1.0 - from_shift))
+    return to_shift * base / (1.0 + (to_shift - 1.0) * base)
+
+
+def time_shift_slope(sigma, from_shift, to_shift):
+    """d(sigma_to)/d(sigma_from) at the same base-grid point.
+
+    Scaling a stream's returned velocity by this slope makes the flat ODE that
+    any sampler integrates on the from-schedule equal to that stream's true ODE
+    on its own schedule.
+    """
+    base = sigma / (from_shift + sigma * (1.0 - from_shift))
+    return (to_shift * (1.0 + (from_shift - 1.0) * base) ** 2) / (from_shift * (1.0 + (to_shift - 1.0) * base) ** 2)
+
+
+def patchify_video(latent, patch_size=(1, 2, 2)):
+    # [B, C, T, H, W] -> [B*t*h*w, C*pt*ph*pw]
+    b, c, t_full, h_full, w_full = latent.shape
+    pt, ph, pw = patch_size
+    t, h, w = t_full // pt, h_full // ph, w_full // pw
+    x = latent.reshape(b, c, t, pt, h, ph, w, pw)
+    x = torch.einsum("nctrhpwq->nthwcrpq", x)
+    return x.reshape(b * t * h * w, c * pt * ph * pw)
+
+
+def unpatchify_video(rows, t, h, w, c=24, patch_size=(1, 2, 2)):
+    pt, ph, pw = patch_size
+    x = rows.reshape(-1, t, h, w, c, pt, ph, pw)
+    x = torch.einsum("nthwcrpq->nctrhpwq", x)
+    return x.reshape(-1, c, t * pt, h * ph, w * pw)
+
+
+def pack_audio(latent):
+    # [B, C=32, ch=2, T] -> [ch*T, 32] channel-major (ch0 t0..T-1, ch1 t0..T-1)
+    b, c, ch, t = latent.shape
+    return latent[0].permute(1, 2, 0).reshape(ch * t, c)
+
+
+def unpack_audio(rows, ch=2):
+    t = rows.shape[0] // ch
+    return rows.reshape(ch, t, rows.shape[-1]).permute(2, 0, 1).unsqueeze(0)
+
+
+def _axis_from_sqrt_area(dim, patch, sqrt_area):
+    # linspace((1 - ratio) / 2, (1 + ratio) / 2, dim // patch, endpoint=False) * 32
+    ratio = dim / sqrt_area
+    n = dim // patch
+    return (torch.arange(n, dtype=torch.float64) * (ratio / n) + (1.0 - ratio) / 2.0) * 32.0
+
+
+def _frame_grid(h, w):
+    # area-normalized (h, w) coordinates of one latent frame's 2x2-patch rows
+    area = math.sqrt(h * w)
+    hh, ww = torch.meshgrid(_axis_from_sqrt_area(h, 2, area), _axis_from_sqrt_area(w, 2, area), indexing="ij")
+    return torch.stack([hh.reshape(-1), ww.reshape(-1)], dim=-1), _axis_from_sqrt_area(w, 2, area)
+
+
+def _video_t_spans(n):
+    return [FRAME_RESCALE * FRAME_PER_TOKEN[k % 5] for k in range(n)]
+
+
+def _video_t_grid(n, origin):
+    # origin + exclusive cumsum
+    spans = torch.tensor(_video_t_spans(n), dtype=torch.float64)
+    return float(origin) + torch.cat([torch.zeros(1, dtype=torch.float64), spans[:-1].cumsum(0)])
+
+
+def _audio_grid(cursor, t, w_low, w_high):
+    # channel-major stereo rows: t advances per latent frame, w pinned to the grid extremes per stereo channel, h stays 0
+    g = torch.zeros(t * 2, 3, dtype=torch.float64)
+    g[:, 0] = (cursor + torch.arange(t, dtype=torch.float64)).repeat(2)
+    g[:t, 2] = w_low
+    g[t:, 2] = w_high
+    return g
+
+
+def _video_grid(vt, frame, cursor):
+    g = torch.empty(vt, frame.shape[0], 3, dtype=torch.float64)
+    g[:, :, 0] = _video_t_grid(vt, cursor)[:, None]
+    g[:, :, 1:] = frame[None]
+    return g.reshape(-1, 3)
+
+
+class TimeEmbedder(nn.Module):
+    def __init__(self, freq_dim, hidden, out, dtype=None, device=None, operations=None):
+        super().__init__()
+        self.freq_dim = freq_dim
+        self.proj_in = operations.Linear(freq_dim, hidden, bias=True, dtype=dtype, device=device)
+        self.proj_out = operations.Linear(hidden, out, bias=True, dtype=dtype, device=device)
+
+    def forward(self, t):
+        # t: [M] in [0, 1]; fp32 throughout, cos before sin
+        half = self.freq_dim // 2
+        freqs = torch.exp(-math.log(10000.0) * torch.arange(half, dtype=torch.float32, device=t.device) / half)
+        args = t.to(torch.float32)[:, None] * freqs[None]
+        emb = torch.cat([torch.cos(args), torch.sin(args)], dim=-1)
+        return self.proj_out(nn.functional.silu(self.proj_in(emb)))
+
+
+def rope_rotation_table(angles, dtype):
+    """[S, rot_dim] pair angles -> [1, S, 1, rot_dim/2, 2, 2] rotation matrices."""
+    half = angles.shape[-1] // 2
+    ang = angles[:, :half]  # duplicated halves: [:, :half] == [:, half:]
+    c, s = torch.cos(ang), torch.sin(ang)
+    table = torch.stack([c, -s, s, c], dim=-1).reshape(1, angles.shape[0], 1, half, 2, 2)
+    return table.to(dtype)
+
+
+class Attention(nn.Module):
+    def __init__(self, hidden, heads, head_dim, eps, dtype=None, device=None, operations=None):
+        super().__init__()
+        self.heads = heads
+        self.head_dim = head_dim
+        inner = heads * head_dim
+        self.qkv_proj = operations.Linear(hidden, inner * 3, bias=False, dtype=dtype, device=device)
+        self.q_norm = operations.RMSNorm(head_dim, eps=eps, dtype=dtype, device=device)
+        self.k_norm = operations.RMSNorm(head_dim, eps=eps, dtype=dtype, device=device)
+        self.out_proj = operations.Linear(inner, hidden, bias=False, dtype=dtype, device=device)
+
+    def forward(self, x, rope_freqs=None, transformer_options={}):
+        s = x.shape[0]
+        q, k, v = self.qkv_proj(x).split(self.heads * self.head_dim, dim=-1)
+        v = v.view(s, self.heads, self.head_dim)
+        if rope_freqs is not None:
+            # fused per-head RMSNorm + partial split-half rope, in place on the qkv buffer
+            q = q.view(1, s, self.heads, self.head_dim)
+            k = k.view(1, s, self.heads, self.head_dim)
+            qw = comfy.model_management.cast_to(self.q_norm.weight, device=x.device)
+            kw = comfy.model_management.cast_to(self.k_norm.weight, device=x.device)
+            rot = rope_freqs.shape[-3] * 2
+            if comfy.model_management.in_training:
+                q, k = comfy.quant_ops.ck.rms_rope_split_half(
+                    q, k, rope_freqs, qw, kw, epsilon=self.q_norm.eps, rot_dim=rot)
+            else:
+                comfy.quant_ops.ck.rms_rope_split_half_(
+                    q, k, rope_freqs, qw, kw, epsilon=self.q_norm.eps, rot_dim=rot)
+            q = q[0]
+            k = k[0]
+        else:
+            q = self.q_norm(q.view(s, self.heads, self.head_dim))
+            k = self.k_norm(k.view(s, self.heads, self.head_dim))
+        q = q.transpose(0, 1).unsqueeze(0)
+        k = k.transpose(0, 1).unsqueeze(0)
+        v = v.transpose(0, 1).unsqueeze(0)
+        out = optimized_attention(q, k, v, self.heads, mask=None, skip_reshape=True, transformer_options=transformer_options)
+        return self.out_proj(out.squeeze(0))
+
+
+class MLP(nn.Module):
+    def __init__(self, hidden, ffn, dtype=None, device=None, operations=None):
+        super().__init__()
+        self.fc1 = operations.Linear(hidden, ffn * 2, bias=False, dtype=dtype, device=device)
+        self.fc2 = operations.Linear(ffn, hidden, bias=False, dtype=dtype, device=device)
+
+    def forward(self, x):
+        return comfy.ops.linear_input_act(self.fc2, self.fc1(x), "swiglu")
+
+
+class AdalnProj(nn.Module):
+    def __init__(self, t_dim, hidden, expand, modalities, apply_silu=True,
+                 dtype=None, device=None, operations=None):
+        super().__init__()
+        self.expand = expand
+        self.modalities = modalities
+        self.hidden = hidden
+        self.apply_silu = apply_silu
+        self.linear = operations.Linear(t_dim, expand * hidden * modalities, bias=True, dtype=dtype, device=device)
+
+    def forward(self, t_emb):
+        # [M, t_dim] -> expand tensors of [M*modalities, hidden]
+        x = self.linear(nn.functional.silu(t_emb) if self.apply_silu else t_emb)
+        x = x.view(x.shape[0] * self.modalities, self.expand * self.hidden)
+        return x.chunk(self.expand, dim=-1)
+
+
+def _mod_scale_shift(h, shift, scale, segments):
+    # segments: [(start, stop, mod_row)] covering h contiguously.
+    for a, b, row in segments:
+        h[a:b].mul_(1.0 + scale[row].to(h.dtype)).add_(shift[row].to(h.dtype))
+    return h
+
+
+def _mod_gate(x, gate, other, segments):
+    # other is the fresh attn/mlp output: accumulate the gated residual into the stream in place, one fused kernel per segment
+    for a, b, row in segments:
+        x[a:b].addcmul_(other[a:b], gate[row].to(x.dtype))
+    return x
+
+
+class RefinerBlock(nn.Module):
+    def __init__(self, hidden, heads, head_dim, ffn, eps, qk_eps, dtype=None, device=None, operations=None):
+        super().__init__()
+        self.norm1 = operations.RMSNorm(hidden, eps=eps, dtype=dtype, device=device)
+        self.norm2 = operations.RMSNorm(hidden, eps=eps, dtype=dtype, device=device)
+        self.attn = Attention(hidden, heads, head_dim, qk_eps, dtype=dtype, device=device, operations=operations)
+        self.mlp = MLP(hidden, ffn, dtype=dtype, device=device, operations=operations)
+
+    def forward(self, x, transformer_options={}):
+        # attn/mlp outputs are fresh: accumulate residuals in place
+        x = self.attn(self.norm1(x), transformer_options=transformer_options).add_(x)
+        return self.mlp(self.norm2(x)).add_(x)
+
+
+class TokenRefiner(nn.Module):
+    def __init__(self, num_layers, hidden, heads, head_dim, ffn, eps, qk_eps, final_eps,
+                 dtype=None, device=None, operations=None):
+        super().__init__()
+        self.blocks = nn.ModuleList([
+            RefinerBlock(hidden, heads, head_dim, ffn, eps, qk_eps, dtype=dtype, device=device, operations=operations)
+            for _ in range(num_layers)])
+        self.final_norm = operations.RMSNorm(hidden, eps=final_eps, dtype=dtype, device=device)
+
+    def forward(self, x, transformer_options={}):
+        for block in self.blocks:
+            x = block(x, transformer_options=transformer_options)
+        return self.final_norm(x)
+
+
+class DiTBlock(nn.Module):
+    def __init__(self, hidden, heads, head_dim, ffn, t_dim, eps, qk_eps,
+                 apply_silu=True, adaln_dtype=None, dtype=None, device=None, operations=None):
+        super().__init__()
+        self.norm1 = operations.RMSNorm(hidden, eps=eps, dtype=dtype, device=device)
+        self.norm2 = operations.RMSNorm(hidden, eps=eps, dtype=dtype, device=device)
+        self.attn = Attention(hidden, heads, head_dim, qk_eps, dtype=dtype, device=device, operations=operations)
+        self.mlp = MLP(hidden, ffn, dtype=dtype, device=device, operations=operations)
+        self.adaln_proj = AdalnProj(t_dim, hidden, 6, 3, apply_silu=apply_silu,
+                                    dtype=adaln_dtype if adaln_dtype is not None else dtype,
+                                    device=device, operations=operations)
+
+    def forward(self, x, t_emb, mod_segments, rope_freqs, transformer_options={}):
+        shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.adaln_proj(t_emb)
+        h = _mod_scale_shift(self.norm1(x), shift_msa, scale_msa, mod_segments)
+        x = _mod_gate(x, gate_msa, self.attn(h, rope_freqs=rope_freqs, transformer_options=transformer_options), mod_segments)
+        h = _mod_scale_shift(self.norm2(x), shift_mlp, scale_mlp, mod_segments)
+        return _mod_gate(x, gate_mlp, self.mlp(h), mod_segments)
+
+
+class FinalLayer(nn.Module):
+    def __init__(self, hidden, t_dim, video_dim, audio_dim, eps, apply_silu=True, adaln_dtype=None,
+                 dtype=None, device=None, operations=None):
+        super().__init__()
+        self.norm = operations.RMSNorm(hidden, eps=eps, dtype=dtype, device=device)
+        self.adaln_proj = AdalnProj(t_dim, hidden, 2, 1, apply_silu=apply_silu,
+                                    dtype=adaln_dtype if adaln_dtype is not None else dtype,
+                                    device=device, operations=operations)
+        # output heads are the checkpoint's fp32 island; norm/adaln are stored at model dtype
+        self.video_out = operations.Linear(hidden, video_dim, bias=True, dtype=torch.float32, device=device)
+        self.audio_out = operations.Linear(hidden, audio_dim, bias=True, dtype=torch.float32, device=device)
+
+    def forward(self, x, t_emb, video_seg, audio_seg):
+        # video_seg / audio_seg: (start, stop, timestep_row) of the target streams
+        shift, scale = self.adaln_proj(t_emb)
+        va, vb, vrow = video_seg
+        aa, ab, arow = audio_seg
+        hv = (self.norm(x[va:vb]) * (1.0 + scale[vrow]) + shift[vrow]).to(torch.float32)
+        ha = (self.norm(x[aa:ab]) * (1.0 + scale[arow]) + shift[arow]).to(torch.float32)
+        return self.video_out(hv), self.audio_out(ha)
+
+
+class PackedLayout:
+    """Static packed-sequence structure for one shape/conditioning signature."""
+
+    def __init__(self, text_len, latent_t, latent_h, latent_w, audio_t, keyframes=None, refs=None, frame_count=None):
+        frame, w_grid = _frame_grid(latent_h, latent_w)
+        frame_rows = frame.shape[0]
+
+        segments = [("text", text_len)]  # (kind, n_rows)
+        g = torch.zeros(text_len, 3, dtype=torch.float64)
+        g[:, 0] = torch.arange(text_len, dtype=torch.float64)
+        pos = [g]  # per segment: [n, 3] float64 (t, h, w)
+
+        img_pos, img_update = [], []
+        audio_pos, audio_update = [], []
+        cursor = text_len
+        row = text_len
+
+        if keyframes:
+            # fl2va: keyframe cond rows right after text, sharing the target spatial grid
+            for kf in keyframes:
+                pixel_index = kf["resolved_frame_index"]
+                if pixel_index == 0:
+                    cond_t = float(text_len)
+                elif frame_count is not None and pixel_index == frame_count - 1:
+                    cond_t = float(text_len) + sum(_video_t_spans(latent_t)) - FRAME_RESCALE
+                else:
+                    raise ValueError("only first/last keyframe anchors are supported")
+                g = torch.empty(frame_rows, 3, dtype=torch.float64)
+                g[:, 0] = cond_t
+                g[:, 1:] = frame
+                segments.append(("cond", frame_rows))
+                pos.append(g)
+                img_pos.append(torch.arange(row, row + frame_rows))
+                img_update.append(torch.zeros(frame_rows, dtype=torch.bool))
+                row += frame_rows
+
+        target_audio_w = (float(w_grid[0]), float(w_grid[-1]))
+        if refs:
+            cursor = float(text_len)
+            for blk in refs:
+                kind = blk["kind"]
+                if kind == "image":
+                    r_frame, _ = _frame_grid(blk["latent_h"], blk["latent_w"])
+                    n = r_frame.shape[0]
+                    g = torch.empty(n, 3, dtype=torch.float64)
+                    g[:, 0] = cursor
+                    g[:, 1:] = r_frame
+                    segments.append(("ref_img", n))
+                    pos.append(g)
+                    img_pos.append(torch.arange(row, row + n))
+                    img_update.append(torch.zeros(n, dtype=torch.bool))
+                    row += n
+                    cursor += 1.0
+                elif kind == "audio":
+                    rt = blk["ref_audio_t"]
+                    if rt > 0:
+                        segments.append(("ref_audio", rt * 2))
+                        pos.append(_audio_grid(cursor, rt, *target_audio_w))
+                        audio_pos.append(torch.arange(row, row + rt * 2))
+                        audio_update.append(torch.zeros(rt * 2, dtype=torch.bool))
+                        row += rt * 2
+                    cursor += float(rt)
+                elif kind in ("video", "video_audio"):
+                    # the block's audio rows pack immediately before its video
+                    # rows, both sharing the cursor origin
+                    rt = blk["ref_audio_t"]
+                    vt = blk["latent_t"]
+                    r_frame, r_w_grid = _frame_grid(blk["latent_h"], blk["latent_w"])
+                    if rt > 0:
+                        segments.append(("ref_audio", rt * 2))
+                        pos.append(_audio_grid(cursor, rt, float(r_w_grid[0]), float(r_w_grid[-1])))
+                        audio_pos.append(torch.arange(row, row + rt * 2))
+                        audio_update.append(torch.zeros(rt * 2, dtype=torch.bool))
+                        row += rt * 2
+                    n = vt * r_frame.shape[0]
+                    segments.append(("ref_img", n))
+                    pos.append(_video_grid(vt, r_frame, cursor))
+                    img_pos.append(torch.arange(row, row + n))
+                    img_update.append(torch.zeros(n, dtype=torch.bool))
+                    row += n
+                    cursor += max(float(rt), sum(_video_t_spans(vt)))
+
+        # target audio then target video, always the last two segments
+        segments.append(("audio", audio_t * 2))
+        pos.append(_audio_grid(cursor, audio_t, *target_audio_w))
+        audio_pos.append(torch.arange(row, row + audio_t * 2))
+        audio_update.append(torch.ones(audio_t * 2, dtype=torch.bool))
+        row += audio_t * 2
+
+        n_video = latent_t * frame_rows
+        segments.append(("video", n_video))
+        pos.append(_video_grid(latent_t, frame, cursor))
+        img_pos.append(torch.arange(row, row + n_video))
+        img_update.append(torch.ones(n_video, dtype=torch.bool))
+        row += n_video
+
+        self.seq_len = row
+        self.position_ids = torch.cat(pos)  # [S, 3] float64
+        self.img_pos = torch.cat(img_pos)
+        self.img_update = torch.cat(img_update)
+        self.audio_pos = torch.cat(audio_pos)
+        self.audio_update = torch.cat(audio_update)
+        self.signature = (text_len, latent_t, latent_h, latent_w, audio_t)
+        # contiguous segment table (start, stop, kind)
+        # kinds: text / cond / ref_img / ref_audio / audio / video
+        # the packed sequence is uniform per segment in (modality tag, timestep class),
+        # except the text span (tag runs resolved at forward time from the presentation tags)
+        seg_abs = []
+        off = 0
+        for kind, n in segments:
+            seg_abs.append((off, off + n, kind))
+            off += n
+        self.segments = seg_abs
+
+
+class MiniMaxH3Model(nn.Module):
+    def __init__(self, hidden_size=5376, num_layers=50, token_refiner_num_layers=2,
+                 num_attention_heads=56, attention_head_dim=128, ffn_hidden_size=14336,
+                 latents_dim=24, audio_latents_dim=32, patch_size=(1, 2, 2), text_dim=5120,
+                 timestep_input_dim=256, time_embed_hidden_size=5376, time_embed_dim=2688,
+                 rope_inv_freq_len=16, norm_eps=1e-5, qk_norm_eps=1e-5, final_norm_eps=1e-5,
+                 sigma_shift_video=12.0, sigma_shift_audio=3.0,
+                 adaln_curve_grid=None,
+                 image_model=None, dtype=None, device=None, operations=None, **kwargs):
+        super().__init__()
+        self.dtype = dtype
+        self.hidden_size = hidden_size
+        self.patch_size = tuple(patch_size)
+        self.latents_dim = latents_dim
+        self.audio_latents_dim = audio_latents_dim
+        self.sigma_shift_video = sigma_shift_video
+        self.sigma_shift_audio = sigma_shift_audio
+        self.use_adaln_curves = adaln_curve_grid is not None
+        # curve-form checkpoints replace the time embedder and full-width adaln weights with a small shared basis of the time-embedding curve
+        curve = {"apply_silu": not self.use_adaln_curves,
+                 "adaln_dtype": torch.float32 if self.use_adaln_curves else dtype}
+        video_patch_dim = latents_dim * self.patch_size[0] * self.patch_size[1] * self.patch_size[2]
+
+        self.video_patch_proj = operations.Linear(video_patch_dim, hidden_size, bias=True, dtype=torch.float32, device=device)
+        self.audio_patch_proj = operations.Linear(audio_latents_dim, hidden_size, bias=True, dtype=torch.float32, device=device)
+        self.condition_proj = operations.Linear(text_dim, hidden_size, bias=True, dtype=dtype, device=device)
+        if self.use_adaln_curves:
+            self.register_buffer("adaln_t_table", torch.empty(adaln_curve_grid, time_embed_dim, dtype=torch.float32))
+        else:
+            self.time_embedder = TimeEmbedder(timestep_input_dim, time_embed_hidden_size, time_embed_dim,
+                                              dtype=torch.float32, device=device, operations=operations)
+        self.rope = nn.Module()
+        self.rope.register_buffer("inv_freq", torch.empty(rope_inv_freq_len, dtype=torch.float32))
+        self.token_refiner = TokenRefiner(token_refiner_num_layers, hidden_size, num_attention_heads,
+                                          attention_head_dim, ffn_hidden_size, norm_eps, qk_norm_eps,
+                                          final_norm_eps, dtype=dtype, device=device, operations=operations)
+        self.blocks = nn.ModuleList([
+            DiTBlock(hidden_size, num_attention_heads, attention_head_dim, ffn_hidden_size,
+                     time_embed_dim, norm_eps, qk_norm_eps, **curve, dtype=dtype, device=device, operations=operations)
+            for _ in range(num_layers)])
+        self.final_layer = FinalLayer(hidden_size, time_embed_dim, video_patch_dim, audio_latents_dim,
+                                      final_norm_eps, **curve, dtype=dtype, device=device, operations=operations)
+
+    def preprocess_text_embeds(self, text_states):
+        """[B, L, text_dim] Qwen states -> [B, L, hidden] refined text embeds."""
+        if text_states.shape[-1] == self.hidden_size:
+            return text_states
+        return self.token_refiner(self.condition_proj(text_states[0])).unsqueeze(0)
+
+    def rope_freqs(self, position_ids, device):
+        # [S, 3] float64 -> [S, 96] fp32
+        pos = position_ids.to(torch.float32).to(device)
+        inv = comfy.model_management.cast_to(self.rope.inv_freq, device=device)
+        per_axis = pos.unsqueeze(-1) * inv.view(1, 1, -1)      # [S, 3, 16]
+        t_f, h_f, w_f = per_axis.unbind(dim=1)
+        half = torch.cat((t_f, h_f, w_f), dim=-1)              # [S, 48]
+        return torch.cat((half, half), dim=-1)                 # [S, 96]
+
+    def _cond_video_rows(self, payload, device):
+        """Concatenated visual condition rows (normalized latents -> patchified), with condition noise augmentation."""
+        rows = []
+        aug = payload.get("visual_cond_noise_aug", VISUAL_COND_TIMESTEP)
+        seed = int(payload.get("seed", 0))
+        # every condition intentionally restarts the same RNG stream
+        for z in payload.get("cond_video_latents", []):
+            r = patchify_video(z.to(torch.float32), self.patch_size)
+            if aug < 1.0:
+                gen = torch.Generator("cpu").manual_seed(seed)
+                noise = torch.randn(r.shape, generator=gen, dtype=torch.float32)
+                r = aug * r + (1.0 - aug) * noise.to(r.device)
+            rows.append(r.to(device))
+        return torch.cat(rows, dim=0) if rows else None
+
+    def _cond_audio_rows(self, payload, device):
+        rows = []
+        aug = payload.get("audio_cond_noise_aug", AUDIO_COND_TIMESTEP)
+        seed = int(payload.get("seed", 0)) + 1
+        for z in payload.get("cond_audio_latents", []):
+            r = pack_audio(z.to(torch.float32))
+            if aug < 1.0:
+                gen = torch.Generator("cpu").manual_seed(seed)
+                noise = torch.randn(r.shape, generator=gen, dtype=torch.float32)
+                r = aug * r + (1.0 - aug) * noise.to(r.device)
+            rows.append(r.to(device))
+        return torch.cat(rows, dim=0) if rows else None
+
+    def forward(self, x, timestep, context, transformer_options={}, minimax_payload=None, **kwargs):
+        return comfy.patcher_extension.WrapperExecutor.new_class_executor(
+            self._forward,
+            self,
+            comfy.patcher_extension.get_all_wrappers(comfy.patcher_extension.WrappersMP.DIFFUSION_MODEL, transformer_options)
+        ).execute(x, timestep, context, transformer_options, minimax_payload=minimax_payload, **kwargs)
+
+    def _forward(self, x, timestep, context, transformer_options={}, minimax_payload=None, **kwargs):
+        video_x, audio_x = x[0], x[1]
+        orig_t, orig_h, orig_w = video_x.shape[2], video_x.shape[3], video_x.shape[4]
+        video_x = comfy.ldm.common_dit.pad_to_patch_size(video_x, self.patch_size)
+        if video_x.shape[0] != 1:
+            raise ValueError("MiniMax H3 supports batch size 1")
+        payload = minimax_payload or {}
+        device = video_x.device
+        dtype = context.dtype  # compute dtype
+
+        latent_t, lat_h, lat_w = video_x.shape[2], video_x.shape[3], video_x.shape[4]
+        audio_t = audio_x.shape[-1]
+        text_len = context.shape[1]
+        # extra_conds prebuilds the layout once per sampling run
+        layout = payload.get("layout")
+        if layout is None or layout.signature != (text_len, latent_t, lat_h, lat_w, audio_t):
+            layout = PackedLayout(text_len, latent_t, lat_h, lat_w, audio_t,
+                                  keyframes=payload.get("keyframes"),
+                                  refs=payload.get("refs"),
+                                  frame_count=payload.get("frame_count"))
+
+        # model_base passes model_sampling.timestep(sigma) = sigma * 1000
+        shift_v = float(transformer_options.get("minimax_h3_sigma_shift_video", self.sigma_shift_video))
+        shift_a = float(transformer_options.get("minimax_h3_sigma_shift_audio", self.sigma_shift_audio))
+        sigma_v = (timestep.flatten()[0] / 1000.0).float().clamp(min=1e-6)
+        t_v = float(1.0 - sigma_v)
+        t_a = float(1.0 - time_shift_sigma(sigma_v, shift_v, shift_a))
+
+        # distinct timesteps are known analytically: text/pad follow video, cond rows pin near 1
+        vis_aug = float(payload.get("visual_cond_noise_aug", VISUAL_COND_TIMESTEP))
+        aud_aug = float(payload.get("audio_cond_noise_aug", AUDIO_COND_TIMESTEP))
+        has_vis_cond = any(k in ("cond", "ref_img") for _, _, k in layout.segments)
+        has_aud_cond = any(k == "ref_audio" for _, _, k in layout.segments)
+        seg_t = {"text": t_v, "video": t_v, "audio": t_a,
+                 "cond": max(t_v, vis_aug), "ref_img": max(t_v, vis_aug),
+                 "ref_audio": max(t_a, aud_aug)}
+        unique_t = sorted({t_v, t_a} | ({seg_t["cond"]} if has_vis_cond else set())
+                          | ({seg_t["ref_audio"]} if has_aud_cond else set()))
+        t_row = {t: i for i, t in enumerate(unique_t)}
+        seg_tag = {"text": 1, "video": 0, "audio": 2, "cond": 0, "ref_img": 0, "ref_audio": 2}
+
+        text_tags = payload.get("text_token_tags")
+        mod_segments = []
+        for a, b, kind in layout.segments:
+            row_base = t_row[seg_t[kind]] * 3
+            if kind == "text" and text_tags is not None:
+                # the presentation text span mixes tags (vision pads carry the video modality) split into tag runs
+                tags = text_tags.view(-1).tolist()
+                run_start = 0
+                for i in range(1, b - a + 1):
+                    if i == b - a or tags[i] != tags[run_start]:
+                        mod_segments.append((a + run_start, a + i, row_base + int(tags[run_start])))
+                        run_start = i
+            else:
+                mod_segments.append((a, b, row_base + seg_tag[kind]))
+
+        # embed
+        img_update = layout.img_update.to(device)
+        audio_update = layout.audio_update.to(device)
+        video_rows = patchify_video(video_x.to(torch.float32), self.patch_size)
+        audio_rows = pack_audio(audio_x.to(torch.float32))
+        cond_video_rows = self._cond_video_rows(payload, device)
+        cond_audio_rows = self._cond_audio_rows(payload, device)
+
+        all_video_rows = video_rows
+        if cond_video_rows is not None:
+            all_video_rows = torch.empty(img_update.shape[0], video_rows.shape[1], dtype=torch.float32, device=device)
+            all_video_rows[~img_update] = cond_video_rows
+            all_video_rows[img_update] = video_rows
+        all_audio_rows = audio_rows
+        if cond_audio_rows is not None:
+            all_audio_rows = torch.empty(audio_update.shape[0], audio_rows.shape[1], dtype=torch.float32, device=device)
+            all_audio_rows[~audio_update] = cond_audio_rows
+            all_audio_rows[audio_update] = audio_rows
+
+        video_embed = self.video_patch_proj(all_video_rows).to(dtype)
+        audio_embed = self.audio_patch_proj(all_audio_rows).to(dtype)
+        text_states = context[0]
+        if text_states.shape[-1] != self.hidden_size:
+            text_states = self.token_refiner(self.condition_proj(text_states),
+                                             transformer_options=transformer_options)
+
+        # segments are contiguous: assemble by slices, embed rows follow segment order
+        h = torch.empty(layout.seq_len, self.hidden_size, dtype=dtype, device=device)
+        voff = aoff = 0
+        for a, b, kind in layout.segments:
+            n = b - a
+            if kind == "text":
+                h[a:b] = text_states
+            elif kind in ("cond", "ref_img", "video"):
+                h[a:b] = video_embed[voff:voff + n]
+                voff += n
+            else:  # ref_audio / audio
+                h[a:b] = audio_embed[aoff:aoff + n]
+                aoff += n
+
+        t_vals = torch.tensor(unique_t, dtype=torch.float32, device=device)
+        if self.use_adaln_curves:
+            # adaln projections consume interpolated coordinates of the time-embedding curve
+            table = comfy.model_management.cast_to(self.adaln_t_table, device=device)
+            pos = t_vals.clamp(0.0, 1.0) * (table.shape[0] - 1)     # t in [0,1] -> fractional grid index, out-of-range t clamps to the curve ends
+            i0 = pos.floor().long().clamp(max=table.shape[0] - 2)   # lower grid row, max-clamp keeps t=1.0 on the last interval instead of reading past the table
+            t_emb = torch.lerp(table[i0], table[i0 + 1], (pos - i0).unsqueeze(1))  # blend the two rows by the fractional part
+        else:
+            t_emb = self.time_embedder(t_vals).to(dtype)
+
+        # rotation table computed once per forward, consumed by the kitchen split-half rope
+        rope_freqs = rope_rotation_table(self.rope_freqs(layout.position_ids, device), dtype)
+
+        # blocks
+        patches_replace = transformer_options.get("patches_replace", {})
+        blocks_replace = patches_replace.get("dit", {})
+        prefetch_queue = comfy.model_prefetch.make_prefetch_queue(list(self.blocks), device, transformer_options)
+        for i, block in enumerate(self.blocks):
+            comfy.model_prefetch.prefetch_queue_pop(prefetch_queue, device, block)
+            if ("double_block", i) in blocks_replace:
+                def block_wrap(args):
+                    return {"img": block(args["img"], args["t_emb"], args["mod_segments"], args["rope_freqs"],
+                                         transformer_options=args["transformer_options"])}
+                h = blocks_replace[("double_block", i)](
+                    {"img": h, "t_emb": t_emb, "mod_segments": mod_segments, "rope_freqs": rope_freqs,
+                     "transformer_options": transformer_options},
+                    {"original_block": block_wrap})["img"]
+            else:
+                h = block(h, t_emb, mod_segments, rope_freqs, transformer_options=transformer_options)
+        if prefetch_queue is not None:
+            comfy.model_prefetch.prefetch_queue_pop(prefetch_queue, device, None)
+
+        # target streams are single contiguous segments (audio then video, last two)
+        video_seg = next((a, b, t_row[seg_t["video"]]) for a, b, k in layout.segments if k == "video")
+        audio_seg = next((a, b, t_row[seg_t["audio"]]) for a, b, k in layout.segments if k == "audio")
+        v, a = self.final_layer(h, t_emb, video_seg, audio_seg)
+
+        video_out = unpatchify_video(v, latent_t, lat_h // 2, lat_w // 2, self.latents_dim, self.patch_size)
+        video_out = video_out[:, :, :orig_t, :orig_h, :orig_w]
+        audio_out = unpack_audio(a)
+
+        # The sampler integrates the flat ODE dX/dsigma_v = (X - denoised)/sigma_v.
+        # Scaling the audio velocity by d(sigma_a)/d(sigma_v) makes that ODE equal
+        # to the audio stream's true ODE on its own shifted schedule.
+        slope_a = time_shift_slope(sigma_v, shift_v, shift_a).to(audio_out.dtype)
+        return [-video_out.to(video_x.dtype), (-slope_a) * audio_out.to(audio_x.dtype)]

comfy/ldm/minimax/vae.py

@@ -0,0 +1,694 @@
+# MiniMax H3 video VAE: 3D causal CNN encoder + ViT3D decoder.
+
+import math
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+import comfy.ops
+import comfy.quant_ops
+import comfy.rmsnorm
+from comfy.ldm.modules.attention import optimized_attention
+
+ops = comfy.ops.disable_weight_init
+
+IMAGENET_MEAN = (0.485, 0.456, 0.406)
+IMAGENET_STD = (0.229, 0.224, 0.225)
+
+LATENTS_MEAN = [
+    0.858090341091156, -0.9606591463088989, 1.0661640167236328, -0.5090325474739075,
+    -0.2727581858634949, -1.3675414323806763, -0.2553254961967468, -0.26907554268836975,
+    -0.5376840829849243, -0.0464097298681736, 0.6657370328903198, 0.19690127670764923,
+    -0.5460608005523682, -0.4035342037677765, -0.23683024942874908, 0.25928452610969543,
+    -0.30133944749832153, 0.211341992020607, -1.1206848621368408, 0.3581933379173279,
+    -0.04225143790245056, 0.2604829967021942, 0.22864092886447906, 0.7056031823158264,
+]
+
+LATENTS_STD = [
+    1.2223774194717407, 1.2767263650894165, 1.68317747116088865, 1.7549455165863037,
+    1.5636216402053833, 2.194143533706665, 0.96531379222869875, 1.05698859691619875,
+    0.841948926448822, 0.7729952931404114, 1.8955937623977661, 0.946841835975647,
+    0.7996809482574463, 0.44988900423049925, 0.7197399735450745, 0.69362932443618775,
+    2.961095094680786, 2.7694199085235595, 3.0496184825897215, 2.1088054180145265,
+    3.276226282119751, 3.1627357006073, 2.28168129920959475, 2.6127843856811525,
+]
+
+
+# 3D causal CNN encoder
+
+class CausalConv3d(ops.Conv3d):
+    # Reflect spatial padding, causal (zeros, front-only) temporal padding.
+    def __init__(self, in_channels, out_channels, kernel_size, stride=1, padding=0):
+        super().__init__(in_channels, out_channels, kernel_size=kernel_size, stride=stride)
+        self.causal_padding = (padding,) * 3 if isinstance(padding, int) else tuple(padding)
+
+    def forward(self, x):
+        if sum(self.causal_padding) == 0:
+            return super().forward(x)
+
+        x = F.pad(x, (self.causal_padding[2], self.causal_padding[2], self.causal_padding[1], self.causal_padding[1], 0, 0),  mode="reflect")
+        if x.shape[2] == 1:
+            # single frame: the causal front padding is all zeros truncate the temporal taps instead of convolving zero frames
+            return super().forward(x, autopad="causal_zero")
+        x = F.pad(x, (0, 0, 0, 0, self.causal_padding[0] * 2, 0), mode="constant")
+        return super().forward(x)
+
+
+class TemporalIsolatedGroupNorm(ops.GroupNorm):
+    # GroupNorm with statistics computed per frame (time merged into batch).
+    def forward(self, x):
+        if x.dim() == 5:
+            b, c, t, h, w = x.shape
+            x = x.permute(0, 2, 1, 3, 4).contiguous().view(b * t, c, 1, h, w)
+            x = super().forward(x)
+            return x.view(b, t, c, h, w).permute(0, 2, 1, 3, 4).contiguous()
+        return super().forward(x)
+
+
+def group_norm_3d(num_channels):
+    return TemporalIsolatedGroupNorm(num_groups=32, num_channels=num_channels, eps=1e-6, affine=True)
+
+
+class Downsample3D(nn.Module):
+    def __init__(self, in_channels, out_channels, time_stride=1, space_stride=2):
+        super().__init__()
+        self.space_stride = space_stride
+        self.conv = CausalConv3d(
+            in_channels,
+            out_channels,
+            kernel_size=3,
+            padding=(1, 0, 0),
+            stride=(time_stride, space_stride, space_stride),
+        )
+
+    def forward(self, x):
+        if self.space_stride == 2:
+            x = F.pad(x, (0, 1, 0, 1, 0, 0), mode="reflect")
+        return self.conv(x)
+
+
+class ResnetBlock3D(nn.Module):
+    def __init__(self, in_channels, out_channels=None):
+        super().__init__()
+        self.in_channels = in_channels
+        out_channels = in_channels if out_channels is None else out_channels
+        self.out_channels = out_channels
+
+        self.norm1 = group_norm_3d(in_channels)
+        self.norm2 = group_norm_3d(out_channels)
+        self.conv1 = CausalConv3d(in_channels, out_channels, kernel_size=3, padding=1)
+        self.conv2 = CausalConv3d(out_channels, out_channels, kernel_size=3, padding=1)
+        if in_channels != out_channels:
+            self.nin_shortcut = CausalConv3d(in_channels, out_channels, kernel_size=1)
+
+    def forward(self, x):
+        h = self.conv1(F.silu(self.norm1(x), inplace=True))
+        h = self.conv2(F.silu(self.norm2(h), inplace=True))
+        if self.in_channels != self.out_channels:
+            x = self.nin_shortcut(x)
+        return h.add_(x)
+
+
+class EncoderFCN3D(nn.Module):
+    def __init__(self, ch, ch_mult, space_down, time_down, num_res_blocks, in_channels, z_channels, double_z=True):
+        super().__init__()
+        self.num_levels = len(ch_mult)
+        if isinstance(num_res_blocks, int):
+            num_res_blocks = [num_res_blocks] * self.num_levels
+        self.num_res_blocks = num_res_blocks
+
+        block_mid = [ch * ch_mult[i] for i in range(self.num_levels)]
+        block_in = [block_mid[0]] + block_mid[:-1]
+        block_out = block_mid
+
+        self.conv_in = CausalConv3d(in_channels, block_in[0], kernel_size=3, padding=1)
+
+        self.down = nn.ModuleList()
+        for i_level in range(self.num_levels):
+            down = nn.Module()
+            down.block = nn.ModuleList()
+            for i in range(self.num_res_blocks[i_level]):
+                down.block.append(
+                    ResnetBlock3D(
+                        in_channels=block_in[i_level] if i == 0 else block_mid[i_level],
+                        out_channels=block_mid[i_level],
+                    )
+                )
+            if space_down[i_level] * time_down[i_level] > 1:
+                down.downsample = Downsample3D(
+                    block_mid[i_level],
+                    block_out[i_level],
+                    time_stride=time_down[i_level],
+                    space_stride=space_down[i_level],
+                )
+            self.down.append(down)
+
+        self.norm_out = group_norm_3d(block_out[-1])
+        self.conv_out = CausalConv3d(
+            block_out[-1],
+            2 * z_channels if double_z else z_channels,
+            kernel_size=3,
+            padding=1,
+        )
+
+    def forward(self, x):
+        h = self.conv_in(x)
+        for i_level in range(self.num_levels):
+            for i_block in range(self.num_res_blocks[i_level]):
+                h = self.down[i_level].block[i_block](h)
+            if hasattr(self.down[i_level], "downsample"):
+                h = self.down[i_level].downsample(h)
+        h = F.silu(self.norm_out(h))
+        return self.conv_out(h)
+
+
+# ViT3D decoder
+
+def create_token_ids(patch_dims, device, dtype):
+    coords_list = []
+    for dim_size in patch_dims:
+        coords = torch.arange(0.5, dim_size, dtype=dtype, device=device)
+        coords = coords / dim_size
+        coords = 2.0 * coords - 1.0
+        coords_list.append(coords)
+    coords = torch.stack(torch.meshgrid(*coords_list, indexing="ij"), dim=-1)
+    return coords.flatten(0, len(patch_dims) - 1).unsqueeze(0)
+
+
+class RotaryEmbeddingND(nn.Module):
+    def __init__(self, dim, rotary_base=100.0, n_dim=3):
+        super().__init__()
+        self.n_dim = n_dim
+        self.angle_scale = 2.0 * math.pi
+        inv_freq = 1 / rotary_base ** torch.arange(0, 1, 2 * n_dim / dim, dtype=torch.float32)
+        self.register_buffer("inv_freq", inv_freq, persistent=False)
+
+    def forward(self, img_ids):
+        # [B, S, n_dim] -> [B, S, 1, pairs, 2, 2] rotation table for the kitchen split-half rope
+        angles = (
+            self.angle_scale
+            * img_ids[:, :, :, None].float()
+            * self.inv_freq.to(img_ids.device)[None, None, None, :]
+        )
+        angles = angles.flatten(2, 3)
+        c, s = torch.cos(angles), torch.sin(angles)
+        table = torch.stack([c, -s, s, c], dim=-1).reshape(*angles.shape[:2], 1, angles.shape[-1], 2, 2)
+        return table.to(img_ids.dtype)
+
+
+class FeedForward(nn.Module):
+    # Gated SiLU FFN.
+    def __init__(self, dim, mult=4, bias=True):
+        super().__init__()
+        inner_dim = dim * mult
+        self.w1 = ops.Linear(dim, inner_dim * 2, bias=bias)
+        self.w2 = ops.Linear(inner_dim, dim, bias=bias)
+
+    def forward(self, x):
+        gate, x = self.w1(x).chunk(2, dim=-1)
+        return self.w2(F.silu(gate).mul_(x))
+
+
+class Attention(nn.Module):
+    def __init__(self, heads, dim_head, bias=True, eps=1e-5):
+        super().__init__()
+        self.dim_head = dim_head
+        self.heads = heads
+        inner_dim = dim_head * heads
+        self.norm_q = ops.RMSNorm(dim_head, eps=eps, elementwise_affine=False)
+        self.norm_k = ops.RMSNorm(dim_head, eps=eps, elementwise_affine=False)
+        self.to_qkv = ops.Linear(inner_dim, inner_dim * 3, bias=bias)
+        self.to_out = ops.Linear(inner_dim, inner_dim, bias=bias)
+
+    def forward(self, x, rotary_pos_emb=None):
+        batch_size, seq_len, _ = x.shape
+
+        qkv = self.to_qkv(x)
+        qkv = qkv.view(batch_size, seq_len, -1, 3 * self.dim_head)
+        query, key, value = torch.chunk(qkv, 3, dim=-1)
+
+        query = comfy.rmsnorm.rms_norm(query, self.norm_q.weight, self.norm_q.eps)
+        key = comfy.rmsnorm.rms_norm(key, self.norm_k.weight, self.norm_k.eps)
+
+        if rotary_pos_emb is not None:
+            rot = rotary_pos_emb.shape[-3] * 2
+            query[..., :rot], key[..., :rot] = comfy.quant_ops.ck.apply_rope_split_half(
+                query[..., :rot], key[..., :rot], rotary_pos_emb)
+
+        out = optimized_attention(query.transpose(1, 2), key.transpose(1, 2), value.transpose(1, 2),
+                                  self.heads, skip_reshape=True).nan_to_num_(0.0)
+        return self.to_out(out)
+
+
+class TransformerBlock(nn.Module):
+    def __init__(self, heads, dim_head, bias=True, eps=1e-5):
+        super().__init__()
+        dim = heads * dim_head
+        self.norm1 = ops.RMSNorm(dim, elementwise_affine=True, eps=eps)
+        self.attn = Attention(heads=heads, dim_head=dim_head, bias=bias, eps=eps)
+        self.scale1 = nn.Parameter(torch.empty(dim))
+        self.norm2 = ops.RMSNorm(dim, elementwise_affine=True, eps=eps)
+        self.ff = FeedForward(dim=dim, bias=bias)
+        self.scale2 = nn.Parameter(torch.empty(dim))
+
+    def forward(self, x, rotary_pos_emb=None):
+        x = x.addcmul_(self.attn(comfy.rmsnorm.rms_norm(x, self.norm1.weight, self.norm1.eps), rotary_pos_emb), self.scale1)
+        return x.addcmul_(self.ff(comfy.rmsnorm.rms_norm(x, self.norm2.weight, self.norm2.eps)), self.scale2)
+
+
+class ViT3DDecoder(nn.Module):
+    def __init__(self, patch_size=16, patch_size_t=4, in_channels=24, out_channels=3, num_layers=36, heads=32, dim_head=64, rope_theta=100.0,
+                 rope_dim_ratio=0.75, bias=True, eps=1e-5, num_register_tokens=4):
+        super().__init__()
+        dim = heads * dim_head
+        self.patch_size = patch_size
+        self.patch_size_t = patch_size_t
+        self.out_channels = out_channels
+        self.num_register_tokens = num_register_tokens
+
+        self.pos_embed = RotaryEmbeddingND(int(dim_head * rope_dim_ratio), rope_theta, n_dim=3)
+        self.x_embedder = ops.Linear(in_channels, dim)
+        self.register_tokens = nn.Parameter(torch.empty(1, num_register_tokens, dim))
+        # unused at inference; kept so the checkpoint loads without leftover keys
+        self.register_buffer("mask_token", torch.empty(1, 1, dim))
+
+        self.transformer_blocks = nn.ModuleList(
+            [TransformerBlock(heads=heads, dim_head=dim_head, bias=bias, eps=eps)
+             for _ in range(num_layers)]
+        )
+
+        self.norm_out = ops.LayerNorm(dim, elementwise_affine=True, eps=eps)
+        self.proj_out = ops.Linear(dim, out_channels * patch_size_t * patch_size * patch_size)
+
+    def forward(self, x):
+        B, C, latent_T, latent_H, latent_W = x.shape
+
+        h = self.x_embedder(x.flatten(2).transpose(1, 2))  # [B, T*H*W, C]
+
+        num_patches = h.shape[1]
+        num_suffix = 1 + self.num_register_tokens
+
+        h = torch.cat([h, self.register_tokens.expand(B, -1, -1), torch.zeros_like(h[:, 0:1, :])], dim=1)
+
+        img_ids = create_token_ids((latent_T, latent_H, latent_W), x.device, x.dtype).expand(B, -1, -1)
+        suffix_ids = torch.zeros((B, num_suffix, 3), device=x.device, dtype=img_ids.dtype)
+        img_ids = torch.cat([img_ids, suffix_ids], dim=1)
+
+        rotary_pos_emb = self.pos_embed(img_ids)
+
+        for block in self.transformer_blocks:
+            h = block(h, rotary_pos_emb)
+
+        output = self.proj_out(self.norm_out(h))
+
+        output = output[:, :num_patches, :]
+
+        output = output.view(
+            B, latent_T, latent_H, latent_W,
+            self.out_channels, self.patch_size_t, self.patch_size, self.patch_size,
+        )
+        output = output.permute(0, 4, 1, 5, 2, 6, 3, 7).contiguous()
+        output = output.reshape(
+            B, self.out_channels,
+            latent_T * self.patch_size_t,
+            latent_H * self.patch_size,
+            latent_W * self.patch_size,
+        )
+        return output
+
+
+# Full VAE
+
+class MiniMaxH3VideoVAE(nn.Module):
+    def __init__(
+        self,
+        in_channels=3,
+        out_ch=3,
+        ch=128,
+        embed_dim=24,
+        z_channels=24,
+        ch_mult=(1, 2, 2, 4, 4, 8),
+        num_res_blocks=2,
+        space_down=(2, 2, 2, 2, 1, 1),
+        time_down=(1, 2, 2, 1, 1, 1),
+        clip_length=17,
+        token_drop=3,
+        tile_size=256,
+        tile_overlap_min=64,
+        tiling=True,
+    ):
+        super().__init__()
+        self.vae_ratio = int(math.prod(space_down))
+        self.vae_ratio_t = int(math.prod(time_down))
+
+        # temporal chunking parameters
+        self.clip_length = clip_length
+        self.token_drop = token_drop
+        self.frame_pre_padding = (-clip_length) % self.vae_ratio_t
+        self.tokens_chunk_size = math.ceil(clip_length / self.vae_ratio_t)
+        self.token_overlap = (-token_drop) % self.tokens_chunk_size
+        self.frame_overlap = max(self.token_overlap * self.vae_ratio_t - self.frame_pre_padding, 0)
+
+        # spatial tiling parameters
+        self.tiling = tiling
+        self.tile_size = tile_size
+        self.tile_overlap_min = tile_overlap_min
+
+        self.encoder = EncoderFCN3D(
+            ch=ch,
+            ch_mult=list(ch_mult),
+            space_down=list(space_down),
+            time_down=list(time_down),
+            num_res_blocks=num_res_blocks,
+            in_channels=in_channels,
+            z_channels=z_channels,
+            double_z=True,
+        )
+        self.quant_conv = ops.Conv3d(z_channels * 2, 2 * embed_dim, 1)
+        self.post_quant_conv = ops.Conv3d(embed_dim, z_channels, 1)
+        self.decoder = ViT3DDecoder(
+            patch_size=self.vae_ratio,
+            patch_size_t=self.vae_ratio_t,
+            in_channels=z_channels,
+            out_channels=out_ch,
+        )
+
+        self.register_buffer("latents_mean", torch.tensor(LATENTS_MEAN))
+        self.register_buffer("latents_std", torch.tensor(LATENTS_STD))
+        self.register_buffer("pixel_mean", torch.tensor(IMAGENET_MEAN).view(1, 3, 1, 1, 1), persistent=False)
+        self.register_buffer("pixel_std", torch.tensor(IMAGENET_STD).view(1, 3, 1, 1, 1), persistent=False)
+
+    # single-shot forward
+
+    def _encode_moments(self, x):
+        return self.quant_conv(self.encoder(x))
+
+    def _decode_pixels(self, z):
+        return self.decoder(self.post_quant_conv(z))
+
+    def _adaptive_encode(self, x):
+        if self.tiling:
+            return self.tiled_encode(x)
+        return self._encode_moments(x)
+
+    def _adaptive_decode(self, z):
+        if self.tiling:
+            return self.tiled_decode(z)
+        return self._decode_pixels(z)
+
+    # spatial tiling
+
+    def split_tiles(self, input_len):
+        tile_size = self.tile_size
+        if tile_size >= input_len:
+            return [0], [input_len], []
+
+        N = math.ceil(input_len / tile_size)
+        while True:
+            overlaps = [self.tile_overlap_min] * (N - 1)
+            remaining = tile_size * N - sum(overlaps) - input_len
+            if remaining < 0:
+                N += 1
+            else:
+                break
+
+        remaining_units = remaining // self.vae_ratio
+        for i in range(remaining_units):
+            overlaps[i % (N - 1)] += self.vae_ratio
+
+        tile_start_idx = [0]
+        for i in range(N - 1):
+            tile_start_idx.append(tile_start_idx[-1] + tile_size - overlaps[i])
+
+        return tile_start_idx, [tile_size] * N, overlaps
+
+    def blend(self, a, b, blend_extent, dim):
+        blend_extent = min(a.shape[dim], b.shape[dim], blend_extent)
+
+        positions = torch.arange(blend_extent, device=b.device, dtype=b.dtype)
+        weight_a = 1 - positions / blend_extent
+        weight_b = positions / blend_extent
+
+        shape = [1] * a.ndim
+        shape[dim] = blend_extent
+        weight_a = weight_a.view(shape)
+        weight_b = weight_b.view(shape)
+
+        slice_a = [slice(None)] * a.ndim
+        slice_a[dim] = slice(-blend_extent, None)
+        slice_b = [slice(None)] * b.ndim
+        slice_b[dim] = slice(0, blend_extent)
+
+        blended = a[tuple(slice_a)] * weight_a + b[tuple(slice_b)] * weight_b
+
+        if blend_extent < b.shape[dim]:
+            slice_b_rest = [slice(None)] * b.ndim
+            slice_b_rest[dim] = slice(blend_extent, None)
+            return torch.cat([blended, b[tuple(slice_b_rest)]], dim=dim)
+        return blended
+
+    def tiled_encode(self, x):
+        height, width = x.shape[-2], x.shape[-1]
+        y_idx, y_len, y_overlap = self.split_tiles(height)
+        x_idx, x_len, x_overlap = self.split_tiles(width)
+
+        rows = []
+        for i_pos, i_len in zip(y_idx, y_len):
+            row = []
+            for j_pos, j_len in zip(x_idx, x_len):
+                tile = x[..., i_pos:i_pos + i_len, j_pos:j_pos + j_len]
+                row.append(self._encode_moments(tile))
+            rows.append(row)
+
+        latent_y_overlap = [o // self.vae_ratio for o in y_overlap]
+        latent_x_overlap = [o // self.vae_ratio for o in x_overlap]
+
+        result_rows = []
+        for i, row in enumerate(rows):
+            result_row = []
+            for j, tile in enumerate(row):
+                if i > 0:
+                    tile = self.blend(rows[i - 1][j], tile, latent_y_overlap[i - 1], dim=-2)
+                if j > 0:
+                    tile = self.blend(row[j - 1], tile, latent_x_overlap[j - 1], dim=-1)
+                if i < len(rows) - 1:
+                    tile = tile[..., :-latent_y_overlap[i], :]
+                if j < len(row) - 1:
+                    tile = tile[..., :, :-latent_x_overlap[j]]
+                result_row.append(tile)
+            result_rows.append(torch.cat(result_row, dim=-1))
+        return torch.cat(result_rows, dim=-2)
+
+    def tiled_decode(self, z):
+        height, width = z.shape[-2] * self.vae_ratio, z.shape[-1] * self.vae_ratio
+        y_idx, y_len, y_overlap = self.split_tiles(height)
+        x_idx, x_len, x_overlap = self.split_tiles(width)
+
+        # Blended tiles are written straight into a pre-allocated canvas.
+        canvas = None
+        row_tails = []
+        out_y = 0
+        for i, (i_pos, i_len) in enumerate(zip(y_idx, y_len)):
+            zi, zl = i_pos // self.vae_ratio, i_len // self.vae_ratio
+            new_tails = []
+            left_tail = None
+            out_x = 0
+            for j, (j_pos, j_len) in enumerate(zip(x_idx, x_len)):
+                zj, zw = j_pos // self.vae_ratio, j_len // self.vae_ratio
+                tile = self._decode_pixels(z[..., zi:zi + zl, zj:zj + zw])
+                if i < len(y_idx) - 1:
+                    new_tails.append(tile[..., -y_overlap[i]:, :].clone())
+                next_left_tail = tile[..., :, -x_overlap[j]:].clone() if j < len(x_idx) - 1 else None
+                if i > 0:
+                    tile = self.blend(row_tails[j], tile, y_overlap[i - 1], dim=-2)
+                if j > 0:
+                    tile = self.blend(left_tail, tile, x_overlap[j - 1], dim=-1)
+                left_tail = next_left_tail
+                if i < len(y_idx) - 1:
+                    tile = tile[..., :-y_overlap[i], :]
+                if j < len(x_idx) - 1:
+                    tile = tile[..., :, :-x_overlap[j]]
+                if canvas is None:
+                    canvas = torch.empty(*tile.shape[:-2], height, width, dtype=tile.dtype, device=tile.device)
+                canvas[..., out_y:out_y + tile.shape[-2], out_x:out_x + tile.shape[-1]].copy_(tile)
+                out_x += tile.shape[-1]
+            row_tails = new_tails
+            out_y += tile.shape[-2]
+        return canvas
+
+    # temporal chunking
+
+    def encode_temporal(self, x):
+        if x.shape[2] % self.clip_length != 0:
+            pad_size = (-x.shape[2]) % self.clip_length
+            pad_frames = x[:, :, -1:].repeat(1, 1, pad_size, 1, 1)
+            x = torch.cat([x, pad_frames], dim=2)
+
+        num_chunks = x.shape[2] // self.clip_length
+
+        z_list = []
+        for i in range(num_chunks):
+            clip_x = x[:, :, i * self.clip_length:(i + 1) * self.clip_length, :, :]
+            z_list.append(self._adaptive_encode(clip_x))
+
+        z = torch.cat(z_list, dim=2)
+        if self.token_drop > 0:
+            z = z[:, :, :-self.token_drop]
+        return z
+
+    def _decode_temporal_pad_frames(self, z_len, pad_tokens):
+        if pad_tokens <= 0:
+            return 0
+        intra_tail = self.clip_length % self.vae_ratio_t
+        if intra_tail == 0:
+            return pad_tokens * self.vae_ratio_t
+
+        z_len_before_pad = z_len - pad_tokens
+        return sum(
+            (intra_tail if (z_len_before_pad + k) % self.tokens_chunk_size == 0
+             else self.vae_ratio_t)
+            for k in range(pad_tokens)
+        )
+
+    def _decode_temporal_frame_plan(self, z_len, num_chunks, pad_tokens):
+        chunk_dec = self.tokens_chunk_size * self.vae_ratio_t
+        split_count = int(self.token_drop > 0) + 1
+        total_frames = 0
+        final_overlap_frames = 0
+
+        for i in range(num_chunks):
+            t_start_idx = i * self.tokens_chunk_size
+            t_end_idx = t_start_idx + self.tokens_chunk_size + self.token_overlap
+            clip_token_len = max(0, min(t_end_idx, z_len) - min(t_start_idx, z_len))
+            clip_frame_len = clip_token_len * self.vae_ratio_t
+
+            for j in range(split_count):
+                f_start_idx = j * chunk_dec
+                f_end_idx = min(f_start_idx + chunk_dec, clip_frame_len)
+                chunk_frames = max(0, f_end_idx - f_start_idx - self.frame_pre_padding)
+                if j == 0:
+                    total_frames += chunk_frames
+                else:
+                    final_overlap_frames = chunk_frames
+
+        total_frames += final_overlap_frames
+        return total_frames - self._decode_temporal_pad_frames(z_len, pad_tokens)
+
+    def decode_temporal(self, z):
+        chunk_dec = self.tokens_chunk_size * self.vae_ratio_t
+        split_count = int(self.token_drop > 0) + 1
+
+        pseudo_total_tokens = z.shape[2] + self.token_drop
+
+        pad_tokens = 0
+        remainder = pseudo_total_tokens % self.tokens_chunk_size
+        if remainder != 0:
+            pad_tokens = self.tokens_chunk_size - remainder
+            pseudo_total_tokens += pad_tokens
+
+        num_chunks = pseudo_total_tokens // self.tokens_chunk_size - int(self.token_drop > 0)
+        if num_chunks < 1:
+            # too few tokens for one chunk (e.g. T_lat == 2): pad one extra chunk
+            pad_tokens += self.tokens_chunk_size
+            num_chunks += 1
+
+        if pad_tokens > 0:
+            pad_z = z[:, :, -1:, :, :].repeat(1, 1, pad_tokens, 1, 1)
+            z = torch.cat([z, pad_z], dim=2)
+
+        output_frames = self._decode_temporal_frame_plan(z.shape[2], num_chunks, pad_tokens)
+
+        dec = None
+        dec_overlap = None
+        write_pos = 0
+
+        def write_part(part):
+            nonlocal dec, write_pos
+            part_frames = part.shape[2]
+            if part_frames <= 0:
+                return
+            if dec is None:
+                out_shape = list(part.shape)
+                out_shape[2] = output_frames
+                dec = torch.empty(out_shape, dtype=part.dtype, device=part.device)
+            copy_frames = min(part_frames, max(0, dec.shape[2] - write_pos))
+            if copy_frames > 0:
+                dec[:, :, write_pos:write_pos + copy_frames, :, :].copy_(
+                    part[:, :, :copy_frames, :, :]
+                )
+                write_pos += copy_frames
+
+        for i in range(num_chunks):
+            t_start_idx = i * self.tokens_chunk_size
+            t_end_idx = t_start_idx + self.tokens_chunk_size + self.token_overlap
+            clip_z = z[:, :, t_start_idx:t_end_idx, :, :]
+
+            clip_dec = self._adaptive_decode(clip_z)
+
+            for j in range(split_count):
+                f_start_idx = j * chunk_dec
+                f_end_idx = min(f_start_idx + chunk_dec, clip_dec.shape[2])
+                clip_dec_chunk = clip_dec[:, :, f_start_idx:f_end_idx, :, :]
+                clip_dec_chunk = clip_dec_chunk[:, :, self.frame_pre_padding:, :, :]
+
+                if j == 0:
+                    if dec_overlap is not None:
+                        clip_dec_chunk = self.blend(
+                            dec_overlap, clip_dec_chunk, self.frame_overlap, dim=-3
+                        )
+                        dec_overlap = None
+                    write_part(clip_dec_chunk)
+                else:
+                    dec_overlap = clip_dec_chunk.contiguous()
+
+            if i == num_chunks - 1 and dec_overlap is not None:
+                write_part(dec_overlap)
+                dec_overlap = None
+
+            del clip_dec, clip_z
+
+        return dec
+
+
+    def encode(self, x):
+        # x: [B, 3, T, H, W] in [-1, 1] -> normalized latents [B, 24, T_lat, H/16, W/16]
+        if x.ndim == 4:
+            x = x.unsqueeze(2)
+
+        x = x.add(1.0).mul_(0.5).sub_(self.pixel_mean.to(x)).div_(self.pixel_std.to(x))
+
+        if x.shape[2] == 1:
+            moments = self._adaptive_encode(x)
+            moments = moments[:, :, -1:, :, :]
+        else:
+            moments = self.encode_temporal(x)
+
+        mean = torch.chunk(moments.float(), 2, dim=1)[0]
+
+        latents_mean = self.latents_mean.view(1, -1, 1, 1, 1).to(mean)
+        latents_std = self.latents_std.view(1, -1, 1, 1, 1).to(mean)
+        return (mean - latents_mean) / latents_std
+
+    def encode_tiled(self, x, **kwargs):
+        # tiling is always on internally with the reference's semantic tile sizes, ignore tiling fallbacks
+        return self.encode(x)
+
+    def decode_tiled(self, z, **kwargs):
+        return self.decode(z)
+
+    def decode(self, z):
+        # z: [B, 24, T_lat, H_lat, W_lat] normalized latents -> pixels [B, 3, T, H, W] in [-1, 1]
+        latents_mean = self.latents_mean.view(1, -1, 1, 1, 1).to(z)
+        latents_std = self.latents_std.view(1, -1, 1, 1, 1).to(z)
+        z = z * latents_std + latents_mean
+
+        if z.shape[2] == 1:
+            dec = self._adaptive_decode(z)
+            dec = dec[:, :, -1:, :, :]
+        else:
+            dec = self.decode_temporal(z)
+
+        dec = dec.float()
+        dec.mul_(self.pixel_std.to(dec)).add_(self.pixel_mean.to(dec)).clamp_(0.0, 1.0).mul_(2.0).sub_(1.0)
+        return dec

comfy/model_base.py

@@ -21,6 +21,7 @@ import comfy.ldm.hunyuan3dv2_1.hunyuandit
 import torch
 import logging
 import comfy.ldm.lightricks.av_model
+import comfy.ldm.minimax.model
 import comfy.ldm.lightricks.symmetric_patchifier
 import comfy.context_windows
 from comfy.ldm.modules.diffusionmodules.openaimodel import UNetModel, Timestep
@@ -2063,6 +2064,57 @@ class Hunyuan3Dv2_1(BaseModel):
             out['guidance'] = comfy.conds.CONDRegular(torch.FloatTensor([guidance]))
         return out
 
+class MiniMaxH3(BaseModel):
+    def __init__(self, model_config, model_type=ModelType.FLOW, device=None):
+        super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.minimax.model.MiniMaxH3Model)
+
+    def extra_conds(self, **kwargs):
+        out = super().extra_conds(**kwargs)
+        cross_attn = kwargs.get("cross_attn", None)
+        if cross_attn is not None:
+            # run condition_proj + token refiner once per sampling instead of per step
+            cross_attn = self.diffusion_model.preprocess_text_embeds(
+                cross_attn.to(device=kwargs["device"], dtype=self.get_dtype_inference()))
+            out['c_crossattn'] = comfy.conds.CONDRegular(cross_attn)
+
+        latent_shapes = kwargs.get("latent_shapes", None)
+        if latent_shapes is not None:
+            out['latent_shapes'] = comfy.conds.CONDConstant(latent_shapes)
+
+        # Everything H3-specific rides in one dict so _apply_model's dtype cast
+        # (which would flatten fp32 cond latents and long tags to bf16) skips it.
+        payload = {}
+        tags = kwargs.get("minimax_token_tags", None)
+        if tags is not None:
+            payload["text_token_tags"] = tags
+        keyframes = kwargs.get("minimax_keyframes", None)
+        if keyframes is not None:
+            payload["keyframes"] = keyframes
+            payload["frame_count"] = kwargs.get("minimax_frame_count", None)
+            payload["cond_video_latents"] = [kf["latent"] for kf in keyframes]
+        refs = kwargs.get("minimax_refs", None)
+        if refs is not None:
+            payload["refs"] = refs
+            payload["cond_video_latents"] = [r["latent"] for r in refs if "latent" in r]
+            payload["cond_audio_latents"] = [r["audio_latent"] for r in refs if r.get("audio_latent") is not None]
+        if kwargs.get("minimax_visual_cond_noise_aug", None) is not None:
+            payload["visual_cond_noise_aug"] = kwargs["minimax_visual_cond_noise_aug"]
+        if kwargs.get("minimax_audio_cond_noise_aug", None) is not None:
+            payload["audio_cond_noise_aug"] = kwargs["minimax_audio_cond_noise_aug"]
+        payload["seed"] = kwargs.get("seed", 0)
+        if cross_attn is not None and latent_shapes is not None and len(latent_shapes) > 1:
+            # packed layout built once per sampling run, h/w rounded up to the DiT's 2x2 patch
+            vs = latent_shapes[0]
+            payload["layout"] = comfy.ldm.minimax.model.PackedLayout(
+                cross_attn.shape[1], vs[2], (vs[3] + 1) // 2 * 2, (vs[4] + 1) // 2 * 2,
+                latent_shapes[1][-1], keyframes=payload.get("keyframes"),
+                refs=payload.get("refs"), frame_count=payload.get("frame_count"))
+        out['minimax_payload'] = comfy.conds.CONDConstant(payload)
+        return out
+
+    def scale_latent_inpaint(self, sigma, noise, latent_image, **kwargs):
+        return latent_image
+
 class TripoSplat(BaseModel):
     def __init__(self, model_config, model_type=ModelType.FLOW, device=None):
         super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.triposplat.model.LatentSeqMMFlowModel)

comfy/model_detection.py

@@ -359,6 +359,35 @@ def detect_unet_config(state_dict, key_prefix, metadata=None):
         # PixArt diffusers
         return None
 
+    if '{}video_patch_proj.weight'.format(key_prefix) in state_dict_keys and '{}audio_patch_proj.weight'.format(key_prefix) in state_dict_keys: # MiniMax H3
+        dit_config = {}
+        dit_config["image_model"] = "minimax_h3"
+        dit_config["num_layers"] = count_blocks(state_dict_keys, '{}blocks.'.format(key_prefix) + '{}.')
+        dit_config["token_refiner_num_layers"] = count_blocks(state_dict_keys, '{}token_refiner.blocks.'.format(key_prefix) + '{}.')
+        dit_config["hidden_size"] = state_dict['{}video_patch_proj.weight'.format(key_prefix)].shape[0]
+        dit_config["latents_dim"] = state_dict['{}final_layer.video_out.weight'.format(key_prefix)].shape[0] // 4  # patch 1x2x2
+        dit_config["audio_latents_dim"] = state_dict['{}final_layer.audio_out.weight'.format(key_prefix)].shape[0]
+        dit_config["attention_head_dim"] = state_dict['{}blocks.0.attn.q_norm.weight'.format(key_prefix)].shape[0]
+        qkv = state_dict['{}blocks.0.attn.qkv_proj.weight'.format(key_prefix)]
+        dit_config["num_attention_heads"] = qkv.shape[0] // (3 * dit_config["attention_head_dim"])
+        dit_config["ffn_hidden_size"] = state_dict['{}blocks.0.mlp.fc1.weight'.format(key_prefix)].shape[0] // 2
+        dit_config["text_dim"] = state_dict['{}condition_proj.weight'.format(key_prefix)].shape[1]
+        table_key = '{}adaln_t_table'.format(key_prefix)
+        if table_key in state_dict_keys:
+            # adaln shipped over a precomputed curve basis: the adaln linears span a small shared basis of the time-embedding curve (no time embedder)
+            table = state_dict[table_key].shape  # [grid, k]
+            dit_config["adaln_curve_grid"] = table[0]
+            dit_config["time_embed_dim"] = table[1]
+        else:
+            te = state_dict['{}time_embedder.proj_in.weight'.format(key_prefix)]
+            dit_config["timestep_input_dim"] = te.shape[1]
+            dit_config["time_embed_hidden_size"] = te.shape[0]
+            dit_config["time_embed_dim"] = state_dict['{}time_embedder.proj_out.weight'.format(key_prefix)].shape[0]
+        dit_config["rope_inv_freq_len"] = state_dict['{}rope.inv_freq'.format(key_prefix)].shape[0]
+        if metadata is not None and "config" in metadata:
+            dit_config.update(json.loads(metadata["config"]).get("transformer", {}))
+        return dit_config
+
     if '{}adaln_single.emb.timestep_embedder.linear_1.bias'.format(key_prefix) in state_dict_keys: #Lightricks ltxv
         dit_config = {}
         dit_config["image_model"] = "ltxav" if f'{key_prefix}audio_adaln_single.linear.weight' in state_dict_keys else "ltxv"

comfy/ops.py

@@ -943,13 +943,61 @@ if CUBLAS_IS_AVAILABLE:
 # ==============================================================================
 # Mixed Precision Operations
 # ==============================================================================
+from . import quant_ops
 from .quant_ops import (
     QuantizedTensor,
     QUANT_ALGOS,
     TensorCoreFP8Layout,
+    TensorWiseINT8Layout,
     get_layout_class,
 )
 
+def _swiglu_eager(x):
+    gate, up = x.chunk(2, dim=-1)
+    return torch.nn.functional.silu(gate).mul_(up)
+
+
+INPUT_ACT_EAGER = {
+    "gelu_tanh": lambda x: torch.nn.functional.gelu(x, approximate="tanh"),
+    "swiglu": _swiglu_eager,
+}
+
+
+def linear_input_act(linear, x, input_act):
+    """``linear(act(x))``, with ``act`` folded into an INT8 activation quantizer.
+
+    An INT8 linear quantizes its input anyway, so an elementwise activation can
+    ride along inside that kernel instead of writing a full-size intermediate to
+    HBM and reading it straight back. Worth it for an MLP's down-projection,
+    where the intermediate is several times the hidden size.
+
+    """
+    weight = linear.weight
+    if (comfy.model_management.in_training
+            or not isinstance(weight, QuantizedTensor)
+            or weight._layout_cls != "TensorWiseINT8Layout"
+            or getattr(weight._params, "transposed", False)):
+        return linear(INPUT_ACT_EAGER[input_act](x))
+
+    # want_requant keeps a vbar-streamed layer on the INT8 path when a LoRA is
+    # patched in on the fly; without it the cast hands back a dequantized weight.
+    weight, bias, offload_stream = cast_bias_weight(
+        linear, x, offloadable=True, compute_dtype=x.dtype, want_requant=True)
+    try:
+        if not isinstance(weight, QuantizedTensor):
+            # A LoRA weight_function, or activations whose dtype differs from the
+            # weight's, make the cast hand back a dequantized tensor.
+            return torch.nn.functional.linear(INPUT_ACT_EAGER[input_act](x), weight, bias)
+        qdata, scale = TensorWiseINT8Layout.get_plain_tensors(weight)
+        return quant_ops.ck.int8_linear(
+            x, qdata, scale, bias, x.dtype,
+            convrot=getattr(weight._params, "convrot", False),
+            convrot_groupsize=getattr(weight._params, "convrot_groupsize", 256),
+            input_act=input_act,
+        )
+    finally:
+        uncast_bias_weight(linear, weight, bias, offload_stream)
+
 
 class QuantLinearFunc(torch.autograd.Function):
     """Custom autograd function for quantized linear: quantized forward, optionally FP8 backward.
@@ -1259,7 +1307,7 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_prec
 
             def state_dict(self, *args, destination=None, prefix="", **kwargs):
                 sd = destination if destination is not None else {}
-                return _quantized_weight_state_dict(self, sd, prefix, extra_quant_params=("input_scale",))
+                return _quantized_weight_state_dict(self, sd, prefix, extra_quant_params=("input_scale", "pre_quant_scale"))
 
             def _forward(self, input, weight, bias):
                 return torch.nn.functional.linear(input, weight, bias)
@@ -1298,6 +1346,11 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_prec
             def forward(self, input, *args, **kwargs):
                 run_every_op()
 
+                # ModelOpt AWQ-style smoothing
+                pre_quant_scale = getattr(self, 'pre_quant_scale', None)
+                if pre_quant_scale is not None:
+                    input = input * comfy.model_management.cast_to_device(pre_quant_scale, input.device, input.dtype)
+
                 input_shape = input.shape
                 reshaped_nd = False
                 #If cast needs to apply lora, it should be done in the compute dtype

comfy/quant_ops.py

@@ -240,7 +240,7 @@ QUANT_ALGOS = {
     },
     "nvfp4": {
         "storage_t": torch.uint8,
-        "parameters": {"weight_scale", "weight_scale_2", "input_scale"},
+        "parameters": {"weight_scale", "weight_scale_2", "input_scale", "pre_quant_scale"},
         "comfy_tensor_layout": "TensorCoreNVFP4Layout",
         "group_size": 16,
     },

comfy/sd.py

@@ -72,6 +72,9 @@ import comfy.text_encoders.ace15
 import comfy.text_encoders.longcat_image
 import comfy.text_encoders.qwen35
 import comfy.text_encoders.qwen3vl
+import comfy.text_encoders.minimax
+import comfy.ldm.minimax.vae
+import comfy.ldm.minimax.audio_vae
 import comfy.text_encoders.boogu
 import comfy.text_encoders.ernie
 import comfy.text_encoders.gemma4
@@ -936,6 +939,50 @@ class VAE:
                 #Force cast it for --disable-dynamic-vram users until there is a true core fix.
                 if not comfy.memory_management.aimdo_enabled:
                     self.disable_offload = True
+            elif "decoder.transformer_blocks.0.scale1" in sd and "encoder.down.5.block.0.conv1.weight" in sd:  # MiniMax H3 video VAE
+                self.first_stage_model = comfy.ldm.minimax.vae.MiniMaxH3VideoVAE()
+                self.latent_channels = 24
+                self.latent_dim = 3
+                # frames 17k+5 <-> latents 5k+2, 16x spatial
+                self.upscale_ratio = (lambda a: max(1, (a - 2) // 5 * 17 + 5), 16, 16)
+                self.upscale_index_formula = (4, 16, 16)
+                self.downscale_ratio = (lambda a: max(1, (a - 5) // 17 * 5 + 2) if a > 1 else 1, 16, 16)
+                self.downscale_index_formula = (4, 16, 16)
+                self.working_dtypes = [torch.float16, torch.float32]
+                # the model tiles internally (256px spatial, 17-frame temporal chunks)
+                self.handles_tiling = True
+                def estimate_encode_memory(frames, height, width, dtype):
+                    fixed = 110_000_000 if frames == 1 else 1_300_000_000
+                    elements_per_pixel = 7 if frames == 1 else 9.5
+                    return (elements_per_pixel * frames * height * width + fixed) * model_management.dtype_size(dtype) * 1.03
+
+                def estimate_decode_memory(frames, height, width, dtype):
+                    fixed = 110_000_000 if frames <= 22 else 270_000_000
+                    return (9.5 * frames * height * width + fixed) * model_management.dtype_size(dtype) * 1.03
+
+                self.memory_used_encode = lambda shape, dtype: estimate_encode_memory(shape[2], shape[3], shape[4], dtype)
+                self.memory_used_decode = lambda shape, dtype: estimate_decode_memory(self.upscale_ratio[0](shape[2]), shape[3] * self.upscale_ratio[1], shape[4] * self.upscale_ratio[2], dtype)
+            elif "pre_block.attn.zero_k_bias" in sd:  # MiniMax H3 audio VAE (DAC encoder + BigVGAN decoder)
+                self.first_stage_model = comfy.ldm.minimax.audio_vae.MiniMaxH3AudioVAE()
+                self.latent_channels = 32
+                self.output_channels = 2
+                self.pad_channel_value = "replicate"
+                self.audio_sample_rate = 32000
+                self.upscale_ratio = 800
+                self.downscale_ratio = 800
+                self.latent_dim = 2  # [B, 32, stereo 2, T]
+                self.process_output = lambda audio: audio
+                self.process_input = lambda audio: audio
+                self.working_dtypes = [torch.float32]
+                # encode gets the waveform shape [B, 2, samples], decode the latent shape [B, 32, 2, T]
+                def estimate_encode_memory(samples, dtype):
+                    return (900 * samples + 105_000_000) * model_management.dtype_size(dtype) * 1.03
+
+                def estimate_decode_memory(samples, dtype):
+                    return max(42_000_000, 220 * samples + 20_000_000) * model_management.dtype_size(dtype) * 1.03
+
+                self.memory_used_encode = lambda shape, dtype: estimate_encode_memory(shape[2], dtype)
+                self.memory_used_decode = lambda shape, dtype: estimate_decode_memory(shape[-1] * self.upscale_ratio, dtype)
             elif "gs.base_offset_scale" in sd and "octree.out_proj.weight" in sd:  # TripoSplat octree gaussian decoder
                 self.first_stage_model = comfy.ldm.triposplat.vae.OctreeGaussianDecoder()
                 self.latent_channels = 16
@@ -1393,6 +1440,7 @@ class CLIPType(Enum):
     KREA2 = 32
     JOYIMAGE = 33
     MAGE = 34
+    MINIMAX = 35
 
 
 
@@ -1449,6 +1497,7 @@ class TEModel(Enum):
     QWEN3VL_4B = 34
     QWEN3VL_8B = 35
     GEMMA_4_12B = 36
+    QWEN3VL_32B = 37
 
 
 def detect_te_model(sd):
@@ -1515,6 +1564,9 @@ def detect_te_model(sd):
         return TEModel.QWEN35_2B
     if "model.visual.deepstack_merger_list.0.norm.weight" in sd:  # DeepStack is unique to Qwen3-VL
         return TEModel.QWEN3VL_4B if sd["model.visual.merger.linear_fc2.weight"].shape[0] == 2560 else TEModel.QWEN3VL_8B
+    if "visual.deepstack_merger_list.0.norm.weight" in sd and "model.layers.49.self_attn.q_proj.weight" in sd:
+        # MiniMax H3 conditioning encoder: Qwen3-VL-32B, truncated to 50 layers
+        return TEModel.QWEN3VL_32B
     if "model.layers.0.post_attention_layernorm.weight" in sd:
         weight = sd['model.layers.0.post_attention_layernorm.weight']
         if 'model.layers.0.self_attn.q_norm.weight' in sd:
@@ -1744,6 +1796,9 @@ def load_text_encoder_state_dicts(state_dicts=[], embedding_directory=None, clip
                 qwen3vl_type = {TEModel.QWEN3VL_4B: "qwen3vl_4b", TEModel.QWEN3VL_8B: "qwen3vl_8b"}[te_model]
                 clip_target.clip = comfy.text_encoders.qwen3vl.te(**llama_detect(clip_data), model_type=qwen3vl_type)
                 clip_target.tokenizer = comfy.text_encoders.qwen3vl.tokenizer(model_type=qwen3vl_type)
+        elif te_model == TEModel.QWEN3VL_32B:
+            clip_target.clip = comfy.text_encoders.minimax.te(**llama_detect(clip_data))
+            clip_target.tokenizer = comfy.text_encoders.minimax.MiniMaxH3Tokenizer
         elif te_model == TEModel.QWEN3_06B:
             clip_target.clip = comfy.text_encoders.anima.te(**llama_detect(clip_data))
             clip_target.tokenizer = comfy.text_encoders.anima.AnimaTokenizer

comfy/supported_models.py

@@ -15,6 +15,7 @@ import comfy.text_encoders.flux
 import comfy.text_encoders.genmo
 import comfy.text_encoders.lt
 import comfy.text_encoders.hunyuan_video
+import comfy.text_encoders.minimax
 import comfy.text_encoders.cosmos
 import comfy.text_encoders.lumina2
 import comfy.text_encoders.wan
@@ -955,6 +956,33 @@ class LTXAV(LTXV):
         out = model_base.LTXAV(self, device=device)
         return out
 
+class MiniMaxH3(supported_models_base.BASE):
+    unet_config = {
+        "image_model": "minimax_h3",
+    }
+
+    sampling_settings = {
+        "shift": 12.0,
+    }
+
+    unet_extra_config = {}
+    latent_format = latent_formats.MiniMaxH3AV
+
+    memory_usage_factor = 0.114
+
+    supported_inference_dtypes = [torch.bfloat16, torch.float32]
+
+    vae_key_prefix = ["vae."]
+    text_encoder_key_prefix = ["text_encoders."]
+
+    def get_model(self, state_dict, prefix="", device=None):
+        return model_base.MiniMaxH3(self, device=device)
+
+    def clip_target(self, state_dict={}, prefix=""):
+        pref = self.text_encoder_key_prefix[0]
+        detect = comfy.text_encoders.hunyuan_video.llama_detect(state_dict, "{}qwen3vl_32b.transformer.".format(pref))
+        return supported_models_base.ClipTarget(comfy.text_encoders.minimax.MiniMaxH3Tokenizer, comfy.text_encoders.minimax.te(**detect))
+
 class HunyuanVideo(supported_models_base.BASE):
     unet_config = {
         "image_model": "hunyuan_video",
@@ -2407,6 +2435,7 @@ models = [
     GenmoMochi,
     LTXV,
     LTXAV,
+    MiniMaxH3,
     HunyuanVideo15_SR_Distilled,
     HunyuanVideo15,
     HunyuanImage21Refiner,

comfy/text_encoders/llama.py

@@ -264,6 +264,17 @@ class Qwen3VL_4BConfig(Qwen3VL_8BConfig):
     intermediate_size: int = 9728
     lm_head: bool = False  # 4B ties word embeddings
 
+@dataclass
+class Qwen3VL_32BConfig(Qwen3VL_8BConfig):
+    # MiniMax H3 conditioning checkpoint: truncated to the first 50 of 64 layers,
+    # consumed as the unnormalized hidden state after layer 50 (no final norm, no lm_head)
+    hidden_size: int = 5120
+    intermediate_size: int = 25600
+    num_hidden_layers: int = 50
+    num_attention_heads: int = 64
+    lm_head: bool = False
+    final_norm: bool = False
+
 @dataclass
 class Ovis25_2BConfig:
     vocab_size: int = 151936

comfy/text_encoders/minimax.py

@@ -0,0 +1,201 @@
+"""MiniMax H3 text/vision conditioning: Qwen3-VL-32B (truncated to 50 layers).
+
+The H3 presentation is NOT chat-templated: token ids are raw prompt/label text
+(no special tokens) with explicit vision blocks spliced in:
+
+  t2va:   <prompt>
+  fl2va:  "<Picture 1>: " <vision block> ["<Picture 2>: " <vision block>] <prompt>
+  ref2va: per condition in request order (1-based ordinals per type):
+            image -> "<Picture i>: " <vision block>
+            audio -> "<Audio j>: "              (audio never enters Qwen)
+            video -> "<Video k>: " then per 2-frame temporal block
+                     "<T.T seconds>" <vision block(2 frames)>
+          then <prompt>
+
+The conditioning is the unnormalized hidden state after LM layer 50 (the
+converted checkpoint is truncated there, so this is simply the last-layer
+output with no final norm). Vision-pad positions carry adaLN token tag 0
+(video modality) in the DiT; text positions carry tag 1 — the tags are
+returned alongside the embeddings as "minimax_token_tags".
+"""
+
+import math
+
+import torch
+import torch.nn.functional as F
+import comfy.sd1_clip
+from .qwen3vl import Qwen3VL, Qwen3VLSDTokenizer
+
+VISION_START = 151652
+VISION_END = 151653
+QWEN_IMAGE_MEAN = [0.5, 0.5, 0.5]
+QWEN_IMAGE_STD = [0.5, 0.5, 0.5]
+
+
+def process_video_block(frames, patch_size=16, temporal_patch_size=2, merge_size=2,
+                        min_pixels=3136, max_pixels=12845056):
+    """[2, H, W, C] frame pair -> (flatten_patches, grid_thw) with grid_t=1.
+
+    Same resize/normalize policy as process_qwen2vl_images, but the two frames
+    fill the temporal patch instead of repeating a single frame.
+    """
+    t, height, width, _ = frames.shape
+    imgs = frames.permute(0, 3, 1, 2)
+    factor = patch_size * merge_size
+    h_bar = round(height / factor) * factor
+    w_bar = round(width / factor) * factor
+    if h_bar * w_bar > max_pixels:
+        beta = math.sqrt((height * width) / max_pixels)
+        h_bar = max(factor, math.floor(height / beta / factor) * factor)
+        w_bar = max(factor, math.floor(width / beta / factor) * factor)
+    elif h_bar * w_bar < min_pixels:
+        beta = math.sqrt(min_pixels / (height * width))
+        h_bar = math.ceil(height * beta / factor) * factor
+        w_bar = math.ceil(width * beta / factor) * factor
+
+    imgs = F.interpolate(imgs, size=(h_bar, w_bar), mode="bilinear", align_corners=False)
+    mean = torch.tensor(QWEN_IMAGE_MEAN, device=imgs.device).view(1, 3, 1, 1)
+    std = torch.tensor(QWEN_IMAGE_STD, device=imgs.device).view(1, 3, 1, 1)
+    imgs = (imgs - mean) / std
+
+    grid_h = h_bar // patch_size
+    grid_w = w_bar // patch_size
+    patches = imgs.reshape(1, temporal_patch_size, 3, grid_h // merge_size, merge_size,
+                           patch_size, grid_w // merge_size, merge_size, patch_size)
+    patches = patches.permute(0, 3, 6, 4, 7, 2, 1, 5, 8)
+    flatten = patches.reshape(grid_h * grid_w, 3 * temporal_patch_size * patch_size * patch_size)
+    grid_thw = torch.stack([torch.tensor([1, grid_h, grid_w], device=frames.device, dtype=torch.long)])
+    return flatten, grid_thw
+
+
+def token_tags_from_embeds_info(seq_len, embeds_info):
+    # whole vision block VIDEO(0), including the flanking <|vision_start|>/<|vision_end|> tokens
+    # embeds_info spans cover only the expanded embeddings, so widen by one on each side.
+    tags = torch.ones(seq_len, dtype=torch.long)
+    for e in embeds_info:
+        if e.get("type") == "image":
+            tags[max(0, e["index"] - 1):e["index"] + e["size"] + 1] = 0
+    return tags
+
+
+class MiniMaxQwen3VL(Qwen3VL):
+    model_type = "qwen3vl_32b"
+
+    def preprocess_embed(self, embed, device):
+        if embed["type"] == "image" and embed.get("minimax_video_block", False):
+            flatten, grid = process_video_block(embed["data"])
+            merged, deepstack = self.visual(flatten.to(device, dtype=torch.float32), grid)
+            return merged, {"grid": grid, "deepstack": deepstack}
+        return super().preprocess_embed(embed, device)
+
+    def forward(self, input_ids, attention_mask=None, embeds=None, num_tokens=None,
+                intermediate_output=None, final_layer_norm_intermediate=True,
+                dtype=None, embeds_info=[], **kwargs):
+        seq = embeds.shape[1] if embeds is not None else input_ids.shape[1]
+        self.last_token_tags = token_tags_from_embeds_info(seq, embeds_info)
+        return super().forward(input_ids, attention_mask=attention_mask, embeds=embeds,
+                               num_tokens=num_tokens, intermediate_output=intermediate_output,
+                               final_layer_norm_intermediate=final_layer_norm_intermediate,
+                               dtype=dtype, embeds_info=embeds_info, **kwargs)
+
+
+class MiniMaxH3ClipModel(comfy.sd1_clip.SDClipModel):
+    def __init__(self, device="cpu", layer="last", layer_idx=None, dtype=None, model_options={}):
+        super().__init__(device=device, layer="last", layer_idx=None, textmodel_json_config={},
+                         dtype=dtype, special_tokens={"pad": 151643}, layer_norm_hidden_state=False,
+                         model_class=MiniMaxQwen3VL, enable_attention_masks=False,
+                         return_attention_masks=False, model_options=model_options)
+
+    def encode_token_weights(self, token_weight_pairs):
+        out = super().encode_token_weights(token_weight_pairs)
+        tags = getattr(self.transformer, "last_token_tags", None)
+        if tags is not None:
+            extra = out[2] if len(out) > 2 and isinstance(out[2], dict) else {}
+            extra["minimax_token_tags"] = tags
+            out = (out[0], out[1], extra)
+        return out
+
+
+class MiniMaxH3TEModel(comfy.sd1_clip.SD1ClipModel):
+    def __init__(self, device="cpu", dtype=None, model_options={}):
+        super().__init__(device=device, dtype=dtype, name="qwen3vl_32b",
+                         clip_model=MiniMaxH3ClipModel, model_options=model_options)
+
+
+class MiniMaxH3Tokenizer(comfy.sd1_clip.SD1Tokenizer):
+    def __init__(self, embedding_directory=None, tokenizer_data={}):
+        tokenizer = lambda *a, **kw: Qwen3VLSDTokenizer(*a, **kw, embedding_size=5120, embedding_key="qwen3vl_32b")
+        super().__init__(embedding_directory=embedding_directory, tokenizer_data=tokenizer_data, name="qwen3vl_32b", tokenizer=tokenizer)
+
+    def _text_ids(self, text):
+        tok = self.qwen3vl_32b.tokenizer
+        return tok(text, add_special_tokens=False)["input_ids"]
+
+    @staticmethod
+    def _vision_entry(data, video_block=False):
+        emb = {"type": "image", "data": data, "original_type": "image"}
+        if video_block:
+            emb["minimax_video_block"] = True
+        return emb
+
+    def tokenize_with_weights(self, text, return_word_ids=False, images=[],
+                              minimax_ref_items=None, **kwargs):
+        entries = []
+
+        def add_text(s):
+            entries.extend((tid, 1.0) for tid in self._text_ids(s))
+
+        def add_vision(data, video_block=False):
+            entries.append((VISION_START, 1.0))
+            entries.append((self._vision_entry(data, video_block), 1.0))
+            entries.append((VISION_END, 1.0))
+
+        if minimax_ref_items:
+            counters = {"image": 0, "audio": 0, "video": 0}
+            for item in minimax_ref_items:
+                kind = item["type"]
+                counters[kind] += 1
+                if kind == "image":
+                    add_text("<Picture %d>: " % counters["image"])
+                    add_vision(item["data"])
+                elif kind == "audio":
+                    add_text("<Audio %d>: " % counters["audio"])
+                elif kind == "video":
+                    frames = item["data"]  # [T, H, W, C], sampled at 2 fps
+                    timestamps = item.get("timestamps")
+                    if timestamps is None:
+                        timestamps = [i / 2.0 for i in range(frames.shape[0])]
+                    if frames.shape[0] % 2 == 1:  # repeat-pad to temporal patch of 2
+                        frames = torch.cat([frames, frames[-1:]], dim=0)
+                        timestamps = list(timestamps) + [timestamps[-1]]
+                    add_text("<Video %d>: " % counters["video"])
+                    for i in range(0, frames.shape[0], 2):
+                        block_ts = (timestamps[i] + timestamps[i + 1]) / 2.0
+                        add_text("<%.1f seconds>" % block_ts)
+                        add_vision(frames[i:i + 2], video_block=True)
+        else:
+            for i, img in enumerate(images):
+                add_text("<Picture %d>: " % (i + 1))
+                add_vision(img)
+
+        add_text(text)
+        if len(entries) == 0:
+            entries.append((151643, 1.0))
+        if return_word_ids:
+            entries = [t + (0,) for t in entries]
+        return {"qwen3vl_32b": [entries]}
+
+    def untokenize(self, token_weight_pair):
+        return self.qwen3vl_32b.untokenize(token_weight_pair)
+
+
+def te(dtype_llama=None, llama_quantization_metadata=None, **kwargs):
+    class MiniMaxH3TEModel_(MiniMaxH3TEModel):
+        def __init__(self, device="cpu", dtype=None, model_options={}):
+            if dtype_llama is not None:
+                dtype = dtype_llama
+            if llama_quantization_metadata is not None:
+                model_options = model_options.copy()
+                model_options["quantization_metadata"] = llama_quantization_metadata
+            super().__init__(device=device, dtype=dtype, model_options=model_options)
+    return MiniMaxH3TEModel_

comfy/text_encoders/qwen3vl.py

@@ -8,17 +8,18 @@ from transformers import Qwen2Tokenizer
 from comfy import sd1_clip
 import comfy.text_encoders.qwen_vl
 from .qwen35 import Qwen35VisionModel
-from .llama import BaseLlama, BaseQwen3, BaseGenerate, Llama2_, Qwen3VL_4BConfig, Qwen3VL_8BConfig
+from .llama import BaseLlama, BaseQwen3, BaseGenerate, Llama2_, Qwen3VL_4BConfig, Qwen3VL_8BConfig, Qwen3VL_32BConfig
 
 
 QWEN3VL_VISION = {
     "qwen3vl_4b": dict(hidden_size=1024, intermediate_size=4096, depth=24, deepstack_visual_indexes=[5, 11, 17]),
     "qwen3vl_8b": dict(hidden_size=1152, intermediate_size=4304, depth=27, deepstack_visual_indexes=[8, 16, 24]),
+    "qwen3vl_32b": dict(hidden_size=1152, intermediate_size=4304, depth=27, deepstack_visual_indexes=[8, 16, 24]),
 }
 QWEN3VL_VISION_COMMON = dict(num_heads=16, patch_size=16, temporal_patch_size=2, in_channels=3,
                              spatial_merge_size=2, num_position_embeddings=2304)
 
-QWEN3VL_CONFIGS = {"qwen3vl_4b": Qwen3VL_4BConfig, "qwen3vl_8b": Qwen3VL_8BConfig}
+QWEN3VL_CONFIGS = {"qwen3vl_4b": Qwen3VL_4BConfig, "qwen3vl_8b": Qwen3VL_8BConfig, "qwen3vl_32b": Qwen3VL_32BConfig}
 
 
 class Qwen3VLDeepstackMerger(nn.Module):

comfy_extras/nodes_lt.py

@@ -746,6 +746,8 @@ class LTXVConcatAVLatent(io.ComfyNode):
     def define_schema(cls):
         return io.Schema(
             node_id="LTXVConcatAVLatent",
+            display_name="Concat AV Latent",
+            description="Merge a video latent and an audio latent into a joint AV latent (any AV model, e.g. LTXV or MiniMax H3).",
             category="model/latent/ltxv",
             inputs=[
                 io.Latent.Input("video_latent"),
@@ -781,8 +783,9 @@ class LTXVSeparateAVLatent(io.ComfyNode):
     def define_schema(cls):
         return io.Schema(
             node_id="LTXVSeparateAVLatent",
+            display_name="Separate AV Latent",
             category="model/latent/ltxv",
-            description="LTXV Separate AV Latent",
+            description="Split a joint AV latent into its video and audio latents (any AV model, e.g. LTXV or MiniMax H3).",
             inputs=[
                 io.Latent.Input("av_latent"),
             ],

comfy_extras/nodes_minimax_h3.py

@@ -0,0 +1,337 @@
+"""MiniMax H3 nodes: AV latent creation and task conditioning (t2va / fl2va / ref2va).
+
+The H3 packed-DiT consumes, via conditioning:
+- Qwen3-VL-32B hidden states with per-token modality tags (from the minimax CLIP)
+- keyframe / reference condition latents, re-injected every step (never denoised)
+
+Latents are NestedTensor pairs (video [B,24,T,H/16,W/16], audio [B,32,2,T40]);
+sampling runs on the flat pack with any stock sampler (the model handles the
+audio stream's shifted schedule internally).
+"""
+
+import math
+
+import torch
+import torchaudio
+
+import nodes
+import comfy.model_management
+import comfy.model_sampling
+import comfy.nested_tensor
+import comfy.utils
+import node_helpers
+from comfy_api.latest import ComfyExtension, io
+
+CANVAS_MULTIPLE = 32
+BASE_SHORT_EDGE = 768
+MAX_PIXELS = 768 * 1344
+REF_IMAGE_SHORT_EDGE = 2048
+FPS = 24
+AUDIO_LATENT_FPS = 40
+
+
+def align_frame_count(n):
+    while n % 17 != 5:
+        n += 1
+    return n
+
+
+def video_latent_t(frame_count):
+    return 2 if frame_count <= 5 else ((frame_count - 5) // 17) * 5 + 2
+
+
+def temporal_shape(length):
+    frame_count = align_frame_count(max(5, length))
+    duration = frame_count / FPS
+    return frame_count, video_latent_t(frame_count), round(duration * AUDIO_LATENT_FPS)
+
+
+def adapt_canvas(width, height):
+    """768-short-edge canvas with 768*1344 area cap, per-axis round to 32."""
+    ratio = width / height
+    if ratio >= 1.0:
+        nom_w, nom_h = BASE_SHORT_EDGE * ratio, BASE_SHORT_EDGE
+    else:
+        nom_w, nom_h = BASE_SHORT_EDGE, BASE_SHORT_EDGE / ratio
+    if nom_w * nom_h > MAX_PIXELS:
+        s = math.sqrt(MAX_PIXELS / (nom_w * nom_h))
+        nom_w, nom_h = nom_w * s, nom_h * s
+    return (max(CANVAS_MULTIPLE, round(nom_w / CANVAS_MULTIPLE) * CANVAS_MULTIPLE),
+            max(CANVAS_MULTIPLE, round(nom_h / CANVAS_MULTIPLE) * CANVAS_MULTIPLE))
+
+
+def _resize(image, width, height, crop):
+    # image [B, H, W, C] -> [B, height, width, 3]
+    samples = image[..., :3].movedim(-1, 1)
+    samples = comfy.utils.common_upscale(samples, width, height, "lanczos", crop)
+    return samples.movedim(1, -1)
+
+
+def _empty_av_latent(width, height, length, batch_size=1):
+    frame_count, latent_t, audio_t = temporal_shape(length)
+    video = torch.zeros([batch_size, 24, latent_t, height // 16, width // 16],
+                        device=comfy.model_management.intermediate_device())
+    audio = torch.zeros([batch_size, 32, 2, audio_t],
+                        device=comfy.model_management.intermediate_device())
+    return {"samples": comfy.nested_tensor.NestedTensor((video, audio))}, frame_count
+
+
+class EmptyMiniMaxH3LatentAV(io.ComfyNode):
+    @classmethod
+    def define_schema(cls):
+        return io.Schema(
+            node_id="EmptyMiniMaxH3LatentAV",
+            display_name="Empty MiniMax H3 AV Latent",
+            category="model/latent/minimax",
+            description="Joint video+audio latent for MiniMax H3. Duration snaps to the model's 17k+5 frame grid at 24 fps.",
+            inputs=[
+                io.Int.Input("width", default=1344, min=32, max=nodes.MAX_RESOLUTION, step=32),
+                io.Int.Input("height", default=768, min=32, max=nodes.MAX_RESOLUTION, step=32),
+                io.Int.Input("length", default=124, min=5, max=3600, step=17, tooltip="Frame count at 24 fps, snapped up to the model's 17k+5 grid (124 = ~5s; trained range is ~124-362, longer is untested)"),
+            ],
+            outputs=[io.Latent.Output()],
+        )
+
+    @classmethod
+    def execute(cls, width, height, length) -> io.NodeOutput:
+        latent, _ = _empty_av_latent(width, height, length)
+        return io.NodeOutput(latent)
+
+
+class MiniMaxH3ImageToVideo(io.ComfyNode):
+    """t2va and fl2va: prompt (+ optional first/last keyframes) -> conditioning + AV latent."""
+
+    @classmethod
+    def define_schema(cls):
+        return io.Schema(
+            node_id="MiniMaxH3ImageToVideo",
+            display_name="MiniMax H3 Image to Video",
+            category="model/conditioning/minimax",
+            inputs=[
+                io.Clip.Input("clip"),
+                io.Vae.Input("vae"),
+                io.String.Input("prompt", multiline=True, dynamic_prompts=True),
+                io.Int.Input("width", default=1344, min=32, max=nodes.MAX_RESOLUTION, step=32),
+                io.Int.Input("height", default=768, min=32, max=nodes.MAX_RESOLUTION, step=32),
+                io.Int.Input("length", default=124, min=5, max=3600, step=17, tooltip="Frame count at 24 fps, snapped up to the model's 17k+5 grid (124 = ~5s; trained range is ~124-362, longer is untested)"),
+                io.Image.Input("first_frame", optional=True),
+                io.Image.Input("last_frame", optional=True),
+            ],
+            outputs=[io.Conditioning.Output(display_name="positive"), io.Latent.Output()],
+        )
+
+    @classmethod
+    def execute(cls, clip, vae, prompt, width, height, length,
+                first_frame=None, last_frame=None) -> io.NodeOutput:
+        latent, frame_count = _empty_av_latent(width, height, length)
+
+        images = []
+        keyframes = []
+        if first_frame is not None:
+            # geometry anchor: plain stretch to canvas
+            img = _resize(first_frame[:1], width, height, "disabled")
+            images.append(img)
+            keyframes.append({"resolved_frame_index": 0, "image": img})
+        if last_frame is not None:
+            # follower: aspect-preserving cover-crop
+            img = _resize(last_frame[:1], width, height, "center")
+            images.append(img)
+            keyframes.append({"resolved_frame_index": frame_count - 1, "image": img})
+
+        tokens = clip.tokenize(prompt, images=images)
+        cond = clip.encode_from_tokens_scheduled(tokens)
+
+        if keyframes:
+            for kf in keyframes:
+                kf["latent"] = vae.encode(kf.pop("image"))
+            cond = node_helpers.conditioning_set_values(cond, {
+                "minimax_keyframes": keyframes,
+                "minimax_frame_count": frame_count,
+            })
+        return io.NodeOutput(cond, latent)
+
+
+class MiniMaxH3ReferenceToVideo(io.ComfyNode):
+    """ref2va: prompt + reference images / videos / audio -> conditioning + AV latent.
+
+    References enter the presentation in fixed order: images, then videos (each
+    soundtrack's <Audio j> label right before its <Video k>), then standalone
+    audio. Ordinals are 1-based per type, so the prompt refers to them as
+    <Picture i> / <Video k> / <Audio j>.
+    """
+
+    @classmethod
+    def define_schema(cls):
+        return io.Schema(
+            node_id="MiniMaxH3ReferenceToVideo",
+            description="<Picture i> / <Video k> / <Audio j> reference conditioning for MiniMax H3. Use the same tags when prompting.",
+            display_name="MiniMax H3 Reference to Video",
+            category="model/conditioning/minimax",
+            inputs=[
+                io.Clip.Input("clip"),
+                io.Vae.Input("vae"),
+                io.Vae.Input("audio_vae"),
+                io.String.Input("prompt", multiline=True, dynamic_prompts=True),
+                io.Int.Input("width", default=1344, min=32, max=nodes.MAX_RESOLUTION, step=32),
+                io.Int.Input("height", default=768, min=32, max=nodes.MAX_RESOLUTION, step=32),
+                io.Int.Input("length", default=124, min=5, max=3600, step=17, tooltip="Frame count at 24 fps, (124 = ~5s, trained range is ~124-362)"),
+                io.Combo.Input("ref_image_size", options=["match", "max"], default="match",
+                    tooltip="Reference image sizing. 'match' scales each ref (down only, keeping aspect) to the generation's pixel area; 'max' uses the reference pipeline's 2048px short edge for best identity fidelity. Reference tokens ride through every sampling step, so 'max' can be several times slower."),
+                io.Autogrow.Input("ref_images", optional=True,
+                    template=io.Autogrow.TemplatePrefix(
+                        input=io.Image.Input("ref_image", tooltip="Reference image (downscaled to 2048 short edge if larger, never upscaled)"),
+                        prefix="ref_image_", min=0, max=9)),
+                io.Autogrow.Input("ref_videos", optional=True,
+                    template=io.Autogrow.TemplatePrefix(
+                        input=io.Image.Input("ref_video", tooltip="Reference video frames at 24 fps (2-15s)"),
+                        prefix="ref_video_", min=0, max=3)),
+                io.Autogrow.Input("ref_video_audios", optional=True,
+                    template=io.Autogrow.TemplatePrefix(
+                        input=io.Audio.Input("ref_video_audio", tooltip="Soundtrack of the same-numbered reference video"),
+                        prefix="ref_video_audio_", min=0, max=3)),
+                io.Autogrow.Input("ref_audios", optional=True,
+                    template=io.Autogrow.TemplatePrefix(
+                        input=io.Audio.Input("ref_audio", tooltip="Standalone reference audio"),
+                        prefix="ref_audio_", min=0, max=3)),
+            ],
+            outputs=[io.Conditioning.Output(display_name="positive"), io.Latent.Output()],
+        )
+
+    @staticmethod
+    def _encode_ref_audio(audio_vae, audio):
+        waveform = audio["waveform"]  # [B, C, L]
+        sr = audio["sample_rate"]
+        vae_sr = getattr(audio_vae, "audio_sample_rate", 32000)
+        if sr != vae_sr:
+            waveform = torchaudio.functional.resample(waveform, sr, vae_sr)
+        z = audio_vae.encode(waveform[:1].movedim(1, -1))  # [1, 32, 2, T]
+        return z, z.shape[-1]
+
+    @classmethod
+    def execute(cls, clip, vae, audio_vae, prompt, width, height, length, ref_image_size="match",
+                ref_images=None, ref_videos=None, ref_video_audios=None, ref_audios=None) -> io.NodeOutput:
+        latent, frame_count = _empty_av_latent(width, height, length)
+
+        ref_items = []   # for the tokenizer presentation, in request order
+        ref_blocks = []  # for the DiT payload, same order
+
+        for img in (ref_images or {}).values():
+            if img is None:
+                continue
+            h, w = img.shape[1], img.shape[2]
+            if ref_image_size == "match":
+                # aspect-preserving scale (down only) to the generation's pixel area
+                scale = min(1.0, math.sqrt((width * height) / (w * h)))
+            else:
+                scale = min(1.0, REF_IMAGE_SHORT_EDGE / min(w, h))
+            tw = max(CANVAS_MULTIPLE, round(w * scale / CANVAS_MULTIPLE) * CANVAS_MULTIPLE)
+            th = max(CANVAS_MULTIPLE, round(h * scale / CANVAS_MULTIPLE) * CANVAS_MULTIPLE)
+            resized = _resize(img[:1], tw, th, "disabled")
+            z = vae.encode(resized)
+            ref_items.append({"type": "image", "data": resized})
+            ref_blocks.append({"kind": "image", "latent_h": th // 16, "latent_w": tw // 16, "latent": z})
+
+        ref_video_audios = ref_video_audios or {}
+        for name, video_frames in (ref_videos or {}).items():
+            if video_frames is None:
+                continue
+            # index-paired soundtrack: ref_video_audio_N belongs to ref_video_N
+            soundtrack = ref_video_audios.get("ref_video_audio_" + name.rsplit("_", 1)[-1])
+            vh, vw = video_frames.shape[1], video_frames.shape[2]
+            cw, ch = adapt_canvas(vw, vh)
+            if vw * vh < cw * ch:
+                cw = max(CANVAS_MULTIPLE, round(vw / CANVAS_MULTIPLE) * CANVAS_MULTIPLE)
+                ch = max(CANVAS_MULTIPLE, round(vh / CANVAS_MULTIPLE) * CANVAS_MULTIPLE)
+            frames = _resize(video_frames, cw, ch, "disabled")
+            if frames.shape[0] > frame_count:
+                frames = frames[:frame_count]
+            n = frames.shape[0]
+            if n < 5:
+                raise ValueError("MiniMax H3 reference videos need at least 5 frames (~0.2s at 24 fps)")
+            while n % 17 != 5:
+                n -= 1
+            frames = frames[:n]
+            z = vae.encode(frames)
+            audio_latent, ref_audio_t = (None, 0)
+            if soundtrack is not None:
+                audio_latent, ref_audio_t = cls._encode_ref_audio(audio_vae, soundtrack)
+                # the soundtrack gets its own <Audio j> label, emitted before <Video k>
+                ref_items.append({"type": "audio"})
+            # Qwen sees the video at 2 fps with timestamps
+            sample_idx = list(range(0, frames.shape[0], FPS // 2))
+            qwen_frames = frames[sample_idx]
+            ref_items.append({"type": "video", "data": qwen_frames,
+                              "timestamps": [i / 2.0 for i in range(len(sample_idx))]})
+            ref_blocks.append({"kind": "video_audio" if ref_audio_t else "video",
+                               "latent_t": z.shape[2], "latent_h": ch // 16, "latent_w": cw // 16,
+                               "ref_audio_t": ref_audio_t, "latent": z, "audio_latent": audio_latent})
+
+        for audio in (ref_audios or {}).values():
+            if audio is None:
+                continue
+            audio_latent, ref_audio_t = cls._encode_ref_audio(audio_vae, audio)
+            ref_items.append({"type": "audio"})
+            ref_blocks.append({"kind": "audio", "ref_audio_t": ref_audio_t, "audio_latent": audio_latent})
+
+        tokens = clip.tokenize(prompt, minimax_ref_items=ref_items)
+        cond = clip.encode_from_tokens_scheduled(tokens)
+        if ref_blocks:
+            cond = node_helpers.conditioning_set_values(cond, {"minimax_refs": ref_blocks})
+        return io.NodeOutput(cond, latent)
+
+
+class MiniMaxH3SigmaShift(io.ComfyNode):
+    """Set the video/audio flow shifts coherently.
+
+    The video shift drives the sampler's sigma schedule; both values are also
+    handed to the DiT, which inverts the video schedule to the shared base grid
+    and derives the audio schedule from it.
+    """
+
+    @classmethod
+    def define_schema(cls):
+        return io.Schema(
+            node_id="MiniMaxH3SigmaShift",
+            description="Set the video/audio flow shifts.",
+            display_name="MiniMax H3 Sigma Shift",
+            category="model/patch/minimax",
+            inputs=[
+                io.Model.Input("model"),
+                io.Float.Input("shift_video", default=12.0, min=0.01, max=100.0, step=0.01),
+                io.Float.Input("shift_audio", default=3.0, min=0.01, max=100.0, step=0.01),
+            ],
+            outputs=[io.Model.Output()],
+        )
+
+    @classmethod
+    def execute(cls, model, shift_video, shift_audio) -> io.NodeOutput:
+        m = model.clone()
+
+        class ModelSamplingAdvanced(comfy.model_sampling.ModelSamplingDiscreteFlow, comfy.model_sampling.CONST):
+            pass
+
+        original = m.get_model_object("model_sampling")
+        model_sampling = ModelSamplingAdvanced(model.model.model_config)
+        model_sampling.set_parameters(shift=shift_video)
+        if hasattr(original, "noise_scale"):
+            model_sampling.set_noise_scale(original.noise_scale)
+        m.add_object_patch("model_sampling", model_sampling)
+
+        to = m.model_options["transformer_options"] = m.model_options.get("transformer_options", {}).copy()
+        to["minimax_h3_sigma_shift_video"] = shift_video
+        to["minimax_h3_sigma_shift_audio"] = shift_audio
+        return io.NodeOutput(m)
+
+
+class MiniMaxH3Extension(ComfyExtension):
+    async def get_node_list(self):
+        return [
+            EmptyMiniMaxH3LatentAV,
+            MiniMaxH3ImageToVideo,
+            MiniMaxH3ReferenceToVideo,
+            MiniMaxH3SigmaShift
+            ]
+
+
+async def comfy_entrypoint() -> MiniMaxH3Extension:
+    return MiniMaxH3Extension()

nodes.py

@@ -992,7 +992,7 @@ class CLIPLoader:
     @classmethod
     def INPUT_TYPES(s):
         return {"required": { "clip_name": (folder_paths.get_filename_list("text_encoders"), ),
-                              "type": (["stable_diffusion", "stable_cascade", "sd3", "stable_audio", "mochi", "ltxv", "pixart", "cosmos", "lumina2", "wan", "hidream", "chroma", "ace", "omnigen2", "qwen_image", "hunyuan_image", "flux2", "ovis", "longcat_image", "cogvideox", "lens", "pixeldit", "ideogram4", "boogu", "krea2", "joyimage", "mage"], ),
+                              "type": (["stable_diffusion", "stable_cascade", "sd3", "stable_audio", "mochi", "ltxv", "pixart", "cosmos", "lumina2", "wan", "hidream", "chroma", "ace", "omnigen2", "qwen_image", "hunyuan_image", "flux2", "ovis", "longcat_image", "cogvideox", "lens", "pixeldit", "ideogram4", "boogu", "krea2", "joyimage", "mage", "minimax"], ),
                               },
                 "optional": {
                               "device": (["default", "cpu"], {"advanced": True}),
@@ -2436,6 +2436,7 @@ async def init_builtin_extra_nodes():
         "nodes_mahiro.py",
         "nodes_lt_upsampler.py",
         "nodes_lt_audio.py",
+        "nodes_minimax_h3.py",
         "nodes_lt.py",
         "nodes_hooks.py",
         "nodes_multigpu.py",