From 831710d25740257ce84691f1367e5e0eae9c3e33 Mon Sep 17 00:00:00 2001 From: blepping <157360029+blepping@users.noreply.github.com> Date: Fri, 31 Jul 2026 10:12:17 -0600 Subject: [PATCH 01/63] Don't assume sampler_function has a __name__ attribute in detail logging (#15179) --- comfy/samplers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/comfy/samplers.py b/comfy/samplers.py index 9f571ece9..29e6bffb3 100755 --- a/comfy/samplers.py +++ b/comfy/samplers.py @@ -997,7 +997,7 @@ class KSAMPLER(Sampler): def k_callback(x): nonlocal first_step if first_step: - detail("First sampler step: model=%s sampler=%s step=%s total_steps=%s cfg=%s seed=%s sigma=%s sigma_hat=%s latent_shape=%s denoised_shape=%s", model_wrap.model_patcher.model.__class__.__name__, self.sampler_function.__name__, x["i"], total_steps, model_wrap.cfg, extra_args.get("seed"), x.get("sigma"), x.get("sigma_hat"), tuple(x["x"].shape), tuple(x["denoised"].shape)) + detail("First sampler step: model=%s sampler=%s step=%s total_steps=%s cfg=%s seed=%s sigma=%s sigma_hat=%s latent_shape=%s denoised_shape=%s", model_wrap.model_patcher.model.__class__.__name__, getattr(self.sampler_function, "__name__", "unknown"), x["i"], total_steps, model_wrap.cfg, extra_args.get("seed"), x.get("sigma"), x.get("sigma_hat"), tuple(x["x"].shape), tuple(x["denoised"].shape)) first_step = False if callback is not None: callback(x["i"], x["denoised"], x["x"], total_steps) From 9bd9815eabaf02feb6adb07ec3b5e8123fb14cd6 Mon Sep 17 00:00:00 2001 From: Alexander Piskun <13381981+bigcat88@users.noreply.github.com> Date: Fri, 31 Jul 2026 19:59:53 +0300 Subject: [PATCH 02/63] [Partner Nodes] chore(Bria): adjust pricing for other video endpoints (#15171) Signed-off-by: bigcat88 --- comfy_api_nodes/nodes_bria.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/comfy_api_nodes/nodes_bria.py b/comfy_api_nodes/nodes_bria.py index 8077f1398..9e5a93330 100644 --- a/comfy_api_nodes/nodes_bria.py +++ b/comfy_api_nodes/nodes_bria.py @@ -357,7 +357,7 @@ class BriaVideoGreenScreen(IO.ComfyNode): ], is_api_node=True, price_badge=IO.PriceBadge( - expr="""{"type":"usd","usd":0.0042,"format":{"suffix":"/second"}}""", + expr="""{"type":"usd","usd":0.005,"format":{"suffix":"/second"}}""", ), ) @@ -433,7 +433,7 @@ class BriaVideoReplaceBackground(IO.ComfyNode): ], is_api_node=True, price_badge=IO.PriceBadge( - expr="""{"type":"usd","usd":0.0042,"format":{"suffix":"/second"}}""", + expr="""{"type":"usd","usd":0.005,"format":{"suffix":"/second"}}""", ), ) From de5625a64aba29080dc8cbea3a85167da9b8e77f Mon Sep 17 00:00:00 2001 From: rattus <46076784+rattus128@users.noreply.github.com> Date: Sat, 1 Aug 2026 03:29:47 +1000 Subject: [PATCH 03/63] Delay dynamic pin cleanup until model destruction (#15183) --- comfy/model_patcher.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/comfy/model_patcher.py b/comfy/model_patcher.py index e44322e72..6b698f767 100644 --- a/comfy/model_patcher.py +++ b/comfy/model_patcher.py @@ -558,12 +558,9 @@ class ModelPatcher: new_multigpu_models = [] for mm in multigpu_models: # clone main model, but bring over relevant props from existing multigpu clone - n = self.clone() + n = self.clone(model_override=mm.get_clone_model_override()) n.load_device = mm.load_device - n.backup = mm.backup - n.object_patches_backup = mm.object_patches_backup n.hook_backup = mm.hook_backup - n.model = mm.model n.is_multigpu_base_clone = mm.is_multigpu_base_clone n.remove_additional_models("multigpu") orig_additional_models: dict[str, list[ModelPatcher]] = comfy.patcher_extension.copy_nested_dicts(n.additional_models) @@ -1758,6 +1755,9 @@ class ModelPatcherDynamic(ModelPatcher): self.register_load_device(self.load_device) self.non_dynamic_delegate_model = None assert load_device is not None + if not hasattr(self.model, "dynamic_patchers"): + self.model.dynamic_patchers = set() + self.model.dynamic_patchers.add(id(self)) def register_load_device(self, device): """Ensure dynamic_pins has an entry for *device*. @@ -1813,6 +1813,18 @@ class ModelPatcherDynamic(ModelPatcher): def unpin_all_weights(self): self.partially_unload_ram(1e32) + def __del__(self): + model = getattr(self, "model", None) + dynamic_patchers = getattr(model, "dynamic_patchers", None) + if dynamic_patchers is None or id(self) not in dynamic_patchers: + return + dynamic_patchers.discard(id(self)) + try: + if not dynamic_patchers: + self.unpin_all_weights() + finally: + self.detach(unpatch_all=False) + def memory_required(self, input_shape): #Pad this significantly. We are trying to get away from precise estimates. This #estimate is only used when using the ModelPatcherDynamic after ModelPatcher. If you From 7c806288d5210b497cedfca0d111bb56d84a20de Mon Sep 17 00:00:00 2001 From: rattus <46076784+rattus128@users.noreply.github.com> Date: Sat, 1 Aug 2026 03:44:09 +1000 Subject: [PATCH 04/63] ops: apply the custom placeholder logic to Linux too (#15181) Windows has proven this logic works for a long time and there are corner cases where this materialization actual consumes real RAM on linux. Its not as bad as the original windows commit charge surge, but its still a detectable transient leak. So simplify and unify. --- comfy/ops.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/comfy/ops.py b/comfy/ops.py index 9d692dcc7..f4bdd0aef 100644 --- a/comfy/ops.py +++ b/comfy/ops.py @@ -464,8 +464,7 @@ class disable_weight_init: def __init__(self, in_features, out_features, bias=True, device=None, dtype=None): # don't trust subclasses that BYO state dict loader to call us. - if (not comfy.model_management.WINDOWS - or not comfy.memory_management.aimdo_enabled + if (not comfy.memory_management.aimdo_enabled or type(self)._load_from_state_dict is not disable_weight_init.Linear._load_from_state_dict): super().__init__(in_features, out_features, bias, device, dtype) return @@ -487,8 +486,7 @@ class disable_weight_init: def _load_from_state_dict(self, state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs): - if (not comfy.model_management.WINDOWS - or not comfy.memory_management.aimdo_enabled + if (not comfy.memory_management.aimdo_enabled or type(self)._load_from_state_dict is not disable_weight_init.Linear._load_from_state_dict): return super()._load_from_state_dict(state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs) @@ -716,8 +714,7 @@ class disable_weight_init: norm_type=2.0, scale_grad_by_freq=False, sparse=False, _weight=None, _freeze=False, device=None, dtype=None): # don't trust subclasses that BYO state dict loader to call us. - if (not comfy.model_management.WINDOWS - or not comfy.memory_management.aimdo_enabled + if (not comfy.memory_management.aimdo_enabled or type(self)._load_from_state_dict is not disable_weight_init.Embedding._load_from_state_dict): super().__init__(num_embeddings, embedding_dim, padding_idx, max_norm, norm_type, scale_grad_by_freq, sparse, _weight, @@ -744,8 +741,7 @@ class disable_weight_init: def _load_from_state_dict(self, state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs): - if (not comfy.model_management.WINDOWS - or not comfy.memory_management.aimdo_enabled + if (not comfy.memory_management.aimdo_enabled or type(self)._load_from_state_dict is not disable_weight_init.Embedding._load_from_state_dict): return super()._load_from_state_dict(state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs) From 0b73ae83a1284b2e288aae656d88dede6dbb623b Mon Sep 17 00:00:00 2001 From: "Daxiong (Lin)" Date: Sat, 1 Aug 2026 02:20:55 +0800 Subject: [PATCH 05/63] chore: update workflow templates to v0.11.23 (#15187) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 35dd5e613..af414d413 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ comfyui-frontend-package==1.47.11 -comfyui-workflow-templates==0.11.20 +comfyui-workflow-templates==0.11.23 comfyui-embedded-docs==0.5.9 torch torchsde From 6cedd34343ba3214ca9591397bd106e02ef2acf6 Mon Sep 17 00:00:00 2001 From: Alexander Piskun <13381981+bigcat88@users.noreply.github.com> Date: Fri, 31 Jul 2026 21:57:45 +0300 Subject: [PATCH 06/63] [Partner Nodes] fix(ByteDance): encode stereo reference audio without doubling its duration (#15177) * [Partner Nodes] fix(ByteDance): encode stereo reference audio without doubling its duration Signed-off-by: bigcat88 --- comfy_api_nodes/util/conversions.py | 23 +++--- .../audio_conversions_test.py | 78 +++++++++++++++++++ 2 files changed, 89 insertions(+), 12 deletions(-) create mode 100644 tests-unit/comfy_api_nodes_test/audio_conversions_test.py diff --git a/comfy_api_nodes/util/conversions.py b/comfy_api_nodes/util/conversions.py index 9cd644fc0..f46cac3f8 100644 --- a/comfy_api_nodes/util/conversions.py +++ b/comfy_api_nodes/util/conversions.py @@ -266,16 +266,14 @@ def audio_tensor_to_contiguous_ndarray(waveform: torch.Tensor) -> np.ndarray: waveform: a tensor of shape (1, channels, samples) derived from a Comfy `AUDIO` type. Returns: - Contiguous numpy array of the audio waveform. If the audio was batched, - the first item is taken. + Contiguous numpy array of the audio waveform. + + Raises: + ValueError: If the waveform is not shaped (1, channels, samples). """ if waveform.ndim != 3 or waveform.shape[0] != 1: raise ValueError("Expected waveform tensor shape (1, channels, samples)") - # If batch is > 1, take first item - if waveform.shape[0] > 1: - waveform = waveform[0] - # Prepare for av: remove batch dim, move to CPU, make contiguous, convert to numpy array audio_data_np = waveform.squeeze(0).cpu().contiguous().numpy() if audio_data_np.dtype != np.float32: @@ -285,20 +283,21 @@ def audio_tensor_to_contiguous_ndarray(waveform: torch.Tensor) -> np.ndarray: def audio_input_to_mp3(audio: Input.Audio) -> BytesIO: - waveform = audio["waveform"].cpu() + audio_data_np = audio_tensor_to_contiguous_ndarray(audio["waveform"]) + sample_rate = int(audio["sample_rate"]) output_buffer = BytesIO() output_container = av.open(output_buffer, mode="w", format="mp3") - out_stream = output_container.add_stream("libmp3lame", rate=audio["sample_rate"]) + out_stream = output_container.add_stream("libmp3lame", rate=sample_rate) out_stream.bit_rate = 320000 frame = av.AudioFrame.from_ndarray( - waveform.movedim(0, 1).reshape(1, -1).float().numpy(), - format="flt", - layout="mono" if waveform.shape[0] == 1 else "stereo", + audio_data_np, + format="fltp", + layout="stereo" if audio_data_np.shape[0] > 1 else "mono", ) - frame.sample_rate = audio["sample_rate"] + frame.sample_rate = sample_rate frame.pts = 0 output_container.mux(out_stream.encode(frame)) output_container.mux(out_stream.encode(None)) diff --git a/tests-unit/comfy_api_nodes_test/audio_conversions_test.py b/tests-unit/comfy_api_nodes_test/audio_conversions_test.py new file mode 100644 index 000000000..d7c9b3899 --- /dev/null +++ b/tests-unit/comfy_api_nodes_test/audio_conversions_test.py @@ -0,0 +1,78 @@ +import math + +import av +import numpy as np +import pytest +import torch + +from comfy.cli_args import args + +if not torch.cuda.is_available(): + args.cpu = True + +from comfy_api_nodes.util.conversions import audio_input_to_mp3 # noqa: E402 + +SAMPLE_RATE = 48000 +DURATION = 2.0 +LEFT_HZ = 440.0 +RIGHT_HZ = 880.0 + + +def tone(freq, duration=DURATION, sample_rate=SAMPLE_RATE): + t = torch.arange(int(sample_rate * duration), dtype=torch.float32) / sample_rate + return 0.5 * torch.sin(2 * math.pi * freq * t) + + +@pytest.fixture +def stereo_audio(): + """Comfy AUDIO with two tones that stay distinguishable through mp3.""" + waveform = torch.stack([tone(LEFT_HZ), tone(RIGHT_HZ)]).unsqueeze(0) + return {"waveform": waveform, "sample_rate": SAMPLE_RATE} + + +@pytest.fixture +def mono_audio(): + return {"waveform": tone(LEFT_HZ).unsqueeze(0).unsqueeze(0), "sample_rate": SAMPLE_RATE} + + +def decode(buffer): + """(planes[C, N], sample_rate, channels) of an encoded mp3 buffer""" + buffer.seek(0) + with av.open(buffer, mode="r") as container: + stream = container.streams.audio[0] + planes = [] + for frame in container.decode(audio=0): + array = frame.to_ndarray() + if frame.format.is_planar: + planes.append(array) + else: + planes.append(array.reshape(-1, len(frame.layout.channels)).T) + return np.concatenate(planes, axis=1), stream.codec_context.sample_rate, len(stream.layout.channels) + + +def dominant_hz(signal, sample_rate): + """Peak frequency, ignoring the encoder's padding at either edge""" + edge = int(0.2 * sample_rate) + window = signal[edge:-edge] + spectrum = np.abs(np.fft.rfft(window * np.hanning(window.size))) + return np.fft.rfftfreq(window.size, 1.0 / sample_rate)[int(np.argmax(spectrum))] + + +def test_stereo_duration_is_preserved(stereo_audio): + planes, sample_rate, channels = decode(audio_input_to_mp3(stereo_audio)) + assert channels == 2 + assert sample_rate == SAMPLE_RATE + assert planes.shape[1] / sample_rate == pytest.approx(DURATION, abs=0.15) + + +def test_stereo_channels_are_not_concatenated(stereo_audio): + """The channels must be interleaved; concatenating them plays the clip twice.""" + planes, sample_rate, _ = decode(audio_input_to_mp3(stereo_audio)) + assert dominant_hz(planes[0], sample_rate) == pytest.approx(LEFT_HZ, abs=15) + assert dominant_hz(planes[1], sample_rate) == pytest.approx(RIGHT_HZ, abs=15) + + +def test_mono_duration_is_preserved(mono_audio): + planes, sample_rate, _ = decode(audio_input_to_mp3(mono_audio)) + assert planes.shape[1] / sample_rate == pytest.approx(DURATION, abs=0.15) + assert dominant_hz(planes[0], sample_rate) == pytest.approx(LEFT_HZ, abs=15) From a1c421994cdcc5044dbce2bb7628e89386311cc5 Mon Sep 17 00:00:00 2001 From: comfyanonymous <121283862+comfyanonymous@users.noreply.github.com> Date: Fri, 31 Jul 2026 15:17:56 -0700 Subject: [PATCH 07/63] Expand k, v when attention backend would fall back to math because gqa. (#15190) --- comfy/ldm/modules/attention.py | 34 ++++++------------------- comfy/ops.py | 46 ++++++++++++++++++++++++++++------ 2 files changed, 46 insertions(+), 34 deletions(-) diff --git a/comfy/ldm/modules/attention.py b/comfy/ldm/modules/attention.py index e6500cff4..2c549e095 100644 --- a/comfy/ldm/modules/attention.py +++ b/comfy/ldm/modules/attention.py @@ -90,22 +90,6 @@ def default(val, d): return val return d -def _gqa_repeat_factor(query_heads, key_heads, value_heads): - if key_heads != value_heads: - raise ValueError(f"Key/value head count mismatch for GQA: {key_heads} != {value_heads}") - if query_heads == key_heads: - return 1 - if query_heads % key_heads != 0: - raise ValueError(f"Query heads must be divisible by key/value heads for GQA: {query_heads} vs {key_heads}") - return query_heads // key_heads - -def _repeat_kv_for_gqa(k, v, query_heads, head_dim): - n_rep = _gqa_repeat_factor(query_heads, k.shape[head_dim], v.shape[head_dim]) - if n_rep > 1: - k = k.repeat_interleave(n_rep, dim=head_dim) - v = v.repeat_interleave(n_rep, dim=head_dim) - return k, v - def _heads_from_dim(tensor, dim_head, name): inner_dim = tensor.shape[-1] if inner_dim % dim_head != 0: @@ -122,10 +106,8 @@ def _reshape_qkv_to_heads(q, k, v, b, heads, dim_head, enable_gqa=False, expand_ value_heads = heads k = k.unsqueeze(3).reshape(b, -1, key_heads, dim_head) v = v.unsqueeze(3).reshape(b, -1, value_heads, dim_head) - if enable_gqa: - _gqa_repeat_factor(heads, key_heads, value_heads) - if expand_kv: - k, v = _repeat_kv_for_gqa(k, v, heads, -2) + if enable_gqa and expand_kv: + k, v = comfy.ops.repeat_kv_for_gqa(k, v, heads, -2) return q, k, v @@ -196,7 +178,7 @@ def attention_basic(q, k, v, heads, mask=None, attn_precision=None, skip_reshape h = heads if skip_reshape: if kwargs.get("enable_gqa", False): - k, v = _repeat_kv_for_gqa(k, v, q.shape[-3], -3) + k, v = comfy.ops.repeat_kv_for_gqa(k, v, q.shape[-3], -3) q, k, v = map( lambda t: t.reshape(b * heads, -1, dim_head), (q, k, v), @@ -262,7 +244,7 @@ def attention_sub_quad(query, key, value, heads, mask=None, attn_precision=None, if skip_reshape: if kwargs.get("enable_gqa", False): - key, value = _repeat_kv_for_gqa(key, value, query.shape[-3], -3) + key, value = comfy.ops.repeat_kv_for_gqa(key, value, query.shape[-3], -3) query = query.reshape(b * heads, -1, dim_head) value = value.reshape(b * heads, -1, dim_head) key = key.reshape(b * heads, -1, dim_head).movedim(1, 2) @@ -338,7 +320,7 @@ def attention_split(q, k, v, heads, mask=None, attn_precision=None, skip_reshape if skip_reshape: if kwargs.get("enable_gqa", False): - k, v = _repeat_kv_for_gqa(k, v, q.shape[-3], -3) + k, v = comfy.ops.repeat_kv_for_gqa(k, v, q.shape[-3], -3) q, k, v = map( lambda t: t.reshape(b * heads, -1, dim_head), (q, k, v), @@ -476,7 +458,7 @@ def attention_xformers(q, k, v, heads, mask=None, attn_precision=None, skip_resh (q, k, v), ) if kwargs.get("enable_gqa", False): - k, v = _repeat_kv_for_gqa(k, v, q.shape[-2], -2) + k, v = comfy.ops.repeat_kv_for_gqa(k, v, q.shape[-2], -2) # actually do the reshaping else: dim_head //= heads @@ -573,7 +555,7 @@ def attention_sage(q, k, v, heads, mask=None, attn_precision=None, skip_reshape= b, _, _, dim_head = q.shape tensor_layout = "HND" if kwargs.get("enable_gqa", False): - k, v = _repeat_kv_for_gqa(k, v, q.shape[-3], -3) + k, v = comfy.ops.repeat_kv_for_gqa(k, v, q.shape[-3], -3) else: b, _, dim_head = q.shape dim_head //= heads @@ -671,7 +653,7 @@ def attention3_sage(q, k, v, heads, mask=None, attn_precision=None, skip_reshape if skip_reshape: q_s = q if kwargs.get("enable_gqa", False): - k_s, v_s = _repeat_kv_for_gqa(k, v, H, -3) + k_s, v_s = comfy.ops.repeat_kv_for_gqa(k, v, H, -3) else: k_s, v_s = k, v else: diff --git a/comfy/ops.py b/comfy/ops.py index f4bdd0aef..6c3845eef 100644 --- a/comfy/ops.py +++ b/comfy/ops.py @@ -19,6 +19,7 @@ import torch import logging import contextlib +import inspect import comfy.model_management from comfy.cli_args import args, PerformanceFeature import comfy.float @@ -36,30 +37,59 @@ def run_every_op(): comfy.model_management.throw_exception_if_processing_interrupted() +def gqa_repeat_factor(query_heads, key_heads, value_heads): + if key_heads != value_heads: + raise ValueError(f"Key/value head count mismatch for GQA: {key_heads} != {value_heads}") + if query_heads == key_heads: + return 1 + if query_heads % key_heads != 0: + raise ValueError(f"Query heads must be divisible by key/value heads for GQA: {query_heads} vs {key_heads}") + return query_heads // key_heads + +def repeat_kv_for_gqa(k, v, query_heads, head_dim): + n_rep = gqa_repeat_factor(query_heads, k.shape[head_dim], v.shape[head_dim]) + if n_rep > 1: + k = k.repeat_interleave(n_rep, dim=head_dim) + v = v.repeat_interleave(n_rep, dim=head_dim) + return k, v + def scaled_dot_product_attention(q, k, v, *args, **kwargs): + attn_mask = args[0] if len(args) > 0 else kwargs.get("attn_mask") + if kwargs.get("enable_gqa", False) and attn_mask is not None: + k, v = repeat_kv_for_gqa(k, v, q.shape[-3], -3) + kwargs["enable_gqa"] = False return torch.nn.functional.scaled_dot_product_attention(q, k, v, *args, **kwargs) try: if torch.cuda.is_available(): from torch.nn.attention import SDPBackend, sdpa_kernel - import inspect if "set_priority" in inspect.signature(sdpa_kernel).parameters: SDPA_BACKEND_PRIORITY = [ SDPBackend.FLASH_ATTENTION, + SDPBackend.CUDNN_ATTENTION, SDPBackend.EFFICIENT_ATTENTION, SDPBackend.MATH, ] - if comfy.model_management.WINDOWS: - SDPA_BACKEND_PRIORITY.insert(0, SDPBackend.CUDNN_ATTENTION) - else: - SDPA_BACKEND_PRIORITY.insert(1, SDPBackend.CUDNN_ATTENTION) - def scaled_dot_product_attention(q, k, v, *args, **kwargs): - if q.nelement() < 1024 * 128: # arbitrary number, for small inputs cudnn attention seems slower - return torch.nn.functional.scaled_dot_product_attention(q, k, v, *args, **kwargs) + attn_mask = args[0] if len(args) > 0 else kwargs.get("attn_mask") + if kwargs.get("enable_gqa", False) and attn_mask is not None and not comfy.model_management.is_nvidia(): + k, v = repeat_kv_for_gqa(k, v, q.shape[-3], -3) + kwargs["enable_gqa"] = False with sdpa_kernel(SDPA_BACKEND_PRIORITY, set_priority=True): + if kwargs.get("enable_gqa", False) and attn_mask is not None and q.shape[-3] != k.shape[-3]: + dropout_p = args[1] if len(args) > 1 else kwargs.get("dropout_p", 0.0) + is_causal = args[2] if len(args) > 2 else kwargs.get("is_causal", False) + params = torch.backends.cuda.SDPAParams(q, k, v, attn_mask, dropout_p, is_causal, True) + supports_native_gqa = ( + torch.backends.cuda.can_use_flash_attention(params) + or torch.backends.cuda.can_use_cudnn_attention(params) + or torch.backends.cuda.can_use_efficient_attention(params) + ) + if not supports_native_gqa: + k, v = repeat_kv_for_gqa(k, v, q.shape[-3], -3) + kwargs["enable_gqa"] = False return torch.nn.functional.scaled_dot_product_attention(q, k, v, *args, **kwargs) else: logging.warning("Torch version too old to set sdpa backend priority.") From 235b466a0cb26d47c24f2ab66d1a8c5e70b21070 Mon Sep 17 00:00:00 2001 From: comfyanonymous <121283862+comfyanonymous@users.noreply.github.com> Date: Fri, 31 Jul 2026 21:27:48 -0700 Subject: [PATCH 08/63] Add crf option to save video node. (#15191) --- comfy_api/latest/_input/video_types.py | 2 ++ comfy_api/latest/_input_impl/video_types.py | 11 +++++- comfy_extras/nodes_video.py | 35 ++++++++++++++++--- tests-unit/comfy_api_test/video_types_test.py | 18 ++++++++++ 4 files changed, 61 insertions(+), 5 deletions(-) diff --git a/comfy_api/latest/_input/video_types.py b/comfy_api/latest/_input/video_types.py index e2e99521f..b700d44f5 100644 --- a/comfy_api/latest/_input/video_types.py +++ b/comfy_api/latest/_input/video_types.py @@ -29,11 +29,13 @@ class VideoInput(ABC): codec: VideoCodec = VideoCodec.AUTO, metadata: Optional[dict] = None, bit_depth: int | None = None, + crf: float | None = None, ): """ Abstract method to save the video input to a file. bit_depth selects the encoded bit depth; None keeps the video's native depth. + crf selects the H.264 constant rate factor; None uses the encoder default. """ pass diff --git a/comfy_api/latest/_input_impl/video_types.py b/comfy_api/latest/_input_impl/video_types.py index f5af41973..14d663881 100644 --- a/comfy_api/latest/_input_impl/video_types.py +++ b/comfy_api/latest/_input_impl/video_types.py @@ -460,6 +460,7 @@ class VideoFromFile(VideoInput): codec: VideoCodec = VideoCodec.AUTO, metadata: Optional[dict] = None, bit_depth: int | None = None, + crf: float | None = None, ): if isinstance(self.__file, io.BytesIO): self.__file.seek(0) # Reset the BytesIO object to the beginning @@ -475,13 +476,15 @@ class VideoFromFile(VideoInput): reuse_streams = False if bit_depth is not None and video_encoding is not None and bit_depth != source_bit_depth: reuse_streams = False + if crf is not None: + reuse_streams = False if self.__start_time or self.__duration: reuse_streams = False if not reuse_streams: if bit_depth is None: bit_depth = source_bit_depth - return self._save_transcoded(container, path, format=format, codec=codec, metadata=metadata, bit_depth=bit_depth) + return self._save_transcoded(container, path, format=format, codec=codec, metadata=metadata, bit_depth=bit_depth, crf=crf) streams = container.streams @@ -514,6 +517,7 @@ class VideoFromFile(VideoInput): codec: VideoCodec, metadata: dict | None, bit_depth: int, + crf: float | None = None, ): """Re-encode to H.264/AAC one frame at a time; peak memory does not scale with video length.""" open_kwargs = mp4_output_open_kwargs(path, format, codec) @@ -659,6 +663,8 @@ class VideoFromFile(VideoInput): out_video.width = out_width out_video.height = out_height out_video.pix_fmt = pix_fmt + if crf is not None: + out_video.options = {"crf": str(crf)} # source pts pass through (rebased to 0), so variable frame rate survives out_video.codec_context.time_base = video_stream.time_base if audio_stream is not None: @@ -827,6 +833,7 @@ class VideoFromComponents(VideoInput): codec: VideoCodec = VideoCodec.AUTO, metadata: Optional[dict] = None, bit_depth: int | None = None, + crf: float | None = None, ): """Save the video to a file path or BytesIO buffer.""" open_kwargs = mp4_output_open_kwargs(path, format, codec) @@ -847,6 +854,8 @@ class VideoFromComponents(VideoInput): video_stream.width = self.__components.images.shape[2] video_stream.height = self.__components.images.shape[1] video_stream.pix_fmt = pix_fmt + if crf is not None: + video_stream.options = {"crf": str(crf)} # Create an audio stream audio_sample_rate = 1 diff --git a/comfy_extras/nodes_video.py b/comfy_extras/nodes_video.py index 3bfd00be4..45394ce4d 100644 --- a/comfy_extras/nodes_video.py +++ b/comfy_extras/nodes_video.py @@ -86,7 +86,31 @@ class SaveVideo(io.ComfyNode): io.Video.Input("video", tooltip="The video to save."), io.String.Input("filename_prefix", default="video/ComfyUI", tooltip="The prefix for the file to save. This may include formatting information such as %date:yyyy-MM-dd% or %Empty Latent Image.width% to include values from nodes."), io.Combo.Input("format", options=Types.VideoContainer.as_input(), default="auto", tooltip="The format to save the video as."), - io.Combo.Input("codec", options=Types.VideoCodec.as_input(), default="auto", tooltip="The codec to use for the video."), + io.DynamicCombo.Input( + "codec", + options=[ + io.DynamicCombo.Option("auto", []), + io.DynamicCombo.Option( + "h264", + [ + io.DynamicCombo.Input( + "encoding", + display_name="encoding mode", + options=[ + io.DynamicCombo.Option("auto", []), + io.DynamicCombo.Option( + "re-encode", + [io.Float.Input("crf", default=23.0, min=0.0, max=51.0, step=1.0, tooltip="Lower values produce higher quality and larger files.")], + ), + ], + optional=True, + tooltip="Automatic preserves compatible H.264 streams. Re-encode applies a custom CRF.", + ), + ], + ), + ], + tooltip="The codec to use for the video.", + ), ], hidden=[io.Hidden.prompt, io.Hidden.extra_pnginfo], is_output_node=True, @@ -94,7 +118,9 @@ class SaveVideo(io.ComfyNode): ) @classmethod - def execute(cls, video: Input.Video, filename_prefix, format: str, codec) -> io.NodeOutput: + def execute(cls, video: Input.Video, filename_prefix, format: str, codec: io.DynamicCombo.Type) -> io.NodeOutput: + codec_name = codec["codec"] + encoding = codec.get("encoding") or {} width, height = video.get_dimensions() full_output_folder, filename, counter, subfolder, filename_prefix = folder_paths.get_save_image_path( filename_prefix, @@ -115,8 +141,9 @@ class SaveVideo(io.ComfyNode): video.save_to( os.path.join(full_output_folder, file), format=Types.VideoContainer(format), - codec=codec, - metadata=saved_metadata + codec=codec_name, + metadata=saved_metadata, + crf=encoding.get("crf"), ) return io.NodeOutput(video, ui=ui.PreviewVideo([ui.SavedResult(file, subfolder, io.FolderType.output)])) diff --git a/tests-unit/comfy_api_test/video_types_test.py b/tests-unit/comfy_api_test/video_types_test.py index ae758bd40..dd95dc843 100644 --- a/tests-unit/comfy_api_test/video_types_test.py +++ b/tests-unit/comfy_api_test/video_types_test.py @@ -240,6 +240,24 @@ def test_duration_consistency(video_components): assert duration == pytest.approx(manual_duration) +def test_save_to_h264_crf_controls_quality(tmp_path): + generator = torch.Generator().manual_seed(7) + components = VideoComponents( + images=torch.rand(12, 64, 64, 3, generator=generator), + frame_rate=Fraction(30), + ) + high_quality = str(tmp_path / "high_quality.mp4") + low_quality = str(tmp_path / "low_quality.mp4") + transcoded = str(tmp_path / "transcoded.mp4") + + VideoFromComponents(components).save_to(high_quality, codec=VideoCodec.H264, crf=0) + VideoFromComponents(components).save_to(low_quality, codec=VideoCodec.H264, crf=51) + assert os.path.getsize(high_quality) > os.path.getsize(low_quality) + + VideoFromFile(high_quality).save_to(transcoded, codec=VideoCodec.H264, crf=51) + assert os.path.getsize(transcoded) < os.path.getsize(high_quality) + + def create_transcode_source( width=64, height=64, frames=30, fps=30, audio_streams=1, undecodable_audio=0, rotation=False, container_format="mov", audio_codec="pcm_s16le", From 2881e6161081439b1c3fb3b6c1f51b3d272da710 Mon Sep 17 00:00:00 2001 From: comfyanonymous <121283862+comfyanonymous@users.noreply.github.com> Date: Sat, 1 Aug 2026 00:21:28 -0700 Subject: [PATCH 09/63] Store mp4 metadata at the beginning of the file when possible. (#15195) --- comfy_api/latest/_input_impl/video_types.py | 10 +++++++--- tests-unit/comfy_api_test/input_impl_test.py | 5 +++-- tests-unit/comfy_api_test/video_types_test.py | 12 ++++++++++++ 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/comfy_api/latest/_input_impl/video_types.py b/comfy_api/latest/_input_impl/video_types.py index 14d663881..cf4119250 100644 --- a/comfy_api/latest/_input_impl/video_types.py +++ b/comfy_api/latest/_input_impl/video_types.py @@ -36,13 +36,15 @@ def get_open_write_kwargs( dest: str | io.BytesIO, container_format: str, to_format: str | None ) -> dict: """Get kwargs for writing a `VideoFromFile` to a file/stream with `av.open`""" + is_write_to_buffer = isinstance(dest, io.BytesIO) + is_mp4_file = not is_write_to_buffer and os.path.splitext(dest)[1].lower() == ".mp4" + movflags = "use_metadata_tags+faststart" if is_mp4_file else "use_metadata_tags" open_kwargs = { "mode": "w", # If isobmff, preserve custom metadata tags (workflow, prompt, extra_pnginfo) - "options": {"movflags": "use_metadata_tags"}, + "options": {"movflags": movflags}, } - is_write_to_buffer = isinstance(dest, io.BytesIO) if is_write_to_buffer: # Set output format explicitly, since it cannot be inferred from file extension if to_format == VideoContainer.AUTO: @@ -103,7 +105,9 @@ def mp4_output_open_kwargs(path: str | io.BytesIO, format: VideoContainer, codec raise ValueError("Only MP4 format is supported for now") if codec != VideoCodec.AUTO and codec != VideoCodec.H264: raise ValueError("Only H264 codec is supported for now") - open_kwargs = {"mode": "w", "options": {"movflags": "use_metadata_tags"}} + # FFmpeg's faststart pass reopens the output by filename, so it cannot be used with file-like objects. + movflags = "use_metadata_tags+faststart" if isinstance(path, (str, os.PathLike)) else "use_metadata_tags" + open_kwargs = {"mode": "w", "options": {"movflags": movflags}} if isinstance(format, VideoContainer) and format != VideoContainer.AUTO: open_kwargs["format"] = format.value elif isinstance(path, io.BytesIO): diff --git a/tests-unit/comfy_api_test/input_impl_test.py b/tests-unit/comfy_api_test/input_impl_test.py index 5fc21a9a7..f1924f163 100644 --- a/tests-unit/comfy_api_test/input_impl_test.py +++ b/tests-unit/comfy_api_test/input_impl_test.py @@ -36,6 +36,7 @@ def test_get_open_write_kwargs_filepath_no_format(): kwargs_specific = get_open_write_kwargs("output.avi", "mp4", "avi") fail_msg = "Format should not be set for file paths (Specific)" assert "format" not in kwargs_specific, fail_msg + assert kwargs_specific["options"]["movflags"] == "use_metadata_tags" def test_get_open_write_kwargs_base_options_mode(): @@ -43,9 +44,9 @@ def test_get_open_write_kwargs_base_options_mode(): kwargs = get_open_write_kwargs("output.mp4", "mp4", VideoContainer.AUTO) assert kwargs["mode"] == "w", "mode should be set to write" - fail_msg = "movflags should be set to preserve custom metadata tags" + fail_msg = "movflags should preserve custom metadata tags and enable faststart for MP4 files" assert "movflags" in kwargs["options"], fail_msg - assert kwargs["options"]["movflags"] == "use_metadata_tags", fail_msg + assert kwargs["options"]["movflags"] == "use_metadata_tags+faststart", fail_msg def test_get_open_write_kwargs_bytesio_auto_format(): diff --git a/tests-unit/comfy_api_test/video_types_test.py b/tests-unit/comfy_api_test/video_types_test.py index dd95dc843..f688d3eca 100644 --- a/tests-unit/comfy_api_test/video_types_test.py +++ b/tests-unit/comfy_api_test/video_types_test.py @@ -258,6 +258,18 @@ def test_save_to_h264_crf_controls_quality(tmp_path): assert os.path.getsize(transcoded) < os.path.getsize(high_quality) +def test_save_to_mp4_writes_metadata_before_media(video_components, tmp_path): + encoded = tmp_path / "encoded.mp4" + remuxed = tmp_path / "remuxed.mp4" + + VideoFromComponents(video_components).save_to(str(encoded), metadata={"prompt": {"test": "value"}}) + VideoFromFile(str(encoded)).save_to(str(remuxed), metadata={"prompt": {"test": "value"}}) + + for path in (encoded, remuxed): + data = path.read_bytes() + assert data.index(b"moov") < data.index(b"mdat") + + def create_transcode_source( width=64, height=64, frames=30, fps=30, audio_streams=1, undecodable_audio=0, rotation=False, container_format="mov", audio_codec="pcm_s16le", From d8e6aa55f377d9914e64bf07d7633dfe6abbf643 Mon Sep 17 00:00:00 2001 From: Alexander Piskun <13381981+bigcat88@users.noreply.github.com> Date: Sat, 1 Aug 2026 20:30:00 +0300 Subject: [PATCH 10/63] [Partner Nodes] feat(xAI): update nodes for grok-imagine-video-1.5 model (#15197) * [Partner Nodes] feat(xAI): update nodes for grok-imagine-video-1.5 model Signed-off-by: Alexander Piskun --- comfy_api_nodes/apis/grok.py | 5 ++ comfy_api_nodes/nodes_grok.py | 158 +++++++++++++++++++++++++++++++--- 2 files changed, 152 insertions(+), 11 deletions(-) diff --git a/comfy_api_nodes/apis/grok.py b/comfy_api_nodes/apis/grok.py index fbedb53e0..526d8c8ab 100644 --- a/comfy_api_nodes/apis/grok.py +++ b/comfy_api_nodes/apis/grok.py @@ -15,6 +15,10 @@ class InputUrlObject(BaseModel): url: str = Field(...) +class VoiceReferenceObject(BaseModel): + voice_id: str = Field(...) + + class ImageEditRequest(BaseModel): model: str = Field(...) images: list[InputUrlObject] = Field(...) @@ -31,6 +35,7 @@ class VideoGenerationRequest(BaseModel): prompt: str = Field(...) image: InputUrlObject | None = Field(None) reference_images: list[InputUrlObject] | None = Field(None) + reference_audios: list[VoiceReferenceObject] | None = Field(None) duration: int = Field(...) aspect_ratio: str | None = Field(...) resolution: str = Field(...) diff --git a/comfy_api_nodes/nodes_grok.py b/comfy_api_nodes/nodes_grok.py index a95b35917..672a3e537 100644 --- a/comfy_api_nodes/nodes_grok.py +++ b/comfy_api_nodes/nodes_grok.py @@ -1,3 +1,5 @@ +import re + import torch from typing_extensions import override @@ -12,6 +14,7 @@ from comfy_api_nodes.apis.grok import ( VideoGenerationRequest, VideoGenerationResponse, VideoStatusResponse, + VoiceReferenceObject, ) from comfy_api_nodes.util import ( ApiEndpoint, @@ -33,6 +36,75 @@ _GROK_VIDEO_MODEL_API_IDS = { "grok-imagine-video-1.5": "grok-imagine-video-1.5", } +_GROK_VOICE_OPTIONS = [ + "none", + "ara", + "eve", + "leo", + "rex", + "sal", + "carina", + "zagan", + "helix", + "orion", + "luna", + "iris", + "altair", + "zenith", + "perseus", + "helios", + "lux", + "kepler", + "rigel", + "cosmo", + "celeste", + "ursa", + "sirius", + "lumen", + "castor", + "naksh", + "atlas", +] + + +_GROK_REF_TAG_RE = re.compile(r"(?\d*)(?!\w)", re.IGNORECASE | re.ASCII) + + +def _normalize_grok_reference_prompt(prompt: str, total_images: int, voices: list[str]) -> str: + """Rewrite @Image1/@Audio1 style references (1-based, shared partner-node syntax) + into Grok's native / tags; an unnumbered @image/@audio means the first one. + Native tags pass through untouched. @ImageN refers to the Nth reference image overall, in + input order — a batched input contributes one number per image. @AudioN refers to the + 'voice_N' widget; the API only accepts compact arrays, so voices are remapped to array + positions and 'none' slots between selected voices are harmless. Substitution repeats until + stable so adjacent tags like '@Image1@Image2' all resolve.""" + audio_indices: dict[int, int] = {} + for slot, voice in enumerate(voices, start=1): + if voice != "none": + audio_indices[slot] = len(audio_indices) + + def repl(match: re.Match) -> str: + kind = match.group(1).lower() + idx = int(match.group("idx") or 1) + if kind == "image": + if not 1 <= idx <= total_images: + raise ValueError( + f"The prompt references @Image{idx}, but only {total_images} " + f"reference images are connected (a batched input counts once per image)." + ) + return f"" + if idx not in audio_indices: + if 1 <= idx <= len(voices): + raise ValueError(f"The prompt references @Audio{idx}, but 'voice_{idx}' is set to 'none'.") + raise ValueError(f"The prompt references @Audio{idx}, but only voices 1..{len(voices)} exist.") + return f"" + + prev = None + while prev != prompt: + prev = prompt + prompt = _GROK_REF_TAG_RE.sub(repl, prompt) + return prompt + def _extract_grok_price(response) -> float | None: if response.usage and response.usage.cost_in_usd_ticks is not None: @@ -509,12 +581,13 @@ class GrokVideoNode(IO.ComfyNode): IO.Combo.Input( "model", options=["grok-imagine-video", "grok-imagine-video-1.5"], - tooltip="grok-imagine-video-1.5 currently always requires an input image.", + tooltip="The model to use for video generation.", ), IO.String.Input( "prompt", multiline=True, - tooltip="Text description of the desired video.", + tooltip="Text description of the desired video. " + "Optional for grok-imagine-video-1.5 when an input image is provided.", ), IO.Combo.Input( "resolution", @@ -549,7 +622,7 @@ class GrokVideoNode(IO.ComfyNode): IO.Image.Input( "image", optional=True, - tooltip="Optional starting image for grok-imagine-video. Required for grok-imagine-video-1.5.", + tooltip="Optional starting image. If omitted, the video is generated from the text prompt alone.", ), ], outputs=[ @@ -589,8 +662,6 @@ class GrokVideoNode(IO.ComfyNode): seed: int, image: Input.Image | None = None, ) -> IO.NodeOutput: - if image is None and model == "grok-imagine-video-1.5": - raise ValueError(f"The '{model}' model requires an input image; connect one to the 'image' input.") if resolution == "1080p" and model != "grok-imagine-video-1.5": raise ValueError(f"1080p resolution is only available for grok-imagine-video-1.5, not '{model}'.") image_url = None @@ -598,7 +669,8 @@ class GrokVideoNode(IO.ComfyNode): if get_number_of_images(image) != 1: raise ValueError("Only one input image is supported.") image_url = InputUrlObject(url=f"data:image/png;base64,{tensor_to_base64_string(image)}") - validate_string(prompt, strip_whitespace=True, min_length=1) + if image is None or model != "grok-imagine-video-1.5": + validate_string(prompt, strip_whitespace=True, min_length=1) initial_response = await sync_op( cls, ApiEndpoint(path="/proxy/xai/v1/videos/generations", method="POST"), @@ -709,7 +781,7 @@ class GrokVideoReferenceNode(IO.ComfyNode): node_id="GrokVideoReferenceNode", display_name="Grok Reference-to-Video", category="partner/video/Grok", - description="Generate video guided by reference images as style and content references.", + description="Generate video guided by reference images, with optional preset voice references.", inputs=[ IO.String.Input( "prompt", @@ -719,6 +791,57 @@ class GrokVideoReferenceNode(IO.ComfyNode): IO.DynamicCombo.Input( "model", options=[ + IO.DynamicCombo.Option( + "grok-imagine-video-1.5", + [ + IO.Autogrow.Input( + "reference_images", + template=IO.Autogrow.TemplateNames( + IO.Image.Input("image"), + names=[f"reference_{i}" for i in range(1, 8)], + min=1, + ), + tooltip="Up to 7 reference images to guide the video generation. " + "Refer to them in the prompt as @Image1 ... @Image7, numbered " + "in input order; a batched input counts once per image.", + ), + IO.Combo.Input( + "voice_1", + options=_GROK_VOICE_OPTIONS, + tooltip="Optional preset voice reference; refer to it in the prompt as @Audio1. " + "The API supports only these preset voices, not custom audio.", + ), + IO.Combo.Input( + "voice_2", + options=_GROK_VOICE_OPTIONS, + tooltip="Optional second voice reference; @Audio2 in the prompt.", + ), + IO.Combo.Input( + "voice_3", + options=_GROK_VOICE_OPTIONS, + tooltip="Optional third voice reference; @Audio3 in the prompt.", + ), + IO.Combo.Input( + "resolution", + options=["480p", "720p"], + tooltip="The resolution of the output video.", + ), + IO.Combo.Input( + "aspect_ratio", + options=["16:9", "4:3", "3:2", "1:1", "2:3", "3:4", "9:16"], + tooltip="The aspect ratio of the output video.", + ), + IO.Int.Input( + "duration", + default=6, + min=1, + max=15, + step=1, + tooltip="The duration of the output video in seconds.", + display_mode=IO.NumberDisplay.slider, + ), + ], + ), IO.DynamicCombo.Option( "grok-imagine-video", [ @@ -779,16 +902,20 @@ class GrokVideoReferenceNode(IO.ComfyNode): is_api_node=True, price_badge=IO.PriceBadge( depends_on=IO.PriceBadgeDepends( - widgets=["model.duration", "model.resolution"], + widgets=["model", "model.duration", "model.resolution"], input_groups=["model.reference_images"], ), expr=""" ( + $is15 := $contains(widgets.model, "1.5"); $res := $lookup(widgets, "model.resolution"); $dur := $lookup(widgets, "model.duration"); $refs := $lookup(inputGroups, "model.reference_images"); - $rate := $res = "720p" ? 0.07 : 0.05; - $price := ($rate * $dur + 0.002 * $refs) * 1.43; + $rate := $is15 + ? ($res = "720p" ? 0.14 : 0.08) + : ($res = "720p" ? 0.07 : 0.05); + $imgCost := $is15 ? 0.01 : 0.002; + $price := ($rate * $dur + $imgCost * $refs) * 1.43; {"type":"usd","usd": $price} ) """, @@ -803,6 +930,14 @@ class GrokVideoReferenceNode(IO.ComfyNode): seed: int, ) -> IO.NodeOutput: validate_string(prompt, strip_whitespace=True, min_length=1) + total_images = sum(get_number_of_images(t) for t in model["reference_images"].values()) + if total_images > 7: + raise ValueError(f"A maximum of 7 reference images is supported; {total_images} are connected.") + reference_audios = None + if model["model"] == "grok-imagine-video-1.5": + voices = [model.get(f"voice_{i}", "none") for i in range(1, 4)] + reference_audios = [VoiceReferenceObject(voice_id=v) for v in voices if v != "none"] or None + prompt = _normalize_grok_reference_prompt(prompt, total_images=total_images, voices=voices) ref_image_urls = await upload_images_to_comfyapi( cls, list(model["reference_images"].values()), @@ -814,8 +949,9 @@ class GrokVideoReferenceNode(IO.ComfyNode): cls, ApiEndpoint(path="/proxy/xai/v1/videos/generations", method="POST"), data=VideoGenerationRequest( - model=model["model"], + model=_GROK_VIDEO_MODEL_API_IDS.get(model["model"], model["model"]), reference_images=[InputUrlObject(url=i) for i in ref_image_urls], + reference_audios=reference_audios, prompt=prompt, resolution=model["resolution"], duration=model["duration"], From 1091c47b3ae9ac423097649a5b7593d369d03bec Mon Sep 17 00:00:00 2001 From: Alexander Piskun <13381981+bigcat88@users.noreply.github.com> Date: Sat, 1 Aug 2026 20:43:42 +0300 Subject: [PATCH 11/63] [Partner Nodes] chore(Bria): increase price for video endpoints (#15186) Signed-off-by: Alexander Piskun --- comfy_api_nodes/nodes_bria.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/comfy_api_nodes/nodes_bria.py b/comfy_api_nodes/nodes_bria.py index 9e5a93330..77f780a3b 100644 --- a/comfy_api_nodes/nodes_bria.py +++ b/comfy_api_nodes/nodes_bria.py @@ -289,7 +289,7 @@ class BriaRemoveVideoBackground(IO.ComfyNode): ], is_api_node=True, price_badge=IO.PriceBadge( - expr="""{"type":"usd","usd":0.005,"format":{"suffix":"/second"}}""", + expr="""{"type":"usd","usd":0.05,"format":{"suffix":"/second"}}""", ), ) @@ -357,7 +357,7 @@ class BriaVideoGreenScreen(IO.ComfyNode): ], is_api_node=True, price_badge=IO.PriceBadge( - expr="""{"type":"usd","usd":0.005,"format":{"suffix":"/second"}}""", + expr="""{"type":"usd","usd":0.05,"format":{"suffix":"/second"}}""", ), ) @@ -433,7 +433,7 @@ class BriaVideoReplaceBackground(IO.ComfyNode): ], is_api_node=True, price_badge=IO.PriceBadge( - expr="""{"type":"usd","usd":0.005,"format":{"suffix":"/second"}}""", + expr="""{"type":"usd","usd":0.05,"format":{"suffix":"/second"}}""", ), ) @@ -533,7 +533,7 @@ class BriaTransparentVideoBackground(IO.ComfyNode): ], is_api_node=True, price_badge=IO.PriceBadge( - expr="""{"type":"usd","usd":0.005,"format":{"suffix":"/second"}}""", + expr="""{"type":"usd","usd":0.05,"format":{"suffix":"/second"}}""", ), ) From e8e233cdf61c19a76fa60918efe2a6a6172f88cd Mon Sep 17 00:00:00 2001 From: comfyanonymous <121283862+comfyanonymous@users.noreply.github.com> Date: Sat, 1 Aug 2026 12:10:39 -0700 Subject: [PATCH 12/63] Update comfy-kitchen version to 0.2.26 (#15208) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index af414d413..aa3fbba8e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,7 +22,7 @@ alembic SQLAlchemy>=2.0.0 filelock av>=16.0.0 -comfy-kitchen==0.2.25 +comfy-kitchen==0.2.26 comfy-aimdo==0.4.10 requests simpleeval>=1.0.0 From e803f24ea090de7108772d65957fd6388d3f2085 Mon Sep 17 00:00:00 2001 From: comfyanonymous <121283862+comfyanonymous@users.noreply.github.com> Date: Sat, 1 Aug 2026 13:38:42 -0700 Subject: [PATCH 13/63] Let the VAEDecodeAudio node decode nested audio. (#15211) --- comfy_extras/nodes_audio.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/comfy_extras/nodes_audio.py b/comfy_extras/nodes_audio.py index 4ac5ced53..1c6e91fe4 100644 --- a/comfy_extras/nodes_audio.py +++ b/comfy_extras/nodes_audio.py @@ -96,10 +96,14 @@ class VAEEncodeAudio(IO.ComfyNode): def vae_decode_audio(vae, samples, tile=None, overlap=None): + latent = samples["samples"] + if latent.is_nested: + latent = latent.unbind()[-1] + if tile is not None: - audio = vae.decode_tiled(samples["samples"], tile_x=tile, tile_y=tile, overlap=overlap).movedim(-1, 1) + audio = vae.decode_tiled(latent, tile_x=tile, tile_y=tile, overlap=overlap).movedim(-1, 1) else: - audio = vae.decode(samples["samples"]).movedim(-1, 1) + audio = vae.decode(latent).movedim(-1, 1) std = torch.std(audio, dim=[1, 2], keepdim=True) * 5.0 std[std < 1.0] = 1.0 From 49a74228920714b3756d206d2fec8ff61145dd2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Sepp=C3=A4nen?= <40791699+kijai@users.noreply.github.com> Date: Sun, 2 Aug 2026 02:57:15 +0300 Subject: [PATCH 14/63] Support latent previews for nested latents (#15196) --- comfy/latent_formats.py | 134 ++++++++++++++++++++++++++- comfy/nested_tensor.py | 3 + comfy/samplers.py | 8 ++ comfy_extras/nodes_custom_sampler.py | 14 +-- latent_preview.py | 2 + 5 files changed, 153 insertions(+), 8 deletions(-) diff --git a/comfy/latent_formats.py b/comfy/latent_formats.py index 8a16cfe55..63eefd420 100644 --- a/comfy/latent_formats.py +++ b/comfy/latent_formats.py @@ -434,8 +434,138 @@ class LTXV(LatentFormat): class LTXAV(LTXV): def __init__(self): - self.latent_rgb_factors = None - self.latent_rgb_factors_bias = None + # video-stream preview factors for the packed AV latent (audio stream is not previewed) + self.latent_rgb_factors = [ + [ 0.001135, -0.010555, -0.004925], + [-0.008019, -0.006231, -0.005564], + [ 0.012637, 0.005605, 0.012713], + [ 0.023454, 0.020771, 0.017844], + [-0.011940, -0.000932, 0.009292], + [ 0.018602, 0.011018, 0.013969], + [-0.036369, -0.046631, -0.057898], + [-0.031919, 0.000131, 0.015214], + [ 0.014519, 0.021041, 0.015325], + [ 0.018889, 0.016149, -0.002836], + [-0.003784, -0.006057, -0.008195], + [ 0.013262, 0.030259, 0.029775], + [ 0.050465, 0.050366, 0.025255], + [ 0.018628, 0.007691, 0.002893], + [-0.015698, -0.008451, -0.000676], + [-0.013600, -0.012587, -0.004437], + [ 0.012482, 0.021469, 0.027913], + [-0.018241, -0.013488, -0.010975], + [ 0.013828, 0.012568, 0.021984], + [ 0.017911, 0.006552, 0.005567], + [ 0.026769, 0.006803, -0.009360], + [-0.006794, -0.008447, -0.013921], + [ 0.029708, 0.018671, 0.022811], + [-0.014732, -0.019169, 0.000903], + [ 0.019607, 0.032595, 0.053409], + [-0.003721, 0.003976, 0.010364], + [-0.020193, -0.026076, -0.036068], + [-0.002328, 0.006527, 0.013052], + [ 0.017171, 0.009224, 0.006548], + [ 0.001104, -0.000591, 0.000147], + [-0.000217, 0.011834, 0.017945], + [-0.015329, -0.012463, -0.006178], + [-0.009478, -0.008680, -0.004107], + [-0.005565, -0.006006, -0.001493], + [ 0.009451, 0.008794, 0.013207], + [-0.009989, -0.008027, -0.009568], + [-0.001505, -0.008805, -0.006828], + [ 0.001105, 0.008999, 0.009079], + [ 0.025935, 0.016426, 0.008036], + [ 0.006313, 0.000694, -0.006039], + [-0.001893, -0.006951, -0.009560], + [-0.007082, -0.002566, -0.007152], + [-0.005231, 0.004829, 0.008220], + [-0.004333, 0.001251, -0.004852], + [-0.017024, -0.012730, -0.007457], + [ 0.024988, 0.032963, 0.036556], + [ 0.013697, 0.012278, 0.009979], + [-0.013751, -0.008369, -0.015446], + [-0.009348, -0.001047, 0.007622], + [-0.003135, -0.003350, -0.003766], + [ 0.007436, 0.004957, 0.010480], + [ 0.018315, 0.022066, 0.021104], + [-0.005621, -0.006770, -0.008219], + [-0.007427, 0.001911, -0.001231], + [-0.007413, 0.000486, -0.006039], + [-0.014698, -0.007160, 0.006509], + [ 0.013775, 0.014185, 0.008203], + [ 0.060246, 0.069787, 0.072833], + [ 0.009861, 0.004870, 0.001194], + [-0.003660, 0.003251, 0.008015], + [ 0.003696, -0.003680, -0.008851], + [ 0.014924, 0.006196, 0.005282], + [-0.006740, -0.004319, -0.006729], + [ 0.020635, 0.015163, 0.012385], + [-0.032623, -0.006105, 0.010436], + [-0.058988, -0.030162, -0.037961], + [-0.035614, -0.021929, -0.011062], + [-0.023412, -0.011305, -0.005054], + [-0.002716, -0.005184, -0.004084], + [ 0.014591, 0.015294, 0.014045], + [ 0.008310, 0.002466, -0.003225], + [ 0.005176, 0.001119, 0.000695], + [-0.021569, -0.030886, -0.044732], + [ 0.007517, 0.003891, 0.000551], + [-0.006793, 0.004059, 0.010184], + [-0.086481, -0.082033, -0.083414], + [ 0.004192, 0.000762, -0.008658], + [ 0.010970, 0.009002, 0.007384], + [ 0.004042, -0.006732, -0.011031], + [ 0.012164, 0.006401, 0.007483], + [ 0.029252, 0.013990, 0.011128], + [ 0.048452, 0.034648, 0.016269], + [ 0.024104, 0.012647, 0.011754], + [-0.013216, -0.020192, -0.019752], + [-0.010799, -0.008535, -0.005467], + [ 0.005823, 0.001403, 0.001890], + [ 0.052393, 0.044771, 0.032777], + [ 0.007576, -0.008080, -0.012453], + [ 0.009830, 0.004244, 0.001213], + [-0.025867, -0.013169, -0.010636], + [ 0.008494, 0.003135, 0.000790], + [ 0.003969, -0.002625, -0.010204], + [ 0.006509, 0.008272, 0.020819], + [-0.004943, -0.013424, -0.015351], + [ 0.005541, 0.009136, -0.003666], + [-0.014300, -0.015864, -0.016853], + [ 0.002650, 0.028393, 0.014125], + [-0.027661, -0.045422, -0.064995], + [ 0.009220, 0.015522, 0.010574], + [-0.002236, 0.002915, 0.004557], + [-0.020269, -0.008212, -0.000532], + [ 0.019294, 0.003655, -0.002809], + [ 0.007116, -0.002784, 0.000017], + [ 0.057277, 0.073270, 0.074401], + [-0.002616, -0.001696, -0.000498], + [ 0.007248, 0.009793, 0.022829], + [-0.002590, -0.005601, -0.000436], + [-0.007681, 0.003893, -0.004119], + [-0.057392, -0.045545, -0.025290], + [ 0.045188, 0.047985, 0.054059], + [ 0.000937, -0.008861, -0.038406], + [-0.010192, -0.008036, -0.005385], + [-0.030222, -0.027498, -0.030765], + [-0.008359, 0.013247, 0.010918], + [ 0.004102, 0.002093, 0.006934], + [ 0.039461, 0.027339, 0.008284], + [-0.075747, -0.076340, -0.071625], + [ 0.002692, 0.005096, -0.002247], + [-0.002453, -0.002785, -0.010483], + [ 0.012265, 0.005481, 0.001729], + [ 0.017755, 0.008655, 0.003532], + [ 0.055560, 0.049128, 0.044137], + [-0.025861, -0.023798, -0.018815], + [-0.014876, -0.010770, -0.010713], + [-0.017315, -0.012599, -0.008661], + [-0.008461, -0.006210, -0.007744], + [-0.040175, -0.042255, -0.048119], + [-0.019355, -0.021055, -0.021919], + ] + self.latent_rgb_factors_bias = [-0.347892, -0.363814, -0.370287] class HunyuanVideo(LatentFormat): latent_channels = 16 diff --git a/comfy/nested_tensor.py b/comfy/nested_tensor.py index b700816fa..08c7133f8 100644 --- a/comfy/nested_tensor.py +++ b/comfy/nested_tensor.py @@ -51,6 +51,9 @@ class NestedTensor: def float(self): return self.to(dtype=torch.float) + def cpu(self): + return self.to(device="cpu") + def chunk(self, *args, **kwargs): return self.apply_operation(None, lambda x, y: x.chunk(*args, **kwargs)) diff --git a/comfy/samplers.py b/comfy/samplers.py index 29e6bffb3..6fea0913e 100755 --- a/comfy/samplers.py +++ b/comfy/samplers.py @@ -1284,6 +1284,14 @@ class CFGGuider: sampler_shapes = [tuple(latent_image.shape)] detail("Sampler: model=%s latent_shapes=%s", self.model_patcher.model.__class__.__name__, sampler_shapes) + if len(latent_shapes) > 1 and callback is not None: + # samplers run on the flat pack, hand callbacks (previews, x0 output) the nested view + packed_callback = callback + def callback(step, x0, x, total_steps): + x0 = comfy.nested_tensor.NestedTensor(comfy.utils.unpack_latents(x0, latent_shapes)) + x = comfy.nested_tensor.NestedTensor(comfy.utils.unpack_latents(x, latent_shapes)) + return packed_callback(step, x0, x, total_steps) + if denoise_mask is not None: if denoise_mask.is_nested: denoise_masks = denoise_mask.unbind() diff --git a/comfy_extras/nodes_custom_sampler.py b/comfy_extras/nodes_custom_sampler.py index 56ef5f526..e81b6328b 100644 --- a/comfy_extras/nodes_custom_sampler.py +++ b/comfy_extras/nodes_custom_sampler.py @@ -775,10 +775,11 @@ class SamplerCustom(io.ComfyNode): out.pop("downscale_ratio_temporal", None) out["samples"] = samples if "x0" in x0_output: - x0_out = model.model.process_latent_out(x0_output["x0"].cpu()) - if samples.is_nested: + x0 = x0_output["x0"] + if samples.is_nested and not x0.is_nested: latent_shapes = [x.shape for x in samples.unbind()] - x0_out = comfy.nested_tensor.NestedTensor(comfy.utils.unpack_latents(x0_out, latent_shapes)) + x0 = comfy.nested_tensor.NestedTensor(comfy.utils.unpack_latents(x0, latent_shapes)) + x0_out = model.model.process_latent_out(x0.cpu()) out_denoised = latent.copy() out_denoised["samples"] = x0_out else: @@ -1053,10 +1054,11 @@ class SamplerCustomAdvanced(io.ComfyNode): out.pop("downscale_ratio_temporal", None) out["samples"] = samples if "x0" in x0_output: - x0_out = guider.model_patcher.model.process_latent_out(x0_output["x0"].cpu()) - if samples.is_nested: + x0 = x0_output["x0"] + if samples.is_nested and not x0.is_nested: latent_shapes = [x.shape for x in samples.unbind()] - x0_out = comfy.nested_tensor.NestedTensor(comfy.utils.unpack_latents(x0_out, latent_shapes)) + x0 = comfy.nested_tensor.NestedTensor(comfy.utils.unpack_latents(x0, latent_shapes)) + x0_out = guider.model_patcher.model.process_latent_out(x0.cpu()) out_denoised = latent.copy() out_denoised["samples"] = x0_out else: diff --git a/latent_preview.py b/latent_preview.py index a9d777661..6bf2c1869 100644 --- a/latent_preview.py +++ b/latent_preview.py @@ -123,6 +123,8 @@ def prepare_callback(model, steps, x0_output_dict=None): preview_bytes = None if previewer: + if x0.is_nested: + x0 = x0.tensors[0] preview_bytes = previewer.decode_latent_to_preview_image(preview_format, x0) pbar.update_absolute(step + 1, total_steps, preview_bytes) return callback From 41a3e160d714cd99a42fa34bbfa3058839c2ceb1 Mon Sep 17 00:00:00 2001 From: comfyanonymous <121283862+comfyanonymous@users.noreply.github.com> Date: Sat, 1 Aug 2026 19:15:17 -0700 Subject: [PATCH 15/63] Use the actual function to check if the path is within the dir. (#15216) --- folder_paths.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/folder_paths.py b/folder_paths.py index df53542dc..6723e52c8 100644 --- a/folder_paths.py +++ b/folder_paths.py @@ -549,11 +549,10 @@ def get_save_image_path(filename_prefix: str, output_dir: str, image_width=0, im full_output_folder = os.path.join(output_dir, subfolder) - if os.path.commonpath((output_dir, os.path.abspath(full_output_folder))) != output_dir: + if not is_within_directory(output_dir, full_output_folder): err = "**** ERROR: Saving image outside the output folder is not allowed." + \ "\n full_output_folder: " + os.path.abspath(full_output_folder) + \ - "\n output_dir: " + output_dir + \ - "\n commonpath: " + os.path.commonpath((output_dir, os.path.abspath(full_output_folder))) + "\n output_dir: " + output_dir logging.error(err) raise Exception(err) From e0951a9c7e47f534d47ec1ab089c94faf484cbc0 Mon Sep 17 00:00:00 2001 From: comfyanonymous <121283862+comfyanonymous@users.noreply.github.com> Date: Sat, 1 Aug 2026 19:27:37 -0700 Subject: [PATCH 16/63] Let the save text node save in csv format. (#15217) --- comfy_extras/nodes_text.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/comfy_extras/nodes_text.py b/comfy_extras/nodes_text.py index a485f5df8..82efc247f 100644 --- a/comfy_extras/nodes_text.py +++ b/comfy_extras/nodes_text.py @@ -8,6 +8,13 @@ import folder_paths class SaveTextNode(io.ComfyNode): """Save text content to .txt, .md, or .json.""" + FORMAT_EXTENSIONS = { + "txt": "txt", + "csv": "csv", + "md": "md", + "json": "json", + } + @classmethod def define_schema(cls): return io.Schema( @@ -19,7 +26,7 @@ class SaveTextNode(io.ComfyNode): inputs=[ io.String.Input("text", force_input=True), io.String.Input("filename_prefix", default="ComfyUI"), - io.Combo.Input("format", options=["txt", "md", "json"], default="txt"), + io.Combo.Input("format", options=list(cls.FORMAT_EXTENSIONS), default="txt"), ], outputs=[io.String.Output(display_name="text")], is_output_node=True, @@ -27,6 +34,10 @@ class SaveTextNode(io.ComfyNode): @classmethod def execute(cls, text, filename_prefix, format): + extension = cls.FORMAT_EXTENSIONS.get(format) + if extension is None: + raise ValueError(f"Unsupported text format: {format!r}") + full_output_folder, filename, counter, subfolder, filename_prefix = folder_paths.get_save_image_path( filename_prefix, folder_paths.get_output_directory(), @@ -34,10 +45,10 @@ class SaveTextNode(io.ComfyNode): 1, ) - file = f"{filename}_{counter:05}.{format}" + file = f"{filename}_{counter:05}.{extension}" filepath = os.path.join(full_output_folder, file) - if format == "json": + if extension == "json": # tries to pretty print otherwise saves normally try: data = json.loads(text) From 532a16f3b9557f49ef8e3d5dc7978d2040a0f4a1 Mon Sep 17 00:00:00 2001 From: comfyanonymous <121283862+comfyanonymous@users.noreply.github.com> Date: Sat, 1 Aug 2026 20:15:51 -0700 Subject: [PATCH 17/63] Disable gradients on diffusion models. (#15218) --- comfy/model_base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/comfy/model_base.py b/comfy/model_base.py index ee6dc57a2..7c68c6eba 100644 --- a/comfy/model_base.py +++ b/comfy/model_base.py @@ -169,6 +169,7 @@ class BaseModel(torch.nn.Module): else: operations = model_config.custom_operations self.diffusion_model = unet_model(**unet_config, device=device, operations=operations) + self.diffusion_model.requires_grad_(False) self.diffusion_model.eval() if comfy.model_management.force_channels_last(): self.diffusion_model.to(memory_format=torch.channels_last) From f06a187f50f896e4a0ba5be1ce1f2d2dcd13b77b Mon Sep 17 00:00:00 2001 From: comfyanonymous <121283862+comfyanonymous@users.noreply.github.com> Date: Sat, 1 Aug 2026 22:12:12 -0700 Subject: [PATCH 18/63] Handle case where swap memory query fails on windows. (#15219) --- comfy/model_management.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/comfy/model_management.py b/comfy/model_management.py index f7351224d..e3c94c15a 100644 --- a/comfy/model_management.py +++ b/comfy/model_management.py @@ -684,7 +684,11 @@ def should_free_pins_for_ram_pressure(shortfall): return True if psutil.virtual_memory().available < WINDOWS_PIN_EVICTION_EMERGENCY_AVAILABLE: return True - return psutil.swap_memory().percent >= WINDOWS_PIN_EVICTION_SWAP_PERCENT + try: + return psutil.swap_memory().percent >= WINDOWS_PIN_EVICTION_SWAP_PERCENT + except RuntimeError as err: + logging.warning("Could not read Windows swap usage; falling back to RAM-pressure pin eviction: %s", err) + return True def ensure_pin_budget(size, evict_active=False, loaded=False): if args.high_ram: From 8084083d4b085e0ba4bf09b31a050e48ea2a970a Mon Sep 17 00:00:00 2001 From: rattus <46076784+rattus128@users.noreply.github.com> Date: Sun, 2 Aug 2026 20:55:37 +1000 Subject: [PATCH 19/63] comfy-aimdo 0.4.11 (#15215) Changes: Remove sequential scan hint Prefer NVML pressure on windows Add async malloc clamp option (unused by comfy so far) Workaround AMD windows GPU virtual address space leak The largest change is the NVML pressure, which works around a cuMemGetInfo drift from actual VRAM in some circumstances. --- comfy/cli_args.py | 1 + main.py | 11 ++++++++--- requirements.txt | 2 +- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/comfy/cli_args.py b/comfy/cli_args.py index 792148f0a..ee9e1ce9f 100644 --- a/comfy/cli_args.py +++ b/comfy/cli_args.py @@ -172,6 +172,7 @@ vram_group.add_argument("--cpu", action="store_true", help="To use the CPU for e parser.add_argument("--reserve-vram", type=float, default=None, help="Set the amount of vram in GB you want to reserve for use by your OS/other software. By default some amount is reserved depending on your OS.") parser.add_argument("--vram-headroom", type=float, default=0, help="Set the amount of vram in GB for DynamicVRAM to maintain as extra headroom above default. ComfyUI will try and keep this much VRAM completely free and unused, even counting VRAM from other apps.") +parser.add_argument("--disable-nvml-pressure", action="store_true", help="Use CUDA instead of NVML for DynamicVRAM memory pressure.") parser.add_argument("--async-offload", nargs='?', const=2, type=int, default=None, metavar="NUM_STREAMS", help="Use async weight offloading. An optional argument controls the amount of offload streams. Default is 2. Enabled by default on Nvidia.") parser.add_argument("--disable-async-offload", action="store_true", help="Disable async weight offloading.") diff --git a/main.py b/main.py index 9c318fafe..361b1fc89 100644 --- a/main.py +++ b/main.py @@ -58,11 +58,16 @@ if __name__ == "__main__" and args.debug_hang: import comfy_aimdo.control if enables_dynamic_vram(): + simple_vram_headroom = None if args.reserve_vram is None else int(args.reserve_vram * 1024 ** 3) try: - comfy_aimdo.control.init(simple_vram_headroom=None if args.reserve_vram is None else int(args.reserve_vram * 1024 ** 3)) + comfy_aimdo.control.init(simple_vram_headroom=simple_vram_headroom, nvml_pressure=not args.disable_nvml_pressure) except TypeError: - # comfy-aimdo 0.4.9 protocol. - comfy_aimdo.control.init() + # comfy-aimdo 0.4.10 protocol. + try: + comfy_aimdo.control.init(simple_vram_headroom=simple_vram_headroom) + except TypeError: + # comfy-aimdo 0.4.9 protocol. + comfy_aimdo.control.init() if os.name == "nt": os.environ['MIMALLOC_PURGE_DELAY'] = '0' diff --git a/requirements.txt b/requirements.txt index aa3fbba8e..c1eeb9752 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,7 +23,7 @@ SQLAlchemy>=2.0.0 filelock av>=16.0.0 comfy-kitchen==0.2.26 -comfy-aimdo==0.4.10 +comfy-aimdo==0.4.11 requests simpleeval>=1.0.0 blake3 From 364081170f1c15853608db1f24ed9897763bdd09 Mon Sep 17 00:00:00 2001 From: Alexander Piskun <13381981+bigcat88@users.noreply.github.com> Date: Sun, 2 Aug 2026 17:10:47 +0300 Subject: [PATCH 20/63] [Partner Nodes] feat(Minimax): add 768P resolution for H3 model (#15227) Signed-off-by: Alexander Piskun --- comfy_api_nodes/nodes_minimax.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/comfy_api_nodes/nodes_minimax.py b/comfy_api_nodes/nodes_minimax.py index 2d7aef654..3c1d29257 100644 --- a/comfy_api_nodes/nodes_minimax.py +++ b/comfy_api_nodes/nodes_minimax.py @@ -467,7 +467,7 @@ def _hailuo03_model_inputs(include_ratio: bool = True, allow_adaptive: bool = Tr ), IO.Combo.Input( "resolution", - options=["2K"], + options=["768P", "2K"], tooltip="Resolution of the output video.", ), ] @@ -578,11 +578,12 @@ class MinimaxHailuo03TextToVideoNode(IO.ComfyNode): ], is_api_node=True, price_badge=IO.PriceBadge( - depends_on=IO.PriceBadgeDepends(widgets=["model.duration"]), + depends_on=IO.PriceBadgeDepends(widgets=["model.resolution", "model.duration"]), expr=""" ( $dur := $lookup(widgets, "model.duration"); - {"type": "usd", "usd": $dur * 0.1859} + $rate := $lookup(widgets, "model.resolution") = "768p" ? 0.1287 : 0.1859; + {"type": "usd", "usd": $dur * $rate} ) """, ), @@ -660,11 +661,12 @@ class MinimaxHailuo03FirstLastFrameNode(IO.ComfyNode): ], is_api_node=True, price_badge=IO.PriceBadge( - depends_on=IO.PriceBadgeDepends(widgets=["model.duration"]), + depends_on=IO.PriceBadgeDepends(widgets=["model.resolution", "model.duration"]), expr=""" ( $dur := $lookup(widgets, "model.duration"); - {"type": "usd", "usd": $dur * 0.1859} + $rate := $lookup(widgets, "model.resolution") = "768p" ? 0.1287 : 0.1859; + {"type": "usd", "usd": $dur * $rate} ) """, ), @@ -818,20 +820,21 @@ class MinimaxHailuo03ReferenceNode(IO.ComfyNode): is_api_node=True, price_badge=IO.PriceBadge( depends_on=IO.PriceBadgeDepends( - widgets=["model.duration"], + widgets=["model.resolution", "model.duration"], input_groups=["model.reference_images", "model.reference_videos"], ), expr=""" ( $dur := $lookup(widgets, "model.duration"); + $rate := $lookup(widgets, "model.resolution") = "768p" ? 0.1287 : 0.1859; $imgsRaw := $lookup(inputGroups, "model.reference_images"); $imgs := $imgsRaw ? $imgsRaw : 0; $vidsRaw := $lookup(inputGroups, "model.reference_videos"); $vids := $vidsRaw ? $vidsRaw : 0; - $base := $dur * 0.1859 + ($imgs > 5 ? ($imgs - 5) * 0.0572 : 0); + $base := $dur * $rate + ($imgs > 5 ? ($imgs - 5) * 0.0572 : 0); $vids > 0 - ? {"type": "range_usd", "min_usd": $base + $vids * 2 * 0.1859, - "max_usd": $base + 15 * 0.1859, "format": {"approximate": true}} + ? {"type": "range_usd", "min_usd": $base + $vids * 2 * $rate, + "max_usd": $base + 15 * $rate, "format": {"approximate": true}} : {"type": "usd", "usd": $base} ) """, From 611f2a4e0f30ea7f50451fd34ab25b8f9365ff2f Mon Sep 17 00:00:00 2001 From: rattus <46076784+rattus128@users.noreply.github.com> Date: Mon, 3 Aug 2026 01:16:06 +1000 Subject: [PATCH 21/63] fix pin registration priority (#15226) This priority scheme was broken in the case where you have pin registration exhaustion while loading a VBAR that gets a big evicition. The weight would stay in the loaded set but inherit the MRU priority against other workflow models WRT pin registration which leads to async offload without pinning. Fix by universally promiting active pin registration above workflow pins without concern for the weights/weights-loaded split. This diverges from the actual budgeting where the split still makes sense. --- comfy/model_management.py | 21 +++++++++++++++++---- comfy/pinned_memory.py | 6 +++--- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/comfy/model_management.py b/comfy/model_management.py index e3c94c15a..1000f69e1 100644 --- a/comfy/model_management.py +++ b/comfy/model_management.py @@ -671,6 +671,19 @@ def pin_eviction_tiers(loaded, evict_active): tiers.append((PIN_SUBSETS, True, True)) return tiers +def registration_eviction_tiers(evict_active): + subsets = PIN_SUBSETS + LOADED_PIN_SUBSETS + tiers = [ + (subsets, False, False), + (subsets, True, False), + ] + if evict_active: + tiers.extend([ + (subsets, False, True), + (subsets, True, True), + ]) + return tiers + def free_pins(size, evict_active=False, loaded=False): freed = 0 for subsets, current_prompt, active in pin_eviction_tiers(loaded, evict_active): @@ -703,19 +716,19 @@ def ensure_pin_budget(size, evict_active=False, loaded=False): to_free = shortfall + PIN_PRESSURE_HYSTERESIS return free_pins(to_free, evict_active=evict_active, loaded=loaded) >= shortfall -def free_registrations(shortfall, evict_active=True, loaded=False): +def free_registrations(shortfall, evict_active=True): if MAX_PINNED_MEMORY <= 0: return False if shortfall <= 0: return True shortfall += REGISTERABLE_PIN_HYSTERESIS - for subsets, current_prompt, active in pin_eviction_tiers(loaded, evict_active): + for subsets, current_prompt, active in registration_eviction_tiers(evict_active): shortfall -= free_model_pins(shortfall, subsets, current_prompt, active, registrations=True) return shortfall <= REGISTERABLE_PIN_HYSTERESIS -def ensure_pin_registerable(size, evict_active=True, loaded=False): - return free_registrations(TOTAL_PINNED_MEMORY + size - MAX_PINNED_MEMORY, evict_active=evict_active, loaded=loaded) +def ensure_pin_registerable(size, evict_active=True): + return free_registrations(TOTAL_PINNED_MEMORY + size - MAX_PINNED_MEMORY, evict_active=evict_active) class LoadedModel: def __init__(self, model: ModelPatcher): diff --git a/comfy/pinned_memory.py b/comfy/pinned_memory.py index d78ab3c76..e9a9a70e2 100644 --- a/comfy/pinned_memory.py +++ b/comfy/pinned_memory.py @@ -56,7 +56,7 @@ def get_pin(module, subset="weights"): _, _, stack_split, pinned_size, *_ = module._pin_state[subset] size = pin.nbytes - comfy.model_management.ensure_pin_registerable(size, loaded=subset.endswith("-loaded")) + comfy.model_management.ensure_pin_registerable(size) if torch.cuda.cudart().cudaHostRegister(pin.data_ptr(), size, 1) != 0: comfy.model_management.discard_cuda_async_error() @@ -93,7 +93,7 @@ def pin_memory(module, subset="weights", size=None): comfy.memory_management.extra_ram_release(comfy.memory_management.RAM_CACHE_HEADROOM) if (not comfy.model_management.ensure_pin_budget(size, loaded=loaded) or - not comfy.model_management.ensure_pin_registerable(registerable_size, loaded=loaded)): + not comfy.model_management.ensure_pin_registerable(registerable_size)): return _steal_pin(module, stack, buckets, size, priority, subset) offset = hostbuf.size @@ -105,7 +105,7 @@ def pin_memory(module, subset="weights", size=None): pin.untyped_storage()._comfy_hostbuf = hostbuf if torch.cuda.cudart().cudaHostRegister(pin.data_ptr(), size, 1) != 0: comfy.model_management.discard_cuda_async_error() - comfy.model_management.free_registrations(size, loaded=loaded) + comfy.model_management.free_registrations(size) if torch.cuda.cudart().cudaHostRegister(pin.data_ptr(), size, 1) != 0: comfy.model_management.discard_cuda_async_error() del pin From b53e247c94f9225dc206bcfef5d64a2f7bc85232 Mon Sep 17 00:00:00 2001 From: Oliver Freyermuth Date: Sun, 2 Aug 2026 22:20:27 +0200 Subject: [PATCH 22/63] rename comfy/logging.py to comfy/internal_logging.py (#15231) This avoids name collision (circular imports) for external custom nodes, for which the comfy path is pushed into sys.path so Python's own logging module is shadowed otherwise. fixes: #15229 --- app/logger.py | 4 ++-- comfy/{logging.py => internal_logging.py} | 0 comfy/model_management.py | 2 +- comfy/model_patcher.py | 2 +- comfy/samplers.py | 2 +- execution.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) rename comfy/{logging.py => internal_logging.py} (100%) diff --git a/app/logger.py b/app/logger.py index 2e6116813..fe82c40c9 100644 --- a/app/logger.py +++ b/app/logger.py @@ -6,7 +6,7 @@ import os import sys import threading -import comfy.logging +import comfy.internal_logging ANSI_NAMED_COLORS = { 'black': '\033[30m', @@ -91,7 +91,7 @@ def on_flush(callback): def get_log_level(level): - return comfy.logging.DETAIL if level == "DETAIL" else logging.getLevelName(level) + return comfy.internal_logging.DETAIL if level == "DETAIL" else logging.getLevelName(level) def setup_logger(log_level: str = 'INFO', file_outputs=None, capacity: int = 300, use_stdout: bool = False): diff --git a/comfy/logging.py b/comfy/internal_logging.py similarity index 100% rename from comfy/logging.py rename to comfy/internal_logging.py diff --git a/comfy/model_management.py b/comfy/model_management.py index 1000f69e1..ec91fb929 100644 --- a/comfy/model_management.py +++ b/comfy/model_management.py @@ -34,7 +34,7 @@ import comfy.utils import comfy.quant_ops import comfy_aimdo.host_buffer import comfy_aimdo.vram_buffer -from comfy.logging import detail +from comfy.internal_logging import detail from typing import TYPE_CHECKING if TYPE_CHECKING: diff --git a/comfy/model_patcher.py b/comfy/model_patcher.py index 6b698f767..ae3f0191d 100644 --- a/comfy/model_patcher.py +++ b/comfy/model_patcher.py @@ -38,7 +38,7 @@ import comfy.patcher_extension import comfy.utils import comfy_aimdo.host_buffer from comfy.comfy_types import UnetWrapperFunction -from comfy.logging import detail +from comfy.internal_logging import detail from comfy.quant_ops import QuantizedTensor from comfy.patcher_extension import CallbacksMP, PatcherInjection, WrappersMP diff --git a/comfy/samplers.py b/comfy/samplers.py index 6fea0913e..a280f3bb6 100755 --- a/comfy/samplers.py +++ b/comfy/samplers.py @@ -20,7 +20,7 @@ import comfy.hooks import comfy.context_windows import comfy.multigpu import comfy.utils -from comfy.logging import detail +from comfy.internal_logging import detail import scipy.stats import numpy diff --git a/execution.py b/execution.py index 7cab4b331..0b858969d 100644 --- a/execution.py +++ b/execution.py @@ -19,7 +19,7 @@ import comfy.model_management import comfy.model_patcher import comfy.model_prefetch import comfy_aimdo.model_vbar -from comfy.logging import detail +from comfy.internal_logging import detail from latent_preview import set_preview_method import nodes From 57500fc5bc92566a63f2046824f522cd55c335ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Sepp=C3=A4nen?= <40791699+kijai@users.noreply.github.com> Date: Mon, 3 Aug 2026 05:28:29 +0300 Subject: [PATCH 23/63] feat: Support MiniMax-H3 (CORE-375) (#15224) --- comfy/latent_formats.py | 39 ++ comfy/ldm/minimax/audio_vae.py | 442 ++++++++++++++++++++ comfy/ldm/minimax/model.py | 646 ++++++++++++++++++++++++++++ comfy/ldm/minimax/vae.py | 694 +++++++++++++++++++++++++++++++ comfy/model_base.py | 52 +++ comfy/model_detection.py | 29 ++ comfy/ops.py | 55 ++- comfy/quant_ops.py | 2 +- comfy/sd.py | 55 +++ comfy/supported_models.py | 29 ++ comfy/text_encoders/llama.py | 11 + comfy/text_encoders/minimax.py | 201 +++++++++ comfy/text_encoders/qwen3vl.py | 5 +- comfy_extras/nodes_lt.py | 5 +- comfy_extras/nodes_minimax_h3.py | 337 +++++++++++++++ nodes.py | 3 +- 16 files changed, 2599 insertions(+), 6 deletions(-) create mode 100644 comfy/ldm/minimax/audio_vae.py create mode 100644 comfy/ldm/minimax/model.py create mode 100644 comfy/ldm/minimax/vae.py create mode 100644 comfy/text_encoders/minimax.py create mode 100644 comfy_extras/nodes_minimax_h3.py diff --git a/comfy/latent_formats.py b/comfy/latent_formats.py index 63eefd420..c4270022b 100644 --- a/comfy/latent_formats.py +++ b/comfy/latent_formats.py @@ -567,6 +567,45 @@ class LTXAV(LTXV): ] self.latent_rgb_factors_bias = [-0.347892, -0.363814, -0.370287] +class MiniMaxH3Video(LatentFormat): + latent_channels = 24 + latent_dimensions = 3 + spacial_downscale_ratio = 16 + temporal_downscale_ratio = 4 + scale_factor = 1.0 + + latent_rgb_factors = [ + [-0.018555, 0.024344, -0.017536], + [ 0.150164, 0.137244, 0.129221], + [ 0.027367, -0.050369, -0.208606], + [-0.000793, -0.164622, -0.323161], + [-0.048556, 0.013970, -0.074286], + [ 0.011740, 0.014172, -0.006906], + [ 0.061517, 0.061212, 0.110025], + [ 0.035321, 0.086879, 0.110059], + [-0.017426, 0.002997, 0.035356], + [ 0.531539, 0.548819, 0.624404], + [-0.024968, -0.040234, -0.034302], + [-0.032549, -0.029096, -0.017221], + [ 0.022609, 0.020286, 0.050661], + [-0.084001, -0.038131, -0.020805], + [-0.018830, 0.010412, 0.061120], + [ 0.020777, 0.011196, -0.030994], + [-0.008390, -0.012201, -0.025687], + [-0.013281, -0.002924, 0.006331], + [ 0.000260, 0.001833, -0.011038], + [ 0.105471, 0.100482, 0.132106], + [ 0.016529, 0.015213, 0.009999], + [-0.014015, -0.017438, -0.019134], + [-0.033787, -0.009984, -0.019725], + [ 0.004224, 0.017284, 0.027196], + ] + latent_rgb_factors_bias = [ 0.057426, -0.022078, -0.071449] + +class MiniMaxH3AV(MiniMaxH3Video): + # max channels across the two streams (video 24, audio 32) so per-stream slices keep both streams whole + latent_channels = 32 + class HunyuanVideo(LatentFormat): latent_channels = 16 latent_dimensions = 3 diff --git a/comfy/ldm/minimax/audio_vae.py b/comfy/ldm/minimax/audio_vae.py new file mode 100644 index 000000000..033ae6966 --- /dev/null +++ b/comfy/ldm/minimax/audio_vae.py @@ -0,0 +1,442 @@ +# MiniMax H3 audio VAE: DAC-lineage waveform encoder + BigVGAN decoder. +# Weight-norm parametrizations are folded into plain conv weights, so this +# module uses ordinary ops.Conv1d / ops.ConvTranspose1d and loads the converted +# checkpoint (plain "*.weight" tensors) with strict=True. +# +# Lineage / licenses of the reference implementation: +# DAC encoder: descript-audio-codec (MIT) +# BigVGAN decoder: NVIDIA BigVGAN (MIT), adapted from hifi-gan (MIT) +# Alias-free ops: junjun3518/alias-free-torch (Apache-2.0), julius (MIT) + +import math + +import torch +import torch.nn as nn +import torch.nn.functional as F + +import comfy.ops + +ops = comfy.ops.disable_weight_init + + +# Snake activations + +def snake(x, alpha, beta): + # x + 1/beta * sin^2(alpha * x) + t = torch.sin(alpha * x) + return t.mul_(t).mul_((beta + 1e-9).reciprocal()).add_(x) + + +class Snake1d(nn.Module): + """Snake activation with per-channel alpha (encoder side).""" + + def __init__(self, channels): + super().__init__() + self.alpha = nn.Parameter(torch.empty(1, channels, 1)) + + def forward(self, x): + return snake(x, self.alpha, self.alpha) + + +class SnakeBeta(nn.Module): + """SnakeBeta := x + 1/beta * sin^2(alpha * x); alpha/beta stored in log scale.""" + + def __init__(self, in_features): + super().__init__() + self.alpha = nn.Parameter(torch.empty(in_features)) + self.beta = nn.Parameter(torch.empty(in_features)) + + def forward(self, x): + alpha = torch.exp(self.alpha).view(1, -1, 1) + beta = torch.exp(self.beta).view(1, -1, 1) + return snake(x, alpha, beta) + + +# Alias-free (anti-aliased) activation: kaiser-windowed sinc resampling + +def kaiser_sinc_filter1d(cutoff, half_width, kernel_size): + # returns filter [1, 1, kernel_size] + even = kernel_size % 2 == 0 + half_size = kernel_size // 2 + + # kaiser window design + delta_f = 4 * half_width + A = 2.285 * (half_size - 1) * math.pi * delta_f + 7.95 + if A > 50.0: + beta = 0.1102 * (A - 8.7) + elif A >= 21.0: + beta = 0.5842 * (A - 21) ** 0.4 + 0.07886 * (A - 21.0) + else: + beta = 0.0 + window = torch.kaiser_window(kernel_size, beta=beta, periodic=False) + + if even: + time = torch.arange(-half_size, half_size) + 0.5 + else: + time = torch.arange(kernel_size) - half_size + + filter_ = 2 * cutoff * window * torch.sinc(2 * cutoff * time) + # Normalize filter to have sum = 1, otherwise there is a small leakage of + # the constant component in the input signal. + filter_ /= filter_.sum() + return filter_.view(1, 1, kernel_size) + + +class UpSample1d(nn.Module): + def __init__(self, ratio=2, kernel_size=12): + super().__init__() + self.ratio = ratio + self.stride = ratio + self.pad = kernel_size // ratio - 1 + self.pad_left = self.pad * ratio + (kernel_size - ratio) // 2 + self.pad_right = self.pad * ratio + (kernel_size - ratio + 1) // 2 + self.register_buffer( + "filter", + kaiser_sinc_filter1d(cutoff=0.5 / ratio, half_width=0.6 / ratio, kernel_size=kernel_size), + ) + + def forward(self, x): + _, C, _ = x.shape + x = F.pad(x, (self.pad, self.pad), mode="replicate") + x = F.conv_transpose1d(x, self.filter.expand(C, -1, -1).to(x.dtype), stride=self.stride, groups=C).mul_(self.ratio) + x = x[..., self.pad_left:-self.pad_right] + return x + + +class LowPassFilter1d(nn.Module): + def __init__(self, cutoff=0.5, half_width=0.6, stride=1, kernel_size=12): + super().__init__() + self.pad_left = kernel_size // 2 - int(kernel_size % 2 == 0) + self.pad_right = kernel_size // 2 + self.stride = stride + self.register_buffer("filter", kaiser_sinc_filter1d(cutoff, half_width, kernel_size)) + + def forward(self, x): + _, C, _ = x.shape + x = F.pad(x, (self.pad_left, self.pad_right), mode="replicate") + return F.conv1d(x, self.filter.expand(C, -1, -1).to(x.dtype), stride=self.stride, groups=C) + + +class DownSample1d(nn.Module): + def __init__(self, ratio=2, kernel_size=12): + super().__init__() + self.ratio = ratio + self.kernel_size = kernel_size + self.lowpass = LowPassFilter1d( + cutoff=0.5 / ratio, + half_width=0.6 / ratio, + stride=ratio, + kernel_size=self.kernel_size, + ) + + def forward(self, x): + return self.lowpass(x) + + +class Activation1d(nn.Module): + """upsample x2 -> pointwise activation -> downsample x2 (anti-aliased).""" + + def __init__(self, activation, up_ratio=2, down_ratio=2, up_kernel_size=12, down_kernel_size=12): + super().__init__() + self.act = activation + self.upsample = UpSample1d(up_ratio, up_kernel_size) + self.downsample = DownSample1d(down_ratio, down_kernel_size) + + def forward(self, x): + x = self.upsample(x) + x = self.act(x) + x = self.downsample(x) + return x + + +# DAC encoder + +class ResidualUnit(nn.Module): + def __init__(self, dim=16, dilation=1): + super().__init__() + pad = ((7 - 1) * dilation) // 2 + self.block = nn.Sequential( + Snake1d(dim), + ops.Conv1d(dim, dim, kernel_size=7, dilation=dilation, padding=pad), + Snake1d(dim), + ops.Conv1d(dim, dim, kernel_size=1), + ) + + def forward(self, x): + y = self.block(x) + pad = (x.shape[-1] - y.shape[-1]) // 2 + if pad > 0: + x = x[..., pad:-pad] + return y.add_(x) + + +class EncoderBlock(nn.Module): + def __init__(self, dim=16, stride=1): + super().__init__() + self.block = nn.Sequential( + ResidualUnit(dim // 2, dilation=1), + ResidualUnit(dim // 2, dilation=3), + ResidualUnit(dim // 2, dilation=9), + Snake1d(dim // 2), + ops.Conv1d( + dim // 2, + dim, + kernel_size=2 * stride, + stride=stride, + padding=math.ceil(stride / 2), + ), + ) + + def forward(self, x): + return self.block(x) + + +class Encoder(nn.Module): + def __init__(self, d_model=64, strides=(2, 4, 4, 5, 5), d_latent=2048): + super().__init__() + block = [ops.Conv1d(1, d_model, kernel_size=7, padding=3)] + for stride in strides: + d_model *= 2 + block += [EncoderBlock(d_model, stride=stride)] + block += [ + Snake1d(d_model), + ops.Conv1d(d_model, d_latent, kernel_size=3, padding=1), + ] + self.block = nn.Sequential(*block) + + def forward(self, x): + return self.block(x) + + +# Attention projection (encoder posterior head) + +class GeGluMlp(nn.Module): + def __init__(self, in_features, hidden_features): + super().__init__() + self.norm = ops.LayerNorm(in_features) + self.act = nn.GELU(approximate="tanh") + self.w0 = ops.Linear(in_features, hidden_features) + self.w1 = ops.Linear(in_features, hidden_features) + self.w2 = ops.Linear(hidden_features, in_features) + + def forward(self, x): + x = self.norm(x) + return self.w2(self.act(self.w0(x)).mul_(self.w1(x))) + + +class CausalAttention(nn.Module): + def __init__(self, in_dim, out_dim, num_heads): + super().__init__() + self.head_dim = in_dim // num_heads + self.num_heads = num_heads + self.out_dim = out_dim + self.qkv = ops.Linear(in_dim, in_dim * 3, bias=False) + self.q_bias = nn.Parameter(torch.empty(in_dim)) + self.v_bias = nn.Parameter(torch.empty(in_dim)) + self.register_buffer("zero_k_bias", torch.empty(in_dim)) + self.proj = ops.Linear(out_dim, out_dim) + + def forward(self, x): + B, N, C = x.shape + weight, _, offload_stream = comfy.ops.cast_bias_weight(self.qkv, x, offloadable=True) + qkv = F.linear(x, weight=weight, bias=torch.cat((self.q_bias, self.zero_k_bias, self.v_bias))) + comfy.ops.uncast_bias_weight(self.qkv, weight, None, offload_stream) + q, k, v = qkv.reshape(B, N, 3, self.num_heads, self.head_dim).permute(2, 0, 3, 1, 4).unbind(0) + + # mean over heads then pool down to the latent width (in_dim >> out_dim) + x = comfy.ops.scaled_dot_product_attention(q, k, v, is_causal=True) + x = F.adaptive_avg_pool1d(torch.mean(x, dim=1), self.out_dim) + return self.proj(x) + + +class AttnProjection(nn.Module): + def __init__(self, in_dim, out_dim, num_heads, mlp_ratio=2): + super().__init__() + self.norm1 = ops.LayerNorm(in_dim) + self.attn = CausalAttention(in_dim, out_dim, num_heads) + self.proj = ops.Linear(in_dim, out_dim) + self.norm3 = ops.LayerNorm(in_dim) + + self.norm2 = ops.LayerNorm(out_dim) + hidden_dim = int(out_dim * mlp_ratio) + self.mlp = GeGluMlp(in_features=out_dim, hidden_features=hidden_dim) + + def forward(self, x): + # x: [B, T, in_dim] + x = self.proj(self.norm3(x)).add_(self.attn(self.norm1(x))) + return x.add_(self.mlp(self.norm2(x))) + + +# BigVGAN decoder + +def get_padding(kernel_size, dilation=1): + return int((kernel_size * dilation - dilation) / 2) + + +class AMPBlock1(nn.Module): + def __init__(self, channels, kernel_size=3, dilation=(1, 3, 5)): + super().__init__() + self.convs1 = nn.ModuleList( + [ + ops.Conv1d(channels, channels, kernel_size, stride=1, dilation=d, padding=get_padding(kernel_size, d)) + for d in dilation + ] + ) + self.convs2 = nn.ModuleList( + [ + ops.Conv1d(channels, channels, kernel_size, stride=1, dilation=1, padding=get_padding(kernel_size, 1)) + for _ in range(len(dilation)) + ] + ) + self.num_layers = len(self.convs1) + len(self.convs2) + self.activations = nn.ModuleList( + [Activation1d(activation=SnakeBeta(channels)) for _ in range(self.num_layers)] + ) + + def forward(self, x): + acts1, acts2 = self.activations[::2], self.activations[1::2] + for c1, c2, a1, a2 in zip(self.convs1, self.convs2, acts1, acts2): + xt = a1(x) + xt = c1(xt) + xt = a2(xt) + xt = c2(xt) + x = xt.add_(x) + return x + + +class BigVGAN(nn.Module): + """BigVGAN vocoder (MiniMax H3 32 kHz configuration). + + use_bias_at_final=False, use_tanh_at_final=False (output clamped to [-1, 1]). + """ + + def __init__( + self, + num_mels=2048, + upsample_initial_channel=1024, + upsample_rates=(5, 5, 2, 2, 2, 2, 2), + upsample_kernel_sizes=(9, 9, 4, 4, 4, 4, 4), + resblock_kernel_sizes=(3, 7, 11), + resblock_dilation_sizes=((1, 3, 5), (1, 3, 5), (1, 3, 5)), + ): + super().__init__() + self.num_kernels = len(resblock_kernel_sizes) + self.num_upsamples = len(upsample_rates) + + self.conv_pre = ops.Conv1d(num_mels, upsample_initial_channel, 7, 1, padding=3) + + self.ups = nn.ModuleList() + for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)): + self.ups.append( + nn.ModuleList( + [ + ops.ConvTranspose1d( + upsample_initial_channel // (2 ** i), + upsample_initial_channel // (2 ** (i + 1)), + k, + u, + padding=(k - u) // 2, + ) + ] + ) + ) + + self.resblocks = nn.ModuleList() + for i in range(len(self.ups)): + ch = upsample_initial_channel // (2 ** (i + 1)) + for k, d in zip(resblock_kernel_sizes, resblock_dilation_sizes): + self.resblocks.append(AMPBlock1(ch, k, d)) + + self.activation_post = Activation1d(activation=SnakeBeta(ch)) + self.conv_post = ops.Conv1d(ch, 1, 7, 1, padding=3, bias=False) + + def forward(self, x): + x = self.conv_pre(x) + + for i in range(self.num_upsamples): + for i_up in range(len(self.ups[i])): + x = self.ups[i][i_up](x) + xs = None + for j in range(self.num_kernels): + if xs is None: + xs = self.resblocks[i * self.num_kernels + j](x) + else: + xs += self.resblocks[i * self.num_kernels + j](x) + x = xs.div_(self.num_kernels) + + x = self.activation_post(x) + return self.conv_post(x).clamp_(-1.0, 1.0) + + +# Top-level VAE + +class MiniMaxH3AudioVAE(nn.Module): + """MiniMax H3 stereo audio VAE at 32 kHz. + + Latents are [B, 32, 2, T]: 32 channels, 2 stereo channels, T frames at + 40 latent frames per second (800 audio samples per latent frame). The + stereo channels are processed independently by the mono encoder/decoder. + Latents are normalized with the stored per-channel latents_mean/std. + """ + + def __init__( + self, + encoder_dim=64, + encoder_rates=(2, 4, 4, 5, 5), + latent_dim=2048, + decoder_dim=1024, + vae_latent_channels=32, + ): + super().__init__() + self.sample_rate = 32000 + + self.hop_length = 1 + for r in encoder_rates: + self.hop_length *= r + self.samples_per_latent = self.hop_length # 800 + self.latents_per_second = self.sample_rate // self.hop_length # 40 + self.output_sample_rate = self.sample_rate # read by LTXVAudioVAEDecode + + self.encoder = Encoder(encoder_dim, encoder_rates, latent_dim) + + self.pre_block = AttnProjection(latent_dim, vae_latent_channels, num_heads=8) + + self.mean_proj = ops.Conv1d(vae_latent_channels, vae_latent_channels, 1) + # logs_proj exists in the checkpoint but is unused at inference + # (encode returns the posterior mean, no sampling). + self.logs_proj = ops.Conv1d(vae_latent_channels, vae_latent_channels, 1) + + self.dec_in_proj = ops.Conv1d(vae_latent_channels, latent_dim, 1) + self.decoder = BigVGAN(num_mels=latent_dim, upsample_initial_channel=decoder_dim) + + self.register_buffer("latents_mean", torch.empty(vae_latent_channels)) + self.register_buffer("latents_std", torch.empty(vae_latent_channels)) + + def decode(self, z): + """Decode normalized latents [B, 32, 2, T] to stereo waveforms [B, 2, L] at 32 kHz.""" + b, c, s, t = z.shape + z = z.permute(0, 2, 1, 3).reshape(b * s, c, t) + mean = self.latents_mean.view(1, -1, 1).to(device=z.device, dtype=z.dtype) + std = self.latents_std.view(1, -1, 1).to(device=z.device, dtype=z.dtype) + z = z * std + mean + x = self.dec_in_proj(z) + x = self.decoder(x) # [b * s, 1, L], already clamped to [-1, 1] + return x.reshape(b, s, -1) + + def encode(self, waveform): + """Encode stereo waveforms [B, 2, L] at 32 kHz (in [-1, 1]) to normalized latents [B, 32, 2, T]. + + L is right-padded with zeros to a multiple of 800 samples; the returned + posterior mean is used directly (no sampling). + """ + b, s, length = waveform.shape + right_pad = math.ceil(length / self.hop_length) * self.hop_length - length + waveform = F.pad(waveform, (0, right_pad)) + x = waveform.reshape(b * s, 1, -1) + x = self.encoder(x) # [b * s, latent_dim, T] + x = self.pre_block(x.transpose(1, 2)).transpose(1, 2) # [b * s, 32, T] + z = self.mean_proj(x) + mean = self.latents_mean.view(1, -1, 1).to(device=z.device, dtype=z.dtype) + std = self.latents_std.view(1, -1, 1).to(device=z.device, dtype=z.dtype) + z = (z - mean) / std + return z.reshape(b, s, z.shape[1], z.shape[2]).permute(0, 2, 1, 3) diff --git a/comfy/ldm/minimax/model.py b/comfy/ldm/minimax/model.py new file mode 100644 index 000000000..494350d40 --- /dev/null +++ b/comfy/ldm/minimax/model.py @@ -0,0 +1,646 @@ +"""MiniMax H3 audio-video DiT. + +Single-stream packed-token transformer denoising video (24ch, patch 1x2x2) and +stereo audio (32ch, 40 Hz) latents jointly, conditioned on Qwen3-VL layer-50 hidden states. +The packed sequence is: +[text | cond rows | audio | video] for t2va/fl2va +[text | reference blocks | audio | video] for ref2va + +Timestep domain: the model receives the *video* sigma from the sampler and +derives per-token timesteps t = 1 - sigma internally; the audio stream runs on +its own shifted schedule (sigma_shift video 12.0 / audio 3.0), mapped from the +video sigma in closed form. The audio velocity is returned scaled by the +schedule map's derivative d(sigma_a)/d(sigma_v). +""" + +import math + +import torch +import torch.nn as nn + +import comfy.ldm.common_dit +import comfy.model_management +import comfy.model_prefetch +import comfy.ops +import comfy.patcher_extension +import comfy.quant_ops +from comfy.ldm.modules.attention import optimized_attention + +FRAME_PER_TOKEN = (1, 4, 4, 4, 4) +FRAME_RESCALE = 5.0 / 3.0 +VISUAL_COND_TIMESTEP = 0.999 +AUDIO_COND_TIMESTEP = 1.0 + + +def time_shift_sigma(sigma, from_shift, to_shift): + # invert sigma = s*b/(1+(s-1)*b) to the base grid, re-apply the other shift + base = sigma / (from_shift + sigma * (1.0 - from_shift)) + return to_shift * base / (1.0 + (to_shift - 1.0) * base) + + +def time_shift_slope(sigma, from_shift, to_shift): + """d(sigma_to)/d(sigma_from) at the same base-grid point. + + Scaling a stream's returned velocity by this slope makes the flat ODE that + any sampler integrates on the from-schedule equal to that stream's true ODE + on its own schedule. + """ + base = sigma / (from_shift + sigma * (1.0 - from_shift)) + return (to_shift * (1.0 + (from_shift - 1.0) * base) ** 2) / (from_shift * (1.0 + (to_shift - 1.0) * base) ** 2) + + +def patchify_video(latent, patch_size=(1, 2, 2)): + # [B, C, T, H, W] -> [B*t*h*w, C*pt*ph*pw] + b, c, t_full, h_full, w_full = latent.shape + pt, ph, pw = patch_size + t, h, w = t_full // pt, h_full // ph, w_full // pw + x = latent.reshape(b, c, t, pt, h, ph, w, pw) + x = torch.einsum("nctrhpwq->nthwcrpq", x) + return x.reshape(b * t * h * w, c * pt * ph * pw) + + +def unpatchify_video(rows, t, h, w, c=24, patch_size=(1, 2, 2)): + pt, ph, pw = patch_size + x = rows.reshape(-1, t, h, w, c, pt, ph, pw) + x = torch.einsum("nthwcrpq->nctrhpwq", x) + return x.reshape(-1, c, t * pt, h * ph, w * pw) + + +def pack_audio(latent): + # [B, C=32, ch=2, T] -> [ch*T, 32] channel-major (ch0 t0..T-1, ch1 t0..T-1) + b, c, ch, t = latent.shape + return latent[0].permute(1, 2, 0).reshape(ch * t, c) + + +def unpack_audio(rows, ch=2): + t = rows.shape[0] // ch + return rows.reshape(ch, t, rows.shape[-1]).permute(2, 0, 1).unsqueeze(0) + + +def _axis_from_sqrt_area(dim, patch, sqrt_area): + # linspace((1 - ratio) / 2, (1 + ratio) / 2, dim // patch, endpoint=False) * 32 + ratio = dim / sqrt_area + n = dim // patch + return (torch.arange(n, dtype=torch.float64) * (ratio / n) + (1.0 - ratio) / 2.0) * 32.0 + + +def _frame_grid(h, w): + # area-normalized (h, w) coordinates of one latent frame's 2x2-patch rows + area = math.sqrt(h * w) + hh, ww = torch.meshgrid(_axis_from_sqrt_area(h, 2, area), _axis_from_sqrt_area(w, 2, area), indexing="ij") + return torch.stack([hh.reshape(-1), ww.reshape(-1)], dim=-1), _axis_from_sqrt_area(w, 2, area) + + +def _video_t_spans(n): + return [FRAME_RESCALE * FRAME_PER_TOKEN[k % 5] for k in range(n)] + + +def _video_t_grid(n, origin): + # origin + exclusive cumsum + spans = torch.tensor(_video_t_spans(n), dtype=torch.float64) + return float(origin) + torch.cat([torch.zeros(1, dtype=torch.float64), spans[:-1].cumsum(0)]) + + +def _audio_grid(cursor, t, w_low, w_high): + # channel-major stereo rows: t advances per latent frame, w pinned to the grid extremes per stereo channel, h stays 0 + g = torch.zeros(t * 2, 3, dtype=torch.float64) + g[:, 0] = (cursor + torch.arange(t, dtype=torch.float64)).repeat(2) + g[:t, 2] = w_low + g[t:, 2] = w_high + return g + + +def _video_grid(vt, frame, cursor): + g = torch.empty(vt, frame.shape[0], 3, dtype=torch.float64) + g[:, :, 0] = _video_t_grid(vt, cursor)[:, None] + g[:, :, 1:] = frame[None] + return g.reshape(-1, 3) + + +class TimeEmbedder(nn.Module): + def __init__(self, freq_dim, hidden, out, dtype=None, device=None, operations=None): + super().__init__() + self.freq_dim = freq_dim + self.proj_in = operations.Linear(freq_dim, hidden, bias=True, dtype=dtype, device=device) + self.proj_out = operations.Linear(hidden, out, bias=True, dtype=dtype, device=device) + + def forward(self, t): + # t: [M] in [0, 1]; fp32 throughout, cos before sin + half = self.freq_dim // 2 + freqs = torch.exp(-math.log(10000.0) * torch.arange(half, dtype=torch.float32, device=t.device) / half) + args = t.to(torch.float32)[:, None] * freqs[None] + emb = torch.cat([torch.cos(args), torch.sin(args)], dim=-1) + return self.proj_out(nn.functional.silu(self.proj_in(emb))) + + +def rope_rotation_table(angles, dtype): + """[S, rot_dim] pair angles -> [1, S, 1, rot_dim/2, 2, 2] rotation matrices.""" + half = angles.shape[-1] // 2 + ang = angles[:, :half] # duplicated halves: [:, :half] == [:, half:] + c, s = torch.cos(ang), torch.sin(ang) + table = torch.stack([c, -s, s, c], dim=-1).reshape(1, angles.shape[0], 1, half, 2, 2) + return table.to(dtype) + + +class Attention(nn.Module): + def __init__(self, hidden, heads, head_dim, eps, dtype=None, device=None, operations=None): + super().__init__() + self.heads = heads + self.head_dim = head_dim + inner = heads * head_dim + self.qkv_proj = operations.Linear(hidden, inner * 3, bias=False, dtype=dtype, device=device) + self.q_norm = operations.RMSNorm(head_dim, eps=eps, dtype=dtype, device=device) + self.k_norm = operations.RMSNorm(head_dim, eps=eps, dtype=dtype, device=device) + self.out_proj = operations.Linear(inner, hidden, bias=False, dtype=dtype, device=device) + + def forward(self, x, rope_freqs=None, transformer_options={}): + s = x.shape[0] + q, k, v = self.qkv_proj(x).split(self.heads * self.head_dim, dim=-1) + v = v.view(s, self.heads, self.head_dim) + if rope_freqs is not None: + # fused per-head RMSNorm + partial split-half rope, in place on the qkv buffer + q = q.view(1, s, self.heads, self.head_dim) + k = k.view(1, s, self.heads, self.head_dim) + qw = comfy.model_management.cast_to(self.q_norm.weight, device=x.device) + kw = comfy.model_management.cast_to(self.k_norm.weight, device=x.device) + rot = rope_freqs.shape[-3] * 2 + if comfy.model_management.in_training: + q, k = comfy.quant_ops.ck.rms_rope_split_half( + q, k, rope_freqs, qw, kw, epsilon=self.q_norm.eps, rot_dim=rot) + else: + comfy.quant_ops.ck.rms_rope_split_half_( + q, k, rope_freqs, qw, kw, epsilon=self.q_norm.eps, rot_dim=rot) + q = q[0] + k = k[0] + else: + q = self.q_norm(q.view(s, self.heads, self.head_dim)) + k = self.k_norm(k.view(s, self.heads, self.head_dim)) + q = q.transpose(0, 1).unsqueeze(0) + k = k.transpose(0, 1).unsqueeze(0) + v = v.transpose(0, 1).unsqueeze(0) + out = optimized_attention(q, k, v, self.heads, mask=None, skip_reshape=True, transformer_options=transformer_options) + return self.out_proj(out.squeeze(0)) + + +class MLP(nn.Module): + def __init__(self, hidden, ffn, dtype=None, device=None, operations=None): + super().__init__() + self.fc1 = operations.Linear(hidden, ffn * 2, bias=False, dtype=dtype, device=device) + self.fc2 = operations.Linear(ffn, hidden, bias=False, dtype=dtype, device=device) + + def forward(self, x): + return comfy.ops.linear_input_act(self.fc2, self.fc1(x), "swiglu") + + +class AdalnProj(nn.Module): + def __init__(self, t_dim, hidden, expand, modalities, apply_silu=True, + dtype=None, device=None, operations=None): + super().__init__() + self.expand = expand + self.modalities = modalities + self.hidden = hidden + self.apply_silu = apply_silu + self.linear = operations.Linear(t_dim, expand * hidden * modalities, bias=True, dtype=dtype, device=device) + + def forward(self, t_emb): + # [M, t_dim] -> expand tensors of [M*modalities, hidden] + x = self.linear(nn.functional.silu(t_emb) if self.apply_silu else t_emb) + x = x.view(x.shape[0] * self.modalities, self.expand * self.hidden) + return x.chunk(self.expand, dim=-1) + + +def _mod_scale_shift(h, shift, scale, segments): + # segments: [(start, stop, mod_row)] covering h contiguously. + for a, b, row in segments: + h[a:b].mul_(1.0 + scale[row].to(h.dtype)).add_(shift[row].to(h.dtype)) + return h + + +def _mod_gate(x, gate, other, segments): + # other is the fresh attn/mlp output: accumulate the gated residual into the stream in place, one fused kernel per segment + for a, b, row in segments: + x[a:b].addcmul_(other[a:b], gate[row].to(x.dtype)) + return x + + +class RefinerBlock(nn.Module): + def __init__(self, hidden, heads, head_dim, ffn, eps, qk_eps, dtype=None, device=None, operations=None): + super().__init__() + self.norm1 = operations.RMSNorm(hidden, eps=eps, dtype=dtype, device=device) + self.norm2 = operations.RMSNorm(hidden, eps=eps, dtype=dtype, device=device) + self.attn = Attention(hidden, heads, head_dim, qk_eps, dtype=dtype, device=device, operations=operations) + self.mlp = MLP(hidden, ffn, dtype=dtype, device=device, operations=operations) + + def forward(self, x, transformer_options={}): + # attn/mlp outputs are fresh: accumulate residuals in place + x = self.attn(self.norm1(x), transformer_options=transformer_options).add_(x) + return self.mlp(self.norm2(x)).add_(x) + + +class TokenRefiner(nn.Module): + def __init__(self, num_layers, hidden, heads, head_dim, ffn, eps, qk_eps, final_eps, + dtype=None, device=None, operations=None): + super().__init__() + self.blocks = nn.ModuleList([ + RefinerBlock(hidden, heads, head_dim, ffn, eps, qk_eps, dtype=dtype, device=device, operations=operations) + for _ in range(num_layers)]) + self.final_norm = operations.RMSNorm(hidden, eps=final_eps, dtype=dtype, device=device) + + def forward(self, x, transformer_options={}): + for block in self.blocks: + x = block(x, transformer_options=transformer_options) + return self.final_norm(x) + + +class DiTBlock(nn.Module): + def __init__(self, hidden, heads, head_dim, ffn, t_dim, eps, qk_eps, + apply_silu=True, adaln_dtype=None, dtype=None, device=None, operations=None): + super().__init__() + self.norm1 = operations.RMSNorm(hidden, eps=eps, dtype=dtype, device=device) + self.norm2 = operations.RMSNorm(hidden, eps=eps, dtype=dtype, device=device) + self.attn = Attention(hidden, heads, head_dim, qk_eps, dtype=dtype, device=device, operations=operations) + self.mlp = MLP(hidden, ffn, dtype=dtype, device=device, operations=operations) + self.adaln_proj = AdalnProj(t_dim, hidden, 6, 3, apply_silu=apply_silu, + dtype=adaln_dtype if adaln_dtype is not None else dtype, + device=device, operations=operations) + + def forward(self, x, t_emb, mod_segments, rope_freqs, transformer_options={}): + shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.adaln_proj(t_emb) + h = _mod_scale_shift(self.norm1(x), shift_msa, scale_msa, mod_segments) + x = _mod_gate(x, gate_msa, self.attn(h, rope_freqs=rope_freqs, transformer_options=transformer_options), mod_segments) + h = _mod_scale_shift(self.norm2(x), shift_mlp, scale_mlp, mod_segments) + return _mod_gate(x, gate_mlp, self.mlp(h), mod_segments) + + +class FinalLayer(nn.Module): + def __init__(self, hidden, t_dim, video_dim, audio_dim, eps, apply_silu=True, adaln_dtype=None, + dtype=None, device=None, operations=None): + super().__init__() + self.norm = operations.RMSNorm(hidden, eps=eps, dtype=dtype, device=device) + self.adaln_proj = AdalnProj(t_dim, hidden, 2, 1, apply_silu=apply_silu, + dtype=adaln_dtype if adaln_dtype is not None else dtype, + device=device, operations=operations) + # output heads are the checkpoint's fp32 island; norm/adaln are stored at model dtype + self.video_out = operations.Linear(hidden, video_dim, bias=True, dtype=torch.float32, device=device) + self.audio_out = operations.Linear(hidden, audio_dim, bias=True, dtype=torch.float32, device=device) + + def forward(self, x, t_emb, video_seg, audio_seg): + # video_seg / audio_seg: (start, stop, timestep_row) of the target streams + shift, scale = self.adaln_proj(t_emb) + va, vb, vrow = video_seg + aa, ab, arow = audio_seg + hv = (self.norm(x[va:vb]) * (1.0 + scale[vrow]) + shift[vrow]).to(torch.float32) + ha = (self.norm(x[aa:ab]) * (1.0 + scale[arow]) + shift[arow]).to(torch.float32) + return self.video_out(hv), self.audio_out(ha) + + +class PackedLayout: + """Static packed-sequence structure for one shape/conditioning signature.""" + + def __init__(self, text_len, latent_t, latent_h, latent_w, audio_t, keyframes=None, refs=None, frame_count=None): + frame, w_grid = _frame_grid(latent_h, latent_w) + frame_rows = frame.shape[0] + + segments = [("text", text_len)] # (kind, n_rows) + g = torch.zeros(text_len, 3, dtype=torch.float64) + g[:, 0] = torch.arange(text_len, dtype=torch.float64) + pos = [g] # per segment: [n, 3] float64 (t, h, w) + + img_pos, img_update = [], [] + audio_pos, audio_update = [], [] + cursor = text_len + row = text_len + + if keyframes: + # fl2va: keyframe cond rows right after text, sharing the target spatial grid + for kf in keyframes: + pixel_index = kf["resolved_frame_index"] + if pixel_index == 0: + cond_t = float(text_len) + elif frame_count is not None and pixel_index == frame_count - 1: + cond_t = float(text_len) + sum(_video_t_spans(latent_t)) - FRAME_RESCALE + else: + raise ValueError("only first/last keyframe anchors are supported") + g = torch.empty(frame_rows, 3, dtype=torch.float64) + g[:, 0] = cond_t + g[:, 1:] = frame + segments.append(("cond", frame_rows)) + pos.append(g) + img_pos.append(torch.arange(row, row + frame_rows)) + img_update.append(torch.zeros(frame_rows, dtype=torch.bool)) + row += frame_rows + + target_audio_w = (float(w_grid[0]), float(w_grid[-1])) + if refs: + cursor = float(text_len) + for blk in refs: + kind = blk["kind"] + if kind == "image": + r_frame, _ = _frame_grid(blk["latent_h"], blk["latent_w"]) + n = r_frame.shape[0] + g = torch.empty(n, 3, dtype=torch.float64) + g[:, 0] = cursor + g[:, 1:] = r_frame + segments.append(("ref_img", n)) + pos.append(g) + img_pos.append(torch.arange(row, row + n)) + img_update.append(torch.zeros(n, dtype=torch.bool)) + row += n + cursor += 1.0 + elif kind == "audio": + rt = blk["ref_audio_t"] + if rt > 0: + segments.append(("ref_audio", rt * 2)) + pos.append(_audio_grid(cursor, rt, *target_audio_w)) + audio_pos.append(torch.arange(row, row + rt * 2)) + audio_update.append(torch.zeros(rt * 2, dtype=torch.bool)) + row += rt * 2 + cursor += float(rt) + elif kind in ("video", "video_audio"): + # the block's audio rows pack immediately before its video + # rows, both sharing the cursor origin + rt = blk["ref_audio_t"] + vt = blk["latent_t"] + r_frame, r_w_grid = _frame_grid(blk["latent_h"], blk["latent_w"]) + if rt > 0: + segments.append(("ref_audio", rt * 2)) + pos.append(_audio_grid(cursor, rt, float(r_w_grid[0]), float(r_w_grid[-1]))) + audio_pos.append(torch.arange(row, row + rt * 2)) + audio_update.append(torch.zeros(rt * 2, dtype=torch.bool)) + row += rt * 2 + n = vt * r_frame.shape[0] + segments.append(("ref_img", n)) + pos.append(_video_grid(vt, r_frame, cursor)) + img_pos.append(torch.arange(row, row + n)) + img_update.append(torch.zeros(n, dtype=torch.bool)) + row += n + cursor += max(float(rt), sum(_video_t_spans(vt))) + + # target audio then target video, always the last two segments + segments.append(("audio", audio_t * 2)) + pos.append(_audio_grid(cursor, audio_t, *target_audio_w)) + audio_pos.append(torch.arange(row, row + audio_t * 2)) + audio_update.append(torch.ones(audio_t * 2, dtype=torch.bool)) + row += audio_t * 2 + + n_video = latent_t * frame_rows + segments.append(("video", n_video)) + pos.append(_video_grid(latent_t, frame, cursor)) + img_pos.append(torch.arange(row, row + n_video)) + img_update.append(torch.ones(n_video, dtype=torch.bool)) + row += n_video + + self.seq_len = row + self.position_ids = torch.cat(pos) # [S, 3] float64 + self.img_pos = torch.cat(img_pos) + self.img_update = torch.cat(img_update) + self.audio_pos = torch.cat(audio_pos) + self.audio_update = torch.cat(audio_update) + self.signature = (text_len, latent_t, latent_h, latent_w, audio_t) + # contiguous segment table (start, stop, kind) + # kinds: text / cond / ref_img / ref_audio / audio / video + # the packed sequence is uniform per segment in (modality tag, timestep class), + # except the text span (tag runs resolved at forward time from the presentation tags) + seg_abs = [] + off = 0 + for kind, n in segments: + seg_abs.append((off, off + n, kind)) + off += n + self.segments = seg_abs + + +class MiniMaxH3Model(nn.Module): + def __init__(self, hidden_size=5376, num_layers=50, token_refiner_num_layers=2, + num_attention_heads=56, attention_head_dim=128, ffn_hidden_size=14336, + latents_dim=24, audio_latents_dim=32, patch_size=(1, 2, 2), text_dim=5120, + timestep_input_dim=256, time_embed_hidden_size=5376, time_embed_dim=2688, + rope_inv_freq_len=16, norm_eps=1e-5, qk_norm_eps=1e-5, final_norm_eps=1e-5, + sigma_shift_video=12.0, sigma_shift_audio=3.0, + adaln_curve_grid=None, + image_model=None, dtype=None, device=None, operations=None, **kwargs): + super().__init__() + self.dtype = dtype + self.hidden_size = hidden_size + self.patch_size = tuple(patch_size) + self.latents_dim = latents_dim + self.audio_latents_dim = audio_latents_dim + self.sigma_shift_video = sigma_shift_video + self.sigma_shift_audio = sigma_shift_audio + self.use_adaln_curves = adaln_curve_grid is not None + # curve-form checkpoints replace the time embedder and full-width adaln weights with a small shared basis of the time-embedding curve + curve = {"apply_silu": not self.use_adaln_curves, + "adaln_dtype": torch.float32 if self.use_adaln_curves else dtype} + video_patch_dim = latents_dim * self.patch_size[0] * self.patch_size[1] * self.patch_size[2] + + self.video_patch_proj = operations.Linear(video_patch_dim, hidden_size, bias=True, dtype=torch.float32, device=device) + self.audio_patch_proj = operations.Linear(audio_latents_dim, hidden_size, bias=True, dtype=torch.float32, device=device) + self.condition_proj = operations.Linear(text_dim, hidden_size, bias=True, dtype=dtype, device=device) + if self.use_adaln_curves: + self.register_buffer("adaln_t_table", torch.empty(adaln_curve_grid, time_embed_dim, dtype=torch.float32)) + else: + self.time_embedder = TimeEmbedder(timestep_input_dim, time_embed_hidden_size, time_embed_dim, + dtype=torch.float32, device=device, operations=operations) + self.rope = nn.Module() + self.rope.register_buffer("inv_freq", torch.empty(rope_inv_freq_len, dtype=torch.float32)) + self.token_refiner = TokenRefiner(token_refiner_num_layers, hidden_size, num_attention_heads, + attention_head_dim, ffn_hidden_size, norm_eps, qk_norm_eps, + final_norm_eps, dtype=dtype, device=device, operations=operations) + self.blocks = nn.ModuleList([ + DiTBlock(hidden_size, num_attention_heads, attention_head_dim, ffn_hidden_size, + time_embed_dim, norm_eps, qk_norm_eps, **curve, dtype=dtype, device=device, operations=operations) + for _ in range(num_layers)]) + self.final_layer = FinalLayer(hidden_size, time_embed_dim, video_patch_dim, audio_latents_dim, + final_norm_eps, **curve, dtype=dtype, device=device, operations=operations) + + def preprocess_text_embeds(self, text_states): + """[B, L, text_dim] Qwen states -> [B, L, hidden] refined text embeds.""" + if text_states.shape[-1] == self.hidden_size: + return text_states + return self.token_refiner(self.condition_proj(text_states[0])).unsqueeze(0) + + def rope_freqs(self, position_ids, device): + # [S, 3] float64 -> [S, 96] fp32 + pos = position_ids.to(torch.float32).to(device) + inv = comfy.model_management.cast_to(self.rope.inv_freq, device=device) + per_axis = pos.unsqueeze(-1) * inv.view(1, 1, -1) # [S, 3, 16] + t_f, h_f, w_f = per_axis.unbind(dim=1) + half = torch.cat((t_f, h_f, w_f), dim=-1) # [S, 48] + return torch.cat((half, half), dim=-1) # [S, 96] + + def _cond_video_rows(self, payload, device): + """Concatenated visual condition rows (normalized latents -> patchified), with condition noise augmentation.""" + rows = [] + aug = payload.get("visual_cond_noise_aug", VISUAL_COND_TIMESTEP) + seed = int(payload.get("seed", 0)) + # every condition intentionally restarts the same RNG stream + for z in payload.get("cond_video_latents", []): + r = patchify_video(z.to(torch.float32), self.patch_size) + if aug < 1.0: + gen = torch.Generator("cpu").manual_seed(seed) + noise = torch.randn(r.shape, generator=gen, dtype=torch.float32) + r = aug * r + (1.0 - aug) * noise.to(r.device) + rows.append(r.to(device)) + return torch.cat(rows, dim=0) if rows else None + + def _cond_audio_rows(self, payload, device): + rows = [] + aug = payload.get("audio_cond_noise_aug", AUDIO_COND_TIMESTEP) + seed = int(payload.get("seed", 0)) + 1 + for z in payload.get("cond_audio_latents", []): + r = pack_audio(z.to(torch.float32)) + if aug < 1.0: + gen = torch.Generator("cpu").manual_seed(seed) + noise = torch.randn(r.shape, generator=gen, dtype=torch.float32) + r = aug * r + (1.0 - aug) * noise.to(r.device) + rows.append(r.to(device)) + return torch.cat(rows, dim=0) if rows else None + + def forward(self, x, timestep, context, transformer_options={}, minimax_payload=None, **kwargs): + return comfy.patcher_extension.WrapperExecutor.new_class_executor( + self._forward, + self, + comfy.patcher_extension.get_all_wrappers(comfy.patcher_extension.WrappersMP.DIFFUSION_MODEL, transformer_options) + ).execute(x, timestep, context, transformer_options, minimax_payload=minimax_payload, **kwargs) + + def _forward(self, x, timestep, context, transformer_options={}, minimax_payload=None, **kwargs): + video_x, audio_x = x[0], x[1] + orig_t, orig_h, orig_w = video_x.shape[2], video_x.shape[3], video_x.shape[4] + video_x = comfy.ldm.common_dit.pad_to_patch_size(video_x, self.patch_size) + if video_x.shape[0] != 1: + raise ValueError("MiniMax H3 supports batch size 1") + payload = minimax_payload or {} + device = video_x.device + dtype = context.dtype # compute dtype + + latent_t, lat_h, lat_w = video_x.shape[2], video_x.shape[3], video_x.shape[4] + audio_t = audio_x.shape[-1] + text_len = context.shape[1] + # extra_conds prebuilds the layout once per sampling run + layout = payload.get("layout") + if layout is None or layout.signature != (text_len, latent_t, lat_h, lat_w, audio_t): + layout = PackedLayout(text_len, latent_t, lat_h, lat_w, audio_t, + keyframes=payload.get("keyframes"), + refs=payload.get("refs"), + frame_count=payload.get("frame_count")) + + # model_base passes model_sampling.timestep(sigma) = sigma * 1000 + shift_v = float(transformer_options.get("minimax_h3_sigma_shift_video", self.sigma_shift_video)) + shift_a = float(transformer_options.get("minimax_h3_sigma_shift_audio", self.sigma_shift_audio)) + sigma_v = (timestep.flatten()[0] / 1000.0).float().clamp(min=1e-6) + t_v = float(1.0 - sigma_v) + t_a = float(1.0 - time_shift_sigma(sigma_v, shift_v, shift_a)) + + # distinct timesteps are known analytically: text/pad follow video, cond rows pin near 1 + vis_aug = float(payload.get("visual_cond_noise_aug", VISUAL_COND_TIMESTEP)) + aud_aug = float(payload.get("audio_cond_noise_aug", AUDIO_COND_TIMESTEP)) + has_vis_cond = any(k in ("cond", "ref_img") for _, _, k in layout.segments) + has_aud_cond = any(k == "ref_audio" for _, _, k in layout.segments) + seg_t = {"text": t_v, "video": t_v, "audio": t_a, + "cond": max(t_v, vis_aug), "ref_img": max(t_v, vis_aug), + "ref_audio": max(t_a, aud_aug)} + unique_t = sorted({t_v, t_a} | ({seg_t["cond"]} if has_vis_cond else set()) + | ({seg_t["ref_audio"]} if has_aud_cond else set())) + t_row = {t: i for i, t in enumerate(unique_t)} + seg_tag = {"text": 1, "video": 0, "audio": 2, "cond": 0, "ref_img": 0, "ref_audio": 2} + + text_tags = payload.get("text_token_tags") + mod_segments = [] + for a, b, kind in layout.segments: + row_base = t_row[seg_t[kind]] * 3 + if kind == "text" and text_tags is not None: + # the presentation text span mixes tags (vision pads carry the video modality) split into tag runs + tags = text_tags.view(-1).tolist() + run_start = 0 + for i in range(1, b - a + 1): + if i == b - a or tags[i] != tags[run_start]: + mod_segments.append((a + run_start, a + i, row_base + int(tags[run_start]))) + run_start = i + else: + mod_segments.append((a, b, row_base + seg_tag[kind])) + + # embed + img_update = layout.img_update.to(device) + audio_update = layout.audio_update.to(device) + video_rows = patchify_video(video_x.to(torch.float32), self.patch_size) + audio_rows = pack_audio(audio_x.to(torch.float32)) + cond_video_rows = self._cond_video_rows(payload, device) + cond_audio_rows = self._cond_audio_rows(payload, device) + + all_video_rows = video_rows + if cond_video_rows is not None: + all_video_rows = torch.empty(img_update.shape[0], video_rows.shape[1], dtype=torch.float32, device=device) + all_video_rows[~img_update] = cond_video_rows + all_video_rows[img_update] = video_rows + all_audio_rows = audio_rows + if cond_audio_rows is not None: + all_audio_rows = torch.empty(audio_update.shape[0], audio_rows.shape[1], dtype=torch.float32, device=device) + all_audio_rows[~audio_update] = cond_audio_rows + all_audio_rows[audio_update] = audio_rows + + video_embed = self.video_patch_proj(all_video_rows).to(dtype) + audio_embed = self.audio_patch_proj(all_audio_rows).to(dtype) + text_states = context[0] + if text_states.shape[-1] != self.hidden_size: + text_states = self.token_refiner(self.condition_proj(text_states), + transformer_options=transformer_options) + + # segments are contiguous: assemble by slices, embed rows follow segment order + h = torch.empty(layout.seq_len, self.hidden_size, dtype=dtype, device=device) + voff = aoff = 0 + for a, b, kind in layout.segments: + n = b - a + if kind == "text": + h[a:b] = text_states + elif kind in ("cond", "ref_img", "video"): + h[a:b] = video_embed[voff:voff + n] + voff += n + else: # ref_audio / audio + h[a:b] = audio_embed[aoff:aoff + n] + aoff += n + + t_vals = torch.tensor(unique_t, dtype=torch.float32, device=device) + if self.use_adaln_curves: + # adaln projections consume interpolated coordinates of the time-embedding curve + table = comfy.model_management.cast_to(self.adaln_t_table, device=device) + pos = t_vals.clamp(0.0, 1.0) * (table.shape[0] - 1) # t in [0,1] -> fractional grid index, out-of-range t clamps to the curve ends + i0 = pos.floor().long().clamp(max=table.shape[0] - 2) # lower grid row, max-clamp keeps t=1.0 on the last interval instead of reading past the table + t_emb = torch.lerp(table[i0], table[i0 + 1], (pos - i0).unsqueeze(1)) # blend the two rows by the fractional part + else: + t_emb = self.time_embedder(t_vals).to(dtype) + + # rotation table computed once per forward, consumed by the kitchen split-half rope + rope_freqs = rope_rotation_table(self.rope_freqs(layout.position_ids, device), dtype) + + # blocks + patches_replace = transformer_options.get("patches_replace", {}) + blocks_replace = patches_replace.get("dit", {}) + prefetch_queue = comfy.model_prefetch.make_prefetch_queue(list(self.blocks), device, transformer_options) + for i, block in enumerate(self.blocks): + comfy.model_prefetch.prefetch_queue_pop(prefetch_queue, device, block) + if ("double_block", i) in blocks_replace: + def block_wrap(args): + return {"img": block(args["img"], args["t_emb"], args["mod_segments"], args["rope_freqs"], + transformer_options=args["transformer_options"])} + h = blocks_replace[("double_block", i)]( + {"img": h, "t_emb": t_emb, "mod_segments": mod_segments, "rope_freqs": rope_freqs, + "transformer_options": transformer_options}, + {"original_block": block_wrap})["img"] + else: + h = block(h, t_emb, mod_segments, rope_freqs, transformer_options=transformer_options) + if prefetch_queue is not None: + comfy.model_prefetch.prefetch_queue_pop(prefetch_queue, device, None) + + # target streams are single contiguous segments (audio then video, last two) + video_seg = next((a, b, t_row[seg_t["video"]]) for a, b, k in layout.segments if k == "video") + audio_seg = next((a, b, t_row[seg_t["audio"]]) for a, b, k in layout.segments if k == "audio") + v, a = self.final_layer(h, t_emb, video_seg, audio_seg) + + video_out = unpatchify_video(v, latent_t, lat_h // 2, lat_w // 2, self.latents_dim, self.patch_size) + video_out = video_out[:, :, :orig_t, :orig_h, :orig_w] + audio_out = unpack_audio(a) + + # The sampler integrates the flat ODE dX/dsigma_v = (X - denoised)/sigma_v. + # Scaling the audio velocity by d(sigma_a)/d(sigma_v) makes that ODE equal + # to the audio stream's true ODE on its own shifted schedule. + slope_a = time_shift_slope(sigma_v, shift_v, shift_a).to(audio_out.dtype) + return [-video_out.to(video_x.dtype), (-slope_a) * audio_out.to(audio_x.dtype)] diff --git a/comfy/ldm/minimax/vae.py b/comfy/ldm/minimax/vae.py new file mode 100644 index 000000000..aeb3421a2 --- /dev/null +++ b/comfy/ldm/minimax/vae.py @@ -0,0 +1,694 @@ +# MiniMax H3 video VAE: 3D causal CNN encoder + ViT3D decoder. + +import math + +import torch +import torch.nn as nn +import torch.nn.functional as F + +import comfy.ops +import comfy.quant_ops +import comfy.rmsnorm +from comfy.ldm.modules.attention import optimized_attention + +ops = comfy.ops.disable_weight_init + +IMAGENET_MEAN = (0.485, 0.456, 0.406) +IMAGENET_STD = (0.229, 0.224, 0.225) + +LATENTS_MEAN = [ + 0.858090341091156, -0.9606591463088989, 1.0661640167236328, -0.5090325474739075, + -0.2727581858634949, -1.3675414323806763, -0.2553254961967468, -0.26907554268836975, + -0.5376840829849243, -0.0464097298681736, 0.6657370328903198, 0.19690127670764923, + -0.5460608005523682, -0.4035342037677765, -0.23683024942874908, 0.25928452610969543, + -0.30133944749832153, 0.211341992020607, -1.1206848621368408, 0.3581933379173279, + -0.04225143790245056, 0.2604829967021942, 0.22864092886447906, 0.7056031823158264, +] + +LATENTS_STD = [ + 1.2223774194717407, 1.2767263650894165, 1.68317747116088865, 1.7549455165863037, + 1.5636216402053833, 2.194143533706665, 0.96531379222869875, 1.05698859691619875, + 0.841948926448822, 0.7729952931404114, 1.8955937623977661, 0.946841835975647, + 0.7996809482574463, 0.44988900423049925, 0.7197399735450745, 0.69362932443618775, + 2.961095094680786, 2.7694199085235595, 3.0496184825897215, 2.1088054180145265, + 3.276226282119751, 3.1627357006073, 2.28168129920959475, 2.6127843856811525, +] + + +# 3D causal CNN encoder + +class CausalConv3d(ops.Conv3d): + # Reflect spatial padding, causal (zeros, front-only) temporal padding. + def __init__(self, in_channels, out_channels, kernel_size, stride=1, padding=0): + super().__init__(in_channels, out_channels, kernel_size=kernel_size, stride=stride) + self.causal_padding = (padding,) * 3 if isinstance(padding, int) else tuple(padding) + + def forward(self, x): + if sum(self.causal_padding) == 0: + return super().forward(x) + + x = F.pad(x, (self.causal_padding[2], self.causal_padding[2], self.causal_padding[1], self.causal_padding[1], 0, 0), mode="reflect") + if x.shape[2] == 1: + # single frame: the causal front padding is all zeros truncate the temporal taps instead of convolving zero frames + return super().forward(x, autopad="causal_zero") + x = F.pad(x, (0, 0, 0, 0, self.causal_padding[0] * 2, 0), mode="constant") + return super().forward(x) + + +class TemporalIsolatedGroupNorm(ops.GroupNorm): + # GroupNorm with statistics computed per frame (time merged into batch). + def forward(self, x): + if x.dim() == 5: + b, c, t, h, w = x.shape + x = x.permute(0, 2, 1, 3, 4).contiguous().view(b * t, c, 1, h, w) + x = super().forward(x) + return x.view(b, t, c, h, w).permute(0, 2, 1, 3, 4).contiguous() + return super().forward(x) + + +def group_norm_3d(num_channels): + return TemporalIsolatedGroupNorm(num_groups=32, num_channels=num_channels, eps=1e-6, affine=True) + + +class Downsample3D(nn.Module): + def __init__(self, in_channels, out_channels, time_stride=1, space_stride=2): + super().__init__() + self.space_stride = space_stride + self.conv = CausalConv3d( + in_channels, + out_channels, + kernel_size=3, + padding=(1, 0, 0), + stride=(time_stride, space_stride, space_stride), + ) + + def forward(self, x): + if self.space_stride == 2: + x = F.pad(x, (0, 1, 0, 1, 0, 0), mode="reflect") + return self.conv(x) + + +class ResnetBlock3D(nn.Module): + def __init__(self, in_channels, out_channels=None): + super().__init__() + self.in_channels = in_channels + out_channels = in_channels if out_channels is None else out_channels + self.out_channels = out_channels + + self.norm1 = group_norm_3d(in_channels) + self.norm2 = group_norm_3d(out_channels) + self.conv1 = CausalConv3d(in_channels, out_channels, kernel_size=3, padding=1) + self.conv2 = CausalConv3d(out_channels, out_channels, kernel_size=3, padding=1) + if in_channels != out_channels: + self.nin_shortcut = CausalConv3d(in_channels, out_channels, kernel_size=1) + + def forward(self, x): + h = self.conv1(F.silu(self.norm1(x), inplace=True)) + h = self.conv2(F.silu(self.norm2(h), inplace=True)) + if self.in_channels != self.out_channels: + x = self.nin_shortcut(x) + return h.add_(x) + + +class EncoderFCN3D(nn.Module): + def __init__(self, ch, ch_mult, space_down, time_down, num_res_blocks, in_channels, z_channels, double_z=True): + super().__init__() + self.num_levels = len(ch_mult) + if isinstance(num_res_blocks, int): + num_res_blocks = [num_res_blocks] * self.num_levels + self.num_res_blocks = num_res_blocks + + block_mid = [ch * ch_mult[i] for i in range(self.num_levels)] + block_in = [block_mid[0]] + block_mid[:-1] + block_out = block_mid + + self.conv_in = CausalConv3d(in_channels, block_in[0], kernel_size=3, padding=1) + + self.down = nn.ModuleList() + for i_level in range(self.num_levels): + down = nn.Module() + down.block = nn.ModuleList() + for i in range(self.num_res_blocks[i_level]): + down.block.append( + ResnetBlock3D( + in_channels=block_in[i_level] if i == 0 else block_mid[i_level], + out_channels=block_mid[i_level], + ) + ) + if space_down[i_level] * time_down[i_level] > 1: + down.downsample = Downsample3D( + block_mid[i_level], + block_out[i_level], + time_stride=time_down[i_level], + space_stride=space_down[i_level], + ) + self.down.append(down) + + self.norm_out = group_norm_3d(block_out[-1]) + self.conv_out = CausalConv3d( + block_out[-1], + 2 * z_channels if double_z else z_channels, + kernel_size=3, + padding=1, + ) + + def forward(self, x): + h = self.conv_in(x) + for i_level in range(self.num_levels): + for i_block in range(self.num_res_blocks[i_level]): + h = self.down[i_level].block[i_block](h) + if hasattr(self.down[i_level], "downsample"): + h = self.down[i_level].downsample(h) + h = F.silu(self.norm_out(h)) + return self.conv_out(h) + + +# ViT3D decoder + +def create_token_ids(patch_dims, device, dtype): + coords_list = [] + for dim_size in patch_dims: + coords = torch.arange(0.5, dim_size, dtype=dtype, device=device) + coords = coords / dim_size + coords = 2.0 * coords - 1.0 + coords_list.append(coords) + coords = torch.stack(torch.meshgrid(*coords_list, indexing="ij"), dim=-1) + return coords.flatten(0, len(patch_dims) - 1).unsqueeze(0) + + +class RotaryEmbeddingND(nn.Module): + def __init__(self, dim, rotary_base=100.0, n_dim=3): + super().__init__() + self.n_dim = n_dim + self.angle_scale = 2.0 * math.pi + inv_freq = 1 / rotary_base ** torch.arange(0, 1, 2 * n_dim / dim, dtype=torch.float32) + self.register_buffer("inv_freq", inv_freq, persistent=False) + + def forward(self, img_ids): + # [B, S, n_dim] -> [B, S, 1, pairs, 2, 2] rotation table for the kitchen split-half rope + angles = ( + self.angle_scale + * img_ids[:, :, :, None].float() + * self.inv_freq.to(img_ids.device)[None, None, None, :] + ) + angles = angles.flatten(2, 3) + c, s = torch.cos(angles), torch.sin(angles) + table = torch.stack([c, -s, s, c], dim=-1).reshape(*angles.shape[:2], 1, angles.shape[-1], 2, 2) + return table.to(img_ids.dtype) + + +class FeedForward(nn.Module): + # Gated SiLU FFN. + def __init__(self, dim, mult=4, bias=True): + super().__init__() + inner_dim = dim * mult + self.w1 = ops.Linear(dim, inner_dim * 2, bias=bias) + self.w2 = ops.Linear(inner_dim, dim, bias=bias) + + def forward(self, x): + gate, x = self.w1(x).chunk(2, dim=-1) + return self.w2(F.silu(gate).mul_(x)) + + +class Attention(nn.Module): + def __init__(self, heads, dim_head, bias=True, eps=1e-5): + super().__init__() + self.dim_head = dim_head + self.heads = heads + inner_dim = dim_head * heads + self.norm_q = ops.RMSNorm(dim_head, eps=eps, elementwise_affine=False) + self.norm_k = ops.RMSNorm(dim_head, eps=eps, elementwise_affine=False) + self.to_qkv = ops.Linear(inner_dim, inner_dim * 3, bias=bias) + self.to_out = ops.Linear(inner_dim, inner_dim, bias=bias) + + def forward(self, x, rotary_pos_emb=None): + batch_size, seq_len, _ = x.shape + + qkv = self.to_qkv(x) + qkv = qkv.view(batch_size, seq_len, -1, 3 * self.dim_head) + query, key, value = torch.chunk(qkv, 3, dim=-1) + + query = comfy.rmsnorm.rms_norm(query, self.norm_q.weight, self.norm_q.eps) + key = comfy.rmsnorm.rms_norm(key, self.norm_k.weight, self.norm_k.eps) + + if rotary_pos_emb is not None: + rot = rotary_pos_emb.shape[-3] * 2 + query[..., :rot], key[..., :rot] = comfy.quant_ops.ck.apply_rope_split_half( + query[..., :rot], key[..., :rot], rotary_pos_emb) + + out = optimized_attention(query.transpose(1, 2), key.transpose(1, 2), value.transpose(1, 2), + self.heads, skip_reshape=True).nan_to_num_(0.0) + return self.to_out(out) + + +class TransformerBlock(nn.Module): + def __init__(self, heads, dim_head, bias=True, eps=1e-5): + super().__init__() + dim = heads * dim_head + self.norm1 = ops.RMSNorm(dim, elementwise_affine=True, eps=eps) + self.attn = Attention(heads=heads, dim_head=dim_head, bias=bias, eps=eps) + self.scale1 = nn.Parameter(torch.empty(dim)) + self.norm2 = ops.RMSNorm(dim, elementwise_affine=True, eps=eps) + self.ff = FeedForward(dim=dim, bias=bias) + self.scale2 = nn.Parameter(torch.empty(dim)) + + def forward(self, x, rotary_pos_emb=None): + x = x.addcmul_(self.attn(comfy.rmsnorm.rms_norm(x, self.norm1.weight, self.norm1.eps), rotary_pos_emb), self.scale1) + return x.addcmul_(self.ff(comfy.rmsnorm.rms_norm(x, self.norm2.weight, self.norm2.eps)), self.scale2) + + +class ViT3DDecoder(nn.Module): + def __init__(self, patch_size=16, patch_size_t=4, in_channels=24, out_channels=3, num_layers=36, heads=32, dim_head=64, rope_theta=100.0, + rope_dim_ratio=0.75, bias=True, eps=1e-5, num_register_tokens=4): + super().__init__() + dim = heads * dim_head + self.patch_size = patch_size + self.patch_size_t = patch_size_t + self.out_channels = out_channels + self.num_register_tokens = num_register_tokens + + self.pos_embed = RotaryEmbeddingND(int(dim_head * rope_dim_ratio), rope_theta, n_dim=3) + self.x_embedder = ops.Linear(in_channels, dim) + self.register_tokens = nn.Parameter(torch.empty(1, num_register_tokens, dim)) + # unused at inference; kept so the checkpoint loads without leftover keys + self.register_buffer("mask_token", torch.empty(1, 1, dim)) + + self.transformer_blocks = nn.ModuleList( + [TransformerBlock(heads=heads, dim_head=dim_head, bias=bias, eps=eps) + for _ in range(num_layers)] + ) + + self.norm_out = ops.LayerNorm(dim, elementwise_affine=True, eps=eps) + self.proj_out = ops.Linear(dim, out_channels * patch_size_t * patch_size * patch_size) + + def forward(self, x): + B, C, latent_T, latent_H, latent_W = x.shape + + h = self.x_embedder(x.flatten(2).transpose(1, 2)) # [B, T*H*W, C] + + num_patches = h.shape[1] + num_suffix = 1 + self.num_register_tokens + + h = torch.cat([h, self.register_tokens.expand(B, -1, -1), torch.zeros_like(h[:, 0:1, :])], dim=1) + + img_ids = create_token_ids((latent_T, latent_H, latent_W), x.device, x.dtype).expand(B, -1, -1) + suffix_ids = torch.zeros((B, num_suffix, 3), device=x.device, dtype=img_ids.dtype) + img_ids = torch.cat([img_ids, suffix_ids], dim=1) + + rotary_pos_emb = self.pos_embed(img_ids) + + for block in self.transformer_blocks: + h = block(h, rotary_pos_emb) + + output = self.proj_out(self.norm_out(h)) + + output = output[:, :num_patches, :] + + output = output.view( + B, latent_T, latent_H, latent_W, + self.out_channels, self.patch_size_t, self.patch_size, self.patch_size, + ) + output = output.permute(0, 4, 1, 5, 2, 6, 3, 7).contiguous() + output = output.reshape( + B, self.out_channels, + latent_T * self.patch_size_t, + latent_H * self.patch_size, + latent_W * self.patch_size, + ) + return output + + +# Full VAE + +class MiniMaxH3VideoVAE(nn.Module): + def __init__( + self, + in_channels=3, + out_ch=3, + ch=128, + embed_dim=24, + z_channels=24, + ch_mult=(1, 2, 2, 4, 4, 8), + num_res_blocks=2, + space_down=(2, 2, 2, 2, 1, 1), + time_down=(1, 2, 2, 1, 1, 1), + clip_length=17, + token_drop=3, + tile_size=256, + tile_overlap_min=64, + tiling=True, + ): + super().__init__() + self.vae_ratio = int(math.prod(space_down)) + self.vae_ratio_t = int(math.prod(time_down)) + + # temporal chunking parameters + self.clip_length = clip_length + self.token_drop = token_drop + self.frame_pre_padding = (-clip_length) % self.vae_ratio_t + self.tokens_chunk_size = math.ceil(clip_length / self.vae_ratio_t) + self.token_overlap = (-token_drop) % self.tokens_chunk_size + self.frame_overlap = max(self.token_overlap * self.vae_ratio_t - self.frame_pre_padding, 0) + + # spatial tiling parameters + self.tiling = tiling + self.tile_size = tile_size + self.tile_overlap_min = tile_overlap_min + + self.encoder = EncoderFCN3D( + ch=ch, + ch_mult=list(ch_mult), + space_down=list(space_down), + time_down=list(time_down), + num_res_blocks=num_res_blocks, + in_channels=in_channels, + z_channels=z_channels, + double_z=True, + ) + self.quant_conv = ops.Conv3d(z_channels * 2, 2 * embed_dim, 1) + self.post_quant_conv = ops.Conv3d(embed_dim, z_channels, 1) + self.decoder = ViT3DDecoder( + patch_size=self.vae_ratio, + patch_size_t=self.vae_ratio_t, + in_channels=z_channels, + out_channels=out_ch, + ) + + self.register_buffer("latents_mean", torch.tensor(LATENTS_MEAN)) + self.register_buffer("latents_std", torch.tensor(LATENTS_STD)) + self.register_buffer("pixel_mean", torch.tensor(IMAGENET_MEAN).view(1, 3, 1, 1, 1), persistent=False) + self.register_buffer("pixel_std", torch.tensor(IMAGENET_STD).view(1, 3, 1, 1, 1), persistent=False) + + # single-shot forward + + def _encode_moments(self, x): + return self.quant_conv(self.encoder(x)) + + def _decode_pixels(self, z): + return self.decoder(self.post_quant_conv(z)) + + def _adaptive_encode(self, x): + if self.tiling: + return self.tiled_encode(x) + return self._encode_moments(x) + + def _adaptive_decode(self, z): + if self.tiling: + return self.tiled_decode(z) + return self._decode_pixels(z) + + # spatial tiling + + def split_tiles(self, input_len): + tile_size = self.tile_size + if tile_size >= input_len: + return [0], [input_len], [] + + N = math.ceil(input_len / tile_size) + while True: + overlaps = [self.tile_overlap_min] * (N - 1) + remaining = tile_size * N - sum(overlaps) - input_len + if remaining < 0: + N += 1 + else: + break + + remaining_units = remaining // self.vae_ratio + for i in range(remaining_units): + overlaps[i % (N - 1)] += self.vae_ratio + + tile_start_idx = [0] + for i in range(N - 1): + tile_start_idx.append(tile_start_idx[-1] + tile_size - overlaps[i]) + + return tile_start_idx, [tile_size] * N, overlaps + + def blend(self, a, b, blend_extent, dim): + blend_extent = min(a.shape[dim], b.shape[dim], blend_extent) + + positions = torch.arange(blend_extent, device=b.device, dtype=b.dtype) + weight_a = 1 - positions / blend_extent + weight_b = positions / blend_extent + + shape = [1] * a.ndim + shape[dim] = blend_extent + weight_a = weight_a.view(shape) + weight_b = weight_b.view(shape) + + slice_a = [slice(None)] * a.ndim + slice_a[dim] = slice(-blend_extent, None) + slice_b = [slice(None)] * b.ndim + slice_b[dim] = slice(0, blend_extent) + + blended = a[tuple(slice_a)] * weight_a + b[tuple(slice_b)] * weight_b + + if blend_extent < b.shape[dim]: + slice_b_rest = [slice(None)] * b.ndim + slice_b_rest[dim] = slice(blend_extent, None) + return torch.cat([blended, b[tuple(slice_b_rest)]], dim=dim) + return blended + + def tiled_encode(self, x): + height, width = x.shape[-2], x.shape[-1] + y_idx, y_len, y_overlap = self.split_tiles(height) + x_idx, x_len, x_overlap = self.split_tiles(width) + + rows = [] + for i_pos, i_len in zip(y_idx, y_len): + row = [] + for j_pos, j_len in zip(x_idx, x_len): + tile = x[..., i_pos:i_pos + i_len, j_pos:j_pos + j_len] + row.append(self._encode_moments(tile)) + rows.append(row) + + latent_y_overlap = [o // self.vae_ratio for o in y_overlap] + latent_x_overlap = [o // self.vae_ratio for o in x_overlap] + + result_rows = [] + for i, row in enumerate(rows): + result_row = [] + for j, tile in enumerate(row): + if i > 0: + tile = self.blend(rows[i - 1][j], tile, latent_y_overlap[i - 1], dim=-2) + if j > 0: + tile = self.blend(row[j - 1], tile, latent_x_overlap[j - 1], dim=-1) + if i < len(rows) - 1: + tile = tile[..., :-latent_y_overlap[i], :] + if j < len(row) - 1: + tile = tile[..., :, :-latent_x_overlap[j]] + result_row.append(tile) + result_rows.append(torch.cat(result_row, dim=-1)) + return torch.cat(result_rows, dim=-2) + + def tiled_decode(self, z): + height, width = z.shape[-2] * self.vae_ratio, z.shape[-1] * self.vae_ratio + y_idx, y_len, y_overlap = self.split_tiles(height) + x_idx, x_len, x_overlap = self.split_tiles(width) + + # Blended tiles are written straight into a pre-allocated canvas. + canvas = None + row_tails = [] + out_y = 0 + for i, (i_pos, i_len) in enumerate(zip(y_idx, y_len)): + zi, zl = i_pos // self.vae_ratio, i_len // self.vae_ratio + new_tails = [] + left_tail = None + out_x = 0 + for j, (j_pos, j_len) in enumerate(zip(x_idx, x_len)): + zj, zw = j_pos // self.vae_ratio, j_len // self.vae_ratio + tile = self._decode_pixels(z[..., zi:zi + zl, zj:zj + zw]) + if i < len(y_idx) - 1: + new_tails.append(tile[..., -y_overlap[i]:, :].clone()) + next_left_tail = tile[..., :, -x_overlap[j]:].clone() if j < len(x_idx) - 1 else None + if i > 0: + tile = self.blend(row_tails[j], tile, y_overlap[i - 1], dim=-2) + if j > 0: + tile = self.blend(left_tail, tile, x_overlap[j - 1], dim=-1) + left_tail = next_left_tail + if i < len(y_idx) - 1: + tile = tile[..., :-y_overlap[i], :] + if j < len(x_idx) - 1: + tile = tile[..., :, :-x_overlap[j]] + if canvas is None: + canvas = torch.empty(*tile.shape[:-2], height, width, dtype=tile.dtype, device=tile.device) + canvas[..., out_y:out_y + tile.shape[-2], out_x:out_x + tile.shape[-1]].copy_(tile) + out_x += tile.shape[-1] + row_tails = new_tails + out_y += tile.shape[-2] + return canvas + + # temporal chunking + + def encode_temporal(self, x): + if x.shape[2] % self.clip_length != 0: + pad_size = (-x.shape[2]) % self.clip_length + pad_frames = x[:, :, -1:].repeat(1, 1, pad_size, 1, 1) + x = torch.cat([x, pad_frames], dim=2) + + num_chunks = x.shape[2] // self.clip_length + + z_list = [] + for i in range(num_chunks): + clip_x = x[:, :, i * self.clip_length:(i + 1) * self.clip_length, :, :] + z_list.append(self._adaptive_encode(clip_x)) + + z = torch.cat(z_list, dim=2) + if self.token_drop > 0: + z = z[:, :, :-self.token_drop] + return z + + def _decode_temporal_pad_frames(self, z_len, pad_tokens): + if pad_tokens <= 0: + return 0 + intra_tail = self.clip_length % self.vae_ratio_t + if intra_tail == 0: + return pad_tokens * self.vae_ratio_t + + z_len_before_pad = z_len - pad_tokens + return sum( + (intra_tail if (z_len_before_pad + k) % self.tokens_chunk_size == 0 + else self.vae_ratio_t) + for k in range(pad_tokens) + ) + + def _decode_temporal_frame_plan(self, z_len, num_chunks, pad_tokens): + chunk_dec = self.tokens_chunk_size * self.vae_ratio_t + split_count = int(self.token_drop > 0) + 1 + total_frames = 0 + final_overlap_frames = 0 + + for i in range(num_chunks): + t_start_idx = i * self.tokens_chunk_size + t_end_idx = t_start_idx + self.tokens_chunk_size + self.token_overlap + clip_token_len = max(0, min(t_end_idx, z_len) - min(t_start_idx, z_len)) + clip_frame_len = clip_token_len * self.vae_ratio_t + + for j in range(split_count): + f_start_idx = j * chunk_dec + f_end_idx = min(f_start_idx + chunk_dec, clip_frame_len) + chunk_frames = max(0, f_end_idx - f_start_idx - self.frame_pre_padding) + if j == 0: + total_frames += chunk_frames + else: + final_overlap_frames = chunk_frames + + total_frames += final_overlap_frames + return total_frames - self._decode_temporal_pad_frames(z_len, pad_tokens) + + def decode_temporal(self, z): + chunk_dec = self.tokens_chunk_size * self.vae_ratio_t + split_count = int(self.token_drop > 0) + 1 + + pseudo_total_tokens = z.shape[2] + self.token_drop + + pad_tokens = 0 + remainder = pseudo_total_tokens % self.tokens_chunk_size + if remainder != 0: + pad_tokens = self.tokens_chunk_size - remainder + pseudo_total_tokens += pad_tokens + + num_chunks = pseudo_total_tokens // self.tokens_chunk_size - int(self.token_drop > 0) + if num_chunks < 1: + # too few tokens for one chunk (e.g. T_lat == 2): pad one extra chunk + pad_tokens += self.tokens_chunk_size + num_chunks += 1 + + if pad_tokens > 0: + pad_z = z[:, :, -1:, :, :].repeat(1, 1, pad_tokens, 1, 1) + z = torch.cat([z, pad_z], dim=2) + + output_frames = self._decode_temporal_frame_plan(z.shape[2], num_chunks, pad_tokens) + + dec = None + dec_overlap = None + write_pos = 0 + + def write_part(part): + nonlocal dec, write_pos + part_frames = part.shape[2] + if part_frames <= 0: + return + if dec is None: + out_shape = list(part.shape) + out_shape[2] = output_frames + dec = torch.empty(out_shape, dtype=part.dtype, device=part.device) + copy_frames = min(part_frames, max(0, dec.shape[2] - write_pos)) + if copy_frames > 0: + dec[:, :, write_pos:write_pos + copy_frames, :, :].copy_( + part[:, :, :copy_frames, :, :] + ) + write_pos += copy_frames + + for i in range(num_chunks): + t_start_idx = i * self.tokens_chunk_size + t_end_idx = t_start_idx + self.tokens_chunk_size + self.token_overlap + clip_z = z[:, :, t_start_idx:t_end_idx, :, :] + + clip_dec = self._adaptive_decode(clip_z) + + for j in range(split_count): + f_start_idx = j * chunk_dec + f_end_idx = min(f_start_idx + chunk_dec, clip_dec.shape[2]) + clip_dec_chunk = clip_dec[:, :, f_start_idx:f_end_idx, :, :] + clip_dec_chunk = clip_dec_chunk[:, :, self.frame_pre_padding:, :, :] + + if j == 0: + if dec_overlap is not None: + clip_dec_chunk = self.blend( + dec_overlap, clip_dec_chunk, self.frame_overlap, dim=-3 + ) + dec_overlap = None + write_part(clip_dec_chunk) + else: + dec_overlap = clip_dec_chunk.contiguous() + + if i == num_chunks - 1 and dec_overlap is not None: + write_part(dec_overlap) + dec_overlap = None + + del clip_dec, clip_z + + return dec + + + def encode(self, x): + # x: [B, 3, T, H, W] in [-1, 1] -> normalized latents [B, 24, T_lat, H/16, W/16] + if x.ndim == 4: + x = x.unsqueeze(2) + + x = x.add(1.0).mul_(0.5).sub_(self.pixel_mean.to(x)).div_(self.pixel_std.to(x)) + + if x.shape[2] == 1: + moments = self._adaptive_encode(x) + moments = moments[:, :, -1:, :, :] + else: + moments = self.encode_temporal(x) + + mean = torch.chunk(moments.float(), 2, dim=1)[0] + + latents_mean = self.latents_mean.view(1, -1, 1, 1, 1).to(mean) + latents_std = self.latents_std.view(1, -1, 1, 1, 1).to(mean) + return (mean - latents_mean) / latents_std + + def encode_tiled(self, x, **kwargs): + # tiling is always on internally with the reference's semantic tile sizes, ignore tiling fallbacks + return self.encode(x) + + def decode_tiled(self, z, **kwargs): + return self.decode(z) + + def decode(self, z): + # z: [B, 24, T_lat, H_lat, W_lat] normalized latents -> pixels [B, 3, T, H, W] in [-1, 1] + latents_mean = self.latents_mean.view(1, -1, 1, 1, 1).to(z) + latents_std = self.latents_std.view(1, -1, 1, 1, 1).to(z) + z = z * latents_std + latents_mean + + if z.shape[2] == 1: + dec = self._adaptive_decode(z) + dec = dec[:, :, -1:, :, :] + else: + dec = self.decode_temporal(z) + + dec = dec.float() + dec.mul_(self.pixel_std.to(dec)).add_(self.pixel_mean.to(dec)).clamp_(0.0, 1.0).mul_(2.0).sub_(1.0) + return dec diff --git a/comfy/model_base.py b/comfy/model_base.py index 7c68c6eba..6631c9eb0 100644 --- a/comfy/model_base.py +++ b/comfy/model_base.py @@ -21,6 +21,7 @@ import comfy.ldm.hunyuan3dv2_1.hunyuandit import torch import logging import comfy.ldm.lightricks.av_model +import comfy.ldm.minimax.model import comfy.ldm.lightricks.symmetric_patchifier import comfy.context_windows from comfy.ldm.modules.diffusionmodules.openaimodel import UNetModel, Timestep @@ -2063,6 +2064,57 @@ class Hunyuan3Dv2_1(BaseModel): out['guidance'] = comfy.conds.CONDRegular(torch.FloatTensor([guidance])) return out +class MiniMaxH3(BaseModel): + def __init__(self, model_config, model_type=ModelType.FLOW, device=None): + super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.minimax.model.MiniMaxH3Model) + + def extra_conds(self, **kwargs): + out = super().extra_conds(**kwargs) + cross_attn = kwargs.get("cross_attn", None) + if cross_attn is not None: + # run condition_proj + token refiner once per sampling instead of per step + cross_attn = self.diffusion_model.preprocess_text_embeds( + cross_attn.to(device=kwargs["device"], dtype=self.get_dtype_inference())) + out['c_crossattn'] = comfy.conds.CONDRegular(cross_attn) + + latent_shapes = kwargs.get("latent_shapes", None) + if latent_shapes is not None: + out['latent_shapes'] = comfy.conds.CONDConstant(latent_shapes) + + # Everything H3-specific rides in one dict so _apply_model's dtype cast + # (which would flatten fp32 cond latents and long tags to bf16) skips it. + payload = {} + tags = kwargs.get("minimax_token_tags", None) + if tags is not None: + payload["text_token_tags"] = tags + keyframes = kwargs.get("minimax_keyframes", None) + if keyframes is not None: + payload["keyframes"] = keyframes + payload["frame_count"] = kwargs.get("minimax_frame_count", None) + payload["cond_video_latents"] = [kf["latent"] for kf in keyframes] + refs = kwargs.get("minimax_refs", None) + if refs is not None: + payload["refs"] = refs + payload["cond_video_latents"] = [r["latent"] for r in refs if "latent" in r] + payload["cond_audio_latents"] = [r["audio_latent"] for r in refs if r.get("audio_latent") is not None] + if kwargs.get("minimax_visual_cond_noise_aug", None) is not None: + payload["visual_cond_noise_aug"] = kwargs["minimax_visual_cond_noise_aug"] + if kwargs.get("minimax_audio_cond_noise_aug", None) is not None: + payload["audio_cond_noise_aug"] = kwargs["minimax_audio_cond_noise_aug"] + payload["seed"] = kwargs.get("seed", 0) + if cross_attn is not None and latent_shapes is not None and len(latent_shapes) > 1: + # packed layout built once per sampling run, h/w rounded up to the DiT's 2x2 patch + vs = latent_shapes[0] + payload["layout"] = comfy.ldm.minimax.model.PackedLayout( + cross_attn.shape[1], vs[2], (vs[3] + 1) // 2 * 2, (vs[4] + 1) // 2 * 2, + latent_shapes[1][-1], keyframes=payload.get("keyframes"), + refs=payload.get("refs"), frame_count=payload.get("frame_count")) + out['minimax_payload'] = comfy.conds.CONDConstant(payload) + return out + + def scale_latent_inpaint(self, sigma, noise, latent_image, **kwargs): + return latent_image + class TripoSplat(BaseModel): def __init__(self, model_config, model_type=ModelType.FLOW, device=None): super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.triposplat.model.LatentSeqMMFlowModel) diff --git a/comfy/model_detection.py b/comfy/model_detection.py index 39e973d36..103680fd1 100644 --- a/comfy/model_detection.py +++ b/comfy/model_detection.py @@ -359,6 +359,35 @@ def detect_unet_config(state_dict, key_prefix, metadata=None): # PixArt diffusers return None + if '{}video_patch_proj.weight'.format(key_prefix) in state_dict_keys and '{}audio_patch_proj.weight'.format(key_prefix) in state_dict_keys: # MiniMax H3 + dit_config = {} + dit_config["image_model"] = "minimax_h3" + dit_config["num_layers"] = count_blocks(state_dict_keys, '{}blocks.'.format(key_prefix) + '{}.') + dit_config["token_refiner_num_layers"] = count_blocks(state_dict_keys, '{}token_refiner.blocks.'.format(key_prefix) + '{}.') + dit_config["hidden_size"] = state_dict['{}video_patch_proj.weight'.format(key_prefix)].shape[0] + dit_config["latents_dim"] = state_dict['{}final_layer.video_out.weight'.format(key_prefix)].shape[0] // 4 # patch 1x2x2 + dit_config["audio_latents_dim"] = state_dict['{}final_layer.audio_out.weight'.format(key_prefix)].shape[0] + dit_config["attention_head_dim"] = state_dict['{}blocks.0.attn.q_norm.weight'.format(key_prefix)].shape[0] + qkv = state_dict['{}blocks.0.attn.qkv_proj.weight'.format(key_prefix)] + dit_config["num_attention_heads"] = qkv.shape[0] // (3 * dit_config["attention_head_dim"]) + dit_config["ffn_hidden_size"] = state_dict['{}blocks.0.mlp.fc1.weight'.format(key_prefix)].shape[0] // 2 + dit_config["text_dim"] = state_dict['{}condition_proj.weight'.format(key_prefix)].shape[1] + table_key = '{}adaln_t_table'.format(key_prefix) + if table_key in state_dict_keys: + # adaln shipped over a precomputed curve basis: the adaln linears span a small shared basis of the time-embedding curve (no time embedder) + table = state_dict[table_key].shape # [grid, k] + dit_config["adaln_curve_grid"] = table[0] + dit_config["time_embed_dim"] = table[1] + else: + te = state_dict['{}time_embedder.proj_in.weight'.format(key_prefix)] + dit_config["timestep_input_dim"] = te.shape[1] + dit_config["time_embed_hidden_size"] = te.shape[0] + dit_config["time_embed_dim"] = state_dict['{}time_embedder.proj_out.weight'.format(key_prefix)].shape[0] + dit_config["rope_inv_freq_len"] = state_dict['{}rope.inv_freq'.format(key_prefix)].shape[0] + if metadata is not None and "config" in metadata: + dit_config.update(json.loads(metadata["config"]).get("transformer", {})) + return dit_config + if '{}adaln_single.emb.timestep_embedder.linear_1.bias'.format(key_prefix) in state_dict_keys: #Lightricks ltxv dit_config = {} dit_config["image_model"] = "ltxav" if f'{key_prefix}audio_adaln_single.linear.weight' in state_dict_keys else "ltxv" diff --git a/comfy/ops.py b/comfy/ops.py index 6c3845eef..077a42351 100644 --- a/comfy/ops.py +++ b/comfy/ops.py @@ -943,13 +943,61 @@ if CUBLAS_IS_AVAILABLE: # ============================================================================== # Mixed Precision Operations # ============================================================================== +from . import quant_ops from .quant_ops import ( QuantizedTensor, QUANT_ALGOS, TensorCoreFP8Layout, + TensorWiseINT8Layout, get_layout_class, ) +def _swiglu_eager(x): + gate, up = x.chunk(2, dim=-1) + return torch.nn.functional.silu(gate).mul_(up) + + +INPUT_ACT_EAGER = { + "gelu_tanh": lambda x: torch.nn.functional.gelu(x, approximate="tanh"), + "swiglu": _swiglu_eager, +} + + +def linear_input_act(linear, x, input_act): + """``linear(act(x))``, with ``act`` folded into an INT8 activation quantizer. + + An INT8 linear quantizes its input anyway, so an elementwise activation can + ride along inside that kernel instead of writing a full-size intermediate to + HBM and reading it straight back. Worth it for an MLP's down-projection, + where the intermediate is several times the hidden size. + + """ + weight = linear.weight + if (comfy.model_management.in_training + or not isinstance(weight, QuantizedTensor) + or weight._layout_cls != "TensorWiseINT8Layout" + or getattr(weight._params, "transposed", False)): + return linear(INPUT_ACT_EAGER[input_act](x)) + + # want_requant keeps a vbar-streamed layer on the INT8 path when a LoRA is + # patched in on the fly; without it the cast hands back a dequantized weight. + weight, bias, offload_stream = cast_bias_weight( + linear, x, offloadable=True, compute_dtype=x.dtype, want_requant=True) + try: + if not isinstance(weight, QuantizedTensor): + # A LoRA weight_function, or activations whose dtype differs from the + # weight's, make the cast hand back a dequantized tensor. + return torch.nn.functional.linear(INPUT_ACT_EAGER[input_act](x), weight, bias) + qdata, scale = TensorWiseINT8Layout.get_plain_tensors(weight) + return quant_ops.ck.int8_linear( + x, qdata, scale, bias, x.dtype, + convrot=getattr(weight._params, "convrot", False), + convrot_groupsize=getattr(weight._params, "convrot_groupsize", 256), + input_act=input_act, + ) + finally: + uncast_bias_weight(linear, weight, bias, offload_stream) + class QuantLinearFunc(torch.autograd.Function): """Custom autograd function for quantized linear: quantized forward, optionally FP8 backward. @@ -1259,7 +1307,7 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_prec def state_dict(self, *args, destination=None, prefix="", **kwargs): sd = destination if destination is not None else {} - return _quantized_weight_state_dict(self, sd, prefix, extra_quant_params=("input_scale",)) + return _quantized_weight_state_dict(self, sd, prefix, extra_quant_params=("input_scale", "pre_quant_scale")) def _forward(self, input, weight, bias): return torch.nn.functional.linear(input, weight, bias) @@ -1298,6 +1346,11 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_prec def forward(self, input, *args, **kwargs): run_every_op() + # ModelOpt AWQ-style smoothing + pre_quant_scale = getattr(self, 'pre_quant_scale', None) + if pre_quant_scale is not None: + input = input * comfy.model_management.cast_to_device(pre_quant_scale, input.device, input.dtype) + input_shape = input.shape reshaped_nd = False #If cast needs to apply lora, it should be done in the compute dtype diff --git a/comfy/quant_ops.py b/comfy/quant_ops.py index 15f9b1fdb..53586956a 100644 --- a/comfy/quant_ops.py +++ b/comfy/quant_ops.py @@ -240,7 +240,7 @@ QUANT_ALGOS = { }, "nvfp4": { "storage_t": torch.uint8, - "parameters": {"weight_scale", "weight_scale_2", "input_scale"}, + "parameters": {"weight_scale", "weight_scale_2", "input_scale", "pre_quant_scale"}, "comfy_tensor_layout": "TensorCoreNVFP4Layout", "group_size": 16, }, diff --git a/comfy/sd.py b/comfy/sd.py index caf78222d..8d670106e 100644 --- a/comfy/sd.py +++ b/comfy/sd.py @@ -72,6 +72,9 @@ import comfy.text_encoders.ace15 import comfy.text_encoders.longcat_image import comfy.text_encoders.qwen35 import comfy.text_encoders.qwen3vl +import comfy.text_encoders.minimax +import comfy.ldm.minimax.vae +import comfy.ldm.minimax.audio_vae import comfy.text_encoders.boogu import comfy.text_encoders.ernie import comfy.text_encoders.gemma4 @@ -936,6 +939,50 @@ class VAE: #Force cast it for --disable-dynamic-vram users until there is a true core fix. if not comfy.memory_management.aimdo_enabled: self.disable_offload = True + elif "decoder.transformer_blocks.0.scale1" in sd and "encoder.down.5.block.0.conv1.weight" in sd: # MiniMax H3 video VAE + self.first_stage_model = comfy.ldm.minimax.vae.MiniMaxH3VideoVAE() + self.latent_channels = 24 + self.latent_dim = 3 + # frames 17k+5 <-> latents 5k+2, 16x spatial + self.upscale_ratio = (lambda a: max(1, (a - 2) // 5 * 17 + 5), 16, 16) + self.upscale_index_formula = (4, 16, 16) + self.downscale_ratio = (lambda a: max(1, (a - 5) // 17 * 5 + 2) if a > 1 else 1, 16, 16) + self.downscale_index_formula = (4, 16, 16) + self.working_dtypes = [torch.float16, torch.float32] + # the model tiles internally (256px spatial, 17-frame temporal chunks) + self.handles_tiling = True + def estimate_encode_memory(frames, height, width, dtype): + fixed = 110_000_000 if frames == 1 else 1_300_000_000 + elements_per_pixel = 7 if frames == 1 else 9.5 + return (elements_per_pixel * frames * height * width + fixed) * model_management.dtype_size(dtype) * 1.03 + + def estimate_decode_memory(frames, height, width, dtype): + fixed = 110_000_000 if frames <= 22 else 270_000_000 + return (9.5 * frames * height * width + fixed) * model_management.dtype_size(dtype) * 1.03 + + self.memory_used_encode = lambda shape, dtype: estimate_encode_memory(shape[2], shape[3], shape[4], dtype) + self.memory_used_decode = lambda shape, dtype: estimate_decode_memory(self.upscale_ratio[0](shape[2]), shape[3] * self.upscale_ratio[1], shape[4] * self.upscale_ratio[2], dtype) + elif "pre_block.attn.zero_k_bias" in sd: # MiniMax H3 audio VAE (DAC encoder + BigVGAN decoder) + self.first_stage_model = comfy.ldm.minimax.audio_vae.MiniMaxH3AudioVAE() + self.latent_channels = 32 + self.output_channels = 2 + self.pad_channel_value = "replicate" + self.audio_sample_rate = 32000 + self.upscale_ratio = 800 + self.downscale_ratio = 800 + self.latent_dim = 2 # [B, 32, stereo 2, T] + self.process_output = lambda audio: audio + self.process_input = lambda audio: audio + self.working_dtypes = [torch.float32] + # encode gets the waveform shape [B, 2, samples], decode the latent shape [B, 32, 2, T] + def estimate_encode_memory(samples, dtype): + return (900 * samples + 105_000_000) * model_management.dtype_size(dtype) * 1.03 + + def estimate_decode_memory(samples, dtype): + return max(42_000_000, 220 * samples + 20_000_000) * model_management.dtype_size(dtype) * 1.03 + + self.memory_used_encode = lambda shape, dtype: estimate_encode_memory(shape[2], dtype) + self.memory_used_decode = lambda shape, dtype: estimate_decode_memory(shape[-1] * self.upscale_ratio, dtype) elif "gs.base_offset_scale" in sd and "octree.out_proj.weight" in sd: # TripoSplat octree gaussian decoder self.first_stage_model = comfy.ldm.triposplat.vae.OctreeGaussianDecoder() self.latent_channels = 16 @@ -1393,6 +1440,7 @@ class CLIPType(Enum): KREA2 = 32 JOYIMAGE = 33 MAGE = 34 + MINIMAX = 35 @@ -1449,6 +1497,7 @@ class TEModel(Enum): QWEN3VL_4B = 34 QWEN3VL_8B = 35 GEMMA_4_12B = 36 + QWEN3VL_32B = 37 def detect_te_model(sd): @@ -1515,6 +1564,9 @@ def detect_te_model(sd): return TEModel.QWEN35_2B if "model.visual.deepstack_merger_list.0.norm.weight" in sd: # DeepStack is unique to Qwen3-VL return TEModel.QWEN3VL_4B if sd["model.visual.merger.linear_fc2.weight"].shape[0] == 2560 else TEModel.QWEN3VL_8B + if "visual.deepstack_merger_list.0.norm.weight" in sd and "model.layers.49.self_attn.q_proj.weight" in sd: + # MiniMax H3 conditioning encoder: Qwen3-VL-32B, truncated to 50 layers + return TEModel.QWEN3VL_32B if "model.layers.0.post_attention_layernorm.weight" in sd: weight = sd['model.layers.0.post_attention_layernorm.weight'] if 'model.layers.0.self_attn.q_norm.weight' in sd: @@ -1744,6 +1796,9 @@ def load_text_encoder_state_dicts(state_dicts=[], embedding_directory=None, clip qwen3vl_type = {TEModel.QWEN3VL_4B: "qwen3vl_4b", TEModel.QWEN3VL_8B: "qwen3vl_8b"}[te_model] clip_target.clip = comfy.text_encoders.qwen3vl.te(**llama_detect(clip_data), model_type=qwen3vl_type) clip_target.tokenizer = comfy.text_encoders.qwen3vl.tokenizer(model_type=qwen3vl_type) + elif te_model == TEModel.QWEN3VL_32B: + clip_target.clip = comfy.text_encoders.minimax.te(**llama_detect(clip_data)) + clip_target.tokenizer = comfy.text_encoders.minimax.MiniMaxH3Tokenizer elif te_model == TEModel.QWEN3_06B: clip_target.clip = comfy.text_encoders.anima.te(**llama_detect(clip_data)) clip_target.tokenizer = comfy.text_encoders.anima.AnimaTokenizer diff --git a/comfy/supported_models.py b/comfy/supported_models.py index ca89850a5..51b58ed1e 100644 --- a/comfy/supported_models.py +++ b/comfy/supported_models.py @@ -15,6 +15,7 @@ import comfy.text_encoders.flux import comfy.text_encoders.genmo import comfy.text_encoders.lt import comfy.text_encoders.hunyuan_video +import comfy.text_encoders.minimax import comfy.text_encoders.cosmos import comfy.text_encoders.lumina2 import comfy.text_encoders.wan @@ -955,6 +956,33 @@ class LTXAV(LTXV): out = model_base.LTXAV(self, device=device) return out +class MiniMaxH3(supported_models_base.BASE): + unet_config = { + "image_model": "minimax_h3", + } + + sampling_settings = { + "shift": 12.0, + } + + unet_extra_config = {} + latent_format = latent_formats.MiniMaxH3AV + + memory_usage_factor = 0.114 + + supported_inference_dtypes = [torch.bfloat16, torch.float32] + + vae_key_prefix = ["vae."] + text_encoder_key_prefix = ["text_encoders."] + + def get_model(self, state_dict, prefix="", device=None): + return model_base.MiniMaxH3(self, device=device) + + def clip_target(self, state_dict={}, prefix=""): + pref = self.text_encoder_key_prefix[0] + detect = comfy.text_encoders.hunyuan_video.llama_detect(state_dict, "{}qwen3vl_32b.transformer.".format(pref)) + return supported_models_base.ClipTarget(comfy.text_encoders.minimax.MiniMaxH3Tokenizer, comfy.text_encoders.minimax.te(**detect)) + class HunyuanVideo(supported_models_base.BASE): unet_config = { "image_model": "hunyuan_video", @@ -2407,6 +2435,7 @@ models = [ GenmoMochi, LTXV, LTXAV, + MiniMaxH3, HunyuanVideo15_SR_Distilled, HunyuanVideo15, HunyuanImage21Refiner, diff --git a/comfy/text_encoders/llama.py b/comfy/text_encoders/llama.py index 40d04007e..f5c5597ef 100644 --- a/comfy/text_encoders/llama.py +++ b/comfy/text_encoders/llama.py @@ -264,6 +264,17 @@ class Qwen3VL_4BConfig(Qwen3VL_8BConfig): intermediate_size: int = 9728 lm_head: bool = False # 4B ties word embeddings +@dataclass +class Qwen3VL_32BConfig(Qwen3VL_8BConfig): + # MiniMax H3 conditioning checkpoint: truncated to the first 50 of 64 layers, + # consumed as the unnormalized hidden state after layer 50 (no final norm, no lm_head) + hidden_size: int = 5120 + intermediate_size: int = 25600 + num_hidden_layers: int = 50 + num_attention_heads: int = 64 + lm_head: bool = False + final_norm: bool = False + @dataclass class Ovis25_2BConfig: vocab_size: int = 151936 diff --git a/comfy/text_encoders/minimax.py b/comfy/text_encoders/minimax.py new file mode 100644 index 000000000..c2dc47f7f --- /dev/null +++ b/comfy/text_encoders/minimax.py @@ -0,0 +1,201 @@ +"""MiniMax H3 text/vision conditioning: Qwen3-VL-32B (truncated to 50 layers). + +The H3 presentation is NOT chat-templated: token ids are raw prompt/label text +(no special tokens) with explicit vision blocks spliced in: + + t2va: + fl2va: ": " [": " ] + ref2va: per condition in request order (1-based ordinals per type): + image -> ": " + audio -> "