diff --git a/comfy/ldm/wan/model.py b/comfy/ldm/wan/model.py index 1c9782a38..9cde1347a 100644 --- a/comfy/ldm/wan/model.py +++ b/comfy/ldm/wan/model.py @@ -552,6 +552,7 @@ class WanModel(torch.nn.Module): List of denoised video tensors with original input shapes [C_out, F, H / 8, W / 8] """ # embeddings + x_input = x x = self.patch_embedding(x.float()).to(x.dtype) grid_sizes = x.shape[2:] transformer_options["grid_sizes"] = grid_sizes @@ -564,11 +565,13 @@ class WanModel(torch.nn.Module): e0 = self.time_projection(e).unflatten(2, (6, self.dim)) full_ref = None + img_offset = 0 if self.ref_conv is not None: full_ref = kwargs.get("reference_latent", None) if full_ref is not None: full_ref = self.ref_conv(full_ref).flatten(2).transpose(1, 2) x = torch.concat((full_ref, x), dim=1) + img_offset = full_ref.shape[1] # In-context reference (Bernini) context_latents = kwargs.get("context_latents", None) @@ -589,6 +592,7 @@ class WanModel(torch.nn.Module): context_img_len = clip_fea.shape[-2] patches_replace = transformer_options.get("patches_replace", {}) + patches = transformer_options.get("patches", {}) blocks_replace = patches_replace.get("dit", {}) transformer_options["total_blocks"] = len(self.blocks) transformer_options["block_type"] = "double" @@ -604,6 +608,11 @@ class WanModel(torch.nn.Module): else: x = block(x, e=e0, freqs=freqs, context=context, context_img_len=context_img_len, transformer_options=transformer_options) + if "double_block" in patches: + for p in patches["double_block"]: + out = p({"img": x, "x": x_input, "vec": e, "block_index": i, "img_offset": img_offset, "transformer_options": transformer_options}) + x = out["img"] + # head x = self.head(x, e) @@ -777,6 +786,7 @@ class VaceWanModel(WanModel): **kwargs, ): # embeddings + x_input = x x = self.patch_embedding(x.float()).to(x.dtype) grid_sizes = x.shape[2:] transformer_options["grid_sizes"] = grid_sizes @@ -807,6 +817,7 @@ class VaceWanModel(WanModel): x_orig = x patches_replace = transformer_options.get("patches_replace", {}) + patches = transformer_options.get("patches", {}) blocks_replace = patches_replace.get("dit", {}) transformer_options["total_blocks"] = len(self.blocks) transformer_options["block_type"] = "double" @@ -822,6 +833,11 @@ class VaceWanModel(WanModel): else: x = block(x, e=e0, freqs=freqs, context=context, context_img_len=context_img_len, transformer_options=transformer_options) + if "double_block" in patches: + for p in patches["double_block"]: + out = p({"img": x, "x": x_input, "vec": e, "block_index": i, "img_offset": 0, "transformer_options": transformer_options}) + x = out["img"] + ii = self.vace_layers_mapping.get(i, None) if ii is not None: for iii in range(len(c)): diff --git a/comfy/ldm/wan/model_animate.py b/comfy/ldm/wan/model_animate.py index 84d7adec4..411f2ab17 100644 --- a/comfy/ldm/wan/model_animate.py +++ b/comfy/ldm/wan/model_animate.py @@ -493,9 +493,11 @@ class AnimateWanModel(WanModel): **kwargs, ): # embeddings + x_input = x x = self.patch_embedding(x.float()).to(x.dtype) x, motion_vec = self.after_patch_embedding(x, pose_latents, face_pixel_values) grid_sizes = x.shape[2:] + transformer_options["grid_sizes"] = grid_sizes x = x.flatten(2).transpose(1, 2) # time embeddings @@ -505,11 +507,13 @@ class AnimateWanModel(WanModel): e0 = self.time_projection(e).unflatten(2, (6, self.dim)) full_ref = None + img_offset = 0 if self.ref_conv is not None: full_ref = kwargs.get("reference_latent", None) if full_ref is not None: full_ref = self.ref_conv(full_ref).flatten(2).transpose(1, 2) x = torch.concat((full_ref, x), dim=1) + img_offset = full_ref.shape[1] # context context = self.text_embedding(context) @@ -522,6 +526,7 @@ class AnimateWanModel(WanModel): context_img_len = clip_fea.shape[-2] patches_replace = transformer_options.get("patches_replace", {}) + patches = transformer_options.get("patches", {}) blocks_replace = patches_replace.get("dit", {}) transformer_options["total_blocks"] = len(self.blocks) transformer_options["block_type"] = "double" @@ -537,6 +542,11 @@ class AnimateWanModel(WanModel): else: x = block(x, e=e0, freqs=freqs, context=context, context_img_len=context_img_len, transformer_options=transformer_options) + if "double_block" in patches: + for p in patches["double_block"]: + out = p({"img": x, "x": x_input, "vec": e, "block_index": i, "img_offset": img_offset, "transformer_options": transformer_options}) + x = out["img"] + if i % 5 == 0 and motion_vec is not None: x = x + self.face_adapter.fuser_blocks[i // 5](x, motion_vec) diff --git a/comfy/ldm/wan/uni3c.py b/comfy/ldm/wan/uni3c.py new file mode 100644 index 000000000..b42c212d3 --- /dev/null +++ b/comfy/ldm/wan/uni3c.py @@ -0,0 +1,162 @@ +# Uni3C controlnet for Wan 2.1: https://github.com/ewrfcas/Uni3C +# Converted from the original diffusers based implementation. +import torch +import torch.nn as nn + +from comfy.ldm.flux.layers import EmbedND +from .model import WanSelfAttention + + +class Uni3CLayerNormZero(nn.Module): + def __init__( + self, + conditioning_dim, + embedding_dim, + eps=1e-5, + device=None, dtype=None, operations=None + ): + super().__init__() + self.silu = nn.SiLU() + self.linear = operations.Linear(conditioning_dim, 3 * embedding_dim, device=device, dtype=dtype) + self.norm = operations.LayerNorm(embedding_dim, eps=eps, elementwise_affine=True, device=device, dtype=dtype) + + def forward(self, x, temb): + shift, scale, gate = self.linear(self.silu(temb)).chunk(3, dim=1) + x = self.norm(x) * (1 + scale)[:, None, :] + shift[:, None, :] + return x, gate[:, None, :] + + +class Uni3CAttentionBlock(nn.Module): + def __init__( + self, + dim, + ffn_dim, + num_heads, + time_embed_dim=5120, + eps=1e-6, + device=None, dtype=None, operations=None + ): + super().__init__() + operation_settings = {"operations": operations, "device": device, "dtype": dtype} + self.norm1 = Uni3CLayerNormZero(time_embed_dim, dim, device=device, dtype=dtype, operations=operations) + self.self_attn = WanSelfAttention(dim, num_heads, qk_norm=True, eps=eps, operation_settings=operation_settings) + self.norm2 = Uni3CLayerNormZero(time_embed_dim, dim, device=device, dtype=dtype, operations=operations) + self.ffn = nn.Sequential( + operations.Linear(dim, ffn_dim, device=device, dtype=dtype), nn.GELU(approximate='tanh'), + operations.Linear(ffn_dim, dim, device=device, dtype=dtype)) + + def forward(self, x, temb, freqs, transformer_options={}): + norm_x, gate_msa = self.norm1(x, temb) + x = x + gate_msa * self.self_attn(norm_x, freqs, transformer_options=transformer_options) + norm_x, gate_ff = self.norm2(x, temb) + x = x + gate_ff * self.ffn(norm_x) + return x + + +class MaskCamEmbed(nn.Module): + def __init__( + self, + add_channels=7, + mid_channels=256, + conv_out_dim=5120, + interp=False, + device=None, dtype=None, operations=None + ): + super().__init__() + if interp: + self.mask_padding = [0, 0, 0, 0, 3, 3] # first and last frame conditioning + else: + self.mask_padding = [0, 0, 0, 0, 3, 0] # first frame conditioning + self.mask_proj = nn.Sequential( + operations.Conv3d(add_channels, mid_channels, kernel_size=(4, 8, 8), stride=(4, 8, 8), device=device, dtype=dtype), + operations.GroupNorm(mid_channels // 8, mid_channels, device=device, dtype=dtype), + nn.SiLU()) + self.mask_zero_proj = operations.Conv3d(mid_channels, conv_out_dim, kernel_size=(1, 2, 2), stride=(1, 2, 2), device=device, dtype=dtype) + + def forward(self, add_inputs): + add_padded = torch.nn.functional.pad(add_inputs, self.mask_padding, mode="constant", value=0) + add_embeds = self.mask_proj(add_padded) + add_embeds = self.mask_zero_proj(add_embeds) + add_embeds = add_embeds.flatten(2).transpose(1, 2) + return add_embeds + + +class WanUni3CControlnet(nn.Module): + def __init__( + self, + in_channels=36, + conv_out_dim=5120, + dim=1024, + ffn_dim=8192, + num_heads=16, + num_layers=20, + time_embed_dim=5120, + out_proj_dim=5120, + add_channels=7, + mid_channels=256, + device=None, dtype=None, operations=None + ): + super().__init__() + self.dtype = dtype + self.patch_size = (1, 2, 2) + self.in_channels = in_channels + self.num_layers = num_layers + + self.controlnet_patch_embedding = operations.Conv3d( + in_channels, conv_out_dim, kernel_size=self.patch_size, stride=self.patch_size, device=device, dtype=torch.float32) + self.controlnet_mask_embedding = MaskCamEmbed(add_channels, mid_channels, conv_out_dim, device=device, dtype=dtype, operations=operations) + + if conv_out_dim != dim: + self.proj_in = operations.Linear(conv_out_dim, dim, device=device, dtype=dtype) + else: + self.proj_in = nn.Identity() + + self.controlnet_blocks = nn.ModuleList([ + Uni3CAttentionBlock(dim, ffn_dim, num_heads, time_embed_dim, device=device, dtype=dtype, operations=operations) + for _ in range(num_layers)]) + self.proj_out = nn.ModuleList([ + operations.Linear(dim, out_proj_dim, device=device, dtype=dtype) + for _ in range(num_layers)]) + + head_dim = dim // num_heads + self.rope_embedder = EmbedND(dim=head_dim, theta=10000.0, axes_dim=[head_dim - 4 * (head_dim // 6), 2 * (head_dim // 6), 2 * (head_dim // 6)]) + + def rope_encode(self, t_len, h_len, w_len, device=None, dtype=None): + img_ids = torch.zeros((t_len, h_len, w_len, 3), device=device, dtype=dtype) + img_ids[:, :, :, 0] = img_ids[:, :, :, 0] + torch.arange(t_len, device=device, dtype=dtype).reshape(-1, 1, 1) + img_ids[:, :, :, 1] = img_ids[:, :, :, 1] + torch.arange(h_len, device=device, dtype=dtype).reshape(1, -1, 1) + img_ids[:, :, :, 2] = img_ids[:, :, :, 2] + torch.arange(w_len, device=device, dtype=dtype).reshape(1, 1, -1) + img_ids = img_ids.reshape(1, -1, img_ids.shape[-1]) + freqs = self.rope_embedder(img_ids).movedim(1, 2) + return freqs + + def process_input(self, control_input, render_mask=None, camera_embedding=None): + hidden = self.controlnet_patch_embedding(control_input.float()).to(control_input.dtype) + t_len, h_len, w_len = hidden.shape[2:] + freqs = self.rope_encode(t_len, h_len, w_len, device=hidden.device, dtype=hidden.dtype) + hidden = hidden.flatten(2).transpose(1, 2) + + add_inputs = None + if camera_embedding is not None and render_mask is not None: + add_inputs = torch.cat([render_mask, camera_embedding], dim=1) + elif render_mask is not None: + add_inputs = render_mask + + if add_inputs is not None: + hidden = hidden + self.controlnet_mask_embedding(add_inputs.to(hidden.dtype)) + + hidden = self.proj_in(hidden) + return hidden, freqs + + def forward_block(self, block_index, hidden, temb, freqs, transformer_options={}): + hidden = self.controlnet_blocks[block_index](hidden, temb, freqs, transformer_options=transformer_options) + residual = self.proj_out[block_index](hidden) + return hidden, residual + + def forward(self, control_input, temb, render_mask=None, camera_embedding=None, transformer_options={}): + hidden, freqs = self.process_input(control_input, render_mask=render_mask, camera_embedding=camera_embedding) + controlnet_states = [] + for i in range(self.num_layers): + hidden, residual = self.forward_block(i, hidden, temb, freqs, transformer_options=transformer_options) + controlnet_states.append(residual) + return controlnet_states diff --git a/comfy_extras/nodes_model_patch.py b/comfy_extras/nodes_model_patch.py index 3f785c8b5..5b55b1ffd 100644 --- a/comfy_extras/nodes_model_patch.py +++ b/comfy_extras/nodes_model_patch.py @@ -8,6 +8,7 @@ import comfy.ldm.common_dit import comfy.latent_formats import comfy.ldm.lumina.controlnet import comfy.ldm.supir.supir_modules +import comfy.ldm.wan.uni3c from comfy.ldm.wan.model_multitalk import WanMultiTalkAttentionBlock, MultiTalkAudioProjModel from comfy_api.latest import io from comfy.ldm.supir.supir_patch import SUPIRPatch @@ -261,6 +262,32 @@ class ModelPatchLoader: if torch.count_nonzero(ref_weight) == 0: config['broken'] = True model = comfy.ldm.lumina.controlnet.ZImage_Control(device=comfy.model_management.unet_offload_device(), dtype=dtype, operations=comfy.ops.manual_cast, **config) + elif 'controlnet_patch_embedding.weight' in sd: # Uni3C controlnet for Wan + attn_key_replace = {".self_attn.to_q.": ".self_attn.q.", + ".self_attn.to_k.": ".self_attn.k.", + ".self_attn.to_v.": ".self_attn.v.", + ".self_attn.to_out.0.": ".self_attn.o."} + converted_sd = {} + for k, w in sd.items(): + for r, rr in attn_key_replace.items(): + k = k.replace(r, rr) + converted_sd[k] = w + sd = converted_sd + + num_layers = sum(1 for k in sd if k.startswith("proj_out.") and k.endswith(".weight")) + model = comfy.ldm.wan.uni3c.WanUni3CControlnet( + in_channels=sd["controlnet_patch_embedding.weight"].shape[1], + conv_out_dim=sd["controlnet_patch_embedding.weight"].shape[0], + dim=sd["proj_out.0.weight"].shape[1], + ffn_dim=sd["controlnet_blocks.0.ffn.0.bias"].shape[0], + num_layers=num_layers, + time_embed_dim=sd["controlnet_blocks.0.norm1.linear.weight"].shape[1], + out_proj_dim=sd["proj_out.0.weight"].shape[0], + add_channels=sd["controlnet_mask_embedding.mask_proj.0.weight"].shape[1], + mid_channels=sd["controlnet_mask_embedding.mask_proj.0.weight"].shape[0], + device=comfy.model_management.unet_offload_device(), + dtype=dtype, + operations=comfy.ops.manual_cast) elif "audio_proj.proj1.weight" in sd: model = MultiTalkModelPatch( audio_window=5, context_tokens=32, vae_scale=4, @@ -514,6 +541,99 @@ class ZImageFunControlnet(QwenImageDiffsynthControlnet): CATEGORY = "model/patch/z-image" +class WanUni3CCnetPatch: + def __init__(self, model_patch, render_latent, strength, sigma_start, sigma_end): + self.model_patch = model_patch + self.render_latent = render_latent + self.strength = strength + self.sigma_start = sigma_start + self.sigma_end = sigma_end + self.temp_data = None + + def build_controlnet_input(self, x, dtype): + # first 20 channels of the model input: noise latent + I2V mask (zero padded for T2V) + hidden = x[:1, :20].to(dtype) + if hidden.shape[1] < 20: + pad_shape = list(hidden.shape) + pad_shape[1] = 20 - hidden.shape[1] + hidden = torch.cat([hidden, torch.zeros(pad_shape, dtype=hidden.dtype, device=hidden.device)], dim=1) + + render = self.render_latent.to(device=hidden.device, dtype=hidden.dtype) + if render.shape[2:] != hidden.shape[2:]: + render = torch.nn.functional.interpolate(render, size=hidden.shape[2:], mode="trilinear", align_corners=False) + return torch.cat([hidden, render], dim=1) + + def __call__(self, kwargs): + img = kwargs.get("img") + block_index = kwargs.get("block_index") + transformer_options = kwargs.get("transformer_options", {}) + + if block_index == 0: + self.temp_data = None + active = True + sigmas = transformer_options.get("sigmas", None) + if sigmas is not None and sigmas.numel() > 0: + sigma = sigmas[0].item() + if sigma > self.sigma_start or sigma < self.sigma_end: + active = False + if active: + controlnet_input = self.build_controlnet_input(kwargs.get("x"), img.dtype) + temb = kwargs.get("vec")[:1] + if temb.ndim == 3: + temb = temb[:, 0] + hidden, freqs = self.model_patch.model.process_input(controlnet_input) + self.temp_data = (hidden, temb.to(img.dtype), freqs) + + if self.temp_data is not None and block_index < self.model_patch.model.num_layers: + hidden, temb, freqs = self.temp_data + hidden, residual = self.model_patch.model.forward_block(block_index, hidden, temb, freqs, transformer_options=transformer_options) + img_offset = kwargs.get("img_offset", 0) + img[:, img_offset:img_offset + residual.shape[1]] += residual.to(img.dtype) * self.strength + if block_index >= self.model_patch.model.num_layers - 1: + self.temp_data = None + else: + self.temp_data = (hidden, temb, freqs) + + return kwargs + + def to(self, device_or_dtype): + if isinstance(device_or_dtype, torch.device): + self.render_latent = self.render_latent.to(device_or_dtype) + self.temp_data = None + return self + + def models(self): + return [self.model_patch] + + +class WanUni3CControlnetApply: + @classmethod + def INPUT_TYPES(s): + return {"required": { "model": ("MODEL",), + "model_patch": ("MODEL_PATCH",), + "vae": ("VAE",), + "render_video": ("IMAGE", {"tooltip": "The guidance video rendered from the camera trajectory, most commonly warped point cloud renders of the input image."}), + "strength": ("FLOAT", {"default": 1.0, "min": -10.0, "max": 10.0, "step": 0.01}), + "start_percent": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001}), + "end_percent": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.001}), + }} + RETURN_TYPES = ("MODEL",) + FUNCTION = "apply_patch" + EXPERIMENTAL = True + + CATEGORY = "model/patch/wan" + + def apply_patch(self, model, model_patch, vae, render_video, strength, start_percent, end_percent): + model_patched = model.clone() + render_latent = vae.encode(render_video[:, :, :, :3]) + render_latent = model.get_model_object("latent_format").process_in(render_latent) + model_sampling = model.get_model_object("model_sampling") + sigma_start = model_sampling.percent_to_sigma(start_percent) + sigma_end = model_sampling.percent_to_sigma(end_percent) + model_patched.set_model_double_block_patch(WanUni3CCnetPatch(model_patch, render_latent, strength, sigma_start, sigma_end)) + return (model_patched,) + + class UsoStyleProjectorPatch: def __init__(self, model_patch, encoded_image): self.model_patch = model_patch @@ -672,6 +792,7 @@ NODE_CLASS_MAPPINGS = { "ModelPatchLoader": ModelPatchLoader, "QwenImageDiffsynthControlnet": QwenImageDiffsynthControlnet, "ZImageFunControlnet": ZImageFunControlnet, + "WanUni3CControlnetApply": WanUni3CControlnetApply, "USOStyleReference": USOStyleReference, "SUPIRApply": SUPIRApply, } @@ -680,6 +801,7 @@ NODE_DISPLAY_NAME_MAPPINGS = { "ModelPatchLoader": "Load Model Patch", "QwenImageDiffsynthControlnet": "Apply Qwen Image DiffSynth ControlNet", "ZImageFunControlnet": "Apply Z-Image Fun ControlNet", + "WanUni3CControlnetApply": "Apply Wan Uni3C ControlNet", "USOStyleReference": "Apply USO Style Reference", "SUPIRApply": "Apply SUPIR Patch", }