RomanNum3ral
/
ComfyUI
mirror of https://github.com/Comfy-Org/ComfyUI.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into codex/continuous-batching

This commit is contained in:
野生の男 2026-07-19 19:54:26 +09:00
parent 0c4ae6c329 c9602625e4
commit bff8cefed1
13 changed files with 2012 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

278
comfy/ldm/anima/lllite.py Normal file
View File

@ -0,0 +1,278 @@
import re
import torch
from torch import nn
import torch.nn.functional as F
import comfy.ops
import comfy.utils
MODULE_PATTERN = re.compile(r"lllite_dit_blocks_(\d+)_(self_attn_[qkv]_proj|cross_attn_q_proj|mlp_layer1)$")
def _group_norm(channels, device=None, dtype=None, operations=None):
groups = 8
while groups > 1 and channels % groups != 0:
groups //= 2
return operations.GroupNorm(groups, channels, device=device, dtype=dtype)
class AnimaLLLiteResBlock(nn.Module):
def __init__(self, channels, device=None, dtype=None, operations=None):
super().__init__()
self.norm1 = _group_norm(channels, device=device, dtype=dtype, operations=operations)
self.conv1 = operations.Conv2d(channels, channels, kernel_size=3, padding=1, device=device, dtype=dtype)
self.norm2 = _group_norm(channels, device=device, dtype=dtype, operations=operations)
self.conv2 = operations.Conv2d(channels, channels, kernel_size=3, padding=1, device=device, dtype=dtype)
def forward(self, x):
h = self.conv1(F.silu(self.norm1(x)))
h = self.conv2(F.silu(self.norm2(h)))
return x + h
class AnimaLLLiteASPP(nn.Module):
def __init__(self, channels, dilations, device=None, dtype=None, operations=None):
super().__init__()
branches = []
for dilation in dilations:
if dilation == 1:
conv = operations.Conv2d(channels, channels, kernel_size=1, device=device, dtype=dtype)
else:
conv = operations.Conv2d(channels, channels, kernel_size=3, padding=dilation, dilation=dilation, device=device, dtype=dtype)
branches.append(nn.Sequential(conv, _group_norm(channels, device=device, dtype=dtype, operations=operations), nn.SiLU()))
self.branches = nn.ModuleList(branches)
self.global_pool = nn.AdaptiveAvgPool2d(1)
self.global_conv = nn.Sequential(
operations.Conv2d(channels, channels, kernel_size=1, device=device, dtype=dtype),
_group_norm(channels, device=device, dtype=dtype, operations=operations),
nn.SiLU(),
)
self.proj = nn.Sequential(
operations.Conv2d(channels * (len(dilations) + 1), channels, kernel_size=1, device=device, dtype=dtype),
_group_norm(channels, device=device, dtype=dtype, operations=operations),
nn.SiLU(),
)
def forward(self, x):
height, width = x.shape[-2:]
outputs = [branch(x) for branch in self.branches]
pooled = self.global_conv(self.global_pool(x))
outputs.append(F.interpolate(pooled, size=(height, width), mode="bilinear", align_corners=False))
return self.proj(torch.cat(outputs, dim=1))
class AnimaLLLiteConditioning(nn.Module):
def __init__(self, cond_in_channels, cond_dim, cond_emb_dim, cond_resblocks, aspp_dilations, device=None, dtype=None, operations=None):
super().__init__()
half_dim = cond_dim // 2
self.conv1 = operations.Conv2d(cond_in_channels, half_dim, kernel_size=4, stride=4, device=device, dtype=dtype)
self.norm1 = _group_norm(half_dim, device=device, dtype=dtype, operations=operations)
self.conv2 = operations.Conv2d(half_dim, half_dim, kernel_size=3, padding=1, device=device, dtype=dtype)
self.norm2 = _group_norm(half_dim, device=device, dtype=dtype, operations=operations)
self.conv3 = operations.Conv2d(half_dim, cond_dim, kernel_size=4, stride=4, device=device, dtype=dtype)
self.norm3 = _group_norm(cond_dim, device=device, dtype=dtype, operations=operations)
self.resblocks = nn.ModuleList([
AnimaLLLiteResBlock(cond_dim, device=device, dtype=dtype, operations=operations)
for _ in range(cond_resblocks)
])
self.aspp = AnimaLLLiteASPP(cond_dim, aspp_dilations, device=device, dtype=dtype, operations=operations) if aspp_dilations else None
self.proj = operations.Conv2d(cond_dim, cond_emb_dim, kernel_size=1, device=device, dtype=dtype)
self.out_norm = operations.LayerNorm(cond_emb_dim, device=device, dtype=dtype)
def forward(self, x):
x = F.silu(self.norm1(self.conv1(x)))
x = F.silu(self.norm2(self.conv2(x)))
x = F.silu(self.norm3(self.conv3(x)))
for block in self.resblocks:
x = block(x)
if self.aspp is not None:
x = self.aspp(x)
x = self.proj(x).flatten(2).transpose(1, 2).contiguous()
return self.out_norm(x)
class AnimaLLLiteModule(nn.Module):
def __init__(self, in_dim, cond_emb_dim, mlp_dim, device=None, dtype=None, operations=None):
super().__init__()
self.down = operations.Linear(in_dim, mlp_dim, device=device, dtype=dtype)
self.mid = operations.Linear(mlp_dim + cond_emb_dim, mlp_dim, device=device, dtype=dtype)
self.cond_to_film = operations.Linear(cond_emb_dim, 2 * mlp_dim, device=device, dtype=dtype)
self.up = operations.Linear(mlp_dim, in_dim, device=device, dtype=dtype)
self.depth_embed = nn.Parameter(torch.empty(cond_emb_dim, device=device, dtype=dtype), requires_grad=False)
def forward(self, x, cond_emb, strength):
original_shape = x.shape
if x.ndim == 5:
x = x.flatten(1, 3)
if x.shape[0] != cond_emb.shape[0]:
if x.shape[0] % cond_emb.shape[0] != 0:
raise ValueError(f"Anima LLLite batch mismatch: model input batch {x.shape[0]}, control batch {cond_emb.shape[0]}")
cond_emb = cond_emb.repeat(x.shape[0] // cond_emb.shape[0], 1, 1)
if x.shape[1] != cond_emb.shape[1]:
raise ValueError(f"Anima LLLite sequence mismatch: model input has {x.shape[1]} tokens, control has {cond_emb.shape[1]}")
cond_local = cond_emb + comfy.ops.cast_to_input(self.depth_embed, cond_emb)
hidden = F.silu(self.down(x))
gamma, beta = self.cond_to_film(cond_local).chunk(2, dim=-1)
hidden = self.mid(torch.cat((cond_local, hidden), dim=-1))
hidden = F.silu(hidden * (1 + gamma) + beta)
x = x + self.up(hidden) * strength
if len(original_shape) == 5:
x = x.reshape(original_shape)
return x
class AnimaLLLite(nn.Module):
def __init__(self, state_dict, metadata, device=None, dtype=None, operations=None):
super().__init__()
metadata = metadata or {}
version = metadata.get("lllite.version", "2")
if version != "2":
raise ValueError(f"Unsupported Anima LLLite version {version!r}; only named-key v2 checkpoints are supported")
module_names = sorted({key.split(".", 1)[0] for key in state_dict if key.startswith("lllite_dit_blocks_")})
if not module_names:
raise ValueError("Anima LLLite checkpoint has no lllite_dit_blocks_* modules")
cond_in_channels = state_dict["lllite_conditioning1.conv1.weight"].shape[1]
cond_dim = state_dict["lllite_conditioning1.conv3.weight"].shape[0]
cond_emb_dim = state_dict["lllite_conditioning1.proj.weight"].shape[0]
resblock_ids = {int(key.split(".")[2]) for key in state_dict if key.startswith("lllite_conditioning1.resblocks.")}
cond_resblocks = max(resblock_ids) + 1 if resblock_ids else 0
use_aspp = any(key.startswith("lllite_conditioning1.aspp.") for key in state_dict)
dilation_string = metadata.get("lllite.aspp_dilations", "1,2,4,8")
aspp_dilations = tuple(int(value) for value in dilation_string.split(",") if value.strip()) if use_aspp else ()
self.cond_in_channels = cond_in_channels
self.inpaint_masked_input = metadata.get("lllite.inpaint_masked_input", "false").lower() == "true"
self.lllite_conditioning1 = AnimaLLLiteConditioning(
cond_in_channels, cond_dim, cond_emb_dim, cond_resblocks, aspp_dilations,
device=device, dtype=dtype, operations=operations,
)
self.module_names = set()
self.block_count = 0
self.model_dim = None
for name in module_names:
match = MODULE_PATTERN.fullmatch(name)
if match is None:
raise ValueError(f"Unsupported Anima LLLite module name: {name}")
down_shape = state_dict[f"{name}.down.weight"].shape
mlp_dim, in_dim = down_shape
module_cond_dim = state_dict[f"{name}.cond_to_film.weight"].shape[1]
if module_cond_dim != cond_emb_dim:
raise ValueError(f"Anima LLLite conditioning dimension mismatch in {name}: {module_cond_dim} != {cond_emb_dim}")
if self.model_dim is None:
self.model_dim = in_dim
elif self.model_dim != in_dim:
raise ValueError(f"Anima LLLite model dimension mismatch in {name}: {in_dim} != {self.model_dim}")
self.add_module(name, AnimaLLLiteModule(in_dim, cond_emb_dim, mlp_dim, device=device, dtype=dtype, operations=operations))
self.module_names.add(name)
self.block_count = max(self.block_count, int(match.group(1)) + 1)
def encode_conditioning(self, image):
return self.lllite_conditioning1(image)
def apply(self, x, cond_emb, block_index, target, strength):
name = f"lllite_dit_blocks_{block_index}_{target}"
if name not in self.module_names:
return x
return self.get_submodule(name)(x, cond_emb, strength)
class AnimaLLLitePatch:
def __init__(self, model_patch, image, mask, strength, sigma_start, sigma_end):
self.model_patch = model_patch
self.image = image
self.mask = mask
self.strength = strength
self.sigma_start = sigma_start
self.sigma_end = sigma_end
def __call__(self, args):
x = args["x"]
transformer_options = args["transformer_options"]
if self.strength == 0.0:
return args
sigmas = transformer_options.get("sigmas")
if sigmas is not None:
sigma = float(sigmas.max().item())
if not self.sigma_end <= sigma <= self.sigma_start:
return args
if x.shape[2] != 1:
raise ValueError(f"Anima LLLite only supports T=1, got T={x.shape[2]}")
target_height = x.shape[-2] * 8
target_width = x.shape[-1] * 8
image = comfy.utils.common_upscale(
self.image.movedim(-1, 1), target_width, target_height, "bicubic", crop="center"
).clamp(0.0, 1.0)
image = image.to(device=x.device, dtype=x.dtype) * 2.0 - 1.0
if self.model_patch.model.cond_in_channels == 4:
mask = self.mask
if mask.ndim == 3:
mask = mask.unsqueeze(1)
if mask.ndim != 4 or mask.shape[1] != 1:
raise ValueError(f"Anima LLLite mask must have one channel, got shape {tuple(mask.shape)}")
mask = comfy.utils.common_upscale(
mask.float(), target_width, target_height, "nearest-exact", crop="center"
)
if mask.shape[0] != image.shape[0]:
if image.shape[0] % mask.shape[0] != 0:
raise ValueError(
f"Anima LLLite mask batch {mask.shape[0]} cannot be broadcast to image batch {image.shape[0]}"
)
mask = mask.repeat(image.shape[0] // mask.shape[0], 1, 1, 1)
mask = (mask >= 0.5).to(device=x.device, dtype=x.dtype)
if self.model_patch.model.inpaint_masked_input:
image = image * (mask < 0.5).to(image.dtype)
image = torch.cat((image, mask * 2.0 - 1.0), dim=1)
cond_emb = self.model_patch.model.encode_conditioning(image)
transformer_options["model_patch_data"][self] = cond_emb
return args
def to(self, device_or_dtype):
return self
def models(self):
return [self.model_patch]
class AnimaLLLiteAttentionPatch:
def __init__(self, patch, targets):
self.patch = patch
self.targets = targets
def __call__(self, q, k, v, pe=None, attn_mask=None, extra_options=None):
cond_emb = extra_options["model_patch_data"].get(self.patch)
if cond_emb is None:
return {"q": q, "k": k, "v": v, "pe": pe, "attn_mask": attn_mask}
block_index = extra_options["block_index"]
values = {"q": q, "k": k, "v": v}
for value_name, target in self.targets.items():
values[value_name] = self.patch.model_patch.model.apply(
values[value_name], cond_emb, block_index, target, self.patch.strength
)
return {"q": values["q"], "k": values["k"], "v": values["v"], "pe": pe, "attn_mask": attn_mask}
class AnimaLLLiteMLPPatch:
def __init__(self, patch):
self.patch = patch
def __call__(self, args):
cond_emb = args["transformer_options"]["model_patch_data"].get(self.patch)
if cond_emb is None:
return args
args["x"] = self.patch.model_patch.model.apply(
args["x"], cond_emb, args["transformer_options"]["block_index"], "mlp_layer1", self.patch.strength
)
return args

64
comfy/ldm/cosmos/predict2.py
View File

@ -14,6 +14,7 @@ from torchvision import transforms
import comfy.patcher_extension
from comfy.ldm.modules.attention import optimized_attention
import comfy.ldm.common_dit
import comfy.ops
import comfy.quant_ops
@ -148,11 +149,29 @@ class Attention(nn.Module):
x: torch.Tensor,
context: Optional[torch.Tensor] = None,
rope_emb: Optional[torch.Tensor] = None,
transformer_options: Optional[dict] = {},
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
q = self.q_proj(x)
context = x if context is None else context
k = self.k_proj(context)
v = self.v_proj(context)
q_input = x
k_input = context
v_input = context
transformer_patches = transformer_options.get("patches", {})
patch_name = "attn1_patch" if self.is_selfattn else "attn2_patch"
if patch_name in transformer_patches:
extra_options = transformer_options.copy()
extra_options["n_heads"] = self.n_heads
extra_options["dim_head"] = self.head_dim
for patch in transformer_patches[patch_name]:
out = patch(q_input, k_input, v_input, pe=rope_emb, attn_mask=None, extra_options=extra_options)
q_input = out.get("q", q_input)
k_input = out.get("k", k_input)
v_input = out.get("v", v_input)
rope_emb = out.get("pe", rope_emb)
q = self.q_proj(q_input)
k = self.k_proj(k_input)
v = self.v_proj(v_input)
q, k, v = map(
lambda t: rearrange(t, "b ... (h d) -> b ... h d", h=self.n_heads, d=self.head_dim),
(q, k, v),
@ -161,11 +180,16 @@ class Attention(nn.Module):
def apply_norm_and_rotary_pos_emb(
q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, rope_emb: Optional[torch.Tensor]
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
q = self.q_norm(q)
k = self.k_norm(k)
v = self.v_norm(v)
if self.is_selfattn and rope_emb is not None: # only apply to self-attention!
q, k = comfy.quant_ops.ck.apply_rope_split_half(q, k, rope_emb)
q_scale, _, q_offload_stream = comfy.ops.cast_bias_weight(self.q_norm, q, offloadable=True)
k_scale, _, k_offload_stream = comfy.ops.cast_bias_weight(self.k_norm, k, offloadable=True)
q, k = comfy.quant_ops.ck.rms_rope_split_half(q, k, rope_emb, q_scale, k_scale, self.q_norm.eps)
comfy.ops.uncast_bias_weight(self.q_norm, q_scale, None, q_offload_stream)
comfy.ops.uncast_bias_weight(self.k_norm, k_scale, None, k_offload_stream)
else:
q = self.q_norm(q)
k = self.k_norm(k)
return q, k, v
q, k, v = apply_norm_and_rotary_pos_emb(q, k, v, rope_emb)
@ -188,7 +212,7 @@ class Attention(nn.Module):
x (Tensor): The query tensor of shape [B, Mq, K]
context (Optional[Tensor]): The key tensor of shape [B, Mk, K] or use x as context [self attention] if None
"""
q, k, v = self.compute_qkv(x, context, rope_emb=rope_emb)
q, k, v = self.compute_qkv(x, context, rope_emb=rope_emb, transformer_options=transformer_options)
return self.compute_attention(q, k, v, transformer_options=transformer_options)
@ -555,8 +579,14 @@ class Block(nn.Module):
self.layer_norm_mlp,
scale_mlp_B_T_1_1_D,
shift_mlp_B_T_1_1_D,
)
result_B_T_H_W_D = self.mlp(normalized_x_B_T_H_W_D.to(compute_dtype))
).to(compute_dtype)
patches = transformer_options.get("patches", {})
if "mlp_patch" in patches:
args = {"x": normalized_x_B_T_H_W_D, "transformer_options": transformer_options}
for patch in patches["mlp_patch"]:
args = patch(args)
normalized_x_B_T_H_W_D = args["x"]
result_B_T_H_W_D = self.mlp(normalized_x_B_T_H_W_D)
x_B_T_H_W_D = torch.addcmul(x_B_T_H_W_D, gate_mlp_B_T_1_1_D.to(residual_dtype), result_B_T_H_W_D.to(residual_dtype))
return x_B_T_H_W_D
@ -863,11 +893,22 @@ class MiniTrainDIT(nn.Module):
x_B_T_H_W_D.shape == extra_pos_emb_B_T_H_W_D_or_T_H_W_B_D.shape
), f"{x_B_T_H_W_D.shape} != {extra_pos_emb_B_T_H_W_D_or_T_H_W_B_D.shape}"
transformer_options = kwargs.get("transformer_options", {})
patches = transformer_options.get("patches", {})
if "post_input" in patches:
transformer_options = transformer_options.copy()
transformer_options["model_patch_data"] = {}
if "post_input" in patches:
for patch in patches["post_input"]:
out = patch({"img": x_B_T_H_W_D, "x": x_B_C_T_H_W, "transformer_options": transformer_options})
x_B_T_H_W_D = out["img"]
block_kwargs = {
"rope_emb_L_1_1_D": rope_emb_L_1_1_D.unsqueeze(1).unsqueeze(0),
"adaln_lora_B_T_3D": adaln_lora_B_T_3D,
"extra_per_block_pos_emb": extra_pos_emb_B_T_H_W_D_or_T_H_W_B_D,
"transformer_options": kwargs.get("transformer_options", {}),
"transformer_options": transformer_options,
}
# The residual stream for this model has large values. To make fp16 compute_dtype work, we keep the residual stream
@ -877,7 +918,8 @@ class MiniTrainDIT(nn.Module):
if x_B_T_H_W_D.dtype == torch.float16:
x_B_T_H_W_D = x_B_T_H_W_D.float()
for block in self.blocks:
for block_index, block in enumerate(self.blocks):
transformer_options["block_index"] = block_index
x_B_T_H_W_D = block(
x_B_T_H_W_D,
t_embedding_B_T_D,

147
comfy/ldm/krea2/model.py
View File

@ -15,6 +15,7 @@ from einops import rearrange
import comfy.model_management
import comfy.patcher_extension
import comfy.ldm.common_dit
import comfy.utils
from comfy.ldm.flux.layers import EmbedND, timestep_embedding
from comfy.ldm.flux.math import apply_rope
from comfy.ldm.modules.attention import optimized_attention_masked
@ -73,11 +74,20 @@ class Attention(nn.Module):
self.wo = operations.Linear(dim, dim, bias=bias, device=device, dtype=dtype)
def forward(self, x, freqs=None, mask=None, transformer_options={}):
transformer_patches = transformer_options.get("patches", {})
extra_options = transformer_options.copy()
q, k, v, gate = self.wq(x), self.wk(x), self.wv(x), self.gate(x)
q = rearrange(q, "B L (H D) -> B H L D", H=self.heads)
k = rearrange(k, "B L (H D) -> B H L D", H=self.kvheads)
v = rearrange(v, "B L (H D) -> B H L D", H=self.kvheads)
q, k = self.qknorm(q, k)
if "block_index" in transformer_options and "attn1_patch" in transformer_patches:
for p in transformer_patches["attn1_patch"]:
out = p(q, k, v, pe=freqs, attn_mask=mask, extra_options=extra_options)
q, k, v = out.get("q", q), out.get("k", k), out.get("v", v)
freqs, mask = out.get("pe", freqs), out.get("attn_mask", mask)
if freqs is not None:
q, k = apply_rope(q, k, freqs)
if self.kvheads != self.heads:
@ -86,6 +96,11 @@ class Attention(nn.Module):
v = v.repeat_interleave(rep, dim=1)
out = optimized_attention_masked(q, k, v, self.heads, mask=mask, skip_reshape=True,
transformer_options=transformer_options)
if "block_index" in transformer_options and "attn1_output_patch" in transformer_patches:
for p in transformer_patches["attn1_output_patch"]:
out = p(out, extra_options)
return self.wo(out * F.sigmoid(gate))
@ -158,8 +173,44 @@ class SingleStreamBlock(nn.Module):
self.attn = Attention(features, heads, kvheads=kvheads, bias=bias, device=device, dtype=dtype, operations=operations)
self.mlp = SwiGLU(features, multiplier, bias, device=device, dtype=dtype, operations=operations)
def forward(self, x, vec, freqs, mask=None, transformer_options={}):
def forward(self, x, vec, freqs, mask=None, timestep_zero_index=None, transformer_options={}):
prescale, preshift, pregate, postscale, postshift, postgate = self.mod(vec)
if timestep_zero_index is not None:
bs = x.shape[0]
ref_prescale = prescale[bs:]
ref_preshift = preshift[bs:]
ref_pregate = pregate[bs:]
ref_postscale = postscale[bs:]
ref_postshift = postshift[bs:]
ref_postgate = postgate[bs:]
prescale = prescale[:bs]
preshift = preshift[:bs]
pregate = pregate[:bs]
postscale = postscale[:bs]
postshift = postshift[:bs]
postgate = postgate[:bs]
pre = self.prenorm(x)
pre[:, :timestep_zero_index].mul_(1 + prescale).add_(preshift)
pre[:, timestep_zero_index:].mul_(1 + ref_prescale).add_(ref_preshift)
attn = self.attn(pre, freqs, mask, transformer_options=transformer_options)
del pre
attn[:, :timestep_zero_index].mul_(pregate)
attn[:, timestep_zero_index:].mul_(ref_pregate)
x = x + attn
del attn
post = self.postnorm(x)
post[:, :timestep_zero_index].mul_(1 + postscale).add_(postshift)
post[:, timestep_zero_index:].mul_(1 + ref_postscale).add_(ref_postshift)
mlp = self.mlp(post)
del post
mlp[:, :timestep_zero_index].mul_(postgate)
mlp[:, timestep_zero_index:].mul_(ref_postgate)
x = x + mlp
del mlp
return x
x = x + pregate * self.attn((1 + prescale) * self.prenorm(x) + preshift, freqs, mask, transformer_options=transformer_options)
x = x + postgate * self.mlp((1 + postscale) * self.postnorm(x) + postshift)
return x
@ -181,7 +232,7 @@ class LastLayer(nn.Module):
class SingleStreamDiT(nn.Module):
def __init__(self, features=6144, tdim=256, txtdim=2560, heads=48, kvheads=12, multiplier=4,
layers=28, patch=2, channels=16, bias=False, theta=1e3, txtlayers=12,
txtheads=20, txtkvheads=20, image_model=None,
txtheads=20, txtkvheads=20, default_ref_method=None, image_model=None,
device=None, dtype=None, operations=None, **kwargs):
super().__init__()
self.dtype = dtype
@ -191,6 +242,7 @@ class SingleStreamDiT(nn.Module):
self.heads = heads
self.txtdim = txtdim
self.txtlayers = txtlayers
self.default_ref_method = default_ref_method
headdim = features // heads
axes = [headdim - 12 * (headdim // 16), 6 * (headdim // 16), 6 * (headdim // 16)]
@ -221,61 +273,110 @@ class SingleStreamDiT(nn.Module):
operations.Linear(features, features * 6, device=device, dtype=dtype),
)
def forward(self, x, timesteps, context, attention_mask=None, transformer_options={}, **kwargs):
def forward(self, x, timesteps, context, attention_mask=None, ref_latents=None, transformer_options={}, **kwargs):
return comfy.patcher_extension.WrapperExecutor.new_class_executor(
self._forward,
self,
comfy.patcher_extension.get_all_wrappers(comfy.patcher_extension.WrappersMP.DIFFUSION_MODEL, transformer_options),
).execute(x, timesteps, context, attention_mask, transformer_options, **kwargs)
).execute(x, timesteps, context, attention_mask, ref_latents, transformer_options, **kwargs)
def _forward(self, x, timesteps, context, attention_mask=None, transformer_options={}, **kwargs):
def process_img(self, x, index=0):
patch = self.patch
x = comfy.ldm.common_dit.pad_to_patch_size(x, (patch, patch))
h, w = x.shape[-2] // patch, x.shape[-1] // patch
img = rearrange(x, "b c (h ph) (w pw) -> b (h w) (c ph pw)", ph=patch, pw=patch)
img_ids = torch.zeros(h, w, 3, device=x.device, dtype=torch.float32)
img_ids[..., 0] = index
img_ids[..., 1] = torch.arange(h, device=x.device, dtype=torch.float32)[:, None]
img_ids[..., 2] = torch.arange(w, device=x.device, dtype=torch.float32)[None, :]
return img, img_ids.reshape(1, h * w, 3).repeat(x.shape[0], 1, 1), h, w
def _forward(self, x, timesteps, context, attention_mask=None, ref_latents=None, transformer_options={}, **kwargs):
transformer_options = transformer_options.copy()
temporal = x.ndim == 5
if temporal:
b5, c5, t5, h5, w5 = x.shape
x = x.reshape(b5 * t5, c5, h5, w5)
bs, c, H_orig, W_orig = x.shape
bs, _, h_orig, w_orig = x.shape
patch = self.patch
# Pad the latent up to a multiple of patch (as Flux/Lumina/QwenImage do); crop back at the end.
x = comfy.ldm.common_dit.pad_to_patch_size(x, (patch, patch))
H, W = x.shape[-2], x.shape[-1]
h_, w_ = H // patch, W // patch
# context arrives as (B, seq, txtlayers*txtdim); reshape to (B, txtlayers, seq, txtdim).
context = self._unpack_context(context)
img = rearrange(x, "b c (h ph) (w pw) -> b (h w) (c ph pw)", ph=patch, pw=patch)
img, imgpos, h_, w_ = self.process_img(x)
img_tokens = img.shape[1]
timestep_zero_index = None
ref_method = kwargs.get("ref_latents_method", self.default_ref_method)
if ref_method is not None and ref_latents is not None and len(ref_latents) > 0:
ref_tokens = []
ref_pos = []
ref_num_tokens = []
for index, ref in enumerate(ref_latents, 1):
if ref.ndim == 5:
rb, rc, rt, rh5, rw5 = ref.shape
ref = ref.reshape(rb * rt, rc, rh5, rw5)
ref = comfy.utils.repeat_to_batch_size(ref, bs)
kontext, kontext_ids, _, _ = self.process_img(ref, index=index)
ref_tokens.append(kontext)
ref_pos.append(kontext_ids)
ref_num_tokens.append(kontext.shape[1])
img = torch.cat([img] + ref_tokens, dim=1)
imgpos = torch.cat([imgpos] + ref_pos, dim=1)
del ref_tokens, ref_pos
if ref_method == "index_timestep_zero":
timestep_zero_index = img_tokens
transformer_options["reference_image_num_tokens"] = ref_num_tokens
img = self.first(img)
t = self.tmlp(timestep_embedding(timesteps, self.tdim).unsqueeze(1).to(img.dtype))
tvec = self.tproj(t)
if timestep_zero_index is not None:
t0 = self.tmlp(timestep_embedding(torch.zeros_like(timesteps), self.tdim).unsqueeze(1).to(img.dtype))
tvec = torch.cat((tvec, self.tproj(t0)), dim=0)
context = self.txtfusion(context, mask=None, transformer_options=transformer_options)
context = self.txtmlp(context)
txtlen, imglen = context.shape[1], img.shape[1]
txtlen = context.shape[1]
device = context.device
txtpos = torch.zeros(bs, txtlen, 3, device=device, dtype=torch.float32)
patches = transformer_options.get("patches", {})
if "post_input" in patches:
for p in patches["post_input"]:
out = p({"img": img, "txt": context, "img_ids": imgpos, "txt_ids": txtpos, "transformer_options": transformer_options})
img, context = out["img"], out["txt"]
imgpos, txtpos = out["img_ids"], out["txt_ids"]
combined = torch.cat((context, img), dim=1)
del context, img
if timestep_zero_index is not None:
timestep_zero_index += txtlen
# Position ids: text at 0, image at (0, h_idx, w_idx).
device = combined.device
txtpos = torch.zeros(bs, txtlen, 3, device=device, dtype=torch.float32)
imgids = torch.zeros(h_, w_, 3, device=device, dtype=torch.float32)
imgids[..., 1] = torch.arange(h_, device=device, dtype=torch.float32)[:, None]
imgids[..., 2] = torch.arange(w_, device=device, dtype=torch.float32)[None, :]
imgpos = imgids.reshape(1, h_ * w_, 3).repeat(bs, 1, 1)
pos = torch.cat((txtpos, imgpos), dim=1)
del txtpos, imgpos
freqs = self.pe_embedder(pos)
del pos
for block in self.blocks:
combined = block(combined, tvec, freqs, None, transformer_options=transformer_options)
transformer_options["total_blocks"] = len(self.blocks)
transformer_options["block_type"] = "single"
transformer_options["img_slice"] = [txtlen, combined.shape[1]]
for i, block in enumerate(self.blocks):
transformer_options["block_index"] = i
combined = block(combined, tvec, freqs, None, timestep_zero_index=timestep_zero_index, transformer_options=transformer_options)
final = self.last(combined, t)
out = final[:, txtlen:txtlen + imglen, :]
del combined
out = final[:, txtlen:txtlen + img_tokens, :]
out = rearrange(out, "b (h w) (c ph pw) -> b c (h ph) (w pw)",
h=h_, w=w_, ph=patch, pw=patch, c=self.channels)
out = out[:, :, :H_orig, :W_orig] # crop padding back off
out = out[:, :, :h_orig, :w_orig] # crop padding back off
if temporal:
out = out.reshape(b5, t5, self.channels, H_orig, W_orig).movedim(1, 2)
out = out.reshape(b5, t5, self.channels, h_orig, w_orig).movedim(1, 2)
return out
def _unpack_context(self, context):

23
comfy/model_base.py
View File

@ -2227,10 +2227,7 @@ class Omnigen2(BaseModel):
out['c_crossattn'] = comfy.conds.CONDRegular(cross_attn)
ref_latents = kwargs.get("reference_latents", None)
if ref_latents is not None:
latents = []
for lat in ref_latents:
latents.append(self.process_latent_in(lat))
out['ref_latents'] = comfy.conds.CONDList(latents)
out['ref_latents'] = comfy.conds.CONDList([self.process_latent_in(lat) for lat in ref_latents])
return out
def extra_conds_shapes(self, **kwargs):
@ -2317,12 +2314,30 @@ class Ideogram4(BaseModel):
class Krea2(BaseModel):
def __init__(self, model_config, model_type=ModelType.FLUX, device=None):
super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.krea2.model.SingleStreamDiT)
self.memory_usage_factor_conds = ("ref_latents",)
def extra_conds(self, **kwargs):
out = super().extra_conds(**kwargs)
cross_attn = kwargs.get("cross_attn", None)
if cross_attn is not None:
out['c_crossattn'] = comfy.conds.CONDRegular(cross_attn)
ref_latents = kwargs.get("reference_latents", None)
if ref_latents is not None:
latents = []
for lat in ref_latents:
latents.append(self.process_latent_in(lat))
out['ref_latents'] = comfy.conds.CONDList(latents)
ref_latents_method = kwargs.get("reference_latents_method", None)
if ref_latents_method is not None:
out['ref_latents_method'] = comfy.conds.CONDConstant(ref_latents_method)
return out
def extra_conds_shapes(self, **kwargs):
out = {}
ref_latents = kwargs.get("reference_latents", None)
if ref_latents is not None:
out['ref_latents'] = list([1, 16, sum(map(lambda a: math.prod(a.size()), ref_latents)) // 16])
return out
class HunyuanImage21(BaseModel):

59
comfy_api_nodes/apis/gemini.py
View File

@ -1,6 +1,6 @@
from datetime import date
from enum import Enum
from typing import Any
from typing import Any, Literal
from pydantic import BaseModel, Field
@ -242,3 +242,60 @@ class GeminiGenerateContentResponse(BaseModel):
promptFeedback: GeminiPromptFeedback | None = Field(None)
usageMetadata: GeminiUsageMetadata | None = Field(None)
modelVersion: str | None = Field(None)
class GeminiInteractionTextPart(BaseModel):
type: Literal["text"] = "text"
text: str = Field(...)
class GeminiInteractionMediaPart(BaseModel):
type: str = Field(..., description="One of: image, video, audio, document.")
data: str | None = Field(None, description="Base64-encoded media bytes.")
uri: str | None = Field(None, description="URI of the media, as an alternative to inline data.")
mime_type: str | None = Field(None)
class GeminiInteractionGenerationConfig(BaseModel):
temperature: float | None = Field(None, ge=0.0, le=2.0)
top_p: float | None = Field(None, ge=0.0, le=1.0)
class GeminiInteractionRequest(BaseModel):
model: str = Field(...)
input: list[GeminiInteractionTextPart | GeminiInteractionMediaPart] = Field(...)
generation_config: GeminiInteractionGenerationConfig | None = Field(None)
class GeminiInteractionModalityTokens(BaseModel):
modality: str | None = Field(None, description="One of: text, image, audio, video, document.")
tokens: int | None = Field(None)
class GeminiInteractionUsage(BaseModel):
input_tokens_by_modality: list[GeminiInteractionModalityTokens] | None = Field(None)
output_tokens_by_modality: list[GeminiInteractionModalityTokens] | None = Field(None)
total_thought_tokens: int | None = Field(None)
class GeminiInteractionContent(BaseModel):
type: str | None = Field(None)
text: str | None = Field(None)
data: str | None = Field(None)
uri: str | None = Field(None)
mime_type: str | None = Field(None)
class GeminiInteractionStep(BaseModel):
type: str | None = Field(None)
content: list[GeminiInteractionContent] | None = Field(None)
class GeminiInteraction(BaseModel):
id: str | None = Field(None)
status: str | None = Field(
None,
description="One of: in_progress, requires_action, completed, failed, cancelled, incomplete.",
)
steps: list[GeminiInteractionStep] | None = Field(None)
usage: GeminiInteractionUsage | None = Field(None)

452
comfy_api_nodes/apis/heygen.py Normal file
View File

@ -0,0 +1,452 @@
# (label, avatar_id, avatar_type, supported engines)
HEYGEN_AVATAR_LOOKS: list[tuple[str, str, str, tuple[str, ...]]] = [
(
"Annie Lounge Standing Side",
"Annie_Lounge_Standing_Side_public",
"studio_avatar",
("avatar_v", "avatar_iv", "avatar_iii"),
),
(
"Yara Modern Lecture Hall",
"fd6814ecc5e143cd899e615a80eaa2dc",
"photo_avatar",
("avatar_v", "avatar_iv", "avatar_iii"),
),
(
"Brandon Business Sitting Front",
"Brandon_Business_Sitting_Front_public",
"studio_avatar",
("avatar_v", "avatar_iv", "avatar_iii"),
),
(
"Caroline Business Sitting Side",
"Caroline_Business_Sitting_Side_public",
"studio_avatar",
("avatar_v", "avatar_iv", "avatar_iii"),
),
(
"Ursula Lawyer Angle 4",
"f7173d2bb8584c00bfec6905c5e9a492",
"photo_avatar",
("avatar_v", "avatar_iv", "avatar_iii"),
),
(
"Sofia Corporate Presenter 01 Angle 3",
"fe563971fd2d438e957372dac9e2be8c",
"photo_avatar",
("avatar_v", "avatar_iv", "avatar_iii"),
),
(
"Seoyeon Health Nutrition Coach Angle 3",
"fe3c5d5028d941398d064b8fc64a2dea",
"photo_avatar",
("avatar_v", "avatar_iv", "avatar_iii"),
),
(
"Sanne Fitness Coach Angle 4",
"d967f935a8bf4a0c8f0bccfd66c501d2",
"photo_avatar",
("avatar_v", "avatar_iv", "avatar_iii"),
),
("Sander", "f5cd7b94056f495ca0610602d64a9aa3", "photo_avatar", ("avatar_v", "avatar_iv", "avatar_iii")),
(
"Rupert Personal Development Coach Angle 4",
"f57b3e626adb4bc997b38f64884adce4",
"photo_avatar",
("avatar_v", "avatar_iv", "avatar_iii"),
),
(
"Olivier Professor Angle 2",
"f6659bbb094b459c87c967edbb9ee481",
"photo_avatar",
("avatar_v", "avatar_iv", "avatar_iii"),
),
(
"Obi Health Nutrition Coach Angle 5",
"f3dc2c38201d414382f506d2d8e8d029",
"photo_avatar",
("avatar_v", "avatar_iv", "avatar_iii"),
),
(
"Matilda Modern Office Setting",
"fda889ac354a440da8dbecc410981273",
"photo_avatar",
("avatar_v", "avatar_iv", "avatar_iii"),
),
(
"Mateo Traditional Law Office",
"ff172d6c499c4e47ba6fcc5de631e9fc",
"photo_avatar",
("avatar_v", "avatar_iv", "avatar_iii"),
),
(
"Marlon Inviting Armchair Setting",
"f5a57db099ab462daa3e7c604a05dacc",
"photo_avatar",
("avatar_v", "avatar_iv", "avatar_iii"),
),
(
"Margaret Professor Angle 1",
"fb472bc29ab04bcca576e3703978fecb",
"photo_avatar",
("avatar_v", "avatar_iv", "avatar_iii"),
),
(
"Marek Therapy Coach Angle 3",
"e197768703f1463a93dc25ada1f421fb",
"photo_avatar",
("avatar_v", "avatar_iv", "avatar_iii"),
),
(
"Maeve Warm, Professional Setting",
"faf66681d8cc48dc82c4283200b3e782",
"photo_avatar",
("avatar_v", "avatar_iv", "avatar_iii"),
),
(
"Lorenzo Professor Angle 5",
"fc268dc244bb40d7a554663ce723dcf0",
"photo_avatar",
("avatar_v", "avatar_iv", "avatar_iii"),
),
("Luca", "Luca_public", "studio_avatar", ("avatar_iii",)),
("Bruce", "Bruce_public", "studio_avatar", ("avatar_iii",)),
("Nico", "Nico_public", "studio_avatar", ("avatar_iii",)),
("Lisa", "Lisa_public", "studio_avatar", ("avatar_iii",)),
("Sophie", "Sophie_public", "studio_avatar", ("avatar_iii",)),
("Aiko", "Aiko_public", "studio_avatar", ("avatar_iii",)),
("Rebecca (portrait)", "Rebecca_public", "studio_avatar", ("avatar_iii",)),
("Daphne in Grey blazer (portrait)", "Daphne_public_1", "studio_avatar", ("avatar_iii",)),
("Bryce in Black t-shirt", "Bryce_public_5", "studio_avatar", ("avatar_iii",)),
("Diora in White shirt", "Diora_public_3", "studio_avatar", ("avatar_iii",)),
("Freja in White blazer", "Freja_public_1", "studio_avatar", ("avatar_iii",)),
("Albert in Blue blazer", "Albert_public_2", "studio_avatar", ("avatar_iii",)),
("Emery in Red blazer", "Emery_public_1", "studio_avatar", ("avatar_iii",)),
("Minho in Blue shirt", "Minho_public_6", "studio_avatar", ("avatar_iii",)),
("Aditya in Brown blazer", "Aditya_public_4", "studio_avatar", ("avatar_iii",)),
("Nadim in Blue blazer", "Nadim_public_1", "studio_avatar", ("avatar_iii",)),
("Iker in Black blazer", "Iker_public_1", "studio_avatar", ("avatar_iii",)),
("Nour in Black blazer", "Nour_public_1", "studio_avatar", ("avatar_iii",)),
("Saskia in Blue blazer", "Saskia_public_1", "studio_avatar", ("avatar_iii",)),
("Lucien in Blue blazer", "Lucien_public_1", "studio_avatar", ("avatar_iii",)),
("Esmond in Blue suit", "Esmond_public_3", "studio_avatar", ("avatar_iii",)),
("Jinwoo in Blue suit", "Jinwoo_public_5", "studio_avatar", ("avatar_iii",)),
("Annelore in Red sweater (portrait)", "Annelore_public_3", "studio_avatar", ("avatar_iii",)),
("Bastien in Blue shirt", "Bastien_public_4", "studio_avatar", ("avatar_iii",)),
("Zosia in Khaki blazer", "Zosia_public_3", "studio_avatar", ("avatar_iii",)),
("Tahlia in Dark blue suit", "Tahlia_public_4", "studio_avatar", ("avatar_iii",)),
]
HEYGEN_AVATAR_OPTIONS = [x[0] for x in HEYGEN_AVATAR_LOOKS]
HEYGEN_AVATAR_MAP = {x[0]: (x[1], x[2], x[3]) for x in HEYGEN_AVATAR_LOOKS}
# (label, voice_id) — Starfish-compatible voices for the TTS endpoint
HEYGEN_VOICE_TTS: list[tuple[str, str]] = [
("Chill Brian (English, male)", "d2f4f24783d04e22ab49ee8fdc3715e0"),
("Zain (English, female)", "0047732240584155b1588455313e78ec"),
("Narrator Mateo - Excited 🤩 (Spanish, male)", "0077225a877e457db4572ccaf245910b"),
("Aria (English, female)", "007e1378fc454a9f976db570ba6164a7"),
("Caryns (English, female)", "0082e70326864107823605db0d77f5e0"),
("Klara (English, female)", "01209fdcd1c24a109c86dc24ee0f71c0"),
("Bold Kasia - Excited 🤩 (Polish, female)", "015482a78b9a46ebae74bd0beb17765b"),
("Shaun (English, male)", "01c42cddcfdc4665a57b8d89cba8ffc1"),
("Senthil (English, male)", "01d674cfd32b4728a3fddd21b7e7d543"),
("Cody (English, male)", "01f98ed43e6140349f47dbd37a416827"),
("Saffron (English, female)", "0258bbc2cd8648cfa357adfb833f6d7b"),
("Blanka - Lifelike (English, female)", "02880d1c6fd94b7799d91135581ed810"),
("Rami (English, male)", "02d5366a90af4c7a87157808ff352e33"),
("Rhodes (English, female)", "02dce0a169b3460084b6c914d18fb2c8"),
("Michelle - Voice 1 (English, female)", "02X8sHnuxFpsq1caYWN0"),
("Autumn - UGC 3 (English, female)", "03dca9ebfca441dba55fb14afa0791b7"),
("Reassuring Rupert (English, male)", "03fcf8ecb0a94b6b94e9007edb7c35f8"),
("Rose - UGC -2 (English, female)", "0495e14c2bd74eb3aeeef03583e0bce5"),
("Derya - Lifelike - Broadcaster 🎙️ (English, female)", "04d0ae1d0af2489ca7d3bb402a39a890"),
("Dynamic Derek (English, male)", "0516c2d857eb425c94e90b068241914e"),
("Lotte (English, female)", "052fcfb83d1a4c2f8d0368c226fea4b9"),
("Thanos - Broadcaster 🎙️ (English, male)", "054af44a167344d0af2722fdfef08d17"),
("Marcia (English, female)", "05f19352e8f74b0392a8f411eba40de1"),
("Camden (English, male)", "06468055edd4458aa131a1dfd813c1e9"),
("Rumi (English, female)", "06672207805f41a9ad0af6797f8aa14b"),
("Pippa (English, female)", "06b68c4dbb544935b9af984e80efa4fb"),
("William Prescott - Broadcaster 🎙️ (English, male)", "06c816b952f14fa9b3a6c42aa151f731"),
("Sammy (English, female)", "06e6facd99654b9dbb9308f67bf3a31c"),
("Breezy Bagus (Indonesian, male)", "06e81a5d7c8b41818d3f0b38f7cf15a1"),
("Ben (English, male)", "07ca39b243184dbcb82e7e0f0e524b21"),
("Smooth Dev (English, male)", "07d2ba65847541feb97abc9b60181555"),
("Daran inside booth (English, male)", "080d9383c0314056aef392892e009806"),
("Peppy Stella (English, female)", "084760b4922a44599575c770070ec2d7"),
("Silas (English, male)", "08f561403ec846dbbd8c691cc448f45a"),
("Aditya (English, male)", "09c3d65e44e247dd8b78a97a903feb58"),
("Christy (English, female)", "09d88c036bf449fa905900c08b235a37"),
("Elio (English, male)", "0a0b38624ac64ec6afcd5842a977ca10"),
("Luminous Laksh (Hindi, male)", "0adc547b76a5401c856274c379904eb7"),
("Jeff (English, male)", "0add542e349f4ccaba6ecb3b7ced6034"),
("Tahlia Brooks - Excited 🤩 (English, female)", "0b440d1ac2454d69a73302fc806522b1"),
("Riya Mehta (Hindi, female)", "0b464b2f4e2249a4b5a05e60eaf41e7e"),
("Ben Hart (English, male)", "0b47b5a637e944f9bfd49913999b344b"),
("Skylar (English, female)", "0bbfbda5aa924a68a9d1da7b8496052a"),
("Relaxed Reece (English, male)", "0c2151d538844c70a8b096de533f2828"),
("Daniel (English, male)", "0c23804af39a4946ac6fda42bfff2738"),
("Melani (English, female)", "0c54c6399ad64551a304e1a346677723"),
("Clover (English, female)", "0ccb0bea067d4449ad367baeed7ea2e9"),
("Pedro Lima - Serious 😐 (Portuguese, male)", "0d0e23e8170446e38b18a7380b2d30a8"),
("Ana Carvalho (Portuguese, female)", "0d23c5b2f6004e909802a2e8bfcd52c2"),
("Confident Connor - Excited 🤩 (English, male)", "0dd34c3eb79247238219eea35aeb58cd"),
("Vibrant Victor (Spanish, male)", "1062976ea8bf42f4adc27c7e868b8fde"),
("Young Olivier (French, male)", "1c5dc9a8f8cf4de0932f91d75f43a15d"),
("Émile Noir (French, male)", "25a6a67280574d3da78e97b1935ebfc7"),
("Steadfast Stefan (German, male)", "0eb85e6e8710473b82f7e88609ba3053"),
("Deep Dieter (German, male)", "118949676b0a46629d1ad52981c3ef84"),
("Serene Marco (Italian, male)", "72e922488a614041b5ab5f6ee07e3deb"),
("Murmuring Matteo (Italian, male)", "755902b751654f30a6ef49e8bbcacfec"),
("Gail in car (Multilingual, female)", "0214ac51f93e420f8711d568dcfbc50e"),
("Daran outside walking (Multilingual, male)", "0ac81e725f4948dfa9638ceca216bcfa"),
("BOB - Voice 1 (Chinese, unknown)", "dMkR1XwIkarpNqWUJLnX"),
("Hakeem Hassan (Arabic, male)", "61a4359785664d01a59664ceb87ce6d4"),
("Rami Idris (Arabic, male)", "a0bd2e5d41a74643be47ac75ca9171a2"),
("Bold Kasia - Friendly 😊 (Polish, female)", "331624aec8b24a6c9287b8e16bdf54e8"),
("Tranquil Tulin (Turkish, female)", "61646c861eb64e2d9036d8db51385356"),
("Dynamic Derya (Turkish, female)", "664b73058b784aa89ddb2924c141d441"),
("Quiet Dewa (Indonesian, male)", "1fa1193cf1d74f27ba58531c07ef9862"),
("Cuong (Vietnamese, male)", "8af68d7ea38f4e7ca05cf46c3f7a590b"),
]
HEYGEN_VOICE_TTS_OPTIONS = [x[0] for x in HEYGEN_VOICE_TTS]
HEYGEN_VOICE_TTS_MAP = dict(HEYGEN_VOICE_TTS)
# (label, voice_id) — top-ranked voices for video narration (any engine)
HEYGEN_VOICE_GENERAL: list[tuple[str, str]] = [
("Cassidy (English, female)", "16a09e4706f74997ba4ed05ea11470f6"),
("Hope (English, female)", "42d00d4aac5441279d8536cd6b52c53c"),
("Archer (English, male)", "453c20e1525a429080e2ad9e4b26f2cd"),
("Brittney (English, female)", "4754e1ec667544b0bd18cdf4bec7d6a7"),
("Mark (English, male)", "5d8c378ba8c3434586081a52ac368738"),
("Andrew (English, male)", "6be73833ef9a4eb0aeee399b8fe9d62b"),
("Spuds Oxley (English, male)", "76940a9adcd0490a9ce2cfe9a64a2664"),
("Patrick (English, male)", "7e157ec62c9c45f1adca12faae72c86f"),
("David Castlemore (English, male)", "828b59f834fd4c7188da322b6d9b6c75"),
("Michael C (English, male)", "8661cd40d6c44c709e2d0031c0186ada"),
("Adam Stone (English, male)", "88bb9ee1c81b466eb2a08fdde86d3619"),
("Alex (English, male)", "897d6a9b2c844f56aa077238768fe10a"),
("Monika Sogam (English, female)", "97dd67ab8ce242b6a9e7689cb00c6414"),
("Jessica Anne Bogart (English, female)", "b966c31caf124c2a99f19ff1479c964f"),
("John Doe (English, male)", "c4a8ceb7a2954500bc047fb092bcff3f"),
("Ivy (English, female)", "cef3bc4e0a84424cafcde6f2cf466c97"),
("Chill Brian (English, male)", "d2f4f24783d04e22ab49ee8fdc3715e0"),
("Allison (English, female)", "f8c69e517f424cafaecde32dde57096b"),
("Mia Starset (Norwegian, female)", "000466f8ac6d47a49f5743d50b3778de"),
("William Shanks (Spanish, male)", "001248bb63f847888d37b766ee8b3a47"),
("Zain (English, female)", "0047732240584155b1588455313e78ec"),
("Jora Slobod (Romanian, male)", "00631519159a402ab5d8f719e51532bb"),
("Narrator Mateo - Excited 🤩 (Spanish, male)", "0077225a877e457db4572ccaf245910b"),
("Aria (English, female)", "007e1378fc454a9f976db570ba6164a7"),
("Caryns (English, female)", "0082e70326864107823605db0d77f5e0"),
("Klara (English, female)", "01209fdcd1c24a109c86dc24ee0f71c0"),
("Son Tran (Vietnamese, male)", "0132f85950a94d11ba180f885101bf84"),
("Bold Kasia - Excited 🤩 (Polish, female)", "015482a78b9a46ebae74bd0beb17765b"),
("Marc Aurèle (French, male)", "018a94cf15574491a0bab7f6799ac15b"),
("Shaun (English, male)", "01c42cddcfdc4665a57b8d89cba8ffc1"),
("Senthil (English, male)", "01d674cfd32b4728a3fddd21b7e7d543"),
("Cody (English, male)", "01f98ed43e6140349f47dbd37a416827"),
("Saffron (English, female)", "0258bbc2cd8648cfa357adfb833f6d7b"),
("Blanka - Lifelike (English, female)", "02880d1c6fd94b7799d91135581ed810"),
("Rami (English, male)", "02d5366a90af4c7a87157808ff352e33"),
("Rhodes (English, female)", "02dce0a169b3460084b6c914d18fb2c8"),
("Michelle - Voice 1 (English, female)", "02X8sHnuxFpsq1caYWN0"),
("Tuba (, female)", "034ca0c32b6542028748d6d365d90d6a"),
("Autumn - UGC 3 (English, female)", "03dca9ebfca441dba55fb14afa0791b7"),
("Reassuring Rupert (English, male)", "03fcf8ecb0a94b6b94e9007edb7c35f8"),
]
HEYGEN_VOICE_GENERAL_OPTIONS = [x[0] for x in HEYGEN_VOICE_GENERAL]
HEYGEN_VOICE_GENERAL_MAP = dict(HEYGEN_VOICE_GENERAL)
HEYGEN_TRANSLATE_LANGUAGES = [
"English",
"Spanish",
"Spanish (Spain)",
"Spanish (Mexico)",
"French",
"French (France)",
"German",
"German (Germany)",
"Portuguese",
"Portuguese (Brazil)",
"Italian",
"Italian (Italy)",
"Japanese",
"Japanese (Japan)",
"Korean",
"Chinese (Mandarin, Simplified)",
"Arabic",
"Hindi",
"Hindi (India)",
"Russian",
"Russian (Russia)",
"Dutch",
"Polish",
"Turkish",
"Indonesian",
"Vietnamese",
"Ukrainian",
"Afrikaans (South Africa)",
"Albanian (Albania)",
"Amharic (Ethiopia)",
"Arabic (Algeria)",
"Arabic (Bahrain)",
"Arabic (Egypt)",
"Arabic (Iraq)",
"Arabic (Jordan)",
"Arabic (Kuwait)",
"Arabic (Lebanon)",
"Arabic (Libya)",
"Arabic (Morocco)",
"Arabic (Oman)",
"Arabic (Qatar)",
"Arabic (Saudi Arabia)",
"Arabic (Syria)",
"Arabic (Tunisia)",
"Arabic (United Arab Emirates)",
"Arabic (World)",
"Arabic (Yemen)",
"Armenian (Armenia)",
"Azerbaijani (Latin, Azerbaijan)",
"Bangla (Bangladesh)",
"Basque",
"Belarusian (Belarus)",
"Bengali (India)",
"Bosnian (Bosnia and Herzegovina)",
"Bulgarian",
"Bulgarian (Bulgaria)",
"Burmese (Myanmar)",
"Catalan",
"Chinese (Cantonese, Traditional)",
"Chinese (Jilu Mandarin, Simplified)",
"Chinese (Northeastern Mandarin, Simplified)",
"Chinese (Southwestern Mandarin, Simplified)",
"Chinese (Taiwanese Mandarin, Traditional)",
"Chinese (Wu, Simplified)",
"Chinese (Zhongyuan Mandarin Henan, Simplified)",
"Chinese (Zhongyuan Mandarin Shaanxi, Simplified)",
"Croatian",
"Croatian (Croatia)",
"Czech",
"Czech (Czechia)",
"Danish",
"Danish (Denmark)",
"Dutch (Belgium)",
"Dutch (Netherlands)",
"English (Australia)",
"English (Canada)",
"English (Hong Kong SAR)",
"English (India)",
"English (Ireland)",
"English (Kenya)",
"English (New Zealand)",
"English (Nigeria)",
"English (Philippines)",
"English (Singapore)",
"English (South Africa)",
"English (Tanzania)",
"English (UK)",
"English (United States)",
"Estonian (Estonia)",
"Filipino",
"Filipino (Cebuano)",
"Filipino (Philippines)",
"Finnish",
"Finnish (Finland)",
"French (Belgium)",
"French (Canada)",
"French (Switzerland)",
"Galician",
"Georgian (Georgia)",
"German (Austria)",
"German (Switzerland)",
"Greek",
"Greek (Greece)",
"Gujarati (India)",
"Haitian Creole (Haiti)",
"Hebrew (Israel)",
"Hungarian (Hungary)",
"Icelandic (Iceland)",
"Indonesian (Indonesia)",
"Irish (Ireland)",
"Javanese (Latin, Indonesia)",
"Kannada (India)",
"Kazakh (Kazakhstan)",
"Khmer (Cambodia)",
"Konkani (India)",
"Korean (Korea)",
"Lao (Laos)",
"Latin (Vatican City)",
"Latvian (Latvia)",
"Lithuanian (Lithuania)",
"Luxembourgish (Luxembourg)",
"Macedonian (North Macedonia)",
"Maithili (India)",
"Malagasy (Madagascar)",
"Malay",
"Malay (Malaysia)",
"Malayalam (India)",
"Maltese (Malta)",
"Mandarin",
"Marathi (India)",
"Mongolian (Mongolia)",
"Nepali (Nepal)",
"Norwegian Bokmål (Norway)",
"Norwegian Nynorsk (Norway)",
"Odia (India)",
"Pashto (Afghanistan)",
"Persian (Iran)",
"Polish (Poland)",
"Portuguese (Portugal)",
"Punjabi (India)",
"Romanian",
"Romanian (Romania)",
"Serbian (Latin, Serbia)",
"Sindhi (India)",
"Sinhala (Sri Lanka)",
"Slovak",
"Slovak (Slovakia)",
"Slovenian (Slovenia)",
"Somali (Somalia)",
"Spanish (Argentina)",
"Spanish (Bolivia)",
"Spanish (Chile)",
"Spanish (Colombia)",
"Spanish (Costa Rica)",
"Spanish (Cuba)",
"Spanish (Dominican Republic)",
"Spanish (Ecuador)",
"Spanish (El Salvador)",
"Spanish (Equatorial Guinea)",
"Spanish (Guatemala)",
"Spanish (Honduras)",
"Spanish (Latin America)",
"Spanish (Nicaragua)",
"Spanish (Panama)",
"Spanish (Paraguay)",
"Spanish (Peru)",
"Spanish (Puerto Rico)",
"Spanish (United States)",
"Spanish (Uruguay)",
"Spanish (Venezuela)",
"Sundanese (Indonesia)",
"Swahili (Kenya)",
"Swahili (Tanzania)",
"Swedish",
"Swedish (Sweden)",
"Tamil",
"Tamil (India)",
"Tamil (Malaysia)",
"Tamil (Singapore)",
"Tamil (Sri Lanka)",
"Telugu (India)",
"Thai (Thailand)",
"Turkish (Türkiye)",
"Ukrainian (Ukraine)",
"Urdu (India)",
"Urdu (Pakistan)",
"Uzbek (Latin, Uzbekistan)",
"Vietnamese (Vietnam)",
"Welsh (United Kingdom)",
"Zulu (South Africa)",
]

2
comfy_api_nodes/apis/openai.py
View File

@ -128,7 +128,7 @@ class OpenAIResponse(ModelResponseProperties, ResponseProperties):
parallel_tool_calls: bool | None = Field(True)
status: str | None = Field(
None,
description="One of `completed`, `failed`, `in_progress`, or `incomplete`.",
description="One of `completed`, `failed`, `in_progress`, `incomplete`, `queued`, or `cancelled`.",
)
usage: ResponseUsage | None = Field(None)

151
comfy_api_nodes/nodes_gemini.py
View File

@ -24,6 +24,11 @@ from comfy_api_nodes.apis.gemini import (
GeminiImageGenerateContentRequest,
GeminiImageGenerationConfig,
GeminiInlineData,
GeminiInteraction,
GeminiInteractionGenerationConfig,
GeminiInteractionMediaPart,
GeminiInteractionRequest,
GeminiInteractionTextPart,
GeminiMimeType,
GeminiPart,
GeminiRole,
@ -51,6 +56,7 @@ from comfy_api_nodes.util import (
)
GEMINI_BASE_ENDPOINT = "/proxy/vertexai/gemini"
GEMINI_INTERACTIONS_ENDPOINT = "/proxy/gemini-interactions"
GEMINI_MAX_INPUT_FILE_SIZE = 20 * 1024 * 1024 # 20 MB
GEMINI_URL_INPUT_BUDGET = 10
GEMINI_MAX_INLINE_BYTES = 18 * 1024 * 1024
@ -231,29 +237,10 @@ async def get_image_from_response(response: GeminiGenerateContentResponse, thoug
return torch.cat(image_tensors, dim=0)
async def get_video_from_response(
response: GeminiGenerateContentResponse, cls: type[IO.ComfyNode] | None = None
) -> InputImpl.VideoFromFile:
parts = get_parts_by_type(response, "video/*")
for part in parts:
if part.inlineData and part.inlineData.data:
return InputImpl.VideoFromFile(BytesIO(base64.b64decode(part.inlineData.data)))
if part.fileData and part.fileData.fileUri:
return await download_url_to_video_output(part.fileData.fileUri, cls=cls)
model_message = get_text_from_response(response).strip()
if model_message:
raise ValueError(f"Gemini did not generate a video. Model response: {model_message}")
raise ValueError(
"Gemini did not generate a video. Try rephrasing your prompt, "
"shortening the requested duration, or reducing the number of input images/videos."
)
def calculate_tokens_price(response: GeminiGenerateContentResponse) -> float | None:
if not response.modelVersion:
return None
# Define prices (Cost per 1,000,000 tokens), see https://cloud.google.com/vertex-ai/generative-ai/pricing
output_video_tokens_price = 0.0
if response.modelVersion == "gemini-2.5-pro":
input_tokens_price = 1.25
output_text_tokens_price = 10.0
@ -274,6 +261,10 @@ def calculate_tokens_price(response: GeminiGenerateContentResponse) -> float | N
input_tokens_price = 0.25
output_text_tokens_price = 1.50
output_image_tokens_price = 0.0
elif response.modelVersion == "gemini-3.5-flash":
input_tokens_price = 1.50
output_text_tokens_price = 9.0
output_image_tokens_price = 0.0
elif response.modelVersion in ("gemini-3-pro-image-preview", "gemini-3-pro-image"):
input_tokens_price = 2
output_text_tokens_price = 12.0
@ -286,11 +277,6 @@ def calculate_tokens_price(response: GeminiGenerateContentResponse) -> float | N
input_tokens_price = 0.25
output_text_tokens_price = 1.50
output_image_tokens_price = 30.0
elif response.modelVersion == "gemini-omni-flash-preview":
input_tokens_price = 2.145
output_text_tokens_price = 12.87
output_image_tokens_price = 0.0
output_video_tokens_price = 25.025
else:
return None
final_price = response.usageMetadata.promptTokenCount * input_tokens_price
@ -298,8 +284,6 @@ def calculate_tokens_price(response: GeminiGenerateContentResponse) -> float | N
for i in response.usageMetadata.candidatesTokensDetails:
if i.modality == Modality.IMAGE:
final_price += output_image_tokens_price * i.tokenCount # for Nano Banana models
elif i.modality == Modality.VIDEO:
final_price += output_video_tokens_price * i.tokenCount # for Omni Flash
else:
final_price += output_text_tokens_price * i.tokenCount
if response.usageMetadata.thoughtsTokenCount:
@ -307,6 +291,58 @@ def calculate_tokens_price(response: GeminiGenerateContentResponse) -> float | N
return final_price / 1_000_000.0
def get_text_from_interaction(interaction: GeminiInteraction) -> str:
"""Extract and concatenate all model output text from an Interactions API response."""
texts = []
for step in interaction.steps or []:
if step.type != "model_output":
continue
for content in step.content or []:
if content.type == "text" and content.text:
texts.append(content.text)
return "\n".join(texts)
async def get_video_from_interaction(
interaction: GeminiInteraction, cls: type[IO.ComfyNode] | None = None
) -> InputImpl.VideoFromFile:
for step in interaction.steps or []:
if step.type != "model_output":
continue
for content in step.content or []:
if content.type != "video":
continue
if content.data:
return InputImpl.VideoFromFile(BytesIO(base64.b64decode(content.data)))
if content.uri:
return await download_url_to_video_output(content.uri, cls=cls)
model_message = get_text_from_interaction(interaction).strip()
if model_message:
raise ValueError(f"Gemini did not generate a video. Model response: {model_message}")
raise ValueError(
"Gemini did not generate a video. Try rephrasing your prompt, "
"shortening the requested duration, or reducing the number of input images/videos."
)
def calculate_interaction_tokens_price(interaction: GeminiInteraction) -> float | None:
if interaction.usage is None:
return None
input_tokens_price = 1.5
output_tokens_prices = {"text": 9.0, "video": 17.5}
thoughts_tokens_price = 9.0
final_price = 0.0
for i in interaction.usage.input_tokens_by_modality or []:
if i.tokens:
final_price += input_tokens_price * i.tokens
for i in interaction.usage.output_tokens_by_modality or []:
if i.tokens and i.modality in output_tokens_prices:
final_price += output_tokens_prices[i.modality] * i.tokens
if interaction.usage.total_thought_tokens:
final_price += thoughts_tokens_price * interaction.usage.total_thought_tokens
return final_price / 1_000_000.0
def create_video_parts(video_input: Input.Video) -> list[GeminiPart]:
"""Convert a single video input to Gemini API compatible parts (inline MP4/H.264)."""
base_64_string = video_to_base64_string(
@ -441,6 +477,15 @@ async def build_gemini_media_parts(
return parts
def to_interaction_media_part(part: GeminiPart) -> GeminiInteractionMediaPart:
"""Convert a fileData/inlineData GeminiPart into an Interactions API media part."""
if part.fileData:
mime = part.fileData.mimeType.value
return GeminiInteractionMediaPart(type=mime.split("/")[0], uri=part.fileData.fileUri, mime_type=mime)
mime = part.inlineData.mimeType.value
return GeminiInteractionMediaPart(type=mime.split("/")[0], data=part.inlineData.data, mime_type=mime)
class GeminiNode(IO.ComfyNode):
"""
Node to generate text responses from a Gemini model.
@ -619,11 +664,12 @@ class GeminiNode(IO.ComfyNode):
GEMINI_V2_MODELS: dict[str, str] = {
"Gemini 3.1 Pro": "gemini-3.1-pro-preview",
"Gemini 3.5 Flash": "gemini-3.5-flash",
"Gemini 3.1 Flash-Lite": "gemini-3.1-flash-lite-preview",
}
def _gemini_text_model_inputs(thinking_default: str) -> list[Input]:
def _gemini_text_model_inputs(thinking_default: str, thinking_options: list[str] | None = None) -> list[Input]:
"""Per-model inputs revealed by the model DynamicCombo (shared media + sampling controls)."""
return [
IO.Autogrow.Input(
@ -661,7 +707,7 @@ def _gemini_text_model_inputs(thinking_default: str) -> list[Input]:
),
IO.Combo.Input(
"thinking_level",
options=["LOW", "HIGH"],
options=thinking_options or ["LOW", "HIGH"],
default=thinking_default,
tooltip="How hard the model reasons internally before answering. "
"HIGH improves quality on difficult tasks but costs more (thinking) tokens and is slower.",
@ -719,6 +765,10 @@ class GeminiNodeV2(IO.ComfyNode):
IO.DynamicCombo.Input(
"model",
options=[
IO.DynamicCombo.Option(
"Gemini 3.5 Flash",
_gemini_text_model_inputs("MEDIUM", ["MINIMAL", "LOW", "MEDIUM", "HIGH"]),
),
IO.DynamicCombo.Option("Gemini 3.1 Pro", _gemini_text_model_inputs("HIGH")),
IO.DynamicCombo.Option("Gemini 3.1 Flash-Lite", _gemini_text_model_inputs("LOW")),
],
@ -759,7 +809,13 @@ class GeminiNodeV2(IO.ComfyNode):
"type": "list_usd",
"usd": [0.00025, 0.0015],
"format": { "approximate": true, "separator": "-", "suffix": " per 1K tokens" }
} : {
}
: $contains($m, "3.5 flash") ? {
"type": "list_usd",
"usd": [0.0015, 0.009],
"format": { "approximate": true, "separator": "-", "suffix": " per 1K tokens" }
}
: {
"type": "list_usd",
"usd": [0.002, 0.012],
"format": { "approximate": true, "separator": "-", "suffix": " per 1K tokens" }
@ -1661,7 +1717,7 @@ class GeminiVideoOmni(IO.ComfyNode):
],
is_api_node=True,
price_badge=IO.PriceBadge(
expr='{"type":"usd","usd":0.146,"format":{"suffix":"/second","approximate":true}}'
expr='{"type":"usd","usd":0.101,"format":{"suffix":"/second","approximate":true}}'
),
)
@ -1680,27 +1736,34 @@ class GeminiVideoOmni(IO.ComfyNode):
for video in videos:
validate_video_duration(video, max_duration=10)
parts: list[GeminiPart] = []
parts: list[GeminiInteractionTextPart | GeminiInteractionMediaPart] = []
if images or videos:
parts.extend(await build_gemini_media_parts(cls, images, [], videos))
parts.append(GeminiPart(text=prompt))
response = await sync_op(
media_parts = await build_gemini_media_parts(cls, images, [], videos)
parts.extend(to_interaction_media_part(p) for p in media_parts)
parts.append(GeminiInteractionTextPart(text=prompt))
interaction = await sync_op(
cls,
ApiEndpoint(path=f"{GEMINI_BASE_ENDPOINT}/{model_id}", method="POST"),
data=GeminiGenerateContentRequest(
contents=[GeminiContent(role=GeminiRole.user, parts=parts)],
generationConfig=GeminiGenerationConfig(
responseModalities=["TEXT", "VIDEO"],
ApiEndpoint(path=GEMINI_INTERACTIONS_ENDPOINT, method="POST"),
data=GeminiInteractionRequest(
model=model_id,
input=parts,
generation_config=GeminiInteractionGenerationConfig(
temperature=model.get("temperature", 1.0),
topP=model.get("top_p", 0.95),
top_p=model.get("top_p", 0.95),
),
),
response_model=GeminiGenerateContentResponse,
price_extractor=calculate_tokens_price,
response_model=GeminiInteraction,
price_extractor=calculate_interaction_tokens_price,
)
if interaction.status != "completed":
model_message = get_text_from_interaction(interaction).strip()
raise ValueError(
f"Gemini interaction did not complete (status: {interaction.status})."
+ (f" Model response: {model_message}" if model_message else "")
)
return IO.NodeOutput(
await get_video_from_response(response, cls=cls),
get_text_from_response(response),
await get_video_from_interaction(interaction, cls=cls),
get_text_from_interaction(interaction),
)

799
comfy_api_nodes/nodes_heygen.py Normal file
View File

@ -0,0 +1,799 @@
import uuid
import torch
from typing_extensions import override
from comfy_api.latest import IO, ComfyExtension, Input
from comfy_api_nodes.apis.heygen import (
HEYGEN_AVATAR_MAP,
HEYGEN_AVATAR_OPTIONS,
HEYGEN_TRANSLATE_LANGUAGES,
HEYGEN_VOICE_GENERAL_MAP,
HEYGEN_VOICE_GENERAL_OPTIONS,
HEYGEN_VOICE_TTS_MAP,
HEYGEN_VOICE_TTS_OPTIONS,
)
from comfy_api_nodes.util import (
ApiEndpoint,
audio_bytes_to_audio_input,
download_url_as_bytesio,
download_url_to_image_tensor,
download_url_to_video_output,
downscale_image_tensor_by_max_side,
get_number_of_images,
poll_op_raw,
sync_op_raw,
upload_audio_to_comfyapi,
upload_image_to_comfyapi,
upload_images_to_comfyapi,
upload_video_to_comfyapi,
validate_string,
)
from server import PromptServer
_VIDEOS_PATH = "/proxy/heygen/v3/videos"
_TRANSLATIONS_PATH = "/proxy/heygen/v3/video-translations"
_SPEECH_PATH = "/proxy/heygen/v3/voices/speech"
_AVATARS_PATH = "/proxy/heygen/v3/avatars"
_LOOKS_PATH = "/proxy/heygen/v3/avatars/looks"
_DEFAULT_VOICE_OPTION = "(avatar's default voice)"
_AVATARS_BY_ENGINE = {
e: [label for label, (_aid, _atype, engines) in HEYGEN_AVATAR_MAP.items() if e in engines]
for e in ("avatar_iv", "avatar_iii", "avatar_v")
}
async def _apply_speech_source(cls: type[IO.ComfyNode], payload: dict, speech: dict, require_voice: bool) -> None:
"""Fill script/audio speech fields of a /v3/videos payload from the DynamicCombo dict."""
if speech["speech"] == "audio":
payload["audio_url"] = await upload_audio_to_comfyapi(
cls, speech["audio"], container_format="mp3", codec_name="libmp3lame", mime_type="audio/mpeg"
)
elif speech["speech"] == "script":
validate_string(speech["text"], strip_whitespace=True, min_length=1, max_length=5000)
payload["script"] = speech["text"]
voice_id = speech.get("custom_voice_id", "").strip()
if not voice_id and speech["voice"] != _DEFAULT_VOICE_OPTION:
voice_id = HEYGEN_VOICE_GENERAL_MAP[speech["voice"]]
if voice_id:
payload["voice_id"] = voice_id
elif require_voice:
raise ValueError("A voice is required when driving the video with a text script.")
speed = speech.get("voice_speed", 1.0)
if speed != 1.0:
payload["voice_settings"] = {"speed": round(speed, 2)}
async def _create_and_poll_video(cls: type[IO.ComfyNode], payload: dict) -> dict:
"""POST a /v3/videos payload, poll until terminal, and return the final video data."""
created = await sync_op_raw(
cls,
ApiEndpoint(path=_VIDEOS_PATH, method="POST", headers={"Idempotency-Key": uuid.uuid4().hex}),
data=payload,
)
video_id = (created.get("data") or {}).get("video_id")
if not video_id:
raise ValueError(f"HeyGen did not return a video_id: {created}")
final = await poll_op_raw(
cls,
ApiEndpoint(path=f"{_VIDEOS_PATH}/{video_id}"),
status_extractor=lambda r: (r.get("data") or {}).get("status"),
queued_statuses=["pending", "waiting"],
poll_interval=5.0,
)
data = final["data"]
if not data.get("video_url"):
raise ValueError(f"HeyGen returned no video_url for video {video_id}.")
return data
async def _resolve_avatar(
cls: type[IO.ComfyNode], avatar_label: str, custom_avatar_id: str, engine_choice: str
) -> tuple[str, str | None]:
"""Resolve (avatar_id, engine_type) from the combo/override + engine widgets."""
custom_avatar_id = custom_avatar_id.strip()
if custom_avatar_id:
look = (
await sync_op_raw(
cls,
ApiEndpoint(path=f"{_LOOKS_PATH}/{custom_avatar_id}"),
final_label_on_success=None,
)
).get("data") or {}
avatar_id = custom_avatar_id
avatar_label = look.get("name") or custom_avatar_id
supported = look.get("supported_api_engines") or []
else:
avatar_id, avatar_type, supported = HEYGEN_AVATAR_MAP[avatar_label]
if engine_choice == "auto":
engine = next((e for e in ("avatar_iv", "avatar_iii", "avatar_v") if e in supported), None)
else:
engine = engine_choice
if supported and engine not in supported:
raise ValueError(
f"Avatar '{avatar_label}' does not support the {engine} engine "
f"(supported: {', '.join(supported)}). Set engine to 'auto' to pick "
"a compatible engine automatically."
)
return avatar_id, engine
class HeyGenTalkingPhotoNode(IO.ComfyNode):
"""Animate a still image of a person into a lip-synced talking video."""
@classmethod
def define_schema(cls) -> IO.Schema:
return IO.Schema(
node_id="HeyGenTalkingPhotoNode",
display_name="HeyGen Talking Photo",
category="partner/video/HeyGen",
description="Animate any image of a person into a lip-synced talking video "
"(HeyGen Avatar IV). Drive it with a text script or your own audio.",
inputs=[
IO.Image.Input(
"image",
tooltip="Image of a person to animate. Downscaled automatically if larger than 2K.",
),
IO.DynamicCombo.Input(
"speech",
display_name="speech source",
options=[
IO.DynamicCombo.Option(
"script",
[
IO.String.Input(
"text",
multiline=True,
default="",
tooltip="Text for the avatar to speak (up to 5000 characters). "
"The generated speech must be at least 1 second long.",
),
IO.Combo.Input(
"voice",
options=HEYGEN_VOICE_GENERAL_OPTIONS,
tooltip="Voice for the script (HeyGen's most popular voices).",
),
IO.String.Input(
"custom_voice_id",
default="",
optional=True,
tooltip="Optional HeyGen voice ID. When set, overrides the voice selected above. "
"Any voice from HeyGen's library (2000+) can be used.",
),
IO.Float.Input(
"voice_speed",
default=1.0,
min=0.5,
max=1.5,
step=0.05,
optional=True,
tooltip="Speech speed multiplier.",
),
],
),
IO.DynamicCombo.Option(
"audio",
[
IO.Audio.Input(
"audio",
tooltip="Audio for the avatar to lip-sync, up to 10 minutes.",
),
],
),
],
tooltip="Drive the avatar with a text script (HeyGen text-to-speech) or your own audio.",
),
IO.Combo.Input(
"resolution",
options=["720p", "1080p"],
default="1080p",
optional=True,
tooltip="Output video resolution.",
),
IO.Combo.Input(
"aspect_ratio",
options=["auto", "16:9", "9:16", "1:1", "4:5", "5:4"],
default="auto",
optional=True,
tooltip="Output aspect ratio. 'auto' follows the input image.",
),
IO.Combo.Input(
"expressiveness",
options=["low", "medium", "high"],
default="low",
optional=True,
tooltip="How expressive the animated face and gestures are.",
),
IO.Int.Input(
"seed",
default=42,
min=0,
max=2147483647,
control_after_generate=True,
optional=True,
tooltip="Not sent to HeyGen; change it to force a re-run.",
),
],
outputs=[IO.Video.Output()],
hidden=[
IO.Hidden.auth_token_comfy_org,
IO.Hidden.api_key_comfy_org,
IO.Hidden.unique_id,
],
is_api_node=True,
price_badge=IO.PriceBadge(
expr="""{"type":"usd","usd":0.0715,"format":{"suffix":"/second"}}""",
),
)
@classmethod
async def execute(
cls,
image: Input.Image,
speech: dict,
resolution: str = "1080p",
aspect_ratio: str = "auto",
expressiveness: str = "low",
seed: int = 0,
) -> IO.NodeOutput:
image = downscale_image_tensor_by_max_side(image, max_side=2000)
image_url = await upload_image_to_comfyapi(cls, image, mime_type="image/png", total_pixels=None)
payload = {
"type": "image",
"image": {"type": "url", "url": image_url},
"resolution": resolution,
"aspect_ratio": aspect_ratio,
"expressiveness": expressiveness,
"title": "ComfyUI Talking Photo",
}
await _apply_speech_source(cls, payload, speech, require_voice=True)
video = await _create_and_poll_video(cls, payload)
return IO.NodeOutput(await download_url_to_video_output(video["video_url"]))
class HeyGenAvatarVideoNode(IO.ComfyNode):
"""Generate a presenter video from a HeyGen avatar look."""
@classmethod
def define_schema(cls) -> IO.Schema:
return IO.Schema(
node_id="HeyGenAvatarVideoNode",
display_name="HeyGen Avatar Video",
category="partner/video/HeyGen",
description="Generate a talking-presenter video from a HeyGen avatar. "
"Includes HeyGen's most popular public avatars; any look ID can be supplied as an override.",
inputs=[
IO.DynamicCombo.Input(
"engine",
options=[
IO.DynamicCombo.Option(
"auto",
[
IO.Combo.Input(
"avatar",
options=HEYGEN_AVATAR_OPTIONS,
tooltip="Avatar look to present the video (curated from HeyGen's "
"public library). The best engine the look supports is chosen "
"automatically.",
),
],
),
IO.DynamicCombo.Option(
"avatar_iv",
[
IO.Combo.Input(
"avatar",
options=_AVATARS_BY_ENGINE["avatar_iv"],
tooltip="Avatar looks that support the Avatar IV engine.",
),
],
),
IO.DynamicCombo.Option(
"avatar_iii",
[
IO.Combo.Input(
"avatar",
options=_AVATARS_BY_ENGINE["avatar_iii"],
tooltip="Avatar looks that support the Avatar III engine.",
),
],
),
IO.DynamicCombo.Option(
"avatar_v",
[
IO.Combo.Input(
"avatar",
options=_AVATARS_BY_ENGINE["avatar_v"],
tooltip="Avatar looks that support the Avatar V engine.",
),
],
),
],
tooltip="Rendering engine; each choice lists only the avatars that support it. "
"'auto' offers every avatar and picks its best engine (Avatar IV preferred). "
"Avatar V is highest fidelity, Avatar III is the most affordable.",
),
IO.String.Input(
"custom_avatar_id",
default="",
optional=True,
tooltip="Optional HeyGen avatar look ID. When set, overrides the avatar selected above. "
"Any of HeyGen's 3000+ public looks (or your private avatars) can be used.",
),
IO.DynamicCombo.Input(
"speech",
display_name="speech source",
options=[
IO.DynamicCombo.Option(
"script",
[
IO.String.Input(
"text",
multiline=True,
default="",
tooltip="Text for the avatar to speak (up to 5000 characters). "
"The generated speech must be at least 1 second long.",
),
IO.Combo.Input(
"voice",
options=[_DEFAULT_VOICE_OPTION] + HEYGEN_VOICE_GENERAL_OPTIONS,
tooltip="Voice for the script. The default option uses the voice HeyGen assigned to the avatar.",
),
IO.String.Input(
"custom_voice_id",
default="",
optional=True,
tooltip="Optional HeyGen voice ID. When set, overrides the voice selected above. "
"Any voice from HeyGen's library (2000+) can be used.",
),
IO.Float.Input(
"voice_speed",
default=1.0,
min=0.5,
max=1.5,
step=0.05,
optional=True,
tooltip="Speech speed multiplier.",
),
],
),
IO.DynamicCombo.Option(
"audio",
[
IO.Audio.Input(
"audio",
tooltip="Audio for the avatar to lip-sync, up to 10 minutes.",
),
],
),
],
tooltip="Drive the avatar with a text script (HeyGen text-to-speech) or your own audio.",
),
IO.Combo.Input(
"resolution",
options=["720p", "1080p"],
default="1080p",
optional=True,
tooltip="Output video resolution.",
),
IO.Combo.Input(
"aspect_ratio",
options=["auto", "16:9", "9:16", "1:1", "4:5", "5:4"],
default="auto",
optional=True,
tooltip="Output aspect ratio. 'auto' follows the avatar's source footage.",
),
IO.String.Input(
"background_color",
default="",
optional=True,
tooltip="Optional solid background color as a hex code (e.g. '#00ff00'). "
"Leave empty for the avatar's own background.",
),
IO.Int.Input(
"seed",
default=42,
min=0,
max=2147483647,
control_after_generate=True,
optional=True,
tooltip="Not sent to HeyGen; change it to force a re-run.",
),
],
outputs=[IO.Video.Output()],
hidden=[
IO.Hidden.auth_token_comfy_org,
IO.Hidden.api_key_comfy_org,
IO.Hidden.unique_id,
],
is_api_node=True,
price_badge=IO.PriceBadge(
depends_on=IO.PriceBadgeDepends(widgets=["engine"]),
expr="""
widgets.engine = "avatar_iii"
? {"type":"range_usd","min_usd":0.023881,"max_usd":0.061919,"format":{"suffix":"/second"}}
: widgets.engine = "avatar_v"
? {"type":"usd","usd":0.095381,"format":{"suffix":"/second"}}
: widgets.engine = "avatar_iv"
? {"type":"range_usd","min_usd":0.0715,"max_usd":0.095381,"format":{"suffix":"/second"}}
: {"type":"range_usd","min_usd":0.023881,"max_usd":0.095381,"format":{"suffix":"/second"}}
""",
),
)
@classmethod
async def execute(
cls,
engine: dict,
speech: dict,
custom_avatar_id: str = "",
resolution: str = "1080p",
aspect_ratio: str = "auto",
background_color: str = "",
seed: int = 0,
) -> IO.NodeOutput:
avatar_id, engine_type = await _resolve_avatar(cls, engine["avatar"], custom_avatar_id, engine["engine"])
payload = {
"type": "avatar",
"avatar_id": avatar_id,
"resolution": resolution,
"aspect_ratio": aspect_ratio,
"title": "ComfyUI Avatar Video",
}
if engine_type:
payload["engine"] = {"type": engine_type}
background_color = background_color.strip()
if background_color:
if not background_color.startswith("#"):
raise ValueError("background_color must be a hex color code like '#00ff00'.")
payload["background"] = {"type": "color", "value": background_color}
await _apply_speech_source(cls, payload, speech, require_voice=False)
video = await _create_and_poll_video(cls, payload)
return IO.NodeOutput(await download_url_to_video_output(video["video_url"]))
class HeyGenCreateAvatarNode(IO.ComfyNode):
"""Create a reusable HeyGen avatar from a photo or a text prompt."""
@classmethod
def define_schema(cls) -> IO.Schema:
return IO.Schema(
node_id="HeyGenCreateAvatarNode",
display_name="HeyGen Create Avatar",
category="partner/video/HeyGen",
description="Create your own reusable HeyGen avatar from a photo of a person or "
"from a text prompt (a generated character). Feed the resulting avatar_id into "
"HeyGen Avatar Video's custom_avatar_id — and save the ID somewhere to reuse the "
"avatar in future workflows.",
inputs=[
IO.DynamicCombo.Input(
"source",
options=[
IO.DynamicCombo.Option(
"prompt",
[
IO.String.Input(
"prompt",
multiline=True,
default="",
tooltip="Description of the avatar to generate (up to 1000 characters).",
),
IO.Autogrow.Input(
"reference_images",
template=IO.Autogrow.TemplateNames(
IO.Image.Input("ref_image"),
names=[f"ref_image_{i}" for i in range(1, 4)],
min=0,
),
tooltip="Up to 3 reference images guiding the generated look.",
),
],
),
IO.DynamicCombo.Option(
"photo",
[
IO.Image.Input(
"identity_photo",
tooltip="Photo of the person to turn into an avatar. "
"Downscaled automatically if larger than 2K.",
),
],
),
],
tooltip="Generate a new character from a text prompt, or create the avatar "
"from a connected photo of a person.",
),
],
outputs=[
IO.String.Output(
display_name="avatar_id",
tooltip="Avatar look ID. Pass it to HeyGen Avatar Video's custom_avatar_id; "
"save it to reuse the avatar later.",
),
IO.Image.Output(display_name="preview"),
],
hidden=[
IO.Hidden.auth_token_comfy_org,
IO.Hidden.api_key_comfy_org,
IO.Hidden.unique_id,
],
is_api_node=True,
price_badge=IO.PriceBadge(
expr="""{"type":"usd","usd":1.43}""",
),
)
@classmethod
async def execute(
cls,
source: dict,
) -> IO.NodeOutput:
payload: dict = {"name": "ComfyUI Avatar"}
if source["source"] == "photo":
image = downscale_image_tensor_by_max_side(source["identity_photo"], max_side=2000)
image_url = await upload_image_to_comfyapi(cls, image, mime_type="image/png", total_pixels=None)
payload["type"] = "photo"
payload["file"] = {"type": "url", "url": image_url}
else:
validate_string(source["prompt"], strip_whitespace=True, min_length=1, max_length=1000)
payload["type"] = "prompt"
payload["prompt"] = source["prompt"]
ref_tensors = [t for t in (source.get("reference_images") or {}).values() if t is not None]
if ref_tensors:
n_images = sum(get_number_of_images(t) for t in ref_tensors)
if n_images > 3:
raise ValueError(f"HeyGen accepts at most 3 reference images; got {n_images}.")
scaled = [downscale_image_tensor_by_max_side(t, max_side=2000) for t in ref_tensors]
ref_urls = await upload_images_to_comfyapi(
cls, scaled, max_images=3, mime_type="image/png", total_pixels=None
)
payload["reference_images"] = [{"type": "url", "url": u} for u in ref_urls]
created = await sync_op_raw(
cls,
ApiEndpoint(path=_AVATARS_PATH, method="POST"),
data=payload,
)
look_id = ((created.get("data") or {}).get("avatar_item") or {}).get("id")
if not look_id:
raise ValueError(f"HeyGen did not return an avatar: {created}")
final = await poll_op_raw(
cls,
ApiEndpoint(path=f"{_LOOKS_PATH}/{look_id}"),
# A missing status means the look needed no training and is ready.
status_extractor=lambda r: (r.get("data") or {}).get("status") or "completed",
failed_statuses=["failed", "pending_consent"],
poll_interval=5.0,
)
data = final["data"]
if data.get("preview_image_url"):
preview = await download_url_to_image_tensor(data["preview_image_url"])
else:
preview = torch.zeros(1, 64, 64, 3)
PromptServer.instance.send_progress_text(
f"Please save the avatar_id for reuse.\n\navatar_id: {look_id}",
cls.hidden.unique_id,
)
return IO.NodeOutput(look_id, preview)
class HeyGenVideoTranslateNode(IO.ComfyNode):
"""Translate a spoken video into another language with voice cloning and lip sync."""
@classmethod
def define_schema(cls) -> IO.Schema:
return IO.Schema(
node_id="HeyGenVideoTranslateNode",
display_name="HeyGen Video Translate",
category="partner/video/HeyGen",
description="Translate a spoken video into another language. Clones the original "
"speaker's voice and re-animates the mouth to match the translated speech.",
inputs=[
IO.Video.Input(
"video",
tooltip="Video with speech to translate.",
),
IO.Combo.Input(
"output_language",
options=HEYGEN_TRANSLATE_LANGUAGES,
tooltip="Target language for the translated video.",
),
IO.Combo.Input(
"mode",
options=["speed", "precision"],
default="speed",
tooltip="'speed' is faster; 'precision' produces higher-quality lip sync at twice the price.",
),
IO.Boolean.Input(
"translate_audio_only",
default=False,
optional=True,
tooltip="Only swap the audio track, keeping the original mouth movements (no lip sync).",
),
IO.Int.Input(
"speaker_count",
default=0,
min=0,
max=10,
optional=True,
tooltip="Number of speakers in the video. 0 = detect automatically.",
),
IO.Int.Input(
"seed",
default=42,
min=0,
max=2147483647,
control_after_generate=True,
optional=True,
tooltip="Not sent to HeyGen; change it to force a re-run.",
),
],
outputs=[IO.Video.Output()],
hidden=[
IO.Hidden.auth_token_comfy_org,
IO.Hidden.api_key_comfy_org,
IO.Hidden.unique_id,
],
is_api_node=True,
price_badge=IO.PriceBadge(
depends_on=IO.PriceBadgeDepends(widgets=["mode"]),
expr="""{"type":"usd","usd": widgets.mode = "precision" ? 0.095381 : 0.047619,"""
""""format":{"suffix":"/second"}}""",
),
)
@classmethod
async def execute(
cls,
video: Input.Video,
output_language: str,
mode: str,
translate_audio_only: bool = False,
speaker_count: int = 0,
seed: int = 0,
) -> IO.NodeOutput:
video_url = await upload_video_to_comfyapi(cls, video)
payload = {
"video": {"type": "url", "url": video_url},
"output_languages": [output_language],
"mode": mode,
"translate_audio_only": translate_audio_only,
"title": "ComfyUI Video Translate",
}
if speaker_count > 0:
payload["speaker_num"] = speaker_count
created = await sync_op_raw(
cls,
ApiEndpoint(path=_TRANSLATIONS_PATH, method="POST"),
data=payload,
)
translation_ids = (created.get("data") or {}).get("video_translation_ids") or []
if not translation_ids:
raise ValueError(f"HeyGen did not return a translation ID: {created}")
final = await poll_op_raw(
cls,
ApiEndpoint(path=f"{_TRANSLATIONS_PATH}/{translation_ids[0]}"),
status_extractor=lambda r: (r.get("data") or {}).get("status"),
queued_statuses=["pending"],
poll_interval=5.0,
)
data = final["data"]
if not data.get("video_url"):
raise ValueError(f"HeyGen returned no video_url for translation {translation_ids[0]}.")
return IO.NodeOutput(await download_url_to_video_output(data["video_url"]))
class HeyGenTextToSpeechNode(IO.ComfyNode):
"""Synthesize speech audio from text with HeyGen's Starfish TTS engine."""
@classmethod
def define_schema(cls) -> IO.Schema:
return IO.Schema(
node_id="HeyGenTextToSpeechNode",
display_name="HeyGen Text to Speech",
category="partner/audio/HeyGen",
description="Generate speech audio from text using HeyGen's Starfish TTS engine. "
"Includes HeyGen's most popular voices across 17 languages.",
inputs=[
IO.String.Input(
"text",
multiline=True,
default="",
tooltip="Text to synthesize (up to 5000 characters). The generated speech "
"must be at least 1 second long.",
),
IO.Combo.Input(
"voice",
options=HEYGEN_VOICE_TTS_OPTIONS,
tooltip="Voice to use (curated from HeyGen's most popular Starfish-compatible voices).",
),
IO.String.Input(
"custom_voice_id",
default="",
optional=True,
tooltip="Optional HeyGen voice ID. When set, overrides the voice selected above. "
"The voice must support the Starfish engine.",
),
IO.Float.Input(
"speed",
default=1.0,
min=0.5,
max=2.0,
step=0.05,
optional=True,
tooltip="Speech speed multiplier.",
),
IO.Boolean.Input(
"ssml",
default=False,
optional=True,
tooltip="Treat the text as SSML markup (for pauses, emphasis, and pronunciation control).",
),
IO.Int.Input(
"seed",
default=42,
min=0,
max=2147483647,
control_after_generate=True,
optional=True,
tooltip="Not sent to HeyGen; change it to force a re-run.",
),
],
outputs=[IO.Audio.Output()],
hidden=[
IO.Hidden.auth_token_comfy_org,
IO.Hidden.api_key_comfy_org,
IO.Hidden.unique_id,
],
is_api_node=True,
price_badge=IO.PriceBadge(
expr="""{"type":"usd","usd":0.00095381,"format":{"approximate":true,"suffix":"/second"}}""",
),
)
@classmethod
async def execute(
cls,
text: str,
voice: str,
custom_voice_id: str = "",
speed: float = 1.0,
ssml: bool = False,
seed: int = 0,
) -> IO.NodeOutput:
validate_string(text, strip_whitespace=True, min_length=1, max_length=5000)
payload = {
"text": text,
"voice_id": custom_voice_id.strip() or HEYGEN_VOICE_TTS_MAP[voice],
"speed": round(speed, 2),
}
if ssml:
payload["input_type"] = "ssml"
response = await sync_op_raw(
cls,
ApiEndpoint(path=_SPEECH_PATH, method="POST"),
data=payload,
)
audio_url = (response.get("data") or {}).get("audio_url")
if not audio_url:
raise ValueError(f"HeyGen did not return an audio_url: {response}")
audio_bytes = await download_url_as_bytesio(audio_url)
return IO.NodeOutput(audio_bytes_to_audio_input(audio_bytes.getvalue()))
class HeyGenExtension(ComfyExtension):
@override
async def get_node_list(self) -> list[type[IO.ComfyNode]]:
return [
HeyGenTalkingPhotoNode,
HeyGenAvatarVideoNode,
HeyGenCreateAvatarNode,
HeyGenVideoTranslateNode,
HeyGenTextToSpeechNode,
]
async def comfy_entrypoint() -> HeyGenExtension:
return HeyGenExtension()

18
comfy_api_nodes/nodes_openai.py
View File

@ -41,6 +41,9 @@ STARTING_POINT_ID_PATTERN = r"<starting_point_id:(.*)>"
class SupportedOpenAIModel(str, Enum):
gpt_5_6_sol = "gpt-5.6-sol"
gpt_5_6_terra = "gpt-5.6-terra"
gpt_5_6_luna = "gpt-5.6-luna"
gpt_5_5_pro = "gpt-5.5-pro"
gpt_5_5 = "gpt-5.5"
gpt_5 = "gpt-5"
@ -1063,6 +1066,21 @@ class OpenAIChatNode(IO.ComfyNode):
"usd": [0.002, 0.008],
"format": { "approximate": true, "separator": "-", "suffix": " per 1K tokens" }
}
: $contains($m, "gpt-5.6-terra") ? {
"type": "list_usd",
"usd": [0.0025, 0.015],
"format": { "approximate": true, "separator": "-", "suffix": " per 1K tokens" }
}
: $contains($m, "gpt-5.6-luna") ? {
"type": "list_usd",
"usd": [0.001, 0.006],
"format": { "approximate": true, "separator": "-", "suffix": " per 1K tokens" }
}
: $contains($m, "gpt-5.6") ? {
"type": "list_usd",
"usd": [0.005, 0.03],
"format": { "approximate": true, "separator": "-", "suffix": " per 1K tokens" }
}
: $contains($m, "gpt-5.5-pro") ? {
"type": "list_usd",
"usd": [0.03, 0.18],

53
comfy_extras/nodes_model_patch.py
View File

@ -8,6 +8,7 @@ import comfy.ldm.common_dit
import comfy.latent_formats
import comfy.ldm.lumina.controlnet
import comfy.ldm.supir.supir_modules
import comfy.ldm.anima.lllite
from comfy.ldm.wan.model_multitalk import WanMultiTalkAttentionBlock, MultiTalkAudioProjModel
from comfy_api.latest import io
from comfy.ldm.supir.supir_patch import SUPIRPatch
@ -236,10 +237,12 @@ class ModelPatchLoader:
def load_model_patch(self, name):
model_patch_path = folder_paths.get_full_path_or_raise("model_patches", name)
sd = comfy.utils.load_torch_file(model_patch_path, safe_load=True)
sd, metadata = comfy.utils.load_torch_file(model_patch_path, safe_load=True, return_metadata=True)
dtype = comfy.utils.weight_dtype(sd)
if 'controlnet_blocks.0.y_rms.weight' in sd:
if 'lllite_conditioning1.conv1.weight' in sd:
model = comfy.ldm.anima.lllite.AnimaLLLite(sd, metadata, device=comfy.model_management.unet_offload_device(), dtype=dtype, operations=comfy.ops.manual_cast)
elif 'controlnet_blocks.0.y_rms.weight' in sd:
additional_in_dim = sd["img_in.weight"].shape[1] - 64
model = QwenImageBlockWiseControlNet(additional_in_dim=additional_in_dim, device=comfy.model_management.unet_offload_device(), dtype=dtype, operations=comfy.ops.manual_cast)
elif 'feature_embedder.mid_layer_norm.bias' in sd:
@ -296,6 +299,50 @@ class ModelPatchLoader:
return (model_patcher,)
class AnimaLLLiteApply:
@classmethod
def INPUT_TYPES(s):
return {"required": {"model": ("MODEL",),
"model_patch": ("MODEL_PATCH",),
"image": ("IMAGE",),
"strength": ("FLOAT", {"default": 1.0, "min": -10.0, "max": 10.0, "step": 0.01}),
"start_percent": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001}),
"end_percent": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.001}),
},
"optional": {"mask": ("MASK",),
}}
RETURN_TYPES = ("MODEL",)
FUNCTION = "apply_patch"
EXPERIMENTAL = True
CATEGORY = "model_patches/anima"
def apply_patch(self, model, model_patch, image, strength, start_percent, end_percent, mask=None):
image = image[..., :3]
if model_patch.model.cond_in_channels == 4 and mask is None:
mask = torch.zeros_like(image[..., 0])
elif model_patch.model.cond_in_channels != 4:
mask = None
model_sampling = model.get_model_object("model_sampling")
sigma_start = float(model_sampling.percent_to_sigma(start_percent))
sigma_end = float(model_sampling.percent_to_sigma(end_percent))
patch = comfy.ldm.anima.lllite.AnimaLLLitePatch(model_patch, image, mask, strength, sigma_start, sigma_end)
model_patched = model.clone()
model_patched.set_model_post_input_patch(patch)
model_patched.set_model_attn1_patch(comfy.ldm.anima.lllite.AnimaLLLiteAttentionPatch(
patch,
{"q": "self_attn_q_proj", "k": "self_attn_k_proj", "v": "self_attn_v_proj"},
))
model_patched.set_model_attn2_patch(comfy.ldm.anima.lllite.AnimaLLLiteAttentionPatch(
patch,
{"q": "cross_attn_q_proj"},
))
model_patched.set_model_patch(comfy.ldm.anima.lllite.AnimaLLLiteMLPPatch(patch), "mlp_patch")
return (model_patched,)
class DiffSynthCnetPatch:
def __init__(self, model_patch, vae, image, strength, mask=None):
self.model_patch = model_patch
@ -674,6 +721,7 @@ NODE_CLASS_MAPPINGS = {
"ZImageFunControlnet": ZImageFunControlnet,
"USOStyleReference": USOStyleReference,
"SUPIRApply": SUPIRApply,
"AnimaLLLiteApply": AnimaLLLiteApply,
}
NODE_DISPLAY_NAME_MAPPINGS = {
@ -682,4 +730,5 @@ NODE_DISPLAY_NAME_MAPPINGS = {
"ZImageFunControlnet": "Apply Z-Image Fun ControlNet",
"USOStyleReference": "Apply USO Style Reference",
"SUPIRApply": "Apply SUPIR Patch",
"AnimaLLLiteApply": "Apply Anima LLLite",
}

51
openapi.yaml
View File

@ -530,6 +530,10 @@ components:
description: Job creation timestamp (Unix timestamp in milliseconds)
format: int64
type: integer
execution_end_time:
description: Workflow execution completion timestamp (Unix milliseconds, only present for terminal states)
format: int64
type: integer
execution_error:
allOf:
- $ref: '#/components/schemas/ExecutionError'
@ -538,6 +542,10 @@ components:
additionalProperties: true
description: Node-level execution metadata (only for terminal states)
type: object
execution_start_time:
description: Workflow execution start timestamp (Unix milliseconds, only present once execution has started)
format: int64
type: integer
execution_status:
additionalProperties: true
description: ComfyUI execution status and timeline (only for terminal states)
@ -570,6 +578,12 @@ components:
description: Last update timestamp (Unix timestamp in milliseconds)
format: int64
type: integer
user_id:
description: |
ID of the user that owns this job (see the `workspace_id`
description above for why this is always the caller's own id
on a successful response).
type: string
workflow:
additionalProperties: true
description: |
@ -583,6 +597,18 @@ components:
workflow_id:
description: UUID identifying the workflow graph definition
type: string
workspace_id:
description: |
ID of the workspace that owns this job. A successful (200)
response from this operation is only ever returned for the
caller's own job (see this operation's ownership-scoped
query), so this is always the caller's own workspace —
consumers that also need to correlate this job to its
live-progress broadcast channel (workspace+user scoped; see
the internal common/gateways/broadcast package) can use this
value directly rather than resolving their own identity a
second way.
type: string
required:
- id
- status
@ -1565,7 +1591,13 @@ paths:
schema:
default: true
type: boolean
- description: Filter assets by exact content hash.
- description: |
Filter assets by content hash, in the canonical `blake3:<hex>`
form. Matches regardless of which of this asset store's two
internal hash storage formats the matching row was written
under (the canonical form used by from-hash-created references,
or the raw `<hex>.<ext>`/bare `<hex>` storage key used by direct
uploads) — both represent the same content hash.
in: query
name: hash
schema:
@ -2464,6 +2496,23 @@ paths:
schema:
additionalProperties: true
properties:
free_tier_balance:
description: Free-tier job allowance for an authenticated non-paid (FREE-tier) user in the rollout. Absent for paid users and unauthenticated requests. Synthesized from config before a grant row exists so a brand-new user still sees their full allowance.
properties:
allowance:
description: Total free jobs granted for the current period
type: integer
remaining:
description: Free jobs remaining (allowance - used, floored at 0)
type: integer
used:
description: Free jobs consumed so far
type: integer
required:
- allowance
- used
- remaining
type: object
max_upload_size:
description: Maximum upload size in bytes
type: integer

4
requirements.txt
View File

@ -1,5 +1,5 @@
comfyui-frontend-package==1.45.21
comfyui-workflow-templates==0.11.9
comfyui-workflow-templates==0.11.12
comfyui-embedded-docs==0.5.8
torch
torchsde
@ -22,7 +22,7 @@ alembic
SQLAlchemy>=2.0.0
filelock
av>=16.0.0
comfy-kitchen==0.2.20
comfy-kitchen==0.2.22
comfy-aimdo==0.4.10
requests
simpleeval>=1.0.0