Merge branch 'master' into minimax-h3-latent-noise-masks
This commit is contained in:
Barish Ozbay
GitHub
commit
877b5b8862
|
|
@ -11,6 +11,10 @@ reviews:
|
|||
review_status: false
|
||||
review_details: true
|
||||
commit_status: true
|
||||
# Without this, a review that never happened (rate limit, internal error)
|
||||
# still posts a green "CodeRabbit" commit status, so a throttled review is
|
||||
# indistinguishable from a clean one.
|
||||
fail_commit_status: true
|
||||
collapse_walkthrough: true
|
||||
changed_files_summary: false
|
||||
sequence_diagrams: false
|
||||
|
|
|
|||
|
|
@ -99,7 +99,7 @@ class UpSample1d(nn.Module):
|
|||
def forward(self, x):
|
||||
_, C, _ = x.shape
|
||||
x = F.pad(x, (self.pad, self.pad), mode="replicate")
|
||||
x = F.conv_transpose1d(x, self.filter.expand(C, -1, -1).to(x.dtype), stride=self.stride, groups=C).mul_(self.ratio)
|
||||
x = F.conv_transpose1d(x, comfy.ops.cast_to_input(self.filter.expand(C, -1, -1), x), stride=self.stride, groups=C).mul_(self.ratio)
|
||||
x = x[..., self.pad_left:-self.pad_right]
|
||||
return x
|
||||
|
||||
|
|
@ -115,7 +115,7 @@ class LowPassFilter1d(nn.Module):
|
|||
def forward(self, x):
|
||||
_, C, _ = x.shape
|
||||
x = F.pad(x, (self.pad_left, self.pad_right), mode="replicate")
|
||||
return F.conv1d(x, self.filter.expand(C, -1, -1).to(x.dtype), stride=self.stride, groups=C)
|
||||
return F.conv1d(x, comfy.ops.cast_to_input(self.filter.expand(C, -1, -1), x), stride=self.stride, groups=C)
|
||||
|
||||
|
||||
class DownSample1d(nn.Module):
|
||||
|
|
|
|||
|
|
@ -509,14 +509,14 @@ class MiniMaxH3Model(nn.Module):
|
|||
# the sampler carries the audio as (sigma_v / sigma_a) * x_audio; undo it outside
|
||||
# the wrappers so they and the network see the stream's own latent and velocity
|
||||
scale = float((minimax_payload or {}).get("audio_scale", 1.0))
|
||||
audio_x = x[1]
|
||||
audio_src = x[1]
|
||||
if scale != 1.0:
|
||||
shift_v = float(transformer_options.get("minimax_h3_sigma_shift_video", self.sigma_shift_video))
|
||||
shift_a = float(transformer_options.get("minimax_h3_sigma_shift_audio", self.sigma_shift_audio))
|
||||
sigma_v = (timestep.flatten()[0] / 1000.0).float().clamp(min=1e-6)
|
||||
sigma_a = time_shift_sigma(sigma_v, shift_v, shift_a)
|
||||
audio_x = audio_x * (sigma_a / sigma_v).to(audio_x.dtype)
|
||||
x = [x[0], audio_x]
|
||||
carry = (sigma_a / sigma_v).to(audio_src.dtype)
|
||||
x = [x[0], audio_src * carry]
|
||||
|
||||
out = comfy.patcher_extension.WrapperExecutor.new_class_executor(
|
||||
self._forward,
|
||||
|
|
@ -527,7 +527,7 @@ class MiniMaxH3Model(nn.Module):
|
|||
|
||||
if scale != 1.0:
|
||||
# d/d(sigma_v) of the carried variable
|
||||
out[1] = ((1.0 - scale) * audio_x
|
||||
out[1] = ((1.0 - scale) * (audio_src * carry)
|
||||
+ (1.0 + (scale - 1.0) * sigma_a).to(out[1].dtype) * out[1])
|
||||
return out
|
||||
|
||||
|
|
|
|||
|
|
@ -147,7 +147,8 @@ class WanI2VCrossAttention(WanSelfAttention):
|
|||
v = self.v(context)
|
||||
k_img = self.norm_k_img(self.k_img(context_img))
|
||||
v_img = self.v_img(context_img)
|
||||
img_x = optimized_attention(q, k_img, v_img, heads=self.num_heads, transformer_options=transformer_options)
|
||||
# Sageattn can cause Nans here, don't allow it as there is no speed difference anyway as img attention is tiny.
|
||||
img_x = optimized_attention(q, k_img, v_img, heads=self.num_heads, transformer_options=transformer_options, low_precision_attention=False)
|
||||
# compute attention
|
||||
x = optimized_attention(q, k, v, heads=self.num_heads, transformer_options=transformer_options)
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,387 @@
|
|||
# Wan-Animate-2: https://github.com/Wan-Video/Wan-Animate-2
|
||||
"""Wan2.1-I2V-14B weights, driven by a video instead of a motion extractor.
|
||||
|
||||
A pose branch over the pose video's latents runs in lockstep with the generation
|
||||
branch, feeding it K/V per block. The reference image is one extra latent frame at the
|
||||
front of the generation branch, trimmed off by the caller. Upstream calls the pose video
|
||||
the driving video and its branch forward_ref, not to be confused with the reference image.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
import comfy.ldm.common_dit
|
||||
import comfy.model_management
|
||||
import comfy.quant_ops
|
||||
import comfy.utils
|
||||
from comfy.ldm.flux.math import apply_rope1
|
||||
from comfy.ldm.modules.attention import optimized_attention
|
||||
|
||||
from .model import WanAttentionBlock, WanModel, WanSelfAttention, repeat_e, sinusoidal_embedding_1d
|
||||
|
||||
|
||||
class WanAnimate2SelfAttention(WanSelfAttention):
|
||||
|
||||
def qkv(self, x, freqs):
|
||||
b, s, n, d = *x.shape[:2], self.num_heads, self.head_dim
|
||||
q = apply_rope1(self.norm_q(self.q(x)).view(b, s, n, d), freqs)
|
||||
k = apply_rope1(self.norm_k(self.k(x)).view(b, s, n, d), freqs)
|
||||
return q, k, self.v(x).view(b, s, n, d)
|
||||
|
||||
def _attn1_patch(self, x, q, k, transformer_options):
|
||||
for p in transformer_options.get("patches", {}).get("attn1_patch", []):
|
||||
x = p({"x": x, "q": q, "k": k, "transformer_options": transformer_options})
|
||||
return x
|
||||
|
||||
def kv(self, x, freqs):
|
||||
b, s, n, d = *x.shape[:2], self.num_heads, self.head_dim
|
||||
return apply_rope1(self.norm_k(self.k(x)).view(b, s, n, d), freqs), self.v(x).view(b, s, n, d)
|
||||
|
||||
def forward_pose(self, x, freqs, transformer_options={}):
|
||||
b, s, n, d = *x.shape[:2], self.num_heads, self.head_dim
|
||||
q, k, v = self.qkv(x, freqs)
|
||||
out = optimized_attention(q.reshape(b, s, n * d), k.reshape(b, s, n * d), v.reshape(b, s, n * d), heads=self.num_heads, transformer_options=transformer_options)
|
||||
return self.o(self._attn1_patch(out, q, k, transformer_options)), k, v
|
||||
|
||||
def forward_gen(self, x, freqs, k_pose, v_pose, f_gen, hw, buffers, ref_strength=1.0, transformer_options={}):
|
||||
# frame j attends every gen token plus pose frame j-1 (frame 0 is the reference slot and has none)
|
||||
b, s, n, d = *x.shape[:2], self.num_heads, self.head_dim
|
||||
q, k, v = self.qkv(x, freqs)
|
||||
if ref_strength != 1.0:
|
||||
v[:, :hw] *= ref_strength # frame 0 is the reference image's slot
|
||||
|
||||
if k_pose is None: # pose influence windowed out: plain self-attention, no per-frame loop
|
||||
out = optimized_attention(q.reshape(b, s, n * d), k.reshape(b, s, n * d), v.reshape(b, s, n * d), heads=self.num_heads, transformer_options=transformer_options)
|
||||
return self.o(self._attn1_patch(out, q, k, transformer_options))
|
||||
|
||||
# gen half is the same every frame; only the hw-token pose tail is rewritten
|
||||
kbuf, vbuf, out = buffers
|
||||
kbuf[:, :s] = k
|
||||
vbuf[:, :s] = v
|
||||
|
||||
for j in range(f_gen):
|
||||
q_j = q[:, j * hw:(j + 1) * hw].reshape(b, hw, n * d)
|
||||
if j == 0:
|
||||
kk, vv = k, v
|
||||
else:
|
||||
kbuf[:, s:] = k_pose[:, (j - 1) * hw:j * hw]
|
||||
vbuf[:, s:] = v_pose[:, (j - 1) * hw:j * hw]
|
||||
kk, vv = kbuf, vbuf
|
||||
out[:, j * hw:(j + 1) * hw] = optimized_attention(q_j, kk.reshape(b, kk.shape[1], n * d), vv.reshape(b, kk.shape[1], n * d), heads=self.num_heads, transformer_options=transformer_options)
|
||||
return self.o(self._attn1_patch(out, q, k, transformer_options))
|
||||
|
||||
|
||||
class WanAnimate2Block(WanAttentionBlock):
|
||||
|
||||
def __init__(self, cross_attn_type, dim, ffn_dim, num_heads, window_size=(-1, -1), qk_norm=True, cross_attn_norm=False, eps=1e-6, operation_settings={}):
|
||||
super().__init__(cross_attn_type, dim, ffn_dim, num_heads, window_size, qk_norm, cross_attn_norm, eps, operation_settings=operation_settings)
|
||||
self.self_attn = WanAnimate2SelfAttention(dim, num_heads, window_size, qk_norm, eps, operation_settings=operation_settings)
|
||||
|
||||
def _modulation(self, e, x):
|
||||
if e.ndim < 4:
|
||||
return (comfy.model_management.cast_to(self.modulation, dtype=x.dtype, device=x.device) + e).chunk(6, dim=1)
|
||||
return (comfy.model_management.cast_to(self.modulation, dtype=x.dtype, device=x.device).unsqueeze(0) + e).unbind(2)
|
||||
|
||||
def _cross_attn_ffn(self, x, e, context, context_img_len, transformer_options):
|
||||
x = x + self.cross_attn(self.norm3(x), context, context_img_len=context_img_len, transformer_options=transformer_options)
|
||||
for p in transformer_options.get("patches", {}).get("attn2_patch", []):
|
||||
x = p({"x": x, "transformer_options": transformer_options})
|
||||
y = self.ffn(torch.addcmul(repeat_e(e[3], x), self.norm2(x), 1 + repeat_e(e[4], x)))
|
||||
return torch.addcmul(x, y, repeat_e(e[5], x))
|
||||
|
||||
def forward_pose(self, x, e, freqs, context, context_img_len=257, transformer_options={}):
|
||||
e = self._modulation(e, x)
|
||||
x = x.contiguous()
|
||||
y, k, v = self.self_attn.forward_pose(torch.addcmul(repeat_e(e[0], x), self.norm1(x), 1 + repeat_e(e[1], x)), freqs, transformer_options=transformer_options)
|
||||
x = torch.addcmul(x, y, repeat_e(e[2], x))
|
||||
del y
|
||||
return self._cross_attn_ffn(x, e, context, context_img_len, transformer_options), k, v
|
||||
|
||||
def kv_from_input(self, x_pose, e, freqs, transformer_options={}):
|
||||
e = self._modulation(e, x_pose)
|
||||
x_pose = x_pose.contiguous()
|
||||
return self.self_attn.kv(torch.addcmul(repeat_e(e[0], x_pose), self.norm1(x_pose), 1 + repeat_e(e[1], x_pose)), freqs)
|
||||
|
||||
def forward_gen(self, x, e, freqs, context, k_pose, v_pose, f_gen, hw, buffers, ref_strength=1.0, context_img_len=257, transformer_options={}):
|
||||
e = self._modulation(e, x)
|
||||
x = x.contiguous()
|
||||
y = self.self_attn.forward_gen(torch.addcmul(repeat_e(e[0], x), self.norm1(x), 1 + repeat_e(e[1], x)), freqs, k_pose, v_pose, f_gen, hw, buffers, ref_strength=ref_strength, transformer_options=transformer_options)
|
||||
x = torch.addcmul(x, y, repeat_e(e[2], x))
|
||||
del y
|
||||
return self._cross_attn_ffn(x, e, context, context_img_len, transformer_options)
|
||||
|
||||
|
||||
class PoseBranchCache:
|
||||
"""Pose-branch block inputs, reused across the sampling steps of one execution.
|
||||
|
||||
Caching the block input rather than its K/V halves the memory; reprojecting K/V on read
|
||||
costs ~4% of re-running the block. One slot per distinct pose sequence, so under
|
||||
context windows each window keeps its own; least recently used slots are evicted when
|
||||
the store device runs low on memory. Created and freed by WanAnimate2Cache.
|
||||
"""
|
||||
|
||||
CONVROT_GROUPSIZE = 256
|
||||
|
||||
def __init__(self, store_device=None, dtype="default"):
|
||||
self.store_device = torch.device(store_device) if store_device is not None else torch.device("cpu")
|
||||
self.dtype = dtype
|
||||
self.slots = [] # most recently used last
|
||||
self.slot = None
|
||||
self._pending = {}
|
||||
self._staging = {}
|
||||
|
||||
def select(self, pose_latents):
|
||||
# select runs at a forward boundary: an interrupted forward can leave copies in flight that a different slot's forward would then mistake for its own
|
||||
if self._pending:
|
||||
for t, stream in self._pending.values():
|
||||
if stream is not None:
|
||||
stream.synchronize()
|
||||
self._pending = {}
|
||||
# keyed on batch element 0, so a cond batch size change mid-run stays valid
|
||||
k = pose_latents[:1]
|
||||
for s in self.slots:
|
||||
if s["key"].shape == k.shape and torch.equal(s["key"], k.to(s["key"].device)):
|
||||
self.slots.remove(s)
|
||||
self.slots.append(s)
|
||||
self.slot = s
|
||||
return
|
||||
# cache what fits: a filled slot is the size estimate for the next one, and least recently used slots make room when the store device runs low
|
||||
est = max((self._slot_bytes(s) for s in self.slots), default=0) * 1.5
|
||||
while self.slots and comfy.model_management.get_free_memory(self.store_device) < est:
|
||||
self._free_slot(self.slots.pop(0))
|
||||
self.slot = {"key": k.clone().to(self.store_device), "blocks": {}, "params": {}, "shape": None, "pinned": []}
|
||||
self.slots.append(self.slot)
|
||||
|
||||
def _free_slot(self, s):
|
||||
for t, stream in self._pending.values():
|
||||
if stream is not None:
|
||||
stream.synchronize() # an aborted forward can leave a copy in flight, still reading memory we are about to unpin
|
||||
self._pending = {}
|
||||
for t in s["pinned"]:
|
||||
comfy.model_management.unpin_memory(t)
|
||||
|
||||
def free(self):
|
||||
for s in self.slots:
|
||||
self._free_slot(s)
|
||||
self.slots = []
|
||||
self.slot = None
|
||||
self._staging = {}
|
||||
|
||||
def filled(self, num_blocks):
|
||||
return self.slot is not None and len(self.slot["blocks"]) == num_blocks
|
||||
|
||||
def put(self, i, x_pose):
|
||||
t = x_pose[:1]
|
||||
params = None
|
||||
if self.dtype in ("int8", "int4"):
|
||||
# convrot is what lets low-bit survive the ~125x per-channel outliers here, and over a [tokens, dim] view per-row scale means per-token. The kernels want 2D and a power-of-4 group that divides dim.
|
||||
self.slot["shape"] = t.shape
|
||||
g = self.CONVROT_GROUPSIZE
|
||||
while g > 4 and t.shape[-1] % g:
|
||||
g //= 4
|
||||
if self.dtype == "int4":
|
||||
t, params = comfy.quant_ops.TensorCoreConvRotW4A4Layout.quantize(t.reshape(-1, t.shape[-1]), convrot_groupsize=g)
|
||||
else:
|
||||
t, params = comfy.quant_ops.TensorWiseINT8Layout.quantize(t.reshape(-1, t.shape[-1]), is_weight=True, per_channel=True, convrot=True, convrot_groupsize=g)
|
||||
|
||||
t = t.to(self.store_device, copy=True)
|
||||
if comfy.model_management.pin_memory(t):
|
||||
self.slot["pinned"].append(t)
|
||||
self.slot["blocks"][i] = t
|
||||
# the scales follow the blocks off the GPU: per-window slots would otherwise pile them up in VRAM (~200 MB per window at 480p int4)
|
||||
self.slot["params"][i] = params if params is None else params.to_device(self.store_device)
|
||||
|
||||
def prefetch(self, i, device, dtype):
|
||||
# call before the compute this should overlap, so the stream waits only on work already enqueued
|
||||
if i not in self.slot["blocks"] or i in self._pending:
|
||||
return
|
||||
t = self.slot["blocks"][i]
|
||||
cast_dtype = None if self.slot["params"][i] is not None else dtype # int8 entries move in their stored dtype and widen in take()
|
||||
stream = None
|
||||
r = None
|
||||
if t.device != device:
|
||||
stream = comfy.model_management.get_offload_stream(device)
|
||||
cs = comfy.model_management.current_stream(device)
|
||||
if stream is not None and cs is not None:
|
||||
# the handed-out stream last waited on the main stream a full rotation ago, which does not cover the previous consumer's reads of this slot; wait now so the copy cannot overwrite a slot still being read
|
||||
stream.wait_stream(cs)
|
||||
# two persistent staging buffers per tensor shape instead of a fresh allocation per block (~29 GB of churn per pass at 720p); windows of different lengths get their own pair
|
||||
buf_key = (tuple(t.shape), cast_dtype if cast_dtype is not None else t.dtype)
|
||||
if buf_key not in self._staging:
|
||||
self._staging[buf_key] = [torch.empty(t.shape, dtype=buf_key[1], device=device) for _ in range(2)]
|
||||
r = self._staging[buf_key][i % 2]
|
||||
self._pending[i] = (comfy.model_management.cast_to(t, cast_dtype, device, non_blocking=True, stream=stream, r=r), stream)
|
||||
|
||||
def take(self, i, device, dtype, batch_size):
|
||||
if i not in self._pending:
|
||||
self.prefetch(i, device, dtype)
|
||||
t, stream = self._pending.pop(i)
|
||||
comfy.model_management.sync_stream(device, stream)
|
||||
params = self.slot["params"][i]
|
||||
if params is not None:
|
||||
layout = comfy.quant_ops.TensorCoreConvRotW4A4Layout if self.dtype == "int4" else comfy.quant_ops.TensorWiseINT8Layout
|
||||
t = layout.dequantize(t, params.to_device(t.device)).reshape(self.slot["shape"]).to(dtype)
|
||||
return comfy.utils.repeat_to_batch_size(t, batch_size)
|
||||
|
||||
def _slot_bytes(self, s):
|
||||
return sum(t.numel() * t.element_size() for t in s["blocks"].values())
|
||||
|
||||
def memory_bytes(self):
|
||||
return sum(self._slot_bytes(s) for s in self.slots)
|
||||
|
||||
|
||||
class WanAnimate2Model(WanModel):
|
||||
|
||||
def __init__(self,
|
||||
model_type='animate2',
|
||||
patch_size=(1, 2, 2),
|
||||
text_len=512,
|
||||
in_dim=36,
|
||||
dim=5120,
|
||||
ffn_dim=13824,
|
||||
freq_dim=256,
|
||||
text_dim=4096,
|
||||
out_dim=16,
|
||||
num_heads=40,
|
||||
num_layers=40,
|
||||
window_size=(-1, -1),
|
||||
qk_norm=True,
|
||||
cross_attn_norm=True,
|
||||
eps=1e-6,
|
||||
flf_pos_embed_token_number=None,
|
||||
in_dim_ref_conv=None,
|
||||
image_model=None,
|
||||
device=None, dtype=None, operations=None,
|
||||
):
|
||||
# model_type is 'animate2' in unet_config, but the checkpoint is i2v-shaped
|
||||
super().__init__(model_type='i2v', patch_size=patch_size, text_len=text_len, in_dim=in_dim, dim=dim, ffn_dim=ffn_dim, freq_dim=freq_dim,
|
||||
text_dim=text_dim, out_dim=out_dim, num_heads=num_heads, num_layers=num_layers, window_size=window_size, qk_norm=qk_norm,
|
||||
cross_attn_norm=cross_attn_norm, eps=eps, flf_pos_embed_token_number=flf_pos_embed_token_number, in_dim_ref_conv=in_dim_ref_conv,
|
||||
wan_attn_block_class=WanAnimate2Block, image_model=image_model, device=device, dtype=dtype, operations=operations)
|
||||
|
||||
def rope_encode_pose(self, t, h, w, w_patches, device=None, dtype=None):
|
||||
# t_start=1 lines pose frame j up with gen frame j+1, past the reference slot; shift_x parks it in its own strip of rope space.
|
||||
# The caller's rope_options are a user scaling knob and deliberately not forwarded.
|
||||
return super().rope_encode(t, h, w, t_start=1, device=device, dtype=dtype, transformer_options={"rope_options": {"shift_x": float(w_patches)}})
|
||||
|
||||
def _forward(self, x, timestep, context, clip_fea=None, time_dim_concat=None, transformer_options={}, pose_latents=None, clip_fea_pose=None, context_pose=None, **kwargs):
|
||||
bs, c, t, h, w = x.shape
|
||||
x = comfy.ldm.common_dit.pad_to_patch_size(x, self.patch_size)
|
||||
|
||||
# h/w pre-pad: rope_encode's rounding reproduces the post-pad grid
|
||||
freqs = self.rope_encode(t, h, w, device=x.device, dtype=x.dtype, transformer_options=transformer_options)
|
||||
freqs_pose = None
|
||||
if pose_latents is not None: # absent when the node's timestep window excludes this step
|
||||
pose_latents = comfy.ldm.common_dit.pad_to_patch_size(pose_latents.to(x.dtype), self.patch_size)
|
||||
w_patches = (w + (self.patch_size[2] // 2)) // self.patch_size[2]
|
||||
freqs_pose = self.rope_encode_pose(pose_latents.shape[2], h, w, w_patches, device=x.device, dtype=x.dtype)
|
||||
|
||||
return self.forward_orig(x, timestep, context, clip_fea=clip_fea, freqs=freqs, freqs_pose=freqs_pose, pose_latents=pose_latents,
|
||||
clip_fea_pose=clip_fea_pose, context_pose=context_pose, transformer_options=transformer_options, **kwargs)[:, :, :t, :h, :w]
|
||||
|
||||
def forward_orig(self, x, t, context, clip_fea=None, freqs=None, freqs_pose=None, pose_latents=None, clip_fea_pose=None, context_pose=None, pose_strength=1.0, reference_strength=1.0, transformer_options={}, **kwargs):
|
||||
x_input = x[:, :, 1:] # video-only: frame 0 is the reference slot, offset past it below
|
||||
x = self.patch_embedding(x.float()).to(x.dtype)
|
||||
grid_sizes = x.shape[2:]
|
||||
transformer_options["grid_sizes"] = grid_sizes
|
||||
f_gen, gh, gw = grid_sizes
|
||||
hw = gh * gw
|
||||
x = x.flatten(2).transpose(1, 2)
|
||||
|
||||
# the node windows the pose influence via cond timestep ranges: outside the window the cond carries no pose latents, and the branch, its cache traffic and the per-frame attention loop are all skipped
|
||||
apply_pose = pose_latents is not None
|
||||
if apply_pose and pose_latents.shape[2] != f_gen - 1: # before cache.select, which would otherwise keep an empty slot keyed to the rejected latents
|
||||
raise ValueError("pose branch has {} latent frames, expected {} (generation frames minus the reference-image slot)".format(pose_latents.shape[2], f_gen - 1))
|
||||
|
||||
cache = transformer_options.get("animate2_cache", None) if apply_pose else None
|
||||
if cache is not None:
|
||||
cache.select(pose_latents)
|
||||
cached = cache is not None and cache.filled(len(self.blocks))
|
||||
|
||||
x_pose = None
|
||||
if not cached and apply_pose:
|
||||
# 36ch = [latents(16) | mask(4) | latents(16)]; latents twice, and the mask is all ones since every pose frame is known
|
||||
x_pose = self.patch_embedding(torch.cat([pose_latents, torch.ones_like(pose_latents[:, :4]), pose_latents], dim=1).float()).to(x.dtype)
|
||||
x_pose = x_pose.flatten(2).transpose(1, 2)
|
||||
|
||||
# time embeddings
|
||||
e = self.time_embedding(sinusoidal_embedding_1d(self.freq_dim, t.flatten()).to(dtype=x.dtype))
|
||||
e = e.reshape(t.shape[0], -1, e.shape[-1])
|
||||
e0 = self.time_projection(e).unflatten(2, (6, self.dim))
|
||||
|
||||
e0_pose = None
|
||||
if apply_pose:
|
||||
t_pose = torch.ones_like(t.flatten())
|
||||
e_pose = self.time_embedding(sinusoidal_embedding_1d(self.freq_dim, t_pose).to(dtype=x.dtype))
|
||||
e_pose = e_pose.reshape(t.shape[0], -1, e_pose.shape[-1])
|
||||
e0_pose = self.time_projection(e_pose).unflatten(2, (6, self.dim))
|
||||
|
||||
context_gen = self.text_embedding(context)
|
||||
|
||||
context_img_len = None
|
||||
if clip_fea is not None:
|
||||
if self.img_emb is not None:
|
||||
context_gen = torch.cat([self.img_emb(clip_fea), context_gen], dim=1)
|
||||
context_img_len = clip_fea.shape[-2]
|
||||
|
||||
context_img_len_pose = None
|
||||
if not cached and apply_pose:
|
||||
context_pose = self.text_embedding(context if context_pose is None else context_pose)
|
||||
clip_fea_pose = clip_fea if clip_fea_pose is None else clip_fea_pose
|
||||
if clip_fea_pose is not None:
|
||||
if self.img_emb is not None:
|
||||
context_pose = torch.cat([self.img_emb(clip_fea_pose), context_pose], dim=1)
|
||||
context_img_len_pose = clip_fea_pose.shape[-2]
|
||||
|
||||
patches_replace = transformer_options.get("patches_replace", {})
|
||||
patches = transformer_options.get("patches", {})
|
||||
blocks_replace = patches_replace.get("dit", {})
|
||||
transformer_options["total_blocks"] = len(self.blocks)
|
||||
transformer_options["block_type"] = "double"
|
||||
|
||||
if cache is not None and not cached and apply_pose and "context_window" in transformer_options:
|
||||
# pose-only prepass, to avoid inflating dynamic VRAM calibration when using multiple context windows
|
||||
for i, block in enumerate(self.blocks):
|
||||
transformer_options["block_index"] = i
|
||||
cache.put(i, x_pose)
|
||||
x_pose = block.forward_pose(x_pose, e0_pose, freqs_pose, context_pose, context_img_len=context_img_len_pose, transformer_options=transformer_options)[0]
|
||||
x_pose = None
|
||||
cached = True
|
||||
|
||||
buffers = None
|
||||
if apply_pose:
|
||||
# allocated once and reused by every block
|
||||
n, d = self.num_heads, self.dim // self.num_heads
|
||||
buffers = (x.new_empty(x.shape[0], x.shape[1] + hw, n, d), x.new_empty(x.shape[0], x.shape[1] + hw, n, d), x.new_empty(x.shape[0], x.shape[1], self.dim))
|
||||
|
||||
for i, block in enumerate(self.blocks):
|
||||
transformer_options["block_index"] = i
|
||||
|
||||
if not apply_pose:
|
||||
k_pose = v_pose = None
|
||||
elif cached:
|
||||
x_pose_in = cache.take(i, x.device, x.dtype, x.shape[0])
|
||||
cache.prefetch(i + 1, x.device, x.dtype) # queue the next block before the gen compute it should overlap
|
||||
k_pose, v_pose = block.kv_from_input(x_pose_in, e0_pose, freqs_pose, transformer_options=transformer_options)
|
||||
del x_pose_in
|
||||
else:
|
||||
if cache is not None:
|
||||
cache.put(i, x_pose)
|
||||
# runs even under a block replace: its state has to reach block i+1
|
||||
x_pose, k_pose, v_pose = block.forward_pose(x_pose, e0_pose, freqs_pose, context_pose, context_img_len=context_img_len_pose, transformer_options=transformer_options)
|
||||
if v_pose is not None and pose_strength != 1.0:
|
||||
v_pose = v_pose * pose_strength
|
||||
|
||||
if ("double_block", i) in blocks_replace:
|
||||
def block_wrap(args, block=block, k_pose=k_pose, v_pose=v_pose):
|
||||
return {"img": block.forward_gen(args["img"], args["vec"], args["pe"], args["txt"], k_pose, v_pose, f_gen, hw, buffers, ref_strength=reference_strength, context_img_len=context_img_len, transformer_options=args["transformer_options"])}
|
||||
out = blocks_replace[("double_block", i)]({"img": x, "txt": context_gen, "vec": e0, "pe": freqs, "transformer_options": transformer_options}, {"original_block": block_wrap})
|
||||
x = out["img"]
|
||||
else:
|
||||
x = block.forward_gen(x, e0, freqs, context_gen, k_pose, v_pose, f_gen, hw, buffers, ref_strength=reference_strength, context_img_len=context_img_len, transformer_options=transformer_options)
|
||||
|
||||
if "double_block" in patches:
|
||||
for p in patches["double_block"]:
|
||||
out = p({"img": x, "x": x_input, "vec": e, "block_index": i, "img_offset": hw, "transformer_options": transformer_options})
|
||||
x = out["img"]
|
||||
|
||||
return self.unpatchify(self.head(x, e), grid_sizes)
|
||||
|
|
@ -46,6 +46,7 @@ import comfy.ldm.cosmos.predict2
|
|||
import comfy.ldm.lumina.model
|
||||
import comfy.ldm.wan.model
|
||||
import comfy.ldm.wan.model_animate
|
||||
import comfy.ldm.wan.model_animate2
|
||||
import comfy.ldm.wan.ar_model
|
||||
import comfy.ldm.wan.model_wandancer
|
||||
import comfy.ldm.hunyuan3d.model
|
||||
|
|
@ -1816,6 +1817,41 @@ class WAN22_Animate(WAN21):
|
|||
return comfy.context_windows.slice_cond(cond_value, window, x_in, device, temporal_dim=2, temporal_offset=1)
|
||||
return super().resize_cond_for_context_window(cond_key, cond_value, window, x_in, device, retain_index_list=retain_index_list)
|
||||
|
||||
class WAN_Animate2(WAN21):
|
||||
def __init__(self, model_config, model_type=ModelType.FLOW, device=None):
|
||||
super(WAN21, self).__init__(model_config, model_type, device=device, unet_model=comfy.ldm.wan.model_animate2.WanAnimate2Model)
|
||||
self.image_to_video = True
|
||||
|
||||
def extra_conds(self, **kwargs):
|
||||
out = super().extra_conds(**kwargs)
|
||||
|
||||
pose_video_latent = kwargs.get("pose_video_latent", None)
|
||||
if pose_video_latent is not None:
|
||||
out['pose_latents'] = comfy.conds.CONDRegular(self.process_latent_in(pose_video_latent))
|
||||
|
||||
clip_vision_output_pose = kwargs.get("clip_vision_output_pose", None)
|
||||
if clip_vision_output_pose is not None:
|
||||
out['clip_fea_pose'] = comfy.conds.CONDRegular(clip_vision_output_pose.penultimate_hidden_states)
|
||||
|
||||
cross_attn_pose = kwargs.get("cross_attn_pose", None)
|
||||
if cross_attn_pose is not None:
|
||||
out['context_pose'] = comfy.conds.CONDRegular(cross_attn_pose)
|
||||
|
||||
pose_strength = kwargs.get("pose_strength", 1.0)
|
||||
if pose_strength != 1.0:
|
||||
out['pose_strength'] = comfy.conds.CONDConstant(pose_strength)
|
||||
|
||||
reference_strength = kwargs.get("reference_strength", 1.0)
|
||||
if reference_strength != 1.0:
|
||||
out['reference_strength'] = comfy.conds.CONDConstant(reference_strength)
|
||||
|
||||
return out
|
||||
|
||||
def resize_cond_for_context_window(self, cond_key, cond_value, window, x_in, device, retain_index_list=[]):
|
||||
if cond_key == "pose_latents":
|
||||
return comfy.context_windows.slice_cond(cond_value, window, x_in, device, temporal_dim=2, temporal_offset=1)
|
||||
return super().resize_cond_for_context_window(cond_key, cond_value, window, x_in, device, retain_index_list=retain_index_list)
|
||||
|
||||
class WAN22_S2V(WAN21):
|
||||
def __init__(self, model_config, model_type=ModelType.FLOW, device=None):
|
||||
super(WAN21, self).__init__(model_config, model_type, device=device, unet_model=comfy.ldm.wan.model.WanModel_S2V)
|
||||
|
|
|
|||
23
comfy/ops.py
23
comfy/ops.py
|
|
@ -1211,6 +1211,26 @@ def _load_quantized_module(module, super_load, state_dict, prefix, local_metadat
|
|||
"quant_group_size": 64,
|
||||
"linear_dtype": layer_conf.get("linear_dtype", params_conf.get("linear_dtype", "int4")),
|
||||
}
|
||||
elif module.quant_format == "asym_w4a8_int8":
|
||||
# int4 weight (packed int8 [N,K/2]) + fp8 per-group scale (weight_s_rel),
|
||||
# fp32 per-channel scale (weight_s_channel) + optional Lloyd-Max codebook.
|
||||
scale = pop_scale("weight_s_rel")
|
||||
if scale is None:
|
||||
raise ValueError(f"Missing W4A8 group scale (weight_s_rel) for layer {layer_name}")
|
||||
if scale.dtype == torch.uint8:
|
||||
scale = scale.view(torch.float8_e4m3fn)
|
||||
params_conf = layer_conf.get("params", {})
|
||||
if not isinstance(params_conf, dict):
|
||||
params_conf = {}
|
||||
scales = {
|
||||
"scale": scale,
|
||||
"s_channel": pop_scale("weight_s_channel"),
|
||||
"codebook": pop_scale("weight_codebook"),
|
||||
"group_size": int(layer_conf.get("group_size", params_conf.get("group_size", 16))),
|
||||
"convrot_groupsize": int(
|
||||
layer_conf.get("convrot_groupsize", params_conf.get("convrot_groupsize", 256))
|
||||
),
|
||||
}
|
||||
else:
|
||||
raise ValueError(f"Unsupported quantization format: {module.quant_format}")
|
||||
|
||||
|
|
@ -1262,6 +1282,9 @@ def _quantized_weight_state_dict(module, sd, prefix, extra_quant_conf=None, extr
|
|||
linear_dtype = getattr(params, "linear_dtype", "int4")
|
||||
if linear_dtype != "int4":
|
||||
quant_conf["linear_dtype"] = linear_dtype
|
||||
elif module.quant_format == "asym_w4a8_int8":
|
||||
quant_conf["group_size"] = getattr(params, "group_size", 16)
|
||||
quant_conf["convrot_groupsize"] = getattr(params, "convrot_groupsize", 256)
|
||||
if extra_quant_conf:
|
||||
quant_conf.update(extra_quant_conf)
|
||||
sd[f"{prefix}comfy_quant"] = torch.tensor(list(json.dumps(quant_conf).encode("utf-8")), dtype=torch.uint8)
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ try:
|
|||
TensorCoreNVFP4Layout as _CKNvfp4Layout,
|
||||
TensorCoreConvRotW4A4Layout as _CKTensorCoreConvRotW4A4Layout,
|
||||
TensorWiseINT8Layout as _CKTensorWiseINT8Layout,
|
||||
AsymW4A8Int8Layout as _CKAsymW4A8Int8Layout,
|
||||
register_layout_op,
|
||||
register_layout_class,
|
||||
get_layout_class,
|
||||
|
|
@ -83,6 +84,9 @@ except ImportError as e:
|
|||
class _CKTensorCoreConvRotW4A4Layout:
|
||||
pass
|
||||
|
||||
class _CKAsymW4A8Int8Layout:
|
||||
pass
|
||||
|
||||
def register_layout_class(name, cls):
|
||||
pass
|
||||
|
||||
|
|
@ -212,7 +216,7 @@ class TensorCoreFP8E5M2Layout(_TensorCoreFP8LayoutBase):
|
|||
TensorCoreFP8Layout = TensorCoreFP8E4M3Layout
|
||||
TensorWiseINT8Layout = _CKTensorWiseINT8Layout
|
||||
TensorCoreConvRotW4A4Layout = _CKTensorCoreConvRotW4A4Layout
|
||||
|
||||
AsymW4A8Int8Layout = _CKAsymW4A8Int8Layout
|
||||
|
||||
# ==============================================================================
|
||||
# Registry
|
||||
|
|
@ -226,6 +230,7 @@ register_layout_class("TensorWiseINT8Layout", _CKTensorWiseINT8Layout)
|
|||
register_layout_class("TensorCoreConvRotW4A4Layout", _CKTensorCoreConvRotW4A4Layout)
|
||||
if _CK_MXFP8_AVAILABLE:
|
||||
register_layout_class("TensorCoreMXFP8Layout", TensorCoreMXFP8Layout)
|
||||
register_layout_class("AsymW4A8Int8Layout", _CKAsymW4A8Int8Layout)
|
||||
|
||||
QUANT_ALGOS = {
|
||||
"float8_e4m3fn": {
|
||||
|
|
@ -268,6 +273,13 @@ QUANT_ALGOS["convrot_w4a4"] = {
|
|||
"quantize_input": False,
|
||||
}
|
||||
|
||||
QUANT_ALGOS["asym_w4a8_int8"] = {
|
||||
"storage_t": torch.int8,
|
||||
"parameters": {"weight_scale"},
|
||||
"comfy_tensor_layout": "AsymW4A8Int8Layout",
|
||||
"quantize_input": False,
|
||||
}
|
||||
|
||||
|
||||
# ==============================================================================
|
||||
# Re-exports for backward compatibility
|
||||
|
|
@ -282,6 +294,7 @@ __all__ = [
|
|||
"TensorCoreNVFP4Layout",
|
||||
"TensorCoreConvRotW4A4Layout",
|
||||
"TensorWiseINT8Layout",
|
||||
"AsymW4A8Int8Layout",
|
||||
"QUANT_ALGOS",
|
||||
"register_layout_op",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -1450,6 +1450,20 @@ class WAN22_Animate(WAN21_T2V):
|
|||
out = model_base.WAN22_Animate(self, device=device)
|
||||
return out
|
||||
|
||||
class WAN_Animate2(WAN21_T2V):
|
||||
unet_config = {
|
||||
"image_model": "wan2.1",
|
||||
"model_type": "animate2",
|
||||
}
|
||||
|
||||
sampling_settings = {
|
||||
"shift": 5.0,
|
||||
}
|
||||
|
||||
def get_model(self, state_dict, prefix="", device=None):
|
||||
out = model_base.WAN_Animate2(self, device=device)
|
||||
return out
|
||||
|
||||
class WAN22_T2V(WAN21_T2V):
|
||||
unet_config = {
|
||||
"image_model": "wan2.1",
|
||||
|
|
@ -2464,6 +2478,7 @@ models = [
|
|||
WAN22_S2V,
|
||||
WAN21_HuMo,
|
||||
WAN22_Animate,
|
||||
WAN_Animate2,
|
||||
WAN21_FlowRVS,
|
||||
WAN21_SCAIL,
|
||||
WAN21_SCAIL2,
|
||||
|
|
|
|||
|
|
@ -95,9 +95,10 @@ class Seedance2TaskCreationRequest(BaseModel):
|
|||
generate_audio: bool | None = Field(None)
|
||||
resolution: str | None = Field(None)
|
||||
ratio: str | None = Field(None)
|
||||
duration: int | None = Field(None, ge=4, le=15)
|
||||
duration: int | None = Field(None)
|
||||
seed: int | None = Field(None, ge=0, le=2147483647)
|
||||
watermark: bool | None = Field(None)
|
||||
output_format: str | None = Field(None)
|
||||
|
||||
|
||||
class TaskCreationResponse(BaseModel):
|
||||
|
|
@ -186,6 +187,10 @@ SEEDANCE2_PRICE_PER_1K_TOKENS = {
|
|||
("dreamina-seedance-2-0-mini", True, "480p"): 0.0021,
|
||||
("dreamina-seedance-2-0-mini", False, "720p"): 0.0035,
|
||||
("dreamina-seedance-2-0-mini", True, "720p"): 0.0021,
|
||||
("dreamina-seedance-2-5-260628", False, "480p"): 0.0107,
|
||||
("dreamina-seedance-2-5-260628", True, "480p"): 0.0064,
|
||||
("dreamina-seedance-2-5-260628", False, "720p"): 0.0107,
|
||||
("dreamina-seedance-2-5-260628", True, "720p"): 0.0064,
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -304,8 +309,31 @@ SEEDANCE2_REF_VIDEO_PIXEL_LIMITS = {
|
|||
"480p": {"min": 409_600, "max": 927_408},
|
||||
"720p": {"min": 409_600, "max": 927_408},
|
||||
},
|
||||
"dreamina-seedance-2-5-260628": {
|
||||
"480p": {"min": 409_600, "max": 8_295_044},
|
||||
"720p": {"min": 409_600, "max": 8_295_044},
|
||||
},
|
||||
}
|
||||
|
||||
SEEDANCE2_REFERENCE_LIMITS_DEFAULT = {
|
||||
"max_images": 9,
|
||||
"max_videos": 3,
|
||||
"max_audios": 3,
|
||||
"max_total_seconds": 15.1,
|
||||
}
|
||||
SEEDANCE2_REFERENCE_LIMITS = {
|
||||
"dreamina-seedance-2-5-260628": {
|
||||
"max_images": 30,
|
||||
"max_videos": 10,
|
||||
"max_audios": 10,
|
||||
"max_total_seconds": 30.1,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def seedance2_reference_limits(model_id: str) -> dict:
|
||||
return SEEDANCE2_REFERENCE_LIMITS.get(model_id, SEEDANCE2_REFERENCE_LIMITS_DEFAULT)
|
||||
|
||||
# The time in this dictionary are given for 10 seconds duration.
|
||||
VIDEO_TASKS_EXECUTION_TIME = {
|
||||
"seedance-1-0-lite-t2v-250428": {
|
||||
|
|
|
|||
|
|
@ -47,6 +47,7 @@ from comfy_api_nodes.apis.bytedance import (
|
|||
Text2ImageTaskCreationRequest,
|
||||
Text2VideoTaskCreationRequest,
|
||||
seedance2_price_per_1k_tokens,
|
||||
seedance2_reference_limits,
|
||||
)
|
||||
from comfy_api_nodes.util import (
|
||||
ApiEndpoint,
|
||||
|
|
@ -101,11 +102,18 @@ BYTEPLUS_TASK_STATUS_ENDPOINT = "/proxy/byteplus/api/v3/contents/generations/tas
|
|||
BYTEPLUS_SEEDANCE2_TASK_STATUS_ENDPOINT = "/proxy/byteplus-seedance2/api/v3/contents/generations/tasks" # + /{task_id}
|
||||
|
||||
SEEDANCE_MODELS = {
|
||||
"Seedance 2.5": "dreamina-seedance-2-5-260628",
|
||||
"Seedance 2.0": "dreamina-seedance-2-0-260128",
|
||||
"Seedance 2.0 Fast": "dreamina-seedance-2-0-fast-260128",
|
||||
"Seedance 2.0 Mini": "dreamina-seedance-2-0-mini",
|
||||
}
|
||||
|
||||
SEEDANCE_MODEL_TOOLTIP = (
|
||||
"Seedance 2.5 for the newest model, videos up to 30 seconds and mp4/mov output; "
|
||||
"Seedance 2.0 for maximum quality and 1080p/4k; Fast for speed optimization; "
|
||||
"Mini for the fastest, lowest-cost generation."
|
||||
)
|
||||
|
||||
DEPRECATED_MODELS = {"seedance-1-0-lite-t2v-250428", "seedance-1-0-lite-i2v-250428"}
|
||||
|
||||
|
||||
|
|
@ -1692,25 +1700,272 @@ def _seedance2_text_inputs(resolutions: list[str], default_ratio: str = "16:9"):
|
|||
]
|
||||
|
||||
|
||||
def _seedance25_text_inputs(with_ratio: bool = True, with_video_editing: bool = False):
|
||||
return [
|
||||
IO.String.Input(
|
||||
"prompt",
|
||||
multiline=True,
|
||||
default="",
|
||||
tooltip="Text prompt for video generation. Put spoken lines in double quotes to steer "
|
||||
"the generated dialogue.",
|
||||
),
|
||||
IO.Combo.Input(
|
||||
"resolution",
|
||||
options=["480p", "720p"],
|
||||
default="720p",
|
||||
tooltip="Resolution of the output video.",
|
||||
),
|
||||
*(
|
||||
[
|
||||
IO.Combo.Input(
|
||||
"ratio",
|
||||
options=["16:9", "4:3", "1:1", "3:4", "9:16", "21:9", "adaptive"],
|
||||
default="16:9",
|
||||
tooltip="Aspect ratio of the output video.",
|
||||
)
|
||||
]
|
||||
if with_ratio
|
||||
else []
|
||||
),
|
||||
IO.Int.Input(
|
||||
"duration",
|
||||
default=5,
|
||||
min=4,
|
||||
max=30,
|
||||
step=1,
|
||||
tooltip="Duration of the output video in seconds (4-30).",
|
||||
display_mode=IO.NumberDisplay.slider,
|
||||
),
|
||||
IO.Boolean.Input(
|
||||
"generate_audio",
|
||||
default=True,
|
||||
tooltip="Enable audio generation for the output video.",
|
||||
),
|
||||
*(
|
||||
[
|
||||
IO.Boolean.Input(
|
||||
"video_editing",
|
||||
default=False,
|
||||
tooltip="Enable when the prompt edits a connected reference video, for example "
|
||||
"replacing an object in it. The output then keeps the source clip's own length "
|
||||
"and aspect ratio, and the duration and ratio widgets are ignored. Leave "
|
||||
"disabled to generate a new video, or to extend one to the duration you set.",
|
||||
)
|
||||
]
|
||||
if with_video_editing
|
||||
else []
|
||||
),
|
||||
IO.Combo.Input(
|
||||
"output_format",
|
||||
options=["mp4"],
|
||||
default="mp4",
|
||||
tooltip="Container format of the output video.",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _seedance25_reference_inputs():
|
||||
return [
|
||||
*_seedance25_text_inputs(with_video_editing=True),
|
||||
IO.Autogrow.Input(
|
||||
"reference_images",
|
||||
template=IO.Autogrow.TemplateNames(
|
||||
IO.Image.Input("reference_image"),
|
||||
names=[f"image_{i}" for i in range(1, 31)],
|
||||
min=0,
|
||||
),
|
||||
),
|
||||
IO.Autogrow.Input(
|
||||
"reference_videos",
|
||||
template=IO.Autogrow.TemplateNames(
|
||||
IO.Video.Input("reference_video"),
|
||||
names=[f"video_{i}" for i in range(1, 11)],
|
||||
min=0,
|
||||
),
|
||||
),
|
||||
IO.Autogrow.Input(
|
||||
"reference_audios",
|
||||
template=IO.Autogrow.TemplateNames(
|
||||
IO.Audio.Input("reference_audio"),
|
||||
names=[f"audio_{i}" for i in range(1, 11)],
|
||||
min=0,
|
||||
),
|
||||
),
|
||||
IO.Boolean.Input(
|
||||
"auto_downscale",
|
||||
default=True,
|
||||
optional=True,
|
||||
tooltip="Automatically downscale reference videos that exceed the model's pixel budget "
|
||||
"for the selected resolution. Aspect ratio is preserved; videos already within limits are untouched.",
|
||||
),
|
||||
IO.Boolean.Input(
|
||||
"auto_upscale",
|
||||
default=False,
|
||||
advanced=True,
|
||||
optional=True,
|
||||
tooltip="Automatically upscale reference videos that are below the model's minimum pixel count "
|
||||
"for the selected resolution. Aspect ratio is preserved; videos already meeting the minimum are "
|
||||
"untouched. Note: upscaling a low-resolution source does not add real detail and may produce "
|
||||
"lower-quality generations.",
|
||||
),
|
||||
IO.Autogrow.Input(
|
||||
"reference_assets",
|
||||
template=IO.Autogrow.TemplateNames(
|
||||
IO.String.Input("reference_asset"),
|
||||
names=[f"asset_{i}" for i in range(1, 31)],
|
||||
min=0,
|
||||
),
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _seedance2_build_request(
|
||||
model: dict,
|
||||
model_id: str,
|
||||
content: list,
|
||||
seed: int,
|
||||
watermark: bool,
|
||||
ratio: str,
|
||||
) -> Seedance2TaskCreationRequest:
|
||||
video_editing = bool(model.get("video_editing"))
|
||||
return Seedance2TaskCreationRequest(
|
||||
model=model_id,
|
||||
content=content,
|
||||
generate_audio=model["generate_audio"],
|
||||
resolution=model["resolution"],
|
||||
ratio="adaptive" if video_editing else ratio,
|
||||
duration=-1 if video_editing else model["duration"],
|
||||
seed=seed,
|
||||
watermark=watermark,
|
||||
output_format=model.get("output_format"),
|
||||
)
|
||||
|
||||
|
||||
_SEEDANCE2_PRICE_EXPR_TEMPLATE = """
|
||||
(
|
||||
$m := widgets.model;
|
||||
$res := $lookup(widgets, "model.resolution");
|
||||
$ratio := $lookup(widgets, "model.ratio");
|
||||
$dur := $lookup(widgets, "model.duration");
|
||||
$auto := $lookup(widgets, "model.video_editing") = true;
|
||||
$hasVideo := __HAS_VIDEO__;
|
||||
$ready := $type($m) = "string" and $type($res) = "string" and ($auto or $type($dur) = "number");
|
||||
$ready ? (
|
||||
$contains($m, "2.5") ? (
|
||||
$is480 := $res = "480p";
|
||||
$perFrame := $ratio = "1:1" ? ($is480 ? 400 : 900) :
|
||||
$ratio = "4:3" ? ($is480 ? 411.25 : 905.6719) :
|
||||
$ratio = "3:4" ? ($is480 ? 411.25 : 905.6719) :
|
||||
$ratio = "21:9" ? ($is480 ? 418.5 : 904.3945) :
|
||||
($is480 ? 400.3125 : 900);
|
||||
$price := $hasVideo ? 0.009152 : 0.015301;
|
||||
$costFor := function($d) { $floor($perFrame * (24 * $d + 1)) / 1000 * $price };
|
||||
$lo := $costFor($auto ? 4 : $dur);
|
||||
$hi := $costFor(($auto ? 30 : $dur) + ($hasVideo ? 30 : 0));
|
||||
$lo = $hi
|
||||
? {"type": "usd", "usd": $lo, "format": {"approximate": true}}
|
||||
: {"type": "range_usd", "min_usd": $lo, "max_usd": $hi, "format": {"approximate": true}}
|
||||
) : (
|
||||
$rate := $res = "4k" ? 195200 :
|
||||
$res = "1080p" ? 48800 :
|
||||
$res = "720p" ? 21600 : 10044;
|
||||
$noVideoPrice := $res = "4k" ? 0.00572 :
|
||||
$res = "1080p" ? 0.011011 :
|
||||
$contains($m, "mini") ? 0.005005 :
|
||||
$contains($m, "fast") ? 0.008008 : 0.01001;
|
||||
$videoPrice := $res = "4k" ? 0.003432 :
|
||||
$res = "1080p" ? 0.006721 :
|
||||
$contains($m, "mini") ? 0.003003 :
|
||||
$contains($m, "fast") ? 0.004719 : 0.006149;
|
||||
$hasVideo
|
||||
? {"type": "range_usd",
|
||||
"min_usd": $ceil($dur * 5 / 3) * $rate * $videoPrice / 1000,
|
||||
"max_usd": (15 + $dur) * $rate * $videoPrice / 1000,
|
||||
"format": {"approximate": true}}
|
||||
: {"type": "usd", "usd": $dur * $rate * $noVideoPrice / 1000,
|
||||
"format": {"approximate": true}}
|
||||
)
|
||||
) : undefined
|
||||
)
|
||||
"""
|
||||
|
||||
|
||||
_SEEDANCE_AUDIO_POLICY_CODE = "OutputAudioSensitiveContentDetected.PolicyViolation"
|
||||
_SEEDANCE_TASK_TYPE_CONSTRAINT_CODE = "InvalidParameter.TaskTypeConstraint"
|
||||
|
||||
|
||||
async def _seedance2_poll_video_task(
|
||||
cls: type[IO.ComfyNode],
|
||||
task_id: str,
|
||||
model_id: str,
|
||||
resolution: str,
|
||||
has_video_input: bool,
|
||||
) -> TaskStatusResponse:
|
||||
try:
|
||||
return await poll_op(
|
||||
cls,
|
||||
ApiEndpoint(path=f"{BYTEPLUS_SEEDANCE2_TASK_STATUS_ENDPOINT}/{task_id}"),
|
||||
response_model=TaskStatusResponse,
|
||||
status_extractor=lambda r: r.status,
|
||||
price_extractor=_seedance2_price_extractor(
|
||||
model_id, has_video_input=has_video_input, resolution=resolution
|
||||
),
|
||||
poll_interval=9,
|
||||
)
|
||||
except Exception as exc:
|
||||
if _SEEDANCE_AUDIO_POLICY_CODE in str(exc):
|
||||
raise ValueError(
|
||||
"The provider rejected the audio track this model generated for the video "
|
||||
"(possible copyright match). The video itself was fine. Turn off generate_audio "
|
||||
"to get a silent video, or adjust the prompt and try again."
|
||||
) from exc
|
||||
if _SEEDANCE_TASK_TYPE_CONSTRAINT_CODE in str(exc):
|
||||
raise ValueError(
|
||||
"Seedance read this prompt as editing the reference video, and an edit always "
|
||||
"takes its duration and aspect ratio from that video. Enable video_editing on "
|
||||
"this node and run again, or reword the prompt so it describes a new video "
|
||||
"rather than a change to the reference one."
|
||||
) from exc
|
||||
raise
|
||||
|
||||
|
||||
def _seedance2_price_badge(with_reference_videos: bool) -> IO.PriceBadge:
|
||||
widgets = ["model", "model.resolution", "model.ratio", "model.duration"]
|
||||
if with_reference_videos:
|
||||
widgets.append("model.video_editing")
|
||||
has_video = (
|
||||
'$exists(inputGroups) and $lookup(inputGroups, "model.reference_videos") > 0'
|
||||
if with_reference_videos
|
||||
else "false"
|
||||
)
|
||||
return IO.PriceBadge(
|
||||
depends_on=IO.PriceBadgeDepends(
|
||||
widgets=widgets,
|
||||
input_groups=["model.reference_videos"] if with_reference_videos else [],
|
||||
),
|
||||
expr=_SEEDANCE2_PRICE_EXPR_TEMPLATE.replace("__HAS_VIDEO__", has_video),
|
||||
)
|
||||
|
||||
|
||||
class ByteDance2TextToVideoNode(IO.ComfyNode):
|
||||
|
||||
@classmethod
|
||||
def define_schema(cls):
|
||||
return IO.Schema(
|
||||
node_id="ByteDance2TextToVideoNode",
|
||||
display_name="ByteDance Seedance 2.0 Text to Video",
|
||||
display_name="ByteDance Seedance 2.5 Text to Video",
|
||||
category="partner/video/ByteDance",
|
||||
description="Generate video using Seedance 2.0 models based on a text prompt.",
|
||||
description="Generate video using Seedance 2.5 or 2.0 models based on a text prompt.",
|
||||
inputs=[
|
||||
IO.DynamicCombo.Input(
|
||||
"model",
|
||||
options=[
|
||||
IO.DynamicCombo.Option("Seedance 2.5", _seedance25_text_inputs()),
|
||||
IO.DynamicCombo.Option("Seedance 2.0", _seedance2_text_inputs(["480p", "720p", "1080p", "4k"])),
|
||||
IO.DynamicCombo.Option("Seedance 2.0 Fast", _seedance2_text_inputs(["480p", "720p"])),
|
||||
IO.DynamicCombo.Option("Seedance 2.0 Mini", _seedance2_text_inputs(["480p", "720p"])),
|
||||
],
|
||||
tooltip="Seedance 2.0 for maximum quality; Fast for speed optimization; "
|
||||
"Mini for the fastest, lowest-cost generation.",
|
||||
tooltip=SEEDANCE_MODEL_TOOLTIP,
|
||||
),
|
||||
IO.Int.Input(
|
||||
"seed",
|
||||
|
|
@ -1739,30 +1994,7 @@ class ByteDance2TextToVideoNode(IO.ComfyNode):
|
|||
IO.Hidden.unique_id,
|
||||
],
|
||||
is_api_node=True,
|
||||
price_badge=IO.PriceBadge(
|
||||
depends_on=IO.PriceBadgeDepends(widgets=["model", "model.resolution", "model.duration"]),
|
||||
expr="""
|
||||
(
|
||||
$rate480 := 10044;
|
||||
$rate720 := 21600;
|
||||
$rate1080 := 48800;
|
||||
$rate4k := 195200;
|
||||
$m := widgets.model;
|
||||
$res := $lookup(widgets, "model.resolution");
|
||||
$dur := $lookup(widgets, "model.duration");
|
||||
$pricePer1K := $res = "4k" ? 0.00572 :
|
||||
$res = "1080p" ? 0.011011 :
|
||||
$contains($m, "mini") ? 0.005005 :
|
||||
$contains($m, "fast") ? 0.008008 : 0.01001;
|
||||
$rate := $res = "4k" ? $rate4k :
|
||||
$res = "1080p" ? $rate1080 :
|
||||
$res = "720p" ? $rate720 :
|
||||
$rate480;
|
||||
$cost := $dur * $rate * $pricePer1K / 1000;
|
||||
{"type": "usd", "usd": $cost, "format": {"approximate": true}}
|
||||
)
|
||||
""",
|
||||
),
|
||||
price_badge=_seedance2_price_badge(with_reference_videos=False),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
|
|
@ -1777,25 +2009,18 @@ class ByteDance2TextToVideoNode(IO.ComfyNode):
|
|||
initial_response = await sync_op(
|
||||
cls,
|
||||
ApiEndpoint(path=BYTEPLUS_TASK_ENDPOINT, method="POST"),
|
||||
data=Seedance2TaskCreationRequest(
|
||||
model=model_id,
|
||||
content=[TaskTextContent(text=model["prompt"])],
|
||||
generate_audio=model["generate_audio"],
|
||||
resolution=model["resolution"],
|
||||
data=_seedance2_build_request(
|
||||
model,
|
||||
model_id,
|
||||
[TaskTextContent(text=model["prompt"])],
|
||||
seed,
|
||||
watermark,
|
||||
ratio=model["ratio"],
|
||||
duration=model["duration"],
|
||||
seed=seed,
|
||||
watermark=watermark,
|
||||
),
|
||||
response_model=TaskCreationResponse,
|
||||
)
|
||||
response = await poll_op(
|
||||
cls,
|
||||
ApiEndpoint(path=f"{BYTEPLUS_SEEDANCE2_TASK_STATUS_ENDPOINT}/{initial_response.id}"),
|
||||
response_model=TaskStatusResponse,
|
||||
status_extractor=lambda r: r.status,
|
||||
price_extractor=_seedance2_price_extractor(model_id, has_video_input=False, resolution=model["resolution"]),
|
||||
poll_interval=9,
|
||||
response = await _seedance2_poll_video_task(
|
||||
cls, initial_response.id, model_id, model["resolution"], has_video_input=False
|
||||
)
|
||||
return IO.NodeOutput(await download_url_to_video_output(response.content.video_url))
|
||||
|
||||
|
|
@ -1806,13 +2031,15 @@ class ByteDance2FirstLastFrameNode(IO.ComfyNode):
|
|||
def define_schema(cls):
|
||||
return IO.Schema(
|
||||
node_id="ByteDance2FirstLastFrameNode",
|
||||
display_name="ByteDance Seedance 2.0 First-Last-Frame to Video",
|
||||
display_name="ByteDance Seedance 2.5 First-Last-Frame to Video",
|
||||
category="partner/video/ByteDance",
|
||||
description="Generate video using Seedance 2.0 from a first frame image and optional last frame image.",
|
||||
description="Generate video using Seedance 2.5 or 2.0 from a first frame image "
|
||||
"and optional last frame image.",
|
||||
inputs=[
|
||||
IO.DynamicCombo.Input(
|
||||
"model",
|
||||
options=[
|
||||
IO.DynamicCombo.Option("Seedance 2.5", _seedance25_text_inputs(with_ratio=False)),
|
||||
IO.DynamicCombo.Option(
|
||||
"Seedance 2.0",
|
||||
_seedance2_text_inputs(["480p", "720p", "1080p", "4k"], default_ratio="adaptive"),
|
||||
|
|
@ -1826,8 +2053,7 @@ class ByteDance2FirstLastFrameNode(IO.ComfyNode):
|
|||
_seedance2_text_inputs(["480p", "720p"], default_ratio="adaptive"),
|
||||
),
|
||||
],
|
||||
tooltip="Seedance 2.0 for maximum quality; Fast for speed optimization; "
|
||||
"Mini for the fastest, lowest-cost generation.",
|
||||
tooltip=SEEDANCE_MODEL_TOOLTIP,
|
||||
),
|
||||
IO.Image.Input(
|
||||
"first_frame",
|
||||
|
|
@ -1880,30 +2106,7 @@ class ByteDance2FirstLastFrameNode(IO.ComfyNode):
|
|||
IO.Hidden.unique_id,
|
||||
],
|
||||
is_api_node=True,
|
||||
price_badge=IO.PriceBadge(
|
||||
depends_on=IO.PriceBadgeDepends(widgets=["model", "model.resolution", "model.duration"]),
|
||||
expr="""
|
||||
(
|
||||
$rate480 := 10044;
|
||||
$rate720 := 21600;
|
||||
$rate1080 := 48800;
|
||||
$rate4k := 195200;
|
||||
$m := widgets.model;
|
||||
$res := $lookup(widgets, "model.resolution");
|
||||
$dur := $lookup(widgets, "model.duration");
|
||||
$pricePer1K := $res = "4k" ? 0.00572 :
|
||||
$res = "1080p" ? 0.011011 :
|
||||
$contains($m, "mini") ? 0.005005 :
|
||||
$contains($m, "fast") ? 0.008008 : 0.01001;
|
||||
$rate := $res = "4k" ? $rate4k :
|
||||
$res = "1080p" ? $rate1080 :
|
||||
$res = "720p" ? $rate720 :
|
||||
$rate480;
|
||||
$cost := $dur * $rate * $pricePer1K / 1000;
|
||||
{"type": "usd", "usd": $cost, "format": {"approximate": true}}
|
||||
)
|
||||
""",
|
||||
),
|
||||
price_badge=_seedance2_price_badge(with_reference_videos=False),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
|
|
@ -1930,8 +2133,17 @@ class ByteDance2FirstLastFrameNode(IO.ComfyNode):
|
|||
if last_frame is not None and last_frame_asset_id:
|
||||
raise ValueError("Provide only one of last_frame or last_frame_asset_id, not both.")
|
||||
|
||||
request_ratio = model["ratio"]
|
||||
if first_frame_asset_id or last_frame_asset_id:
|
||||
if model_id == "dreamina-seedance-2-5-260628":
|
||||
# 2.5 accepts ratio="adaptive" only here and keeps the first frame's own aspect
|
||||
# (a 1920x1088 frame yields 850x482, not a grid ratio), so pre-sizing the frames to a
|
||||
# supported pixel pair would crop framing the model would otherwise have preserved.
|
||||
request_ratio = "adaptive"
|
||||
if first_frame is not None:
|
||||
first_frame = _prepare_seedance_image(first_frame)
|
||||
if last_frame is not None:
|
||||
last_frame = _prepare_seedance_image(last_frame)
|
||||
elif first_frame_asset_id or last_frame_asset_id:
|
||||
request_ratio = model["ratio"]
|
||||
if first_frame is not None:
|
||||
first_frame = _prepare_seedance_image(first_frame)
|
||||
if last_frame is not None:
|
||||
|
|
@ -1997,25 +2209,11 @@ class ByteDance2FirstLastFrameNode(IO.ComfyNode):
|
|||
initial_response = await sync_op(
|
||||
cls,
|
||||
ApiEndpoint(path=BYTEPLUS_TASK_ENDPOINT, method="POST"),
|
||||
data=Seedance2TaskCreationRequest(
|
||||
model=model_id,
|
||||
content=content,
|
||||
generate_audio=model["generate_audio"],
|
||||
resolution=model["resolution"],
|
||||
ratio=request_ratio,
|
||||
duration=model["duration"],
|
||||
seed=seed,
|
||||
watermark=watermark,
|
||||
),
|
||||
data=_seedance2_build_request(model, model_id, content, seed, watermark, ratio=request_ratio),
|
||||
response_model=TaskCreationResponse,
|
||||
)
|
||||
response = await poll_op(
|
||||
cls,
|
||||
ApiEndpoint(path=f"{BYTEPLUS_SEEDANCE2_TASK_STATUS_ENDPOINT}/{initial_response.id}"),
|
||||
response_model=TaskStatusResponse,
|
||||
status_extractor=lambda r: r.status,
|
||||
price_extractor=_seedance2_price_extractor(model_id, has_video_input=False, resolution=model["resolution"]),
|
||||
poll_interval=9,
|
||||
response = await _seedance2_poll_video_task(
|
||||
cls, initial_response.id, model_id, model["resolution"], has_video_input=False
|
||||
)
|
||||
return IO.NodeOutput(await download_url_to_video_output(response.content.video_url))
|
||||
|
||||
|
|
@ -2101,14 +2299,15 @@ class ByteDance2ReferenceNode(IO.ComfyNode):
|
|||
def define_schema(cls):
|
||||
return IO.Schema(
|
||||
node_id="ByteDance2ReferenceNode",
|
||||
display_name="ByteDance Seedance 2.0 Reference to Video",
|
||||
display_name="ByteDance Seedance 2.5 Reference to Video",
|
||||
category="partner/video/ByteDance",
|
||||
description="Generate, edit, or extend video using Seedance 2.0 with reference images, "
|
||||
"videos, and audio. Supports multimodal reference, video editing, and video extension.",
|
||||
description="Generate, edit, or extend video using Seedance 2.5 or 2.0 with reference "
|
||||
"images, videos, and audio. Supports multimodal reference, video editing, and video extension.",
|
||||
inputs=[
|
||||
IO.DynamicCombo.Input(
|
||||
"model",
|
||||
options=[
|
||||
IO.DynamicCombo.Option("Seedance 2.5", _seedance25_reference_inputs()),
|
||||
IO.DynamicCombo.Option(
|
||||
"Seedance 2.0",
|
||||
_seedance2_reference_inputs(["480p", "720p", "1080p", "4k"], default_ratio="adaptive"),
|
||||
|
|
@ -2122,8 +2321,7 @@ class ByteDance2ReferenceNode(IO.ComfyNode):
|
|||
_seedance2_reference_inputs(["480p", "720p"], default_ratio="adaptive"),
|
||||
),
|
||||
],
|
||||
tooltip="Seedance 2.0 for maximum quality; Fast for speed optimization; "
|
||||
"Mini for the fastest, lowest-cost generation.",
|
||||
tooltip=SEEDANCE_MODEL_TOOLTIP,
|
||||
),
|
||||
IO.Int.Input(
|
||||
"seed",
|
||||
|
|
@ -2152,52 +2350,7 @@ class ByteDance2ReferenceNode(IO.ComfyNode):
|
|||
IO.Hidden.unique_id,
|
||||
],
|
||||
is_api_node=True,
|
||||
price_badge=IO.PriceBadge(
|
||||
depends_on=IO.PriceBadgeDepends(
|
||||
widgets=["model", "model.resolution", "model.duration"],
|
||||
input_groups=["model.reference_videos"],
|
||||
),
|
||||
expr="""
|
||||
(
|
||||
$rate480 := 10044;
|
||||
$rate720 := 21600;
|
||||
$rate1080 := 48800;
|
||||
$rate4k := 195200;
|
||||
$m := widgets.model;
|
||||
$hasVideo := $lookup(inputGroups, "model.reference_videos") > 0;
|
||||
$res := $lookup(widgets, "model.resolution");
|
||||
$dur := $lookup(widgets, "model.duration");
|
||||
$noVideoPricePer1K := $res = "4k" ? 0.00572 :
|
||||
$res = "1080p" ? 0.011011 :
|
||||
$contains($m, "mini") ? 0.005005 :
|
||||
$contains($m, "fast") ? 0.008008 : 0.01001;
|
||||
$videoPricePer1K := $res = "4k" ? 0.003432 :
|
||||
$res = "1080p" ? 0.006721 :
|
||||
$contains($m, "mini") ? 0.003003 :
|
||||
$contains($m, "fast") ? 0.004719 : 0.006149;
|
||||
$rate := $res = "4k" ? $rate4k :
|
||||
$res = "1080p" ? $rate1080 :
|
||||
$res = "720p" ? $rate720 :
|
||||
$rate480;
|
||||
$noVideoCost := $dur * $rate * $noVideoPricePer1K / 1000;
|
||||
$minVideoFactor := $ceil($dur * 5 / 3);
|
||||
$minVideoCost := $minVideoFactor * $rate * $videoPricePer1K / 1000;
|
||||
$maxVideoCost := (15 + $dur) * $rate * $videoPricePer1K / 1000;
|
||||
$hasVideo
|
||||
? {
|
||||
"type": "range_usd",
|
||||
"min_usd": $minVideoCost,
|
||||
"max_usd": $maxVideoCost,
|
||||
"format": {"approximate": true}
|
||||
}
|
||||
: {
|
||||
"type": "usd",
|
||||
"usd": $noVideoCost,
|
||||
"format": {"approximate": true}
|
||||
}
|
||||
)
|
||||
""",
|
||||
),
|
||||
price_badge=_seedance2_price_badge(with_reference_videos=True),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
|
|
@ -2218,32 +2371,38 @@ class ByteDance2ReferenceNode(IO.ComfyNode):
|
|||
cls, list(reference_assets.values())
|
||||
)
|
||||
|
||||
model_id = SEEDANCE_MODELS[model["model"]]
|
||||
limits = seedance2_reference_limits(model_id)
|
||||
|
||||
if not reference_images and not reference_videos and not reference_image_assets and not reference_video_assets:
|
||||
raise ValueError("At least one reference image or video or asset is required.")
|
||||
if model_id != "dreamina-seedance-2-5-260628" or not (reference_audios or reference_audio_assets):
|
||||
raise ValueError("At least one reference image or video or asset is required.")
|
||||
|
||||
total_images = len(reference_images) + len(reference_image_assets)
|
||||
if total_images > 9:
|
||||
if total_images > limits["max_images"]:
|
||||
raise ValueError(
|
||||
f"Too many reference images: {total_images} "
|
||||
f"(images={len(reference_images)}, image assets={len(reference_image_assets)}). Maximum is 9."
|
||||
f"(images={len(reference_images)}, image assets={len(reference_image_assets)}). "
|
||||
f"Maximum is {limits['max_images']}."
|
||||
)
|
||||
total_videos = len(reference_videos) + len(reference_video_assets)
|
||||
if total_videos > 3:
|
||||
if total_videos > limits["max_videos"]:
|
||||
raise ValueError(
|
||||
f"Too many reference videos: {total_videos} "
|
||||
f"(videos={len(reference_videos)}, video assets={len(reference_video_assets)}). Maximum is 3."
|
||||
f"(videos={len(reference_videos)}, video assets={len(reference_video_assets)}). "
|
||||
f"Maximum is {limits['max_videos']}."
|
||||
)
|
||||
total_audios = len(reference_audios) + len(reference_audio_assets)
|
||||
if total_audios > 3:
|
||||
if total_audios > limits["max_audios"]:
|
||||
raise ValueError(
|
||||
f"Too many reference audios: {total_audios} "
|
||||
f"(audios={len(reference_audios)}, audio assets={len(reference_audio_assets)}). Maximum is 3."
|
||||
f"(audios={len(reference_audios)}, audio assets={len(reference_audio_assets)}). "
|
||||
f"Maximum is {limits['max_audios']}."
|
||||
)
|
||||
|
||||
for key in reference_images:
|
||||
reference_images[key] = _prepare_seedance_image(reference_images[key])
|
||||
|
||||
model_id = SEEDANCE_MODELS[model["model"]]
|
||||
has_video_input = total_videos > 0
|
||||
|
||||
if model.get("auto_downscale") and reference_videos:
|
||||
|
|
@ -2271,8 +2430,11 @@ class ByteDance2ReferenceNode(IO.ComfyNode):
|
|||
raise
|
||||
except Exception:
|
||||
pass
|
||||
if total_video_duration > 15.1:
|
||||
raise ValueError(f"Total reference video duration is {total_video_duration:.1f}s. Maximum is 15.1 seconds.")
|
||||
if total_video_duration > limits["max_total_seconds"]:
|
||||
raise ValueError(
|
||||
f"Total reference video duration is {total_video_duration:.1f}s. "
|
||||
f"Maximum is {limits['max_total_seconds']} seconds."
|
||||
)
|
||||
|
||||
total_audio_duration = 0.0
|
||||
for i, key in enumerate(reference_audios, 1):
|
||||
|
|
@ -2281,8 +2443,11 @@ class ByteDance2ReferenceNode(IO.ComfyNode):
|
|||
if dur < 1.8:
|
||||
raise ValueError(f"Reference audio {i} is too short: {dur:.1f}s. Minimum duration is 1.8 seconds.")
|
||||
total_audio_duration += dur
|
||||
if total_audio_duration > 15.1:
|
||||
raise ValueError(f"Total reference audio duration is {total_audio_duration:.1f}s. Maximum is 15.1 seconds.")
|
||||
if total_audio_duration > limits["max_total_seconds"]:
|
||||
raise ValueError(
|
||||
f"Total reference audio duration is {total_audio_duration:.1f}s. "
|
||||
f"Maximum is {limits['max_total_seconds']} seconds."
|
||||
)
|
||||
|
||||
asset_labels = _build_asset_labels(
|
||||
reference_assets,
|
||||
|
|
@ -2355,27 +2520,11 @@ class ByteDance2ReferenceNode(IO.ComfyNode):
|
|||
initial_response = await sync_op(
|
||||
cls,
|
||||
ApiEndpoint(path=BYTEPLUS_TASK_ENDPOINT, method="POST"),
|
||||
data=Seedance2TaskCreationRequest(
|
||||
model=model_id,
|
||||
content=content,
|
||||
generate_audio=model["generate_audio"],
|
||||
resolution=model["resolution"],
|
||||
ratio=model["ratio"],
|
||||
duration=model["duration"],
|
||||
seed=seed,
|
||||
watermark=watermark,
|
||||
),
|
||||
data=_seedance2_build_request(model, model_id, content, seed, watermark, ratio=model["ratio"]),
|
||||
response_model=TaskCreationResponse,
|
||||
)
|
||||
response = await poll_op(
|
||||
cls,
|
||||
ApiEndpoint(path=f"{BYTEPLUS_SEEDANCE2_TASK_STATUS_ENDPOINT}/{initial_response.id}"),
|
||||
response_model=TaskStatusResponse,
|
||||
status_extractor=lambda r: r.status,
|
||||
price_extractor=_seedance2_price_extractor(
|
||||
model_id, has_video_input=has_video_input, resolution=model["resolution"]
|
||||
),
|
||||
poll_interval=9,
|
||||
response = await _seedance2_poll_video_task(
|
||||
cls, initial_response.id, model_id, model["resolution"], has_video_input=has_video_input
|
||||
)
|
||||
return IO.NodeOutput(await download_url_to_video_output(response.content.video_url))
|
||||
|
||||
|
|
|
|||
|
|
@ -76,8 +76,8 @@ GEMINI_IMAGE_2_PRICE_BADGE = IO.PriceBadge(
|
|||
$m := widgets.model;
|
||||
$r := widgets.resolution;
|
||||
$isFlash := $contains($m, "nano banana 2");
|
||||
$flashPrices := {"1k": 0.0696, "2k": 0.1014, "4k": 0.154};
|
||||
$proPrices := {"1k": 0.134, "2k": 0.134, "4k": 0.24};
|
||||
$flashPrices := {"1k": 0.0835, "2k": 0.1217, "4k": 0.1848};
|
||||
$proPrices := {"1k": 0.1608, "2k": 0.1608, "4k": 0.288};
|
||||
$prices := $isFlash ? $flashPrices : $proPrices;
|
||||
{"type":"usd","usd": $lookup($prices, $r), "format":{"suffix":"/Image","approximate":true}}
|
||||
)
|
||||
|
|
@ -1464,10 +1464,10 @@ class GeminiNanoBanana2V2(IO.ComfyNode):
|
|||
expr="""
|
||||
(
|
||||
$contains(widgets.model, "lite")
|
||||
? {"type":"usd","usd": 0.034, "format":{"suffix":"/Image","approximate":true}}
|
||||
? {"type":"usd","usd": 0.0408, "format":{"suffix":"/Image","approximate":true}}
|
||||
: (
|
||||
$r := $lookup(widgets, "model.resolution");
|
||||
$prices := {"1k": 0.0696, "2k": 0.1014, "4k": 0.154};
|
||||
$prices := {"1k": 0.0835, "2k": 0.1217, "4k": 0.1848};
|
||||
{"type":"usd","usd": $lookup($prices, $r), "format":{"suffix":"/Image","approximate":true}}
|
||||
)
|
||||
)
|
||||
|
|
|
|||
|
|
@ -494,9 +494,9 @@ class OpenAIGPTImage1(IO.ComfyNode):
|
|||
"high": [0.133, 0.22]
|
||||
},
|
||||
"gpt-image-2": {
|
||||
"low": [0.0048, 0.019],
|
||||
"medium": [0.041, 0.168],
|
||||
"high": [0.165, 0.67]
|
||||
"low": [0.0058, 0.0228],
|
||||
"medium": [0.0492, 0.2016],
|
||||
"high": [0.198, 0.804]
|
||||
}
|
||||
};
|
||||
$range := $lookup($lookup($ranges, widgets.model), widgets.quality);
|
||||
|
|
@ -792,9 +792,9 @@ class OpenAIGPTImageNodeV2(IO.ComfyNode):
|
|||
"high": [0.133, 0.22]
|
||||
},
|
||||
"gpt-image-2": {
|
||||
"low": [0.0048, 0.019],
|
||||
"medium": [0.041, 0.168],
|
||||
"high": [0.165, 0.67]
|
||||
"low": [0.0058, 0.0228],
|
||||
"medium": [0.0492, 0.2016],
|
||||
"high": [0.198, 0.804]
|
||||
}
|
||||
};
|
||||
$range := $lookup($lookup($ranges, widgets.model), $lookup(widgets, "model.quality"));
|
||||
|
|
|
|||
|
|
@ -6,6 +6,8 @@ import comfy.model_management
|
|||
import comfy.utils
|
||||
import comfy.latent_formats
|
||||
import comfy.clip_vision
|
||||
import comfy.ldm.wan.model_animate2
|
||||
import comfy.patcher_extension
|
||||
import json
|
||||
import numpy as np
|
||||
from typing import Tuple, TypedDict
|
||||
|
|
@ -1248,6 +1250,167 @@ class WanAnimateToVideo(io.ComfyNode):
|
|||
out_latent["samples"] = latent
|
||||
return io.NodeOutput(positive, negative, out_latent, trim_latent, max(0, ref_motion_latent_length * 4 - 3), video_frame_offset + length)
|
||||
|
||||
class WanAnimate2ToVideo(io.ComfyNode):
|
||||
|
||||
CONTINUE_MOTION_FRAMES = 1
|
||||
|
||||
@classmethod
|
||||
def define_schema(cls):
|
||||
return io.Schema(
|
||||
node_id="WanAnimate2ToVideo",
|
||||
category="model/conditioning/wan/animate",
|
||||
description="Animate a character in a reference image using a video, effectively replicating the facial animation, body motion and hands gesture from the video.",
|
||||
inputs=[
|
||||
io.Conditioning.Input("positive"),
|
||||
io.Conditioning.Input("negative"),
|
||||
io.Vae.Input("vae"),
|
||||
io.Int.Input("width", default=832, min=16, max=nodes.MAX_RESOLUTION, step=16, tooltip="Output video width in pixels."),
|
||||
io.Int.Input("height", default=480, min=16, max=nodes.MAX_RESOLUTION, step=16, tooltip="Output video height in pixels."),
|
||||
io.Int.Input("length", default=81, min=1, max=nodes.MAX_RESOLUTION, step=4, tooltip="Number of frames to generate."),
|
||||
io.Int.Input("batch_size", default=1, min=1, max=4096, tooltip="Number of videos to generate simultaneously."),
|
||||
io.Image.Input("reference_image", optional=True, tooltip="The character to animate."),
|
||||
io.Image.Input("pose_video", optional=True, tooltip="The video whose motion is transferred to the reference character."),
|
||||
io.ClipVisionOutput.Input("clip_vision_output", optional=True, tooltip="CLIP vision of the reference image."),
|
||||
io.Conditioning.Input("positive_pose", optional=True, tooltip="Prompt for the pose-video branch, describing the motion rather than the character. Defaults to positive. Used for both the cond and uncond passes."),
|
||||
io.ClipVisionOutput.Input("clip_vision_output_pose", optional=True, tooltip="CLIP vision of the pose video's first frame. Defaults to clip_vision_output."),
|
||||
io.Image.Input("continue_motion", optional=True, tooltip="Previous motion sequence to continue from for temporal consistency."),
|
||||
io.Int.Input("video_frame_offset", default=0, min=0, max=nodes.MAX_RESOLUTION, step=1, tooltip="Frames to seek into the pose video. Connect to the video_frame_offset output of the previous node when extending."),
|
||||
io.Float.Input("pose_strength", default=1.0, min=0.0, max=10.0, step=0.01, tooltip="Scales the pose video's influence on the motion. 1.0 is the trained behavior; below weakens adherence, above amplifies. 0.0 mutes it but does not fully remove it."),
|
||||
io.Float.Input("pose_start_percent", default=0.0, min=0.0, max=1.0, step=0.01, tooltip="Sampling percent at which the pose influence starts. Outside the window the pose branch is skipped entirely, which also speeds those steps up."),
|
||||
io.Float.Input("pose_end_percent", default=1.0, min=0.0, max=1.0, step=0.01, tooltip="Sampling percent at which the pose influence ends. Motion is mostly established early, so e.g. 0.7 can loosen fine detail while keeping the choreography."),
|
||||
io.Float.Input("reference_image_strength", default=1.0, min=0.0, max=10.0, step=0.01, tooltip="Scales how strongly generated frames attend to the reference image's latent frame. Below 1.0 loosens identity/appearance adherence (e.g. to let the prompt restyle), above tightens it against drift."),
|
||||
],
|
||||
outputs=[
|
||||
io.Conditioning.Output(display_name="positive"),
|
||||
io.Conditioning.Output(display_name="negative"),
|
||||
io.Latent.Output(display_name="latent"),
|
||||
io.Int.Output(display_name="trim_latent", tooltip="Number of latent frames that should be trimmed before decoding."),
|
||||
io.Int.Output(display_name="trim_image", tooltip="Number of overlapping image frames when extending a video."),
|
||||
io.Int.Output(display_name="video_frame_offset", tooltip="Frames to seek into the pose video."),
|
||||
],
|
||||
is_experimental=True,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def execute(cls, positive, negative, vae, width, height, length, batch_size, video_frame_offset, reference_image=None, pose_video=None, clip_vision_output=None, positive_pose=None, clip_vision_output_pose=None, continue_motion=None, pose_strength=1.0, pose_start_percent=0.0, pose_end_percent=1.0, reference_image_strength=1.0) -> io.NodeOutput:
|
||||
if pose_start_percent > pose_end_percent:
|
||||
raise ValueError("pose_start_percent ({}) must not be greater than pose_end_percent ({}).".format(pose_start_percent, pose_end_percent))
|
||||
latent_length = ((length - 1) // 4) + 1
|
||||
latent_width = width // 8
|
||||
latent_height = height // 8
|
||||
|
||||
if reference_image is None:
|
||||
reference_image = torch.zeros((1, height, width, 3))
|
||||
|
||||
ref_image = comfy.utils.common_upscale(reference_image[:1].movedim(-1, 1), width, height, "area", "center").movedim(1, -1)
|
||||
ref_latent = vae.encode(ref_image[:, :, :, :3])
|
||||
trim_latent = ref_latent.shape[2]
|
||||
|
||||
ref_motion_latent_length = 0
|
||||
if continue_motion is None:
|
||||
image = torch.ones((length, height, width, 3)) * 0.5
|
||||
else:
|
||||
continue_motion = continue_motion[-cls.CONTINUE_MOTION_FRAMES:]
|
||||
video_frame_offset = max(0, video_frame_offset - continue_motion.shape[0])
|
||||
continue_motion = comfy.utils.common_upscale(continue_motion[-length:].movedim(-1, 1), width, height, "area", "center").movedim(1, -1)
|
||||
# 0.5 is mid-grey, matching upstream's zeros in [-1, 1] pixel space
|
||||
image = torch.ones((length, height, width, continue_motion.shape[-1]), device=continue_motion.device, dtype=continue_motion.dtype) * 0.5
|
||||
image[:continue_motion.shape[0]] = continue_motion
|
||||
ref_motion_latent_length += ((continue_motion.shape[0] - 1) // 4) + 1
|
||||
|
||||
concat_latent_image = torch.cat((ref_latent, vae.encode(image[:, :, :, :3])), dim=2)
|
||||
|
||||
# 1-channel, 0 == known; concat_cond inverts and repeats it to the 4 mask channels
|
||||
mask = torch.ones((1, 1, latent_length + trim_latent, latent_height, latent_width), device=concat_latent_image.device, dtype=concat_latent_image.dtype)
|
||||
mask[:, :, :trim_latent + ref_motion_latent_length] = 0.0
|
||||
|
||||
positive = node_helpers.conditioning_set_values(positive, {"concat_latent_image": concat_latent_image, "concat_mask": mask})
|
||||
negative = node_helpers.conditioning_set_values(negative, {"concat_latent_image": concat_latent_image, "concat_mask": mask})
|
||||
|
||||
if clip_vision_output is not None:
|
||||
positive = node_helpers.conditioning_set_values(positive, {"clip_vision_output": clip_vision_output})
|
||||
negative = node_helpers.conditioning_set_values(negative, {"clip_vision_output": clip_vision_output})
|
||||
|
||||
# not windowed with the pose values: the reference frame is part of the latent on every step
|
||||
if reference_image_strength != 1.0:
|
||||
positive = node_helpers.conditioning_set_values(positive, {"reference_strength": reference_image_strength})
|
||||
negative = node_helpers.conditioning_set_values(negative, {"reference_strength": reference_image_strength})
|
||||
|
||||
# set on the negative too: upstream runs the pose branch once, outside the CFG loop, so it never sees the negative prompt
|
||||
pose_values = {}
|
||||
if pose_video is not None:
|
||||
if pose_video.shape[0] <= video_frame_offset:
|
||||
raise ValueError("pose_video has {} frames but video_frame_offset is {} -- nothing left to read.".format(pose_video.shape[0], video_frame_offset))
|
||||
pose_video = pose_video[video_frame_offset:]
|
||||
pose_video = comfy.utils.common_upscale(pose_video[:length].movedim(-1, 1), width, height, "area", "center").movedim(1, -1)
|
||||
if pose_video.shape[0] < length: # hold the last frame, as upstream pads its clips
|
||||
pose_video = torch.cat((pose_video,) + (pose_video[-1:],) * (length - pose_video.shape[0]), dim=0)
|
||||
pose_values["pose_video_latent"] = vae.encode(pose_video[:, :, :, :3])
|
||||
|
||||
pose_clip = clip_vision_output_pose if clip_vision_output_pose is not None else clip_vision_output
|
||||
if pose_clip is not None:
|
||||
pose_values["clip_vision_output_pose"] = pose_clip
|
||||
|
||||
pose_cond = positive_pose if positive_pose is not None else positive
|
||||
if len(pose_cond) > 0:
|
||||
pose_values["cross_attn_pose"] = pose_cond[0][0]
|
||||
|
||||
if pose_strength != 1.0:
|
||||
pose_values["pose_strength"] = pose_strength
|
||||
|
||||
if pose_start_percent > 0.0 or pose_end_percent < 1.0:
|
||||
# windowed via cond timestep ranges: the pose values ride a cond limited to the window, and complement conds without them cover the rest, where the model runs without the pose branch at all
|
||||
def windowed(cond):
|
||||
parts = node_helpers.conditioning_set_values(cond, {**pose_values, "start_percent": pose_start_percent, "end_percent": pose_end_percent})
|
||||
if pose_start_percent > 0.0:
|
||||
parts = parts + node_helpers.conditioning_set_values(cond, {"start_percent": 0.0, "end_percent": pose_start_percent})
|
||||
if pose_end_percent < 1.0:
|
||||
parts = parts + node_helpers.conditioning_set_values(cond, {"start_percent": pose_end_percent, "end_percent": 1.0})
|
||||
return parts
|
||||
positive = windowed(positive)
|
||||
negative = windowed(negative)
|
||||
else:
|
||||
positive = node_helpers.conditioning_set_values(positive, pose_values)
|
||||
negative = node_helpers.conditioning_set_values(negative, pose_values)
|
||||
|
||||
latent = torch.zeros([batch_size, 16, latent_length + trim_latent, latent_height, latent_width], device=comfy.model_management.intermediate_device())
|
||||
out_latent = {}
|
||||
out_latent["samples"] = latent
|
||||
return io.NodeOutput(positive, negative, out_latent, trim_latent, max(0, ref_motion_latent_length * 4 - 3), video_frame_offset + length)
|
||||
|
||||
|
||||
class WanAnimate2Cache(io.ComfyNode):
|
||||
@classmethod
|
||||
def define_schema(cls):
|
||||
return io.Schema(
|
||||
node_id="WanAnimate2Cache",
|
||||
category="model/conditioning/wan/animate",
|
||||
description=(
|
||||
"Caches the pose-video's per-block activations so it runs once instead of on every sampling step. Roughly halves generation time "
|
||||
"Tradeoff is ~12.5 GB of system RAM at 480x832/81 frames in bf16 (scales with resolution and length). "
|
||||
"With context windows each window is cached separately, so RAM scales with the window count; use the static_standard schedule, as uniform schedules shift the windows every step and nothing ever recurs to hit the cache."
|
||||
),
|
||||
inputs=[
|
||||
io.Model.Input("model"),
|
||||
io.Combo.Input("device", options=["cpu", "gpu"], default="cpu",
|
||||
tooltip="Where to keep the cache. cpu (RAM) is the safe choice, the cache will not fit in VRAM alongside the model at typical sizes. gpu (VRAM) can be faster if it fits."),
|
||||
io.Combo.Input("dtype", options=["default", "int8", "int4"], default="default",
|
||||
tooltip="Storage precision. default stores the activations in the model's compute dtype. int8 halves the cache, int4 quarters it, convrot is used to retain accuracy."),
|
||||
],
|
||||
outputs=[io.Model.Output()],
|
||||
is_experimental=True,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def execute(cls, model, device, dtype="default") -> io.NodeOutput:
|
||||
store = comfy.model_management.get_torch_device() if device == "gpu" else torch.device("cpu")
|
||||
cache = comfy.ldm.wan.model_animate2.PoseBranchCache(store_device=store, dtype=dtype)
|
||||
m = model.clone()
|
||||
m.model_options["transformer_options"]["animate2_cache"] = cache
|
||||
m.add_callback(comfy.patcher_extension.CallbacksMP.ON_CLEANUP, lambda patcher: cache.free())
|
||||
return io.NodeOutput(m)
|
||||
|
||||
|
||||
class Wan22ImageToVideoLatent(io.ComfyNode):
|
||||
@classmethod
|
||||
def define_schema(cls):
|
||||
|
|
@ -1475,6 +1638,8 @@ class WanExtension(ComfyExtension):
|
|||
WanSoundImageToVideoExtend,
|
||||
WanHuMoImageToVideo,
|
||||
WanAnimateToVideo,
|
||||
WanAnimate2ToVideo,
|
||||
WanAnimate2Cache,
|
||||
Wan22ImageToVideoLatent,
|
||||
WanInfiniteTalkToVideo,
|
||||
]
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ alembic
|
|||
SQLAlchemy>=2.0.0
|
||||
filelock
|
||||
av>=16.0.0
|
||||
comfy-kitchen==0.2.26
|
||||
comfy-kitchen==0.2.27
|
||||
comfy-aimdo==0.4.13
|
||||
requests
|
||||
simpleeval>=1.0.0
|
||||
|
|
|
|||
Loading…
Reference in New Issue