From d8e6aa55f377d9914e64bf07d7633dfe6abbf643 Mon Sep 17 00:00:00 2001 From: Alexander Piskun <13381981+bigcat88@users.noreply.github.com> Date: Sat, 1 Aug 2026 20:30:00 +0300 Subject: [PATCH] [Partner Nodes] feat(xAI): update nodes for grok-imagine-video-1.5 model (#15197) * [Partner Nodes] feat(xAI): update nodes for grok-imagine-video-1.5 model Signed-off-by: Alexander Piskun --- comfy_api_nodes/apis/grok.py | 5 ++ comfy_api_nodes/nodes_grok.py | 158 +++++++++++++++++++++++++++++++--- 2 files changed, 152 insertions(+), 11 deletions(-) diff --git a/comfy_api_nodes/apis/grok.py b/comfy_api_nodes/apis/grok.py index fbedb53e0..526d8c8ab 100644 --- a/comfy_api_nodes/apis/grok.py +++ b/comfy_api_nodes/apis/grok.py @@ -15,6 +15,10 @@ class InputUrlObject(BaseModel): url: str = Field(...) +class VoiceReferenceObject(BaseModel): + voice_id: str = Field(...) + + class ImageEditRequest(BaseModel): model: str = Field(...) images: list[InputUrlObject] = Field(...) @@ -31,6 +35,7 @@ class VideoGenerationRequest(BaseModel): prompt: str = Field(...) image: InputUrlObject | None = Field(None) reference_images: list[InputUrlObject] | None = Field(None) + reference_audios: list[VoiceReferenceObject] | None = Field(None) duration: int = Field(...) aspect_ratio: str | None = Field(...) resolution: str = Field(...) diff --git a/comfy_api_nodes/nodes_grok.py b/comfy_api_nodes/nodes_grok.py index a95b35917..672a3e537 100644 --- a/comfy_api_nodes/nodes_grok.py +++ b/comfy_api_nodes/nodes_grok.py @@ -1,3 +1,5 @@ +import re + import torch from typing_extensions import override @@ -12,6 +14,7 @@ from comfy_api_nodes.apis.grok import ( VideoGenerationRequest, VideoGenerationResponse, VideoStatusResponse, + VoiceReferenceObject, ) from comfy_api_nodes.util import ( ApiEndpoint, @@ -33,6 +36,75 @@ _GROK_VIDEO_MODEL_API_IDS = { "grok-imagine-video-1.5": "grok-imagine-video-1.5", } +_GROK_VOICE_OPTIONS = [ + "none", + "ara", + "eve", + "leo", + "rex", + "sal", + "carina", + "zagan", + "helix", + "orion", + "luna", + "iris", + "altair", + "zenith", + "perseus", + "helios", + "lux", + "kepler", + "rigel", + "cosmo", + "celeste", + "ursa", + "sirius", + "lumen", + "castor", + "naksh", + "atlas", +] + + +_GROK_REF_TAG_RE = re.compile(r"(?\d*)(?!\w)", re.IGNORECASE | re.ASCII) + + +def _normalize_grok_reference_prompt(prompt: str, total_images: int, voices: list[str]) -> str: + """Rewrite @Image1/@Audio1 style references (1-based, shared partner-node syntax) + into Grok's native / tags; an unnumbered @image/@audio means the first one. + Native tags pass through untouched. @ImageN refers to the Nth reference image overall, in + input order — a batched input contributes one number per image. @AudioN refers to the + 'voice_N' widget; the API only accepts compact arrays, so voices are remapped to array + positions and 'none' slots between selected voices are harmless. Substitution repeats until + stable so adjacent tags like '@Image1@Image2' all resolve.""" + audio_indices: dict[int, int] = {} + for slot, voice in enumerate(voices, start=1): + if voice != "none": + audio_indices[slot] = len(audio_indices) + + def repl(match: re.Match) -> str: + kind = match.group(1).lower() + idx = int(match.group("idx") or 1) + if kind == "image": + if not 1 <= idx <= total_images: + raise ValueError( + f"The prompt references @Image{idx}, but only {total_images} " + f"reference images are connected (a batched input counts once per image)." + ) + return f"" + if idx not in audio_indices: + if 1 <= idx <= len(voices): + raise ValueError(f"The prompt references @Audio{idx}, but 'voice_{idx}' is set to 'none'.") + raise ValueError(f"The prompt references @Audio{idx}, but only voices 1..{len(voices)} exist.") + return f"" + + prev = None + while prev != prompt: + prev = prompt + prompt = _GROK_REF_TAG_RE.sub(repl, prompt) + return prompt + def _extract_grok_price(response) -> float | None: if response.usage and response.usage.cost_in_usd_ticks is not None: @@ -509,12 +581,13 @@ class GrokVideoNode(IO.ComfyNode): IO.Combo.Input( "model", options=["grok-imagine-video", "grok-imagine-video-1.5"], - tooltip="grok-imagine-video-1.5 currently always requires an input image.", + tooltip="The model to use for video generation.", ), IO.String.Input( "prompt", multiline=True, - tooltip="Text description of the desired video.", + tooltip="Text description of the desired video. " + "Optional for grok-imagine-video-1.5 when an input image is provided.", ), IO.Combo.Input( "resolution", @@ -549,7 +622,7 @@ class GrokVideoNode(IO.ComfyNode): IO.Image.Input( "image", optional=True, - tooltip="Optional starting image for grok-imagine-video. Required for grok-imagine-video-1.5.", + tooltip="Optional starting image. If omitted, the video is generated from the text prompt alone.", ), ], outputs=[ @@ -589,8 +662,6 @@ class GrokVideoNode(IO.ComfyNode): seed: int, image: Input.Image | None = None, ) -> IO.NodeOutput: - if image is None and model == "grok-imagine-video-1.5": - raise ValueError(f"The '{model}' model requires an input image; connect one to the 'image' input.") if resolution == "1080p" and model != "grok-imagine-video-1.5": raise ValueError(f"1080p resolution is only available for grok-imagine-video-1.5, not '{model}'.") image_url = None @@ -598,7 +669,8 @@ class GrokVideoNode(IO.ComfyNode): if get_number_of_images(image) != 1: raise ValueError("Only one input image is supported.") image_url = InputUrlObject(url=f"data:image/png;base64,{tensor_to_base64_string(image)}") - validate_string(prompt, strip_whitespace=True, min_length=1) + if image is None or model != "grok-imagine-video-1.5": + validate_string(prompt, strip_whitespace=True, min_length=1) initial_response = await sync_op( cls, ApiEndpoint(path="/proxy/xai/v1/videos/generations", method="POST"), @@ -709,7 +781,7 @@ class GrokVideoReferenceNode(IO.ComfyNode): node_id="GrokVideoReferenceNode", display_name="Grok Reference-to-Video", category="partner/video/Grok", - description="Generate video guided by reference images as style and content references.", + description="Generate video guided by reference images, with optional preset voice references.", inputs=[ IO.String.Input( "prompt", @@ -719,6 +791,57 @@ class GrokVideoReferenceNode(IO.ComfyNode): IO.DynamicCombo.Input( "model", options=[ + IO.DynamicCombo.Option( + "grok-imagine-video-1.5", + [ + IO.Autogrow.Input( + "reference_images", + template=IO.Autogrow.TemplateNames( + IO.Image.Input("image"), + names=[f"reference_{i}" for i in range(1, 8)], + min=1, + ), + tooltip="Up to 7 reference images to guide the video generation. " + "Refer to them in the prompt as @Image1 ... @Image7, numbered " + "in input order; a batched input counts once per image.", + ), + IO.Combo.Input( + "voice_1", + options=_GROK_VOICE_OPTIONS, + tooltip="Optional preset voice reference; refer to it in the prompt as @Audio1. " + "The API supports only these preset voices, not custom audio.", + ), + IO.Combo.Input( + "voice_2", + options=_GROK_VOICE_OPTIONS, + tooltip="Optional second voice reference; @Audio2 in the prompt.", + ), + IO.Combo.Input( + "voice_3", + options=_GROK_VOICE_OPTIONS, + tooltip="Optional third voice reference; @Audio3 in the prompt.", + ), + IO.Combo.Input( + "resolution", + options=["480p", "720p"], + tooltip="The resolution of the output video.", + ), + IO.Combo.Input( + "aspect_ratio", + options=["16:9", "4:3", "3:2", "1:1", "2:3", "3:4", "9:16"], + tooltip="The aspect ratio of the output video.", + ), + IO.Int.Input( + "duration", + default=6, + min=1, + max=15, + step=1, + tooltip="The duration of the output video in seconds.", + display_mode=IO.NumberDisplay.slider, + ), + ], + ), IO.DynamicCombo.Option( "grok-imagine-video", [ @@ -779,16 +902,20 @@ class GrokVideoReferenceNode(IO.ComfyNode): is_api_node=True, price_badge=IO.PriceBadge( depends_on=IO.PriceBadgeDepends( - widgets=["model.duration", "model.resolution"], + widgets=["model", "model.duration", "model.resolution"], input_groups=["model.reference_images"], ), expr=""" ( + $is15 := $contains(widgets.model, "1.5"); $res := $lookup(widgets, "model.resolution"); $dur := $lookup(widgets, "model.duration"); $refs := $lookup(inputGroups, "model.reference_images"); - $rate := $res = "720p" ? 0.07 : 0.05; - $price := ($rate * $dur + 0.002 * $refs) * 1.43; + $rate := $is15 + ? ($res = "720p" ? 0.14 : 0.08) + : ($res = "720p" ? 0.07 : 0.05); + $imgCost := $is15 ? 0.01 : 0.002; + $price := ($rate * $dur + $imgCost * $refs) * 1.43; {"type":"usd","usd": $price} ) """, @@ -803,6 +930,14 @@ class GrokVideoReferenceNode(IO.ComfyNode): seed: int, ) -> IO.NodeOutput: validate_string(prompt, strip_whitespace=True, min_length=1) + total_images = sum(get_number_of_images(t) for t in model["reference_images"].values()) + if total_images > 7: + raise ValueError(f"A maximum of 7 reference images is supported; {total_images} are connected.") + reference_audios = None + if model["model"] == "grok-imagine-video-1.5": + voices = [model.get(f"voice_{i}", "none") for i in range(1, 4)] + reference_audios = [VoiceReferenceObject(voice_id=v) for v in voices if v != "none"] or None + prompt = _normalize_grok_reference_prompt(prompt, total_images=total_images, voices=voices) ref_image_urls = await upload_images_to_comfyapi( cls, list(model["reference_images"].values()), @@ -814,8 +949,9 @@ class GrokVideoReferenceNode(IO.ComfyNode): cls, ApiEndpoint(path="/proxy/xai/v1/videos/generations", method="POST"), data=VideoGenerationRequest( - model=model["model"], + model=_GROK_VIDEO_MODEL_API_IDS.get(model["model"], model["model"]), reference_images=[InputUrlObject(url=i) for i in ref_image_urls], + reference_audios=reference_audios, prompt=prompt, resolution=model["resolution"], duration=model["duration"],