diff --git a/comfy_api_nodes/apis/minimax.py b/comfy_api_nodes/apis/minimax.py index bac4572d4..12a0853ac 100644 --- a/comfy_api_nodes/apis/minimax.py +++ b/comfy_api_nodes/apis/minimax.py @@ -161,12 +161,30 @@ class Hailuo03TaskCreationRequest(BaseModel): ..., min_length=1 ) resolution: str = Field(...) - duration: int = Field(..., ge=5, le=15) + duration: int = Field(..., ge=4, le=15) ratio: str | None = Field(None) seed: int | None = Field(None, ge=0, le=4294967295) aigc_watermark: bool | None = Field(None) +class Hailuo03ContextIRRequest(BaseModel): + model: str = Field(...) + content: list[Hailuo03TextContent | Hailuo03ImageContent | Hailuo03VideoContent | Hailuo03AudioContent] = Field( + ..., min_length=1 + ) + duration: int = Field(..., ge=4, le=15) + ratio: str | None = Field(None) + + +class Hailuo03RegenerationRequest(BaseModel): + model: str = Field(...) + content: list[Hailuo03TextContent | Hailuo03ImageContent | Hailuo03VideoContent | Hailuo03AudioContent] = Field( + ..., min_length=1 + ) + resolution: str = Field(...) + aigc_watermark: bool | None = Field(None) + + class Hailuo03TaskCreationResponse(BaseModel): task_id: str = Field(...) @@ -178,6 +196,7 @@ class Hailuo03TaskError(BaseModel): class Hailuo03TaskContent(BaseModel): url: str | None = Field(None) + prompt: str | None = Field(None) class Hailuo03TaskUsage(BaseModel): diff --git a/comfy_api_nodes/nodes_minimax.py b/comfy_api_nodes/nodes_minimax.py index 3c1d29257..de3895221 100644 --- a/comfy_api_nodes/nodes_minimax.py +++ b/comfy_api_nodes/nodes_minimax.py @@ -3,12 +3,14 @@ from typing import Optional import torch from typing_extensions import override -from comfy_api.latest import IO, ComfyExtension +from comfy_api.latest import IO, ComfyExtension, Input from comfy_api_nodes.apis.minimax import ( Hailuo03AudioContent, Hailuo03AudioContentUrl, + Hailuo03ContextIRRequest, Hailuo03ImageContent, Hailuo03ImageContentUrl, + Hailuo03RegenerationRequest, Hailuo03TaskCreationRequest, Hailuo03TaskCreationResponse, Hailuo03TaskQueryResponse, @@ -456,6 +458,9 @@ HAILUO_03_QUERY_ENDPOINT = "/proxy/minimax/v2/query/video_generation" # + /{tas HAILUO_03_MODELS = {"MiniMax H3": "MiniMax-H3"} HAILUO_03_FAILED_STATUSES = ["failed", "cancelled", "expired"] +HAILUO_03_CONTEXT_IR_ENDPOINT = "/proxy/minimax/v2/h3_context_ir" +HAILUO_03_REGENERATION_ENDPOINT = "/proxy/minimax/v2/video_regeneration" + def _hailuo03_model_inputs(include_ratio: bool = True, allow_adaptive: bool = True): inputs = [ @@ -487,10 +492,10 @@ def _hailuo03_model_inputs(include_ratio: bool = True, allow_adaptive: bool = Tr IO.Int.Input( "duration", default=5, - min=5, + min=4, max=15, step=1, - tooltip="Duration of the output video in seconds (5-15).", + tooltip="Duration of the output video in seconds (4-15).", display_mode=IO.NumberDisplay.slider, ) ) @@ -939,6 +944,592 @@ class MinimaxHailuo03ReferenceNode(IO.ComfyNode): ) +class MinimaxHailuo03ContextIRNode(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="MinimaxHailuo03ContextIRNode", + display_name="MiniMax H3 Context IR (Prompt Enhancer)", + category="partner/video/MiniMax", + description="Analyze text and media context with MiniMax H3 Context IR and produce an enhanced, " + "structured video prompt. Feed the output into the prompt of a MiniMax H3 video node and attach " + "the same media there in the same order, because the enhanced prompt refers to the attached " + "media by position.", + inputs=[ + IO.DynamicCombo.Input( + "model", + options=[ + IO.DynamicCombo.Option( + "MiniMax H3", + [ + IO.String.Input( + "prompt", + multiline=True, + default="", + tooltip="Description of the video you intend to generate.", + ), + IO.Int.Input( + "duration", + default=5, + min=4, + max=15, + step=1, + tooltip="Duration of the video you intend to generate, in seconds (4-15).", + display_mode=IO.NumberDisplay.slider, + ), + IO.Combo.Input( + "ratio", + options=["adaptive", "16:9", "4:3", "1:1", "3:4", "9:16", "21:9"], + default="adaptive", + tooltip="Aspect ratio of the video you intend to generate. 'adaptive' " + "requires at least one image, video, or audio input.", + ), + IO.Autogrow.Input( + "reference_images", + template=IO.Autogrow.TemplateNames( + IO.Image.Input("reference_image"), + names=[ + "image_1", + "image_2", + "image_3", + "image_4", + "image_5", + "image_6", + "image_7", + "image_8", + "image_9", + ], + min=0, + ), + tooltip="Subject or style reference images, referred to in the prompt " + "as 'Image 1'..'Image 9' in connection order. Up to 9 images.", + ), + IO.Autogrow.Input( + "reference_videos", + template=IO.Autogrow.TemplateNames( + IO.Video.Input("reference_video"), + names=["video_1", "video_2", "video_3"], + min=0, + ), + tooltip="Motion or scene reference videos, referred to in the prompt " + "as 'Video 1'..'Video 3' in connection order. Up to 3 videos, " + "2-15 seconds each, 15 seconds in total.", + ), + IO.Autogrow.Input( + "reference_audios", + template=IO.Autogrow.TemplateNames( + IO.Audio.Input("reference_audio"), + names=["audio_1", "audio_2", "audio_3"], + min=0, + ), + tooltip="Audio references, referred to in the prompt as " + "'Audio 1'..'Audio 3' in connection order. Up to 3 clips, " + "2-15 seconds each, 15 seconds in total. Cannot be used without " + "a reference image or video.", + ), + ], + ) + ], + tooltip="Model to use for prompt enhancement.", + ), + IO.Image.Input( + "first_frame", + tooltip="First frame of the video you intend to generate. Cannot be combined with " + "reference media.", + optional=True, + ), + IO.Image.Input( + "last_frame", + tooltip="Last frame of the video you intend to generate. Cannot be combined with " + "reference media.", + optional=True, + ), + ], + outputs=[ + IO.String.Output(), + ], + hidden=[ + IO.Hidden.auth_token_comfy_org, + IO.Hidden.api_key_comfy_org, + IO.Hidden.unique_id, + ], + is_api_node=True, + price_badge=IO.PriceBadge( + depends_on=IO.PriceBadgeDepends( + inputs=["first_frame", "last_frame"], + input_groups=["model.reference_images", "model.reference_videos", "model.reference_audios"], + ), + expr=""" + ( + $imgsRaw := $lookup(inputGroups, "model.reference_images"); + $imgs := $imgsRaw ? $imgsRaw : 0; + $vidsRaw := $lookup(inputGroups, "model.reference_videos"); + $vids := $vidsRaw ? $vidsRaw : 0; + $audsRaw := $lookup(inputGroups, "model.reference_audios"); + $auds := $audsRaw ? $audsRaw : 0; + $frames := (inputs.first_frame.connected ? 1 : 0) + (inputs.last_frame.connected ? 1 : 0); + ($imgs + $vids + $auds) > 0 + ? {"type": "range_usd", "min_usd": 0.06, "max_usd": 0.11, "format": {"approximate": true}} + : $frames > 0 + ? {"type": "usd", "usd": 0.05, "format": {"approximate": true}} + : {"type": "usd", "usd": 0.02, "format": {"approximate": true}} + ) + """, + ), + ) + + @classmethod + async def execute( + cls, + model: dict, + first_frame: torch.Tensor | None = None, + last_frame: torch.Tensor | None = None, + ) -> IO.NodeOutput: + validate_string(model["prompt"], strip_whitespace=True, min_length=1) + + reference_images = {k: v for k, v in (model.get("reference_images") or {}).items() if v is not None} + reference_videos = {k: v for k, v in (model.get("reference_videos") or {}).items() if v is not None} + reference_audios = {k: v for k, v in (model.get("reference_audios") or {}).items() if v is not None} + has_frames = first_frame is not None or last_frame is not None + has_references = bool(reference_images) or bool(reference_videos) or bool(reference_audios) + if has_frames and has_references: + raise ValueError( + "First/last frame and reference media are mutually exclusive. Use frames for an " + "image-to-video prompt, or reference media for a reference-to-video prompt." + ) + if reference_audios and not reference_images and not reference_videos: + raise ValueError("Reference audio cannot be used without a reference image or video.") + if not has_frames and not has_references and model["ratio"] == "adaptive": + raise ValueError( + "Ratio 'adaptive' is not supported for text-only requests; select an explicit aspect ratio." + ) + + for frame in (first_frame, last_frame): + if frame is not None: + validate_image_aspect_ratio(frame, (2, 5), (5, 2), strict=False) # 0.4 to 2.5 + validate_image_dimensions(frame, min_width=256, min_height=256) + for image in reference_images.values(): + validate_image_aspect_ratio(image, (2, 5), (5, 2), strict=False) # 0.4 to 2.5 + validate_image_dimensions(image, min_width=256, min_height=256) + + total_video_duration = 0.0 + for i, video in enumerate(reference_videos.values(), 1): + try: + fps = float(video.get_frame_rate()) + except Exception: + fps = 0.0 + if fps and not (23.9 <= fps <= 60.5): + raise ValueError(f"Reference video {i} is {fps:.2f} FPS. Supported range is 23.976-60 FPS.") + try: + dur = video.get_duration() + except Exception: + continue + if dur < 1.8: + raise ValueError(f"Reference video {i} is too short: {dur:.1f}s. Minimum duration is 2 seconds.") + total_video_duration += dur + if total_video_duration > 15.1: + raise ValueError( + f"Total reference video duration is {total_video_duration:.1f}s. Maximum is 15 seconds." + ) + + total_audio_duration = 0.0 + for i, audio in enumerate(reference_audios.values(), 1): + dur = int(audio["waveform"].shape[-1]) / int(audio["sample_rate"]) + if dur < 1.8: + raise ValueError(f"Reference audio {i} is too short: {dur:.1f}s. Minimum duration is 2 seconds.") + total_audio_duration += dur + if total_audio_duration > 15.1: + raise ValueError( + f"Total reference audio duration is {total_audio_duration:.1f}s. Maximum is 15 seconds." + ) + + content: list = [Hailuo03TextContent(text=model["prompt"])] + if first_frame is not None: + content.append( + Hailuo03ImageContent( + image_url=Hailuo03ImageContentUrl( + url=( + await upload_images_to_comfyapi( + cls, first_frame, max_images=1, wait_label="Uploading first frame" + ) + )[0], + ), + role="first_frame", + ) + ) + if last_frame is not None: + content.append( + Hailuo03ImageContent( + image_url=Hailuo03ImageContentUrl( + url=( + await upload_images_to_comfyapi( + cls, last_frame, max_images=1, wait_label="Uploading last frame" + ) + )[0], + ), + role="last_frame", + ) + ) + for i, image in enumerate(reference_images.values(), 1): + content.append( + Hailuo03ImageContent( + image_url=Hailuo03ImageContentUrl( + url=( + await upload_images_to_comfyapi( + cls, image, max_images=1, wait_label=f"Uploading image {i}" + ) + )[0], + ), + role="reference_image", + ) + ) + for i, video in enumerate(reference_videos.values(), 1): + content.append( + Hailuo03VideoContent( + video_url=Hailuo03VideoContentUrl( + url=await upload_video_to_comfyapi(cls, video, wait_label=f"Uploading video {i}"), + ), + ) + ) + for audio in reference_audios.values(): + content.append( + Hailuo03AudioContent( + audio_url=Hailuo03AudioContentUrl( + url=await upload_audio_to_comfyapi( + cls, + audio, + container_format="mp3", + codec_name="libmp3lame", + mime_type="audio/mpeg", + ), + ), + ) + ) + + response = await sync_op( + cls, + ApiEndpoint(path=HAILUO_03_CONTEXT_IR_ENDPOINT, method="POST"), + response_model=Hailuo03TaskCreationResponse, + data=Hailuo03ContextIRRequest( + model=HAILUO_03_MODELS[model["model"]], + content=content, + duration=model["duration"], + ratio=None if model["ratio"] == "adaptive" else model["ratio"], + ), + ) + task_result = await poll_op( + cls, + ApiEndpoint(path=f"{HAILUO_03_QUERY_ENDPOINT}/{response.task_id}"), + response_model=Hailuo03TaskQueryResponse, + status_extractor=lambda r: r.task.status, + failed_statuses=HAILUO_03_FAILED_STATUSES, + poll_interval=5, + ) + prompt = task_result.task.content.prompt if task_result.task.content else None + if not prompt: + raise Exception(f"No enhanced prompt in the response: {task_result.model_dump()}") + return IO.NodeOutput(prompt) + + +class MinimaxHailuo03RegenerateNode(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="MinimaxHailuo03RegenerateNode", + display_name="MiniMax H3 Regenerate to 2K", + category="partner/video/MiniMax", + description="Re-render a MiniMax H3 768P output at 2K resolution. Connect the unmodified 768P " + "video and the exact prompt used to generate it; if the original generation used first/last " + "frames or reference media, attach the same inputs.", + inputs=[ + IO.DynamicCombo.Input( + "model", + options=[ + IO.DynamicCombo.Option( + "MiniMax H3", + [ + IO.String.Input( + "prompt", + multiline=True, + default="", + tooltip="The exact prompt used to generate the source video.", + ), + IO.Combo.Input( + "resolution", + options=["2K"], + tooltip="Resolution to re-render the source video at.", + ), + IO.Autogrow.Input( + "reference_images", + template=IO.Autogrow.TemplateNames( + IO.Image.Input("reference_image"), + names=[ + "image_1", + "image_2", + "image_3", + "image_4", + "image_5", + "image_6", + "image_7", + "image_8", + "image_9", + ], + min=0, + ), + tooltip="Reference images from the original generation, in the same " + "order. Up to 9 images.", + ), + IO.Autogrow.Input( + "reference_videos", + template=IO.Autogrow.TemplateNames( + IO.Video.Input("reference_video"), + names=["video_1", "video_2", "video_3"], + min=0, + ), + tooltip="Reference videos from the original generation, in the same " + "order. Up to 3 videos, 2-15 seconds each, 15 seconds in total.", + ), + IO.Autogrow.Input( + "reference_audios", + template=IO.Autogrow.TemplateNames( + IO.Audio.Input("reference_audio"), + names=["audio_1", "audio_2", "audio_3"], + min=0, + ), + tooltip="Audio references from the original generation, in the same " + "order. Up to 3 clips, 2-15 seconds each, 15 seconds in total. " + "Cannot be used without a reference image or video.", + ), + ], + ) + ], + tooltip="Model to use for video regeneration.", + ), + IO.Video.Input( + "video", + tooltip="The MiniMax H3 768P output video to re-render. Connect the unmodified output " + "of a MiniMax H3 video node (24 FPS, 4-15 seconds). 2K outputs cannot be used.", + ), + IO.Image.Input( + "first_frame", + tooltip="First frame image from the original generation, if one was used.", + optional=True, + ), + IO.Image.Input( + "last_frame", + tooltip="Last frame image from the original generation, if one was used.", + optional=True, + ), + IO.Boolean.Input( + "watermark", + default=False, + tooltip="Whether to add an AIGC watermark to the video.", + advanced=True, + ), + ], + outputs=[ + IO.Video.Output(), + ], + hidden=[ + IO.Hidden.auth_token_comfy_org, + IO.Hidden.api_key_comfy_org, + IO.Hidden.unique_id, + ], + is_api_node=True, + price_badge=IO.PriceBadge( + expr="""{"type": "usd", "usd": 0.0715, "format": {"suffix": "/second"}}""", + ), + ) + + @classmethod + async def execute( + cls, + model: dict, + video: Input.Video, + watermark: bool, + first_frame: torch.Tensor | None = None, + last_frame: torch.Tensor | None = None, + ) -> IO.NodeOutput: + validate_string(model["prompt"], strip_whitespace=True, min_length=1) + + try: + fps = float(video.get_frame_rate()) + except Exception: + fps = 0.0 + if fps and not (23.9 <= fps <= 24.1): + raise ValueError( + f"The source video is {fps:.2f} FPS. Regeneration accepts unmodified MiniMax H3 768P " + "outputs, which are 24 FPS." + ) + try: + width, height = video.get_dimensions() + except Exception: + width = height = 0 + if width and height and (width % 32 or height % 32 or width * height > 1_032_192): + raise ValueError( + f"The source video is {width}x{height}. Regeneration accepts MiniMax H3 768P outputs " + "(width and height divisible by 32, at most 1,032,192 total pixels); 2K outputs cannot " + "be used as a source." + ) + try: + frame_count = video.get_frame_count() + except Exception: + frame_count = 0 + if frame_count and (frame_count < 107 or frame_count > 362 or (frame_count - 107) % 17): + raise ValueError( + f"The source video has {frame_count} frames. Regeneration accepts unmodified " + "MiniMax H3 outputs, whose length is 107 to 362 frames in steps of 17 " + "(4 to 15 seconds at 24 FPS)." + ) + + reference_images = {k: v for k, v in (model.get("reference_images") or {}).items() if v is not None} + reference_videos = {k: v for k, v in (model.get("reference_videos") or {}).items() if v is not None} + reference_audios = {k: v for k, v in (model.get("reference_audios") or {}).items() if v is not None} + if (first_frame is not None or last_frame is not None) and ( + reference_images or reference_videos or reference_audios + ): + raise ValueError( + "First/last frame and reference media are mutually exclusive. Use frames for an " + "image-to-video prompt, or reference media for a reference-to-video prompt." + ) + if reference_audios and not reference_images and not reference_videos: + raise ValueError("Reference audio cannot be used without a reference image or video.") + + for frame in (first_frame, last_frame): + if frame is not None: + validate_image_aspect_ratio(frame, (2, 5), (5, 2), strict=False) # 0.4 to 2.5 + validate_image_dimensions(frame, min_width=256, min_height=256) + for image in reference_images.values(): + validate_image_aspect_ratio(image, (2, 5), (5, 2), strict=False) # 0.4 to 2.5 + validate_image_dimensions(image, min_width=256, min_height=256) + + total_video_duration = 0.0 + for i, ref_video in enumerate(reference_videos.values(), 1): + try: + ref_fps = float(ref_video.get_frame_rate()) + except Exception: + ref_fps = 0.0 + if ref_fps and not (23.9 <= ref_fps <= 60.5): + raise ValueError(f"Reference video {i} is {ref_fps:.2f} FPS. Supported range is 23.976-60 FPS.") + try: + dur = ref_video.get_duration() + except Exception: + continue + if dur < 1.8: + raise ValueError(f"Reference video {i} is too short: {dur:.1f}s. Minimum duration is 2 seconds.") + total_video_duration += dur + if total_video_duration > 15.1: + raise ValueError( + f"Total reference video duration is {total_video_duration:.1f}s. Maximum is 15 seconds." + ) + + total_audio_duration = 0.0 + for i, audio in enumerate(reference_audios.values(), 1): + dur = int(audio["waveform"].shape[-1]) / int(audio["sample_rate"]) + if dur < 1.8: + raise ValueError(f"Reference audio {i} is too short: {dur:.1f}s. Minimum duration is 2 seconds.") + total_audio_duration += dur + if total_audio_duration > 15.1: + raise ValueError( + f"Total reference audio duration is {total_audio_duration:.1f}s. Maximum is 15 seconds." + ) + + content: list = [ + Hailuo03VideoContent( + video_url=Hailuo03VideoContentUrl( + url=await upload_video_to_comfyapi(cls, video, wait_label="Uploading source video"), + ), + role="base_video", + ), + Hailuo03TextContent(text=model["prompt"]), + ] + if first_frame is not None: + content.append( + Hailuo03ImageContent( + image_url=Hailuo03ImageContentUrl( + url=( + await upload_images_to_comfyapi( + cls, first_frame, max_images=1, wait_label="Uploading first frame" + ) + )[0], + ), + role="first_frame", + ) + ) + if last_frame is not None: + content.append( + Hailuo03ImageContent( + image_url=Hailuo03ImageContentUrl( + url=( + await upload_images_to_comfyapi( + cls, last_frame, max_images=1, wait_label="Uploading last frame" + ) + )[0], + ), + role="last_frame", + ) + ) + for i, image in enumerate(reference_images.values(), 1): + content.append( + Hailuo03ImageContent( + image_url=Hailuo03ImageContentUrl( + url=( + await upload_images_to_comfyapi( + cls, image, max_images=1, wait_label=f"Uploading image {i}" + ) + )[0], + ), + role="reference_image", + ) + ) + for i, ref_video in enumerate(reference_videos.values(), 1): + content.append( + Hailuo03VideoContent( + video_url=Hailuo03VideoContentUrl( + url=await upload_video_to_comfyapi(cls, ref_video, wait_label=f"Uploading video {i}"), + ), + ) + ) + for audio in reference_audios.values(): + content.append( + Hailuo03AudioContent( + audio_url=Hailuo03AudioContentUrl( + url=await upload_audio_to_comfyapi( + cls, + audio, + container_format="mp3", + codec_name="libmp3lame", + mime_type="audio/mpeg", + ), + ), + ) + ) + + response = await sync_op( + cls, + ApiEndpoint(path=HAILUO_03_REGENERATION_ENDPOINT, method="POST"), + response_model=Hailuo03TaskCreationResponse, + data=Hailuo03RegenerationRequest( + model=HAILUO_03_MODELS[model["model"]], + content=content, + resolution=model["resolution"], + aigc_watermark=watermark, + ), + ) + task_result = await poll_op( + cls, + ApiEndpoint(path=f"{HAILUO_03_QUERY_ENDPOINT}/{response.task_id}"), + response_model=Hailuo03TaskQueryResponse, + status_extractor=lambda r: r.task.status, + failed_statuses=HAILUO_03_FAILED_STATUSES, + poll_interval=10, + ) + video_url = task_result.task.content.url if task_result.task.content else None + if not video_url: + raise Exception(f"No video URL in the response: {task_result.model_dump()}") + return IO.NodeOutput(await download_url_to_video_output(video_url)) + + class MinimaxExtension(ComfyExtension): @override async def get_node_list(self) -> list[type[IO.ComfyNode]]: @@ -950,6 +1541,8 @@ class MinimaxExtension(ComfyExtension): MinimaxHailuo03TextToVideoNode, MinimaxHailuo03FirstLastFrameNode, MinimaxHailuo03ReferenceNode, + MinimaxHailuo03ContextIRNode, + MinimaxHailuo03RegenerateNode, ]