diff --git a/.coderabbit.yaml b/.coderabbit.yaml index 0d1e49270..08629ed8e 100644 --- a/.coderabbit.yaml +++ b/.coderabbit.yaml @@ -4,12 +4,12 @@ early_access: false tone_instructions: "Only comment on issues introduced by this PR's changes. Do not flag pre-existing problems in moved, re-indented, or reformatted code." reviews: - profile: "chill" - request_changes_workflow: false + profile: "assertive" + request_changes_workflow: true high_level_summary: false poem: false review_status: false - review_details: false + review_details: true commit_status: true collapse_walkthrough: true changed_files_summary: false @@ -39,6 +39,14 @@ reviews: - path: "**" instructions: | IMPORTANT: Only comment on issues directly introduced by this PR's code changes. + Treat AGENTS.md as mandatory repository policy, not optional style guidance. + Flag PR changes that violate AGENTS.md even when the code is otherwise functional. + In particular, enforce architecture boundaries, dtype/device/memory rules, + interface contracts, import style, no unnecessary try/except blocks, no inline + imports, no outbound internet paths in core ComfyUI, and narrow scoped fixes. + Prefer direct findings over suggestions when a rule is violated. Only ignore + AGENTS.md when it clearly conflicts with a newer explicit maintainer instruction + in the PR. Do NOT flag pre-existing issues in code that was merely moved, re-indented, de-indented, or reformatted without logic changes. If code appears in the diff only due to whitespace or structural reformatting (e.g., removing a `with:` block), @@ -123,5 +131,10 @@ chat: knowledge_base: opt_out: false + code_guidelines: + enabled: true + filePatterns: + - files: "AGENTS.md" + applyTo: "**" learnings: scope: "auto" diff --git a/.github/workflows/ci-cursor-review.yml b/.github/workflows/ci-cursor-review.yml new file mode 100644 index 000000000..2312c0ccd --- /dev/null +++ b/.github/workflows/ci-cursor-review.yml @@ -0,0 +1,38 @@ +name: CI - Cursor Review + +# Thin caller for the shared reusable cursor-review workflow in +# Comfy-Org/github-workflows. The review logic (panel matrix, judge +# consolidation, prompts, extract/post/notify scripts) lives there as the +# single source of truth, so this repo only carries the repo-specific diff +# excludes. + +on: + pull_request: + types: [labeled, unlabeled] + +concurrency: + group: cursor-review-pr-${{ github.event.pull_request.number }}-${{ github.event.label.name }} + cancel-in-progress: true + +jobs: + cursor-review: + if: github.event.label.name == 'cursor-review' + permissions: + contents: read + pull-requests: write + # SHA-pinned per zizmor `unpinned-uses: hash-pin`. Bump this SHA to pick up + # upstream changes; keep `workflows_ref` matching so prompts/scripts load + # from the same commit as the workflow definition. + uses: Comfy-Org/github-workflows/.github/workflows/cursor-review.yml@047ca48febe3a6647608ed2e0c4331b491cb9d6a # github-workflows#9 + with: + workflows_ref: 047ca48febe3a6647608ed2e0c4331b491cb9d6a + diff_excludes: >- + :!**/.claude/** + :!**/dist/** + :!**/vendor/** + :!**/*.generated.* + :!**/*.min.js + :!**/*.min.css + secrets: + CURSOR_API_KEY: ${{ secrets.CURSOR_API_KEY }} + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} diff --git a/.github/workflows/cla.yml b/.github/workflows/cla.yml new file mode 100644 index 000000000..b75397e50 --- /dev/null +++ b/.github/workflows/cla.yml @@ -0,0 +1,91 @@ +name: CLA Assistant + +on: + issue_comment: + types: [created] + pull_request_target: + types: [opened, synchronize, closed] + +permissions: + actions: write + contents: read # 'read' is enough because signatures live in a REMOTE repo + pull-requests: write + statuses: write + +jobs: + cla-assistant: + runs-on: ubuntu-latest + steps: + # The CLA action normally requires every commit author in a PR to sign. + # We only want the PR author to sign, so we allowlist all other committers + # by computing them from the PR's commits and excluding the PR author. + - name: Build author-only allowlist + id: allowlist + if: > + github.event_name == 'pull_request_target' || + (github.event_name == 'issue_comment' && github.event.issue.pull_request && ( + github.event.comment.body == 'recheck' || + github.event.comment.body == 'I have read and agree to the Contributor License Agreement' + )) + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ github.event.pull_request.number || github.event.issue.number }} + PR_AUTHOR: ${{ github.event.pull_request.user.login || github.event.issue.user.login }} + BASE_ALLOWLIST: action@github.com,actions-user,ampagent,claude,comfy-pr-bot,GitHub Action,github-actions,github-actions[bot],Glary Bot,Glary-Bot,*[bot] + run: | + others=$(gh api "repos/${{ github.repository }}/pulls/${PR_NUMBER}/commits" --paginate \ + --jq '.[] | (.author.login // empty), (.committer.login // empty)' \ + | sort -u | grep -vix "${PR_AUTHOR}" | paste -sd, -) + if [ -n "$others" ]; then + echo "allowlist=${BASE_ALLOWLIST},${others}" >> "$GITHUB_OUTPUT" + else + echo "allowlist=${BASE_ALLOWLIST}" >> "$GITHUB_OUTPUT" + fi + + - name: CLA Assistant + # Run on PR events, on "recheck" comment, or when someone posts the exact signing phrase. + # IMPORTANT: this phrase must match `custom-pr-sign-comment` below. + if: > + github.event_name == 'pull_request_target' || + (github.event_name == 'issue_comment' && github.event.issue.pull_request && ( + github.event.comment.body == 'recheck' || + github.event.comment.body == 'I have read and agree to the Contributor License Agreement' + )) + uses: contributor-assistant/github-action@ca4a40a7d1004f18d9960b404b97e5f30a505a08 # v2.6.1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # PAT required to write to the centralized signatures repo. + PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} + with: + # Where the CLA document lives (shown to contributors) + path-to-document: https://github.com/Comfy-Org/comfy-cla/blob/main/comfyui_icla.md + + # Centralized signature storage + remote-organization-name: comfy-org + remote-repository-name: comfy-cla + path-to-signatures: signatures/cla.json + branch: main + + # Only the PR author must sign: bots plus every non-author committer + # are allowlisted via the "Build author-only allowlist" step above. + # *[bot] is a catch-all for any GitHub App bot account. + allowlist: ${{ steps.allowlist.outputs.allowlist }} + + # Custom PR comment messages + custom-notsigned-prcomment: | + 🎉 Thank you for your contribution, we really appreciate it! 🎉 + + Like many open source projects, we require contributors to sign our [Contributor License Agreement (CLA)](https://github.com/Comfy-Org/comfy-cla/blob/main/comfyui_icla.md). A CLA makes the ownership of contributions explicit, so contributors and the project share a clear understanding of how the code can be used. By signing, you: + + - Confirm that you own your contribution. + - Keep the right to reuse your own code. + - Grant us a copyright license to include and share it within our projects. + + CLAs are standard practice across major open source projects including those under the Apache Software Foundation and the Linux Foundation. Ours is based on the Apache Software Foundation's CLA. Most importantly, it would enable us to relicense the project under a more permissive license in the future, giving the project and its community greater flexibility. + + ✍ **To sign, please post a new comment on this PR with exactly the following text:** ✍ + + custom-pr-sign-comment: I have read and agree to the Contributor License Agreement + + custom-allsigned-prcomment: | + ✅ All contributors have signed the CLA. Thank you! This PR is ready to be merged. diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 000000000..a8bacbd5e --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,294 @@ +## Engineering Style + +- Keep changes small and direct. Most fixes should touch the narrowest code path + that explains the bug, performance issue, dtype issue, model-format issue, or + user-facing behavior. +- Change the least amount of files possible. A change that touches many files is + more likely to be a bad change than a good one unless the broader scope is + directly required. +- Prefer practical fixes over broad architecture work. Add abstractions only + when they remove real repeated logic or match an existing ComfyUI pattern. +- Prefer fewer dependencies. Do not add new dependencies to ComfyUI unless they + are absolutely necessary. +- Delete obsolete code aggressively when newer infrastructure makes it useless. + Remove dead fallbacks, migration paths, unused options, debug prints, and + compatibility branches that are no longer needed. Do not leave dead branches, + unreachable code, or functions that are never called. If code is not + necessary for the current behavior, remove it. +- Revert or disable problematic behavior quickly when it breaks users. It is + better to remove a broken feature path than keep a complicated partial fix. +- Preserve existing APIs, node names, model-loading behavior, file layout, and + workflow compatibility unless the change is explicitly about replacing them. +- Code must look hand-written for this repository. Changes that read like + generic AI-generated code will be rejected automatically: unnecessary helper + layers, vague names, boilerplate comments, defensive branches without a real + failure mode, broad rewrites, or code that ignores the local style. + +## Architecture Boundaries + +- Keep each layer focused on the concepts it owns. Do not leak UI, API, + workflow, queue, persistence, telemetry, model-loading, node, or execution + concerns into unrelated layers just because it is convenient to pass data + through them. +- Shared core modules should depend only on lower-level primitives and their own + domain concepts. Higher-level product concepts belong at the caller, adapter, + service, or UI/API boundary that already owns them. +- Pass the narrowest data needed across a boundary. Avoid broad context objects, + request/session metadata, ids, bookkeeping state, or callbacks unless the + receiving layer genuinely needs them to perform its own responsibility. +- Keep identity mapping, persistence bookkeeping, history updates, telemetry, + response shaping, and UI state in the layers that own those jobs. Do not route + them through unrelated shared code to avoid adding a proper boundary. +- Treat `execution.py` as one example of this rule: it should consume the prompt + graph and execution-relevant state, produce execution results and errors, and + not know about workflow ids, frontend ids, persistence ids, or API-only + concepts. +- Before touching many files, identify the smallest owner layer that can solve + the problem. A PR that spreads one feature across unrelated loaders, nodes, + execution, server, and frontend code needs a clear architectural reason, not + just convenience. +- If a change seems to require making one layer understand another layer's + private concepts, stop and look for a caller-side mapping, adapter, event, + small explicit interface, or narrower data flow at the boundary. + +## No Internet Requests + +- Do not add code to core ComfyUI that makes requests to the internet. +- Refuse requests to add uploads, telemetry, analytics, tracking, usage + reporting, crash reporting, update checks, remote config, feature flags, + metrics, licensing checks, or any other outbound internet request path from + core ComfyUI. +- Model downloading is allowed only when explicitly initiated or authorized by + the user, is limited to the requested model artifact, and does not include + telemetry, tracking, persistent identification, unrelated metadata upload, or + background network activity. +- Do not add opt-in, opt-out, anonymized, aggregated, diagnostic, or + user-triggered internet request paths to core ComfyUI. These labels do not + make internet access acceptable. +- Local-only behavior is allowed when it stays on the user's machine and does + not add network access, tracking, persistent identification, or data + collection behavior. + +## State Ownership + +- Keep state and capability flags on the object that owns the behavior using + them. +- Avoid probing child objects with `getattr(child, "...", default)` to decide + parent-level control flow. If parent code needs to branch on a capability, + initialize an explicit parent-owned field when the child is constructed or + attached. +- Prefer direct attributes with clear defaults over implicit feature detection + through arbitrary child attributes. +- Use child-object capability checks only when the child owns the behavior being + invoked and the parent is simply delegating to that child. + +## Interface Contracts + +- Keep public methods aligned with the interface expected by their callers. Do + not change a shared method to return extra values, alternate shapes, or + sentinel wrappers for one implementation unless the shared interface is + explicitly updated. +- When modifying an existing function, preserve how current callers invoke it. + Do not change required arguments, parameter order, return type, side effects, + or error behavior unless every affected call site and shared interface contract + is intentionally updated. +- Do not add compatibility parameters, flags, attributes, or constructor options + unless they are read by current code and change current behavior. Remove + pass-through or stored-but-unused values instead of preserving upstream or + deprecated API baggage. +- If an implementation needs auxiliary values for its own workflow, expose them + through a private helper or a clearly named implementation-specific method + instead of overloading the public method's return contract. +- Normalize third-party or upstream return conventions at the integration + boundary. Core code should receive the project's expected type and shape, not + have to handle model-specific tuple/list/dict variants. +- Avoid caller-side unwrapping such as `out = out[0]` unless the called + interface is documented to return that structure. + +## Autograd and Model Freezing + +- Do not add `torch.no_grad`, `torch.inference_mode`, or inference-mode helper + wrappers in ComfyUI code. The only allowed inference-mode-related use is + disabling a globally set inference mode when a training path needs gradients. +- Do not add freeze, unfreeze, or trainability toggles to model classes. ComfyUI + models are always treated as frozen for inference, so explicit freeze + functionality is redundant and should not be added. +- Remove training-only behavior such as dropout from inference model code, but + preserve checkpoint and state-dict compatibility when doing so. If deleting a + module would change state-dict keys, module ordering, or checkpoint loading + behavior, replace it with a no-op such as `nn.Identity` instead of removing the + slot outright. + +## Python Style + +- Keep imports at module scope. Avoid inline imports unless they are already part + of an established optional-backend probe or are needed to avoid an import + cycle. +- Do not add unnecessary `try`/`except` blocks. Use them for optional dependency, + platform, or backend capability detection only when the program has a useful + fallback. Prefer specific exception types when changing new code. +- Remove any workarounds for PyTorch versions that ComfyUI no longer officially + supports. Deprecated workarounds include catching an exception and rerunning + the same op with the input cast to float. If a workaround does not have a + comment naming the exact PyTorch version or versions that still need it, + remove it. +- Let unsupported model formats, invalid quantization metadata, and bad states + fail with clear errors instead of silently producing lower quality output. +- Match the existing local style in the file you edit. This codebase tolerates + long lines, simple helper functions, module-level state, and direct tensor + operations when they make the code easier to follow. +- Keep comments sparse and useful. Strip useless comments that restate the code + or describe obvious behavior. Short TODOs are fine when they name the concrete + missing follow-up. + +## Model, Device, and Memory Behavior + +- Treat dtype, device placement, VRAM usage, and offloading behavior as core + correctness concerns. Check CPU, CUDA, ROCm, MPS, DirectML, XPU, NPU, and low + VRAM implications when touching shared execution or loading code. +- Prefer native ComfyUI formats and existing quantization/offload helpers over + adding parallel code paths. Use `comfy.quant_ops`, `comfy.model_management`, + `comfy.memory_management`, `comfy.pinned_memory`, `comfy_aimdo`, and + `comfy-kitchen` helpers where they already solve the problem. +- Use optimized comfy-kitchen ops in places where they improve performance + without changing the expected dtype, device, memory, or interface behavior. +- All models should use the optimized attention function selected by ComfyUI. + Treat optimized backend functions, dispatch helpers, and capability-selected + callables as opaque. Higher-level code must not inspect function identity, + names, modules, or implementation details to decide behavior. +- Apply the same opacity rule to similar patterns beyond attention: callers + should depend on the documented interface and result contract, not on which + backend implementation was selected underneath. +- Do not use custom inference ops that only duplicate an existing op while + upcasting to float32, such as custom RMSNorm variants. Use the generic ComfyUI + ops and/or native torch ops instead. +- If a model class `__init__` has an `operations` parameter, assume + `operations` is never `None`. Do not add fallback branches or default torch + ops for a missing `operations` object. +- Do not add unnecessary parameters to model, model block, or model ops related + classes. Constructor and forward signatures should carry only values that are + actually needed by that object for inference. +- Reuse existing model classes, blocks, ops, and helper modules when appropriate. + Before implementing a new version of a model component, search the existing + model code for a class or helper that already provides the behavior. +- Model detection code that inspects linear weight shapes should only use the + first dimension. The second dimension may be half the original size for + NVFP4 or other 4-bit quantized models. +- Avoid adding `einops` usage in core inference code. Use native torch tensor + ops such as `reshape`, `view`, `permute`, `transpose`, `flatten`, `unflatten`, + `unsqueeze`, and `squeeze` instead. +- Do not use tensors as general-purpose Python data structures. Keep metadata, + bookkeeping, counters, flags, shape math, padding math, index planning, memory + estimates, and control-flow decisions in plain Python values unless the data + must participate directly in tensor computation. Do not create tensors for + structural metadata that is only used for Python-side control flow. Sequence + lengths, cumulative offsets, split indices, window counts, slice boundaries, + and repeat counts should be kept as Python ints/lists from the point they are + computed. Do not build them as CPU/GPU tensors and then cast, move, validate, + or convert them back to Python for `split`, `tensor_split`, indexing plans, + loops, or cache keys. Avoid creating temporary tensors just to use tensor + methods for scalar or structural calculations. +- Avoid unnecessary casts and transfers. Preserve the intended compute dtype, + storage dtype, bias dtype, and original tensor shape metadata. +- Keep model-native latent layout handling inside the model or latent-format + owner, not in helper nodes. Do not collapse, expand, pack, or unpack latent + dimensions in nodes or other caller-side adapters just to satisfy a model + forward; the model path should consume and return the native latent shape for + that model family. +- Assume inputs to the main model forward are already in the compute dtype by + default, except integer inputs such as some model timestep tensors. Do not add + defensive or convenience casts in model code; it is better for invalid dtype + plumbing to error clearly than to hide it with unnecessary casts. +- Raw model parameters that are not owned by an op and may be initialized in a + dtype different from the compute dtype should be cast at use in forward or + inference code with `comfy.ops.cast_to_input` or + `comfy.model_management.cast_to` to avoid dtype mismatches. +- Model code should not care what dtype it is initialized in, and model + `__init__` methods should not contain workarounds for specific dtypes. Dtype + workaround code, such as making a model work with fp16 compute, belongs in the + execution or model-management layer that owns compute policy. +- Model code should not perform unnecessary device-to-CPU or CPU-to-device + transfers. New allocations must be created on the correct device and dtype; + never allocate on CPU and then move to GPU, or allocate in one dtype and then + convert to another. +- Model code itself should not perform memory management. Loading, unloading, + offloading, device movement, VRAM policy, cache lifetime, and cleanup belong + in the relevant model-management and execution layers, not inside model + implementations. +- Do not add global, module-level, class-level, singleton, or model-owned stores + for tensors or other large memory that persist across executions. Temporary + caches must be scoped to a single execution or forward/encode/decode call: + allocate them in the owning top-level call, pass them explicitly through the + call stack, and let them be discarded when that call returns. +- Follow the Wan VAE temporal cache pattern for temporary caches: create a local + cache such as `feat_map` for the encode/decode operation, pass it into the + blocks that need it, and do not retain it on the model or in global state. +- In model init code, prefer `torch.empty` for parameter/buffer placeholders + that are populated from the model state dict instead of zero-initializing with + `torch.zeros` or similar. If an allocation is not loaded from the state dict + and is useless for inference, do not include it. +- `nn.Parameter` tensors that are stored in and populated from the model state + dict should be initialized with `torch.empty`, not with zero, random, or + otherwise meaningful initialization. +- Model initialization should describe module structure, not fabricate + checkpoint-owned tensor contents. Parameters and buffers that are loaded from + the state dict must not be manually initialized, reassigned, or filled with + fallback values unless that value is actually used when no checkpoint key + exists. +- When slicing large tensors, copy the slice if the sliced tensor's lifetime + exceeds the current function scope. Do not keep a long-lived view into a large + backing tensor when a smaller copy would release memory sooner. +- Use fused or compound torch operations such as `addcmul` when they naturally + match the math. Reducing Python and torch dispatch overhead is a valid + optimization when it does not obscure the code or change dtype/device + behavior. +- Avoid caches that persist across different executions as much as possible. + Persistent caches are acceptable only when they use a very minimal amount of + memory and have a clear ownership and invalidation story. +- When optimizing, favor small measurable changes: fewer allocations, fewer + device transfers, less peak memory, better batching, or use of a faster + existing backend op. + +## Nodes and User-Facing Behavior + +- Follow existing node conventions: `INPUT_TYPES`, `RETURN_TYPES`, `FUNCTION`, + `CATEGORY`, and registration through the local mapping used by that file. +- Keep node changes backward compatible by default. Add inputs with sensible + defaults and avoid changing output types unless the request requires it. +- Model implementations should add the minimal number of ComfyUI nodes required + to run the model. Reuse existing nodes as much as possible; adapting the model + to work with existing nodes is strongly preferred over creating new nodes. +- Nodes should output only values they own. Do not add pass-through outputs for + workflow convenience unless the node is explicitly an output node. Existing + models, latents, conditioning, or other inputs should flow directly to the + next consumer instead of being re-emitted unchanged. +- Nodes should expose only inputs they actually read to produce current + behavior. Do not add placeholder, pass-through, compatibility, or + workflow-shaping inputs that are ignored or could flow directly to another + node. +- Node-level code must not patch model code directly. Any node behavior that + modifies, wraps, hooks, or changes model behavior must go through the model + patcher class instead of reaching into model internals. +- The official mascot of ComfyUI is a very cute anime girl with massive fennec + ears, a big fluffy tail, long blonde wavy hair, and blue eyes. Feel free to + use her in ComfyUI materials, UI text, examples, tests, generated assets, or + comments, but do not disrespect her. +- Warning and info messages should be short and actionable. Remove noisy or + misleading messages rather than adding more logging. +- Documentation and README edits should be concise, factual, and tied to the + changed behavior. + +## Commit and Review Habits + +- If asked to write commit messages, use short direct subjects like the existing + history: `Fix ...`, `Add ...`, `Support ...`, `Remove ...`, `Update ...`, + `Make ...`, `Use ...`, `Disable ...`, `Bump ...`, or `Revert ...`. +- Keep PR descriptions short and reviewable. State the problem, the behavioral + change, and the tests run; avoid long narrative explanations, implementation + diaries, or exhaustive file-by-file summaries unless the reviewer explicitly + needs that context. +- Prefer one coherent behavioral change per commit. Dependency pins, tests, and + the code that needs them may be in the same commit when they are inseparable. +- In reviews, prioritize real user impact: crashes, wrong dtype/device behavior, + memory regressions, broken model loading, workflow incompatibility, and noisy + or misleading user-facing output. diff --git a/README.md b/README.md index dc2389266..bcec86377 100644 --- a/README.md +++ b/README.md @@ -140,7 +140,7 @@ ComfyUI follows a weekly release cycle targeting Monday but this regularly chang - Commits outside of the stable release tags may be very unstable and break many custom nodes. - Serves as the foundation for the desktop release -2. **[ComfyUI Desktop](https://github.com/Comfy-Org/desktop)** +2. **[Comfy Desktop](https://github.com/Comfy-Org/Comfy-Desktop)** - Builds a new release using the latest stable core version 3. **[ComfyUI Frontend](https://github.com/Comfy-Org/ComfyUI_frontend)** @@ -309,7 +309,7 @@ After this you should have everything installed and can proceed to running Comfy #### Apple Mac silicon -You can install ComfyUI in Apple Mac silicon (M1 or M2) with any recent macOS version. +You can install ComfyUI in Apple Mac silicon (M1, M2, M3 or M4) with any recent macOS version. 1. Install pytorch nightly. For instructions, read the [Accelerated PyTorch training on Mac](https://developer.apple.com/metal/pytorch/) Apple Developer guide (make sure to install the latest pytorch nightly). 1. Follow the [ComfyUI manual installation](#manual-install-windows-linux) instructions for Windows and Linux. @@ -364,7 +364,7 @@ For models compatible with Iluvatar Extension for PyTorch. Here's a step-by-step | Flag | Description | |------|-------------| | `--enable-manager` | Enable ComfyUI-Manager | -| `--enable-manager-legacy-ui` | Use the legacy manager UI instead of the new UI (requires `--enable-manager`) | +| `--enable-manager-legacy-ui` | Use the legacy manager UI instead of the new UI (implies `--enable-manager`) | | `--disable-manager-ui` | Disable the manager UI and endpoints while keeping background features like security checks and scheduled installation completion (requires `--enable-manager`) | @@ -382,11 +382,7 @@ For AMD 7600 and maybe other RDNA3 cards: ```HSA_OVERRIDE_GFX_VERSION=11.0.0 pyt ### AMD ROCm Tips -You can enable experimental memory efficient attention on recent pytorch in ComfyUI on some AMD GPUs using this command, it should already be enabled by default on RDNA3. If this improves speed for you on latest pytorch on your GPU please report it so that I can enable it by default. - -```TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL=1 python main.py --use-pytorch-cross-attention``` - -You can also try setting this env variable `PYTORCH_TUNABLEOP_ENABLED=1` which might speed things up at the cost of a very slow initial run. +You can try setting this env variable `PYTORCH_TUNABLEOP_ENABLED=1` which might speed things up at the cost of a very slow initial run. # Notes @@ -462,16 +458,6 @@ To use the most up-to-date frontend version: This approach allows you to easily switch between the stable fortnightly release and the cutting-edge daily updates, or even specific versions for testing purposes. -### Accessing the Legacy Frontend - -If you need to use the legacy frontend for any reason, you can access it using the following command line argument: - -``` ---front-end-version Comfy-Org/ComfyUI_legacy_frontend@latest -``` - -This will use a snapshot of the legacy frontend preserved in the [ComfyUI Legacy Frontend repository](https://github.com/Comfy-Org/ComfyUI_legacy_frontend). - # QA ### Which GPU should I buy for this? diff --git a/app/assets/api/routes.py b/app/assets/api/routes.py index 7ef462f5c..53c84eff3 100644 --- a/app/assets/api/routes.py +++ b/app/assets/api/routes.py @@ -306,12 +306,15 @@ async def download_asset_content(request: web.Request) -> web.Response: 404, "FILE_NOT_FOUND", "Underlying file not found on disk." ) - _DANGEROUS_MIME_TYPES = { - "text/html", "text/html-sandboxed", "application/xhtml+xml", - "text/javascript", "text/css", - } - if content_type in _DANGEROUS_MIME_TYPES: + # User-controlled asset content must never render inline in the app origin + # (stored XSS via SVG/HTML/XML). Force dangerous types to download and + # override any requested inline disposition. Centralised through + # folder_paths.is_dangerous_content_type so this can't drift from /view and + # /userdata (the previous inline set here omitted image/svg+xml and missed + # the charset/casing/+xml-dialect bypasses). + if folder_paths.is_dangerous_content_type(content_type): content_type = "application/octet-stream" + disposition = "attachment" safe_name = (filename or "").replace("\r", "").replace("\n", "") encoded = urllib.parse.quote(safe_name) diff --git a/app/model_manager.py b/app/model_manager.py index 8f6e34b33..b0329ce17 100644 --- a/app/model_manager.py +++ b/app/model_manager.py @@ -50,21 +50,45 @@ class ModelFileManager: @routes.get("/experiment/models/preview/{folder}/{path_index}/{filename:.*}") async def get_model_preview(request): folder_name = request.match_info.get("folder", None) - path_index = int(request.match_info.get("path_index", None)) filename = request.match_info.get("filename", None) if folder_name not in folder_paths.folder_names_and_paths: return web.Response(status=404) + # The "{filename:.*}" capture also matches the empty string, which + # would resolve to the folder itself; reject it explicitly. + if not filename: + return web.Response(status=400) + + try: + path_index = int(request.match_info.get("path_index", None)) + except (TypeError, ValueError): + return web.Response(status=400) + folders = folder_paths.folder_names_and_paths[folder_name] + if path_index < 0 or path_index >= len(folders[0]): + return web.Response(status=404) folder = folders[0][path_index] - full_filename = os.path.join(folder, filename) + full_filename = os.path.normpath(os.path.join(folder, filename)) + + # Prevent path traversal: the requested file must stay within the + # configured model folder. `filename` is an unrestricted ".*" capture, + # so values like "../../../../etc/passwd" would otherwise escape it. + if not folder_paths.is_within_directory(folder, full_filename): + return web.Response(status=403) previews = self.get_model_previews(full_filename) default_preview = previews[0] if len(previews) > 0 else None if default_preview is None or (isinstance(default_preview, str) and not os.path.isfile(default_preview)): return web.Response(status=404) + # The preview is selected by a glob inside get_model_previews, so a + # companion file (e.g. "model.preview.png") could itself be a symlink + # resolving outside the model folder. Re-validate the file actually + # opened: is_within_directory realpaths it, catching symlink escape. + if isinstance(default_preview, str) and not folder_paths.is_within_directory(folder, default_preview): + return web.Response(status=403) + try: with Image.open(default_preview) as img: img_bytes = BytesIO() diff --git a/app/user_manager.py b/app/user_manager.py index 7b11e381c..de261ad39 100644 --- a/app/user_manager.py +++ b/app/user_manager.py @@ -6,6 +6,7 @@ import glob import shutil import logging import tempfile +import mimetypes from aiohttp import web from urllib import parse from comfy.cli_args import args @@ -336,7 +337,20 @@ class UserManager(): if not isinstance(path, str): return path - return web.FileResponse(path) + # User data files are arbitrary user-supplied content and are never + # meant to render inline. Disable MIME sniffing and force a download + # so uploaded markup/scripts can't execute in the app origin (stored + # XSS). Content-Disposition: attachment is the load-bearing guard; + # the content-type override and nosniff are defence in depth. + content_type = mimetypes.guess_type(path)[0] or 'application/octet-stream' + if folder_paths.is_dangerous_content_type(content_type): + content_type = 'application/octet-stream' + + return web.FileResponse(path, headers={ + "Content-Type": content_type, + "X-Content-Type-Options": "nosniff", + "Content-Disposition": "attachment", + }) @routes.post("/userdata/{file}") async def post_userdata(request): diff --git a/blueprints/Character Replacement (SCAIL-2 Base).json b/blueprints/Character Replacement (SCAIL-2 Base).json new file mode 100644 index 000000000..61803df65 --- /dev/null +++ b/blueprints/Character Replacement (SCAIL-2 Base).json @@ -0,0 +1,4191 @@ +{ + "revision": 0, + "last_node_id": 410, + "last_link_id": 0, + "nodes": [ + { + "id": 410, + "type": "35331397-69fb-40ad-b99a-7f17b1a53017", + "pos": [ + 2450, + 5670 + ], + "size": [ + 490, + 1120 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "label": "pose_video", + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": null + }, + { + "label": "reference_image", + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "link": null + }, + { + "label": "prompt", + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + }, + { + "label": "segment_index", + "name": "value", + "type": "INT", + "widget": { + "name": "value" + }, + "link": null + }, + { + "label": "replace_mode", + "name": "value_2", + "type": "BOOLEAN", + "widget": { + "name": "value_2" + }, + "link": null + }, + { + "label": "width", + "name": "value_3", + "type": "INT", + "widget": { + "name": "value_3" + }, + "link": null + }, + { + "label": "height", + "name": "value_4", + "type": "INT", + "widget": { + "name": "value_4" + }, + "link": null + }, + { + "label": "frame_count", + "name": "length", + "type": "INT", + "widget": { + "name": "length" + }, + "link": null + }, + { + "name": "previous_frame_count", + "type": "INT", + "widget": { + "name": "previous_frame_count" + }, + "link": null + }, + { + "name": "pose_strength", + "type": "FLOAT", + "widget": { + "name": "pose_strength" + }, + "link": null + }, + { + "name": "pose_start", + "type": "FLOAT", + "widget": { + "name": "pose_start" + }, + "link": null + }, + { + "name": "pose_end", + "type": "FLOAT", + "widget": { + "name": "pose_end" + }, + "link": null + }, + { + "label": "turbo_mode", + "name": "value_5", + "type": "BOOLEAN", + "widget": { + "name": "value_5" + }, + "link": null + }, + { + "name": "unet_name", + "type": "COMBO", + "widget": { + "name": "unet_name" + }, + "link": null + }, + { + "label": "distill_lora", + "name": "lora_name", + "type": "COMBO", + "widget": { + "name": "lora_name" + }, + "link": null + }, + { + "label": "dpo_lora", + "name": "lora_name_1", + "type": "COMBO", + "widget": { + "name": "lora_name_1" + }, + "link": null + }, + { + "name": "clip_name", + "type": "COMBO", + "widget": { + "name": "clip_name" + }, + "link": null + }, + { + "name": "vae_name", + "type": "COMBO", + "widget": { + "name": "vae_name" + }, + "link": null + }, + { + "label": "clip_vision", + "name": "clip_name_1", + "type": "COMBO", + "widget": { + "name": "clip_name_1" + }, + "link": null + }, + { + "label": "sam3_video_object", + "name": "text_1", + "type": "STRING", + "widget": { + "name": "text_1" + }, + "link": null + }, + { + "label": "sam3_image_object", + "name": "text_2", + "type": "STRING", + "widget": { + "name": "text_2" + }, + "link": null + }, + { + "label": "sam3_model", + "name": "ckpt_name", + "type": "COMBO", + "widget": { + "name": "ckpt_name" + }, + "link": null + }, + { + "name": "noise_seed", + "type": "INT", + "widget": { + "name": "noise_seed" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "output", + "name": "output", + "type": "IMAGE", + "links": [] + } + ], + "properties": { + "proxyWidgets": [ + [ + "405", + "text" + ], + [ + "391", + "value" + ], + [ + "398", + "value" + ], + [ + "387", + "value" + ], + [ + "388", + "value" + ], + [ + "386", + "length" + ], + [ + "406", + "previous_frame_count" + ], + [ + "406", + "pose_strength" + ], + [ + "406", + "pose_start" + ], + [ + "406", + "pose_end" + ], + [ + "402", + "value" + ], + [ + "374", + "unet_name" + ], + [ + "367", + "lora_name" + ], + [ + "408", + "lora_name" + ], + [ + "372", + "clip_name" + ], + [ + "368", + "vae_name" + ], + [ + "373", + "clip_name" + ], + [ + "393", + "text" + ], + [ + "407", + "text" + ], + [ + "394", + "ckpt_name" + ], + [ + "378", + "noise_seed" + ], + [ + "399", + "$$canvas-image-preview" + ], + [ + "400", + "$$canvas-image-preview" + ] + ], + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [], + "title": "Character Replacement (SCAIL-2 Base)" + } + ], + "links": [], + "version": 0.4, + "definitions": { + "subgraphs": [ + { + "id": "35331397-69fb-40ad-b99a-7f17b1a53017", + "version": 1, + "state": { + "lastGroupId": 17, + "lastNodeId": 410, + "lastLinkId": 570, + "lastRerouteId": 0 + }, + "revision": 0, + "config": {}, + "name": "Character Replacement (SCAIL-2 Base)", + "inputNode": { + "id": -10, + "bounding": [ + -440, + 4190, + 173.015625, + 508 + ] + }, + "outputNode": { + "id": -20, + "bounding": [ + 3980, + 3870, + 128, + 68 + ] + }, + "inputs": [ + { + "id": "e9aedfaf-1991-4a75-ad9b-8c3a279b0488", + "name": "video", + "type": "VIDEO", + "linkIds": [ + 409 + ], + "localized_name": "video", + "label": "pose_video", + "pos": [ + -290.984375, + 4214 + ] + }, + { + "id": "0badb26e-9abd-4e9c-b221-aab1237b8773", + "name": "images", + "type": "IMAGE", + "linkIds": [ + 469, + 408, + 473 + ], + "localized_name": "images", + "label": "reference_image", + "pos": [ + -290.984375, + 4234 + ] + }, + { + "id": "098f15aa-a066-422e-b491-eaf140bafb2c", + "name": "text", + "type": "STRING", + "linkIds": [ + 499 + ], + "label": "prompt", + "pos": [ + -290.984375, + 4254 + ] + }, + { + "id": "861ea850-8329-491b-9413-2588cdd77bbd", + "name": "value", + "type": "INT", + "linkIds": [ + 502 + ], + "label": "segment_index", + "pos": [ + -290.984375, + 4274 + ] + }, + { + "id": "dcf3ebf8-1263-458a-966b-9e77a5ad704a", + "name": "value_2", + "type": "BOOLEAN", + "linkIds": [ + 504 + ], + "label": "replace_mode", + "pos": [ + -290.984375, + 4294 + ] + }, + { + "id": "dbee10ce-bb98-4733-88a0-d4cd5b13c691", + "name": "value_3", + "type": "INT", + "linkIds": [ + 505 + ], + "label": "width", + "pos": [ + -290.984375, + 4314 + ] + }, + { + "id": "363e3ac7-0096-4480-b875-1d6ff1b8a2d9", + "name": "value_4", + "type": "INT", + "linkIds": [ + 506 + ], + "label": "height", + "pos": [ + -290.984375, + 4334 + ] + }, + { + "id": "4017e1ed-96cd-4c1c-9949-dc1a8c30386a", + "name": "length", + "type": "INT", + "linkIds": [ + 522, + 541 + ], + "label": "frame_count", + "pos": [ + -290.984375, + 4354 + ] + }, + { + "id": "617809fd-9647-4067-8f44-10d8acc8af82", + "name": "previous_frame_count", + "type": "INT", + "linkIds": [ + 542, + 543 + ], + "pos": [ + -290.984375, + 4374 + ] + }, + { + "id": "36ab49f6-528f-4301-a37a-0108aa7f5bec", + "name": "pose_strength", + "type": "FLOAT", + "linkIds": [ + 547 + ], + "pos": [ + -290.984375, + 4394 + ] + }, + { + "id": "24c2e93f-7bbc-4c6d-898e-36b440b8469b", + "name": "pose_start", + "type": "FLOAT", + "linkIds": [ + 548 + ], + "pos": [ + -290.984375, + 4414 + ] + }, + { + "id": "d1dbe632-bb2a-4c30-a7ea-5082f21dc686", + "name": "pose_end", + "type": "FLOAT", + "linkIds": [ + 549 + ], + "pos": [ + -290.984375, + 4434 + ] + }, + { + "id": "ddf07ba4-2837-40ef-925f-5996ea436334", + "name": "value_5", + "type": "BOOLEAN", + "linkIds": [ + 507 + ], + "label": "turbo_mode", + "pos": [ + -290.984375, + 4454 + ] + }, + { + "id": "ee8c796d-b326-40c6-9f9d-65f564053974", + "name": "unet_name", + "type": "COMBO", + "linkIds": [ + 508 + ], + "pos": [ + -290.984375, + 4474 + ] + }, + { + "id": "6ccae991-51d7-4c8e-adc8-c7f6377d681b", + "name": "lora_name", + "type": "COMBO", + "linkIds": [ + 509 + ], + "label": "distill_lora", + "pos": [ + -290.984375, + 4494 + ] + }, + { + "id": "30772913-ecf9-44f4-a638-2824404b24fb", + "name": "lora_name_1", + "type": "COMBO", + "linkIds": [ + 565 + ], + "label": "dpo_lora", + "pos": [ + -290.984375, + 4514 + ] + }, + { + "id": "2e9c2347-93fe-462f-b5ef-51613fb52c85", + "name": "clip_name", + "type": "COMBO", + "linkIds": [ + 510 + ], + "pos": [ + -290.984375, + 4534 + ] + }, + { + "id": "bc1822c0-7484-47b7-8d59-cf94788290f7", + "name": "vae_name", + "type": "COMBO", + "linkIds": [ + 511 + ], + "pos": [ + -290.984375, + 4554 + ] + }, + { + "id": "bbf2ee19-58b3-46f4-af6a-cd1557d60c8d", + "name": "clip_name_1", + "type": "COMBO", + "linkIds": [ + 512 + ], + "label": "clip_vision", + "pos": [ + -290.984375, + 4574 + ] + }, + { + "id": "556415bd-8ef3-4cfa-b182-0d7e3e4cd572", + "name": "text_1", + "type": "STRING", + "linkIds": [ + 513 + ], + "label": "sam3_video_object", + "pos": [ + -290.984375, + 4594 + ] + }, + { + "id": "c577930c-85a3-4e92-ac44-8ed639937217", + "name": "text_2", + "type": "STRING", + "linkIds": [ + 514 + ], + "label": "sam3_image_object", + "pos": [ + -290.984375, + 4614 + ] + }, + { + "id": "8f542c5e-da91-4dad-8ff8-4e81a42d5140", + "name": "ckpt_name", + "type": "COMBO", + "linkIds": [ + 515 + ], + "label": "sam3_model", + "pos": [ + -290.984375, + 4634 + ] + }, + { + "id": "3064d232-69a3-43b1-b350-05d8ca0e7c9f", + "name": "noise_seed", + "type": "INT", + "linkIds": [ + 558 + ], + "pos": [ + -290.984375, + 4654 + ] + } + ], + "outputs": [ + { + "id": "7ae88834-7553-45ec-a4e8-bab7d5276b45", + "name": "output", + "type": "IMAGE", + "linkIds": [ + 544 + ], + "localized_name": "output", + "pos": [ + 4004, + 3894 + ] + } + ], + "widgets": [], + "nodes": [ + { + "id": 367, + "type": "LoraLoaderModelOnly", + "pos": [ + 300, + 3590 + ], + "size": [ + 590, + 140 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 563 + }, + { + "localized_name": "lora_name", + "name": "lora_name", + "type": "COMBO", + "widget": { + "name": "lora_name" + }, + "link": 509 + }, + { + "localized_name": "strength_model", + "name": "strength_model", + "type": "FLOAT", + "widget": { + "name": "strength_model" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MODEL", + "name": "MODEL", + "type": "MODEL", + "links": [ + 398, + 425 + ] + } + ], + "properties": { + "Node name for S&R": "LoraLoaderModelOnly", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "models": [ + { + "name": "lightx2v_I2V_14B_480p_cfg_step_distill_rank64_bf16.safetensors", + "url": "https://huggingface.co/Kijai/WanVideo_comfy/resolve/main/Lightx2v/lightx2v_I2V_14B_480p_cfg_step_distill_rank64_bf16.safetensors", + "directory": "loras" + } + ] + }, + "widgets_values": [ + "lightx2v_I2V_14B_480p_cfg_step_distill_rank64_bf16.safetensors", + 0.8 + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 368, + "type": "VAELoader", + "pos": [ + 300, + 4020 + ], + "size": [ + 590, + 140 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "localized_name": "vae_name", + "name": "vae_name", + "type": "COMBO", + "widget": { + "name": "vae_name" + }, + "link": 511 + } + ], + "outputs": [ + { + "localized_name": "VAE", + "name": "VAE", + "type": "VAE", + "links": [ + 406, + 407 + ] + } + ], + "properties": { + "Node name for S&R": "VAELoader", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "models": [ + { + "name": "Wan2_1_VAE_bf16.safetensors", + "url": "https://huggingface.co/Kijai/WanVideo_comfy/resolve/main/Wan2_1_VAE_bf16.safetensors", + "directory": "vae" + } + ] + }, + "widgets_values": [ + "Wan2_1_VAE_bf16.safetensors" + ] + }, + { + "id": 369, + "type": "ResizeImageMaskNode", + "pos": [ + 1280, + 4860 + ], + "size": [ + 270, + 160 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "localized_name": "input", + "name": "input", + "type": "IMAGE,MASK", + "link": 491 + }, + { + "localized_name": "resize_type", + "name": "resize_type", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "resize_type" + }, + "link": null + }, + { + "localized_name": "width", + "name": "resize_type.width", + "type": "INT", + "widget": { + "name": "resize_type.width" + }, + "link": 444 + }, + { + "localized_name": "height", + "name": "resize_type.height", + "type": "INT", + "widget": { + "name": "resize_type.height" + }, + "link": 446 + }, + { + "localized_name": "crop", + "name": "resize_type.crop", + "type": "COMBO", + "widget": { + "name": "resize_type.crop" + }, + "link": null + }, + { + "localized_name": "scale_method", + "name": "scale_method", + "type": "COMBO", + "widget": { + "name": "scale_method" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "resized", + "name": "resized", + "type": "*", + "links": [ + 420, + 492, + 493 + ] + } + ], + "properties": { + "Node name for S&R": "ResizeImageMaskNode", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + "scale dimensions", + 512, + 512, + "center", + "area" + ] + }, + { + "id": 370, + "type": "GetVideoComponents", + "pos": [ + 270, + 4500 + ], + "size": [ + 230, + 90 + ], + "flags": { + "collapsed": true + }, + "order": 8, + "mode": 0, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 409 + } + ], + "outputs": [ + { + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "links": [ + 490 + ] + }, + { + "localized_name": "audio", + "name": "audio", + "type": "AUDIO", + "links": null + }, + { + "localized_name": "fps", + "name": "fps", + "type": "FLOAT", + "links": [] + }, + { + "localized_name": "bit_depth", + "name": "bit_depth", + "type": "INT", + "links": null + } + ], + "properties": { + "Node name for S&R": "GetVideoComponents", + "cnr_id": "comfy-core", + "ver": "0.24.0" + } + }, + { + "id": 371, + "type": "GetImageSize", + "pos": [ + 1640, + 4500 + ], + "size": [ + 240, + 190 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "localized_name": "image", + "name": "image", + "type": "IMAGE", + "link": 492 + } + ], + "outputs": [ + { + "localized_name": "width", + "name": "width", + "type": "INT", + "links": [ + 414 + ] + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "links": [ + 415 + ] + }, + { + "localized_name": "batch_size", + "name": "batch_size", + "type": "INT", + "links": [ + 416 + ] + } + ], + "properties": { + "Node name for S&R": "GetImageSize", + "cnr_id": "comfy-core", + "ver": "0.24.0" + } + }, + { + "id": 372, + "type": "CLIPLoader", + "pos": [ + 300, + 3790 + ], + "size": [ + 590, + 170 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "localized_name": "clip_name", + "name": "clip_name", + "type": "COMBO", + "widget": { + "name": "clip_name" + }, + "link": 510 + }, + { + "localized_name": "type", + "name": "type", + "type": "COMBO", + "widget": { + "name": "type" + }, + "link": null + }, + { + "localized_name": "device", + "name": "device", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "device" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "CLIP", + "name": "CLIP", + "type": "CLIP", + "links": [ + 7, + 8 + ] + } + ], + "properties": { + "Node name for S&R": "CLIPLoader", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "models": [ + { + "name": "umt5_xxl_fp8_e4m3fn_scaled.safetensors", + "url": "https://huggingface.co/Comfy-Org/Wan_2.1_ComfyUI_repackaged/resolve/main/split_files/text_encoders/umt5_xxl_fp8_e4m3fn_scaled.safetensors", + "directory": "text_encoders" + } + ] + }, + "widgets_values": [ + "umt5_xxl_fp8_e4m3fn_scaled.safetensors", + "wan", + "default" + ] + }, + { + "id": 373, + "type": "CLIPVisionLoader", + "pos": [ + 300, + 4230 + ], + "size": [ + 590, + 110 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "localized_name": "clip_name", + "name": "clip_name", + "type": "COMBO", + "widget": { + "name": "clip_name" + }, + "link": 512 + } + ], + "outputs": [ + { + "localized_name": "CLIP_VISION", + "name": "CLIP_VISION", + "type": "CLIP_VISION", + "links": [ + 196 + ] + } + ], + "properties": { + "Node name for S&R": "CLIPVisionLoader", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "models": [ + { + "name": "clip_vision_h.safetensors", + "url": "https://huggingface.co/Comfy-Org/Wan_2.1_ComfyUI_repackaged/resolve/main/split_files/clip_vision/clip_vision_h.safetensors", + "directory": "clip_vision" + } + ] + }, + "widgets_values": [ + "clip_vision_h.safetensors" + ] + }, + { + "id": 374, + "type": "UNETLoader", + "pos": [ + 300, + 3190 + ], + "size": [ + 590, + 140 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "localized_name": "unet_name", + "name": "unet_name", + "type": "COMBO", + "widget": { + "name": "unet_name" + }, + "link": 508 + }, + { + "localized_name": "weight_dtype", + "name": "weight_dtype", + "type": "COMBO", + "widget": { + "name": "weight_dtype" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MODEL", + "name": "MODEL", + "type": "MODEL", + "links": [ + 562 + ] + } + ], + "properties": { + "Node name for S&R": "UNETLoader", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "models": [ + { + "name": "wan2.1_14B_SCAIL_2_fp16.safetensors", + "url": "https://huggingface.co/Comfy-Org/SCAIL-2/resolve/main/diffusion_models/wan2.1_14B_SCAIL_2_fp16.safetensors", + "directory": "diffusion_models" + } + ] + }, + "widgets_values": [ + "wan2.1_14B_SCAIL_2_fp16.safetensors", + "default" + ] + }, + { + "id": 375, + "type": "ModelSamplingSD3", + "pos": [ + 2760, + 3370 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 424 + }, + { + "localized_name": "shift", + "name": "shift", + "type": "FLOAT", + "widget": { + "name": "shift" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MODEL", + "name": "MODEL", + "type": "MODEL", + "links": [ + 417 + ] + } + ], + "properties": { + "Node name for S&R": "ModelSamplingSD3", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 5 + ] + }, + { + "id": 376, + "type": "KSamplerSelect", + "pos": [ + 2770, + 3520 + ], + "size": [ + 260, + 110 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [ + { + "localized_name": "sampler_name", + "name": "sampler_name", + "type": "COMBO", + "widget": { + "name": "sampler_name" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "SAMPLER", + "name": "SAMPLER", + "type": "SAMPLER", + "links": [ + 419 + ] + } + ], + "properties": { + "Node name for S&R": "KSamplerSelect", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + "euler" + ] + }, + { + "id": 377, + "type": "BasicScheduler", + "pos": [ + 2770, + 3670 + ], + "size": [ + 260, + 170 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 398 + }, + { + "localized_name": "scheduler", + "name": "scheduler", + "type": "COMBO", + "widget": { + "name": "scheduler" + }, + "link": null + }, + { + "localized_name": "steps", + "name": "steps", + "type": "INT", + "widget": { + "name": "steps" + }, + "link": 428 + }, + { + "localized_name": "denoise", + "name": "denoise", + "type": "FLOAT", + "widget": { + "name": "denoise" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "SIGMAS", + "name": "SIGMAS", + "type": "SIGMAS", + "links": [ + 418 + ] + } + ], + "properties": { + "Node name for S&R": "BasicScheduler", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + "simple", + 6, + 1 + ] + }, + { + "id": 378, + "type": "SamplerCustom", + "pos": [ + 3080, + 3370 + ], + "size": [ + 270, + 670 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 417 + }, + { + "localized_name": "positive", + "name": "positive", + "type": "CONDITIONING", + "link": 333 + }, + { + "localized_name": "negative", + "name": "negative", + "type": "CONDITIONING", + "link": 334 + }, + { + "localized_name": "sampler", + "name": "sampler", + "type": "SAMPLER", + "link": 419 + }, + { + "localized_name": "sigmas", + "name": "sigmas", + "type": "SIGMAS", + "link": 418 + }, + { + "localized_name": "latent_image", + "name": "latent_image", + "type": "LATENT", + "link": 335 + }, + { + "localized_name": "add_noise", + "name": "add_noise", + "type": "BOOLEAN", + "widget": { + "name": "add_noise" + }, + "link": null + }, + { + "localized_name": "noise_seed", + "name": "noise_seed", + "type": "INT", + "widget": { + "name": "noise_seed" + }, + "link": 558 + }, + { + "localized_name": "cfg", + "name": "cfg", + "type": "FLOAT", + "widget": { + "name": "cfg" + }, + "link": 431 + } + ], + "outputs": [ + { + "localized_name": "output", + "name": "output", + "type": "LATENT", + "links": [] + }, + { + "localized_name": "denoised_output", + "name": "denoised_output", + "type": "LATENT", + "links": [ + 124 + ] + } + ], + "properties": { + "Node name for S&R": "SamplerCustom", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + true, + 1, + "fixed", + 1 + ] + }, + { + "id": 379, + "type": "PrimitiveInt", + "pos": [ + 2060, + 3750 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "INT", + "widget": { + "name": "value" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 427 + ] + } + ], + "title": "Int (Steps)", + "properties": { + "Node name for S&R": "PrimitiveInt", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 6, + "fixed" + ] + }, + { + "id": 380, + "type": "PrimitiveFloat", + "pos": [ + 2060, + 3920 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "FLOAT", + "widget": { + "name": "value" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "FLOAT", + "name": "FLOAT", + "type": "FLOAT", + "links": [ + 430 + ] + } + ], + "title": "Float (CFG)", + "properties": { + "Node name for S&R": "PrimitiveFloat", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 1 + ] + }, + { + "id": 381, + "type": "PrimitiveInt", + "pos": [ + 2060, + 3380 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "INT", + "widget": { + "name": "value" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 426 + ] + } + ], + "title": "Int (Steps)", + "properties": { + "Node name for S&R": "PrimitiveInt", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 40, + "fixed" + ] + }, + { + "id": 382, + "type": "PrimitiveFloat", + "pos": [ + 2060, + 3530 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "FLOAT", + "widget": { + "name": "value" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "FLOAT", + "name": "FLOAT", + "type": "FLOAT", + "links": [ + 429 + ] + } + ], + "title": "Float (CFG)", + "properties": { + "Node name for S&R": "PrimitiveFloat", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 5 + ] + }, + { + "id": 383, + "type": "ComfySwitchNode", + "pos": [ + 2400, + 3360 + ], + "size": [ + 270, + 130 + ], + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [ + { + "localized_name": "on_false", + "name": "on_false", + "type": "*", + "link": 564 + }, + { + "localized_name": "on_true", + "name": "on_true", + "type": "*", + "link": 425 + }, + { + "localized_name": "switch", + "name": "switch", + "type": "BOOLEAN", + "widget": { + "name": "switch" + }, + "link": 432 + } + ], + "outputs": [ + { + "localized_name": "output", + "name": "output", + "type": "*", + "links": [ + 424 + ] + } + ], + "title": "Switch (Model)", + "properties": { + "Node name for S&R": "ComfySwitchNode", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + false + ] + }, + { + "id": 384, + "type": "ComfySwitchNode", + "pos": [ + 2400, + 3550 + ], + "size": [ + 270, + 130 + ], + "flags": {}, + "order": 17, + "mode": 0, + "inputs": [ + { + "localized_name": "on_false", + "name": "on_false", + "type": "*", + "link": 426 + }, + { + "localized_name": "on_true", + "name": "on_true", + "type": "*", + "link": 427 + }, + { + "localized_name": "switch", + "name": "switch", + "type": "BOOLEAN", + "widget": { + "name": "switch" + }, + "link": 433 + } + ], + "outputs": [ + { + "localized_name": "output", + "name": "output", + "type": "*", + "links": [ + 428 + ] + } + ], + "title": "Switch (Steps)", + "properties": { + "Node name for S&R": "ComfySwitchNode", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + false + ] + }, + { + "id": 385, + "type": "ComfySwitchNode", + "pos": [ + 2400, + 3730 + ], + "size": [ + 270, + 130 + ], + "flags": {}, + "order": 18, + "mode": 0, + "inputs": [ + { + "localized_name": "on_false", + "name": "on_false", + "type": "*", + "link": 429 + }, + { + "localized_name": "on_true", + "name": "on_true", + "type": "*", + "link": 430 + }, + { + "localized_name": "switch", + "name": "switch", + "type": "BOOLEAN", + "widget": { + "name": "switch" + }, + "link": 434 + } + ], + "outputs": [ + { + "localized_name": "output", + "name": "output", + "type": "*", + "links": [ + 431 + ] + } + ], + "title": "Switch (Steps)", + "properties": { + "Node name for S&R": "ComfySwitchNode", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + false + ] + }, + { + "id": 386, + "type": "ImageFromBatch", + "pos": [ + 880, + 4490 + ], + "size": [ + 270, + 140 + ], + "flags": {}, + "order": 19, + "mode": 0, + "inputs": [ + { + "localized_name": "image", + "name": "image", + "type": "IMAGE", + "link": 490 + }, + { + "localized_name": "batch_index", + "name": "batch_index", + "type": "INT", + "widget": { + "name": "batch_index" + }, + "link": 450 + }, + { + "localized_name": "length", + "name": "length", + "type": "INT", + "widget": { + "name": "length" + }, + "link": 522 + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 491 + ] + } + ], + "properties": { + "Node name for S&R": "ImageFromBatch", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 0, + 81 + ] + }, + { + "id": 387, + "type": "PrimitiveInt", + "pos": [ + 250, + 4820 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 20, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "INT", + "widget": { + "name": "value" + }, + "link": 505 + } + ], + "outputs": [ + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 441 + ] + } + ], + "title": "Int (Width)", + "properties": { + "Node name for S&R": "PrimitiveInt", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 896, + "fixed" + ] + }, + { + "id": 388, + "type": "PrimitiveInt", + "pos": [ + 250, + 5000 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 21, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "INT", + "widget": { + "name": "value" + }, + "link": 506 + } + ], + "outputs": [ + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 445 + ] + } + ], + "title": "Int (Height)", + "properties": { + "Node name for S&R": "PrimitiveInt", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 512, + "fixed" + ] + }, + { + "id": 389, + "type": "ComfyMathExpression", + "pos": [ + 690, + 4850 + ], + "size": [ + 230, + 80 + ], + "flags": { + "collapsed": true + }, + "order": 22, + "mode": 0, + "inputs": [ + { + "label": "a", + "localized_name": "values.a", + "name": "values.a", + "type": "FLOAT,INT,BOOLEAN", + "link": 441 + }, + { + "label": "b", + "localized_name": "values.b", + "name": "values.b", + "shape": 7, + "type": "FLOAT,INT,BOOLEAN", + "link": null + }, + { + "localized_name": "expression", + "name": "expression", + "type": "STRING", + "widget": { + "name": "expression" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "FLOAT", + "name": "FLOAT", + "type": "FLOAT", + "links": [] + }, + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 444 + ] + }, + { + "localized_name": "BOOL", + "name": "BOOL", + "type": "BOOLEAN", + "links": null + } + ], + "properties": { + "Node name for S&R": "ComfyMathExpression", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + "(a // 32) * 32" + ] + }, + { + "id": 390, + "type": "ComfyMathExpression", + "pos": [ + 690, + 5030 + ], + "size": [ + 230, + 80 + ], + "flags": { + "collapsed": true + }, + "order": 23, + "mode": 0, + "inputs": [ + { + "label": "a", + "localized_name": "values.a", + "name": "values.a", + "type": "FLOAT,INT,BOOLEAN", + "link": 445 + }, + { + "label": "b", + "localized_name": "values.b", + "name": "values.b", + "shape": 7, + "type": "FLOAT,INT,BOOLEAN", + "link": null + }, + { + "localized_name": "expression", + "name": "expression", + "type": "STRING", + "widget": { + "name": "expression" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "FLOAT", + "name": "FLOAT", + "type": "FLOAT", + "links": [] + }, + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 446 + ] + }, + { + "localized_name": "BOOL", + "name": "BOOL", + "type": "BOOLEAN", + "links": null + } + ], + "properties": { + "Node name for S&R": "ComfyMathExpression", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + "(a // 32) * 32" + ] + }, + { + "id": 391, + "type": "PrimitiveInt", + "pos": [ + 260, + 4570 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 24, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "INT", + "widget": { + "name": "value" + }, + "link": 502 + } + ], + "outputs": [ + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 447 + ] + } + ], + "title": "Int (segment index)", + "properties": { + "Node name for S&R": "PrimitiveInt", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 1, + "fixed" + ] + }, + { + "id": 392, + "type": "ComfyMathExpression", + "pos": [ + 580, + 4590 + ], + "size": [ + 230, + 220 + ], + "flags": { + "collapsed": false + }, + "order": 25, + "mode": 0, + "inputs": [ + { + "label": "a", + "localized_name": "values.a", + "name": "values.a", + "type": "FLOAT,INT,BOOLEAN", + "link": 447 + }, + { + "label": "b", + "localized_name": "values.b", + "name": "values.b", + "shape": 7, + "type": "FLOAT,INT,BOOLEAN", + "link": 541 + }, + { + "label": "c", + "localized_name": "values.c", + "name": "values.c", + "shape": 7, + "type": "FLOAT,INT,BOOLEAN", + "link": 543 + }, + { + "label": "d", + "localized_name": "values.d", + "name": "values.d", + "shape": 7, + "type": "FLOAT,INT,BOOLEAN", + "link": null + }, + { + "localized_name": "expression", + "name": "expression", + "type": "STRING", + "widget": { + "name": "expression" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "FLOAT", + "name": "FLOAT", + "type": "FLOAT", + "links": null + }, + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 450 + ] + }, + { + "localized_name": "BOOL", + "name": "BOOL", + "type": "BOOLEAN", + "links": null + } + ], + "properties": { + "Node name for S&R": "ComfyMathExpression", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + "(b - c) * (a - 1)" + ] + }, + { + "id": 393, + "type": "CLIPTextEncode", + "pos": [ + 660, + 5360 + ], + "size": [ + 380, + 160 + ], + "flags": {}, + "order": 26, + "mode": 0, + "inputs": [ + { + "localized_name": "clip", + "name": "clip", + "type": "CLIP", + "link": 454 + }, + { + "localized_name": "text", + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": 513 + } + ], + "outputs": [ + { + "localized_name": "CONDITIONING", + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 461 + ] + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode", + "cnr_id": "comfy-core", + "ver": "0.19.3", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "" + ] + }, + { + "id": 394, + "type": "CheckpointLoaderSimple", + "pos": [ + 270, + 5340 + ], + "size": [ + 330, + 160 + ], + "flags": {}, + "order": 27, + "mode": 0, + "inputs": [ + { + "localized_name": "ckpt_name", + "name": "ckpt_name", + "type": "COMBO", + "widget": { + "name": "ckpt_name" + }, + "link": 515 + } + ], + "outputs": [ + { + "localized_name": "MODEL", + "name": "MODEL", + "type": "MODEL", + "links": [ + 458, + 463 + ] + }, + { + "localized_name": "CLIP", + "name": "CLIP", + "type": "CLIP", + "links": [ + 454, + 489 + ] + }, + { + "localized_name": "VAE", + "name": "VAE", + "type": "VAE", + "links": [] + } + ], + "properties": { + "Node name for S&R": "CheckpointLoaderSimple", + "cnr_id": "comfy-core", + "ver": "0.19.3", + "models": [ + { + "name": "sam3.1_multiplex_fp16.safetensors", + "url": "https://huggingface.co/Comfy-Org/sam3.1/resolve/main/checkpoints/sam3.1_multiplex_fp16.safetensors", + "directory": "checkpoints" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "sam3.1_multiplex_fp16.safetensors" + ] + }, + { + "id": 395, + "type": "SAM3_VideoTrack", + "pos": [ + 1190, + 5320 + ], + "size": [ + 280, + 250 + ], + "flags": {}, + "order": 28, + "mode": 0, + "inputs": [ + { + "label": "images", + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "link": 493 + }, + { + "label": "model", + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 458 + }, + { + "label": "initial_mask", + "localized_name": "initial_mask", + "name": "initial_mask", + "shape": 7, + "type": "MASK", + "link": null + }, + { + "label": "conditioning", + "localized_name": "conditioning", + "name": "conditioning", + "shape": 7, + "type": "CONDITIONING", + "link": 461 + }, + { + "localized_name": "detection_threshold", + "name": "detection_threshold", + "type": "FLOAT", + "widget": { + "name": "detection_threshold" + }, + "link": null + }, + { + "localized_name": "max_objects", + "name": "max_objects", + "type": "INT", + "widget": { + "name": "max_objects" + }, + "link": null + }, + { + "localized_name": "detect_interval", + "name": "detect_interval", + "type": "INT", + "widget": { + "name": "detect_interval" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "track_data", + "name": "track_data", + "type": "SAM3_TRACK_DATA", + "links": [ + 460 + ] + } + ], + "properties": { + "Node name for S&R": "SAM3_VideoTrack", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 0.5, + 4, + 1 + ] + }, + { + "id": 396, + "type": "SCAIL2ColoredMask", + "pos": [ + 1550, + 5470 + ], + "size": [ + 370, + 200 + ], + "flags": {}, + "order": 29, + "mode": 0, + "inputs": [ + { + "localized_name": "driving_track_data", + "name": "driving_track_data", + "type": "SAM3_TRACK_DATA", + "link": 460 + }, + { + "localized_name": "ref_track_data", + "name": "ref_track_data", + "shape": 7, + "type": "SAM3_TRACK_DATA", + "link": 464 + }, + { + "localized_name": "object_indices", + "name": "object_indices", + "type": "STRING", + "widget": { + "name": "object_indices" + }, + "link": null + }, + { + "localized_name": "sort_by", + "name": "sort_by", + "type": "COMBO", + "widget": { + "name": "sort_by" + }, + "link": null + }, + { + "localized_name": "replacement_mode", + "name": "replacement_mode", + "type": "BOOLEAN", + "widget": { + "name": "replacement_mode" + }, + "link": 476 + } + ], + "outputs": [ + { + "localized_name": "pose_video_mask", + "name": "pose_video_mask", + "type": "IMAGE", + "links": [ + 466, + 467 + ] + }, + { + "localized_name": "reference_image_mask", + "name": "reference_image_mask", + "type": "IMAGE", + "links": [ + 465, + 472 + ] + } + ], + "properties": { + "Node name for S&R": "SCAIL2ColoredMask", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + "", + "left_to_right", + true + ] + }, + { + "id": 397, + "type": "SAM3_VideoTrack", + "pos": [ + 1190, + 5620 + ], + "size": [ + 280, + 250 + ], + "flags": {}, + "order": 30, + "mode": 0, + "inputs": [ + { + "label": "images", + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "link": 469 + }, + { + "label": "model", + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 463 + }, + { + "label": "initial_mask", + "localized_name": "initial_mask", + "name": "initial_mask", + "shape": 7, + "type": "MASK", + "link": null + }, + { + "label": "conditioning", + "localized_name": "conditioning", + "name": "conditioning", + "shape": 7, + "type": "CONDITIONING", + "link": 488 + }, + { + "localized_name": "detection_threshold", + "name": "detection_threshold", + "type": "FLOAT", + "widget": { + "name": "detection_threshold" + }, + "link": null + }, + { + "localized_name": "max_objects", + "name": "max_objects", + "type": "INT", + "widget": { + "name": "max_objects" + }, + "link": null + }, + { + "localized_name": "detect_interval", + "name": "detect_interval", + "type": "INT", + "widget": { + "name": "detect_interval" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "track_data", + "name": "track_data", + "type": "SAM3_TRACK_DATA", + "links": [ + 464 + ] + } + ], + "properties": { + "Node name for S&R": "SAM3_VideoTrack", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 0.5, + 4, + 1 + ] + }, + { + "id": 398, + "type": "PrimitiveBoolean", + "pos": [ + 1660, + 4200 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 31, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "BOOLEAN", + "widget": { + "name": "value" + }, + "link": 504 + } + ], + "outputs": [ + { + "localized_name": "BOOLEAN", + "name": "BOOLEAN", + "type": "BOOLEAN", + "links": [ + 475, + 476 + ] + } + ], + "title": "Boolean （Replace Mode）", + "properties": { + "Node name for S&R": "PrimitiveBoolean", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + true + ] + }, + { + "id": 399, + "type": "PreviewImage", + "pos": [ + 2040, + 4470 + ], + "size": [ + 350, + 1190 + ], + "flags": {}, + "order": 32, + "mode": 4, + "inputs": [ + { + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "link": 466 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage", + "cnr_id": "comfy-core", + "ver": "0.24.0" + } + }, + { + "id": 400, + "type": "PreviewImage", + "pos": [ + 2440, + 4470 + ], + "size": [ + 230, + 310 + ], + "flags": {}, + "order": 33, + "mode": 4, + "inputs": [ + { + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "link": 465 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage", + "cnr_id": "comfy-core", + "ver": "0.24.0" + } + }, + { + "id": 401, + "type": "VAEDecode", + "pos": [ + 3080, + 4120 + ], + "size": [ + 270, + 100 + ], + "flags": { + "collapsed": false + }, + "order": 34, + "mode": 0, + "inputs": [ + { + "localized_name": "samples", + "name": "samples", + "type": "LATENT", + "link": 124 + }, + { + "localized_name": "vae", + "name": "vae", + "type": "VAE", + "link": 407 + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 544 + ] + } + ], + "properties": { + "Node name for S&R": "VAEDecode", + "cnr_id": "comfy-core", + "ver": "0.24.0" + } + }, + { + "id": 402, + "type": "PrimitiveBoolean", + "pos": [ + 2410, + 3940 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 35, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "BOOLEAN", + "widget": { + "name": "value" + }, + "link": 507 + } + ], + "outputs": [ + { + "localized_name": "BOOLEAN", + "name": "BOOLEAN", + "type": "BOOLEAN", + "links": [ + 432, + 433, + 434 + ] + } + ], + "title": "Boolean (Enable Distill LoRA?)", + "properties": { + "Node name for S&R": "PrimitiveBoolean", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + true + ] + }, + { + "id": 403, + "type": "CLIPVisionEncode", + "pos": [ + 1200, + 4260 + ], + "size": [ + 230, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 36, + "mode": 0, + "inputs": [ + { + "localized_name": "clip_vision", + "name": "clip_vision", + "type": "CLIP_VISION", + "link": 196 + }, + { + "localized_name": "image", + "name": "image", + "type": "IMAGE", + "link": 408 + }, + { + "localized_name": "crop", + "name": "crop", + "type": "COMBO", + "widget": { + "name": "crop" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "CLIP_VISION_OUTPUT", + "name": "CLIP_VISION_OUTPUT", + "type": "CLIP_VISION_OUTPUT", + "links": [ + 404 + ] + } + ], + "properties": { + "Node name for S&R": "CLIPVisionEncode", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + "none" + ] + }, + { + "id": 404, + "type": "CLIPTextEncode", + "pos": [ + 1000, + 3970 + ], + "size": [ + 520, + 210 + ], + "flags": {}, + "order": 37, + "mode": 0, + "inputs": [ + { + "localized_name": "clip", + "name": "clip", + "type": "CLIP", + "link": 8 + }, + { + "localized_name": "text", + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "CONDITIONING", + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 326 + ] + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + "" + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 405, + "type": "CLIPTextEncode", + "pos": [ + 990, + 3170 + ], + "size": [ + 550, + 750 + ], + "flags": {}, + "order": 38, + "mode": 0, + "inputs": [ + { + "localized_name": "clip", + "name": "clip", + "type": "CLIP", + "link": 7 + }, + { + "localized_name": "text", + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": 499 + } + ], + "outputs": [ + { + "localized_name": "CONDITIONING", + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 325 + ] + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + "A young woman with dark hair tied in a neat high bun, with a few loose strands framing her face, is dancing outdoors on a sunny coastal hillside. She has a normal-sized head and a slim face, with no hat, no headwear, and no oversized hair volume. She wears a fitted black long-sleeve crop top with a shoulder cutout, extremely baggy black cargo pants with straps and pockets, and chunky black combat boots. She performs energetic dance moves with one leg lifted and arms extended, moving naturally in front of a large tree, a small white stone house with a terracotta roof, and a bright blue sea under a clear sky with light clouds." + ], + "color": "#232", + "bgcolor": "#353" + }, + { + "id": 406, + "type": "WanSCAILToVideo", + "pos": [ + 1640, + 3400 + ], + "size": [ + 310, + 580 + ], + "flags": {}, + "order": 39, + "mode": 0, + "inputs": [ + { + "localized_name": "positive", + "name": "positive", + "type": "CONDITIONING", + "link": 325 + }, + { + "localized_name": "negative", + "name": "negative", + "type": "CONDITIONING", + "link": 326 + }, + { + "localized_name": "vae", + "name": "vae", + "type": "VAE", + "link": 406 + }, + { + "localized_name": "pose_video", + "name": "pose_video", + "shape": 7, + "type": "IMAGE", + "link": 420 + }, + { + "localized_name": "pose_video_mask", + "name": "pose_video_mask", + "shape": 7, + "type": "IMAGE", + "link": 467 + }, + { + "localized_name": "reference_image", + "name": "reference_image", + "shape": 7, + "type": "IMAGE", + "link": 473 + }, + { + "localized_name": "reference_image_mask", + "name": "reference_image_mask", + "shape": 7, + "type": "IMAGE", + "link": 472 + }, + { + "localized_name": "clip_vision_output", + "name": "clip_vision_output", + "shape": 7, + "type": "CLIP_VISION_OUTPUT", + "link": 404 + }, + { + "localized_name": "previous_frames", + "name": "previous_frames", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": 414 + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": 415 + }, + { + "localized_name": "length", + "name": "length", + "type": "INT", + "widget": { + "name": "length" + }, + "link": 416 + }, + { + "localized_name": "batch_size", + "name": "batch_size", + "type": "INT", + "widget": { + "name": "batch_size" + }, + "link": null + }, + { + "localized_name": "pose_strength", + "name": "pose_strength", + "type": "FLOAT", + "widget": { + "name": "pose_strength" + }, + "link": 547 + }, + { + "localized_name": "pose_start", + "name": "pose_start", + "type": "FLOAT", + "widget": { + "name": "pose_start" + }, + "link": 548 + }, + { + "localized_name": "pose_end", + "name": "pose_end", + "type": "FLOAT", + "widget": { + "name": "pose_end" + }, + "link": 549 + }, + { + "localized_name": "video_frame_offset", + "name": "video_frame_offset", + "type": "INT", + "widget": { + "name": "video_frame_offset" + }, + "link": null + }, + { + "localized_name": "previous_frame_count", + "name": "previous_frame_count", + "type": "INT", + "widget": { + "name": "previous_frame_count" + }, + "link": 542 + }, + { + "localized_name": "replacement_mode", + "name": "replacement_mode", + "shape": 7, + "type": "BOOLEAN", + "widget": { + "name": "replacement_mode" + }, + "link": 475 + } + ], + "outputs": [ + { + "localized_name": "positive", + "name": "positive", + "type": "CONDITIONING", + "links": [ + 333 + ] + }, + { + "localized_name": "negative", + "name": "negative", + "type": "CONDITIONING", + "links": [ + 334 + ] + }, + { + "localized_name": "latent", + "name": "latent", + "type": "LATENT", + "links": [ + 335 + ] + }, + { + "localized_name": "video_frame_offset", + "name": "video_frame_offset", + "type": "INT", + "links": [] + } + ], + "properties": { + "Node name for S&R": "WanSCAILToVideo", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 512, + 896, + 65, + 1, + 1, + 0, + 1, + 0, + 5, + true + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 407, + "type": "CLIPTextEncode", + "pos": [ + 670, + 5690 + ], + "size": [ + 380, + 160 + ], + "flags": {}, + "order": 40, + "mode": 0, + "inputs": [ + { + "localized_name": "clip", + "name": "clip", + "type": "CLIP", + "link": 489 + }, + { + "localized_name": "text", + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": 514 + } + ], + "outputs": [ + { + "localized_name": "CONDITIONING", + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 488 + ] + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode", + "cnr_id": "comfy-core", + "ver": "0.19.3", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "" + ] + }, + { + "id": 408, + "type": "LoraLoaderModelOnly", + "pos": [ + 310, + 3390 + ], + "size": [ + 580, + 140 + ], + "flags": {}, + "order": 41, + "mode": 0, + "inputs": [ + { + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 562 + }, + { + "localized_name": "lora_name", + "name": "lora_name", + "type": "COMBO", + "widget": { + "name": "lora_name" + }, + "link": 565 + }, + { + "localized_name": "strength_model", + "name": "strength_model", + "type": "FLOAT", + "widget": { + "name": "strength_model" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MODEL", + "name": "MODEL", + "type": "MODEL", + "links": [ + 563, + 564 + ] + } + ], + "properties": { + "Node name for S&R": "LoraLoaderModelOnly", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "models": [ + { + "name": "wan2.1_SCAIL_2_DPO_lora_bf16.safetensors", + "url": "https://huggingface.co/Comfy-Org/SCAIL-2/resolve/main/loras/wan2.1_SCAIL_2_DPO_lora_bf16.safetensors", + "directory": "loras" + } + ] + }, + "widgets_values": [ + "wan2.1_SCAIL_2_DPO_lora_bf16.safetensors", + 1 + ] + } + ], + "groups": [ + { + "id": 3, + "title": "Models", + "bounding": [ + 240, + 3100, + 710, + 1270 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 4, + "title": "Prompt", + "bounding": [ + 980, + 3100, + 570, + 1270 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 5, + "title": "Sampling", + "bounding": [ + 2730, + 3100, + 660, + 1270 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 6, + "title": "SCAIL Conditioning", + "bounding": [ + 1580, + 3100, + 420, + 1270 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 9, + "title": "Switch Settings", + "bounding": [ + 2030, + 3100, + 670, + 1270 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 7, + "title": "Original Settings", + "bounding": [ + 2050, + 3320, + 300, + 340 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 8, + "title": "Distill LoRA Settings", + "bounding": [ + 2050, + 3680, + 300, + 370 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 10, + "title": "Create Mask", + "bounding": [ + 240, + 5230, + 1760, + 670 + ], + "flags": {} + }, + { + "id": 11, + "title": "Resize Images", + "bounding": [ + 240, + 4750, + 1760, + 450 + ], + "flags": {} + }, + { + "id": 12, + "title": "Trim Frames from Video", + "bounding": [ + 240, + 4410, + 1760, + 310 + ], + "flags": {} + }, + { + "id": 13, + "title": "Preview Masks", + "bounding": [ + 2030, + 4410, + 660, + 1490 + ], + "flags": {} + } + ], + "links": [ + { + "id": 491, + "origin_id": 386, + "origin_slot": 0, + "target_id": 369, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 444, + "origin_id": 389, + "origin_slot": 1, + "target_id": 369, + "target_slot": 2, + "type": "INT" + }, + { + "id": 446, + "origin_id": 390, + "origin_slot": 1, + "target_id": 369, + "target_slot": 3, + "type": "INT" + }, + { + "id": 492, + "origin_id": 369, + "origin_slot": 0, + "target_id": 371, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 424, + "origin_id": 383, + "origin_slot": 0, + "target_id": 375, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 398, + "origin_id": 367, + "origin_slot": 0, + "target_id": 377, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 428, + "origin_id": 384, + "origin_slot": 0, + "target_id": 377, + "target_slot": 2, + "type": "INT" + }, + { + "id": 417, + "origin_id": 375, + "origin_slot": 0, + "target_id": 378, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 333, + "origin_id": 406, + "origin_slot": 0, + "target_id": 378, + "target_slot": 1, + "type": "CONDITIONING" + }, + { + "id": 334, + "origin_id": 406, + "origin_slot": 1, + "target_id": 378, + "target_slot": 2, + "type": "CONDITIONING" + }, + { + "id": 419, + "origin_id": 376, + "origin_slot": 0, + "target_id": 378, + "target_slot": 3, + "type": "SAMPLER" + }, + { + "id": 418, + "origin_id": 377, + "origin_slot": 0, + "target_id": 378, + "target_slot": 4, + "type": "SIGMAS" + }, + { + "id": 335, + "origin_id": 406, + "origin_slot": 2, + "target_id": 378, + "target_slot": 5, + "type": "LATENT" + }, + { + "id": 431, + "origin_id": 385, + "origin_slot": 0, + "target_id": 378, + "target_slot": 8, + "type": "FLOAT" + }, + { + "id": 425, + "origin_id": 367, + "origin_slot": 0, + "target_id": 383, + "target_slot": 1, + "type": "MODEL" + }, + { + "id": 432, + "origin_id": 402, + "origin_slot": 0, + "target_id": 383, + "target_slot": 2, + "type": "BOOLEAN" + }, + { + "id": 426, + "origin_id": 381, + "origin_slot": 0, + "target_id": 384, + "target_slot": 0, + "type": "INT" + }, + { + "id": 427, + "origin_id": 379, + "origin_slot": 0, + "target_id": 384, + "target_slot": 1, + "type": "INT" + }, + { + "id": 433, + "origin_id": 402, + "origin_slot": 0, + "target_id": 384, + "target_slot": 2, + "type": "BOOLEAN" + }, + { + "id": 429, + "origin_id": 382, + "origin_slot": 0, + "target_id": 385, + "target_slot": 0, + "type": "FLOAT" + }, + { + "id": 430, + "origin_id": 380, + "origin_slot": 0, + "target_id": 385, + "target_slot": 1, + "type": "FLOAT" + }, + { + "id": 434, + "origin_id": 402, + "origin_slot": 0, + "target_id": 385, + "target_slot": 2, + "type": "BOOLEAN" + }, + { + "id": 490, + "origin_id": 370, + "origin_slot": 0, + "target_id": 386, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 450, + "origin_id": 392, + "origin_slot": 1, + "target_id": 386, + "target_slot": 1, + "type": "INT" + }, + { + "id": 441, + "origin_id": 387, + "origin_slot": 0, + "target_id": 389, + "target_slot": 0, + "type": "INT" + }, + { + "id": 445, + "origin_id": 388, + "origin_slot": 0, + "target_id": 390, + "target_slot": 0, + "type": "INT" + }, + { + "id": 447, + "origin_id": 391, + "origin_slot": 0, + "target_id": 392, + "target_slot": 0, + "type": "INT" + }, + { + "id": 454, + "origin_id": 394, + "origin_slot": 1, + "target_id": 393, + "target_slot": 0, + "type": "CLIP" + }, + { + "id": 493, + "origin_id": 369, + "origin_slot": 0, + "target_id": 395, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 458, + "origin_id": 394, + "origin_slot": 0, + "target_id": 395, + "target_slot": 1, + "type": "MODEL" + }, + { + "id": 461, + "origin_id": 393, + "origin_slot": 0, + "target_id": 395, + "target_slot": 3, + "type": "CONDITIONING" + }, + { + "id": 460, + "origin_id": 395, + "origin_slot": 0, + "target_id": 396, + "target_slot": 0, + "type": "SAM3_TRACK_DATA" + }, + { + "id": 464, + "origin_id": 397, + "origin_slot": 0, + "target_id": 396, + "target_slot": 1, + "type": "SAM3_TRACK_DATA" + }, + { + "id": 476, + "origin_id": 398, + "origin_slot": 0, + "target_id": 396, + "target_slot": 4, + "type": "BOOLEAN" + }, + { + "id": 463, + "origin_id": 394, + "origin_slot": 0, + "target_id": 397, + "target_slot": 1, + "type": "MODEL" + }, + { + "id": 488, + "origin_id": 407, + "origin_slot": 0, + "target_id": 397, + "target_slot": 3, + "type": "CONDITIONING" + }, + { + "id": 466, + "origin_id": 396, + "origin_slot": 0, + "target_id": 399, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 465, + "origin_id": 396, + "origin_slot": 1, + "target_id": 400, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 124, + "origin_id": 378, + "origin_slot": 1, + "target_id": 401, + "target_slot": 0, + "type": "LATENT" + }, + { + "id": 407, + "origin_id": 368, + "origin_slot": 0, + "target_id": 401, + "target_slot": 1, + "type": "VAE" + }, + { + "id": 196, + "origin_id": 373, + "origin_slot": 0, + "target_id": 403, + "target_slot": 0, + "type": "CLIP_VISION" + }, + { + "id": 8, + "origin_id": 372, + "origin_slot": 0, + "target_id": 404, + "target_slot": 0, + "type": "CLIP" + }, + { + "id": 7, + "origin_id": 372, + "origin_slot": 0, + "target_id": 405, + "target_slot": 0, + "type": "CLIP" + }, + { + "id": 325, + "origin_id": 405, + "origin_slot": 0, + "target_id": 406, + "target_slot": 0, + "type": "CONDITIONING" + }, + { + "id": 326, + "origin_id": 404, + "origin_slot": 0, + "target_id": 406, + "target_slot": 1, + "type": "CONDITIONING" + }, + { + "id": 406, + "origin_id": 368, + "origin_slot": 0, + "target_id": 406, + "target_slot": 2, + "type": "VAE" + }, + { + "id": 420, + "origin_id": 369, + "origin_slot": 0, + "target_id": 406, + "target_slot": 3, + "type": "IMAGE" + }, + { + "id": 467, + "origin_id": 396, + "origin_slot": 0, + "target_id": 406, + "target_slot": 4, + "type": "IMAGE" + }, + { + "id": 472, + "origin_id": 396, + "origin_slot": 1, + "target_id": 406, + "target_slot": 6, + "type": "IMAGE" + }, + { + "id": 404, + "origin_id": 403, + "origin_slot": 0, + "target_id": 406, + "target_slot": 7, + "type": "CLIP_VISION_OUTPUT" + }, + { + "id": 414, + "origin_id": 371, + "origin_slot": 0, + "target_id": 406, + "target_slot": 9, + "type": "INT" + }, + { + "id": 415, + "origin_id": 371, + "origin_slot": 1, + "target_id": 406, + "target_slot": 10, + "type": "INT" + }, + { + "id": 416, + "origin_id": 371, + "origin_slot": 2, + "target_id": 406, + "target_slot": 11, + "type": "INT" + }, + { + "id": 475, + "origin_id": 398, + "origin_slot": 0, + "target_id": 406, + "target_slot": 18, + "type": "BOOLEAN" + }, + { + "id": 489, + "origin_id": 394, + "origin_slot": 1, + "target_id": 407, + "target_slot": 0, + "type": "CLIP" + }, + { + "id": 409, + "origin_id": -10, + "origin_slot": 0, + "target_id": 370, + "target_slot": 0, + "type": "VIDEO" + }, + { + "id": 469, + "origin_id": -10, + "origin_slot": 1, + "target_id": 397, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 408, + "origin_id": -10, + "origin_slot": 1, + "target_id": 403, + "target_slot": 1, + "type": "IMAGE" + }, + { + "id": 473, + "origin_id": -10, + "origin_slot": 1, + "target_id": 406, + "target_slot": 5, + "type": "IMAGE" + }, + { + "id": 499, + "origin_id": -10, + "origin_slot": 2, + "target_id": 405, + "target_slot": 1, + "type": "STRING" + }, + { + "id": 502, + "origin_id": -10, + "origin_slot": 3, + "target_id": 391, + "target_slot": 0, + "type": "INT" + }, + { + "id": 504, + "origin_id": -10, + "origin_slot": 4, + "target_id": 398, + "target_slot": 0, + "type": "BOOLEAN" + }, + { + "id": 505, + "origin_id": -10, + "origin_slot": 5, + "target_id": 387, + "target_slot": 0, + "type": "INT" + }, + { + "id": 506, + "origin_id": -10, + "origin_slot": 6, + "target_id": 388, + "target_slot": 0, + "type": "INT" + }, + { + "id": 507, + "origin_id": -10, + "origin_slot": 12, + "target_id": 402, + "target_slot": 0, + "type": "BOOLEAN" + }, + { + "id": 508, + "origin_id": -10, + "origin_slot": 13, + "target_id": 374, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 509, + "origin_id": -10, + "origin_slot": 14, + "target_id": 367, + "target_slot": 1, + "type": "COMBO" + }, + { + "id": 510, + "origin_id": -10, + "origin_slot": 16, + "target_id": 372, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 511, + "origin_id": -10, + "origin_slot": 17, + "target_id": 368, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 512, + "origin_id": -10, + "origin_slot": 18, + "target_id": 373, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 513, + "origin_id": -10, + "origin_slot": 19, + "target_id": 393, + "target_slot": 1, + "type": "STRING" + }, + { + "id": 514, + "origin_id": -10, + "origin_slot": 20, + "target_id": 407, + "target_slot": 1, + "type": "STRING" + }, + { + "id": 515, + "origin_id": -10, + "origin_slot": 21, + "target_id": 394, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 522, + "origin_id": -10, + "origin_slot": 7, + "target_id": 386, + "target_slot": 2, + "type": "INT" + }, + { + "id": 541, + "origin_id": -10, + "origin_slot": 7, + "target_id": 392, + "target_slot": 1, + "type": "FLOAT,INT,BOOLEAN" + }, + { + "id": 542, + "origin_id": -10, + "origin_slot": 8, + "target_id": 406, + "target_slot": 17, + "type": "INT" + }, + { + "id": 543, + "origin_id": -10, + "origin_slot": 8, + "target_id": 392, + "target_slot": 2, + "type": "FLOAT,INT,BOOLEAN" + }, + { + "id": 544, + "origin_id": 401, + "origin_slot": 0, + "target_id": -20, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 547, + "origin_id": -10, + "origin_slot": 9, + "target_id": 406, + "target_slot": 13, + "type": "FLOAT" + }, + { + "id": 548, + "origin_id": -10, + "origin_slot": 10, + "target_id": 406, + "target_slot": 14, + "type": "FLOAT" + }, + { + "id": 549, + "origin_id": -10, + "origin_slot": 11, + "target_id": 406, + "target_slot": 15, + "type": "FLOAT" + }, + { + "id": 558, + "origin_id": -10, + "origin_slot": 22, + "target_id": 378, + "target_slot": 7, + "type": "INT" + }, + { + "id": 562, + "origin_id": 374, + "origin_slot": 0, + "target_id": 408, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 563, + "origin_id": 408, + "origin_slot": 0, + "target_id": 367, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 564, + "origin_id": 408, + "origin_slot": 0, + "target_id": 383, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 565, + "origin_id": -10, + "origin_slot": 15, + "target_id": 408, + "target_slot": 1, + "type": "COMBO" + } + ], + "extra": {}, + "category": "Video generation and editing/Video Edit", + "description": "Replaces a character in a video with a reference image using the SCAIL-2 model for end-to-end controlled animation without intermediate pose maps. Key inputs include a source video, a reference character image, and optional text prompts for style or context. Suitable for animated or live-action footage, multi-character scenes, and creative video editing where direct pose-free animation is needed; works best with moderate-length videos." + } + ] + }, + "extra": { + "BlueprintDescription": "Replaces a character in a video with a reference image using the SCAIL-2 model for end-to-end controlled animation without intermediate pose maps. Key inputs include a source video, a reference character image, and optional text prompts for style or context. Suitable for animated or live-action footage, multi-character scenes, and creative video editing where direct pose-free animation is needed; works best with moderate-length videos.", + "BlueprintSearchAliases": [ + "character replacement", + "SCAIL-2", + "pose-free animation", + "video editing", + "cross-identity replacement" + ] + } +} \ No newline at end of file diff --git a/blueprints/Character Replacement (SCAIL-2 Extend).json b/blueprints/Character Replacement (SCAIL-2 Extend).json new file mode 100644 index 000000000..bf2f67dfd --- /dev/null +++ b/blueprints/Character Replacement (SCAIL-2 Extend).json @@ -0,0 +1,4461 @@ +{ + "revision": 0, + "last_node_id": 409, + "last_link_id": 0, + "nodes": [ + { + "id": 409, + "type": "2ebfb952-bd3b-43c3-9390-9ea73d41bd1f", + "pos": [ + 3420, + 5580 + ], + "size": [ + 530, + 1140 + ], + "flags": { + "collapsed": false + }, + "order": 4, + "mode": 0, + "inputs": [ + { + "label": "pose_video", + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": null + }, + { + "label": "reference_image", + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "link": null + }, + { + "label": "previous_frames", + "localized_name": "image", + "name": "image", + "type": "IMAGE", + "link": null + }, + { + "label": "prompt", + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + }, + { + "label": "segment_index", + "name": "value", + "type": "INT", + "widget": { + "name": "value" + }, + "link": null + }, + { + "label": "replace_mode", + "name": "value_2", + "type": "BOOLEAN", + "widget": { + "name": "value_2" + }, + "link": null + }, + { + "label": "width", + "name": "value_3", + "type": "INT", + "widget": { + "name": "value_3" + }, + "link": null + }, + { + "label": "height", + "name": "value_4", + "type": "INT", + "widget": { + "name": "value_4" + }, + "link": null + }, + { + "label": "frame_count", + "name": "length", + "type": "INT", + "widget": { + "name": "length" + }, + "link": null + }, + { + "name": "previous_frame_count", + "type": "INT", + "widget": { + "name": "previous_frame_count" + }, + "link": null + }, + { + "name": "pose_strength", + "type": "FLOAT", + "widget": { + "name": "pose_strength" + }, + "link": null + }, + { + "name": "pose_start", + "type": "FLOAT", + "widget": { + "name": "pose_start" + }, + "link": null + }, + { + "name": "pose_end", + "type": "FLOAT", + "widget": { + "name": "pose_end" + }, + "link": null + }, + { + "label": "turbo_mode", + "name": "value_5", + "type": "BOOLEAN", + "widget": { + "name": "value_5" + }, + "link": null + }, + { + "name": "unet_name", + "type": "COMBO", + "widget": { + "name": "unet_name" + }, + "link": null + }, + { + "label": "distill_lora", + "name": "lora_name", + "type": "COMBO", + "widget": { + "name": "lora_name" + }, + "link": null + }, + { + "label": "dpo_lora", + "name": "lora_name_1", + "type": "COMBO", + "widget": { + "name": "lora_name_1" + }, + "link": null + }, + { + "name": "clip_name", + "type": "COMBO", + "widget": { + "name": "clip_name" + }, + "link": null + }, + { + "name": "vae_name", + "type": "COMBO", + "widget": { + "name": "vae_name" + }, + "link": null + }, + { + "label": "clip_vision", + "name": "clip_name_1", + "type": "COMBO", + "widget": { + "name": "clip_name_1" + }, + "link": null + }, + { + "label": "sam3_video_object", + "name": "text_1", + "type": "STRING", + "widget": { + "name": "text_1" + }, + "link": null + }, + { + "label": "sam3_image_object", + "name": "text_2", + "type": "STRING", + "widget": { + "name": "text_2" + }, + "link": null + }, + { + "label": "sam3_model", + "name": "ckpt_name", + "type": "COMBO", + "widget": { + "name": "ckpt_name" + }, + "link": null + }, + { + "name": "noise_seed", + "type": "INT", + "widget": { + "name": "noise_seed" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "output", + "name": "output", + "type": "IMAGE", + "links": [] + } + ], + "properties": { + "proxyWidgets": [ + [ + "363", + "text" + ], + [ + "346", + "value" + ], + [ + "353", + "value" + ], + [ + "342", + "value" + ], + [ + "343", + "value" + ], + [ + "341", + "length" + ], + [ + "364", + "previous_frame_count" + ], + [ + "364", + "pose_strength" + ], + [ + "364", + "pose_start" + ], + [ + "364", + "pose_end" + ], + [ + "360", + "value" + ], + [ + "329", + "unet_name" + ], + [ + "322", + "lora_name" + ], + [ + "366", + "lora_name" + ], + [ + "327", + "clip_name" + ], + [ + "323", + "vae_name" + ], + [ + "328", + "clip_name" + ], + [ + "348", + "text" + ], + [ + "365", + "text" + ], + [ + "349", + "ckpt_name" + ], + [ + "333", + "noise_seed" + ], + [ + "354", + "$$canvas-image-preview" + ], + [ + "355", + "$$canvas-image-preview" + ] + ], + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [], + "title": "Character Replacement (SCAIL-2 Extend)" + } + ], + "links": [], + "version": 0.4, + "definitions": { + "subgraphs": [ + { + "id": "2ebfb952-bd3b-43c3-9390-9ea73d41bd1f", + "version": 1, + "state": { + "lastGroupId": 17, + "lastNodeId": 410, + "lastLinkId": 570, + "lastRerouteId": 0 + }, + "revision": 0, + "config": {}, + "name": "Character Replacement (SCAIL-2 Extend)", + "inputNode": { + "id": -10, + "bounding": [ + -300, + 4240, + 173.015625, + 528 + ] + }, + "outputNode": { + "id": -20, + "bounding": [ + 3570, + 4500, + 128, + 68 + ] + }, + "inputs": [ + { + "id": "e9aedfaf-1991-4a75-ad9b-8c3a279b0488", + "name": "video", + "type": "VIDEO", + "linkIds": [ + 409 + ], + "localized_name": "video", + "label": "pose_video", + "pos": [ + -150.984375, + 4264 + ] + }, + { + "id": "0badb26e-9abd-4e9c-b221-aab1237b8773", + "name": "images", + "type": "IMAGE", + "linkIds": [ + 469, + 408, + 473 + ], + "localized_name": "images", + "label": "reference_image", + "pos": [ + -150.984375, + 4284 + ] + }, + { + "id": "9f424e9e-d5a2-4dd9-9934-cc6f6fc8da09", + "name": "image", + "type": "IMAGE", + "linkIds": [ + 486, + 485 + ], + "localized_name": "image", + "label": "previous_frames", + "pos": [ + -150.984375, + 4304 + ] + }, + { + "id": "098f15aa-a066-422e-b491-eaf140bafb2c", + "name": "text", + "type": "STRING", + "linkIds": [ + 499 + ], + "label": "prompt", + "pos": [ + -150.984375, + 4324 + ] + }, + { + "id": "861ea850-8329-491b-9413-2588cdd77bbd", + "name": "value", + "type": "INT", + "linkIds": [ + 502 + ], + "label": "segment_index", + "pos": [ + -150.984375, + 4344 + ] + }, + { + "id": "dcf3ebf8-1263-458a-966b-9e77a5ad704a", + "name": "value_2", + "type": "BOOLEAN", + "linkIds": [ + 504 + ], + "label": "replace_mode", + "pos": [ + -150.984375, + 4364 + ] + }, + { + "id": "dbee10ce-bb98-4733-88a0-d4cd5b13c691", + "name": "value_3", + "type": "INT", + "linkIds": [ + 505 + ], + "label": "width", + "pos": [ + -150.984375, + 4384 + ] + }, + { + "id": "363e3ac7-0096-4480-b875-1d6ff1b8a2d9", + "name": "value_4", + "type": "INT", + "linkIds": [ + 506 + ], + "label": "height", + "pos": [ + -150.984375, + 4404 + ] + }, + { + "id": "dc19eaf1-dd24-4c1e-9b08-6d8d98e0e7f2", + "name": "length", + "type": "INT", + "linkIds": [ + 550, + 557 + ], + "label": "frame_count", + "pos": [ + -150.984375, + 4424 + ] + }, + { + "id": "39d2f098-1740-419e-9c29-e4c4e4a7c3fd", + "name": "previous_frame_count", + "type": "INT", + "linkIds": [ + 551, + 556 + ], + "pos": [ + -150.984375, + 4444 + ] + }, + { + "id": "8d248225-a7f5-42c5-86bd-31eaf695f66e", + "name": "pose_strength", + "type": "FLOAT", + "linkIds": [ + 552 + ], + "pos": [ + -150.984375, + 4464 + ] + }, + { + "id": "04c56360-963a-48ba-944e-4864e2c0349c", + "name": "pose_start", + "type": "FLOAT", + "linkIds": [ + 553 + ], + "pos": [ + -150.984375, + 4484 + ] + }, + { + "id": "1ecd0315-d61f-4986-837c-27fb3f2d0470", + "name": "pose_end", + "type": "FLOAT", + "linkIds": [ + 554 + ], + "pos": [ + -150.984375, + 4504 + ] + }, + { + "id": "ddf07ba4-2837-40ef-925f-5996ea436334", + "name": "value_5", + "type": "BOOLEAN", + "linkIds": [ + 507 + ], + "label": "turbo_mode", + "pos": [ + -150.984375, + 4524 + ] + }, + { + "id": "ee8c796d-b326-40c6-9f9d-65f564053974", + "name": "unet_name", + "type": "COMBO", + "linkIds": [ + 508 + ], + "pos": [ + -150.984375, + 4544 + ] + }, + { + "id": "6ccae991-51d7-4c8e-adc8-c7f6377d681b", + "name": "lora_name", + "type": "COMBO", + "linkIds": [ + 509 + ], + "label": "distill_lora", + "pos": [ + -150.984375, + 4564 + ] + }, + { + "id": "1f1ac950-6ce5-4253-a266-edba58acd135", + "name": "lora_name_1", + "type": "COMBO", + "linkIds": [ + 569 + ], + "label": "dpo_lora", + "pos": [ + -150.984375, + 4584 + ] + }, + { + "id": "2e9c2347-93fe-462f-b5ef-51613fb52c85", + "name": "clip_name", + "type": "COMBO", + "linkIds": [ + 510 + ], + "pos": [ + -150.984375, + 4604 + ] + }, + { + "id": "bc1822c0-7484-47b7-8d59-cf94788290f7", + "name": "vae_name", + "type": "COMBO", + "linkIds": [ + 511 + ], + "pos": [ + -150.984375, + 4624 + ] + }, + { + "id": "bbf2ee19-58b3-46f4-af6a-cd1557d60c8d", + "name": "clip_name_1", + "type": "COMBO", + "linkIds": [ + 512 + ], + "label": "clip_vision", + "pos": [ + -150.984375, + 4644 + ] + }, + { + "id": "556415bd-8ef3-4cfa-b182-0d7e3e4cd572", + "name": "text_1", + "type": "STRING", + "linkIds": [ + 513 + ], + "label": "sam3_video_object", + "pos": [ + -150.984375, + 4664 + ] + }, + { + "id": "c577930c-85a3-4e92-ac44-8ed639937217", + "name": "text_2", + "type": "STRING", + "linkIds": [ + 514 + ], + "label": "sam3_image_object", + "pos": [ + -150.984375, + 4684 + ] + }, + { + "id": "8f542c5e-da91-4dad-8ff8-4e81a42d5140", + "name": "ckpt_name", + "type": "COMBO", + "linkIds": [ + 515 + ], + "label": "sam3_model", + "pos": [ + -150.984375, + 4704 + ] + }, + { + "id": "9764a497-f066-429c-b77c-bcd5b8c7a68d", + "name": "noise_seed", + "type": "INT", + "linkIds": [ + 559 + ], + "pos": [ + -150.984375, + 4724 + ] + } + ], + "outputs": [ + { + "id": "7ae88834-7553-45ec-a4e8-bab7d5276b45", + "name": "output", + "type": "IMAGE", + "linkIds": [ + 546 + ], + "localized_name": "output", + "pos": [ + 3594, + 4524 + ] + } + ], + "widgets": [], + "nodes": [ + { + "id": 322, + "type": "LoraLoaderModelOnly", + "pos": [ + 300, + 3590 + ], + "size": [ + 590, + 140 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 566 + }, + { + "localized_name": "lora_name", + "name": "lora_name", + "type": "COMBO", + "widget": { + "name": "lora_name" + }, + "link": 509 + }, + { + "localized_name": "strength_model", + "name": "strength_model", + "type": "FLOAT", + "widget": { + "name": "strength_model" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MODEL", + "name": "MODEL", + "type": "MODEL", + "links": [ + 398, + 425 + ] + } + ], + "properties": { + "Node name for S&R": "LoraLoaderModelOnly", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "models": [ + { + "name": "lightx2v_I2V_14B_480p_cfg_step_distill_rank64_bf16.safetensors", + "url": "https://huggingface.co/Kijai/WanVideo_comfy/resolve/main/Lightx2v/lightx2v_I2V_14B_480p_cfg_step_distill_rank64_bf16.safetensors", + "directory": "loras" + } + ] + }, + "widgets_values": [ + "lightx2v_I2V_14B_480p_cfg_step_distill_rank64_bf16.safetensors", + 0.8 + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 323, + "type": "VAELoader", + "pos": [ + 300, + 4020 + ], + "size": [ + 590, + 140 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "localized_name": "vae_name", + "name": "vae_name", + "type": "COMBO", + "widget": { + "name": "vae_name" + }, + "link": 511 + } + ], + "outputs": [ + { + "localized_name": "VAE", + "name": "VAE", + "type": "VAE", + "links": [ + 406, + 407 + ] + } + ], + "properties": { + "Node name for S&R": "VAELoader", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "models": [ + { + "name": "Wan2_1_VAE_bf16.safetensors", + "url": "https://huggingface.co/Kijai/WanVideo_comfy/resolve/main/Wan2_1_VAE_bf16.safetensors", + "directory": "vae" + } + ] + }, + "widgets_values": [ + "Wan2_1_VAE_bf16.safetensors" + ] + }, + { + "id": 324, + "type": "ResizeImageMaskNode", + "pos": [ + 1280, + 4880 + ], + "size": [ + 270, + 160 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "localized_name": "input", + "name": "input", + "type": "IMAGE,MASK", + "link": 491 + }, + { + "localized_name": "resize_type", + "name": "resize_type", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "resize_type" + }, + "link": null + }, + { + "localized_name": "width", + "name": "resize_type.width", + "type": "INT", + "widget": { + "name": "resize_type.width" + }, + "link": 444 + }, + { + "localized_name": "height", + "name": "resize_type.height", + "type": "INT", + "widget": { + "name": "resize_type.height" + }, + "link": 446 + }, + { + "localized_name": "crop", + "name": "resize_type.crop", + "type": "COMBO", + "widget": { + "name": "resize_type.crop" + }, + "link": null + }, + { + "localized_name": "scale_method", + "name": "scale_method", + "type": "COMBO", + "widget": { + "name": "scale_method" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "resized", + "name": "resized", + "type": "*", + "links": [ + 420, + 492, + 493 + ] + } + ], + "properties": { + "Node name for S&R": "ResizeImageMaskNode", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + "scale dimensions", + 512, + 512, + "center", + "area" + ] + }, + { + "id": 325, + "type": "GetVideoComponents", + "pos": [ + 270, + 4520 + ], + "size": [ + 230, + 90 + ], + "flags": { + "collapsed": true + }, + "order": 8, + "mode": 0, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 409 + } + ], + "outputs": [ + { + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "links": [ + 490 + ] + }, + { + "localized_name": "audio", + "name": "audio", + "type": "AUDIO", + "links": null + }, + { + "localized_name": "fps", + "name": "fps", + "type": "FLOAT", + "links": [] + }, + { + "localized_name": "bit_depth", + "name": "bit_depth", + "type": "INT", + "links": null + } + ], + "properties": { + "Node name for S&R": "GetVideoComponents", + "cnr_id": "comfy-core", + "ver": "0.24.0" + } + }, + { + "id": 326, + "type": "GetImageSize", + "pos": [ + 1640, + 4520 + ], + "size": [ + 240, + 190 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "localized_name": "image", + "name": "image", + "type": "IMAGE", + "link": 492 + } + ], + "outputs": [ + { + "localized_name": "width", + "name": "width", + "type": "INT", + "links": [ + 414 + ] + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "links": [ + 415 + ] + }, + { + "localized_name": "batch_size", + "name": "batch_size", + "type": "INT", + "links": [ + 416 + ] + } + ], + "properties": { + "Node name for S&R": "GetImageSize", + "cnr_id": "comfy-core", + "ver": "0.24.0" + } + }, + { + "id": 327, + "type": "CLIPLoader", + "pos": [ + 300, + 3790 + ], + "size": [ + 590, + 170 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "localized_name": "clip_name", + "name": "clip_name", + "type": "COMBO", + "widget": { + "name": "clip_name" + }, + "link": 510 + }, + { + "localized_name": "type", + "name": "type", + "type": "COMBO", + "widget": { + "name": "type" + }, + "link": null + }, + { + "localized_name": "device", + "name": "device", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "device" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "CLIP", + "name": "CLIP", + "type": "CLIP", + "links": [ + 7, + 8 + ] + } + ], + "properties": { + "Node name for S&R": "CLIPLoader", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "models": [ + { + "name": "umt5_xxl_fp8_e4m3fn_scaled.safetensors", + "url": "https://huggingface.co/Comfy-Org/Wan_2.1_ComfyUI_repackaged/resolve/main/split_files/text_encoders/umt5_xxl_fp8_e4m3fn_scaled.safetensors", + "directory": "text_encoders" + } + ] + }, + "widgets_values": [ + "umt5_xxl_fp8_e4m3fn_scaled.safetensors", + "wan", + "default" + ] + }, + { + "id": 328, + "type": "CLIPVisionLoader", + "pos": [ + 300, + 4230 + ], + "size": [ + 590, + 110 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "localized_name": "clip_name", + "name": "clip_name", + "type": "COMBO", + "widget": { + "name": "clip_name" + }, + "link": 512 + } + ], + "outputs": [ + { + "localized_name": "CLIP_VISION", + "name": "CLIP_VISION", + "type": "CLIP_VISION", + "links": [ + 196 + ] + } + ], + "properties": { + "Node name for S&R": "CLIPVisionLoader", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "models": [ + { + "name": "clip_vision_h.safetensors", + "url": "https://huggingface.co/Comfy-Org/Wan_2.1_ComfyUI_repackaged/resolve/main/split_files/clip_vision/clip_vision_h.safetensors", + "directory": "clip_vision" + } + ] + }, + "widgets_values": [ + "clip_vision_h.safetensors" + ] + }, + { + "id": 329, + "type": "UNETLoader", + "pos": [ + 300, + 3190 + ], + "size": [ + 590, + 140 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "localized_name": "unet_name", + "name": "unet_name", + "type": "COMBO", + "widget": { + "name": "unet_name" + }, + "link": 508 + }, + { + "localized_name": "weight_dtype", + "name": "weight_dtype", + "type": "COMBO", + "widget": { + "name": "weight_dtype" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MODEL", + "name": "MODEL", + "type": "MODEL", + "links": [ + 568 + ] + } + ], + "properties": { + "Node name for S&R": "UNETLoader", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "models": [ + { + "name": "wan2.1_14B_SCAIL_2_fp16.safetensors", + "url": "https://huggingface.co/Comfy-Org/SCAIL-2/resolve/main/diffusion_models/wan2.1_14B_SCAIL_2_fp16.safetensors", + "directory": "diffusion_models" + } + ] + }, + "widgets_values": [ + "wan2.1_14B_SCAIL_2_fp16.safetensors", + "default" + ] + }, + { + "id": 330, + "type": "ModelSamplingSD3", + "pos": [ + 2820, + 3430 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 424 + }, + { + "localized_name": "shift", + "name": "shift", + "type": "FLOAT", + "widget": { + "name": "shift" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MODEL", + "name": "MODEL", + "type": "MODEL", + "links": [ + 417 + ] + } + ], + "properties": { + "Node name for S&R": "ModelSamplingSD3", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 5 + ] + }, + { + "id": 331, + "type": "KSamplerSelect", + "pos": [ + 2830, + 3580 + ], + "size": [ + 260, + 110 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [ + { + "localized_name": "sampler_name", + "name": "sampler_name", + "type": "COMBO", + "widget": { + "name": "sampler_name" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "SAMPLER", + "name": "SAMPLER", + "type": "SAMPLER", + "links": [ + 419 + ] + } + ], + "properties": { + "Node name for S&R": "KSamplerSelect", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + "euler" + ] + }, + { + "id": 332, + "type": "BasicScheduler", + "pos": [ + 2830, + 3730 + ], + "size": [ + 260, + 170 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 398 + }, + { + "localized_name": "scheduler", + "name": "scheduler", + "type": "COMBO", + "widget": { + "name": "scheduler" + }, + "link": null + }, + { + "localized_name": "steps", + "name": "steps", + "type": "INT", + "widget": { + "name": "steps" + }, + "link": 428 + }, + { + "localized_name": "denoise", + "name": "denoise", + "type": "FLOAT", + "widget": { + "name": "denoise" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "SIGMAS", + "name": "SIGMAS", + "type": "SIGMAS", + "links": [ + 418 + ] + } + ], + "properties": { + "Node name for S&R": "BasicScheduler", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + "simple", + 6, + 1 + ] + }, + { + "id": 333, + "type": "SamplerCustom", + "pos": [ + 3140, + 3430 + ], + "size": [ + 270, + 670 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 417 + }, + { + "localized_name": "positive", + "name": "positive", + "type": "CONDITIONING", + "link": 333 + }, + { + "localized_name": "negative", + "name": "negative", + "type": "CONDITIONING", + "link": 334 + }, + { + "localized_name": "sampler", + "name": "sampler", + "type": "SAMPLER", + "link": 419 + }, + { + "localized_name": "sigmas", + "name": "sigmas", + "type": "SIGMAS", + "link": 418 + }, + { + "localized_name": "latent_image", + "name": "latent_image", + "type": "LATENT", + "link": 335 + }, + { + "localized_name": "add_noise", + "name": "add_noise", + "type": "BOOLEAN", + "widget": { + "name": "add_noise" + }, + "link": null + }, + { + "localized_name": "noise_seed", + "name": "noise_seed", + "type": "INT", + "widget": { + "name": "noise_seed" + }, + "link": 559 + }, + { + "localized_name": "cfg", + "name": "cfg", + "type": "FLOAT", + "widget": { + "name": "cfg" + }, + "link": 431 + } + ], + "outputs": [ + { + "localized_name": "output", + "name": "output", + "type": "LATENT", + "links": [] + }, + { + "localized_name": "denoised_output", + "name": "denoised_output", + "type": "LATENT", + "links": [ + 124 + ] + } + ], + "properties": { + "Node name for S&R": "SamplerCustom", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + true, + 1, + "fixed", + 1 + ] + }, + { + "id": 334, + "type": "PrimitiveInt", + "pos": [ + 2090, + 3760 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "INT", + "widget": { + "name": "value" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 427 + ] + } + ], + "title": "Int (Steps)", + "properties": { + "Node name for S&R": "PrimitiveInt", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 6, + "fixed" + ] + }, + { + "id": 335, + "type": "PrimitiveFloat", + "pos": [ + 2090, + 3930 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "FLOAT", + "widget": { + "name": "value" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "FLOAT", + "name": "FLOAT", + "type": "FLOAT", + "links": [ + 430 + ] + } + ], + "title": "Float (CFG)", + "properties": { + "Node name for S&R": "PrimitiveFloat", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 1 + ] + }, + { + "id": 336, + "type": "PrimitiveInt", + "pos": [ + 2090, + 3390 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "INT", + "widget": { + "name": "value" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 426 + ] + } + ], + "title": "Int (Steps)", + "properties": { + "Node name for S&R": "PrimitiveInt", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 40, + "fixed" + ] + }, + { + "id": 337, + "type": "PrimitiveFloat", + "pos": [ + 2090, + 3540 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "FLOAT", + "widget": { + "name": "value" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "FLOAT", + "name": "FLOAT", + "type": "FLOAT", + "links": [ + 429 + ] + } + ], + "title": "Float (CFG)", + "properties": { + "Node name for S&R": "PrimitiveFloat", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 5 + ] + }, + { + "id": 338, + "type": "ComfySwitchNode", + "pos": [ + 2430, + 3370 + ], + "size": [ + 270, + 130 + ], + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [ + { + "localized_name": "on_false", + "name": "on_false", + "type": "*", + "link": 567 + }, + { + "localized_name": "on_true", + "name": "on_true", + "type": "*", + "link": 425 + }, + { + "localized_name": "switch", + "name": "switch", + "type": "BOOLEAN", + "widget": { + "name": "switch" + }, + "link": 432 + } + ], + "outputs": [ + { + "localized_name": "output", + "name": "output", + "type": "*", + "links": [ + 424 + ] + } + ], + "title": "Switch (Model)", + "properties": { + "Node name for S&R": "ComfySwitchNode", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + false + ] + }, + { + "id": 339, + "type": "ComfySwitchNode", + "pos": [ + 2430, + 3560 + ], + "size": [ + 270, + 130 + ], + "flags": {}, + "order": 17, + "mode": 0, + "inputs": [ + { + "localized_name": "on_false", + "name": "on_false", + "type": "*", + "link": 426 + }, + { + "localized_name": "on_true", + "name": "on_true", + "type": "*", + "link": 427 + }, + { + "localized_name": "switch", + "name": "switch", + "type": "BOOLEAN", + "widget": { + "name": "switch" + }, + "link": 433 + } + ], + "outputs": [ + { + "localized_name": "output", + "name": "output", + "type": "*", + "links": [ + 428 + ] + } + ], + "title": "Switch (Steps)", + "properties": { + "Node name for S&R": "ComfySwitchNode", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + false + ] + }, + { + "id": 340, + "type": "ComfySwitchNode", + "pos": [ + 2430, + 3740 + ], + "size": [ + 270, + 130 + ], + "flags": {}, + "order": 18, + "mode": 0, + "inputs": [ + { + "localized_name": "on_false", + "name": "on_false", + "type": "*", + "link": 429 + }, + { + "localized_name": "on_true", + "name": "on_true", + "type": "*", + "link": 430 + }, + { + "localized_name": "switch", + "name": "switch", + "type": "BOOLEAN", + "widget": { + "name": "switch" + }, + "link": 434 + } + ], + "outputs": [ + { + "localized_name": "output", + "name": "output", + "type": "*", + "links": [ + 431 + ] + } + ], + "title": "Switch (Steps)", + "properties": { + "Node name for S&R": "ComfySwitchNode", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + false + ] + }, + { + "id": 341, + "type": "ImageFromBatch", + "pos": [ + 880, + 4510 + ], + "size": [ + 270, + 140 + ], + "flags": {}, + "order": 19, + "mode": 0, + "inputs": [ + { + "localized_name": "image", + "name": "image", + "type": "IMAGE", + "link": 490 + }, + { + "localized_name": "batch_index", + "name": "batch_index", + "type": "INT", + "widget": { + "name": "batch_index" + }, + "link": 450 + }, + { + "localized_name": "length", + "name": "length", + "type": "INT", + "widget": { + "name": "length" + }, + "link": 550 + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 491 + ] + } + ], + "properties": { + "Node name for S&R": "ImageFromBatch", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 0, + 81 + ] + }, + { + "id": 342, + "type": "PrimitiveInt", + "pos": [ + 250, + 4840 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 20, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "INT", + "widget": { + "name": "value" + }, + "link": 505 + } + ], + "outputs": [ + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 441 + ] + } + ], + "title": "Int (Width)", + "properties": { + "Node name for S&R": "PrimitiveInt", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 896, + "fixed" + ] + }, + { + "id": 343, + "type": "PrimitiveInt", + "pos": [ + 250, + 5020 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 21, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "INT", + "widget": { + "name": "value" + }, + "link": 506 + } + ], + "outputs": [ + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 445 + ] + } + ], + "title": "Int (Height)", + "properties": { + "Node name for S&R": "PrimitiveInt", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 512, + "fixed" + ] + }, + { + "id": 344, + "type": "ComfyMathExpression", + "pos": [ + 690, + 4870 + ], + "size": [ + 230, + 80 + ], + "flags": { + "collapsed": true + }, + "order": 22, + "mode": 0, + "inputs": [ + { + "label": "a", + "localized_name": "values.a", + "name": "values.a", + "type": "FLOAT,INT,BOOLEAN", + "link": 441 + }, + { + "label": "b", + "localized_name": "values.b", + "name": "values.b", + "shape": 7, + "type": "FLOAT,INT,BOOLEAN", + "link": null + }, + { + "localized_name": "expression", + "name": "expression", + "type": "STRING", + "widget": { + "name": "expression" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "FLOAT", + "name": "FLOAT", + "type": "FLOAT", + "links": [] + }, + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 444 + ] + }, + { + "localized_name": "BOOL", + "name": "BOOL", + "type": "BOOLEAN", + "links": null + } + ], + "properties": { + "Node name for S&R": "ComfyMathExpression", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + "(a // 32) * 32" + ] + }, + { + "id": 345, + "type": "ComfyMathExpression", + "pos": [ + 690, + 5050 + ], + "size": [ + 230, + 80 + ], + "flags": { + "collapsed": true + }, + "order": 23, + "mode": 0, + "inputs": [ + { + "label": "a", + "localized_name": "values.a", + "name": "values.a", + "type": "FLOAT,INT,BOOLEAN", + "link": 445 + }, + { + "label": "b", + "localized_name": "values.b", + "name": "values.b", + "shape": 7, + "type": "FLOAT,INT,BOOLEAN", + "link": null + }, + { + "localized_name": "expression", + "name": "expression", + "type": "STRING", + "widget": { + "name": "expression" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "FLOAT", + "name": "FLOAT", + "type": "FLOAT", + "links": [] + }, + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 446 + ] + }, + { + "localized_name": "BOOL", + "name": "BOOL", + "type": "BOOLEAN", + "links": null + } + ], + "properties": { + "Node name for S&R": "ComfyMathExpression", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + "(a // 32) * 32" + ] + }, + { + "id": 346, + "type": "PrimitiveInt", + "pos": [ + 270, + 4580 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 24, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "INT", + "widget": { + "name": "value" + }, + "link": 502 + } + ], + "outputs": [ + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 447 + ] + } + ], + "title": "Int (segment index)", + "properties": { + "Node name for S&R": "PrimitiveInt", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 2, + "fixed" + ] + }, + { + "id": 347, + "type": "ComfyMathExpression", + "pos": [ + 580, + 4610 + ], + "size": [ + 230, + 120 + ], + "flags": { + "collapsed": true + }, + "order": 25, + "mode": 0, + "inputs": [ + { + "label": "a", + "localized_name": "values.a", + "name": "values.a", + "type": "FLOAT,INT,BOOLEAN", + "link": 447 + }, + { + "label": "b", + "localized_name": "values.b", + "name": "values.b", + "shape": 7, + "type": "FLOAT,INT,BOOLEAN", + "link": 557 + }, + { + "label": "c", + "localized_name": "values.c", + "name": "values.c", + "shape": 7, + "type": "FLOAT,INT,BOOLEAN", + "link": 556 + }, + { + "label": "d", + "localized_name": "values.d", + "name": "values.d", + "shape": 7, + "type": "FLOAT,INT,BOOLEAN", + "link": null + }, + { + "localized_name": "expression", + "name": "expression", + "type": "STRING", + "widget": { + "name": "expression" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "FLOAT", + "name": "FLOAT", + "type": "FLOAT", + "links": null + }, + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 450 + ] + }, + { + "localized_name": "BOOL", + "name": "BOOL", + "type": "BOOLEAN", + "links": null + } + ], + "properties": { + "Node name for S&R": "ComfyMathExpression", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + "(b - c) * (a - 1)" + ] + }, + { + "id": 348, + "type": "CLIPTextEncode", + "pos": [ + 660, + 5380 + ], + "size": [ + 380, + 160 + ], + "flags": {}, + "order": 26, + "mode": 0, + "inputs": [ + { + "localized_name": "clip", + "name": "clip", + "type": "CLIP", + "link": 454 + }, + { + "localized_name": "text", + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": 513 + } + ], + "outputs": [ + { + "localized_name": "CONDITIONING", + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 461 + ] + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode", + "cnr_id": "comfy-core", + "ver": "0.19.3", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "" + ] + }, + { + "id": 349, + "type": "CheckpointLoaderSimple", + "pos": [ + 270, + 5360 + ], + "size": [ + 330, + 160 + ], + "flags": {}, + "order": 27, + "mode": 0, + "inputs": [ + { + "localized_name": "ckpt_name", + "name": "ckpt_name", + "type": "COMBO", + "widget": { + "name": "ckpt_name" + }, + "link": 515 + } + ], + "outputs": [ + { + "localized_name": "MODEL", + "name": "MODEL", + "type": "MODEL", + "links": [ + 458, + 463 + ] + }, + { + "localized_name": "CLIP", + "name": "CLIP", + "type": "CLIP", + "links": [ + 454, + 489 + ] + }, + { + "localized_name": "VAE", + "name": "VAE", + "type": "VAE", + "links": [] + } + ], + "properties": { + "Node name for S&R": "CheckpointLoaderSimple", + "cnr_id": "comfy-core", + "ver": "0.19.3", + "models": [ + { + "name": "sam3.1_multiplex_fp16.safetensors", + "url": "https://huggingface.co/Comfy-Org/sam3.1/resolve/main/checkpoints/sam3.1_multiplex_fp16.safetensors", + "directory": "checkpoints" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "sam3.1_multiplex_fp16.safetensors" + ] + }, + { + "id": 350, + "type": "SAM3_VideoTrack", + "pos": [ + 1190, + 5340 + ], + "size": [ + 280, + 250 + ], + "flags": {}, + "order": 28, + "mode": 0, + "inputs": [ + { + "label": "images", + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "link": 493 + }, + { + "label": "model", + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 458 + }, + { + "label": "initial_mask", + "localized_name": "initial_mask", + "name": "initial_mask", + "shape": 7, + "type": "MASK", + "link": null + }, + { + "label": "conditioning", + "localized_name": "conditioning", + "name": "conditioning", + "shape": 7, + "type": "CONDITIONING", + "link": 461 + }, + { + "localized_name": "detection_threshold", + "name": "detection_threshold", + "type": "FLOAT", + "widget": { + "name": "detection_threshold" + }, + "link": null + }, + { + "localized_name": "max_objects", + "name": "max_objects", + "type": "INT", + "widget": { + "name": "max_objects" + }, + "link": null + }, + { + "localized_name": "detect_interval", + "name": "detect_interval", + "type": "INT", + "widget": { + "name": "detect_interval" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "track_data", + "name": "track_data", + "type": "SAM3_TRACK_DATA", + "links": [ + 460 + ] + } + ], + "properties": { + "Node name for S&R": "SAM3_VideoTrack", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 0.5, + 4, + 1 + ] + }, + { + "id": 351, + "type": "SCAIL2ColoredMask", + "pos": [ + 1550, + 5490 + ], + "size": [ + 370, + 200 + ], + "flags": {}, + "order": 29, + "mode": 0, + "inputs": [ + { + "localized_name": "driving_track_data", + "name": "driving_track_data", + "type": "SAM3_TRACK_DATA", + "link": 460 + }, + { + "localized_name": "ref_track_data", + "name": "ref_track_data", + "shape": 7, + "type": "SAM3_TRACK_DATA", + "link": 464 + }, + { + "localized_name": "object_indices", + "name": "object_indices", + "type": "STRING", + "widget": { + "name": "object_indices" + }, + "link": null + }, + { + "localized_name": "sort_by", + "name": "sort_by", + "type": "COMBO", + "widget": { + "name": "sort_by" + }, + "link": null + }, + { + "localized_name": "replacement_mode", + "name": "replacement_mode", + "type": "BOOLEAN", + "widget": { + "name": "replacement_mode" + }, + "link": 476 + } + ], + "outputs": [ + { + "localized_name": "pose_video_mask", + "name": "pose_video_mask", + "type": "IMAGE", + "links": [ + 466, + 467 + ] + }, + { + "localized_name": "reference_image_mask", + "name": "reference_image_mask", + "type": "IMAGE", + "links": [ + 465, + 472 + ] + } + ], + "properties": { + "Node name for S&R": "SCAIL2ColoredMask", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + "", + "left_to_right", + true + ] + }, + { + "id": 352, + "type": "SAM3_VideoTrack", + "pos": [ + 1190, + 5640 + ], + "size": [ + 280, + 250 + ], + "flags": {}, + "order": 30, + "mode": 0, + "inputs": [ + { + "label": "images", + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "link": 469 + }, + { + "label": "model", + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 463 + }, + { + "label": "initial_mask", + "localized_name": "initial_mask", + "name": "initial_mask", + "shape": 7, + "type": "MASK", + "link": null + }, + { + "label": "conditioning", + "localized_name": "conditioning", + "name": "conditioning", + "shape": 7, + "type": "CONDITIONING", + "link": 488 + }, + { + "localized_name": "detection_threshold", + "name": "detection_threshold", + "type": "FLOAT", + "widget": { + "name": "detection_threshold" + }, + "link": null + }, + { + "localized_name": "max_objects", + "name": "max_objects", + "type": "INT", + "widget": { + "name": "max_objects" + }, + "link": null + }, + { + "localized_name": "detect_interval", + "name": "detect_interval", + "type": "INT", + "widget": { + "name": "detect_interval" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "track_data", + "name": "track_data", + "type": "SAM3_TRACK_DATA", + "links": [ + 464 + ] + } + ], + "properties": { + "Node name for S&R": "SAM3_VideoTrack", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 0.5, + 4, + 1 + ] + }, + { + "id": 353, + "type": "PrimitiveBoolean", + "pos": [ + 1660, + 4030 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 31, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "BOOLEAN", + "widget": { + "name": "value" + }, + "link": 504 + } + ], + "outputs": [ + { + "localized_name": "BOOLEAN", + "name": "BOOLEAN", + "type": "BOOLEAN", + "links": [ + 475, + 476 + ] + } + ], + "title": "Boolean （Replace Mode）", + "properties": { + "Node name for S&R": "PrimitiveBoolean", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + true + ] + }, + { + "id": 354, + "type": "PreviewImage", + "pos": [ + 2060, + 4500 + ], + "size": [ + 350, + 1190 + ], + "flags": {}, + "order": 32, + "mode": 4, + "inputs": [ + { + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "link": 466 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage", + "cnr_id": "comfy-core", + "ver": "0.24.0" + } + }, + { + "id": 355, + "type": "PreviewImage", + "pos": [ + 2460, + 4500 + ], + "size": [ + 230, + 310 + ], + "flags": {}, + "order": 33, + "mode": 4, + "inputs": [ + { + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "link": 465 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage", + "cnr_id": "comfy-core", + "ver": "0.24.0" + } + }, + { + "id": 356, + "type": "VAEDecode", + "pos": [ + 2920, + 4510 + ], + "size": [ + 270, + 100 + ], + "flags": { + "collapsed": false + }, + "order": 34, + "mode": 0, + "inputs": [ + { + "localized_name": "samples", + "name": "samples", + "type": "LATENT", + "link": 124 + }, + { + "localized_name": "vae", + "name": "vae", + "type": "VAE", + "link": 407 + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 483 + ] + } + ], + "properties": { + "Node name for S&R": "VAEDecode", + "cnr_id": "comfy-core", + "ver": "0.24.0" + } + }, + { + "id": 357, + "type": "ImageFromBatch", + "pos": [ + 2920, + 4680 + ], + "size": [ + 270, + 140 + ], + "flags": {}, + "order": 35, + "mode": 0, + "inputs": [ + { + "localized_name": "image", + "name": "image", + "type": "IMAGE", + "link": 483 + }, + { + "localized_name": "batch_index", + "name": "batch_index", + "type": "INT", + "widget": { + "name": "batch_index" + }, + "link": null + }, + { + "localized_name": "length", + "name": "length", + "type": "INT", + "widget": { + "name": "length" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 484 + ] + } + ], + "properties": { + "Node name for S&R": "ImageFromBatch", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 5, + 4096 + ] + }, + { + "id": 358, + "type": "ColorTransfer", + "pos": [ + 2920, + 5050 + ], + "size": [ + 270, + 130 + ], + "flags": {}, + "order": 36, + "mode": 0, + "inputs": [ + { + "localized_name": "image_target", + "name": "image_target", + "type": "IMAGE", + "link": 484 + }, + { + "localized_name": "image_ref", + "name": "image_ref", + "type": "IMAGE", + "link": 482 + }, + { + "localized_name": "method", + "name": "method", + "type": "COMBO", + "widget": { + "name": "method" + }, + "link": null + }, + { + "localized_name": "source_stats", + "name": "source_stats", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "source_stats" + }, + "link": null + }, + { + "localized_name": "strength", + "name": "strength", + "type": "FLOAT", + "widget": { + "name": "strength" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "image", + "name": "image", + "type": "IMAGE", + "links": [ + 546 + ] + } + ], + "properties": { + "Node name for S&R": "ColorTransfer", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + "reinhard_lab", + "per_frame", + 1 + ] + }, + { + "id": 359, + "type": "ImageFromBatch", + "pos": [ + 2920, + 4870 + ], + "size": [ + 270, + 140 + ], + "flags": {}, + "order": 37, + "mode": 0, + "inputs": [ + { + "localized_name": "image", + "name": "image", + "type": "IMAGE", + "link": 486 + }, + { + "localized_name": "batch_index", + "name": "batch_index", + "type": "INT", + "widget": { + "name": "batch_index" + }, + "link": null + }, + { + "localized_name": "length", + "name": "length", + "type": "INT", + "widget": { + "name": "length" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 482 + ] + } + ], + "properties": { + "Node name for S&R": "ImageFromBatch", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + -1, + 1 + ] + }, + { + "id": 360, + "type": "PrimitiveBoolean", + "pos": [ + 2440, + 3950 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 38, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "BOOLEAN", + "widget": { + "name": "value" + }, + "link": 507 + } + ], + "outputs": [ + { + "localized_name": "BOOLEAN", + "name": "BOOLEAN", + "type": "BOOLEAN", + "links": [ + 432, + 433, + 434 + ] + } + ], + "title": "Boolean (Enable Distill LoRA?)", + "properties": { + "Node name for S&R": "PrimitiveBoolean", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + true + ] + }, + { + "id": 361, + "type": "CLIPVisionEncode", + "pos": [ + 1230, + 4310 + ], + "size": [ + 230, + 60 + ], + "flags": { + "collapsed": true + }, + "order": 39, + "mode": 0, + "inputs": [ + { + "localized_name": "clip_vision", + "name": "clip_vision", + "type": "CLIP_VISION", + "link": 196 + }, + { + "localized_name": "image", + "name": "image", + "type": "IMAGE", + "link": 408 + }, + { + "localized_name": "crop", + "name": "crop", + "type": "COMBO", + "widget": { + "name": "crop" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "CLIP_VISION_OUTPUT", + "name": "CLIP_VISION_OUTPUT", + "type": "CLIP_VISION_OUTPUT", + "links": [ + 404 + ] + } + ], + "properties": { + "Node name for S&R": "CLIPVisionEncode", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + "none" + ] + }, + { + "id": 362, + "type": "CLIPTextEncode", + "pos": [ + 1030, + 4020 + ], + "size": [ + 520, + 210 + ], + "flags": {}, + "order": 40, + "mode": 0, + "inputs": [ + { + "localized_name": "clip", + "name": "clip", + "type": "CLIP", + "link": 8 + }, + { + "localized_name": "text", + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "CONDITIONING", + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 326 + ] + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + "" + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 363, + "type": "CLIPTextEncode", + "pos": [ + 1020, + 3180 + ], + "size": [ + 520, + 720 + ], + "flags": {}, + "order": 41, + "mode": 0, + "inputs": [ + { + "localized_name": "clip", + "name": "clip", + "type": "CLIP", + "link": 7 + }, + { + "localized_name": "text", + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": 499 + } + ], + "outputs": [ + { + "localized_name": "CONDITIONING", + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 325 + ] + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + "A young woman with dark hair tied in a neat high bun, with a few loose strands framing her face, is dancing outdoors on a sunny coastal hillside. She has a normal-sized head and a slim face, with no hat, no headwear, and no oversized hair volume. She wears a fitted black long-sleeve crop top with a shoulder cutout, extremely baggy black cargo pants with straps and pockets, and chunky black combat boots. She performs energetic dance moves with one leg lifted and arms extended, moving naturally in front of a large tree, a small white stone house with a terracotta roof, and a bright blue sea under a clear sky with light clouds." + ], + "color": "#232", + "bgcolor": "#353" + }, + { + "id": 364, + "type": "WanSCAILToVideo", + "pos": [ + 1650, + 3310 + ], + "size": [ + 310, + 580 + ], + "flags": {}, + "order": 42, + "mode": 0, + "inputs": [ + { + "localized_name": "positive", + "name": "positive", + "type": "CONDITIONING", + "link": 325 + }, + { + "localized_name": "negative", + "name": "negative", + "type": "CONDITIONING", + "link": 326 + }, + { + "localized_name": "vae", + "name": "vae", + "type": "VAE", + "link": 406 + }, + { + "localized_name": "pose_video", + "name": "pose_video", + "shape": 7, + "type": "IMAGE", + "link": 420 + }, + { + "localized_name": "pose_video_mask", + "name": "pose_video_mask", + "shape": 7, + "type": "IMAGE", + "link": 467 + }, + { + "localized_name": "reference_image", + "name": "reference_image", + "shape": 7, + "type": "IMAGE", + "link": 473 + }, + { + "localized_name": "reference_image_mask", + "name": "reference_image_mask", + "shape": 7, + "type": "IMAGE", + "link": 472 + }, + { + "localized_name": "clip_vision_output", + "name": "clip_vision_output", + "shape": 7, + "type": "CLIP_VISION_OUTPUT", + "link": 404 + }, + { + "localized_name": "previous_frames", + "name": "previous_frames", + "shape": 7, + "type": "IMAGE", + "link": 485 + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": 414 + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": 415 + }, + { + "localized_name": "length", + "name": "length", + "type": "INT", + "widget": { + "name": "length" + }, + "link": 416 + }, + { + "localized_name": "batch_size", + "name": "batch_size", + "type": "INT", + "widget": { + "name": "batch_size" + }, + "link": null + }, + { + "localized_name": "pose_strength", + "name": "pose_strength", + "type": "FLOAT", + "widget": { + "name": "pose_strength" + }, + "link": 552 + }, + { + "localized_name": "pose_start", + "name": "pose_start", + "type": "FLOAT", + "widget": { + "name": "pose_start" + }, + "link": 553 + }, + { + "localized_name": "pose_end", + "name": "pose_end", + "type": "FLOAT", + "widget": { + "name": "pose_end" + }, + "link": 554 + }, + { + "localized_name": "video_frame_offset", + "name": "video_frame_offset", + "type": "INT", + "widget": { + "name": "video_frame_offset" + }, + "link": null + }, + { + "localized_name": "previous_frame_count", + "name": "previous_frame_count", + "type": "INT", + "widget": { + "name": "previous_frame_count" + }, + "link": 551 + }, + { + "localized_name": "replacement_mode", + "name": "replacement_mode", + "shape": 7, + "type": "BOOLEAN", + "widget": { + "name": "replacement_mode" + }, + "link": 475 + } + ], + "outputs": [ + { + "localized_name": "positive", + "name": "positive", + "type": "CONDITIONING", + "links": [ + 333 + ] + }, + { + "localized_name": "negative", + "name": "negative", + "type": "CONDITIONING", + "links": [ + 334 + ] + }, + { + "localized_name": "latent", + "name": "latent", + "type": "LATENT", + "links": [ + 335 + ] + }, + { + "localized_name": "video_frame_offset", + "name": "video_frame_offset", + "type": "INT", + "links": [] + } + ], + "properties": { + "Node name for S&R": "WanSCAILToVideo", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 512, + 896, + 65, + 1, + 1, + 0, + 1, + 0, + 5, + true + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 365, + "type": "CLIPTextEncode", + "pos": [ + 670, + 5710 + ], + "size": [ + 380, + 160 + ], + "flags": {}, + "order": 43, + "mode": 0, + "inputs": [ + { + "localized_name": "clip", + "name": "clip", + "type": "CLIP", + "link": 489 + }, + { + "localized_name": "text", + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": 514 + } + ], + "outputs": [ + { + "localized_name": "CONDITIONING", + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 488 + ] + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode", + "cnr_id": "comfy-core", + "ver": "0.19.3", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "" + ] + }, + { + "id": 366, + "type": "LoraLoaderModelOnly", + "pos": [ + 300, + 3390 + ], + "size": [ + 580, + 140 + ], + "flags": {}, + "order": 44, + "mode": 0, + "inputs": [ + { + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 568 + }, + { + "localized_name": "lora_name", + "name": "lora_name", + "type": "COMBO", + "widget": { + "name": "lora_name" + }, + "link": 569 + }, + { + "localized_name": "strength_model", + "name": "strength_model", + "type": "FLOAT", + "widget": { + "name": "strength_model" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MODEL", + "name": "MODEL", + "type": "MODEL", + "links": [ + 566, + 567 + ] + } + ], + "properties": { + "Node name for S&R": "LoraLoaderModelOnly", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "models": [ + { + "name": "wan2.1_SCAIL_2_DPO_lora_bf16.safetensors", + "url": "https://huggingface.co/Comfy-Org/SCAIL-2/resolve/main/loras/wan2.1_SCAIL_2_DPO_lora_bf16.safetensors", + "directory": "loras" + } + ] + }, + "widgets_values": [ + "wan2.1_SCAIL_2_DPO_lora_bf16.safetensors", + 1 + ] + } + ], + "groups": [ + { + "id": 3, + "title": "Models", + "bounding": [ + 240, + 3100, + 720, + 1300 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 4, + "title": "Prompt", + "bounding": [ + 990, + 3100, + 580, + 1300 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 5, + "title": "Sampling", + "bounding": [ + 2770, + 3100, + 700, + 1300 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 6, + "title": "SCAIL Conditioning", + "bounding": [ + 1590, + 3100, + 430, + 1300 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 9, + "title": "Switch Settings", + "bounding": [ + 2050, + 3100, + 690, + 1300 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 7, + "title": "Original Settings", + "bounding": [ + 2080, + 3330, + 300, + 340 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 8, + "title": "Distill LoRA Settings", + "bounding": [ + 2080, + 3690, + 300, + 370 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 10, + "title": "Create Mask", + "bounding": [ + 240, + 5250, + 1780, + 670 + ], + "flags": {} + }, + { + "id": 11, + "title": "Resize Images", + "bounding": [ + 240, + 4770, + 1780, + 450 + ], + "flags": {} + }, + { + "id": 12, + "title": "Trim Frames from Video", + "bounding": [ + 240, + 4430, + 1780, + 310 + ], + "flags": {} + }, + { + "id": 13, + "title": "Preview Masks", + "bounding": [ + 2050, + 4430, + 690, + 1490 + ], + "flags": {} + }, + { + "id": 14, + "title": "Group", + "bounding": [ + 2770, + 4430, + 700, + 1490 + ], + "color": "#3f789e", + "flags": {} + } + ], + "links": [ + { + "id": 491, + "origin_id": 341, + "origin_slot": 0, + "target_id": 324, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 444, + "origin_id": 344, + "origin_slot": 1, + "target_id": 324, + "target_slot": 2, + "type": "INT" + }, + { + "id": 446, + "origin_id": 345, + "origin_slot": 1, + "target_id": 324, + "target_slot": 3, + "type": "INT" + }, + { + "id": 492, + "origin_id": 324, + "origin_slot": 0, + "target_id": 326, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 424, + "origin_id": 338, + "origin_slot": 0, + "target_id": 330, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 398, + "origin_id": 322, + "origin_slot": 0, + "target_id": 332, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 428, + "origin_id": 339, + "origin_slot": 0, + "target_id": 332, + "target_slot": 2, + "type": "INT" + }, + { + "id": 417, + "origin_id": 330, + "origin_slot": 0, + "target_id": 333, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 333, + "origin_id": 364, + "origin_slot": 0, + "target_id": 333, + "target_slot": 1, + "type": "CONDITIONING" + }, + { + "id": 334, + "origin_id": 364, + "origin_slot": 1, + "target_id": 333, + "target_slot": 2, + "type": "CONDITIONING" + }, + { + "id": 419, + "origin_id": 331, + "origin_slot": 0, + "target_id": 333, + "target_slot": 3, + "type": "SAMPLER" + }, + { + "id": 418, + "origin_id": 332, + "origin_slot": 0, + "target_id": 333, + "target_slot": 4, + "type": "SIGMAS" + }, + { + "id": 335, + "origin_id": 364, + "origin_slot": 2, + "target_id": 333, + "target_slot": 5, + "type": "LATENT" + }, + { + "id": 431, + "origin_id": 340, + "origin_slot": 0, + "target_id": 333, + "target_slot": 8, + "type": "FLOAT" + }, + { + "id": 425, + "origin_id": 322, + "origin_slot": 0, + "target_id": 338, + "target_slot": 1, + "type": "MODEL" + }, + { + "id": 432, + "origin_id": 360, + "origin_slot": 0, + "target_id": 338, + "target_slot": 2, + "type": "BOOLEAN" + }, + { + "id": 426, + "origin_id": 336, + "origin_slot": 0, + "target_id": 339, + "target_slot": 0, + "type": "INT" + }, + { + "id": 427, + "origin_id": 334, + "origin_slot": 0, + "target_id": 339, + "target_slot": 1, + "type": "INT" + }, + { + "id": 433, + "origin_id": 360, + "origin_slot": 0, + "target_id": 339, + "target_slot": 2, + "type": "BOOLEAN" + }, + { + "id": 429, + "origin_id": 337, + "origin_slot": 0, + "target_id": 340, + "target_slot": 0, + "type": "FLOAT" + }, + { + "id": 430, + "origin_id": 335, + "origin_slot": 0, + "target_id": 340, + "target_slot": 1, + "type": "FLOAT" + }, + { + "id": 434, + "origin_id": 360, + "origin_slot": 0, + "target_id": 340, + "target_slot": 2, + "type": "BOOLEAN" + }, + { + "id": 490, + "origin_id": 325, + "origin_slot": 0, + "target_id": 341, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 450, + "origin_id": 347, + "origin_slot": 1, + "target_id": 341, + "target_slot": 1, + "type": "INT" + }, + { + "id": 441, + "origin_id": 342, + "origin_slot": 0, + "target_id": 344, + "target_slot": 0, + "type": "INT" + }, + { + "id": 445, + "origin_id": 343, + "origin_slot": 0, + "target_id": 345, + "target_slot": 0, + "type": "INT" + }, + { + "id": 447, + "origin_id": 346, + "origin_slot": 0, + "target_id": 347, + "target_slot": 0, + "type": "INT" + }, + { + "id": 454, + "origin_id": 349, + "origin_slot": 1, + "target_id": 348, + "target_slot": 0, + "type": "CLIP" + }, + { + "id": 493, + "origin_id": 324, + "origin_slot": 0, + "target_id": 350, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 458, + "origin_id": 349, + "origin_slot": 0, + "target_id": 350, + "target_slot": 1, + "type": "MODEL" + }, + { + "id": 461, + "origin_id": 348, + "origin_slot": 0, + "target_id": 350, + "target_slot": 3, + "type": "CONDITIONING" + }, + { + "id": 460, + "origin_id": 350, + "origin_slot": 0, + "target_id": 351, + "target_slot": 0, + "type": "SAM3_TRACK_DATA" + }, + { + "id": 464, + "origin_id": 352, + "origin_slot": 0, + "target_id": 351, + "target_slot": 1, + "type": "SAM3_TRACK_DATA" + }, + { + "id": 476, + "origin_id": 353, + "origin_slot": 0, + "target_id": 351, + "target_slot": 4, + "type": "BOOLEAN" + }, + { + "id": 463, + "origin_id": 349, + "origin_slot": 0, + "target_id": 352, + "target_slot": 1, + "type": "MODEL" + }, + { + "id": 488, + "origin_id": 365, + "origin_slot": 0, + "target_id": 352, + "target_slot": 3, + "type": "CONDITIONING" + }, + { + "id": 466, + "origin_id": 351, + "origin_slot": 0, + "target_id": 354, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 465, + "origin_id": 351, + "origin_slot": 1, + "target_id": 355, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 124, + "origin_id": 333, + "origin_slot": 1, + "target_id": 356, + "target_slot": 0, + "type": "LATENT" + }, + { + "id": 407, + "origin_id": 323, + "origin_slot": 0, + "target_id": 356, + "target_slot": 1, + "type": "VAE" + }, + { + "id": 483, + "origin_id": 356, + "origin_slot": 0, + "target_id": 357, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 484, + "origin_id": 357, + "origin_slot": 0, + "target_id": 358, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 482, + "origin_id": 359, + "origin_slot": 0, + "target_id": 358, + "target_slot": 1, + "type": "IMAGE" + }, + { + "id": 196, + "origin_id": 328, + "origin_slot": 0, + "target_id": 361, + "target_slot": 0, + "type": "CLIP_VISION" + }, + { + "id": 8, + "origin_id": 327, + "origin_slot": 0, + "target_id": 362, + "target_slot": 0, + "type": "CLIP" + }, + { + "id": 7, + "origin_id": 327, + "origin_slot": 0, + "target_id": 363, + "target_slot": 0, + "type": "CLIP" + }, + { + "id": 325, + "origin_id": 363, + "origin_slot": 0, + "target_id": 364, + "target_slot": 0, + "type": "CONDITIONING" + }, + { + "id": 326, + "origin_id": 362, + "origin_slot": 0, + "target_id": 364, + "target_slot": 1, + "type": "CONDITIONING" + }, + { + "id": 406, + "origin_id": 323, + "origin_slot": 0, + "target_id": 364, + "target_slot": 2, + "type": "VAE" + }, + { + "id": 420, + "origin_id": 324, + "origin_slot": 0, + "target_id": 364, + "target_slot": 3, + "type": "IMAGE" + }, + { + "id": 467, + "origin_id": 351, + "origin_slot": 0, + "target_id": 364, + "target_slot": 4, + "type": "IMAGE" + }, + { + "id": 472, + "origin_id": 351, + "origin_slot": 1, + "target_id": 364, + "target_slot": 6, + "type": "IMAGE" + }, + { + "id": 404, + "origin_id": 361, + "origin_slot": 0, + "target_id": 364, + "target_slot": 7, + "type": "CLIP_VISION_OUTPUT" + }, + { + "id": 414, + "origin_id": 326, + "origin_slot": 0, + "target_id": 364, + "target_slot": 9, + "type": "INT" + }, + { + "id": 415, + "origin_id": 326, + "origin_slot": 1, + "target_id": 364, + "target_slot": 10, + "type": "INT" + }, + { + "id": 416, + "origin_id": 326, + "origin_slot": 2, + "target_id": 364, + "target_slot": 11, + "type": "INT" + }, + { + "id": 475, + "origin_id": 353, + "origin_slot": 0, + "target_id": 364, + "target_slot": 18, + "type": "BOOLEAN" + }, + { + "id": 489, + "origin_id": 349, + "origin_slot": 1, + "target_id": 365, + "target_slot": 0, + "type": "CLIP" + }, + { + "id": 409, + "origin_id": -10, + "origin_slot": 0, + "target_id": 325, + "target_slot": 0, + "type": "VIDEO" + }, + { + "id": 469, + "origin_id": -10, + "origin_slot": 1, + "target_id": 352, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 408, + "origin_id": -10, + "origin_slot": 1, + "target_id": 361, + "target_slot": 1, + "type": "IMAGE" + }, + { + "id": 473, + "origin_id": -10, + "origin_slot": 1, + "target_id": 364, + "target_slot": 5, + "type": "IMAGE" + }, + { + "id": 486, + "origin_id": -10, + "origin_slot": 2, + "target_id": 359, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 485, + "origin_id": -10, + "origin_slot": 2, + "target_id": 364, + "target_slot": 8, + "type": "IMAGE" + }, + { + "id": 499, + "origin_id": -10, + "origin_slot": 3, + "target_id": 363, + "target_slot": 1, + "type": "STRING" + }, + { + "id": 502, + "origin_id": -10, + "origin_slot": 4, + "target_id": 346, + "target_slot": 0, + "type": "INT" + }, + { + "id": 504, + "origin_id": -10, + "origin_slot": 5, + "target_id": 353, + "target_slot": 0, + "type": "BOOLEAN" + }, + { + "id": 505, + "origin_id": -10, + "origin_slot": 6, + "target_id": 342, + "target_slot": 0, + "type": "INT" + }, + { + "id": 506, + "origin_id": -10, + "origin_slot": 7, + "target_id": 343, + "target_slot": 0, + "type": "INT" + }, + { + "id": 507, + "origin_id": -10, + "origin_slot": 13, + "target_id": 360, + "target_slot": 0, + "type": "BOOLEAN" + }, + { + "id": 508, + "origin_id": -10, + "origin_slot": 14, + "target_id": 329, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 509, + "origin_id": -10, + "origin_slot": 15, + "target_id": 322, + "target_slot": 1, + "type": "COMBO" + }, + { + "id": 510, + "origin_id": -10, + "origin_slot": 17, + "target_id": 327, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 511, + "origin_id": -10, + "origin_slot": 18, + "target_id": 323, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 512, + "origin_id": -10, + "origin_slot": 19, + "target_id": 328, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 513, + "origin_id": -10, + "origin_slot": 20, + "target_id": 348, + "target_slot": 1, + "type": "STRING" + }, + { + "id": 514, + "origin_id": -10, + "origin_slot": 21, + "target_id": 365, + "target_slot": 1, + "type": "STRING" + }, + { + "id": 515, + "origin_id": -10, + "origin_slot": 22, + "target_id": 349, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 546, + "origin_id": 358, + "origin_slot": 0, + "target_id": -20, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 550, + "origin_id": -10, + "origin_slot": 8, + "target_id": 341, + "target_slot": 2, + "type": "INT" + }, + { + "id": 551, + "origin_id": -10, + "origin_slot": 9, + "target_id": 364, + "target_slot": 17, + "type": "INT" + }, + { + "id": 552, + "origin_id": -10, + "origin_slot": 10, + "target_id": 364, + "target_slot": 13, + "type": "FLOAT" + }, + { + "id": 553, + "origin_id": -10, + "origin_slot": 11, + "target_id": 364, + "target_slot": 14, + "type": "FLOAT" + }, + { + "id": 554, + "origin_id": -10, + "origin_slot": 12, + "target_id": 364, + "target_slot": 15, + "type": "FLOAT" + }, + { + "id": 556, + "origin_id": -10, + "origin_slot": 9, + "target_id": 347, + "target_slot": 2, + "type": "FLOAT,INT,BOOLEAN" + }, + { + "id": 557, + "origin_id": -10, + "origin_slot": 8, + "target_id": 347, + "target_slot": 1, + "type": "FLOAT,INT,BOOLEAN" + }, + { + "id": 559, + "origin_id": -10, + "origin_slot": 23, + "target_id": 333, + "target_slot": 7, + "type": "INT" + }, + { + "id": 566, + "origin_id": 366, + "origin_slot": 0, + "target_id": 322, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 567, + "origin_id": 366, + "origin_slot": 0, + "target_id": 338, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 568, + "origin_id": 329, + "origin_slot": 0, + "target_id": 366, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 569, + "origin_id": -10, + "origin_slot": 16, + "target_id": 366, + "target_slot": 1, + "type": "COMBO" + } + ], + "extra": {}, + "category": "Video generation and editing/Video Edit", + "description": "Replaces a character in a video with a reference image using the SCAIL-2 model for end-to-end controlled animation without intermediate pose maps. Key inputs include a source video, a reference character image, and optional text prompts for style or context. Suitable for animated or live-action footage, multi-character scenes, and creative video editing where direct pose-free animation is needed; works best with moderate-length videos." + } + ] + }, + "extra": { + "BlueprintDescription": "Replaces a character in a video with a reference image using the SCAIL-2 model for end-to-end controlled animation without intermediate pose maps. Key inputs include a source video, a reference character image, and optional text prompts for style or context. Suitable for animated or live-action footage, multi-character scenes, and creative video editing where direct pose-free animation is needed; works best with moderate-length videos.", + "BlueprintSearchAliases": [ + "character replacement", + "SCAIL-2 extend", + "video character swap" + ] + } +} \ No newline at end of file diff --git a/blueprints/Image Depth Estimation (Depth Anything 3).json b/blueprints/Image Depth Estimation (Depth Anything 3).json new file mode 100644 index 000000000..3c5f24845 --- /dev/null +++ b/blueprints/Image Depth Estimation (Depth Anything 3).json @@ -0,0 +1,569 @@ +{ + "revision": 0, + "last_node_id": 89, + "last_link_id": 0, + "nodes": [ + { + "id": 89, + "type": "85e595bd-af9e-40ee-85c5-b98bb15da47a", + "pos": [ + 320, + 520 + ], + "size": [ + 400, + 360 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "localized_name": "image", + "name": "image", + "type": "IMAGE", + "link": null + }, + { + "name": "resolution", + "type": "INT", + "widget": { + "name": "resolution" + }, + "link": null + }, + { + "name": "resize_method", + "type": "COMBO", + "widget": { + "name": "resize_method" + }, + "link": null + }, + { + "label": "output_type", + "name": "output", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "output" + }, + "link": null + }, + { + "label": "output_normalization", + "name": "output.normalization", + "type": "COMBO", + "widget": { + "name": "output.normalization" + }, + "link": null + }, + { + "label": "apply_sky_clip", + "name": "output.apply_sky_clip", + "type": "BOOLEAN", + "widget": { + "name": "output.apply_sky_clip" + }, + "link": null + }, + { + "name": "model_name", + "type": "COMBO", + "widget": { + "name": "model_name" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [] + } + ], + "properties": { + "proxyWidgets": [ + [ + "87", + "resolution" + ], + [ + "87", + "resize_method" + ], + [ + "86", + "output" + ], + [ + "86", + "output.normalization" + ], + [ + "86", + "output.apply_sky_clip" + ], + [ + "88", + "model_name" + ] + ], + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [], + "title": "Image Depth Estimation (Depth Anything 3)" + } + ], + "links": [], + "version": 0.4, + "definitions": { + "subgraphs": [ + { + "id": "85e595bd-af9e-40ee-85c5-b98bb15da47a", + "version": 1, + "state": { + "lastGroupId": 4, + "lastNodeId": 89, + "lastLinkId": 109, + "lastRerouteId": 0 + }, + "revision": 2, + "config": {}, + "name": "Image Depth Estimation (Depth Anything 3)", + "inputNode": { + "id": -10, + "bounding": [ + 400, + 90, + 166.998046875, + 188 + ] + }, + "outputNode": { + "id": -20, + "bounding": [ + 1250, + 146, + 128, + 68 + ] + }, + "inputs": [ + { + "id": "43cf3118-495a-487d-8eb3-a17c7e92f64f", + "name": "image", + "type": "IMAGE", + "linkIds": [ + 19 + ], + "localized_name": "image", + "pos": [ + 542.998046875, + 114 + ] + }, + { + "id": "1089a0a1-6db1-45a8-84b0-0bfdc2ed920a", + "name": "resolution", + "type": "INT", + "linkIds": [ + 22 + ], + "pos": [ + 542.998046875, + 134 + ] + }, + { + "id": "25fb64ac-26d5-466d-995b-6d51b9afa2c4", + "name": "resize_method", + "type": "COMBO", + "linkIds": [ + 23 + ], + "pos": [ + 542.998046875, + 154 + ] + }, + { + "id": "8acafb7c-6c8b-46b3-9d74-c563498a3af1", + "name": "output", + "type": "COMFY_DYNAMICCOMBO_V3", + "linkIds": [ + 24 + ], + "label": "output_type", + "pos": [ + 542.998046875, + 174 + ] + }, + { + "id": "1da5009b-4648-43e8-a257-16426630cf22", + "name": "output.normalization", + "type": "COMBO", + "linkIds": [ + 25 + ], + "label": "output_normalization", + "pos": [ + 542.998046875, + 194 + ] + }, + { + "id": "fd7edb33-5fb1-4538-a411-26e5039a9321", + "name": "output.apply_sky_clip", + "type": "BOOLEAN", + "linkIds": [ + 26 + ], + "label": "apply_sky_clip", + "pos": [ + 542.998046875, + 214 + ] + }, + { + "id": "b5be4c8a-b833-4f1e-8c94-3ed1dd722190", + "name": "model_name", + "type": "COMBO", + "linkIds": [ + 106 + ], + "pos": [ + 542.998046875, + 234 + ] + } + ], + "outputs": [ + { + "id": "478ab537-63bc-4d74-a9f0-c975f550880f", + "name": "IMAGE", + "type": "IMAGE", + "linkIds": [ + 7 + ], + "localized_name": "IMAGE", + "pos": [ + 1274, + 170 + ] + } + ], + "widgets": [], + "nodes": [ + { + "id": 86, + "type": "DA3Render", + "pos": [ + 800, + 310 + ], + "size": [ + 380, + 130 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [ + { + "localized_name": "da3_geometry", + "name": "da3_geometry", + "type": "DA3_GEOMETRY", + "link": 12 + }, + { + "localized_name": "output", + "name": "output", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "output" + }, + "link": 24 + }, + { + "localized_name": "output.normalization", + "name": "output.normalization", + "type": "COMBO", + "widget": { + "name": "output.normalization" + }, + "link": 25 + }, + { + "localized_name": "output.apply_sky_clip", + "name": "output.apply_sky_clip", + "type": "BOOLEAN", + "widget": { + "name": "output.apply_sky_clip" + }, + "link": 26 + }, + { + "name": "geometry", + "type": "DA3_GEOMETRY", + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 7 + ] + } + ], + "properties": { + "Node name for S&R": "DA3Render", + "cnr_id": "comfy-core", + "ver": "0.19.0" + }, + "widgets_values": [ + "depth", + "v2_style", + false + ] + }, + { + "id": 87, + "type": "DA3Inference", + "pos": [ + 800, + 50 + ], + "size": [ + 390, + 130 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "localized_name": "da3_model", + "name": "da3_model", + "type": "DA3_MODEL", + "link": 107 + }, + { + "localized_name": "image", + "name": "image", + "type": "IMAGE", + "link": 19 + }, + { + "localized_name": "resolution", + "name": "resolution", + "type": "INT", + "widget": { + "name": "resolution" + }, + "link": 22 + }, + { + "localized_name": "resize_method", + "name": "resize_method", + "type": "COMBO", + "widget": { + "name": "resize_method" + }, + "link": 23 + }, + { + "localized_name": "mode", + "name": "mode", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "mode" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "da3_geometry", + "name": "da3_geometry", + "type": "DA3_GEOMETRY", + "slot_index": 0, + "links": [ + 12 + ] + } + ], + "properties": { + "Node name for S&R": "DA3Inference", + "cnr_id": "comfy-core", + "ver": "0.19.0" + }, + "widgets_values": [ + 504, + "upper_bound_resize", + "mono" + ] + }, + { + "id": 88, + "type": "LoadDA3Model", + "pos": [ + 810, + -160 + ], + "size": [ + 400, + 140 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "localized_name": "model_name", + "name": "model_name", + "type": "COMBO", + "widget": { + "name": "model_name" + }, + "link": 106 + }, + { + "localized_name": "weight_dtype", + "name": "weight_dtype", + "type": "COMBO", + "widget": { + "name": "weight_dtype" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "DA3_MODEL", + "name": "DA3_MODEL", + "type": "DA3_MODEL", + "links": [ + 107 + ] + } + ], + "properties": { + "Node name for S&R": "LoadDA3Model", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "models": [ + { + "name": "depth_anything_3_mono_large.safetensors", + "url": "https://huggingface.co/Comfy-Org/Depth-Anything-3/resolve/main/geometry_estimation/depth_anything_3_mono_large.safetensors", + "directory": "geometry_estimation" + } + ] + }, + "widgets_values": [ + "depth_anything_3_mono_large.safetensors", + "default" + ] + } + ], + "groups": [], + "links": [ + { + "id": 12, + "origin_id": 87, + "origin_slot": 0, + "target_id": 86, + "target_slot": 0, + "type": "DA3_GEOMETRY" + }, + { + "id": 19, + "origin_id": -10, + "origin_slot": 0, + "target_id": 87, + "target_slot": 1, + "type": "IMAGE" + }, + { + "id": 7, + "origin_id": 86, + "origin_slot": 0, + "target_id": -20, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 22, + "origin_id": -10, + "origin_slot": 1, + "target_id": 87, + "target_slot": 2, + "type": "INT" + }, + { + "id": 23, + "origin_id": -10, + "origin_slot": 2, + "target_id": 87, + "target_slot": 3, + "type": "COMBO" + }, + { + "id": 24, + "origin_id": -10, + "origin_slot": 3, + "target_id": 86, + "target_slot": 1, + "type": "COMFY_DYNAMICCOMBO_V3" + }, + { + "id": 25, + "origin_id": -10, + "origin_slot": 4, + "target_id": 86, + "target_slot": 2, + "type": "COMBO" + }, + { + "id": 26, + "origin_id": -10, + "origin_slot": 5, + "target_id": 86, + "target_slot": 3, + "type": "BOOLEAN" + }, + { + "id": 106, + "origin_id": -10, + "origin_slot": 6, + "target_id": 88, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 107, + "origin_id": 88, + "origin_slot": 0, + "target_id": 87, + "target_slot": 0, + "type": "DA3_MODEL" + } + ], + "extra": {}, + "category": "Conditioning & Preprocessors/Depth", + "description": "This subgraph takes an input image and produces a depth map using the Depth Anything 3 model, which recovers spatially consistent geometry from any number of views. It is ideal for single or multi-view images, videos, and 3D scenes where accurate depth estimation is needed for tasks like SLAM, novel view synthesis, or spatial perception. The model uses a plain transformer backbone and supports both monocular and multi-view inputs without." + } + ] + }, + "extra": { + "BlueprintDescription": "This subgraph takes an input image and produces a depth map using the Depth Anything 3 model, which recovers spatially consistent geometry from any number of views. It is ideal for single or multi-view images, videos, and 3D scenes where accurate depth estimation is needed for tasks like SLAM, novel view synthesis, or spatial perception. The model uses a plain transformer backbone and supports both monocular and multi-view inputs without." + } +} \ No newline at end of file diff --git a/blueprints/Image Edit (Bernini-R).json b/blueprints/Image Edit (Bernini-R).json new file mode 100644 index 000000000..9ad7867a4 --- /dev/null +++ b/blueprints/Image Edit (Bernini-R).json @@ -0,0 +1,3549 @@ +{ + "revision": 0, + "last_node_id": 76, + "last_link_id": 0, + "nodes": [ + { + "id": 76, + "type": "64670aa5-a932-4e9f-a299-3bfef6ebc043", + "pos": [ + 3240, + 4340 + ], + "size": [ + 510, + 850 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "label": "source_image", + "name": "source_video", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "reference_video", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "label": "reference_image", + "name": "reference_images.reference_image_0", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "label": "prompt", + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + }, + { + "label": "task_type", + "name": "choice", + "type": "COMBO", + "widget": { + "name": "choice" + }, + "link": null + }, + { + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + }, + { + "name": "ref_max_size", + "shape": 7, + "type": "INT", + "widget": { + "name": "ref_max_size" + }, + "link": null + }, + { + "label": "enable_turbo_mode", + "name": "value", + "type": "BOOLEAN", + "widget": { + "name": "value" + }, + "link": null + }, + { + "label": "high_noise_model", + "name": "unet_name", + "type": "COMBO", + "widget": { + "name": "unet_name" + }, + "link": null + }, + { + "label": "low_noise_model", + "name": "unet_name_1", + "type": "COMBO", + "widget": { + "name": "unet_name_1" + }, + "link": null + }, + { + "label": "lightning_lora", + "name": "lora_name", + "type": "COMBO", + "widget": { + "name": "lora_name" + }, + "link": null + }, + { + "name": "clip_name", + "type": "COMBO", + "widget": { + "name": "clip_name" + }, + "link": null + }, + { + "name": "vae_name", + "type": "COMBO", + "widget": { + "name": "vae_name" + }, + "link": null + }, + { + "name": "noise_seed", + "type": "INT", + "widget": { + "name": "noise_seed" + }, + "link": null + }, + { + "name": "length", + "type": "INT", + "widget": { + "name": "length" + }, + "link": null + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [] + } + ], + "properties": { + "proxyWidgets": [ + [ + "120", + "value" + ], + [ + "54", + "choice" + ], + [ + "50", + "width" + ], + [ + "50", + "height" + ], + [ + "50", + "ref_max_size" + ], + [ + "70", + "value" + ], + [ + "5", + "unet_name" + ], + [ + "12", + "unet_name" + ], + [ + "11", + "lora_name" + ], + [ + "9", + "clip_name" + ], + [ + "7", + "vae_name" + ], + [ + "19", + "noise_seed" + ], + [ + "50", + "length" + ] + ], + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [], + "title": "Image Edit (Bernini-R)" + } + ], + "links": [], + "version": 0.4, + "definitions": { + "subgraphs": [ + { + "id": "64670aa5-a932-4e9f-a299-3bfef6ebc043", + "version": 1, + "state": { + "lastGroupId": 9, + "lastNodeId": 157, + "lastLinkId": 308, + "lastRerouteId": 0 + }, + "revision": 0, + "config": {}, + "name": "Image Edit (Bernini-R)", + "inputNode": { + "id": -10, + "bounding": [ + -60, + 4230, + 159.744140625, + 368 + ] + }, + "outputNode": { + "id": -20, + "bounding": [ + 4720, + 4220, + 128, + 68 + ] + }, + "inputs": [ + { + "id": "3df44016-bb49-40a2-a1dc-47d750440516", + "name": "source_video", + "type": "IMAGE", + "linkIds": [ + 294 + ], + "label": "source_image", + "pos": [ + 75.744140625, + 4254 + ] + }, + { + "id": "9e960570-35a6-4f48-bfa3-5f6ded6ede10", + "name": "reference_video", + "type": "IMAGE", + "linkIds": [ + 162 + ], + "pos": [ + 75.744140625, + 4274 + ] + }, + { + "id": "4ddf8f1f-b92a-4802-96cd-1379ce4dcba3", + "name": "reference_images.reference_image_0", + "type": "IMAGE", + "linkIds": [ + 163 + ], + "label": "reference_image", + "pos": [ + 75.744140625, + 4294 + ] + }, + { + "id": "fb6d520b-6281-4b05-8a36-64ed7ef58850", + "name": "text", + "type": "STRING", + "linkIds": [ + 308 + ], + "label": "prompt", + "pos": [ + 75.744140625, + 4314 + ] + }, + { + "id": "8005758c-ac19-4baf-8895-4661b2bf9327", + "name": "choice", + "type": "COMBO", + "linkIds": [ + 165 + ], + "label": "task_type", + "pos": [ + 75.744140625, + 4334 + ] + }, + { + "id": "8f7316d5-b98d-4702-83db-6f57fd8804c6", + "name": "width", + "type": "INT", + "linkIds": [ + 166 + ], + "pos": [ + 75.744140625, + 4354 + ] + }, + { + "id": "2dc22817-de2d-4d0f-8763-71795668c1d3", + "name": "height", + "type": "INT", + "linkIds": [ + 167 + ], + "pos": [ + 75.744140625, + 4374 + ] + }, + { + "id": "ff81d4ba-46f0-40e3-995d-50df973d951c", + "name": "ref_max_size", + "type": "INT", + "linkIds": [ + 169 + ], + "pos": [ + 75.744140625, + 4394 + ] + }, + { + "id": "2c12a5f5-e9d9-45e7-bb62-bbf7a352731f", + "name": "value", + "type": "BOOLEAN", + "linkIds": [ + 170 + ], + "label": "enable_turbo_mode", + "pos": [ + 75.744140625, + 4414 + ] + }, + { + "id": "ff09c771-83a5-4d92-bc16-006e6eb19406", + "name": "unet_name", + "type": "COMBO", + "linkIds": [ + 171 + ], + "label": "high_noise_model", + "pos": [ + 75.744140625, + 4434 + ] + }, + { + "id": "67aed2f7-f407-4f55-add0-6ae669a069dc", + "name": "unet_name_1", + "type": "COMBO", + "linkIds": [ + 173 + ], + "label": "low_noise_model", + "pos": [ + 75.744140625, + 4454 + ] + }, + { + "id": "7f456845-cdba-46f0-829c-31b72d4a9038", + "name": "lora_name", + "type": "COMBO", + "linkIds": [ + 174, + 175 + ], + "label": "lightning_lora", + "pos": [ + 75.744140625, + 4474 + ] + }, + { + "id": "b7332485-fd38-4522-8192-ea862afe9a35", + "name": "clip_name", + "type": "COMBO", + "linkIds": [ + 176 + ], + "pos": [ + 75.744140625, + 4494 + ] + }, + { + "id": "3328f3bc-c4ce-467e-830e-0e394c8da2b7", + "name": "vae_name", + "type": "COMBO", + "linkIds": [ + 177 + ], + "pos": [ + 75.744140625, + 4514 + ] + }, + { + "id": "c8a71e1e-2d0b-4f9a-afd8-4186ba4cc241", + "name": "noise_seed", + "type": "INT", + "linkIds": [ + 283 + ], + "pos": [ + 75.744140625, + 4534 + ] + }, + { + "id": "3663f935-0959-4cb3-923d-d44736198056", + "name": "length", + "type": "INT", + "linkIds": [ + 295 + ], + "pos": [ + 75.744140625, + 4554 + ] + } + ], + "outputs": [ + { + "id": "5621984a-f548-41d5-b812-481a9122dd81", + "name": "IMAGE", + "type": "IMAGE", + "linkIds": [ + 297 + ], + "pos": [ + 4744, + 4244 + ] + } + ], + "widgets": [], + "nodes": [ + { + "id": 9, + "type": "CLIPLoader", + "pos": [ + 890, + 5030 + ], + "size": [ + 670, + 170 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "localized_name": "clip_name", + "name": "clip_name", + "type": "COMBO", + "widget": { + "name": "clip_name" + }, + "link": 176 + }, + { + "localized_name": "type", + "name": "type", + "type": "COMBO", + "widget": { + "name": "type" + }, + "link": null + }, + { + "localized_name": "device", + "name": "device", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "device" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "CLIP", + "name": "CLIP", + "type": "CLIP", + "links": [ + 7, + 8 + ] + } + ], + "properties": { + "Node name for S&R": "CLIPLoader", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "models": [ + { + "name": "umt5_xxl_fp8_e4m3fn_scaled.safetensors", + "url": "https://huggingface.co/Comfy-Org/Wan_2.1_ComfyUI_repackaged/resolve/main/split_files/text_encoders/umt5_xxl_fp8_e4m3fn_scaled.safetensors?download=true", + "directory": "text_encoders" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "umt5_xxl_fp8_e4m3fn_scaled.safetensors", + "wan", + "default" + ] + }, + { + "id": 7, + "type": "VAELoader", + "pos": [ + 890, + 5260 + ], + "size": [ + 670, + 110 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "localized_name": "vae_name", + "name": "vae_name", + "type": "COMBO", + "widget": { + "name": "vae_name" + }, + "link": 177 + } + ], + "outputs": [ + { + "localized_name": "VAE", + "name": "VAE", + "type": "VAE", + "links": [ + 99, + 119 + ] + } + ], + "properties": { + "Node name for S&R": "VAELoader", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "models": [ + { + "name": "Wan2_1_VAE_bf16.safetensors", + "url": "https://huggingface.co/Kijai/WanVideo_comfy/resolve/main/Wan2_1_VAE_bf16.safetensors?download=true", + "directory": "vae" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "Wan2_1_VAE_bf16.safetensors" + ] + }, + { + "id": 29, + "type": "LoraLoaderModelOnly", + "pos": [ + 890, + 4810 + ], + "size": [ + 670, + 170 + ], + "flags": {}, + "order": 20, + "mode": 0, + "inputs": [ + { + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 128 + }, + { + "localized_name": "lora_name", + "name": "lora_name", + "type": "COMBO", + "widget": { + "name": "lora_name" + }, + "link": 175 + }, + { + "localized_name": "strength_model", + "name": "strength_model", + "type": "FLOAT", + "widget": { + "name": "strength_model" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MODEL", + "name": "MODEL", + "type": "MODEL", + "links": [ + 132 + ] + } + ], + "properties": { + "Node name for S&R": "LoraLoaderModelOnly", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "models": [ + { + "name": "lightx2v_T2V_14B_cfg_step_distill_v2_lora_rank64_bf16.safetensors", + "url": "https://huggingface.co/Kijai/WanVideo_comfy/resolve/main/Lightx2v/lightx2v_T2V_14B_cfg_step_distill_v2_lora_rank64_bf16.safetensors?download=true", + "directory": "loras" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "lightx2v_T2V_14B_cfg_step_distill_v2_lora_rank64_bf16.safetensors", + 1.5 + ], + "color": "#332922", + "bgcolor": "#593930" + }, + { + "id": 4, + "type": "CLIPTextEncode", + "pos": [ + 1670, + 4670 + ], + "size": [ + 700, + 240 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "localized_name": "clip", + "name": "clip", + "type": "CLIP", + "link": 8 + }, + { + "localized_name": "text", + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "CONDITIONING", + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 118 + ] + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 17, + "type": "SplitSigmas", + "pos": [ + 3240, + 4450 + ], + "size": [ + 270, + 130 + ], + "flags": {}, + "order": 17, + "mode": 0, + "inputs": [ + { + "localized_name": "sigmas", + "name": "sigmas", + "type": "SIGMAS", + "link": 33 + }, + { + "localized_name": "step", + "name": "step", + "type": "INT", + "widget": { + "name": "step" + }, + "link": 146 + } + ], + "outputs": [ + { + "localized_name": "high_sigmas", + "name": "high_sigmas", + "type": "SIGMAS", + "links": [ + 41 + ] + }, + { + "localized_name": "low_sigmas", + "name": "low_sigmas", + "type": "SIGMAS", + "links": [ + 42 + ] + } + ], + "properties": { + "Node name for S&R": "SplitSigmas", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + 3 + ] + }, + { + "id": 27, + "type": "KSamplerSelect", + "pos": [ + 3240, + 4640 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [ + { + "localized_name": "sampler_name", + "name": "sampler_name", + "type": "COMBO", + "widget": { + "name": "sampler_name" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "SAMPLER", + "name": "SAMPLER", + "type": "SAMPLER", + "links": [ + 53, + 56 + ] + } + ], + "properties": { + "Node name for S&R": "KSamplerSelect", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "res_multistep" + ] + }, + { + "id": 5, + "type": "UNETLoader", + "pos": [ + 890, + 4200 + ], + "size": [ + 670, + 140 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "localized_name": "unet_name", + "name": "unet_name", + "type": "COMBO", + "widget": { + "name": "unet_name" + }, + "link": 171 + }, + { + "localized_name": "weight_dtype", + "name": "weight_dtype", + "type": "COMBO", + "widget": { + "name": "weight_dtype" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MODEL", + "name": "MODEL", + "type": "MODEL", + "links": [ + 127, + 130 + ] + } + ], + "properties": { + "Node name for S&R": "UNETLoader", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "models": [ + { + "name": "wan2.2_bernini_r_high_noise_fp8_scaled.safetensors", + "url": "https://huggingface.co/Comfy-Org/Bernini-R/resolve/main/diffusion_models/wan2.2_bernini_r_high_noise_fp8_scaled.safetensors", + "directory": "diffusion_models" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "wan2.2_bernini_r_high_noise_fp8_scaled.safetensors", + "default" + ] + }, + { + "id": 11, + "type": "LoraLoaderModelOnly", + "pos": [ + 890, + 4390 + ], + "size": [ + 670, + 170 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 127 + }, + { + "localized_name": "lora_name", + "name": "lora_name", + "type": "COMBO", + "widget": { + "name": "lora_name" + }, + "link": 174 + }, + { + "localized_name": "strength_model", + "name": "strength_model", + "type": "FLOAT", + "widget": { + "name": "strength_model" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MODEL", + "name": "MODEL", + "type": "MODEL", + "links": [ + 129 + ] + } + ], + "properties": { + "Node name for S&R": "LoraLoaderModelOnly", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "models": [ + { + "name": "lightx2v_T2V_14B_cfg_step_distill_v2_lora_rank64_bf16.safetensors", + "url": "https://huggingface.co/Kijai/WanVideo_comfy/resolve/main/Lightx2v/lightx2v_T2V_14B_cfg_step_distill_v2_lora_rank64_bf16.safetensors?download=true", + "directory": "loras" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "lightx2v_T2V_14B_cfg_step_distill_v2_lora_rank64_bf16.safetensors", + 3 + ], + "color": "#332922", + "bgcolor": "#593930" + }, + { + "id": 12, + "type": "UNETLoader", + "pos": [ + 890, + 4620 + ], + "size": [ + 670, + 140 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "localized_name": "unet_name", + "name": "unet_name", + "type": "COMBO", + "widget": { + "name": "unet_name" + }, + "link": 173 + }, + { + "localized_name": "weight_dtype", + "name": "weight_dtype", + "type": "COMBO", + "widget": { + "name": "weight_dtype" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MODEL", + "name": "MODEL", + "type": "MODEL", + "links": [ + 34, + 128, + 131 + ] + } + ], + "properties": { + "Node name for S&R": "UNETLoader", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "models": [ + { + "name": "wan2.2_bernini_r_low_noise_fp8_scaled.safetensors", + "url": "https://huggingface.co/Comfy-Org/Bernini-R/resolve/main/diffusion_models/wan2.2_bernini_r_low_noise_fp8_scaled.safetensors", + "directory": "diffusion_models" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "wan2.2_bernini_r_low_noise_fp8_scaled.safetensors", + "default" + ] + }, + { + "id": 16, + "type": "VAEDecode", + "pos": [ + 4410, + 4220 + ], + "size": [ + 250, + 100 + ], + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [ + { + "localized_name": "samples", + "name": "samples", + "type": "LATENT", + "link": 31 + }, + { + "localized_name": "vae", + "name": "vae", + "type": "VAE", + "link": 99 + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 297 + ] + } + ], + "properties": { + "Node name for S&R": "VAEDecode", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + } + }, + { + "id": 19, + "type": "SamplerCustom", + "pos": [ + 3580, + 4220 + ], + "size": [ + 280, + 680 + ], + "flags": {}, + "order": 19, + "mode": 0, + "showAdvanced": false, + "inputs": [ + { + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 133 + }, + { + "localized_name": "positive", + "name": "positive", + "type": "CONDITIONING", + "link": 121 + }, + { + "localized_name": "negative", + "name": "negative", + "type": "CONDITIONING", + "link": 123 + }, + { + "localized_name": "sampler", + "name": "sampler", + "type": "SAMPLER", + "link": 53 + }, + { + "localized_name": "sigmas", + "name": "sigmas", + "type": "SIGMAS", + "link": 41 + }, + { + "localized_name": "latent_image", + "name": "latent_image", + "type": "LATENT", + "link": 125 + }, + { + "localized_name": "add_noise", + "name": "add_noise", + "type": "BOOLEAN", + "widget": { + "name": "add_noise" + }, + "link": null + }, + { + "localized_name": "noise_seed", + "name": "noise_seed", + "type": "INT", + "widget": { + "name": "noise_seed" + }, + "link": 283 + }, + { + "localized_name": "cfg", + "name": "cfg", + "type": "FLOAT", + "widget": { + "name": "cfg" + }, + "link": 153 + } + ], + "outputs": [ + { + "localized_name": "output", + "name": "output", + "type": "LATENT", + "links": [ + 40 + ] + }, + { + "localized_name": "denoised_output", + "name": "denoised_output", + "type": "LATENT", + "links": [] + } + ], + "properties": { + "Node name for S&R": "SamplerCustom", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + true, + 283365432432581, + "randomize", + 1 + ] + }, + { + "id": 18, + "type": "BasicScheduler", + "pos": [ + 3240, + 4230 + ], + "size": [ + 270, + 170 + ], + "flags": {}, + "order": 18, + "mode": 0, + "inputs": [ + { + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 34 + }, + { + "localized_name": "scheduler", + "name": "scheduler", + "type": "COMBO", + "widget": { + "name": "scheduler" + }, + "link": null + }, + { + "localized_name": "steps", + "name": "steps", + "type": "INT", + "widget": { + "name": "steps" + }, + "link": 143 + }, + { + "localized_name": "denoise", + "name": "denoise", + "type": "FLOAT", + "widget": { + "name": "denoise" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "SIGMAS", + "name": "SIGMAS", + "type": "SIGMAS", + "links": [ + 33 + ] + } + ], + "properties": { + "Node name for S&R": "BasicScheduler", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "simple", + 6, + 1 + ] + }, + { + "id": 15, + "type": "SamplerCustom", + "pos": [ + 3910, + 4220 + ], + "size": [ + 280, + 680 + ], + "flags": {}, + "order": 15, + "mode": 0, + "showAdvanced": false, + "inputs": [ + { + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 134 + }, + { + "localized_name": "positive", + "name": "positive", + "type": "CONDITIONING", + "link": 122 + }, + { + "localized_name": "negative", + "name": "negative", + "type": "CONDITIONING", + "link": 124 + }, + { + "localized_name": "sampler", + "name": "sampler", + "type": "SAMPLER", + "link": 56 + }, + { + "localized_name": "sigmas", + "name": "sigmas", + "type": "SIGMAS", + "link": 42 + }, + { + "localized_name": "latent_image", + "name": "latent_image", + "type": "LATENT", + "link": 40 + }, + { + "localized_name": "add_noise", + "name": "add_noise", + "type": "BOOLEAN", + "widget": { + "name": "add_noise" + }, + "link": null + }, + { + "localized_name": "noise_seed", + "name": "noise_seed", + "type": "INT", + "widget": { + "name": "noise_seed" + }, + "link": null + }, + { + "localized_name": "cfg", + "name": "cfg", + "type": "FLOAT", + "widget": { + "name": "cfg" + }, + "link": 154 + } + ], + "outputs": [ + { + "localized_name": "output", + "name": "output", + "type": "LATENT", + "links": [ + 31 + ] + }, + { + "localized_name": "denoised_output", + "name": "denoised_output", + "type": "LATENT", + "links": null + } + ], + "properties": { + "Node name for S&R": "SamplerCustom", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + false, + 0, + "fixed", + 1 + ] + }, + { + "id": 50, + "type": "BerniniConditioning", + "pos": [ + 1880, + 4990 + ], + "size": [ + 310, + 380 + ], + "flags": {}, + "order": 21, + "mode": 0, + "inputs": [ + { + "localized_name": "positive", + "name": "positive", + "type": "CONDITIONING", + "link": 117 + }, + { + "localized_name": "negative", + "name": "negative", + "type": "CONDITIONING", + "link": 118 + }, + { + "localized_name": "vae", + "name": "vae", + "type": "VAE", + "link": 119 + }, + { + "localized_name": "source_video", + "name": "source_video", + "shape": 7, + "type": "IMAGE", + "link": 294 + }, + { + "localized_name": "reference_video", + "name": "reference_video", + "shape": 7, + "type": "IMAGE", + "link": 162 + }, + { + "label": "reference_image_0", + "localized_name": "reference_images.reference_image_0", + "name": "reference_images.reference_image_0", + "shape": 7, + "type": "IMAGE", + "link": 163 + }, + { + "label": "reference_image_1", + "localized_name": "reference_images.reference_image_1", + "name": "reference_images.reference_image_1", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": 166 + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": 167 + }, + { + "localized_name": "length", + "name": "length", + "type": "INT", + "widget": { + "name": "length" + }, + "link": 295 + }, + { + "localized_name": "batch_size", + "name": "batch_size", + "type": "INT", + "widget": { + "name": "batch_size" + }, + "link": null + }, + { + "localized_name": "ref_max_size", + "name": "ref_max_size", + "shape": 7, + "type": "INT", + "widget": { + "name": "ref_max_size" + }, + "link": 169 + } + ], + "outputs": [ + { + "localized_name": "positive", + "name": "positive", + "type": "CONDITIONING", + "links": [ + 121, + 122 + ] + }, + { + "localized_name": "negative", + "name": "negative", + "type": "CONDITIONING", + "links": [ + 123, + 124 + ] + }, + { + "localized_name": "latent", + "name": "latent", + "type": "LATENT", + "links": [ + 125 + ] + } + ], + "properties": { + "Node name for S&R": "BerniniConditioning", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + 928, + 1280, + 1, + 1, + 848 + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 3, + "type": "CLIPTextEncode", + "pos": [ + 1670, + 4210 + ], + "size": [ + 710, + 390 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "localized_name": "clip", + "name": "clip", + "type": "CLIP", + "link": 7 + }, + { + "localized_name": "text", + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": 140 + } + ], + "outputs": [ + { + "localized_name": "CONDITIONING", + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 117 + ] + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "" + ], + "color": "#232", + "bgcolor": "#353" + }, + { + "id": 53, + "type": "ComfySwitchNode", + "pos": [ + 2860, + 4520 + ], + "size": [ + 270, + 130 + ], + "flags": {}, + "order": 23, + "mode": 0, + "inputs": [ + { + "localized_name": "on_false", + "name": "on_false", + "type": "*", + "link": 131 + }, + { + "localized_name": "on_true", + "name": "on_true", + "type": "*", + "link": 132 + }, + { + "localized_name": "switch", + "name": "switch", + "type": "BOOLEAN", + "widget": { + "name": "switch" + }, + "link": 148 + } + ], + "outputs": [ + { + "localized_name": "output", + "name": "output", + "type": "*", + "links": [ + 134 + ] + } + ], + "title": "Switch (Low Noise)", + "properties": { + "Node name for S&R": "ComfySwitchNode", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + false + ] + }, + { + "id": 54, + "type": "CustomCombo", + "pos": [ + 890, + 3070 + ], + "size": [ + 230, + 350 + ], + "flags": {}, + "order": 24, + "mode": 0, + "inputs": [ + { + "localized_name": "choice", + "name": "choice", + "type": "COMBO", + "widget": { + "name": "choice" + }, + "link": 165 + } + ], + "outputs": [ + { + "localized_name": "STRING", + "name": "STRING", + "type": "STRING", + "links": null + }, + { + "localized_name": "INDEX", + "name": "INDEX", + "type": "INT", + "links": [ + 135 + ] + } + ], + "properties": { + "Node name for S&R": "CustomCombo", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + "Default", + 0, + "Default", + "Text to Image", + "Text to Video", + "Image Editing", + "Subject to Image", + "" + ] + }, + { + "id": 57, + "type": "c39e0ea5-b767-460c-b394-b09703772fa6", + "pos": [ + 1400, + 3070 + ], + "size": [ + 390, + 440 + ], + "flags": {}, + "order": 25, + "mode": 0, + "inputs": [ + { + "localized_name": "text_per_line", + "name": "text_per_line", + "type": "STRING", + "widget": { + "name": "text_per_line" + }, + "link": null + }, + { + "localized_name": "index", + "name": "index", + "type": "INT", + "widget": { + "name": "index" + }, + "link": 135 + } + ], + "outputs": [ + { + "localized_name": "selected_line", + "name": "selected_line", + "type": "STRING", + "links": [ + 137 + ] + } + ], + "properties": { + "proxyWidgets": [ + [ + "2", + "string" + ], + [ + "56", + "value" + ] + ], + "cnr_id": "comfy-core", + "ver": "0.19.0", + "ue_properties": { + "widget_ue_connectable": {}, + "input_ue_unconnectable": {} + } + }, + "widgets_values": [] + }, + { + "id": 59, + "type": "StringConcatenate", + "pos": [ + 1410, + 3770 + ], + "size": [ + 400, + 250 + ], + "flags": {}, + "order": 26, + "mode": 0, + "inputs": [ + { + "localized_name": "string_a", + "name": "string_a", + "type": "STRING", + "widget": { + "name": "string_a" + }, + "link": 137 + }, + { + "localized_name": "string_b", + "name": "string_b", + "type": "STRING", + "widget": { + "name": "string_b" + }, + "link": 307 + }, + { + "localized_name": "delimiter", + "name": "delimiter", + "type": "STRING", + "widget": { + "name": "delimiter" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "STRING", + "name": "STRING", + "type": "STRING", + "links": [ + 140 + ] + } + ], + "properties": { + "Node name for S&R": "StringConcatenate", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + "", + "", + "" + ] + }, + { + "id": 62, + "type": "PrimitiveInt", + "pos": [ + 2460, + 4240 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "INT", + "widget": { + "name": "value" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 141 + ] + } + ], + "title": "Int (Steps)", + "properties": { + "Node name for S&R": "PrimitiveInt", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 40, + "fixed" + ] + }, + { + "id": 63, + "type": "PrimitiveInt", + "pos": [ + 2470, + 4780 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "INT", + "widget": { + "name": "value" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 142 + ] + } + ], + "title": "Int (Steps)", + "properties": { + "Node name for S&R": "PrimitiveInt", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 6, + "fixed" + ] + }, + { + "id": 64, + "type": "ComfySwitchNode", + "pos": [ + 2860, + 4710 + ], + "size": [ + 270, + 130 + ], + "flags": {}, + "order": 27, + "mode": 0, + "inputs": [ + { + "localized_name": "on_false", + "name": "on_false", + "type": "*", + "link": 141 + }, + { + "localized_name": "on_true", + "name": "on_true", + "type": "*", + "link": 142 + }, + { + "localized_name": "switch", + "name": "switch", + "type": "BOOLEAN", + "widget": { + "name": "switch" + }, + "link": 149 + } + ], + "outputs": [ + { + "localized_name": "output", + "name": "output", + "type": "*", + "links": [ + 143 + ] + } + ], + "title": "Switch (Steps)", + "properties": { + "Node name for S&R": "ComfySwitchNode", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + false + ] + }, + { + "id": 66, + "type": "PrimitiveInt", + "pos": [ + 2460, + 4400 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "INT", + "widget": { + "name": "value" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 144 + ] + } + ], + "title": "Int (Split Steps)", + "properties": { + "Node name for S&R": "PrimitiveInt", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 20, + "fixed" + ] + }, + { + "id": 67, + "type": "PrimitiveInt", + "pos": [ + 2470, + 4950 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "INT", + "widget": { + "name": "value" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 145 + ] + } + ], + "title": "Int (Split Steps)", + "properties": { + "Node name for S&R": "PrimitiveInt", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 3, + "fixed" + ] + }, + { + "id": 68, + "type": "ComfySwitchNode", + "pos": [ + 2860, + 4910 + ], + "size": [ + 270, + 130 + ], + "flags": {}, + "order": 28, + "mode": 0, + "inputs": [ + { + "localized_name": "on_false", + "name": "on_false", + "type": "*", + "link": 144 + }, + { + "localized_name": "on_true", + "name": "on_true", + "type": "*", + "link": 145 + }, + { + "localized_name": "switch", + "name": "switch", + "type": "BOOLEAN", + "widget": { + "name": "switch" + }, + "link": 150 + } + ], + "outputs": [ + { + "localized_name": "output", + "name": "output", + "type": "*", + "links": [ + 146 + ] + } + ], + "title": "Switch (Low Steps)", + "properties": { + "Node name for S&R": "ComfySwitchNode", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + false + ] + }, + { + "id": 72, + "type": "ComfySwitchNode", + "pos": [ + 2860, + 5100 + ], + "size": [ + 270, + 130 + ], + "flags": {}, + "order": 30, + "mode": 0, + "inputs": [ + { + "localized_name": "on_false", + "name": "on_false", + "type": "*", + "link": 151 + }, + { + "localized_name": "on_true", + "name": "on_true", + "type": "*", + "link": 152 + }, + { + "localized_name": "switch", + "name": "switch", + "type": "BOOLEAN", + "widget": { + "name": "switch" + }, + "link": 155 + } + ], + "outputs": [ + { + "localized_name": "output", + "name": "output", + "type": "*", + "links": [ + 153, + 154 + ] + } + ], + "title": "Switch (CFG)", + "properties": { + "Node name for S&R": "ComfySwitchNode", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + false + ] + }, + { + "id": 73, + "type": "PrimitiveFloat", + "pos": [ + 2470, + 5110 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "FLOAT", + "widget": { + "name": "value" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "FLOAT", + "name": "FLOAT", + "type": "FLOAT", + "links": [ + 152 + ] + } + ], + "title": "Float (CFG)", + "properties": { + "Node name for S&R": "PrimitiveFloat", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 1 + ] + }, + { + "id": 71, + "type": "PrimitiveFloat", + "pos": [ + 2460, + 4560 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "FLOAT", + "widget": { + "name": "value" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "FLOAT", + "name": "FLOAT", + "type": "FLOAT", + "links": [ + 151 + ] + } + ], + "title": "Float (CFG)", + "properties": { + "Node name for S&R": "PrimitiveFloat", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 5 + ] + }, + { + "id": 70, + "type": "PrimitiveBoolean", + "pos": [ + 2470, + 5290 + ], + "size": [ + 270, + 100 + ], + "flags": {}, + "order": 29, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "BOOLEAN", + "widget": { + "name": "value" + }, + "link": 170 + } + ], + "outputs": [ + { + "localized_name": "BOOLEAN", + "name": "BOOLEAN", + "type": "BOOLEAN", + "links": [ + 147, + 148, + 149, + 150, + 155 + ] + } + ], + "title": "Boolean (Enable Turbo LoRA?)", + "properties": { + "Node name for S&R": "PrimitiveBoolean", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + true + ] + }, + { + "id": 52, + "type": "ComfySwitchNode", + "pos": [ + 2860, + 4330 + ], + "size": [ + 270, + 130 + ], + "flags": {}, + "order": 22, + "mode": 0, + "inputs": [ + { + "localized_name": "on_false", + "name": "on_false", + "type": "*", + "link": 130 + }, + { + "localized_name": "on_true", + "name": "on_true", + "type": "*", + "link": 129 + }, + { + "localized_name": "switch", + "name": "switch", + "type": "BOOLEAN", + "widget": { + "name": "switch" + }, + "link": 147 + } + ], + "outputs": [ + { + "localized_name": "output", + "name": "output", + "type": "*", + "links": [ + 133 + ] + } + ], + "title": "Switch (High Noise)", + "properties": { + "Node name for S&R": "ComfySwitchNode", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + false + ] + }, + { + "id": 75, + "type": "MarkdownNote", + "pos": [ + 2450, + 3950 + ], + "size": [ + 340, + 150 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": {}, + "widgets_values": [ + "You can find the original settings here: https://github.com/bytedance/Bernini/blob/main/gradio_demo.py" + ], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 120, + "type": "PrimitiveStringMultiline", + "pos": [ + 890, + 3750 + ], + "size": [ + 470, + 290 + ], + "flags": {}, + "order": 31, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "STRING", + "widget": { + "name": "value" + }, + "link": 308 + } + ], + "outputs": [ + { + "localized_name": "STRING", + "name": "STRING", + "type": "STRING", + "links": [ + 307 + ] + } + ], + "title": "Text Multiline (User Prompt)", + "properties": { + "Node name for S&R": "PrimitiveStringMultiline", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + "" + ] + } + ], + "groups": [ + { + "id": 1, + "title": "Prompt & Conditioning", + "bounding": [ + 1650, + 4130, + 760, + 1270 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 2, + "title": "Models", + "bounding": [ + 870, + 4130, + 750, + 1270 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 3, + "title": "Sampling", + "bounding": [ + 3210, + 4130, + 1160, + 1270 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 8, + "title": "Prompt Construction", + "bounding": [ + 870, + 2960, + 960, + 1120 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 4, + "title": "System Prompt (Based on task type)", + "bounding": [ + 880, + 3000, + 920, + 680 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 7, + "title": "Switch Settings", + "bounding": [ + 2440, + 4130, + 740, + 1270 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 5, + "title": "Distill LoRA settings", + "bounding": [ + 2450, + 4710, + 310, + 530 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 6, + "title": "Original Settings", + "bounding": [ + 2450, + 4170, + 300, + 504 + ], + "color": "#3f789e", + "flags": {} + } + ], + "links": [ + { + "id": 128, + "origin_id": 12, + "origin_slot": 0, + "target_id": 29, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 8, + "origin_id": 9, + "origin_slot": 0, + "target_id": 4, + "target_slot": 0, + "type": "CLIP" + }, + { + "id": 33, + "origin_id": 18, + "origin_slot": 0, + "target_id": 17, + "target_slot": 0, + "type": "SIGMAS" + }, + { + "id": 146, + "origin_id": 68, + "origin_slot": 0, + "target_id": 17, + "target_slot": 1, + "type": "INT" + }, + { + "id": 127, + "origin_id": 5, + "origin_slot": 0, + "target_id": 11, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 31, + "origin_id": 15, + "origin_slot": 0, + "target_id": 16, + "target_slot": 0, + "type": "LATENT" + }, + { + "id": 99, + "origin_id": 7, + "origin_slot": 0, + "target_id": 16, + "target_slot": 1, + "type": "VAE" + }, + { + "id": 133, + "origin_id": 52, + "origin_slot": 0, + "target_id": 19, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 121, + "origin_id": 50, + "origin_slot": 0, + "target_id": 19, + "target_slot": 1, + "type": "CONDITIONING" + }, + { + "id": 123, + "origin_id": 50, + "origin_slot": 1, + "target_id": 19, + "target_slot": 2, + "type": "CONDITIONING" + }, + { + "id": 53, + "origin_id": 27, + "origin_slot": 0, + "target_id": 19, + "target_slot": 3, + "type": "SAMPLER" + }, + { + "id": 41, + "origin_id": 17, + "origin_slot": 0, + "target_id": 19, + "target_slot": 4, + "type": "SIGMAS" + }, + { + "id": 125, + "origin_id": 50, + "origin_slot": 2, + "target_id": 19, + "target_slot": 5, + "type": "LATENT" + }, + { + "id": 153, + "origin_id": 72, + "origin_slot": 0, + "target_id": 19, + "target_slot": 8, + "type": "FLOAT" + }, + { + "id": 34, + "origin_id": 12, + "origin_slot": 0, + "target_id": 18, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 143, + "origin_id": 64, + "origin_slot": 0, + "target_id": 18, + "target_slot": 2, + "type": "INT" + }, + { + "id": 134, + "origin_id": 53, + "origin_slot": 0, + "target_id": 15, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 122, + "origin_id": 50, + "origin_slot": 0, + "target_id": 15, + "target_slot": 1, + "type": "CONDITIONING" + }, + { + "id": 124, + "origin_id": 50, + "origin_slot": 1, + "target_id": 15, + "target_slot": 2, + "type": "CONDITIONING" + }, + { + "id": 56, + "origin_id": 27, + "origin_slot": 0, + "target_id": 15, + "target_slot": 3, + "type": "SAMPLER" + }, + { + "id": 42, + "origin_id": 17, + "origin_slot": 1, + "target_id": 15, + "target_slot": 4, + "type": "SIGMAS" + }, + { + "id": 40, + "origin_id": 19, + "origin_slot": 0, + "target_id": 15, + "target_slot": 5, + "type": "LATENT" + }, + { + "id": 154, + "origin_id": 72, + "origin_slot": 0, + "target_id": 15, + "target_slot": 8, + "type": "FLOAT" + }, + { + "id": 117, + "origin_id": 3, + "origin_slot": 0, + "target_id": 50, + "target_slot": 0, + "type": "CONDITIONING" + }, + { + "id": 118, + "origin_id": 4, + "origin_slot": 0, + "target_id": 50, + "target_slot": 1, + "type": "CONDITIONING" + }, + { + "id": 119, + "origin_id": 7, + "origin_slot": 0, + "target_id": 50, + "target_slot": 2, + "type": "VAE" + }, + { + "id": 7, + "origin_id": 9, + "origin_slot": 0, + "target_id": 3, + "target_slot": 0, + "type": "CLIP" + }, + { + "id": 140, + "origin_id": 59, + "origin_slot": 0, + "target_id": 3, + "target_slot": 1, + "type": "STRING" + }, + { + "id": 131, + "origin_id": 12, + "origin_slot": 0, + "target_id": 53, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 132, + "origin_id": 29, + "origin_slot": 0, + "target_id": 53, + "target_slot": 1, + "type": "MODEL" + }, + { + "id": 148, + "origin_id": 70, + "origin_slot": 0, + "target_id": 53, + "target_slot": 2, + "type": "BOOLEAN" + }, + { + "id": 135, + "origin_id": 54, + "origin_slot": 1, + "target_id": 57, + "target_slot": 1, + "type": "INT" + }, + { + "id": 137, + "origin_id": 57, + "origin_slot": 0, + "target_id": 59, + "target_slot": 0, + "type": "STRING" + }, + { + "id": 141, + "origin_id": 62, + "origin_slot": 0, + "target_id": 64, + "target_slot": 0, + "type": "INT" + }, + { + "id": 142, + "origin_id": 63, + "origin_slot": 0, + "target_id": 64, + "target_slot": 1, + "type": "INT" + }, + { + "id": 149, + "origin_id": 70, + "origin_slot": 0, + "target_id": 64, + "target_slot": 2, + "type": "BOOLEAN" + }, + { + "id": 144, + "origin_id": 66, + "origin_slot": 0, + "target_id": 68, + "target_slot": 0, + "type": "INT" + }, + { + "id": 145, + "origin_id": 67, + "origin_slot": 0, + "target_id": 68, + "target_slot": 1, + "type": "INT" + }, + { + "id": 150, + "origin_id": 70, + "origin_slot": 0, + "target_id": 68, + "target_slot": 2, + "type": "BOOLEAN" + }, + { + "id": 151, + "origin_id": 71, + "origin_slot": 0, + "target_id": 72, + "target_slot": 0, + "type": "FLOAT" + }, + { + "id": 152, + "origin_id": 73, + "origin_slot": 0, + "target_id": 72, + "target_slot": 1, + "type": "FLOAT" + }, + { + "id": 155, + "origin_id": 70, + "origin_slot": 0, + "target_id": 72, + "target_slot": 2, + "type": "BOOLEAN" + }, + { + "id": 130, + "origin_id": 5, + "origin_slot": 0, + "target_id": 52, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 129, + "origin_id": 11, + "origin_slot": 0, + "target_id": 52, + "target_slot": 1, + "type": "MODEL" + }, + { + "id": 147, + "origin_id": 70, + "origin_slot": 0, + "target_id": 52, + "target_slot": 2, + "type": "BOOLEAN" + }, + { + "id": 162, + "origin_id": -10, + "origin_slot": 1, + "target_id": 50, + "target_slot": 4, + "type": "IMAGE" + }, + { + "id": 163, + "origin_id": -10, + "origin_slot": 2, + "target_id": 50, + "target_slot": 5, + "type": "IMAGE" + }, + { + "id": 165, + "origin_id": -10, + "origin_slot": 4, + "target_id": 54, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 166, + "origin_id": -10, + "origin_slot": 5, + "target_id": 50, + "target_slot": 7, + "type": "INT" + }, + { + "id": 167, + "origin_id": -10, + "origin_slot": 6, + "target_id": 50, + "target_slot": 8, + "type": "INT" + }, + { + "id": 169, + "origin_id": -10, + "origin_slot": 7, + "target_id": 50, + "target_slot": 11, + "type": "INT" + }, + { + "id": 170, + "origin_id": -10, + "origin_slot": 8, + "target_id": 70, + "target_slot": 0, + "type": "BOOLEAN" + }, + { + "id": 171, + "origin_id": -10, + "origin_slot": 9, + "target_id": 5, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 173, + "origin_id": -10, + "origin_slot": 10, + "target_id": 12, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 174, + "origin_id": -10, + "origin_slot": 11, + "target_id": 11, + "target_slot": 1, + "type": "COMBO" + }, + { + "id": 175, + "origin_id": -10, + "origin_slot": 11, + "target_id": 29, + "target_slot": 1, + "type": "COMBO" + }, + { + "id": 176, + "origin_id": -10, + "origin_slot": 12, + "target_id": 9, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 177, + "origin_id": -10, + "origin_slot": 13, + "target_id": 7, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 283, + "origin_id": -10, + "origin_slot": 14, + "target_id": 19, + "target_slot": 7, + "type": "INT" + }, + { + "id": 294, + "origin_id": -10, + "origin_slot": 0, + "target_id": 50, + "target_slot": 3, + "type": "IMAGE" + }, + { + "id": 295, + "origin_id": -10, + "origin_slot": 15, + "target_id": 50, + "target_slot": 9, + "type": "INT" + }, + { + "id": 297, + "origin_id": 16, + "origin_slot": 0, + "target_id": -20, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 307, + "origin_id": 120, + "origin_slot": 0, + "target_id": 59, + "target_slot": 1, + "type": "STRING" + }, + { + "id": 308, + "origin_id": -10, + "origin_slot": 3, + "target_id": 120, + "target_slot": 0, + "type": "STRING" + } + ], + "extra": {}, + "category": "Image generation and editing/Edit image", + "description": "Edits a single image using a text prompt, leveraging Bernini-R's latent semantic planning for changes like object addition, removal, or style transfer. Ideal for creative edits requiring precise semantic understanding, such as adding a snowman to a scene or altering an object's appearance." + }, + { + "id": "c39e0ea5-b767-460c-b394-b09703772fa6", + "version": 1, + "state": { + "lastGroupId": 9, + "lastNodeId": 157, + "lastLinkId": 308, + "lastRerouteId": 0 + }, + "revision": 0, + "config": {}, + "name": "Select Per-Line Text by Index", + "description": "Selects one line from multiline text by zero-based index for batch or list-driven prompt workflows.", + "inputNode": { + "id": -10, + "bounding": [ + -990, + 8595, + 128, + 88 + ] + }, + "outputNode": { + "id": -20, + "bounding": [ + 710, + 8585, + 128, + 68 + ] + }, + "inputs": [ + { + "id": "75417d82-a934-4ac9-b667-d8dcd5a3bfb3", + "name": "text_per_line", + "type": "STRING", + "linkIds": [ + 13 + ], + "localized_name": "text_per_line", + "pos": [ + -886, + 8619 + ] + }, + { + "id": "46e69a73-1804-4ca6-9175-31445bf0be96", + "name": "index", + "type": "INT", + "linkIds": [ + 14 + ], + "localized_name": "index", + "pos": [ + -886, + 8639 + ] + } + ], + "outputs": [ + { + "id": "e34e8ad1-84d2-4bd2-a460-eb7de6067c10", + "name": "selected_line", + "type": "STRING", + "linkIds": [ + 10 + ], + "localized_name": "selected_line", + "pos": [ + 734, + 8609 + ] + } + ], + "widgets": [], + "nodes": [ + { + "id": 1, + "type": "PreviewAny", + "pos": [ + -500, + 8400 + ], + "size": [ + 230, + 180 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [ + { + "localized_name": "source", + "name": "source", + "type": "*", + "link": 1 + } + ], + "outputs": [ + { + "localized_name": "STRING", + "name": "STRING", + "type": "STRING", + "links": [ + 6 + ] + } + ], + "properties": { + "Node name for S&R": "PreviewAny", + "cnr_id": "comfy-core", + "ver": "0.19.0", + "ue_properties": { + "widget_ue_connectable": {}, + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + null, + null, + null + ] + }, + { + "id": 2, + "type": "RegexExtract", + "pos": [ + -240, + 8740 + ], + "size": [ + 470, + 460 + ], + "flags": {}, + "order": 1, + "mode": 0, + "showAdvanced": false, + "inputs": [ + { + "localized_name": "string", + "name": "string", + "type": "STRING", + "widget": { + "name": "string" + }, + "link": 13 + }, + { + "localized_name": "regex_pattern", + "name": "regex_pattern", + "type": "STRING", + "widget": { + "name": "regex_pattern" + }, + "link": 9 + }, + { + "localized_name": "mode", + "name": "mode", + "type": "COMBO", + "widget": { + "name": "mode" + }, + "link": null + }, + { + "localized_name": "case_insensitive", + "name": "case_insensitive", + "type": "BOOLEAN", + "widget": { + "name": "case_insensitive" + }, + "link": null + }, + { + "localized_name": "multiline", + "name": "multiline", + "type": "BOOLEAN", + "widget": { + "name": "multiline" + }, + "link": null + }, + { + "localized_name": "dotall", + "name": "dotall", + "type": "BOOLEAN", + "widget": { + "name": "dotall" + }, + "link": null + }, + { + "localized_name": "group_index", + "name": "group_index", + "type": "INT", + "widget": { + "name": "group_index" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "STRING", + "name": "STRING", + "type": "STRING", + "links": [ + 10 + ] + } + ], + "properties": { + "Node name for S&R": "RegexExtract", + "cnr_id": "comfy-core", + "ver": "0.19.0", + "ue_properties": { + "widget_ue_connectable": {}, + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + "You are a helpful assistant.\nYou are a helpful assistant specialized in text-to-image generation.\nYou are a helpful assistant specialized in text-to-video generation.\nYou are a helpful assistant specialized in image editing.\nYou are a helpful assistant specialized in subject-to-image generation.", + "", + "First Group", + false, + false, + false, + 1 + ] + }, + { + "id": 56, + "type": "PrimitiveInt", + "pos": [ + -810, + 8400 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "INT", + "widget": { + "name": "value" + }, + "link": 14 + } + ], + "outputs": [ + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 1 + ] + } + ], + "title": "Int (line index)", + "properties": { + "Node name for S&R": "Int (line index)", + "cnr_id": "comfy-core", + "ver": "0.19.0", + "ue_properties": { + "widget_ue_connectable": {}, + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + 0, + "fixed" + ] + }, + { + "id": 8, + "type": "StringReplace", + "pos": [ + -240, + 8400 + ], + "size": [ + 400, + 280 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "localized_name": "string", + "name": "string", + "type": "STRING", + "widget": { + "name": "string" + }, + "link": null + }, + { + "localized_name": "find", + "name": "find", + "type": "STRING", + "widget": { + "name": "find" + }, + "link": null + }, + { + "localized_name": "replace", + "name": "replace", + "type": "STRING", + "widget": { + "name": "replace" + }, + "link": 6 + } + ], + "outputs": [ + { + "localized_name": "STRING", + "name": "STRING", + "type": "STRING", + "links": [ + 9 + ] + } + ], + "properties": { + "Node name for S&R": "StringReplace", + "cnr_id": "comfy-core", + "ver": "0.19.0", + "ue_properties": { + "widget_ue_connectable": {}, + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + "^(?:[^\\n]*\\n){index}([^\\n]*)(?:\\n|$)", + "index", + "" + ] + } + ], + "groups": [], + "links": [ + { + "id": 1, + "origin_id": 56, + "origin_slot": 0, + "target_id": 1, + "target_slot": 0, + "type": "INT" + }, + { + "id": 9, + "origin_id": 8, + "origin_slot": 0, + "target_id": 2, + "target_slot": 1, + "type": "STRING" + }, + { + "id": 6, + "origin_id": 1, + "origin_slot": 0, + "target_id": 8, + "target_slot": 2, + "type": "STRING" + }, + { + "id": 10, + "origin_id": 2, + "origin_slot": 0, + "target_id": -20, + "target_slot": 0, + "type": "STRING" + }, + { + "id": 13, + "origin_id": -10, + "origin_slot": 0, + "target_id": 2, + "target_slot": 0, + "type": "STRING" + }, + { + "id": 14, + "origin_id": -10, + "origin_slot": 1, + "target_id": 56, + "target_slot": 0, + "type": "INT" + } + ], + "extra": { + "ue_links": [], + "links_added_by_ue": [] + } + } + ] + }, + "extra": { + "BlueprintDescription": "Edits a single image using a text prompt, leveraging Bernini-R's latent semantic planning for changes like object addition, removal, or style transfer. Ideal for creative edits requiring precise semantic understanding, such as adding a snowman to a scene or altering an object's appearance." + } +} \ No newline at end of file diff --git a/blueprints/Image to Gaussian Splat (TripoSplat).json b/blueprints/Image to Gaussian Splat (TripoSplat).json new file mode 100644 index 000000000..13da12be3 --- /dev/null +++ b/blueprints/Image to Gaussian Splat (TripoSplat).json @@ -0,0 +1,1983 @@ +{ + "revision": 0, + "last_node_id": 118, + "last_link_id": 0, + "nodes": [ + { + "id": 118, + "type": "6c0b94f2-a3d2-4056-93f7-4072126afe39", + "pos": [ + 790, + 1900 + ], + "size": [ + 430, + 670 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "localized_name": "image", + "name": "image", + "type": "IMAGE", + "link": null + }, + { + "label": "auto_remove_background", + "name": "switch", + "type": "BOOLEAN", + "widget": { + "name": "switch" + }, + "link": null + }, + { + "label": "num_gaussians", + "name": "num_gaussians_1", + "type": "INT", + "widget": { + "name": "num_gaussians_1" + }, + "link": null + }, + { + "name": "seed", + "type": "INT", + "widget": { + "name": "seed" + }, + "link": null + }, + { + "name": "unet_name", + "type": "COMBO", + "widget": { + "name": "unet_name" + }, + "link": null + }, + { + "name": "clip_name", + "type": "COMBO", + "widget": { + "name": "clip_name" + }, + "link": null + }, + { + "label": "splat_vae", + "name": "vae_name", + "type": "COMBO", + "widget": { + "name": "vae_name" + }, + "link": null + }, + { + "label": "flux2_vae", + "name": "vae_name_1", + "type": "COMBO", + "widget": { + "name": "vae_name_1" + }, + "link": null + }, + { + "label": "bg_removal_model", + "name": "bg_removal_name", + "type": "COMBO", + "widget": { + "name": "bg_removal_name" + }, + "link": null + }, + { + "label": "mask", + "name": "on_false", + "type": "MASK", + "link": null + }, + { + "label": "enable_preview", + "name": "switch_1", + "type": "BOOLEAN", + "widget": { + "name": "switch_1" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "splat", + "name": "splat", + "type": "SPLAT", + "links": [] + } + ], + "properties": { + "proxyWidgets": [ + [ + "103", + "switch" + ], + [ + "108", + "num_gaussians" + ], + [ + "102", + "seed" + ], + [ + "111", + "unet_name" + ], + [ + "105", + "clip_name" + ], + [ + "106", + "vae_name" + ], + [ + "107", + "vae_name" + ], + [ + "109", + "bg_removal_name" + ], + [ + "112", + "switch" + ], + [ + "101", + "$$canvas-image-preview" + ] + ], + "cnr_id": "comfy-core", + "ver": "0.22.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [], + "title": "Image to Gaussian Splat (TripoSplat)" + } + ], + "links": [], + "version": 0.4, + "definitions": { + "subgraphs": [ + { + "id": "6c0b94f2-a3d2-4056-93f7-4072126afe39", + "version": 1, + "state": { + "lastGroupId": 6, + "lastNodeId": 118, + "lastLinkId": 219, + "lastRerouteId": 0 + }, + "revision": 0, + "config": {}, + "name": "Image to Gaussian Splat (TripoSplat)", + "inputNode": { + "id": -10, + "bounding": [ + -110, + 1050, + 194.666015625, + 268 + ] + }, + "outputNode": { + "id": -20, + "bounding": [ + 2250, + 700, + 128, + 68 + ] + }, + "inputs": [ + { + "id": "736cc4fd-77da-4fb7-8db7-d94296654571", + "name": "image", + "type": "IMAGE", + "linkIds": [ + 165, + 195 + ], + "localized_name": "image", + "pos": [ + 60.666015625, + 1074 + ] + }, + { + "id": "bc100771-6c6a-4223-a6b1-62f8c7b1c5b6", + "name": "switch", + "type": "BOOLEAN", + "linkIds": [ + 184 + ], + "label": "auto_remove_background", + "pos": [ + 60.666015625, + 1094 + ] + }, + { + "id": "13a86171-bf5d-4328-ae51-116502beb274", + "name": "num_gaussians_1", + "type": "INT", + "linkIds": [ + 196 + ], + "label": "num_gaussians", + "pos": [ + 60.666015625, + 1114 + ] + }, + { + "id": "38ebeca8-4674-4a73-a686-f8b9444c7bb6", + "name": "seed", + "type": "INT", + "linkIds": [ + 197 + ], + "pos": [ + 60.666015625, + 1134 + ] + }, + { + "id": "6952068a-04ba-4e56-b702-0986f69b0d96", + "name": "unet_name", + "type": "COMBO", + "linkIds": [ + 201 + ], + "pos": [ + 60.666015625, + 1154 + ] + }, + { + "id": "206819d5-2ecb-4788-8dcf-134138e46bbf", + "name": "clip_name", + "type": "COMBO", + "linkIds": [ + 202 + ], + "pos": [ + 60.666015625, + 1174 + ] + }, + { + "id": "83a8d3d1-4c75-4ce0-a03c-face70f57ac9", + "name": "vae_name", + "type": "COMBO", + "linkIds": [ + 203 + ], + "label": "splat_vae", + "pos": [ + 60.666015625, + 1194 + ] + }, + { + "id": "dce14e7e-37ca-493b-8dbe-24333d7cee63", + "name": "vae_name_1", + "type": "COMBO", + "linkIds": [ + 204 + ], + "label": "flux2_vae", + "pos": [ + 60.666015625, + 1214 + ] + }, + { + "id": "d9c843e7-567b-4acc-813d-752579cb89b5", + "name": "bg_removal_name", + "type": "COMBO", + "linkIds": [ + 205 + ], + "label": "bg_removal_model", + "pos": [ + 60.666015625, + 1234 + ] + }, + { + "id": "07d672fb-6e35-44d8-b973-7577c3468208", + "name": "on_false", + "type": "MASK", + "linkIds": [ + 209 + ], + "label": "mask", + "pos": [ + 60.666015625, + 1254 + ] + }, + { + "id": "8ae4f3b4-e3d2-4d42-84fa-4e9af9ffc3e7", + "name": "switch_1", + "type": "BOOLEAN", + "linkIds": [ + 212 + ], + "label": "enable_preview", + "pos": [ + 60.666015625, + 1274 + ] + } + ], + "outputs": [ + { + "id": "4185ff5d-4179-4938-9fb6-cea3a8322606", + "name": "splat", + "type": "SPLAT", + "linkIds": [ + 156, + 156 + ], + "localized_name": "splat", + "pos": [ + 2274, + 724 + ] + } + ], + "widgets": [], + "nodes": [ + { + "id": 100, + "type": "TripoSplatConditioning", + "pos": [ + 940, + 800 + ], + "size": [ + 290, + 120 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [ + { + "localized_name": "clip_vision", + "name": "clip_vision", + "type": "CLIP_VISION", + "link": 45 + }, + { + "localized_name": "vae", + "name": "vae", + "type": "VAE", + "link": 46 + }, + { + "localized_name": "image", + "name": "image", + "type": "IMAGE", + "link": 47 + } + ], + "outputs": [ + { + "localized_name": "positive", + "name": "positive", + "type": "CONDITIONING", + "links": [ + 48 + ] + }, + { + "localized_name": "negative", + "name": "negative", + "type": "CONDITIONING", + "links": [ + 49 + ] + }, + { + "localized_name": "latent", + "name": "latent", + "type": "LATENT", + "links": [ + 54 + ] + } + ], + "properties": { + "Node name for S&R": "TripoSplatConditioning", + "cnr_id": "comfy-core", + "ver": "0.22.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + } + }, + { + "id": 101, + "type": "PreviewImage", + "pos": [ + 1360, + 1290 + ], + "size": [ + 440, + 530 + ], + "flags": {}, + "order": 1, + "mode": 4, + "inputs": [ + { + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "link": 38 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage", + "cnr_id": "comfy-core", + "ver": "0.22.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + } + }, + { + "id": 102, + "type": "KSampler", + "pos": [ + 1350, + 570 + ], + "size": [ + 290, + 590 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 211 + }, + { + "localized_name": "positive", + "name": "positive", + "type": "CONDITIONING", + "link": 48 + }, + { + "localized_name": "negative", + "name": "negative", + "type": "CONDITIONING", + "link": 49 + }, + { + "localized_name": "latent_image", + "name": "latent_image", + "type": "LATENT", + "link": 54 + }, + { + "localized_name": "seed", + "name": "seed", + "type": "INT", + "widget": { + "name": "seed" + }, + "link": 197 + }, + { + "localized_name": "steps", + "name": "steps", + "type": "INT", + "widget": { + "name": "steps" + }, + "link": null + }, + { + "localized_name": "cfg", + "name": "cfg", + "type": "FLOAT", + "widget": { + "name": "cfg" + }, + "link": null + }, + { + "localized_name": "sampler_name", + "name": "sampler_name", + "type": "COMBO", + "widget": { + "name": "sampler_name" + }, + "link": null + }, + { + "localized_name": "scheduler", + "name": "scheduler", + "type": "COMBO", + "widget": { + "name": "scheduler" + }, + "link": null + }, + { + "localized_name": "denoise", + "name": "denoise", + "type": "FLOAT", + "widget": { + "name": "denoise" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "LATENT", + "name": "LATENT", + "type": "LATENT", + "links": [ + 103 + ] + } + ], + "properties": { + "Node name for S&R": "KSampler", + "cnr_id": "comfy-core", + "ver": "0.22.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + 46, + "fixed", + 20, + 3, + "dpmpp_2m", + "simple", + 1 + ] + }, + { + "id": 103, + "type": "ComfySwitchNode", + "pos": [ + 1020, + 1330 + ], + "size": [ + 270, + 130 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "localized_name": "on_false", + "name": "on_false", + "type": "*", + "link": 208 + }, + { + "localized_name": "on_true", + "name": "on_true", + "type": "*", + "link": 170 + }, + { + "localized_name": "switch", + "name": "switch", + "type": "BOOLEAN", + "widget": { + "name": "switch" + }, + "link": 184 + } + ], + "outputs": [ + { + "localized_name": "output", + "name": "output", + "type": "*", + "links": [ + 95 + ] + } + ], + "title": "Switch: Mask Source", + "properties": { + "Node name for S&R": "ComfySwitchNode", + "cnr_id": "comfy-core", + "ver": "0.22.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + true + ] + }, + { + "id": 104, + "type": "TripoSplatPreprocessImage", + "pos": [ + 940, + 990 + ], + "size": [ + 300, + 160 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "localized_name": "image", + "name": "image", + "type": "IMAGE", + "link": 195 + }, + { + "localized_name": "mask", + "name": "mask", + "type": "MASK", + "link": 95 + }, + { + "localized_name": "erode_radius", + "name": "erode_radius", + "type": "INT", + "widget": { + "name": "erode_radius" + }, + "link": null + }, + { + "localized_name": "size", + "name": "size", + "type": "INT", + "widget": { + "name": "size" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "image", + "name": "image", + "type": "IMAGE", + "links": [ + 38, + 47 + ] + } + ], + "properties": { + "Node name for S&R": "TripoSplatPreprocessImage", + "cnr_id": "comfy-core", + "ver": "0.22.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + 1, + 1024 + ] + }, + { + "id": 105, + "type": "CLIPVisionLoader", + "pos": [ + 410, + 760 + ], + "size": [ + 420, + 140 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "localized_name": "clip_name", + "name": "clip_name", + "type": "COMBO", + "widget": { + "name": "clip_name" + }, + "link": 202 + } + ], + "outputs": [ + { + "localized_name": "CLIP_VISION", + "name": "CLIP_VISION", + "type": "CLIP_VISION", + "links": [ + 45 + ] + } + ], + "properties": { + "Node name for S&R": "CLIPVisionLoader", + "cnr_id": "comfy-core", + "ver": "0.22.0", + "models": [ + { + "name": "dino_v3_vit_h.safetensors", + "url": "https://huggingface.co/VAST-AI/TripoSplat/resolve/main/clip_vision/dino_v3_vit_h.safetensors", + "directory": "clip_vision" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "dino_v3_vit_h.safetensors" + ] + }, + { + "id": 106, + "type": "VAELoader", + "pos": [ + 420, + 930 + ], + "size": [ + 410, + 110 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "localized_name": "vae_name", + "name": "vae_name", + "type": "COMBO", + "widget": { + "name": "vae_name" + }, + "link": 203 + } + ], + "outputs": [ + { + "localized_name": "VAE", + "name": "VAE", + "type": "VAE", + "links": [ + 104, + 216 + ] + } + ], + "properties": { + "Node name for S&R": "VAELoader", + "cnr_id": "comfy-core", + "ver": "0.22.0", + "models": [ + { + "name": "triposplat_vae_decoder_fp16.safetensors", + "url": "https://huggingface.co/VAST-AI/TripoSplat/resolve/main/vae/triposplat_vae_decoder_fp16.safetensors", + "directory": "vae" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "triposplat_vae_decoder_fp16.safetensors" + ] + }, + { + "id": 107, + "type": "VAELoader", + "pos": [ + 410, + 1120 + ], + "size": [ + 420, + 110 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "localized_name": "vae_name", + "name": "vae_name", + "type": "COMBO", + "widget": { + "name": "vae_name" + }, + "link": 204 + } + ], + "outputs": [ + { + "localized_name": "VAE", + "name": "VAE", + "type": "VAE", + "links": [ + 46 + ] + } + ], + "properties": { + "Node name for S&R": "VAELoader", + "cnr_id": "comfy-core", + "ver": "0.22.0", + "models": [ + { + "name": "flux2-vae.safetensors", + "url": "https://huggingface.co/VAST-AI/TripoSplat/resolve/main/vae/flux2-vae.safetensors", + "directory": "vae" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "flux2-vae.safetensors" + ] + }, + { + "id": 108, + "type": "VAEDecodeTripoSplat", + "pos": [ + 1730, + 570 + ], + "size": [ + 430, + 160 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "localized_name": "samples", + "name": "samples", + "type": "LATENT", + "link": 103 + }, + { + "localized_name": "vae", + "name": "vae", + "type": "VAE", + "link": 104 + }, + { + "localized_name": "num_gaussians", + "name": "num_gaussians", + "type": "INT", + "widget": { + "name": "num_gaussians" + }, + "link": 196 + }, + { + "localized_name": "seed", + "name": "seed", + "type": "INT", + "widget": { + "name": "seed" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "splat", + "name": "splat", + "type": "SPLAT", + "links": [ + 156 + ] + } + ], + "properties": { + "Node name for S&R": "VAEDecodeTripoSplat", + "cnr_id": "comfy-core", + "ver": "0.22.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + 262144, + 790219963981395, + "fixed" + ] + }, + { + "id": 109, + "type": "e527b93c-76f7-485d-b285-fcf78914a4d5", + "pos": [ + 410, + 1330 + ], + "size": [ + 350, + 160 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "localized_name": "image", + "name": "image", + "type": "IMAGE", + "link": 165 + }, + { + "name": "bg_removal_name", + "type": "COMBO", + "widget": { + "name": "bg_removal_name" + }, + "link": 205 + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 166, + 194 + ] + }, + { + "name": "mask", + "type": "MASK", + "links": [ + 170 + ] + } + ], + "properties": { + "proxyWidgets": [ + [ + "115", + "bg_removal_name" + ] + ], + "cnr_id": "comfy-core", + "ver": "0.22.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [] + }, + { + "id": 110, + "type": "InvertMask", + "pos": [ + 430, + 1540 + ], + "size": [ + 230, + 80 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "localized_name": "mask", + "name": "mask", + "type": "MASK", + "link": 209 + } + ], + "outputs": [ + { + "localized_name": "MASK", + "name": "MASK", + "type": "MASK", + "links": [ + 208 + ] + } + ], + "properties": { + "Node name for S&R": "InvertMask", + "cnr_id": "comfy-core", + "ver": "0.22.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + } + }, + { + "id": 111, + "type": "UNETLoader", + "pos": [ + 410, + 560 + ], + "size": [ + 410, + 140 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "localized_name": "unet_name", + "name": "unet_name", + "type": "COMBO", + "widget": { + "name": "unet_name" + }, + "link": 201 + }, + { + "localized_name": "weight_dtype", + "name": "weight_dtype", + "type": "COMBO", + "widget": { + "name": "weight_dtype" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MODEL", + "name": "MODEL", + "type": "MODEL", + "links": [ + 213, + 215 + ] + } + ], + "properties": { + "Node name for S&R": "UNETLoader", + "cnr_id": "comfy-core", + "ver": "0.22.0", + "models": [ + { + "name": "triposplat_fp16.safetensors", + "url": "https://huggingface.co/VAST-AI/TripoSplat/resolve/main/diffusion_models/triposplat_fp16.safetensors", + "directory": "diffusion_models" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "triposplat_fp16.safetensors", + "default" + ] + }, + { + "id": 112, + "type": "ComfySwitchNode", + "pos": [ + 930, + 370 + ], + "size": [ + 300, + 140 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "localized_name": "on_false", + "name": "on_false", + "type": "*", + "link": 213 + }, + { + "localized_name": "on_true", + "name": "on_true", + "type": "*", + "link": 217 + }, + { + "localized_name": "switch", + "name": "switch", + "type": "BOOLEAN", + "widget": { + "name": "switch" + }, + "link": 212 + } + ], + "outputs": [ + { + "localized_name": "output", + "name": "output", + "type": "*", + "links": [ + 211 + ] + } + ], + "properties": { + "Node name for S&R": "ComfySwitchNode", + "cnr_id": "comfy-core", + "ver": "0.22.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + true + ] + }, + { + "id": 113, + "type": "TripoSplatSamplingPreview", + "pos": [ + 940, + 560 + ], + "size": [ + 290, + 190 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 215 + }, + { + "localized_name": "vae", + "name": "vae", + "type": "VAE", + "link": 216 + }, + { + "localized_name": "octree_level", + "name": "octree_level", + "type": "INT", + "widget": { + "name": "octree_level" + }, + "link": null + }, + { + "localized_name": "num_gaussians", + "name": "num_gaussians", + "type": "INT", + "widget": { + "name": "num_gaussians" + }, + "link": null + }, + { + "localized_name": "yaw", + "name": "yaw", + "type": "FLOAT", + "widget": { + "name": "yaw" + }, + "link": null + }, + { + "localized_name": "pitch", + "name": "pitch", + "type": "FLOAT", + "widget": { + "name": "pitch" + }, + "link": null + }, + { + "localized_name": "point_size", + "name": "point_size", + "type": "INT", + "widget": { + "name": "point_size" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MODEL", + "name": "MODEL", + "type": "MODEL", + "links": [ + 217 + ] + } + ], + "properties": { + "Node name for S&R": "TripoSplatSamplingPreview", + "cnr_id": "comfy-core", + "ver": "0.22.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + 5, + 16384, + 90, + 15, + 2 + ] + } + ], + "groups": [ + { + "id": 1, + "title": "Models", + "bounding": [ + 400, + 490, + 440, + 748.625 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 2, + "title": "Image Preprocessing", + "bounding": [ + 400, + 1260, + 910, + 370 + ], + "color": "#3f789e", + "flags": {} + } + ], + "links": [ + { + "id": 45, + "origin_id": 105, + "origin_slot": 0, + "target_id": 100, + "target_slot": 0, + "type": "CLIP_VISION" + }, + { + "id": 46, + "origin_id": 107, + "origin_slot": 0, + "target_id": 100, + "target_slot": 1, + "type": "VAE" + }, + { + "id": 47, + "origin_id": 104, + "origin_slot": 0, + "target_id": 100, + "target_slot": 2, + "type": "IMAGE" + }, + { + "id": 38, + "origin_id": 104, + "origin_slot": 0, + "target_id": 101, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 103, + "origin_id": 102, + "origin_slot": 0, + "target_id": 108, + "target_slot": 0, + "type": "LATENT" + }, + { + "id": 104, + "origin_id": 106, + "origin_slot": 0, + "target_id": 108, + "target_slot": 1, + "type": "VAE" + }, + { + "id": 48, + "origin_id": 100, + "origin_slot": 0, + "target_id": 102, + "target_slot": 1, + "type": "CONDITIONING" + }, + { + "id": 49, + "origin_id": 100, + "origin_slot": 1, + "target_id": 102, + "target_slot": 2, + "type": "CONDITIONING" + }, + { + "id": 54, + "origin_id": 100, + "origin_slot": 2, + "target_id": 102, + "target_slot": 3, + "type": "LATENT" + }, + { + "id": 170, + "origin_id": 109, + "origin_slot": 1, + "target_id": 103, + "target_slot": 1, + "type": "MASK" + }, + { + "id": 95, + "origin_id": 103, + "origin_slot": 0, + "target_id": 104, + "target_slot": 1, + "type": "MASK" + }, + { + "id": 165, + "origin_id": -10, + "origin_slot": 0, + "target_id": 109, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 156, + "origin_id": 108, + "origin_slot": 0, + "target_id": -20, + "target_slot": 0, + "type": "SPLAT" + }, + { + "id": 184, + "origin_id": -10, + "origin_slot": 1, + "target_id": 103, + "target_slot": 2, + "type": "BOOLEAN" + }, + { + "id": 195, + "origin_id": -10, + "origin_slot": 0, + "target_id": 104, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 196, + "origin_id": -10, + "origin_slot": 2, + "target_id": 108, + "target_slot": 2, + "type": "INT" + }, + { + "id": 197, + "origin_id": -10, + "origin_slot": 3, + "target_id": 102, + "target_slot": 4, + "type": "INT" + }, + { + "id": 201, + "origin_id": -10, + "origin_slot": 4, + "target_id": 111, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 202, + "origin_id": -10, + "origin_slot": 5, + "target_id": 105, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 203, + "origin_id": -10, + "origin_slot": 6, + "target_id": 106, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 204, + "origin_id": -10, + "origin_slot": 7, + "target_id": 107, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 205, + "origin_id": -10, + "origin_slot": 8, + "target_id": 109, + "target_slot": 1, + "type": "COMBO" + }, + { + "id": 208, + "origin_id": 110, + "origin_slot": 0, + "target_id": 103, + "target_slot": 0, + "type": "MASK" + }, + { + "id": 209, + "origin_id": -10, + "origin_slot": 9, + "target_id": 110, + "target_slot": 0, + "type": "MASK" + }, + { + "id": 211, + "origin_id": 112, + "origin_slot": 0, + "target_id": 102, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 212, + "origin_id": -10, + "origin_slot": 10, + "target_id": 112, + "target_slot": 2, + "type": "BOOLEAN" + }, + { + "id": 213, + "origin_id": 111, + "origin_slot": 0, + "target_id": 112, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 215, + "origin_id": 111, + "origin_slot": 0, + "target_id": 113, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 216, + "origin_id": 106, + "origin_slot": 0, + "target_id": 113, + "target_slot": 1, + "type": "VAE" + }, + { + "id": 217, + "origin_id": 113, + "origin_slot": 0, + "target_id": 112, + "target_slot": 1, + "type": "MODEL" + } + ], + "extra": {}, + "category": "3D", + "description": "This subgraph takes a single 2D image as input and generates a variable number of 3D Gaussians (up to 262,144) as output, enabling high-quality 3D reconstruction. It is ideal for asset creation, AR/VR, game development, and simulation environments, handling diverse image styles from photos to illustrations." + }, + { + "id": "e527b93c-76f7-485d-b285-fcf78914a4d5", + "version": 1, + "state": { + "lastGroupId": 6, + "lastNodeId": 118, + "lastLinkId": 219, + "lastRerouteId": 0 + }, + "revision": 0, + "config": {}, + "name": "Remove Background (BiRefNet)", + "description": "Removes or replaces image backgrounds using BiRefNet segmentation and alpha compositing.", + "inputNode": { + "id": -10, + "bounding": [ + -6810, + 1480, + 150.9140625, + 88 + ] + }, + "outputNode": { + "id": -20, + "bounding": [ + -6169.049695722246, + 1475.2619799128663, + 128, + 88 + ] + }, + "inputs": [ + { + "id": "7bc321cd-df31-4c39-aaf7-7f0d01326189", + "name": "image", + "type": "IMAGE", + "linkIds": [ + 5, + 7 + ], + "localized_name": "image", + "pos": [ + -6683.0859375, + 1504 + ] + }, + { + "id": "e89d2cd8-daa3-4e29-8a69-851db85072cb", + "name": "bg_removal_name", + "type": "COMBO", + "linkIds": [ + 12 + ], + "pos": [ + -6683.0859375, + 1524 + ] + } + ], + "outputs": [ + { + "id": "16e7863c-4c38-46c2-aa74-e82991fbfe8d", + "name": "IMAGE", + "type": "IMAGE", + "linkIds": [ + 8 + ], + "localized_name": "IMAGE", + "pos": [ + -6145.049695722246, + 1499.2619799128663 + ] + }, + { + "id": "f7240c19-5b80-406e-a8e2-9b12440ee2d6", + "name": "mask", + "type": "MASK", + "linkIds": [ + 11 + ], + "pos": [ + -6145.049695722246, + 1519.2619799128663 + ] + } + ], + "widgets": [], + "nodes": [ + { + "id": 114, + "type": "RemoveBackground", + "pos": [ + -6540, + 1440 + ], + "size": [ + 310, + 100 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [ + { + "localized_name": "bg_removal_model", + "name": "bg_removal_model", + "type": "BACKGROUND_REMOVAL", + "link": 3 + }, + { + "localized_name": "image", + "name": "image", + "type": "IMAGE", + "link": 5 + } + ], + "outputs": [ + { + "localized_name": "mask", + "name": "mask", + "type": "MASK", + "links": [ + 4, + 11 + ] + } + ], + "properties": { + "Node name for S&R": "RemoveBackground", + "cnr_id": "comfy-core", + "ver": "0.22.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + } + }, + { + "id": 115, + "type": "LoadBackgroundRemovalModel", + "pos": [ + -6540, + 1300 + ], + "size": [ + 320, + 110 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "localized_name": "bg_removal_name", + "name": "bg_removal_name", + "type": "COMBO", + "widget": { + "name": "bg_removal_name" + }, + "link": 12 + } + ], + "outputs": [ + { + "localized_name": "bg_model", + "name": "bg_model", + "type": "BACKGROUND_REMOVAL", + "links": [ + 3 + ] + } + ], + "properties": { + "Node name for S&R": "LoadBackgroundRemovalModel", + "cnr_id": "comfy-core", + "ver": "0.22.0", + "models": [ + { + "name": "birefnet.safetensors", + "url": "https://huggingface.co/Comfy-Org/BiRefNet/resolve/main/background_removal/birefnet.safetensors", + "directory": "background_removal" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "birefnet.safetensors" + ] + }, + { + "id": 116, + "type": "InvertMask", + "pos": [ + -6530, + 1570 + ], + "size": [ + 290, + 80 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "localized_name": "mask", + "name": "mask", + "type": "MASK", + "link": 4 + } + ], + "outputs": [ + { + "localized_name": "MASK", + "name": "MASK", + "type": "MASK", + "links": [ + 6 + ] + } + ], + "properties": { + "Node name for S&R": "InvertMask", + "cnr_id": "comfy-core", + "ver": "0.22.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + } + }, + { + "id": 117, + "type": "JoinImageWithAlpha", + "pos": [ + -6530, + 1670 + ], + "size": [ + 290, + 100 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "localized_name": "image", + "name": "image", + "type": "IMAGE", + "link": 7 + }, + { + "localized_name": "alpha", + "name": "alpha", + "type": "MASK", + "link": 6 + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 8 + ] + } + ], + "properties": { + "Node name for S&R": "JoinImageWithAlpha", + "cnr_id": "comfy-core", + "ver": "0.22.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + } + } + ], + "groups": [], + "links": [ + { + "id": 3, + "origin_id": 115, + "origin_slot": 0, + "target_id": 114, + "target_slot": 0, + "type": "BACKGROUND_REMOVAL" + }, + { + "id": 4, + "origin_id": 114, + "origin_slot": 0, + "target_id": 116, + "target_slot": 0, + "type": "MASK" + }, + { + "id": 6, + "origin_id": 116, + "origin_slot": 0, + "target_id": 117, + "target_slot": 1, + "type": "MASK" + }, + { + "id": 5, + "origin_id": -10, + "origin_slot": 0, + "target_id": 114, + "target_slot": 1, + "type": "IMAGE" + }, + { + "id": 7, + "origin_id": -10, + "origin_slot": 0, + "target_id": 117, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 8, + "origin_id": 117, + "origin_slot": 0, + "target_id": -20, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 11, + "origin_id": 114, + "origin_slot": 0, + "target_id": -20, + "target_slot": 1, + "type": "MASK" + }, + { + "id": 12, + "origin_id": -10, + "origin_slot": 1, + "target_id": 115, + "target_slot": 0, + "type": "COMBO" + } + ], + "extra": {} + } + ] + }, + "extra": { + "BlueprintDescription": "This subgraph takes a single 2D image as input and generates a variable number of 3D Gaussians (up to 262,144) as output, enabling high-quality 3D reconstruction. It is ideal for asset creation, AR/VR, game development, and simulation environments, handling diverse image styles from photos to illustrations." + } +} \ No newline at end of file diff --git a/blueprints/Text to Image (Anima Base 1.0).json b/blueprints/Text to Image (Anima Base 1.0).json new file mode 100644 index 000000000..379e7c605 --- /dev/null +++ b/blueprints/Text to Image (Anima Base 1.0).json @@ -0,0 +1,1088 @@ +{ + "revision": 0, + "last_node_id": 60, + "last_link_id": 0, + "nodes": [ + { + "id": 60, + "type": "a3c0dab6-b250-4585-a0f9-8fb8b074fb2f", + "pos": [ + -10, + 130 + ], + "size": [ + 500, + 640 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "label": "prompt", + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + }, + { + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + }, + { + "name": "steps", + "type": "INT", + "widget": { + "name": "steps" + }, + "link": null + }, + { + "name": "cfg", + "type": "FLOAT", + "widget": { + "name": "cfg" + }, + "link": null + }, + { + "name": "seed", + "type": "INT", + "widget": { + "name": "seed" + }, + "link": null + }, + { + "name": "unet_name", + "type": "COMBO", + "widget": { + "name": "unet_name" + }, + "link": null + }, + { + "name": "clip_name", + "type": "COMBO", + "widget": { + "name": "clip_name" + }, + "link": null + }, + { + "name": "vae_name", + "type": "COMBO", + "widget": { + "name": "vae_name" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [] + } + ], + "title": "Text to Image (Anima Base 1.0)", + "properties": { + "proxyWidgets": [ + [ + "11", + "text" + ], + [ + "28", + "width" + ], + [ + "28", + "height" + ], + [ + "19", + "steps" + ], + [ + "19", + "cfg" + ], + [ + "19", + "seed" + ], + [ + "44", + "unet_name" + ], + [ + "45", + "clip_name" + ], + [ + "15", + "vae_name" + ] + ], + "cnr_id": "comfy-core", + "ver": "0.18.1", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [] + } + ], + "links": [], + "version": 0.4, + "definitions": { + "subgraphs": [ + { + "id": "a3c0dab6-b250-4585-a0f9-8fb8b074fb2f", + "version": 1, + "state": { + "lastGroupId": 3, + "lastNodeId": 70, + "lastLinkId": 104, + "lastRerouteId": 0 + }, + "revision": 0, + "config": {}, + "name": "Text to Image (Anima Base 1.0)", + "inputNode": { + "id": -10, + "bounding": [ + -330, + 530, + 120, + 220 + ] + }, + "outputNode": { + "id": -20, + "bounding": [ + 1229.9999873482075, + 505, + 120, + 60 + ] + }, + "inputs": [ + { + "id": "4693f350-6ba0-446d-80d4-3038c661d26c", + "name": "text", + "type": "STRING", + "linkIds": [ + 95 + ], + "label": "prompt", + "pos": [ + -230, + 550 + ] + }, + { + "id": "4a7886a9-4ed7-49bb-afc2-977bb78a303d", + "name": "width", + "type": "INT", + "linkIds": [ + 96 + ], + "pos": [ + -230, + 570 + ] + }, + { + "id": "f6c04461-d29e-49e3-8790-07bb662bbbfe", + "name": "height", + "type": "INT", + "linkIds": [ + 97 + ], + "pos": [ + -230, + 590 + ] + }, + { + "id": "7a24f998-3808-4837-8bff-52304ad09fb6", + "name": "steps", + "type": "INT", + "linkIds": [ + 98 + ], + "pos": [ + -230, + 610 + ] + }, + { + "id": "aaa99698-b222-40fe-b946-28067528a85c", + "name": "cfg", + "type": "FLOAT", + "linkIds": [ + 99 + ], + "pos": [ + -230, + 630 + ] + }, + { + "id": "053df9ae-7311-4816-aa23-7fa13c656ced", + "name": "seed", + "type": "INT", + "linkIds": [ + 100 + ], + "pos": [ + -230, + 650 + ] + }, + { + "id": "c59194ea-015c-41a7-8edd-ae7ffc220b63", + "name": "unet_name", + "type": "COMBO", + "linkIds": [ + 101 + ], + "pos": [ + -230, + 670 + ] + }, + { + "id": "e655aa3b-2db7-4e25-9ea2-61550fa7ae2d", + "name": "clip_name", + "type": "COMBO", + "linkIds": [ + 102 + ], + "pos": [ + -230, + 690 + ] + }, + { + "id": "94965a7a-74dd-4f5a-87e3-9f87995d554f", + "name": "vae_name", + "type": "COMBO", + "linkIds": [ + 103 + ], + "pos": [ + -230, + 710 + ] + } + ], + "outputs": [ + { + "id": "ef85ac0a-2152-4232-bfa1-929cfc913718", + "name": "IMAGE", + "type": "IMAGE", + "linkIds": [ + 82 + ], + "localized_name": "IMAGE", + "pos": [ + 1249.9999873482075, + 525 + ] + } + ], + "widgets": [], + "nodes": [ + { + "id": 45, + "type": "CLIPLoader", + "pos": [ + -60, + 380 + ], + "size": [ + 310, + 150 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "localized_name": "clip_name", + "name": "clip_name", + "type": "COMBO", + "widget": { + "name": "clip_name" + }, + "link": 102 + }, + { + "localized_name": "type", + "name": "type", + "type": "COMBO", + "widget": { + "name": "type" + }, + "link": null + }, + { + "localized_name": "device", + "name": "device", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "device" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "CLIP", + "name": "CLIP", + "type": "CLIP", + "links": [ + 80, + 81 + ] + } + ], + "properties": { + "Node name for S&R": "CLIPLoader", + "cnr_id": "comfy-core", + "ver": "0.11.0", + "models": [ + { + "name": "qwen_3_06b_base.safetensors", + "url": "https://huggingface.co/circlestone-labs/Anima/resolve/main/split_files/text_encoders/qwen_3_06b_base.safetensors", + "directory": "text_encoders" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "qwen_3_06b_base.safetensors", + "stable_diffusion", + "default" + ] + }, + { + "id": 15, + "type": "VAELoader", + "pos": [ + -50, + 610 + ], + "size": [ + 310, + 100 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "localized_name": "vae_name", + "name": "vae_name", + "type": "COMBO", + "widget": { + "name": "vae_name" + }, + "link": 103 + } + ], + "outputs": [ + { + "localized_name": "VAE", + "name": "VAE", + "type": "VAE", + "links": [ + 11 + ] + } + ], + "properties": { + "Node name for S&R": "VAELoader", + "cnr_id": "comfy-core", + "ver": "0.3.40", + "models": [ + { + "name": "qwen_image_vae.safetensors", + "url": "https://huggingface.co/circlestone-labs/Anima/resolve/main/split_files/vae/qwen_image_vae.safetensors", + "directory": "vae" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "qwen_image_vae.safetensors" + ] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [ + 880, + 840 + ], + "size": [ + 230, + 90 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [ + { + "localized_name": "samples", + "name": "samples", + "type": "LATENT", + "link": 10 + }, + { + "localized_name": "vae", + "name": "vae", + "type": "VAE", + "link": 11 + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 82 + ] + } + ], + "properties": { + "Node name for S&R": "VAEDecode", + "cnr_id": "comfy-core", + "ver": "0.3.40", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + } + }, + { + "id": 28, + "type": "EmptyLatentImage", + "pos": [ + -50, + 830 + ], + "size": [ + 310, + 150 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": 96 + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": 97 + }, + { + "localized_name": "batch_size", + "name": "batch_size", + "type": "INT", + "widget": { + "name": "batch_size" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "LATENT", + "name": "LATENT", + "type": "LATENT", + "links": [ + 78 + ] + } + ], + "properties": { + "Node name for S&R": "EmptyLatentImage", + "cnr_id": "comfy-core", + "ver": "0.3.40", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + 1024, + 1024, + 1 + ] + }, + { + "id": 12, + "type": "CLIPTextEncode", + "pos": [ + 330, + 830 + ], + "size": [ + 490, + 140 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "localized_name": "clip", + "name": "clip", + "type": "CLIP", + "link": 81 + }, + { + "localized_name": "text", + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "CONDITIONING", + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 40 + ] + } + ], + "title": "CLIP Text Encode (Negative Prompt)", + "properties": { + "Node name for S&R": "CLIPTextEncode", + "cnr_id": "comfy-core", + "ver": "0.3.65", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "worst quality, low quality, score_1, score_2, score_3, blurry, jpeg artifacts, sepia" + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 19, + "type": "KSampler", + "pos": [ + 870, + 120 + ], + "size": [ + 300, + 620 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 79 + }, + { + "localized_name": "positive", + "name": "positive", + "type": "CONDITIONING", + "link": 39 + }, + { + "localized_name": "negative", + "name": "negative", + "type": "CONDITIONING", + "link": 40 + }, + { + "localized_name": "latent_image", + "name": "latent_image", + "type": "LATENT", + "link": 78 + }, + { + "localized_name": "seed", + "name": "seed", + "type": "INT", + "widget": { + "name": "seed" + }, + "link": 100 + }, + { + "localized_name": "steps", + "name": "steps", + "type": "INT", + "widget": { + "name": "steps" + }, + "link": 98 + }, + { + "localized_name": "cfg", + "name": "cfg", + "type": "FLOAT", + "widget": { + "name": "cfg" + }, + "link": 99 + }, + { + "localized_name": "sampler_name", + "name": "sampler_name", + "type": "COMBO", + "widget": { + "name": "sampler_name" + }, + "link": null + }, + { + "localized_name": "scheduler", + "name": "scheduler", + "type": "COMBO", + "widget": { + "name": "scheduler" + }, + "link": null + }, + { + "localized_name": "denoise", + "name": "denoise", + "type": "FLOAT", + "widget": { + "name": "denoise" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "LATENT", + "name": "LATENT", + "type": "LATENT", + "slot_index": 0, + "links": [ + 10 + ] + } + ], + "properties": { + "Node name for S&R": "KSampler", + "cnr_id": "comfy-core", + "ver": "0.3.40", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + 875817230929465, + "fixed", + 30, + 4, + "er_sde", + "simple", + 1 + ] + }, + { + "id": 11, + "type": "CLIPTextEncode", + "pos": [ + 320, + 170 + ], + "size": [ + 490, + 610 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "localized_name": "clip", + "name": "clip", + "type": "CLIP", + "link": 80 + }, + { + "localized_name": "text", + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": 95 + } + ], + "outputs": [ + { + "localized_name": "CONDITIONING", + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 39 + ] + } + ], + "title": "CLIP Text Encode (Positive Prompt)", + "properties": { + "Node name for S&R": "CLIPTextEncode", + "cnr_id": "comfy-core", + "ver": "0.3.65", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "" + ], + "color": "#232", + "bgcolor": "#353" + }, + { + "id": 44, + "type": "UNETLoader", + "pos": [ + -50, + 170 + ], + "size": [ + 310, + 130 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "localized_name": "unet_name", + "name": "unet_name", + "type": "COMBO", + "widget": { + "name": "unet_name" + }, + "link": 101 + }, + { + "localized_name": "weight_dtype", + "name": "weight_dtype", + "type": "COMBO", + "widget": { + "name": "weight_dtype" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MODEL", + "name": "MODEL", + "type": "MODEL", + "links": [ + 79 + ] + } + ], + "properties": { + "Node name for S&R": "UNETLoader", + "cnr_id": "comfy-core", + "ver": "0.11.0", + "models": [ + { + "name": "anima-base-v1.0.safetensors", + "url": "https://huggingface.co/circlestone-labs/Anima/resolve/main/split_files/diffusion_models/anima-base-v1.0.safetensors", + "directory": "diffusion_models" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "anima-base-v1.0.safetensors", + "default" + ] + } + ], + "groups": [ + { + "id": 1, + "title": "Model", + "bounding": [ + -80, + 80, + 360, + 640 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 2, + "title": "Image Size(1MP)", + "bounding": [ + -80, + 750, + 360, + 240 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 3, + "title": "Prompt", + "bounding": [ + 300, + 80, + 530, + 910 + ], + "color": "#3f789e", + "flags": {} + } + ], + "links": [ + { + "id": 10, + "origin_id": 19, + "origin_slot": 0, + "target_id": 8, + "target_slot": 0, + "type": "LATENT" + }, + { + "id": 11, + "origin_id": 15, + "origin_slot": 0, + "target_id": 8, + "target_slot": 1, + "type": "VAE" + }, + { + "id": 81, + "origin_id": 45, + "origin_slot": 0, + "target_id": 12, + "target_slot": 0, + "type": "CLIP" + }, + { + "id": 79, + "origin_id": 44, + "origin_slot": 0, + "target_id": 19, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 39, + "origin_id": 11, + "origin_slot": 0, + "target_id": 19, + "target_slot": 1, + "type": "CONDITIONING" + }, + { + "id": 40, + "origin_id": 12, + "origin_slot": 0, + "target_id": 19, + "target_slot": 2, + "type": "CONDITIONING" + }, + { + "id": 78, + "origin_id": 28, + "origin_slot": 0, + "target_id": 19, + "target_slot": 3, + "type": "LATENT" + }, + { + "id": 80, + "origin_id": 45, + "origin_slot": 0, + "target_id": 11, + "target_slot": 0, + "type": "CLIP" + }, + { + "id": 82, + "origin_id": 8, + "origin_slot": 0, + "target_id": -20, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 95, + "origin_id": -10, + "origin_slot": 0, + "target_id": 11, + "target_slot": 1, + "type": "STRING" + }, + { + "id": 96, + "origin_id": -10, + "origin_slot": 1, + "target_id": 28, + "target_slot": 0, + "type": "INT" + }, + { + "id": 97, + "origin_id": -10, + "origin_slot": 2, + "target_id": 28, + "target_slot": 1, + "type": "INT" + }, + { + "id": 98, + "origin_id": -10, + "origin_slot": 3, + "target_id": 19, + "target_slot": 5, + "type": "INT" + }, + { + "id": 99, + "origin_id": -10, + "origin_slot": 4, + "target_id": 19, + "target_slot": 6, + "type": "FLOAT" + }, + { + "id": 100, + "origin_id": -10, + "origin_slot": 5, + "target_id": 19, + "target_slot": 4, + "type": "INT" + }, + { + "id": 101, + "origin_id": -10, + "origin_slot": 6, + "target_id": 44, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 102, + "origin_id": -10, + "origin_slot": 7, + "target_id": 45, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 103, + "origin_id": -10, + "origin_slot": 8, + "target_id": 15, + "target_slot": 0, + "type": "COMBO" + } + ], + "extra": {}, + "category": "Image generation and editing/Text to image", + "description": "This subgraph generates non-photorealistic illustrations from text prompts using a 2-billion-parameter model optimized for anime concepts, characters, and styles. It is ideal for creating artistic images, concept art, or stylized illustrations where photorealism is not required. The model excels with anime and artistic content but performs poorly on realistic subjects." + } + ] + }, + "extra": { + "BlueprintDescription": "This subgraph generates non-photorealistic illustrations from text prompts using a 2-billion-parameter model optimized for anime concepts, characters, and styles. It is ideal for creating artistic images, concept art, or stylized illustrations where photorealism is not required. The model excels with anime and artistic content but performs poorly on realistic subjects." + } +} \ No newline at end of file diff --git a/blueprints/Text to Image (Anima).json b/blueprints/Text to Image (Anima).json index 787908ca9..dcf6e5973 100644 --- a/blueprints/Text to Image (Anima).json +++ b/blueprints/Text to Image (Anima).json @@ -1077,9 +1077,12 @@ } ], "extra": {}, - "category": "Image generation and editing/Text to image" + "category": "Image generation and editing/Text to image", + "description": "This subgraph converts text prompts into non-photorealistic illustrations using a 2-billion-parameter model optimized for anime and artistic styles. It is ideal for generating concept art, character designs, or stylized illustrations where photorealism is not required. The model excels with anime and artistic content but performs poorly on realistic subjects." } ] }, - "extra": {} + "extra": { + "BlueprintDescription": "This subgraph converts text prompts into non-photorealistic illustrations using a 2-billion-parameter model optimized for anime and artistic styles. It is ideal for generating concept art, character designs, or stylized illustrations where photorealism is not required. The model excels with anime and artistic content but performs poorly on realistic subjects." + } } \ No newline at end of file diff --git a/blueprints/Text to Image (Ideogram v4).json b/blueprints/Text to Image (Ideogram v4).json new file mode 100644 index 000000000..0d5c1d7c7 --- /dev/null +++ b/blueprints/Text to Image (Ideogram v4).json @@ -0,0 +1,2473 @@ +{ + "revision": 0, + "last_node_id": 204, + "last_link_id": 0, + "nodes": [ + { + "id": 204, + "type": "3aa9dcf6-e101-4b91-abee-6d7a7c10023b", + "pos": [ + 6380, + 1410 + ], + "size": [ + 440, + 690 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "label": "prompt", + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + }, + { + "label": "width", + "name": "value", + "type": "INT", + "widget": { + "name": "value" + }, + "link": null + }, + { + "label": "height", + "name": "value_1", + "type": "INT", + "widget": { + "name": "value_1" + }, + "link": null + }, + { + "name": "noise_seed", + "type": "INT", + "widget": { + "name": "noise_seed" + }, + "link": null + }, + { + "name": "unet_name", + "type": "COMBO", + "widget": { + "name": "unet_name" + }, + "link": null + }, + { + "name": "clip_name", + "type": "COMBO", + "widget": { + "name": "clip_name" + }, + "link": null + }, + { + "name": "vae_name", + "type": "COMBO", + "widget": { + "name": "vae_name" + }, + "link": null + }, + { + "label": "unconditional_unet", + "name": "unet_name_1", + "type": "COMBO", + "widget": { + "name": "unet_name_1" + }, + "link": null + }, + { + "label": "mode", + "name": "choice", + "type": "COMBO", + "widget": { + "name": "choice" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [] + } + ], + "properties": { + "proxyWidgets": [ + [ + "186", + "text" + ], + [ + "188", + "value" + ], + [ + "189", + "value" + ], + [ + "184", + "noise_seed" + ], + [ + "185", + "unet_name" + ], + [ + "187", + "clip_name" + ], + [ + "177", + "vae_name" + ], + [ + "200", + "unet_name" + ], + [ + "202", + "choice" + ] + ], + "cnr_id": "comfy-core", + "ver": "0.23.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [], + "title": "Text to Image (Ideogram v4)" + } + ], + "links": [], + "version": 0.4, + "definitions": { + "subgraphs": [ + { + "id": "3aa9dcf6-e101-4b91-abee-6d7a7c10023b", + "version": 1, + "state": { + "lastGroupId": 9, + "lastNodeId": 204, + "lastLinkId": 252, + "lastRerouteId": 0 + }, + "revision": 0, + "config": {}, + "name": "Text to Image (Ideogram v4)", + "inputNode": { + "id": -10, + "bounding": [ + 3490, + 920, + 154.921875, + 228 + ] + }, + "outputNode": { + "id": -20, + "bounding": [ + 6850, + 936, + 128, + 68 + ] + }, + "inputs": [ + { + "id": "4bc742d1-7b4b-452c-90d9-0d76ebcdae76", + "name": "text", + "type": "STRING", + "linkIds": [ + 152 + ], + "label": "prompt", + "pos": [ + 3620.921875, + 944 + ] + }, + { + "id": "8d4038eb-73c7-45e9-bba1-f068f55e8d32", + "name": "value", + "type": "INT", + "linkIds": [ + 153 + ], + "label": "width", + "pos": [ + 3620.921875, + 964 + ] + }, + { + "id": "281550e6-6acf-4cbe-aec1-9eb803b4dec1", + "name": "value_1", + "type": "INT", + "linkIds": [ + 154 + ], + "label": "height", + "pos": [ + 3620.921875, + 984 + ] + }, + { + "id": "fae56884-2f1a-470b-a25f-40e7a87ef69d", + "name": "noise_seed", + "type": "INT", + "linkIds": [ + 155 + ], + "pos": [ + 3620.921875, + 1004 + ] + }, + { + "id": "3497309c-a7d7-4e28-9330-142c15881632", + "name": "unet_name", + "type": "COMBO", + "linkIds": [ + 156 + ], + "pos": [ + 3620.921875, + 1024 + ] + }, + { + "id": "e87126db-7147-465e-b129-370ed2c6cc22", + "name": "clip_name", + "type": "COMBO", + "linkIds": [ + 157 + ], + "pos": [ + 3620.921875, + 1044 + ] + }, + { + "id": "a1e6c080-b11b-4d5c-a3a8-fcf4df654cf7", + "name": "vae_name", + "type": "COMBO", + "linkIds": [ + 158 + ], + "pos": [ + 3620.921875, + 1064 + ] + }, + { + "id": "b0d16516-95de-44d9-bea8-3cd2e7c78e9a", + "name": "unet_name_1", + "type": "COMBO", + "linkIds": [ + 216 + ], + "label": "unconditional_unet", + "pos": [ + 3620.921875, + 1084 + ] + }, + { + "id": "249fd825-e6b3-489d-a341-6d8050500f5e", + "name": "choice", + "type": "COMBO", + "linkIds": [ + 219 + ], + "label": "mode", + "pos": [ + 3620.921875, + 1104 + ] + } + ], + "outputs": [ + { + "id": "b81e4f60-e543-4f02-875b-b0f1bdc274f2", + "name": "IMAGE", + "type": "IMAGE", + "linkIds": [ + 25 + ], + "localized_name": "IMAGE", + "pos": [ + 6874, + 960 + ] + } + ], + "widgets": [], + "nodes": [ + { + "id": 177, + "type": "VAELoader", + "pos": [ + 4730, + 1220 + ], + "size": [ + 470, + 110 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "localized_name": "vae_name", + "name": "vae_name", + "type": "COMBO", + "widget": { + "name": "vae_name" + }, + "link": 158 + } + ], + "outputs": [ + { + "localized_name": "VAE", + "name": "VAE", + "type": "VAE", + "links": [ + 17 + ] + } + ], + "properties": { + "Node name for S&R": "VAELoader", + "cnr_id": "comfy-core", + "ver": "0.8.2", + "models": [ + { + "name": "flux2-vae.safetensors", + "url": "https://huggingface.co/Comfy-Org/flux2-dev/resolve/main/split_files/vae/flux2-vae.safetensors", + "directory": "vae" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "flux2-vae.safetensors" + ] + }, + { + "id": 178, + "type": "ConditioningZeroOut", + "pos": [ + 5450, + 960 + ], + "size": [ + 250, + 80 + ], + "flags": { + "collapsed": false + }, + "order": 2, + "mode": 0, + "inputs": [ + { + "localized_name": "conditioning", + "name": "conditioning", + "type": "CONDITIONING", + "link": 8 + } + ], + "outputs": [ + { + "localized_name": "CONDITIONING", + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 214 + ] + } + ], + "properties": { + "Node name for S&R": "ConditioningZeroOut", + "cnr_id": "comfy-core", + "ver": "0.9.1", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + } + }, + { + "id": 179, + "type": "EmptyFlux2LatentImage", + "pos": [ + 5330, + 1180 + ], + "size": [ + 270, + 170 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": 33 + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": 36 + }, + { + "localized_name": "batch_size", + "name": "batch_size", + "type": "INT", + "widget": { + "name": "batch_size" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "LATENT", + "name": "LATENT", + "type": "LATENT", + "links": [ + 15 + ] + } + ], + "properties": { + "Node name for S&R": "EmptyFlux2LatentImage", + "cnr_id": "comfy-core", + "ver": "0.8.2", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + 1024, + 1024, + 1 + ] + }, + { + "id": 180, + "type": "SamplerCustomAdvanced", + "pos": [ + 6160, + 500 + ], + "size": [ + 290, + 170 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "localized_name": "noise", + "name": "noise", + "type": "NOISE", + "link": 11 + }, + { + "localized_name": "guider", + "name": "guider", + "type": "GUIDER", + "link": 215 + }, + { + "localized_name": "sampler", + "name": "sampler", + "type": "SAMPLER", + "link": 13 + }, + { + "localized_name": "sigmas", + "name": "sigmas", + "type": "SIGMAS", + "link": 14 + }, + { + "localized_name": "latent_image", + "name": "latent_image", + "type": "LATENT", + "link": 15 + } + ], + "outputs": [ + { + "localized_name": "output", + "name": "output", + "type": "LATENT", + "links": [ + 16 + ] + }, + { + "localized_name": "denoised_output", + "name": "denoised_output", + "type": "LATENT", + "links": [] + } + ], + "properties": { + "Node name for S&R": "SamplerCustomAdvanced", + "cnr_id": "comfy-core", + "ver": "0.8.2", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + } + }, + { + "id": 181, + "type": "VAEDecode", + "pos": [ + 6560, + 500 + ], + "size": [ + 230, + 100 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "localized_name": "samples", + "name": "samples", + "type": "LATENT", + "link": 16 + }, + { + "localized_name": "vae", + "name": "vae", + "type": "VAE", + "link": 17 + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 25 + ] + } + ], + "properties": { + "Node name for S&R": "VAEDecode", + "cnr_id": "comfy-core", + "ver": "0.8.2", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + } + }, + { + "id": 182, + "type": "KSamplerSelect", + "pos": [ + 5790, + 1100 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [ + { + "localized_name": "sampler_name", + "name": "sampler_name", + "type": "COMBO", + "widget": { + "name": "sampler_name" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "SAMPLER", + "name": "SAMPLER", + "type": "SAMPLER", + "links": [ + 13 + ] + } + ], + "properties": { + "Node name for S&R": "KSamplerSelect", + "cnr_id": "comfy-core", + "ver": "0.8.2", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "euler" + ] + }, + { + "id": 183, + "type": "Ideogram4Scheduler", + "pos": [ + 5790, + 1260 + ], + "size": [ + 270, + 240 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "localized_name": "steps", + "name": "steps", + "type": "INT", + "widget": { + "name": "steps" + }, + "link": 207 + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": 34 + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": 37 + }, + { + "localized_name": "mu", + "name": "mu", + "type": "FLOAT", + "widget": { + "name": "mu" + }, + "link": 208 + }, + { + "localized_name": "std", + "name": "std", + "type": "FLOAT", + "widget": { + "name": "std" + }, + "link": 209 + } + ], + "outputs": [ + { + "localized_name": "SIGMAS", + "name": "SIGMAS", + "type": "SIGMAS", + "links": [ + 14 + ] + } + ], + "properties": { + "Node name for S&R": "Ideogram4Scheduler", + "cnr_id": "comfy-core", + "ver": "0.23.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + 20, + 1024, + 1024, + 0.5, + 1.75 + ] + }, + { + "id": 184, + "type": "RandomNoise", + "pos": [ + 5780, + 490 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "localized_name": "noise_seed", + "name": "noise_seed", + "type": "INT", + "widget": { + "name": "noise_seed" + }, + "link": 155 + } + ], + "outputs": [ + { + "localized_name": "NOISE", + "name": "NOISE", + "type": "NOISE", + "links": [ + 11 + ] + } + ], + "properties": { + "Node name for S&R": "RandomNoise", + "cnr_id": "comfy-core", + "ver": "0.8.2", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + 885894517601261, + "randomize" + ] + }, + { + "id": 185, + "type": "UNETLoader", + "pos": [ + 4720, + 520 + ], + "size": [ + 470, + 170 + ], + "flags": {}, + "order": 8, + "mode": 0, + "showAdvanced": true, + "inputs": [ + { + "localized_name": "unet_name", + "name": "unet_name", + "type": "COMBO", + "widget": { + "name": "unet_name" + }, + "link": 156 + }, + { + "localized_name": "weight_dtype", + "name": "weight_dtype", + "type": "COMBO", + "widget": { + "name": "weight_dtype" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MODEL", + "name": "MODEL", + "type": "MODEL", + "links": [ + 222 + ] + } + ], + "properties": { + "Node name for S&R": "UNETLoader", + "cnr_id": "comfy-core", + "ver": "0.8.2", + "models": [ + { + "name": "ideogram4_fp8_scaled.safetensors", + "url": "https://huggingface.co/Comfy-Org/Ideogram-4/resolve/main/diffusion_models/ideogram4_fp8_scaled.safetensors", + "directory": "diffusion_models" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "ideogram4_fp8_scaled.safetensors", + "default" + ] + }, + { + "id": 186, + "type": "CLIPTextEncode", + "pos": [ + 5270, + 500 + ], + "size": [ + 430, + 420 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "localized_name": "clip", + "name": "clip", + "type": "CLIP", + "link": 24 + }, + { + "localized_name": "text", + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": 152 + } + ], + "outputs": [ + { + "localized_name": "CONDITIONING", + "name": "CONDITIONING", + "type": "CONDITIONING", + "slot_index": 0, + "links": [ + 8, + 213 + ] + } + ], + "title": "CLIP Text Encode (Positive Prompt)", + "properties": { + "Node name for S&R": "CLIPTextEncode", + "cnr_id": "comfy-core", + "ver": "0.8.2", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "" + ] + }, + { + "id": 187, + "type": "CLIPLoader", + "pos": [ + 4730, + 990 + ], + "size": [ + 470, + 170 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "localized_name": "clip_name", + "name": "clip_name", + "type": "COMBO", + "widget": { + "name": "clip_name" + }, + "link": 157 + }, + { + "localized_name": "type", + "name": "type", + "type": "COMBO", + "widget": { + "name": "type" + }, + "link": null + }, + { + "localized_name": "device", + "name": "device", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "device" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "CLIP", + "name": "CLIP", + "type": "CLIP", + "links": [ + 24 + ] + } + ], + "properties": { + "Node name for S&R": "CLIPLoader", + "cnr_id": "comfy-core", + "ver": "0.8.2", + "models": [ + { + "name": "qwen3vl_8b_fp8_scaled.safetensors", + "url": "https://huggingface.co/Comfy-Org/Qwen3-VL/resolve/main/text_encoders/qwen3vl_8b_fp8_scaled.safetensors", + "directory": "text_encoders" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "qwen3vl_8b_fp8_scaled.safetensors", + "ideogram4", + "default" + ] + }, + { + "id": 188, + "type": "PrimitiveInt", + "pos": [ + 4240, + 1610 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "INT", + "widget": { + "name": "value" + }, + "link": 153 + } + ], + "outputs": [ + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 32 + ] + } + ], + "title": "Int (Width)", + "properties": { + "Node name for S&R": "PrimitiveInt", + "cnr_id": "comfy-core", + "ver": "0.23.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + 1024, + "fixed" + ] + }, + { + "id": 189, + "type": "PrimitiveInt", + "pos": [ + 4250, + 1800 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "INT", + "widget": { + "name": "value" + }, + "link": 154 + } + ], + "outputs": [ + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 35 + ] + } + ], + "title": "Int (Height)", + "properties": { + "Node name for S&R": "PrimitiveInt", + "cnr_id": "comfy-core", + "ver": "0.23.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + 1024, + "fixed" + ] + }, + { + "id": 190, + "type": "ComfyMathExpression", + "pos": [ + 5340, + 1400 + ], + "size": [ + 230, + 80 + ], + "flags": { + "collapsed": true + }, + "order": 13, + "mode": 0, + "inputs": [ + { + "label": "a", + "localized_name": "values.a", + "name": "values.a", + "type": "FLOAT,INT,BOOLEAN", + "link": 32 + }, + { + "label": "b", + "localized_name": "values.b", + "name": "values.b", + "shape": 7, + "type": "FLOAT,INT,BOOLEAN", + "link": null + }, + { + "localized_name": "expression", + "name": "expression", + "type": "STRING", + "widget": { + "name": "expression" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "FLOAT", + "name": "FLOAT", + "type": "FLOAT", + "links": null + }, + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 33, + 34 + ] + }, + { + "localized_name": "BOOL", + "name": "BOOL", + "type": "BOOLEAN", + "links": null + } + ], + "properties": { + "Node name for S&R": "ComfyMathExpression", + "cnr_id": "comfy-core", + "ver": "0.23.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "max(((a + 15) // 16) * 16, 256)" + ] + }, + { + "id": 191, + "type": "ComfyMathExpression", + "pos": [ + 5350, + 1470 + ], + "size": [ + 230, + 80 + ], + "flags": { + "collapsed": true + }, + "order": 14, + "mode": 0, + "inputs": [ + { + "label": "a", + "localized_name": "values.a", + "name": "values.a", + "type": "FLOAT,INT,BOOLEAN", + "link": 35 + }, + { + "label": "b", + "localized_name": "values.b", + "name": "values.b", + "shape": 7, + "type": "FLOAT,INT,BOOLEAN", + "link": null + }, + { + "localized_name": "expression", + "name": "expression", + "type": "STRING", + "widget": { + "name": "expression" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "FLOAT", + "name": "FLOAT", + "type": "FLOAT", + "links": null + }, + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 36, + 37 + ] + }, + { + "localized_name": "BOOL", + "name": "BOOL", + "type": "BOOLEAN", + "links": null + } + ], + "properties": { + "Node name for S&R": "ComfyMathExpression", + "cnr_id": "comfy-core", + "ver": "0.23.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "max(((a + 15) // 16) * 16, 256)" + ] + }, + { + "id": 192, + "type": "ComfyNumberConvert", + "pos": [ + 5920, + 1870 + ], + "size": [ + 230, + 100 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "label": "value", + "localized_name": "value", + "name": "value", + "type": "INT,FLOAT,STRING,BOOLEAN", + "link": 195 + } + ], + "outputs": [ + { + "localized_name": "FLOAT", + "name": "FLOAT", + "type": "FLOAT", + "links": [ + 208 + ] + }, + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": null + } + ], + "properties": { + "Node name for S&R": "ComfyNumberConvert", + "cnr_id": "comfy-core", + "ver": "0.23.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + } + }, + { + "id": 193, + "type": "JsonExtractString", + "pos": [ + 5450, + 1870 + ], + "size": [ + 400, + 200 + ], + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [ + { + "localized_name": "json_string", + "name": "json_string", + "type": "STRING", + "widget": { + "name": "json_string" + }, + "link": 196 + }, + { + "localized_name": "key", + "name": "key", + "type": "STRING", + "widget": { + "name": "key" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "STRING", + "name": "STRING", + "type": "STRING", + "links": [ + 195 + ] + } + ], + "properties": { + "Node name for S&R": "JsonExtractString", + "cnr_id": "comfy-core", + "ver": "0.23.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "", + "mu" + ] + }, + { + "id": 194, + "type": "ComfyNumberConvert", + "pos": [ + 5930, + 2110 + ], + "size": [ + 230, + 100 + ], + "flags": {}, + "order": 17, + "mode": 0, + "inputs": [ + { + "label": "value", + "localized_name": "value", + "name": "value", + "type": "INT,FLOAT,STRING,BOOLEAN", + "link": 197 + } + ], + "outputs": [ + { + "localized_name": "FLOAT", + "name": "FLOAT", + "type": "FLOAT", + "links": [ + 209 + ] + }, + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": null + } + ], + "properties": { + "Node name for S&R": "ComfyNumberConvert", + "cnr_id": "comfy-core", + "ver": "0.23.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + } + }, + { + "id": 195, + "type": "JsonExtractString", + "pos": [ + 5010, + 1630 + ], + "size": [ + 410, + 470 + ], + "flags": {}, + "order": 18, + "mode": 0, + "inputs": [ + { + "localized_name": "json_string", + "name": "json_string", + "type": "STRING", + "widget": { + "name": "json_string" + }, + "link": null + }, + { + "localized_name": "key", + "name": "key", + "type": "STRING", + "widget": { + "name": "key" + }, + "link": 218 + } + ], + "outputs": [ + { + "localized_name": "STRING", + "name": "STRING", + "type": "STRING", + "links": [ + 199 + ] + } + ], + "properties": { + "Node name for S&R": "JsonExtractString", + "cnr_id": "comfy-core", + "ver": "0.23.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "{\n \"Quality\": {\n \"num_steps\": 48,\n \"mu\": 0.0,\n \"std\": 1.5,\n \"preset_id\": \"V4_QUALITY_48\"\n },\n \"Default\": {\n \"num_steps\": 20,\n \"mu\": 0.0,\n \"std\": 1.75,\n \"preset_id\": \"V4_DEFAULT_20\"\n },\n \"Turbo\": {\n \"num_steps\": 12,\n \"mu\": 0.5,\n \"std\": 1.75,\n \"preset_id\": \"V4_TURBO_12\"\n }\n}", + "Default" + ] + }, + { + "id": 196, + "type": "StringReplace", + "pos": [ + 5050, + 2150 + ], + "size": [ + 230, + 40 + ], + "flags": { + "collapsed": true + }, + "order": 19, + "mode": 0, + "inputs": [ + { + "localized_name": "string", + "name": "string", + "type": "STRING", + "widget": { + "name": "string" + }, + "link": 199 + }, + { + "localized_name": "find", + "name": "find", + "type": "STRING", + "widget": { + "name": "find" + }, + "link": null + }, + { + "localized_name": "replace", + "name": "replace", + "type": "STRING", + "widget": { + "name": "replace" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "STRING", + "name": "STRING", + "type": "STRING", + "links": [ + 196, + 200, + 201 + ] + } + ], + "properties": { + "Node name for S&R": "StringReplace", + "cnr_id": "comfy-core", + "ver": "0.23.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "", + "'", + "\"" + ] + }, + { + "id": 197, + "type": "JsonExtractString", + "pos": [ + 5460, + 1610 + ], + "size": [ + 400, + 200 + ], + "flags": {}, + "order": 20, + "mode": 0, + "inputs": [ + { + "localized_name": "json_string", + "name": "json_string", + "type": "STRING", + "widget": { + "name": "json_string" + }, + "link": 200 + }, + { + "localized_name": "key", + "name": "key", + "type": "STRING", + "widget": { + "name": "key" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "STRING", + "name": "STRING", + "type": "STRING", + "links": [ + 202 + ] + } + ], + "properties": { + "Node name for S&R": "JsonExtractString", + "cnr_id": "comfy-core", + "ver": "0.23.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "", + "num_steps" + ] + }, + { + "id": 198, + "type": "JsonExtractString", + "pos": [ + 5450, + 2110 + ], + "size": [ + 400, + 200 + ], + "flags": {}, + "order": 21, + "mode": 0, + "inputs": [ + { + "localized_name": "json_string", + "name": "json_string", + "type": "STRING", + "widget": { + "name": "json_string" + }, + "link": 201 + }, + { + "localized_name": "key", + "name": "key", + "type": "STRING", + "widget": { + "name": "key" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "STRING", + "name": "STRING", + "type": "STRING", + "links": [ + 197 + ] + } + ], + "properties": { + "Node name for S&R": "JsonExtractString", + "cnr_id": "comfy-core", + "ver": "0.23.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "", + "std" + ] + }, + { + "id": 199, + "type": "ComfyNumberConvert", + "pos": [ + 5920, + 1620 + ], + "size": [ + 230, + 100 + ], + "flags": {}, + "order": 22, + "mode": 0, + "inputs": [ + { + "label": "value", + "localized_name": "value", + "name": "value", + "type": "INT,FLOAT,STRING,BOOLEAN", + "link": 202 + } + ], + "outputs": [ + { + "localized_name": "FLOAT", + "name": "FLOAT", + "type": "FLOAT", + "links": [] + }, + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 207 + ] + } + ], + "properties": { + "Node name for S&R": "ComfyNumberConvert", + "cnr_id": "comfy-core", + "ver": "0.23.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + } + }, + { + "id": 200, + "type": "UNETLoader", + "pos": [ + 4730, + 740 + ], + "size": [ + 470, + 170 + ], + "flags": {}, + "order": 23, + "mode": 0, + "showAdvanced": true, + "inputs": [ + { + "localized_name": "unet_name", + "name": "unet_name", + "type": "COMBO", + "widget": { + "name": "unet_name" + }, + "link": 216 + }, + { + "localized_name": "weight_dtype", + "name": "weight_dtype", + "type": "COMBO", + "widget": { + "name": "weight_dtype" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MODEL", + "name": "MODEL", + "type": "MODEL", + "links": [ + 211 + ] + } + ], + "properties": { + "Node name for S&R": "UNETLoader", + "cnr_id": "comfy-core", + "ver": "0.8.2", + "models": [ + { + "name": "ideogram4_unconditional_fp8_scaled.safetensors", + "url": "https://huggingface.co/Comfy-Org/Ideogram-4/resolve/main/diffusion_models/ideogram4_unconditional_fp8_scaled.safetensors", + "directory": "diffusion_models" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "ideogram4_unconditional_fp8_scaled.safetensors", + "default" + ] + }, + { + "id": 201, + "type": "DualModelGuider", + "pos": [ + 5790, + 870 + ], + "size": [ + 270, + 180 + ], + "flags": {}, + "order": 24, + "mode": 0, + "inputs": [ + { + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 223 + }, + { + "localized_name": "positive", + "name": "positive", + "type": "CONDITIONING", + "link": 213 + }, + { + "localized_name": "model_negative", + "name": "model_negative", + "shape": 7, + "type": "MODEL", + "link": 211 + }, + { + "localized_name": "negative", + "name": "negative", + "shape": 7, + "type": "CONDITIONING", + "link": 214 + }, + { + "localized_name": "cfg", + "name": "cfg", + "type": "FLOAT", + "widget": { + "name": "cfg" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "GUIDER", + "name": "GUIDER", + "type": "GUIDER", + "links": [ + 215 + ] + } + ], + "properties": { + "Node name for S&R": "DualModelGuider", + "cnr_id": "comfy-core", + "ver": "0.23.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + 7 + ] + }, + { + "id": 202, + "type": "CustomCombo", + "pos": [ + 4720, + 1630 + ], + "size": [ + 270, + 280 + ], + "flags": {}, + "order": 25, + "mode": 0, + "inputs": [ + { + "localized_name": "choice", + "name": "choice", + "type": "COMBO", + "widget": { + "name": "choice" + }, + "link": 219 + } + ], + "outputs": [ + { + "localized_name": "STRING", + "name": "STRING", + "type": "STRING", + "links": [ + 218 + ] + }, + { + "localized_name": "INDEX", + "name": "INDEX", + "type": "INT", + "links": null + } + ], + "properties": { + "Node name for S&R": "CustomCombo", + "cnr_id": "comfy-core", + "ver": "0.23.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "Default", + 1, + "Quality", + "Default", + "Turbo", + "" + ] + }, + { + "id": 203, + "type": "CFGOverride", + "pos": [ + 5790, + 650 + ], + "size": [ + 260, + 170 + ], + "flags": {}, + "order": 26, + "mode": 0, + "inputs": [ + { + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 222 + }, + { + "localized_name": "cfg", + "name": "cfg", + "type": "FLOAT", + "widget": { + "name": "cfg" + }, + "link": null + }, + { + "localized_name": "start_percent", + "name": "start_percent", + "type": "FLOAT", + "widget": { + "name": "start_percent" + }, + "link": null + }, + { + "localized_name": "end_percent", + "name": "end_percent", + "type": "FLOAT", + "widget": { + "name": "end_percent" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MODEL", + "name": "MODEL", + "type": "MODEL", + "links": [ + 223 + ] + } + ], + "properties": { + "Node name for S&R": "CFGOverride", + "cnr_id": "comfy-core", + "ver": "0.23.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + 3, + 0.7, + 1 + ] + } + ], + "groups": [ + { + "id": 1, + "title": "Models", + "bounding": [ + 4700, + 420, + 530, + 1100 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 2, + "title": "Latent Size", + "bounding": [ + 5260, + 1080, + 450, + 440 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 3, + "title": "Sampling", + "bounding": [ + 5740, + 420, + 780, + 1100 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 4, + "title": "Prompt", + "bounding": [ + 5260, + 420, + 450, + 640 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 5, + "title": "Image Size", + "bounding": [ + 4130, + 1540, + 530, + 420 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 9, + "title": "Preset", + "bounding": [ + 4700, + 1540, + 1820, + 780 + ], + "color": "#3f789e", + "flags": {} + } + ], + "links": [ + { + "id": 8, + "origin_id": 186, + "origin_slot": 0, + "target_id": 178, + "target_slot": 0, + "type": "CONDITIONING" + }, + { + "id": 33, + "origin_id": 190, + "origin_slot": 1, + "target_id": 179, + "target_slot": 0, + "type": "INT" + }, + { + "id": 36, + "origin_id": 191, + "origin_slot": 1, + "target_id": 179, + "target_slot": 1, + "type": "INT" + }, + { + "id": 11, + "origin_id": 184, + "origin_slot": 0, + "target_id": 180, + "target_slot": 0, + "type": "NOISE" + }, + { + "id": 13, + "origin_id": 182, + "origin_slot": 0, + "target_id": 180, + "target_slot": 2, + "type": "SAMPLER" + }, + { + "id": 14, + "origin_id": 183, + "origin_slot": 0, + "target_id": 180, + "target_slot": 3, + "type": "SIGMAS" + }, + { + "id": 15, + "origin_id": 179, + "origin_slot": 0, + "target_id": 180, + "target_slot": 4, + "type": "LATENT" + }, + { + "id": 16, + "origin_id": 180, + "origin_slot": 0, + "target_id": 181, + "target_slot": 0, + "type": "LATENT" + }, + { + "id": 17, + "origin_id": 177, + "origin_slot": 0, + "target_id": 181, + "target_slot": 1, + "type": "VAE" + }, + { + "id": 34, + "origin_id": 190, + "origin_slot": 1, + "target_id": 183, + "target_slot": 1, + "type": "INT" + }, + { + "id": 37, + "origin_id": 191, + "origin_slot": 1, + "target_id": 183, + "target_slot": 2, + "type": "INT" + }, + { + "id": 24, + "origin_id": 187, + "origin_slot": 0, + "target_id": 186, + "target_slot": 0, + "type": "CLIP" + }, + { + "id": 32, + "origin_id": 188, + "origin_slot": 0, + "target_id": 190, + "target_slot": 0, + "type": "INT" + }, + { + "id": 35, + "origin_id": 189, + "origin_slot": 0, + "target_id": 191, + "target_slot": 0, + "type": "INT" + }, + { + "id": 25, + "origin_id": 181, + "origin_slot": 0, + "target_id": -20, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 152, + "origin_id": -10, + "origin_slot": 0, + "target_id": 186, + "target_slot": 1, + "type": "STRING" + }, + { + "id": 153, + "origin_id": -10, + "origin_slot": 1, + "target_id": 188, + "target_slot": 0, + "type": "INT" + }, + { + "id": 154, + "origin_id": -10, + "origin_slot": 2, + "target_id": 189, + "target_slot": 0, + "type": "INT" + }, + { + "id": 155, + "origin_id": -10, + "origin_slot": 3, + "target_id": 184, + "target_slot": 0, + "type": "INT" + }, + { + "id": 156, + "origin_id": -10, + "origin_slot": 4, + "target_id": 185, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 157, + "origin_id": -10, + "origin_slot": 5, + "target_id": 187, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 158, + "origin_id": -10, + "origin_slot": 6, + "target_id": 177, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 195, + "origin_id": 193, + "origin_slot": 0, + "target_id": 192, + "target_slot": 0, + "type": "STRING" + }, + { + "id": 196, + "origin_id": 196, + "origin_slot": 0, + "target_id": 193, + "target_slot": 0, + "type": "STRING" + }, + { + "id": 197, + "origin_id": 198, + "origin_slot": 0, + "target_id": 194, + "target_slot": 0, + "type": "STRING" + }, + { + "id": 199, + "origin_id": 195, + "origin_slot": 0, + "target_id": 196, + "target_slot": 0, + "type": "STRING" + }, + { + "id": 200, + "origin_id": 196, + "origin_slot": 0, + "target_id": 197, + "target_slot": 0, + "type": "STRING" + }, + { + "id": 201, + "origin_id": 196, + "origin_slot": 0, + "target_id": 198, + "target_slot": 0, + "type": "STRING" + }, + { + "id": 202, + "origin_id": 197, + "origin_slot": 0, + "target_id": 199, + "target_slot": 0, + "type": "STRING" + }, + { + "id": 207, + "origin_id": 199, + "origin_slot": 1, + "target_id": 183, + "target_slot": 0, + "type": "INT" + }, + { + "id": 208, + "origin_id": 192, + "origin_slot": 0, + "target_id": 183, + "target_slot": 3, + "type": "FLOAT" + }, + { + "id": 209, + "origin_id": 194, + "origin_slot": 0, + "target_id": 183, + "target_slot": 4, + "type": "FLOAT" + }, + { + "id": 211, + "origin_id": 200, + "origin_slot": 0, + "target_id": 201, + "target_slot": 2, + "type": "MODEL" + }, + { + "id": 213, + "origin_id": 186, + "origin_slot": 0, + "target_id": 201, + "target_slot": 1, + "type": "CONDITIONING" + }, + { + "id": 214, + "origin_id": 178, + "origin_slot": 0, + "target_id": 201, + "target_slot": 3, + "type": "CONDITIONING" + }, + { + "id": 215, + "origin_id": 201, + "origin_slot": 0, + "target_id": 180, + "target_slot": 1, + "type": "GUIDER" + }, + { + "id": 216, + "origin_id": -10, + "origin_slot": 7, + "target_id": 200, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 218, + "origin_id": 202, + "origin_slot": 0, + "target_id": 195, + "target_slot": 1, + "type": "STRING" + }, + { + "id": 219, + "origin_id": -10, + "origin_slot": 8, + "target_id": 202, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 222, + "origin_id": 185, + "origin_slot": 0, + "target_id": 203, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 223, + "origin_id": 203, + "origin_slot": 0, + "target_id": 201, + "target_slot": 0, + "type": "MODEL" + } + ], + "extra": {}, + "category": "Image generation and editing/Text to image", + "description": "This subgraph generates images using Ideogram v4, accepting plain text or structured JSON prompts for precise layout and style control. It suits detailed illustrations, concept art, or marketing visuals needing predictable composition and color palettes. The model uses flow-matching with asymmetric guidance, so no negative prompt is needed, but JSON prompts yield the best results." + } + ] + }, + "extra": { + "BlueprintDescription": "This subgraph generates images using Ideogram v4, accepting plain text or structured JSON prompts for precise layout and style control. It suits detailed illustrations, concept art, or marketing visuals needing predictable composition and color palettes. The model uses flow-matching with asymmetric guidance, so no negative prompt is needed, but JSON prompts yield the best results." + } +} \ No newline at end of file diff --git a/blueprints/Video Depth Estimation (Depth Anything 3).json b/blueprints/Video Depth Estimation (Depth Anything 3).json new file mode 100644 index 000000000..88084a393 --- /dev/null +++ b/blueprints/Video Depth Estimation (Depth Anything 3).json @@ -0,0 +1,825 @@ +{ + "revision": 0, + "last_node_id": 97, + "last_link_id": 0, + "nodes": [ + { + "id": 97, + "type": "253ec5ca-8333-4ddf-a036-9fc0923651b9", + "pos": [ + 410, + 500 + ], + "size": [ + 400, + 400 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "video", + "type": "VIDEO", + "link": null + }, + { + "name": "start_time", + "type": "FLOAT", + "widget": { + "name": "start_time" + }, + "link": null + }, + { + "name": "duration", + "type": "FLOAT", + "widget": { + "name": "duration" + }, + "link": null + }, + { + "name": "resolution", + "type": "INT", + "widget": { + "name": "resolution" + }, + "link": null + }, + { + "name": "resize_method", + "type": "COMBO", + "widget": { + "name": "resize_method" + }, + "link": null + }, + { + "label": "output_type", + "name": "output", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "output" + }, + "link": null + }, + { + "label": "normalization", + "name": "output.normalization", + "type": "COMBO", + "widget": { + "name": "output.normalization" + }, + "link": null + }, + { + "name": "output.apply_sky_clip", + "type": "BOOLEAN", + "widget": { + "name": "output.apply_sky_clip" + }, + "link": null + }, + { + "name": "model_name", + "type": "COMBO", + "widget": { + "name": "model_name" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [] + }, + { + "name": "audio", + "type": "AUDIO", + "links": [] + }, + { + "name": "fps", + "type": "FLOAT", + "links": [] + } + ], + "properties": { + "proxyWidgets": [ + [ + "96", + "start_time" + ], + [ + "96", + "duration" + ], + [ + "93", + "resolution" + ], + [ + "93", + "resize_method" + ], + [ + "92", + "output" + ], + [ + "92", + "output.normalization" + ], + [ + "92", + "output.apply_sky_clip" + ], + [ + "94", + "model_name" + ] + ], + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [], + "title": "Video Depth Estimation (Depth Anything 3)" + } + ], + "links": [], + "version": 0.4, + "definitions": { + "subgraphs": [ + { + "id": "253ec5ca-8333-4ddf-a036-9fc0923651b9", + "version": 1, + "state": { + "lastGroupId": 4, + "lastNodeId": 97, + "lastLinkId": 129, + "lastRerouteId": 0 + }, + "revision": 2, + "config": {}, + "name": "Video Depth Estimation (Depth Anything 3)", + "inputNode": { + "id": -10, + "bounding": [ + -230, + 130, + 167.912109375, + 228 + ] + }, + "outputNode": { + "id": -20, + "bounding": [ + 1520, + 140, + 128, + 108 + ] + }, + "inputs": [ + { + "id": "698c28c6-cf92-4039-8b39-f3062868ea7c", + "name": "video", + "type": "VIDEO", + "linkIds": [ + 119 + ], + "pos": [ + -86.087890625, + 154 + ] + }, + { + "id": "97a1f63e-1585-4a40-9dec-e2700120d84a", + "name": "start_time", + "type": "FLOAT", + "linkIds": [ + 121 + ], + "pos": [ + -86.087890625, + 174 + ] + }, + { + "id": "4dbbd3b3-c5ee-4a56-a0d3-3268d3b2fd64", + "name": "duration", + "type": "FLOAT", + "linkIds": [ + 122 + ], + "pos": [ + -86.087890625, + 194 + ] + }, + { + "id": "16f55101-f99d-4c0c-bebf-c3b31c54f13e", + "name": "resolution", + "type": "INT", + "linkIds": [ + 124 + ], + "pos": [ + -86.087890625, + 214 + ] + }, + { + "id": "d9cd7693-4bb3-4ed7-9a75-276b997abcd9", + "name": "resize_method", + "type": "COMBO", + "linkIds": [ + 125 + ], + "pos": [ + -86.087890625, + 234 + ] + }, + { + "id": "a6e90532-323b-462e-ba9c-1672384d5b31", + "name": "output", + "type": "COMFY_DYNAMICCOMBO_V3", + "linkIds": [ + 126 + ], + "label": "output_type", + "pos": [ + -86.087890625, + 254 + ] + }, + { + "id": "69e6aeef-437d-4fde-b2fc-d5ab9369238d", + "name": "output.normalization", + "type": "COMBO", + "linkIds": [ + 127 + ], + "label": "normalization", + "pos": [ + -86.087890625, + 274 + ] + }, + { + "id": "73206f72-f89a-4698-885e-5d9277df2998", + "name": "output.apply_sky_clip", + "type": "BOOLEAN", + "linkIds": [ + 128 + ], + "pos": [ + -86.087890625, + 294 + ] + }, + { + "id": "dddbc7fc-9431-448a-9ed3-9aa62404288b", + "name": "model_name", + "type": "COMBO", + "linkIds": [ + 129 + ], + "pos": [ + -86.087890625, + 314 + ] + } + ], + "outputs": [ + { + "id": "478ab537-63bc-4d74-a9f0-c975f550880f", + "name": "IMAGE", + "type": "IMAGE", + "linkIds": [ + 7 + ], + "localized_name": "IMAGE", + "pos": [ + 1544, + 164 + ] + }, + { + "id": "cdaf037e-79bc-4a94-b06c-0fd32e76f615", + "name": "audio", + "type": "AUDIO", + "linkIds": [ + 112 + ], + "pos": [ + 1544, + 184 + ] + }, + { + "id": "4c0e5484-d193-49c7-b107-92619628880a", + "name": "fps", + "type": "FLOAT", + "linkIds": [ + 113 + ], + "pos": [ + 1544, + 204 + ] + } + ], + "widgets": [], + "nodes": [ + { + "id": 92, + "type": "DA3Render", + "pos": [ + 740, + 230 + ], + "size": [ + 380, + 130 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [ + { + "localized_name": "da3_geometry", + "name": "da3_geometry", + "type": "DA3_GEOMETRY", + "link": 12 + }, + { + "localized_name": "output", + "name": "output", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "output" + }, + "link": 126 + }, + { + "localized_name": "output.normalization", + "name": "output.normalization", + "type": "COMBO", + "widget": { + "name": "output.normalization" + }, + "link": 127 + }, + { + "localized_name": "output.apply_sky_clip", + "name": "output.apply_sky_clip", + "type": "BOOLEAN", + "widget": { + "name": "output.apply_sky_clip" + }, + "link": 128 + }, + { + "name": "geometry", + "type": "DA3_GEOMETRY", + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 7 + ] + } + ], + "properties": { + "Node name for S&R": "DA3Render", + "cnr_id": "comfy-core", + "ver": "0.19.0" + }, + "widgets_values": [ + "depth", + "v2_style", + false + ] + }, + { + "id": 93, + "type": "DA3Inference", + "pos": [ + 740, + -30 + ], + "size": [ + 390, + 130 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "localized_name": "da3_model", + "name": "da3_model", + "type": "DA3_MODEL", + "link": 107 + }, + { + "localized_name": "image", + "name": "image", + "type": "IMAGE", + "link": 111 + }, + { + "localized_name": "resolution", + "name": "resolution", + "type": "INT", + "widget": { + "name": "resolution" + }, + "link": 124 + }, + { + "localized_name": "resize_method", + "name": "resize_method", + "type": "COMBO", + "widget": { + "name": "resize_method" + }, + "link": 125 + }, + { + "localized_name": "mode", + "name": "mode", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "mode" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "da3_geometry", + "name": "da3_geometry", + "type": "DA3_GEOMETRY", + "slot_index": 0, + "links": [ + 12 + ] + } + ], + "properties": { + "Node name for S&R": "DA3Inference", + "cnr_id": "comfy-core", + "ver": "0.19.0" + }, + "widgets_values": [ + 504, + "lower_bound_resize", + "mono" + ] + }, + { + "id": 94, + "type": "LoadDA3Model", + "pos": [ + 50, + 410 + ], + "size": [ + 400, + 140 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "localized_name": "model_name", + "name": "model_name", + "type": "COMBO", + "widget": { + "name": "model_name" + }, + "link": 129 + }, + { + "localized_name": "weight_dtype", + "name": "weight_dtype", + "type": "COMBO", + "widget": { + "name": "weight_dtype" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "DA3_MODEL", + "name": "DA3_MODEL", + "type": "DA3_MODEL", + "links": [ + 107 + ] + } + ], + "properties": { + "Node name for S&R": "LoadDA3Model", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "models": [ + { + "name": "depth_anything_3_mono_large.safetensors", + "url": "https://huggingface.co/Comfy-Org/Depth-Anything-3/resolve/main/geometry_estimation/depth_anything_3_mono_large.safetensors", + "directory": "geometry_estimation" + } + ] + }, + "widgets_values": [ + "depth_anything_3_mono_large.safetensors", + "default" + ] + }, + { + "id": 95, + "type": "GetVideoComponents", + "pos": [ + 70, + -140 + ], + "size": [ + 260, + 120 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 120 + } + ], + "outputs": [ + { + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "links": [ + 111 + ] + }, + { + "localized_name": "audio", + "name": "audio", + "type": "AUDIO", + "links": [ + 112 + ] + }, + { + "localized_name": "fps", + "name": "fps", + "type": "FLOAT", + "links": [ + 113 + ] + }, + { + "localized_name": "bit_depth", + "name": "bit_depth", + "type": "INT", + "links": null + } + ], + "properties": { + "Node name for S&R": "GetVideoComponents", + "cnr_id": "comfy-core", + "ver": "0.24.0" + } + }, + { + "id": 96, + "type": "Video Slice", + "pos": [ + 70, + -360 + ], + "size": [ + 270, + 170 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 119 + }, + { + "localized_name": "start_time", + "name": "start_time", + "type": "FLOAT", + "widget": { + "name": "start_time" + }, + "link": 121 + }, + { + "localized_name": "duration", + "name": "duration", + "type": "FLOAT", + "widget": { + "name": "duration" + }, + "link": 122 + }, + { + "localized_name": "strict_duration", + "name": "strict_duration", + "type": "BOOLEAN", + "widget": { + "name": "strict_duration" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "VIDEO", + "name": "VIDEO", + "type": "VIDEO", + "links": [ + 120 + ] + } + ], + "properties": { + "Node name for S&R": "Video Slice", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 0, + 5, + false + ] + } + ], + "groups": [], + "links": [ + { + "id": 12, + "origin_id": 93, + "origin_slot": 0, + "target_id": 92, + "target_slot": 0, + "type": "DA3_GEOMETRY" + }, + { + "id": 7, + "origin_id": 92, + "origin_slot": 0, + "target_id": -20, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 107, + "origin_id": 94, + "origin_slot": 0, + "target_id": 93, + "target_slot": 0, + "type": "DA3_MODEL" + }, + { + "id": 111, + "origin_id": 95, + "origin_slot": 0, + "target_id": 93, + "target_slot": 1, + "type": "IMAGE" + }, + { + "id": 112, + "origin_id": 95, + "origin_slot": 1, + "target_id": -20, + "target_slot": 1, + "type": "AUDIO" + }, + { + "id": 113, + "origin_id": 95, + "origin_slot": 2, + "target_id": -20, + "target_slot": 2, + "type": "FLOAT" + }, + { + "id": 119, + "origin_id": -10, + "origin_slot": 0, + "target_id": 96, + "target_slot": 0, + "type": "VIDEO" + }, + { + "id": 120, + "origin_id": 96, + "origin_slot": 0, + "target_id": 95, + "target_slot": 0, + "type": "VIDEO" + }, + { + "id": 121, + "origin_id": -10, + "origin_slot": 1, + "target_id": 96, + "target_slot": 1, + "type": "FLOAT" + }, + { + "id": 122, + "origin_id": -10, + "origin_slot": 2, + "target_id": 96, + "target_slot": 2, + "type": "FLOAT" + }, + { + "id": 124, + "origin_id": -10, + "origin_slot": 3, + "target_id": 93, + "target_slot": 2, + "type": "INT" + }, + { + "id": 125, + "origin_id": -10, + "origin_slot": 4, + "target_id": 93, + "target_slot": 3, + "type": "COMBO" + }, + { + "id": 126, + "origin_id": -10, + "origin_slot": 5, + "target_id": 92, + "target_slot": 1, + "type": "COMFY_DYNAMICCOMBO_V3" + }, + { + "id": 127, + "origin_id": -10, + "origin_slot": 6, + "target_id": 92, + "target_slot": 2, + "type": "COMBO" + }, + { + "id": 128, + "origin_id": -10, + "origin_slot": 7, + "target_id": 92, + "target_slot": 3, + "type": "BOOLEAN" + }, + { + "id": 129, + "origin_id": -10, + "origin_slot": 8, + "target_id": 94, + "target_slot": 0, + "type": "COMBO" + } + ], + "extra": {}, + "category": "Conditioning & Preprocessors/Depth", + "description": "This subgraph processes a video input through Depth Anything 3 to produce temporally consistent depth maps for each frame, outputting a depth video. It is ideal for video content requiring spatial geometry estimation, such as 3D reconstruction, SLAM, or novel view synthesis from moving cameras. The model uses a plain transformer backbone trained with a depth-ray representation, supporting any number of views without requiring known camera poses." + } + ] + }, + "extra": { + "BlueprintDescription": "This subgraph processes a video input through Depth Anything 3 to produce temporally consistent depth maps for each frame, outputting a depth video. It is ideal for video content requiring spatial geometry estimation, such as 3D reconstruction, SLAM, or novel view synthesis from moving cameras. The model uses a plain transformer backbone trained with a depth-ray representation, supporting any number of views without requiring known camera poses." + } +} \ No newline at end of file diff --git a/blueprints/Video Edit (Bernini-R).json b/blueprints/Video Edit (Bernini-R).json new file mode 100644 index 000000000..2f09db5fa --- /dev/null +++ b/blueprints/Video Edit (Bernini-R).json @@ -0,0 +1,3732 @@ +{ + "revision": 0, + "last_node_id": 376, + "last_link_id": 0, + "nodes": [ + { + "id": 376, + "type": "70d8911e-8530-4a3d-9889-b39e8fbd131b", + "pos": [ + 4090, + 4890 + ], + "size": [ + 480, + 740 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "label": "source_video", + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": null + }, + { + "name": "reference_video", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "label": "reference_images", + "name": "reference_images.reference_image_0", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "value", + "type": "STRING", + "widget": { + "name": "value" + }, + "link": null + }, + { + "label": "task_type", + "name": "choice", + "type": "COMBO", + "widget": { + "name": "choice" + }, + "link": null + }, + { + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + }, + { + "label": "noise_seed", + "name": "noise_seed", + "type": "INT", + "widget": { + "name": "noise_seed" + }, + "link": null + }, + { + "name": "length", + "type": "INT", + "widget": { + "name": "length" + }, + "link": null + }, + { + "name": "ref_max_size", + "shape": 7, + "type": "INT", + "widget": { + "name": "ref_max_size" + }, + "link": null + }, + { + "label": "turbo_mode", + "name": "value_1", + "type": "BOOLEAN", + "widget": { + "name": "value_1" + }, + "link": null + }, + { + "label": "high_noise_model", + "name": "unet_name", + "type": "COMBO", + "widget": { + "name": "unet_name" + }, + "link": null + }, + { + "label": "low_noise_model", + "name": "unet_name_1", + "type": "COMBO", + "widget": { + "name": "unet_name_1" + }, + "link": null + }, + { + "label": "distill_lora", + "name": "lora_name", + "type": "COMBO", + "widget": { + "name": "lora_name" + }, + "link": null + }, + { + "name": "clip_name", + "type": "COMBO", + "widget": { + "name": "clip_name" + }, + "link": null + }, + { + "name": "vae_name", + "type": "COMBO", + "widget": { + "name": "vae_name" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "VIDEO", + "name": "VIDEO", + "type": "VIDEO", + "links": [] + } + ], + "properties": { + "proxyWidgets": [ + [ + "371", + "value" + ], + [ + "356", + "choice" + ], + [ + "352", + "width" + ], + [ + "352", + "height" + ], + [ + "349", + "noise_seed" + ], + [ + "352", + "length" + ], + [ + "352", + "ref_max_size" + ], + [ + "368", + "value" + ], + [ + "344", + "unet_name" + ], + [ + "346", + "unet_name" + ], + [ + "345", + "lora_name" + ], + [ + "338", + "clip_name" + ], + [ + "339", + "vae_name" + ] + ], + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [], + "title": "Video Edit (Bernini-R)" + } + ], + "links": [], + "version": 0.4, + "definitions": { + "subgraphs": [ + { + "id": "70d8911e-8530-4a3d-9889-b39e8fbd131b", + "version": 1, + "state": { + "lastGroupId": 9, + "lastNodeId": 376, + "lastLinkId": 496, + "lastRerouteId": 0 + }, + "revision": 0, + "config": {}, + "name": "Video Edit (Bernini-R)", + "inputNode": { + "id": -10, + "bounding": [ + 5240, + 5360, + 149.689453125, + 368 + ] + }, + "outputNode": { + "id": -20, + "bounding": [ + 10004.984375, + 5000.9921875, + 128, + 68 + ] + }, + "inputs": [ + { + "id": "15c3bfa3-6844-473a-a927-a50284131356", + "name": "video", + "type": "VIDEO", + "linkIds": [ + 457 + ], + "localized_name": "video", + "label": "source_video", + "pos": [ + 5365.689453125, + 5384 + ] + }, + { + "id": "c02ac440-58ae-4415-bb24-dabb61b91f69", + "name": "reference_video", + "type": "IMAGE", + "linkIds": [ + 465 + ], + "pos": [ + 5365.689453125, + 5404 + ] + }, + { + "id": "c46d38f5-b0e8-4620-bd0e-4e86db7bdb1b", + "name": "reference_images.reference_image_0", + "type": "IMAGE", + "linkIds": [ + 466 + ], + "label": "reference_images", + "pos": [ + 5365.689453125, + 5424 + ] + }, + { + "id": "565fc711-6052-4c07-b638-403e01fcf7f8", + "name": "value", + "type": "STRING", + "linkIds": [ + 467 + ], + "pos": [ + 5365.689453125, + 5444 + ] + }, + { + "id": "bfcd6148-0bae-43b1-9440-a133fbc2663c", + "name": "choice", + "type": "COMBO", + "linkIds": [ + 468 + ], + "label": "task_type", + "pos": [ + 5365.689453125, + 5464 + ] + }, + { + "id": "28fedb03-828c-49d9-91ec-d3669a10c3b9", + "name": "width", + "type": "INT", + "linkIds": [ + 469 + ], + "pos": [ + 5365.689453125, + 5484 + ] + }, + { + "id": "69994b05-577e-486f-bd10-3360261d7bb8", + "name": "height", + "type": "INT", + "linkIds": [ + 470 + ], + "pos": [ + 5365.689453125, + 5504 + ] + }, + { + "id": "db300c09-5b92-41ad-990b-9dd8dad35f86", + "name": "noise_seed", + "type": "INT", + "linkIds": [ + 480 + ], + "label": "noise_seed", + "pos": [ + 5365.689453125, + 5524 + ] + }, + { + "id": "db664a31-39e5-4d6a-a5f1-3abac65b35d1", + "name": "length", + "type": "INT", + "linkIds": [ + 481 + ], + "pos": [ + 5365.689453125, + 5544 + ] + }, + { + "id": "26aa0c9e-9daa-4302-ab5c-5ac9141b9e20", + "name": "ref_max_size", + "type": "INT", + "linkIds": [ + 482 + ], + "pos": [ + 5365.689453125, + 5564 + ] + }, + { + "id": "ee527a92-8cc9-4b16-9858-9daab9ef2c45", + "name": "value_1", + "type": "BOOLEAN", + "linkIds": [ + 488 + ], + "label": "turbo_mode", + "pos": [ + 5365.689453125, + 5584 + ] + }, + { + "id": "8a6c9f3d-e24e-4d40-9c03-864bd4458376", + "name": "unet_name", + "type": "COMBO", + "linkIds": [ + 489 + ], + "label": "high_noise_model", + "pos": [ + 5365.689453125, + 5604 + ] + }, + { + "id": "f35afd1c-c183-4d18-8672-314527728e9b", + "name": "unet_name_1", + "type": "COMBO", + "linkIds": [ + 490 + ], + "label": "low_noise_model", + "pos": [ + 5365.689453125, + 5624 + ] + }, + { + "id": "fff89f1d-615c-436e-b28a-3b7f915d0b05", + "name": "lora_name", + "type": "COMBO", + "linkIds": [ + 491, + 492 + ], + "label": "distill_lora", + "pos": [ + 5365.689453125, + 5644 + ] + }, + { + "id": "d76ff30d-c865-49b4-bccb-fb6e0a9b4f34", + "name": "clip_name", + "type": "COMBO", + "linkIds": [ + 493 + ], + "pos": [ + 5365.689453125, + 5664 + ] + }, + { + "id": "0850a515-4051-4de3-9343-7db929548ada", + "name": "vae_name", + "type": "COMBO", + "linkIds": [ + 494 + ], + "pos": [ + 5365.689453125, + 5684 + ] + } + ], + "outputs": [ + { + "id": "7d994238-c919-43c6-9d97-340c9e383743", + "name": "VIDEO", + "type": "VIDEO", + "linkIds": [ + 458 + ], + "localized_name": "VIDEO", + "pos": [ + 10028.984375, + 5024.9921875 + ] + } + ], + "widgets": [], + "nodes": [ + { + "id": 338, + "type": "CLIPLoader", + "pos": [ + 6170, + 5760 + ], + "size": [ + 670, + 170 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "localized_name": "clip_name", + "name": "clip_name", + "type": "COMBO", + "widget": { + "name": "clip_name" + }, + "link": 493 + }, + { + "localized_name": "type", + "name": "type", + "type": "COMBO", + "widget": { + "name": "type" + }, + "link": null + }, + { + "localized_name": "device", + "name": "device", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "device" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "CLIP", + "name": "CLIP", + "type": "CLIP", + "links": [ + 409, + 438 + ] + } + ], + "properties": { + "Node name for S&R": "CLIPLoader", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "models": [ + { + "name": "umt5_xxl_fp8_e4m3fn_scaled.safetensors", + "url": "https://huggingface.co/Comfy-Org/Wan_2.1_ComfyUI_repackaged/resolve/main/split_files/text_encoders/umt5_xxl_fp8_e4m3fn_scaled.safetensors?download=true", + "directory": "text_encoders" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "umt5_xxl_fp8_e4m3fn_scaled.safetensors", + "wan", + "default" + ] + }, + { + "id": 339, + "type": "VAELoader", + "pos": [ + 6170, + 5990 + ], + "size": [ + 670, + 110 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "localized_name": "vae_name", + "name": "vae_name", + "type": "COMBO", + "widget": { + "name": "vae_name" + }, + "link": 494 + } + ], + "outputs": [ + { + "localized_name": "VAE", + "name": "VAE", + "type": "VAE", + "links": [ + 414, + 436 + ] + } + ], + "properties": { + "Node name for S&R": "VAELoader", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "models": [ + { + "name": "Wan2_1_VAE_bf16.safetensors", + "url": "https://huggingface.co/Kijai/WanVideo_comfy/resolve/main/Wan2_1_VAE_bf16.safetensors?download=true", + "directory": "vae" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "Wan2_1_VAE_bf16.safetensors" + ] + }, + { + "id": 340, + "type": "LoraLoaderModelOnly", + "pos": [ + 6180, + 5550 + ], + "size": [ + 670, + 170 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 408 + }, + { + "localized_name": "lora_name", + "name": "lora_name", + "type": "COMBO", + "widget": { + "name": "lora_name" + }, + "link": 492 + }, + { + "localized_name": "strength_model", + "name": "strength_model", + "type": "FLOAT", + "widget": { + "name": "strength_model" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MODEL", + "name": "MODEL", + "type": "MODEL", + "links": [ + 441 + ] + } + ], + "properties": { + "Node name for S&R": "LoraLoaderModelOnly", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "models": [ + { + "name": "lightx2v_T2V_14B_cfg_step_distill_v2_lora_rank64_bf16.safetensors", + "url": "https://huggingface.co/Kijai/WanVideo_comfy/resolve/main/Lightx2v/lightx2v_T2V_14B_cfg_step_distill_v2_lora_rank64_bf16.safetensors?download=true", + "directory": "loras" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "lightx2v_T2V_14B_cfg_step_distill_v2_lora_rank64_bf16.safetensors", + 1.5 + ], + "color": "#332922", + "bgcolor": "#593930" + }, + { + "id": 341, + "type": "CLIPTextEncode", + "pos": [ + 6950, + 5400 + ], + "size": [ + 700, + 240 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "localized_name": "clip", + "name": "clip", + "type": "CLIP", + "link": 409 + }, + { + "localized_name": "text", + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "CONDITIONING", + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 435 + ] + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 342, + "type": "SplitSigmas", + "pos": [ + 8520, + 5180 + ], + "size": [ + 270, + 130 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "localized_name": "sigmas", + "name": "sigmas", + "type": "SIGMAS", + "link": 410 + }, + { + "localized_name": "step", + "name": "step", + "type": "INT", + "widget": { + "name": "step" + }, + "link": 411 + } + ], + "outputs": [ + { + "localized_name": "high_sigmas", + "name": "high_sigmas", + "type": "SIGMAS", + "links": [ + 422 + ] + }, + { + "localized_name": "low_sigmas", + "name": "low_sigmas", + "type": "SIGMAS", + "links": [ + 431 + ] + } + ], + "properties": { + "Node name for S&R": "SplitSigmas", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + 3 + ] + }, + { + "id": 343, + "type": "KSamplerSelect", + "pos": [ + 8520, + 5370 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [ + { + "localized_name": "sampler_name", + "name": "sampler_name", + "type": "COMBO", + "widget": { + "name": "sampler_name" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "SAMPLER", + "name": "SAMPLER", + "type": "SAMPLER", + "links": [ + 421, + 430 + ] + } + ], + "properties": { + "Node name for S&R": "KSamplerSelect", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "res_multistep" + ] + }, + { + "id": 344, + "type": "UNETLoader", + "pos": [ + 6170, + 4930 + ], + "size": [ + 670, + 140 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "localized_name": "unet_name", + "name": "unet_name", + "type": "COMBO", + "widget": { + "name": "unet_name" + }, + "link": 489 + }, + { + "localized_name": "weight_dtype", + "name": "weight_dtype", + "type": "COMBO", + "widget": { + "name": "weight_dtype" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MODEL", + "name": "MODEL", + "type": "MODEL", + "links": [ + 412, + 454 + ] + } + ], + "properties": { + "Node name for S&R": "UNETLoader", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "models": [ + { + "name": "wan2.2_bernini_r_high_noise_fp8_scaled.safetensors", + "url": "https://huggingface.co/Comfy-Org/Bernini-R/resolve/main/diffusion_models/wan2.2_bernini_r_high_noise_fp8_scaled.safetensors", + "directory": "diffusion_models" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "wan2.2_bernini_r_high_noise_fp8_scaled.safetensors", + "default" + ] + }, + { + "id": 345, + "type": "LoraLoaderModelOnly", + "pos": [ + 6170, + 5120 + ], + "size": [ + 670, + 170 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 412 + }, + { + "localized_name": "lora_name", + "name": "lora_name", + "type": "COMBO", + "widget": { + "name": "lora_name" + }, + "link": 491 + }, + { + "localized_name": "strength_model", + "name": "strength_model", + "type": "FLOAT", + "widget": { + "name": "strength_model" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MODEL", + "name": "MODEL", + "type": "MODEL", + "links": [ + 455 + ] + } + ], + "properties": { + "Node name for S&R": "LoraLoaderModelOnly", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "models": [ + { + "name": "lightx2v_T2V_14B_cfg_step_distill_v2_lora_rank64_bf16.safetensors", + "url": "https://huggingface.co/Kijai/WanVideo_comfy/resolve/main/Lightx2v/lightx2v_T2V_14B_cfg_step_distill_v2_lora_rank64_bf16.safetensors?download=true", + "directory": "loras" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "lightx2v_T2V_14B_cfg_step_distill_v2_lora_rank64_bf16.safetensors", + 3 + ], + "color": "#332922", + "bgcolor": "#593930" + }, + { + "id": 346, + "type": "UNETLoader", + "pos": [ + 6170, + 5350 + ], + "size": [ + 670, + 140 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "localized_name": "unet_name", + "name": "unet_name", + "type": "COMBO", + "widget": { + "name": "unet_name" + }, + "link": 490 + }, + { + "localized_name": "weight_dtype", + "name": "weight_dtype", + "type": "COMBO", + "widget": { + "name": "weight_dtype" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MODEL", + "name": "MODEL", + "type": "MODEL", + "links": [ + 408, + 425, + 440 + ] + } + ], + "properties": { + "Node name for S&R": "UNETLoader", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "models": [ + { + "name": "wan2.2_bernini_r_low_noise_fp8_scaled.safetensors", + "url": "https://huggingface.co/Comfy-Org/Bernini-R/resolve/main/diffusion_models/wan2.2_bernini_r_low_noise_fp8_scaled.safetensors", + "directory": "diffusion_models" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "wan2.2_bernini_r_low_noise_fp8_scaled.safetensors", + "default" + ] + }, + { + "id": 347, + "type": "VAEDecode", + "pos": [ + 9690, + 4950 + ], + "size": [ + 250, + 100 + ], + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [ + { + "localized_name": "samples", + "name": "samples", + "type": "LATENT", + "link": 413 + }, + { + "localized_name": "vae", + "name": "vae", + "type": "VAE", + "link": 414 + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 415 + ] + } + ], + "properties": { + "Node name for S&R": "VAEDecode", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + } + }, + { + "id": 348, + "type": "CreateVideo", + "pos": [ + 9690, + 5120 + ], + "size": [ + 260, + 160 + ], + "flags": {}, + "order": 17, + "mode": 0, + "inputs": [ + { + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "link": 415 + }, + { + "localized_name": "audio", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": 416 + }, + { + "localized_name": "fps", + "name": "fps", + "type": "FLOAT", + "widget": { + "name": "fps" + }, + "link": 417 + }, + { + "localized_name": "bit_depth", + "name": "bit_depth", + "shape": 7, + "type": "INT", + "widget": { + "name": "bit_depth" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "VIDEO", + "name": "VIDEO", + "type": "VIDEO", + "links": [ + 458 + ] + } + ], + "properties": { + "Node name for S&R": "CreateVideo", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + 24, + 8 + ] + }, + { + "id": 349, + "type": "SamplerCustom", + "pos": [ + 8860, + 4960 + ], + "size": [ + 280, + 680 + ], + "flags": {}, + "order": 18, + "mode": 0, + "showAdvanced": false, + "inputs": [ + { + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 418 + }, + { + "localized_name": "positive", + "name": "positive", + "type": "CONDITIONING", + "link": 419 + }, + { + "localized_name": "negative", + "name": "negative", + "type": "CONDITIONING", + "link": 420 + }, + { + "localized_name": "sampler", + "name": "sampler", + "type": "SAMPLER", + "link": 421 + }, + { + "localized_name": "sigmas", + "name": "sigmas", + "type": "SIGMAS", + "link": 422 + }, + { + "localized_name": "latent_image", + "name": "latent_image", + "type": "LATENT", + "link": 423 + }, + { + "localized_name": "add_noise", + "name": "add_noise", + "type": "BOOLEAN", + "widget": { + "name": "add_noise" + }, + "link": null + }, + { + "localized_name": "noise_seed", + "name": "noise_seed", + "type": "INT", + "widget": { + "name": "noise_seed" + }, + "link": 480 + }, + { + "localized_name": "cfg", + "name": "cfg", + "type": "FLOAT", + "widget": { + "name": "cfg" + }, + "link": 424 + } + ], + "outputs": [ + { + "localized_name": "output", + "name": "output", + "type": "LATENT", + "links": [ + 432 + ] + }, + { + "localized_name": "denoised_output", + "name": "denoised_output", + "type": "LATENT", + "links": [] + } + ], + "properties": { + "Node name for S&R": "SamplerCustom", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + true, + 852303761886160, + "randomize", + 1 + ] + }, + { + "id": 350, + "type": "BasicScheduler", + "pos": [ + 8520, + 4960 + ], + "size": [ + 270, + 170 + ], + "flags": {}, + "order": 19, + "mode": 0, + "inputs": [ + { + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 425 + }, + { + "localized_name": "scheduler", + "name": "scheduler", + "type": "COMBO", + "widget": { + "name": "scheduler" + }, + "link": null + }, + { + "localized_name": "steps", + "name": "steps", + "type": "INT", + "widget": { + "name": "steps" + }, + "link": 426 + }, + { + "localized_name": "denoise", + "name": "denoise", + "type": "FLOAT", + "widget": { + "name": "denoise" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "SIGMAS", + "name": "SIGMAS", + "type": "SIGMAS", + "links": [ + 410 + ] + } + ], + "properties": { + "Node name for S&R": "BasicScheduler", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "simple", + 6, + 1 + ] + }, + { + "id": 351, + "type": "SamplerCustom", + "pos": [ + 9190, + 4950 + ], + "size": [ + 280, + 680 + ], + "flags": {}, + "order": 20, + "mode": 0, + "showAdvanced": false, + "inputs": [ + { + "localized_name": "model", + "name": "model", + "type": "MODEL", + "link": 427 + }, + { + "localized_name": "positive", + "name": "positive", + "type": "CONDITIONING", + "link": 428 + }, + { + "localized_name": "negative", + "name": "negative", + "type": "CONDITIONING", + "link": 429 + }, + { + "localized_name": "sampler", + "name": "sampler", + "type": "SAMPLER", + "link": 430 + }, + { + "localized_name": "sigmas", + "name": "sigmas", + "type": "SIGMAS", + "link": 431 + }, + { + "localized_name": "latent_image", + "name": "latent_image", + "type": "LATENT", + "link": 432 + }, + { + "localized_name": "add_noise", + "name": "add_noise", + "type": "BOOLEAN", + "widget": { + "name": "add_noise" + }, + "link": null + }, + { + "localized_name": "noise_seed", + "name": "noise_seed", + "type": "INT", + "widget": { + "name": "noise_seed" + }, + "link": null + }, + { + "localized_name": "cfg", + "name": "cfg", + "type": "FLOAT", + "widget": { + "name": "cfg" + }, + "link": 433 + } + ], + "outputs": [ + { + "localized_name": "output", + "name": "output", + "type": "LATENT", + "links": [ + 413 + ] + }, + { + "localized_name": "denoised_output", + "name": "denoised_output", + "type": "LATENT", + "links": [] + } + ], + "properties": { + "Node name for S&R": "SamplerCustom", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + false, + 0, + "fixed", + 1 + ] + }, + { + "id": 352, + "type": "BerniniConditioning", + "pos": [ + 7160, + 5720 + ], + "size": [ + 310, + 380 + ], + "flags": {}, + "order": 21, + "mode": 0, + "inputs": [ + { + "localized_name": "positive", + "name": "positive", + "type": "CONDITIONING", + "link": 434 + }, + { + "localized_name": "negative", + "name": "negative", + "type": "CONDITIONING", + "link": 435 + }, + { + "localized_name": "vae", + "name": "vae", + "type": "VAE", + "link": 436 + }, + { + "localized_name": "source_video", + "name": "source_video", + "shape": 7, + "type": "IMAGE", + "link": 437 + }, + { + "localized_name": "reference_video", + "name": "reference_video", + "shape": 7, + "type": "IMAGE", + "link": 465 + }, + { + "label": "reference_image_0", + "localized_name": "reference_images.reference_image_0", + "name": "reference_images.reference_image_0", + "shape": 7, + "type": "IMAGE", + "link": 466 + }, + { + "label": "reference_image_1", + "localized_name": "reference_images.reference_image_1", + "name": "reference_images.reference_image_1", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": 469 + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": 470 + }, + { + "localized_name": "length", + "name": "length", + "type": "INT", + "widget": { + "name": "length" + }, + "link": 481 + }, + { + "localized_name": "batch_size", + "name": "batch_size", + "type": "INT", + "widget": { + "name": "batch_size" + }, + "link": null + }, + { + "localized_name": "ref_max_size", + "name": "ref_max_size", + "shape": 7, + "type": "INT", + "widget": { + "name": "ref_max_size" + }, + "link": 482 + } + ], + "outputs": [ + { + "localized_name": "positive", + "name": "positive", + "type": "CONDITIONING", + "links": [ + 419, + 428 + ] + }, + { + "localized_name": "negative", + "name": "negative", + "type": "CONDITIONING", + "links": [ + 420, + 429 + ] + }, + { + "localized_name": "latent", + "name": "latent", + "type": "LATENT", + "links": [ + 423 + ] + } + ], + "properties": { + "Node name for S&R": "BerniniConditioning", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + 480, + 832, + 81, + 1, + 848 + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 353, + "type": "GetVideoComponents", + "pos": [ + 6170, + 6220 + ], + "size": [ + 230, + 150 + ], + "flags": {}, + "order": 22, + "mode": 0, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 457 + } + ], + "outputs": [ + { + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "links": [ + 437 + ] + }, + { + "localized_name": "audio", + "name": "audio", + "type": "AUDIO", + "links": [ + 416 + ] + }, + { + "localized_name": "fps", + "name": "fps", + "type": "FLOAT", + "links": [ + 417 + ] + }, + { + "localized_name": "bit_depth", + "name": "bit_depth", + "type": "INT", + "links": [] + } + ], + "properties": { + "Node name for S&R": "GetVideoComponents", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + } + }, + { + "id": 354, + "type": "CLIPTextEncode", + "pos": [ + 6950, + 4940 + ], + "size": [ + 710, + 390 + ], + "flags": {}, + "order": 23, + "mode": 0, + "inputs": [ + { + "localized_name": "clip", + "name": "clip", + "type": "CLIP", + "link": 438 + }, + { + "localized_name": "text", + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": 439 + } + ], + "outputs": [ + { + "localized_name": "CONDITIONING", + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 434 + ] + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode", + "cnr_id": "comfy-core", + "ver": "0.24.0", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65 + }, + "widgets_values": [ + "" + ], + "color": "#232", + "bgcolor": "#353" + }, + { + "id": 355, + "type": "ComfySwitchNode", + "pos": [ + 8140, + 5250 + ], + "size": [ + 270, + 130 + ], + "flags": {}, + "order": 24, + "mode": 0, + "inputs": [ + { + "localized_name": "on_false", + "name": "on_false", + "type": "*", + "link": 440 + }, + { + "localized_name": "on_true", + "name": "on_true", + "type": "*", + "link": 441 + }, + { + "localized_name": "switch", + "name": "switch", + "type": "BOOLEAN", + "widget": { + "name": "switch" + }, + "link": 442 + } + ], + "outputs": [ + { + "localized_name": "output", + "name": "output", + "type": "*", + "links": [ + 427 + ] + } + ], + "title": "Switch (Low Noise)", + "properties": { + "Node name for S&R": "ComfySwitchNode", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + false + ] + }, + { + "id": 356, + "type": "CustomCombo", + "pos": [ + 6170, + 3800 + ], + "size": [ + 460, + 600 + ], + "flags": {}, + "order": 25, + "mode": 0, + "inputs": [ + { + "localized_name": "choice", + "name": "choice", + "type": "COMBO", + "widget": { + "name": "choice" + }, + "link": 468 + } + ], + "outputs": [ + { + "localized_name": "STRING", + "name": "STRING", + "type": "STRING", + "links": [] + }, + { + "localized_name": "INDEX", + "name": "INDEX", + "type": "INT", + "links": [ + 443 + ] + } + ], + "properties": { + "Node name for S&R": "CustomCombo", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + "Video Editing (Content Propagation)", + 7, + "Default", + "Text to Image", + "Text to Video", + "Image Editing", + "Subject to Image", + "Image to Video", + "Video Editing", + "Video Editing (Content Propagation)", + "Video Editing with Reference", + "Ads / Content Insertion", + "Video Editing (Action / Position)", + "Video Editing (Style / Motion)", + "" + ] + }, + { + "id": 357, + "type": "a98d3dcb-12b1-467c-94b8-723a89533c30", + "pos": [ + 6680, + 3800 + ], + "size": [ + 390, + 440 + ], + "flags": {}, + "order": 26, + "mode": 0, + "inputs": [ + { + "localized_name": "text_per_line", + "name": "text_per_line", + "type": "STRING", + "widget": { + "name": "text_per_line" + }, + "link": null + }, + { + "localized_name": "index", + "name": "index", + "type": "INT", + "widget": { + "name": "index" + }, + "link": 443 + } + ], + "outputs": [ + { + "localized_name": "selected_line", + "name": "selected_line", + "type": "STRING", + "links": [ + 444 + ] + } + ], + "properties": { + "proxyWidgets": [ + [ + "373", + "string" + ], + [ + "374", + "value" + ] + ], + "cnr_id": "comfy-core", + "ver": "0.19.0", + "ue_properties": { + "widget_ue_connectable": {}, + "input_ue_unconnectable": {} + } + }, + "widgets_values": [] + }, + { + "id": 358, + "type": "StringConcatenate", + "pos": [ + 6680, + 4500 + ], + "size": [ + 400, + 250 + ], + "flags": {}, + "order": 27, + "mode": 0, + "inputs": [ + { + "localized_name": "string_a", + "name": "string_a", + "type": "STRING", + "widget": { + "name": "string_a" + }, + "link": 444 + }, + { + "localized_name": "string_b", + "name": "string_b", + "type": "STRING", + "widget": { + "name": "string_b" + }, + "link": 459 + }, + { + "localized_name": "delimiter", + "name": "delimiter", + "type": "STRING", + "widget": { + "name": "delimiter" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "STRING", + "name": "STRING", + "type": "STRING", + "links": [ + 439 + ] + } + ], + "properties": { + "Node name for S&R": "StringConcatenate", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + "", + "", + "" + ] + }, + { + "id": 359, + "type": "PrimitiveInt", + "pos": [ + 7740, + 4970 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "INT", + "widget": { + "name": "value" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 445 + ] + } + ], + "title": "Int (Steps)", + "properties": { + "Node name for S&R": "PrimitiveInt", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 40, + "fixed" + ] + }, + { + "id": 360, + "type": "PrimitiveInt", + "pos": [ + 7750, + 5510 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "INT", + "widget": { + "name": "value" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 446 + ] + } + ], + "title": "Int (Steps)", + "properties": { + "Node name for S&R": "PrimitiveInt", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 6, + "fixed" + ] + }, + { + "id": 361, + "type": "ComfySwitchNode", + "pos": [ + 8140, + 5440 + ], + "size": [ + 270, + 130 + ], + "flags": {}, + "order": 28, + "mode": 0, + "inputs": [ + { + "localized_name": "on_false", + "name": "on_false", + "type": "*", + "link": 445 + }, + { + "localized_name": "on_true", + "name": "on_true", + "type": "*", + "link": 446 + }, + { + "localized_name": "switch", + "name": "switch", + "type": "BOOLEAN", + "widget": { + "name": "switch" + }, + "link": 447 + } + ], + "outputs": [ + { + "localized_name": "output", + "name": "output", + "type": "*", + "links": [ + 426 + ] + } + ], + "title": "Switch (Steps)", + "properties": { + "Node name for S&R": "ComfySwitchNode", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + false + ] + }, + { + "id": 362, + "type": "PrimitiveInt", + "pos": [ + 7740, + 5130 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "INT", + "widget": { + "name": "value" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 448 + ] + } + ], + "title": "Int (Split Steps)", + "properties": { + "Node name for S&R": "PrimitiveInt", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 20, + "fixed" + ] + }, + { + "id": 363, + "type": "PrimitiveInt", + "pos": [ + 7750, + 5680 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "INT", + "widget": { + "name": "value" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 449 + ] + } + ], + "title": "Int (Split Steps)", + "properties": { + "Node name for S&R": "PrimitiveInt", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 3, + "fixed" + ] + }, + { + "id": 364, + "type": "ComfySwitchNode", + "pos": [ + 8140, + 5640 + ], + "size": [ + 270, + 130 + ], + "flags": {}, + "order": 29, + "mode": 0, + "inputs": [ + { + "localized_name": "on_false", + "name": "on_false", + "type": "*", + "link": 448 + }, + { + "localized_name": "on_true", + "name": "on_true", + "type": "*", + "link": 449 + }, + { + "localized_name": "switch", + "name": "switch", + "type": "BOOLEAN", + "widget": { + "name": "switch" + }, + "link": 450 + } + ], + "outputs": [ + { + "localized_name": "output", + "name": "output", + "type": "*", + "links": [ + 411 + ] + } + ], + "title": "Switch (Low Steps)", + "properties": { + "Node name for S&R": "ComfySwitchNode", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + false + ] + }, + { + "id": 365, + "type": "ComfySwitchNode", + "pos": [ + 8140, + 5830 + ], + "size": [ + 270, + 130 + ], + "flags": {}, + "order": 30, + "mode": 0, + "inputs": [ + { + "localized_name": "on_false", + "name": "on_false", + "type": "*", + "link": 451 + }, + { + "localized_name": "on_true", + "name": "on_true", + "type": "*", + "link": 452 + }, + { + "localized_name": "switch", + "name": "switch", + "type": "BOOLEAN", + "widget": { + "name": "switch" + }, + "link": 453 + } + ], + "outputs": [ + { + "localized_name": "output", + "name": "output", + "type": "*", + "links": [ + 424, + 433 + ] + } + ], + "title": "Switch (CFG)", + "properties": { + "Node name for S&R": "ComfySwitchNode", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + false + ] + }, + { + "id": 366, + "type": "PrimitiveFloat", + "pos": [ + 7750, + 5840 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "FLOAT", + "widget": { + "name": "value" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "FLOAT", + "name": "FLOAT", + "type": "FLOAT", + "links": [ + 452 + ] + } + ], + "title": "Float (CFG)", + "properties": { + "Node name for S&R": "PrimitiveFloat", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 1 + ] + }, + { + "id": 367, + "type": "PrimitiveFloat", + "pos": [ + 7740, + 5290 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "FLOAT", + "widget": { + "name": "value" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "FLOAT", + "name": "FLOAT", + "type": "FLOAT", + "links": [ + 451 + ] + } + ], + "title": "Float (CFG)", + "properties": { + "Node name for S&R": "PrimitiveFloat", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + 5 + ] + }, + { + "id": 368, + "type": "PrimitiveBoolean", + "pos": [ + 7750, + 6020 + ], + "size": [ + 270, + 100 + ], + "flags": {}, + "order": 31, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "BOOLEAN", + "widget": { + "name": "value" + }, + "link": 488 + } + ], + "outputs": [ + { + "localized_name": "BOOLEAN", + "name": "BOOLEAN", + "type": "BOOLEAN", + "links": [ + 442, + 447, + 450, + 453, + 456 + ] + } + ], + "title": "Boolean (Enable Turbo LoRA?)", + "properties": { + "Node name for S&R": "PrimitiveBoolean", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + true + ] + }, + { + "id": 369, + "type": "ComfySwitchNode", + "pos": [ + 8140, + 5060 + ], + "size": [ + 270, + 130 + ], + "flags": {}, + "order": 32, + "mode": 0, + "inputs": [ + { + "localized_name": "on_false", + "name": "on_false", + "type": "*", + "link": 454 + }, + { + "localized_name": "on_true", + "name": "on_true", + "type": "*", + "link": 455 + }, + { + "localized_name": "switch", + "name": "switch", + "type": "BOOLEAN", + "widget": { + "name": "switch" + }, + "link": 456 + } + ], + "outputs": [ + { + "localized_name": "output", + "name": "output", + "type": "*", + "links": [ + 418 + ] + } + ], + "title": "Switch (High Noise)", + "properties": { + "Node name for S&R": "ComfySwitchNode", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + false + ] + }, + { + "id": 370, + "type": "MarkdownNote", + "pos": [ + 7730, + 4680 + ], + "size": [ + 340, + 150 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": {}, + "widgets_values": [ + "You can find the original settings here: https://github.com/bytedance/Bernini/blob/main/gradio_demo.py" + ], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 371, + "type": "PrimitiveStringMultiline", + "pos": [ + 6160, + 4510 + ], + "size": [ + 470, + 230 + ], + "flags": {}, + "order": 33, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "STRING", + "widget": { + "name": "value" + }, + "link": 467 + } + ], + "outputs": [ + { + "localized_name": "STRING", + "name": "STRING", + "type": "STRING", + "links": [ + 459 + ] + } + ], + "properties": { + "Node name for S&R": "PrimitiveStringMultiline", + "cnr_id": "comfy-core", + "ver": "0.24.0" + }, + "widgets_values": [ + "Replace the gray studio backdrop with a daytime urban street: brick buildings, shop windows, sidewalk, and soft overcast light. Keep the model's outfit, accessories, body pose, motion, and full-body framing unchanged. Only the environment behind the subject should change." + ] + } + ], + "groups": [ + { + "id": 1, + "title": "Prompt & Conditioning", + "bounding": [ + 6930, + 4860, + 760, + 1270 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 2, + "title": "Models", + "bounding": [ + 6150, + 4860, + 750, + 1270 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 3, + "title": "Sampling", + "bounding": [ + 8490, + 4860, + 1160, + 1270 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 8, + "title": "Prompt Construction", + "bounding": [ + 6150, + 3690, + 960, + 1120 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 4, + "title": "System Prompt (Based on task type)", + "bounding": [ + 6160, + 3730, + 920, + 680 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 7, + "title": "Switch Settings", + "bounding": [ + 7720, + 4860, + 740, + 1270 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 5, + "title": "Distill LoRA settings", + "bounding": [ + 7730, + 5440, + 310, + 530 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 6, + "title": "Original Settings", + "bounding": [ + 7730, + 4900, + 300, + 504 + ], + "color": "#3f789e", + "flags": {} + } + ], + "links": [ + { + "id": 408, + "origin_id": 346, + "origin_slot": 0, + "target_id": 340, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 409, + "origin_id": 338, + "origin_slot": 0, + "target_id": 341, + "target_slot": 0, + "type": "CLIP" + }, + { + "id": 410, + "origin_id": 350, + "origin_slot": 0, + "target_id": 342, + "target_slot": 0, + "type": "SIGMAS" + }, + { + "id": 411, + "origin_id": 364, + "origin_slot": 0, + "target_id": 342, + "target_slot": 1, + "type": "INT" + }, + { + "id": 412, + "origin_id": 344, + "origin_slot": 0, + "target_id": 345, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 413, + "origin_id": 351, + "origin_slot": 0, + "target_id": 347, + "target_slot": 0, + "type": "LATENT" + }, + { + "id": 414, + "origin_id": 339, + "origin_slot": 0, + "target_id": 347, + "target_slot": 1, + "type": "VAE" + }, + { + "id": 415, + "origin_id": 347, + "origin_slot": 0, + "target_id": 348, + "target_slot": 0, + "type": "IMAGE" + }, + { + "id": 416, + "origin_id": 353, + "origin_slot": 1, + "target_id": 348, + "target_slot": 1, + "type": "AUDIO" + }, + { + "id": 417, + "origin_id": 353, + "origin_slot": 2, + "target_id": 348, + "target_slot": 2, + "type": "FLOAT" + }, + { + "id": 418, + "origin_id": 369, + "origin_slot": 0, + "target_id": 349, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 419, + "origin_id": 352, + "origin_slot": 0, + "target_id": 349, + "target_slot": 1, + "type": "CONDITIONING" + }, + { + "id": 420, + "origin_id": 352, + "origin_slot": 1, + "target_id": 349, + "target_slot": 2, + "type": "CONDITIONING" + }, + { + "id": 421, + "origin_id": 343, + "origin_slot": 0, + "target_id": 349, + "target_slot": 3, + "type": "SAMPLER" + }, + { + "id": 422, + "origin_id": 342, + "origin_slot": 0, + "target_id": 349, + "target_slot": 4, + "type": "SIGMAS" + }, + { + "id": 423, + "origin_id": 352, + "origin_slot": 2, + "target_id": 349, + "target_slot": 5, + "type": "LATENT" + }, + { + "id": 424, + "origin_id": 365, + "origin_slot": 0, + "target_id": 349, + "target_slot": 8, + "type": "FLOAT" + }, + { + "id": 425, + "origin_id": 346, + "origin_slot": 0, + "target_id": 350, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 426, + "origin_id": 361, + "origin_slot": 0, + "target_id": 350, + "target_slot": 2, + "type": "INT" + }, + { + "id": 427, + "origin_id": 355, + "origin_slot": 0, + "target_id": 351, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 428, + "origin_id": 352, + "origin_slot": 0, + "target_id": 351, + "target_slot": 1, + "type": "CONDITIONING" + }, + { + "id": 429, + "origin_id": 352, + "origin_slot": 1, + "target_id": 351, + "target_slot": 2, + "type": "CONDITIONING" + }, + { + "id": 430, + "origin_id": 343, + "origin_slot": 0, + "target_id": 351, + "target_slot": 3, + "type": "SAMPLER" + }, + { + "id": 431, + "origin_id": 342, + "origin_slot": 1, + "target_id": 351, + "target_slot": 4, + "type": "SIGMAS" + }, + { + "id": 432, + "origin_id": 349, + "origin_slot": 0, + "target_id": 351, + "target_slot": 5, + "type": "LATENT" + }, + { + "id": 433, + "origin_id": 365, + "origin_slot": 0, + "target_id": 351, + "target_slot": 8, + "type": "FLOAT" + }, + { + "id": 434, + "origin_id": 354, + "origin_slot": 0, + "target_id": 352, + "target_slot": 0, + "type": "CONDITIONING" + }, + { + "id": 435, + "origin_id": 341, + "origin_slot": 0, + "target_id": 352, + "target_slot": 1, + "type": "CONDITIONING" + }, + { + "id": 436, + "origin_id": 339, + "origin_slot": 0, + "target_id": 352, + "target_slot": 2, + "type": "VAE" + }, + { + "id": 437, + "origin_id": 353, + "origin_slot": 0, + "target_id": 352, + "target_slot": 3, + "type": "IMAGE" + }, + { + "id": 438, + "origin_id": 338, + "origin_slot": 0, + "target_id": 354, + "target_slot": 0, + "type": "CLIP" + }, + { + "id": 439, + "origin_id": 358, + "origin_slot": 0, + "target_id": 354, + "target_slot": 1, + "type": "STRING" + }, + { + "id": 440, + "origin_id": 346, + "origin_slot": 0, + "target_id": 355, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 441, + "origin_id": 340, + "origin_slot": 0, + "target_id": 355, + "target_slot": 1, + "type": "MODEL" + }, + { + "id": 442, + "origin_id": 368, + "origin_slot": 0, + "target_id": 355, + "target_slot": 2, + "type": "BOOLEAN" + }, + { + "id": 443, + "origin_id": 356, + "origin_slot": 1, + "target_id": 357, + "target_slot": 1, + "type": "INT" + }, + { + "id": 444, + "origin_id": 357, + "origin_slot": 0, + "target_id": 358, + "target_slot": 0, + "type": "STRING" + }, + { + "id": 459, + "origin_id": 371, + "origin_slot": 0, + "target_id": 358, + "target_slot": 1, + "type": "STRING" + }, + { + "id": 445, + "origin_id": 359, + "origin_slot": 0, + "target_id": 361, + "target_slot": 0, + "type": "INT" + }, + { + "id": 446, + "origin_id": 360, + "origin_slot": 0, + "target_id": 361, + "target_slot": 1, + "type": "INT" + }, + { + "id": 447, + "origin_id": 368, + "origin_slot": 0, + "target_id": 361, + "target_slot": 2, + "type": "BOOLEAN" + }, + { + "id": 448, + "origin_id": 362, + "origin_slot": 0, + "target_id": 364, + "target_slot": 0, + "type": "INT" + }, + { + "id": 449, + "origin_id": 363, + "origin_slot": 0, + "target_id": 364, + "target_slot": 1, + "type": "INT" + }, + { + "id": 450, + "origin_id": 368, + "origin_slot": 0, + "target_id": 364, + "target_slot": 2, + "type": "BOOLEAN" + }, + { + "id": 451, + "origin_id": 367, + "origin_slot": 0, + "target_id": 365, + "target_slot": 0, + "type": "FLOAT" + }, + { + "id": 452, + "origin_id": 366, + "origin_slot": 0, + "target_id": 365, + "target_slot": 1, + "type": "FLOAT" + }, + { + "id": 453, + "origin_id": 368, + "origin_slot": 0, + "target_id": 365, + "target_slot": 2, + "type": "BOOLEAN" + }, + { + "id": 454, + "origin_id": 344, + "origin_slot": 0, + "target_id": 369, + "target_slot": 0, + "type": "MODEL" + }, + { + "id": 455, + "origin_id": 345, + "origin_slot": 0, + "target_id": 369, + "target_slot": 1, + "type": "MODEL" + }, + { + "id": 456, + "origin_id": 368, + "origin_slot": 0, + "target_id": 369, + "target_slot": 2, + "type": "BOOLEAN" + }, + { + "id": 457, + "origin_id": -10, + "origin_slot": 0, + "target_id": 353, + "target_slot": 0, + "type": "VIDEO" + }, + { + "id": 458, + "origin_id": 348, + "origin_slot": 0, + "target_id": -20, + "target_slot": 0, + "type": "VIDEO" + }, + { + "id": 465, + "origin_id": -10, + "origin_slot": 1, + "target_id": 352, + "target_slot": 4, + "type": "IMAGE" + }, + { + "id": 466, + "origin_id": -10, + "origin_slot": 2, + "target_id": 352, + "target_slot": 5, + "type": "IMAGE" + }, + { + "id": 467, + "origin_id": -10, + "origin_slot": 3, + "target_id": 371, + "target_slot": 0, + "type": "STRING" + }, + { + "id": 468, + "origin_id": -10, + "origin_slot": 4, + "target_id": 356, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 469, + "origin_id": -10, + "origin_slot": 5, + "target_id": 352, + "target_slot": 7, + "type": "INT" + }, + { + "id": 470, + "origin_id": -10, + "origin_slot": 6, + "target_id": 352, + "target_slot": 8, + "type": "INT" + }, + { + "id": 480, + "origin_id": -10, + "origin_slot": 7, + "target_id": 349, + "target_slot": 7, + "type": "INT" + }, + { + "id": 481, + "origin_id": -10, + "origin_slot": 8, + "target_id": 352, + "target_slot": 9, + "type": "INT" + }, + { + "id": 482, + "origin_id": -10, + "origin_slot": 9, + "target_id": 352, + "target_slot": 11, + "type": "INT" + }, + { + "id": 488, + "origin_id": -10, + "origin_slot": 10, + "target_id": 368, + "target_slot": 0, + "type": "BOOLEAN" + }, + { + "id": 489, + "origin_id": -10, + "origin_slot": 11, + "target_id": 344, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 490, + "origin_id": -10, + "origin_slot": 12, + "target_id": 346, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 491, + "origin_id": -10, + "origin_slot": 13, + "target_id": 345, + "target_slot": 1, + "type": "COMBO" + }, + { + "id": 492, + "origin_id": -10, + "origin_slot": 13, + "target_id": 340, + "target_slot": 1, + "type": "COMBO" + }, + { + "id": 493, + "origin_id": -10, + "origin_slot": 14, + "target_id": 338, + "target_slot": 0, + "type": "COMBO" + }, + { + "id": 494, + "origin_id": -10, + "origin_slot": 15, + "target_id": 339, + "target_slot": 0, + "type": "COMBO" + } + ], + "extra": {}, + "category": "Video generation and editing/Video Edit", + "description": "This subgraph uses Depth Anything 3 to predict spatially consistent geometry from any number of images or video frames, with or without known camera poses. It outputs depth maps, camera poses, and optionally 3D Gaussian parameters for novel view synthesis." + }, + { + "id": "a98d3dcb-12b1-467c-94b8-723a89533c30", + "version": 1, + "state": { + "lastGroupId": 9, + "lastNodeId": 376, + "lastLinkId": 496, + "lastRerouteId": 0 + }, + "revision": 0, + "config": {}, + "name": "Select Per-Line Text by Index", + "description": "Selects one line from multiline text by zero-based index for batch or list-driven prompt workflows.", + "inputNode": { + "id": -10, + "bounding": [ + -990, + 8595, + 128, + 88 + ] + }, + "outputNode": { + "id": -20, + "bounding": [ + 710, + 8585, + 128, + 68 + ] + }, + "inputs": [ + { + "id": "75417d82-a934-4ac9-b667-d8dcd5a3bfb3", + "name": "text_per_line", + "type": "STRING", + "linkIds": [ + 13 + ], + "localized_name": "text_per_line", + "pos": [ + -886, + 8619 + ] + }, + { + "id": "46e69a73-1804-4ca6-9175-31445bf0be96", + "name": "index", + "type": "INT", + "linkIds": [ + 14 + ], + "localized_name": "index", + "pos": [ + -886, + 8639 + ] + } + ], + "outputs": [ + { + "id": "e34e8ad1-84d2-4bd2-a460-eb7de6067c10", + "name": "selected_line", + "type": "STRING", + "linkIds": [ + 10 + ], + "localized_name": "selected_line", + "pos": [ + 734, + 8609 + ] + } + ], + "widgets": [], + "nodes": [ + { + "id": 372, + "type": "PreviewAny", + "pos": [ + -500, + 8400 + ], + "size": [ + 230, + 180 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [ + { + "localized_name": "source", + "name": "source", + "type": "*", + "link": 1 + } + ], + "outputs": [ + { + "localized_name": "STRING", + "name": "STRING", + "type": "STRING", + "links": [ + 6 + ] + } + ], + "properties": { + "Node name for S&R": "PreviewAny", + "cnr_id": "comfy-core", + "ver": "0.19.0", + "ue_properties": { + "widget_ue_connectable": {}, + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + null, + null, + null + ] + }, + { + "id": 373, + "type": "RegexExtract", + "pos": [ + -240, + 8740 + ], + "size": [ + 470, + 460 + ], + "flags": {}, + "order": 1, + "mode": 0, + "showAdvanced": false, + "inputs": [ + { + "localized_name": "string", + "name": "string", + "type": "STRING", + "widget": { + "name": "string" + }, + "link": 13 + }, + { + "localized_name": "regex_pattern", + "name": "regex_pattern", + "type": "STRING", + "widget": { + "name": "regex_pattern" + }, + "link": 9 + }, + { + "localized_name": "mode", + "name": "mode", + "type": "COMBO", + "widget": { + "name": "mode" + }, + "link": null + }, + { + "localized_name": "case_insensitive", + "name": "case_insensitive", + "type": "BOOLEAN", + "widget": { + "name": "case_insensitive" + }, + "link": null + }, + { + "localized_name": "multiline", + "name": "multiline", + "type": "BOOLEAN", + "widget": { + "name": "multiline" + }, + "link": null + }, + { + "localized_name": "dotall", + "name": "dotall", + "type": "BOOLEAN", + "widget": { + "name": "dotall" + }, + "link": null + }, + { + "localized_name": "group_index", + "name": "group_index", + "type": "INT", + "widget": { + "name": "group_index" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "STRING", + "name": "STRING", + "type": "STRING", + "links": [ + 10 + ] + } + ], + "properties": { + "Node name for S&R": "RegexExtract", + "cnr_id": "comfy-core", + "ver": "0.19.0", + "ue_properties": { + "widget_ue_connectable": {}, + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + "You are a helpful assistant.\nYou are a helpful assistant specialized in text-to-image generation.\nYou are a helpful assistant specialized in text-to-video generation.\nYou are a helpful assistant specialized in image editing.\nYou are a helpful assistant specialized in subject-to-image generation.\nYou are a helpful assistant specialized in image-to-video generation.\nYou are a helpful assistant specialized in video editing.\nYou are a helpful assistant specialized in video editing on content propagation.\nYou are a helpful assistant specialized in video editing with reference.\nYou are a helpful assistant specialized in ads insertion.\nYou are a helpful assistant for editing. You may need to adjust the subject's action or position.\nYou are a helpful assistant for editing. You might need to adjust the video's style, lighting, colors, textures, and the subject's pose or action.", + "", + "First Group", + false, + false, + false, + 1 + ] + }, + { + "id": 374, + "type": "PrimitiveInt", + "pos": [ + -810, + 8400 + ], + "size": [ + 270, + 110 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "INT", + "widget": { + "name": "value" + }, + "link": 14 + } + ], + "outputs": [ + { + "localized_name": "INT", + "name": "INT", + "type": "INT", + "links": [ + 1 + ] + } + ], + "title": "Int (line index)", + "properties": { + "Node name for S&R": "Int (line index)", + "cnr_id": "comfy-core", + "ver": "0.19.0", + "ue_properties": { + "widget_ue_connectable": {}, + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + 0, + "fixed" + ] + }, + { + "id": 375, + "type": "StringReplace", + "pos": [ + -240, + 8400 + ], + "size": [ + 400, + 280 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "localized_name": "string", + "name": "string", + "type": "STRING", + "widget": { + "name": "string" + }, + "link": null + }, + { + "localized_name": "find", + "name": "find", + "type": "STRING", + "widget": { + "name": "find" + }, + "link": null + }, + { + "localized_name": "replace", + "name": "replace", + "type": "STRING", + "widget": { + "name": "replace" + }, + "link": 6 + } + ], + "outputs": [ + { + "localized_name": "STRING", + "name": "STRING", + "type": "STRING", + "links": [ + 9 + ] + } + ], + "properties": { + "Node name for S&R": "StringReplace", + "cnr_id": "comfy-core", + "ver": "0.19.0", + "ue_properties": { + "widget_ue_connectable": {}, + "input_ue_unconnectable": {} + } + }, + "widgets_values": [ + "^(?:[^\\n]*\\n){index}([^\\n]*)(?:\\n|$)", + "index", + "" + ] + } + ], + "groups": [], + "links": [ + { + "id": 1, + "origin_id": 374, + "origin_slot": 0, + "target_id": 372, + "target_slot": 0, + "type": "INT" + }, + { + "id": 9, + "origin_id": 375, + "origin_slot": 0, + "target_id": 373, + "target_slot": 1, + "type": "STRING" + }, + { + "id": 6, + "origin_id": 372, + "origin_slot": 0, + "target_id": 375, + "target_slot": 2, + "type": "STRING" + }, + { + "id": 10, + "origin_id": 373, + "origin_slot": 0, + "target_id": -20, + "target_slot": 0, + "type": "STRING" + }, + { + "id": 13, + "origin_id": -10, + "origin_slot": 0, + "target_id": 373, + "target_slot": 0, + "type": "STRING" + }, + { + "id": 14, + "origin_id": -10, + "origin_slot": 1, + "target_id": 374, + "target_slot": 0, + "type": "INT" + } + ], + "extra": { + "ue_links": [], + "links_added_by_ue": [] + } + } + ] + }, + "extra": { + "BlueprintDescription": "This subgraph uses Depth Anything 3 to predict spatially consistent geometry from any number of images or video frames, with or without known camera poses. It outputs depth maps, camera poses, and optionally 3D Gaussian parameters for novel view synthesis." + } +} \ No newline at end of file diff --git a/comfy/cli_args.py b/comfy/cli_args.py index cba0dfa34..4bef096fb 100644 --- a/comfy/cli_args.py +++ b/comfy/cli_args.py @@ -115,6 +115,7 @@ cache_group.add_argument("--cache-ram", nargs='*', type=float, default=[], metav cache_group.add_argument("--cache-classic", action="store_true", help="Use the old style (aggressive) caching.") cache_group.add_argument("--cache-lru", type=int, default=0, help="Use LRU caching with a maximum of N node results cached. May use more RAM/VRAM.") cache_group.add_argument("--cache-none", action="store_true", help="Reduced RAM/VRAM usage at the expense of executing every node for each run.") +cache_group.add_argument("--high-ram", action="store_true", help="Can improve performance slightly on high RAM or on systems where pagefile use is preferred over model loading.") attn_group = parser.add_mutually_exclusive_group() attn_group.add_argument("--use-split-cross-attention", action="store_true", help="Use the split cross attention optimization. Ignored when xformers is used.") @@ -133,7 +134,7 @@ upcast.add_argument("--dont-upcast-attention", action="store_true", help="Disabl parser.add_argument("--enable-manager", action="store_true", help="Enable the ComfyUI-Manager feature.") manager_group = parser.add_mutually_exclusive_group() manager_group.add_argument("--disable-manager-ui", action="store_true", help="Disables only the ComfyUI-Manager UI and endpoints. Scheduled installations and similar background tasks will still operate.") -manager_group.add_argument("--enable-manager-legacy-ui", action="store_true", help="Enables the legacy UI of ComfyUI-Manager") +manager_group.add_argument("--enable-manager-legacy-ui", action="store_true", help="Enables the legacy UI of ComfyUI-Manager. Implies --enable-manager.") vram_group = parser.add_mutually_exclusive_group() @@ -144,6 +145,7 @@ vram_group.add_argument("--novram", action="store_true", help="When lowvram isn' vram_group.add_argument("--cpu", action="store_true", help="To use the CPU for everything (slow).") parser.add_argument("--reserve-vram", type=float, default=None, help="Set the amount of vram in GB you want to reserve for use by your OS/other software. By default some amount is reserved depending on your OS.") +parser.add_argument("--vram-headroom", type=float, default=0, help="Set the amount of vram in GB for DynamicVRAM to maintain as extra headroom above default. ComfyUI will try and keep this much VRAM completely free and unused, even counting VRAM from other apps.") parser.add_argument("--async-offload", nargs='?', const=2, type=int, default=None, metavar="NUM_STREAMS", help="Use async weight offloading. An optional argument controls the amount of offload streams. Default is 2. Enabled by default on Nvidia.") parser.add_argument("--disable-async-offload", action="store_true", help="Disable async weight offloading.") @@ -238,6 +240,7 @@ database_default_path = os.path.abspath( ) parser.add_argument("--database-url", type=str, default=f"sqlite:///{database_default_path}", help="Specify the database URL, e.g. for an in-memory database you can use 'sqlite:///:memory:'.") parser.add_argument("--enable-assets", action="store_true", help="Enable the assets system (API routes, database synchronization, and background scanning).") +parser.add_argument("--enable-asset-hashing", action="store_true", help="Compute blake3 content hashes when scanning assets. Hashing enables future asset-portability features (deduplication, cross-machine model resolution) but adds startup cost and per-output cost on large models directories. Off by default; enable to opt in.") parser.add_argument("--feature-flag", type=str, action='append', default=[], metavar="KEY[=VALUE]", help="Set a server feature flag. Use KEY=VALUE to set an explicit value, or bare KEY to set it to true. Can be specified multiple times. Boolean values (true/false) and numbers are auto-converted. Examples: --feature-flag show_signin_button=true or --feature-flag show_signin_button") parser.add_argument("--list-feature-flags", action="store_true", help="Print the registry of known CLI-settable feature flags as JSON and exit.") @@ -249,6 +252,9 @@ else: if args.cache_ram is not None and len(args.cache_ram) > 2: parser.error("--cache-ram accepts at most two values: active GB and inactive GB") +if args.high_ram: + args.cache_classic = True + if args.windows_standalone_build: args.auto_launch = True @@ -258,6 +264,10 @@ if args.disable_auto_launch: if args.force_fp16: args.fp16_unet = True +# '--enable-manager-legacy-ui' is meaningless unless the manager is enabled, so imply '--enable-manager'. +if args.enable_manager_legacy_ui: + args.enable_manager = True + # '--fast' is not provided, use an empty set if args.fast is None: diff --git a/comfy/context_windows.py b/comfy/context_windows.py index db57537a2..5f9899c67 100644 --- a/comfy/context_windows.py +++ b/comfy/context_windows.py @@ -8,6 +8,8 @@ from abc import ABC, abstractmethod import logging import comfy.model_management import comfy.patcher_extension +import comfy.utils +import comfy.conds if TYPE_CHECKING: from comfy.model_base import BaseModel from comfy.model_patcher import ModelPatcher @@ -51,12 +53,18 @@ class ContextHandlerABC(ABC): class IndexListContextWindow(ContextWindowABC): - def __init__(self, index_list: list[int], dim: int=0, total_frames: int=0): + def __init__(self, index_list: list[int], dim: int=0, total_frames: int=0, modality_windows: dict=None, context_overlap: int=0): self.index_list = index_list self.context_length = len(index_list) + self.context_overlap = context_overlap self.dim = dim self.total_frames = total_frames self.center_ratio = (min(index_list) + max(index_list)) / (2 * total_frames) + self.modality_windows = modality_windows # dict of {mod_idx: IndexListContextWindow} + self.guide_frames_indices: list[int] = [] + self.guide_overlap_info: list[tuple[int, int]] = [] + self.guide_kf_local_positions: list[int] = [] + self.guide_downscale_factors: list[int] = [] def get_tensor(self, full: torch.Tensor, device=None, dim=None, retain_index_list=[]) -> torch.Tensor: if dim is None: @@ -85,6 +93,11 @@ class IndexListContextWindow(ContextWindowABC): region_idx = int(self.center_ratio * num_regions) return min(max(region_idx, 0), num_regions - 1) + def get_window_for_modality(self, modality_idx: int) -> 'IndexListContextWindow': + if modality_idx == 0: + return self + return self.modality_windows[modality_idx] + class IndexListCallbacks: EVALUATE_CONTEXT_WINDOWS = "evaluate_context_windows" @@ -148,6 +161,172 @@ def slice_cond(cond_value, window: IndexListContextWindow, x_in: torch.Tensor, d return cond_value._copy_with(sliced) +def compute_guide_overlap(guide_entries: list[dict], keyframe_idxs: torch.Tensor, temporal_downscale_ratio: int, window_index_list: list[int]): + """Compute which concatenated guide frames overlap with a context window. + + Each guide's latent-space start is derived from its first token's pixel-t-start + in keyframe_idxs (shape (B, [t,h,w], num_tokens, [start, end])), divided by the + model's temporal_downscale_ratio. + + Args: + guide_entries: list of guide_attention_entry dicts + keyframe_idxs: per-token pixel coords cond tensor for the modality + temporal_downscale_ratio: model's pixel-to-latent temporal compression ratio + window_index_list: the window's frame indices into the video portion + + Returns: + suffix_indices: indices into the guide_frames tensor for frame selection + overlap_info: list of (entry_idx, overlap_count) for guide_attention_entries adjustment + kf_local_positions: window-local frame positions for keyframe_idxs regeneration + total_overlap: total number of overlapping guide frames + """ + window_set = set(window_index_list) + window_list = list(window_index_list) + suffix_indices = [] + overlap_info = [] + kf_local_positions = [] + suffix_base = 0 + token_offset = 0 + + for entry_idx, entry in enumerate(guide_entries): + first_t_pixel = int(keyframe_idxs[0, 0, token_offset, 0].item()) + latent_start = (first_t_pixel + temporal_downscale_ratio - 1) // temporal_downscale_ratio + guide_len = entry["latent_shape"][0] + entry_overlap = 0 + + for local_offset in range(guide_len): + video_pos = latent_start + local_offset + if video_pos in window_set: + suffix_indices.append(suffix_base + local_offset) + kf_local_positions.append(window_list.index(video_pos)) + entry_overlap += 1 + + if entry_overlap > 0: + overlap_info.append((entry_idx, entry_overlap)) + suffix_base += guide_len + token_offset += entry["pre_filter_count"] + + return suffix_indices, overlap_info, kf_local_positions, len(suffix_indices) + + +@dataclass +class WindowingState: + """Per-modality context windowing state for each step, + built using IndexListContextHandler._build_window_state(). + For non-multimodal models the lists are length 1 + """ + latents: list[torch.Tensor] # per-modality working latents (guide frames stripped) + guide_latents: list[torch.Tensor | None] # per-modality guide frames stripped from latents + guide_entries: list[list[dict] | None] # per-modality guide_attention_entry metadata + keyframe_idxs: list[torch.Tensor | None] # per-modality keyframe_idxs tensor for guide latent_start derivation + latent_shapes: list | None # original packed shapes for unpack/pack (None if not multimodal) + dim: int = 0 # primary modality temporal dim for context windowing + is_multimodal: bool = False + temporal_downscale_ratio: int = 1 # model's pixel-to-latent temporal compression ratio + + def prepare_window(self, window: IndexListContextWindow, model) -> IndexListContextWindow: + """Reformat window for multimodal contexts by deriving per-modality index lists. + Non-multimodal contexts return the input window unchanged. + """ + if not self.is_multimodal: + return window + + x = self.latents[0] + primary_total = self.latent_shapes[0][self.dim] + primary_overlap = window.context_overlap + map_shapes = self.latent_shapes + if x.size(self.dim) != primary_total: + map_shapes = list(self.latent_shapes) + video_shape = list(self.latent_shapes[0]) + video_shape[self.dim] = x.size(self.dim) + map_shapes[0] = torch.Size(video_shape) + try: + per_modality_indices = model.map_context_window_to_modalities( + window.index_list, map_shapes, self.dim) + except AttributeError: + raise NotImplementedError( + f"{type(model).__name__} must implement map_context_window_to_modalities for multimodal context windows.") + modality_windows = {} + for mod_idx in range(1, len(self.latents)): + modality_total_frames = self.latents[mod_idx].shape[self.dim] + ratio = modality_total_frames / primary_total if primary_total > 0 else 1 + modality_overlap = max(round(primary_overlap * ratio), 0) + modality_windows[mod_idx] = IndexListContextWindow( + per_modality_indices[mod_idx], dim=self.dim, + total_frames=modality_total_frames, + context_overlap=modality_overlap) + return IndexListContextWindow( + window.index_list, dim=self.dim, total_frames=x.shape[self.dim], + modality_windows=modality_windows, context_overlap=primary_overlap) + + def slice_for_window(self, window: IndexListContextWindow, retain_index_list: list[int], device=None) -> tuple[list[torch.Tensor], list[int]]: + """Slice latents for a context window, injecting guide frames where applicable. + For multimodal contexts, uses the modality-specific windows derived in prepare_window(). + """ + sliced = [] + guide_frame_counts = [] + for idx in range(len(self.latents)): + modality_window = window.get_window_for_modality(idx) + retain = retain_index_list if idx == 0 else [] + s = modality_window.get_tensor(self.latents[idx], device, retain_index_list=retain) + if self.guide_entries[idx] is not None: + s, ng = self._inject_guide_frames(s, modality_window, modality_idx=idx) + else: + ng = 0 + sliced.append(s) + guide_frame_counts.append(ng) + return sliced, guide_frame_counts + + def strip_guide_frames(self, out_per_modality: list[list[torch.Tensor]], guide_frame_counts: list[int], window: IndexListContextWindow): + """Strip injected guide frames from per-cond, per-modality outputs in place.""" + for idx in range(len(self.latents)): + if guide_frame_counts[idx] > 0: + window_len = len(window.get_window_for_modality(idx).index_list) + for ci in range(len(out_per_modality)): + out_per_modality[ci][idx] = out_per_modality[ci][idx].narrow(self.dim, 0, window_len) + + def _inject_guide_frames(self, latent_slice: torch.Tensor, window: IndexListContextWindow, modality_idx: int = 0) -> tuple[torch.Tensor, int]: + guide_entries = self.guide_entries[modality_idx] + guide_frames = self.guide_latents[modality_idx] + keyframe_idxs = self.keyframe_idxs[modality_idx] + suffix_idx, overlap_info, kf_local_pos, guide_frame_count = compute_guide_overlap( + guide_entries, keyframe_idxs, self.temporal_downscale_ratio, window.index_list) + # Shift keyframe positions to account for causal_window_fix anchor occupying sub-pos 0. + anchor_idx = getattr(window, 'causal_anchor_index', None) + if anchor_idx is not None and anchor_idx >= 0: + kf_local_pos = [p + 1 for p in kf_local_pos] + window.guide_frames_indices = suffix_idx + window.guide_overlap_info = overlap_info + window.guide_kf_local_positions = kf_local_pos + + # Derive per-overlap-entry latent_downscale_factor from guide entry latent_shape vs guide frame spatial dims. + # guide_frames has full (post-dilation) spatial dims; entry["latent_shape"] has pre-dilation dims. + guide_downscale_factors = [] + if guide_frame_count > 0: + full_H = guide_frames.shape[3] + for entry_idx, _ in overlap_info: + entry_H = guide_entries[entry_idx]["latent_shape"][1] + guide_downscale_factors.append(full_H // entry_H) + window.guide_downscale_factors = guide_downscale_factors + + if guide_frame_count > 0: + idx = tuple([slice(None)] * self.dim + [suffix_idx]) + return torch.cat([latent_slice, guide_frames[idx]], dim=self.dim), guide_frame_count + return latent_slice, 0 + + def patch_latent_shapes(self, sub_conds, new_shapes): + if not self.is_multimodal: + return + + for cond_list in sub_conds: + if cond_list is None: + continue + for cond_dict in cond_list: + model_conds = cond_dict.get('model_conds', {}) + if 'latent_shapes' in model_conds: + model_conds['latent_shapes'] = comfy.conds.CONDConstant(new_shapes) + + @dataclass class ContextSchedule: name: str @@ -162,7 +341,7 @@ ContextResults = collections.namedtuple("ContextResults", ['window_idx', 'sub_co class IndexListContextHandler(ContextHandlerABC): def __init__(self, context_schedule: ContextSchedule, fuse_method: ContextFuseMethod, context_length: int=1, context_overlap: int=0, context_stride: int=1, closed_loop: bool=False, dim:int=0, freenoise: bool=False, cond_retain_index_list: list[int]=[], split_conds_to_windows: bool=False, - causal_window_fix: bool=True): + latent_retain_index_list: list[int]=[], causal_window_fix: bool=True): self.context_schedule = context_schedule self.fuse_method = fuse_method self.context_length = context_length @@ -174,17 +353,118 @@ class IndexListContextHandler(ContextHandlerABC): self.freenoise = freenoise self.cond_retain_index_list = [int(x.strip()) for x in cond_retain_index_list.split(",")] if cond_retain_index_list else [] self.split_conds_to_windows = split_conds_to_windows + self.latent_retain_index_list = [int(x.strip()) for x in latent_retain_index_list.split(",")] if latent_retain_index_list else [] self.causal_window_fix = causal_window_fix self.callbacks = {} + @staticmethod + def _get_latent_shapes(conds): + for cond_list in conds: + if cond_list is None: + continue + for cond_dict in cond_list: + model_conds = cond_dict.get('model_conds', {}) + if 'latent_shapes' in model_conds: + return model_conds['latent_shapes'].cond + return None + + @staticmethod + def _get_guide_entries(conds): + for cond_list in conds: + if cond_list is None: + continue + for cond_dict in cond_list: + model_conds = cond_dict.get('model_conds', {}) + entries = model_conds.get('guide_attention_entries') + if entries is not None and hasattr(entries, 'cond') and entries.cond: + return entries.cond + return None + + @staticmethod + def _get_keyframe_idxs(conds): + for cond_list in conds: + if cond_list is None: + continue + for cond_dict in cond_list: + model_conds = cond_dict.get('model_conds', {}) + kf = model_conds.get('keyframe_idxs') + if kf is not None and hasattr(kf, 'cond') and kf.cond is not None: + return kf.cond + return None + + def _apply_freenoise(self, noise: torch.Tensor, conds: list[list[dict]], seed: int) -> torch.Tensor: + """Apply FreeNoise shuffling, scaling context length/overlap per-modality by frame ratio. + If guide frames are present on the primary modality, only the video portion is shuffled. + """ + guide_entries = self._get_guide_entries(conds) + guide_count = sum(e["latent_shape"][0] for e in guide_entries) if guide_entries else 0 + + latent_shapes = self._get_latent_shapes(conds) + if latent_shapes is not None and len(latent_shapes) > 1: + modalities = comfy.utils.unpack_latents(noise, latent_shapes) + primary_total = latent_shapes[0][self.dim] + primary_video_count = modalities[0].size(self.dim) - guide_count + apply_freenoise(modalities[0].narrow(self.dim, 0, primary_video_count), self.dim, self.context_length, self.context_overlap, seed) + for i in range(1, len(modalities)): + mod_total = latent_shapes[i][self.dim] + ratio = mod_total / primary_total if primary_total > 0 else 1 + mod_ctx_len = max(round(self.context_length * ratio), 1) + mod_ctx_overlap = max(round(self.context_overlap * ratio), 0) + modalities[i] = apply_freenoise(modalities[i], self.dim, mod_ctx_len, mod_ctx_overlap, seed) + noise, _ = comfy.utils.pack_latents(modalities) + return noise + video_count = noise.size(self.dim) - guide_count + apply_freenoise(noise.narrow(self.dim, 0, video_count), self.dim, self.context_length, self.context_overlap, seed) + return noise + + def _build_window_state(self, x_in: torch.Tensor, conds: list[list[dict]], model: BaseModel) -> WindowingState: + """Build windowing state for the current step, including unpacking latents and extracting guide frame info from conds.""" + latent_shapes = self._get_latent_shapes(conds) + is_multimodal = latent_shapes is not None and len(latent_shapes) > 1 + unpacked_latents = comfy.utils.unpack_latents(x_in, latent_shapes) if is_multimodal else [x_in] + + unpacked_latents_list = list(unpacked_latents) + guide_latents_list = [None] * len(unpacked_latents) + guide_entries_list = [None] * len(unpacked_latents) + keyframe_idxs_list = [None] * len(unpacked_latents) + + extracted_guide_entries = self._get_guide_entries(conds) + extracted_keyframe_idxs = self._get_keyframe_idxs(conds) + + # Strip guide frames (only from first modality for now) + if extracted_guide_entries is not None: + guide_count = sum(e["latent_shape"][0] for e in extracted_guide_entries) + if guide_count > 0: + x = unpacked_latents[0] + latent_count = x.size(self.dim) - guide_count + unpacked_latents_list[0] = x.narrow(self.dim, 0, latent_count) + guide_latents_list[0] = x.narrow(self.dim, latent_count, guide_count) + guide_entries_list[0] = extracted_guide_entries + keyframe_idxs_list[0] = extracted_keyframe_idxs + + + return WindowingState( + latents=unpacked_latents_list, + guide_latents=guide_latents_list, + guide_entries=guide_entries_list, + keyframe_idxs=keyframe_idxs_list, + latent_shapes=latent_shapes, + dim=self.dim, + is_multimodal=is_multimodal, + temporal_downscale_ratio=model.latent_format.temporal_downscale_ratio) + def should_use_context(self, model: BaseModel, conds: list[list[dict]], x_in: torch.Tensor, timestep: torch.Tensor, model_options: dict[str]) -> bool: - # for now, assume first dim is batch - should have stored on BaseModel in actual implementation - if x_in.size(self.dim) > self.context_length: - logging.info(f"Using context windows {self.context_length} with overlap {self.context_overlap} for {x_in.size(self.dim)} frames.") + window_state = self._build_window_state(x_in, conds, model) # build window_state to check frame counts, will be built again in execute + total_frame_count = window_state.latents[0].size(self.dim) + if total_frame_count > self.context_length: + logging.info(f"\nUsing context windows: Context length {self.context_length} with overlap {self.context_overlap} for {total_frame_count} frames.") if self.cond_retain_index_list: logging.info(f"Retaining original cond for indexes: {self.cond_retain_index_list}") + if self.latent_retain_index_list: + logging.info(f"Retaining original latent for indexes: {self.latent_retain_index_list}") return True + logging.info(f"\nNot using context windows since context length ({self.context_length}) exceeds input frames ({total_frame_count}).") return False def prepare_control_objects(self, control: ControlBase, device=None) -> ControlBase: @@ -275,7 +555,9 @@ class IndexListContextHandler(ContextHandlerABC): return resized_cond def set_step(self, timestep: torch.Tensor, model_options: dict[str]): - mask = torch.isclose(model_options["transformer_options"]["sample_sigmas"], timestep[0], rtol=0.0001) + sample_sigmas = model_options["transformer_options"]["sample_sigmas"] + current_timestep = timestep[0].to(sample_sigmas.dtype) + mask = torch.isclose(sample_sigmas, current_timestep, rtol=0.0001) matches = torch.nonzero(mask) if torch.numel(matches) == 0: return # substep from multi-step sampler: keep self._step from the last full step @@ -284,54 +566,98 @@ class IndexListContextHandler(ContextHandlerABC): def get_context_windows(self, model: BaseModel, x_in: torch.Tensor, model_options: dict[str]) -> list[IndexListContextWindow]: full_length = x_in.size(self.dim) # TODO: choose dim based on model context_windows = self.context_schedule.func(full_length, self, model_options) - context_windows = [IndexListContextWindow(window, dim=self.dim, total_frames=full_length) for window in context_windows] + context_windows = [IndexListContextWindow(window, dim=self.dim, total_frames=full_length, context_overlap=self.context_overlap) for window in context_windows] return context_windows def execute(self, calc_cond_batch: Callable, model: BaseModel, conds: list[list[dict]], x_in: torch.Tensor, timestep: torch.Tensor, model_options: dict[str]): self._model = model self.set_step(timestep, model_options) - context_windows = self.get_context_windows(model, x_in, model_options) - enumerated_context_windows = list(enumerate(context_windows)) - conds_final = [torch.zeros_like(x_in) for _ in conds] + window_state = self._build_window_state(x_in, conds, model) + num_modalities = len(window_state.latents) + + context_windows = self.get_context_windows(model, window_state.latents[0], model_options) + enumerated_context_windows = list(enumerate(context_windows)) + total_windows = len(enumerated_context_windows) + + # Initialize per-modality accumulators (length 1 for single-modality) + accum = [[torch.zeros_like(m) for _ in conds] for m in window_state.latents] if self.fuse_method.name == ContextFuseMethods.RELATIVE: - counts_final = [torch.ones(get_shape_for_dim(x_in, self.dim), device=x_in.device) for _ in conds] + counts = [[torch.ones(get_shape_for_dim(m, self.dim), device=m.device) for _ in conds] for m in window_state.latents] else: - counts_final = [torch.zeros(get_shape_for_dim(x_in, self.dim), device=x_in.device) for _ in conds] - biases_final = [([0.0] * x_in.shape[self.dim]) for _ in conds] + counts = [[torch.zeros(get_shape_for_dim(m, self.dim), device=m.device) for _ in conds] for m in window_state.latents] + biases = [[([0.0] * m.shape[self.dim]) for _ in conds] for m in window_state.latents] for callback in comfy.patcher_extension.get_all_callbacks(IndexListCallbacks.EXECUTE_START, self.callbacks): callback(self, model, x_in, conds, timestep, model_options) + # accumulate results from each context window for enum_window in enumerated_context_windows: - results = self.evaluate_context_windows(calc_cond_batch, model, x_in, conds, timestep, [enum_window], model_options) + results = self.evaluate_context_windows( + calc_cond_batch, model, x_in, conds, timestep, [enum_window], + model_options, window_state=window_state, total_windows=total_windows) for result in results: - self.combine_context_window_results(x_in, result.sub_conds_out, result.sub_conds, result.window, result.window_idx, len(enumerated_context_windows), timestep, - conds_final, counts_final, biases_final) + # result.sub_conds_out is per-cond, per-modality: list[list[Tensor]] + for mod_idx in range(num_modalities): + mod_out = [result.sub_conds_out[ci][mod_idx] for ci in range(len(conds))] + modality_window = result.window.get_window_for_modality(mod_idx) + self.combine_context_window_results( + window_state.latents[mod_idx], mod_out, result.sub_conds, modality_window, + result.window_idx, total_windows, timestep, + accum[mod_idx], counts[mod_idx], biases[mod_idx]) + + # fuse accumulated results into final conds try: - # finalize conds - if self.fuse_method.name == ContextFuseMethods.RELATIVE: - # relative is already normalized, so return as is - del counts_final - return conds_final - else: - # normalize conds via division by context usage counts - for i in range(len(conds_final)): - conds_final[i] /= counts_final[i] - del counts_final - return conds_final + result_out = [] + for ci in range(len(conds)): + finalized = [] + for mod_idx in range(num_modalities): + if self.fuse_method.name != ContextFuseMethods.RELATIVE: + accum[mod_idx][ci] /= counts[mod_idx][ci] + f = accum[mod_idx][ci] + + # if guide frames were injected, append them to the end of the fused latents for the next step + if window_state.guide_latents[mod_idx] is not None: + f = torch.cat([f, window_state.guide_latents[mod_idx]], dim=self.dim) + finalized.append(f) + + # pack modalities together if needed + if window_state.is_multimodal and len(finalized) > 1: + packed, _ = comfy.utils.pack_latents(finalized) + else: + packed = finalized[0] + + result_out.append(packed) + return result_out finally: for callback in comfy.patcher_extension.get_all_callbacks(IndexListCallbacks.EXECUTE_CLEANUP, self.callbacks): callback(self, model, x_in, conds, timestep, model_options) - def evaluate_context_windows(self, calc_cond_batch: Callable, model: BaseModel, x_in: torch.Tensor, conds, timestep: torch.Tensor, enumerated_context_windows: list[tuple[int, IndexListContextWindow]], - model_options, device=None, first_device=None): + def evaluate_context_windows(self, calc_cond_batch: Callable, model: BaseModel, x_in: torch.Tensor, conds, + timestep: torch.Tensor, enumerated_context_windows: list[tuple[int, IndexListContextWindow]], + model_options, window_state: WindowingState, total_windows: int = None, + device=None, first_device=None): + """Evaluate context windows and return per-cond, per-modality outputs in ContextResults.sub_conds_out + + For each window: + 1. Builds windows (for each modality if multimodal) + 2. Slices window for each modality + 3. Injects concatenated latent guide frames where present + 4. Packs together if needed and calls model + 5. Unpacks and strips any guides from outputs + """ + x = window_state.latents[0] + results: list[ContextResults] = [] for window_idx, window in enumerated_context_windows: # allow processing to end between context window executions for faster Cancel comfy.model_management.throw_exception_if_processing_interrupted() - # causal_window_fix: prepend a pre-window frame that will be stripped post-forward + # prepare the window accounting for multimodal windows + window = window_state.prepare_window(window, model) + + # causal_window_fix: prepend a pre-window frame that will be stripped post-forward. + # Set anchor before slice_for_window so the latent slice and downstream cond slices both pick it up. anchor_applied = False if self.causal_window_fix: anchor_idx = window.index_list[0] - 1 @@ -339,27 +665,46 @@ class IndexListContextHandler(ContextHandlerABC): window.causal_anchor_index = anchor_idx anchor_applied = True + # slice the window for each modality, injecting guide frames where applicable + sliced, guide_frame_counts_per_modality = window_state.slice_for_window(window, self.latent_retain_index_list, device) + for callback in comfy.patcher_extension.get_all_callbacks(IndexListCallbacks.EVALUATE_CONTEXT_WINDOWS, self.callbacks): callback(self, model, x_in, conds, timestep, model_options, window_idx, window, model_options, device, first_device) - # update exposed params + logging.info(f"Context window {window_idx + 1}/{total_windows or len(enumerated_context_windows)}: frames {window.index_list[0]}-{window.index_list[-1]} of {x.shape[self.dim]}" + + (f" (+{guide_frame_counts_per_modality[0]} guide frames)" if guide_frame_counts_per_modality[0] > 0 else "") + ) + + # if multimodal, pack modalities together + if window_state.is_multimodal and len(sliced) > 1: + sub_x, sub_shapes = comfy.utils.pack_latents(sliced) + else: + sub_x, sub_shapes = sliced[0], [sliced[0].shape] + + # get resized conds for window model_options["transformer_options"]["context_window"] = window - # get subsections of x, timestep, conds - sub_x = window.get_tensor(x_in, device) - sub_timestep = window.get_tensor(timestep, device, dim=0) - sub_conds = [self.get_resized_cond(cond, x_in, window, device) for cond in conds] + sub_timestep = window.get_tensor(timestep, dim=0) + sub_conds = [self.get_resized_cond(cond, x, window) for cond in conds] + # if multimodal, patch latent_shapes in conds for correct unpacking in model + window_state.patch_latent_shapes(sub_conds, sub_shapes) + + # call model on window sub_conds_out = calc_cond_batch(model, sub_conds, sub_x, sub_timestep, model_options) - if device is not None: - for i in range(len(sub_conds_out)): - sub_conds_out[i] = sub_conds_out[i].to(x_in.device) - # strip causal_window_fix anchor if applied + # unpack outputs + out_per_modality = [comfy.utils.unpack_latents(sub_conds_out[i], sub_shapes) for i in range(len(sub_conds_out))] + + # strip causal_window_fix anchor from primary modality before guide strip so window_len math stays correct if anchor_applied: - for i in range(len(sub_conds_out)): - sub_conds_out[i] = sub_conds_out[i].narrow(self.dim, 1, sub_conds_out[i].shape[self.dim] - 1) + for ci in range(len(out_per_modality)): + t = out_per_modality[ci][0] + out_per_modality[ci][0] = t.narrow(self.dim, 1, t.shape[self.dim] - 1) - results.append(ContextResults(window_idx, sub_conds_out, sub_conds, window)) + # strip injected guide frames + window_state.strip_guide_frames(out_per_modality, guide_frame_counts_per_modality, window) + + results.append(ContextResults(window_idx, out_per_modality, sub_conds, window)) return results @@ -383,7 +728,7 @@ class IndexListContextHandler(ContextHandlerABC): biases_final[i][idx] = bias_total + bias else: # add conds and counts based on weights of fuse method - weights = get_context_weights(window.context_length, x_in.shape[self.dim], window.index_list, self, sigma=timestep) + weights = get_context_weights(window.context_length, x_in.shape[self.dim], window.index_list, self, sigma=timestep, context_overlap=window.context_overlap) weights_tensor = match_weights_to_dim(weights, x_in, self.dim, device=x_in.device) for i in range(len(sub_conds_out)): window.add_window(conds_final[i], sub_conds_out[i] * weights_tensor) @@ -393,16 +738,22 @@ class IndexListContextHandler(ContextHandlerABC): callback(self, x_in, sub_conds_out, sub_conds, window, window_idx, total_windows, timestep, conds_final, counts_final, biases_final) -def _prepare_sampling_wrapper(executor, model, noise_shape: torch.Tensor, *args, **kwargs): - # limit noise_shape length to context_length for more accurate vram use estimation +def _prepare_sampling_wrapper(executor, model, noise_shape: torch.Tensor, conds, *args, **kwargs): + # Scale noise_shape to a single context window so VRAM estimation budgets per-window. model_options = kwargs.get("model_options", None) if model_options is None: raise Exception("model_options not found in prepare_sampling_wrapper; this should never happen, something went wrong.") handler: IndexListContextHandler = model_options.get("context_handler", None) if handler is not None: noise_shape = list(noise_shape) - noise_shape[handler.dim] = min(noise_shape[handler.dim], handler.context_length) - return executor(model, noise_shape, *args, **kwargs) + is_packed = len(noise_shape) == 3 and noise_shape[1] == 1 + if is_packed: + # TODO: latent_shapes cond isn't attached yet at this point, so we can't compute a + # per-window flat latent here. Skipping the clamp over-estimates but prevents immediate OOM. + pass + elif handler.dim < len(noise_shape) and noise_shape[handler.dim] > handler.context_length: + noise_shape[handler.dim] = min(noise_shape[handler.dim], handler.context_length) + return executor(model, noise_shape, conds, *args, **kwargs) def create_prepare_sampling_wrapper(model: ModelPatcher): @@ -422,11 +773,12 @@ def _sampler_sample_wrapper(executor, guider, sigmas, extra_args, callback, nois raise Exception("context_handler not found in sampler_sample_wrapper; this should never happen, something went wrong.") if not handler.freenoise: return executor(guider, sigmas, extra_args, callback, noise, *args, **kwargs) - noise = apply_freenoise(noise, handler.dim, handler.context_length, handler.context_overlap, extra_args["seed"]) + + conds = [guider.conds.get('positive', guider.conds.get('negative', []))] + noise = handler._apply_freenoise(noise, conds, extra_args["seed"]) return executor(guider, sigmas, extra_args, callback, noise, *args, **kwargs) - def create_sampler_sample_wrapper(model: ModelPatcher): model.add_wrapper_with_key( comfy.patcher_extension.WrappersMP.SAMPLER_SAMPLE, @@ -434,7 +786,6 @@ def create_sampler_sample_wrapper(model: ModelPatcher): _sampler_sample_wrapper ) - def match_weights_to_dim(weights: list[float], x_in: torch.Tensor, dim: int, device=None) -> torch.Tensor: total_dims = len(x_in.shape) weights_tensor = torch.Tensor(weights).to(device=device) @@ -580,8 +931,9 @@ def get_matching_context_schedule(context_schedule: str) -> ContextSchedule: return ContextSchedule(context_schedule, func) -def get_context_weights(length: int, full_length: int, idxs: list[int], handler: IndexListContextHandler, sigma: torch.Tensor=None): - return handler.fuse_method.func(length, sigma=sigma, handler=handler, full_length=full_length, idxs=idxs) +def get_context_weights(length: int, full_length: int, idxs: list[int], handler: IndexListContextHandler, sigma: torch.Tensor=None, context_overlap: int=None): + context_overlap = handler.context_overlap if context_overlap is None else context_overlap + return handler.fuse_method.func(length, sigma=sigma, handler=handler, full_length=full_length, idxs=idxs, context_overlap=context_overlap) def create_weights_flat(length: int, **kwargs) -> list[float]: @@ -599,18 +951,18 @@ def create_weights_pyramid(length: int, **kwargs) -> list[float]: weight_sequence = list(range(1, max_weight, 1)) + [max_weight] + list(range(max_weight - 1, 0, -1)) return weight_sequence -def create_weights_overlap_linear(length: int, full_length: int, idxs: list[int], handler: IndexListContextHandler, **kwargs): +def create_weights_overlap_linear(length: int, full_length: int, idxs: list[int], context_overlap: int, **kwargs): # based on code in Kijai's WanVideoWrapper: https://github.com/kijai/ComfyUI-WanVideoWrapper/blob/dbb2523b37e4ccdf45127e5ae33e31362f755c8e/nodes.py#L1302 # only expected overlap is given different weights weights_torch = torch.ones((length)) # blend left-side on all except first window if min(idxs) > 0: - ramp_up = torch.linspace(1e-37, 1, handler.context_overlap) - weights_torch[:handler.context_overlap] = ramp_up + ramp_up = torch.linspace(1e-37, 1, context_overlap) + weights_torch[:context_overlap] = ramp_up # blend right-side on all except last window if max(idxs) < full_length-1: - ramp_down = torch.linspace(1, 1e-37, handler.context_overlap) - weights_torch[-handler.context_overlap:] = ramp_down + ramp_down = torch.linspace(1, 1e-37, context_overlap) + weights_torch[-context_overlap:] = ramp_down return weights_torch class ContextFuseMethods: diff --git a/comfy/ldm/boogu/model.py b/comfy/ldm/boogu/model.py new file mode 100644 index 000000000..966f3c583 --- /dev/null +++ b/comfy/ldm/boogu/model.py @@ -0,0 +1,321 @@ +# Boogu-Image-0.1 transformer +# Architecture is an OmniGen2 derivative (see comfy/ldm/omnigen/omnigen2.py) with an +# added dual-stream ("double_stream") stage before the single-stream layers, conditioned +# by a Qwen3-VL multimodal LLM. Reuses the OmniGen2/Lumina building blocks and the Flux +# RoPE core, the only new component is the double-stream block + the hybrid forward order. + +from typing import Optional, Tuple + +import torch +import torch.nn as nn +from einops import rearrange + +import comfy.ldm.common_dit +import comfy.ldm.omnigen.omnigen2 +from comfy.ldm.modules.attention import optimized_attention_masked +from comfy.ldm.omnigen.omnigen2 import ( + OmniGen2RotaryPosEmbed, + Lumina2CombinedTimestepCaptionEmbedding, + LuminaRMSNormZero, + LuminaLayerNormContinuous, + LuminaFeedForward, + Attention, + OmniGen2TransformerBlock, + apply_rotary_emb, +) + +class BooguDoubleStreamProcessor(nn.Module): + # Joint attention over [instruct ; img] with separate per-stream q/k/v and output projections. + def __init__(self, dim, head_dim, heads, kv_heads, dtype=None, device=None, operations=None): + super().__init__() + query_dim = head_dim * heads + kv_dim = head_dim * kv_heads + + self.img_to_q = operations.Linear(query_dim, query_dim, bias=False, dtype=dtype, device=device) + self.img_to_k = operations.Linear(query_dim, kv_dim, bias=False, dtype=dtype, device=device) + self.img_to_v = operations.Linear(query_dim, kv_dim, bias=False, dtype=dtype, device=device) + + self.instruct_to_q = operations.Linear(query_dim, query_dim, bias=False, dtype=dtype, device=device) + self.instruct_to_k = operations.Linear(query_dim, kv_dim, bias=False, dtype=dtype, device=device) + self.instruct_to_v = operations.Linear(query_dim, kv_dim, bias=False, dtype=dtype, device=device) + + self.instruct_out = operations.Linear(query_dim, query_dim, bias=False, dtype=dtype, device=device) + self.img_out = operations.Linear(query_dim, query_dim, bias=False, dtype=dtype, device=device) + + def forward(self, attn, img_hidden_states, instruct_hidden_states, rotary_emb, attention_mask=None, transformer_options={}): + batch_size = img_hidden_states.shape[0] + L_instruct = instruct_hidden_states.shape[1] + + img_q = self.img_to_q(img_hidden_states) + img_k = self.img_to_k(img_hidden_states) + img_v = self.img_to_v(img_hidden_states) + + instruct_q = self.instruct_to_q(instruct_hidden_states) + instruct_k = self.instruct_to_k(instruct_hidden_states) + instruct_v = self.instruct_to_v(instruct_hidden_states) + + # Concatenate instruction first, then image (matches reference processor order). + query = torch.cat([instruct_q, img_q], dim=1) + key = torch.cat([instruct_k, img_k], dim=1) + value = torch.cat([instruct_v, img_v], dim=1) + + query = query.view(batch_size, -1, attn.heads, attn.dim_head) + key = key.view(batch_size, -1, attn.kv_heads, attn.dim_head) + value = value.view(batch_size, -1, attn.kv_heads, attn.dim_head) + + query = attn.norm_q(query) + key = attn.norm_k(key) + + if rotary_emb is not None: + query = apply_rotary_emb(query, rotary_emb) + key = apply_rotary_emb(key, rotary_emb) + + query = query.transpose(1, 2) + key = key.transpose(1, 2) + value = value.transpose(1, 2) + + if attn.kv_heads < attn.heads: + key = key.repeat_interleave(attn.heads // attn.kv_heads, dim=1) + value = value.repeat_interleave(attn.heads // attn.kv_heads, dim=1) + + hidden_states = optimized_attention_masked(query, key, value, attn.heads, attention_mask, skip_reshape=True, transformer_options=transformer_options) + + # Split back to instruction/image, apply per-stream output projections, recombine. + instruct_hidden_states = self.instruct_out(hidden_states[:, :L_instruct]) + img_hidden_states = self.img_out(hidden_states[:, L_instruct:]) + hidden_states = torch.cat([instruct_hidden_states, img_hidden_states], dim=1) + + hidden_states = attn.to_out[0](hidden_states) + return hidden_states + + +class BooguJointAttention(nn.Module): + # Holds the shared q/k RMSNorm + final output projection + def __init__(self, dim, head_dim, heads, kv_heads, eps=1e-5, dtype=None, device=None, operations=None): + super().__init__() + self.heads = heads + self.kv_heads = kv_heads + self.dim_head = head_dim + self.scale = head_dim ** -0.5 + + self.norm_q = operations.RMSNorm(head_dim, eps=eps, dtype=dtype, device=device) + self.norm_k = operations.RMSNorm(head_dim, eps=eps, dtype=dtype, device=device) + self.to_out = nn.Sequential( + operations.Linear(heads * head_dim, dim, bias=False, dtype=dtype, device=device), + nn.Dropout(0.0), + ) + self.processor = BooguDoubleStreamProcessor(dim, head_dim, heads, kv_heads, dtype=dtype, device=device, operations=operations) + + def forward(self, img_hidden_states, instruct_hidden_states, rotary_emb, attention_mask=None, transformer_options={}): + return self.processor(self, img_hidden_states, instruct_hidden_states, rotary_emb, attention_mask, transformer_options=transformer_options) + + +class BooguDoubleStreamBlock(nn.Module): + # Dual-stream block: joint attention over [instruct ; img] + image self-attention, each stream with its own modulation/MLP. + def __init__(self, dim, num_attention_heads, num_kv_heads, multiple_of, ffn_dim_multiplier, norm_eps, dtype=None, device=None, operations=None): + super().__init__() + head_dim = dim // num_attention_heads + + self.img_instruct_attn = BooguJointAttention(dim, head_dim, num_attention_heads, num_kv_heads, eps=1e-5, dtype=dtype, device=device, operations=operations) + self.img_self_attn = Attention( + query_dim=dim, dim_head=head_dim, heads=num_attention_heads, kv_heads=num_kv_heads, + eps=1e-5, bias=False, dtype=dtype, device=device, operations=operations, + ) + + self.img_feed_forward = LuminaFeedForward(dim=dim, inner_dim=4 * dim, multiple_of=multiple_of, dtype=dtype, device=device, operations=operations) + self.instruct_feed_forward = LuminaFeedForward(dim=dim, inner_dim=4 * dim, multiple_of=multiple_of, dtype=dtype, device=device, operations=operations) + + self.img_norm1 = LuminaRMSNormZero(embedding_dim=dim, norm_eps=norm_eps, dtype=dtype, device=device, operations=operations) + self.img_norm2 = LuminaRMSNormZero(embedding_dim=dim, norm_eps=norm_eps, dtype=dtype, device=device, operations=operations) + self.img_norm3 = LuminaRMSNormZero(embedding_dim=dim, norm_eps=norm_eps, dtype=dtype, device=device, operations=operations) + self.instruct_norm1 = LuminaRMSNormZero(embedding_dim=dim, norm_eps=norm_eps, dtype=dtype, device=device, operations=operations) + self.instruct_norm2 = LuminaRMSNormZero(embedding_dim=dim, norm_eps=norm_eps, dtype=dtype, device=device, operations=operations) + + self.img_attn_norm = operations.RMSNorm(dim, eps=norm_eps, dtype=dtype, device=device) + self.img_self_attn_norm = operations.RMSNorm(dim, eps=norm_eps, dtype=dtype, device=device) + self.img_ffn_norm1 = operations.RMSNorm(dim, eps=norm_eps, dtype=dtype, device=device) + self.img_ffn_norm2 = operations.RMSNorm(dim, eps=norm_eps, dtype=dtype, device=device) + + self.instruct_attn_norm = operations.RMSNorm(dim, eps=norm_eps, dtype=dtype, device=device) + self.instruct_ffn_norm1 = operations.RMSNorm(dim, eps=norm_eps, dtype=dtype, device=device) + self.instruct_ffn_norm2 = operations.RMSNorm(dim, eps=norm_eps, dtype=dtype, device=device) + + def forward(self, img_hidden_states, instruct_hidden_states, joint_rotary_emb, img_rotary_emb, temb, joint_attention_mask=None, img_attention_mask=None, transformer_options={}): + L_instruct = instruct_hidden_states.shape[1] + + img_norm1_out, img_gate_msa, img_scale_mlp, img_gate_mlp = self.img_norm1(img_hidden_states, temb) + img_norm2_out, img_shift_mlp, _, _ = self.img_norm2(img_hidden_states, temb) + img_norm3_out, img_gate_self, _, _ = self.img_norm3(img_hidden_states, temb) + + instruct_norm1_out, instruct_gate_msa, instruct_scale_mlp, instruct_gate_mlp = self.instruct_norm1(instruct_hidden_states, temb) + instruct_norm2_out, instruct_shift_mlp, _, _ = self.instruct_norm2(instruct_hidden_states, temb) + + joint_attn_out = self.img_instruct_attn(img_norm1_out, instruct_norm1_out, joint_rotary_emb, joint_attention_mask, transformer_options=transformer_options) + instruct_attn_out = joint_attn_out[:, :L_instruct] + img_attn_out = joint_attn_out[:, L_instruct:] + + img_self_attn_out = self.img_self_attn(img_norm3_out, img_norm3_out, img_attention_mask, img_rotary_emb, transformer_options=transformer_options) + + img_hidden_states = img_hidden_states + img_gate_msa.unsqueeze(1).tanh() * self.img_attn_norm(img_attn_out) + img_hidden_states = img_hidden_states + img_gate_self.unsqueeze(1).tanh() * self.img_self_attn_norm(img_self_attn_out) + img_mlp_input = (1 + img_scale_mlp.unsqueeze(1)) * img_norm2_out + img_shift_mlp.unsqueeze(1) + img_mlp_out = self.img_feed_forward(self.img_ffn_norm1(img_mlp_input)) + img_hidden_states = img_hidden_states + img_gate_mlp.unsqueeze(1).tanh() * self.img_ffn_norm2(img_mlp_out) + + instruct_hidden_states = instruct_hidden_states + instruct_gate_msa.unsqueeze(1).tanh() * self.instruct_attn_norm(instruct_attn_out) + instruct_mlp_input = (1 + instruct_scale_mlp.unsqueeze(1)) * instruct_norm2_out + instruct_shift_mlp.unsqueeze(1) + instruct_mlp_out = self.instruct_feed_forward(self.instruct_ffn_norm1(instruct_mlp_input)) + instruct_hidden_states = instruct_hidden_states + instruct_gate_mlp.unsqueeze(1).tanh() * self.instruct_ffn_norm2(instruct_mlp_out) + + return img_hidden_states, instruct_hidden_states + + +class BooguTransformer2DModel(nn.Module): + def __init__( + self, + patch_size: int = 2, + in_channels: int = 16, + out_channels: Optional[int] = None, + hidden_size: int = 3360, + num_layers: int = 32, + num_double_stream_layers: int = 8, + num_refiner_layers: int = 2, + num_attention_heads: int = 28, + num_kv_heads: int = 7, + multiple_of: int = 256, + ffn_dim_multiplier: Optional[float] = None, + norm_eps: float = 1e-5, + axes_dim_rope: Tuple[int, int, int] = (40, 40, 40), + axes_lens: Tuple[int, int, int] = (2048, 1664, 1664), + instruction_feat_dim: int = 4096, + timestep_scale: float = 1000.0, + image_model=None, + device=None, dtype=None, operations=None, + ): + super().__init__() + + self.patch_size = patch_size + self.out_channels = out_channels or in_channels + self.hidden_size = hidden_size + self.dtype = dtype + + self.rope_embedder = OmniGen2RotaryPosEmbed( + theta=10000, + axes_dim=axes_dim_rope, + axes_lens=axes_lens, + patch_size=patch_size, + ) + + self.x_embedder = operations.Linear(patch_size * patch_size * in_channels, hidden_size, dtype=dtype, device=device) + self.ref_image_patch_embedder = operations.Linear(patch_size * patch_size * in_channels, hidden_size, dtype=dtype, device=device) + + self.time_caption_embed = Lumina2CombinedTimestepCaptionEmbedding( + hidden_size=hidden_size, + text_feat_dim=instruction_feat_dim, + norm_eps=norm_eps, + timestep_scale=timestep_scale, dtype=dtype, device=device, operations=operations + ) + + self.noise_refiner = nn.ModuleList([ + OmniGen2TransformerBlock(hidden_size, num_attention_heads, num_kv_heads, multiple_of, ffn_dim_multiplier, norm_eps, modulation=True, dtype=dtype, device=device, operations=operations) + for _ in range(num_refiner_layers) + ]) + + self.ref_image_refiner = nn.ModuleList([ + OmniGen2TransformerBlock(hidden_size, num_attention_heads, num_kv_heads, multiple_of, ffn_dim_multiplier, norm_eps, modulation=True, dtype=dtype, device=device, operations=operations) + for _ in range(num_refiner_layers) + ]) + + self.context_refiner = nn.ModuleList([ + OmniGen2TransformerBlock(hidden_size, num_attention_heads, num_kv_heads, multiple_of, ffn_dim_multiplier, norm_eps, modulation=False, dtype=dtype, device=device, operations=operations) + for _ in range(num_refiner_layers) + ]) + + self.double_stream_layers = nn.ModuleList([ + BooguDoubleStreamBlock(hidden_size, num_attention_heads, num_kv_heads, multiple_of, ffn_dim_multiplier, norm_eps, dtype=dtype, device=device, operations=operations) + for _ in range(num_double_stream_layers) + ]) + + self.single_stream_layers = nn.ModuleList([ + OmniGen2TransformerBlock(hidden_size, num_attention_heads, num_kv_heads, multiple_of, ffn_dim_multiplier, norm_eps, modulation=True, dtype=dtype, device=device, operations=operations) + for _ in range(num_layers) + ]) + + self.norm_out = LuminaLayerNormContinuous( + embedding_dim=hidden_size, + conditioning_embedding_dim=min(hidden_size, 1024), + elementwise_affine=False, + eps=1e-6, + out_dim=patch_size * patch_size * self.out_channels, dtype=dtype, device=device, operations=operations + ) + + self.image_index_embedding = nn.Parameter(torch.empty(5, hidden_size, device=device, dtype=dtype)) + + # Patchify/refine helpers are identical to OmniGen2; reuse via bound methods. + flat_and_pad_to_seq = comfy.ldm.omnigen.omnigen2.OmniGen2Transformer2DModel.flat_and_pad_to_seq + img_patch_embed_and_refine = comfy.ldm.omnigen.omnigen2.OmniGen2Transformer2DModel.img_patch_embed_and_refine + + def forward(self, x, timesteps, context, num_tokens, ref_latents=None, attention_mask=None, transformer_options={}, **kwargs): + B, C, H, W = x.shape + hidden_states = comfy.ldm.common_dit.pad_to_patch_size(x, (self.patch_size, self.patch_size)) + _, _, H_padded, W_padded = hidden_states.shape + timestep = 1.0 - timesteps + text_hidden_states = context + text_attention_mask = attention_mask + ref_image_hidden_states = ref_latents + device = hidden_states.device + + temb, text_hidden_states = self.time_caption_embed(timestep, text_hidden_states, hidden_states[0].dtype) + + ( + hidden_states, ref_image_hidden_states, + img_mask, ref_img_mask, + l_effective_ref_img_len, l_effective_img_len, + ref_img_sizes, img_sizes, + ) = self.flat_and_pad_to_seq(hidden_states, ref_image_hidden_states) + + ( + context_rotary_emb, ref_img_rotary_emb, noise_rotary_emb, + rotary_emb, encoder_seq_lengths, seq_lengths, + ) = self.rope_embedder( + hidden_states.shape[0], text_hidden_states.shape[1], [num_tokens] * text_hidden_states.shape[0], + l_effective_ref_img_len, l_effective_img_len, + ref_img_sizes, img_sizes, device, + ) + + for layer in self.context_refiner: + text_hidden_states = layer(text_hidden_states, text_attention_mask, context_rotary_emb, transformer_options=transformer_options) + + img_len = hidden_states.shape[1] + combined_img_hidden_states = self.img_patch_embed_and_refine( + hidden_states, ref_image_hidden_states, + img_mask, ref_img_mask, + noise_rotary_emb, ref_img_rotary_emb, + l_effective_ref_img_len, l_effective_img_len, + temb, + transformer_options=transformer_options, + ) + + # Double-stream stage: the image self-attention only sees the [ref ; noise] tokens, + # which sit after the instruction tokens in the joint rope. + L_instruct = text_hidden_states.shape[1] + combined_img_rotary_emb = rotary_emb[:, L_instruct:] + for layer in self.double_stream_layers: + combined_img_hidden_states, text_hidden_states = layer( + combined_img_hidden_states, text_hidden_states, + rotary_emb, combined_img_rotary_emb, temb, + joint_attention_mask=None, img_attention_mask=None, + transformer_options=transformer_options, + ) + + hidden_states = torch.cat([text_hidden_states, combined_img_hidden_states], dim=1) + + for layer in self.single_stream_layers: + hidden_states = layer(hidden_states, None, rotary_emb, temb, transformer_options=transformer_options) + + hidden_states = self.norm_out(hidden_states, temb) + + p = self.patch_size + output = rearrange(hidden_states[:, -img_len:], 'b (h w) (p1 p2 c) -> b c (h p1) (w p2)', h=H_padded // p, w=W_padded // p, p1=p, p2=p)[:, :, :H, :W] + + return -output diff --git a/comfy/ldm/cosmos/predict2.py b/comfy/ldm/cosmos/predict2.py index 671fe834d..aec874815 100644 --- a/comfy/ldm/cosmos/predict2.py +++ b/comfy/ldm/cosmos/predict2.py @@ -515,7 +515,7 @@ class Block(nn.Module): h=H, w=W, ) - x_B_T_H_W_D = x_B_T_H_W_D + gate_self_attn_B_T_1_1_D.to(residual_dtype) * result_B_T_H_W_D.to(residual_dtype) + x_B_T_H_W_D = torch.addcmul(x_B_T_H_W_D, gate_self_attn_B_T_1_1_D.to(residual_dtype), result_B_T_H_W_D.to(residual_dtype)) def _x_fn( _x_B_T_H_W_D: torch.Tensor, @@ -548,7 +548,7 @@ class Block(nn.Module): shift_cross_attn_B_T_1_1_D, transformer_options=transformer_options, ) - x_B_T_H_W_D = result_B_T_H_W_D.to(residual_dtype) * gate_cross_attn_B_T_1_1_D.to(residual_dtype) + x_B_T_H_W_D + x_B_T_H_W_D = torch.addcmul(x_B_T_H_W_D, gate_cross_attn_B_T_1_1_D.to(residual_dtype), result_B_T_H_W_D.to(residual_dtype)) normalized_x_B_T_H_W_D = _fn( x_B_T_H_W_D, @@ -557,7 +557,7 @@ class Block(nn.Module): shift_mlp_B_T_1_1_D, ) result_B_T_H_W_D = self.mlp(normalized_x_B_T_H_W_D.to(compute_dtype)) - x_B_T_H_W_D = x_B_T_H_W_D + gate_mlp_B_T_1_1_D.to(residual_dtype) * result_B_T_H_W_D.to(residual_dtype) + x_B_T_H_W_D = torch.addcmul(x_B_T_H_W_D, gate_mlp_B_T_1_1_D.to(residual_dtype), result_B_T_H_W_D.to(residual_dtype)) return x_B_T_H_W_D diff --git a/comfy/ldm/ideogram4/model.py b/comfy/ldm/ideogram4/model.py index b86c65bf0..4ea5b8aaf 100644 --- a/comfy/ldm/ideogram4/model.py +++ b/comfy/ldm/ideogram4/model.py @@ -106,11 +106,11 @@ class Ideogram4EmbedScalar(nn.Module): self.mlp_in = operations.Linear(dim, dim, bias=True, dtype=dtype, device=device) self.mlp_out = operations.Linear(dim, dim, bias=True, dtype=dtype, device=device) - def forward(self, x): + def forward(self, x, dtype): x = x.to(torch.float32) scaled = 1e4 * (x - self.range_min) / (self.range_max - self.range_min) emb = _sinusoidal_embedding(scaled, self.dim) - emb = emb.to(self.mlp_in.weight.dtype) + emb = emb.to(dtype) emb = F.silu(self.mlp_in(emb)) return self.mlp_out(emb) @@ -161,7 +161,7 @@ class Ideogram4Transformer(nn.Module): x = x * output_image_mask h = self.input_proj(x) * output_image_mask - t_cond = self.t_embedding(t) + t_cond = self.t_embedding(t, dtype=x.dtype) if t.dim() == 1: t_cond = t_cond.unsqueeze(1) adaln_input = F.silu(self.adaln_proj(t_cond)) diff --git a/comfy/ldm/krea2/model.py b/comfy/ldm/krea2/model.py new file mode 100644 index 000000000..ecb16254f --- /dev/null +++ b/comfy/ldm/krea2/model.py @@ -0,0 +1,290 @@ +"""Krea 2 (K2) — single-stream MMDiT. + +Text tokens produced by a Qwen3-VL-4B 12-layer ``txtfusion`` adapter and patchified image tokens are +concatenated into one sequence and run through ``layers`` shared transformer blocks with +AdaLN-single modulation, GQA + per-head QK-norm + sigmoid-gated attention, SwiGLU MLP, and 3-axis RoPE. +""" + +from typing import Optional + +import torch +import torch.nn as nn +import torch.nn.functional as F +from einops import rearrange + +import comfy.model_management +import comfy.patcher_extension +import comfy.ldm.common_dit +from comfy.ldm.flux.layers import EmbedND, timestep_embedding +from comfy.ldm.flux.math import apply_rope +from comfy.ldm.modules.attention import optimized_attention_masked + + +class RMSNorm(nn.Module): + """RMSNorm with the reference ``(1 + scale)`` weight convention (scale stored zero-centered).""" + + def __init__(self, features: int, eps: float = 1e-5, device=None, dtype=None, operations=None): + super().__init__() + self.eps = eps + self.scale = nn.Parameter(torch.empty(features, device=device, dtype=dtype)) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + dtype = x.dtype + weight = comfy.model_management.cast_to(self.scale, dtype=torch.float32, device=x.device) + 1.0 + return F.rms_norm(x.float(), (x.shape[-1],), weight=weight, eps=self.eps).to(dtype) + + +class QKNorm(nn.Module): + def __init__(self, dim: int, device=None, dtype=None, operations=None): + super().__init__() + self.qnorm = RMSNorm(dim, device=device, dtype=dtype, operations=operations) + self.knorm = RMSNorm(dim, device=device, dtype=dtype, operations=operations) + + def forward(self, q, k): + return self.qnorm(q), self.knorm(k) + + +class SwiGLU(nn.Module): + def __init__(self, features: int, multiplier: int, bias: bool = False, multiple: int = 128, + device=None, dtype=None, operations=None): + super().__init__() + mlpdim = int(2 * features / 3) * multiplier + mlpdim = multiple * ((mlpdim + multiple - 1) // multiple) + self.gate = operations.Linear(features, mlpdim, bias=bias, device=device, dtype=dtype) + self.up = operations.Linear(features, mlpdim, bias=bias, device=device, dtype=dtype) + self.down = operations.Linear(mlpdim, features, bias=bias, device=device, dtype=dtype) + + def forward(self, x): + return self.down(F.silu(self.gate(x)).mul_(self.up(x))) + + +class Attention(nn.Module): + def __init__(self, dim: int, heads: int, kvheads: Optional[int] = None, bias: bool = False, + device=None, dtype=None, operations=None): + super().__init__() + self.heads = heads + self.kvheads = kvheads if kvheads is not None else heads + self.headdim = dim // self.heads + self.wq = operations.Linear(dim, self.headdim * self.heads, bias=bias, device=device, dtype=dtype) + self.wk = operations.Linear(dim, self.headdim * self.kvheads, bias=bias, device=device, dtype=dtype) + self.wv = operations.Linear(dim, self.headdim * self.kvheads, bias=bias, device=device, dtype=dtype) + self.gate = operations.Linear(dim, dim, bias=bias, device=device, dtype=dtype) + self.qknorm = QKNorm(self.headdim, device=device, dtype=dtype, operations=operations) + self.wo = operations.Linear(dim, dim, bias=bias, device=device, dtype=dtype) + + def forward(self, x, freqs=None, mask=None, transformer_options={}): + q, k, v, gate = self.wq(x), self.wk(x), self.wv(x), self.gate(x) + q = rearrange(q, "B L (H D) -> B H L D", H=self.heads) + k = rearrange(k, "B L (H D) -> B H L D", H=self.kvheads) + v = rearrange(v, "B L (H D) -> B H L D", H=self.kvheads) + q, k = self.qknorm(q, k) + if freqs is not None: + q, k = apply_rope(q, k, freqs) + if self.kvheads != self.heads: + rep = self.heads // self.kvheads + k = k.repeat_interleave(rep, dim=1) + v = v.repeat_interleave(rep, dim=1) + out = optimized_attention_masked(q, k, v, self.heads, mask=mask, skip_reshape=True, + transformer_options=transformer_options) + return self.wo(out * F.sigmoid(gate)) + + +class SimpleModulation(nn.Module): + def __init__(self, dim: int, device=None, dtype=None, operations=None): + super().__init__() + self.lin = nn.Parameter(torch.empty(2, dim, device=device, dtype=dtype)) + + def forward(self, vec): + out = vec + comfy.model_management.cast_to(self.lin, dtype=vec.dtype, device=vec.device).unsqueeze(0) + scale, shift = out.chunk(2, dim=1) + return scale, shift + + +class DoubleSharedModulation(nn.Module): + def __init__(self, dim: int, device=None, dtype=None, operations=None): + super().__init__() + self.lin = nn.Parameter(torch.empty(6 * dim, device=device, dtype=dtype)) + + def forward(self, vec): + out = vec + comfy.model_management.cast_to(self.lin, dtype=vec.dtype, device=vec.device) + return out.chunk(6, dim=-1) + + +class TextFusionBlock(nn.Module): + def __init__(self, features, heads, multiplier, bias=False, kvheads=None, device=None, dtype=None, operations=None): + super().__init__() + self.prenorm = RMSNorm(features, device=device, dtype=dtype, operations=operations) + self.postnorm = RMSNorm(features, device=device, dtype=dtype, operations=operations) + self.attn = Attention(features, heads, kvheads=kvheads, bias=bias, device=device, dtype=dtype, operations=operations) + self.mlp = SwiGLU(features, multiplier, bias, device=device, dtype=dtype, operations=operations) + + def forward(self, x, mask=None, transformer_options={}): + x = x + self.attn(self.prenorm(x), mask=mask, transformer_options=transformer_options) + x = x + self.mlp(self.postnorm(x)) + return x + + +class TextFusionTransformer(nn.Module): + def __init__(self, num_txt_layers, txt_dim, heads, multiplier, bias=False, kvheads=None, device=None, dtype=None, operations=None): + super().__init__() + self.layerwise_blocks = nn.ModuleList([ + TextFusionBlock(txt_dim, heads, multiplier, bias, kvheads, device=device, dtype=dtype, operations=operations) + for _ in range(2) + ]) + self.projector = operations.Linear(num_txt_layers, 1, bias=False, device=device, dtype=dtype) + self.refiner_blocks = nn.ModuleList([ + TextFusionBlock(txt_dim, heads, multiplier, bias, kvheads, device=device, dtype=dtype, operations=operations) + for _ in range(2) + ]) + + def forward(self, x, mask=None, transformer_options={}): + b, l, n, d = x.shape + x = x.reshape(b * l, n, d) + for block in self.layerwise_blocks: + x = block(x.contiguous(), mask=None, transformer_options=transformer_options) + x = rearrange(x, "(b l) n d -> b l d n", b=b, l=l) + x = self.projector(x).squeeze(-1) + for block in self.refiner_blocks: + x = block(x, mask=mask, transformer_options=transformer_options) + return x + + +class SingleStreamBlock(nn.Module): + def __init__(self, features, heads, multiplier, bias=False, kvheads=None, device=None, dtype=None, operations=None): + super().__init__() + self.mod = DoubleSharedModulation(features, device=device, dtype=dtype, operations=operations) + self.prenorm = RMSNorm(features, device=device, dtype=dtype, operations=operations) + self.postnorm = RMSNorm(features, device=device, dtype=dtype, operations=operations) + self.attn = Attention(features, heads, kvheads=kvheads, bias=bias, device=device, dtype=dtype, operations=operations) + self.mlp = SwiGLU(features, multiplier, bias, device=device, dtype=dtype, operations=operations) + + def forward(self, x, vec, freqs, mask=None, transformer_options={}): + prescale, preshift, pregate, postscale, postshift, postgate = self.mod(vec) + x = x + pregate * self.attn((1 + prescale) * self.prenorm(x) + preshift, freqs, mask, transformer_options=transformer_options) + x = x + postgate * self.mlp((1 + postscale) * self.postnorm(x) + postshift) + return x + + +class LastLayer(nn.Module): + def __init__(self, features, patch, channels, device=None, dtype=None, operations=None): + super().__init__() + self.norm = RMSNorm(features, device=device, dtype=dtype, operations=operations) + self.linear = operations.Linear(features, patch * patch * channels, bias=True, device=device, dtype=dtype) + self.modulation = SimpleModulation(features, device=device, dtype=dtype, operations=operations) + + def forward(self, x, tvec): + scale, shift = self.modulation(tvec) + x = (1 + scale) * self.norm(x) + shift + return self.linear(x) + + +class SingleStreamDiT(nn.Module): + def __init__(self, features=6144, tdim=256, txtdim=2560, heads=48, kvheads=12, multiplier=4, + layers=28, patch=2, channels=16, bias=False, theta=1e3, txtlayers=12, + txtheads=20, txtkvheads=20, image_model=None, + device=None, dtype=None, operations=None, **kwargs): + super().__init__() + self.dtype = dtype + self.patch = patch + self.channels = channels + self.tdim = tdim + self.heads = heads + self.txtdim = txtdim + self.txtlayers = txtlayers + + headdim = features // heads + axes = [headdim - 12 * (headdim // 16), 6 * (headdim // 16), 6 * (headdim // 16)] + assert sum(axes) == headdim, f"axes {axes} sum != headdim {headdim}" + self.pe_embedder = EmbedND(dim=headdim, theta=int(theta), axes_dim=axes) + + self.first = operations.Linear(channels * patch ** 2, features, bias=True, device=device, dtype=dtype) + self.blocks = nn.ModuleList([ + SingleStreamBlock(features, heads, multiplier, bias, kvheads, device=device, dtype=dtype, operations=operations) + for _ in range(layers) + ]) + self.tmlp = nn.Sequential( + operations.Linear(tdim, features, device=device, dtype=dtype), + nn.GELU(approximate="tanh"), + operations.Linear(features, features, device=device, dtype=dtype), + ) + self.txtfusion = TextFusionTransformer(txtlayers, txtdim, txtheads, multiplier, bias, txtkvheads, + device=device, dtype=dtype, operations=operations) + self.txtmlp = nn.Sequential( + RMSNorm(txtdim, device=device, dtype=dtype, operations=operations), + operations.Linear(txtdim, features, device=device, dtype=dtype), + nn.GELU(approximate="tanh"), + operations.Linear(features, features, device=device, dtype=dtype), + ) + self.last = LastLayer(features, patch, channels, device=device, dtype=dtype, operations=operations) + self.tproj = nn.Sequential( + nn.GELU(approximate="tanh"), + operations.Linear(features, features * 6, device=device, dtype=dtype), + ) + + def forward(self, x, timesteps, context, attention_mask=None, transformer_options={}, **kwargs): + return comfy.patcher_extension.WrapperExecutor.new_class_executor( + self._forward, + self, + comfy.patcher_extension.get_all_wrappers(comfy.patcher_extension.WrappersMP.DIFFUSION_MODEL, transformer_options), + ).execute(x, timesteps, context, attention_mask, transformer_options, **kwargs) + + def _forward(self, x, timesteps, context, attention_mask=None, transformer_options={}, **kwargs): + temporal = x.ndim == 5 + if temporal: + b5, c5, t5, h5, w5 = x.shape + x = x.reshape(b5 * t5, c5, h5, w5) + bs, c, H_orig, W_orig = x.shape + patch = self.patch + # Pad the latent up to a multiple of patch (as Flux/Lumina/QwenImage do); crop back at the end. + x = comfy.ldm.common_dit.pad_to_patch_size(x, (patch, patch)) + H, W = x.shape[-2], x.shape[-1] + h_, w_ = H // patch, W // patch + + # context arrives as (B, seq, txtlayers*txtdim); reshape to (B, txtlayers, seq, txtdim). + context = self._unpack_context(context) + + img = rearrange(x, "b c (h ph) (w pw) -> b (h w) (c ph pw)", ph=patch, pw=patch) + img = self.first(img) + + t = self.tmlp(timestep_embedding(timesteps, self.tdim).unsqueeze(1).to(img.dtype)) + tvec = self.tproj(t) + + context = self.txtfusion(context, mask=None, transformer_options=transformer_options) + context = self.txtmlp(context) + + txtlen, imglen = context.shape[1], img.shape[1] + combined = torch.cat((context, img), dim=1) + + # Position ids: text at 0, image at (0, h_idx, w_idx). + device = combined.device + txtpos = torch.zeros(bs, txtlen, 3, device=device, dtype=torch.float32) + imgids = torch.zeros(h_, w_, 3, device=device, dtype=torch.float32) + imgids[..., 1] = torch.arange(h_, device=device, dtype=torch.float32)[:, None] + imgids[..., 2] = torch.arange(w_, device=device, dtype=torch.float32)[None, :] + imgpos = imgids.reshape(1, h_ * w_, 3).repeat(bs, 1, 1) + pos = torch.cat((txtpos, imgpos), dim=1) + + freqs = self.pe_embedder(pos) + + for block in self.blocks: + combined = block(combined, tvec, freqs, None, transformer_options=transformer_options) + + final = self.last(combined, t) + out = final[:, txtlen:txtlen + imglen, :] + out = rearrange(out, "b (h w) (c ph pw) -> b c (h ph) (w pw)", + h=h_, w=w_, ph=patch, pw=patch, c=self.channels) + out = out[:, :, :H_orig, :W_orig] # crop padding back off + if temporal: + out = out.reshape(b5, t5, self.channels, H_orig, W_orig).movedim(1, 2) + return out + + def _unpack_context(self, context): + # context: (B, seq, txtlayers*txtdim) -> (B, seq, txtlayers, txtdim). + b, seq, fused = context.shape + if fused != self.txtlayers * self.txtdim: + raise ValueError( + f"Krea2 expects conditioning with {self.txtlayers}x{self.txtdim}={self.txtlayers * self.txtdim} " + f"features (a {self.txtlayers}-layer Qwen3-VL stack) but got {fused}. " + f"Load the text encoder with CLIPLoader type 'krea2'." + ) + return context.reshape(b, seq, self.txtlayers, self.txtdim) diff --git a/comfy/ldm/lightricks/model.py b/comfy/ldm/lightricks/model.py index e0a4a0f9b..9953b6679 100644 --- a/comfy/ldm/lightricks/model.py +++ b/comfy/ldm/lightricks/model.py @@ -1085,7 +1085,7 @@ class LTXVModel(LTXBaseModel): ) grid_mask = None - if keyframe_idxs is not None: + if keyframe_idxs is not None and keyframe_idxs.shape[2] > 0: additional_args.update({ "orig_patchified_shape": list(x.shape)}) denoise_mask = self.patchifier.patchify(denoise_mask)[0] grid_mask = ~torch.any(denoise_mask < 0, dim=-1)[0] @@ -1330,7 +1330,7 @@ class LTXVModel(LTXBaseModel): x = x * (1 + scale) + shift x = self.proj_out(x) - if keyframe_idxs is not None: + if keyframe_idxs is not None and keyframe_idxs.shape[2] > 0: grid_mask = kwargs["grid_mask"] orig_patchified_shape = kwargs["orig_patchified_shape"] full_x = torch.zeros(orig_patchified_shape, dtype=x.dtype, device=x.device) diff --git a/comfy/ldm/omnigen/omnigen2.py b/comfy/ldm/omnigen/omnigen2.py index 82edc92da..b8da4cf39 100644 --- a/comfy/ldm/omnigen/omnigen2.py +++ b/comfy/ldm/omnigen/omnigen2.py @@ -8,6 +8,7 @@ import torch.nn.functional as F from einops import rearrange, repeat from comfy.ldm.lightricks.model import Timesteps from comfy.ldm.flux.layers import EmbedND +from comfy.ldm.flux.math import apply_rope1 from comfy.ldm.modules.attention import optimized_attention_masked import comfy.model_management import comfy.ldm.common_dit @@ -17,13 +18,11 @@ def apply_rotary_emb(x, freqs_cis): if x.shape[1] == 0: return x - t_ = x.reshape(*x.shape[:-1], -1, 1, 2) - t_out = freqs_cis[..., 0] * t_[..., 0] + freqs_cis[..., 1] * t_[..., 1] - return t_out.reshape(*x.shape).to(dtype=x.dtype) + return apply_rope1(x, freqs_cis) def swiglu(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor: - return F.silu(x) * y + return F.silu(x, inplace=True).mul_(y) class TimestepEmbedding(nn.Module): diff --git a/comfy/ldm/wan/model.py b/comfy/ldm/wan/model.py index 282408891..1c9782a38 100644 --- a/comfy/ldm/wan/model.py +++ b/comfy/ldm/wan/model.py @@ -1665,7 +1665,7 @@ class SCAILWanModel(WanModel): # embeddings x = self.patch_embedding(x.float()).to(x.dtype) - if ref_mask_latents is not None: # SCAIL-2 additive mask stream + if ref_mask_latents is not None: # SCAIL-2 additive mask stream (one identity mask frame per reference, then video) x = x + self.patch_embedding_mask(ref_mask_latents.float()).to(x.dtype) grid_sizes = x.shape[2:] transformer_options["grid_sizes"] = grid_sizes @@ -1728,22 +1728,25 @@ class SCAILWanModel(WanModel): # ref_mask_flag is a scalar bool (CONDConstant, SCAIL-2 only). False => replacement mode, # which places ref/pose via H/W rope shifts instead of the animation-mode temporal offset. + # reference_latent may stack several frames: the last is the primary reference adjacent to the video, the earlier frames are additional references. def rope_encode(self, t, h, w, t_start=0, steps_t=None, steps_h=None, steps_w=None, device=None, dtype=None, pose_latents=None, reference_latent=None, ref_mask_flag=None, transformer_options={}): + ref_t_patches = 0 + if reference_latent is not None: + ref_t_patches = (reference_latent.shape[2] + (self.patch_size[0] // 2)) // self.patch_size[0] + if ref_mask_flag is not None and not bool(ref_mask_flag): REF_ROPE_H = 120.0 POSE_ROPE_W = 120.0 - ref_t_patches = 0 - if reference_latent is not None: - ref_t_patches = (reference_latent.shape[2] + (self.patch_size[0] // 2)) // self.patch_size[0] main_t_patches = t - ref_t_patches + video_t_start = max(ref_t_patches - 1, 0) parts = [] if ref_t_patches > 0: ref_tf = {"rope_options": {"shift_y": REF_ROPE_H, "shift_x": 0.0, "scale_y": 1.0, "scale_x": 1.0}} parts.append(super().rope_encode(ref_t_patches, h, w, t_start=0, device=device, dtype=dtype, transformer_options=ref_tf)) if main_t_patches > 0: - parts.append(super().rope_encode(main_t_patches, h, w, t_start=0, device=device, dtype=dtype, transformer_options=transformer_options)) + parts.append(super().rope_encode(main_t_patches, h, w, t_start=video_t_start, device=device, dtype=dtype, transformer_options=transformer_options)) if pose_latents is not None: F_pose, H_pose, W_pose = pose_latents.shape[-3], pose_latents.shape[-2], pose_latents.shape[-1] @@ -1752,7 +1755,7 @@ class SCAILWanModel(WanModel): h_shift = (h_scale - 1) / 2 w_shift = (w_scale - 1) / 2 pose_tf = {"rope_options": {"shift_y": h_shift, "shift_x": POSE_ROPE_W + w_shift, "scale_y": h_scale, "scale_x": w_scale}} - parts.append(super().rope_encode(F_pose, H_pose, W_pose, t_start=0, device=device, dtype=dtype, transformer_options=pose_tf)) + parts.append(super().rope_encode(F_pose, H_pose, W_pose, t_start=video_t_start, device=device, dtype=dtype, transformer_options=pose_tf)) return torch.cat(parts, dim=1) @@ -1761,10 +1764,6 @@ class SCAILWanModel(WanModel): if pose_latents is None: return main_freqs - ref_t_patches = 0 - if reference_latent is not None: - ref_t_patches = (reference_latent.shape[2] + (self.patch_size[0] // 2)) // self.patch_size[0] - F_pose, H_pose, W_pose = pose_latents.shape[-3], pose_latents.shape[-2], pose_latents.shape[-1] # if pose is at half resolution, scale_y/scale_x=2 stretches the position range to cover the same RoPE extent as the main frames diff --git a/comfy/lora.py b/comfy/lora.py index 2c8d0f0bf..427cf98aa 100644 --- a/comfy/lora.py +++ b/comfy/lora.py @@ -326,6 +326,17 @@ def model_lora_keys_unet(model, key_map={}): key_map["transformer.{}".format(key_lora)] = k key_map["lycoris_{}".format(key_lora.replace(".", "_"))] = k #SimpleTuner lycoris format + if isinstance(model, comfy.model_base.Krea2): + diffusers_keys = comfy.utils.krea2_to_diffusers(model.model_config.unet_config, output_prefix="diffusion_model.") + for k in diffusers_keys: + if k.endswith(".weight"): + to = diffusers_keys[k] + key_lora = k[:-len(".weight")] + key_map["diffusion_model.{}".format(key_lora)] = to + key_map["transformer.{}".format(key_lora)] = to + key_map["lycoris_{}".format(key_lora.replace(".", "_"))] = to + key_map[key_lora] = to + if isinstance(model, comfy.model_base.Lumina2): diffusers_keys = comfy.utils.z_image_to_diffusers(model.model_config.unet_config, output_prefix="diffusion_model.") for k in diffusers_keys: diff --git a/comfy/model_base.py b/comfy/model_base.py index 2289e0812..dcfa555dc 100644 --- a/comfy/model_base.py +++ b/comfy/model_base.py @@ -21,6 +21,7 @@ import comfy.ldm.hunyuan3dv2_1.hunyuandit import torch import logging import comfy.ldm.lightricks.av_model +import comfy.ldm.lightricks.symmetric_patchifier import comfy.context_windows from comfy.ldm.modules.diffusionmodules.openaimodel import UNetModel, Timestep from comfy.ldm.cascade.stage_c import StageC @@ -54,8 +55,10 @@ import comfy.ldm.pixeldit.model import comfy.ldm.pixeldit.pid import comfy.ldm.ace.model import comfy.ldm.omnigen.omnigen2 +import comfy.ldm.boogu.model import comfy.ldm.qwen_image.model import comfy.ldm.ideogram4.model +import comfy.ldm.krea2.model import comfy.ldm.kandinsky5.model import comfy.ldm.anima.model import comfy.ldm.ace.ace_step15 @@ -1203,6 +1206,127 @@ class LTXAV(BaseModel): def scale_latent_inpaint(self, sigma, noise, latent_image, **kwargs): return latent_image + def map_context_window_to_modalities(self, primary_indices, latent_shapes, dim): + result = [primary_indices] + if len(latent_shapes) < 2: + return result + + video_total = latent_shapes[0][dim] + + for i in range(1, len(latent_shapes)): + mod_total = latent_shapes[i][dim] + # Map each primary index to its proportional range of modality indices and + # concatenate in order. Preserves wrapped/strided geometry so the modality + # attends to the same temporal regions as the primary window. + mod_indices = [] + seen = set() + for v_idx in primary_indices: + a_start = min(int(round(v_idx * mod_total / video_total)), mod_total - 1) + a_end = min(int(round((v_idx + 1) * mod_total / video_total)), mod_total) + if a_end <= a_start: + a_end = a_start + 1 + for a in range(a_start, a_end): + if a not in seen: + seen.add(a) + mod_indices.append(a) + result.append(mod_indices) + + return result + + @staticmethod + def _get_guide_entries(conds): + for cond_list in conds: + if cond_list is None: + continue + for cond_dict in cond_list: + model_conds = cond_dict.get('model_conds', {}) + entries = model_conds.get('guide_attention_entries') + if entries is not None and hasattr(entries, 'cond') and entries.cond: + return entries.cond + return None + + def resize_cond_for_context_window(self, cond_key, cond_value, window, x_in, device, retain_index_list=[]): + # Audio denoise mask — slice using audio modality window + if cond_key == "audio_denoise_mask" and hasattr(window, 'modality_windows') and window.modality_windows: + audio_window = window.modality_windows.get(1) + if audio_window is not None and hasattr(cond_value, "cond") and isinstance(cond_value.cond, torch.Tensor): + sliced = audio_window.get_tensor(cond_value.cond, device, dim=2) + return cond_value._copy_with(sliced) + + # Video denoise mask — split into video + guide portions, slice each + if cond_key == "denoise_mask" and hasattr(cond_value, "cond") and isinstance(cond_value.cond, torch.Tensor): + cond_tensor = cond_value.cond + guide_count = cond_tensor.size(window.dim) - x_in.size(window.dim) + if guide_count > 0: + T_video = x_in.size(window.dim) + video_mask = cond_tensor.narrow(window.dim, 0, T_video) + guide_mask = cond_tensor.narrow(window.dim, T_video, guide_count) + sliced_video = window.get_tensor(video_mask, device, retain_index_list=retain_index_list) + suffix_indices = window.guide_frames_indices + if suffix_indices: + idx = tuple([slice(None)] * window.dim + [suffix_indices]) + sliced_guide = guide_mask[idx].to(device) + return cond_value._copy_with(torch.cat([sliced_video, sliced_guide], dim=window.dim)) + else: + return cond_value._copy_with(sliced_video) + + # Keyframe indices — regenerate pixel coords for window, select guide positions + if cond_key == "keyframe_idxs": + kf_local_pos = window.guide_kf_local_positions + if not kf_local_pos: + return cond_value._copy_with(cond_value.cond[:, :, :0, :]) # empty + H, W = x_in.shape[3], x_in.shape[4] + window_len = len(window.index_list) + # account for causal_window_fix anchor in coord space size + anchor_idx = getattr(window, 'causal_anchor_index', None) + if anchor_idx is not None and anchor_idx >= 0: + window_len += 1 + patchifier = self.diffusion_model.patchifier + latent_coords = patchifier.get_latent_coords(window_len, H, W, 1, cond_value.cond.device) + scale_factors = self.diffusion_model.vae_scale_factors + pixel_coords = comfy.ldm.lightricks.symmetric_patchifier.latent_to_pixel_coords( + latent_coords, + scale_factors, + causal_fix=self.diffusion_model.causal_temporal_positioning) + tokens = [] + for pos in kf_local_pos: + tokens.extend(range(pos * H * W, (pos + 1) * H * W)) + pixel_coords = pixel_coords[:, :, tokens, :] + + # Adjust spatial end positions for dilated (downscaled) guides. + # Each guide entry may have a different downscale factor; expand the + # per-entry factor to cover all tokens belonging to that entry. + downscale_factors = window.guide_downscale_factors + overlap_info = window.guide_overlap_info + if downscale_factors: + per_token_factor = [] + for (entry_idx, overlap_count), dsf in zip(overlap_info, downscale_factors): + per_token_factor.extend([dsf] * (overlap_count * H * W)) + factor_tensor = torch.tensor(per_token_factor, device=pixel_coords.device, dtype=pixel_coords.dtype) + spatial_end_offset = (factor_tensor.unsqueeze(0).unsqueeze(0).unsqueeze(-1) - 1) * torch.tensor( + scale_factors[1:], device=pixel_coords.device, dtype=pixel_coords.dtype, + ).view(1, -1, 1, 1) + pixel_coords[:, 1:, :, 1:] += spatial_end_offset + + B = cond_value.cond.shape[0] + if B > 1: + pixel_coords = pixel_coords.expand(B, -1, -1, -1) + return cond_value._copy_with(pixel_coords) + + # Guide attention entries — adjust per-guide counts based on window overlap + if cond_key == "guide_attention_entries": + overlap_info = window.guide_overlap_info + H, W = x_in.shape[3], x_in.shape[4] + new_entries = [] + for entry_idx, overlap_count in overlap_info: + e = cond_value.cond[entry_idx] + new_entries.append({**e, + "pre_filter_count": overlap_count * H * W, + "latent_shape": [overlap_count, H, W]}) + return cond_value._copy_with(new_entries) + + return None + class HunyuanVideo(BaseModel): def __init__(self, model_config, model_type=ModelType.FLOW, device=None): super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.hunyuan_video.model.HunyuanVideo) @@ -1747,10 +1871,14 @@ class WAN21_SCAIL(WAN21): reference_latents = kwargs.get("reference_latents", None) if reference_latents is not None: - ref_latent = self.process_latent_in(reference_latents[-1]) - ref_mask = torch.ones_like(ref_latent[:, :4]) - ref_latent = torch.cat([ref_latent, ref_mask], dim=1) - out['reference_latent'] = comfy.conds.CONDRegular(ref_latent) + # SCAIL-2 multi-reference: reference_latents[0] is the primary ref, [1:] are additional + # references. Stack as [additional..., primary] so the primary stays adjacent to the video. + ordered = list(reference_latents[1:]) + list(reference_latents[:1]) + stacked = [] + for lat in ordered: + lat = self.process_latent_in(lat) + stacked.append(torch.cat([lat, torch.ones_like(lat[:, :4])], dim=1)) + out['reference_latent'] = comfy.conds.CONDRegular(torch.cat(stacked, dim=2)) pose_latents = kwargs.get("pose_video_latent", None) if pose_latents is not None: @@ -1792,6 +1920,7 @@ class WAN21_SCAIL2(WAN21_SCAIL): if driving_mask_28ch is not None: out['sam_latents'] = comfy.conds.CONDRegular(driving_mask_28ch.movedim(1, 2).contiguous()) + # ref_mask_28ch holds one identity mask per stacked reference frame (additional refs first, then the primary ref), followed by zeros over the video frames. ref_mask_28ch = kwargs.get("ref_mask_28ch", None) if ref_mask_28ch is not None: out['ref_mask_latents'] = comfy.conds.CONDRegular(ref_mask_28ch.movedim(1, 2).contiguous()) @@ -1816,7 +1945,25 @@ class WAN21_SCAIL2(WAN21_SCAIL): def resize_cond_for_context_window(self, cond_key, cond_value, window, x_in, device, retain_index_list=[]): if cond_key in ("sam_latents", "pose_latents"): - return comfy.context_windows.slice_cond(cond_value, window, x_in, device, temporal_dim=2, temporal_offset=1) + # Return sliced view omitting retain_index_list + return comfy.context_windows.slice_cond(cond_value, window, x_in, device, temporal_dim=2, temporal_offset=0) + if cond_key == "ref_mask_latents" and hasattr(cond_value, "cond") and isinstance(cond_value.cond, torch.Tensor): + # The ref mask is N leading ref frames padded with frames of zeros, so just grab the first frames for all windows + full_ref_mask = cond_value.cond + video_frame_count = x_in.shape[2] + ref_frame_count = full_ref_mask.shape[2] - video_frame_count + if ref_frame_count < 1: + return None + window_length = len(window.index_list) + + # Account for the causal anchor frame if it exists + anchor_index = getattr(window, "causal_anchor_index", None) + if anchor_index is not None and anchor_index >= 0: + window_length += 1 + + window_ref_mask = full_ref_mask[:, :, :window_length + ref_frame_count].to(device) + return cond_value._copy_with(window_ref_mask) + return super().resize_cond_for_context_window(cond_key, cond_value, window, x_in, device, retain_index_list=retain_index_list) def concat_cond(self, **kwargs): @@ -2080,6 +2227,11 @@ class Omnigen2(BaseModel): out['ref_latents'] = list([1, 16, sum(map(lambda a: math.prod(a.size()), ref_latents)) // 16]) return out +class Boogu(Omnigen2): + def __init__(self, model_config, model_type=ModelType.FLOW, device=None): + super(Omnigen2, self).__init__(model_config, model_type, device=device, unet_model=comfy.ldm.boogu.model.BooguTransformer2DModel) + self.memory_usage_factor_conds = ("ref_latents",) + class QwenImage(BaseModel): def __init__(self, model_config, model_type=ModelType.FLUX, device=None): super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.qwen_image.model.QwenImageTransformer2DModel) @@ -2127,6 +2279,17 @@ class Ideogram4(BaseModel): out['c_crossattn'] = comfy.conds.CONDRegular(cross_attn) return out +class Krea2(BaseModel): + def __init__(self, model_config, model_type=ModelType.FLUX, device=None): + super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.krea2.model.SingleStreamDiT) + + def extra_conds(self, **kwargs): + out = super().extra_conds(**kwargs) + cross_attn = kwargs.get("cross_attn", None) + if cross_attn is not None: + out['c_crossattn'] = comfy.conds.CONDRegular(cross_attn) + return out + class HunyuanImage21(BaseModel): def __init__(self, model_config, model_type=ModelType.FLOW, device=None): super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.hunyuan_video.model.HunyuanVideo) diff --git a/comfy/model_detection.py b/comfy/model_detection.py index 7d0cab308..e53d848c9 100644 --- a/comfy/model_detection.py +++ b/comfy/model_detection.py @@ -761,6 +761,16 @@ def detect_unet_config(state_dict, key_prefix, metadata=None): return dit_config + if '{}double_stream_layers.0.img_instruct_attn.processor.img_to_q.weight'.format(key_prefix) in state_dict_keys: # Boogu-Image (OmniGen2 derivative + dual-stream stage) + dit_config = {} + dit_config["image_model"] = "boogu" + dit_config["hidden_size"] = state_dict['{}x_embedder.weight'.format(key_prefix)].shape[0] + dit_config["num_layers"] = count_blocks(state_dict_keys, '{}single_stream_layers.'.format(key_prefix) + '{}.') + dit_config["num_double_stream_layers"] = count_blocks(state_dict_keys, '{}double_stream_layers.'.format(key_prefix) + '{}.') + dit_config["num_refiner_layers"] = count_blocks(state_dict_keys, '{}noise_refiner.'.format(key_prefix) + '{}.') + dit_config["instruction_feat_dim"] = state_dict['{}time_caption_embed.caption_embedder.0.weight'.format(key_prefix)].shape[0] + return dit_config + if '{}time_caption_embed.timestep_embedder.linear_1.bias'.format(key_prefix) in state_dict_keys: # Omnigen2 dit_config = {} dit_config["image_model"] = "omnigen2" @@ -824,6 +834,21 @@ def detect_unet_config(state_dict, key_prefix, metadata=None): dit_config["num_layers"] = count_blocks(state_dict_keys, '{}layers.'.format(key_prefix) + '{}.') return dit_config + if '{}txtfusion.projector.weight'.format(key_prefix) in state_dict_keys: # Krea 2 (K2) + dit_config = {} + dit_config["image_model"] = "krea2" + head_dim = 128 + first_w = state_dict['{}first.weight'.format(key_prefix)] # (features, channels*patch^2) + dit_config["features"] = first_w.shape[0] + dit_config["channels"] = first_w.shape[1] // (2 * 2) # patch=2 + dit_config["patch"] = 2 + dit_config["layers"] = count_blocks(state_dict_keys, '{}blocks.'.format(key_prefix) + '{}.') + dit_config["heads"] = state_dict['{}blocks.0.attn.wq.weight'.format(key_prefix)].shape[0] // head_dim + dit_config["kvheads"] = state_dict['{}blocks.0.attn.wk.weight'.format(key_prefix)].shape[0] // head_dim + dit_config["txtlayers"] = state_dict['{}txtfusion.projector.weight'.format(key_prefix)].shape[1] + dit_config["txtdim"] = state_dict['{}txtfusion.layerwise_blocks.0.prenorm.scale'.format(key_prefix)].shape[0] + return dit_config + if '{}visual_transformer_blocks.0.cross_attention.key_norm.weight'.format(key_prefix) in state_dict_keys: # Kandinsky 5 dit_config = {} model_dim = state_dict['{}visual_embeddings.in_layer.bias'.format(key_prefix)].shape[0] diff --git a/comfy/model_management.py b/comfy/model_management.py index 9dc0a4e13..b15d08ba1 100644 --- a/comfy/model_management.py +++ b/comfy/model_management.py @@ -534,8 +534,10 @@ try: except: pass -if torch.cuda.is_available() and torch.backends.cudnn.is_available() and PerformanceFeature.AutoTune in args.fast: - torch.backends.cudnn.benchmark = True + +def set_cudnn_benchmark(): + if torch.cuda.is_available() and torch.backends.cudnn.is_available(): + torch.backends.cudnn.benchmark = PerformanceFeature.AutoTune in args.fast try: if torch_version_numeric >= (2, 5): @@ -641,6 +643,8 @@ def free_pins(size, evict_active=False): return freed_total def ensure_pin_budget(size, evict_active=False): + if args.high_ram: + return True if args.fast_disk: shortfall = TOTAL_PINNED_MEMORY + size - MAX_PINNED_MEMORY else: @@ -1494,6 +1498,8 @@ if not args.disable_pinned_memory: PINNING_ALLOWED_TYPES = set(["Tensor", "Parameter", "QuantizedTensor"]) def pinned_hostbuf_size(size): + if args.high_ram: + return max(0, int(size * 2)) return max(0, int(min(size, MAX_PINNED_MEMORY) * 2)) def discard_cuda_async_error(): diff --git a/comfy/ops.py b/comfy/ops.py index 119177c37..69d32e254 100644 --- a/comfy/ops.py +++ b/comfy/ops.py @@ -180,7 +180,7 @@ def cast_modules_with_vbar(comfy_modules, dtype, device, bias_dtype, non_blockin if pin is not None: cast_maybe_lowvram_patch([pin], dest, offload_stream) return - if signature is None: + if signature is None or args.high_ram: comfy.pinned_memory.pin_memory(m, subset=subset, size=size) pin = comfy.pinned_memory.get_pin(m, subset=subset) cast_maybe_lowvram_patch(source, pin, offload_stream, xfer_dest2=dest) @@ -256,7 +256,7 @@ def resolve_cast_module_with_vbar(s, dtype, device, bias_dtype, compute_dtype, w if (want_requant and len(fns) == 0 or update_weight): seed = comfy.utils.string_to_seed(s.seed_key) if isinstance(orig, QuantizedTensor): - y = QuantizedTensor.from_float(x, s.layout_type, scale="recalculate", stochastic_rounding=seed) + y = orig.requantize_from_float(x, scale="recalculate", stochastic_rounding=seed) else: y = comfy.float.stochastic_rounding(x, orig.dtype, seed=seed) if want_requant and len(fns) == 0: @@ -299,21 +299,21 @@ def cast_bias_weight(s, input=None, dtype=None, device=None, bias_dtype=None, of non_blocking = comfy.model_management.device_supports_non_blocking(device) - if hasattr(s, "_v"): + if hasattr(s, "_v") and comfy.model_management.is_device_cpu(device): #vbar doesn't support CPU weights, but some custom nodes have weird paths #that might switch the layer to the CPU and expect it to work. We have to take #a clone conservatively as we are mmapped and some SFT files are packed misaligned #If you are a custom node author reading this, please move your layer to the GPU #or declare your ModelPatcher as CPU in the first place. - if comfy.model_management.is_device_cpu(device): - materialize_meta_param(s, ["weight", "bias"]) - weight = s.weight.to(dtype=dtype, copy=True) - if isinstance(weight, QuantizedTensor): - weight = weight.dequantize() - bias = s.bias.to(dtype=bias_dtype, copy=True) if s.bias is not None else None - return format_return((weight, bias, (None, None, None)), offloadable) + materialize_meta_param(s, ["weight", "bias"]) + weight = s.weight.to(dtype=dtype, copy=True) + if isinstance(weight, QuantizedTensor): + weight = weight.dequantize() + bias = s.bias.to(dtype=bias_dtype, copy=True) if s.bias is not None else None + return format_return((weight, bias, (None, None, None)), offloadable) + elif hasattr(s, "_v") and s.weight.device != device: prefetched = hasattr(s, "_prefetch") offload_stream = None offload_device = None @@ -1089,6 +1089,19 @@ def _load_quantized_module(module, super_load, state_dict, prefix, local_metadat if ts is None or bs is None: raise ValueError(f"Missing NVFP4 scales for layer {layer_name}") scales = {"scale": ts, "block_scale": bs} + elif module.quant_format == "int8_tensorwise": + scale = pop_scale("weight_scale") + if scale is None: + raise ValueError(f"Missing INT8 weight scale for layer {layer_name}") + scales = {"scale": scale} + params_conf = layer_conf.get("params", {}) + if not isinstance(params_conf, dict): + params_conf = {} + if layer_conf.get("convrot", params_conf.get("convrot", False)): + scales["convrot"] = True + scales["convrot_groupsize"] = int( + layer_conf.get("convrot_groupsize", params_conf.get("convrot_groupsize", 256)) + ) else: raise ValueError(f"Unsupported quantization format: {module.quant_format}") @@ -1131,6 +1144,10 @@ def _quantized_weight_state_dict(module, sd, prefix, extra_quant_conf=None, extr quant_conf = {"format": module.quant_format} if getattr(module, '_full_precision_mm_config', False): quant_conf["full_precision_matrix_mult"] = True + params = getattr(module.weight, "_params", None) + if module.quant_format == "int8_tensorwise" and getattr(params, "convrot", False): + quant_conf["convrot"] = True + quant_conf["convrot_groupsize"] = getattr(params, "convrot_groupsize", 256) if extra_quant_conf: quant_conf.update(extra_quant_conf) sd[f"{prefix}comfy_quant"] = torch.tensor(list(json.dumps(quant_conf).encode("utf-8")), dtype=torch.uint8) @@ -1183,8 +1200,33 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_prec def _forward(self, input, weight, bias): return torch.nn.functional.linear(input, weight, bias) - def forward_comfy_cast_weights(self, input, compute_dtype=None, want_requant=False): - weight, bias, offload_stream = cast_bias_weight(self, input, offloadable=True, compute_dtype=compute_dtype, want_requant=want_requant) + def forward_comfy_cast_weights( + self, + input, + compute_dtype=None, + want_requant=False, + weight_only_quant=False, + ): + if weight_only_quant: + weight, bias, offload_stream = cast_bias_weight( + self, + input=None, + dtype=self.weight.dtype, + device=input.device, + bias_dtype=input.dtype, + offloadable=True, + compute_dtype=compute_dtype, + want_requant=True, + ) + weight = weight.to(dtype=input.dtype) + else: + weight, bias, offload_stream = cast_bias_weight( + self, + input, + offloadable=True, + compute_dtype=compute_dtype, + want_requant=want_requant, + ) x = self._forward(input, weight, bias) uncast_bias_weight(self, weight, bias, offload_stream) return x @@ -1203,9 +1245,10 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_prec not getattr(self, 'comfy_force_cast_weights', False) and len(self.weight_function) == 0 and len(self.bias_function) == 0 ) + quantize_input = QUANT_ALGOS.get(getattr(self, 'quant_format', None), {}).get("quantize_input", True) # Training path: quantized forward with compute_dtype backward via autograd function - if (input.requires_grad and _use_quantized): + if (input.requires_grad and _use_quantized and quantize_input): weight, bias, offload_stream = cast_bias_weight( self, @@ -1227,7 +1270,7 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_prec return output # Inference path (unchanged) - if _use_quantized: + if _use_quantized and quantize_input: # Reshape 3D tensors to 2D for quantization (needed for NVFP4 and others) input_reshaped = input.reshape(-1, input_shape[2]) if input.ndim == 3 else input @@ -1241,7 +1284,13 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_prec scale = comfy.model_management.cast_to_device(scale, input.device, None) input = QuantizedTensor.from_float(input_reshaped, self.layout_type, scale=scale) - output = self.forward_comfy_cast_weights(input, compute_dtype, want_requant=isinstance(input, QuantizedTensor)) + weight_only_quant = _use_quantized and not quantize_input and isinstance(self.weight, QuantizedTensor) + output = self.forward_comfy_cast_weights( + input, + compute_dtype, + want_requant=isinstance(input, QuantizedTensor), + weight_only_quant=weight_only_quant, + ) # Reshape output back to 3D if input was 3D if reshaped_3d: @@ -1257,8 +1306,7 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_prec def set_weight(self, weight, inplace_update=False, seed=None, return_weight=False, **kwargs): if getattr(self, 'layout_type', None) is not None: - # dtype is now implicit in the layout class - weight = QuantizedTensor.from_float(weight, self.layout_type, scale="recalculate", stochastic_rounding=seed, inplace_ops=True).to(self.weight.dtype) + weight = self.weight.requantize_from_float(weight, scale="recalculate", stochastic_rounding=seed, inplace_ops=True).to(self.weight.dtype) else: weight = weight.to(self.weight.dtype) if return_weight: diff --git a/comfy/quant_ops.py b/comfy/quant_ops.py index b90bcfd25..44f25a97e 100644 --- a/comfy/quant_ops.py +++ b/comfy/quant_ops.py @@ -10,6 +10,7 @@ try: QuantizedLayout, TensorCoreFP8Layout as _CKFp8Layout, TensorCoreNVFP4Layout as _CKNvfp4Layout, + TensorWiseINT8Layout as _CKTensorWiseINT8Layout, register_layout_op, register_layout_class, get_layout_class, @@ -47,6 +48,9 @@ except ImportError as e: class _CKNvfp4Layout: pass + class _CKTensorWiseINT8Layout: + pass + def register_layout_class(name, cls): pass @@ -174,6 +178,7 @@ class TensorCoreFP8E5M2Layout(_TensorCoreFP8LayoutBase): # Backward compatibility alias - default to E4M3 TensorCoreFP8Layout = TensorCoreFP8E4M3Layout +TensorWiseINT8Layout = _CKTensorWiseINT8Layout # ============================================================================== @@ -184,6 +189,7 @@ register_layout_class("TensorCoreFP8Layout", TensorCoreFP8Layout) register_layout_class("TensorCoreFP8E4M3Layout", TensorCoreFP8E4M3Layout) register_layout_class("TensorCoreFP8E5M2Layout", TensorCoreFP8E5M2Layout) register_layout_class("TensorCoreNVFP4Layout", TensorCoreNVFP4Layout) +register_layout_class("TensorWiseINT8Layout", _CKTensorWiseINT8Layout) if _CK_MXFP8_AVAILABLE: register_layout_class("TensorCoreMXFP8Layout", TensorCoreMXFP8Layout) @@ -214,6 +220,13 @@ if _CK_MXFP8_AVAILABLE: "group_size": 32, } +QUANT_ALGOS["int8_tensorwise"] = { + "storage_t": torch.int8, + "parameters": {"weight_scale"}, + "comfy_tensor_layout": "TensorWiseINT8Layout", + "quantize_input": False, +} + # ============================================================================== # Re-exports for backward compatibility @@ -226,6 +239,7 @@ __all__ = [ "TensorCoreFP8E4M3Layout", "TensorCoreFP8E5M2Layout", "TensorCoreNVFP4Layout", + "TensorWiseINT8Layout", "QUANT_ALGOS", "register_layout_op", ] diff --git a/comfy/sd.py b/comfy/sd.py index a66ba1bfb..faf3104f3 100644 --- a/comfy/sd.py +++ b/comfy/sd.py @@ -58,6 +58,7 @@ import comfy.text_encoders.omnigen2 import comfy.text_encoders.qwen_image import comfy.text_encoders.hunyuan_image import comfy.text_encoders.z_image +import comfy.text_encoders.krea2 import comfy.text_encoders.ideogram4 import comfy.text_encoders.ovis import comfy.text_encoders.kandinsky5 @@ -67,6 +68,8 @@ import comfy.text_encoders.anima import comfy.text_encoders.ace15 import comfy.text_encoders.longcat_image import comfy.text_encoders.qwen35 +import comfy.text_encoders.qwen3vl +import comfy.text_encoders.boogu import comfy.text_encoders.ernie import comfy.text_encoders.gemma4 import comfy.text_encoders.cogvideo @@ -465,6 +468,9 @@ class CLIP: def decode(self, token_ids, skip_special_tokens=True): return self.tokenizer.decode(token_ids, skip_special_tokens=skip_special_tokens) + def is_dynamic(self): + return self.patcher.is_dynamic() + class VAE: def __init__(self, sd=None, device=None, config=None, dtype=None, metadata=None): if 'decoder.up_blocks.0.resnets.0.norm1.weight' in sd.keys(): #diffusers format @@ -1248,6 +1254,8 @@ class VAE: except: return None + def is_dynamic(self): + return self.patcher.is_dynamic() class StyleModel: def __init__(self, model, device="cpu"): @@ -1300,6 +1308,8 @@ class CLIPType(Enum): LENS = 28 PIXELDIT = 29 IDEOGRAM4 = 30 + BOOGU = 31 + KREA2 = 32 @@ -1353,6 +1363,8 @@ class TEModel(Enum): GEMMA_4_31B = 31 T5_GEMMA = 32 GPT_OSS_20B = 33 + QWEN3VL_4B = 34 + QWEN3VL_8B = 35 def detect_te_model(sd): @@ -1414,6 +1426,8 @@ def detect_te_model(sd): if weight.shape[0] == 5120: return TEModel.QWEN35_27B return TEModel.QWEN35_2B + if "model.visual.deepstack_merger_list.0.norm.weight" in sd: # DeepStack is unique to Qwen3-VL + return TEModel.QWEN3VL_4B if sd["model.visual.merger.linear_fc2.weight"].shape[0] == 2560 else TEModel.QWEN3VL_8B if "model.layers.0.post_attention_layernorm.weight" in sd: weight = sd['model.layers.0.post_attention_layernorm.weight'] if 'model.layers.0.self_attn.q_norm.weight' in sd: @@ -1612,6 +1626,28 @@ def load_text_encoder_state_dicts(state_dicts=[], embedding_directory=None, clip qwen35_type = {TEModel.QWEN35_08B: "qwen35_08b", TEModel.QWEN35_2B: "qwen35_2b", TEModel.QWEN35_4B: "qwen35_4b", TEModel.QWEN35_9B: "qwen35_9b", TEModel.QWEN35_27B: "qwen35_27b"}[te_model] clip_target.clip = comfy.text_encoders.qwen35.te(**llama_detect(clip_data), model_type=qwen35_type) clip_target.tokenizer = comfy.text_encoders.qwen35.tokenizer(model_type=qwen35_type) + elif te_model in (TEModel.QWEN3VL_4B, TEModel.QWEN3VL_8B): + if clip_type == CLIPType.IDEOGRAM4 and te_model == TEModel.QWEN3VL_8B: # Ideogram4 reuses the full Qwen3-VL-8B (13-layer tap for conditioning + multimodal generate). + clip_data[0] = comfy.utils.state_dict_prefix_replace(clip_data[0], {"model.language_model.": "model.", "model.visual.": "visual.", "lm_head.": "model.lm_head."}) + clip_target.clip = comfy.text_encoders.ideogram4.te_qwen3vl(**llama_detect(clip_data)) + clip_target.tokenizer = comfy.text_encoders.ideogram4.Ideogram4Qwen3VLTokenizer + elif clip_type == CLIPType.BOOGU and te_model == TEModel.QWEN3VL_8B: # Boogu-Image: full Qwen3-VL-8B, last hidden state, no-think template. + clip_data[0] = comfy.utils.state_dict_prefix_replace(clip_data[0], {"model.language_model.": "model.", "model.visual.": "visual.", "lm_head.": "model.lm_head."}) + clip_target.clip = comfy.text_encoders.boogu.te(**llama_detect(clip_data)) + clip_target.tokenizer = comfy.text_encoders.boogu.BooguTokenizer + elif clip_type == CLIPType.KREA2 and te_model == TEModel.QWEN3VL_4B: # Krea2: full Qwen3-VL-4B (12-layer tap for conditioning + multimodal generate). + clip_data[0] = comfy.utils.state_dict_prefix_replace(clip_data[0], {"model.language_model.": "model.", "model.visual.": "visual.", "lm_head.": "model.lm_head."}) + clip_target.clip = comfy.text_encoders.krea2.te(**llama_detect(clip_data)) + clip_target.tokenizer = comfy.text_encoders.krea2.Krea2Tokenizer + elif clip_type in (CLIPType.FLUX, CLIPType.FLUX2): # Flux2 Klein reuses the Qwen3-VL LM (3-layer tap -> 12288); visual unused. + klein_model_type = "qwen3_8b" if te_model == TEModel.QWEN3VL_8B else "qwen3_4b" + clip_target.clip = comfy.text_encoders.flux.klein_te(**llama_detect(clip_data), model_type=klein_model_type) + clip_target.tokenizer = comfy.text_encoders.flux.KleinTokenizer8B if te_model == TEModel.QWEN3VL_8B else comfy.text_encoders.flux.KleinTokenizer + else: + clip_data[0] = comfy.utils.state_dict_prefix_replace(clip_data[0], {"model.language_model.": "model.", "model.visual.": "visual.", "lm_head.": "model.lm_head."}) + qwen3vl_type = {TEModel.QWEN3VL_4B: "qwen3vl_4b", TEModel.QWEN3VL_8B: "qwen3vl_8b"}[te_model] + clip_target.clip = comfy.text_encoders.qwen3vl.te(**llama_detect(clip_data), model_type=qwen3vl_type) + clip_target.tokenizer = comfy.text_encoders.qwen3vl.tokenizer(model_type=qwen3vl_type) elif te_model == TEModel.QWEN3_06B: clip_target.clip = comfy.text_encoders.anima.te(**llama_detect(clip_data)) clip_target.tokenizer = comfy.text_encoders.anima.AnimaTokenizer diff --git a/comfy/sd1_clip.py b/comfy/sd1_clip.py index 897186bba..f0fdf1aa5 100644 --- a/comfy/sd1_clip.py +++ b/comfy/sd1_clip.py @@ -543,18 +543,24 @@ class SDTokenizer: def _try_get_embedding(self, embedding_name:str): ''' Takes a potential embedding name and tries to retrieve it. - Returns a Tuple consisting of the embedding and any leftover string, embedding can be None. + Returns a Tuple consisting of the embedding, the cleaned embedding name, and any leftover string, embedding can be None. ''' split_embed = embedding_name.split() embedding_name = split_embed[0] leftover = ' '.join(split_embed[1:]) + + match = re.search(r'[<\[]', embedding_name) + if match is not None: + leftover = embedding_name[match.start():] + (" " + leftover if leftover else "") + embedding_name = embedding_name[:match.start()] + embed = load_embed(embedding_name, self.embedding_directory, self.embedding_size, self.embedding_key) if embed is None: stripped = embedding_name.strip(',') if len(stripped) < len(embedding_name): embed = load_embed(stripped, self.embedding_directory, self.embedding_size, self.embedding_key) - return (embed, "{} {}".format(embedding_name[len(stripped):], leftover)) - return (embed, leftover) + return (embed, embedding_name, "{} {}".format(embedding_name[len(stripped):], leftover)) + return (embed, embedding_name, leftover) def pad_tokens(self, tokens, amount): if self.pad_left: @@ -585,7 +591,7 @@ class SDTokenizer: tokens = [] for weighted_segment, weight in parsed_weights: to_tokenize = unescape_important(weighted_segment) - split = re.split(' {0}|\n{0}'.format(self.embedding_identifier), to_tokenize) + split = re.split(r'(?<=\s){}'.format(re.escape(self.embedding_identifier)), to_tokenize) to_tokenize = [split[0]] for i in range(1, len(split)): to_tokenize.append("{}{}".format(self.embedding_identifier, split[i])) @@ -595,7 +601,7 @@ class SDTokenizer: # if we find an embedding, deal with the embedding if word.startswith(self.embedding_identifier) and self.embedding_directory is not None: embedding_name = word[len(self.embedding_identifier):].strip('\n') - embed, leftover = self._try_get_embedding(embedding_name) + embed, embedding_name, leftover = self._try_get_embedding(embedding_name) if embed is None: logging.warning(f"warning, embedding:{embedding_name} does not exist, ignoring") else: diff --git a/comfy/supported_models.py b/comfy/supported_models.py index 3be935577..afb66e6f3 100644 --- a/comfy/supported_models.py +++ b/comfy/supported_models.py @@ -25,6 +25,8 @@ import comfy.text_encoders.hunyuan_image import comfy.text_encoders.kandinsky5 import comfy.text_encoders.z_image import comfy.text_encoders.ideogram4 +import comfy.text_encoders.boogu +import comfy.text_encoders.krea2 import comfy.text_encoders.anima import comfy.text_encoders.ace15 import comfy.text_encoders.longcat_image @@ -1758,6 +1760,27 @@ class Omnigen2(supported_models_base.BASE): hunyuan_detect = comfy.text_encoders.hunyuan_video.llama_detect(state_dict, "{}qwen25_3b.transformer.".format(pref)) return supported_models_base.ClipTarget(comfy.text_encoders.omnigen2.Omnigen2Tokenizer, comfy.text_encoders.omnigen2.te(**hunyuan_detect)) +class Boogu(Omnigen2): + unet_config = { + "image_model": "boogu", + } + + sampling_settings = { + "multiplier": 1.0, + "shift": 3.16, + } + + memory_usage_factor = 2.15 + + def get_model(self, state_dict, prefix="", device=None): + out = model_base.Boogu(self, device=device) + return out + + def clip_target(self, state_dict={}): + pref = self.text_encoder_key_prefix[0] + hunyuan_detect = comfy.text_encoders.hunyuan_video.llama_detect(state_dict, "{}qwen3vl_8b.transformer.".format(pref)) + return supported_models_base.ClipTarget(comfy.text_encoders.boogu.BooguTokenizer, comfy.text_encoders.boogu.te(**hunyuan_detect)) + class Ideogram4(supported_models_base.BASE): unet_config = { "image_model": "ideogram4", @@ -1796,6 +1819,35 @@ class Ideogram4(supported_models_base.BASE): hunyuan_detect = comfy.text_encoders.hunyuan_video.llama_detect(state_dict, "{}qwen3vl_8b.transformer.".format(pref)) return supported_models_base.ClipTarget(comfy.text_encoders.ideogram4.Ideogram4Tokenizer, comfy.text_encoders.ideogram4.te(**hunyuan_detect)) + +class Krea2(supported_models_base.BASE): + unet_config = { + "image_model": "krea2", + } + + sampling_settings = { + "multiplier": 1.0, + "shift": 1.15, + } + + memory_usage_factor = 2.2 + + latent_format = latent_formats.Wan21 + + supported_inference_dtypes = [torch.bfloat16, torch.float16, torch.float32] + + vae_key_prefix = ["vae."] + text_encoder_key_prefix = ["text_encoders."] + + def get_model(self, state_dict, prefix="", device=None): + out = model_base.Krea2(self, device=device) + return out + + def clip_target(self, state_dict={}): + pref = self.text_encoder_key_prefix[0] + hunyuan_detect = comfy.text_encoders.hunyuan_video.llama_detect(state_dict, "{}qwen3vl_4b.transformer.".format(pref)) + return supported_models_base.ClipTarget(comfy.text_encoders.krea2.Krea2Tokenizer, comfy.text_encoders.krea2.te(**hunyuan_detect)) + class QwenImage(supported_models_base.BASE): unet_config = { "image_model": "qwen_image", @@ -2300,8 +2352,10 @@ models = [ ACEStep, ACEStep15, Omnigen2, + Boogu, QwenImage, Ideogram4, + Krea2, Flux2, Lens, Kandinsky5Image, diff --git a/comfy/text_encoders/boogu.py b/comfy/text_encoders/boogu.py new file mode 100644 index 000000000..d9de92f10 --- /dev/null +++ b/comfy/text_encoders/boogu.py @@ -0,0 +1,58 @@ +"""Boogu-Image text encoder: full Qwen3-VL-8B, last hidden state (4096-dim). + +Boogu uses the final hidden state of Qwen3-VL as the per-token instruction feature +(num_instruction_feature_layers=1, reduce_type=mean -> just the last layer). +The model itself is the standard Qwen3-VL TE, only the chat template differs +(a fixed system prompt and no block). +""" + +import comfy.text_encoders.qwen3vl +from comfy import sd1_clip + + +# System prompts from the reference pipeline (pipeline_boogu.py). +# T2I (non-empty instruction, no image) uses the helpful-assistant prompt +# everything else (the CFG negative / "drop" condition, and any image case) uses the TI2I "describe" prompt. +BOOGU_T2I_SYSTEM = "You are a helpful assistant that generates high-quality images based on user instructions. The instructions are as follows." +BOOGU_DROP_SYSTEM = "Describe the key features of the input image (color, shape, size, texture, objects, background), then explain how the user's text instruction should alter or modify the image. Generate a new image that meets the user's requirements while maintaining consistency with the original input where appropriate." + + +class BooguTokenizer(comfy.text_encoders.qwen3vl.Qwen3VLTokenizer): + def __init__(self, embedding_directory=None, tokenizer_data={}): + super().__init__(embedding_directory=embedding_directory, tokenizer_data=tokenizer_data, model_type="qwen3vl_8b") + # apply_chat_template without add_generation_prompt + self.llama_template = "<|im_start|>system\n" + BOOGU_T2I_SYSTEM + "<|im_end|>\n<|im_start|>user\n{}<|im_end|>\n" + self.llama_template_images = "<|im_start|>system\n" + BOOGU_DROP_SYSTEM + "<|im_end|>\n<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>{}<|im_end|>\n" + # Reference SYSTEM_PROMPT_DROP: used for the empty negative/uncond instruction. + self.llama_template_drop = "<|im_start|>system\n" + BOOGU_DROP_SYSTEM + "<|im_end|>\n<|im_start|>user\n{}<|im_end|>\n" + + def tokenize_with_weights(self, text, return_word_ids=False, llama_template=None, images=[], prevent_empty_text=False, thinking=True, **kwargs): + if llama_template is None and len(images) == 0 and text.strip() == "": + llama_template = self.llama_template_drop + # Boogu conditions on the no-think template; thinking=True drops the empty block qwen3vl adds by default. + return super().tokenize_with_weights(text, return_word_ids=return_word_ids, llama_template=llama_template, images=images, prevent_empty_text=prevent_empty_text, thinking=thinking, **kwargs) + + +class BooguQwen3VLClipModel(comfy.text_encoders.qwen3vl.Qwen3VLClipModel): + def __init__(self, device="cpu", dtype=None, attention_mask=True, model_options={}, model_type="qwen3vl_8b"): + super().__init__(device=device, dtype=dtype, attention_mask=attention_mask, model_options=model_options, model_type=model_type) + # apply the final RMSNorm to the tapped last layer + self.layer_norm_hidden_state = True + + +class BooguTEModel(sd1_clip.SD1ClipModel): + def __init__(self, device="cpu", dtype=None, model_options={}): + clip_model = lambda **kw: BooguQwen3VLClipModel(**kw, model_type="qwen3vl_8b") + super().__init__(device=device, dtype=dtype, name="qwen3vl_8b", clip_model=clip_model, model_options=model_options) + + +def te(dtype_llama=None, llama_quantization_metadata=None): + class BooguTEModel_(BooguTEModel): + def __init__(self, device="cpu", dtype=None, model_options={}): + if dtype_llama is not None: + dtype = dtype_llama + if llama_quantization_metadata is not None: + model_options = model_options.copy() + model_options["quantization_metadata"] = llama_quantization_metadata + super().__init__(device=device, dtype=dtype, model_options=model_options) + return BooguTEModel_ diff --git a/comfy/text_encoders/ideogram4.py b/comfy/text_encoders/ideogram4.py index 84243772d..151b43c53 100644 --- a/comfy/text_encoders/ideogram4.py +++ b/comfy/text_encoders/ideogram4.py @@ -9,6 +9,7 @@ import os from transformers import Qwen2Tokenizer import comfy.text_encoders.llama +import comfy.text_encoders.qwen3vl from comfy import sd1_clip # Reference taps outputs of layers (0,3,...,35); comfy captures layer inputs, offset by +1. @@ -77,3 +78,43 @@ def te(dtype_llama=None, llama_quantization_metadata=None): model_options["quantization_metadata"] = llama_quantization_metadata super().__init__(device=device, dtype=dtype, model_options=model_options) return Ideogram4TEModel_ + + +# Full Qwen3-VL-8B variant with vision + +class Ideogram4Qwen3VLClipModel(comfy.text_encoders.qwen3vl.Qwen3VLClipModel): + def __init__(self, device="cpu", dtype=None, attention_mask=True, model_options={}): + super().__init__(device=device, layer=IDEOGRAM4_TAP_LAYERS, layer_idx=None, dtype=dtype, + attention_mask=attention_mask, model_options=model_options, model_type="qwen3vl_8b") + + +class Ideogram4Qwen3VLTEModel(sd1_clip.SD1ClipModel): + def __init__(self, device="cpu", dtype=None, model_options={}): + super().__init__(device=device, dtype=dtype, name="qwen3vl_8b", clip_model=Ideogram4Qwen3VLClipModel, model_options=model_options) + + def encode_token_weights(self, token_weight_pairs): + out, pooled, extra = super().encode_token_weights(token_weight_pairs) + b, n, seq, h = out.shape # (B, n_taps=13, seq, 4096), ascending layer order. + out = out.permute(0, 2, 3, 1).reshape(b, seq, h * n) # (B, seq, 4096*13 = 53248). + return out, pooled, extra + + +class Ideogram4Qwen3VLTokenizer(comfy.text_encoders.qwen3vl.Qwen3VLTokenizer): + def __init__(self, embedding_directory=None, tokenizer_data={}): + super().__init__(embedding_directory=embedding_directory, tokenizer_data=tokenizer_data, model_type="qwen3vl_8b") + + def tokenize_with_weights(self, text, return_word_ids=False, llama_template=None, images=[], prevent_empty_text=False, thinking=True, **kwargs): + # Ideogram 4 conditions on the no-think template; default thinking=True drops the empty think block qwen3vl adds. + return super().tokenize_with_weights(text, return_word_ids=return_word_ids, llama_template=llama_template, images=images, prevent_empty_text=prevent_empty_text, thinking=thinking, **kwargs) + + +def te_qwen3vl(dtype_llama=None, llama_quantization_metadata=None): + class Ideogram4Qwen3VLTEModel_(Ideogram4Qwen3VLTEModel): + def __init__(self, device="cpu", dtype=None, model_options={}): + if dtype_llama is not None: + dtype = dtype_llama + if llama_quantization_metadata is not None: + model_options = model_options.copy() + model_options["quantization_metadata"] = llama_quantization_metadata + super().__init__(device=device, dtype=dtype, model_options=model_options) + return Ideogram4Qwen3VLTEModel_ diff --git a/comfy/text_encoders/krea2.py b/comfy/text_encoders/krea2.py new file mode 100644 index 000000000..408a03566 --- /dev/null +++ b/comfy/text_encoders/krea2.py @@ -0,0 +1,84 @@ +"""Krea 2 (K2) text encoder: Qwen3-VL-4B, 12-layer tap. + +K2 conditions on a stack of hidden states from 12 layers of Qwen3-VL-4B +(reference taps ``hidden_states[2,5,8,...,35]``), kept as a ``(B, 12, seq, 2560)`` tensor and +consumed by the DiT's internal ``txtfusion`` adapter. Comfy carries conditioning as a 3D tensor, +so the 12-layer stack is flattened to ``(B, seq, 12*2560)`` here and unpacked inside the model. +""" + +import numbers + +import torch + +import comfy.text_encoders.qwen3vl +from comfy import sd1_clip + +# tap k == hidden_states[k] (no offset). +KREA2_TAP_LAYERS = [2, 5, 8, 11, 14, 17, 20, 23, 26, 29, 32, 35] + +# Identical system template to Qwen-Image; Krea2 strips the system+user-opening prefix. +KREA2_TEMPLATE = "<|im_start|>system\nDescribe the image by detailing the color, shape, size, texture, quantity, text, spatial relationships of the objects and background:<|im_end|>\n<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n" + + +class Krea2Tokenizer(comfy.text_encoders.qwen3vl.Qwen3VLTokenizer): + def __init__(self, embedding_directory=None, tokenizer_data={}): + super().__init__(embedding_directory=embedding_directory, tokenizer_data=tokenizer_data, model_type="qwen3vl_4b") + self.llama_template = KREA2_TEMPLATE # conditioning template; image text-gen uses qwen3vl's default image template. + + def tokenize_with_weights(self, text, return_word_ids=False, llama_template=None, images=[], prevent_empty_text=False, thinking=True, **kwargs): + # Krea2 conditions on the no-think template; thinking=True drops the empty block qwen3vl adds. + return super().tokenize_with_weights(text, return_word_ids=return_word_ids, llama_template=llama_template, images=images, prevent_empty_text=prevent_empty_text, thinking=thinking, **kwargs) + + +class Krea2Qwen3VLClipModel(comfy.text_encoders.qwen3vl.Qwen3VLClipModel): + def __init__(self, device="cpu", dtype=None, attention_mask=True, model_options={}): + super().__init__(device=device, layer=KREA2_TAP_LAYERS, layer_idx=None, dtype=dtype, + attention_mask=attention_mask, model_options=model_options, model_type="qwen3vl_4b") + + +class Krea2TEModel(sd1_clip.SD1ClipModel): + def __init__(self, device="cpu", dtype=None, model_options={}): + super().__init__(device=device, dtype=dtype, name="qwen3vl_4b", clip_model=Krea2Qwen3VLClipModel, model_options=model_options) + + def encode_token_weights(self, token_weight_pairs, template_end=-1): + out, pooled, extra = super().encode_token_weights(token_weight_pairs) # out: (B, 12, seq, 2560) + tok_pairs = token_weight_pairs["qwen3vl_4b"][0] + + # Strip the system + user-opening prefix + count_im_start = 0 + if template_end == -1: + for i, v in enumerate(tok_pairs): + elem = v[0] + if not torch.is_tensor(elem) and isinstance(elem, numbers.Integral): + if elem == 151644 and count_im_start < 2: + template_end = i + count_im_start += 1 + if out.shape[2] > (template_end + 3): + if tok_pairs[template_end + 1][0] == 872: # "user" + if tok_pairs[template_end + 2][0] == 198: # "\n" + template_end += 3 + + out = out[:, :, template_end:] + + b, n, seq, h = out.shape + # Flatten the 12-layer axis into the feature dim: (B, seq, 12*2560). Unpacked in the model. + out = out.permute(0, 2, 1, 3).reshape(b, seq, n * h) + + if "attention_mask" in extra: + extra["attention_mask"] = extra["attention_mask"][:, template_end:] + if extra["attention_mask"].sum() == torch.numel(extra["attention_mask"]): + extra.pop("attention_mask") + + return out, pooled, extra + + +def te(dtype_llama=None, llama_quantization_metadata=None): + class Krea2TEModel_(Krea2TEModel): + def __init__(self, device="cpu", dtype=None, model_options={}): + if llama_quantization_metadata is not None: + model_options = model_options.copy() + model_options["quantization_metadata"] = llama_quantization_metadata + if dtype_llama is not None: + dtype = dtype_llama + super().__init__(device=device, dtype=dtype, model_options=model_options) + return Krea2TEModel_ diff --git a/comfy/text_encoders/llama.py b/comfy/text_encoders/llama.py index 5087228ca..7403a60b8 100644 --- a/comfy/text_encoders/llama.py +++ b/comfy/text_encoders/llama.py @@ -251,6 +251,19 @@ class Qwen3_8BConfig: lm_head: bool = True stop_tokens = [151643, 151645] +@dataclass +class Qwen3VL_8BConfig(Qwen3_8BConfig): + max_position_embeddings: int = 262144 + rope_theta: float = 5000000.0 + rope_dims = [24, 20, 20] + interleaved_mrope = True + +@dataclass +class Qwen3VL_4BConfig(Qwen3VL_8BConfig): + hidden_size: int = 2560 + intermediate_size: int = 9728 + lm_head: bool = False # 4B ties word embeddings + @dataclass class Ovis25_2BConfig: vocab_size: int = 151936 @@ -703,7 +716,8 @@ class Llama2_(nn.Module): interleaved_mrope=getattr(self.config, "interleaved_mrope", False), device=device) - def forward(self, x, attention_mask=None, embeds=None, num_tokens=None, intermediate_output=None, final_layer_norm_intermediate=True, dtype=None, position_ids=None, embeds_info=[], past_key_values=None, input_ids=None): + def forward(self, x, attention_mask=None, embeds=None, num_tokens=None, intermediate_output=None, final_layer_norm_intermediate=True, + dtype=None, position_ids=None, embeds_info=[], past_key_values=None, input_ids=None,deepstack_embeds=None, visual_pos_masks=None): if embeds is not None: x = embeds else: @@ -767,6 +781,10 @@ class Llama2_(nn.Module): if current_kv is not None: next_key_values.append(current_kv) + # DeepStack: add per-layer visual features into the first len() decoder layers at image positions (Qwen3-VL) + if deepstack_embeds is not None and i < len(deepstack_embeds): + x[visual_pos_masks] = x[visual_pos_masks] + deepstack_embeds[i].to(x) + if i == intermediate_output: intermediate = x.clone() @@ -860,7 +878,7 @@ class BaseGenerate: torch.empty([batch, model_config.num_key_value_heads, max_cache_len, model_config.head_dim], device=device, dtype=execution_dtype), 0)) return past_key_values - def generate(self, embeds=None, do_sample=True, max_length=256, temperature=1.0, top_k=50, top_p=0.9, min_p=0.0, repetition_penalty=1.0, seed=42, stop_tokens=None, initial_tokens=[], execution_dtype=None, min_tokens=0, presence_penalty=0.0, initial_input_ids=None): + def generate(self, embeds=None, do_sample=True, max_length=256, temperature=1.0, top_k=50, top_p=0.9, min_p=0.0, repetition_penalty=1.0, seed=42, stop_tokens=None, initial_tokens=[], execution_dtype=None, min_tokens=0, presence_penalty=0.0, initial_input_ids=None, position_ids=None, deepstack_embeds=None, visual_pos_masks=None): device = embeds.device if stop_tokens is None: @@ -884,10 +902,18 @@ class BaseGenerate: generated_token_ids = [] pbar = comfy.utils.ProgressBar(max_length) + # MRoPE: prefill uses explicit 3D position_ids, decode continues from the last position + next_pos = int(position_ids[:, -1].max()) + 1 if position_ids is not None else None + # Generation loop current_input_ids = initial_input_ids for step in tqdm(range(max_length), desc="Generating tokens"): - x, _, past_key_values = self.model.forward(None, embeds=embeds, attention_mask=None, past_key_values=past_key_values, input_ids=current_input_ids) + # DeepStack visual features are injected on the prefill only; gemma4's forward lacks these kwargs. + extra = {} + if step == 0 and deepstack_embeds is not None: + extra["deepstack_embeds"] = deepstack_embeds + extra["visual_pos_masks"] = visual_pos_masks + x, _, past_key_values = self.model.forward(None, embeds=embeds, attention_mask=None, past_key_values=past_key_values, input_ids=current_input_ids, position_ids=position_ids, **extra) logits = self.logits(x)[:, -1] next_token = self.sample_token(logits, temperature, top_k, top_p, min_p, repetition_penalty, initial_tokens + generated_token_ids, generator, do_sample=do_sample, presence_penalty=presence_penalty) token_id = next_token[0].item() @@ -895,6 +921,9 @@ class BaseGenerate: embeds = self.model.embed_tokens(next_token).to(execution_dtype) current_input_ids = next_token if initial_input_ids is not None else None + if next_pos is not None: # advance MRoPE position for the next (decode) step + position_ids = torch.tensor([[next_pos]], device=device) + next_pos += 1 pbar.update(1) if token_id in stop_tokens: @@ -908,22 +937,41 @@ class BaseGenerate: return torch.argmax(logits, dim=-1, keepdim=True) # Sampling mode - if repetition_penalty != 1.0: - for i in range(logits.shape[0]): - for token_id in set(token_history): - logits[i, token_id] *= repetition_penalty if logits[i, token_id] < 0 else 1/repetition_penalty - - if presence_penalty is not None and presence_penalty != 0.0: - for i in range(logits.shape[0]): - for token_id in set(token_history): - logits[i, token_id] -= presence_penalty + if len(token_history) > 0 and (repetition_penalty != 1.0 or (presence_penalty is not None and presence_penalty != 0.0)): + token_ids = torch.tensor(list(set(token_history)), device=logits.device) + token_logits = logits[:, token_ids] + if repetition_penalty != 1.0: + token_logits = torch.where(token_logits < 0, token_logits * repetition_penalty, token_logits / repetition_penalty) + if presence_penalty is not None and presence_penalty != 0.0: + token_logits = token_logits - presence_penalty + logits[:, token_ids] = token_logits if temperature != 1.0: logits = logits / temperature if top_k > 0: - indices_to_remove = logits < torch.topk(logits, top_k)[0][..., -1, None] - logits[indices_to_remove] = torch.finfo(logits.dtype).min + top_k = min(top_k, logits.shape[-1]) + logits, top_indices = torch.topk(logits, top_k) + + if min_p > 0.0: + probs_before_filter = torch.nn.functional.softmax(logits, dim=-1) + top_probs, _ = probs_before_filter.max(dim=-1, keepdim=True) + min_threshold = min_p * top_probs + indices_to_remove = probs_before_filter < min_threshold + logits[indices_to_remove] = torch.finfo(logits.dtype).min + + if top_p < 1.0: + sorted_logits, sorted_indices = torch.sort(logits, descending=True) + cumulative_probs = torch.cumsum(torch.nn.functional.softmax(sorted_logits, dim=-1), dim=-1) + sorted_indices_to_remove = cumulative_probs > top_p + sorted_indices_to_remove[..., 0] = False + indices_to_remove = torch.zeros_like(logits, dtype=torch.bool) + indices_to_remove.scatter_(1, sorted_indices, sorted_indices_to_remove) + logits[indices_to_remove] = torch.finfo(logits.dtype).min + + probs = torch.nn.functional.softmax(logits, dim=-1) + next_token = torch.multinomial(probs, num_samples=1, generator=generator) + return top_indices.gather(1, next_token) if min_p > 0.0: probs_before_filter = torch.nn.functional.softmax(logits, dim=-1) diff --git a/comfy/text_encoders/qwen35.py b/comfy/text_encoders/qwen35.py index 416ce9d18..71a17990f 100644 --- a/comfy/text_encoders/qwen35.py +++ b/comfy/text_encoders/qwen35.py @@ -3,7 +3,6 @@ import torch.nn as nn import torch.nn.functional as F from dataclasses import dataclass, field import os -import math import comfy.model_management from comfy.ldm.modules.attention import optimized_attention_for_device @@ -563,6 +562,8 @@ class Qwen35VisionModel(nn.Module): for _ in range(config["depth"]) ]) self.merger = Qwen35VisionPatchMerger(self.hidden_size, self.spatial_merge_size, config["out_hidden_size"], device=device, dtype=dtype, ops=ops) + self.deepstack_visual_indexes = [] # DeepStack, per-layer visual features (Qwen3-VL) + self.deepstack_merger_list = None def rot_pos_emb(self, grid_thw): merge_size = self.spatial_merge_size @@ -664,9 +665,14 @@ class Qwen35VisionModel(nn.Module): ).cumsum(dim=0, dtype=torch.int32) cu_seqlens = F.pad(cu_seqlens, (1, 0), value=0) optimized_attention = optimized_attention_for_device(x.device, mask=False, small_input=True) - for blk in self.blocks: + deepstack_features = [] + for layer_num, blk in enumerate(self.blocks): x = blk(x, cu_seqlens=cu_seqlens, position_embeddings=position_embeddings, optimized_attention=optimized_attention) + if self.deepstack_merger_list is not None and layer_num in self.deepstack_visual_indexes: + deepstack_features.append(self.deepstack_merger_list[self.deepstack_visual_indexes.index(layer_num)](x)) merged = self.merger(x) + if self.deepstack_merger_list is not None: + return merged, deepstack_features return merged # Model Wrapper @@ -690,30 +696,7 @@ class Qwen35(BaseLlama, BaseGenerate, torch.nn.Module): return None, None def forward(self, x, attention_mask=None, embeds=None, num_tokens=None, intermediate_output=None, final_layer_norm_intermediate=True, dtype=None, embeds_info=[], past_key_values=None): - grid = None - position_ids = None - offset = 0 - for e in embeds_info: - if e.get("type") == "image": - grid = e.get("extra", None) - start = e.get("index") - if position_ids is None: - position_ids = torch.zeros((3, embeds.shape[1]), device=embeds.device) - position_ids[:, :start] = torch.arange(0, start, device=embeds.device) - end = e.get("size") + start - len_max = int(grid.max()) // 2 - start_next = len_max + start - position_ids[:, end:] = torch.arange(start_next + offset, start_next + (embeds.shape[1] - end) + offset, device=embeds.device) - position_ids[0, start:end] = start + offset - max_d = int(grid[0][1]) // 2 - position_ids[1, start:end] = torch.arange(start + offset, start + max_d + offset, device=embeds.device).unsqueeze(1).repeat(1, math.ceil((end - start) / max_d)).flatten(0)[:end - start] - max_d = int(grid[0][2]) // 2 - position_ids[2, start:end] = torch.arange(start + offset, start + max_d + offset, device=embeds.device).unsqueeze(0).repeat(math.ceil((end - start) / max_d), 1).flatten(0)[:end - start] - offset += len_max - (end - start) - - if grid is None: - position_ids = None - + position_ids = comfy.text_encoders.qwen_vl.qwen2vl_mrope_position_ids(embeds_info, embeds.shape[1], embeds.device) return super().forward(x, attention_mask=attention_mask, embeds=embeds, num_tokens=num_tokens, intermediate_output=intermediate_output, final_layer_norm_intermediate=final_layer_norm_intermediate, dtype=dtype, position_ids=position_ids, past_key_values=past_key_values) def init_kv_cache(self, batch, max_cache_len, device, execution_dtype): diff --git a/comfy/text_encoders/qwen3vl.py b/comfy/text_encoders/qwen3vl.py new file mode 100644 index 000000000..2082c42e7 --- /dev/null +++ b/comfy/text_encoders/qwen3vl.py @@ -0,0 +1,193 @@ +import os + +import torch +import torch.nn as nn +import torch.nn.functional as F +from transformers import Qwen2Tokenizer + +from comfy import sd1_clip +import comfy.text_encoders.qwen_vl +from .qwen35 import Qwen35VisionModel +from .llama import BaseLlama, BaseQwen3, BaseGenerate, Llama2_, Qwen3VL_4BConfig, Qwen3VL_8BConfig + + +QWEN3VL_VISION = { + "qwen3vl_4b": dict(hidden_size=1024, intermediate_size=4096, depth=24, deepstack_visual_indexes=[5, 11, 17]), + "qwen3vl_8b": dict(hidden_size=1152, intermediate_size=4304, depth=27, deepstack_visual_indexes=[8, 16, 24]), +} +QWEN3VL_VISION_COMMON = dict(num_heads=16, patch_size=16, temporal_patch_size=2, in_channels=3, + spatial_merge_size=2, num_position_embeddings=2304) + +QWEN3VL_CONFIGS = {"qwen3vl_4b": Qwen3VL_4BConfig, "qwen3vl_8b": Qwen3VL_8BConfig} + + +class Qwen3VLDeepstackMerger(nn.Module): + # DeepStack merger: postshuffle LayerNorm (applied after spatial merge), unlike the main merger. + def __init__(self, hidden_size, spatial_merge_size, out_hidden_size, device=None, dtype=None, ops=None): + super().__init__() + self.merge_dim = hidden_size * (spatial_merge_size ** 2) + self.norm = ops.LayerNorm(self.merge_dim, eps=1e-6, device=device, dtype=dtype) + self.linear_fc1 = ops.Linear(self.merge_dim, self.merge_dim, device=device, dtype=dtype) + self.linear_fc2 = ops.Linear(self.merge_dim, out_hidden_size, device=device, dtype=dtype) + + def forward(self, x): + x = self.norm(x.view(-1, self.merge_dim)) + return self.linear_fc2(F.gelu(self.linear_fc1(x))) + + +class Qwen3VLVisionModel(Qwen35VisionModel): + # Qwen3.5 vision + DeepStack + def __init__(self, config, device=None, dtype=None, ops=None): + super().__init__(config, device=device, dtype=dtype, ops=ops) + self.deepstack_visual_indexes = config["deepstack_visual_indexes"] + self.deepstack_merger_list = nn.ModuleList([ + Qwen3VLDeepstackMerger(self.hidden_size, self.spatial_merge_size, config["out_hidden_size"], device=device, dtype=dtype, ops=ops) + for _ in self.deepstack_visual_indexes + ]) + + +class Qwen3VL(BaseLlama, BaseQwen3, BaseGenerate, torch.nn.Module): + model_type = "qwen3vl_8b" + + def __init__(self, config_dict, dtype, device, operations): + super().__init__() + config = QWEN3VL_CONFIGS[self.model_type](**config_dict) + self.num_layers = config.num_hidden_layers + self.model = Llama2_(config, device=device, dtype=dtype, ops=operations) + vision_config = {**QWEN3VL_VISION_COMMON, **QWEN3VL_VISION[self.model_type], "out_hidden_size": config.hidden_size} + self.visual = Qwen3VLVisionModel(vision_config, device=device, dtype=dtype, ops=operations) + self.dtype = dtype + + def preprocess_embed(self, embed, device): + if embed["type"] == "image": + # Qwen3-VL normalizes to [-1, 1] (mean/std 0.5), unlike Qwen2.5-VL's CLIP normalization. + image, grid = comfy.text_encoders.qwen_vl.process_qwen2vl_images(embed["data"], patch_size=16, image_mean=[0.5, 0.5, 0.5], image_std=[0.5, 0.5, 0.5]) + merged, deepstack = self.visual(image.to(device, dtype=torch.float32), grid) + return merged, {"grid": grid, "deepstack": deepstack} + return None, None + + def build_image_inputs(self, embeds, embeds_info): + # Returns (position_ids, visual_pos_masks, deepstack) for the prompt + images = sorted([e for e in embeds_info if e.get("type") == "image"], key=lambda e: e["index"]) + if len(images) == 0: + return None, None, None + + device = embeds.device + seq = embeds.shape[1] + position_ids = comfy.text_encoders.qwen_vl.qwen2vl_mrope_position_ids(embeds_info, seq, device) + + # DeepStack: mask of image positions + per-vision-layer features to inject there. + visual_pos_masks = torch.zeros((1, seq), dtype=torch.bool, device=device) + deepstack = None + for e in images: + start = e["index"] + end = e["size"] + start + visual_pos_masks[0, start:end] = True + ds = e["extra"]["deepstack"] + if deepstack is None: + deepstack = [d for d in ds] + else: + deepstack = [torch.cat([deepstack[i], ds[i]], dim=0) for i in range(len(ds))] + return position_ids, visual_pos_masks, deepstack + + +def _make_qwen3vl_model(model_type): + class Qwen3VL_(Qwen3VL): + pass + Qwen3VL_.model_type = model_type + return Qwen3VL_ + + +class Qwen3VLClipModel(sd1_clip.SDClipModel): + def __init__(self, device="cpu", layer="hidden", layer_idx=-1, dtype=None, attention_mask=True, model_options={}, model_type="qwen3vl_8b"): + super().__init__(device=device, layer=layer, layer_idx=layer_idx, textmodel_json_config={}, + dtype=dtype, special_tokens={"pad": 151643}, layer_norm_hidden_state=False, + model_class=_make_qwen3vl_model(model_type), enable_attention_masks=attention_mask, + return_attention_masks=attention_mask, model_options=model_options) + + def generate(self, tokens, do_sample, max_length, temperature, top_k, top_p, min_p, repetition_penalty, seed, presence_penalty=0.0): + if isinstance(tokens, dict): + tokens = next(iter(tokens.values())) + tokens_only = [[t[0] for t in b] for b in tokens] + embeds, _, _, embeds_info = self.process_tokens(tokens_only, self.execution_device) + position_ids, visual_pos_masks, deepstack = self.transformer.build_image_inputs(embeds, embeds_info) + return self.transformer.generate(embeds, do_sample, max_length, temperature, top_k, top_p, min_p, repetition_penalty, seed, + presence_penalty=presence_penalty, position_ids=position_ids, + visual_pos_masks=visual_pos_masks, deepstack_embeds=deepstack) + + +class Qwen3VLTEModel(sd1_clip.SD1ClipModel): + def __init__(self, device="cpu", dtype=None, model_options={}, model_type="qwen3vl_8b"): + clip_model = lambda **kw: Qwen3VLClipModel(**kw, model_type=model_type) + super().__init__(device=device, dtype=dtype, name=model_type, clip_model=clip_model, model_options=model_options) + + +class Qwen3VLSDTokenizer(sd1_clip.SDTokenizer): + def __init__(self, embedding_directory=None, tokenizer_data={}, embedding_size=4096, embedding_key="qwen3vl_8b"): + tokenizer_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "qwen25_tokenizer") + super().__init__(tokenizer_path, pad_with_end=False, embedding_directory=embedding_directory, embedding_size=embedding_size, embedding_key=embedding_key, tokenizer_class=Qwen2Tokenizer, + has_start_token=False, has_end_token=False, pad_to_max_length=False, max_length=99999999, min_length=1, pad_token=151643, tokenizer_data=tokenizer_data) + + +class Qwen3VLTokenizer(sd1_clip.SD1Tokenizer): + def __init__(self, embedding_directory=None, tokenizer_data={}, model_type="qwen3vl_8b"): + embedding_size = 2560 if model_type == "qwen3vl_4b" else 4096 + tokenizer = lambda *a, **kw: Qwen3VLSDTokenizer(*a, **kw, embedding_size=embedding_size, embedding_key=model_type) + super().__init__(embedding_directory=embedding_directory, tokenizer_data=tokenizer_data, name=model_type, tokenizer=tokenizer) + self.llama_template = "<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n" + self.llama_template_images = "<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>{}<|im_end|>\n<|im_start|>assistant\n" + + def tokenize_with_weights(self, text, return_word_ids=False, llama_template=None, images=[], prevent_empty_text=False, thinking=False, **kwargs): + image = kwargs.get("image", None) + if image is not None and len(images) == 0: + images = [image[i:i + 1] for i in range(image.shape[0])] + + skip_template = text.startswith('<|im_start|>') + if prevent_empty_text and text == '': + text = ' ' + + if skip_template: + llama_text = text + else: + if llama_template is not None: + template = llama_template + elif len(images) == 0: + template = self.llama_template + else: + template = self.llama_template_images + if len(images) > 1: + vision_block = "<|vision_start|><|image_pad|><|vision_end|>" + template = template.replace(vision_block, vision_block * len(images), 1) + llama_text = template.format(text) + if not thinking: # Qwen3 convention: empty think block suppresses reasoning + llama_text += "\n\n\n\n" + + tokens = super().tokenize_with_weights(llama_text, return_word_ids=return_word_ids, disable_weights=True, **kwargs) + key_name = next(iter(tokens)) + embed_count = 0 + for r in tokens[key_name]: + for i in range(len(r)): + if isinstance(r[i][0], (int, float)) and r[i][0] == 151655: # <|image_pad|> + if len(images) > embed_count: + r[i] = ({"type": "image", "data": images[embed_count], "original_type": "image"},) + r[i][1:] + embed_count += 1 + return tokens + + +def tokenizer(model_type="qwen3vl_8b"): + class Qwen3VLTokenizer_(Qwen3VLTokenizer): + def __init__(self, embedding_directory=None, tokenizer_data={}): + super().__init__(embedding_directory=embedding_directory, tokenizer_data=tokenizer_data, model_type=model_type) + return Qwen3VLTokenizer_ + + +def te(dtype_llama=None, llama_quantization_metadata=None, model_type="qwen3vl_8b"): + class Qwen3VLTEModel_(Qwen3VLTEModel): + def __init__(self, device="cpu", dtype=None, model_options={}): + if dtype_llama is not None: + dtype = dtype_llama + if llama_quantization_metadata is not None: + model_options = model_options.copy() + model_options["quantization_metadata"] = llama_quantization_metadata + super().__init__(device=device, dtype=dtype, model_options=model_options, model_type=model_type) + return Qwen3VLTEModel_ diff --git a/comfy/text_encoders/qwen_vl.py b/comfy/text_encoders/qwen_vl.py index 98c350a12..924eb6ad8 100644 --- a/comfy/text_encoders/qwen_vl.py +++ b/comfy/text_encoders/qwen_vl.py @@ -88,6 +88,32 @@ def process_qwen2vl_images( return flatten_patches, image_grid_thw +def qwen2vl_mrope_position_ids(embeds_info, seq_len, device): + # (3, seq_len) T/H/W MRoPE position ids: text runs sequentially, each image span gets its grid positions. + # Returns None when there are no image embeds. `extra` is the image grid_thw, or a dict carrying it under "grid". + position_ids = None + offset = 0 + for e in embeds_info: + if e.get("type") == "image": + extra = e.get("extra", None) + grid = extra["grid"] if isinstance(extra, dict) else extra + start = e.get("index") + if position_ids is None: + position_ids = torch.zeros((3, seq_len), device=device) + position_ids[:, :start] = torch.arange(0, start, device=device) + end = e.get("size") + start + len_max = int(grid.max()) // 2 + start_next = len_max + start + position_ids[:, end:] = torch.arange(start_next + offset, start_next + (seq_len - end) + offset, device=device) + position_ids[0, start:end] = start + offset + max_d = int(grid[0][1]) // 2 + position_ids[1, start:end] = torch.arange(start + offset, start + max_d + offset, device=device).unsqueeze(1).repeat(1, math.ceil((end - start) / max_d)).flatten(0)[:end - start] + max_d = int(grid[0][2]) // 2 + position_ids[2, start:end] = torch.arange(start + offset, start + max_d + offset, device=device).unsqueeze(0).repeat(math.ceil((end - start) / max_d), 1).flatten(0)[:end - start] + offset += len_max - (end - start) + return position_ids + + class VisionPatchEmbed(nn.Module): def __init__( self, diff --git a/comfy/utils.py b/comfy/utils.py index 09d783fff..61c2a22dd 100644 --- a/comfy/utils.py +++ b/comfy/utils.py @@ -818,6 +818,44 @@ def z_image_to_diffusers(mmdit_config, output_prefix=""): return key_map +def krea2_to_diffusers(mmdit_config, output_prefix=""): + n_layers = mmdit_config.get("layers", 0) + n_txt_layerwise = 2 # TextFusionTransformer hardcodes 2 layerwise + 2 refiner blocks + n_txt_refiner = 2 + key_map = {} + + def add_block(prefix_to, prefix_from): + block_map = { + "attn.to_q": "attn.wq", "attn.to_k": "attn.wk", "attn.to_v": "attn.wv", + "attn.to_gate": "attn.gate", "attn.to_out.0": "attn.wo", + "attn.to_out": "attn.wo", # some tools drop the ".0" on to_out + "ff.gate": "mlp.gate", "ff.up": "mlp.up", "ff.down": "mlp.down", + } + for d, c in block_map.items(): + key_map["{}.{}.weight".format(prefix_to, d)] = "{}{}.{}.weight".format(output_prefix, prefix_from, c) + + for i in range(n_layers): + add_block("transformer_blocks.{}".format(i), "blocks.{}".format(i)) + for i in range(n_txt_layerwise): + add_block("text_fusion.layerwise_blocks.{}".format(i), "txtfusion.layerwise_blocks.{}".format(i)) + for i in range(n_txt_refiner): + add_block("text_fusion.refiner_blocks.{}".format(i), "txtfusion.refiner_blocks.{}".format(i)) + + MAP_BASIC = [ + ("img_in", "first"), + ("time_embed.linear_1", "tmlp.0"), + ("time_embed.linear_2", "tmlp.2"), + ("time_mod_proj", "tproj.1"), + ("txt_in.linear_1", "txtmlp.1"), + ("txt_in.linear_2", "txtmlp.3"), + ("text_fusion.projector", "txtfusion.projector"), + ("final_layer.linear", "last.linear"), + ] + for d, c in MAP_BASIC: + key_map["{}.weight".format(d)] = "{}{}.weight".format(output_prefix, c) + + return key_map + def repeat_to_batch_size(tensor, batch_size, dim=0): if tensor.shape[dim] > batch_size: return tensor.narrow(dim, 0, batch_size) diff --git a/comfy_api/feature_flags.py b/comfy_api/feature_flags.py index adb5a3144..0f30608a9 100644 --- a/comfy_api/feature_flags.py +++ b/comfy_api/feature_flags.py @@ -25,6 +25,11 @@ CLI_FEATURE_FLAG_REGISTRY: dict[str, FeatureFlagInfo] = { "default": False, "description": "Show the sign-in button in the frontend even when not signed in", }, + "enable_telemetry": { + "type": "bool", + "default": False, + "description": "Signal the frontend that telemetry collection is enabled", + }, } diff --git a/comfy_api/latest/_input/video_types.py b/comfy_api/latest/_input/video_types.py index 8fff52c16..e2e99521f 100644 --- a/comfy_api/latest/_input/video_types.py +++ b/comfy_api/latest/_input/video_types.py @@ -27,10 +27,13 @@ class VideoInput(ABC): path: Union[str, IO[bytes]], format: VideoContainer = VideoContainer.AUTO, codec: VideoCodec = VideoCodec.AUTO, - metadata: Optional[dict] = None + metadata: Optional[dict] = None, + bit_depth: int | None = None, ): """ Abstract method to save the video input to a file. + + bit_depth selects the encoded bit depth; None keeps the video's native depth. """ pass @@ -83,6 +86,14 @@ class VideoInput(ABC): components = self.get_components() return components.images.shape[2], components.images.shape[1] + def get_bit_depth(self) -> int: + """ + Returns the bit depth of the video (e.g. 8 or 10). + + Default implementation returns 8; subclasses report their real depth. + """ + return 8 + def get_duration(self) -> float: """ Returns the duration of the video in seconds. diff --git a/comfy_api/latest/_input_impl/video_types.py b/comfy_api/latest/_input_impl/video_types.py index 4a12ff9c1..bc95a5b99 100644 --- a/comfy_api/latest/_input_impl/video_types.py +++ b/comfy_api/latest/_input_impl/video_types.py @@ -52,6 +52,12 @@ def get_open_write_kwargs( return open_kwargs +def video_stream_bit_depth(stream) -> int: + if stream is None or stream.format is None or not stream.format.components: + return 8 + return max(component.bits for component in stream.format.components) + + class VideoFromFile(VideoInput): """ Class representing video input from a file. @@ -97,6 +103,13 @@ class VideoFromFile(VideoInput): return stream.width, stream.height raise ValueError(f"No video stream found in file '{self.__file}'") + def get_bit_depth(self) -> int: + if isinstance(self.__file, io.BytesIO): + self.__file.seek(0) # Reset the BytesIO object to the beginning + with av.open(self.__file, mode="r") as container: + video_stream = container.streams.video[0] if len(container.streams.video) > 0 else None + return video_stream_bit_depth(video_stream) + def get_duration(self) -> float: """ Returns the duration of the video in seconds. @@ -257,6 +270,7 @@ class VideoFromFile(VideoInput): image_format = 'gbrpf32le' process_image_format = lambda a: a + align_graph = None audio = None streams = [video_stream] @@ -267,11 +281,18 @@ class VideoFromFile(VideoInput): video_done = False audio_done = True - if len(container.streams.audio): - audio_stream = container.streams.audio[-1] + # Use the last decodable audio stream. Streams FFmpeg has no decoder for have no codec context, + # and decoding their packets crashes the process. (e.g. APAC spatial-audio track in iPhone) + audio_stream = next( + (s for s in reversed(container.streams.audio) if s.codec_context is not None), + None, + ) + if audio_stream is not None: streams += [audio_stream] resampler = av.audio.resampler.AudioResampler(format='fltp') audio_done = False + elif len(container.streams.audio): + logging.warning("No decodable audio stream found in video; ignoring audio.") for packet in container.demux(*streams): if video_done and audio_done: @@ -310,7 +331,28 @@ class VideoFromFile(VideoInput): checked_alpha = True - img = frame.to_ndarray(format=image_format) # shape: (H, W, 4) + # Fix non-deterministic video decode when the video width is not a multiple of 32 + # For non-yuvj pixel formats: most H.264/H.265 video and static images (e.g. lossy WebP via LoadImage) + # Pad both axes to a multiple of 32 and smear the border so the alignment padding never bleeds into the cropped edges + if image_format in ('gbrpf32le', 'gbrapf32le') and frame.width % 32 != 0: + if align_graph is None: + pad_w = ((frame.width + 31) // 32) * 32 + pad_h = ((frame.height + 31) // 32) * 32 + g = av.filter.Graph() + g_src = g.add_buffer(width=frame.width, height=frame.height, + format=frame.format.name, time_base=video_stream.time_base) + g_pad = g.add('pad', f'{pad_w}:{pad_h}:0:0') + g_fill = g.add('fillborders', f'left=0:right={pad_w - frame.width}:top=0:bottom={pad_h - frame.height}:mode=smear') + g_sink = g.add('buffersink') + g_src.link_to(g_pad) + g_pad.link_to(g_fill) + g_fill.link_to(g_sink) + g.configure() + align_graph = (g, g_src, g_sink) + align_graph[1].push(frame) + img = np.ascontiguousarray(align_graph[2].pull().to_ndarray(format=image_format)[:frame.height, :frame.width]) + else: + img = frame.to_ndarray(format=image_format) if frame.rotation != 0: k = int(round(frame.rotation // 90)) img = np.rot90(img, k=k, axes=(0, 1)).copy() @@ -377,25 +419,32 @@ class VideoFromFile(VideoInput): format: VideoContainer = VideoContainer.AUTO, codec: VideoCodec = VideoCodec.AUTO, metadata: Optional[dict] = None, + bit_depth: int | None = None, ): if isinstance(self.__file, io.BytesIO): self.__file.seek(0) # Reset the BytesIO object to the beginning with av.open(self.__file, mode='r') as container: container_format = container.format.name - video_encoding = container.streams.video[0].codec.name if len(container.streams.video) > 0 else None + video_stream = container.streams.video[0] if len(container.streams.video) > 0 else None + video_encoding = video_stream.codec.name if video_stream is not None else None + source_bit_depth = video_stream_bit_depth(video_stream) reuse_streams = True if format != VideoContainer.AUTO and format not in container_format.split(","): reuse_streams = False if codec != VideoCodec.AUTO and codec != video_encoding and video_encoding is not None: reuse_streams = False + if bit_depth is not None and video_encoding is not None and bit_depth != source_bit_depth: + reuse_streams = False if self.__start_time or self.__duration: reuse_streams = False if not reuse_streams: + if bit_depth is None: + bit_depth = source_bit_depth components = self.get_components_internal(container) video = VideoFromComponents(components) return video.save_to( - path, format=format, codec=codec, metadata=metadata + path, format=format, codec=codec, metadata=metadata, bit_depth=bit_depth, ) streams = container.streams @@ -415,10 +464,13 @@ class VideoFromFile(VideoInput): else: output_container.metadata[key] = json.dumps(value) - # Add streams to the new container + # Add streams to the new container. Streams with no codec context cannot be used as an output template. stream_map = {} for stream in streams: if isinstance(stream, (av.VideoStream, av.AudioStream, SubtitleStream)): + if stream.codec_context is None: + logging.warning("Skipping %s stream %d with unsupported codec", stream.type, stream.index) + continue out_stream = output_container.add_stream_from_template(template=stream, opaque=True) stream_map[stream] = out_stream @@ -451,8 +503,10 @@ class VideoFromComponents(VideoInput): Class representing video input from tensors. """ - def __init__(self, components: VideoComponents): + def __init__(self, components: VideoComponents, bit_depth: int = 8): self.__components = components + # Tensor components have no inherent bit depth; this is the depth used when encoding. + self.__bit_depth = bit_depth def get_components(self) -> VideoComponents: return VideoComponents( @@ -461,18 +515,26 @@ class VideoFromComponents(VideoInput): frame_rate=self.__components.frame_rate, ) + def get_bit_depth(self) -> int: + return self.__bit_depth + def save_to( self, path: str, format: VideoContainer = VideoContainer.AUTO, codec: VideoCodec = VideoCodec.AUTO, metadata: Optional[dict] = None, + bit_depth: int | None = None, ): """Save the video to a file path or BytesIO buffer.""" if format != VideoContainer.AUTO and format != VideoContainer.MP4: raise ValueError("Only MP4 format is supported for now") if codec != VideoCodec.AUTO and codec != VideoCodec.H264: raise ValueError("Only H264 codec is supported for now") + # None means "use the depth this video was created with" (CreateVideo's choice). + if bit_depth is None: + bit_depth = self.__bit_depth + is_10bit = bit_depth >= 10 extra_kwargs = {} if isinstance(format, VideoContainer) and format != VideoContainer.AUTO: extra_kwargs["format"] = format.value @@ -488,10 +550,11 @@ class VideoFromComponents(VideoInput): frame_rate = Fraction(round(self.__components.frame_rate * 1000), 1000) # Create a video stream + pix_fmt = "yuv420p10le" if is_10bit else "yuv420p" video_stream = output.add_stream('h264', rate=frame_rate) video_stream.width = self.__components.images.shape[2] video_stream.height = self.__components.images.shape[1] - video_stream.pix_fmt = 'yuv420p' + video_stream.pix_fmt = pix_fmt # Create an audio stream audio_sample_rate = 1 @@ -505,9 +568,14 @@ class VideoFromComponents(VideoInput): # Encode video for i, frame in enumerate(self.__components.images): - img = (frame * 255).clamp(0, 255).byte().cpu().numpy() # shape: (H, W, 3) - frame = av.VideoFrame.from_ndarray(img, format='rgb24') - frame = frame.reformat(format='yuv420p') # Convert to YUV420P as required by h264 + if is_10bit: + # 16-bit RGB keeps float precision through the conversion to 10-bit YUV. + img = (frame.float() * 65535).clamp(0, 65535).cpu().numpy().astype(np.uint16) # shape: (H, W, 3) + frame = av.VideoFrame.from_ndarray(img, format="rgb48le") + else: + img = (frame * 255).clamp(0, 255).byte().cpu().numpy() # shape: (H, W, 3) + frame = av.VideoFrame.from_ndarray(img, format='rgb24') + frame = frame.reformat(format=pix_fmt) packet = video_stream.encode(frame) output.mux(packet) diff --git a/comfy_api/latest/_io.py b/comfy_api/latest/_io.py index 37614a4c3..58e49d8e2 100644 --- a/comfy_api/latest/_io.py +++ b/comfy_api/latest/_io.py @@ -891,6 +891,14 @@ class Tracks(ComfyTypeIO): track_visibility: torch.Tensor Type = TrackDict +@comfytype(io_type="DICT") +class Dict(ComfyTypeIO): + Type = dict + +@comfytype(io_type="ARRAY") +class Array(ComfyTypeIO): + Type = list + @comfytype(io_type="COMFY_MULTITYPED_V3") class MultiType: Type = Any @@ -1279,6 +1287,19 @@ class Color(ComfyTypeIO): def as_dict(self): return super().as_dict() + +@comfytype(io_type="COLORS") +class Colors(ComfyTypeIO): + Type = list[Color.Type] + + class Input(WidgetInput): + def __init__(self, id: str, display_name: str=None, optional=False, tooltip: str=None, + socketless: bool=True, default: list[str]=None, advanced: bool=None): + super().__init__(id, display_name, optional, tooltip, None, default, socketless, None, None, None, None, advanced) + if default is None: + self.default = [] + + @comfytype(io_type="BOUNDING_BOX") class BoundingBox(ComfyTypeIO): class BoundingBoxDict(TypedDict): @@ -1326,6 +1347,20 @@ class Curve(ComfyTypeIO): return d +@comfytype(io_type="BOUNDING_BOXES") +class BoundingBoxes(ComfyTypeIO): + class BoundingBoxWithMetadata(BoundingBox.BoundingBoxDict): + metadata: dict + Type = list[BoundingBoxWithMetadata] + + class Input(WidgetInput): + def __init__(self, id: str, display_name: str=None, optional=False, tooltip: str=None, + socketless: bool=True, default: list[dict]=None, advanced: bool=None): + super().__init__(id, display_name, optional, tooltip, None, default, socketless, None, None, None, None, advanced) + if default is None: + self.default = [] + + @comfytype(io_type="HISTOGRAM") class Histogram(ComfyTypeIO): """A histogram represented as a list of bin counts.""" @@ -1400,7 +1435,8 @@ class V3Data(TypedDict): class HiddenHolder: def __init__(self, unique_id: str, prompt: Any, extra_pnginfo: Any, dynprompt: Any, - auth_token_comfy_org: str, api_key_comfy_org: str, **kwargs): + auth_token_comfy_org: str, api_key_comfy_org: str, + comfy_usage_source: str = None, **kwargs): self.unique_id = unique_id """UNIQUE_ID is the unique identifier of the node, and matches the id property of the node on the client side. It is commonly used in client-server communications (see messages).""" self.prompt = prompt @@ -1413,6 +1449,8 @@ class HiddenHolder: """AUTH_TOKEN_COMFY_ORG is a token acquired from signing into a ComfyOrg account on frontend.""" self.api_key_comfy_org = api_key_comfy_org """API_KEY_COMFY_ORG is an API Key generated by ComfyOrg that allows skipping signing into a ComfyOrg account on frontend.""" + self.comfy_usage_source = comfy_usage_source + """COMFY_USAGE_SOURCE identifies the client that submitted the prompt (e.g. comfyui-frontend, comfy-cli, comfyui-mcp); forwarded to API nodes' upstream requests via the Comfy-Usage-Source header.""" def __getattr__(self, key: str): '''If hidden variable not found, return None.''' @@ -1429,6 +1467,7 @@ class HiddenHolder: dynprompt=d.get(Hidden.dynprompt, None), auth_token_comfy_org=d.get(Hidden.auth_token_comfy_org, None), api_key_comfy_org=d.get(Hidden.api_key_comfy_org, None), + comfy_usage_source=d.get(Hidden.comfy_usage_source, None), ) @classmethod @@ -1451,6 +1490,8 @@ class Hidden(str, Enum): """AUTH_TOKEN_COMFY_ORG is a token acquired from signing into a ComfyOrg account on frontend.""" api_key_comfy_org = "API_KEY_COMFY_ORG" """API_KEY_COMFY_ORG is an API Key generated by ComfyOrg that allows skipping signing into a ComfyOrg account on frontend.""" + comfy_usage_source = "COMFY_USAGE_SOURCE" + """COMFY_USAGE_SOURCE identifies the client that submitted the prompt (e.g. comfyui-frontend, comfy-cli, comfyui-mcp); forwarded to API nodes' upstream requests via the Comfy-Usage-Source header.""" @dataclass @@ -1654,6 +1695,8 @@ class Schema: self.hidden.append(Hidden.auth_token_comfy_org) if Hidden.api_key_comfy_org not in self.hidden: self.hidden.append(Hidden.api_key_comfy_org) + if Hidden.comfy_usage_source not in self.hidden: + self.hidden.append(Hidden.comfy_usage_source) # if is an output_node, will need prompt and extra_pnginfo if self.is_output_node: if Hidden.prompt not in self.hidden: @@ -2368,6 +2411,8 @@ __all__ = [ "AnyType", "MultiType", "Tracks", + "Dict", + "Array", "Color", # Dynamic Types "MatchType", @@ -2386,6 +2431,8 @@ __all__ = [ "PriceBadgeDepends", "PriceBadge", "BoundingBox", + "BoundingBoxes", + "Colors", "Curve", "Histogram", "Range", diff --git a/comfy_api_nodes/apis/__init__.py b/comfy_api_nodes/apis/__init__.py index 9c4cfb9b6..9a7049ea2 100644 --- a/comfy_api_nodes/apis/__init__.py +++ b/comfy_api_nodes/apis/__init__.py @@ -1310,13 +1310,6 @@ class KlingTaskStatus(str, Enum): failed = 'failed' -class KlingTextToVideoModelName(str, Enum): - kling_v1 = 'kling-v1' - kling_v1_6 = 'kling-v1-6' - kling_v2_1_master = 'kling-v2-1-master' - kling_v2_5_turbo = 'kling-v2-5-turbo' - - class KlingVideoGenAspectRatio(str, Enum): field_16_9 = '16:9' field_9_16 = '9:16' @@ -5179,7 +5172,7 @@ class KlingText2VideoRequest(BaseModel): duration: Optional[KlingVideoGenDuration] = '5' external_task_id: Optional[str] = Field(None, description='Customized Task ID') mode: Optional[KlingVideoGenMode] = 'std' - model_name: Optional[KlingTextToVideoModelName] = 'kling-v1' + model_name: Optional[str] = 'kling-v1' negative_prompt: Optional[str] = Field( None, description='Negative text prompt', max_length=2500 ) diff --git a/comfy_api_nodes/apis/bytedance.py b/comfy_api_nodes/apis/bytedance.py index 47f24586c..5267395a1 100644 --- a/comfy_api_nodes/apis/bytedance.py +++ b/comfy_api_nodes/apis/bytedance.py @@ -1,4 +1,4 @@ -from typing import Literal +from typing import Any, Literal from pydantic import BaseModel, Field @@ -163,15 +163,31 @@ class SeedanceVirtualLibraryCreateAssetRequest(BaseModel): asset_type: str | None = Field(None, description="BytePlus asset type. Defaults to Image server-side when omitted.") -# Dollars per 1K tokens, keyed by (model_id, has_video_input). +# Dollars per 1K tokens, keyed by (model_id, has_video_input, resolution). SEEDANCE2_PRICE_PER_1K_TOKENS = { - ("dreamina-seedance-2-0-260128", False): 0.007, - ("dreamina-seedance-2-0-260128", True): 0.0043, - ("dreamina-seedance-2-0-fast-260128", False): 0.0056, - ("dreamina-seedance-2-0-fast-260128", True): 0.0033, + ("dreamina-seedance-2-0-260128", False, "480p"): 0.007, + ("dreamina-seedance-2-0-260128", True, "480p"): 0.0043, + ("dreamina-seedance-2-0-260128", False, "720p"): 0.007, + ("dreamina-seedance-2-0-260128", True, "720p"): 0.0043, + ("dreamina-seedance-2-0-260128", False, "1080p"): 0.0077, + ("dreamina-seedance-2-0-260128", True, "1080p"): 0.0047, + ("dreamina-seedance-2-0-260128", False, "4k"): 0.004, + ("dreamina-seedance-2-0-260128", True, "4k"): 0.0024, + ("dreamina-seedance-2-0-fast-260128", False, "480p"): 0.0056, + ("dreamina-seedance-2-0-fast-260128", True, "480p"): 0.0033, + ("dreamina-seedance-2-0-fast-260128", False, "720p"): 0.0056, + ("dreamina-seedance-2-0-fast-260128", True, "720p"): 0.0033, + ("dreamina-seedance-2-0-mini", False, "480p"): 0.0035, + ("dreamina-seedance-2-0-mini", True, "480p"): 0.0021, + ("dreamina-seedance-2-0-mini", False, "720p"): 0.0035, + ("dreamina-seedance-2-0-mini", True, "720p"): 0.0021, } +def seedance2_price_per_1k_tokens(model_id: str, has_video_input: bool, resolution: str) -> float | None: + return SEEDANCE2_PRICE_PER_1K_TOKENS.get((model_id, has_video_input, resolution)) + + RECOMMENDED_PRESETS = [ ("1024x1024 (1:1)", 1024, 1024), ("864x1152 (3:4)", 864, 1152), @@ -266,6 +282,10 @@ SEEDANCE2_REF_VIDEO_PIXEL_LIMITS = { "480p": {"min": 409_600, "max": 927_408}, "720p": {"min": 409_600, "max": 927_408}, }, + "dreamina-seedance-2-0-mini": { + "480p": {"min": 409_600, "max": 927_408}, + "720p": {"min": 409_600, "max": 927_408}, + }, } # The time in this dictionary are given for 10 seconds duration. @@ -296,3 +316,36 @@ VIDEO_TASKS_EXECUTION_TIME = { "1080p": 150, }, } + + +class SeedAudioConfig(BaseModel): + format: str = Field(default="mp3") + sample_rate: int = Field(default=24000) + speech_rate: int = Field(default=0) + loudness_rate: int = Field(default=0) + pitch_rate: int = Field(default=0) + + +class SeedAudioReference(BaseModel): + speaker: str | None = Field(default=None) + audio_data: str | None = Field(default=None) + audio_url: str | None = Field(default=None) + image_data: str | None = Field(default=None) + image_url: str | None = Field(default=None) + + +class SeedAudioRequest(BaseModel): + model: str = Field(default="seed-audio-1.0") + text_prompt: str = Field(...) + references: list[SeedAudioReference] | None = Field(default=None) + audio_config: SeedAudioConfig = Field(default_factory=SeedAudioConfig) + watermark: dict[str, Any] = Field(default_factory=dict) + + +class SeedAudioResponse(BaseModel): + audio: str | None = Field(default=None) + url: str | None = Field(default=None) + duration: float | None = Field(default=None) + original_duration: float | None = Field(default=None) + code: int | None = Field(default=None) + message: str | None = Field(default=None) diff --git a/comfy_api_nodes/apis/gemini.py b/comfy_api_nodes/apis/gemini.py index caaba8f36..7b2543270 100644 --- a/comfy_api_nodes/apis/gemini.py +++ b/comfy_api_nodes/apis/gemini.py @@ -121,6 +121,7 @@ class GeminiGenerationConfig(BaseModel): topK: int | None = Field(None, ge=1) topP: float | None = Field(None, ge=0.0, le=1.0) thinkingConfig: GeminiThinkingConfig | None = Field(None) + responseModalities: list[str] | None = Field(None) class GeminiImageOutputOptions(BaseModel): diff --git a/comfy_api_nodes/apis/ideogram.py b/comfy_api_nodes/apis/ideogram.py index c5ad9559f..ee3256e96 100644 --- a/comfy_api_nodes/apis/ideogram.py +++ b/comfy_api_nodes/apis/ideogram.py @@ -33,53 +33,6 @@ class IdeogramColorPalette( ) -class ImageRequest(BaseModel): - aspect_ratio: Optional[str] = Field( - None, - description="Optional. The aspect ratio (e.g., 'ASPECT_16_9', 'ASPECT_1_1'). Cannot be used with resolution. Defaults to 'ASPECT_1_1' if unspecified.", - ) - color_palette: Optional[Dict[str, Any]] = Field( - None, description='Optional. Color palette object. Only for V_2, V_2_TURBO.' - ) - magic_prompt_option: Optional[str] = Field( - None, description="Optional. MagicPrompt usage ('AUTO', 'ON', 'OFF')." - ) - model: str = Field(..., description="The model used (e.g., 'V_2', 'V_2A_TURBO')") - negative_prompt: Optional[str] = Field( - None, - description='Optional. Description of what to exclude. Only for V_1, V_1_TURBO, V_2, V_2_TURBO.', - ) - num_images: Optional[int] = Field( - 1, - description='Optional. Number of images to generate (1-8). Defaults to 1.', - ge=1, - le=8, - ) - prompt: str = Field( - ..., description='Required. The prompt to use to generate the image.' - ) - resolution: Optional[str] = Field( - None, - description="Optional. Resolution (e.g., 'RESOLUTION_1024_1024'). Only for model V_2. Cannot be used with aspect_ratio.", - ) - seed: Optional[int] = Field( - None, - description='Optional. A number between 0 and 2147483647.', - ge=0, - le=2147483647, - ) - style_type: Optional[str] = Field( - None, - description="Optional. Style type ('AUTO', 'GENERAL', 'REALISTIC', 'DESIGN', 'RENDER_3D', 'ANIME'). Only for models V_2 and above.", - ) - - -class IdeogramGenerateRequest(BaseModel): - image_request: ImageRequest = Field( - ..., description='The image generation request parameters.' - ) - - class Datum(BaseModel): is_image_safe: Optional[bool] = Field( None, description='Indicates whether the image is considered safe.' @@ -113,20 +66,6 @@ class StyleCode(RootModel[str]): root: str = Field(..., pattern='^[0-9A-Fa-f]{8}$') -class Datum1(BaseModel): - is_image_safe: Optional[bool] = None - prompt: Optional[str] = None - resolution: Optional[str] = None - seed: Optional[int] = None - style_type: Optional[str] = None - url: Optional[str] = None - - -class IdeogramV3IdeogramResponse(BaseModel): - created: Optional[datetime] = None - data: Optional[List[Datum1]] = None - - class RenderingSpeed1(str, Enum): TURBO = 'TURBO' DEFAULT = 'DEFAULT' diff --git a/comfy_api_nodes/apis/kling.py b/comfy_api_nodes/apis/kling.py index fe0f97cb3..2c98c23b7 100644 --- a/comfy_api_nodes/apis/kling.py +++ b/comfy_api_nodes/apis/kling.py @@ -149,3 +149,59 @@ class MotionControlRequest(BaseModel): character_orientation: str = Field(...) mode: str = Field(..., description="'pro' or 'std'") model_name: str = Field(...) + + +class Kling3TurboSettings(BaseModel): + resolution: str = Field("720p", description="'720p' or '1080p'") + aspect_ratio: str | None = Field(None, description="'16:9'/'9:16'/'1:1'; text-to-video only") + duration: int = Field(5, description="3-15 second") + + +class Kling3TurboText2VideoRequest(BaseModel): + prompt: str = Field(..., description="<=3072 chars; may use multi-shot 'shot n, m, words; ...'") + settings: Kling3TurboSettings | None = Field(None) + + +class Kling3TurboContent(BaseModel): + type: str = Field(..., description="'prompt' or 'first_frame'") + text: str | None = Field(None, description="for type=prompt; <=2500 chars") + url: str | None = Field(None, description="for type=first_frame") + + +class Kling3TurboImage2VideoRequest(BaseModel): + contents: list[Kling3TurboContent] = Field(..., description="prompt + first_frame materials") + settings: Kling3TurboSettings | None = Field(None) + + +class Kling3TurboCreateData(BaseModel): + id: str | None = Field(None, description="Task ID") + status: str | None = Field(None) + message: str | None = Field(None) + + +class Kling3TurboCreateResponse(BaseModel): + code: int | None = Field(None) + message: str | None = Field(None) + request_id: str | None = Field(None) + data: Kling3TurboCreateData | None = Field(None) + + +class Kling3TurboOutput(BaseModel): + type: str | None = Field(None, description="'video', 'image', 'audio', ...") + id: str | None = Field(None) + url: str | None = Field(None) + duration: str | None = Field(None) + + +class Kling3TurboTaskData(BaseModel): + id: str | None = Field(None) + status: str | None = Field(None, description="submitted | processing | succeeded | failed") + message: str | None = Field(None) + outputs: list[Kling3TurboOutput] | None = Field(None) + + +class Kling3TurboQueryResponse(BaseModel): + code: int | None = Field(None) + message: str | None = Field(None) + request_id: str | None = Field(None) + data: list[Kling3TurboTaskData] | None = Field(None) diff --git a/comfy_api_nodes/apis/luma.py b/comfy_api_nodes/apis/luma.py index 8c6db2022..2465c3b37 100644 --- a/comfy_api_nodes/apis/luma.py +++ b/comfy_api_nodes/apis/luma.py @@ -10,6 +10,7 @@ from pydantic import BaseModel, Field, confloat class LumaIO: LUMA_REF = "LUMA_REF" LUMA_CONCEPTS = "LUMA_CONCEPTS" + LUMA_RAY32_KEYFRAME = "LUMA_RAY32_KEYFRAME" class LumaReference: @@ -20,13 +21,14 @@ class LumaReference: def create_api_model(self, download_url: str): return LumaImageRef(url=download_url, weight=self.weight) + class LumaReferenceChain: - def __init__(self, first_ref: LumaReference=None): + def __init__(self, first_ref: LumaReference = None): self.refs: list[LumaReference] = [] if first_ref: self.refs.append(first_ref) - def add(self, luma_ref: LumaReference=None): + def add(self, luma_ref: LumaReference = None): self.refs.append(luma_ref) def create_api_model(self, download_urls: list[str], max_refs=4): @@ -124,7 +126,7 @@ def get_luma_concepts(include_none=False): "pull_out", "aerial", "crane_up", - "eye_level" + "eye_level", ] @@ -162,8 +164,8 @@ class LumaVideoModelOutputDuration(str, Enum): class LumaGenerationType(str, Enum): - video = 'video' - image = 'image' + video = "video" + image = "image" class LumaState(str, Enum): @@ -174,86 +176,109 @@ class LumaState(str, Enum): class LumaAssets(BaseModel): - video: Optional[str] = Field(None, description='The URL of the video') - image: Optional[str] = Field(None, description='The URL of the image') - progress_video: Optional[str] = Field(None, description='The URL of the progress video') + video: Optional[str] = Field(None, description="The URL of the video") + image: Optional[str] = Field(None, description="The URL of the image") + progress_video: Optional[str] = Field(None, description="The URL of the progress video") class LumaImageRef(BaseModel): """Used for image gen""" - url: str = Field(..., description='The URL of the image reference') - weight: confloat(ge=0.0, le=1.0) = Field(..., description='The weight of the image reference') + + url: str = Field(..., description="The URL of the image reference") + weight: confloat(ge=0.0, le=1.0) = Field(..., description="The weight of the image reference") class LumaImageReference(BaseModel): """Used for video gen""" - type: Optional[str] = Field('image', description='Input type, defaults to image') - url: str = Field(..., description='The URL of the image') + + type: Optional[str] = Field("image", description="Input type, defaults to image") + url: str = Field(..., description="The URL of the image") class LumaModifyImageRef(BaseModel): - url: str = Field(..., description='The URL of the image reference') - weight: confloat(ge=0.0, le=1.0) = Field(..., description='The weight of the image reference') + url: str = Field(..., description="The URL of the image reference") + weight: confloat(ge=0.0, le=1.0) = Field(..., description="The weight of the image reference") class LumaCharacterRef(BaseModel): - identity0: LumaImageIdentity = Field(..., description='The image identity object') + identity0: LumaImageIdentity = Field(..., description="The image identity object") class LumaImageIdentity(BaseModel): - images: list[str] = Field(..., description='The URLs of the image identity') + images: list[str] = Field(..., description="The URLs of the image identity") class LumaGenerationReference(BaseModel): - type: str = Field('generation', description='Input type, defaults to generation') - id: str = Field(..., description='The ID of the generation') + type: str = Field("generation", description="Input type, defaults to generation") + id: str = Field(..., description="The ID of the generation") class LumaKeyframes(BaseModel): - frame0: Optional[Union[LumaImageReference, LumaGenerationReference]] = Field(None, description='') - frame1: Optional[Union[LumaImageReference, LumaGenerationReference]] = Field(None, description='') + frame0: Optional[Union[LumaImageReference, LumaGenerationReference]] = Field(None, description="") + frame1: Optional[Union[LumaImageReference, LumaGenerationReference]] = Field(None, description="") class LumaConceptObject(BaseModel): - key: str = Field(..., description='Camera Concept name') + key: str = Field(..., description="Camera Concept name") class LumaImageGenerationRequest(BaseModel): - prompt: str = Field(..., description='The prompt of the generation') - model: LumaImageModel = Field(LumaImageModel.photon_1, description='The image model used for the generation') - aspect_ratio: Optional[LumaAspectRatio] = Field(LumaAspectRatio.ratio_16_9, description='The aspect ratio of the generation') - image_ref: Optional[list[LumaImageRef]] = Field(None, description='List of image reference objects') - style_ref: Optional[list[LumaImageRef]] = Field(None, description='List of style reference objects') - character_ref: Optional[LumaCharacterRef] = Field(None, description='The image identity object') - modify_image_ref: Optional[LumaModifyImageRef] = Field(None, description='The modify image reference object') + prompt: str = Field(..., description="The prompt of the generation") + model: LumaImageModel = Field(LumaImageModel.photon_1, description="The image model used for the generation") + aspect_ratio: Optional[LumaAspectRatio] = Field(LumaAspectRatio.ratio_16_9) + image_ref: Optional[list[LumaImageRef]] = Field(None, description="List of image reference objects") + style_ref: Optional[list[LumaImageRef]] = Field(None, description="List of style reference objects") + character_ref: Optional[LumaCharacterRef] = Field(None, description="The image identity object") + modify_image_ref: Optional[LumaModifyImageRef] = Field(None, description="The modify image reference object") class LumaGenerationRequest(BaseModel): - prompt: str = Field(..., description='The prompt of the generation') - model: LumaVideoModel = Field(LumaVideoModel.ray_2, description='The video model used for the generation') - duration: Optional[LumaVideoModelOutputDuration] = Field(None, description='The duration of the generation') - aspect_ratio: Optional[LumaAspectRatio] = Field(None, description='The aspect ratio of the generation') - resolution: Optional[LumaVideoOutputResolution] = Field(None, description='The resolution of the generation') - loop: Optional[bool] = Field(None, description='Whether to loop the video') - keyframes: Optional[LumaKeyframes] = Field(None, description='The keyframes of the generation') - concepts: Optional[list[LumaConceptObject]] = Field(None, description='Camera Concepts to apply to generation') + prompt: str = Field(..., description="The prompt of the generation") + model: LumaVideoModel = Field(LumaVideoModel.ray_2, description="The video model used for the generation") + duration: Optional[LumaVideoModelOutputDuration] = Field(None, description="The duration of the generation") + aspect_ratio: Optional[LumaAspectRatio] = Field(None, description="The aspect ratio of the generation") + resolution: Optional[LumaVideoOutputResolution] = Field(None, description="The resolution of the generation") + loop: Optional[bool] = Field(None, description="Whether to loop the video") + keyframes: Optional[LumaKeyframes] = Field(None, description="The keyframes of the generation") + concepts: Optional[list[LumaConceptObject]] = Field(None, description="Camera Concepts to apply to generation") class LumaGeneration(BaseModel): - id: str = Field(..., description='The ID of the generation') - generation_type: LumaGenerationType = Field(..., description='Generation type, image or video') - state: LumaState = Field(..., description='The state of the generation') - failure_reason: Optional[str] = Field(None, description='The reason for the state of the generation') - created_at: str = Field(..., description='The date and time when the generation was created') - assets: Optional[LumaAssets] = Field(None, description='The assets of the generation') - model: str = Field(..., description='The model used for the generation') - request: Union[LumaGenerationRequest, LumaImageGenerationRequest] = Field(..., description="The request used for the generation") + id: str = Field(..., description="The ID of the generation") + generation_type: LumaGenerationType = Field(..., description="Generation type, image or video") + state: LumaState = Field(..., description="The state of the generation") + failure_reason: Optional[str] = Field(None, description="The reason for the state of the generation") + created_at: str = Field(..., description="The date and time when the generation was created") + assets: Optional[LumaAssets] = Field(None, description="The assets of the generation") + model: str = Field(..., description="The model used for the generation") + request: Union[LumaGenerationRequest, LumaImageGenerationRequest] = Field(...) class Luma2ImageRef(BaseModel): url: str | None = None data: str | None = None media_type: str | None = None + generation_id: str | None = Field(None, description="reference a prior generation (extend / source reuse)") + + +class Luma2VideoEdit(BaseModel): + """Edit controls for Ray 3.2 ``video_edit`` generations.""" + + auto_controls: bool | None = Field(None, description="derive a conditioning schedule from the source (recommended)") + strength: str | None = Field(None, description="'adhere_1' .. 'reimagine_3'; constrained by IO.Combo") + + +class Luma2VideoOptions(BaseModel): + """Ray 3.2 ``video`` output settings (text / image / keyframe / edit / extend).""" + + resolution: str | None = Field(None, description="360p | 540p | 720p | 1080p") + duration: str | None = Field(None, description="5s | 10s") + loop: bool | None = Field(None) + start_frame: Luma2ImageRef | None = Field(None) + end_frame: Luma2ImageRef | None = Field(None) + keyframes: list[Luma2ImageRef] | None = Field(None) + keyframe_indexes: list[int] | None = Field(None) + edit: Luma2VideoEdit | None = Field(None) class Luma2GenerationRequest(BaseModel): @@ -266,6 +291,7 @@ class Luma2GenerationRequest(BaseModel): web_search: bool | None = None image_ref: list[Luma2ImageRef] | None = None source: Luma2ImageRef | None = None + video: Luma2VideoOptions | None = Field(None) class Luma2Generation(BaseModel): @@ -277,3 +303,31 @@ class Luma2Generation(BaseModel): output: list[LumaImageReference] | None = None failure_reason: str | None = None failure_code: str | None = None + + +# --- Ray 3.2 multi-keyframe chain --- + +LUMA_KEYFRAME_MODE_FRACTION = "fraction" # value in [0.0, 1.0] of the output video duration +LUMA_KEYFRAME_MODE_SECONDS = "seconds" # absolute time, in seconds, from the start of the output + + +class LumaRay32KeyframeItem: + """One guide image anchored at a position on the Ray 3.2 output timeline.""" + + def __init__(self, image: torch.Tensor, mode: str, value: float): + self.image = image + self.mode = mode # LUMA_KEYFRAME_MODE_FRACTION | LUMA_KEYFRAME_MODE_SECONDS + self.value = value + + +class LumaRay32KeyframeChain: + def __init__(self): + self.items: list[LumaRay32KeyframeItem] = [] + + def add(self, item: LumaRay32KeyframeItem) -> None: + self.items.append(item) + + def clone(self) -> "LumaRay32KeyframeChain": + c = LumaRay32KeyframeChain() + c.items = list(self.items) + return c diff --git a/comfy_api_nodes/apis/runway.py b/comfy_api_nodes/apis/runway.py index df6f2b845..6878aa6f0 100644 --- a/comfy_api_nodes/apis/runway.py +++ b/comfy_api_nodes/apis/runway.py @@ -67,15 +67,6 @@ class RunwayImageToVideoResponse(BaseModel): id: Optional[str] = Field(None, description='Task ID') -class RunwayTaskStatusEnum(str, Enum): - SUCCEEDED = 'SUCCEEDED' - RUNNING = 'RUNNING' - FAILED = 'FAILED' - PENDING = 'PENDING' - CANCELLED = 'CANCELLED' - THROTTLED = 'THROTTLED' - - class RunwayTaskStatusResponse(BaseModel): createdAt: datetime = Field(..., description='Task creation timestamp') id: str = Field(..., description='Task ID') @@ -86,7 +77,7 @@ class RunwayTaskStatusResponse(BaseModel): ge=0.0, le=1.0, ) - status: RunwayTaskStatusEnum + status: str = Field(..., description="SUCCEEDED, RUNNING, FAILED, PENDING, CANCELLED or THROTTLED") class Model4(str, Enum): @@ -125,3 +116,144 @@ class RunwayTextToImageRequest(BaseModel): class RunwayTextToImageResponse(BaseModel): id: Optional[str] = Field(None, description='Task ID') + + +class RunwayAleph2IO: + """Custom socket types for chaining Aleph2 guidance images.""" + + KEYFRAME = "RUNWAY_ALEPH2_KEYFRAME" + PROMPT_IMAGE = "RUNWAY_ALEPH2_PROMPT_IMAGE" + + +# Keyframe timing modes (anchored to the INPUT video). Stored on the chain item and used to +# choose the request model below. The values match the Aleph2 keyframe union field names. +KEYFRAME_MODE_SECONDS = "seconds" # absolute time, in seconds, from the start of the input video +KEYFRAME_MODE_AT = "at" # fraction [0.0, 1.0] of the input video duration + +# Prompt-image position modes (anchored to the OUTPUT video). Values match the Aleph2 position `type`. +PROMPT_IMAGE_MODE_TIMESTAMP = "timestamp" # absolute time, in seconds, from the start of the output video +PROMPT_IMAGE_MODE_POSITION = "position" # fraction [0.0, 1.0] of the output video duration + + +class RunwayAleph2KeyframeItem: + """A guidance image anchored to a point of the INPUT video (one Aleph2 ``keyframe``).""" + + def __init__(self, image, mode: str, value: float): + self.image = image + self.mode = mode # KEYFRAME_MODE_SECONDS | KEYFRAME_MODE_AT + self.value = value + + +class RunwayAleph2KeyframeChain: + """An ordered collection of keyframes, built by chaining Runway Aleph2 Keyframe nodes.""" + + def __init__(self): + self.items: list[RunwayAleph2KeyframeItem] = [] + + def add(self, item: RunwayAleph2KeyframeItem) -> None: + self.items.append(item) + + def clone(self) -> "RunwayAleph2KeyframeChain": + c = RunwayAleph2KeyframeChain() + c.items = list(self.items) + return c + + +class RunwayAleph2PromptImageItem: + """A guidance image anchored to a point of the OUTPUT video (one Aleph2 ``promptImage``).""" + + def __init__(self, image, mode: str, value: float): + self.image = image + self.mode = mode # PROMPT_IMAGE_MODE_TIMESTAMP | PROMPT_IMAGE_MODE_POSITION + self.value = value + + +class RunwayAleph2PromptImageChain: + """An ordered collection of prompt images, built by chaining Runway Aleph2 Prompt Image nodes.""" + + def __init__(self): + self.items: list[RunwayAleph2PromptImageItem] = [] + + def add(self, item: RunwayAleph2PromptImageItem) -> None: + self.items.append(item) + + def clone(self) -> "RunwayAleph2PromptImageChain": + c = RunwayAleph2PromptImageChain() + c.items = list(self.items) + return c + + +class RunwayAleph2KeyframeSeconds(BaseModel): + seconds: float = Field( + ..., + description="Absolute timestamp in seconds from the start of the input video when this guidance image should apply.", + ge=0.0, + ) + uri: str = Field(...) + + +class RunwayAleph2KeyframeAt(BaseModel): + at: float = Field( + ..., + description="Position as a fraction [0.0, 1.0] of the input video duration.", + ge=0.0, + le=1.0, + ) + uri: str = Field(...) + + +class RunwayAleph2TimestampPosition(BaseModel): + type: str = Field(default="timestamp") + timestampSeconds: float = Field( + ..., + description="Absolute timestamp in seconds from the start of the output video.", + ge=0.0, + ) + + +class RunwayAleph2RelativePosition(BaseModel): + type: str = Field(default="position") + positionPercentage: float = Field( + ..., + description="Position as a fraction [0.0, 1.0] of the total output video duration.", + ge=0.0, + le=1.0, + ) + + +class RunwayAleph2PromptImage(BaseModel): + position: RunwayAleph2TimestampPosition | RunwayAleph2RelativePosition + uri: str = Field(...) + + +class RunwayAleph2ContentModeration(BaseModel): + publicFigureThreshold: str = Field( + ..., + description='When set to "low", the content moderation system is less strict about ' + 'recognizable public figures. One of "auto" or "low".', + ) + + +class RunwayAleph2Request(BaseModel): + model: str = Field(default="aleph2") + promptText: str = Field( + ..., + description="A non-empty string describing what should appear in the output.", + min_length=1, + max_length=1000, + ) + videoUri: str = Field(...) + seed: int = Field(..., description="Random seed for generation", ge=0, le=4294967295) + contentModeration: RunwayAleph2ContentModeration = Field(...) + keyframes: list[RunwayAleph2KeyframeSeconds | RunwayAleph2KeyframeAt] | None = Field( + None, + description="Timed guidance images placed at specific points in the input video. Up to 5.", + ) + promptImage: list[RunwayAleph2PromptImage] | None = Field( + None, + description="Up to 5 image keyframes for guiding the edit at specific points in the output video.", + ) + + +class RunwayAleph2Response(BaseModel): + id: str | None = Field(None, description="Task ID") diff --git a/comfy_api_nodes/apis/stability.py b/comfy_api_nodes/apis/stability.py deleted file mode 100644 index 5b9b5ac7d..000000000 --- a/comfy_api_nodes/apis/stability.py +++ /dev/null @@ -1,147 +0,0 @@ -from enum import Enum -from typing import Optional - -from pydantic import BaseModel, Field, confloat - - -class StabilityFormat(str, Enum): - png = 'png' - jpeg = 'jpeg' - webp = 'webp' - - -class StabilityAspectRatio(str, Enum): - ratio_1_1 = "1:1" - ratio_16_9 = "16:9" - ratio_9_16 = "9:16" - ratio_3_2 = "3:2" - ratio_2_3 = "2:3" - ratio_5_4 = "5:4" - ratio_4_5 = "4:5" - ratio_21_9 = "21:9" - ratio_9_21 = "9:21" - - -def get_stability_style_presets(include_none=True): - presets = [] - if include_none: - presets.append("None") - return presets + [x.value for x in StabilityStylePreset] - - -class StabilityStylePreset(str, Enum): - _3d_model = "3d-model" - analog_film = "analog-film" - anime = "anime" - cinematic = "cinematic" - comic_book = "comic-book" - digital_art = "digital-art" - enhance = "enhance" - fantasy_art = "fantasy-art" - isometric = "isometric" - line_art = "line-art" - low_poly = "low-poly" - modeling_compound = "modeling-compound" - neon_punk = "neon-punk" - origami = "origami" - photographic = "photographic" - pixel_art = "pixel-art" - tile_texture = "tile-texture" - - -class Stability_SD3_5_Model(str, Enum): - sd3_5_large = "sd3.5-large" - # sd3_5_large_turbo = "sd3.5-large-turbo" - sd3_5_medium = "sd3.5-medium" - - -class Stability_SD3_5_GenerationMode(str, Enum): - text_to_image = "text-to-image" - image_to_image = "image-to-image" - - -class StabilityStable3_5Request(BaseModel): - model: str = Field(...) - mode: str = Field(...) - prompt: str = Field(...) - negative_prompt: Optional[str] = Field(None) - aspect_ratio: Optional[str] = Field(None) - seed: Optional[int] = Field(None) - output_format: Optional[str] = Field(StabilityFormat.png.value) - image: Optional[str] = Field(None) - style_preset: Optional[str] = Field(None) - cfg_scale: float = Field(...) - strength: Optional[confloat(ge=0.0, le=1.0)] = Field(None) - - -class StabilityUpscaleConservativeRequest(BaseModel): - prompt: str = Field(...) - negative_prompt: Optional[str] = Field(None) - seed: Optional[int] = Field(None) - output_format: Optional[str] = Field(StabilityFormat.png.value) - image: Optional[str] = Field(None) - creativity: Optional[confloat(ge=0.2, le=0.5)] = Field(None) - - -class StabilityUpscaleCreativeRequest(BaseModel): - prompt: str = Field(...) - negative_prompt: Optional[str] = Field(None) - seed: Optional[int] = Field(None) - output_format: Optional[str] = Field(StabilityFormat.png.value) - image: Optional[str] = Field(None) - creativity: Optional[confloat(ge=0.1, le=0.5)] = Field(None) - style_preset: Optional[str] = Field(None) - - -class StabilityStableUltraRequest(BaseModel): - prompt: str = Field(...) - negative_prompt: Optional[str] = Field(None) - aspect_ratio: Optional[str] = Field(None) - seed: Optional[int] = Field(None) - output_format: Optional[str] = Field(StabilityFormat.png.value) - image: Optional[str] = Field(None) - style_preset: Optional[str] = Field(None) - strength: Optional[confloat(ge=0.0, le=1.0)] = Field(None) - - -class StabilityStableUltraResponse(BaseModel): - image: Optional[str] = Field(None) - finish_reason: Optional[str] = Field(None) - seed: Optional[int] = Field(None) - - -class StabilityResultsGetResponse(BaseModel): - image: Optional[str] = Field(None) - finish_reason: Optional[str] = Field(None) - seed: Optional[int] = Field(None) - id: Optional[str] = Field(None) - name: Optional[str] = Field(None) - errors: Optional[list[str]] = Field(None) - status: Optional[str] = Field(None) - result: Optional[str] = Field(None) - - -class StabilityAsyncResponse(BaseModel): - id: Optional[str] = Field(None) - - -class StabilityTextToAudioRequest(BaseModel): - model: str = Field(...) - prompt: str = Field(...) - duration: int = Field(190, ge=1, le=190) - seed: int = Field(0, ge=0, le=4294967294) - steps: int = Field(8, ge=4, le=8) - output_format: str = Field("wav") - - -class StabilityAudioToAudioRequest(StabilityTextToAudioRequest): - strength: float = Field(0.01, ge=0.01, le=1.0) - - -class StabilityAudioInpaintRequest(StabilityTextToAudioRequest): - mask_start: int = Field(30, ge=0, le=190) - mask_end: int = Field(190, ge=0, le=190) - - -class StabilityAudioResponse(BaseModel): - audio: Optional[str] = Field(None) diff --git a/comfy_api_nodes/apis/tripo.py b/comfy_api_nodes/apis/tripo.py index 7ac81d42c..79913997a 100644 --- a/comfy_api_nodes/apis/tripo.py +++ b/comfy_api_nodes/apis/tripo.py @@ -208,6 +208,10 @@ class TripoMultiviewToModelRequest(BaseModel): quad: bool | None = Field(False, description="Whether to apply quad to the generated model") +class TripoTexturePrompt(BaseModel): + text: str | None = Field(None, description="Text guidance for texture generation") + + class TripoTextureModelRequest(BaseModel): type: TripoTaskType = Field(TripoTaskType.TEXTURE_MODEL, description="Type of task") original_model_task_id: str = Field(..., description="The task ID of the original model") @@ -219,6 +223,11 @@ class TripoTextureModelRequest(BaseModel): texture_alignment: TripoTextureAlignment | None = Field( TripoTextureAlignment.ORIGINAL_IMAGE, description="The texture alignment method" ) + texture_prompt: TripoTexturePrompt | None = Field( + None, + description="Optional guidance for texturing. Required in practice for imported models, " + "which carry no source image to infer texture from.", + ) class TripoRefineModelRequest(BaseModel): @@ -307,6 +316,17 @@ class TripoP1MultiviewToModelRequest(TripoP1CommonRequest): orientation: str | None = None +class TripoImportModelRequest(BaseModel): + """Request for the comfy-api composite import endpoint (/proxy/tripo/v2/openapi/import). + + The model file is uploaded to ComfyUI API storage first; the backend downloads it from + `url`, re-uploads it to Tripo's storage and creates the import_model task server-side. + """ + + url: str = Field(..., description="ComfyUI API storage download URL of the model file") + format: str = Field(..., description='File format: "glb", "fbx", "obj" or "stl"') + + class TripoTaskOutput(BaseModel): model: str | None = Field(None, description="URL to the model") base_model: str | None = Field(None, description="URL to the base model") diff --git a/comfy_api_nodes/nodes_bria.py b/comfy_api_nodes/nodes_bria.py index e138fafa9..090154afb 100644 --- a/comfy_api_nodes/nodes_bria.py +++ b/comfy_api_nodes/nodes_bria.py @@ -289,7 +289,7 @@ class BriaRemoveVideoBackground(IO.ComfyNode): ], is_api_node=True, price_badge=IO.PriceBadge( - expr="""{"type":"usd","usd":0.14,"format":{"suffix":"/second"}}""", + expr="""{"type":"usd","usd":0.0042,"format":{"suffix":"/second"}}""", ), ) @@ -357,7 +357,7 @@ class BriaVideoGreenScreen(IO.ComfyNode): ], is_api_node=True, price_badge=IO.PriceBadge( - expr="""{"type":"usd","usd":0.14,"format":{"suffix":"/second"}}""", + expr="""{"type":"usd","usd":0.0042,"format":{"suffix":"/second"}}""", ), ) @@ -433,7 +433,7 @@ class BriaVideoReplaceBackground(IO.ComfyNode): ], is_api_node=True, price_badge=IO.PriceBadge( - expr="""{"type":"usd","usd":0.14,"format":{"suffix":"/second"}}""", + expr="""{"type":"usd","usd":0.0042,"format":{"suffix":"/second"}}""", ), ) @@ -452,7 +452,10 @@ class BriaVideoReplaceBackground(IO.ComfyNode): validate_video_duration(background_video, max_duration=60.0) background_url = await upload_video_to_comfyapi(cls, background_video, wait_label="Uploading background") else: - background_url = await upload_image_to_comfyapi(cls, background_image, wait_label="Uploading background") + # Bria's replace_background 500s on RGBA, so drop the alpha channel before upload. + background_url = await upload_image_to_comfyapi( + cls, background_image[:, :, :, :3], wait_label="Uploading background" + ) response = await sync_op( cls, ApiEndpoint(path="/proxy/bria/v2/video/edit/replace_background", method="POST"), @@ -530,7 +533,7 @@ class BriaTransparentVideoBackground(IO.ComfyNode): ], is_api_node=True, price_badge=IO.PriceBadge( - expr="""{"type":"usd","usd":0.14,"format":{"suffix":"/second"}}""", + expr="""{"type":"usd","usd":0.0042,"format":{"suffix":"/second"}}""", ), ) @@ -571,7 +574,7 @@ class BriaExtension(ComfyExtension): BriaRemoveImageBackground, BriaRemoveVideoBackground, BriaVideoGreenScreen, - # BriaVideoReplaceBackground, # server returns Status 500 when we pass background video + BriaVideoReplaceBackground, BriaTransparentVideoBackground, ] diff --git a/comfy_api_nodes/nodes_bytedance.py b/comfy_api_nodes/nodes_bytedance.py index c30ddc446..58307290d 100644 --- a/comfy_api_nodes/nodes_bytedance.py +++ b/comfy_api_nodes/nodes_bytedance.py @@ -1,3 +1,4 @@ +import base64 import hashlib import logging import math @@ -15,12 +16,15 @@ from comfy_api_nodes.apis.bytedance import ( RECOMMENDED_PRESETS_SEEDREAM_4_0, RECOMMENDED_PRESETS_SEEDREAM_4_5, RECOMMENDED_PRESETS_SEEDREAM_5_LITE, - SEEDANCE2_PRICE_PER_1K_TOKENS, SEEDANCE2_REF_VIDEO_PIXEL_LIMITS, VIDEO_TASKS_EXECUTION_TIME, GetAssetResponse, Image2VideoTaskCreationRequest, ImageTaskCreationResponse, + SeedAudioConfig, + SeedAudioReference, + SeedAudioRequest, + SeedAudioResponse, Seedance2TaskCreationRequest, SeedanceCreateAssetRequest, SeedanceCreateAssetResponse, @@ -40,9 +44,12 @@ from comfy_api_nodes.apis.bytedance import ( TaskVideoContentUrl, Text2ImageTaskCreationRequest, Text2VideoTaskCreationRequest, + seedance2_price_per_1k_tokens, ) from comfy_api_nodes.util import ( ApiEndpoint, + audio_bytes_to_audio_input, + audio_input_to_mp3, download_url_to_image_tensor, download_url_to_video_output, downscale_image_tensor_by_max_side, @@ -51,11 +58,14 @@ from comfy_api_nodes.util import ( image_tensor_pair_to_batch, poll_op, sync_op, + tensor_to_base64_string, upload_audio_to_comfyapi, upload_image_to_comfyapi, upload_images_to_comfyapi, upload_video_to_comfyapi, + upscale_image_tensor_to_min_pixels, upscale_video_to_min_pixels, + validate_audio_duration, validate_image_aspect_ratio, validate_image_dimensions, validate_string, @@ -89,6 +99,7 @@ BYTEPLUS_SEEDANCE2_TASK_STATUS_ENDPOINT = "/proxy/byteplus-seedance2/api/v3/cont SEEDANCE_MODELS = { "Seedance 2.0": "dreamina-seedance-2-0-260128", "Seedance 2.0 Fast": "dreamina-seedance-2-0-fast-260128", + "Seedance 2.0 Mini": "dreamina-seedance-2-0-mini", } DEPRECATED_MODELS = {"seedance-1-0-lite-t2v-250428", "seedance-1-0-lite-i2v-250428"} @@ -141,7 +152,7 @@ SEEDANCE2_RATIO_WH = { "9:16": (9, 16), "21:9": (21, 9), } -SEEDANCE2_RES_SHORT_SIDE = {"480p": 480, "720p": 720, "1080p": 1080} +SEEDANCE2_RES_SHORT_SIDE = {"480p": 480, "720p": 720, "1080p": 1080, "4k": 2160} def _seedance2_target_dims(resolution: str, ratio: str, image: torch.Tensor) -> tuple[int, int]: @@ -377,9 +388,9 @@ async def _seedance_virtual_library_upload_video_asset( return f"asset://{create_resp.asset_id}" -def _seedance2_price_extractor(model_id: str, has_video_input: bool): +def _seedance2_price_extractor(model_id: str, has_video_input: bool, resolution: str): """Returns a price_extractor closure for Seedance 2.0 poll_op.""" - rate = SEEDANCE2_PRICE_PER_1K_TOKENS.get((model_id, has_video_input)) + rate = seedance2_price_per_1k_tokens(model_id, has_video_input, resolution) if rate is None: return None @@ -1621,10 +1632,12 @@ class ByteDance2TextToVideoNode(IO.ComfyNode): IO.DynamicCombo.Input( "model", options=[ - IO.DynamicCombo.Option("Seedance 2.0", _seedance2_text_inputs(["480p", "720p", "1080p"])), + IO.DynamicCombo.Option("Seedance 2.0", _seedance2_text_inputs(["480p", "720p", "1080p", "4k"])), IO.DynamicCombo.Option("Seedance 2.0 Fast", _seedance2_text_inputs(["480p", "720p"])), + IO.DynamicCombo.Option("Seedance 2.0 Mini", _seedance2_text_inputs(["480p", "720p"])), ], - tooltip="Seedance 2.0 for maximum quality; Seedance 2.0 Fast for speed optimization.", + tooltip="Seedance 2.0 for maximum quality; Fast for speed optimization; " + "Mini for the fastest, lowest-cost generation.", ), IO.Int.Input( "seed", @@ -1660,11 +1673,16 @@ class ByteDance2TextToVideoNode(IO.ComfyNode): $rate480 := 10044; $rate720 := 21600; $rate1080 := 48800; + $rate4k := 195200; $m := widgets.model; - $pricePer1K := $contains($m, "fast") ? 0.008008 : 0.01001; $res := $lookup(widgets, "model.resolution"); $dur := $lookup(widgets, "model.duration"); - $rate := $res = "1080p" ? $rate1080 : + $pricePer1K := $res = "4k" ? 0.00572 : + $res = "1080p" ? 0.011011 : + $contains($m, "mini") ? 0.005005 : + $contains($m, "fast") ? 0.008008 : 0.01001; + $rate := $res = "4k" ? $rate4k : + $res = "1080p" ? $rate1080 : $res = "720p" ? $rate720 : $rate480; $cost := $dur * $rate * $pricePer1K / 1000; @@ -1703,7 +1721,7 @@ class ByteDance2TextToVideoNode(IO.ComfyNode): ApiEndpoint(path=f"{BYTEPLUS_SEEDANCE2_TASK_STATUS_ENDPOINT}/{initial_response.id}"), response_model=TaskStatusResponse, status_extractor=lambda r: r.status, - price_extractor=_seedance2_price_extractor(model_id, has_video_input=False), + price_extractor=_seedance2_price_extractor(model_id, has_video_input=False, resolution=model["resolution"]), poll_interval=9, ) return IO.NodeOutput(await download_url_to_video_output(response.content.video_url)) @@ -1724,14 +1742,19 @@ class ByteDance2FirstLastFrameNode(IO.ComfyNode): options=[ IO.DynamicCombo.Option( "Seedance 2.0", - _seedance2_text_inputs(["480p", "720p", "1080p"], default_ratio="adaptive"), + _seedance2_text_inputs(["480p", "720p", "1080p", "4k"], default_ratio="adaptive"), ), IO.DynamicCombo.Option( "Seedance 2.0 Fast", _seedance2_text_inputs(["480p", "720p"], default_ratio="adaptive"), ), + IO.DynamicCombo.Option( + "Seedance 2.0 Mini", + _seedance2_text_inputs(["480p", "720p"], default_ratio="adaptive"), + ), ], - tooltip="Seedance 2.0 for maximum quality; Seedance 2.0 Fast for speed optimization.", + tooltip="Seedance 2.0 for maximum quality; Fast for speed optimization; " + "Mini for the fastest, lowest-cost generation.", ), IO.Image.Input( "first_frame", @@ -1791,11 +1814,16 @@ class ByteDance2FirstLastFrameNode(IO.ComfyNode): $rate480 := 10044; $rate720 := 21600; $rate1080 := 48800; + $rate4k := 195200; $m := widgets.model; - $pricePer1K := $contains($m, "fast") ? 0.008008 : 0.01001; $res := $lookup(widgets, "model.resolution"); $dur := $lookup(widgets, "model.duration"); - $rate := $res = "1080p" ? $rate1080 : + $pricePer1K := $res = "4k" ? 0.00572 : + $res = "1080p" ? 0.011011 : + $contains($m, "mini") ? 0.005005 : + $contains($m, "fast") ? 0.008008 : 0.01001; + $rate := $res = "4k" ? $rate4k : + $res = "1080p" ? $rate1080 : $res = "720p" ? $rate720 : $rate480; $cost := $dur * $rate * $pricePer1K / 1000; @@ -1913,7 +1941,7 @@ class ByteDance2FirstLastFrameNode(IO.ComfyNode): ApiEndpoint(path=f"{BYTEPLUS_SEEDANCE2_TASK_STATUS_ENDPOINT}/{initial_response.id}"), response_model=TaskStatusResponse, status_extractor=lambda r: r.status, - price_extractor=_seedance2_price_extractor(model_id, has_video_input=False), + price_extractor=_seedance2_price_extractor(model_id, has_video_input=False, resolution=model["resolution"]), poll_interval=9, ) return IO.NodeOutput(await download_url_to_video_output(response.content.video_url)) @@ -2010,14 +2038,19 @@ class ByteDance2ReferenceNode(IO.ComfyNode): options=[ IO.DynamicCombo.Option( "Seedance 2.0", - _seedance2_reference_inputs(["480p", "720p", "1080p"], default_ratio="adaptive"), + _seedance2_reference_inputs(["480p", "720p", "1080p", "4k"], default_ratio="adaptive"), ), IO.DynamicCombo.Option( "Seedance 2.0 Fast", _seedance2_reference_inputs(["480p", "720p"], default_ratio="adaptive"), ), + IO.DynamicCombo.Option( + "Seedance 2.0 Mini", + _seedance2_reference_inputs(["480p", "720p"], default_ratio="adaptive"), + ), ], - tooltip="Seedance 2.0 for maximum quality; Seedance 2.0 Fast for speed optimization.", + tooltip="Seedance 2.0 for maximum quality; Fast for speed optimization; " + "Mini for the fastest, lowest-cost generation.", ), IO.Int.Input( "seed", @@ -2056,13 +2089,21 @@ class ByteDance2ReferenceNode(IO.ComfyNode): $rate480 := 10044; $rate720 := 21600; $rate1080 := 48800; + $rate4k := 195200; $m := widgets.model; $hasVideo := $lookup(inputGroups, "model.reference_videos") > 0; - $noVideoPricePer1K := $contains($m, "fast") ? 0.008008 : 0.01001; - $videoPricePer1K := $contains($m, "fast") ? 0.004719 : 0.006149; $res := $lookup(widgets, "model.resolution"); $dur := $lookup(widgets, "model.duration"); - $rate := $res = "1080p" ? $rate1080 : + $noVideoPricePer1K := $res = "4k" ? 0.00572 : + $res = "1080p" ? 0.011011 : + $contains($m, "mini") ? 0.005005 : + $contains($m, "fast") ? 0.008008 : 0.01001; + $videoPricePer1K := $res = "4k" ? 0.003432 : + $res = "1080p" ? 0.006721 : + $contains($m, "mini") ? 0.003003 : + $contains($m, "fast") ? 0.004719 : 0.006149; + $rate := $res = "4k" ? $rate4k : + $res = "1080p" ? $rate1080 : $res = "720p" ? $rate720 : $rate480; $noVideoCost := $dur * $rate * $noVideoPricePer1K / 1000; @@ -2258,7 +2299,9 @@ class ByteDance2ReferenceNode(IO.ComfyNode): ApiEndpoint(path=f"{BYTEPLUS_SEEDANCE2_TASK_STATUS_ENDPOINT}/{initial_response.id}"), response_model=TaskStatusResponse, status_extractor=lambda r: r.status, - price_extractor=_seedance2_price_extractor(model_id, has_video_input=has_video_input), + price_extractor=_seedance2_price_extractor( + model_id, has_video_input=has_video_input, resolution=model["resolution"] + ), poll_interval=9, ) return IO.NodeOutput(await download_url_to_video_output(response.content.video_url)) @@ -2441,6 +2484,311 @@ class ByteDanceCreateVideoAsset(IO.ComfyNode): return IO.NodeOutput(asset_id, resolved_group) +MODE_TEXT = "text only" +MODE_AUDIO = "audio reference" +MODE_IMAGE = "image reference" +MODE_SPEAKER = "preset voice" + +# (speaker_id, display_label) for built-in TTS 2.0 voices; resolvable ids are account-scoped. +SEED_AUDIO_PRESET_VOICES: list[tuple[str, str]] = [ + ("zh_female_vv_uranus_bigtts", "Vivi (Female, multilingual)"), + ("zh_female_xiaohe_uranus_bigtts", "Mindy (Female, multilingual)"), + ("en_female_stokie_uranus_bigtts", "Stokie (Female, English)"), + ("en_female_dacey_uranus_bigtts", "Dacey (Female, English)"), + ("en_male_tim_uranus_bigtts", "Tim (Male, English)"), + ("zh_male_m191_uranus_bigtts", "Kian (Male, multilingual)"), + ("zh_male_taocheng_uranus_bigtts", "Cedric (Male, multilingual)"), + ("zh_male_sophie_uranus_bigtts", "Sophie (Female, multilingual)"), + ("zh_female_yingyujiaoxue_uranus_bigtts", "Jean (Female, multilingual)"), + ("zh_male_dayi_uranus_bigtts", "Magnus (Male, multilingual)"), + ("zh_female_mizai_uranus_bigtts", "Mabel (Female, multilingual)"), + ("zh_female_jitangnv_uranus_bigtts", "Nadia (Female, multilingual)"), + ("zh_female_meilinvyou_uranus_bigtts", "Opal (Female, multilingual)"), + ("zh_female_liuchangnv_uranus_bigtts", "Pearl (Female, multilingual)"), + ("zh_male_ruyayichen_uranus_bigtts", "Quentin (Male, multilingual)"), + ("zh_female_vivo_uranus_bigtts", "Vienna (Female, multilingual)"), + ("zh_female_xiaoai_uranus_bigtts", "Alina (Female, multilingual)"), + ("zh_female_cancan_uranus_bigtts", "Corinne (Female, multilingual)"), + ("zh_female_tianmeixiaoyuan_uranus_bigtts", "Esther (Female, multilingual)"), + ("zh_female_tianmeitaozi_uranus_bigtts", "Freya (Female, multilingual)"), + ("zh_female_shuangkuaisisi_uranus_bigtts", "Gigi (Female, multilingual)"), + ("zh_female_peiqi_uranus_bigtts", "Holly (Female, multilingual)"), + ("zh_female_xiaoxue_uranus_bigtts", "Lyla (Female, multilingual)"), + ("zh_female_yuanqi_uranus_bigtts", "Daisy (Female, multilingual)"), + ("zh_female_kefunvsheng_uranus_bigtts", "Tracy (Female, multilingual)"), + ("zh_male_shaonianzixin_uranus_bigtts", "Jess (Male, multilingual)"), + ("zh_female_linjianvhai_uranus_bigtts", "Pinky (Female, multilingual)"), + ("zh_female_kiwi_uranus_bigtts", "Sweety (Female, multilingual)"), + ("zh_female_sajiaoxuemei_uranus_bigtts", "Sandy (Female, multilingual)"), + ("de_male_seven_uranus_bigtts", "Sven (Male, German)"), + ("jp_female_minimi_uranus_bigtts", "Minimi (Female, Japanese)"), + ("fr_male_usseau_uranus_bigtts", "Usseau (Male, French)"), + ("es_male_felipe_uranus_bigtts", "Felipe (Male, Spanish)"), + ("id_male_han_uranus_bigtts", "Han (Male, Indonesian)"), + ("pt_male_martins_uranus_bigtts", "Martins (Male, Portuguese)"), + ("it_male_enzo_uranus_bigtts", "Enzo (Male, Italian)"), + ("kr_male_shane_uranus_bigtts", "Shane (Male, Korean)"), + ("zh_male_liufei_uranus_bigtts", "Felix (Male, Chinese)"), + ("zh_female_qingxinnvsheng_uranus_bigtts", "Celeste (Female, Chinese)"), + ("zh_male_sunwukong_uranus_bigtts", "Monkey King (Male, Chinese)"), +] +SEED_AUDIO_VOICE_OPTIONS = [label for _, label in SEED_AUDIO_PRESET_VOICES] +SEED_AUDIO_VOICE_MAP = {label: speaker_id for speaker_id, label in SEED_AUDIO_PRESET_VOICES} + +_AUDIO_TAG_RE = re.compile(r"@Audio(\d+)", re.IGNORECASE) + + +def max_audio_tag(prompt: str) -> int: + """Highest N referenced as @AudioN in the prompt (0 if none).""" + nums = [int(m) for m in _AUDIO_TAG_RE.findall(prompt or "")] + return max(nums) if nums else 0 + + +def connected_audio_indices(reference_mode: dict) -> list[int]: + """Indices (1-based) of connected reference_audio sockets, in order.""" + return [ + i + for i in range(1, 3 + 1) + if reference_mode.get(f"reference_audio_{i}") is not None + ] + + +def validate_seed_audio_inputs( + text_prompt: str, + mode: str, + audio_indices: list[int], + has_image: bool, + preset_voice: str | None = None, +) -> None: + validate_string(text_prompt, field_name="text_prompt", min_length=1, max_length=3000) + max_tag = max_audio_tag(text_prompt) + + if mode == MODE_TEXT: + if max_tag: + raise ValueError( + f"The prompt references @Audio{max_tag}, but reference mode is '{MODE_TEXT}'. " + f"Switch to '{MODE_AUDIO}' and connect the reference clip(s)." + ) + elif mode == MODE_AUDIO: + if not audio_indices: + raise ValueError( + f"Reference mode '{MODE_AUDIO}' requires at least one reference_audio input " + f"(or switch to '{MODE_TEXT}')." + ) + if audio_indices != list(range(1, len(audio_indices) + 1)): + raise ValueError( + "Connect reference_audio inputs in order without gaps: reference_audio_1, then _2, then _3." + ) + if max_tag > len(audio_indices): + raise ValueError( + f"The prompt references @Audio{max_tag}, but only {len(audio_indices)} " + f"reference audio(s) are connected." + ) + elif mode == MODE_IMAGE: + if not has_image: + raise ValueError(f"Reference mode '{MODE_IMAGE}' requires a reference_image input.") + if max_tag: + raise ValueError( + f"@AudioN tags are not used in '{MODE_IMAGE}' mode; the prompt should contain " + f"only the text to synthesize." + ) + elif mode == MODE_SPEAKER: + if not preset_voice or preset_voice not in SEED_AUDIO_VOICE_MAP: + raise ValueError(f"Reference mode '{MODE_SPEAKER}' requires selecting a preset voice.") + if max_tag > 1: + raise ValueError( + f"'{MODE_SPEAKER}' mode uses a single voice, so @Audio{max_tag} is out of range. " + f"Remove the @AudioN tags — the whole prompt is read in the selected voice." + ) + else: + raise ValueError(f"Unknown reference mode: {mode!r}") + + +class ByteDanceSeedAudioNode(IO.ComfyNode): + + @classmethod + def define_schema(cls) -> IO.Schema: + return IO.Schema( + node_id="ByteDanceSeedAudio", + display_name="ByteDance Seed Audio 1.0", + category="partner/audio/ByteDance", + description=( + "Generate speech, music, sound effects and multi-speaker dialogue from a single prompt " + "with ByteDance Seed Audio 1.0. Describe the voice(s), emotion, ambience, background music " + "and sound effects in the prompt, and include the lines to speak. Optionally pick a built-in " + "preset voice, clone voices from up to 3 reference clips (tagged @Audio1-3 in the prompt), " + "or derive a voice from a character image. Up to 2 minutes of audio per run." + ), + inputs=[ + IO.String.Input( + "text_prompt", + multiline=True, + default="", + tooltip=( + "Describe the voice(s), emotion, pacing, ambience, background music and sound " + "effects, and include the lines to speak (name characters inline for dialogue). " + "In 'audio reference' mode, refer to connected clips by order as @Audio1, @Audio2, " + "@Audio3. Maximum 3000 characters." + ), + ), + IO.DynamicCombo.Input( + "reference_mode", + options=[ + IO.DynamicCombo.Option(MODE_TEXT, []), + IO.DynamicCombo.Option( + MODE_AUDIO, + [ + IO.Audio.Input( + "reference_audio_1", + optional=True, + tooltip="Reference clip for voice cloning, tagged @Audio1 in the prompt. " + "Up to 30s.", + ), + IO.Audio.Input( + "reference_audio_2", + optional=True, + tooltip="Reference clip tagged @Audio2 in the prompt. Up to 30s.", + ), + IO.Audio.Input( + "reference_audio_3", + optional=True, + tooltip="Reference clip tagged @Audio3 in the prompt. Up to 30s.", + ), + ], + ), + IO.DynamicCombo.Option( + MODE_IMAGE, + [ + IO.Image.Input( + "reference_image", + optional=True, + tooltip="A single character image; the model derives a voice from it. " + "Cannot be combined with reference audio.", + ), + ], + ), + IO.DynamicCombo.Option( + MODE_SPEAKER, + [ + IO.Combo.Input( + "preset_voice", + options=SEED_AUDIO_VOICE_OPTIONS, + default=SEED_AUDIO_VOICE_OPTIONS[0], + tooltip="A built-in TTS 2.0 voice that reads the prompt. No reference " + "clip needed, and @AudioN tags are not used in this mode.", + ), + ], + ), + ], + tooltip=( + "How to condition the voice: 'text only' (describe everything in the prompt), " + "'audio reference' (clone up to 3 voices, tagged @Audio1-3), 'image reference' " + "(derive a voice from one character image), or 'preset voice' (pick a built-in " + "named voice that reads the prompt)." + ), + ), + IO.Combo.Input( + "sample_rate", + options=["8000", "16000", "24000", "32000", "44100", "48000"], + default="24000", + tooltip="Output sample rate in Hz.", + ), + IO.Int.Input( + "speech_rate", + default=0, + min=-50, + max=100, + tooltip="Speaking speed. 0 = normal, 100 = 2.0x, -50 = 0.5x.", + ), + IO.Int.Input( + "loudness_rate", + default=0, + min=-50, + max=100, + tooltip="Loudness. 0 = normal, 100 = 2.0x, -50 = 0.5x.", + ), + IO.Int.Input( + "pitch_rate", + default=0, + min=-12, + max=12, + tooltip="Pitch shift in semitones (-12 to 12).", + ), + IO.Int.Input( + "seed", + default=42, + min=0, + max=2147483647, + control_after_generate=True, + tooltip="Seed controls whether the node should re-run; " + "results are non-deterministic regardless of seed.", + ), + ], + outputs=[IO.Audio.Output()], + hidden=[ + IO.Hidden.auth_token_comfy_org, + IO.Hidden.api_key_comfy_org, + IO.Hidden.unique_id, + ], + is_api_node=True, + price_badge=IO.PriceBadge( + expr="""{"type":"usd","usd": 0.2145, "format":{"suffix":"/minute","approximate":true}}""", + ), + ) + + @classmethod + async def execute( + cls, + text_prompt: str, + reference_mode: dict, + sample_rate: str, + speech_rate: int, + loudness_rate: int, + pitch_rate: int, + seed: int, + ) -> IO.NodeOutput: + mode = reference_mode["reference_mode"] + audio_indices = connected_audio_indices(reference_mode) + image = reference_mode.get("reference_image") + preset_voice = reference_mode.get("preset_voice") + validate_seed_audio_inputs(text_prompt, mode, audio_indices, image is not None, preset_voice) + + references: list[SeedAudioReference] | None = None + if mode == MODE_AUDIO: + references = [] + for i in audio_indices: + clip = reference_mode[f"reference_audio_{i}"] + validate_audio_duration(clip, max_duration=30.0) + mp3_bytes = audio_input_to_mp3(clip).getvalue() + references.append(SeedAudioReference(audio_data=base64.b64encode(mp3_bytes).decode("utf-8"))) + elif mode == MODE_IMAGE: + image = upscale_image_tensor_to_min_pixels(image, 160_000) + references = [SeedAudioReference(image_data=tensor_to_base64_string(image, mime_type="image/png"))] + elif mode == MODE_SPEAKER: + references = [SeedAudioReference(speaker=SEED_AUDIO_VOICE_MAP[preset_voice])] + + response = await sync_op( + cls, + ApiEndpoint(path="/proxy/byteplus/api/v3/tts/create", method="POST"), + response_model=SeedAudioResponse, + data=SeedAudioRequest( + text_prompt=text_prompt, + references=references, + audio_config=SeedAudioConfig( + sample_rate=int(sample_rate), + speech_rate=speech_rate, + loudness_rate=loudness_rate, + pitch_rate=pitch_rate, + ), + ), + ) + if not response.audio: + raise Exception( + f"Seed Audio returned no audio (code={response.code}): {response.message}" + ) + return IO.NodeOutput(audio_bytes_to_audio_input(base64.b64decode(response.audio))) + + class ByteDanceExtension(ComfyExtension): @override async def get_node_list(self) -> list[type[IO.ComfyNode]]: @@ -2457,6 +2805,7 @@ class ByteDanceExtension(ComfyExtension): ByteDance2ReferenceNode, ByteDanceCreateImageAsset, ByteDanceCreateVideoAsset, + ByteDanceSeedAudioNode, ] diff --git a/comfy_api_nodes/nodes_gemini.py b/comfy_api_nodes/nodes_gemini.py index 3d4be6065..aa992802d 100644 --- a/comfy_api_nodes/nodes_gemini.py +++ b/comfy_api_nodes/nodes_gemini.py @@ -5,7 +5,6 @@ See: https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/infer import base64 import os -from enum import Enum from fnmatch import fnmatch from io import BytesIO from typing import Any, Literal @@ -14,7 +13,7 @@ import torch from typing_extensions import override import folder_paths -from comfy_api.latest import IO, ComfyExtension, Input, Types +from comfy_api.latest import IO, ComfyExtension, Input, InputImpl, Types from comfy_api_nodes.apis.gemini import ( GeminiContent, GeminiFileData, @@ -38,6 +37,7 @@ from comfy_api_nodes.util import ( audio_to_base64_string, bytesio_to_image_tensor, download_url_to_image_tensor, + download_url_to_video_output, get_number_of_images, sync_op, tensor_to_base64_string, @@ -46,6 +46,7 @@ from comfy_api_nodes.util import ( upload_images_to_comfyapi, upload_video_to_comfyapi, validate_string, + validate_video_duration, video_to_base64_string, ) @@ -78,15 +79,6 @@ GEMINI_IMAGE_2_PRICE_BADGE = IO.PriceBadge( ) -class GeminiImageModel(str, Enum): - """ - Gemini Image Model Names allowed by comfy-api - """ - - gemini_2_5_flash_image_preview = "gemini-2.5-flash-image-preview" - gemini_2_5_flash_image = "gemini-2.5-flash-image" - - async def create_image_parts( cls: type[IO.ComfyNode], images: Input.Image | list[Input.Image], @@ -239,25 +231,38 @@ async def get_image_from_response(response: GeminiGenerateContentResponse, thoug return torch.cat(image_tensors, dim=0) +async def get_video_from_response( + response: GeminiGenerateContentResponse, cls: type[IO.ComfyNode] | None = None +) -> InputImpl.VideoFromFile: + parts = get_parts_by_type(response, "video/*") + for part in parts: + if part.inlineData and part.inlineData.data: + return InputImpl.VideoFromFile(BytesIO(base64.b64decode(part.inlineData.data))) + if part.fileData and part.fileData.fileUri: + return await download_url_to_video_output(part.fileData.fileUri, cls=cls) + model_message = get_text_from_response(response).strip() + if model_message: + raise ValueError(f"Gemini did not generate a video. Model response: {model_message}") + raise ValueError( + "Gemini did not generate a video. Try rephrasing your prompt, " + "shortening the requested duration, or reducing the number of input images/videos." + ) + + def calculate_tokens_price(response: GeminiGenerateContentResponse) -> float | None: if not response.modelVersion: return None # Define prices (Cost per 1,000,000 tokens), see https://cloud.google.com/vertex-ai/generative-ai/pricing - if response.modelVersion in ("gemini-2.5-pro-preview-05-06", "gemini-2.5-pro"): + output_video_tokens_price = 0.0 + if response.modelVersion == "gemini-2.5-pro": input_tokens_price = 1.25 output_text_tokens_price = 10.0 output_image_tokens_price = 0.0 - elif response.modelVersion in ( - "gemini-2.5-flash-preview-04-17", - "gemini-2.5-flash", - ): + elif response.modelVersion == "gemini-2.5-flash": input_tokens_price = 0.30 output_text_tokens_price = 2.50 output_image_tokens_price = 0.0 - elif response.modelVersion in ( - "gemini-2.5-flash-image-preview", - "gemini-2.5-flash-image", - ): + elif response.modelVersion == "gemini-2.5-flash-image": input_tokens_price = 0.30 output_text_tokens_price = 2.50 output_image_tokens_price = 30.0 @@ -265,18 +270,27 @@ def calculate_tokens_price(response: GeminiGenerateContentResponse) -> float | N input_tokens_price = 2 output_text_tokens_price = 12.0 output_image_tokens_price = 0.0 - elif response.modelVersion == "gemini-3.1-flash-lite-preview": + elif response.modelVersion in ("gemini-3.1-flash-lite-preview", "gemini-3.1-flash-lite"): input_tokens_price = 0.25 output_text_tokens_price = 1.50 output_image_tokens_price = 0.0 - elif response.modelVersion == "gemini-3-pro-image-preview": + elif response.modelVersion in ("gemini-3-pro-image-preview", "gemini-3-pro-image"): input_tokens_price = 2 output_text_tokens_price = 12.0 output_image_tokens_price = 120.0 - elif response.modelVersion == "gemini-3.1-flash-image-preview": + elif response.modelVersion in ("gemini-3.1-flash-image-preview", "gemini-3.1-flash-image"): input_tokens_price = 0.5 output_text_tokens_price = 3.0 output_image_tokens_price = 60.0 + elif response.modelVersion == "gemini-3.1-flash-lite-image": + input_tokens_price = 0.25 + output_text_tokens_price = 1.50 + output_image_tokens_price = 30.0 + elif response.modelVersion == "gemini-omni-flash-preview": + input_tokens_price = 2.145 + output_text_tokens_price = 12.87 + output_image_tokens_price = 0.0 + output_video_tokens_price = 25.025 else: return None final_price = response.usageMetadata.promptTokenCount * input_tokens_price @@ -284,6 +298,8 @@ def calculate_tokens_price(response: GeminiGenerateContentResponse) -> float | N for i in response.usageMetadata.candidatesTokensDetails: if i.modality == Modality.IMAGE: final_price += output_image_tokens_price * i.tokenCount # for Nano Banana models + elif i.modality == Modality.VIDEO: + final_price += output_video_tokens_price * i.tokenCount # for Omni Flash else: final_price += output_text_tokens_price * i.tokenCount if response.usageMetadata.thoughtsTokenCount: @@ -455,8 +471,6 @@ class GeminiNode(IO.ComfyNode): IO.Combo.Input( "model", options=[ - "gemini-2.5-pro-preview-05-06", - "gemini-2.5-flash-preview-04-17", "gemini-2.5-pro", "gemini-2.5-flash", "gemini-3-pro-preview", @@ -904,8 +918,7 @@ class GeminiImage(IO.ComfyNode): ), IO.Combo.Input( "model", - options=GeminiImageModel, - default=GeminiImageModel.gemini_2_5_flash_image, + options=["gemini-2.5-flash-image"], tooltip="The Gemini model to use for generating responses.", ), IO.Int.Input( @@ -1321,7 +1334,7 @@ class GeminiNanoBanana2(IO.ComfyNode): ) -def _nano_banana_2_v2_model_inputs(): +def _nano_banana_2_v2_model_inputs(resolutions: list[str]): return [ IO.Combo.Input( "aspect_ratio", @@ -1348,8 +1361,8 @@ def _nano_banana_2_v2_model_inputs(): ), IO.Combo.Input( "resolution", - options=["1K", "2K", "4K"], - tooltip="Target output resolution. For 2K/4K the native Gemini upscaler is used.", + options=resolutions, + tooltip="Target output resolution.", ), IO.Combo.Input( "thinking_level", @@ -1395,7 +1408,11 @@ class GeminiNanoBanana2V2(IO.ComfyNode): options=[ IO.DynamicCombo.Option( "Nano Banana 2 (Gemini 3.1 Flash Image)", - _nano_banana_2_v2_model_inputs(), + _nano_banana_2_v2_model_inputs(resolutions=["1K", "2K", "4K"]), + ), + IO.DynamicCombo.Option( + "Nano Banana 2 Lite", + _nano_banana_2_v2_model_inputs(resolutions=["1K"]), ), ], ), @@ -1464,9 +1481,13 @@ class GeminiNanoBanana2V2(IO.ComfyNode): depends_on=IO.PriceBadgeDepends(widgets=["model", "model.resolution"]), expr=""" ( - $r := $lookup(widgets, "model.resolution"); - $prices := {"1k": 0.0696, "2k": 0.1014, "4k": 0.154}; - {"type":"usd","usd": $lookup($prices, $r), "format":{"suffix":"/Image","approximate":true}} + $contains(widgets.model, "lite") + ? {"type":"usd","usd": 0.034, "format":{"suffix":"/Image","approximate":true}} + : ( + $r := $lookup(widgets, "model.resolution"); + $prices := {"1k": 0.0696, "2k": 0.1014, "4k": 0.154}; + {"type":"usd","usd": $lookup($prices, $r), "format":{"suffix":"/Image","approximate":true}} + ) ) """, ), @@ -1487,6 +1508,8 @@ class GeminiNanoBanana2V2(IO.ComfyNode): model_choice = model["model"] if model_choice == "Nano Banana 2 (Gemini 3.1 Flash Image)": model_id = "gemini-3.1-flash-image-preview" + elif model_choice == "Nano Banana 2 Lite": + model_id = "gemini-3.1-flash-lite-image" else: model_id = model_choice @@ -1536,6 +1559,149 @@ class GeminiNanoBanana2V2(IO.ComfyNode): ) +OMNI_MAX_IMAGES = 14 +OMNI_MAX_VIDEOS = 3 + +OMNI_MODELS: dict[str, str] = { + "Omni Flash": "gemini-omni-flash-preview", +} + + +def _omni_flash_inputs() -> list[Input]: + """Per-model inputs for the Omni video DynamicCombo (prompt + reference media + sampling).""" + return [ + IO.String.Input( + "prompt", + multiline=True, + default="", + tooltip="Describe the video to generate. Specify the length and aspect ratio directly in the " + 'prompt, e.g. "a 6-second clip in 16:9". Length may be 3-10 seconds; the aspect ratio must be ' + "16:9 (landscape) or 9:16 (portrait). The output is 720p, 24 FPS, with audio.", + ), + IO.Autogrow.Input( + "images", + template=IO.Autogrow.TemplateNames( + IO.Image.Input("image"), + names=[f"image_{i}" for i in range(1, OMNI_MAX_IMAGES + 1)], + min=0, + ), + tooltip=f"Optional reference image(s) to guide or animate the video. Up to {OMNI_MAX_IMAGES} images.", + ), + IO.Autogrow.Input( + "videos", + template=IO.Autogrow.TemplateNames( + IO.Video.Input("video"), + names=[f"video_{i}" for i in range(1, OMNI_MAX_VIDEOS + 1)], + min=0, + ), + tooltip=f"Optional reference video(s) to guide or edit. Up to {OMNI_MAX_VIDEOS} videos, " + f"each up to 10 seconds long.", + ), + IO.Float.Input( + "temperature", + default=1.0, + min=0.0, + max=2.0, + step=0.01, + tooltip="Controls randomness. Lower is more focused/deterministic, higher is more varied.", + advanced=True, + ), + IO.Float.Input( + "top_p", + default=0.95, + min=0.0, + max=1.0, + step=0.01, + tooltip="Nucleus sampling: sample from the smallest token set whose cumulative probability reaches top_p.", + advanced=True, + ), + ] + + +class GeminiVideoOmni(IO.ComfyNode): + + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="GeminiVideoOmni", + display_name="Google Gemini Omni (Video)", + category="partner/video/Gemini", + essentials_category="Video Generation", + description="Generate a video with audio from a text prompt using Google's Gemini Omni Flash model. " + "Optionally provide reference images and/or videos to guide or edit the result. Describe the desired " + "length (3-10s) and aspect ratio (16:9 or 9:16) directly in the prompt.", + inputs=[ + IO.DynamicCombo.Input( + "model", + options=[ + IO.DynamicCombo.Option("Omni Flash", _omni_flash_inputs()), + ], + tooltip="The Gemini video model used to generate the video.", + ), + IO.Int.Input( + "seed", + default=42, + min=0, + max=2147483647, + control_after_generate=True, + tooltip="Seed controls whether the node should re-run; " + "results are non-deterministic regardless of seed.", + ), + ], + outputs=[ + IO.Video.Output(), + IO.String.Output(), + ], + hidden=[ + IO.Hidden.auth_token_comfy_org, + IO.Hidden.api_key_comfy_org, + IO.Hidden.unique_id, + ], + is_api_node=True, + price_badge=IO.PriceBadge( + expr='{"type":"usd","usd":0.146,"format":{"suffix":"/second","approximate":true}}' + ), + ) + + @classmethod + async def execute(cls, model: dict, seed: int) -> IO.NodeOutput: + prompt = model.get("prompt") or "" + validate_string(prompt, strip_whitespace=True, min_length=1) + model_id = OMNI_MODELS[model["model"]] + + images = [t for t in (model.get("images") or {}).values() if t is not None] + videos = [v for v in (model.get("videos") or {}).values() if v is not None] + if sum(get_number_of_images(t) for t in images) > OMNI_MAX_IMAGES: + raise ValueError(f"The current maximum number of supported images is {OMNI_MAX_IMAGES}.") + if len(videos) > OMNI_MAX_VIDEOS: + raise ValueError(f"The current maximum number of supported videos is {OMNI_MAX_VIDEOS}.") + for video in videos: + validate_video_duration(video, max_duration=10) + + parts: list[GeminiPart] = [] + if images or videos: + parts.extend(await build_gemini_media_parts(cls, images, [], videos)) + parts.append(GeminiPart(text=prompt)) + response = await sync_op( + cls, + ApiEndpoint(path=f"{GEMINI_BASE_ENDPOINT}/{model_id}", method="POST"), + data=GeminiGenerateContentRequest( + contents=[GeminiContent(role=GeminiRole.user, parts=parts)], + generationConfig=GeminiGenerationConfig( + responseModalities=["TEXT", "VIDEO"], + temperature=model.get("temperature", 1.0), + topP=model.get("top_p", 0.95), + ), + ), + response_model=GeminiGenerateContentResponse, + price_extractor=calculate_tokens_price, + ) + return IO.NodeOutput( + await get_video_from_response(response, cls=cls), + get_text_from_response(response), + ) + + class GeminiExtension(ComfyExtension): @override async def get_node_list(self) -> list[type[IO.ComfyNode]]: @@ -1546,6 +1712,7 @@ class GeminiExtension(ComfyExtension): GeminiImage2, GeminiNanoBanana2, GeminiNanoBanana2V2, + GeminiVideoOmni, GeminiInputFiles, ] diff --git a/comfy_api_nodes/nodes_grok.py b/comfy_api_nodes/nodes_grok.py index 2ae529813..dc484536e 100644 --- a/comfy_api_nodes/nodes_grok.py +++ b/comfy_api_nodes/nodes_grok.py @@ -30,7 +30,7 @@ from comfy_api_nodes.util import ( _GROK_VIDEO_MODEL_API_IDS = { - "grok-imagine-video-1.5": "grok-imagine-video-1.5-preview", + "grok-imagine-video-1.5": "grok-imagine-video-1.5", } @@ -521,8 +521,8 @@ class GrokVideoNode(IO.ComfyNode): ), IO.Combo.Input( "resolution", - options=["480p", "720p"], - tooltip="The resolution of the output video.", + options=["480p", "720p", "1080p"], + tooltip="The resolution of the output video. 1080p is only available for grok-imagine-video-1.5.", ), IO.Combo.Input( "aspect_ratio", @@ -570,11 +570,12 @@ class GrokVideoNode(IO.ComfyNode): ( $is15 := $contains(widgets.model, "1.5"); $rate := $is15 - ? (widgets.resolution = "720p" ? 0.2002 : 0.1144) + ? (widgets.resolution = "1080p" ? 0.25 : (widgets.resolution = "720p" ? 0.14 : 0.08)) : (widgets.resolution = "720p" ? 0.07 : 0.05); - $imgCost := $is15 ? 0.0143 : 0.002; + $imgCost := $is15 ? 0.01 : 0.002; $base := $rate * widgets.duration; - {"type":"usd","usd": inputs.image.connected ? $base + $imgCost : $base} + $total := inputs.image.connected ? $base + $imgCost : $base; + {"type":"usd","usd": $is15 ? $total * 1.43 : $total} ) """, ), @@ -593,6 +594,8 @@ class GrokVideoNode(IO.ComfyNode): ) -> IO.NodeOutput: if image is None and model == "grok-imagine-video-1.5": raise ValueError(f"The '{model}' model requires an input image; connect one to the 'image' input.") + if resolution == "1080p" and model != "grok-imagine-video-1.5": + raise ValueError(f"1080p resolution is only available for grok-imagine-video-1.5, not '{model}'.") image_url = None if image is not None: if get_number_of_images(image) != 1: diff --git a/comfy_api_nodes/nodes_ideogram.py b/comfy_api_nodes/nodes_ideogram.py index 3b914a850..cc0467987 100644 --- a/comfy_api_nodes/nodes_ideogram.py +++ b/comfy_api_nodes/nodes_ideogram.py @@ -5,9 +5,7 @@ from PIL import Image import numpy as np import torch from comfy_api_nodes.apis.ideogram import ( - IdeogramGenerateRequest, IdeogramGenerateResponse, - ImageRequest, IdeogramV3Request, IdeogramV3EditRequest, IdeogramV4Request, @@ -21,101 +19,6 @@ from comfy_api_nodes.util import ( validate_string, ) -V1_V1_RES_MAP = { - "Auto":"AUTO", - "512 x 1536":"RESOLUTION_512_1536", - "576 x 1408":"RESOLUTION_576_1408", - "576 x 1472":"RESOLUTION_576_1472", - "576 x 1536":"RESOLUTION_576_1536", - "640 x 1024":"RESOLUTION_640_1024", - "640 x 1344":"RESOLUTION_640_1344", - "640 x 1408":"RESOLUTION_640_1408", - "640 x 1472":"RESOLUTION_640_1472", - "640 x 1536":"RESOLUTION_640_1536", - "704 x 1152":"RESOLUTION_704_1152", - "704 x 1216":"RESOLUTION_704_1216", - "704 x 1280":"RESOLUTION_704_1280", - "704 x 1344":"RESOLUTION_704_1344", - "704 x 1408":"RESOLUTION_704_1408", - "704 x 1472":"RESOLUTION_704_1472", - "720 x 1280":"RESOLUTION_720_1280", - "736 x 1312":"RESOLUTION_736_1312", - "768 x 1024":"RESOLUTION_768_1024", - "768 x 1088":"RESOLUTION_768_1088", - "768 x 1152":"RESOLUTION_768_1152", - "768 x 1216":"RESOLUTION_768_1216", - "768 x 1232":"RESOLUTION_768_1232", - "768 x 1280":"RESOLUTION_768_1280", - "768 x 1344":"RESOLUTION_768_1344", - "832 x 960":"RESOLUTION_832_960", - "832 x 1024":"RESOLUTION_832_1024", - "832 x 1088":"RESOLUTION_832_1088", - "832 x 1152":"RESOLUTION_832_1152", - "832 x 1216":"RESOLUTION_832_1216", - "832 x 1248":"RESOLUTION_832_1248", - "864 x 1152":"RESOLUTION_864_1152", - "896 x 960":"RESOLUTION_896_960", - "896 x 1024":"RESOLUTION_896_1024", - "896 x 1088":"RESOLUTION_896_1088", - "896 x 1120":"RESOLUTION_896_1120", - "896 x 1152":"RESOLUTION_896_1152", - "960 x 832":"RESOLUTION_960_832", - "960 x 896":"RESOLUTION_960_896", - "960 x 1024":"RESOLUTION_960_1024", - "960 x 1088":"RESOLUTION_960_1088", - "1024 x 640":"RESOLUTION_1024_640", - "1024 x 768":"RESOLUTION_1024_768", - "1024 x 832":"RESOLUTION_1024_832", - "1024 x 896":"RESOLUTION_1024_896", - "1024 x 960":"RESOLUTION_1024_960", - "1024 x 1024":"RESOLUTION_1024_1024", - "1088 x 768":"RESOLUTION_1088_768", - "1088 x 832":"RESOLUTION_1088_832", - "1088 x 896":"RESOLUTION_1088_896", - "1088 x 960":"RESOLUTION_1088_960", - "1120 x 896":"RESOLUTION_1120_896", - "1152 x 704":"RESOLUTION_1152_704", - "1152 x 768":"RESOLUTION_1152_768", - "1152 x 832":"RESOLUTION_1152_832", - "1152 x 864":"RESOLUTION_1152_864", - "1152 x 896":"RESOLUTION_1152_896", - "1216 x 704":"RESOLUTION_1216_704", - "1216 x 768":"RESOLUTION_1216_768", - "1216 x 832":"RESOLUTION_1216_832", - "1232 x 768":"RESOLUTION_1232_768", - "1248 x 832":"RESOLUTION_1248_832", - "1280 x 704":"RESOLUTION_1280_704", - "1280 x 720":"RESOLUTION_1280_720", - "1280 x 768":"RESOLUTION_1280_768", - "1280 x 800":"RESOLUTION_1280_800", - "1312 x 736":"RESOLUTION_1312_736", - "1344 x 640":"RESOLUTION_1344_640", - "1344 x 704":"RESOLUTION_1344_704", - "1344 x 768":"RESOLUTION_1344_768", - "1408 x 576":"RESOLUTION_1408_576", - "1408 x 640":"RESOLUTION_1408_640", - "1408 x 704":"RESOLUTION_1408_704", - "1472 x 576":"RESOLUTION_1472_576", - "1472 x 640":"RESOLUTION_1472_640", - "1472 x 704":"RESOLUTION_1472_704", - "1536 x 512":"RESOLUTION_1536_512", - "1536 x 576":"RESOLUTION_1536_576", - "1536 x 640":"RESOLUTION_1536_640", -} - -V1_V2_RATIO_MAP = { - "1:1":"ASPECT_1_1", - "4:3":"ASPECT_4_3", - "3:4":"ASPECT_3_4", - "16:9":"ASPECT_16_9", - "9:16":"ASPECT_9_16", - "2:1":"ASPECT_2_1", - "1:2":"ASPECT_1_2", - "3:2":"ASPECT_3_2", - "2:3":"ASPECT_2_3", - "4:5":"ASPECT_4_5", - "5:4":"ASPECT_5_4", -} V3_RATIO_MAP = { "1:3":"1x3", @@ -229,298 +132,6 @@ async def download_and_process_images(image_urls): return stacked_tensors -class IdeogramV1(IO.ComfyNode): - - @classmethod - def define_schema(cls): - return IO.Schema( - node_id="IdeogramV1", - display_name="Ideogram V1", - category="partner/image/Ideogram", - description="Generates images using the Ideogram V1 model.", - inputs=[ - IO.String.Input( - "prompt", - multiline=True, - default="", - tooltip="Prompt for the image generation", - ), - IO.Boolean.Input( - "turbo", - default=False, - tooltip="Whether to use turbo mode (faster generation, potentially lower quality)", - ), - IO.Combo.Input( - "aspect_ratio", - options=list(V1_V2_RATIO_MAP.keys()), - default="1:1", - tooltip="The aspect ratio for image generation.", - optional=True, - ), - IO.Combo.Input( - "magic_prompt_option", - options=["AUTO", "ON", "OFF"], - default="AUTO", - tooltip="Determine if MagicPrompt should be used in generation", - optional=True, - advanced=True, - ), - IO.Int.Input( - "seed", - default=0, - min=0, - max=2147483647, - step=1, - control_after_generate=True, - display_mode=IO.NumberDisplay.number, - optional=True, - ), - IO.String.Input( - "negative_prompt", - multiline=True, - default="", - tooltip="Description of what to exclude from the image", - optional=True, - ), - IO.Int.Input( - "num_images", - default=1, - min=1, - max=8, - step=1, - display_mode=IO.NumberDisplay.number, - optional=True, - ), - ], - outputs=[ - IO.Image.Output(), - ], - hidden=[ - IO.Hidden.auth_token_comfy_org, - IO.Hidden.api_key_comfy_org, - IO.Hidden.unique_id, - ], - is_api_node=True, - price_badge=IO.PriceBadge( - depends_on=IO.PriceBadgeDepends(widgets=["num_images", "turbo"]), - expr=""" - ( - $n := widgets.num_images; - $base := (widgets.turbo = true) ? 0.0286 : 0.0858; - {"type":"usd","usd": $round($base * $n, 2)} - ) - """, - ), - ) - - @classmethod - async def execute( - cls, - prompt, - turbo=False, - aspect_ratio="1:1", - magic_prompt_option="AUTO", - seed=0, - negative_prompt="", - num_images=1, - ): - # Determine the model based on turbo setting - aspect_ratio = V1_V2_RATIO_MAP.get(aspect_ratio, None) - model = "V_1_TURBO" if turbo else "V_1" - - response = await sync_op( - cls, - ApiEndpoint(path="/proxy/ideogram/generate", method="POST"), - response_model=IdeogramGenerateResponse, - data=IdeogramGenerateRequest( - image_request=ImageRequest( - prompt=prompt, - model=model, - num_images=num_images, - seed=seed, - aspect_ratio=aspect_ratio if aspect_ratio != "ASPECT_1_1" else None, - magic_prompt_option=(magic_prompt_option if magic_prompt_option != "AUTO" else None), - negative_prompt=negative_prompt if negative_prompt else None, - ) - ), - max_retries=1, - ) - - if not response.data or len(response.data) == 0: - raise Exception("No images were generated in the response") - - image_urls = [image_data.url for image_data in response.data if image_data.url] - if not image_urls: - raise Exception("No image URLs were generated in the response") - return IO.NodeOutput(await download_and_process_images(image_urls)) - - -class IdeogramV2(IO.ComfyNode): - - @classmethod - def define_schema(cls): - return IO.Schema( - node_id="IdeogramV2", - display_name="Ideogram V2", - category="partner/image/Ideogram", - description="Generates images using the Ideogram V2 model.", - inputs=[ - IO.String.Input( - "prompt", - multiline=True, - default="", - tooltip="Prompt for the image generation", - ), - IO.Boolean.Input( - "turbo", - default=False, - tooltip="Whether to use turbo mode (faster generation, potentially lower quality)", - ), - IO.Combo.Input( - "aspect_ratio", - options=list(V1_V2_RATIO_MAP.keys()), - default="1:1", - tooltip="The aspect ratio for image generation. Ignored if resolution is not set to AUTO.", - optional=True, - ), - IO.Combo.Input( - "resolution", - options=list(V1_V1_RES_MAP.keys()), - default="Auto", - tooltip="The resolution for image generation. " - "If not set to AUTO, this overrides the aspect_ratio setting.", - optional=True, - ), - IO.Combo.Input( - "magic_prompt_option", - options=["AUTO", "ON", "OFF"], - default="AUTO", - tooltip="Determine if MagicPrompt should be used in generation", - optional=True, - advanced=True, - ), - IO.Int.Input( - "seed", - default=0, - min=0, - max=2147483647, - step=1, - control_after_generate=True, - display_mode=IO.NumberDisplay.number, - optional=True, - ), - IO.Combo.Input( - "style_type", - options=["AUTO", "GENERAL", "REALISTIC", "DESIGN", "RENDER_3D", "ANIME"], - default="NONE", - tooltip="Style type for generation (V2 only)", - optional=True, - advanced=True, - ), - IO.String.Input( - "negative_prompt", - multiline=True, - default="", - tooltip="Description of what to exclude from the image", - optional=True, - ), - IO.Int.Input( - "num_images", - default=1, - min=1, - max=8, - step=1, - display_mode=IO.NumberDisplay.number, - optional=True, - ), - #"color_palette": ( - # IO.STRING, - # { - # "multiline": False, - # "default": "", - # "tooltip": "Color palette preset name or hex colors with weights", - # }, - #), - ], - outputs=[ - IO.Image.Output(), - ], - hidden=[ - IO.Hidden.auth_token_comfy_org, - IO.Hidden.api_key_comfy_org, - IO.Hidden.unique_id, - ], - is_api_node=True, - price_badge=IO.PriceBadge( - depends_on=IO.PriceBadgeDepends(widgets=["num_images", "turbo"]), - expr=""" - ( - $n := widgets.num_images; - $base := (widgets.turbo = true) ? 0.0715 : 0.1144; - {"type":"usd","usd": $round($base * $n, 2)} - ) - """, - ), - ) - - @classmethod - async def execute( - cls, - prompt, - turbo=False, - aspect_ratio="1:1", - resolution="Auto", - magic_prompt_option="AUTO", - seed=0, - style_type="NONE", - negative_prompt="", - num_images=1, - color_palette="", - ): - aspect_ratio = V1_V2_RATIO_MAP.get(aspect_ratio, None) - resolution = V1_V1_RES_MAP.get(resolution, None) - # Determine the model based on turbo setting - model = "V_2_TURBO" if turbo else "V_2" - - # Handle resolution vs aspect_ratio logic - # If resolution is not AUTO, it overrides aspect_ratio - final_resolution = None - final_aspect_ratio = None - - if resolution != "AUTO": - final_resolution = resolution - else: - final_aspect_ratio = aspect_ratio if aspect_ratio != "ASPECT_1_1" else None - - response = await sync_op( - cls, - endpoint=ApiEndpoint(path="/proxy/ideogram/generate", method="POST"), - response_model=IdeogramGenerateResponse, - data=IdeogramGenerateRequest( - image_request=ImageRequest( - prompt=prompt, - model=model, - num_images=num_images, - seed=seed, - aspect_ratio=final_aspect_ratio, - resolution=final_resolution, - magic_prompt_option=(magic_prompt_option if magic_prompt_option != "AUTO" else None), - style_type=style_type if style_type != "NONE" else None, - negative_prompt=negative_prompt if negative_prompt else None, - color_palette=color_palette if color_palette else None, - ) - ), - max_retries=1, - ) - if not response.data or len(response.data) == 0: - raise Exception("No images were generated in the response") - - image_urls = [image_data.url for image_data in response.data if image_data.url] - if not image_urls: - raise Exception("No image URLs were generated in the response") - return IO.NodeOutput(await download_and_process_images(image_urls)) - - class IdeogramV3(IO.ComfyNode): @classmethod @@ -917,8 +528,6 @@ class IdeogramExtension(ComfyExtension): @override async def get_node_list(self) -> list[type[IO.ComfyNode]]: return [ - IdeogramV1, - IdeogramV2, IdeogramV3, IdeogramV4, ] diff --git a/comfy_api_nodes/nodes_kling.py b/comfy_api_nodes/nodes_kling.py index d11e42540..b27de2549 100644 --- a/comfy_api_nodes/nodes_kling.py +++ b/comfy_api_nodes/nodes_kling.py @@ -60,6 +60,12 @@ from comfy_api_nodes.apis.kling import ( OmniProImageRequest, OmniProReferences2VideoRequest, OmniProText2VideoRequest, + Kling3TurboSettings, + Kling3TurboText2VideoRequest, + Kling3TurboContent, + Kling3TurboImage2VideoRequest, + Kling3TurboCreateResponse, + Kling3TurboQueryResponse, TaskStatusResponse, TextToVideoWithAudioRequest, ) @@ -436,7 +442,7 @@ async def execute_text2video( negative_prompt=negative_prompt if negative_prompt else None, duration=KlingVideoGenDuration(duration), mode=KlingVideoGenMode(model_mode), - model_name=KlingVideoGenModelName(model_name), + model_name=model_name, cfg_scale=cfg_scale, aspect_ratio=KlingVideoGenAspectRatio(aspect_ratio), camera_control=camera_control, @@ -2847,6 +2853,67 @@ class MotionControl(IO.ComfyNode): return IO.NodeOutput(await download_url_to_video_output(final_response.data.task_result.videos[0].url)) +def build_turbo_shot_prompt(multi_prompt: list[MultiPromptEntry]) -> str: + """Render storyboard entries into the Turbo multi-shot prompt 'shot n, m, words; ...'.""" + return "; ".join(f"shot {i}, {int(e.duration)}, {e.prompt}" for i, e in enumerate(multi_prompt, 1)) + ";" + + +def _turbo_video_url(response: Kling3TurboQueryResponse) -> str: + """Extract the result video URL from a /tasks response (data[].outputs[] where type == 'video').""" + task = response.data[0] if response.data else None + if task and task.outputs: + for output in task.outputs: + if output.type == "video" and output.url: + return output.url + raise RuntimeError(f"Kling 3.0 Turbo task finished without a video output: {response.model_dump()}") + + +async def execute_kling_turbo( + cls: type[IO.ComfyNode], + *, + prompt: str, + resolution: str, + aspect_ratio: str, + duration: int, + start_frame: torch.Tensor | None, +) -> IO.NodeOutput: + """Create + poll a Kling 3.0 Turbo task. Image-to-video when start_frame is given, else text-to-video.""" + if start_frame is not None: + validate_image_dimensions(start_frame, min_width=300, min_height=300) + validate_image_aspect_ratio(start_frame, (1, 2.5), (2.5, 1)) + contents = [Kling3TurboContent(type="first_frame", url=tensor_to_base64_string(start_frame))] + if prompt: + contents.insert(0, Kling3TurboContent(type="prompt", text=prompt)) + create = await sync_op( + cls, + ApiEndpoint(path="/proxy/kling/image-to-video/kling-3.0-turbo", method="POST"), + response_model=Kling3TurboCreateResponse, + data=Kling3TurboImage2VideoRequest( + contents=contents, + settings=Kling3TurboSettings(resolution=resolution, duration=duration), # i2v: no aspect_ratio + ), + ) + else: + create = await sync_op( + cls, + ApiEndpoint(path="/proxy/kling/text-to-video/kling-3.0-turbo", method="POST"), + response_model=Kling3TurboCreateResponse, + data=Kling3TurboText2VideoRequest( + prompt=prompt, + settings=Kling3TurboSettings(resolution=resolution, aspect_ratio=aspect_ratio, duration=duration), + ), + ) + if not (create.data and create.data.id): + raise RuntimeError(f"Kling 3.0 Turbo create failed. Code: {create.code}, Message: {create.message}") + final_response = await poll_op( + cls, + ApiEndpoint(path="/proxy/kling/tasks", query_params={"task_ids": create.data.id}), + response_model=Kling3TurboQueryResponse, + status_extractor=lambda r: (r.data[0].status if r.data else None), + ) + return IO.NodeOutput(await download_url_to_video_output(_turbo_video_url(final_response))) + + class KlingVideoNode(IO.ComfyNode): @classmethod @@ -2884,7 +2951,11 @@ class KlingVideoNode(IO.ComfyNode): ], tooltip="Generate a series of video segments with individual prompts and durations.", ), - IO.Boolean.Input("generate_audio", default=True), + IO.Boolean.Input( + "generate_audio", + default=True, + tooltip="'kling-3.0-turbo' always generates native audio, so the audio toggle is ignored.", + ), IO.DynamicCombo.Input( "model", options=[ @@ -2899,6 +2970,17 @@ class KlingVideoNode(IO.ComfyNode): ), ], ), + IO.DynamicCombo.Option( + "kling-3.0-turbo", + [ + IO.Combo.Input("resolution", options=["1080p", "720p"], default="720p"), + IO.Combo.Input( + "aspect_ratio", + options=["16:9", "9:16", "1:1"], + tooltip="Ignored in image-to-video mode.", + ), + ], + ), ], tooltip="Model and generation settings.", ), @@ -2930,6 +3012,7 @@ class KlingVideoNode(IO.ComfyNode): price_badge=IO.PriceBadge( depends_on=IO.PriceBadgeDepends( widgets=[ + "model", "model.resolution", "generate_audio", "multi_shot", @@ -2944,14 +3027,7 @@ class KlingVideoNode(IO.ComfyNode): ), expr=""" ( - $rates := { - "4k": {"off": 0.42, "on": 0.42}, - "1080p": {"off": 0.112, "on": 0.168}, - "720p": {"off": 0.084, "on": 0.126} - }; $res := $lookup(widgets, "model.resolution"); - $audio := widgets.generate_audio ? "on" : "off"; - $rate := $lookup($lookup($rates, $res), $audio); $ms := widgets.multi_shot; $isSb := $ms != "disabled"; $n := $isSb ? $number($substring($ms, 0, 1)) : 0; @@ -2962,7 +3038,18 @@ class KlingVideoNode(IO.ComfyNode): $d5 := $n >= 5 ? $lookup(widgets, "multi_shot.storyboard_5_duration") : 0; $d6 := $n >= 6 ? $lookup(widgets, "multi_shot.storyboard_6_duration") : 0; $dur := $isSb ? $d1 + $d2 + $d3 + $d4 + $d5 + $d6 : $lookup(widgets, "multi_shot.duration"); - {"type":"usd","usd": $rate * $dur} + widgets.model = "kling-3.0-turbo" + ? {"type":"usd","usd": ($res = "1080p" ? 0.14 : 0.112) * $dur} + : ( + $rates := { + "4k": {"off": 0.42, "on": 0.42}, + "1080p": {"off": 0.112, "on": 0.168}, + "720p": {"off": 0.084, "on": 0.126} + }; + $audio := widgets.generate_audio ? "on" : "off"; + $rate := $lookup($lookup($rates, $res), $audio); + {"type":"usd","usd": $rate * $dur} + ) ) """, ), @@ -3015,6 +3102,17 @@ class KlingVideoNode(IO.ComfyNode): duration = multi_shot["duration"] validate_string(multi_shot["prompt"], min_length=1, max_length=2500) + if model["model"] == "kling-3.0-turbo": + turbo_prompt = build_turbo_shot_prompt(multi_prompt_list) if custom_multi_shot else multi_shot["prompt"] + return await execute_kling_turbo( + cls, + prompt=turbo_prompt, + resolution=model["resolution"], + aspect_ratio=model["aspect_ratio"], + duration=duration, + start_frame=start_frame, + ) + if start_frame is not None: validate_image_dimensions(start_frame, min_width=300, min_height=300) validate_image_aspect_ratio(start_frame, (1, 2.5), (2.5, 1)) diff --git a/comfy_api_nodes/nodes_luma.py b/comfy_api_nodes/nodes_luma.py index 0d31ac77e..cdfa32d8b 100644 --- a/comfy_api_nodes/nodes_luma.py +++ b/comfy_api_nodes/nodes_luma.py @@ -3,9 +3,13 @@ from typing_extensions import override from comfy_api.latest import IO, ComfyExtension, Input from comfy_api_nodes.apis.luma import ( + LUMA_KEYFRAME_MODE_FRACTION, + LUMA_KEYFRAME_MODE_SECONDS, Luma2Generation, Luma2GenerationRequest, Luma2ImageRef, + Luma2VideoEdit, + Luma2VideoOptions, LumaAspectRatio, LumaCharacterRef, LumaConceptChain, @@ -18,6 +22,8 @@ from comfy_api_nodes.apis.luma import ( LumaIO, LumaKeyframes, LumaModifyImageRef, + LumaRay32KeyframeChain, + LumaRay32KeyframeItem, LumaReference, LumaReferenceChain, LumaVideoModel, @@ -33,6 +39,7 @@ from comfy_api_nodes.util import ( sync_op, upload_image_to_comfyapi, upload_images_to_comfyapi, + upload_video_to_comfyapi, validate_string, ) @@ -692,7 +699,10 @@ async def _luma2_upload_image_refs( async def _luma2_submit_and_poll( cls: type[IO.ComfyNode], request: Luma2GenerationRequest, -) -> Input.Image: + *, + estimated_duration: int | None = None, +) -> Luma2Generation: + """Submit a Luma Agents generation and poll until done; returns the completed generation.""" initial = await sync_op( cls, ApiEndpoint(path="/proxy/luma_2/generations", method="POST"), @@ -700,21 +710,21 @@ async def _luma2_submit_and_poll( data=request, ) if not initial.id: - raise RuntimeError("Luma 2 API did not return a generation id.") + raise RuntimeError("Luma API did not return a generation id.") final = await poll_op( cls, ApiEndpoint(path=f"/proxy/luma_2/generations/{initial.id}", method="GET"), response_model=Luma2Generation, status_extractor=lambda r: r.state, progress_extractor=lambda r: None, + estimated_duration=estimated_duration, ) - if not final.output: + if not final.output or not final.output[0].url: msg = final.failure_reason or "no output returned" - raise RuntimeError(f"Luma 2 generation failed: {msg}") - url = final.output[0].url - if not url: - raise RuntimeError("Luma 2 generation completed without an output URL.") - return await download_url_to_image_tensor(url) + if final.failure_code: + msg = f"{msg} [{final.failure_code}]" + raise RuntimeError(f"Luma generation failed: {msg}") + return final class LumaImageNode(IO.ComfyNode): @@ -843,7 +853,8 @@ class LumaImageNode(IO.ComfyNode): web_search=model["web_search"], image_ref=await _luma2_upload_image_refs(cls, model.get("image_ref"), max_count=9), ) - return IO.NodeOutput(await _luma2_submit_and_poll(cls, request)) + final = await _luma2_submit_and_poll(cls, request) + return IO.NodeOutput(await download_url_to_image_tensor(final.output[0].url)) class LumaImageEditNode(IO.ComfyNode): @@ -929,7 +940,533 @@ class LumaImageEditNode(IO.ComfyNode): web_search=model["web_search"], image_ref=await _luma2_upload_image_refs(cls, model.get("image_ref"), max_count=8), ) - return IO.NodeOutput(await _luma2_submit_and_poll(cls, request)) + final = await _luma2_submit_and_poll(cls, request) + return IO.NodeOutput(await download_url_to_image_tensor(final.output[0].url)) + + +_BADGE_RAY32_VIDEO = IO.PriceBadge( + depends_on=IO.PriceBadgeDepends(widgets=["resolution", "duration"]), + expr=""" + ( + $p := { + "360p": {"5s": 0.06, "10s": 0.18}, + "540p": {"5s": 0.15, "10s": 0.45}, + "720p": {"5s": 0.3, "10s": 0.9}, + "1080p": {"5s": 1.2, "10s": 3.6} + }; + {"type": "usd", "usd": $lookup($lookup($p, widgets.resolution), widgets.duration)} + ) + """, +) + +_BADGE_RAY32_VIDEO_5S = IO.PriceBadge( + depends_on=IO.PriceBadgeDepends(widgets=["resolution"]), + expr=""" + ( + $p := {"360p": 0.06, "540p": 0.15, "720p": 0.3, "1080p": 1.2}; + {"type": "usd", "usd": $lookup($p, widgets.resolution)} + ) + """, +) + +_BADGE_RAY32_EDIT = IO.PriceBadge( + depends_on=IO.PriceBadgeDepends(widgets=["resolution"]), + expr=""" + ( + $p := { + "360p": {"min": 0.54, "max": 1.08}, + "540p": {"min": 0.72, "max": 1.44}, + "720p": {"min": 1.08, "max": 2.16}, + "1080p": {"min": 2.16, "max": 4.32} + }; + $r := $lookup($p, widgets.resolution); + {"type": "range_usd", "min_usd": $r.min, "max_usd": $r.max, "format": {"note": "(by source length)"}} + ) + """, +) + +_BADGE_RAY32_REFRAME = IO.PriceBadge( + depends_on=IO.PriceBadgeDepends(widgets=["resolution"]), + expr=""" + ( + $p := {"360p": 0.03, "540p": 0.06, "720p": 0.12, "1080p": 0.36}; + {"type": "usd", "usd": $lookup($p, widgets.resolution), "format": {"suffix": "/second"}} + ) + """, +) + + +def _ray32_seed_input() -> IO.Input: + return IO.Int.Input( + "seed", + default=0, + min=0, + max=0xFFFFFFFFFFFFFFFF, + control_after_generate=True, + tooltip="Seed to determine if node should re-run; results are nondeterministic regardless of seed.", + ) + + +async def _ray32_generate(cls: type[IO.ComfyNode], request: Luma2GenerationRequest) -> IO.NodeOutput: + """Run a ray-3.2 generation and return (video, generation_id).""" + final = await _luma2_submit_and_poll(cls, request, estimated_duration=120) + video = await download_url_to_video_output(final.output[0].url) + return IO.NodeOutput(video, final.id or "") + + +class LumaRay32TextToVideoNode(IO.ComfyNode): + @classmethod + def define_schema(cls) -> IO.Schema: + return IO.Schema( + node_id="LumaRay32TextToVideoNode", + display_name="Luma Ray 3.2 Text to Video", + category="partner/video/Luma", + description="Generate a video from a text prompt using Luma's Ray 3.2 model.", + inputs=[ + IO.String.Input("prompt", multiline=True, default="", tooltip="Text prompt for the video generation."), + IO.Combo.Input("aspect_ratio", options=["16:9", "9:16", "1:1", "4:3", "3:4", "21:9"]), + IO.Combo.Input("resolution", options=["360p", "540p", "720p", "1080p"], default="720p"), + IO.Combo.Input("duration", options=["5s", "10s"]), + IO.Boolean.Input( + "loop", + default=False, + tooltip="Make the video loop seamlessly. Only available with 5s duration.", + ), + _ray32_seed_input(), + ], + outputs=[IO.Video.Output(), IO.String.Output(display_name="generation_id")], + hidden=[ + IO.Hidden.auth_token_comfy_org, + IO.Hidden.api_key_comfy_org, + IO.Hidden.unique_id, + ], + is_api_node=True, + price_badge=_BADGE_RAY32_VIDEO, + ) + + @classmethod + async def execute( + cls, prompt: str, aspect_ratio: str, resolution: str, duration: str, loop: bool, seed: int + ) -> IO.NodeOutput: + validate_string(prompt, strip_whitespace=True, min_length=1, max_length=6000) + if loop and duration == "10s": + raise ValueError("Looping is only available with 5s duration on Ray 3.2.") + request = Luma2GenerationRequest( + prompt=prompt, + model="ray-3.2", + type="video", + aspect_ratio=aspect_ratio, + video=Luma2VideoOptions(resolution=resolution, duration=duration, loop=loop or None), + ) + return await _ray32_generate(cls, request) + + +class LumaRay32ImageToVideoNode(IO.ComfyNode): + @classmethod + def define_schema(cls) -> IO.Schema: + return IO.Schema( + node_id="LumaRay32ImageToVideoNode", + display_name="Luma Ray 3.2 Image to Video", + category="partner/video/Luma", + description="Generate a video from a start and/or end frame using Luma's Ray 3.2 model. " + "Image-anchored generations are always 5 seconds.", + inputs=[ + IO.String.Input("prompt", multiline=True, default="", tooltip="Text prompt for the video generation."), + IO.Combo.Input("resolution", options=["360p", "540p", "720p", "1080p"], default="720p"), + IO.Boolean.Input( + "loop", + default=False, + tooltip="Make the video loop seamlessly. Not available when an end_frame is set.", + ), + _ray32_seed_input(), + IO.Image.Input("start_frame", optional=True, tooltip="First frame of the generated video."), + IO.Image.Input("end_frame", optional=True, tooltip="Last frame of the generated video."), + ], + outputs=[IO.Video.Output(), IO.String.Output(display_name="generation_id")], + hidden=[ + IO.Hidden.auth_token_comfy_org, + IO.Hidden.api_key_comfy_org, + IO.Hidden.unique_id, + ], + is_api_node=True, + price_badge=_BADGE_RAY32_VIDEO_5S, + ) + + @classmethod + async def execute( + cls, + prompt: str, + resolution: str, + loop: bool, + seed: int, + start_frame: torch.Tensor | None = None, + end_frame: torch.Tensor | None = None, + ) -> IO.NodeOutput: + validate_string(prompt, strip_whitespace=True, min_length=1, max_length=6000) + if start_frame is None and end_frame is None: + raise ValueError("Provide at least one of start_frame / end_frame.") + if loop and end_frame is not None: + raise ValueError("Looping is not available when an end_frame is set.") + video = Luma2VideoOptions(resolution=resolution, duration="5s", loop=loop or None) + if start_frame is not None: + url = await upload_image_to_comfyapi(cls, start_frame, mime_type="image/png") + video.start_frame = Luma2ImageRef(url=url) + if end_frame is not None: + url = await upload_image_to_comfyapi(cls, end_frame, mime_type="image/png") + video.end_frame = Luma2ImageRef(url=url) + request = Luma2GenerationRequest(prompt=prompt, model="ray-3.2", type="video", video=video) + return await _ray32_generate(cls, request) + + +class LumaRay32KeyframeNode(IO.ComfyNode): + @classmethod + def define_schema(cls) -> IO.Schema: + return IO.Schema( + node_id="LumaRay32KeyframeNode", + display_name="Luma Ray 3.2 Keyframe", + category="partner/video/Luma", + description="Anchor a guide image to a position on the Ray 3.2 output video timeline. Connect this to " + "the 'keyframes' input of the Luma Ray 3.2 Keyframes to Video node; chain several together via the " + "optional 'keyframes' input below.", + inputs=[ + IO.Image.Input("image", tooltip="Guide image to place at the chosen moment of the output video."), + IO.DynamicCombo.Input( + "position", + options=[ + IO.DynamicCombo.Option( + "Fraction of duration (0.0-1.0)", + [ + IO.Float.Input( + "fraction", + default=0.0, + min=0.0, + max=1.0, + step=0.01, + display_mode=IO.NumberDisplay.number, + tooltip="Where in the output video this image applies " "(0.0 = start, 1.0 = end).", + ), + ], + ), + IO.DynamicCombo.Option( + "Absolute time (seconds)", + [ + IO.Float.Input( + "seconds", + default=0.0, + min=0.0, + max=10.0, + step=0.1, + display_mode=IO.NumberDisplay.number, + tooltip="Time in seconds from the start of the output video where this " + "image applies.", + ), + ], + ), + ], + tooltip="How to place this image on the output video's timeline.", + ), + IO.Custom(LumaIO.LUMA_RAY32_KEYFRAME).Input( + "keyframes", + optional=True, + tooltip="Optional earlier keyframes to chain with this one.", + ), + ], + outputs=[IO.Custom(LumaIO.LUMA_RAY32_KEYFRAME).Output(display_name="keyframes")], + ) + + @classmethod + def execute( + cls, + image: torch.Tensor, + position: dict, + keyframes: LumaRay32KeyframeChain | None = None, + ) -> IO.NodeOutput: + chain = keyframes.clone() if keyframes is not None else LumaRay32KeyframeChain() + if position["position"] == "Absolute time (seconds)": + mode, value = LUMA_KEYFRAME_MODE_SECONDS, float(position["seconds"]) + else: + mode, value = LUMA_KEYFRAME_MODE_FRACTION, float(position["fraction"]) + chain.add(LumaRay32KeyframeItem(image=image, mode=mode, value=value)) + return IO.NodeOutput(chain) + + +class LumaRay32KeyframesToVideoNode(IO.ComfyNode): + @classmethod + def define_schema(cls) -> IO.Schema: + return IO.Schema( + node_id="LumaRay32KeyframesToVideoNode", + display_name="Luma Ray 3.2 Keyframes to Video", + category="partner/video/Luma", + description="Generate a video that interpolates through a sequence of guide images, each anchored to a " + "position on the timeline, using Luma Ray 3.2. Build the sequence with Luma Ray 3.2 Keyframe nodes " + "(at least 2).", + inputs=[ + IO.String.Input("prompt", multiline=True, default="", tooltip="Text prompt for the video generation."), + IO.Combo.Input("resolution", options=["360p", "540p", "720p", "1080p"], default="720p"), + IO.Combo.Input("duration", options=["5s", "10s"]), + _ray32_seed_input(), + IO.Custom(LumaIO.LUMA_RAY32_KEYFRAME).Input( + "keyframes", + tooltip="Keyframe sequence from Luma Ray 3.2 Keyframe nodes (at least 2).", + ), + ], + outputs=[IO.Video.Output(), IO.String.Output(display_name="generation_id")], + hidden=[ + IO.Hidden.auth_token_comfy_org, + IO.Hidden.api_key_comfy_org, + IO.Hidden.unique_id, + ], + is_api_node=True, + price_badge=_BADGE_RAY32_VIDEO, + ) + + @classmethod + async def execute( + cls, + prompt: str, + resolution: str, + duration: str, + seed: int, + keyframes: LumaRay32KeyframeChain | None = None, + ) -> IO.NodeOutput: + validate_string(prompt, strip_whitespace=True, min_length=1, max_length=6000) + items = keyframes.items if keyframes is not None else [] + if len(items) < 2: + raise ValueError( + "Connect at least 2 Luma Ray 3.2 Keyframe nodes " + "(use Luma Ray 3.2 Image to Video for a single start/end frame)." + ) + if len(items) > 64: + raise ValueError(f"Ray 3.2 supports at most 64 keyframes; got {len(items)}.") + maxframe = 120 if duration == "5s" else 240 + duration_seconds = maxframe / 24 # 5.0 or 10.0 + # Resolve each keyframe to an output-frame index, then order by position + # (so the user can chain keyframes in any order — the position is what places them) + placed: list[tuple[int, torch.Tensor]] = [] + for item in items: + if item.mode == LUMA_KEYFRAME_MODE_SECONDS: + if item.value > duration_seconds: + raise ValueError( + f"Keyframe position {item.value:g}s is past the end of the {duration} video; " + f"use 0-{duration_seconds:g}s (or switch the keyframe to fraction mode)." + ) + idx = round(item.value * 24) + else: + idx = round(item.value * maxframe) + placed.append((max(0, min(maxframe, idx)), item.image)) + placed.sort(key=lambda p: p[0]) + indexes = [idx for idx, _ in placed] + for a, b in zip(indexes, indexes[1:]): + if a == b: + raise ValueError( + f"Two keyframes resolve to the same output frame ({a}) for a {duration} video " + f"(valid range 0-{maxframe}); give each keyframe a distinct position." + ) + refs: list[Luma2ImageRef] = [] + for _, image in placed: + url = await upload_image_to_comfyapi(cls, image, mime_type="image/png") + refs.append(Luma2ImageRef(url=url)) + request = Luma2GenerationRequest( + prompt=prompt, + model="ray-3.2", + type="video", + video=Luma2VideoOptions(resolution=resolution, duration=duration, keyframes=refs, keyframe_indexes=indexes), + ) + return await _ray32_generate(cls, request) + + +class LumaRay32VideoEditNode(IO.ComfyNode): + @classmethod + def define_schema(cls) -> IO.Schema: + return IO.Schema( + node_id="LumaRay32VideoEditNode", + display_name="Luma Ray 3.2 Video Edit", + category="partner/video/Luma", + description="Re-render an existing video under a new prompt using Luma Ray 3.2 (restyle, relight, add " + "or remove elements) while keeping the original motion. Source video up to 18 seconds; the edited " + "video keeps the source's length.", + inputs=[ + IO.Video.Input("video", tooltip="Source video to edit. Up to 18 seconds."), + IO.String.Input("prompt", multiline=True, default="", tooltip="Describes the desired edit."), + IO.Combo.Input("resolution", options=["360p", "540p", "720p", "1080p"], default="720p"), + IO.Combo.Input( + "strength", + options=[ + "auto", + "adhere_1", + "adhere_2", + "adhere_3", + "flex_1", + "flex_2", + "flex_3", + "reimagine_1", + "reimagine_2", + "reimagine_3", + ], + default="auto", + tooltip="How strongly to preserve vs. reimagine the source. 'auto' lets Ray 3.2 choose; " + "adhere_* preserves the most, flex_* is balanced, reimagine_* changes the most.", + ), + _ray32_seed_input(), + ], + outputs=[ + IO.Video.Output(), + IO.String.Output(display_name="generation_id"), + ], + hidden=[ + IO.Hidden.auth_token_comfy_org, + IO.Hidden.api_key_comfy_org, + IO.Hidden.unique_id, + ], + is_api_node=True, + price_badge=_BADGE_RAY32_EDIT, + ) + + @classmethod + async def execute( + cls, video: Input.Video, prompt: str, resolution: str, strength: str, seed: int + ) -> IO.NodeOutput: + validate_string(prompt, strip_whitespace=True, min_length=1, max_length=6000) + try: + duration = "5s" if video.get_duration() <= 5.0 else "10s" + except Exception: + duration = "10s" + source_url = await upload_video_to_comfyapi(cls, video, max_duration=18) + edit = Luma2VideoEdit(auto_controls=True) if strength == "auto" else Luma2VideoEdit(strength=strength) + request = Luma2GenerationRequest( + prompt=prompt, + model="ray-3.2", + type="video_edit", + source=Luma2ImageRef(url=source_url, media_type="video/mp4"), + video=Luma2VideoOptions(resolution=resolution, duration=duration, edit=edit), + ) + return await _ray32_generate(cls, request) + + +class LumaRay32VideoReframeNode(IO.ComfyNode): + @classmethod + def define_schema(cls) -> IO.Schema: + return IO.Schema( + node_id="LumaRay32VideoReframeNode", + display_name="Luma Ray 3.2 Video Reframe", + category="partner/video/Luma", + description="Change the aspect ratio of an existing video, using Luma Ray 3.2 to fill the newly " + "exposed canvas areas. Source video up to 30 seconds. Billed per second of output.", + inputs=[ + IO.Video.Input("video", tooltip="Source video to reframe. Up to 30 seconds."), + IO.String.Input( + "prompt", + multiline=True, + default="", + tooltip="Describes how the newly exposed canvas areas should be filled.", + ), + IO.Combo.Input("aspect_ratio", options=["16:9", "9:16", "1:1", "4:3", "3:4", "21:9"]), + IO.Combo.Input("resolution", options=["360p", "540p", "720p", "1080p"], default="720p"), + _ray32_seed_input(), + ], + outputs=[ + IO.Video.Output(), + IO.String.Output(display_name="generation_id"), + ], + hidden=[ + IO.Hidden.auth_token_comfy_org, + IO.Hidden.api_key_comfy_org, + IO.Hidden.unique_id, + ], + is_api_node=True, + price_badge=_BADGE_RAY32_REFRAME, + ) + + @classmethod + async def execute( + cls, video: Input.Video, prompt: str, aspect_ratio: str, resolution: str, seed: int + ) -> IO.NodeOutput: + validate_string(prompt, strip_whitespace=False, min_length=1, max_length=6000) + if resolution == "1080p" and aspect_ratio in {"9:16", "3:4"}: + raise ValueError("1080p is not available for vertical aspect ratios (9:16, 3:4) when reframing.") + source_url = await upload_video_to_comfyapi(cls, video, max_duration=30) + request = Luma2GenerationRequest( + prompt=prompt, + model="ray-3.2", + type="video_reframe", + aspect_ratio=aspect_ratio, + source=Luma2ImageRef(url=source_url, media_type="video/mp4"), + video=Luma2VideoOptions(resolution=resolution), + ) + return await _ray32_generate(cls, request) + + +class LumaRay32ExtendVideoNode(IO.ComfyNode): + @classmethod + def define_schema(cls) -> IO.Schema: + return IO.Schema( + node_id="LumaRay32ExtendVideoNode", + display_name="Luma Ray 3.2 Extend Video", + category="partner/video/Luma", + description="Extend a previous Ray 3.2 generation forward (continue after it) or backward (lead-in " + "before it). Connect the generation_id output of a prior Luma Ray 3.2 node." + " Extensions are always 5 seconds.", + inputs=[ + IO.String.Input( + "source_generation_id", + default="", + tooltip="generation_id of the prior Ray 3.2 video to extend." + " Connect the generation_id output of another Luma Ray 3.2 node.", + ), + IO.DynamicCombo.Input( + "direction", + options=[ + IO.DynamicCombo.Option( + "Forward (continue after)", + [ + IO.Boolean.Input( + "loop", + default=False, + tooltip="Loop the extended video seamlessly (forward extend only).", + ), + ], + ), + IO.DynamicCombo.Option("Backward (lead-in before)", []), + ], + tooltip="Forward continues after the prior clip; backward is prepended before it.", + ), + IO.String.Input("prompt", multiline=True, default="", tooltip="Text prompt for the new content."), + IO.Combo.Input("resolution", options=["540p", "720p", "1080p"], default="720p"), + _ray32_seed_input(), + ], + outputs=[ + IO.Video.Output(), + IO.String.Output(display_name="generation_id"), + ], + hidden=[ + IO.Hidden.auth_token_comfy_org, + IO.Hidden.api_key_comfy_org, + IO.Hidden.unique_id, + ], + is_api_node=True, + price_badge=_BADGE_RAY32_VIDEO_5S, + ) + + @classmethod + async def execute( + cls, source_generation_id: str, direction: dict, prompt: str, resolution: str, seed: int + ) -> IO.NodeOutput: + validate_string(prompt, strip_whitespace=False, min_length=1, max_length=6000) + gen_id = (source_generation_id or "").strip() + if not gen_id: + raise ValueError( + "source_generation_id is required (connect the generation_id output of a prior Luma Ray 3.2 node)." + ) + video = Luma2VideoOptions(resolution=resolution, duration="5s") + ref = Luma2ImageRef(generation_id=gen_id) + if direction["direction"] == "Forward (continue after)": + video.start_frame = ref + if direction.get("loop"): + video.loop = True + else: + video.end_frame = ref + request = Luma2GenerationRequest(prompt=prompt, model="ray-3.2", type="video", video=video) + return await _ray32_generate(cls, request) class LumaExtension(ComfyExtension): @@ -944,6 +1481,13 @@ class LumaExtension(ComfyExtension): LumaConceptsNode, LumaImageNode, LumaImageEditNode, + LumaRay32TextToVideoNode, + LumaRay32ImageToVideoNode, + LumaRay32KeyframeNode, + LumaRay32KeyframesToVideoNode, + LumaRay32VideoEditNode, + LumaRay32VideoReframeNode, + LumaRay32ExtendVideoNode, ] diff --git a/comfy_api_nodes/nodes_openai.py b/comfy_api_nodes/nodes_openai.py index 0fe5fb9d0..ad62f2164 100644 --- a/comfy_api_nodes/nodes_openai.py +++ b/comfy_api_nodes/nodes_openai.py @@ -9,6 +9,7 @@ from PIL import Image from typing_extensions import override import folder_paths +from comfy.utils import common_upscale from comfy_api.latest import IO, ComfyExtension, Input from comfy_api_nodes.apis.openai import ( InputFileContent, @@ -62,7 +63,8 @@ async def validate_and_cast_response(response, timeout: int = None) -> torch.Ten timeout: Request timeout in seconds. Defaults to None (no timeout). Returns: - A torch.Tensor representing the image (1, H, W, C). + A torch.Tensor of shape (N, H, W, C) with all returned images; images whose + dimensions differ from the first image's are resized to match it. Raises: ValueError: If the response is not valid. @@ -89,6 +91,14 @@ async def validate_and_cast_response(response, timeout: int = None) -> torch.Ten arr = np.asarray(pil_img).astype(np.float32) / 255.0 image_tensors.append(torch.from_numpy(arr)) + # With size="auto" the API can return images whose dimensions differ by a few pixels within a single response + # resize them to the first image's dimensions so they can be stacked into one batch. + ref_h, ref_w = image_tensors[0].shape[:2] + for i, t in enumerate(image_tensors): + if t.shape[:2] != (ref_h, ref_w): + samples = t.unsqueeze(0).movedim(-1, 1) + samples = common_upscale(samples, ref_w, ref_h, "bilinear", "center") + image_tensors[i] = samples.movedim(1, -1).squeeze(0) return torch.stack(image_tensors, dim=0) diff --git a/comfy_api_nodes/nodes_runway.py b/comfy_api_nodes/nodes_runway.py index b9c5c81a1..013a193d9 100644 --- a/comfy_api_nodes/nodes_runway.py +++ b/comfy_api_nodes/nodes_runway.py @@ -30,13 +30,33 @@ from comfy_api_nodes.apis.runway import ( Model4, ReferenceImage, RunwayTextToImageAspectRatioEnum, + RunwayAleph2IO, + RunwayAleph2KeyframeChain, + RunwayAleph2KeyframeItem, + RunwayAleph2PromptImageChain, + RunwayAleph2PromptImageItem, + RunwayAleph2Request, + RunwayAleph2Response, + RunwayAleph2KeyframeSeconds, + RunwayAleph2KeyframeAt, + RunwayAleph2PromptImage, + RunwayAleph2TimestampPosition, + RunwayAleph2RelativePosition, + RunwayAleph2ContentModeration, + KEYFRAME_MODE_SECONDS, + KEYFRAME_MODE_AT, + PROMPT_IMAGE_MODE_TIMESTAMP, + PROMPT_IMAGE_MODE_POSITION, ) from comfy_api_nodes.util import ( image_tensor_pair_to_batch, validate_string, validate_image_dimensions, validate_image_aspect_ratio, + validate_video_duration, upload_images_to_comfyapi, + upload_image_to_comfyapi, + upload_video_to_comfyapi, download_url_to_video_output, download_url_to_image_tensor, ApiEndpoint, @@ -45,6 +65,7 @@ from comfy_api_nodes.util import ( ) PATH_IMAGE_TO_VIDEO = "/proxy/runway/image_to_video" +PATH_VIDEO_TO_VIDEO = "/proxy/runway/video_to_video" PATH_TEXT_TO_IMAGE = "/proxy/runway/text_to_image" PATH_GET_TASK_STATUS = "/proxy/runway/tasks" @@ -53,12 +74,6 @@ AVERAGE_DURATION_FLF_SECONDS = 256 AVERAGE_DURATION_T2I_SECONDS = 41 -class RunwayApiError(Exception): - """Base exception for Runway API errors.""" - - pass - - class RunwayGen4TurboAspectRatio(str, Enum): """Aspect ratios supported for Image to Video API when using gen4_turbo model.""" @@ -84,14 +99,6 @@ def get_video_url_from_task_status(response: TaskStatusResponse) -> str | None: return None -def extract_progress_from_task_status( - response: TaskStatusResponse, -) -> float | None: - if hasattr(response, "progress") and response.progress is not None: - return response.progress * 100 - return None - - def get_image_url_from_task_status(response: TaskStatusResponse) -> str | None: """Returns the image URL from the task status response if it exists.""" if hasattr(response, "output") and len(response.output) > 0: @@ -102,14 +109,13 @@ def get_image_url_from_task_status(response: TaskStatusResponse) -> str | None: async def get_response( cls: type[IO.ComfyNode], task_id: str, estimated_duration: int | None = None ) -> TaskStatusResponse: - """Poll the task status until it is finished then get the response.""" return await poll_op( cls, ApiEndpoint(path=f"{PATH_GET_TASK_STATUS}/{task_id}"), response_model=TaskStatusResponse, - status_extractor=lambda r: r.status.value, + status_extractor=lambda r: r.status, estimated_duration=estimated_duration, - progress_extractor=extract_progress_from_task_status, + progress_extractor=lambda r: r.progress * 100 if r.progress is not None else None, ) @@ -127,7 +133,7 @@ async def generate_video( final_response = await get_response(cls, initial_response.id, estimated_duration) if not final_response.output: - raise RunwayApiError("Runway task succeeded but no video data found in response.") + raise ValueError("Runway task succeeded but no video data found in response.") video_url = get_video_url_from_task_status(final_response) return await download_url_to_video_output(video_url) @@ -410,7 +416,7 @@ class RunwayFirstLastFrameNode(IO.ComfyNode): mime_type="image/png", ) if len(download_urls) != 2: - raise RunwayApiError("Failed to upload one or more images to comfy api.") + raise ValueError("Failed to upload one or more images to comfy api.") return IO.NodeOutput( await generate_video( @@ -514,11 +520,321 @@ class RunwayTextToImageNode(IO.ComfyNode): estimated_duration=AVERAGE_DURATION_T2I_SECONDS, ) if not final_response.output: - raise RunwayApiError("Runway task succeeded but no image data found in response.") + raise ValueError("Runway task succeeded but no image data found in response.") return IO.NodeOutput(await download_url_to_image_tensor(get_image_url_from_task_status(final_response))) +_TIMING_ABSOLUTE = "Absolute time (seconds)" +_TIMING_FRACTION = "Fraction of duration (0.0-1.0)" + + +class RunwayAleph2KeyframeNode(IO.ComfyNode): + + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="RunwayAleph2KeyframeNode", + display_name="Runway Aleph2 Keyframe", + category="partner/video/Runway", + description="Anchor a guidance image to a moment of the input (source) video, so Aleph2 " + "steers the edit at that point of your footage. Connect this to the 'keyframes' input of " + "the Runway Aleph2 Video to Video node; chain several together (up to 5) via the optional " + "'keyframes' input below.", + inputs=[ + IO.Image.Input( + "image", + tooltip="The guidance image to apply at the chosen moment of the input video.", + ), + IO.DynamicCombo.Input( + "timing", + options=[ + IO.DynamicCombo.Option( + _TIMING_ABSOLUTE, + [ + IO.Float.Input( + "seconds", + default=0.0, + min=0.0, + max=30.0, + step=0.1, + display_mode=IO.NumberDisplay.number, + tooltip="Time in seconds from start of the input video where this image applies.", + ), + ], + ), + IO.DynamicCombo.Option( + _TIMING_FRACTION, + [ + IO.Float.Input( + "fraction", + default=0.0, + min=0.0, + max=1.0, + step=0.01, + display_mode=IO.NumberDisplay.number, + tooltip="Where in the input video this image applies, " + "as a fraction of its duration (0.0 = start, 1.0 = end).", + ), + ], + ), + ], + tooltip="How to place this image on the input video's timeline.", + ), + IO.Custom(RunwayAleph2IO.KEYFRAME).Input( + "keyframes", + optional=True, + tooltip="Optional earlier keyframes to chain with this one.", + ), + ], + outputs=[IO.Custom(RunwayAleph2IO.KEYFRAME).Output(display_name="keyframes")], + ) + + @classmethod + def execute( + cls, + image: Input.Image, + timing: dict, + keyframes: RunwayAleph2KeyframeChain | None = None, + ) -> IO.NodeOutput: + chain = keyframes.clone() if keyframes is not None else RunwayAleph2KeyframeChain() + if timing["timing"] == _TIMING_ABSOLUTE: + mode, value = KEYFRAME_MODE_SECONDS, float(timing["seconds"]) + else: + mode, value = KEYFRAME_MODE_AT, float(timing["fraction"]) + chain.add(RunwayAleph2KeyframeItem(image=image, mode=mode, value=value)) + return IO.NodeOutput(chain) + + +class RunwayAleph2PromptImageNode(IO.ComfyNode): + + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="RunwayAleph2PromptImageNode", + display_name="Runway Aleph2 Prompt Image", + category="partner/video/Runway", + description="Anchor a guidance image to a moment of the output (result) video, to guide what " + "the edited video looks like at that point. Connect this to the 'prompt_images' input of the " + "Runway Aleph2 Video to Video node; chain several together (up to 5) via the optional " + "'prompt_images' input below.", + inputs=[ + IO.Image.Input( + "image", + tooltip="The guidance image to place at the chosen moment of the output video.", + ), + IO.DynamicCombo.Input( + "position", + options=[ + IO.DynamicCombo.Option( + _TIMING_ABSOLUTE, + [ + IO.Float.Input( + "seconds", + default=0.0, + min=0.0, + max=30.0, + step=0.1, + display_mode=IO.NumberDisplay.number, + tooltip="Time in seconds from start of the output video where this image applies.", + ), + ], + ), + IO.DynamicCombo.Option( + _TIMING_FRACTION, + [ + IO.Float.Input( + "fraction", + default=0.0, + min=0.0, + max=1.0, + step=0.01, + display_mode=IO.NumberDisplay.number, + tooltip="Where in the output video this image applies, " + "as a fraction of its duration (0.0 = start, 1.0 = end).", + ), + ], + ), + ], + tooltip="How to place this image on the output video's timeline.", + ), + IO.Custom(RunwayAleph2IO.PROMPT_IMAGE).Input( + "prompt_images", + optional=True, + tooltip="Optional earlier prompt images to chain with this one.", + ), + ], + outputs=[IO.Custom(RunwayAleph2IO.PROMPT_IMAGE).Output(display_name="prompt_images")], + ) + + @classmethod + def execute( + cls, + image: Input.Image, + position: dict, + prompt_images: RunwayAleph2PromptImageChain | None = None, + ) -> IO.NodeOutput: + chain = prompt_images.clone() if prompt_images is not None else RunwayAleph2PromptImageChain() + if position["position"] == _TIMING_ABSOLUTE: + mode, value = PROMPT_IMAGE_MODE_TIMESTAMP, float(position["seconds"]) + else: + mode, value = PROMPT_IMAGE_MODE_POSITION, float(position["fraction"]) + chain.add(RunwayAleph2PromptImageItem(image=image, mode=mode, value=value)) + return IO.NodeOutput(chain) + + +class RunwayAleph2VideoToVideoNode(IO.ComfyNode): + + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="RunwayAleph2VideoToVideoNode", + display_name="Runway Aleph2 Video to Video", + category="partner/video/Runway", + description="Edit a video with a text prompt using Runway's Aleph2 model. Aleph2 transforms " + "your footage (restyle, relight, add or remove elements, change the viewpoint) while keeping " + "the original motion and timing; the output resolution matches the input video, which must be " + "2-30 seconds at 30 fps or lower. Optionally steer the edit with either keyframes (anchored to " + "the input video) or prompt images (anchored to the output video) - use one or the other, not both.", + inputs=[ + IO.String.Input( + "prompt", + multiline=True, + default="", + tooltip="Describes what should appear in the output (1-1000 characters).", + ), + IO.Video.Input( + "video", + tooltip="Input video to edit. Must be 2-30 seconds at 30 fps or lower.", + ), + IO.Int.Input( + "seed", + default=0, + min=0, + max=4294967295, + step=1, + control_after_generate=True, + display_mode=IO.NumberDisplay.number, + tooltip="Random seed for generation", + ), + IO.Combo.Input( + "public_figure_threshold", + options=["auto", "low"], + default="low", + tooltip="Content moderation for recognizable public figures.", + ), + IO.Custom(RunwayAleph2IO.KEYFRAME).Input( + "keyframes", + optional=True, + tooltip="Guidance images anchored to the input video, from Aleph2 Keyframe nodes (up to 5). " + "Use keyframes or prompt images, not both.", + ), + IO.Custom(RunwayAleph2IO.PROMPT_IMAGE).Input( + "prompt_images", + optional=True, + tooltip="Guidance images anchored to the output video, from Aleph2 Prompt Image nodes (up to 5). " + "Use keyframes or prompt images, not both.", + ), + ], + outputs=[ + IO.Video.Output(), + ], + hidden=[ + IO.Hidden.auth_token_comfy_org, + IO.Hidden.api_key_comfy_org, + IO.Hidden.unique_id, + ], + is_api_node=True, + price_badge=IO.PriceBadge( + expr="""{"type":"usd","usd": 0.4004, "format":{"suffix":"/second"}}""", + ), + ) + + @classmethod + async def execute( + cls, + prompt: str, + video: Input.Video, + seed: int, + public_figure_threshold: str = "low", + keyframes: RunwayAleph2KeyframeChain | None = None, + prompt_images: RunwayAleph2PromptImageChain | None = None, + ) -> IO.NodeOutput: + validate_string(prompt, min_length=1, max_length=1000) + validate_video_duration( + video, + min_duration=2.0, + max_duration=30.0, + ) + try: + fps = float(video.get_frame_rate()) + except Exception: + fps = None + if fps is not None and fps > 30.0 + 0.01: + raise ValueError(f"Input video frame rate ({fps:.2f} fps) exceeds Aleph2's maximum of 30 fps.") + + if (keyframes and keyframes.items) and (prompt_images and prompt_images.items): + raise ValueError("Aleph2 accepts either keyframes or prompt images, not both.") + + video_duration: float | None = None + try: + video_duration = video.get_duration() + except Exception: + video_duration = None + + def _check_seconds(value: float, label: str) -> None: + if video_duration is not None and value > video_duration + 0.0001: + raise ValueError(f"{label} {value:.2f}s exceeds the input video duration ({video_duration:.2f}s).") + + video_url = await upload_video_to_comfyapi(cls, video) + + keyframe_models: list[RunwayAleph2KeyframeSeconds | RunwayAleph2KeyframeAt] = [] + if keyframes is not None: + if len(keyframes.items) > 5: + raise ValueError("Aleph2 supports at most 5 keyframes.") + for item in keyframes.items: + image_url = await upload_image_to_comfyapi(cls, item.image, mime_type="image/png") + if item.mode == KEYFRAME_MODE_SECONDS: + _check_seconds(item.value, "Keyframe timestamp") + keyframe_models.append(RunwayAleph2KeyframeSeconds(seconds=item.value, uri=image_url)) + else: + keyframe_models.append(RunwayAleph2KeyframeAt(at=item.value, uri=image_url)) + + prompt_image_models: list[RunwayAleph2PromptImage] = [] + if prompt_images is not None: + if len(prompt_images.items) > 5: + raise ValueError("Aleph2 supports at most 5 prompt images.") + for item in prompt_images.items: + image_url = await upload_image_to_comfyapi(cls, item.image, mime_type="image/png") + position: RunwayAleph2TimestampPosition | RunwayAleph2RelativePosition + if item.mode == PROMPT_IMAGE_MODE_TIMESTAMP: + _check_seconds(item.value, "Prompt image timestamp") + position = RunwayAleph2TimestampPosition(timestampSeconds=item.value) + else: + position = RunwayAleph2RelativePosition(positionPercentage=item.value) + prompt_image_models.append(RunwayAleph2PromptImage(position=position, uri=image_url)) + + initial_response = await sync_op( + cls, + endpoint=ApiEndpoint(path=PATH_VIDEO_TO_VIDEO, method="POST"), + response_model=RunwayAleph2Response, + data=RunwayAleph2Request( + promptText=prompt, + videoUri=video_url, + seed=seed, + contentModeration=RunwayAleph2ContentModeration(publicFigureThreshold=public_figure_threshold), + keyframes=keyframe_models or None, + promptImage=prompt_image_models or None, + ), + ) + + final_response = await get_response(cls, initial_response.id) + if not final_response.output: + raise ValueError("Runway task succeeded but no video data found in response.") + + return IO.NodeOutput(await download_url_to_video_output(get_video_url_from_task_status(final_response))) + + class RunwayExtension(ComfyExtension): @override async def get_node_list(self) -> list[type[IO.ComfyNode]]: @@ -527,6 +843,9 @@ class RunwayExtension(ComfyExtension): RunwayImageToVideoNodeGen3a, RunwayImageToVideoNodeGen4, RunwayTextToImageNode, + RunwayAleph2VideoToVideoNode, + RunwayAleph2KeyframeNode, + RunwayAleph2PromptImageNode, ] diff --git a/comfy_api_nodes/nodes_sonilo.py b/comfy_api_nodes/nodes_sonilo.py index 9ce896ed0..2ad35531a 100644 --- a/comfy_api_nodes/nodes_sonilo.py +++ b/comfy_api_nodes/nodes_sonilo.py @@ -16,7 +16,7 @@ from comfy_api_nodes.util import ( ) from comfy_api_nodes.util._helpers import ( default_base_url, - get_auth_header, + get_comfy_api_headers, get_node_id, is_processing_interrupted, ) @@ -100,8 +100,7 @@ class SoniloTextToMusic(IO.ComfyNode): node_id="SoniloTextToMusic", display_name="Sonilo Text to Music", category="partner/audio/Sonilo", - description="Generate music from a text prompt using Sonilo's AI model. " - "Leave duration at 0 to let the model infer it from the prompt.", + description="Generate music from a text prompt using Sonilo's AI model.", inputs=[ IO.String.Input( "prompt", @@ -111,11 +110,10 @@ class SoniloTextToMusic(IO.ComfyNode): ), IO.Int.Input( "duration", - default=0, - min=0, + default=30, + min=1, max=360, - tooltip="Target duration in seconds. Set to 0 to let the model " - "infer the duration from the prompt. Maximum: 6 minutes.", + tooltip="Target duration in seconds. Maximum: 6 minutes.", ), IO.Int.Input( "seed", @@ -136,13 +134,7 @@ class SoniloTextToMusic(IO.ComfyNode): is_api_node=True, price_badge=IO.PriceBadge( depends_on=IO.PriceBadgeDepends(widgets=["duration"]), - expr=""" - ( - widgets.duration > 0 - ? {"type":"usd","usd": 0.005 * widgets.duration} - : {"type":"usd","usd": 0.005, "format":{"suffix":"/second"}} - ) - """, + expr='{"type":"usd","usd": 0.0025 * widgets.duration}', ), ) @@ -150,14 +142,13 @@ class SoniloTextToMusic(IO.ComfyNode): async def execute( cls, prompt: str, - duration: int = 0, + duration: int = 1, seed: int = 0, ) -> IO.NodeOutput: - validate_string(prompt, strip_whitespace=True, min_length=1) + validate_string(prompt, strip_whitespace=True, min_length=1, max_length=1000) form = aiohttp.FormData() form.add_field("prompt", prompt) - if duration > 0: - form.add_field("duration", str(duration)) + form.add_field("duration", str(duration)) audio_bytes = await _stream_sonilo_music( cls, ApiEndpoint(path="/proxy/sonilo/t2m/generate", method="POST"), @@ -174,8 +165,7 @@ async def _stream_sonilo_music( """POST ``form`` to Sonilo, read the NDJSON stream, and return the first stream's audio bytes.""" url = urljoin(default_base_url().rstrip("/") + "/", endpoint.path.lstrip("/")) - headers: dict[str, str] = {} - headers.update(get_auth_header(cls)) + headers = get_comfy_api_headers(cls) headers.update(endpoint.headers) node_id = get_node_id(cls) diff --git a/comfy_api_nodes/nodes_stability.py b/comfy_api_nodes/nodes_stability.py deleted file mode 100644 index 9eaba173b..000000000 --- a/comfy_api_nodes/nodes_stability.py +++ /dev/null @@ -1,932 +0,0 @@ -from inspect import cleandoc -from typing import Optional -from typing_extensions import override - -from comfy_api.latest import ComfyExtension, Input, IO -from comfy_api_nodes.apis.stability import ( - StabilityUpscaleConservativeRequest, - StabilityUpscaleCreativeRequest, - StabilityAsyncResponse, - StabilityResultsGetResponse, - StabilityStable3_5Request, - StabilityStableUltraRequest, - StabilityStableUltraResponse, - StabilityAspectRatio, - Stability_SD3_5_Model, - Stability_SD3_5_GenerationMode, - get_stability_style_presets, - StabilityTextToAudioRequest, - StabilityAudioToAudioRequest, - StabilityAudioInpaintRequest, - StabilityAudioResponse, -) -from comfy_api_nodes.util import ( - validate_audio_duration, - validate_string, - audio_input_to_mp3, - bytesio_to_image_tensor, - tensor_to_bytesio, - audio_bytes_to_audio_input, - sync_op, - poll_op, - ApiEndpoint, -) - -import torch -import base64 -from io import BytesIO -from enum import Enum - - -class StabilityPollStatus(str, Enum): - finished = "finished" - in_progress = "in_progress" - failed = "failed" - - -def get_async_dummy_status(x: StabilityResultsGetResponse): - if x.name is not None or x.errors is not None: - return StabilityPollStatus.failed - elif x.finish_reason is not None: - return StabilityPollStatus.finished - return StabilityPollStatus.in_progress - - -class StabilityStableImageUltraNode(IO.ComfyNode): - """ - Generates images synchronously based on prompt and resolution. - """ - - @classmethod - def define_schema(cls): - return IO.Schema( - node_id="StabilityStableImageUltraNode", - display_name="Stability AI Stable Image Ultra", - category="partner/image/Stability AI", - description=cleandoc(cls.__doc__ or ""), - inputs=[ - IO.String.Input( - "prompt", - multiline=True, - default="", - tooltip="What you wish to see in the output image. A strong, descriptive prompt that clearly defines" + - "elements, colors, and subjects will lead to better results. " + - "To control the weight of a given word use the format `(word:weight)`," + - "where `word` is the word you'd like to control the weight of and `weight`" + - "is a value between 0 and 1. For example: `The sky was a crisp (blue:0.3) and (green:0.8)`" + - "would convey a sky that was blue and green, but more green than blue.", - ), - IO.Combo.Input( - "aspect_ratio", - options=StabilityAspectRatio, - default=StabilityAspectRatio.ratio_1_1, - tooltip="Aspect ratio of generated image.", - ), - IO.Combo.Input( - "style_preset", - options=get_stability_style_presets(), - tooltip="Optional desired style of generated image.", - advanced=True, - ), - IO.Int.Input( - "seed", - default=0, - min=0, - max=4294967294, - step=1, - display_mode=IO.NumberDisplay.number, - control_after_generate=True, - tooltip="The random seed used for creating the noise.", - ), - IO.Image.Input( - "image", - optional=True, - ), - IO.String.Input( - "negative_prompt", - default="", - tooltip="A blurb of text describing what you do not wish to see in the output image. This is an advanced feature.", - force_input=True, - optional=True, - advanced=True, - ), - IO.Float.Input( - "image_denoise", - default=0.5, - min=0.0, - max=1.0, - step=0.01, - tooltip="Denoise of input image; 0.0 yields image identical to input, 1.0 is as if no image was provided at all.", - optional=True, - ), - ], - outputs=[ - IO.Image.Output(), - ], - hidden=[ - IO.Hidden.auth_token_comfy_org, - IO.Hidden.api_key_comfy_org, - IO.Hidden.unique_id, - ], - is_api_node=True, - price_badge=IO.PriceBadge( - expr="""{"type":"usd","usd":0.08}""", - ), - ) - - @classmethod - async def execute( - cls, - prompt: str, - aspect_ratio: str, - style_preset: str, - seed: int, - image: Optional[torch.Tensor] = None, - negative_prompt: str = "", - image_denoise: Optional[float] = 0.5, - ) -> IO.NodeOutput: - validate_string(prompt, strip_whitespace=False) - # prepare image binary if image present - image_binary = None - if image is not None: - image_binary = tensor_to_bytesio(image, total_pixels=1504*1504).read() - else: - image_denoise = None - - if not negative_prompt: - negative_prompt = None - if style_preset == "None": - style_preset = None - - files = { - "image": image_binary - } - - response_api = await sync_op( - cls, - ApiEndpoint(path="/proxy/stability/v2beta/stable-image/generate/ultra", method="POST"), - response_model=StabilityStableUltraResponse, - data=StabilityStableUltraRequest( - prompt=prompt, - negative_prompt=negative_prompt, - aspect_ratio=aspect_ratio, - seed=seed, - strength=image_denoise, - style_preset=style_preset, - ), - files=files, - content_type="multipart/form-data", - ) - - if response_api.finish_reason != "SUCCESS": - raise Exception(f"Stable Image Ultra generation failed: {response_api.finish_reason}.") - - image_data = base64.b64decode(response_api.image) - returned_image = bytesio_to_image_tensor(BytesIO(image_data)) - - return IO.NodeOutput(returned_image) - - -class StabilityStableImageSD_3_5Node(IO.ComfyNode): - """ - Generates images synchronously based on prompt and resolution. - """ - - @classmethod - def define_schema(cls): - return IO.Schema( - node_id="StabilityStableImageSD_3_5Node", - display_name="Stability AI Stable Diffusion 3.5 Image", - category="partner/image/Stability AI", - description=cleandoc(cls.__doc__ or ""), - inputs=[ - IO.String.Input( - "prompt", - multiline=True, - default="", - tooltip="What you wish to see in the output image. A strong, descriptive prompt that clearly defines elements, colors, and subjects will lead to better results.", - ), - IO.Combo.Input( - "model", - options=Stability_SD3_5_Model, - ), - IO.Combo.Input( - "aspect_ratio", - options=StabilityAspectRatio, - default=StabilityAspectRatio.ratio_1_1, - tooltip="Aspect ratio of generated image.", - ), - IO.Combo.Input( - "style_preset", - options=get_stability_style_presets(), - tooltip="Optional desired style of generated image.", - advanced=True, - ), - IO.Float.Input( - "cfg_scale", - default=4.0, - min=1.0, - max=10.0, - step=0.1, - tooltip="How strictly the diffusion process adheres to the prompt text (higher values keep your image closer to your prompt)", - ), - IO.Int.Input( - "seed", - default=0, - min=0, - max=4294967294, - step=1, - display_mode=IO.NumberDisplay.number, - control_after_generate=True, - tooltip="The random seed used for creating the noise.", - ), - IO.Image.Input( - "image", - optional=True, - ), - IO.String.Input( - "negative_prompt", - default="", - tooltip="Keywords of what you do not wish to see in the output image. This is an advanced feature.", - force_input=True, - optional=True, - advanced=True, - ), - IO.Float.Input( - "image_denoise", - default=0.5, - min=0.0, - max=1.0, - step=0.01, - tooltip="Denoise of input image; 0.0 yields image identical to input, 1.0 is as if no image was provided at all.", - optional=True, - ), - ], - outputs=[ - IO.Image.Output(), - ], - hidden=[ - IO.Hidden.auth_token_comfy_org, - IO.Hidden.api_key_comfy_org, - IO.Hidden.unique_id, - ], - is_api_node=True, - price_badge=IO.PriceBadge( - depends_on=IO.PriceBadgeDepends(widgets=["model"]), - expr=""" - ( - $contains(widgets.model,"large") - ? {"type":"usd","usd":0.065} - : {"type":"usd","usd":0.035} - ) - """, - ), - ) - - @classmethod - async def execute( - cls, - model: str, - prompt: str, - aspect_ratio: str, - style_preset: str, - seed: int, - cfg_scale: float, - image: Optional[torch.Tensor] = None, - negative_prompt: str = "", - image_denoise: Optional[float] = 0.5, - ) -> IO.NodeOutput: - validate_string(prompt, strip_whitespace=False) - # prepare image binary if image present - image_binary = None - mode = Stability_SD3_5_GenerationMode.text_to_image - if image is not None: - image_binary = tensor_to_bytesio(image, total_pixels=1504*1504).read() - mode = Stability_SD3_5_GenerationMode.image_to_image - aspect_ratio = None - else: - image_denoise = None - - if not negative_prompt: - negative_prompt = None - if style_preset == "None": - style_preset = None - - files = { - "image": image_binary - } - - response_api = await sync_op( - cls, - ApiEndpoint(path="/proxy/stability/v2beta/stable-image/generate/sd3", method="POST"), - response_model=StabilityStableUltraResponse, - data=StabilityStable3_5Request( - prompt=prompt, - negative_prompt=negative_prompt, - aspect_ratio=aspect_ratio, - seed=seed, - strength=image_denoise, - style_preset=style_preset, - cfg_scale=cfg_scale, - model=model, - mode=mode, - ), - files=files, - content_type="multipart/form-data", - ) - - if response_api.finish_reason != "SUCCESS": - raise Exception(f"Stable Diffusion 3.5 Image generation failed: {response_api.finish_reason}.") - - image_data = base64.b64decode(response_api.image) - returned_image = bytesio_to_image_tensor(BytesIO(image_data)) - - return IO.NodeOutput(returned_image) - - -class StabilityUpscaleConservativeNode(IO.ComfyNode): - """ - Upscale image with minimal alterations to 4K resolution. - """ - - @classmethod - def define_schema(cls): - return IO.Schema( - node_id="StabilityUpscaleConservativeNode", - display_name="Stability AI Upscale Conservative", - category="partner/image/Stability AI", - description=cleandoc(cls.__doc__ or ""), - inputs=[ - IO.Image.Input("image"), - IO.String.Input( - "prompt", - multiline=True, - default="", - tooltip="What you wish to see in the output image. A strong, descriptive prompt that clearly defines elements, colors, and subjects will lead to better results.", - ), - IO.Float.Input( - "creativity", - default=0.35, - min=0.2, - max=0.5, - step=0.01, - tooltip="Controls the likelihood of creating additional details not heavily conditioned by the init image.", - ), - IO.Int.Input( - "seed", - default=0, - min=0, - max=4294967294, - step=1, - display_mode=IO.NumberDisplay.number, - control_after_generate=True, - tooltip="The random seed used for creating the noise.", - ), - IO.String.Input( - "negative_prompt", - default="", - tooltip="Keywords of what you do not wish to see in the output image. This is an advanced feature.", - force_input=True, - optional=True, - advanced=True, - ), - ], - outputs=[ - IO.Image.Output(), - ], - hidden=[ - IO.Hidden.auth_token_comfy_org, - IO.Hidden.api_key_comfy_org, - IO.Hidden.unique_id, - ], - is_api_node=True, - price_badge=IO.PriceBadge( - expr="""{"type":"usd","usd":0.4}""", - ), - ) - - @classmethod - async def execute( - cls, - image: torch.Tensor, - prompt: str, - creativity: float, - seed: int, - negative_prompt: str = "", - ) -> IO.NodeOutput: - validate_string(prompt, strip_whitespace=False) - image_binary = tensor_to_bytesio(image, total_pixels=1024*1024).read() - - if not negative_prompt: - negative_prompt = None - - files = { - "image": image_binary - } - - response_api = await sync_op( - cls, - ApiEndpoint(path="/proxy/stability/v2beta/stable-image/upscale/conservative", method="POST"), - response_model=StabilityStableUltraResponse, - data=StabilityUpscaleConservativeRequest( - prompt=prompt, - negative_prompt=negative_prompt, - creativity=round(creativity,2), - seed=seed, - ), - files=files, - content_type="multipart/form-data", - ) - - if response_api.finish_reason != "SUCCESS": - raise Exception(f"Stability Upscale Conservative generation failed: {response_api.finish_reason}.") - - image_data = base64.b64decode(response_api.image) - returned_image = bytesio_to_image_tensor(BytesIO(image_data)) - - return IO.NodeOutput(returned_image) - - -class StabilityUpscaleCreativeNode(IO.ComfyNode): - """ - Upscale image with minimal alterations to 4K resolution. - """ - - @classmethod - def define_schema(cls): - return IO.Schema( - node_id="StabilityUpscaleCreativeNode", - display_name="Stability AI Upscale Creative", - category="partner/image/Stability AI", - description=cleandoc(cls.__doc__ or ""), - inputs=[ - IO.Image.Input("image"), - IO.String.Input( - "prompt", - multiline=True, - default="", - tooltip="What you wish to see in the output image. A strong, descriptive prompt that clearly defines elements, colors, and subjects will lead to better results.", - ), - IO.Float.Input( - "creativity", - default=0.3, - min=0.1, - max=0.5, - step=0.01, - tooltip="Controls the likelihood of creating additional details not heavily conditioned by the init image.", - ), - IO.Combo.Input( - "style_preset", - options=get_stability_style_presets(), - tooltip="Optional desired style of generated image.", - advanced=True, - ), - IO.Int.Input( - "seed", - default=0, - min=0, - max=4294967294, - step=1, - display_mode=IO.NumberDisplay.number, - control_after_generate=True, - tooltip="The random seed used for creating the noise.", - ), - IO.String.Input( - "negative_prompt", - default="", - tooltip="Keywords of what you do not wish to see in the output image. This is an advanced feature.", - force_input=True, - optional=True, - advanced=True, - ), - ], - outputs=[ - IO.Image.Output(), - ], - hidden=[ - IO.Hidden.auth_token_comfy_org, - IO.Hidden.api_key_comfy_org, - IO.Hidden.unique_id, - ], - is_api_node=True, - price_badge=IO.PriceBadge( - expr="""{"type":"usd","usd":0.6}""", - ), - ) - - @classmethod - async def execute( - cls, - image: torch.Tensor, - prompt: str, - creativity: float, - style_preset: str, - seed: int, - negative_prompt: str = "", - ) -> IO.NodeOutput: - validate_string(prompt, strip_whitespace=False) - image_binary = tensor_to_bytesio(image, total_pixels=1024*1024).read() - - if not negative_prompt: - negative_prompt = None - if style_preset == "None": - style_preset = None - - files = { - "image": image_binary - } - - response_api = await sync_op( - cls, - ApiEndpoint(path="/proxy/stability/v2beta/stable-image/upscale/creative", method="POST"), - response_model=StabilityAsyncResponse, - data=StabilityUpscaleCreativeRequest( - prompt=prompt, - negative_prompt=negative_prompt, - creativity=round(creativity,2), - style_preset=style_preset, - seed=seed, - ), - files=files, - content_type="multipart/form-data", - ) - - response_poll = await poll_op( - cls, - ApiEndpoint(path=f"/proxy/stability/v2beta/results/{response_api.id}"), - response_model=StabilityResultsGetResponse, - poll_interval=3, - status_extractor=lambda x: get_async_dummy_status(x), - ) - - if response_poll.finish_reason != "SUCCESS": - raise Exception(f"Stability Upscale Creative generation failed: {response_poll.finish_reason}.") - - image_data = base64.b64decode(response_poll.result) - returned_image = bytesio_to_image_tensor(BytesIO(image_data)) - - return IO.NodeOutput(returned_image) - - -class StabilityUpscaleFastNode(IO.ComfyNode): - """ - Quickly upscales an image via Stability API call to 4x its original size; intended for upscaling low-quality/compressed images. - """ - - @classmethod - def define_schema(cls): - return IO.Schema( - node_id="StabilityUpscaleFastNode", - display_name="Stability AI Upscale Fast", - category="partner/image/Stability AI", - description=cleandoc(cls.__doc__ or ""), - inputs=[ - IO.Image.Input("image"), - ], - outputs=[ - IO.Image.Output(), - ], - hidden=[ - IO.Hidden.auth_token_comfy_org, - IO.Hidden.api_key_comfy_org, - IO.Hidden.unique_id, - ], - is_api_node=True, - price_badge=IO.PriceBadge( - expr="""{"type":"usd","usd":0.02}""", - ), - ) - - @classmethod - async def execute(cls, image: torch.Tensor) -> IO.NodeOutput: - image_binary = tensor_to_bytesio(image, total_pixels=4096*4096).read() - - files = { - "image": image_binary - } - - response_api = await sync_op( - cls, - ApiEndpoint(path="/proxy/stability/v2beta/stable-image/upscale/fast", method="POST"), - response_model=StabilityStableUltraResponse, - files=files, - content_type="multipart/form-data", - ) - - if response_api.finish_reason != "SUCCESS": - raise Exception(f"Stability Upscale Fast failed: {response_api.finish_reason}.") - - image_data = base64.b64decode(response_api.image) - returned_image = bytesio_to_image_tensor(BytesIO(image_data)) - - return IO.NodeOutput(returned_image) - - -class StabilityTextToAudio(IO.ComfyNode): - """Generates high-quality music and sound effects from text descriptions.""" - - @classmethod - def define_schema(cls): - return IO.Schema( - node_id="StabilityTextToAudio", - display_name="Stability AI Text To Audio", - category="partner/audio/Stability AI", - essentials_category="Audio", - description=cleandoc(cls.__doc__ or ""), - inputs=[ - IO.Combo.Input( - "model", - options=["stable-audio-2.5"], - ), - IO.String.Input("prompt", multiline=True, default=""), - IO.Int.Input( - "duration", - default=190, - min=1, - max=190, - step=1, - tooltip="Controls the duration in seconds of the generated audio.", - optional=True, - ), - IO.Int.Input( - "seed", - default=0, - min=0, - max=4294967294, - step=1, - display_mode=IO.NumberDisplay.number, - control_after_generate=True, - tooltip="The random seed used for generation.", - optional=True, - ), - IO.Int.Input( - "steps", - default=8, - min=4, - max=8, - step=1, - tooltip="Controls the number of sampling steps.", - optional=True, - advanced=True, - ), - ], - outputs=[ - IO.Audio.Output(), - ], - hidden=[ - IO.Hidden.auth_token_comfy_org, - IO.Hidden.api_key_comfy_org, - IO.Hidden.unique_id, - ], - is_api_node=True, - price_badge=IO.PriceBadge( - expr="""{"type":"usd","usd":0.2}""", - ), - ) - - @classmethod - async def execute(cls, model: str, prompt: str, duration: int, seed: int, steps: int) -> IO.NodeOutput: - validate_string(prompt, max_length=10000) - payload = StabilityTextToAudioRequest(prompt=prompt, model=model, duration=duration, seed=seed, steps=steps) - response_api = await sync_op( - cls, - ApiEndpoint(path="/proxy/stability/v2beta/audio/stable-audio-2/text-to-audio", method="POST"), - response_model=StabilityAudioResponse, - data=payload, - content_type="multipart/form-data", - ) - if not response_api.audio: - raise ValueError("No audio file was received in response.") - return IO.NodeOutput(audio_bytes_to_audio_input(base64.b64decode(response_api.audio))) - - -class StabilityAudioToAudio(IO.ComfyNode): - """Transforms existing audio samples into new high-quality compositions using text instructions.""" - - @classmethod - def define_schema(cls): - return IO.Schema( - node_id="StabilityAudioToAudio", - display_name="Stability AI Audio To Audio", - category="partner/audio/Stability AI", - description=cleandoc(cls.__doc__ or ""), - inputs=[ - IO.Combo.Input( - "model", - options=["stable-audio-2.5"], - ), - IO.String.Input("prompt", multiline=True, default=""), - IO.Audio.Input("audio", tooltip="Audio must be between 6 and 190 seconds long."), - IO.Int.Input( - "duration", - default=190, - min=1, - max=190, - step=1, - tooltip="Controls the duration in seconds of the generated audio.", - optional=True, - ), - IO.Int.Input( - "seed", - default=0, - min=0, - max=4294967294, - step=1, - display_mode=IO.NumberDisplay.number, - control_after_generate=True, - tooltip="The random seed used for generation.", - optional=True, - ), - IO.Int.Input( - "steps", - default=8, - min=4, - max=8, - step=1, - tooltip="Controls the number of sampling steps.", - optional=True, - advanced=True, - ), - IO.Float.Input( - "strength", - default=1, - min=0.01, - max=1.0, - step=0.01, - display_mode=IO.NumberDisplay.slider, - tooltip="Parameter controls how much influence the audio parameter has on the generated audio.", - optional=True, - ), - ], - outputs=[ - IO.Audio.Output(), - ], - hidden=[ - IO.Hidden.auth_token_comfy_org, - IO.Hidden.api_key_comfy_org, - IO.Hidden.unique_id, - ], - is_api_node=True, - price_badge=IO.PriceBadge( - expr="""{"type":"usd","usd":0.2}""", - ), - ) - - @classmethod - async def execute( - cls, model: str, prompt: str, audio: Input.Audio, duration: int, seed: int, steps: int, strength: float - ) -> IO.NodeOutput: - validate_string(prompt, max_length=10000) - validate_audio_duration(audio, 6, 190) - payload = StabilityAudioToAudioRequest( - prompt=prompt, model=model, duration=duration, seed=seed, steps=steps, strength=strength - ) - response_api = await sync_op( - cls, - ApiEndpoint(path="/proxy/stability/v2beta/audio/stable-audio-2/audio-to-audio", method="POST"), - response_model=StabilityAudioResponse, - data=payload, - content_type="multipart/form-data", - files={"audio": audio_input_to_mp3(audio)}, - ) - if not response_api.audio: - raise ValueError("No audio file was received in response.") - return IO.NodeOutput(audio_bytes_to_audio_input(base64.b64decode(response_api.audio))) - - -class StabilityAudioInpaint(IO.ComfyNode): - """Transforms part of existing audio sample using text instructions.""" - - @classmethod - def define_schema(cls): - return IO.Schema( - node_id="StabilityAudioInpaint", - display_name="Stability AI Audio Inpaint", - category="partner/audio/Stability AI", - description=cleandoc(cls.__doc__ or ""), - inputs=[ - IO.Combo.Input( - "model", - options=["stable-audio-2.5"], - ), - IO.String.Input("prompt", multiline=True, default=""), - IO.Audio.Input("audio", tooltip="Audio must be between 6 and 190 seconds long."), - IO.Int.Input( - "duration", - default=190, - min=1, - max=190, - step=1, - tooltip="Controls the duration in seconds of the generated audio.", - optional=True, - ), - IO.Int.Input( - "seed", - default=0, - min=0, - max=4294967294, - step=1, - display_mode=IO.NumberDisplay.number, - control_after_generate=True, - tooltip="The random seed used for generation.", - optional=True, - ), - IO.Int.Input( - "steps", - default=8, - min=4, - max=8, - step=1, - tooltip="Controls the number of sampling steps.", - optional=True, - advanced=True, - ), - IO.Int.Input( - "mask_start", - default=30, - min=0, - max=190, - step=1, - optional=True, - advanced=True, - ), - IO.Int.Input( - "mask_end", - default=190, - min=0, - max=190, - step=1, - optional=True, - advanced=True, - ), - ], - outputs=[ - IO.Audio.Output(), - ], - hidden=[ - IO.Hidden.auth_token_comfy_org, - IO.Hidden.api_key_comfy_org, - IO.Hidden.unique_id, - ], - is_api_node=True, - price_badge=IO.PriceBadge( - expr="""{"type":"usd","usd":0.2}""", - ), - ) - - @classmethod - async def execute( - cls, - model: str, - prompt: str, - audio: Input.Audio, - duration: int, - seed: int, - steps: int, - mask_start: int, - mask_end: int, - ) -> IO.NodeOutput: - validate_string(prompt, max_length=10000) - if mask_end <= mask_start: - raise ValueError(f"Value of mask_end({mask_end}) should be greater then mask_start({mask_start})") - validate_audio_duration(audio, 6, 190) - - payload = StabilityAudioInpaintRequest( - prompt=prompt, - model=model, - duration=duration, - seed=seed, - steps=steps, - mask_start=mask_start, - mask_end=mask_end, - ) - response_api = await sync_op( - cls, - endpoint=ApiEndpoint(path="/proxy/stability/v2beta/audio/stable-audio-2/inpaint", method="POST"), - response_model=StabilityAudioResponse, - data=payload, - content_type="multipart/form-data", - files={"audio": audio_input_to_mp3(audio)}, - ) - if not response_api.audio: - raise ValueError("No audio file was received in response.") - return IO.NodeOutput(audio_bytes_to_audio_input(base64.b64decode(response_api.audio))) - - -class StabilityExtension(ComfyExtension): - @override - async def get_node_list(self) -> list[type[IO.ComfyNode]]: - return [ - StabilityStableImageUltraNode, - StabilityStableImageSD_3_5Node, - StabilityUpscaleConservativeNode, - StabilityUpscaleCreativeNode, - StabilityUpscaleFastNode, - StabilityTextToAudio, - StabilityAudioToAudio, - StabilityAudioInpaint, - ] - - -async def comfy_entrypoint() -> StabilityExtension: - return StabilityExtension() diff --git a/comfy_api_nodes/nodes_tripo.py b/comfy_api_nodes/nodes_tripo.py index a3f2cb053..228fe8a1d 100644 --- a/comfy_api_nodes/nodes_tripo.py +++ b/comfy_api_nodes/nodes_tripo.py @@ -1,6 +1,6 @@ from typing_extensions import override -from comfy_api.latest import IO, ComfyExtension, Input +from comfy_api.latest import IO, ComfyExtension, Input, Types from comfy_api_nodes.apis.tripo import ( TripoAnimateRetargetRequest, TripoAnimateRigRequest, @@ -8,6 +8,7 @@ from comfy_api_nodes.apis.tripo import ( TripoFileEmptyReference, TripoFileReference, TripoImageToModelRequest, + TripoImportModelRequest, TripoModelVersion, TripoMultiviewToModelRequest, TripoOrientation, @@ -21,6 +22,7 @@ from comfy_api_nodes.apis.tripo import ( TripoTaskType, TripoTextToModelRequest, TripoTextureModelRequest, + TripoTexturePrompt, TripoUrlReference, ) from comfy_api_nodes.util import ( @@ -28,6 +30,7 @@ from comfy_api_nodes.util import ( download_url_to_file_3d, poll_op, sync_op, + upload_3d_model_to_comfyapi, upload_images_to_comfyapi, ) @@ -538,6 +541,14 @@ class TripoTextureNode(IO.ComfyNode): optional=True, advanced=True, ), + IO.String.Input( + "texture_prompt", + default="", + multiline=True, + optional=True, + tooltip="Optional text guidance for texturing. Required in practice for imported " + "models (Tripo: Import Model), which carry no source image to infer colors from.", + ), ], outputs=[ IO.String.Output(display_name="model_file"), # for backward compatibility only @@ -571,6 +582,7 @@ class TripoTextureNode(IO.ComfyNode): texture_seed: int | None = None, texture_quality: str | None = None, texture_alignment: str | None = None, + texture_prompt: str = "", ) -> IO.NodeOutput: response = await sync_op( cls, @@ -583,6 +595,7 @@ class TripoTextureNode(IO.ComfyNode): texture_seed=texture_seed, texture_quality=texture_quality, texture_alignment=texture_alignment, + texture_prompt=TripoTexturePrompt(text=texture_prompt.strip()) if texture_prompt.strip() else None, ), ) return await poll_until_finished(cls, response, average_duration=80) @@ -915,6 +928,90 @@ class TripoConversionNode(IO.ComfyNode): return await poll_until_finished(cls, response, average_duration=30) +class TripoImportModelNode(IO.ComfyNode): + """Imports an external 3D model into Tripo, producing a MODEL_TASK_ID for post-processing nodes.""" + + SUPPORTED_FORMATS = ("glb", "fbx", "obj", "stl") + + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="TripoImportModelNode", + display_name="Tripo: Import Model", + category="partner/3d/Tripo", + description="Import an external 3D model (e.g. from Rodin, Hunyuan3D or a local file) into Tripo " + "to use it with Tripo's post-processing nodes: Texture, Rig, Convert. " + "GLB is recommended: textures survive import only when embedded in the file. " + "Note that texturing an imported model requires a texture prompt.", + inputs=[ + IO.MultiType.Input( + "model_3d", + types=[IO.File3DGLB, IO.File3DFBX, IO.File3DOBJ, IO.File3DSTL, IO.File3DAny], + tooltip="3D model to import (GLB / FBX / OBJ / STL, up to 150 MB). " + "OBJ and STL files carry no embedded textures.", + ), + ], + outputs=[ + IO.Custom("MODEL_TASK_ID").Output(display_name="model task_id"), + ], + hidden=[ + IO.Hidden.auth_token_comfy_org, + IO.Hidden.api_key_comfy_org, + IO.Hidden.unique_id, + ], + is_api_node=True, + price_badge=IO.PriceBadge( + expr="""{"type":"text","text":"Free"}""", + ), + ) + + @classmethod + async def execute(cls, model_3d: Types.File3D) -> IO.NodeOutput: + file_format = (model_3d.format or "").lstrip(".").lower() + if file_format == "gltf": + raise ValueError( + "GLTF (.gltf) references external files and cannot be imported. Export a single-file GLB instead." + ) + if file_format not in cls.SUPPORTED_FORMATS: + raise ValueError( + f"Unsupported 3D format '{file_format or 'unknown'}'. " + f"Tripo import supports: {', '.join(f.upper() for f in cls.SUPPORTED_FORMATS)}." + ) + size = len(model_3d.get_bytes()) + if size > 150 * 1024 * 1024: + raise ValueError(f"Model file is {size / (1024 * 1024):.1f} MB; Tripo import allows up to 150 MB.") + + url = await upload_3d_model_to_comfyapi(cls, model_3d, file_format) + response = await sync_op( + cls, + endpoint=ApiEndpoint(path="/proxy/tripo/v2/openapi/import", method="POST"), + response_model=TripoTaskResponse, + data=TripoImportModelRequest(url=url, format=file_format), + ) + if response.code != 0: + raise RuntimeError(f"Failed to import model: {response.error}") + + task_id = response.data.task_id + response_poll = await poll_op( + cls, + poll_endpoint=ApiEndpoint(path=f"/proxy/tripo/v2/openapi/task/{task_id}"), + response_model=TripoTaskResponse, + failed_statuses=[ + TripoTaskStatus.FAILED, + TripoTaskStatus.CANCELLED, + TripoTaskStatus.UNKNOWN, + TripoTaskStatus.BANNED, + TripoTaskStatus.EXPIRED, + ], + status_extractor=lambda x: x.data.status, + progress_extractor=lambda x: x.data.progress, + estimated_duration=10, + ) + if response_poll.data.status != TripoTaskStatus.SUCCESS: + raise RuntimeError(f"Failed to import model: {response_poll}") + return IO.NodeOutput(task_id) + + def _p1_price_expr(*, geometry_credits: int, textured_credits: int, detailed_credits: int) -> str: return ( "(" @@ -1292,6 +1389,7 @@ class TripoExtension(ComfyExtension): TripoP1TextToModelNode, TripoP1ImageToModelNode, TripoP1MultiviewToModelNode, + TripoImportModelNode, TripoTextureNode, TripoRefineNode, TripoRigNode, diff --git a/comfy_api_nodes/nodes_wan.py b/comfy_api_nodes/nodes_wan.py index b7b97d70f..1782739fd 100644 --- a/comfy_api_nodes/nodes_wan.py +++ b/comfy_api_nodes/nodes_wan.py @@ -48,10 +48,13 @@ from comfy_api_nodes.util import ( upload_image_to_comfyapi, upload_video_to_comfyapi, validate_audio_duration, + validate_image_aspect_ratio, + validate_image_dimensions, validate_string, validate_video_duration, ) + RES_IN_PARENS = re.compile(r"\((\d+)\s*[x×]\s*(\d+)\)") @@ -1657,6 +1660,44 @@ class HappyHorseTextToVideoApi(IO.ComfyNode): IO.DynamicCombo.Input( "model", options=[ + IO.DynamicCombo.Option( + "happyhorse-1.1-t2v", + [ + IO.String.Input( + "prompt", + multiline=True, + default="", + tooltip="Prompt describing the elements and visual features. " + "Supports English and Chinese.", + ), + IO.Combo.Input( + "resolution", + options=["720P", "1080P"], + ), + IO.Combo.Input( + "ratio", + options=[ + "16:9", + "9:16", + "1:1", + "4:3", + "3:4", + "21:9", + "9:21", + "5:4", + "4:5", + ], + ), + IO.Int.Input( + "duration", + default=5, + min=3, + max=15, + step=1, + display_mode=IO.NumberDisplay.number, + ), + ], + ), IO.DynamicCombo.Option( "happyhorse-1.0-t2v", [ @@ -1719,7 +1760,9 @@ class HappyHorseTextToVideoApi(IO.ComfyNode): ( $res := $lookup(widgets, "model.resolution"); $dur := $lookup(widgets, "model.duration"); - $ppsTable := { "720p": 0.14, "1080p": 0.24 }; + $ppsTable := $contains(widgets.model, "1.1") + ? { "720p": 0.2002, "1080p": 0.2574 } + : { "720p": 0.14, "1080p": 0.24 }; $pps := $lookup($ppsTable, $res); { "type": "usd", "usd": $pps * $dur } ) @@ -1781,6 +1824,30 @@ class HappyHorseImageToVideoApi(IO.ComfyNode): IO.DynamicCombo.Input( "model", options=[ + IO.DynamicCombo.Option( + "happyhorse-1.1-i2v", + [ + IO.String.Input( + "prompt", + multiline=True, + default="", + tooltip="Prompt describing the elements and visual features. " + "Supports English and Chinese.", + ), + IO.Combo.Input( + "resolution", + options=["720P", "1080P"], + ), + IO.Int.Input( + "duration", + default=5, + min=3, + max=15, + step=1, + display_mode=IO.NumberDisplay.number, + ), + ], + ), IO.DynamicCombo.Option( "happyhorse-1.0-i2v", [ @@ -1843,7 +1910,9 @@ class HappyHorseImageToVideoApi(IO.ComfyNode): ( $res := $lookup(widgets, "model.resolution"); $dur := $lookup(widgets, "model.duration"); - $ppsTable := { "720p": 0.14, "1080p": 0.24 }; + $ppsTable := $contains(widgets.model, "1.1") + ? { "720p": 0.2002, "1080p": 0.2574 } + : { "720p": 0.14, "1080p": 0.24 }; $pps := $lookup($ppsTable, $res); { "type": "usd", "usd": $pps * $dur } ) @@ -1859,6 +1928,8 @@ class HappyHorseImageToVideoApi(IO.ComfyNode): seed: int, watermark: bool, ): + validate_image_dimensions(first_frame, min_width=300, min_height=300) + validate_image_aspect_ratio(first_frame, (1, 2.5), (2.5, 1), strict=False) media = [ Wan27MediaItem( type="first_frame", @@ -2053,6 +2124,62 @@ class HappyHorseReferenceVideoApi(IO.ComfyNode): IO.DynamicCombo.Input( "model", options=[ + IO.DynamicCombo.Option( + "happyhorse-1.1-r2v", + [ + IO.String.Input( + "prompt", + multiline=True, + default="", + tooltip="Prompt describing the video. Use identifiers such as 'character1' and " + "'character2' to refer to the reference characters.", + ), + IO.Combo.Input( + "resolution", + options=["720P", "1080P"], + ), + IO.Combo.Input( + "ratio", + options=[ + "16:9", + "9:16", + "1:1", + "4:3", + "3:4", + "21:9", + "9:21", + "5:4", + "4:5", + ], + ), + IO.Int.Input( + "duration", + default=5, + min=3, + max=15, + step=1, + display_mode=IO.NumberDisplay.number, + ), + IO.Autogrow.Input( + "reference_images", + template=IO.Autogrow.TemplateNames( + IO.Image.Input("reference_image"), + names=[ + "image1", + "image2", + "image3", + "image4", + "image5", + "image6", + "image7", + "image8", + "image9", + ], + min=1, + ), + ), + ], + ), IO.DynamicCombo.Option( "happyhorse-1.0-r2v", [ @@ -2133,7 +2260,9 @@ class HappyHorseReferenceVideoApi(IO.ComfyNode): ( $res := $lookup(widgets, "model.resolution"); $dur := $lookup(widgets, "model.duration"); - $ppsTable := { "720p": 0.14, "1080p": 0.24 }; + $ppsTable := $contains(widgets.model, "1.1") + ? { "720p": 0.2002, "1080p": 0.2574 } + : { "720p": 0.14, "1080p": 0.24 }; $pps := $lookup($ppsTable, $res); { "type": "usd", "usd": $pps * $dur } ) @@ -2149,8 +2278,11 @@ class HappyHorseReferenceVideoApi(IO.ComfyNode): watermark: bool, ): validate_string(model["prompt"], strip_whitespace=False, min_length=1) - media = [] reference_images = model.get("reference_images", {}) + for key in reference_images: + validate_image_dimensions(reference_images[key], min_width=400, min_height=400) + validate_image_aspect_ratio(reference_images[key], (1, 2.5), (2.5, 1), strict=False) + media = [] for key in reference_images: media.append( Wan27MediaItem( @@ -2159,7 +2291,7 @@ class HappyHorseReferenceVideoApi(IO.ComfyNode): ) ) if not media: - raise ValueError("At least one reference reference image must be provided.") + raise ValueError("At least one reference image must be provided.") initial_response = await sync_op( cls, diff --git a/comfy_api_nodes/util/__init__.py b/comfy_api_nodes/util/__init__.py index 25cb88869..1fb6b96cf 100644 --- a/comfy_api_nodes/util/__init__.py +++ b/comfy_api_nodes/util/__init__.py @@ -26,6 +26,7 @@ from .conversions import ( text_filepath_to_base64_string, text_filepath_to_data_uri, trim_video, + upscale_image_tensor_to_min_pixels, upscale_video_to_min_pixels, video_to_base64_string, ) @@ -99,6 +100,7 @@ __all__ = [ "text_filepath_to_base64_string", "text_filepath_to_data_uri", "trim_video", + "upscale_image_tensor_to_min_pixels", "upscale_video_to_min_pixels", "video_to_base64_string", # Validation utilities diff --git a/comfy_api_nodes/util/_helpers.py b/comfy_api_nodes/util/_helpers.py index 648defe3d..6b8121cab 100644 --- a/comfy_api_nodes/util/_helpers.py +++ b/comfy_api_nodes/util/_helpers.py @@ -4,11 +4,14 @@ import os import re import time from collections.abc import Callable +from datetime import datetime, timezone +from email.utils import parsedate_to_datetime from io import BytesIO from yarl import URL from comfy.cli_args import args +from comfy.deploy_environment import get_deploy_environment from comfy.model_management import processing_interrupted from comfy_api.latest import IO @@ -35,6 +38,30 @@ def get_auth_header(node_cls: type[IO.ComfyNode]) -> dict[str, str]: return {} +def get_usage_source(node_cls: type[IO.ComfyNode]) -> str: + """Source of the prompt that triggered this API node. + + Defaults to "comfyui-api" when the submitting client didn't identify itself, + i.e. a direct API call to this server. + """ + return node_cls.hidden.comfy_usage_source or "comfyui-api" + + +def get_comfy_api_headers(node_cls: type[IO.ComfyNode]) -> dict[str, str]: + """Common headers (auth, deploy environment, usage source) for Comfy API requests. + + Centralizes the shared header set so every Comfy API request sends a consistent + set and new shared headers only need to be added in one place. Intended for + relative/cloud URLs resolved against ``default_base_url()``; because the result + includes auth, callers must not attach it to arbitrary absolute/presigned URLs. + """ + return { + **get_auth_header(node_cls), + "Comfy-Env": get_deploy_environment(), + "Comfy-Usage-Source": get_usage_source(node_cls), + } + + def default_base_url() -> str: return getattr(args, "comfy_api_base", "https://api.comfy.org") @@ -66,6 +93,32 @@ async def sleep_with_interrupt( await asyncio.sleep(min(1.0, end - now)) +def _retry_after_wait(value: str | None, fallback: float, max_wait: float) -> float: + """Delay before the next retry, honoring a server ``Retry-After`` header.""" + + seconds: float | None = None + if value is not None: + value = value.strip() + if value.isascii() and value.isdigit(): + # delay-seconds form. The ASCII-digit guard keeps exotic Unicode "digit" characters away from float() + # an all-digit string always converts (huge values become inf, never raising). + seconds = float(value) + elif value: + # HTTP-date form. parsedate_to_datetime raises OverflowError (not a ValueError) on absurd years/offsets + try: + parsed = parsedate_to_datetime(value) + except (TypeError, ValueError, OverflowError): + parsed = None + if parsed is not None: + if parsed.tzinfo is None: # naive datetime: HTTP-date is UTC + parsed = parsed.replace(tzinfo=timezone.utc) + delta = (parsed - datetime.now(timezone.utc)).total_seconds() + seconds = delta if delta > 0 else 0.0 + if seconds is None: + return fallback + return min(seconds, max_wait) + + def mimetype_to_extension(mime_type: str) -> str: """Converts a MIME type to a file extension.""" return mime_type.split("/")[-1].lower() diff --git a/comfy_api_nodes/util/client.py b/comfy_api_nodes/util/client.py index 57c501724..66aab17f8 100644 --- a/comfy_api_nodes/util/client.py +++ b/comfy_api_nodes/util/client.py @@ -19,12 +19,11 @@ from comfy import utils from comfy_api.latest import IO from server import PromptServer -from comfy.deploy_environment import get_deploy_environment - from . import request_logger from ._helpers import ( + _retry_after_wait, default_base_url, - get_auth_header, + get_comfy_api_headers, get_node_id, is_processing_interrupted, sleep_with_interrupt, @@ -84,6 +83,7 @@ class _PollUIState: _RETRY_STATUS = {408, 500, 502, 503, 504} # status 429 is handled separately +_MAX_RETRY_AFTER_WAIT = 150.0 # Cap a server Retry-After at this many seconds so a large hint can't block execution COMPLETED_STATUSES = ["succeeded", "succeed", "success", "completed", "finished", "done", "complete"] FAILED_STATUSES = ["cancelled", "canceled", "canceling", "fail", "failed", "error"] QUEUED_STATUSES = ["created", "queued", "queueing", "submitted", "initializing", "wait", "in_queue"] @@ -645,8 +645,7 @@ async def _request_base(cfg: _RequestConfig, expect_binary: bool): payload_headers = {"Accept": "*/*"} if expect_binary else {"Accept": "application/json"} if not parsed_url.scheme and not parsed_url.netloc: # is URL relative? - payload_headers.update(get_auth_header(cfg.node_cls)) - payload_headers["Comfy-Env"] = get_deploy_environment() + payload_headers.update(get_comfy_api_headers(cfg.node_cls)) if cfg.endpoint.headers: payload_headers.update(cfg.endpoint.headers) @@ -750,6 +749,7 @@ async def _request_base(cfg: _RequestConfig, expect_binary: bool): should_retry = True if should_retry: + wait_time = _retry_after_wait(resp.headers.get("Retry-After"), wait_time, _MAX_RETRY_AFTER_WAIT) logging.warning( "HTTP %s %s -> %s. Waiting %.2fs (%s).", method, diff --git a/comfy_api_nodes/util/conversions.py b/comfy_api_nodes/util/conversions.py index a1b5d599c..9cd644fc0 100644 --- a/comfy_api_nodes/util/conversions.py +++ b/comfy_api_nodes/util/conversions.py @@ -448,6 +448,15 @@ def _compute_upscale_dims(src_w: int, src_h: int, total_pixels: int) -> tuple[in return new_w, new_h +def upscale_image_tensor_to_min_pixels(image: torch.Tensor, total_pixels: int) -> torch.Tensor: + samples = image.movedim(-1, 1) + dims = _compute_upscale_dims(samples.shape[3], samples.shape[2], int(total_pixels)) + if dims is None: + return image + new_w, new_h = dims + return common_upscale(samples, new_w, new_h, "lanczos", "disabled").movedim(1, -1) + + def upscale_video_to_min_pixels(video: Input.Video, min_pixels: int) -> Input.Video: """Upscale a video to meet at least ``min_pixels`` (w * h), preserving aspect ratio. diff --git a/comfy_api_nodes/util/download_helpers.py b/comfy_api_nodes/util/download_helpers.py index aa588d038..0ec3c6e66 100644 --- a/comfy_api_nodes/util/download_helpers.py +++ b/comfy_api_nodes/util/download_helpers.py @@ -17,7 +17,7 @@ from folder_paths import get_output_directory from . import request_logger from ._helpers import ( default_base_url, - get_auth_header, + get_comfy_api_headers, is_processing_interrupted, sleep_with_interrupt, to_aiohttp_url, @@ -64,7 +64,7 @@ async def download_url_to_bytesio( if cls is None: raise ValueError("For relative 'cloud' paths, the `cls` parameter is required.") url = urljoin(default_base_url().rstrip("/") + "/", url.lstrip("/")) - headers = get_auth_header(cls) + headers = get_comfy_api_headers(cls) while True: attempt += 1 diff --git a/comfy_api_nodes/util/request_logger.py b/comfy_api_nodes/util/request_logger.py index fe0543d9b..70ecaf41a 100644 --- a/comfy_api_nodes/util/request_logger.py +++ b/comfy_api_nodes/util/request_logger.py @@ -9,6 +9,7 @@ from typing import Any import folder_paths logger = logging.getLogger(__name__) +_SENSITIVE_HEADERS = {"authorization", "x-api-key"} def get_log_directory(): @@ -73,6 +74,10 @@ def _format_data_for_logging(data: Any) -> str: return str(data) +def _redact_headers(headers: dict) -> dict: + return {k: ("***" if k.lower() in _SENSITIVE_HEADERS else v) for k, v in headers.items()} + + def log_request_response( operation_id: str, request_method: str, @@ -101,7 +106,7 @@ def log_request_response( log_content.append(f"Method: {request_method}") log_content.append(f"URL: {request_url}") if request_headers: - log_content.append(f"Headers:\n{_format_data_for_logging(request_headers)}") + log_content.append(f"Headers:\n{_format_data_for_logging(_redact_headers(request_headers))}") if request_params: log_content.append(f"Params:\n{_format_data_for_logging(request_params)}") if request_data is not None: diff --git a/comfy_api_nodes/util/upload_helpers.py b/comfy_api_nodes/util/upload_helpers.py index 6d1d107a1..f7029ee78 100644 --- a/comfy_api_nodes/util/upload_helpers.py +++ b/comfy_api_nodes/util/upload_helpers.py @@ -158,7 +158,14 @@ async def upload_video_to_comfyapi( # Convert VideoInput to BytesIO using specified container/codec video_bytes_io = BytesIO() - video.save_to(video_bytes_io, format=container, codec=codec) + try: + video.save_to(video_bytes_io, format=container, codec=codec) + except Exception as e: + raise ValueError( + f"Could not convert the input video to {container.value.upper()} for upload; " + f"the file may be corrupted or use an unsupported codec. " + f"Try re-exporting it as MP4 (H.264). Original error: {e}" + ) from e video_bytes_io.seek(0) return await upload_file_to_comfyapi(cls, video_bytes_io, filename, upload_mime_type, wait_label) diff --git a/comfy_execution/asset_enrichment.py b/comfy_execution/asset_enrichment.py new file mode 100644 index 000000000..38e9496a8 --- /dev/null +++ b/comfy_execution/asset_enrichment.py @@ -0,0 +1,66 @@ +"""Enrich executed-node output entries with asset id.""" +import logging +import os + + +def enrich_output_with_assets(output_ui: dict) -> dict: + """Register file-type output entries as assets and inject their ``id``. + + Runs at output-processing time, once per produced output, when + --enable-assets is set. Returns a new dict; entries without a resolvable + on-disk file path are left unchanged. Errors are caught per-entry so a + failure never blocks execution or the other entries. + """ + from comfy.cli_args import args + if not args.enable_assets: + return output_ui + + import folder_paths + from app.assets.services.ingest import register_file_in_place, DependencyMissingError + + enriched = {} + for key, entries in output_ui.items(): + if not isinstance(entries, list): + enriched[key] = entries + continue + new_entries = [] + for entry in entries: + if not isinstance(entry, dict) or "filename" not in entry or "type" not in entry: + new_entries.append(entry) + continue + try: + base = folder_paths.get_directory_by_type(entry["type"]) + if base is None: + new_entries.append(entry) + continue + base_abs = os.path.abspath(base) + abs_path = os.path.abspath(os.path.join(base_abs, entry.get("subfolder") or "", entry["filename"])) + try: + if os.path.commonpath([base_abs, abs_path]) != base_abs: + raise ValueError("escapes base") + except ValueError: + logging.warning("Asset enrichment skipped (path escapes base): %s", entry.get("filename")) + new_entries.append(entry) + continue + if not os.path.isfile(abs_path): + new_entries.append(entry) + continue + + # Register unconditionally: the file was just produced, and + # register_file_in_place re-hashes so an overwritten path can + # never carry a stale id. + result = register_file_in_place( + abs_path=abs_path, + name=entry["filename"], + tags=[entry["type"]], + ) + + entry = dict(entry) + entry["id"] = result.ref.id + except DependencyMissingError: + logging.warning("Asset enrichment skipped (blake3 not available): %s", entry.get("filename")) + except Exception: + logging.warning("Failed to enrich output entry with asset id: %s", entry.get("filename"), exc_info=True) + new_entries.append(entry) + enriched[key] = new_entries + return enriched diff --git a/comfy_execution/caching.py b/comfy_execution/caching.py index ba1e8bc84..ad75a0e50 100644 --- a/comfy_execution/caching.py +++ b/comfy_execution/caching.py @@ -503,6 +503,21 @@ RAM_CACHE_DEFAULT_RAM_USAGE = 0.05 RAM_CACHE_OLD_WORKFLOW_OOM_MULTIPLIER = 1.3 + +def all_outputs_dynamic(outputs): + if outputs is None: + return False + + for output in outputs: + if isinstance(output, (list, tuple)): + if not all_outputs_dynamic(output): + return False + elif not hasattr(output, "is_dynamic") or not output.is_dynamic(): + return False + + return True + + class RAMPressureCache(LRUCache): def __init__(self, key_class, enable_providers=False): @@ -533,7 +548,11 @@ class RAMPressureCache(LRUCache): for key, cache_entry in self.cache.items(): if not free_active and self.used_generation[key] == self.generation: continue - oom_score = RAM_CACHE_OLD_WORKFLOW_OOM_MULTIPLIER ** (self.generation - self.used_generation[key]) + + if all_outputs_dynamic(cache_entry.outputs) and self.used_generation[key] == self.generation: + continue + + oom_score = RAM_CACHE_OLD_WORKFLOW_OOM_MULTIPLIER ** (self.generation - self.used_generation[key]) ram_usage = RAM_CACHE_DEFAULT_RAM_USAGE def scan_list_for_ram_usage(outputs): diff --git a/comfy_execution/jobs.py b/comfy_execution/jobs.py index fcd7ef735..fa3ab0faf 100644 --- a/comfy_execution/jobs.py +++ b/comfy_execution/jobs.py @@ -3,11 +3,23 @@ Job utilities for the /api/jobs endpoint. Provides normalization and helper functions for job status tracking. """ -from typing import Optional +import uuid +from typing import Callable, Optional from comfy_api.internal import prune_dict +# Result of classifying a job for cancellation. +# 'running' -> job is currently executing (interrupt it) +# 'pending' -> job is queued but not started (dequeue it) +# 'terminal' -> job already finished (present in history); cancel is a no-op +# 'unknown' -> job id is not present anywhere +CANCEL_RUNNING = 'running' +CANCEL_PENDING = 'pending' +CANCEL_TERMINAL = 'terminal' +CANCEL_UNKNOWN = 'unknown' + + class JobStatus: """Job status constants.""" PENDING = 'pending' @@ -19,6 +31,25 @@ class JobStatus: ALL = [PENDING, IN_PROGRESS, COMPLETED, FAILED, CANCELLED] +def validate_job_id(value) -> str: + """Validate a client-supplied job (prompt) id. + + Job ids must be UUIDs in the canonical lowercase hyphenated form. The id + is stored and compared verbatim everywhere downstream — history keys, + websocket events, and /interrupt matching — so accepting another spelling + would silently rewrite the client's id and then miss every exact-match + lookup. Rejecting loudly beats that. + + Returns the id unchanged. Raises ValueError when the value is not a + string in canonical UUID form. + """ + if not isinstance(value, str): + raise ValueError(f"job id must be a string, got {type(value).__name__}") + if str(uuid.UUID(value)) != value: + raise ValueError("job id must be a UUID in canonical lowercase hyphenated form") + return value + + # Media types that can be previewed in the frontend PREVIEWABLE_MEDIA_TYPES = frozenset({'images', 'video', 'audio', '3d', 'text'}) @@ -387,3 +418,71 @@ def get_all_jobs( jobs = jobs[:limit] return (jobs, total_count) + + +def classify_job_for_cancel(prompt_id: str, running: list, queued: list, history: dict) -> str: + """Classify a job id for cancellation. + + Returns one of CANCEL_RUNNING, CANCEL_PENDING, CANCEL_TERMINAL, CANCEL_UNKNOWN. + + Queue items are tuples whose second element (index 1) is the prompt_id. + History is a dict keyed by prompt_id, so a job present there has already + finished and cancelling it is a no-op. + """ + for item in running: + if item[1] == prompt_id: + return CANCEL_RUNNING + for item in queued: + if item[1] == prompt_id: + return CANCEL_PENDING + if prompt_id in history: + return CANCEL_TERMINAL + return CANCEL_UNKNOWN + + +def cancel_job( + prompt_id: str, + running: list, + queued: list, + history: dict, + interrupt: Callable[[str], bool], + dequeue: Callable[[str], bool], +) -> str: + """Cancel a single job by id, regardless of state. + + Maps the cancel onto the runtime's existing mechanics: + - a running job is interrupted via ``interrupt`` + - a pending job is removed from the queue via ``dequeue`` + - a job that already finished (terminal) is a no-op + - an unknown id is a no-op (callers that need fail-fast behaviour should + validate ids up front with ``classify_job_for_cancel``) + + Both ``interrupt`` and ``dequeue`` take the prompt id and return whether + they acted on a job that was *actually* in that state, so the value returned + here reflects what truly happened rather than the (possibly stale) + classification. This matters around the narrow TOCTOU windows where a job + changes state between the caller's snapshot and the action: + + - a job classified RUNNING may have finished before ``interrupt`` fires: + ``interrupt`` returns False and this returns CANCEL_UNKNOWN (no-op). + - a job classified PENDING may have started executing before ``dequeue`` + fires: ``dequeue`` returns False, ``interrupt`` then catches the now- + running job and this returns CANCEL_RUNNING. If it had simply finished + instead, both return False and this returns CANCEL_UNKNOWN. + + ``interrupt`` must be atomic — interrupt the job only if it is still the one + running — so a cancel can never land on an unrelated prompt that started in + the meantime (see ``execution.PromptQueue.interrupt_if_running``). + """ + classification = classify_job_for_cancel(prompt_id, running, queued, history) + if classification == CANCEL_RUNNING: + return CANCEL_RUNNING if interrupt(prompt_id) else CANCEL_UNKNOWN + if classification == CANCEL_PENDING: + if dequeue(prompt_id): + return CANCEL_PENDING + # Left the pending queue between classification and dequeue: if it + # started executing, interrupt the now-running job; otherwise it has + # already finished and the cancel is a genuine no-op. + return CANCEL_RUNNING if interrupt(prompt_id) else CANCEL_UNKNOWN + # CANCEL_TERMINAL and CANCEL_UNKNOWN are intentional no-ops. + return classification diff --git a/comfy_extras/color_util.py b/comfy_extras/color_util.py new file mode 100644 index 000000000..d50795ae3 --- /dev/null +++ b/comfy_extras/color_util.py @@ -0,0 +1,23 @@ +def hex_to_rgb(value: str) -> tuple[int, int, int]: + h = value.lstrip("#") + if len(h) != 6: + return (255, 255, 255) + try: + return (int(h[0:2], 16), int(h[2:4], 16), int(h[4:6], 16)) + except ValueError: + return (255, 255, 255) + + +def readable_color(rgb: tuple[int, int, int]) -> tuple[int, int, int]: + r, g, b = rgb + lum = 0.299 * r + 0.587 * g + 0.114 * b + if lum >= 130: + return (r, g, b) + t = (130 - lum) / (255 - lum) + return (round(r + (255 - r) * t), round(g + (255 - g) * t), round(b + (255 - b) * t)) + + +def normalize_palette(colors) -> list[str]: + if isinstance(colors, dict): + colors = colors.values() + return [c.upper() for c in colors if isinstance(c, str) and c] diff --git a/comfy_extras/nodes_ace.py b/comfy_extras/nodes_ace.py index 044077b18..eaf234d5b 100644 --- a/comfy_extras/nodes_ace.py +++ b/comfy_extras/nodes_ace.py @@ -11,7 +11,7 @@ class TextEncodeAceStepAudio(IO.ComfyNode): def define_schema(cls): return IO.Schema( node_id="TextEncodeAceStepAudio", - category="model/conditioning", + category="model/conditioning/ace", inputs=[ IO.Clip.Input("clip"), IO.String.Input("tags", multiline=True, dynamic_prompts=True), @@ -33,7 +33,7 @@ class TextEncodeAceStepAudio15(IO.ComfyNode): def define_schema(cls): return IO.Schema( node_id="TextEncodeAceStepAudio1.5", - category="model/conditioning", + category="model/conditioning/ace", inputs=[ IO.Clip.Input("clip"), IO.String.Input("tags", multiline=True, dynamic_prompts=True), @@ -67,7 +67,7 @@ class EmptyAceStepLatentAudio(IO.ComfyNode): return IO.Schema( node_id="EmptyAceStepLatentAudio", display_name="Empty Ace Step 1.0 Latent Audio", - category="model/latent/audio", + category="model/latent/ace", inputs=[ IO.Float.Input("seconds", default=120.0, min=1.0, max=1000.0, step=0.1), IO.Int.Input( @@ -90,7 +90,7 @@ class EmptyAceStep15LatentAudio(IO.ComfyNode): return IO.Schema( node_id="EmptyAceStep1.5LatentAudio", display_name="Empty Ace Step 1.5 Latent Audio", - category="model/latent/audio", + category="model/latent/ace", inputs=[ IO.Float.Input("seconds", default=120.0, min=1.0, max=1000.0, step=0.01), IO.Int.Input( @@ -111,8 +111,8 @@ class ReferenceAudio(IO.ComfyNode): def define_schema(cls): return IO.Schema( node_id="ReferenceTimbreAudio", - display_name="Reference Audio", - category="advanced/conditioning/audio", + display_name="Set Reference Audio", + category="model/conditioning", is_experimental=True, description="This node sets the reference audio for ace step 1.5", inputs=[ diff --git a/comfy_extras/nodes_apg.py b/comfy_extras/nodes_apg.py index 4a352038a..6e69b73f7 100644 --- a/comfy_extras/nodes_apg.py +++ b/comfy_extras/nodes_apg.py @@ -16,7 +16,7 @@ class APG(io.ComfyNode): return io.Schema( node_id="APG", display_name="Adaptive Projected Guidance", - category="model/sampling/custom_sampling", + category="model/sampling/custom", inputs=[ io.Model.Input("model"), io.Float.Input( diff --git a/comfy_extras/nodes_ar_video.py b/comfy_extras/nodes_ar_video.py index c22359eb2..9d8f64b20 100644 --- a/comfy_extras/nodes_ar_video.py +++ b/comfy_extras/nodes_ar_video.py @@ -19,7 +19,7 @@ class EmptyARVideoLatent(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="EmptyARVideoLatent", - category="model/latent/video", + category="model/latent/autoregressive", inputs=[ io.Int.Input("width", default=832, min=16, max=8192, step=16), io.Int.Input("height", default=480, min=16, max=8192, step=16), @@ -85,7 +85,7 @@ class ARVideoI2V(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="ARVideoI2V", - category="model/conditioning/video_models", + category="model/conditioning/autoregressive", inputs=[ io.Model.Input("model"), io.Vae.Input("vae"), diff --git a/comfy_extras/nodes_audio.py b/comfy_extras/nodes_audio.py index 1dc97ecd7..6adcc95fa 100644 --- a/comfy_extras/nodes_audio.py +++ b/comfy_extras/nodes_audio.py @@ -16,7 +16,7 @@ class EmptyLatentAudio(IO.ComfyNode): return IO.Schema( node_id="EmptyLatentAudio", display_name="Empty Latent Audio", - category="model/latent/audio", + category="model/latent", essentials_category="Audio", inputs=[ IO.Float.Input("seconds", default=47.6, min=1.0, max=1000.0, step=0.1), @@ -41,7 +41,7 @@ class ConditioningStableAudio(IO.ComfyNode): def define_schema(cls): return IO.Schema( node_id="ConditioningStableAudio", - category="model/conditioning", + category="model/conditioning/stable audio", inputs=[ IO.Conditioning.Input("positive"), IO.Conditioning.Input("negative"), @@ -70,7 +70,7 @@ class VAEEncodeAudio(IO.ComfyNode): node_id="VAEEncodeAudio", search_aliases=["audio to latent"], display_name="VAE Encode Audio", - category="model/latent/audio", + category="model/latent", inputs=[ IO.Audio.Input("audio"), IO.Vae.Input("vae"), @@ -115,7 +115,7 @@ class VAEDecodeAudio(IO.ComfyNode): node_id="VAEDecodeAudio", search_aliases=["latent to audio"], display_name="VAE Decode Audio", - category="model/latent/audio", + category="model/latent", inputs=[ IO.Latent.Input("samples"), IO.Vae.Input("vae"), @@ -137,7 +137,7 @@ class VAEDecodeAudioTiled(IO.ComfyNode): node_id="VAEDecodeAudioTiled", search_aliases=["latent to audio"], display_name="VAE Decode Audio (Tiled)", - category="model/latent/audio", + category="model/latent", inputs=[ IO.Latent.Input("samples"), IO.Vae.Input("vae"), @@ -158,7 +158,7 @@ class SaveAudio(IO.ComfyNode): return IO.Schema( node_id="SaveAudio", search_aliases=["export flac"], - display_name="Save Audio (FLAC) (Deprecated)", + display_name="Save Audio (FLAC) (DEPRECATED)", category="audio", essentials_category="Audio", inputs=[ @@ -166,8 +166,9 @@ class SaveAudio(IO.ComfyNode): IO.String.Input("filename_prefix", default="audio/ComfyUI"), ], hidden=[IO.Hidden.prompt, IO.Hidden.extra_pnginfo], - is_output_node=True, is_deprecated=True, + is_output_node=True, + outputs=[IO.Audio.Output("audio")] ) @classmethod @@ -175,11 +176,10 @@ class SaveAudio(IO.ComfyNode): if audio is None: raise ValueError("SaveAudio: input audio is None (source video may have no audio track).") return IO.NodeOutput( + audio, ui=UI.AudioSaveHelper.get_save_audio_ui(audio, filename_prefix=filename_prefix, cls=cls, format=format) ) - save_flac = execute # TODO: remove - class SaveAudioMP3(IO.ComfyNode): @classmethod @@ -187,7 +187,7 @@ class SaveAudioMP3(IO.ComfyNode): return IO.Schema( node_id="SaveAudioMP3", search_aliases=["export mp3"], - display_name="Save Audio (MP3) (Deprecated)", + display_name="Save Audio (MP3) (DEPRECATED)", category="audio", essentials_category="Audio", inputs=[ @@ -196,8 +196,9 @@ class SaveAudioMP3(IO.ComfyNode): IO.Combo.Input("quality", options=["V0", "128k", "320k"], default="V0"), ], hidden=[IO.Hidden.prompt, IO.Hidden.extra_pnginfo], - is_output_node=True, is_deprecated=True, + is_output_node=True, + outputs=[IO.Audio.Output("audio")] ) @classmethod @@ -205,13 +206,12 @@ class SaveAudioMP3(IO.ComfyNode): if audio is None: raise ValueError("SaveAudioMP3: input audio is None (source video may have no audio track).") return IO.NodeOutput( + audio, ui=UI.AudioSaveHelper.get_save_audio_ui( audio, filename_prefix=filename_prefix, cls=cls, format=format, quality=quality ) ) - save_mp3 = execute # TODO: remove - class SaveAudioOpus(IO.ComfyNode): @classmethod @@ -219,7 +219,7 @@ class SaveAudioOpus(IO.ComfyNode): return IO.Schema( node_id="SaveAudioOpus", search_aliases=["export opus"], - display_name="Save Audio (Opus) (Deprecated)", + display_name="Save Audio (Opus) (DEPRECATED)", category="audio", inputs=[ IO.Audio.Input("audio"), @@ -227,8 +227,9 @@ class SaveAudioOpus(IO.ComfyNode): IO.Combo.Input("quality", options=["64k", "96k", "128k", "192k", "320k"], default="128k"), ], hidden=[IO.Hidden.prompt, IO.Hidden.extra_pnginfo], - is_output_node=True, is_deprecated=True, + is_output_node=True, + outputs=[IO.Audio.Output("audio")] ) @classmethod @@ -236,13 +237,12 @@ class SaveAudioOpus(IO.ComfyNode): if audio is None: raise ValueError("SaveAudioOpus: input audio is None (source video may have no audio track).") return IO.NodeOutput( + audio, ui=UI.AudioSaveHelper.get_save_audio_ui( audio, filename_prefix=filename_prefix, cls=cls, format=format, quality=quality ) ) - save_opus = execute # TODO: remove - class SaveAudioAdvanced(IO.ComfyNode): @classmethod @@ -258,10 +258,7 @@ class SaveAudioAdvanced(IO.ComfyNode): IO.String.Input( "filename_prefix", default="audio/ComfyUI", - tooltip=( - "The prefix for the file to save. May include formatting tokens " - "such as %date:yyyy-MM-dd%." - ), + tooltip=("The prefix for the file to save. May include formatting tokens such as %date:yyyy-MM-dd%."), ), IO.DynamicCombo.Input( "format", @@ -279,6 +276,7 @@ class SaveAudioAdvanced(IO.ComfyNode): ], hidden=[IO.Hidden.prompt, IO.Hidden.extra_pnginfo], is_output_node=True, + outputs=[IO.Audio.Output("audio")], ) @classmethod @@ -289,7 +287,7 @@ class SaveAudioAdvanced(IO.ComfyNode): ui=UI.AudioSaveHelper.get_save_audio_ui(audio, filename_prefix=filename_prefix, cls=cls, format=file_format, quality=quality) else: ui=UI.AudioSaveHelper.get_save_audio_ui(audio, filename_prefix=filename_prefix, cls=cls, format=file_format) - return IO.NodeOutput(ui=ui) + return IO.NodeOutput(audio, ui=ui) class PreviewAudio(IO.ComfyNode): @@ -305,13 +303,14 @@ class PreviewAudio(IO.ComfyNode): ], hidden=[IO.Hidden.prompt, IO.Hidden.extra_pnginfo], is_output_node=True, + outputs=[IO.Audio.Output("audio")] ) @classmethod def execute(cls, audio) -> IO.NodeOutput: if audio is None: raise ValueError("PreviewAudio: input audio is None (source video may have no audio track).") - return IO.NodeOutput(ui=UI.PreviewAudio(audio, cls=cls)) + return IO.NodeOutput(audio, ui=UI.PreviewAudio(audio, cls=cls)) save_flac = execute # TODO: remove diff --git a/comfy_extras/nodes_bernini.py b/comfy_extras/nodes_bernini.py index 227fa5753..0537e0806 100644 --- a/comfy_extras/nodes_bernini.py +++ b/comfy_extras/nodes_bernini.py @@ -39,9 +39,9 @@ class BerniniConditioning(io.ComfyNode): return io.Schema( node_id="BerniniConditioning", display_name="Bernini Conditioning", - category="conditioning/video_models", + category="model/conditioning/bernini", description="Conditioning node for Bernini in-context video/image conditioning. It can be used for the following tasks: t2v (text-to-video), v2v (video-to-video), rv2v (reference-guided video editing), r2v (reference-to-video), ads2v (insert image/video into video)." - "Reference images injected as in-context tokens (r2v, rv2v) are encoded independently at their own native aspect ratio (long edge capped at ref_max_size)", + "Reference images injected as in-context tokens (r2v, rv2v) are encoded independently at their own native aspect ratio (long edge capped at ref_max_size)", inputs=[ io.Conditioning.Input("positive"), io.Conditioning.Input("negative"), @@ -50,14 +50,11 @@ class BerniniConditioning(io.ComfyNode): io.Int.Input("height", default=480, min=16, max=8192, step=16), io.Int.Input("length", default=81, min=1, max=8192, step=4), io.Int.Input("batch_size", default=1, min=1, max=4096), - io.Image.Input("source_video", optional=True, tooltip=( - "Source video to edit or restyle (v2v, rv2v). Resized to width/height and trimmed to length.")), - io.Image.Input("reference_video", optional=True, tooltip=( - "Video to insert into the source video (ads2v).")), + io.Image.Input("source_video", optional=True, tooltip=("Source video to edit or restyle (v2v, rv2v). Resized to width/height and trimmed to length.")), + io.Image.Input("reference_video", optional=True, tooltip=("Video to insert into the source video (ads2v).")), io.Autogrow.Input("reference_images", optional=True, template=io.Autogrow.TemplatePrefix( - input=io.Image.Input("reference_image", tooltip=( - "Reference image injected as an in-context token (r2v, rv2v).")), + input=io.Image.Input("reference_image", tooltip=("Reference image injected as an in-context token (r2v, rv2v).")), prefix="reference_image_", min=0, max=8)), io.Int.Input("ref_max_size", default=848, min=16, max=8192, step=16, optional=True, tooltip=( "Max size for the long edge of reference_video and reference_images. Resized with preserved aspect ratio and snapped to 16px.")), @@ -70,10 +67,8 @@ class BerniniConditioning(io.ComfyNode): ) @classmethod - def execute(cls, positive, negative, vae, width, height, length, batch_size, - source_video=None, reference_video=None, reference_images=None, ref_max_size=848) -> io.NodeOutput: - latent = torch.zeros([batch_size, 16, ((length - 1) // 4) + 1, height // 8, width // 8], - device=comfy.model_management.intermediate_device()) + def execute(cls, positive, negative, vae, width, height, length, batch_size, source_video=None, reference_video=None, reference_images=None, ref_max_size=848) -> io.NodeOutput: + latent = torch.zeros([batch_size, 16, ((length - 1) // 4) + 1, height // 8, width // 8], device=comfy.model_management.intermediate_device()) # source_video (1), reference_video (2), reference_images (3, 4, ...). context = [] @@ -106,9 +101,7 @@ class BerniniConditioning(io.ComfyNode): class BerniniExtension(ComfyExtension): @override async def get_node_list(self) -> list[type[io.ComfyNode]]: - return [ - BerniniConditioning, - ] + return [BerniniConditioning,] async def comfy_entrypoint() -> BerniniExtension: diff --git a/comfy_extras/nodes_boogu.py b/comfy_extras/nodes_boogu.py new file mode 100644 index 000000000..f3951c290 --- /dev/null +++ b/comfy_extras/nodes_boogu.py @@ -0,0 +1,97 @@ +import math + +import node_helpers +import comfy.utils +from typing_extensions import override +from comfy_api.latest import ComfyExtension, io + + +class TextEncodeBooguEdit(io.ComfyNode): + """Boogu-Image Edit conditioning. + + The edit image is used twice, matching the reference pipeline: + - Qwen3-VL vision tokens (instruction understanding) -> positive only + - VAE reference latent (image identity) -> positive and negative + The ref latent is in both conds so it cancels under CFG (identity preserved); + the vision tokens are only in the positive so CFG amplifies the instruction. + The tokenizer selects the right system prompt automatically (image -> TI2I, + empty negative -> DROP), so no template plumbing is needed here. + """ + + @classmethod + def define_schema(cls): + return io.Schema( + node_id="TextEncodeBooguEdit", + category="model/conditioning/boogu", + inputs=[ + io.Clip.Input("clip"), + io.String.Input("prompt", multiline=True, dynamic_prompts=True), + io.String.Input("negative_prompt", multiline=True, dynamic_prompts=True, advanced=True), + io.Vae.Input("vae"), + io.Autogrow.Input( + "images", + template=io.Autogrow.TemplateNames( + io.Image.Input("image"), + names=[f"image_{i}" for i in range(1, 17)], + min=0, + ), + tooltip="Reference image(s) to edit. Boogu focuses on one reference per sample; more are allowed.", + ), + ], + outputs=[ + io.Conditioning.Output(display_name="positive"), + io.Conditioning.Output(display_name="negative"), + ], + ) + + @classmethod + def execute(cls, clip, prompt, negative_prompt, vae=None, images: io.Autogrow.Type = None) -> io.NodeOutput: + ref_latents = [] + images_vl = [] + + images = images or {} + for name in sorted(images, key=lambda n: int(n.rsplit("_", 1)[-1])): + image = images[name] + if image is None: + continue + samples = image.movedim(-1, 1) + + # Vision tower input: the reference caps the VLM image at 384x384 + # (max_vlm_input_pil_pixels in pipeline_boogu.py). + total = int(384 * 384) + scale_by = math.sqrt(total / (samples.shape[3] * samples.shape[2])) + width = round(samples.shape[3] * scale_by) + height = round(samples.shape[2] * scale_by) + s = comfy.utils.common_upscale(samples, width, height, "area", "disabled") + images_vl.append(s.movedim(1, -1)[:, :, :, :3]) + + # Reference latent: align to 16 px (VAE /8 * patch_size 2). + if vae is not None: + total = int(1024 * 1024) + scale_by = math.sqrt(total / (samples.shape[3] * samples.shape[2])) + width = round(samples.shape[3] * scale_by / 16.0) * 16 + height = round(samples.shape[2] * scale_by / 16.0) * 16 + s = comfy.utils.common_upscale(samples, width, height, "area", "disabled") + ref_latents.append(vae.encode(s.movedim(1, -1)[:, :, :, :3])) + + # positive: instruction + vision tokens; negative: empty (no vision). Ref latent on both. + positive = clip.encode_from_tokens_scheduled(clip.tokenize(prompt, images=images_vl)) + negative = clip.encode_from_tokens_scheduled(clip.tokenize(negative_prompt)) + + if len(ref_latents) > 0: + positive = node_helpers.conditioning_set_values(positive, {"reference_latents": ref_latents}, append=True) + negative = node_helpers.conditioning_set_values(negative, {"reference_latents": ref_latents}, append=True) + + return io.NodeOutput(positive, negative) + + +class BooguExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [ + TextEncodeBooguEdit, + ] + + +async def comfy_entrypoint() -> BooguExtension: + return BooguExtension() diff --git a/comfy_extras/nodes_bounding_boxes.py b/comfy_extras/nodes_bounding_boxes.py new file mode 100644 index 000000000..77cbf8649 --- /dev/null +++ b/comfy_extras/nodes_bounding_boxes.py @@ -0,0 +1,253 @@ +import numpy as np +import torch +from PIL import Image, ImageDraw, ImageEnhance, ImageFont +from typing_extensions import override + +from comfy_api.latest import ComfyExtension, io +from comfy_extras.color_util import hex_to_rgb, normalize_palette, readable_color + +_PREVIEW_LONG_EDGE = 1024 +_PREVIEW_DIM = 0.25 + + +def pixels_to_fractions(box: dict, width: int, height: int) -> dict: + w = width or 1 + h = height or 1 + return { + "x": box.get("x", 0) / w, + "y": box.get("y", 0) / h, + "w": box.get("width", 0) / w, + "h": box.get("height", 0) / h, + } + + +def fractions_to_pixels(box: dict, width: int, height: int) -> dict: + x, y = box.get("x", 0.0), box.get("y", 0.0) + w, h = box.get("w", 0.0), box.get("h", 0.0) + if w < 0: + x, w = x + w, -w + if h < 0: + y, h = y + h, -h + return { + "x": round(x * width), + "y": round(y * height), + "width": round(w * width), + "height": round(h * height), + } + + +def fractions_to_bbox_frame(boxes: list, width: int, height: int) -> list: + pixels = [ + fractions_to_pixels(box, width, height) + for box in boxes + if isinstance(box, dict) + ] + return [pixels] if pixels else [] + + +def _font(size: int): + try: + return ImageFont.load_default(size) + except Exception: + return ImageFont.load_default() + + +def _wrap(draw, text: str, font, max_w: float) -> list[str]: + lines = [] + for para in text.split("\n"): + line = "" + for word in para.split(): + test = word if not line else line + " " + word + if line and draw.textlength(test, font=font) > max_w: + lines.append(line) + line = word + else: + line = test + lines.append(line) + return lines + + +def _bg_from_image(image) -> Image.Image | None: + if image is None: + return None + try: + arr = (image[0].detach().cpu().numpy() * 255).clip(0, 255).astype(np.uint8) + return Image.fromarray(arr) + except Exception: + return None + + +def render_preview(regions, width, height, bg=None): + if bg is not None: + iw, ih = bg.size + long_edge = max(iw, ih) or 1 + scale = min(1.0, _PREVIEW_LONG_EDGE / long_edge) + rw, rh = max(1, round(iw * scale)), max(1, round(ih * scale)) + base = bg.convert("RGB").resize((rw, rh), Image.LANCZOS) + base = ImageEnhance.Brightness(base).enhance(_PREVIEW_DIM) + img = base.convert("RGBA") + else: + long_edge = max(width, height) or 1 + scale = min(1.0, _PREVIEW_LONG_EDGE / long_edge) + rw, rh = max(1, round(width * scale)), max(1, round(height * scale)) + grey = round(_PREVIEW_DIM * 128) + img = Image.new("RGBA", (rw, rh), (grey, grey, grey, 255)) + + overlay = Image.new("RGBA", (rw, rh), (0, 0, 0, 0)) + draw = ImageDraw.Draw(overlay) + fs = max(10, round(rh / 64)) + font = _font(fs) + tag_font = _font(max(9, fs - 2)) + line_h = fs + 2 + + for i, region in enumerate(regions): + if not isinstance(region, dict): + continue + palette = [c for c in (region.get("palette") or []) if c] + r, g, b = hex_to_rgb(palette[0]) if palette else (140, 140, 140) + x1 = max(0, min(rw, round(region.get("x", 0) * rw))) + y1 = max(0, min(rh, round(region.get("y", 0) * rh))) + x2 = max(0, min(rw, round((region.get("x", 0) + region.get("w", 0)) * rw))) + y2 = max(0, min(rh, round((region.get("y", 0) + region.get("h", 0)) * rh))) + if x2 < x1: + x1, x2 = x2, x1 + if y2 < y1: + y1, y2 = y2, y1 + + draw.rectangle([x1, y1, x2, y2], outline=(r, g, b, 255), width=2) + + swatches = palette[:5] + if swatches and (x2 - x1) > 2: + sh = max(5, fs // 2) + seg = (x2 - x1) / len(swatches) + for p, hexc in enumerate(swatches): + sx = x1 + round(p * seg) + draw.rectangle([sx, y1, x1 + round((p + 1) * seg), y1 + sh], fill=hex_to_rgb(hexc)) + + etype = "text" if region.get("type") == "text" else "obj" + tag = str(i + 1).zfill(2) + tw = draw.textlength(tag, font=tag_font) + draw.rectangle([x1, y1, x1 + tw + 6, y1 + fs + 2], fill=(r, g, b, 255)) + tag_fill = (0, 0, 0, 255) if (0.299 * r + 0.587 * g + 0.114 * b) > 140 else (255, 255, 255, 255) + draw.text((x1 + 3, y1 + 1), tag, fill=tag_fill, font=tag_font) + + body = region.get("desc", "") or "" + if etype == "text" and region.get("text"): + body = '"%s"%s' % (region["text"], " — " + body if body else "") + if body and (x2 - x1) > 8: + ty = y1 + fs + 5 + for line in _wrap(draw, body, font, x2 - x1 - 8): + if ty > y2: + break + draw.text((x1 + 4, ty), line, fill=readable_color((r, g, b)) + (255,), font=font) + ty += line_h + + composed = Image.alpha_composite(img, overlay).convert("RGB") + arr = np.asarray(composed, dtype=np.float32) / 255.0 + return torch.from_numpy(arr).unsqueeze(0) + + +def boxes_to_regions(boxes, width: int, height: int) -> list: + regions: list = [] + if not isinstance(boxes, list): + return regions + for box in boxes: + if not isinstance(box, dict): + continue + meta = box.get("metadata") + meta = meta if isinstance(meta, dict) else {} + regions.append({ + **pixels_to_fractions(box, width, height), + "type": meta.get("type", "obj"), + "text": meta.get("text", ""), + "desc": meta.get("desc", ""), + "palette": meta.get("palette", []), + }) + return regions + + +def _norm_bbox(region: dict) -> list[int]: + def grid(value: float) -> int: + return max(0, min(1000, round(value * 1000))) + + x, y = region.get("x", 0.0), region.get("y", 0.0) + w, h = region.get("w", 0.0), region.get("h", 0.0) + ymin, xmin, ymax, xmax = grid(y), grid(x), grid(y + h), grid(x + w) + if ymin > ymax: + ymin, ymax = ymax, ymin + if xmin > xmax: + xmin, xmax = xmax, xmin + return [ymin, xmin, ymax, xmax] + + +def build_elements(regions: list) -> list: + elements = [] + for region in regions: + if not isinstance(region, dict): + continue + etype = "text" if region.get("type") == "text" else "obj" + element = {"type": etype} + element["bbox"] = _norm_bbox(region) + if etype == "text": + element["text"] = region.get("text", "") + element["desc"] = region.get("desc", "") + palette = normalize_palette(region.get("palette", [])) + if palette: + element["color_palette"] = palette[:5] + elements.append(element) + return elements + + +class CreateBoundingBoxes(io.ComfyNode): + @classmethod + def define_schema(cls): + editor_state = io.BoundingBoxes.Input( + "editor_state", + socketless=False, + tooltip="Draw bounding boxes and set each box type, text, description, color palette. Start with background element first and foreground last.", + ) + return io.Schema( + node_id="CreateBoundingBoxes", + display_name="Create Bounding Boxes", + category="utilities", + description="Draw bounding boxes in a canvas. Outputs Ideogram prompt elements, pixel-space bounding boxes, and a preview image.", + inputs=[ + io.Image.Input( + "background", + optional=True, + tooltip="Optional image used as background in the canvas and preview.", + ), + io.Int.Input("width", default=1024, min=64, max=16384, step=16, + tooltip="Width of the canvas and the pixel grid for the bounding boxes."), + io.Int.Input("height", default=1024, min=64, max=16384, step=16, + tooltip="Height of the canvas and the pixel grid for the bounding boxes."), + editor_state, + ], + outputs=[ + io.Image.Output(display_name="preview"), + io.BoundingBox.Output(display_name="bboxes"), + io.Array.Output(display_name="elements"), + ], + is_experimental=True, + ) + + @classmethod + def execute(cls, width, height, editor_state=None, background=None) -> io.NodeOutput: + regions = boxes_to_regions(editor_state, width, height) + preview = render_preview(regions, width, height, _bg_from_image(background)) + return io.NodeOutput( + preview, + fractions_to_bbox_frame(regions, width, height), + build_elements(regions), + ui={"dims": [width, height]}, + ) + + +class BoundingBoxesExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [CreateBoundingBoxes] + + +async def comfy_entrypoint() -> BoundingBoxesExtension: + return BoundingBoxesExtension() diff --git a/comfy_extras/nodes_camera_trajectory.py b/comfy_extras/nodes_camera_trajectory.py index 13a1448f4..280d136af 100644 --- a/comfy_extras/nodes_camera_trajectory.py +++ b/comfy_extras/nodes_camera_trajectory.py @@ -153,7 +153,7 @@ class WanCameraEmbedding(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="WanCameraEmbedding", - category="model/conditioning/video_models", + category="model/conditioning/wan/camera", inputs=[ io.Combo.Input( "camera_pose", diff --git a/comfy_extras/nodes_chroma_radiance.py b/comfy_extras/nodes_chroma_radiance.py index a4f673001..059344f3c 100644 --- a/comfy_extras/nodes_chroma_radiance.py +++ b/comfy_extras/nodes_chroma_radiance.py @@ -13,7 +13,7 @@ class EmptyChromaRadianceLatentImage(io.ComfyNode): def define_schema(cls) -> io.Schema: return io.Schema( node_id="EmptyChromaRadianceLatentImage", - category="model/latent/chroma_radiance", + category="model/latent/chroma radiance", inputs=[ io.Int.Input(id="width", default=1024, min=16, max=nodes.MAX_RESOLUTION, step=16), io.Int.Input(id="height", default=1024, min=16, max=nodes.MAX_RESOLUTION, step=16), @@ -33,7 +33,7 @@ class ChromaRadianceOptions(io.ComfyNode): def define_schema(cls) -> io.Schema: return io.Schema( node_id="ChromaRadianceOptions", - category="model/patch/chroma_radiance", + category="model/patch/chroma radiance", description="Allows setting advanced options for the Chroma Radiance model.", inputs=[ io.Model.Input(id="model"), diff --git a/comfy_extras/nodes_clip_sdxl.py b/comfy_extras/nodes_clip_sdxl.py index 7a001af6f..08fbbd827 100644 --- a/comfy_extras/nodes_clip_sdxl.py +++ b/comfy_extras/nodes_clip_sdxl.py @@ -9,7 +9,8 @@ class CLIPTextEncodeSDXLRefiner(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="CLIPTextEncodeSDXLRefiner", - category="advanced/conditioning", + display_name="CLIP Text Encode (SDXL Refiner)", + category="model/conditioning/stable diffusion", inputs=[ io.Float.Input("ascore", default=6.0, min=0.0, max=1000.0, step=0.01), io.Int.Input("width", default=1024, min=0, max=nodes.MAX_RESOLUTION), @@ -30,7 +31,8 @@ class CLIPTextEncodeSDXL(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="CLIPTextEncodeSDXL", - category="advanced/conditioning", + display_name="CLIP Text Encode (SDXL)", + category="model/conditioning/stable diffusion", inputs=[ io.Clip.Input("clip"), io.Int.Input("width", default=1024, min=0, max=nodes.MAX_RESOLUTION), diff --git a/comfy_extras/nodes_color.py b/comfy_extras/nodes_color.py index 688254e4e..6d10b26f4 100644 --- a/comfy_extras/nodes_color.py +++ b/comfy_extras/nodes_color.py @@ -1,5 +1,6 @@ from typing_extensions import override from comfy_api.latest import ComfyExtension, io +from comfy_extras.color_util import hex_to_rgb class ColorToRGBInt(io.ComfyNode): @@ -15,21 +16,30 @@ class ColorToRGBInt(io.ComfyNode): ], outputs=[ io.Int.Output(display_name="rgb_int"), - io.Color.Output(display_name="hex") + io.Color.Output(display_name="hex"), + io.Float.Output(display_name="alpha"), ], ) @classmethod def execute(cls, color: str) -> io.NodeOutput: - # expect format #RRGGBB - if len(color) != 7 or color[0] != "#": - raise ValueError("Color must be in format #RRGGBB") - r = int(color[1:3], 16) - g = int(color[3:5], 16) - b = int(color[5:7], 16) + # expect format #RRGGBB or #RRGGBBAA + if len(color) not in (7, 9) or color[0] != "#": + raise ValueError("Color must be in format #RRGGBB or #RRGGBBAA") + try: + int(color[1:], 16) + except ValueError: + raise ValueError("Color must be in format #RRGGBB or #RRGGBBAA") from None + + alpha = 1.0 + if len(color) == 9: + alpha = int(color[7:9], 16) / 255.0 + color = color[:7] + + r, g, b = hex_to_rgb(color) rgb_int = r * 256 * 256 + g * 256 + b - return io.NodeOutput(rgb_int, color) + return io.NodeOutput(rgb_int, color, alpha) class ColorExtension(ComfyExtension): diff --git a/comfy_extras/nodes_cond.py b/comfy_extras/nodes_cond.py index b745a43af..c8091b7a4 100644 --- a/comfy_extras/nodes_cond.py +++ b/comfy_extras/nodes_cond.py @@ -8,7 +8,8 @@ class CLIPTextEncodeControlnet(io.ComfyNode): def define_schema(cls) -> io.Schema: return io.Schema( node_id="CLIPTextEncodeControlnet", - category="experimental/conditioning", + display_name="CLIP Text Encode (Controlnet)", + category="model/conditioning", inputs=[ io.Clip.Input("clip"), io.Conditioning.Input("conditioning"), @@ -35,11 +36,12 @@ class T5TokenizerOptions(io.ComfyNode): def define_schema(cls) -> io.Schema: return io.Schema( node_id="T5TokenizerOptions", - category="experimental/conditioning", + display_name="T5 Tokenizer Options", + category="model/conditioning", inputs=[ io.Clip.Input("clip"), - io.Int.Input("min_padding", default=0, min=0, max=10000, step=1, advanced=True), - io.Int.Input("min_length", default=0, min=0, max=10000, step=1, advanced=True), + io.Int.Input("min_padding", default=0, min=0, max=10000, step=1), + io.Int.Input("min_length", default=0, min=0, max=10000, step=1), ], outputs=[io.Clip.Output()], is_experimental=True, diff --git a/comfy_extras/nodes_context_windows.py b/comfy_extras/nodes_context_windows.py index d9e32b9d9..15d2dc506 100644 --- a/comfy_extras/nodes_context_windows.py +++ b/comfy_extras/nodes_context_windows.py @@ -13,21 +13,22 @@ class ContextWindowsManualNode(io.ComfyNode): description="Manually set context windows.", inputs=[ io.Model.Input("model", tooltip="The model to apply context windows to during sampling."), - io.Int.Input("context_length", min=1, default=16, tooltip="The length of the context window.", advanced=True), - io.Int.Input("context_overlap", min=0, default=4, tooltip="The overlap of the context window.", advanced=True), + io.Int.Input("context_length", min=1, default=16, tooltip="The length of the context window."), + io.Int.Input("context_overlap", min=0, default=4, tooltip="The overlap of the context window."), io.Combo.Input("context_schedule", options=[ comfy.context_windows.ContextSchedules.STATIC_STANDARD, comfy.context_windows.ContextSchedules.UNIFORM_STANDARD, comfy.context_windows.ContextSchedules.UNIFORM_LOOPED, comfy.context_windows.ContextSchedules.BATCHED, - ], tooltip="The stride of the context window."), - io.Int.Input("context_stride", min=1, default=1, tooltip="The stride of the context window; only applicable to uniform schedules.", advanced=True), + ], default=comfy.context_windows.ContextSchedules.STATIC_STANDARD, tooltip="Step-dependent scheduling algorithm for context windows."), + io.Int.Input("context_stride", min=1, default=1, tooltip="The stride of the context window; only applicable to uniform schedules."), io.Boolean.Input("closed_loop", default=False, tooltip="Whether to close the context window loop; only applicable to looped schedules."), io.Combo.Input("fuse_method", options=comfy.context_windows.ContextFuseMethods.LIST_STATIC, default=comfy.context_windows.ContextFuseMethods.PYRAMID, tooltip="The method to use to fuse the context windows."), io.Int.Input("dim", min=0, max=5, default=0, tooltip="The dimension to apply the context windows to."), io.Boolean.Input("freenoise", default=False, tooltip="Whether to apply FreeNoise noise shuffling, improves window blending."), - io.String.Input("cond_retain_index_list", default="", tooltip="List of latent indices to retain in the conditioning tensors for each window, for example setting this to '0' will use the initial start image for each window."), + io.String.Input("cond_retain_index_list", default="", tooltip="List of latent indices to retain in the conditioning tensors for each window. For concat-style I2V models (e.g. Wan I2V, HunyuanVideo I2V, Cosmos I2V, SVD) the encoded start image lives in the c_concat conditioning channels; setting this to '0' will retain that start image content at sub-pos 0 of every window."), io.Boolean.Input("split_conds_to_windows", default=False, tooltip="Whether to split multiple conditionings (created by ConditionCombine) to each window based on region index."), + io.String.Input("latent_retain_index_list", default="", tooltip="List of latent indices to retain in the noise latent itself for each window. Use for workflows where reference content (e.g. a start image) lives directly in the noise latent rather than in separate conditioning channels (e.g. inplace-style I2V like LTXV, AnimateDiff). Independent of cond_retain_index_list."), io.Boolean.Input("causal_window_fix", default=True, tooltip="Whether to add a causal fix frame to non-0-indexed context windows."), ], outputs=[ @@ -38,7 +39,7 @@ class ContextWindowsManualNode(io.ComfyNode): @classmethod def execute(cls, model: io.Model.Type, context_length: int, context_overlap: int, context_schedule: str, context_stride: int, closed_loop: bool, fuse_method: str, dim: int, freenoise: bool, - cond_retain_index_list: list[int]=[], split_conds_to_windows: bool=False, causal_window_fix: bool=True) -> io.Model: + cond_retain_index_list: list[int]=[], split_conds_to_windows: bool=False, latent_retain_index_list: list[int]=[], causal_window_fix: bool=True) -> io.Model: model = model.clone() model.model_options["context_handler"] = comfy.context_windows.IndexListContextHandler( context_schedule=comfy.context_windows.get_matching_context_schedule(context_schedule), @@ -51,6 +52,7 @@ class ContextWindowsManualNode(io.ComfyNode): freenoise=freenoise, cond_retain_index_list=cond_retain_index_list, split_conds_to_windows=split_conds_to_windows, + latent_retain_index_list=latent_retain_index_list, causal_window_fix=causal_window_fix, ) # make memory usage calculation only take into account the context window latents @@ -65,32 +67,71 @@ class WanContextWindowsManualNode(ContextWindowsManualNode): schema = super().define_schema() schema.node_id = "WanContextWindowsManual" schema.display_name = "WAN Context Windows (Manual)" - schema.description = "Manually set context windows for WAN-like models (dim=2)." + schema.display_name = "Wan Context Windows" + schema.description = "Set context windows for Wan-like models." + schema.category="model/patch/wan" schema.inputs = [ io.Model.Input("model", tooltip="The model to apply context windows to during sampling."), - io.Int.Input("context_length", min=1, max=nodes.MAX_RESOLUTION, step=4, default=81, tooltip="The length of the context window.", advanced=True), - io.Int.Input("context_overlap", min=0, default=30, tooltip="The overlap of the context window.", advanced=True), + io.Int.Input("context_length", min=1, max=nodes.MAX_RESOLUTION, step=4, default=81, tooltip="The length of the context window in real frames. Must be 4*n + 1."), + io.Int.Input("context_overlap", min=0, default=30, tooltip="The overlap of the context window in real frames."), io.Combo.Input("context_schedule", options=[ comfy.context_windows.ContextSchedules.STATIC_STANDARD, comfy.context_windows.ContextSchedules.UNIFORM_STANDARD, comfy.context_windows.ContextSchedules.UNIFORM_LOOPED, comfy.context_windows.ContextSchedules.BATCHED, - ], tooltip="The stride of the context window."), + ], default=comfy.context_windows.ContextSchedules.UNIFORM_STANDARD, tooltip="Step-dependent scheduling algorithm for context windows."), io.Int.Input("context_stride", min=1, default=1, tooltip="The stride of the context window; only applicable to uniform schedules.", advanced=True), - io.Boolean.Input("closed_loop", default=False, tooltip="Whether to close the context window loop; only applicable to looped schedules."), + io.Boolean.Input("closed_loop", default=False, tooltip="Whether to close the context window loop; only applicable to looped schedules.", advanced=True), io.Combo.Input("fuse_method", options=comfy.context_windows.ContextFuseMethods.LIST_STATIC, default=comfy.context_windows.ContextFuseMethods.PYRAMID, tooltip="The method to use to fuse the context windows."), - io.Boolean.Input("freenoise", default=False, tooltip="Whether to apply FreeNoise noise shuffling, improves window blending."), - #io.String.Input("cond_retain_index_list", default="", tooltip="List of latent indices to retain in the conditioning tensors for each window, for example setting this to '0' will use the initial start image for each window."), - #io.Boolean.Input("split_conds_to_windows", default=False, tooltip="Whether to split multiple conditionings (created by ConditionCombine) to each window based on region index."), + io.Boolean.Input("freenoise", default=True, tooltip="Whether to apply FreeNoise noise shuffling, improves window blending.", advanced=True), + io.Boolean.Input("retain_first_frame", default=False, tooltip="Retain the first I2V frame in every context window (may help retain initial reference)."), + io.Boolean.Input("split_conds_to_windows", default=False, tooltip="Whether to split multiple conditionings (created by ConditionCombine) to each window based on region index.", advanced=True), ] return schema @classmethod def execute(cls, model: io.Model.Type, context_length: int, context_overlap: int, context_schedule: str, context_stride: int, closed_loop: bool, fuse_method: str, freenoise: bool, - cond_retain_index_list: list[int]=[], split_conds_to_windows: bool=False) -> io.Model: - context_length = max(((context_length - 1) // 4) + 1, 1) # at least length 1 - context_overlap = max(((context_overlap - 1) // 4) + 1, 0) # at least overlap 0 - return super().execute(model, context_length, context_overlap, context_schedule, context_stride, closed_loop, fuse_method, dim=2, freenoise=freenoise, cond_retain_index_list=cond_retain_index_list, split_conds_to_windows=split_conds_to_windows) + retain_first_frame: bool=False, split_conds_to_windows: bool=False) -> io.Model: + context_length = max(((context_length - 1) // 4) + 1, 1) # at least length 1 + context_overlap = max(context_overlap // 4, 0) # at least overlap 0 + retain_index_list = "0" if retain_first_frame else "" + return super().execute(model, context_length, context_overlap, context_schedule, context_stride, closed_loop, fuse_method, dim=2, freenoise=freenoise, cond_retain_index_list=retain_index_list, split_conds_to_windows=split_conds_to_windows) + + +class LTXVContextWindowsNode(ContextWindowsManualNode): + @classmethod + def define_schema(cls) -> io.Schema: + schema = super().define_schema() + schema.node_id = "LTXVContextWindows" + schema.display_name = "LTXV Context Windows" + schema.description = "Set context windows for LTXV-like models." + schema.inputs = [ + io.Model.Input("model", tooltip="The model to apply context windows to during sampling."), + io.Int.Input("context_length", min=1, max=nodes.MAX_RESOLUTION, step=8, default=145, tooltip="The length of the context window in real frames. Must be 8*n + 1."), + io.Int.Input("context_overlap", min=0, step=8, default=40, tooltip="The overlap of the context window in real frames."), + io.Combo.Input("context_schedule", options=[ + comfy.context_windows.ContextSchedules.STATIC_STANDARD, + comfy.context_windows.ContextSchedules.UNIFORM_STANDARD, + comfy.context_windows.ContextSchedules.UNIFORM_LOOPED, + comfy.context_windows.ContextSchedules.BATCHED, + ], default=comfy.context_windows.ContextSchedules.UNIFORM_STANDARD, tooltip="Step-dependent scheduling algorithm for context windows."), + io.Int.Input("context_stride", min=1, default=1, tooltip="The stride of the context window; only applicable to uniform schedules.", advanced=True), + io.Boolean.Input("closed_loop", default=False, tooltip="Whether to close the context window loop; only applicable to looped schedules.", advanced=True), + io.Combo.Input("fuse_method", options=comfy.context_windows.ContextFuseMethods.LIST_STATIC, default=comfy.context_windows.ContextFuseMethods.PYRAMID, tooltip="The method to use to fuse the context windows."), + io.Boolean.Input("freenoise", default=True, tooltip="Whether to apply FreeNoise noise shuffling, improves window blending.", advanced=True), + io.Boolean.Input("retain_first_frame", default=False, tooltip="Retain the first latent frame in every context window (may help retain initial reference)."), + io.Boolean.Input("split_conds_to_windows", default=False, tooltip="Whether to split multiple conditionings (created by ConditionCombine) to each window based on region index.", advanced=True), + ] + return schema + + @classmethod + def execute(cls, model: io.Model.Type, context_length: int, context_overlap: int, context_schedule: str, fuse_method: str, freenoise: bool, + retain_first_frame: bool=False, split_conds_to_windows: bool=False, context_stride: int=1, closed_loop: bool=False) -> io.Model: + context_length = max(((context_length - 1) // 8) + 1, 1) # at least length 1 + context_overlap = max(context_overlap // 8, 0) # at least overlap 0 + retain_index_list = "0" if retain_first_frame else "" + return super().execute(model, context_length, context_overlap, context_schedule, context_stride, closed_loop, fuse_method, dim=2, freenoise=freenoise, + cond_retain_index_list=retain_index_list, latent_retain_index_list=retain_index_list, split_conds_to_windows=split_conds_to_windows) class ContextWindowsExtension(ComfyExtension): @@ -98,6 +139,7 @@ class ContextWindowsExtension(ComfyExtension): return [ ContextWindowsManualNode, WanContextWindowsManualNode, + LTXVContextWindowsNode, ] def comfy_entrypoint(): diff --git a/comfy_extras/nodes_controlnet.py b/comfy_extras/nodes_controlnet.py index 17d965405..eb476f497 100644 --- a/comfy_extras/nodes_controlnet.py +++ b/comfy_extras/nodes_controlnet.py @@ -9,6 +9,8 @@ class SetUnionControlNetType(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="SetUnionControlNetType", + search_aliases=["set controlnet type", "union controlnet type"], + display_name="Set Union ControlNet Type", category="model/conditioning/controlnet", inputs=[ io.ControlNet.Input("control_net"), @@ -39,6 +41,7 @@ class ControlNetInpaintingAliMamaApply(io.ComfyNode): return io.Schema( node_id="ControlNetInpaintingAliMamaApply", search_aliases=["masked controlnet"], + display_name="Apply ControlNet Inpainting (AliMama)", category="model/conditioning/controlnet", inputs=[ io.Conditioning.Input("positive"), diff --git a/comfy_extras/nodes_cosmos.py b/comfy_extras/nodes_cosmos.py index d754ab442..93cc67a6c 100644 --- a/comfy_extras/nodes_cosmos.py +++ b/comfy_extras/nodes_cosmos.py @@ -13,7 +13,7 @@ class EmptyCosmosLatentVideo(io.ComfyNode): def define_schema(cls) -> io.Schema: return io.Schema( node_id="EmptyCosmosLatentVideo", - category="model/latent/video", + category="model/latent/cosmos", inputs=[ io.Int.Input("width", default=1280, min=16, max=nodes.MAX_RESOLUTION, step=16), io.Int.Input("height", default=704, min=16, max=nodes.MAX_RESOLUTION, step=16), @@ -45,7 +45,7 @@ class CosmosImageToVideoLatent(io.ComfyNode): def define_schema(cls) -> io.Schema: return io.Schema( node_id="CosmosImageToVideoLatent", - category="model/conditioning/inpaint", + category="model/conditioning/cosmos", inputs=[ io.Vae.Input("vae"), io.Int.Input("width", default=1280, min=16, max=nodes.MAX_RESOLUTION, step=16), @@ -88,7 +88,7 @@ class CosmosPredict2ImageToVideoLatent(io.ComfyNode): def define_schema(cls) -> io.Schema: return io.Schema( node_id="CosmosPredict2ImageToVideoLatent", - category="model/conditioning/inpaint", + category="model/conditioning/cosmos", inputs=[ io.Vae.Input("vae"), io.Int.Input("width", default=848, min=16, max=nodes.MAX_RESOLUTION, step=16), diff --git a/comfy_extras/nodes_custom_sampler.py b/comfy_extras/nodes_custom_sampler.py index 3e97084a4..56ef5f526 100644 --- a/comfy_extras/nodes_custom_sampler.py +++ b/comfy_extras/nodes_custom_sampler.py @@ -729,7 +729,7 @@ class SamplerCustom(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="SamplerCustom", - category="model/sampling/custom_sampling", + category="model/sampling/custom", inputs=[ io.Model.Input("model"), io.Boolean.Input("add_noise", default=True, advanced=True), @@ -1015,7 +1015,7 @@ class SamplerCustomAdvanced(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="SamplerCustomAdvanced", - category="model/sampling/custom_sampling", + category="model/sampling/custom", inputs=[ io.Noise.Input("noise"), io.Guider.Input("guider"), @@ -1070,7 +1070,7 @@ class AddNoise(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="AddNoise", - category="experimental/custom_sampling/noise", + category="model/sampling/noise", is_experimental=True, inputs=[ io.Model.Input("model"), @@ -1120,7 +1120,7 @@ class ManualSigmas(io.ComfyNode): return io.Schema( node_id="ManualSigmas", search_aliases=["custom noise schedule", "define sigmas"], - category="experimental/custom_sampling", + category="model/sampling/sigmas", is_experimental=True, inputs=[ io.String.Input("sigmas", default="1, 0.5", multiline=False) @@ -1143,7 +1143,7 @@ class CFGOverride(io.ComfyNode): display_name="CFG Override", description="Override cfg to a fixed value over a [start, end] percent (sigma) range. " "With multiple overrides, the one nearest the sampler wins on overlap.", - category="sampling/custom_sampling", + category="model/sampling/guiders", inputs=[ io.Model.Input("model"), io.Float.Input("cfg", default=1.0, min=0.0, max=100.0, step=0.1, round=0.01), diff --git a/comfy_extras/nodes_dataset.py b/comfy_extras/nodes_dataset.py index 0253b4b4f..73fe75b7f 100644 --- a/comfy_extras/nodes_dataset.py +++ b/comfy_extras/nodes_dataset.py @@ -1583,7 +1583,7 @@ class LoadTrainingDataset(io.ComfyNode): shard_path = os.path.join(dataset_dir, shard_file) with open(shard_path, "rb") as f: - shard_data = torch.load(f) + shard_data = torch.load(f, weights_only=True) all_latents.extend(shard_data["latents"]) all_conditioning.extend(shard_data["conditioning"]) diff --git a/comfy_extras/nodes_easycache.py b/comfy_extras/nodes_easycache.py index 923c2bb05..9e907d371 100644 --- a/comfy_extras/nodes_easycache.py +++ b/comfy_extras/nodes_easycache.py @@ -363,7 +363,7 @@ class EasyCacheNode(io.ComfyNode): node_id="EasyCache", display_name="EasyCache", description="Native EasyCache implementation.", - category="advanced/debug/model", + category="advanced/debug", is_experimental=True, inputs=[ io.Model.Input("model", tooltip="The model to add EasyCache to."), @@ -496,7 +496,7 @@ class LazyCacheNode(io.ComfyNode): node_id="LazyCache", display_name="LazyCache", description="A homebrew version of EasyCache - even 'easier' version of EasyCache to implement. Overall works worse than EasyCache, but better in some rare cases AND universal compatibility with everything in ComfyUI.", - category="advanced/debug/model", + category="advanced/debug", is_experimental=True, inputs=[ io.Model.Input("model", tooltip="The model to add LazyCache to."), diff --git a/comfy_extras/nodes_edit_model.py b/comfy_extras/nodes_edit_model.py index 36da66f34..d0d20ae7a 100644 --- a/comfy_extras/nodes_edit_model.py +++ b/comfy_extras/nodes_edit_model.py @@ -8,7 +8,8 @@ class ReferenceLatent(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="ReferenceLatent", - category="advanced/conditioning/edit_models", + display_name="Set Reference Latent", + category="model/conditioning", description="This node sets the guiding latent for an edit model. If the model supports it you can chain multiple to set multiple reference images.", inputs=[ io.Conditioning.Input("conditioning"), diff --git a/comfy_extras/nodes_flux.py b/comfy_extras/nodes_flux.py index afc663b22..e9986c9e7 100644 --- a/comfy_extras/nodes_flux.py +++ b/comfy_extras/nodes_flux.py @@ -13,7 +13,7 @@ class CLIPTextEncodeFlux(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="CLIPTextEncodeFlux", - category="advanced/conditioning/flux", + category="model/conditioning/flux", inputs=[ io.Clip.Input("clip"), io.String.Input("clip_l", multiline=True, dynamic_prompts=True), @@ -40,7 +40,7 @@ class EmptyFlux2LatentImage(io.ComfyNode): return io.Schema( node_id="EmptyFlux2LatentImage", display_name="Empty Flux 2 Latent", - category="model/latent", + category="model/latent/flux", inputs=[ io.Int.Input("width", default=1024, min=16, max=nodes.MAX_RESOLUTION, step=16), io.Int.Input("height", default=1024, min=16, max=nodes.MAX_RESOLUTION, step=16), @@ -61,7 +61,7 @@ class FluxGuidance(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="FluxGuidance", - category="advanced/conditioning/flux", + category="model/conditioning/flux", inputs=[ io.Conditioning.Input("conditioning"), io.Float.Input("guidance", default=3.5, min=0.0, max=100.0, step=0.1), @@ -84,7 +84,7 @@ class FluxDisableGuidance(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="FluxDisableGuidance", - category="advanced/conditioning/flux", + category="model/conditioning/flux", description="This node completely disables the guidance embed on Flux and Flux like models", inputs=[ io.Conditioning.Input("conditioning"), @@ -128,7 +128,7 @@ class FluxKontextImageScale(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="FluxKontextImageScale", - category="advanced/conditioning/flux", + category="model/conditioning/flux", description="This node resizes the image to one that is more optimal for flux kontext.", inputs=[ io.Image.Input("image"), @@ -156,7 +156,7 @@ class FluxKontextMultiReferenceLatentMethod(io.ComfyNode): return io.Schema( node_id="FluxKontextMultiReferenceLatentMethod", display_name="Edit Model Reference Method", - category="advanced/conditioning/flux", + category="model/conditioning/flux", inputs=[ io.Conditioning.Input("conditioning"), io.Combo.Input( @@ -245,6 +245,11 @@ class KV_Attn_Input: cache_key = "{}_{}".format(extra_options["block_type"], extra_options["block_index"]) if cache_key in self.cache: kk, vv = self.cache[cache_key] + + # Fix batch size changing. + kk = comfy.utils.repeat_to_batch_size(kk, k.shape[0]) + vv = comfy.utils.repeat_to_batch_size(vv, v.shape[0]) + self.set_cache = False return {"q": q, "k": torch.cat((k, kk), dim=2), "v": torch.cat((v, vv), dim=2)} diff --git a/comfy_extras/nodes_frame_interpolation.py b/comfy_extras/nodes_frame_interpolation.py index 4d5bca17e..44708e5ec 100644 --- a/comfy_extras/nodes_frame_interpolation.py +++ b/comfy_extras/nodes_frame_interpolation.py @@ -77,7 +77,7 @@ class FrameInterpolate(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="FrameInterpolate", - display_name="Frame Interpolate", + display_name="Run Frame Interpolation Model", category="video", search_aliases=["rife", "film", "frame interpolation", "slow motion", "interpolate frames", "vfi"], inputs=[ diff --git a/comfy_extras/nodes_glsl.py b/comfy_extras/nodes_glsl.py index ea7420a73..c7161973a 100644 --- a/comfy_extras/nodes_glsl.py +++ b/comfy_extras/nodes_glsl.py @@ -1,85 +1,68 @@ import os import sys import re +import ctypes import logging -import ctypes.util -import importlib.util from typing import TypedDict import numpy as np import torch import nodes +import comfy_angle from comfy_api.latest import ComfyExtension, io, ui from typing_extensions import override -from utils.install_util import get_missing_requirements_message logger = logging.getLogger(__name__) -def _check_opengl_availability(): - """Early check for OpenGL availability. Raises RuntimeError if unlikely to work.""" - logger.debug("_check_opengl_availability: starting") - missing = [] +def _preload_angle(): + egl_path = comfy_angle.get_egl_path() + gles_path = comfy_angle.get_glesv2_path() - # Check Python packages (using find_spec to avoid importing) - logger.debug("_check_opengl_availability: checking for glfw package") - if importlib.util.find_spec("glfw") is None: - missing.append("glfw") + if sys.platform == "win32": + angle_dir = comfy_angle.get_lib_dir() + os.add_dll_directory(angle_dir) + os.environ["PATH"] = angle_dir + os.pathsep + os.environ.get("PATH", "") - logger.debug("_check_opengl_availability: checking for OpenGL package") - if importlib.util.find_spec("OpenGL") is None: - missing.append("PyOpenGL") - - if missing: - raise RuntimeError( - f"OpenGL dependencies not available.\n{get_missing_requirements_message()}\n" - ) - - # On Linux without display, check if headless backends are available - logger.debug(f"_check_opengl_availability: platform={sys.platform}") - if sys.platform.startswith("linux"): - has_display = os.environ.get("DISPLAY") or os.environ.get("WAYLAND_DISPLAY") - logger.debug(f"_check_opengl_availability: has_display={bool(has_display)}") - if not has_display: - # Check for EGL or OSMesa libraries - logger.debug("_check_opengl_availability: checking for EGL library") - has_egl = ctypes.util.find_library("EGL") - logger.debug("_check_opengl_availability: checking for OSMesa library") - has_osmesa = ctypes.util.find_library("OSMesa") - - # Error disabled for CI as it fails this check - # if not has_egl and not has_osmesa: - # raise RuntimeError( - # "GLSL Shader node: No display and no headless backend (EGL/OSMesa) found.\n" - # "See error below for installation instructions." - # ) - logger.debug(f"Headless mode: EGL={'yes' if has_egl else 'no'}, OSMesa={'yes' if has_osmesa else 'no'}") - - logger.debug("_check_opengl_availability: completed") + mode = 0 if sys.platform == "win32" else ctypes.RTLD_GLOBAL + ctypes.CDLL(str(egl_path), mode=mode) + ctypes.CDLL(str(gles_path), mode=mode) -# Run early check at import time -logger.debug("nodes_glsl: running _check_opengl_availability at import time") -_check_opengl_availability() - -# OpenGL modules - initialized lazily when context is created -gl = None -glfw = None -EGL = None +# Pre-load ANGLE *before* any PyOpenGL import so that the EGL platform +# plugin picks up ANGLE's libEGL / libGLESv2 instead of system libs. +_preload_angle() +os.environ.setdefault("PYOPENGL_PLATFORM", "egl") -def _import_opengl(): - """Import OpenGL module. Called after context is created.""" - global gl - if gl is None: - logger.debug("_import_opengl: importing OpenGL.GL") - import OpenGL.GL as _gl - gl = _gl - logger.debug("_import_opengl: import completed") - return gl +import OpenGL +OpenGL.USE_ACCELERATE = False +def _patch_find_library(): + """PyOpenGL's EGL platform looks for 'EGL' and 'GLESv2' by short name + via ctypes.util.find_library, but ANGLE ships as 'libEGL' and + 'libGLESv2'. Patch find_library to return the full ANGLE paths so + PyOpenGL loads the same libraries we pre-loaded.""" + if sys.platform == "linux": + return + import ctypes.util + _orig = ctypes.util.find_library + def _patched(name): + if name == 'EGL': + return comfy_angle.get_egl_path() + if name == 'GLESv2': + return comfy_angle.get_glesv2_path() + return _orig(name) + ctypes.util.find_library = _patched + + +_patch_find_library() + +from OpenGL import EGL +from OpenGL import GLES3 as gl + class SizeModeInput(TypedDict): size_mode: str width: int @@ -102,7 +85,7 @@ MAX_OUTPUTS = 4 # fragColor0-3 (MRT) # (-1,-1)---(3,-1) # # v_texCoord is computed from clip space: * 0.5 + 0.5 maps (-1,1) -> (0,1) -VERTEX_SHADER = """#version 330 core +VERTEX_SHADER = """#version 300 es out vec2 v_texCoord; void main() { vec2 verts[3] = vec2[](vec2(-1, -1), vec2(3, -1), vec2(-1, 3)); @@ -126,14 +109,99 @@ void main() { """ -def _convert_es_to_desktop(source: str) -> str: - """Convert GLSL ES (WebGL) shader source to desktop GLSL 330 core.""" - # Remove any existing #version directive - source = re.sub(r"#version\s+\d+(\s+es)?\s*\n?", "", source, flags=re.IGNORECASE) - # Remove precision qualifiers (not needed in desktop GLSL) - source = re.sub(r"precision\s+(lowp|mediump|highp)\s+\w+\s*;\s*\n?", "", source) - # Prepend desktop GLSL version - return "#version 330 core\n" + source + +def _egl_attribs(*values): + """Build an EGL_NONE-terminated EGLint attribute array.""" + vals = list(values) + [EGL.EGL_NONE] + return (ctypes.c_int32 * len(vals))(*vals) + + +# EGL platform extension constants +EGL_PLATFORM_ANGLE_ANGLE = 0x3202 +EGL_PLATFORM_ANGLE_TYPE_ANGLE = 0x3203 +EGL_PLATFORM_ANGLE_TYPE_VULKAN_ANGLE = 0x3450 +EGL_MESA_PLATFORM_SURFACELESS = 0x31DD + + +_eglGetPlatformDisplayEXT = None + +def _get_egl_platform_display_ext(platform, native_display, attribs): + """Call eglGetPlatformDisplayEXT via ctypes (extension, not in PyOpenGL).""" + global _eglGetPlatformDisplayEXT + if _eglGetPlatformDisplayEXT is None: + from OpenGL import platform as _plat + egl_lib = _plat.PLATFORM.EGL + _get_proc = egl_lib.eglGetProcAddress + _get_proc.restype = ctypes.c_void_p + _get_proc.argtypes = [ctypes.c_char_p] + ptr = _get_proc(b"eglGetPlatformDisplayEXT") + if not ptr: + return None + func_type = ctypes.CFUNCTYPE(ctypes.c_void_p, ctypes.c_uint32, ctypes.c_void_p, ctypes.c_void_p) + _eglGetPlatformDisplayEXT = func_type(ptr) + + raw = _eglGetPlatformDisplayEXT(platform, native_display, attribs) + if not raw: + return None + return ctypes.cast(raw, EGL.EGLDisplay) + + +def _get_egl_display(): + """Get an EGL display, trying the default first then ANGLE's Vulkan + platform for headless environments without a display server.""" + failures = [] + + # Try the default display first (works when X11/Wayland is available) + display = EGL.eglGetDisplay(EGL.EGL_DEFAULT_DISPLAY) + if display: + major, minor = ctypes.c_int32(0), ctypes.c_int32(0) + try: + if EGL.eglInitialize(display, ctypes.byref(major), ctypes.byref(minor)): + return display, major.value, minor.value + except Exception as e: + failures.append(f"default: {e}") + + logger.info("Default EGL display unavailable, trying headless fallbacks") + + # Headless fallback strategies, tried in order: + headless_strategies = [ + ("surfaceless", EGL_MESA_PLATFORM_SURFACELESS, None, None), + ("ANGLE Vulkan", EGL_PLATFORM_ANGLE_ANGLE, None, + _egl_attribs(EGL_PLATFORM_ANGLE_TYPE_ANGLE, EGL_PLATFORM_ANGLE_TYPE_VULKAN_ANGLE)), + ] + + for name, platform, native_display, attribs in headless_strategies: + display = _get_egl_platform_display_ext(platform, native_display, attribs) + if not display: + failures.append(f"{name}: eglGetPlatformDisplayEXT returned no display") + continue + major, minor = ctypes.c_int32(0), ctypes.c_int32(0) + try: + if EGL.eglInitialize(display, ctypes.byref(major), ctypes.byref(minor)): + logger.info(f"Using EGL {name} platform (headless)") + return display, major.value, minor.value + failures.append(f"{name}: eglInitialize returned false") + except Exception as e: + failures.append(f"{name}: {e}") + continue + + details = "\n".join(f" - {f}" for f in failures) + raise RuntimeError( + "Failed to initialize EGL display.\n" + "No display server and no headless EGL platform available.\n" + f"Tried:\n{details}\n" + "Ensure GPU drivers are installed or set DISPLAY for a virtual framebuffer." + ) + + +def _gl_str(name): + """Get an OpenGL string parameter.""" + v = gl.glGetString(name) + if not v: + return "Unknown" + if isinstance(v, bytes): + return v.decode(errors="replace") + return ctypes.string_at(v).decode(errors="replace") def _detect_output_count(source: str) -> int: @@ -159,163 +227,8 @@ def _detect_pass_count(source: str) -> int: return 1 -def _init_glfw(): - """Initialize GLFW. Returns (window, glfw_module). Raises RuntimeError on failure.""" - logger.debug("_init_glfw: starting") - # On macOS, glfw.init() must be called from main thread or it hangs forever - if sys.platform == "darwin": - logger.debug("_init_glfw: skipping on macOS") - raise RuntimeError("GLFW backend not supported on macOS") - - logger.debug("_init_glfw: importing glfw module") - import glfw as _glfw - - logger.debug("_init_glfw: calling glfw.init()") - if not _glfw.init(): - raise RuntimeError("glfw.init() failed") - - try: - logger.debug("_init_glfw: setting window hints") - _glfw.window_hint(_glfw.VISIBLE, _glfw.FALSE) - _glfw.window_hint(_glfw.CONTEXT_VERSION_MAJOR, 3) - _glfw.window_hint(_glfw.CONTEXT_VERSION_MINOR, 3) - _glfw.window_hint(_glfw.OPENGL_PROFILE, _glfw.OPENGL_CORE_PROFILE) - - logger.debug("_init_glfw: calling create_window()") - window = _glfw.create_window(64, 64, "ComfyUI GLSL", None, None) - if not window: - raise RuntimeError("glfw.create_window() failed") - - logger.debug("_init_glfw: calling make_context_current()") - _glfw.make_context_current(window) - logger.debug("_init_glfw: completed successfully") - return window, _glfw - except Exception: - logger.debug("_init_glfw: failed, terminating glfw") - _glfw.terminate() - raise - - -def _init_egl(): - """Initialize EGL for headless rendering. Returns (display, context, surface, EGL_module). Raises RuntimeError on failure.""" - logger.debug("_init_egl: starting") - from OpenGL import EGL as _EGL - from OpenGL.EGL import ( - eglGetDisplay, eglInitialize, eglChooseConfig, eglCreateContext, - eglMakeCurrent, eglCreatePbufferSurface, eglBindAPI, - eglTerminate, eglDestroyContext, eglDestroySurface, - EGL_DEFAULT_DISPLAY, EGL_NO_CONTEXT, EGL_NONE, - EGL_SURFACE_TYPE, EGL_PBUFFER_BIT, EGL_RENDERABLE_TYPE, EGL_OPENGL_BIT, - EGL_RED_SIZE, EGL_GREEN_SIZE, EGL_BLUE_SIZE, EGL_ALPHA_SIZE, EGL_DEPTH_SIZE, - EGL_WIDTH, EGL_HEIGHT, EGL_OPENGL_API, - ) - logger.debug("_init_egl: imports completed") - - display = None - context = None - surface = None - - try: - logger.debug("_init_egl: calling eglGetDisplay()") - display = eglGetDisplay(EGL_DEFAULT_DISPLAY) - if display == _EGL.EGL_NO_DISPLAY: - raise RuntimeError("eglGetDisplay() failed") - - logger.debug("_init_egl: calling eglInitialize()") - major, minor = _EGL.EGLint(), _EGL.EGLint() - if not eglInitialize(display, major, minor): - display = None # Not initialized, don't terminate - raise RuntimeError("eglInitialize() failed") - logger.debug(f"_init_egl: EGL version {major.value}.{minor.value}") - - config_attribs = [ - EGL_SURFACE_TYPE, EGL_PBUFFER_BIT, - EGL_RENDERABLE_TYPE, EGL_OPENGL_BIT, - EGL_RED_SIZE, 8, EGL_GREEN_SIZE, 8, EGL_BLUE_SIZE, 8, EGL_ALPHA_SIZE, 8, - EGL_DEPTH_SIZE, 0, EGL_NONE - ] - configs = (_EGL.EGLConfig * 1)() - num_configs = _EGL.EGLint() - if not eglChooseConfig(display, config_attribs, configs, 1, num_configs) or num_configs.value == 0: - raise RuntimeError("eglChooseConfig() failed") - config = configs[0] - logger.debug(f"_init_egl: config chosen, num_configs={num_configs.value}") - - if not eglBindAPI(EGL_OPENGL_API): - raise RuntimeError("eglBindAPI() failed") - - logger.debug("_init_egl: calling eglCreateContext()") - context_attribs = [ - _EGL.EGL_CONTEXT_MAJOR_VERSION, 3, - _EGL.EGL_CONTEXT_MINOR_VERSION, 3, - _EGL.EGL_CONTEXT_OPENGL_PROFILE_MASK, _EGL.EGL_CONTEXT_OPENGL_CORE_PROFILE_BIT, - EGL_NONE - ] - context = eglCreateContext(display, config, EGL_NO_CONTEXT, context_attribs) - if context == EGL_NO_CONTEXT: - raise RuntimeError("eglCreateContext() failed") - - logger.debug("_init_egl: calling eglCreatePbufferSurface()") - pbuffer_attribs = [EGL_WIDTH, 64, EGL_HEIGHT, 64, EGL_NONE] - surface = eglCreatePbufferSurface(display, config, pbuffer_attribs) - if surface == _EGL.EGL_NO_SURFACE: - raise RuntimeError("eglCreatePbufferSurface() failed") - - logger.debug("_init_egl: calling eglMakeCurrent()") - if not eglMakeCurrent(display, surface, surface, context): - raise RuntimeError("eglMakeCurrent() failed") - - logger.debug("_init_egl: completed successfully") - return display, context, surface, _EGL - - except Exception: - logger.debug("_init_egl: failed, cleaning up") - # Clean up any resources on failure - if surface is not None: - eglDestroySurface(display, surface) - if context is not None: - eglDestroyContext(display, context) - if display is not None: - eglTerminate(display) - raise - - -def _init_osmesa(): - """Initialize OSMesa for software rendering. Returns (context, buffer). Raises RuntimeError on failure.""" - import ctypes - - logger.debug("_init_osmesa: starting") - os.environ["PYOPENGL_PLATFORM"] = "osmesa" - - logger.debug("_init_osmesa: importing OpenGL.osmesa") - from OpenGL import GL as _gl - from OpenGL.osmesa import ( - OSMesaCreateContextExt, OSMesaMakeCurrent, OSMesaDestroyContext, - OSMESA_RGBA, - ) - logger.debug("_init_osmesa: imports completed") - - ctx = OSMesaCreateContextExt(OSMESA_RGBA, 24, 0, 0, None) - if not ctx: - raise RuntimeError("OSMesaCreateContextExt() failed") - - width, height = 64, 64 - buffer = (ctypes.c_ubyte * (width * height * 4))() - - logger.debug("_init_osmesa: calling OSMesaMakeCurrent()") - if not OSMesaMakeCurrent(ctx, buffer, _gl.GL_UNSIGNED_BYTE, width, height): - OSMesaDestroyContext(ctx) - raise RuntimeError("OSMesaMakeCurrent() failed") - - logger.debug("_init_osmesa: completed successfully") - return ctx, buffer - - class GLContext: - """Manages OpenGL context and resources for shader execution. - - Tries backends in order: GLFW (desktop) → EGL (headless GPU) → OSMesa (software). - """ + """Manages an OpenGL ES 3.0 context via EGL/ANGLE (singleton).""" _instance = None _initialized = False @@ -327,131 +240,105 @@ class GLContext: def __init__(self): if GLContext._initialized: - logger.debug("GLContext.__init__: already initialized, skipping") return - logger.debug("GLContext.__init__: starting initialization") - - global glfw, EGL - import time start = time.perf_counter() - self._backend = None - self._window = None - self._egl_display = None - self._egl_context = None - self._egl_surface = None - self._osmesa_ctx = None - self._osmesa_buffer = None + self._display = None + self._surface = None + self._context = None self._vao = None - # Try backends in order: GLFW → EGL → OSMesa - errors = [] - - logger.debug("GLContext.__init__: trying GLFW backend") try: - self._window, glfw = _init_glfw() - self._backend = "glfw" - logger.debug("GLContext.__init__: GLFW backend succeeded") - except Exception as e: - logger.debug(f"GLContext.__init__: GLFW backend failed: {e}") - errors.append(("GLFW", e)) + self._display, self._egl_major, self._egl_minor = _get_egl_display() - if self._backend is None: - logger.debug("GLContext.__init__: trying EGL backend") - try: - self._egl_display, self._egl_context, self._egl_surface, EGL = _init_egl() - self._backend = "egl" - logger.debug("GLContext.__init__: EGL backend succeeded") - except Exception as e: - logger.debug(f"GLContext.__init__: EGL backend failed: {e}") - errors.append(("EGL", e)) + if not EGL.eglBindAPI(EGL.EGL_OPENGL_ES_API): + raise RuntimeError("eglBindAPI(EGL_OPENGL_ES_API) failed") - if self._backend is None: - logger.debug("GLContext.__init__: trying OSMesa backend") - try: - self._osmesa_ctx, self._osmesa_buffer = _init_osmesa() - self._backend = "osmesa" - logger.debug("GLContext.__init__: OSMesa backend succeeded") - except Exception as e: - logger.debug(f"GLContext.__init__: OSMesa backend failed: {e}") - errors.append(("OSMesa", e)) + config = EGL.EGLConfig() + n_configs = ctypes.c_int32(0) + if not EGL.eglChooseConfig( + self._display, + _egl_attribs( + EGL.EGL_RENDERABLE_TYPE, EGL.EGL_OPENGL_ES3_BIT, + EGL.EGL_SURFACE_TYPE, EGL.EGL_PBUFFER_BIT, + EGL.EGL_RED_SIZE, 8, EGL.EGL_GREEN_SIZE, 8, + EGL.EGL_BLUE_SIZE, 8, EGL.EGL_ALPHA_SIZE, 8, + ), + ctypes.byref(config), 1, ctypes.byref(n_configs), + ) or n_configs.value == 0: + raise RuntimeError("eglChooseConfig() failed") - if self._backend is None: - if sys.platform == "win32": - platform_help = ( - "Windows: Ensure GPU drivers are installed and display is available.\n" - " CPU-only/headless mode is not supported on Windows." - ) - elif sys.platform == "darwin": - platform_help = ( - "macOS: GLFW is not supported.\n" - " Install OSMesa via Homebrew: brew install mesa\n" - " Then: pip install PyOpenGL PyOpenGL-accelerate" - ) - else: - platform_help = ( - "Linux: Install one of these backends:\n" - " Desktop: sudo apt install libgl1-mesa-glx libglfw3\n" - " Headless with GPU: sudo apt install libegl1-mesa libgl1-mesa-dri\n" - " Headless (CPU): sudo apt install libosmesa6" - ) - - error_details = "\n".join(f" {name}: {err}" for name, err in errors) - raise RuntimeError( - f"Failed to create OpenGL context.\n\n" - f"Backend errors:\n{error_details}\n\n" - f"{platform_help}" + self._surface = EGL.eglCreatePbufferSurface( + self._display, config, + _egl_attribs(EGL.EGL_WIDTH, 64, EGL.EGL_HEIGHT, 64), ) + if not self._surface: + raise RuntimeError("eglCreatePbufferSurface() failed") - # Now import OpenGL.GL (after context is current) - logger.debug("GLContext.__init__: importing OpenGL.GL") - _import_opengl() + self._context = EGL.eglCreateContext( + self._display, config, EGL.EGL_NO_CONTEXT, + _egl_attribs(EGL.EGL_CONTEXT_CLIENT_VERSION, 3), + ) + if not self._context: + raise RuntimeError("eglCreateContext() failed") - # Create VAO (required for core profile, but OSMesa may use compat profile) - logger.debug("GLContext.__init__: creating VAO") - try: - vao = gl.glGenVertexArrays(1) - gl.glBindVertexArray(vao) - self._vao = vao # Only store after successful bind - logger.debug("GLContext.__init__: VAO created successfully") - except Exception as e: - logger.debug(f"GLContext.__init__: VAO creation failed (may be expected for OSMesa): {e}") - # OSMesa with older Mesa may not support VAOs - # Clean up if we created but couldn't bind - if vao: - try: - gl.glDeleteVertexArrays(1, [vao]) - except Exception: - pass + if not EGL.eglMakeCurrent(self._display, self._surface, self._surface, self._context): + raise RuntimeError("eglMakeCurrent() failed") + + self._vao = gl.glGenVertexArrays(1) + gl.glBindVertexArray(self._vao) + + except Exception: + self._cleanup() + raise elapsed = (time.perf_counter() - start) * 1000 - # Log device info - renderer = gl.glGetString(gl.GL_RENDERER) - vendor = gl.glGetString(gl.GL_VENDOR) - version = gl.glGetString(gl.GL_VERSION) - renderer = renderer.decode() if renderer else "Unknown" - vendor = vendor.decode() if vendor else "Unknown" - version = version.decode() if version else "Unknown" + renderer = _gl_str(gl.GL_RENDERER) + vendor = _gl_str(gl.GL_VENDOR) + version = _gl_str(gl.GL_VERSION) GLContext._initialized = True - logger.info(f"GLSL context initialized in {elapsed:.1f}ms ({self._backend}) - {renderer} ({vendor}), GL {version}") + logger.info(f"GLSL context initialized in {elapsed:.1f}ms - EGL {self._egl_major}.{self._egl_minor}, {renderer} ({vendor}), GL {version}") def make_current(self): - if self._backend == "glfw": - glfw.make_context_current(self._window) - elif self._backend == "egl": - from OpenGL.EGL import eglMakeCurrent - eglMakeCurrent(self._egl_display, self._egl_surface, self._egl_surface, self._egl_context) - elif self._backend == "osmesa": - from OpenGL.osmesa import OSMesaMakeCurrent - OSMesaMakeCurrent(self._osmesa_ctx, self._osmesa_buffer, gl.GL_UNSIGNED_BYTE, 64, 64) - + if not EGL.eglMakeCurrent(self._display, self._surface, self._surface, self._context): + err = EGL.eglGetError() + raise RuntimeError(f"eglMakeCurrent() failed (EGL error: 0x{err:04X})") if self._vao is not None: gl.glBindVertexArray(self._vao) + def _cleanup(self): + if not self._display: + return + try: + if self._vao is not None: + gl.glDeleteVertexArrays(1, [self._vao]) + self._vao = None + except Exception: + pass + try: + EGL.eglMakeCurrent(self._display, EGL.EGL_NO_SURFACE, EGL.EGL_NO_SURFACE, EGL.EGL_NO_CONTEXT) + except Exception: + pass + try: + if self._context: + EGL.eglDestroyContext(self._display, self._context) + except Exception: + pass + try: + if self._surface: + EGL.eglDestroySurface(self._display, self._surface) + except Exception: + pass + try: + EGL.eglTerminate(self._display) + except Exception: + pass + self._display = None + def _compile_shader(source: str, shader_type: int) -> int: """Compile a shader and return its ID.""" @@ -459,8 +346,10 @@ def _compile_shader(source: str, shader_type: int) -> int: gl.glShaderSource(shader, source) gl.glCompileShader(shader) - if gl.glGetShaderiv(shader, gl.GL_COMPILE_STATUS) != gl.GL_TRUE: - error = gl.glGetShaderInfoLog(shader).decode() + if not gl.glGetShaderiv(shader, gl.GL_COMPILE_STATUS): + error = gl.glGetShaderInfoLog(shader) + if isinstance(error, bytes): + error = error.decode(errors="replace") gl.glDeleteShader(shader) raise RuntimeError(f"Shader compilation failed:\n{error}") @@ -484,8 +373,10 @@ def _create_program(vertex_source: str, fragment_source: str) -> int: gl.glDeleteShader(vertex_shader) gl.glDeleteShader(fragment_shader) - if gl.glGetProgramiv(program, gl.GL_LINK_STATUS) != gl.GL_TRUE: - error = gl.glGetProgramInfoLog(program).decode() + if not gl.glGetProgramiv(program, gl.GL_LINK_STATUS): + error = gl.glGetProgramInfoLog(program) + if isinstance(error, bytes): + error = error.decode(errors="replace") gl.glDeleteProgram(program) raise RuntimeError(f"Program linking failed:\n{error}") @@ -530,9 +421,6 @@ def _render_shader_batch( ctx = GLContext() ctx.make_current() - # Convert from GLSL ES to desktop GLSL 330 - fragment_source = _convert_es_to_desktop(fragment_code) - # Detect how many outputs the shader actually uses num_outputs = _detect_output_count(fragment_code) @@ -558,9 +446,9 @@ def _render_shader_batch( try: # Compile shaders (once for all batches) try: - program = _create_program(VERTEX_SHADER, fragment_source) + program = _create_program(VERTEX_SHADER, fragment_code) except RuntimeError: - logger.error(f"Fragment shader:\n{fragment_source}") + logger.error(f"Fragment shader:\n{fragment_code}") raise gl.glUseProgram(program) @@ -723,13 +611,13 @@ def _render_shader_batch( gl.glDrawArrays(gl.GL_TRIANGLES, 0, 3) # Read back outputs for this batch - # (glGetTexImage is synchronous, implicitly waits for rendering) + gl.glBindFramebuffer(gl.GL_FRAMEBUFFER, fbo) batch_outputs = [] - for tex in output_textures: - gl.glBindTexture(gl.GL_TEXTURE_2D, tex) - data = gl.glGetTexImage(gl.GL_TEXTURE_2D, 0, gl.GL_RGBA, gl.GL_FLOAT) - img = np.frombuffer(data, dtype=np.float32).reshape(height, width, 4) - batch_outputs.append(img[::-1, :, :].copy()) + for i in range(num_outputs): + gl.glReadBuffer(gl.GL_COLOR_ATTACHMENT0 + i) + buf = np.empty((height, width, 4), dtype=np.float32) + gl.glReadPixels(0, 0, width, height, gl.GL_RGBA, gl.GL_FLOAT, buf) + batch_outputs.append(buf[::-1, :, :].copy()) # Pad with black images for unused outputs black_img = np.zeros((height, width, 4), dtype=np.float32) @@ -750,18 +638,18 @@ def _render_shader_batch( gl.glBindFramebuffer(gl.GL_FRAMEBUFFER, 0) gl.glUseProgram(0) - for tex in input_textures: - gl.glDeleteTextures(int(tex)) - for tex in curve_textures: - gl.glDeleteTextures(int(tex)) - for tex in output_textures: - gl.glDeleteTextures(int(tex)) - for tex in ping_pong_textures: - gl.glDeleteTextures(int(tex)) + if input_textures: + gl.glDeleteTextures(len(input_textures), input_textures) + if curve_textures: + gl.glDeleteTextures(len(curve_textures), curve_textures) + if output_textures: + gl.glDeleteTextures(len(output_textures), output_textures) + if ping_pong_textures: + gl.glDeleteTextures(len(ping_pong_textures), ping_pong_textures) if fbo is not None: gl.glDeleteFramebuffers(1, [fbo]) - for pp_fbo in ping_pong_fbos: - gl.glDeleteFramebuffers(1, [pp_fbo]) + if ping_pong_fbos: + gl.glDeleteFramebuffers(len(ping_pong_fbos), ping_pong_fbos) if program is not None: gl.glDeleteProgram(program) diff --git a/comfy_extras/nodes_hidream.py b/comfy_extras/nodes_hidream.py index e345fe51d..65248561b 100644 --- a/comfy_extras/nodes_hidream.py +++ b/comfy_extras/nodes_hidream.py @@ -11,8 +11,9 @@ class QuadrupleCLIPLoader(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="QuadrupleCLIPLoader", - category="advanced/loaders", - description="[Recipes]\n\nhidream: long clip-l, long clip-g, t5xxl, llama_8b_3.1_instruct", + display_name="Load CLIP (Quadruple)", + category="model/loaders", + description="Recipes:\nhidream: long clip-l, long clip-g, t5xxl, llama_8b_3.1_instruct", inputs=[ io.Combo.Input("clip_name1", options=folder_paths.get_filename_list("text_encoders")), io.Combo.Input("clip_name2", options=folder_paths.get_filename_list("text_encoders")), @@ -38,8 +39,9 @@ class CLIPTextEncodeHiDream(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="CLIPTextEncodeHiDream", + display_name="CLIP Text Encode (HiDream)", search_aliases=["hidream prompt"], - category="advanced/conditioning", + category="model/conditioning/hidream", inputs=[ io.Clip.Input("clip"), io.String.Input("clip_l", multiline=True, dynamic_prompts=True), diff --git a/comfy_extras/nodes_hidream_o1.py b/comfy_extras/nodes_hidream_o1.py index 8648d2e26..85693fce6 100644 --- a/comfy_extras/nodes_hidream_o1.py +++ b/comfy_extras/nodes_hidream_o1.py @@ -14,7 +14,7 @@ class EmptyHiDreamO1LatentImage(io.ComfyNode): return io.Schema( node_id="EmptyHiDreamO1LatentImage", display_name="Empty HiDream-O1 Latent Image", - category="model/latent/image", + category="model/latent/hidream", description=( "Empty pixel-space latent for HiDream-O1-Image. The model was " "trained at ~4 megapixels; lower resolutions go off-distribution " @@ -47,7 +47,7 @@ class HiDreamO1ReferenceImages(io.ComfyNode): return io.Schema( node_id="HiDreamO1ReferenceImages", display_name="HiDream-O1 Reference Images", - category="model/conditioning/image", + category="model/conditioning/hidream", description=( "Attach 1-10 reference images to conditioning, one for edit instruction" "or multiple for subject-driven personalization." @@ -117,7 +117,7 @@ class HiDreamO1PatchSeamSmoothing(io.ComfyNode): return io.Schema( node_id="HiDreamO1PatchSeamSmoothing", display_name="HiDream-O1 Patch Seam Smoothing", - category="advanced/model", + category="model/patch/hidream", is_experimental=True, description=( "Average the model output across multiple shifted patch-grid " diff --git a/comfy_extras/nodes_hunyuan.py b/comfy_extras/nodes_hunyuan.py index 16fff12af..8df2c8908 100644 --- a/comfy_extras/nodes_hunyuan.py +++ b/comfy_extras/nodes_hunyuan.py @@ -14,7 +14,8 @@ class CLIPTextEncodeHunyuanDiT(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="CLIPTextEncodeHunyuanDiT", - category="advanced/conditioning", + display_name="CLIP Text Encode (Hunyuan Image)", + category="model/conditioning/hunyuan image", inputs=[ io.Clip.Input("clip"), io.String.Input("bert", multiline=True, dynamic_prompts=True), @@ -41,7 +42,7 @@ class EmptyHunyuanLatentVideo(io.ComfyNode): return io.Schema( node_id="EmptyHunyuanLatentVideo", display_name="Empty HunyuanVideo 1.0 Latent", - category="model/latent/video", + category="model/latent/hunyuan video", inputs=[ io.Int.Input("width", default=848, min=16, max=nodes.MAX_RESOLUTION, step=16), io.Int.Input("height", default=480, min=16, max=nodes.MAX_RESOLUTION, step=16), @@ -67,6 +68,7 @@ class EmptyHunyuanVideo15Latent(EmptyHunyuanLatentVideo): schema = super().define_schema() schema.node_id = "EmptyHunyuanVideo15Latent" schema.display_name = "Empty HunyuanVideo 1.5 Latent" + schema.category = "model/latent/hunyuan video" return schema @classmethod @@ -81,7 +83,7 @@ class HunyuanVideo15ImageToVideo(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="HunyuanVideo15ImageToVideo", - category="model/conditioning/video_models", + category="model/conditioning/hunyuan video", inputs=[ io.Conditioning.Input("positive"), io.Conditioning.Input("negative"), @@ -132,7 +134,7 @@ class HunyuanVideo15SuperResolution(io.ComfyNode): return io.Schema( node_id="HunyuanVideo15SuperResolution", display_name="Hunyuan Video 1.5 Super Resolution", - category="model/conditioning/video_models", + category="model/conditioning/hunyuan video", inputs=[ io.Conditioning.Input("positive"), io.Conditioning.Input("negative"), @@ -227,7 +229,7 @@ class HunyuanVideo15LatentUpscaleWithModel(io.ComfyNode): return io.Schema( node_id="HunyuanVideo15LatentUpscaleWithModel", display_name="Hunyuan Video 15 Latent Upscale With Model", - category="model/latent", + category="model/latent/hunyhuan video", inputs=[ io.LatentUpscaleModel.Input("model"), io.Latent.Input("samples"), @@ -276,7 +278,7 @@ class TextEncodeHunyuanVideo_ImageToVideo(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="TextEncodeHunyuanVideo_ImageToVideo", - category="advanced/conditioning", + category="model/conditioning/hunyuan video", inputs=[ io.Clip.Input("clip"), io.ClipVisionOutput.Input("clip_vision_output"), @@ -308,7 +310,7 @@ class HunyuanImageToVideo(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="HunyuanImageToVideo", - category="model/conditioning/video_models", + category="model/conditioning/hunyuan video", inputs=[ io.Conditioning.Input("positive"), io.Vae.Input("vae"), @@ -359,7 +361,7 @@ class EmptyHunyuanImageLatent(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="EmptyHunyuanImageLatent", - category="model/latent", + category="model/latent/hunyuan image", inputs=[ io.Int.Input("width", default=2048, min=64, max=nodes.MAX_RESOLUTION, step=32), io.Int.Input("height", default=2048, min=64, max=nodes.MAX_RESOLUTION, step=32), @@ -384,7 +386,7 @@ class HunyuanRefinerLatent(io.ComfyNode): return io.Schema( node_id="HunyuanRefinerLatent", display_name="Hunyuan Latent Refiner", - category="model/conditioning/video_models", + category="model/conditioning/hunyuan video", inputs=[ io.Conditioning.Input("positive"), io.Conditioning.Input("negative"), diff --git a/comfy_extras/nodes_hunyuan3d.py b/comfy_extras/nodes_hunyuan3d.py index 60e530626..c5fa946cc 100644 --- a/comfy_extras/nodes_hunyuan3d.py +++ b/comfy_extras/nodes_hunyuan3d.py @@ -12,7 +12,7 @@ class EmptyLatentHunyuan3Dv2(IO.ComfyNode): def define_schema(cls): return IO.Schema( node_id="EmptyLatentHunyuan3Dv2", - category="model/latent/3d", + category="model/latent/hunyuan 3d", inputs=[ IO.Int.Input("resolution", default=3072, min=1, max=8192), IO.Int.Input("batch_size", default=1, min=1, max=4096, tooltip="The number of latent images in the batch."), @@ -35,7 +35,7 @@ class Hunyuan3Dv2Conditioning(IO.ComfyNode): def define_schema(cls): return IO.Schema( node_id="Hunyuan3Dv2Conditioning", - category="model/conditioning/3d_models", + category="model/conditioning/hunyuan 3d", inputs=[ IO.ClipVisionOutput.Input("clip_vision_output"), ], @@ -60,7 +60,7 @@ class Hunyuan3Dv2ConditioningMultiView(IO.ComfyNode): def define_schema(cls): return IO.Schema( node_id="Hunyuan3Dv2ConditioningMultiView", - category="model/conditioning/3d_models", + category="model/conditioning/hunyuan 3d", inputs=[ IO.ClipVisionOutput.Input("front", optional=True), IO.ClipVisionOutput.Input("left", optional=True), @@ -97,7 +97,7 @@ class VAEDecodeHunyuan3D(IO.ComfyNode): def define_schema(cls): return IO.Schema( node_id="VAEDecodeHunyuan3D", - category="model/latent/3d", + category="model/latent/hunyuan 3d", inputs=[ IO.Latent.Input("samples"), IO.Vae.Input("vae"), diff --git a/comfy_extras/nodes_ideogram4.py b/comfy_extras/nodes_ideogram4.py index 1c558af46..174fe6f1a 100644 --- a/comfy_extras/nodes_ideogram4.py +++ b/comfy_extras/nodes_ideogram4.py @@ -78,7 +78,7 @@ class Ideogram4Scheduler(io.ComfyNode): return io.Schema( node_id="Ideogram4Scheduler", display_name="Ideogram 4 Scheduler", - category="sampling/custom_sampling/schedulers", + category="model/sampling/schedulers", inputs=[ io.Int.Input("steps", default=20, min=1, max=200), io.Int.Input("width", default=1024, min=256, max=8192, step=16), diff --git a/comfy_extras/nodes_images.py b/comfy_extras/nodes_images.py index 469a7be55..fe1937ba5 100644 --- a/comfy_extras/nodes_images.py +++ b/comfy_extras/nodes_images.py @@ -214,11 +214,13 @@ class SaveAnimatedWEBP(IO.ComfyNode): ], hidden=[IO.Hidden.prompt, IO.Hidden.extra_pnginfo], is_output_node=True, + outputs=[IO.Image.Output(display_name="images")] ) @classmethod def execute(cls, images, fps, filename_prefix, lossless, quality, method, num_frames=0) -> IO.NodeOutput: return IO.NodeOutput( + images, ui=UI.ImageSaveHelper.get_save_animated_webp_ui( images=images, filename_prefix=filename_prefix, @@ -230,8 +232,6 @@ class SaveAnimatedWEBP(IO.ComfyNode): ) ) - save_images = execute # TODO: remove - class SaveAnimatedPNG(IO.ComfyNode): @@ -249,11 +249,13 @@ class SaveAnimatedPNG(IO.ComfyNode): ], hidden=[IO.Hidden.prompt, IO.Hidden.extra_pnginfo], is_output_node=True, + outputs=[IO.Image.Output(display_name="images")] ) @classmethod def execute(cls, images, fps, compress_level, filename_prefix="ComfyUI") -> IO.NodeOutput: return IO.NodeOutput( + images, ui=UI.ImageSaveHelper.get_save_animated_png_ui( images=images, filename_prefix=filename_prefix, @@ -263,8 +265,6 @@ class SaveAnimatedPNG(IO.ComfyNode): ) ) - save_images = execute # TODO: remove - class ImageStitch(IO.ComfyNode): """Upstreamed from https://github.com/kijai/ComfyUI-KJNodes""" @@ -513,6 +513,7 @@ class SaveSVGNode(IO.ComfyNode): ], hidden=[IO.Hidden.prompt, IO.Hidden.extra_pnginfo], is_output_node=True, + outputs=[IO.SVG.Output("svg")], ) @classmethod @@ -562,9 +563,7 @@ class SaveSVGNode(IO.ComfyNode): results.append(UI.SavedResult(filename=file, subfolder=subfolder, type=IO.FolderType.output)) counter += 1 - return IO.NodeOutput(ui={"images": results}) - - save_svg = execute # TODO: remove + return IO.NodeOutput(svg, ui={"images": results}) class GetImageSize(IO.ComfyNode): @@ -1157,40 +1156,27 @@ class SaveImageAdvanced(IO.ComfyNode): IO.String.Input( "filename_prefix", default="ComfyUI", - tooltip=( - "The prefix for the file to save. May include formatting tokens " - "such as %date:yyyy-MM-dd% or %Empty Latent Image.width%." - ), + tooltip=("The prefix for the file to save. May include formatting tokens such as %date:yyyy-MM-dd% or %Empty Latent Image.width%."), ), IO.DynamicCombo.Input( "format", options=[ IO.DynamicCombo.Option("png", [ - IO.Combo.Input("bit_depth", options=["8-bit", "16-bit"], - default="8-bit", advanced=True), - IO.Combo.Input("input_color_space", options=["sRGB"], - default="sRGB", advanced=True), + IO.Combo.Input("bit_depth", options=["8-bit", "16-bit"], default="8-bit", advanced=True), + IO.Combo.Input("input_color_space", options=["sRGB"], default="sRGB", advanced=True), ]), IO.DynamicCombo.Option("exr", [ - IO.Combo.Input("bit_depth", options=["32-bit float"], - default="32-bit float", advanced=True), + IO.Combo.Input("bit_depth", options=["32-bit float"], default="32-bit float", advanced=True), IO.Combo.Input( "input_color_space", options=["sRGB", "HDR", "linear"], default="sRGB", advanced=True, tooltip=( - "Colorspace of the input tensor. The EXR is " - "always written as scene-linear in the matching " - "gamut.\n" - " 'sRGB' — input is sRGB-encoded Rec.709; " - "the inverse sRGB EOTF is applied.\n" - " 'HDR' — input is HLG-encoded Rec.2020 " - "(BT.2100); the inverse HLG OETF is applied " - "to get scene-linear light.\n" - " 'linear' — input is already scene-linear " - "(Rec.709 primaries); written through unchanged. " - "Use this for renderer/compositor output." + "Colorspace of the input tensor. The EXR is always written as scene-linear in the matching gamut.\n" + "sRGB — input is sRGB-encoded Rec.709; the inverse sRGB EOTF is applied.\n" + "HDR — input is HLG-encoded Rec.2020 (BT.2100); the inverse HLG OETF is applied to get scene-linear light.\n" + "linear — input is already scene-linear (Rec.709 primaries); written through unchanged. Use this for renderer/compositor output." ), ), ]), @@ -1200,6 +1186,7 @@ class SaveImageAdvanced(IO.ComfyNode): ], hidden=[IO.Hidden.prompt, IO.Hidden.extra_pnginfo], is_output_node=True, + outputs=[IO.Image.Output(display_name="images")] ) @classmethod @@ -1237,7 +1224,7 @@ class SaveImageAdvanced(IO.ComfyNode): results.append({"filename": file, "subfolder": subfolder, "type": "output"}) counter += 1 - return IO.NodeOutput(ui={"images": results}) + return IO.NodeOutput(images, ui={"images": results}) class ImagesExtension(ComfyExtension): diff --git a/comfy_extras/nodes_json_prompt.py b/comfy_extras/nodes_json_prompt.py new file mode 100644 index 000000000..206f5aa71 --- /dev/null +++ b/comfy_extras/nodes_json_prompt.py @@ -0,0 +1,77 @@ +from typing_extensions import override + +from comfy_api.latest import ComfyExtension, io +from comfy_extras.color_util import normalize_palette + + +class BuildJsonPromptIdeogram(io.ComfyNode): + @classmethod + def define_schema(cls): + color_palette = io.Colors.Input( + "color_palette", + socketless=False, + tooltip="Hex color codes that steer the image's dominant colors. Up to 16 entries.", + ) + return io.Schema( + node_id="BuildJsonPromptIdeogram", + display_name="Build JSON Prompt (Ideogram)", + category="text", + description="Build a JSON prompt for the Ideogram 4 model.", + inputs=[ + io.Array.Input("element", tooltip="Prompt elements from the node Create Bounding Boxes."), + io.String.Input("high_level_description", multiline=True, default="", + tooltip="Optional description of the image in one or two sentences. Strongly recommended."), + io.String.Input("background", multiline=True, default="", + tooltip="Mandatory description of the image background or environment."), + io.DynamicCombo.Input("style", options=[ + io.DynamicCombo.Option("none", []), + io.DynamicCombo.Option("photo", [io.String.Input("photo", default="", tooltip="Camera or lens details for photographic outputs (e.g. 35mm, f/1.4, bokeh).")]), + io.DynamicCombo.Option("art_style", [io.String.Input("art_style", default="", tooltip="Art style description (e.g. flat vector illustration, bold outlines).")]), + ]), + io.String.Input("aesthetics", default="", tooltip="Mandatory aesthetic keywords (e.g. moody, cinematic, desaturated)."), + io.String.Input("lighting", default="", tooltip="Mandatory lighting description (e.g. golden hour, rim light, dramatic shadows)."), + io.String.Input("medium", default="", tooltip="Mandatory medium type (e.g. photograph, illustration, 3d_render, painting, graphic_design). When style = photo, set to photograph."), + color_palette, + ], + outputs=[io.Dict.Output(display_name="prompt")], + is_experimental=True, + ) + + @classmethod + def execute(cls, element, style, high_level_description="", background="", + aesthetics="", lighting="", medium="", color_palette=None) -> io.NodeOutput: + elements = element if isinstance(element, list) else [] + kind = style.get("style", "none") if isinstance(style, dict) else "none" + photo = style.get("photo", "") if isinstance(style, dict) else "" + art_style = style.get("art_style", "") if isinstance(style, dict) else "" + palette = normalize_palette(color_palette or []) + + caption: dict = {} + if high_level_description.strip(): + caption["high_level_description"] = high_level_description + if kind != "none": + style_desc: dict = {"aesthetics": aesthetics, "lighting": lighting} + if kind == "photo": + style_desc["photo"] = photo + style_desc["medium"] = medium + else: + style_desc["medium"] = medium + style_desc["art_style"] = art_style + if palette: + style_desc["color_palette"] = palette + caption["style_description"] = style_desc + caption["compositional_deconstruction"] = { + "background": background, + "elements": elements, + } + return io.NodeOutput(caption) + + +class JsonPromptExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [BuildJsonPromptIdeogram] + + +async def comfy_entrypoint() -> JsonPromptExtension: + return JsonPromptExtension() diff --git a/comfy_extras/nodes_kandinsky5.py b/comfy_extras/nodes_kandinsky5.py index 015965498..96cca0386 100644 --- a/comfy_extras/nodes_kandinsky5.py +++ b/comfy_extras/nodes_kandinsky5.py @@ -13,7 +13,7 @@ class Kandinsky5ImageToVideo(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="Kandinsky5ImageToVideo", - category="model/conditioning/video_models", + category="model/conditioning/kandinsky", inputs=[ io.Conditioning.Input("positive"), io.Conditioning.Input("negative"), @@ -71,7 +71,7 @@ class NormalizeVideoLatentStart(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="NormalizeVideoLatentStart", - category="model/conditioning/video_models", + category="model/conditioning", description="Normalizes the initial frames of a video latent to match the mean and standard deviation of subsequent reference frames. Helps reduce differences between the starting frames and the rest of the video.", inputs=[ io.Latent.Input("latent"), @@ -104,8 +104,9 @@ class CLIPTextEncodeKandinsky5(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="CLIPTextEncodeKandinsky5", + display_name="CLIP Text Encode (Kandinsky 5)", search_aliases=["kandinsky prompt"], - category="advanced/conditioning/kandinsky5", + category="model/conditioning/kandinsky", inputs=[ io.Clip.Input("clip"), io.String.Input("clip_l", multiline=True, dynamic_prompts=True), diff --git a/comfy_extras/nodes_latent.py b/comfy_extras/nodes_latent.py index 32da9e8ac..1f93e34d6 100644 --- a/comfy_extras/nodes_latent.py +++ b/comfy_extras/nodes_latent.py @@ -262,6 +262,7 @@ class LatentBatch(io.ComfyNode): return io.Schema( node_id="LatentBatch", search_aliases=["combine latents", "merge latents", "join latents"], + display_name="Batch Latents (DEPRECATED)", category="model/latent/batch", is_deprecated=True, inputs=[ @@ -447,6 +448,7 @@ class ReplaceVideoLatentFrames(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="ReplaceVideoLatentFrames", + display_name="Replace Video Latent Frames", category="model/latent/batch", inputs=[ io.Latent.Input("destination", tooltip="The destination latent where frames will be replaced."), diff --git a/comfy_extras/nodes_load_3d.py b/comfy_extras/nodes_load_3d.py index 455897859..6e3e88471 100644 --- a/comfy_extras/nodes_load_3d.py +++ b/comfy_extras/nodes_load_3d.py @@ -317,11 +317,74 @@ class PreviewPointCloud(IO.ComfyNode): ) +MESH_EXTENSIONS = {'.gltf', '.glb', '.obj', '.fbx', '.stl'} + + +class Load3DAdvanced(IO.ComfyNode): + @classmethod + def define_schema(cls): + input_dir = os.path.join(folder_paths.get_input_directory(), "3d") + os.makedirs(input_dir, exist_ok=True) + + input_path = Path(input_dir) + base_path = Path(folder_paths.get_input_directory()) + + files = [ + normalize_path(str(file_path.relative_to(base_path))) + for file_path in input_path.rglob("*") + if file_path.suffix.lower() in MESH_EXTENSIONS + ] + return IO.Schema( + node_id="Load3DAdvanced", + display_name="Load 3D (Advanced)", + category="3d", + search_aliases=[ + "load mesh", + "load gltf", + "load glb", + "load obj", + "load fbx", + "load stl", + ], + is_experimental=True, + inputs=[ + IO.Combo.Input("model_file", options=["none"] + sorted(files), upload=IO.UploadType.model), + IO.Load3D.Input("viewport_state"), + IO.Int.Input("width", default=1024, min=1, max=4096, step=1), + IO.Int.Input("height", default=1024, min=1, max=4096, step=1), + ], + outputs=[ + IO.File3DAny.Output(display_name="model_3d"), + IO.Load3DModelInfo.Output(display_name="model_3d_info"), + IO.Load3DCamera.Output(display_name="camera_info"), + IO.Int.Output(display_name="width"), + IO.Int.Output(display_name="height"), + ], + ) + + @classmethod + def validate_inputs(cls, model_file, **kwargs) -> bool | str: + if not model_file or model_file == "none": + return True + if not folder_paths.exists_annotated_filepath(model_file): + return f"Invalid 3D model file: {model_file}" + return True + + @classmethod + def execute(cls, model_file, viewport_state, width: int, height: int, **kwargs) -> IO.NodeOutput: + file_3d = None + if model_file and model_file != "none": + file_3d = Types.File3D(folder_paths.get_annotated_filepath(model_file)) + model_3d_info = viewport_state.get('model_3d_info', []) + return IO.NodeOutput(file_3d, model_3d_info, viewport_state['camera_info'], width, height) + + class Load3DExtension(ComfyExtension): @override async def get_node_list(self) -> list[type[IO.ComfyNode]]: return [ Load3D, + Load3DAdvanced, Preview3D, Preview3DAdvanced, PreviewGaussianSplat, diff --git a/comfy_extras/nodes_logic.py b/comfy_extras/nodes_logic.py index 95f6ab848..13c1685f7 100644 --- a/comfy_extras/nodes_logic.py +++ b/comfy_extras/nodes_logic.py @@ -89,7 +89,8 @@ class SwitchNode(io.ComfyNode): template = io.MatchType.Template("switch") return io.Schema( node_id="ComfySwitchNode", - display_name="Switch", + search_aliases=["if", "then", "switch", "conditional", "branch"], + display_name="If/Else Switch", category="utilities/logic", is_experimental=True, inputs=[ diff --git a/comfy_extras/nodes_lt.py b/comfy_extras/nodes_lt.py index 6d6078abe..85d76ecef 100644 --- a/comfy_extras/nodes_lt.py +++ b/comfy_extras/nodes_lt.py @@ -25,7 +25,7 @@ class GetICLoRAParameters(io.ComfyNode): display_name="Get IC-LoRA Parameters", description="Extracts IC-LoRA parameters from the safetensors metadata of a LoRA-loaded " "model and outputs them for LTXVAddGuide (eg. reference_downscale_factor).", - category="model/conditioning/video_models", + category="model/conditioning/ltxv", search_aliases=["ic-lora", "ic lora", "iclora", "downscale factor", "reference downscale"], inputs=[ io.Model.Input( @@ -62,7 +62,7 @@ class EmptyLTXVLatentVideo(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="EmptyLTXVLatentVideo", - category="model/latent/video/ltxv", + category="model/latent/ltxv", inputs=[ io.Int.Input("width", default=768, min=64, max=nodes.MAX_RESOLUTION, step=32), io.Int.Input("height", default=512, min=64, max=nodes.MAX_RESOLUTION, step=32), @@ -86,7 +86,7 @@ class LTXVImgToVideo(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="LTXVImgToVideo", - category="model/conditioning/video_models", + category="model/conditioning/ltxv", inputs=[ io.Conditioning.Input("positive"), io.Conditioning.Input("negative"), @@ -131,7 +131,7 @@ class LTXVImgToVideoInplace(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="LTXVImgToVideoInplace", - category="model/conditioning/video_models", + category="model/conditioning/ltxv", inputs=[ io.Vae.Input("vae"), io.Image.Input("image"), @@ -251,7 +251,7 @@ class LTXVAddGuide(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="LTXVAddGuide", - category="model/conditioning/video_models", + category="model/conditioning/ltxv", inputs=[ io.Conditioning.Input("positive"), io.Conditioning.Input("negative"), @@ -498,7 +498,7 @@ class LTXVCropGuides(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="LTXVCropGuides", - category="model/conditioning/video_models", + category="model/conditioning/ltxv", inputs=[ io.Conditioning.Input("positive"), io.Conditioning.Input("negative"), @@ -542,7 +542,7 @@ class LTXVConditioning(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="LTXVConditioning", - category="model/conditioning/video_models", + category="model/conditioning/ltxv", inputs=[ io.Conditioning.Input("positive"), io.Conditioning.Input("negative"), @@ -566,7 +566,7 @@ class ModelSamplingLTXV(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="ModelSamplingLTXV", - category="advanced/model", + category="model/patch/ltxv", inputs=[ io.Model.Input("model"), io.Float.Input("max_shift", default=2.05, min=0.0, max=100.0, step=0.01), @@ -746,7 +746,7 @@ class LTXVConcatAVLatent(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="LTXVConcatAVLatent", - category="model/latent/video/ltxv", + category="model/latent/ltxv", inputs=[ io.Latent.Input("video_latent"), io.Latent.Input("audio_latent"), @@ -781,7 +781,7 @@ class LTXVSeparateAVLatent(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="LTXVSeparateAVLatent", - category="model/latent/video/ltxv", + category="model/latent/ltxv", description="LTXV Separate AV Latent", inputs=[ io.Latent.Input("av_latent"), @@ -814,7 +814,7 @@ class LTXVReferenceAudio(io.ComfyNode): return io.Schema( node_id="LTXVReferenceAudio", display_name="LTXV Reference Audio (ID-LoRA)", - category="model/conditioning/audio", + category="model/conditioning/ltxv", description="Set reference audio for ID-LoRA speaker identity transfer. Encodes a reference audio clip into the conditioning and optionally patches the model with identity guidance (extra forward pass without reference, amplifying the speaker identity effect).", inputs=[ io.Model.Input("model"), diff --git a/comfy_extras/nodes_lt_audio.py b/comfy_extras/nodes_lt_audio.py index 052186083..2d774a0a3 100644 --- a/comfy_extras/nodes_lt_audio.py +++ b/comfy_extras/nodes_lt_audio.py @@ -40,7 +40,7 @@ class LTXVAudioVAEEncode(VAEEncodeAudio): return io.Schema( node_id="LTXVAudioVAEEncode", display_name="LTXV Audio VAE Encode", - category="model/latent/audio", + category="model/latent/ltxv", inputs=[ io.Audio.Input("audio", tooltip="The audio to be encoded."), io.Vae.Input( @@ -63,7 +63,7 @@ class LTXVAudioVAEDecode(io.ComfyNode): return io.Schema( node_id="LTXVAudioVAEDecode", display_name="LTXV Audio VAE Decode", - category="model/latent/audio", + category="model/latent/ltxv", inputs=[ io.Latent.Input("samples", tooltip="The latent to be decoded."), io.Vae.Input( @@ -96,7 +96,7 @@ class LTXVEmptyLatentAudio(io.ComfyNode): return io.Schema( node_id="LTXVEmptyLatentAudio", display_name="LTXV Empty Latent Audio", - category="model/latent/audio", + category="model/latent/ltxv", inputs=[ io.Int.Input( "frames_number", @@ -168,9 +168,9 @@ class LTXAVTextEncoderLoader(io.ComfyNode): def define_schema(cls) -> io.Schema: return io.Schema( node_id="LTXAVTextEncoderLoader", - display_name="LTXV Audio Text Encoder Loader", - category="advanced/loaders", - description="[Recipes]\n\nltxav: gemma 3 12B", + display_name="Load LTXV Audio Text Encoder", + category="model/loaders", + description="Recipes:\nltxav: gemma 3 12B", inputs=[ io.Combo.Input( "text_encoder", diff --git a/comfy_extras/nodes_lt_upsampler.py b/comfy_extras/nodes_lt_upsampler.py index be9a36e69..ef36109d1 100644 --- a/comfy_extras/nodes_lt_upsampler.py +++ b/comfy_extras/nodes_lt_upsampler.py @@ -13,7 +13,7 @@ class LTXVLatentUpsampler(IO.ComfyNode): def define_schema(cls): return IO.Schema( node_id="LTXVLatentUpsampler", - category="model/latent/video", + category="model/latent/ltxv", is_experimental=True, inputs=[ IO.Latent.Input("samples"), diff --git a/comfy_extras/nodes_lumina2.py b/comfy_extras/nodes_lumina2.py index c060a86a0..bc543c242 100644 --- a/comfy_extras/nodes_lumina2.py +++ b/comfy_extras/nodes_lumina2.py @@ -9,7 +9,7 @@ class RenormCFG(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="RenormCFG", - category="advanced/model", + category="model/patch", inputs=[ io.Model.Input("model"), io.Float.Input("cfg_trunc", default=100, min=0.0, max=100.0, step=0.01, advanced=True), @@ -80,8 +80,8 @@ class CLIPTextEncodeLumina2(io.ComfyNode): return io.Schema( node_id="CLIPTextEncodeLumina2", search_aliases=["lumina prompt"], - display_name="CLIP Text Encode for Lumina2", - category="model/conditioning", + display_name="CLIP Text Encode (Lumina 2)", + category="model/conditioning/lumina", description="Encodes a system prompt and a user prompt using a CLIP model into an embedding " "that can be used to guide the diffusion model towards generating specific images.", inputs=[ diff --git a/comfy_extras/nodes_mask.py b/comfy_extras/nodes_mask.py index 52484697a..76af338de 100644 --- a/comfy_extras/nodes_mask.py +++ b/comfy_extras/nodes_mask.py @@ -53,6 +53,7 @@ class LatentCompositeMasked(IO.ComfyNode): return IO.Schema( node_id="LatentCompositeMasked", search_aliases=["overlay latent", "layer latent", "paste latent", "inpaint latent"], + display_name="Latent Composite Masked", category="model/latent", inputs=[ IO.Latent.Input("destination"), diff --git a/comfy_extras/nodes_mochi.py b/comfy_extras/nodes_mochi.py index 3dcea6ab3..3aaf23e69 100644 --- a/comfy_extras/nodes_mochi.py +++ b/comfy_extras/nodes_mochi.py @@ -10,7 +10,7 @@ class EmptyMochiLatentVideo(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="EmptyMochiLatentVideo", - category="model/latent/video", + category="model/latent/mochi", inputs=[ io.Int.Input("width", default=848, min=16, max=nodes.MAX_RESOLUTION, step=16), io.Int.Input("height", default=480, min=16, max=nodes.MAX_RESOLUTION, step=16), diff --git a/comfy_extras/nodes_model_advanced.py b/comfy_extras/nodes_model_advanced.py index b27ac1296..a336ba079 100644 --- a/comfy_extras/nodes_model_advanced.py +++ b/comfy_extras/nodes_model_advanced.py @@ -59,7 +59,7 @@ class ModelSamplingDiscrete: RETURN_TYPES = ("MODEL",) FUNCTION = "patch" - CATEGORY = "advanced/model" + CATEGORY = "model/patch" def patch(self, model, sampling, zsnr): m = model.clone() @@ -97,7 +97,7 @@ class ModelSamplingStableCascade: RETURN_TYPES = ("MODEL",) FUNCTION = "patch" - CATEGORY = "advanced/model" + CATEGORY = "model/patch/stable cascade" def patch(self, model, shift): m = model.clone() @@ -123,7 +123,7 @@ class ModelSamplingSD3: RETURN_TYPES = ("MODEL",) FUNCTION = "patch" - CATEGORY = "advanced/model" + CATEGORY = "model/patch/stable diffusion" def patch(self, model, shift, multiplier=1000): m = model.clone() @@ -150,6 +150,7 @@ class ModelSamplingAuraFlow(ModelSamplingSD3): }} FUNCTION = "patch_aura" + CATEGORY = "model/patch" def patch_aura(self, model, shift): return self.patch(model, shift, multiplier=1.0) @@ -167,7 +168,7 @@ class ModelSamplingFlux: RETURN_TYPES = ("MODEL",) FUNCTION = "patch" - CATEGORY = "advanced/model" + CATEGORY = "model/patch/flux" def patch(self, model, max_shift, base_shift, width, height): m = model.clone() @@ -202,7 +203,7 @@ class ModelSamplingContinuousEDM: RETURN_TYPES = ("MODEL",) FUNCTION = "patch" - CATEGORY = "advanced/model" + CATEGORY = "model/patch" def patch(self, model, sampling, sigma_max, sigma_min): m = model.clone() @@ -247,7 +248,7 @@ class ModelSamplingContinuousV: RETURN_TYPES = ("MODEL",) FUNCTION = "patch" - CATEGORY = "advanced/model" + CATEGORY = "model/patch" def patch(self, model, sampling, sigma_max, sigma_min): m = model.clone() @@ -273,7 +274,7 @@ class RescaleCFG: RETURN_TYPES = ("MODEL",) FUNCTION = "patch" - CATEGORY = "advanced/model" + CATEGORY = "model/patch" def patch(self, model, multiplier): def rescale_cfg(args): @@ -314,7 +315,7 @@ class ModelNoiseScale: RETURN_TYPES = ("MODEL",) FUNCTION = "patch" - CATEGORY = "advanced/model" + CATEGORY = "model/patch" def patch(self, model, noise_scale): m = model.clone() @@ -337,7 +338,7 @@ class ModelComputeDtype: RETURN_TYPES = ("MODEL",) FUNCTION = "patch" - CATEGORY = "advanced/debug/model" + CATEGORY = "advanced/debug" def patch(self, model, dtype): m = model.clone() diff --git a/comfy_extras/nodes_model_merging.py b/comfy_extras/nodes_model_merging.py index b6b29e34a..962d2a0bb 100644 --- a/comfy_extras/nodes_model_merging.py +++ b/comfy_extras/nodes_model_merging.py @@ -21,7 +21,7 @@ class ModelMergeSimple: RETURN_TYPES = ("MODEL",) FUNCTION = "merge" - CATEGORY = "advanced/model_merging" + CATEGORY = "model/merging" def merge(self, model1, model2, ratio): m = model1.clone() @@ -40,7 +40,7 @@ class ModelSubtract: RETURN_TYPES = ("MODEL",) FUNCTION = "merge" - CATEGORY = "advanced/model_merging" + CATEGORY = "model/merging" def merge(self, model1, model2, multiplier): m = model1.clone() @@ -58,7 +58,7 @@ class ModelAdd: RETURN_TYPES = ("MODEL",) FUNCTION = "merge" - CATEGORY = "advanced/model_merging" + CATEGORY = "model/merging" def merge(self, model1, model2): m = model1.clone() @@ -78,7 +78,7 @@ class CLIPMergeSimple: RETURN_TYPES = ("CLIP",) FUNCTION = "merge" - CATEGORY = "advanced/model_merging" + CATEGORY = "model/merging" def merge(self, clip1, clip2, ratio): m = clip1.clone() @@ -101,7 +101,7 @@ class CLIPSubtract: RETURN_TYPES = ("CLIP",) FUNCTION = "merge" - CATEGORY = "advanced/model_merging" + CATEGORY = "model/merging" def merge(self, clip1, clip2, multiplier): m = clip1.clone() @@ -123,7 +123,7 @@ class CLIPAdd: RETURN_TYPES = ("CLIP",) FUNCTION = "merge" - CATEGORY = "advanced/model_merging" + CATEGORY = "model/merging" def merge(self, clip1, clip2): m = clip1.clone() @@ -147,7 +147,7 @@ class ModelMergeBlocks: RETURN_TYPES = ("MODEL",) FUNCTION = "merge" - CATEGORY = "advanced/model_merging" + CATEGORY = "model/merging" def merge(self, model1, model2, **kwargs): m = model1.clone() @@ -242,7 +242,7 @@ class CheckpointSave: FUNCTION = "save" OUTPUT_NODE = True - CATEGORY = "advanced/model_merging" + CATEGORY = "model/merging" def save(self, model, clip, vae, filename_prefix, prompt=None, extra_pnginfo=None): save_checkpoint(model, clip=clip, vae=vae, filename_prefix=filename_prefix, output_dir=self.output_dir, prompt=prompt, extra_pnginfo=extra_pnginfo) @@ -261,7 +261,7 @@ class CLIPSave: FUNCTION = "save" OUTPUT_NODE = True - CATEGORY = "advanced/model_merging" + CATEGORY = "model/merging" def save(self, clip, filename_prefix, prompt=None, extra_pnginfo=None): prompt_info = "" @@ -318,7 +318,7 @@ class VAESave: FUNCTION = "save" OUTPUT_NODE = True - CATEGORY = "advanced/model_merging" + CATEGORY = "model/merging" def save(self, vae, filename_prefix, prompt=None, extra_pnginfo=None): full_output_folder, filename, counter, subfolder, filename_prefix = folder_paths.get_save_image_path(filename_prefix, self.output_dir) @@ -353,7 +353,7 @@ class ModelSave: FUNCTION = "save" OUTPUT_NODE = True - CATEGORY = "advanced/model_merging" + CATEGORY = "model/merging" def save(self, model, filename_prefix, prompt=None, extra_pnginfo=None): save_checkpoint(model, filename_prefix=filename_prefix, output_dir=self.output_dir, prompt=prompt, extra_pnginfo=extra_pnginfo) diff --git a/comfy_extras/nodes_model_merging_model_specific.py b/comfy_extras/nodes_model_merging_model_specific.py index 55eb3ccfe..e563d950b 100644 --- a/comfy_extras/nodes_model_merging_model_specific.py +++ b/comfy_extras/nodes_model_merging_model_specific.py @@ -1,7 +1,7 @@ import comfy_extras.nodes_model_merging class ModelMergeSD1(comfy_extras.nodes_model_merging.ModelMergeBlocks): - CATEGORY = "advanced/model_merging/model_specific" + CATEGORY = "model/merging/model specific" @classmethod def INPUT_TYPES(s): arg_dict = { "model1": ("MODEL",), @@ -27,7 +27,7 @@ class ModelMergeSD1(comfy_extras.nodes_model_merging.ModelMergeBlocks): class ModelMergeSDXL(comfy_extras.nodes_model_merging.ModelMergeBlocks): - CATEGORY = "advanced/model_merging/model_specific" + CATEGORY = "model/merging/model specific" @classmethod def INPUT_TYPES(s): @@ -53,7 +53,7 @@ class ModelMergeSDXL(comfy_extras.nodes_model_merging.ModelMergeBlocks): return {"required": arg_dict} class ModelMergeSD3_2B(comfy_extras.nodes_model_merging.ModelMergeBlocks): - CATEGORY = "advanced/model_merging/model_specific" + CATEGORY = "model/merging/model specific" @classmethod def INPUT_TYPES(s): @@ -77,7 +77,7 @@ class ModelMergeSD3_2B(comfy_extras.nodes_model_merging.ModelMergeBlocks): class ModelMergeAuraflow(comfy_extras.nodes_model_merging.ModelMergeBlocks): - CATEGORY = "advanced/model_merging/model_specific" + CATEGORY = "model/merging/model specific" @classmethod def INPUT_TYPES(s): @@ -104,7 +104,7 @@ class ModelMergeAuraflow(comfy_extras.nodes_model_merging.ModelMergeBlocks): return {"required": arg_dict} class ModelMergeFlux1(comfy_extras.nodes_model_merging.ModelMergeBlocks): - CATEGORY = "advanced/model_merging/model_specific" + CATEGORY = "model/merging/model specific" @classmethod def INPUT_TYPES(s): @@ -130,7 +130,7 @@ class ModelMergeFlux1(comfy_extras.nodes_model_merging.ModelMergeBlocks): return {"required": arg_dict} class ModelMergeSD35_Large(comfy_extras.nodes_model_merging.ModelMergeBlocks): - CATEGORY = "advanced/model_merging/model_specific" + CATEGORY = "model/merging/model specific" @classmethod def INPUT_TYPES(s): @@ -153,7 +153,7 @@ class ModelMergeSD35_Large(comfy_extras.nodes_model_merging.ModelMergeBlocks): return {"required": arg_dict} class ModelMergeMochiPreview(comfy_extras.nodes_model_merging.ModelMergeBlocks): - CATEGORY = "advanced/model_merging/model_specific" + CATEGORY = "model/merging/model specific" @classmethod def INPUT_TYPES(s): @@ -175,7 +175,7 @@ class ModelMergeMochiPreview(comfy_extras.nodes_model_merging.ModelMergeBlocks): return {"required": arg_dict} class ModelMergeLTXV(comfy_extras.nodes_model_merging.ModelMergeBlocks): - CATEGORY = "advanced/model_merging/model_specific" + CATEGORY = "model/merging/model specific" @classmethod def INPUT_TYPES(s): @@ -197,7 +197,7 @@ class ModelMergeLTXV(comfy_extras.nodes_model_merging.ModelMergeBlocks): return {"required": arg_dict} class ModelMergeCosmos7B(comfy_extras.nodes_model_merging.ModelMergeBlocks): - CATEGORY = "advanced/model_merging/model_specific" + CATEGORY = "model/merging/model specific" @classmethod def INPUT_TYPES(s): @@ -221,7 +221,7 @@ class ModelMergeCosmos7B(comfy_extras.nodes_model_merging.ModelMergeBlocks): return {"required": arg_dict} class ModelMergeCosmos14B(comfy_extras.nodes_model_merging.ModelMergeBlocks): - CATEGORY = "advanced/model_merging/model_specific" + CATEGORY = "model/merging/model specific" @classmethod def INPUT_TYPES(s): @@ -245,7 +245,7 @@ class ModelMergeCosmos14B(comfy_extras.nodes_model_merging.ModelMergeBlocks): return {"required": arg_dict} class ModelMergeWAN2_1(comfy_extras.nodes_model_merging.ModelMergeBlocks): - CATEGORY = "advanced/model_merging/model_specific" + CATEGORY = "model/merging/model specific" DESCRIPTION = "1.3B model has 30 blocks, 14B model has 40 blocks. Image to video model has the extra img_emb." @classmethod @@ -269,7 +269,7 @@ class ModelMergeWAN2_1(comfy_extras.nodes_model_merging.ModelMergeBlocks): return {"required": arg_dict} class ModelMergeCosmosPredict2_2B(comfy_extras.nodes_model_merging.ModelMergeBlocks): - CATEGORY = "advanced/model_merging/model_specific" + CATEGORY = "model/merging/model specific" @classmethod def INPUT_TYPES(s): @@ -292,7 +292,7 @@ class ModelMergeCosmosPredict2_2B(comfy_extras.nodes_model_merging.ModelMergeBlo return {"required": arg_dict} class ModelMergeCosmosPredict2_14B(comfy_extras.nodes_model_merging.ModelMergeBlocks): - CATEGORY = "advanced/model_merging/model_specific" + CATEGORY = "model/merging/model specific" @classmethod def INPUT_TYPES(s): @@ -315,7 +315,7 @@ class ModelMergeCosmosPredict2_14B(comfy_extras.nodes_model_merging.ModelMergeBl return {"required": arg_dict} class ModelMergeQwenImage(comfy_extras.nodes_model_merging.ModelMergeBlocks): - CATEGORY = "advanced/model_merging/model_specific" + CATEGORY = "model/merging/model specific" @classmethod def INPUT_TYPES(s): @@ -337,6 +337,36 @@ class ModelMergeQwenImage(comfy_extras.nodes_model_merging.ModelMergeBlocks): return {"required": arg_dict} +class ModelMergeKrea2(comfy_extras.nodes_model_merging.ModelMergeBlocks): + CATEGORY = "model/merging/model specific" + + @classmethod + def INPUT_TYPES(s): + arg_dict = { "model1": ("MODEL",), + "model2": ("MODEL",)} + + argument = ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01}) + + arg_dict["first."] = argument + arg_dict["tmlp."] = argument + arg_dict["txtmlp."] = argument + arg_dict["tproj."] = argument + + for i in range(2): + arg_dict["txtfusion.layerwise_blocks.{}.".format(i)] = argument + + arg_dict["txtfusion.projector."] = argument + + for i in range(2): + arg_dict["txtfusion.refiner_blocks.{}.".format(i)] = argument + + for i in range(28): + arg_dict["blocks.{}.".format(i)] = argument + + arg_dict["last."] = argument + + return {"required": arg_dict} + NODE_CLASS_MAPPINGS = { "ModelMergeSD1": ModelMergeSD1, "ModelMergeSD2": ModelMergeSD1, #SD1 and SD2 have the same blocks @@ -353,4 +383,5 @@ NODE_CLASS_MAPPINGS = { "ModelMergeCosmosPredict2_2B": ModelMergeCosmosPredict2_2B, "ModelMergeCosmosPredict2_14B": ModelMergeCosmosPredict2_14B, "ModelMergeQwenImage": ModelMergeQwenImage, + "ModelMergeKrea2": ModelMergeKrea2, } diff --git a/comfy_extras/nodes_model_patch.py b/comfy_extras/nodes_model_patch.py index bdccbf8c4..3f785c8b5 100644 --- a/comfy_extras/nodes_model_patch.py +++ b/comfy_extras/nodes_model_patch.py @@ -232,7 +232,7 @@ class ModelPatchLoader: FUNCTION = "load_model_patch" EXPERIMENTAL = True - CATEGORY = "advanced/loaders" + CATEGORY = "model/loaders" def load_model_patch(self, name): model_patch_path = folder_paths.get_full_path_or_raise("model_patches", name) @@ -479,7 +479,7 @@ class QwenImageDiffsynthControlnet: FUNCTION = "diffsynth_controlnet" EXPERIMENTAL = True - CATEGORY = "advanced/loaders/qwen" + CATEGORY = "model/patch/qwen" def diffsynth_controlnet(self, model, model_patch, vae, image=None, strength=1.0, inpaint_image=None, mask=None): model_patched = model.clone() @@ -512,7 +512,7 @@ class ZImageFunControlnet(QwenImageDiffsynthControlnet): }, "optional": {"image": ("IMAGE",), "inpaint_image": ("IMAGE",), "mask": ("MASK",)}} - CATEGORY = "advanced/loaders/zimage" + CATEGORY = "model/patch/z-image" class UsoStyleProjectorPatch: def __init__(self, model_patch, encoded_image): @@ -675,3 +675,11 @@ NODE_CLASS_MAPPINGS = { "USOStyleReference": USOStyleReference, "SUPIRApply": SUPIRApply, } + +NODE_DISPLAY_NAME_MAPPINGS = { + "ModelPatchLoader": "Load Model Patch", + "QwenImageDiffsynthControlnet": "Apply Qwen Image DiffSynth ControlNet", + "ZImageFunControlnet": "Apply Z-Image Fun ControlNet", + "USOStyleReference": "Apply USO Style Reference", + "SUPIRApply": "Apply SUPIR Patch", +} diff --git a/comfy_extras/nodes_photomaker.py b/comfy_extras/nodes_photomaker.py index 8a2248572..72fad1673 100644 --- a/comfy_extras/nodes_photomaker.py +++ b/comfy_extras/nodes_photomaker.py @@ -123,7 +123,8 @@ class PhotoMakerLoader(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="PhotoMakerLoader", - category="experimental/photomaker", + display_name="Load PhotoMaker Model", + category="model/loaders", inputs=[ io.Combo.Input("photomaker_model_name", options=folder_paths.get_filename_list("photomaker")), ], @@ -149,7 +150,8 @@ class PhotoMakerEncode(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="PhotoMakerEncode", - category="experimental/photomaker", + display_name="PhotoMaker Encode", + category="model/conditioning/photomaker", inputs=[ io.Photomaker.Input("photomaker"), io.Image.Input("image"), diff --git a/comfy_extras/nodes_pid.py b/comfy_extras/nodes_pid.py index 71855254e..a3ffd9671 100644 --- a/comfy_extras/nodes_pid.py +++ b/comfy_extras/nodes_pid.py @@ -14,10 +14,8 @@ class PiDConditioning(io.ComfyNode): return io.Schema( node_id="PiDConditioning", display_name="PiD Conditioning", - category="advanced/conditioning", - description=( - "Attaches a latent and a degrade_sigma scalar to a CONDITIONING for PiD decoding/upscaling" - ), + category="model/conditioning", + description=("Attaches a latent and a degrade_sigma scalar to a CONDITIONING for PiD decoding/upscaling"), inputs=[ io.Conditioning.Input("positive"), io.Latent.Input("latent", tooltip="latent (from VAEEncode or a KSampler)."), diff --git a/comfy_extras/nodes_pixart.py b/comfy_extras/nodes_pixart.py index 2f1b73e60..f878a33b5 100644 --- a/comfy_extras/nodes_pixart.py +++ b/comfy_extras/nodes_pixart.py @@ -7,8 +7,9 @@ class CLIPTextEncodePixArtAlpha(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="CLIPTextEncodePixArtAlpha", + display_name="CLIP Text Encode (PixArt Alpha)", search_aliases=["pixart prompt"], - category="advanced/conditioning", + category="model/conditioning/pixart", description="Encodes text and sets the resolution conditioning for PixArt Alpha. Does not apply to PixArt Sigma.", inputs=[ io.Int.Input("width", default=1024, min=0, max=nodes.MAX_RESOLUTION), diff --git a/comfy_extras/nodes_post_processing.py b/comfy_extras/nodes_post_processing.py index 3e440433e..763b8a52f 100644 --- a/comfy_extras/nodes_post_processing.py +++ b/comfy_extras/nodes_post_processing.py @@ -616,7 +616,7 @@ class BatchLatentsNode(io.ComfyNode): node_id="BatchLatentsNode", search_aliases=["combine latents", "stack latents", "merge latents"], display_name="Batch Latents", - category="model/latent", + category="model/latent/batch", inputs=[ io.Autogrow.Input("latents", template=autogrow_template) ], diff --git a/comfy_extras/nodes_primitive.py b/comfy_extras/nodes_primitive.py index c44b09098..7f90daf14 100644 --- a/comfy_extras/nodes_primitive.py +++ b/comfy_extras/nodes_primitive.py @@ -10,12 +10,11 @@ class String(io.ComfyNode): return io.Schema( node_id="PrimitiveString", search_aliases=["text", "string", "text box", "prompt"], - display_name="Text String", + display_name="Text String (DEPRECATED)", category="utilities/primitive", - inputs=[ - io.String.Input("value"), - ], + inputs=[io.String.Input("value")], outputs=[io.String.Output()], + is_deprecated=True ) @classmethod @@ -29,12 +28,10 @@ class StringMultiline(io.ComfyNode): return io.Schema( node_id="PrimitiveStringMultiline", search_aliases=["text", "string", "text multiline", "string multiline", "text box", "prompt"], - display_name="Text String (Multiline)", + display_name="Input Text", category="utilities/primitive", essentials_category="Basics", - inputs=[ - io.String.Input("value", multiline=True), - ], + inputs=[io.String.Input("value", multiline=True)], outputs=[io.String.Output()], ) diff --git a/comfy_extras/nodes_qwen.py b/comfy_extras/nodes_qwen.py index 5b92814a4..4960774db 100644 --- a/comfy_extras/nodes_qwen.py +++ b/comfy_extras/nodes_qwen.py @@ -12,7 +12,7 @@ class TextEncodeQwenImageEdit(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="TextEncodeQwenImageEdit", - category="advanced/conditioning", + category="model/conditioning/qwen image", inputs=[ io.Clip.Input("clip"), io.String.Input("prompt", multiline=True, dynamic_prompts=True), @@ -55,7 +55,7 @@ class TextEncodeQwenImageEditPlus(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="TextEncodeQwenImageEditPlus", - category="advanced/conditioning", + category="model/conditioning/qwen image", inputs=[ io.Clip.Input("clip"), io.String.Input("prompt", multiline=True, dynamic_prompts=True), diff --git a/comfy_extras/nodes_rtdetr.py b/comfy_extras/nodes_rtdetr.py index e5a9b3902..653f3af2f 100644 --- a/comfy_extras/nodes_rtdetr.py +++ b/comfy_extras/nodes_rtdetr.py @@ -14,7 +14,7 @@ class RTDETR_detect(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="RTDETR_detect", - display_name="RT-DETR Detect", + display_name="Run Real-Time Detection (RT-DETR)", category="image/detection", search_aliases=["bbox", "bounding box", "object detection", "coco"], inputs=[ diff --git a/comfy_extras/nodes_sam3.py b/comfy_extras/nodes_sam3.py index daac52f9b..f88aec925 100644 --- a/comfy_extras/nodes_sam3.py +++ b/comfy_extras/nodes_sam3.py @@ -264,7 +264,7 @@ class SAM3_VideoTrack(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="SAM3_VideoTrack", - display_name="SAM3 Video Track", + display_name="Run SAM3 Video Track", category="image/detection", search_aliases=["sam3", "video", "track", "propagate"], inputs=[ diff --git a/comfy_extras/nodes_scail.py b/comfy_extras/nodes_scail.py index a740442de..55c9897e3 100644 --- a/comfy_extras/nodes_scail.py +++ b/comfy_extras/nodes_scail.py @@ -34,14 +34,20 @@ def _unpack(track_data): return unpack_masks(packed) -def _first_frame_cx_area(masks_bool): - first = masks_bool[0].float() - H, W = first.shape[-2], first.shape[-1] - n_pixels = H * W - grid_x = torch.arange(W, device=first.device, dtype=first.dtype).view(1, W) - area = first.sum(dim=(-1, -2)).clamp_(min=1) - cx = (first * grid_x).sum(dim=(-1, -2)) / area - return (cx / W).tolist(), (area / n_pixels).tolist() +def _first_appearance_cx_area(masks_bool): + """Per object: first frame it appears in, plus centroid-x and area in that frame.""" + m = masks_bool.float() + T, H, W = m.shape[0], m.shape[-2], m.shape[-1] + grid_x = torch.arange(W, device=m.device, dtype=m.dtype).view(1, 1, 1, W) + area_t = m.sum(dim=(-1, -2)) + cx_t = (m * grid_x).sum(dim=(-1, -2)) / area_t.clamp(min=1) + present = area_t > 0 + frame_idx = torch.arange(T, device=m.device).unsqueeze(1) + first_t = torch.where(present, frame_idx, T).amin(dim=0) + sel = first_t.clamp(max=T - 1).unsqueeze(0) + cx = cx_t.gather(0, sel).squeeze(0) + area = area_t.gather(0, sel).squeeze(0) + return first_t.tolist(), (cx / W).tolist(), (area / (H * W)).tolist() def _subset_track_data(track_data, obj_indices): @@ -81,12 +87,26 @@ def _render_colored_masks(track_data, background="black"): masks_full.view(T * N_obj, 1, Hm, Wm), size=(H, W), mode="nearest" ).view(T, N_obj, H, W) > 0.5 any_mask = masks_full.any(dim=1) - obj_idx_map = masks_full.to(torch.uint8).argmax(dim=1) - color_overlay = colors[obj_idx_map] + color_overlay = colors[masks_full.to(torch.uint8).argmax(dim=1)] bg_tensor = torch.tensor(bg_rgb, device=device, dtype=color_overlay.dtype).view(1, 1, 1, 3) return torch.where(any_mask.unsqueeze(-1), color_overlay, bg_tensor.expand_as(color_overlay)) +def _render_mask_as_identity(mask, background="black"): + """Plain comfy MASK (B,H,W) or (H,W) -> (B,H,W,3) rendered as a single identity (palette[0]) + on the given background. A batch is treated as multiple views of that one subject.""" + device = comfy.model_management.intermediate_device() + dtype = comfy.model_management.intermediate_dtype() + if mask.ndim == 2: + mask = mask.unsqueeze(0) + mask = mask.to(device=device, dtype=dtype) + B, H, W = mask.shape + bg_rgb = (1.0, 1.0, 1.0) if background.startswith("white") else (0.0, 0.0, 0.0) + color = torch.tensor(DEFAULT_PALETTE[0], device=device, dtype=dtype).view(1, 1, 1, 3) + bg = torch.tensor(bg_rgb, device=device, dtype=dtype).view(1, 1, 1, 3) + return torch.where((mask > 0.5).unsqueeze(-1), color.expand(B, H, W, 3), bg.expand(B, H, W, 3)) + + def _extract_mask_to_28ch(rgb_video): """Colored RGB mask (T, H, W, 3) in [0, 1] -> SCAIL-2 28-channel binary latent (1, T_lat, 28, H_lat, W_lat). 7 per-color binary channels (white/r/g/b/y/m/c) @@ -123,7 +143,7 @@ class WanSCAILToVideo(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="WanSCAILToVideo", - category="model/conditioning/video_models", + category="model/conditioning/wan/scail", inputs=[ io.Conditioning.Input("positive"), io.Conditioning.Input("negative"), @@ -138,8 +158,8 @@ class WanSCAILToVideo(io.ComfyNode): io.Float.Input("pose_strength", default=1.0, min=0.0, max=10.0, step=0.01, tooltip="Strength of the pose latent."), io.Float.Input("pose_start", default=0.0, min=0.0, max=1.0, step=0.01, tooltip="Start step of the pose conditioning."), io.Float.Input("pose_end", default=1.0, min=0.0, max=1.0, step=0.01, tooltip="End step of the pose conditioning."), - io.Image.Input("reference_image", optional=True, tooltip="Reference image, for multiple references composite all on single image."), - io.Image.Input("reference_image_mask", optional=True, tooltip="SCAIL-2 only. Colored reference mask at the same resolution as reference_image."), + io.Image.Input("reference_image", optional=True, tooltip="Reference image. The first image is the primary reference (composite all identities onto it). SCAIL-2: extra batch images are used as additional views (back view, close-up, occluded background), each needing a matching reference_image_mask in that identity's color."), + io.Image.Input("reference_image_mask", optional=True, tooltip="SCAIL-2 only. Colored reference mask, batch matching reference_image (first = primary reference mask, rest = identity masks for the additional reference_image)."), io.ClipVisionOutput.Input("clip_vision_output", optional=True, tooltip="CLIP vision features for conditioning. Model is trained with stretch resize to aspect ratio."), io.Int.Input("video_frame_offset", default=0, min=0, max=nodes.MAX_RESOLUTION, step=1, tooltip="Cumulative output frame this chunk begins at. Wire from the previous chunk's video_frame_offset output."), io.Int.Input("previous_frame_count", default=5, min=1, max=nodes.MAX_RESOLUTION, step=4, tooltip="Tail frames of previous_frames to anchor. SCAIL-2 trained at 5 (81-frame chunks, 76-frame step)."), @@ -171,19 +191,21 @@ class WanSCAILToVideo(io.ComfyNode): video_frame_offset -= prev_trimmed.shape[0] video_frame_offset = max(0, video_frame_offset) - ref_latent = None if reference_image is not None: - reference_image = comfy.utils.common_upscale(reference_image[:1].movedim(-1, 1), width, height, "bicubic", "center").movedim(1, -1) - # Replacement Mode: composite ref on black bg using reference_image_mask as alpha matte - if replacement_mode and reference_image_mask is not None: - rm = comfy.utils.common_upscale(reference_image_mask[:1].movedim(-1, 1), width, height, "nearest-exact", "center").movedim(1, -1) - is_char = (rm[..., :3].max(dim=-1, keepdim=True).values > 0.1).to(reference_image.dtype) - reference_image = reference_image * is_char - ref_latent = vae.encode(reference_image[:, :, :, :3]) + ref_imgs = comfy.utils.common_upscale(reference_image.movedim(-1, 1), width, height, "bicubic", "center").movedim(1, -1) + n_ref = ref_imgs.shape[0] + # SCAIL-2 multi-reference: the first image is the primary ref, the rest are additional references. - if ref_latent is not None: - positive = node_helpers.conditioning_set_values(positive, {"reference_latents": [ref_latent]}, append=True) - negative = node_helpers.conditioning_set_values(negative, {"reference_latents": [ref_latent]}, append=True) + # Replacement Mode: composite each ref on black bg using its mask as alpha matte + if replacement_mode and reference_image_mask is not None: + rm = comfy.utils.common_upscale(reference_image_mask.movedim(-1, 1), width, height, "nearest-exact", "center").movedim(1, -1) + rm = rm[[min(i, rm.shape[0] - 1) for i in range(n_ref)]] + is_char = (rm[..., :3].max(dim=-1, keepdim=True).values > 0.1).to(ref_imgs.dtype) + ref_imgs = ref_imgs * is_char + # encode each ref individually so each stays a single latent frame (a batched encode would be treated as a video) + ref_latents = [vae.encode(ref_imgs[i:i + 1, :, :, :3]) for i in range(n_ref)] + positive = node_helpers.conditioning_set_values(positive, {"reference_latents": ref_latents}, append=True) + negative = node_helpers.conditioning_set_values(negative, {"reference_latents": ref_latents}, append=True) if clip_vision_output is not None: positive = node_helpers.conditioning_set_values(positive, {"clip_vision_output": clip_vision_output}) @@ -221,11 +243,16 @@ class WanSCAILToVideo(io.ComfyNode): positive = node_helpers.conditioning_set_values(positive, {"driving_mask_28ch": driving_mask_28ch}) negative = node_helpers.conditioning_set_values(negative, {"driving_mask_28ch": driving_mask_28ch}) - if reference_image_mask is not None: - ref_mask_hw = comfy.utils.common_upscale(reference_image_mask[:1].movedim(-1, 1), width, height, "bicubic", "center").movedim(1, -1) - ref_mask_1f = _extract_mask_to_28ch(ref_mask_hw) + # The ref mask binds reference frames to identities, so it only applies when there's a reference image. + if reference_image_mask is not None and reference_image is not None: + ref_mask_hw = comfy.utils.common_upscale(reference_image_mask.movedim(-1, 1), width, height, "nearest-exact", "center").movedim(1, -1) + n_masks = ref_mask_hw.shape[0] + n_ref = reference_image.shape[0] + + add_masks = [_extract_mask_to_28ch(ref_mask_hw[min(i, n_masks - 1)][None]) for i in range(1, n_ref)] + ref_mask_1f = _extract_mask_to_28ch(ref_mask_hw[:1]) zeros = torch.zeros((1, latent.shape[2], 28, ref_mask_1f.shape[-2], ref_mask_1f.shape[-1]), device=ref_mask_1f.device, dtype=ref_mask_1f.dtype) - ref_mask_28ch = torch.cat([ref_mask_1f, zeros], dim=1) + ref_mask_28ch = torch.cat(add_masks + [ref_mask_1f, zeros], dim=1) positive = node_helpers.conditioning_set_values(positive, {"ref_mask_28ch": ref_mask_28ch}) negative = node_helpers.conditioning_set_values(negative, {"ref_mask_28ch": ref_mask_28ch}) @@ -244,12 +271,9 @@ class WanSCAILToVideo(io.ComfyNode): class SCAIL2ColoredMask(io.ComfyNode): - """Render SAM3 tracks for the driving pose video and (optionally) the reference - image into the two colored masks WanSCAILToVideo consumes. Shared `sort_by` - across both outputs guarantees identity K maps to the same color on both - sides, for multi-person workflow consistency. - reference_image_mask is always rendered black-bg (model convention) - pose_video_mask bg follows replacement_mode: black = Animation Mode, white = Replacement Mode + """Render SAM3 tracks for the driving pose video and reference image(s) into the + colored masks WanSCAILToVideo consumes. Shared `sort_by` keeps each identity on the + same color across both outputs. """ @classmethod @@ -257,17 +281,18 @@ class SCAIL2ColoredMask(io.ComfyNode): return io.Schema( node_id="SCAIL2ColoredMask", display_name="Create SCAIL-2 Colored Mask", - category="conditioning/video_models/scail", + category="model/conditioning/wan/scail", inputs=[ SAM3TrackData.Input("driving_track_data", tooltip="SAM3 track of the driving pose video. Will be rendered into the pose_video_mask output."), - SAM3TrackData.Input("ref_track_data", optional=True, - tooltip="SAM3 track of the reference image."), + io.MultiType.Input("ref_track_data", [SAM3TrackData, io.Mask], optional=True, display_name="reference_masks", + tooltip="SAM3 track of the reference image(s) (one identity per object, colored in batch order), or a plain MASK of the reference subject (rendered as a single identity)."), io.String.Input("object_indices", default="", tooltip="Comma-separated list of person indices to include (e.g. '0,2,3'). Applied to both reference and pose video masks. Empty = all."), io.Combo.Input("sort_by", options=["none", "left_to_right", "area"], default="left_to_right", - tooltip="Order in which palette colors are assigned to the tracked objects (applied to both reference and pose video so each identity keeps the same color). left_to_right = leftmost object (by first-frame centroid) gets the first color; area = biggest object (by first-frame mask area) gets the first color; none = keep SAM3's order."), + tooltip="Order in which palette colors are assigned to the tracked objects (applied to both reference and pose video so each identity keeps the same color). Objects that appear in earlier frames always come first; within a frame, left_to_right = leftmost object (by centroid at first appearance) gets the first color, area = biggest object (by mask area at first appearance) gets the first color; none = keep SAM3's order."), io.Boolean.Input("replacement_mode", default=False, - tooltip="False = mask_video has black bg (Animation Mode). True = white bg (Replacement Mode). Set the matching replacement_mode on WanSCAILToVideo. reference_image_mask is always black-bg regardless."), + tooltip="False = Animation Mode (pose_video_mask has black background, reference_image_mask has white background). " + "True = Replacement Mode (pose_video_mask has white background, reference_image_mask has black background)."), ], outputs=[ io.Image.Output("pose_video_mask"), @@ -281,11 +306,11 @@ class SCAIL2ColoredMask(io.ComfyNode): def _prep(td): masks_bool = _unpack(td) if sort_by != "none" and masks_bool is not None: - cx, area = _first_frame_cx_area(masks_bool) + first_t, cx, area = _first_appearance_cx_area(masks_bool) if sort_by == "left_to_right": - order = sorted(range(len(cx)), key=lambda i: cx[i]) + order = sorted(range(len(cx)), key=lambda i: (first_t[i], cx[i])) else: # "area" - order = sorted(range(len(area)), key=lambda i: -area[i]) + order = sorted(range(len(area)), key=lambda i: (first_t[i], -area[i])) td = _subset_track_data(td, order) if object_indices.strip(): indices = [int(i.strip()) for i in object_indices.split(",") if i.strip().isdigit()] @@ -296,14 +321,19 @@ class SCAIL2ColoredMask(io.ComfyNode): return td drv = _prep(driving_track_data) + # Animation: driving=black, ref=white. Replacement: driving=white, ref=black. mask_video = _render_colored_masks(drv, "white" if replacement_mode else "black") + ref_bg = "black" if replacement_mode else "white" if ref_track_data is not None: - ref = _prep(ref_track_data) - reference_image_mask = _render_colored_masks(ref, "black") + if isinstance(ref_track_data, torch.Tensor): # plain comfy MASK + reference_image_mask = _render_mask_as_identity(ref_track_data, ref_bg) + else: + reference_image_mask = _render_colored_masks(_prep(ref_track_data), ref_bg) else: H, W = drv["orig_size"] - reference_image_mask = torch.zeros(1, H, W, 3, device=comfy.model_management.intermediate_device(), dtype=comfy.model_management.intermediate_dtype()) + fill_value = 1.0 if ref_bg == "white" else 0.0 + reference_image_mask = torch.full((1, H, W, 3), fill_value, device=comfy.model_management.intermediate_device(), dtype=comfy.model_management.intermediate_dtype()) return io.NodeOutput(mask_video, reference_image_mask) diff --git a/comfy_extras/nodes_sd3.py b/comfy_extras/nodes_sd3.py index 38cbf117b..40e84656b 100644 --- a/comfy_extras/nodes_sd3.py +++ b/comfy_extras/nodes_sd3.py @@ -13,8 +13,9 @@ class TripleCLIPLoader(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="TripleCLIPLoader", - category="advanced/loaders", - description="[Recipes]\n\nsd3: clip-l, clip-g, t5", + display_name="Load CLIP (Triple)", + category="model/loaders", + description="Recipes:\nsd3: clip-l, clip-g, t5", inputs=[ io.Combo.Input("clip_name1", options=folder_paths.get_filename_list("text_encoders")), io.Combo.Input("clip_name2", options=folder_paths.get_filename_list("text_encoders")), @@ -41,7 +42,7 @@ class EmptySD3LatentImage(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="EmptySD3LatentImage", - category="model/latent/sd3", + category="model/latent/stable diffusion", inputs=[ io.Int.Input("width", default=1024, min=16, max=nodes.MAX_RESOLUTION, step=16), io.Int.Input("height", default=1024, min=16, max=nodes.MAX_RESOLUTION, step=16), @@ -66,7 +67,8 @@ class CLIPTextEncodeSD3(io.ComfyNode): return io.Schema( node_id="CLIPTextEncodeSD3", search_aliases=["sd3 prompt"], - category="advanced/conditioning", + display_name="CLIP Text Encode (SD3)", + category="model/conditioning/stable diffusion", inputs=[ io.Clip.Input("clip"), io.String.Input("clip_l", multiline=True, dynamic_prompts=True), diff --git a/comfy_extras/nodes_sdpose.py b/comfy_extras/nodes_sdpose.py index 20d459b00..d1cbff2a6 100644 --- a/comfy_extras/nodes_sdpose.py +++ b/comfy_extras/nodes_sdpose.py @@ -96,8 +96,12 @@ class KeypointDraw: # Body connections - matching DWPose limbSeq (1-indexed, converted to 0-indexed) self.body_limbSeq = [ [2, 3], [2, 6], [3, 4], [4, 5], [6, 7], [7, 8], [2, 9], [9, 10], - [10, 11], [2, 12], [12, 13], [13, 14], [2, 1], [1, 15], [15, 17], - [1, 16], [16, 18] + [10, 11], [2, 12], [12, 13], [13, 14] + ] + + # Head connections (1-indexed, converted to 0-indexed) + self.head_edges = [ + [2, 1], [1, 15], [15, 17], [1, 16], [16, 18] ] # Colors matching DWPose @@ -215,7 +219,7 @@ class KeypointDraw: return unique_pts if len(unique_pts) > 1 else [[center[0], center[1]], [center[0], center[1]]] def draw_wholebody_keypoints(self, canvas, keypoints, scores=None, threshold=0.3, - draw_body=True, draw_feet=True, draw_face=True, draw_hands=True, stick_width=4, face_point_size=3): + draw_body=True, draw_head=True, draw_feet=True, draw_face=True, draw_hands=True, stick_width=4, face_point_size=3): """ Draw wholebody keypoints (134 keypoints after processing) in DWPose style. @@ -237,9 +241,17 @@ class KeypointDraw: """ H, W, C = canvas.shape - # Draw body limbs - if draw_body and len(keypoints) >= 18: - for i, limb in enumerate(self.body_limbSeq): + # Draw body limbs & head connections + if (draw_body or draw_head) and len(keypoints) >= 18: + colorIndexOffset = 0 + edges = [] + if draw_body: + edges += self.body_limbSeq + else: + colorIndexOffset += len(self.body_limbSeq) + if draw_head: + edges += self.head_edges + for i, limb in enumerate(edges): # Convert from 1-indexed to 0-indexed idx1, idx2 = limb[0] - 1, limb[1] - 1 @@ -262,11 +274,17 @@ class KeypointDraw: polygon = self.draw.ellipse2Poly((int(mY), int(mX)), (int(length / 2), stick_width), int(angle), 0, 360, 1) - self.draw.fillConvexPoly(canvas, polygon, self.colors[i % len(self.colors)]) + self.draw.fillConvexPoly(canvas, polygon, self.colors[(i + colorIndexOffset) % len(self.colors)]) - # Draw body keypoints - if draw_body and len(keypoints) >= 18: + # Draw body & head keypoints + if (draw_body or draw_head) and len(keypoints) >= 18: + head_keypoints = {0, 14, 15, 16, 17} # nose, eyes, ears + neck_point = 1 for i in range(18): + if not draw_head and i in head_keypoints: + continue + if not draw_body and i not in head_keypoints and i != neck_point: + continue if scores is not None and scores[i] < threshold: continue x, y = int(keypoints[i][0]), int(keypoints[i][1]) @@ -365,6 +383,7 @@ class SDPoseDrawKeypoints(io.ComfyNode): io.Int.Input("stick_width", default=4, min=1, max=10, step=1), io.Int.Input("face_point_size", default=3, min=1, max=10, step=1), io.Float.Input("score_threshold", default=0.3, min=0.0, max=1.0, step=0.01), + io.Boolean.Input("draw_head", default=True), ], outputs=[ io.Image.Output(), @@ -372,7 +391,7 @@ class SDPoseDrawKeypoints(io.ComfyNode): ) @classmethod - def execute(cls, keypoints, draw_body, draw_hands, draw_face, draw_feet, stick_width, face_point_size, score_threshold) -> io.NodeOutput: + def execute(cls, keypoints, draw_body, draw_hands, draw_face, draw_feet, stick_width, face_point_size, score_threshold, draw_head) -> io.NodeOutput: if not keypoints: return io.NodeOutput(torch.zeros((1, 64, 64, 3), dtype=torch.float32)) height = keypoints[0]["canvas_height"] @@ -405,7 +424,7 @@ class SDPoseDrawKeypoints(io.ComfyNode): canvas = drawer.draw_wholebody_keypoints( canvas, kp, sc, threshold=score_threshold, - draw_body=draw_body, draw_feet=draw_feet, + draw_body=draw_body, draw_head=draw_head, draw_feet=draw_feet, draw_face=draw_face, draw_hands=draw_hands, stick_width=stick_width, face_point_size=face_point_size, ) diff --git a/comfy_extras/nodes_sdupscale.py b/comfy_extras/nodes_sdupscale.py index ea283e971..5c247fb49 100644 --- a/comfy_extras/nodes_sdupscale.py +++ b/comfy_extras/nodes_sdupscale.py @@ -9,7 +9,7 @@ class SD_4XUpscale_Conditioning(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="SD_4XUpscale_Conditioning", - category="model/conditioning/upscale_diffusion", + category="model/conditioning/stable diffusion upscaler", inputs=[ io.Image.Input("images"), io.Conditioning.Input("positive"), diff --git a/comfy_extras/nodes_seed.py b/comfy_extras/nodes_seed.py new file mode 100644 index 000000000..e64f1d7e3 --- /dev/null +++ b/comfy_extras/nodes_seed.py @@ -0,0 +1,33 @@ +import sys +from typing_extensions import override + +from comfy_api.latest import ComfyExtension, io + + +class SeedNode(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="SeedNode", + display_name="Seed", + search_aliases=["seed", "random"], + category="utilities", + inputs=[ + io.Int.Input("seed", min=0, max=sys.maxsize, control_after_generate=io.ControlAfterGenerate.fixed), + ], + outputs=[io.Int.Output(display_name="seed")], + ) + + @classmethod + def execute(cls, seed: int) -> io.NodeOutput: + return io.NodeOutput(seed) + + +class SeedExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [SeedNode] + + +async def comfy_entrypoint() -> SeedExtension: + return SeedExtension() diff --git a/comfy_extras/nodes_stable3d.py b/comfy_extras/nodes_stable3d.py index 8a6e5b726..b0eba819b 100644 --- a/comfy_extras/nodes_stable3d.py +++ b/comfy_extras/nodes_stable3d.py @@ -27,7 +27,7 @@ class StableZero123_Conditioning(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="StableZero123_Conditioning", - category="model/conditioning/3d_models", + category="model/conditioning/stable zero123", inputs=[ io.ClipVision.Input("clip_vision"), io.Image.Input("init_image"), @@ -65,7 +65,7 @@ class StableZero123_Conditioning_Batched(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="StableZero123_Conditioning_Batched", - category="model/conditioning/3d_models", + category="model/conditioning/stable zero123", inputs=[ io.ClipVision.Input("clip_vision"), io.Image.Input("init_image"), @@ -112,7 +112,7 @@ class SV3D_Conditioning(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="SV3D_Conditioning", - category="model/conditioning/3d_models", + category="model/conditioning/stable video 3d", inputs=[ io.ClipVision.Input("clip_vision"), io.Image.Input("init_image"), diff --git a/comfy_extras/nodes_stable_cascade.py b/comfy_extras/nodes_stable_cascade.py index e55f248ae..ddfb4f2b0 100644 --- a/comfy_extras/nodes_stable_cascade.py +++ b/comfy_extras/nodes_stable_cascade.py @@ -29,7 +29,7 @@ class StableCascade_EmptyLatentImage(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="StableCascade_EmptyLatentImage", - category="model/latent/stable_cascade", + category="model/latent/stable cascade", inputs=[ io.Int.Input("width", default=1024, min=256, max=nodes.MAX_RESOLUTION, step=8), io.Int.Input("height", default=1024, min=256, max=nodes.MAX_RESOLUTION, step=8), @@ -58,7 +58,7 @@ class StableCascade_StageC_VAEEncode(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="StableCascade_StageC_VAEEncode", - category="model/latent/stable_cascade", + category="model/latent/stable cascade", inputs=[ io.Image.Input("image"), io.Vae.Input("vae"), @@ -93,7 +93,7 @@ class StableCascade_StageB_Conditioning(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="StableCascade_StageB_Conditioning", - category="model/conditioning/stable_cascade", + category="model/conditioning/stable cascade", inputs=[ io.Conditioning.Input("conditioning"), io.Latent.Input("stage_c"), @@ -119,7 +119,7 @@ class StableCascade_SuperResolutionControlnet(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="StableCascade_SuperResolutionControlnet", - category="experimental/stable_cascade", + category="experimental/stable cascade", is_experimental=True, inputs=[ io.Image.Input("image"), diff --git a/comfy_extras/nodes_string.py b/comfy_extras/nodes_string.py index 97485c8c5..21929ae63 100644 --- a/comfy_extras/nodes_string.py +++ b/comfy_extras/nodes_string.py @@ -440,6 +440,57 @@ class JsonExtractString(io.ComfyNode): except (json.JSONDecodeError, TypeError): return io.NodeOutput("") + +def _dump_json(value, indent): + return json.dumps(value, ensure_ascii=False, indent=indent or None) + + +class ConvertDictionaryToString(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="ConvertDictionaryToString", + display_name="Convert Dictionary to String", + category="text", + search_aliases=["json", "dict to json", "stringify", "serialize", "dict to string"], + inputs=[ + io.Dict.Input("dictionary"), + io.Int.Input("indent", default=2, min=0, max=8, + tooltip="Spaces per indent level. 0 produces compact single-line string."), + ], + outputs=[ + io.String.Output(), + ], + ) + + @classmethod + def execute(cls, dictionary, indent=2): + return io.NodeOutput(_dump_json(dictionary, indent)) + + +class ConvertArrayToString(io.ComfyNode): + @classmethod + def define_schema(cls): + return io.Schema( + node_id="ConvertArrayToString", + display_name="Convert Array to String", + category="text", + search_aliases=["json", "list to json", "stringify", "serialize", "list to string", "array to json"], + inputs=[ + io.Array.Input("array"), + io.Int.Input("indent", default=2, min=0, max=8, + tooltip="Spaces per indent level. 0 produces compact single-line string."), + ], + outputs=[ + io.String.Output(), + ], + ) + + @classmethod + def execute(cls, array, indent=2): + return io.NodeOutput(_dump_json(array, indent)) + + class StringExtension(ComfyExtension): @override async def get_node_list(self) -> list[type[io.ComfyNode]]: @@ -457,6 +508,8 @@ class StringExtension(ComfyExtension): RegexExtract, RegexReplace, JsonExtractString, + ConvertDictionaryToString, + ConvertArrayToString, ] async def comfy_entrypoint() -> StringExtension: diff --git a/comfy_extras/nodes_text_overlay.py b/comfy_extras/nodes_text_overlay.py new file mode 100644 index 000000000..4c5cdae60 --- /dev/null +++ b/comfy_extras/nodes_text_overlay.py @@ -0,0 +1,150 @@ +import numpy as np +import torch +from PIL import Image as PILImage, ImageColor, ImageDraw, ImageFont +from typing_extensions import override + +from comfy_api.latest import ComfyExtension, IO + + +class TextOverlay(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="TextOverlay", + display_name="Draw Text Overlay", + category="text", + description="Draw text overlay on an image or batch of images.", + search_aliases=["text", "label", "caption", "subtitle", "watermark", "title", "addlabel", "overlay"], + inputs=[ + IO.Image.Input("images"), + IO.String.Input("text", multiline=True, default=""), + IO.Float.Input("font_size", default=5.0, min=0.5, max=50.0, step=0.5, tooltip="Font size as a percentage of the image height."), + IO.Color.Input("color", default="#ffffff", tooltip="Color of the text."), + IO.Combo.Input("position", options=["top", "bottom"], default="top"), + IO.Combo.Input("align", options=["left", "center", "right"], default="left"), + IO.Boolean.Input("outline", default=True, tooltip="Draw a black outline around the text."), + ], + outputs=[IO.Image.Output(display_name="images")], + ) + + @classmethod + def execute(cls, images, text, font_size, color, position, align, outline) -> IO.NodeOutput: + if text.strip() == "": + return IO.NodeOutput(images) + + text = text.replace("\\n", "\n").replace("\\t", "\t") + + text_rgba = cls.parse_color_to_rgba(color) + outline_rgba = (0, 0, 0, 255) if outline else (0, 0, 0, 0) + + # Render the overlay once and composite it across all frames in the batch + height = images.shape[1] + width = images.shape[2] + overlay_rgb, overlay_alpha = cls.render_overlay_text(width, height, text, position, align, font_size, text_rgba, outline_rgba) + overlay_rgb = overlay_rgb.to(device=images.device, dtype=images.dtype) + overlay_alpha = overlay_alpha.to(device=images.device, dtype=images.dtype) + + result = images * (1.0 - overlay_alpha) + overlay_rgb * overlay_alpha + return IO.NodeOutput(result) + + @staticmethod + def parse_color_to_rgba(color_string): + parsed = ImageColor.getrgb(color_string) + + if len(parsed) == 3: + return (*parsed, 255) + + return parsed + + @classmethod + def render_overlay_text(cls, width, height, text, position, align, font_size, text_rgba, outline_rgba): + line_spacing = 1.2 + margin_percent = 1.0 + min_font_percent = 2.0 + min_font_pixels = 10 + outline_thickness_factor = 0.04 + + # Draw onto a transparent layer so the result can be alpha-composited over any frame. + layer = PILImage.new("RGBA", (width, height), (0, 0, 0, 0)) + draw = ImageDraw.Draw(layer) + + margin = int(round(margin_percent / 100.0 * min(width, height))) + max_width = max(1, width - 2 * margin) + max_height = max(1, height - 2 * margin) + + # Font scales with resolution, then shrinks to fit the height. + size = max(1, int(round(font_size / 100.0 * height))) + floor = min(size, max(min_font_pixels, int(round(min_font_percent / 100.0 * height)))) + + while True: + font = ImageFont.load_default(size=size) + stroke = max(1, int(round(size * outline_thickness_factor))) if outline_rgba[3] > 0 else 0 + block = "\n".join(cls.wrap_text(text, font, max_width)) + # convert line spacing to pixel spacing + single = draw.textbbox((0, 0), "Ay", font=font, stroke_width=stroke) + double = draw.multiline_textbbox((0, 0), "Ay\nAy", font=font, spacing=0, stroke_width=stroke) + natural_advance = (double[3] - double[1]) - (single[3] - single[1]) + pixel_spacing = int(round(size * line_spacing - natural_advance)) + box = draw.multiline_textbbox((0, 0), block, font=font, spacing=pixel_spacing, stroke_width=stroke) + block_height = box[3] - box[1] + + if block_height <= max_height or size <= floor: + break + + size = max(floor, int(size * 0.9)) + + anchor_h, x = {"left": ("l", margin), "center": ("m", width / 2), "right": ("r", width - margin)}[align] + + # Offset y so the rendered text sits flush against the margin + if position == "bottom": + y = height - margin - box[3] + else: + y = margin - box[1] + + draw.multiline_text((x, y), block, font=font, fill=text_rgba, anchor=anchor_h + "a", + align=align, spacing=pixel_spacing, stroke_width=stroke, stroke_fill=outline_rgba) + + overlay = np.array(layer).astype(np.float32) / 255.0 + overlay_rgb = torch.from_numpy(overlay[:, :, :3]) + overlay_alpha = torch.from_numpy(overlay[:, :, 3:4]) + return overlay_rgb, overlay_alpha + + @staticmethod + def wrap_text(text, font, max_width): + lines = [] + for raw_line in text.split("\n"): + words = raw_line.split() + if not words: + lines.append("") + continue + current = "" + # Break the line into words and split words that are too long + for word in words: + while font.getlength(word) > max_width and len(word) > 1: + cut = 1 + while cut < len(word) and font.getlength(word[:cut + 1]) <= max_width: + cut += 1 + if current: + lines.append(current) + current = "" + lines.append(word[:cut]) + word = word[cut:] + candidate = word if not current else current + " " + word + if not current or font.getlength(candidate) <= max_width: + current = candidate + else: + lines.append(current) + current = word + if current: + lines.append(current) + return lines + + +class TextOverlayExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[IO.ComfyNode]]: + return [TextOverlay] + + +async def comfy_entrypoint() -> TextOverlayExtension: + return TextOverlayExtension() diff --git a/comfy_extras/nodes_textgen.py b/comfy_extras/nodes_textgen.py index d52faf815..5a947d5c5 100644 --- a/comfy_extras/nodes_textgen.py +++ b/comfy_extras/nodes_textgen.py @@ -35,7 +35,7 @@ class TextGenerate(io.ComfyNode): io.Image.Input("image", optional=True), io.Image.Input("video", optional=True, tooltip="Video frames as image batch. Assumed to be 24 FPS; subsampled to 1 FPS internally."), io.Audio.Input("audio", optional=True), - io.Int.Input("max_length", default=256, min=1, max=2048), + io.Int.Input("max_length", default=512, min=1, max=32768), io.DynamicCombo.Input("sampling_mode", options=sampling_options, display_name="Sampling Mode"), io.Boolean.Input("thinking", optional=True, default=False, tooltip="Operate in thinking mode if the model supports it."), io.Boolean.Input("use_default_template", optional=True, default=True, tooltip="Use the built in system prompt/template if the model has one.", advanced=True), diff --git a/comfy_extras/nodes_train.py b/comfy_extras/nodes_train.py index bb68da6fa..a27217b80 100644 --- a/comfy_extras/nodes_train.py +++ b/comfy_extras/nodes_train.py @@ -1367,7 +1367,7 @@ class SaveLoRA(io.ComfyNode): node_id="SaveLoRA", search_aliases=["export lora"], display_name="Save LoRA Weights", - category="advanced/model_merging", + category="model/merging", is_experimental=True, is_output_node=True, inputs=[ diff --git a/comfy_extras/nodes_triposplat.py b/comfy_extras/nodes_triposplat.py index 1848ad31a..c892213e4 100644 --- a/comfy_extras/nodes_triposplat.py +++ b/comfy_extras/nodes_triposplat.py @@ -65,7 +65,7 @@ class TripoSplatPreprocessImage(IO.ComfyNode): return IO.Schema( node_id="TripoSplatPreprocessImage", display_name="TripoSplat Preprocess Image", - category="3d/conditioning", + category="model/conditioning/triposplat", description="Crop center each image to a square canvas on a black background and add padding.", inputs=[ IO.Image.Input("image"), @@ -95,7 +95,7 @@ class TripoSplatConditioning(IO.ComfyNode): return IO.Schema( node_id="TripoSplatConditioning", display_name="TripoSplat Conditioning", - category="3d/conditioning", + category="model/conditioning/triposplat", description="Encode the image with DINOv3 and the Flux2 VAE into TripoSplat positive/negative " "conditioning, and create the fixed size noise target (latent + camera) for the KSampler", inputs=[ @@ -143,7 +143,7 @@ class VAEDecodeTripoSplat(IO.ComfyNode): return IO.Schema( node_id="VAEDecodeTripoSplat", display_name="TripoSplat Decode", - category="3d/latent", + category="model/latent/triposplat", description="Decode the sampled TripoSplat latent into a 3D gaussian splat. " "Modify the number of gaussians to vary the density.", inputs=[ @@ -188,7 +188,7 @@ class TripoSplatSamplingPreview(IO.ComfyNode): return IO.Schema( node_id="TripoSplatSamplingPreview", display_name="TripoSplat Sampling Preview", - category="3d/latent", + category="model/latent/triposplat", description="Patch the TripoSplat model for the standard Ksampler node to show a live decoded " "gaussian splat preview at each step.", inputs=[ diff --git a/comfy_extras/nodes_video.py b/comfy_extras/nodes_video.py index 6f6c416a6..d3acc9ad0 100644 --- a/comfy_extras/nodes_video.py +++ b/comfy_extras/nodes_video.py @@ -27,6 +27,7 @@ class SaveWEBM(io.ComfyNode): ], hidden=[io.Hidden.prompt, io.Hidden.extra_pnginfo], is_output_node=True, + outputs=[io.Image.Output(display_name="images")] ) @classmethod @@ -69,7 +70,7 @@ class SaveWEBM(io.ComfyNode): container.mux(stream.encode()) container.close() - return io.NodeOutput(ui=ui.PreviewVideo([ui.SavedResult(file, subfolder, io.FolderType.output)])) + return io.NodeOutput(images, ui=ui.PreviewVideo([ui.SavedResult(file, subfolder, io.FolderType.output)])) class SaveVideo(io.ComfyNode): @classmethod @@ -89,6 +90,7 @@ class SaveVideo(io.ComfyNode): ], hidden=[io.Hidden.prompt, io.Hidden.extra_pnginfo], is_output_node=True, + outputs=[io.Video.Output("video")], ) @classmethod @@ -117,7 +119,7 @@ class SaveVideo(io.ComfyNode): metadata=saved_metadata ) - return io.NodeOutput(ui=ui.PreviewVideo([ui.SavedResult(file, subfolder, io.FolderType.output)])) + return io.NodeOutput(video, ui=ui.PreviewVideo([ui.SavedResult(file, subfolder, io.FolderType.output)])) class CreateVideo(io.ComfyNode): @@ -134,6 +136,17 @@ class CreateVideo(io.ComfyNode): io.Image.Input("images", tooltip="The images to create a video from."), io.Float.Input("fps", default=30.0, min=1.0, max=120.0, step=1.0), io.Audio.Input("audio", optional=True, tooltip="The audio to add to the video."), + io.Int.Input( + "bit_depth", + min=8, + max=10, + default=8, + step=2, + tooltip="Bit depth of the created video. 10-bit keeps smoother gradients with less" + " banding, but some players and downstream nodes may not support it.", + optional=True, + display_mode=io.NumberDisplay.number, + ), ], outputs=[ io.Video.Output(), @@ -141,9 +154,14 @@ class CreateVideo(io.ComfyNode): ) @classmethod - def execute(cls, images: Input.Image, fps: float, audio: Optional[Input.Audio] = None) -> io.NodeOutput: + def execute( + cls, images: Input.Image, fps: float, audio: Optional[Input.Audio] = None, bit_depth: int = 8, + ) -> io.NodeOutput: return io.NodeOutput( - InputImpl.VideoFromComponents(Types.VideoComponents(images=images, audio=audio, frame_rate=Fraction(fps))) + InputImpl.VideoFromComponents( + Types.VideoComponents(images=images, audio=audio, frame_rate=Fraction(fps)), + bit_depth=bit_depth, + ) ) class GetVideoComponents(io.ComfyNode): @@ -154,7 +172,7 @@ class GetVideoComponents(io.ComfyNode): search_aliases=["extract frames", "split video", "video to images", "demux"], display_name="Get Video Components", category="video", - description="Extracts all components from a video: frames, audio, and framerate.", + description="Extracts all components from a video: frames, audio, framerate, and bit depth.", inputs=[ io.Video.Input("video", tooltip="The video to extract components from."), ], @@ -162,13 +180,14 @@ class GetVideoComponents(io.ComfyNode): io.Image.Output(display_name="images"), io.Audio.Output(display_name="audio"), io.Float.Output(display_name="fps"), + io.Int.Output(display_name="bit_depth"), ], ) @classmethod def execute(cls, video: Input.Video) -> io.NodeOutput: components = video.get_components() - return io.NodeOutput(components.images, components.audio, float(components.frame_rate)) + return io.NodeOutput(components.images, components.audio, float(components.frame_rate), video.get_bit_depth()) class LoadVideo(io.ComfyNode): @@ -216,13 +235,8 @@ class VideoSlice(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="Video Slice", - display_name="Video Slice", - search_aliases=[ - "trim video duration", - "skip first frames", - "frame load cap", - "start time", - ], + display_name="Trim Video", + search_aliases=["trim video duration", "skip first frames", "frame load cap", "start time"], category="video", essentials_category="Video Tools", inputs=[ diff --git a/comfy_extras/nodes_video_model.py b/comfy_extras/nodes_video_model.py index 0d6cae6a8..01d48d4d4 100644 --- a/comfy_extras/nodes_video_model.py +++ b/comfy_extras/nodes_video_model.py @@ -41,7 +41,7 @@ class SVD_img2vid_Conditioning: FUNCTION = "encode" - CATEGORY = "model/conditioning/video_models" + CATEGORY = "model/conditioning/stable video" def encode(self, clip_vision, init_image, vae, width, height, video_frames, motion_bucket_id, fps, augmentation_level): output = clip_vision.encode_image(init_image) @@ -108,7 +108,7 @@ class VideoTriangleCFGGuidance: return (m, ) class ImageOnlyCheckpointSave(comfy_extras.nodes_model_merging.CheckpointSave): - CATEGORY = "advanced/model_merging" + CATEGORY = "model/merging" @classmethod def INPUT_TYPES(s): @@ -138,7 +138,7 @@ class ConditioningSetAreaPercentageVideo: RETURN_TYPES = ("CONDITIONING",) FUNCTION = "append" - CATEGORY = "model/conditioning" + CATEGORY = "model/conditioning/transform" def append(self, conditioning, width, height, temporal, x, y, z, strength): c = node_helpers.conditioning_set_values(conditioning, {"area": ("percentage", temporal, height, width, z, y, x), @@ -160,4 +160,5 @@ NODE_DISPLAY_NAME_MAPPINGS = { "ImageOnlyCheckpointLoader": "Load Checkpoint Image Only (img2vid model)", "VideoLinearCFGGuidance": "Video Linear CFG Guidance", "VideoTriangleCFGGuidance": "Video Triangle CFG Guidance", + "ConditioningSetAreaPercentageVideo": "Conditioning (Set Area with Percentage for Video)", } diff --git a/comfy_extras/nodes_void.py b/comfy_extras/nodes_void.py index b43154b8d..7527baf43 100644 --- a/comfy_extras/nodes_void.py +++ b/comfy_extras/nodes_void.py @@ -175,7 +175,7 @@ class VOIDInpaintConditioning(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="VOIDInpaintConditioning", - category="model/conditioning/video_models", + category="model/conditioning/void", inputs=[ io.Conditioning.Input("positive"), io.Conditioning.Input("negative"), @@ -288,7 +288,7 @@ class VOIDWarpedNoise(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="VOIDWarpedNoise", - category="model/latent/video", + category="model/latent/void", inputs=[ OpticalFlow.Input( "optical_flow", @@ -393,7 +393,7 @@ class VOIDWarpedNoiseSource(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="VOIDWarpedNoiseSource", - category="model/sampling/noise", + category="model/latent/void", inputs=[ io.Latent.Input("warped_noise", tooltip="Warped noise latent from VOIDWarpedNoise"), diff --git a/comfy_extras/nodes_wan.py b/comfy_extras/nodes_wan.py index d73be8e00..0e47a58df 100644 --- a/comfy_extras/nodes_wan.py +++ b/comfy_extras/nodes_wan.py @@ -18,7 +18,7 @@ class WanImageToVideo(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="WanImageToVideo", - category="model/conditioning/video_models", + category="model/conditioning/wan", inputs=[ io.Conditioning.Input("positive"), io.Conditioning.Input("negative"), @@ -66,7 +66,7 @@ class WanFunControlToVideo(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="WanFunControlToVideo", - category="model/conditioning/video_models", + category="model/conditioning/wan/fun control", inputs=[ io.Conditioning.Input("positive"), io.Conditioning.Input("negative"), @@ -119,7 +119,7 @@ class Wan22FunControlToVideo(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="Wan22FunControlToVideo", - category="model/conditioning/video_models", + category="model/conditioning/wan/fun control", inputs=[ io.Conditioning.Input("positive"), io.Conditioning.Input("negative"), @@ -184,7 +184,7 @@ class WanFirstLastFrameToVideo(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="WanFirstLastFrameToVideo", - category="model/conditioning/video_models", + category="model/conditioning/wan", inputs=[ io.Conditioning.Input("positive"), io.Conditioning.Input("negative"), @@ -256,7 +256,7 @@ class WanFunInpaintToVideo(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="WanFunInpaintToVideo", - category="model/conditioning/video_models", + category="model/conditioning/wan/fun inpaint", inputs=[ io.Conditioning.Input("positive"), io.Conditioning.Input("negative"), @@ -288,7 +288,7 @@ class WanVaceToVideo(io.ComfyNode): return io.Schema( node_id="WanVaceToVideo", search_aliases=["video conditioning", "video control"], - category="model/conditioning/video_models", + category="model/conditioning/wan/vace", inputs=[ io.Conditioning.Input("positive"), io.Conditioning.Input("negative"), @@ -375,7 +375,8 @@ class TrimVideoLatent(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="TrimVideoLatent", - category="model/latent/video", + display_name="Trim Video Latent", + category="model/latent", inputs=[ io.Latent.Input("samples"), io.Int.Input("trim_amount", default=0, min=0, max=99999), @@ -398,7 +399,7 @@ class WanCameraImageToVideo(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="WanCameraImageToVideo", - category="model/conditioning/video_models", + category="model/conditioning/wan/camera", inputs=[ io.Conditioning.Input("positive"), io.Conditioning.Input("negative"), @@ -452,7 +453,7 @@ class WanPhantomSubjectToVideo(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="WanPhantomSubjectToVideo", - category="model/conditioning/video_models", + category="model/conditioning/wan/phantom subject", inputs=[ io.Conditioning.Input("positive"), io.Conditioning.Input("negative"), @@ -707,7 +708,7 @@ class WanTrackToVideo(io.ComfyNode): return io.Schema( node_id="WanTrackToVideo", search_aliases=["motion tracking", "trajectory video", "point tracking", "keypoint animation"], - category="model/conditioning/video_models", + category="model/conditioning/wan/move", inputs=[ io.Conditioning.Input("positive"), io.Conditioning.Input("negative"), @@ -951,7 +952,7 @@ class WanSoundImageToVideo(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="WanSoundImageToVideo", - category="model/conditioning/video_models", + category="model/conditioning/wan/sound", inputs=[ io.Conditioning.Input("positive"), io.Conditioning.Input("negative"), @@ -984,7 +985,7 @@ class WanSoundImageToVideoExtend(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="WanSoundImageToVideoExtend", - category="model/conditioning/video_models", + category="model/conditioning/wan/sound", inputs=[ io.Conditioning.Input("positive"), io.Conditioning.Input("negative"), @@ -1046,7 +1047,7 @@ class WanHuMoImageToVideo(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="WanHuMoImageToVideo", - category="model/conditioning/video_models", + category="model/conditioning/wan/humo", inputs=[ io.Conditioning.Input("positive"), io.Conditioning.Input("negative"), @@ -1112,7 +1113,7 @@ class WanAnimateToVideo(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="WanAnimateToVideo", - category="model/conditioning/video_models", + category="model/conditioning/wan/animate", inputs=[ io.Conditioning.Input("positive"), io.Conditioning.Input("negative"), @@ -1252,7 +1253,7 @@ class Wan22ImageToVideoLatent(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="Wan22ImageToVideoLatent", - category="model/conditioning/inpaint", + category="model/conditioning/wan", inputs=[ io.Vae.Input("vae"), io.Int.Input("width", default=1280, min=32, max=nodes.MAX_RESOLUTION, step=32), @@ -1302,7 +1303,7 @@ class WanInfiniteTalkToVideo(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="WanInfiniteTalkToVideo", - category="model/conditioning/video_models", + category="model/conditioning/wan/infinite talk", inputs=[ io.DynamicCombo.Input("mode", options=[ io.DynamicCombo.Option("single_speaker", []), diff --git a/comfy_extras/nodes_wandancer.py b/comfy_extras/nodes_wandancer.py index a96885745..fdb2b5e57 100644 --- a/comfy_extras/nodes_wandancer.py +++ b/comfy_extras/nodes_wandancer.py @@ -713,7 +713,7 @@ class WanDancerEncodeAudio(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="WanDancerEncodeAudio", - category="model/conditioning/video_models", + category="model/conditioning/wan/dancer", inputs=[ io.Audio.Input("audio"), io.Int.Input("video_frames", default=149, min=1, max=nodes.MAX_RESOLUTION, step=4), @@ -787,7 +787,7 @@ class WanDancerVideo(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="WanDancerVideo", - category="model/conditioning/video_models", + category="model/conditioning/wan/dancer", inputs=[ io.Conditioning.Input("positive"), io.Conditioning.Input("negative"), diff --git a/comfy_extras/nodes_wanmove.py b/comfy_extras/nodes_wanmove.py index 2db064922..d1f924a40 100644 --- a/comfy_extras/nodes_wanmove.py +++ b/comfy_extras/nodes_wanmove.py @@ -247,7 +247,7 @@ class WanMoveVisualizeTracks(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="WanMoveVisualizeTracks", - category="model/conditioning/video_models", + category="model/conditioning/wan/move", inputs=[ io.Image.Input("images"), io.Tracks.Input("tracks", optional=True), @@ -283,7 +283,7 @@ class WanMoveTracksFromCoords(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="WanMoveTracksFromCoords", - category="model/conditioning/video_models", + category="model/conditioning/wan/move", inputs=[ io.String.Input("track_coords", force_input=True, default="[]", optional=True), io.Mask.Input("track_mask", optional=True), @@ -325,7 +325,8 @@ class GenerateTracks(io.ComfyNode): return io.Schema( node_id="GenerateTracks", search_aliases=["motion paths", "camera movement", "trajectory"], - category="model/conditioning/video_models", + display_name="Generate Video Tracks", + category="model/conditioning/wan/move", inputs=[ io.Int.Input("width", default=832, min=16, max=4096, step=16), io.Int.Input("height", default=480, min=16, max=4096, step=16), @@ -434,7 +435,7 @@ class WanMoveConcatTrack(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="WanMoveConcatTrack", - category="model/conditioning/video_models", + category="model/conditioning/wan/move", inputs=[ io.Tracks.Input("tracks_1"), io.Tracks.Input("tracks_2", optional=True), @@ -463,7 +464,7 @@ class WanMoveTrackToVideo(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="WanMoveTrackToVideo", - category="model/conditioning/video_models", + category="model/conditioning/wan/move", inputs=[ io.Conditioning.Input("positive"), io.Conditioning.Input("negative"), diff --git a/comfy_extras/nodes_zimage.py b/comfy_extras/nodes_zimage.py index 70ddc4afa..ce946b377 100644 --- a/comfy_extras/nodes_zimage.py +++ b/comfy_extras/nodes_zimage.py @@ -10,7 +10,7 @@ class TextEncodeZImageOmni(io.ComfyNode): def define_schema(cls): return io.Schema( node_id="TextEncodeZImageOmni", - category="advanced/conditioning", + category="model/conditioning/z-image", is_experimental=True, inputs=[ io.Clip.Input("clip"), diff --git a/comfyui_version.py b/comfyui_version.py index 4e3c924e6..8e9967f1b 100644 --- a/comfyui_version.py +++ b/comfyui_version.py @@ -1,3 +1,3 @@ # This file is automatically generated by the build process when version is # updated in pyproject.toml. -__version__ = "0.24.0" +__version__ = "0.27.0" diff --git a/execution.py b/execution.py index 5246d651c..c45317593 100644 --- a/execution.py +++ b/execution.py @@ -40,6 +40,7 @@ from comfy_execution.graph_utils import GraphBuilder, is_link from comfy_execution.validation import validate_node_input from comfy_execution.progress import get_progress_state, reset_progress_state, add_progress_handler, WebUIProgressHandler from comfy_execution.utils import CurrentNodeContext +from comfy_execution.asset_enrichment import enrich_output_with_assets from comfy_api.internal import _ComfyNodeInternal, _NodeOutputInternal, first_real_override, is_class, make_locked_method_func from comfy_api.latest import io, _io from comfy_execution.cache_provider import _has_cache_providers, _get_cache_providers, _logger as _cache_logger @@ -199,6 +200,8 @@ def get_input_data(inputs, class_def, unique_id, execution_list=None, dynprompt= hidden_inputs_v3[io.Hidden.auth_token_comfy_org] = extra_data.get("auth_token_comfy_org", None) if io.Hidden.api_key_comfy_org.name in hidden: hidden_inputs_v3[io.Hidden.api_key_comfy_org] = extra_data.get("api_key_comfy_org", None) + if io.Hidden.comfy_usage_source.name in hidden: + hidden_inputs_v3[io.Hidden.comfy_usage_source] = extra_data.get("comfy_usage_source", None) else: if "hidden" in valid_inputs: h = valid_inputs["hidden"] @@ -215,6 +218,8 @@ def get_input_data(inputs, class_def, unique_id, execution_list=None, dynprompt= input_data_all[x] = [extra_data.get("auth_token_comfy_org", None)] if h[x] == "API_KEY_COMFY_ORG": input_data_all[x] = [extra_data.get("api_key_comfy_org", None)] + if h[x] == "COMFY_USAGE_SOURCE": + input_data_all[x] = [extra_data.get("comfy_usage_source", None)] v3_data["hidden_inputs"] = hidden_inputs_v3 return input_data_all, missing_keys, v3_data @@ -418,6 +423,7 @@ def _is_intermediate_output(dynprompt, node_id): class_def = nodes.NODE_CLASS_MAPPINGS[class_type] return getattr(class_def, 'HAS_INTERMEDIATE_OUTPUT', False) + def _send_cached_ui(server, node_id, display_node_id, cached, prompt_id, ui_outputs): if server.client_id is None: return @@ -552,6 +558,10 @@ async def execute(server, dynprompt, caches, current_item, extra_data, executed, asyncio.create_task(await_completion()) return (ExecutionResult.PENDING, None, None) if len(output_ui) > 0: + # Enrich at output-processing time (not in the send path) so assets + # are registered even when no client is connected, and the asset id + # flows into ui_outputs and the cache alongside the raw entries. + output_ui = enrich_output_with_assets(output_ui) ui_outputs[unique_id] = { "meta": { "node_id": unique_id, @@ -1298,6 +1308,25 @@ class PromptQueue: queued = copy.copy(self.queue) return (running, queued) + def interrupt_if_running(self, prompt_id): + """Interrupt the running prompt with this id, atomically. + + Checks the live running set and signals the interrupt under the queue + mutex, so the worker cannot move the job to done (and start the next + prompt) in between. Returns True if a matching job was running and an + interrupt was signalled, False otherwise. The atomicity is what keeps a + cancel from landing on an unrelated prompt that started after a separate + is-running check: the global interrupt flag is reset at the start of + every prompt (execute_async), so a job that finishes before consuming + the flag cannot leak the interrupt onto its successor. + """ + with self.mutex: + for item in self.currently_running.values(): + if item[1] == prompt_id: + nodes.interrupt_processing() + return True + return False + def get_tasks_remaining(self): with self.mutex: return len(self.queue) + len(self.currently_running) diff --git a/extra_model_paths.yaml.example b/extra_model_paths.yaml.example index 9c395c0b2..6a31d8a63 100644 --- a/extra_model_paths.yaml.example +++ b/extra_model_paths.yaml.example @@ -8,21 +8,37 @@ # # You can use is_default to mark that these folders should be listed first, and used as the default dirs for eg downloads # #is_default: true # checkpoints: models/checkpoints/ +# configs: models/configs/ +# loras: models/loras/ +# vae: models/vae/ # text_encoders: | # models/text_encoders/ -# models/clip/ # legacy location still supported -# clip_vision: models/clip_vision/ -# configs: models/configs/ -# controlnet: models/controlnet/ +# models/clip/ # diffusion_models: | -# models/diffusion_models -# models/unet +# models/unet/ +# models/diffusion_models/ +# clip_vision: models/clip_vision/ +# style_models: models/style_models/ # embeddings: models/embeddings/ -# loras: models/loras/ +# diffusers: models/diffusers/ +# vae_approx: models/vae_approx/ +# controlnet: | +# models/controlnet/ +# models/t2i_adapter/ +# gligen: models/gligen/ # upscale_models: models/upscale_models/ -# vae: models/vae/ -# audio_encoders: models/audio_encoders/ +# latent_upscale_models: models/latent_upscale_models/ +# custom_nodes: custom_nodes/ +# hypernetworks: models/hypernetworks/ +# photomaker: models/photomaker/ +# classifiers: models/classifiers/ # model_patches: models/model_patches/ +# audio_encoders: models/audio_encoders/ +# background_removal: models/background_removal/ +# frame_interpolation: models/frame_interpolation/ +# geometry_estimation: models/geometry_estimation/ +# optical_flow: models/optical_flow/ +# detection: models/detection/ #config for a1111 ui @@ -45,8 +61,7 @@ # controlnet: models/ControlNet -# For a full list of supported keys (style_models, vae_approx, hypernetworks, photomaker, -# model_patches, audio_encoders, classifiers, etc.) see folder_paths.py. +# For the canonical list of supported keys and extensions, see folder_paths.py. #other_ui: # base_path: path/to/ui diff --git a/folder_paths.py b/folder_paths.py index 7304e1b73..ee048b0f2 100644 --- a/folder_paths.py +++ b/folder_paths.py @@ -264,6 +264,59 @@ def annotated_filepath(name: str) -> tuple[str, str | None]: return name, base_dir +# Content types a browser may execute or render inline. File endpoints that +# serve user-controlled content must force these to download (and ideally set +# Content-Disposition: attachment) to avoid stored XSS. Centralised here so the +# /view and /userdata handlers can't drift apart. mimetypes.guess_type may +# return either the text/* or application/* spelling depending on platform, so +# both are listed. +DANGEROUS_CONTENT_TYPES = { + 'text/html', 'text/html-sandboxed', 'application/xhtml+xml', + 'text/javascript', 'application/javascript', 'application/x-javascript', + 'application/ecmascript', 'text/css', + 'image/svg+xml', 'application/xml', 'text/xml', + # message/rfc822 (.mht/.mhtml) can carry script in some browsers. + 'message/rfc822', +} + + +def is_dangerous_content_type(content_type: str | None) -> bool: + """Return True if a browser may execute or render `content_type` inline. + + Normalises before matching so the check can't be slipped past with a + charset/boundary parameter (``text/html; charset=utf-8``) or casing + (``TEXT/HTML``). Any XML dialect (``*+xml`` or ``*/xml``) is treated as + dangerous because XML can carry inline script via stylesheet/entity tricks, + which also covers the ``application/{xslt,rss,atom,rdf}+xml`` family without + enumerating each one. Endpoints serving user-controlled content should route + a dangerous type to ``application/octet-stream`` + ``Content-Disposition: + attachment`` + ``X-Content-Type-Options: nosniff``. + """ + if not content_type: + return False + normalized = content_type.split(';', 1)[0].strip().lower() + if normalized in DANGEROUS_CONTENT_TYPES: + return True + return normalized.endswith('+xml') or normalized.endswith('/xml') + + +def is_within_directory(directory: str, target: str) -> bool: + """Return True if `target` resolves to a path inside `directory`. + + Uses realpath on both operands so that a symlink placed inside `directory` + that points elsewhere cannot escape the containment check at open time. + """ + try: + directory = os.path.realpath(directory) + target = os.path.realpath(target) + return os.path.commonpath((directory, target)) == directory + except ValueError: + # ValueError is raised by realpath() on a path with an embedded null + # byte, and by commonpath() on Windows when the paths are on different + # drives. In either case the target is not safely within the directory. + return False + + def get_annotated_filepath(name: str, default_dir: str | None=None) -> str: name, base_dir = annotated_filepath(name) @@ -273,7 +326,12 @@ def get_annotated_filepath(name: str, default_dir: str | None=None) -> str: else: base_dir = get_input_directory() # fallback path - return os.path.join(base_dir, name) + filepath = os.path.abspath(os.path.join(base_dir, name)) + # Prevent path traversal: the resolved path must stay within base_dir. + # repr() the name in the message so a crafted value can't inject log lines. + if not is_within_directory(base_dir, filepath): + raise ValueError("Invalid file path: {!r}".format(name)) + return filepath def exists_annotated_filepath(name) -> bool: @@ -282,7 +340,10 @@ def exists_annotated_filepath(name) -> bool: if base_dir is None: base_dir = get_input_directory() # fallback path - filepath = os.path.join(base_dir, name) + filepath = os.path.abspath(os.path.join(base_dir, name)) + # Treat traversal attempts as non-existent rather than probing the filesystem. + if not is_within_directory(base_dir, filepath): + return False return os.path.exists(filepath) diff --git a/main.py b/main.py index 7fcc8e97d..20ec83c9e 100644 --- a/main.py +++ b/main.py @@ -55,7 +55,11 @@ if __name__ == "__main__" and args.debug_hang: import comfy_aimdo.control if enables_dynamic_vram(): - comfy_aimdo.control.init() + try: + comfy_aimdo.control.init(simple_vram_headroom=None if args.reserve_vram is None else int(args.reserve_vram * 1024 ** 3)) + except TypeError: + # comfy-aimdo 0.4.9 protocol. + comfy_aimdo.control.init() if os.name == "nt": os.environ['MIMALLOC_PURGE_DELAY'] = '0' @@ -123,6 +127,10 @@ def apply_custom_paths(): for config_path in itertools.chain(*args.extra_model_paths_config): utils.extra_config.load_extra_path_config(config_path) + # --base-directory + if args.base_directory: + logging.info(f"Setting base directory to: {folder_paths.base_path}") + # --output-directory, --input-directory, --user-directory if args.output_directory: output_dir = os.path.abspath(args.output_directory) @@ -231,23 +239,30 @@ import comfy.model_patcher if args.enable_dynamic_vram or (enables_dynamic_vram() and comfy.model_management.is_nvidia() and not comfy.model_management.is_wsl()): if (not args.enable_dynamic_vram) and (comfy.model_management.torch_version_numeric < (2, 8)): logging.warning("Unsupported Pytorch detected. DynamicVRAM support requires Pytorch version 2.8 or later. Falling back to legacy ModelPatcher. VRAM estimates may be unreliable especially on Windows") - elif comfy_aimdo.control.init_devices(d.index for d in comfy.model_management.get_all_torch_devices()): - if args.verbose == 'DEBUG': - comfy_aimdo.control.set_log_debug() - elif args.verbose == 'CRITICAL': - comfy_aimdo.control.set_log_critical() - elif args.verbose == 'ERROR': - comfy_aimdo.control.set_log_error() - elif args.verbose == 'WARNING': - comfy_aimdo.control.set_log_warning() - else: #INFO - comfy_aimdo.control.set_log_info() - - comfy.model_patcher.CoreModelPatcher = comfy.model_patcher.ModelPatcherDynamic - comfy.memory_management.aimdo_enabled = True - logging.info("DynamicVRAM support detected and enabled") else: - logging.warning("No working comfy-aimdo install detected. DynamicVRAM support disabled. Falling back to legacy ModelPatcher. VRAM estimates may be unreliable especially on Windows") + try: + aimdo_initialized = comfy_aimdo.control.init_devices((d.index, int(args.vram_headroom * 1024 ** 3)) for d in comfy.model_management.get_all_torch_devices()) + except TypeError: + # comfy-aimdo 0.4.9 protocol. + aimdo_initialized = comfy_aimdo.control.init_devices(d.index for d in comfy.model_management.get_all_torch_devices()) + + if aimdo_initialized: + if args.verbose == 'DEBUG': + comfy_aimdo.control.set_log_debug() + elif args.verbose == 'CRITICAL': + comfy_aimdo.control.set_log_critical() + elif args.verbose == 'ERROR': + comfy_aimdo.control.set_log_error() + elif args.verbose == 'WARNING': + comfy_aimdo.control.set_log_warning() + else: #INFO + comfy_aimdo.control.set_log_info() + + comfy.model_patcher.CoreModelPatcher = comfy.model_patcher.ModelPatcherDynamic + comfy.memory_management.aimdo_enabled = True + logging.info("DynamicVRAM support detected and enabled") + else: + logging.warning("No working comfy-aimdo install detected. DynamicVRAM support disabled. Falling back to legacy ModelPatcher. VRAM estimates may be unreliable especially on Windows") def cuda_malloc_warning(): @@ -388,7 +403,7 @@ def prompt_worker(q, server_instance): hook_breaker_ac10a0.restore_functions() if not asset_seeder.is_disabled(): - asset_seeder.enqueue_enrich(roots=("output",), compute_hashes=True) + asset_seeder.enqueue_enrich(roots=("output",), compute_hashes=args.enable_asset_hashing) asset_seeder.resume() @@ -443,7 +458,7 @@ def setup_database(): if dependencies_available(): init_db() if args.enable_assets: - if asset_seeder.start(roots=("models", "input", "output"), prune_first=True, compute_hashes=True): + if asset_seeder.start(roots=("models", "input", "output"), prune_first=True, compute_hashes=args.enable_asset_hashing): logging.info("Background asset scan initiated for models, input, output") except Exception as e: if "database is locked" in str(e): @@ -490,6 +505,11 @@ def start_comfyui(asyncio_loop=None): init_custom_nodes=(not args.disable_all_custom_nodes) or len(args.whitelist_custom_nodes) > 0, init_api_nodes=not args.disable_api_nodes )) + + # Re-apply Comfy's cuDNN benchmark policy after custom-node imports. Benchmark + # mode can request near-card-sized autotune workspaces, and some custom nodes set it at import time. + comfy.model_management.set_cudnn_benchmark() + hook_breaker_ac10a0.restore_functions() cuda_malloc_warning() @@ -537,8 +557,13 @@ if __name__ == "__main__": logging.warning("WARNING: You are using a python version older than 3.10, please upgrade to a newer one. 3.12 and above is recommended.") if args.disable_dynamic_vram: - logging.warning("Dynamic vram disabled with argument. If you have any issues with dynamic vram enabled please give us a detailed reports as this argument will be removed soon.") - + logging.warning( + "Dynamic vram disabled with argument. If you have any issues with " + "dynamic vram enabled please give us a detailed reports as this " + "argument will be removed soon. If you use gguf we recommend keeping " + "dynamic vram enabled and using native ComfyUI model formats instead. " + "ComfyUI native formats like fp8 will be faster even if they are larger than your memory." + ) event_loop, _, start_all_func = start_comfyui() try: x = start_all_func() diff --git a/manager_requirements.txt b/manager_requirements.txt index a079d3492..13786bb35 100644 --- a/manager_requirements.txt +++ b/manager_requirements.txt @@ -1 +1 @@ -comfyui_manager==4.2.1 +comfyui_manager==4.2.2 diff --git a/nodes.py b/nodes.py index 0d422d418..e126576fe 100644 --- a/nodes.py +++ b/nodes.py @@ -20,8 +20,6 @@ from PIL.PngImagePlugin import PngInfo import numpy as np import safetensors.torch -sys.path.insert(0, os.path.join(os.path.dirname(os.path.realpath(__file__)), "comfy")) - import comfy.diffusers_load import comfy.samplers import comfy.sample @@ -87,7 +85,7 @@ class ConditioningCombine: RETURN_TYPES = ("CONDITIONING",) FUNCTION = "combine" - CATEGORY = "model/conditioning" + CATEGORY = "model/conditioning/transform" SEARCH_ALIASES = ["combine", "merge conditioning", "combine prompts", "merge prompts", "mix prompts", "add prompt"] def combine(self, conditioning_1, conditioning_2): @@ -104,7 +102,7 @@ class ConditioningAverage : RETURN_TYPES = ("CONDITIONING",) FUNCTION = "addWeighted" - CATEGORY = "model/conditioning" + CATEGORY = "model/conditioning/transform" def addWeighted(self, conditioning_to, conditioning_from, conditioning_to_strength): out = [] @@ -143,7 +141,7 @@ class ConditioningConcat: RETURN_TYPES = ("CONDITIONING",) FUNCTION = "concat" - CATEGORY = "model/conditioning" + CATEGORY = "model/conditioning/transform" def concat(self, conditioning_to, conditioning_from): out = [] @@ -161,6 +159,29 @@ class ConditioningConcat: return (out, ) +class ConditioningMultiply: + SEARCH_ALIASES = ["scale conditioning", "scale prompt", "multiply conditioning", "multiply prompt"] + + @classmethod + def INPUT_TYPES(cls): + return {"required": {"conditioning": ("CONDITIONING", ), + "multiplier": ("FLOAT", {"default": 1.0, "min": -100.0, "max": 100.0, "step": 0.01}) + }} + RETURN_TYPES = ("CONDITIONING",) + FUNCTION = "multiply" + CATEGORY = "model/conditioning/transform" + + def multiply(self, conditioning, multiplier): + c = [] + for t in conditioning: + values = {} + pooled_output = t[1].get("pooled_output", None) + if pooled_output is not None: + values["pooled_output"] = pooled_output * multiplier + scaled = node_helpers.conditioning_set_values([[t[0] * multiplier, t[1]]], values)[0] + c.append(scaled) + return (c,) + class ConditioningSetArea: SEARCH_ALIASES = ["regional prompt", "area prompt", "spatial conditioning", "localized prompt"] @@ -176,7 +197,7 @@ class ConditioningSetArea: RETURN_TYPES = ("CONDITIONING",) FUNCTION = "append" - CATEGORY = "model/conditioning" + CATEGORY = "model/conditioning/transform" def append(self, conditioning, width, height, x, y, strength): c = node_helpers.conditioning_set_values(conditioning, {"area": (height // 8, width // 8, y // 8, x // 8), @@ -197,7 +218,7 @@ class ConditioningSetAreaPercentage: RETURN_TYPES = ("CONDITIONING",) FUNCTION = "append" - CATEGORY = "model/conditioning" + CATEGORY = "model/conditioning/transform" def append(self, conditioning, width, height, x, y, strength): c = node_helpers.conditioning_set_values(conditioning, {"area": ("percentage", height, width, y, x), @@ -214,7 +235,7 @@ class ConditioningSetAreaStrength: RETURN_TYPES = ("CONDITIONING",) FUNCTION = "append" - CATEGORY = "model/conditioning" + CATEGORY = "model/conditioning/transform" def append(self, conditioning, strength): c = node_helpers.conditioning_set_values(conditioning, {"strength": strength}) @@ -234,7 +255,7 @@ class ConditioningSetMask: RETURN_TYPES = ("CONDITIONING",) FUNCTION = "append" - CATEGORY = "model/conditioning" + CATEGORY = "model/conditioning/transform" def append(self, conditioning, mask, set_cond_area, strength): set_area_to_bounds = False @@ -257,7 +278,7 @@ class ConditioningZeroOut: RETURN_TYPES = ("CONDITIONING",) FUNCTION = "zero_out" - CATEGORY = "advanced/conditioning" + CATEGORY = "model/conditioning/transform" def zero_out(self, conditioning): c = [] @@ -283,11 +304,10 @@ class ConditioningSetTimestepRange: RETURN_TYPES = ("CONDITIONING",) FUNCTION = "set_range" - CATEGORY = "advanced/conditioning" + CATEGORY = "model/conditioning/transform" def set_range(self, conditioning, start, end): - c = node_helpers.conditioning_set_values(conditioning, {"start_percent": start, - "end_percent": end}) + c = node_helpers.conditioning_set_values(conditioning, {"start_percent": start, "end_percent": end}) return (c, ) class VAEDecode: @@ -329,7 +349,7 @@ class VAEDecodeTiled: RETURN_TYPES = ("IMAGE",) FUNCTION = "decode" - CATEGORY = "experimental" + CATEGORY = "model/latent" def decode(self, vae, samples, tile_size, overlap=64, temporal_size=64, temporal_overlap=8): if tile_size < overlap * 4: @@ -376,7 +396,7 @@ class VAEEncodeTiled: RETURN_TYPES = ("LATENT",) FUNCTION = "encode" - CATEGORY = "experimental" + CATEGORY = "model/latent" def encode(self, vae, pixels, tile_size, overlap, temporal_size=64, temporal_overlap=8): t = vae.encode_tiled(pixels, tile_x=tile_size, tile_y=tile_size, overlap=overlap, tile_t=temporal_size, overlap_t=temporal_overlap) @@ -389,7 +409,7 @@ class VAEEncodeForInpaint: RETURN_TYPES = ("LATENT",) FUNCTION = "encode" - CATEGORY = "model/latent/inpaint" + CATEGORY = "model/latent" def encode(self, vae, pixels, mask, grow_mask_by=6): downscale_ratio = vae.spacial_compression_encode() @@ -438,7 +458,7 @@ class InpaintModelConditioning: RETURN_NAMES = ("positive", "negative", "latent") FUNCTION = "encode" - CATEGORY = "model/conditioning/inpaint" + CATEGORY = "model/conditioning" def encode(self, positive, negative, pixels, vae, mask, noise_mask=True): x = (pixels.shape[1] // 8) * 8 @@ -483,16 +503,18 @@ class SaveLatent: @classmethod def INPUT_TYPES(s): - return {"required": { "samples": ("LATENT", ), - "filename_prefix": ("STRING", {"default": "latents/ComfyUI"})}, - "hidden": {"prompt": "PROMPT", "extra_pnginfo": "EXTRA_PNGINFO"}, - } - RETURN_TYPES = () + return { "required": { + "samples": ("LATENT",), + "filename_prefix": ("STRING", {"default": "latents/ComfyUI"})}, + "hidden": {"prompt": "PROMPT", "extra_pnginfo": "EXTRA_PNGINFO"}, + } + RETURN_TYPES = ("LATENT",) + RETURN_NAMES = ("samples",) FUNCTION = "save" OUTPUT_NODE = True - CATEGORY = "experimental" + CATEGORY = "model/latent" def save(self, samples, filename_prefix="ComfyUI", prompt=None, extra_pnginfo=None): full_output_folder, filename, counter, subfolder, filename_prefix = folder_paths.get_save_image_path(filename_prefix, self.output_dir) @@ -525,7 +547,7 @@ class SaveLatent: output["latent_format_version_0"] = torch.tensor([]) comfy.utils.save_torch_file(output, file, metadata=metadata) - return { "ui": { "latents": results } } + return { "ui": { "latents": results }, "result": (samples,) } class LoadLatent: @@ -537,7 +559,7 @@ class LoadLatent: files = [f for f in os.listdir(input_dir) if os.path.isfile(os.path.join(input_dir, f)) and f.endswith(".latent")] return {"required": {"latent": [sorted(files), ]}, } - CATEGORY = "experimental" + CATEGORY = "model/latent" RETURN_TYPES = ("LATENT", ) FUNCTION = "load" @@ -576,7 +598,7 @@ class CheckpointLoader: RETURN_TYPES = ("MODEL", "CLIP", "VAE") FUNCTION = "load_checkpoint" - CATEGORY = "advanced/loaders" + CATEGORY = "model/loaders" DEPRECATED = True def load_checkpoint(self, config_name, ckpt_name): @@ -622,8 +644,9 @@ class DiffusersLoader: return {"required": {"model_path": (paths,), }} RETURN_TYPES = ("MODEL", "CLIP", "VAE") FUNCTION = "load_checkpoint" + DEPRECATED = True - CATEGORY = "advanced/loaders/deprecated" + CATEGORY = "model/loaders" def load_checkpoint(self, model_path, output_vae=True, output_clip=True): for search_path in folder_paths.get_folder_paths("diffusers"): @@ -949,7 +972,7 @@ class UNETLoader: RETURN_TYPES = ("MODEL",) FUNCTION = "load_unet" - CATEGORY = "advanced/loaders" + CATEGORY = "model/loaders" def load_unet(self, unet_name, weight_dtype): model_options = {} @@ -969,7 +992,7 @@ class CLIPLoader: @classmethod def INPUT_TYPES(s): return {"required": { "clip_name": (folder_paths.get_filename_list("text_encoders"), ), - "type": (["stable_diffusion", "stable_cascade", "sd3", "stable_audio", "mochi", "ltxv", "pixart", "cosmos", "lumina2", "wan", "hidream", "chroma", "ace", "omnigen2", "qwen_image", "hunyuan_image", "flux2", "ovis", "longcat_image", "cogvideox", "lens", "pixeldit", "ideogram4"], ), + "type": (["stable_diffusion", "stable_cascade", "sd3", "stable_audio", "mochi", "ltxv", "pixart", "cosmos", "lumina2", "wan", "hidream", "chroma", "ace", "omnigen2", "qwen_image", "hunyuan_image", "flux2", "ovis", "longcat_image", "cogvideox", "lens", "pixeldit", "ideogram4", "boogu", "krea2"], ), }, "optional": { "device": (["default", "cpu"], {"advanced": True}), @@ -977,9 +1000,9 @@ class CLIPLoader: RETURN_TYPES = ("CLIP",) FUNCTION = "load_clip" - CATEGORY = "advanced/loaders" + CATEGORY = "model/loaders" - DESCRIPTION = "[Recipes]\n\nstable_diffusion: clip-l\nstable_cascade: clip-g\nsd3: t5 xxl/ clip-g / clip-l\nstable_audio: t5 base\nmochi: t5 xxl\ncogvideox: t5 xxl (226-token padding)\ncosmos: old t5 xxl\nlumina2: gemma 2 2B\nwan: umt5 xxl\n hidream: llama-3.1 (Recommend) or t5\nomnigen2: qwen vl 2.5 3B\nlens: gpt-oss-20b\n pixeldit: gemma 2 2B elm" + DESCRIPTION = "Recipes:\nsd: clip-l\nstable cascade: clip-g\nsd3: t5 xxl / clip-g / clip-l\nstable audio: t5 base\nmochi: t5 xxl\ncogvideox: t5 xxl (226-token padding)\ncosmos: old t5 xxl\nlumina2: gemma 2 2B\nwan: umt5 xxl\nhidream: llama-3.1 (Recommend) or t5\nomnigen2: qwen vl 2.5 3B\nlens: gpt-oss-20b\npixeldit: gemma 2 2B elm" def load_clip(self, clip_name, type="stable_diffusion", device="default"): clip_type = getattr(comfy.sd.CLIPType, type.upper(), comfy.sd.CLIPType.STABLE_DIFFUSION) @@ -1005,9 +1028,9 @@ class DualCLIPLoader: RETURN_TYPES = ("CLIP",) FUNCTION = "load_clip" - CATEGORY = "advanced/loaders" + CATEGORY = "model/loaders" - DESCRIPTION = "[Recipes]\n\nsdxl: clip-l, clip-g\nsd3: clip-l, clip-g / clip-l, t5 / clip-g, t5\nflux: clip-l, t5\nhidream: at least one of t5 or llama, recommended t5 and llama\nhunyuan_image: qwen2.5vl 7b and byt5 small\nnewbie: gemma-3-4b-it, jina clip v2" + DESCRIPTION = "Recipes:\nsdxl: clip-l, clip-g\nsd3: clip-l, clip-g / clip-l, t5 / clip-g, t5\nflux: clip-l, t5\nhidream: at least one of t5 or llama, recommended t5 and llama\nhunyuan_image: qwen2.5vl 7b and byt5 small\nnewbie: gemma-3-4b-it, jina clip v2" def load_clip(self, clip_name1, clip_name2, type, device="default"): clip_type = getattr(comfy.sd.CLIPType, type.upper(), comfy.sd.CLIPType.STABLE_DIFFUSION) @@ -1088,7 +1111,7 @@ class StyleModelApply: RETURN_TYPES = ("CONDITIONING",) FUNCTION = "apply_stylemodel" - CATEGORY = "model/conditioning/style_model" + CATEGORY = "model/conditioning" def apply_stylemodel(self, conditioning, style_model, clip_vision_output, strength, strength_type): cond = style_model.get_cond(clip_vision_output).flatten(start_dim=0, end_dim=1).unsqueeze(dim=0) @@ -1518,13 +1541,11 @@ class LatentCrop: class SetLatentNoiseMask: @classmethod def INPUT_TYPES(s): - return {"required": { "samples": ("LATENT",), - "mask": ("MASK",), - }} + return {"required": { "samples": ("LATENT",), "mask": ("MASK",), }} RETURN_TYPES = ("LATENT",) FUNCTION = "set_mask" - CATEGORY = "model/latent/inpaint" + CATEGORY = "model/latent" def set_mask(self, samples, mask): s = samples.copy() @@ -1631,14 +1652,18 @@ class SaveImage: return { "required": { "images": ("IMAGE", {"tooltip": "The images to save."}), - "filename_prefix": ("STRING", {"default": "ComfyUI", "tooltip": "The prefix for the file to save. This may include formatting information such as %date:yyyy-MM-dd% or %Empty Latent Image.width% to include values from nodes."}) + "filename_prefix": ("STRING", { + "default": "ComfyUI", + "tooltip": "The prefix for the file to save. This may include formatting information such as %date:yyyy-MM-dd% or %Empty Latent Image.width% to include values from nodes." + }) }, "hidden": { "prompt": "PROMPT", "extra_pnginfo": "EXTRA_PNGINFO" }, } - RETURN_TYPES = () + RETURN_TYPES = ("IMAGE",) + RETURN_NAMES = ("images",) FUNCTION = "save_images" OUTPUT_NODE = True @@ -1674,7 +1699,7 @@ class SaveImage: }) counter += 1 - return { "ui": { "images": results } } + return { "ui": { "images": results }, "result" : (images,) } class PreviewImage(SaveImage): def __init__(self): @@ -2045,9 +2070,10 @@ NODE_CLASS_MAPPINGS = { "ImageBatch": ImageBatch, "ImagePadForOutpaint": ImagePadForOutpaint, "EmptyImage": EmptyImage, - "ConditioningAverage": ConditioningAverage , + "ConditioningAverage": ConditioningAverage, "ConditioningCombine": ConditioningCombine, "ConditioningConcat": ConditioningConcat, + "ConditioningMultiply": ConditioningMultiply, "ConditioningSetArea": ConditioningSetArea, "ConditioningSetAreaPercentage": ConditioningSetAreaPercentage, "ConditioningSetAreaStrength": ConditioningSetAreaStrength, @@ -2101,6 +2127,7 @@ NODE_DISPLAY_NAME_MAPPINGS = { "LoraLoader": "Load LoRA (Model and CLIP)", "LoraLoaderModelOnly": "Load LoRA", "CLIPLoader": "Load CLIP", + "DualCLIPLoader": "Load CLIP (Dual)", "ControlNetLoader": "Load ControlNet Model", "DiffControlNetLoader": "Load ControlNet Model (diff)", "StyleModelLoader": "Load Style Model", @@ -2108,6 +2135,7 @@ NODE_DISPLAY_NAME_MAPPINGS = { "UNETLoader": "Load Diffusion Model", "unCLIPCheckpointLoader": "Load unCLIP Checkpoint", "GLIGENLoader": "Load GLIGEN Model", + "DiffusersLoader": "Load Diffusers Model (DEPRECATED)", # Conditioning "CLIPVisionEncode": "CLIP Vision Encode", "StyleModelApply": "Apply Style Model", @@ -2115,13 +2143,20 @@ NODE_DISPLAY_NAME_MAPPINGS = { "CLIPSetLastLayer": "CLIP Set Last Layer", "ConditioningCombine": "Conditioning (Combine)", "ConditioningAverage ": "Conditioning (Average)", + "ConditioningAverage": "Conditioning (Average)", "ConditioningConcat": "Conditioning (Concat)", + "ConditioningMultiply": "Conditioning (Multiply)", "ConditioningSetArea": "Conditioning (Set Area)", "ConditioningSetAreaPercentage": "Conditioning (Set Area with Percentage)", + "ConditioningSetAreaStrength": "Conditioning (Set Area Strength)", "ConditioningSetMask": "Conditioning (Set Mask)", "ControlNetApply": "Apply ControlNet (DEPRECATED)", "ControlNetApplyAdvanced": "Apply ControlNet", + "GLIGENTextBoxApply": "Apply GLIGEN Text Box", + "ConditioningZeroOut": "Conditioning Zero Out", # Latent + "LoadLatent": "Load Latent", + "SaveLatent": "Save Latent", "VAEEncodeForInpaint": "VAE Encode (for Inpainting)", "SetLatentNoiseMask": "Set Latent Noise Mask", "VAEDecode": "VAE Decode", @@ -2134,7 +2169,7 @@ NODE_DISPLAY_NAME_MAPPINGS = { "LatentUpscaleBy": "Upscale Latent By", "LatentComposite": "Latent Composite", "LatentBlend": "Latent Blend", - "LatentFromBatch" : "Latent From Batch", + "LatentFromBatch" : "Get Latent From Batch", "RepeatLatentBatch": "Repeat Latent Batch", # Image "EmptyImage": "Empty Image", @@ -2156,7 +2191,6 @@ NODE_DISPLAY_NAME_MAPPINGS = { "ImageSharpen": "Sharpen Image", "ImageScaleToTotalPixels": "Scale Image to Total Pixels", "GetImageSize": "Get Image Size", - # experimental "VAEDecodeTiled": "VAE Decode (Tiled)", "VAEEncodeTiled": "VAE Encode (Tiled)", } @@ -2295,6 +2329,9 @@ async def init_external_custom_nodes(): Returns: None """ + # TODO: remove at some point when custom nodes don't break. + sys.path.insert(0, os.path.join(os.path.dirname(os.path.realpath(__file__)), "comfy")) + base_node_names = set(NODE_CLASS_MAPPINGS.keys()) node_paths = folder_paths.get_folder_paths("custom_nodes") node_import_times = [] @@ -2363,6 +2400,8 @@ async def init_builtin_extra_nodes(): "nodes_images.py", "nodes_video_model.py", "nodes_ideogram4.py", + "nodes_bounding_boxes.py", + "nodes_json_prompt.py", "nodes_train.py", "nodes_dataset.py", "nodes_sag.py", @@ -2421,6 +2460,7 @@ async def init_builtin_extra_nodes(): "nodes_tcfg.py", "nodes_context_windows.py", "nodes_qwen.py", + "nodes_boogu.py", "nodes_chroma_radiance.py", "nodes_pid.py", "nodes_model_patch.py", @@ -2438,6 +2478,7 @@ async def init_builtin_extra_nodes(): "nodes_glsl.py", "nodes_lora_debug.py", "nodes_textgen.py", + "nodes_text_overlay.py", "nodes_color.py", "nodes_toolkit.py", "nodes_replacements.py", @@ -2461,6 +2502,7 @@ async def init_builtin_extra_nodes(): "nodes_gaussian_splat.py", "nodes_triposplat.py", "nodes_depth_anything_3.py", + "nodes_seed.py", ] import_failed = [] diff --git a/openapi.yaml b/openapi.yaml index c27ed7adf..c6a8621cc 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -55,6 +55,12 @@ components: description: URL for asset preview/thumbnail format: uri type: string + short_url: + description: Durable, owner-gated short link to this asset's content (relative `/api/s/{id}` path). Stable across the underlying signed URL's expiry — resolving it re-mints a fresh signed URL on every request — so it is safe to persist or share into chat, unlike `preview_url`. Only the minting user can resolve it. Omitted when the short-link surface is disabled or the asset has no resolvable content hash. + nullable: true + type: string + x-runtime: + - cloud size: description: Size of the asset in bytes format: int64 @@ -673,6 +679,35 @@ components: - created_at - updated_at type: object + JobsCancelRequest: + additionalProperties: false + description: Request to cancel multiple jobs by ID. + properties: + job_ids: + description: Job identifiers (UUIDs) to cancel. + items: + format: uuid + type: string + maxItems: 100 + minItems: 1 + type: array + required: + - job_ids + type: object + JobsCancelResponse: + description: Response for POST /api/jobs/cancel. + properties: + cancelled: + description: | + Job IDs for which a cancel event was successfully dispatched by this + call. Jobs already in a terminal or cancelling state are idempotently + skipped and will not appear here. + items: + type: string + type: array + required: + - cancelled + type: object JobsListResponse: description: Paginated list of jobs for the authenticated user. properties: @@ -1006,7 +1041,7 @@ components: description: If true, clear all pending jobs from the queue type: boolean delete: - description: Array of PENDING job IDs to cancel + description: Array of job IDs to cancel; pending and running jobs transition to cancelled items: type: string type: array @@ -1062,6 +1097,9 @@ components: comfyui_version: description: ComfyUI version type: string + deploy_environment: + description: How this ComfyUI instance is deployed (e.g. cloud, local-git, local-portable, local-desktop) + type: string embedded_python: description: Whether using embedded Python type: boolean @@ -1654,6 +1692,12 @@ paths: schema: $ref: '#/components/schemas/ErrorResponse' description: Unsupported media type + "422": + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + description: Validation error (e.g., disallowed model_type tag) "500": content: application/json: @@ -1792,13 +1836,91 @@ paths: application/json: schema: $ref: '#/components/schemas/ErrorResponse' - description: Invalid request (no fields provided) + description: | + Invalid request — no fields provided, or `preview_id` is the zero UUID + (`INVALID_PREVIEW_ID`). "401": content: application/json: schema: $ref: '#/components/schemas/ErrorResponse' description: Unauthorized + "404": + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + description: | + Asset not found — returned both when the asset being updated does + not exist and when `preview_id` does not reference an asset + accessible to the caller. + "500": + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + description: Internal server error + summary: Update asset metadata + tags: + - file + /api/assets/{id}/content: + get: + description: | + Returns the binary content of an asset by ID. + + The contract is the same across runtimes — "GET this path and you + receive the asset's bytes" — but the mechanism differs: + - **Local ComfyUI** streams the bytes directly (`200`, + `application/octet-stream`). + - **Cloud** does not proxy large files; it responds `302` with a + `Location` redirect to a short-lived signed storage URL. Clients that + follow redirects (browsers, `fetch`/XHR, ``/`