From 9c38af2344ea6987fb717c17a940785fd9a6eda6 Mon Sep 17 00:00:00 2001 From: Uxito-Ada <414416158@qq.com> Date: Fri, 13 Nov 2026 04:35:34 +0000 Subject: [PATCH 1/6] feat: integrate MiniMax H3 Image and PAN2 inference Add native H3-Image six-head PDD generation and reference editing, plus PAN2 text-to-video and first-frame conditioning through existing pipeline, stage, module-loading and worker contracts. Preserve checkpoint sampling order, conditioning layouts, FP32 modulation and rotary buffers, and output conversion. Reuse H3 parallel attention and implement PAN2 Ulysses/TP with padding removal, FP8 linear quantization and Sol-Attn. Support repeated model CPU offload with sharded weights. Keep H3 FP8 prefix/correction inputs contiguous at the existing operator call sites. Add no shared framework configuration or service schema. Provide runnable examples, source and checkpoint provenance, focused numerical regressions, and reproducible distributed/worker validation tools. Twenty new unit tests and 41 existing H3/FP8 regressions pass. Tiny SP2, TP2 and SP2xTP2 checks pass with FP8 GEMM, FP8 Sol-Attn and repeated CPU offload. Released conditioner and image VAE outputs match independent references exactly. Released-transformer generation verification remains in progress while its weights download; the validation document records this boundary explicitly. --- README.md | 2 + README_zh.md | 2 + docs/en/h3_image_pan2_validation.md | 132 +++ examples/minimax_h3_image/README.md | 129 +++ examples/minimax_h3_image/common.py | 91 ++ examples/minimax_h3_image/generate.py | 57 ++ examples/pan2/README.md | 123 +++ examples/pan2/common.py | 77 ++ examples/pan2/generate.py | 64 ++ telefuser/models/minimax_h3_dit.py | 10 +- telefuser/models/minimax_h3_image_dit.py | 143 +++ telefuser/models/minimax_h3_image_encoder.py | 26 + telefuser/models/minimax_h3_image_vae.py | 889 ++++++++++++++++++ telefuser/models/pan2_conditioning.py | 74 ++ telefuser/models/pan2_dit.py | 812 ++++++++++++++++ .../pipelines/minimax_h3_image/__init__.py | 3 + telefuser/pipelines/minimax_h3_image/data.py | 179 ++++ .../pipelines/minimax_h3_image/pipeline.py | 134 +++ .../pipelines/minimax_h3_image/stages.py | 206 ++++ telefuser/pipelines/pan2/__init__.py | 3 + telefuser/pipelines/pan2/data.py | 185 ++++ telefuser/pipelines/pan2/pipeline.py | 119 +++ telefuser/pipelines/pan2/stages.py | 162 ++++ .../models/test_h3_image_pan2_conditioning.py | 25 + .../unit/models/test_minimax_h3_image_dit.py | 180 ++++ .../unit/models/test_minimax_h3_image_vae.py | 68 ++ tests/unit/models/test_pan2_dit.py | 73 ++ .../pipelines/test_h3_image_pan2_sampling.py | 167 ++++ .../validate_h3_image_pan2_parallel.py | 181 ++++ .../validate_h3_image_pan2_sampling.py | 159 ++++ 30 files changed, 4472 insertions(+), 3 deletions(-) create mode 100644 docs/en/h3_image_pan2_validation.md create mode 100644 examples/minimax_h3_image/README.md create mode 100644 examples/minimax_h3_image/common.py create mode 100644 examples/minimax_h3_image/generate.py create mode 100644 examples/pan2/README.md create mode 100644 examples/pan2/common.py create mode 100644 examples/pan2/generate.py create mode 100644 telefuser/models/minimax_h3_image_dit.py create mode 100644 telefuser/models/minimax_h3_image_encoder.py create mode 100644 telefuser/models/minimax_h3_image_vae.py create mode 100644 telefuser/models/pan2_conditioning.py create mode 100644 telefuser/models/pan2_dit.py create mode 100644 telefuser/pipelines/minimax_h3_image/__init__.py create mode 100644 telefuser/pipelines/minimax_h3_image/data.py create mode 100644 telefuser/pipelines/minimax_h3_image/pipeline.py create mode 100644 telefuser/pipelines/minimax_h3_image/stages.py create mode 100644 telefuser/pipelines/pan2/__init__.py create mode 100644 telefuser/pipelines/pan2/data.py create mode 100644 telefuser/pipelines/pan2/pipeline.py create mode 100644 telefuser/pipelines/pan2/stages.py create mode 100644 tests/unit/models/test_h3_image_pan2_conditioning.py create mode 100644 tests/unit/models/test_minimax_h3_image_dit.py create mode 100644 tests/unit/models/test_minimax_h3_image_vae.py create mode 100644 tests/unit/models/test_pan2_dit.py create mode 100644 tests/unit/pipelines/test_h3_image_pan2_sampling.py create mode 100644 tools/validation/validate_h3_image_pan2_parallel.py create mode 100644 tools/validation/validate_h3_image_pan2_sampling.py diff --git a/README.md b/README.md index 3c35b0a8..b5ec345d 100644 --- a/README.md +++ b/README.md @@ -264,6 +264,8 @@ telefuser/ | `LTX Video` | I2V + Audio | Unified audio-video generation via [examples/ltx_video/README.md](examples/ltx_video/README.md) | | `LTX-2.5 Distilled` | T2V, I2V + Audio | ModuleManager-backed six-stage pipeline with 1/2/4-H100 Ulysses SP; see [examples/ltx25_distilled/README.md](examples/ltx25_distilled/README.md) | | `MiniMax H3` | T2VA, FL2VA, Ref2VA + Audio | Local 768p joint audio-video generation via [examples/minimax_h3/README.md](examples/minimax_h3/README.md) | +| `MiniMax-H3-Image` | T2I, Edit | Six-step image generation with SP, TP, Sol-Attn and FP8 via [examples/minimax_h3_image/README.md](examples/minimax_h3_image/README.md) | +| `PAN2` | T2V, I2V | Video generation with SP, TP, Sol-Attn and FP8 via [examples/pan2/README.md](examples/pan2/README.md) | | `LongCat-Video` | T2V, I2V, VC | Long-form generation and continuation via [examples/longcat_video/README.md](examples/longcat_video/README.md) | | `LingBot-Video` | T2I, T2V, TI2V, MoE refiner | Dense/MoE generation with native CFG/SP and an in-memory base-to-refiner path; see [examples/lingbot_video/README.md](examples/lingbot_video/README.md) | diff --git a/README_zh.md b/README_zh.md index 05ad6b08..a7c5e0a5 100644 --- a/README_zh.md +++ b/README_zh.md @@ -242,6 +242,8 @@ telefuser/ | `LTX Video` | I2V + Audio | 统一音视频生成,见 [examples/ltx_video/README.md](examples/ltx_video/README.md) | | `LTX-2.5 Distilled` | T2V, I2V + Audio | 基于 ModuleManager 的六阶段 Pipeline,支持 1/2/4 张 H100 的 Ulysses SP,见 [examples/ltx25_distilled/README.md](examples/ltx25_distilled/README.md) | | `MiniMax H3` | T2VA, FL2VA, Ref2VA + Audio | 本地 768p 联合音视频生成,见 [examples/minimax_h3/README.md](examples/minimax_h3/README.md) | +| `MiniMax-H3-Image` | T2I、编辑 | 六步图像生成,支持 SP、TP、Sol-Attn 和 FP8,参见 [examples/minimax_h3_image/README.md](examples/minimax_h3_image/README.md) | +| `PAN2` | T2V、I2V | 视频生成,支持 SP、TP、Sol-Attn 和 FP8,参见 [examples/pan2/README.md](examples/pan2/README.md) | | `LongCat-Video` | T2V, I2V, VC | 长视频生成与续写,见 [examples/longcat_video/README.md](examples/longcat_video/README.md) | | **NEW** `LingBot-Video` | T2I, T2V, TI2V, MoE refiner | 支持原生 CFG/SP 的 Dense/MoE 生成与内存直传 base-to-refiner,见 [examples/lingbot_video/README.md](examples/lingbot_video/README.md) | diff --git a/docs/en/h3_image_pan2_validation.md b/docs/en/h3_image_pan2_validation.md new file mode 100644 index 00000000..8e58092a --- /dev/null +++ b/docs/en/h3_image_pan2_validation.md @@ -0,0 +1,132 @@ +# MiniMax-H3-Image and PAN2 integration validation + +This document distinguishes architecture/parallel numerical checks from full-checkpoint generation. +A successful small-model check does not establish output quality, throughput or memory requirements of the released model. + +## Sources and runtime + +| Item | Revision / value | +| --- | --- | +| TeleFuser fork baseline | `f21457a92e92f4a7570f0cb6b445ee334f1072b0` | +| MiniMax-AI/MiniMax-H3-Image source | `40850901ab938303b5274122b5f305ebc5b75c80` | +| ifm-ai/PAN2 source | `0fb907b6e16a82ead6d691d1de2eef6299c71e9a` | +| Independent Diffusers reference | `843f1743e259e510f22065c0cb917d417b98d9f5` | +| MiniMaxAI/MiniMax-H3-Image weights | `5f90473023975e6271b0ed1dd84c4e88e4bbc6d3` | +| IFM/PAN2 weights | `82f54c6cea3e20a9c451b61d7360016e5970503d` | +| GPU | NVIDIA H100 80GB HBM3, four NVLink-connected devices, 81,559 MiB each | +| Driver | 590.48.01 | +| CPU / container memory limit | Two Intel Xeon Platinum 8468 sockets, 192 logical CPUs / 800 GiB | +| Python / PyTorch | 3.11.13 / 2.11.0+cu130 | +| Transformers / xFormers | 5.14.1 / 0.0.35 (cu130) | +| Accelerate (independent reference loading only) | 1.15.0 | + +Runtime dependencies were installed in an isolated task environment. Existing unrelated GPU workloads were preserved; +full-component tests use the two free devices. Four-device numerical fixtures allocate only small models. + +## Released transformer architecture + +| Model | Parameters | Blocks / refiner | Attention | Latents / patch | Conditioning | +| --- | --- | --- | --- | --- | --- | +| MiniMax-H3-Image | 33,125,224,384 | 50 / 2 | Hidden 5376, 56 heads × 128 | 24 channels, 1×2×2 patch | Qwen3-VL layer 50, 5120 features, six fused PDD heads | +| PAN2 | 29,201,996,992 | 44 / 4 | Hidden 4096, 32 heads × 128 | 48 channels; noisy + condition + mask = 97 input channels, 1×2×2 patch | Provided Qwen3.5 text encoder, hidden_states[-3], 4096 features | + +Counts are obtained from meta-device models built from the released configuration, including H3's six PDD heads. +H3 boundary projections and PAN2 timestep/modulation projections retain their checkpoint FP32 precision. + +## Numerical checks + +- H3-Image DiT: exact agreement with the independent Diffusers tiny model after checkpoint conversion, including + gate/value ordering and native FP32 boundary modules. +- PAN2 DiT: exact FP32 agreement with the independent Diffusers tiny model. +- H3-Image VAE: exact encoder and decoder agreement, including overlapping tile stitching. The full released + mixed-precision VAE also matches the official from_pretrained loader with zero encoder/decoder error. +- PAN2 full text encoder: exact agreement with the official prompt-embedding helper for a short prompt and the + 1477-token released negative caption. H3-Image pure-text and 1024-pixel reference-image conditioning also match + the full 64-layer official Qwen3-VL reference exactly at hidden_states[50]. Non-persistent rotary frequencies remain FP32 when model weights cast to BF16. +- Sampling: H3 positive velocity direction, six step-dependent heads, frozen reference rows, per-row reference timestep, + image packing and optimized-attention padding; PAN2 CFG, conditional first frame, FP32 accumulation and negative-caption + duration are covered by unit tests. +- New architecture, conditioning precision, VAE and sampling tests: 20 passed. Existing H3 model and FP8 smoothing regressions: 41 passed. +- Complete six-head H3-Image edit sampling agrees exactly with the independent reference scheduler and model. +- Actual ParallelWorker sampling (with reference/first-frame inputs, odd text lengths, all sampling steps and two + consecutive requests) passed SP2, TP2 and SP2×TP2 for both models with FP8 linear and FP8 Sol-Attn enabled. + +The deterministic model fixtures use two transformer blocks, one text refiner and seed 2026 (CPU input generator 71). +H3 uses hidden 256, four 128-dimensional heads, FFN 512, latent channels 2 and text features 32; the Sol fixture has +384 packed rows with four text rows. PAN2 uses four 128-dimensional heads, latent output channels 2, five input channels, +text features 32 and FFN ratios 2. Its Sol fixture input is `[1,5,3,16,32]` with five text tokens; dense SP also tests +27 video tokens and five text tokens. These are intentionally reduced architectures. + +The distributed fixture compares an unchanged single-device BF16 model to the same model after actual sharding and +quantization. PAN2 uses odd video/text sequence lengths to exercise padding removal. FP8 checks require materialized +E4M3 weight caches. Repeated-offload checks run two complete CPU unload/reload cycles and require identical outputs. + +| Model | SP | TP | FP8 linear + FP8 Sol-Attn relative L2 | Maximum absolute error | Materialized FP8 modules | +| --- | --- | --- | --- | --- | --- | +| H3-Image tiny | 2 | 1 | 0.004660 | 0.007718 | 14 | +| H3-Image tiny | 1 | 2 | 0.004576 | 0.006151 | 14 | +| H3-Image tiny | 2 | 2 | 0.004576 | 0.006151 | 14 | +| PAN2 tiny | 2 | 1 | 0.003891 | 0.007813 | 30 | +| PAN2 tiny | 1 | 2 | 0.003835 | 0.007813 | 30 | +| PAN2 tiny | 2 | 2 | 0.003835 | 0.007813 | 30 | + +All six configurations also passed the repeated-offload identity check. Sol-Attn used `tau=1`, exact threshold estimation, +no dense warmup, and FP8 attention. These tolerances/results describe numerical fixtures; use calibrated dense warmup +and quality evaluation for real generation. + +Reproduce a fixture from the repository root (use an environment with compatible optional GPU kernels): + +```bash +CUDA_VISIBLE_DEVICES=0,1 NCCL_IB_DISABLE=1 python -m torch.distributed.run \ + --standalone --nproc_per_node=2 tools/validation/validate_h3_image_pan2_parallel.py \ + --model pan2 --sp 2 --tp 1 --fp8 --sol --sol-fp8 --offload \ + --output work_dirs/pan2-parallel.json +``` + +The actual worker sampling path has a separate reproduction command: + +```bash +NCCL_IB_DISABLE=1 python tools/validation/validate_h3_image_pan2_sampling.py \ + --model pan2 --scheduler-config /path/to/PAN2/scheduler/scheduler_config.json \ + --device-ids 0 1 --sp 2 --tp 1 --fp8 --sol --sol-fp8 \ + --output work_dirs/pan2-worker-sampling.json +``` + +For H3-Image use `--model h3-image` and omit `--scheduler-config`. Both checks use random tiny models. + +Replace `pan2` with `h3-image`. For SP2×TP2 use four visible devices, four processes and `--sp 2 --tp 2`. +`NCCL_IB_DISABLE=1` was needed by the validation host; it is an existing NCCL option, not a TeleFuser setting. + +## Full components + +| Component | Check | Result | +| --- | --- | --- | +| PAN2 released Qwen3.5 text encoder | Load all four shards, chat-template prompt and negative prompt, model CPU offload | Passed; output `[1,14,4096]`, negative `[1,9,4096]`; peak allocated 15,938,346,496 bytes | +| H3-Image released Qwen3-VL conditioner | All fourteen shard hashes match H3-Image metadata; encode verbatim prompt through layer 50 and offload | Passed; output `[8,5120]`; peak allocated 51,557,238,784 bytes | +| H3-Image released visual text conditioning | Encode a 1024×1024 reference with image-token presentation and offload | Passed; output `[1043,5120]`; peak allocated 51,845,369,856 bytes | +| H3-Image released image VAE | Mixed precision, encode/decode 256×256 RGB fixture, offload between phases | Passed; output `[1,3,256,256]`; peak allocated 2,305,942,528 bytes | +| PAN2 released Wan2.2 VAE | Full file SHA256 matches the shared Wan2.2 VAE; encode/reconstruct RGB image and decode 9 video frames, with model offload | Passed; latent `[1,48,1,16,16]`, video `[1,3,9,256,256]`; peak allocated 2,211,739,136 bytes | +| Released H3-Image / PAN2 complete generation | Transformer downloads and end-to-end verification | Pending | + +The component peaks are synchronized-process allocator observations for those specific inputs, not advertised minimum +VRAM or full-model throughput. Full-generation quality/performance conclusions require completed transformer weights. + +## Integration scope + +The implementation adds model-family classes, pipeline/stage classes and example options. It adds no shared framework +configuration fields, environment variables, loaders, registries, worker protocols or service schemas. +The existing H3 attention call site makes tensors contiguous before existing FP8 prefix/correction ops; no operator +signature changed. This fixes the SP + FP8 Sol-Attn combination inherited by H3-Image. + +H3-Image uses the native transformer plus fused PDD heads, and the officially supplied `vae_diffusers` image VAE. +PAN2 uses the provided text-only Qwen3.5 checkpoint and the maintained Diffusers Wan VAE inside a TeleFuser BaseModel. +PAN2 follows the pinned official Diffusers inference contract: FP32 initial noise/latent accumulation and separate +conditional/unconditional forwards. The native training repository initializes BF16 noise and batches CFG; therefore +a fixed seed need not reproduce its command line output byte for byte. The dense and optimized cases in this validation +use the same declared noise, conditioning and schedule. + +Hosted prompt rewriting and guardrail services are outside this local generation integration. The command line +examples do not offer LoRA, feature caching, FSDP, Ring SP or serving routes. + +See the [H3-Image example](../../examples/minimax_h3_image/README.md) and +[PAN2 example](../../examples/pan2/README.md) for checkpoint layout and generation commands. diff --git a/examples/minimax_h3_image/README.md b/examples/minimax_h3_image/README.md new file mode 100644 index 00000000..183a2a12 --- /dev/null +++ b/examples/minimax_h3_image/README.md @@ -0,0 +1,129 @@ +# MiniMax-H3-Image Examples + +Generate images from text or edit up to sixteen reference images with the native six-step PDD checkpoint. +The pipeline uses TeleFuser model loading, stages, workers, attention dispatch and quantization. + +## Model Source + +| Model | HuggingFace | ModelScope | Purpose | +| --- | --- | --- | --- | +| MiniMax-H3-Image | [MiniMaxAI/MiniMax-H3-Image](https://huggingface.co/MiniMaxAI/MiniMax-H3-Image) | N/A | Native transformer, fused PDD heads, image VAE and Qwen3-VL conditioner | + +## Feature Support + +| Feature | Support | Notes | +| --- | --- | --- | +| Text to image / reference editing | Supported | One prompt, at most 16 references, fixed six-step schedule | +| Sequence parallel | Supported | Existing Ulysses SP; divides attention heads after TP | +| Tensor parallel | Supported | Existing H3 TP; can combine with Ulysses | +| Sol-Attn | Supported | Existing attention configuration, dense text prefix, optional FP8 Q/K/V | +| FP8 linear quantization | Supported | Existing tf-kernel FP8 backend; FP32 boundary projections retained | +| CPU offload | Supported | Model CPU offload, including sharded DiT; sequential conditioning phases | +| LoRA / feature cache / compilation / FSDP | Unsupported | Not exposed by this initial image integration | +| Server API | Unsupported | Local pipeline and command line example only | + +Numerical feature checks use small models, including SP2, TP2 and SP2×TP2 with FP8 and Sol-Attn together. +See [validation notes](../../docs/en/h3_image_pan2_validation.md) for the distinction between these checks and full weights. + +## Requirements + +- An 80GB CUDA GPU for the BF16 transformer and sequential CPU offload; more devices can reduce TP memory. +- Sufficient host memory for the transformer, Qwen3-VL encoder, VAE and spawned worker storage. +- TeleFuser, a Transformers version supporting Qwen3-VL, Diffusers primitives, and Pillow. +- Sol-Attn and FP8 require a compatible, locally provisioned tf-kernel wheel; use the existing installation instructions. +- Reference inputs are upright RGB images after EXIF orientation is applied. + +Install TeleFuser through the [development setup](../../CONTRIBUTING.md#development-setup). +Validation used PyTorch 2.11.0+cu130, Transformers 5.14.1 and the Diffusers revision recorded in the validation notes. +No new TeleFuser package extra is required. + +## Model Directory + +Download the released native transformer and PDD heads, plus the official **image** VAE in Diffusers format. +The native `vae/` video-style weights are not consumed by this example. + +```text +MiniMax-H3-Image/ +|-- transformer/config.json +|-- transformer/model-*-of-*.safetensors +|-- pdd_config.json +|-- pdd_fused_heads.safetensors +|-- scheduler/scheduler_config.json +|-- processor/ # tokenizer and image processor assets +|-- text_encoder/config.json +|-- text_encoder/model-*-of-*.safetensors +|-- vae_diffusers/config.json +\-- vae_diffusers/diffusion_pytorch_model.safetensors +``` + +`--text_encoder_path` can point to an existing identical H3 Qwen3-VL text encoder to avoid duplicating weights. +It must contain the released encoder configuration and all fourteen shards. + +## Quick Start + +```bash +python examples/minimax_h3_image/generate.py \ + --model_root /path/to/MiniMax-H3-Image \ + --text_encoder_path /path/to/MiniMax-H3-Image/text_encoder \ + --prompt "A red fox in a snowy forest, detailed natural photograph." \ + --output_path work_dirs/h3-image.png +``` + +The command writes a 2048×2048 PNG using all six published heads and the published sigma grid. + +## Examples + +### Image generation and editing + +#### `generate.py` + +Use the same command for text-only generation or image editing; the prompt can refer to ``, etc. + +```bash +python examples/minimax_h3_image/generate.py \ + --model_root /path/to/MiniMax-H3-Image \ + --text_encoder_path /path/to/MiniMax-H3-Image/text_encoder \ + --prompt "Change the sky in to sunset." \ + --image inputs/landscape.png --output_path work_dirs/edit.png + +python examples/minimax_h3_image/generate.py \ + --model_root /path/to/MiniMax-H3-Image \ + --text_encoder_path /path/to/MiniMax-H3-Image/text_encoder \ + --prompt "A red fox in a snowy forest." \ + --device_ids 0 1 2 3 --sp 2 --tp 2 --fp8 --sol --sol_fp8 +``` + +| Option | Default | Description | +| --- | --- | --- | +| `--device_ids` | `0` | Visible CUDA device IDs; count must equal SP×TP | +| `--sp`, `--tp` | `1`, `1` | Ulysses and tensor parallel degrees | +| `--fp8` | off | Online linear weight/activation FP8 through tf-kernel | +| `--sol`, `--sol_fp8` | off | Sol-Attn; optional post-RoPE FP8 attention | +| `--height`, `--width` | `2048`, `2048` | Returned image dimensions | +| `--min_generation_area` | `4194304` | Generation canvas floor; smaller requested images resize after decoding | +| `--seed` | `0` | CPU request generator seed | +| `--image` | empty | Ordered reference paths, at most sixteen | + +## Configuration + +Text is encoded verbatim without a chat template. Image references retain their own aspect ratio and are resized +separately for the VAE (short edge 1024–2048) and vision conditioner (short edge 1024). The image VAE uses ImageNet +pixel normalization, posterior sampling rounded through FP16, per-channel latent statistics and tiled decoding. + +Sampling is guidance-distilled: there is no CFG or arbitrary step-count option. The scheduler must specify +unshifted `velocity` updates. Changing the head count or silently substituting another flow schedule is rejected. +The example enables Sol-Attn after one dense step and one dense layer; adjust the ordinary `AttentionConfig` in +`common.py` for different quality/performance tradeoffs. + +## Troubleshooting + +- Missing PDD heads: load `pdd_fused_heads.safetensors` together with every native transformer shard. +- Out of memory: use TP and model CPU offload; SP alone does not shard parameter storage. +- Worker startup timeout: check the existing `ParallelConfig.timeout`, CUDA visibility and NCCL connectivity. +- Incorrect VAE keys: use `vae_diffusers/`, not the unrelated native video-style `vae/` serialization. + +## Notes + +The source model is [MiniMax-AI/MiniMax-H3-Image](https://github.com/MiniMax-AI/MiniMax-H3-Image). +The model-specific image VAE and packing helpers retain Apache-2.0 notices from their pinned Diffusers source. +This local example does not perform hosted prompt rewriting or expose a service schema. diff --git a/examples/minimax_h3_image/common.py b/examples/minimax_h3_image/common.py new file mode 100644 index 00000000..f1772b2a --- /dev/null +++ b/examples/minimax_h3_image/common.py @@ -0,0 +1,91 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Explicit checkpoint loading through TeleFuser's existing ModuleManager API.""" + +from pathlib import Path + +import torch + +from telefuser.core.config import ( + AttentionConfig, + ModelRuntimeConfig, + OffloadConfig, + ParallelConfig, + QuantConfig, + QuantKernelBackend, + QuantType, + WeightOffloadType, +) +from telefuser.core.module_manager import ModuleManager +from telefuser.models.minimax_h3_image_dit import MiniMaxH3ImageDiT +from telefuser.models.minimax_h3_image_encoder import MiniMaxH3ImageEncoder +from telefuser.models.minimax_h3_image_vae import MiniMaxH3ImageVAE +from telefuser.pipelines.minimax_h3_image import MiniMaxH3ImagePipeline, MiniMaxH3ImagePipelineConfig + + +def build_pipeline( + model_root: str, + text_encoder_path: str, + *, + device_ids: list[int], + sp: int = 1, + tp: int = 1, + fp8: bool = False, + sol: bool = False, + sol_fp8: bool = False, +) -> MiniMaxH3ImagePipeline: + root, encoder = Path(model_root), Path(text_encoder_path) + torch.cuda.set_device(device_ids[0]) + parallel = ParallelConfig(device_ids=device_ids, sp_ulysses_degree=sp, tp_degree=tp) + parallel.validate() + if sol_fp8 and not sol: + raise ValueError("--sol_fp8 requires --sol") + offload = OffloadConfig(offload_type=WeightOffloadType.MODEL_CPU_OFFLOAD, pin_cpu_memory=False) + attention = ( + AttentionConfig.sol_attention(dense_timesteps=1, dense_layers=1, sol_fp8=sol_fp8) if sol else AttentionConfig() + ) + quant = QuantConfig(enabled=fp8, quant_type=QuantType.FP8, kernel_backend=QuantKernelBackend.TF_KERNEL) + runtime = ModelRuntimeConfig( + device_id=device_ids[0], + offload_config=offload, + parallel_config=parallel, + attention_config=attention, + quant_config=quant, + ) + manager = ModuleManager(device="cpu", torch_dtype=torch.bfloat16) + manager.load_model( + sorted(str(p) for p in (root / "transformer").glob("model-*.safetensors")) + + [str(root / "pdd_fused_heads.safetensors")], + name="minimax_h3_image_transformer", + model_class=MiniMaxH3ImageDiT, + converter_kwargs={"config_path": root / "transformer/config.json"}, + low_cpu_mem_usage=True, + ) + manager.load_model( + sorted(str(p) for p in encoder.glob("model-*.safetensors")), + name="minimax_h3_text_encoder", + model_class=MiniMaxH3ImageEncoder, + converter_kwargs={"config_path": encoder}, + low_cpu_mem_usage=True, + ) + manager.load_model( + str(root / "vae_diffusers/diffusion_pytorch_model.safetensors"), + name="minimax_h3_image_vae", + model_class=MiniMaxH3ImageVAE, + model_resource="diffusers", + converter_kwargs={"config_path": root / "vae_diffusers/config.json"}, + torch_dtype=torch.bfloat16, + low_cpu_mem_usage=True, + ) + pipeline = MiniMaxH3ImagePipeline(device=f"cuda:{device_ids[0]}", torch_dtype=torch.bfloat16) + pipeline.init( + manager, + MiniMaxH3ImagePipelineConfig( + processor_path=str(root / "processor"), + pdd_config_path=str(root / "pdd_config.json"), + scheduler_config_path=str(root / "scheduler/scheduler_config.json"), + text_encoder_config=ModelRuntimeConfig(device_id=device_ids[0], offload_config=offload), + dit_config=runtime, + vae_config=ModelRuntimeConfig(device_id=device_ids[0], offload_config=offload, torch_dtype=torch.bfloat16), + ), + ) + return pipeline diff --git a/examples/minimax_h3_image/generate.py b/examples/minimax_h3_image/generate.py new file mode 100644 index 00000000..4f667603 --- /dev/null +++ b/examples/minimax_h3_image/generate.py @@ -0,0 +1,57 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Generate an image or edit up to sixteen references with MiniMax-H3-Image.""" + +import argparse +from pathlib import Path + +from PIL import Image +from common import build_pipeline + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model_root", required=True) + parser.add_argument("--text_encoder_path", required=True) + parser.add_argument("--prompt", required=True) + parser.add_argument("--image", nargs="*", default=[]) + parser.add_argument("--height", type=int, default=2048) + parser.add_argument("--width", type=int, default=2048) + parser.add_argument("--min_generation_area", type=int, default=2048 * 2048) + parser.add_argument("--seed", type=int, default=0) + parser.add_argument("--device_ids", type=int, nargs="+", default=[0]) + parser.add_argument("--sp", type=int, default=1) + parser.add_argument("--tp", type=int, default=1) + parser.add_argument("--fp8", action="store_true") + parser.add_argument("--sol", action="store_true") + parser.add_argument("--sol_fp8", action="store_true") + parser.add_argument("--output_path", default="work_dirs/h3-image.png") + args = parser.parse_args() + pipeline = build_pipeline( + args.model_root, + args.text_encoder_path, + device_ids=args.device_ids, + sp=args.sp, + tp=args.tp, + fp8=args.fp8, + sol=args.sol, + sol_fp8=args.sol_fp8, + ) + try: + images = [Image.open(path) for path in args.image] + output = pipeline( + args.prompt, + image=images, + height=args.height, + width=args.width, + min_generation_area=args.min_generation_area, + seed=args.seed, + ) + path = Path(args.output_path) + path.parent.mkdir(parents=True, exist_ok=True) + output.save(path) + finally: + pipeline.stop() + + +if __name__ == "__main__": + main() diff --git a/examples/pan2/README.md b/examples/pan2/README.md new file mode 100644 index 00000000..a98a6458 --- /dev/null +++ b/examples/pan2/README.md @@ -0,0 +1,123 @@ +# PAN2 Examples + +Generate a video from text or a first frame with PAN2, using TeleFuser model loading, stages and distributed workers. +The transformer supports the existing Ulysses SP, TP, Sol-Attn and tf-kernel FP8 paths. + +## Model Source + +| Model | HuggingFace | ModelScope | Purpose | +| --- | --- | --- | --- | +| PAN2 | [IFM/PAN2](https://huggingface.co/IFM/PAN2) | N/A | Transformer, provided Qwen3.5 text encoder, Wan2.2 VAE and tokenizer | + +## Feature Support + +| Feature | Support | Notes | +| --- | --- | --- | +| Text to video / image to video | Supported | One prompt and optional first RGB frame | +| Sequence parallel | Supported | Ulysses; odd video/text lengths are padded and compacted before attention | +| Tensor parallel | Supported | Joint attention, refiner, FFNs and modulation projections; combines with SP | +| Sol-Attn | Supported | Joint layers; text-only refiner stays dense; optional FP8 attention | +| FP8 linear quantization | Supported | Existing tf-kernel FP8 backend, FP32 modulation retained | +| CPU offload | Supported | Model CPU offload including SP/TP shards and repeated requests | +| LoRA / feature cache / compilation / FSDP / Ring SP | Unsupported | Not exposed by this integration | +| Server API | Unsupported | Local pipeline and command line example only | + +Numerical checks cover SP2, TP2 and SP2×TP2, including FP8+Sol-Attn and repeated CPU offload. +See [validation notes](../../docs/en/h3_image_pan2_validation.md) for full checkpoint verification status. + +## Requirements + +- An 80GB CUDA GPU for BF16 transformer inference with sequential model CPU offload; TP can reduce parameter memory. +- Sufficient host memory for all components and spawned worker storage. +- Transformers with `Qwen3_5TextModel` (tested 5.14.1), Diffusers with `AutoencoderKLWan`, Pillow and an MP4 writer. +- A compatible, locally provisioned tf-kernel wheel for Sol-Attn and FP8. + +Install TeleFuser through the [development setup](../../CONTRIBUTING.md#development-setup). +Validation used PyTorch 2.11.0+cu130 and the Diffusers revision recorded in the validation notes. +The Qwen3.5 text encoder works with Transformers' PyTorch fallback when optional FLA kernels are absent. + +## Model Directory + +```text +PAN2/ +|-- transformer/config.json +|-- transformer/diffusion_pytorch_model-*-of-*.safetensors +|-- text_encoder/config.json +|-- text_encoder/model-*-of-*.safetensors +|-- tokenizer/ # complete released tokenizer assets +|-- scheduler/scheduler_config.json +|-- vae/config.json +\-- vae/diffusion_pytorch_model.safetensors +``` + +Use the **provided text-only encoder** in `IFM/PAN2`; a separate complete Qwen3.5 multimodal checkpoint is unnecessary. + +## Quick Start + +```bash +python examples/pan2/generate.py \ + --model_root /path/to/PAN2 \ + --prompt "A red fox walks through a snowy forest, tracking camera, natural light." \ + --output_path work_dirs/pan2.mp4 +``` + +The command writes a 121-frame RGB MP4 at 24 fps, 704×1248, using 50 steps, CFG 2.5 and the released flow shift 9. +PAN2 was trained with structured captions; the official repository provides richer structured prompt examples. +The example encodes the supplied string directly and does not call an external captioning service. + +## Examples + +### Video generation + +#### `generate.py` + +Pass `--image` for first-frame conditioning. Frame counts must be `4*k+1`; both spatial dimensions must divide 32. + +```bash +python examples/pan2/generate.py \ + --model_root /path/to/PAN2 --image inputs/fox.png \ + --prompt "The fox walks forward through the snow." --output_path work_dirs/pan2-i2v.mp4 + +python examples/pan2/generate.py \ + --model_root /path/to/PAN2 --prompt "A red fox walks through snow." \ + --device_ids 0 1 2 3 --sp 2 --tp 2 --fp8 --sol --sol_fp8 \ + --num_frames 241 --output_path work_dirs/pan2-10s.mp4 +``` + +| Option | Default | Description | +| --- | --- | --- | +| `--device_ids` | `0` | Visible device IDs; count must equal SP×TP | +| `--sp`, `--tp` | `1`, `1` | Ulysses and tensor parallel degrees | +| `--fp8` | off | tf-kernel FP8 linear quantization | +| `--sol`, `--sol_fp8` | off | Sparse joint attention and optional FP8 Q/K/V | +| `--height`, `--width` | `704`, `1248` | Positive multiples of 32 | +| `--num_frames` | `121` | `4*k+1`; 241 frames span 10 seconds at 24 fps | +| `--num_inference_steps` | `50` | Number of flow-matching Euler steps | +| `--guidance_scale` | `2.5` | CFG; values at most one use conditional prediction only | +| `--fps` | `24` | MP4 frame rate and default negative-caption time span | +| `--negative_prompt` | released default | Optional replacement negative caption | +| `--seed` | `0` | CPU noise generator seed | + +## Configuration + +The text conditioner applies the released chat template, reads `hidden_states[-3]` and removes the system/user +header tokens, with the released 1536-token budget. The default negative caption has time ranges filled from the requested clip length. +Sampling retains FP32 latents across Euler updates. The optional first-frame condition uses VAE posterior mode, +per-channel normalization and a channel mask; its predicted first latent frame is not CFG amplified. + +SP compacts padding before attention so padded keys cannot change the softmax. TP row reductions remain outside +linear modules so replacing those modules with FP8 preserves collective semantics. Modulation projections remain +FP32. The example uses five initial dense steps and one dense layer before Sol-Attn. + +## Troubleshooting + +- Missing `Qwen3_5TextModel`: use a compatible Transformers release. +- Out of memory: use TP, reduce video length or resolution, and retain model CPU offload. SP does not shard weights. +- Invalid geometry: use dimensions divisible by 32 and frame counts `4*k+1`. +- Missing optional FLA kernels: Transformers uses its PyTorch fallback; this affects conditioning speed. + +## Notes + +The source model is [ifm-ai/PAN2](https://github.com/ifm-ai/PAN2). +The transformer and preprocessing preserve Apache-2.0 notices from the pinned Diffusers implementation. +This example integrates local generation components; it does not port the hosted prompt-rewriting or guardrail service. diff --git a/examples/pan2/common.py b/examples/pan2/common.py new file mode 100644 index 00000000..4c6b75d8 --- /dev/null +++ b/examples/pan2/common.py @@ -0,0 +1,77 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Load released PAN2 components without adding a framework registry or loader.""" + +from pathlib import Path + +import torch + +from telefuser.core.config import ( + AttentionConfig, + ModelRuntimeConfig, + OffloadConfig, + ParallelConfig, + QuantConfig, + QuantKernelBackend, + QuantType, + WeightOffloadType, +) +from telefuser.core.module_manager import ModuleManager +from telefuser.models.pan2_conditioning import PAN2VAE, PAN2TextEncoder +from telefuser.models.pan2_dit import PAN2DiT +from telefuser.pipelines.pan2 import PAN2Pipeline, PAN2PipelineConfig + + +def build_pipeline( + model_root: str, + *, + device_ids: list[int], + sp: int = 1, + tp: int = 1, + fp8: bool = False, + sol: bool = False, + sol_fp8: bool = False, +) -> PAN2Pipeline: + root = Path(model_root) + torch.cuda.set_device(device_ids[0]) + parallel = ParallelConfig(device_ids=device_ids, sp_ulysses_degree=sp, tp_degree=tp) + parallel.validate() + if sol_fp8 and not sol: + raise ValueError("--sol_fp8 requires --sol") + offload = OffloadConfig(offload_type=WeightOffloadType.MODEL_CPU_OFFLOAD, pin_cpu_memory=False) + attention = ( + AttentionConfig.sol_attention(dense_timesteps=5, dense_layers=1, sol_fp8=sol_fp8) if sol else AttentionConfig() + ) + quant = QuantConfig(enabled=fp8, quant_type=QuantType.FP8, kernel_backend=QuantKernelBackend.TF_KERNEL) + runtime = ModelRuntimeConfig( + device_id=device_ids[0], + offload_config=offload, + parallel_config=parallel, + attention_config=attention, + quant_config=quant, + ) + manager = ModuleManager(device="cpu", torch_dtype=torch.bfloat16) + for component, name, model_class in ( + ("transformer", "pan2_transformer", PAN2DiT), + ("text_encoder", "pan2_text_encoder", PAN2TextEncoder), + ("vae", "pan2_vae", PAN2VAE), + ): + manager.load_model( + sorted(str(p) for p in (root / component).glob("*.safetensors")), + name=name, + model_class=model_class, + model_resource="diffusers", + converter_kwargs={"config_path": root / component / "config.json"}, + low_cpu_mem_usage=True, + ) + pipeline = PAN2Pipeline(device=f"cuda:{device_ids[0]}", torch_dtype=torch.bfloat16) + pipeline.init( + manager, + PAN2PipelineConfig( + tokenizer_path=str(root / "tokenizer"), + scheduler_config_path=str(root / "scheduler/scheduler_config.json"), + dit_config=runtime, + text_encoder_config=ModelRuntimeConfig(device_id=device_ids[0], offload_config=offload), + vae_config=ModelRuntimeConfig(device_id=device_ids[0], offload_config=offload), + ), + ) + return pipeline diff --git a/examples/pan2/generate.py b/examples/pan2/generate.py new file mode 100644 index 00000000..8bd50fd5 --- /dev/null +++ b/examples/pan2/generate.py @@ -0,0 +1,64 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Generate a PAN2 video from a prompt and optional first frame.""" + +import argparse +from pathlib import Path + +from PIL import Image +from common import build_pipeline +from diffusers.utils import export_to_video + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model_root", required=True) + parser.add_argument("--prompt", required=True) + parser.add_argument("--image") + parser.add_argument("--negative_prompt") + parser.add_argument("--height", type=int, default=704) + parser.add_argument("--width", type=int, default=1248) + parser.add_argument("--num_frames", type=int, default=121) + parser.add_argument("--num_inference_steps", type=int, default=50) + parser.add_argument("--guidance_scale", type=float, default=2.5) + parser.add_argument("--fps", type=float, default=24.0) + parser.add_argument("--seed", type=int, default=0) + parser.add_argument("--device_ids", type=int, nargs="+", default=[0]) + parser.add_argument("--sp", type=int, default=1) + parser.add_argument("--tp", type=int, default=1) + parser.add_argument("--fp8", action="store_true") + parser.add_argument("--sol", action="store_true") + parser.add_argument("--sol_fp8", action="store_true") + parser.add_argument("--output_path", default="work_dirs/pan2.mp4") + args = parser.parse_args() + pipeline = build_pipeline( + args.model_root, + device_ids=args.device_ids, + sp=args.sp, + tp=args.tp, + fp8=args.fp8, + sol=args.sol, + sol_fp8=args.sol_fp8, + ) + try: + image = Image.open(args.image).convert("RGB") if args.image else None + frames = pipeline( + args.prompt, + image=image, + negative_prompt=args.negative_prompt, + height=args.height, + width=args.width, + num_frames=args.num_frames, + num_inference_steps=args.num_inference_steps, + guidance_scale=args.guidance_scale, + fps=args.fps, + seed=args.seed, + ) + path = Path(args.output_path) + path.parent.mkdir(parents=True, exist_ok=True) + export_to_video(list(frames.numpy()), str(path), fps=args.fps) + finally: + pipeline.stop() + + +if __name__ == "__main__": + main() diff --git a/telefuser/models/minimax_h3_dit.py b/telefuser/models/minimax_h3_dit.py index d8d57b84..5f225d99 100644 --- a/telefuser/models/minimax_h3_dit.py +++ b/telefuser/models/minimax_h3_dit.py @@ -719,13 +719,17 @@ def forward( live_key.transpose(1, 2), live_value.transpose(1, 2), scale=self.head_dim**-0.5, - ).transpose(1, 2) + ).transpose(1, 2).contiguous() if output_correction is None: live_output = torch.cat((dense_prefix, live_output[:, prefix_tokens:]), dim=1) else: - live_output = merge_fp8_attention_prefix(dense_prefix, live_output, output_correction) + live_output = merge_fp8_attention_prefix( + dense_prefix.contiguous(), live_output.contiguous(), output_correction.contiguous() + ) elif output_correction is not None: - live_output = apply_fp8_attention_output_correction(live_output, output_correction) + live_output = apply_fp8_attention_output_correction( + live_output.contiguous(), output_correction.contiguous() + ) if live_tokens == total_tokens: output = live_output else: diff --git a/telefuser/models/minimax_h3_image_dit.py b/telefuser/models/minimax_h3_image_dit.py new file mode 100644 index 00000000..e38405a0 --- /dev/null +++ b/telefuser/models/minimax_h3_image_dit.py @@ -0,0 +1,143 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Image-only H3 DiT using TeleFuser's maintained H3 SP/TP and attention paths.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import torch +from torch import nn +from torch.nn import functional as F + +from telefuser.core.config import QuantConfig, QuantKernelBackend, QuantType +from telefuser.models.minimax_h3_dit import ( + MiniMaxH3DiT, + MiniMaxH3DiTConfig, + MiniMaxH3DiTStateDictConverter, +) + + +class _AbsentAudioInput(nn.Module): + def __init__(self, hidden_size: int) -> None: + super().__init__() + self.hidden_size = hidden_size + + def forward(self, value: torch.Tensor) -> torch.Tensor: + if value.shape[0]: + raise ValueError("MiniMax-H3-Image has no audio input branch") + return value.new_empty((0, self.hidden_size)) + + +class _AbsentAudioOutput(nn.Module): + def forward(self, value: torch.Tensor) -> torch.Tensor: + # The inherited packed forward selects zero audio rows from this tensor. + return value.new_empty((value.shape[0], 0)) + + +class MiniMaxH3ImagePDDProjection(nn.Module): + """Released fused FP32 output heads; head i belongs to sampling step i.""" + + def __init__(self, hidden_size: int, patch_dim: int, num_heads: int) -> None: + super().__init__() + if num_heads < 1: + raise ValueError("num_heads must be positive") + self.weight = nn.Parameter(torch.empty(num_heads, patch_dim, hidden_size, dtype=torch.float32)) + self.bias = nn.Parameter(torch.zeros(num_heads, patch_dim, dtype=torch.float32)) + nn.init.normal_(self.weight, std=0.02) + self.step_index = 0 + + def forward(self, value: torch.Tensor) -> torch.Tensor: + return F.linear(value.float(), self.weight[self.step_index], self.bias[self.step_index]) + + +class MiniMaxH3ImageDiT(MiniMaxH3DiT): + """H3-Image preserves the three-modality AdaLN table, but has no audio rows. + + Only the input/output branches and the step-dependent PDD projection differ + from the maintained H3 DiT. SP, TP, Sol-Attn and FP8 use the inherited paths. + """ + + def __init__(self, config: MiniMaxH3DiTConfig | None = None, num_pdd_heads: int = 6) -> None: + super().__init__(config) + self.audio_patch_proj = _AbsentAudioInput(self.config.hidden_size) + self.final_layer.audio_out = _AbsentAudioOutput() + self.final_layer.video_out = MiniMaxH3ImagePDDProjection( + self.config.hidden_size, self.config.video_patch_dim, num_pdd_heads + ) + + def forward(self, *, pdd_step_index: int, **kwargs: Any) -> torch.Tensor: + projection = self.final_layer.video_out + if not 0 <= pdd_step_index < projection.weight.shape[0]: + raise ValueError(f"PDD step index {pdd_step_index} is outside the released output-head table") + audio = kwargs.get("audio_x") + if audio is not None and audio.numel(): + raise ValueError("MiniMax-H3-Image does not accept audio") + device = kwargs["x"].device + kwargs["audio_x"] = torch.empty((1, 0, self.config.audio_latents_dim), device=device) + kwargs["audio_pos_info"] = {"position_ids": torch.empty((0,), device=device, dtype=torch.long)} + projection.step_index = int(pdd_step_index) + video, _ = super().forward(**kwargs) + return video + + def enable_quant(self, quant_type: QuantConfig | QuantType | str | torch.dtype) -> None: + if not isinstance(quant_type, QuantConfig): + if quant_type not in (QuantType.FP8, "fp8", torch.float8_e4m3fn): + raise ValueError("Use QuantConfig to select a supported H3-Image quantization backend") + quant_type = QuantConfig( + enabled=True, quant_type=QuantType.FP8, kernel_backend=QuantKernelBackend.TF_KERNEL + ) + super().enable_quant(quant_type) + + @staticmethod + def state_dict_converter(config_path: str | Path | None = None) -> MiniMaxH3ImageDiTStateDictConverter: + return MiniMaxH3ImageDiTStateDictConverter(config_path) + + +class MiniMaxH3ImageDiTStateDictConverter: + def __init__(self, config_path: str | Path | None = None) -> None: + self.config_path = config_path + + def from_official(self, state_dict: dict[str, torch.Tensor]) -> tuple[dict[str, torch.Tensor], dict[str, Any]]: + if self.config_path is None: + raise ValueError("H3-Image loading requires the released transformer/config.json") + if "audio_patch_proj.weight" in state_dict: + raise ValueError("An audio-enabled H3 checkpoint is not an H3-Image checkpoint") + heads = state_dict.get("video_out.weight") + bias = state_dict.get("video_out.bias") + if heads is None or bias is None or heads.ndim != 3 or bias.shape != heads.shape[:2]: + raise ValueError("Load transformer shards together with pdd_fused_heads.safetensors") + converted, kwargs = MiniMaxH3DiTStateDictConverter(self.config_path).from_official(state_dict) + config = kwargs["config"] + if tuple(heads.shape[1:]) != (config.video_patch_dim, config.hidden_size): + raise ValueError("PDD output heads do not match the transformer configuration") + converted.pop("video_out.weight") + converted.pop("video_out.bias") + converted["final_layer.video_out.weight"] = heads + converted["final_layer.video_out.bias"] = bias + kwargs["num_pdd_heads"] = int(heads.shape[0]) + return converted, kwargs + + def from_diffusers(self, state_dict: dict[str, torch.Tensor]) -> tuple[dict[str, torch.Tensor], dict[str, Any]]: + if self.config_path is None: + raise ValueError("H3-Image loading requires the released native transformer/config.json") + regular = {name: value for name, value in state_dict.items() if not name.startswith("pdd_proj_out.")} + converted, kwargs = MiniMaxH3DiTStateDictConverter(self.config_path).from_diffusers(regular) + config = kwargs["config"] + indices = sorted({int(name.split(".")[1]) for name in state_dict if name.startswith("pdd_proj_out.")}) + if not indices or indices != list(range(len(indices))): + raise ValueError("Diffusers PDD output heads must be contiguous and start at zero") + for suffix in ("weight", "bias"): + converted[f"final_layer.video_out.{suffix}"] = torch.stack( + [state_dict[f"pdd_proj_out.{index}.{suffix}"] for index in indices] + ) + # Native H3 stores [gate, value], while Diffusers SwiGLU stores [value, gate]. + for name, value in list(converted.items()): + if name.endswith(".mlp.fc1.weight"): + first, second = value.chunk(2, dim=0) + converted[name] = torch.cat((second, first), dim=0) + converted["rope.inv_freq"] = 10000.0 ** ( + -torch.arange(config.rope_inv_freq_len, dtype=torch.float32) / config.rope_inv_freq_len + ) + kwargs["num_pdd_heads"] = len(indices) + return converted, kwargs diff --git a/telefuser/models/minimax_h3_image_encoder.py b/telefuser/models/minimax_h3_image_encoder.py new file mode 100644 index 00000000..c0a68bdd --- /dev/null +++ b/telefuser/models/minimax_h3_image_encoder.py @@ -0,0 +1,26 @@ +# SPDX-License-Identifier: Apache-2.0 +"""H3-Image Qwen3-VL conditioner with checkpoint-faithful rotary precision.""" + +from __future__ import annotations + +from typing import Any + +import torch + +from telefuser.models.minimax_h3_encoder import MiniMaxH3Encoder + + +class MiniMaxH3ImageEncoder(MiniMaxH3Encoder): + def to(self, *args: Any, **kwargs: Any) -> MiniMaxH3ImageEncoder: + # Transformers from_pretrained keeps these non-persistent buffers in FP32. + preserved = { + name: buffer.detach().clone() + for name, buffer in self.named_buffers() + if name.endswith("inv_freq") and not buffer.is_meta + } + result = super().to(*args, **kwargs) + for name, buffer in result.named_buffers(): + if name.endswith("inv_freq"): + value = preserved.get(name, buffer) + buffer.data = value.to(device=buffer.device, dtype=torch.float32) + return result diff --git a/telefuser/models/minimax_h3_image_vae.py b/telefuser/models/minimax_h3_image_vae.py new file mode 100644 index 00000000..519dd152 --- /dev/null +++ b/telefuser/models/minimax_h3_image_vae.py @@ -0,0 +1,889 @@ +# Copyright 2026 The MiniMax and HuggingFace Teams. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Ported from Diffusers commit 843f1743e259e510f22065c0cb917d417b98d9f5. +from __future__ import annotations + +import math +from pathlib import Path +from typing import Any + +import torch +import torch.nn as nn +import torch.nn.functional as F +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.models.attention import FeedForward +from diffusers.models.autoencoders.vae import DecoderOutput, DiagonalGaussianDistribution +from diffusers.models.modeling_outputs import AutoencoderKLOutput +from diffusers.models.modeling_utils import get_parameter_dtype +from diffusers.utils import logging +from diffusers.utils.accelerate_utils import apply_forward_hook + +from telefuser.core.base_model import BaseModel +from telefuser.ops.attention import attention + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +# Adapted from the corresponding Diffusers MiniMax-H3 autoencoder component. +class MiniMaxH3ImageCausalConv3d(nn.Conv3d): + r""" + 3D convolution used throughout the MiniMax-H3 video encoder. + + Spatial padding is symmetric and uses `spatial_padding_mode` (`"reflect"` in the released checkpoint); temporal + padding is causal, i.e. `kernel_size_t - 1` zero frames are prepended and nothing is appended. + """ + + def __init__( + self, + in_channels: int, + out_channels: int, + kernel_size: int | tuple[int, int, int], + stride: int | tuple[int, int, int] = 1, + spatial_padding: int = 0, + temporal_padding: int = 0, + spatial_padding_mode: str = "reflect", + ) -> None: + super().__init__(in_channels, out_channels, kernel_size=kernel_size, stride=stride, padding=0) + self.spatial_padding = spatial_padding + self.temporal_padding = temporal_padding + self.spatial_padding_mode = spatial_padding_mode + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + if self.spatial_padding > 0: + padding = self.spatial_padding + hidden_states = F.pad( + hidden_states, (padding, padding, padding, padding, 0, 0), mode=self.spatial_padding_mode + ) + if self.temporal_padding > 0: + hidden_states = F.pad(hidden_states, (0, 0, 0, 0, self.temporal_padding, 0), mode="constant") + return F.conv3d(hidden_states, self.weight, self.bias, stride=self.stride, padding=0, dilation=self.dilation) + + +# Adapted from the corresponding Diffusers MiniMax-H3 autoencoder component. +class MiniMaxH3ImageGroupNorm(nn.GroupNorm): + r""" + Group normalization applied to each latent frame in isolation (`use_t_isolated_gn` in the original config): the + temporal axis is folded into the batch axis so statistics never mix across frames. + """ + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + batch_size, num_channels, num_frames, height, width = hidden_states.shape + hidden_states = hidden_states.permute(0, 2, 1, 3, 4).contiguous() + hidden_states = hidden_states.view(batch_size * num_frames, num_channels, 1, height, width) + hidden_states = super().forward(hidden_states) + hidden_states = hidden_states.view(batch_size, num_frames, num_channels, height, width) + return hidden_states.permute(0, 2, 1, 3, 4).contiguous() + + +# Adapted from the corresponding Diffusers MiniMax-H3 autoencoder component. +class MiniMaxH3ImageResnetBlock3d(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + norm_num_groups: int = 32, + norm_eps: float = 1e-6, + spatial_padding_mode: str = "reflect", + ) -> None: + super().__init__() + self.in_channels = in_channels + self.out_channels = out_channels + + self.norm1 = MiniMaxH3ImageGroupNorm(norm_num_groups, in_channels, eps=norm_eps, affine=True) + self.conv1 = MiniMaxH3ImageCausalConv3d( + in_channels, + out_channels, + kernel_size=3, + spatial_padding=1, + temporal_padding=2, + spatial_padding_mode=spatial_padding_mode, + ) + self.norm2 = MiniMaxH3ImageGroupNorm(norm_num_groups, out_channels, eps=norm_eps, affine=True) + self.conv2 = MiniMaxH3ImageCausalConv3d( + out_channels, + out_channels, + kernel_size=3, + spatial_padding=1, + temporal_padding=2, + spatial_padding_mode=spatial_padding_mode, + ) + self.conv_shortcut = None + if in_channels != out_channels: + self.conv_shortcut = MiniMaxH3ImageCausalConv3d(in_channels, out_channels, kernel_size=1) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + residual = hidden_states + hidden_states = F.silu(self.norm1(hidden_states)) + hidden_states = self.conv1(hidden_states) + hidden_states = F.silu(self.norm2(hidden_states)) + hidden_states = self.conv2(hidden_states) + if self.conv_shortcut is not None: + residual = self.conv_shortcut(residual) + return residual + hidden_states + + +# Adapted from the corresponding Diffusers MiniMax-H3 autoencoder component. +class MiniMaxH3ImageDownsample3d(nn.Module): + r""" + Strided 3x3x3 downsampling convolution. A spatial stride of 2 is preceded by an asymmetric bottom/right pad of 1 + (the convolution itself carries no spatial padding), so the output is exactly `ceil(size / 2)`. + """ + + def __init__( + self, + in_channels: int, + out_channels: int, + temporal_stride: int = 1, + spatial_stride: int = 2, + spatial_padding_mode: str = "reflect", + ) -> None: + super().__init__() + self.spatial_stride = spatial_stride + self.spatial_padding_mode = spatial_padding_mode + self.conv = MiniMaxH3ImageCausalConv3d( + in_channels, + out_channels, + kernel_size=3, + stride=(temporal_stride, spatial_stride, spatial_stride), + spatial_padding=0, + temporal_padding=2, + spatial_padding_mode=spatial_padding_mode, + ) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + if self.spatial_stride == 2: + hidden_states = F.pad(hidden_states, (0, 1, 0, 1, 0, 0), mode=self.spatial_padding_mode) + return self.conv(hidden_states) + + +# Adapted from the corresponding Diffusers MiniMax-H3 autoencoder component. +class MiniMaxH3ImageDownBlock3d(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + num_layers: int, + temporal_downsample_factor: int, + spatial_downsample_factor: int, + norm_num_groups: int = 32, + norm_eps: float = 1e-6, + spatial_padding_mode: str = "reflect", + ) -> None: + super().__init__() + self.resnets = nn.ModuleList( + [ + MiniMaxH3ImageResnetBlock3d( + in_channels=in_channels if i == 0 else out_channels, + out_channels=out_channels, + norm_num_groups=norm_num_groups, + norm_eps=norm_eps, + spatial_padding_mode=spatial_padding_mode, + ) + for i in range(num_layers) + ] + ) + self.downsamplers = None + if temporal_downsample_factor * spatial_downsample_factor > 1: + self.downsamplers = nn.ModuleList( + [ + MiniMaxH3ImageDownsample3d( + out_channels, + out_channels, + temporal_stride=temporal_downsample_factor, + spatial_stride=spatial_downsample_factor, + spatial_padding_mode=spatial_padding_mode, + ) + ] + ) + + self.gradient_checkpointing = False + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + for resnet in self.resnets: + if torch.is_grad_enabled() and self.gradient_checkpointing: + hidden_states = self._gradient_checkpointing_func(resnet, hidden_states) + else: + hidden_states = resnet(hidden_states) + if self.downsamplers is not None: + for downsampler in self.downsamplers: + hidden_states = downsampler(hidden_states) + return hidden_states + + +# Adapted from the corresponding Diffusers MiniMax-H3 autoencoder component. +class MiniMaxH3ImageEncoder3d(nn.Module): + r""" + Causal 3D CNN encoder. `block_out_channels` gives the channel count of every level; the per-level + `spatial_downsample_factors` / `temporal_downsample_factors` multiply out to the total compression ratios. + """ + + def __init__( + self, + in_channels: int = 3, + out_channels: int = 48, + block_out_channels: tuple[int, ...] = (128, 256, 256, 512, 512, 1024), + layers_per_block: int = 2, + spatial_downsample_factors: tuple[int, ...] = (2, 2, 2, 2, 1, 1), + temporal_downsample_factors: tuple[int, ...] = (1, 2, 2, 1, 1, 1), + norm_num_groups: int = 32, + norm_eps: float = 1e-6, + spatial_padding_mode: str = "reflect", + ) -> None: + super().__init__() + + self.conv_in = MiniMaxH3ImageCausalConv3d( + in_channels, + block_out_channels[0], + kernel_size=3, + spatial_padding=1, + temporal_padding=2, + spatial_padding_mode=spatial_padding_mode, + ) + + block_in_channels = (block_out_channels[0],) + tuple(block_out_channels[:-1]) + self.down_blocks = nn.ModuleList( + [ + MiniMaxH3ImageDownBlock3d( + in_channels=block_in_channels[i], + out_channels=block_out_channels[i], + num_layers=layers_per_block, + temporal_downsample_factor=temporal_downsample_factors[i], + spatial_downsample_factor=spatial_downsample_factors[i], + norm_num_groups=norm_num_groups, + norm_eps=norm_eps, + spatial_padding_mode=spatial_padding_mode, + ) + for i in range(len(block_out_channels)) + ] + ) + + self.norm_out = MiniMaxH3ImageGroupNorm(norm_num_groups, block_out_channels[-1], eps=norm_eps, affine=True) + self.conv_out = MiniMaxH3ImageCausalConv3d( + block_out_channels[-1], + out_channels, + kernel_size=3, + spatial_padding=1, + temporal_padding=2, + spatial_padding_mode=spatial_padding_mode, + ) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + hidden_states = self.conv_in(hidden_states) + for down_block in self.down_blocks: + hidden_states = down_block(hidden_states) + hidden_states = F.silu(self.norm_out(hidden_states)) + return self.conv_out(hidden_states) + + +# Adapted from the corresponding Diffusers MiniMax-H3 autoencoder component. +class MiniMaxH3ImageRotaryPosEmbed(nn.Module): + r""" + 3-axis rotary embedding for the ViT decoder. Coordinates are length-normalized to `[-1, 1)` per axis and scaled by + `2 * pi`, and the resulting `(t, h, w)` angles are concatenated and then duplicated, so the first `rope_dim_ratio * + attention_head_dim` channels of every head are rotated. + """ + + def __init__(self, dim: int, theta: float = 100.0, num_axes: int = 3) -> None: + super().__init__() + if dim % (2 * num_axes) != 0: + raise ValueError(f"`dim` {dim} must be divisible by `2 * num_axes` {2 * num_axes}.") + inv_freq = 1.0 / theta ** torch.arange(0, 1, 2 * num_axes / dim, dtype=torch.float32) + self.register_buffer("inv_freq", inv_freq, persistent=False) + + def forward(self, position_ids: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + angles = 2.0 * math.pi * position_ids[:, :, :, None] * self.inv_freq[None, None, None, :] + angles = angles.flatten(2, 3).tile(2).unsqueeze(2) + return angles.cos(), angles.sin() + + +# Adapted from the corresponding Diffusers MiniMax-H3 autoencoder component. +class MiniMaxH3ImageAttnProcessor: + _attention_backend = None + _parallel_config = None + + def __call__( + self, + attn: "MiniMaxH3ImageAttention", + hidden_states: torch.Tensor, + rotary_emb: tuple[torch.Tensor, torch.Tensor] | None = None, + ) -> torch.Tensor: + query = attn.to_q(hidden_states).unflatten(2, (attn.heads, -1)) + key = attn.to_k(hidden_states).unflatten(2, (attn.heads, -1)) + value = attn.to_v(hidden_states).unflatten(2, (attn.heads, -1)) + + # The reference normalizes Q/K in float32 regardless of the compute dtype. + query = attn.norm_q(query.float()).to(query.dtype) + key = attn.norm_k(key.float()).to(key.dtype) + + if rotary_emb is not None: + cos, sin = rotary_emb + cos = cos.to(query.dtype) + sin = sin.to(query.dtype) + rotary_dim = cos.shape[-1] + query_rotary, query_pass = query[..., :rotary_dim], query[..., rotary_dim:] + key_rotary, key_pass = key[..., :rotary_dim], key[..., rotary_dim:] + query_first, query_second = query_rotary.chunk(2, dim=-1) + key_first, key_second = key_rotary.chunk(2, dim=-1) + query_rotated = torch.cat([-query_second, query_first], dim=-1) + key_rotated = torch.cat([-key_second, key_first], dim=-1) + query = torch.cat([query_rotary * cos + query_rotated * sin, query_pass], dim=-1) + key = torch.cat([key_rotary * cos + key_rotated * sin, key_pass], dim=-1) + + hidden_states = attention( + query, + key, + value, + attn_mask=None, + ) + hidden_states = hidden_states.flatten(2, 3) + return attn.to_out[0](hidden_states) + + +# Adapted from the corresponding Diffusers MiniMax-H3 autoencoder component. +class MiniMaxH3ImageAttention(nn.Module): + _default_processor_cls = MiniMaxH3ImageAttnProcessor + _available_processors = [MiniMaxH3ImageAttnProcessor] + + def __init__(self, dim: int, heads: int, dim_head: int, eps: float = 1e-5, bias: bool = True) -> None: + super().__init__() + self.heads = heads + self.dim_head = dim_head + self.use_bias = bias + inner_dim = heads * dim_head + + self.norm_q = nn.RMSNorm(dim_head, eps=eps, elementwise_affine=False) + self.norm_k = nn.RMSNorm(dim_head, eps=eps, elementwise_affine=False) + self.to_q = nn.Linear(dim, inner_dim, bias=bias) + self.to_k = nn.Linear(dim, inner_dim, bias=bias) + self.to_v = nn.Linear(dim, inner_dim, bias=bias) + self.to_out = nn.ModuleList([nn.Linear(inner_dim, dim, bias=bias), nn.Dropout(0.0)]) + + self.set_processor(MiniMaxH3ImageAttnProcessor()) + + def set_processor(self, processor: MiniMaxH3ImageAttnProcessor) -> None: + self.processor = processor + + def forward( + self, hidden_states: torch.Tensor, rotary_emb: tuple[torch.Tensor, torch.Tensor] | None = None + ) -> torch.Tensor: + return self.processor(self, hidden_states, rotary_emb) + + +# Adapted from the corresponding Diffusers MiniMax-H3 autoencoder component. +class MiniMaxH3ImageTransformerBlock(nn.Module): + def __init__( + self, + dim: int, + heads: int, + dim_head: int, + ffn_mult: int = 4, + eps: float = 1e-5, + bias: bool = True, + ) -> None: + super().__init__() + self.norm1 = nn.RMSNorm(dim, eps=eps, elementwise_affine=True) + self.attn = MiniMaxH3ImageAttention(dim=dim, heads=heads, dim_head=dim_head, eps=eps, bias=bias) + self.scale1 = nn.Parameter(torch.zeros(dim)) + self.norm2 = nn.RMSNorm(dim, eps=eps, elementwise_affine=True) + self.ff = FeedForward(dim, mult=ffn_mult, activation_fn="swiglu", bias=bias) + self.scale2 = nn.Parameter(torch.zeros(dim)) + + def forward( + self, hidden_states: torch.Tensor, rotary_emb: tuple[torch.Tensor, torch.Tensor] | None = None + ) -> torch.Tensor: + # The reference normalizes in float32 regardless of the compute dtype. The residual stream is float32 too + # because the anchors are, so the normed activation follows the projections instead. + compute_dtype = get_parameter_dtype(self.attn.to_q) + norm_hidden_states = self.norm1(hidden_states.float()).to(compute_dtype) + hidden_states = hidden_states + self.attn(norm_hidden_states, rotary_emb) * self.scale1 + norm_hidden_states = self.norm2(hidden_states.float()).to(compute_dtype) + hidden_states = hidden_states + self.ff(norm_hidden_states) * self.scale2 + return hidden_states + + +class MiniMaxH3ImageViTDecoder2d(nn.Module): + r""" + Non-causal 2D ViT decoder. Every latent pixel becomes one token; `num_register_tokens` learned register tokens plus + a single all-zero token are appended (all at position `0`), attended over with full self-attention, and dropped + again before the patch projection expands each token into a `patch_size x patch_size` pixel block. The rotary + embedding carries `(h, w)` coordinates only. + """ + + def __init__( + self, + in_channels: int = 24, + out_channels: int = 3, + patch_size: int = 16, + num_layers: int = 24, + num_attention_heads: int = 16, + attention_head_dim: int = 64, + num_register_tokens: int = 4, + ffn_mult: int = 4, + rope_theta: float = 100.0, + rope_dim_ratio: float = 0.5, + norm_eps: float = 1e-5, + ) -> None: + super().__init__() + dim = num_attention_heads * attention_head_dim + self.patch_size = patch_size + self.out_channels = out_channels + self.num_register_tokens = num_register_tokens + + self.rope = MiniMaxH3ImageRotaryPosEmbed(int(attention_head_dim * rope_dim_ratio), theta=rope_theta, num_axes=2) + self.proj_in = nn.Linear(in_channels, dim) + self.register_tokens = nn.Parameter(torch.zeros(1, num_register_tokens, dim)) + self.transformer_blocks = nn.ModuleList( + [ + MiniMaxH3ImageTransformerBlock( + dim=dim, + heads=num_attention_heads, + dim_head=attention_head_dim, + ffn_mult=ffn_mult, + eps=norm_eps, + ) + for _ in range(num_layers) + ] + ) + self.norm_out = nn.LayerNorm(dim, elementwise_affine=True, eps=norm_eps) + self.proj_out = nn.Linear(dim, out_channels * patch_size * patch_size) + + self.gradient_checkpointing = False + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + batch_size, num_channels, height, width = hidden_states.shape + + hidden_states = hidden_states.permute(0, 2, 3, 1).reshape(batch_size, height * width, num_channels) + hidden_states = self.proj_in(hidden_states) + num_patches = hidden_states.shape[1] + + # `proj_in` is pinned to float32 and `register_tokens` is not, so align rather than rely on promotion. + register_tokens = self.register_tokens.expand(batch_size, -1, -1).to(hidden_states.dtype) + cls_token = torch.zeros_like(hidden_states[:, :1, :]) + hidden_states = torch.cat([hidden_states, register_tokens, cls_token], dim=1) + + grids = [ + 2.0 * (torch.arange(0.5, size, dtype=torch.float32, device=hidden_states.device) / size) - 1.0 + for size in (height, width) + ] + position_ids = torch.stack(torch.meshgrid(*grids, indexing="ij"), dim=-1).flatten(0, 1) + position_ids = position_ids.unsqueeze(0).expand(batch_size, -1, -1) + suffix_ids = position_ids.new_zeros((batch_size, self.num_register_tokens + 1, 2)) + position_ids = torch.cat([position_ids, suffix_ids], dim=1) + rotary_emb = self.rope(position_ids) + + for block in self.transformer_blocks: + if torch.is_grad_enabled() and self.gradient_checkpointing: + hidden_states = self._gradient_checkpointing_func(block, hidden_states, rotary_emb) + else: + hidden_states = block(hidden_states, rotary_emb) + + # `norm_out` is pinned to float32 and `proj_out` is not. + hidden_states = self.norm_out(hidden_states).to(get_parameter_dtype(self.proj_out)) + hidden_states = self.proj_out(hidden_states) + hidden_states = hidden_states[:, :num_patches, :] + + patch_size = self.patch_size + hidden_states = hidden_states.view(batch_size, height, width, self.out_channels, patch_size, patch_size) + hidden_states = hidden_states.permute(0, 3, 1, 4, 2, 5).contiguous() + return hidden_states.reshape(batch_size, self.out_channels, height * patch_size, width * patch_size) + + +class MiniMaxH3ImageVAE(BaseModel, ConfigMixin): + r""" + A VAE model with a causal 3D CNN encoder and a non-causal 2D ViT decoder, used in + [MiniMax-H3-Image](https://huggingface.co/MiniMaxAI). The encoder is the MiniMax-H3 video encoder run over a single + frame; the decoder is an image-only ViT. + + Checkpoints are loaded through TeleFuser ModuleManager and its model-specific state dictionary converter. + + Latents are normalized with per-channel `latents_mean` / `latents_std` rather than a `scaling_factor`; a pipeline + encodes with `(latent - latents_mean) / latents_std` and decodes with `latent * latents_std + latents_mean`. + + The pixel convention is ImageNet-normalized RGB over a `[0, 1]` base range, not the usual `[-1, 1]`: `encode` + expects `(pixel - imagenet_mean) / imagenet_std` and `decode` returns values in that same space, so a pipeline has + to apply `sample * imagenet_std + imagenet_mean` (mean `(0.485, 0.456, 0.406)`, std `(0.229, 0.224, 0.225)`) and + clamp to `[0, 1]` before postprocessing. + + Unlike most autoencoders in the library, spatial tiling is **on by default**: MiniMax-H3-Image was released with + tiling enabled for both encoding and decoding, and the released images are the blended-tile ones, so disabling + tiling changes the output. Use `enable_tiling` to change the tile geometry, `disable_tiling` to turn it off. + """ + + config_name = "config.json" + _supports_gradient_checkpointing = True + _no_split_modules = ["MiniMaxH3ImageResnetBlock3d", "MiniMaxH3ImageTransformerBlock"] + _repeated_blocks = ["MiniMaxH3ImageTransformerBlock"] + _skip_layerwise_casting_patterns = ["norm"] + # The released checkpoint is float32; only the cast-sensitive modules are pinned so `torch_dtype` still reaches + # the decoder block stack. + _keep_in_fp32_modules = [ + "encoder", + "quant_conv", + "norm1", + "norm2", + "norm_out", + "scale1", + "scale2", + ] + + @register_to_config + def __init__( + self, + in_channels: int = 3, + out_channels: int = 3, + latent_channels: int = 24, + block_out_channels: tuple[int, ...] = (128, 256, 256, 512, 512, 1024), + layers_per_block: int = 2, + spatial_downsample_factors: tuple[int, ...] = (2, 2, 2, 2, 1, 1), + temporal_downsample_factors: tuple[int, ...] = (1, 2, 2, 1, 1, 1), + norm_num_groups: int = 32, + norm_eps: float = 1e-6, + spatial_padding_mode: str = "reflect", + decoder_num_layers: int = 24, + decoder_num_attention_heads: int = 16, + decoder_attention_head_dim: int = 64, + decoder_num_register_tokens: int = 4, + decoder_ffn_mult: int = 4, + decoder_rope_theta: float = 100.0, + decoder_rope_dim_ratio: float = 0.5, + decoder_norm_eps: float = 1e-5, + latents_mean: tuple[float, ...] = (0.0,) * 24, + latents_std: tuple[float, ...] = (1.0,) * 24, + ) -> None: + super().__init__() + + self.spatial_compression_ratio = math.prod(spatial_downsample_factors) + + self.encoder = MiniMaxH3ImageEncoder3d( + in_channels=in_channels, + out_channels=2 * latent_channels, + block_out_channels=block_out_channels, + layers_per_block=layers_per_block, + spatial_downsample_factors=spatial_downsample_factors, + temporal_downsample_factors=temporal_downsample_factors, + norm_num_groups=norm_num_groups, + norm_eps=norm_eps, + spatial_padding_mode=spatial_padding_mode, + ) + self.quant_conv = nn.Conv3d(2 * latent_channels, 2 * latent_channels, kernel_size=1) + self.post_quant_conv = nn.Conv2d(latent_channels, latent_channels, kernel_size=1) + self.decoder = MiniMaxH3ImageViTDecoder2d( + in_channels=latent_channels, + out_channels=out_channels, + patch_size=self.spatial_compression_ratio, + num_layers=decoder_num_layers, + num_attention_heads=decoder_num_attention_heads, + attention_head_dim=decoder_attention_head_dim, + num_register_tokens=decoder_num_register_tokens, + ffn_mult=decoder_ffn_mult, + rope_theta=decoder_rope_theta, + rope_dim_ratio=decoder_rope_dim_ratio, + norm_eps=decoder_norm_eps, + ) + + # When decoding a batch of latents at a time, one can save memory by slicing across the batch dimension to + # perform decoding of a single latent at a time. + self.use_slicing = False + + # Large images are encoded and decoded in overlapping tiles that are linearly blended back together. + # MiniMax-H3-Image ships with tiling enabled. + self.use_tiling = True + + # The tile size in pixel space, and the minimum overlap between two neighbouring tiles. The actual overlaps are + # widened (in multiples of `spatial_compression_ratio`) so that the tiles cover the image exactly. + self.tile_sample_min_height = 256 + self.tile_sample_min_width = 256 + self.tile_sample_min_overlap_height = 64 + self.tile_sample_min_overlap_width = 64 + + def to(self, *args: Any, **kwargs: Any) -> MiniMaxH3ImageVAE: + def keep(name: str) -> bool: + return any(part in self._keep_in_fp32_modules for part in name.split(".")) + + preserved = { + name: parameter.detach().clone() + for name, parameter in self.named_parameters() + if keep(name) and not parameter.is_meta + } + rope = self.decoder.rope.inv_freq + preserved_rope = None if rope.is_meta else rope.detach().clone() + result = super().to(*args, **kwargs) + for name, parameter in result.named_parameters(): + if name in preserved: + parameter.data = preserved[name].to(device=parameter.device, dtype=torch.float32) + elif keep(name): + parameter.data = parameter.data.float() + rope = result.decoder.rope.inv_freq + rope.data = (rope if preserved_rope is None else preserved_rope.to(rope.device)).float() + return result + + # Copied from diffusers.models.autoencoders.autoencoder_kl_minimax_h3.AutoencoderKLMiniMaxH3.enable_tiling + def enable_tiling( + self, + tile_sample_min_height: int | None = None, + tile_sample_min_width: int | None = None, + tile_sample_min_overlap_height: int | None = None, + tile_sample_min_overlap_width: int | None = None, + ) -> None: + r""" + Enable tiled VAE encoding/decoding. When this option is enabled, the VAE splits the frames into tiles, encodes + or decodes each tile separately and linearly blends the overlaps back together. This lowers the memory + requirement and allows processing larger frames. + + Args: + tile_sample_min_height (`int`, *optional*): + The tile height in pixel space. Frames taller than this are split along the height dimension. + tile_sample_min_width (`int`, *optional*): + The tile width in pixel space. Frames wider than this are split along the width dimension. + tile_sample_min_overlap_height (`int`, *optional*): + The minimum overlap, in pixels, between two consecutive vertical tiles. + tile_sample_min_overlap_width (`int`, *optional*): + The minimum overlap, in pixels, between two consecutive horizontal tiles. + """ + self.use_tiling = True + self.tile_sample_min_height = tile_sample_min_height or self.tile_sample_min_height + self.tile_sample_min_width = tile_sample_min_width or self.tile_sample_min_width + self.tile_sample_min_overlap_height = tile_sample_min_overlap_height or self.tile_sample_min_overlap_height + self.tile_sample_min_overlap_width = tile_sample_min_overlap_width or self.tile_sample_min_overlap_width + + # Copied from diffusers.models.autoencoders.autoencoder_kl_minimax_h3.AutoencoderKLMiniMaxH3._split_tiles + def _split_tiles(self, length: int, tile_size: int, min_overlap: int) -> tuple[list[int], list[int], list[int]]: + r""" + Lay `tile_size`-wide tiles over `length` pixels. The number of tiles is the smallest one whose union can cover + `length` while keeping every overlap at least `min_overlap`; the slack is then distributed round-robin over the + overlaps in whole `spatial_compression_ratio` steps so that every tile boundary stays latent-aligned. + """ + if tile_size >= length: + return [0], [length], [] + + num_tiles = math.ceil(length / tile_size) + while tile_size * num_tiles - min_overlap * (num_tiles - 1) - length < 0: + num_tiles += 1 + + overlaps = [min_overlap] * (num_tiles - 1) + remaining = tile_size * num_tiles - sum(overlaps) - length + for i in range(remaining // self.spatial_compression_ratio): + overlaps[i % (num_tiles - 1)] += self.spatial_compression_ratio + + tile_start_indices = [0] + for i in range(num_tiles - 1): + tile_start_indices.append(tile_start_indices[-1] + tile_size - overlaps[i]) + return tile_start_indices, [tile_size] * num_tiles, overlaps + + # Copied from diffusers.models.autoencoders.autoencoder_kl_minimax_h3.AutoencoderKLMiniMaxH3._blend + def _blend(self, a: torch.Tensor, b: torch.Tensor, blend_extent: int, dim: int) -> torch.Tensor: + blend_extent = min(a.shape[dim], b.shape[dim], blend_extent) + positions = torch.arange(blend_extent, device=b.device, dtype=b.dtype) + shape = [1] * a.ndim + shape[dim] = blend_extent + weight_a = (1 - positions / blend_extent).view(shape) + weight_b = (positions / blend_extent).view(shape) + + slice_a = [slice(None)] * a.ndim + slice_a[dim] = slice(-blend_extent, None) + slice_b = [slice(None)] * b.ndim + slice_b[dim] = slice(0, blend_extent) + blended = a[tuple(slice_a)] * weight_a + b[tuple(slice_b)] * weight_b + + if blend_extent == b.shape[dim]: + return blended + slice_rest = [slice(None)] * b.ndim + slice_rest[dim] = slice(blend_extent, None) + return torch.cat([blended, b[tuple(slice_rest)]], dim=dim) + + # Copied from diffusers.models.autoencoders.autoencoder_kl_minimax_h3.AutoencoderKLMiniMaxH3._stitch_tiles + def _stitch_tiles( + self, + tiles: list[list[torch.Tensor]], + height_overlaps: list[int], + width_overlaps: list[int], + ) -> torch.Tensor: + result_rows = [] + for i, row in enumerate(tiles): + result_row = [] + for j, tile in enumerate(row): + if i > 0: + tile = self._blend(tiles[i - 1][j], tile, height_overlaps[i - 1], dim=-2) + if j > 0: + tile = self._blend(row[j - 1], tile, width_overlaps[j - 1], dim=-1) + if i < len(tiles) - 1: + tile = tile[..., : -height_overlaps[i], :] + if j < len(row) - 1: + tile = tile[..., :, : -width_overlaps[j]] + result_row.append(tile) + result_rows.append(torch.cat(result_row, dim=-1)) + return torch.cat(result_rows, dim=-2) + + def _encode(self, x: torch.Tensor) -> torch.Tensor: + # The encoder is a causal video encoder; an image is a single frame, which maps to a single latent frame. + if not self.use_tiling: + return self.quant_conv(self.encoder(x[:, :, None]))[:, :, -1] + + height, width = x.shape[-2], x.shape[-1] + y_indices, y_lengths, y_overlaps = self._split_tiles( + height, self.tile_sample_min_height, self.tile_sample_min_overlap_height + ) + x_indices, x_lengths, x_overlaps = self._split_tiles( + width, self.tile_sample_min_width, self.tile_sample_min_overlap_width + ) + + rows = [] + for i_pos, i_len in zip(y_indices, y_lengths): + row = [] + for j_pos, j_len in zip(x_indices, x_lengths): + tile = x[..., i_pos : i_pos + i_len, j_pos : j_pos + j_len] + row.append(self.quant_conv(self.encoder(tile[:, :, None]))[:, :, -1]) + rows.append(row) + + latent_y_overlaps = [overlap // self.spatial_compression_ratio for overlap in y_overlaps] + latent_x_overlaps = [overlap // self.spatial_compression_ratio for overlap in x_overlaps] + return self._stitch_tiles(rows, latent_y_overlaps, latent_x_overlaps) + + def _decode(self, z: torch.Tensor) -> torch.Tensor: + if not self.use_tiling: + return self.decoder(self.post_quant_conv(z)) + + # Tiles are laid out in pixel space and then mapped back onto the latent grid. + height = z.shape[-2] * self.spatial_compression_ratio + width = z.shape[-1] * self.spatial_compression_ratio + y_indices, y_lengths, y_overlaps = self._split_tiles( + height, self.tile_sample_min_height, self.tile_sample_min_overlap_height + ) + x_indices, x_lengths, x_overlaps = self._split_tiles( + width, self.tile_sample_min_width, self.tile_sample_min_overlap_width + ) + + ratio = self.spatial_compression_ratio + rows = [] + for i_pos, i_len in zip(y_indices, y_lengths): + row = [] + for j_pos, j_len in zip(x_indices, x_lengths): + tile = z[ + ..., + i_pos // ratio : i_pos // ratio + i_len // ratio, + j_pos // ratio : j_pos // ratio + j_len // ratio, + ] + row.append(self.decoder(self.post_quant_conv(tile))) + rows.append(row) + + return self._stitch_tiles(rows, y_overlaps, x_overlaps) + + @apply_forward_hook + def encode(self, x: torch.Tensor, return_dict: bool = True) -> AutoencoderKLOutput | tuple[torch.Tensor]: + r""" + Encode a batch of images into latents. + + Args: + x (`torch.Tensor`): + Input batch of images, shape `(batch_size, in_channels, height, width)`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether to return a [`~models.autoencoders.autoencoder_kl.AutoencoderKLOutput`] instead of a plain + tuple. + + Returns: + The latent distribution of the encoded images, over latents of shape `(batch_size, latent_channels, height + // spatial_compression_ratio, width // spatial_compression_ratio)`. Note that MiniMax-H3-Image normalizes + the sampled latents with `latents_mean` / `latents_std` afterwards. + """ + # The encoder is pinned to float32 by `_keep_in_fp32_modules`; align lower-precision pixels with the weights. + x = x.to(get_parameter_dtype(self.encoder)) + if self.use_slicing and x.shape[0] > 1: + moments = torch.cat([self._encode(x_slice) for x_slice in x.split(1)]) + else: + moments = self._encode(x) + posterior = DiagonalGaussianDistribution(moments) + if not return_dict: + return (posterior,) + return AutoencoderKLOutput(latent_dist=posterior) + + @apply_forward_hook + def decode(self, z: torch.Tensor, return_dict: bool = True) -> DecoderOutput | tuple[torch.Tensor]: + r""" + Decode a batch of latents. + + Args: + z (`torch.Tensor`): + Input batch of latents, shape `(batch_size, latent_channels, height, width)`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether to return a [`~models.autoencoders.vae.DecoderOutput`] instead of a plain tuple. + + Returns: + [`~models.autoencoders.vae.DecoderOutput`] or `tuple`: + The decoded images, shape `(batch_size, out_channels, height * spatial_compression_ratio, width * + spatial_compression_ratio)`. + """ + # The decoder is mixed precision, so align with the first module that consumes the latents. + z = z.to(get_parameter_dtype(self.post_quant_conv)) + if self.use_slicing and z.shape[0] > 1: + decoded = torch.cat([self._decode(z_slice) for z_slice in z.split(1)]) + else: + decoded = self._decode(z) + if not return_dict: + return (decoded,) + return DecoderOutput(sample=decoded) + + def forward( + self, + sample: torch.Tensor, + sample_posterior: bool = False, + generator: torch.Generator | None = None, + return_dict: bool = True, + ) -> DecoderOutput | tuple[torch.Tensor]: + r""" + Encode then decode a batch of images. + + Args: + sample (`torch.Tensor`): + Input batch of images, shape `(batch_size, in_channels, height, width)`. + sample_posterior (`bool`, *optional*, defaults to `False`): + Whether to sample the posterior instead of taking its mode. + generator (`torch.Generator`, *optional*): + Generator used when `sample_posterior=True`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether to return a [`~models.autoencoders.vae.DecoderOutput`] instead of a plain tuple. + + Returns: + [`~models.autoencoders.vae.DecoderOutput`] or `tuple`: + The round-tripped images, shape `(batch_size, out_channels, height, width)`. + """ + posterior = self.encode(sample).latent_dist + z = posterior.sample(generator=generator) if sample_posterior else posterior.mode() + return self.decode(z, return_dict=return_dict) + + @staticmethod + def state_dict_converter(config_path: str | Path | None = None) -> MiniMaxH3ImageVAEStateDictConverter: + return MiniMaxH3ImageVAEStateDictConverter(config_path) + + +class MiniMaxH3ImageVAEStateDictConverter: + def __init__(self, config_path: str | Path | None) -> None: + if config_path is None: + raise ValueError("H3-Image VAE requires vae_diffusers/config.json") + from pathlib import Path + + self.config_path = Path(config_path) + if self.config_path.is_dir(): + self.config_path = self.config_path / "config.json" + + def from_official(self, state_dict: dict[str, torch.Tensor]) -> tuple[dict[str, torch.Tensor], dict[str, Any]]: + import inspect + import json + + config = json.loads(self.config_path.read_text()) + accepted = inspect.signature(MiniMaxH3ImageVAE.__init__).parameters + return state_dict, {key: value for key, value in config.items() if key in accepted} + + def from_diffusers(self, state_dict: dict[str, torch.Tensor]) -> tuple[dict[str, torch.Tensor], dict[str, Any]]: + return self.from_official(state_dict) diff --git a/telefuser/models/pan2_conditioning.py b/telefuser/models/pan2_conditioning.py new file mode 100644 index 00000000..6851f8a5 --- /dev/null +++ b/telefuser/models/pan2_conditioning.py @@ -0,0 +1,74 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Checkpoint-compatible conditioning components for PAN2.""" + +from __future__ import annotations + +import inspect +import json +from pathlib import Path +from typing import Any + +import torch + +from telefuser.core.base_model import BaseModel + + +class PAN2TextEncoder(BaseModel): + def __init__(self, config: dict[str, Any]) -> None: + super().__init__() + from transformers import Qwen3_5TextConfig, Qwen3_5TextModel + + self.model = Qwen3_5TextModel(Qwen3_5TextConfig(**config)) + + def to(self, *args: Any, **kwargs: Any) -> PAN2TextEncoder: + # Match the Transformers loader: rotary frequencies remain FP32 rather than checkpoint dtype. + preserved = { + name: buffer.detach().clone() + for name, buffer in self.named_buffers() + if name.endswith("inv_freq") and not buffer.is_meta + } + result = super().to(*args, **kwargs) + for name, buffer in result.named_buffers(): + if name.endswith("inv_freq"): + buffer.data = preserved.get(name, buffer).to(device=buffer.device, dtype=torch.float32) + return result + + def forward(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.model(input_ids=input_ids, output_hidden_states=True, use_cache=False).hidden_states[-3] + + @staticmethod + def state_dict_converter(config_path: str | Path) -> PAN2ConditioningConverter: + return PAN2ConditioningConverter(config_path) + + +class PAN2VAE(BaseModel): + def __init__(self, config: dict[str, Any]) -> None: + super().__init__() + from diffusers import AutoencoderKLWan + + allowed = inspect.signature(AutoencoderKLWan.__init__).parameters + self.model = AutoencoderKLWan(**{k: v for k, v in config.items() if k in allowed}) + self.config = self.model.config + + def encode(self, pixels: torch.Tensor) -> torch.Tensor: + return self.model.encode(pixels).latent_dist.mode() + + def decode(self, latents: torch.Tensor) -> torch.Tensor: + return self.model.decode(latents).sample + + @staticmethod + def state_dict_converter(config_path: str | Path) -> PAN2ConditioningConverter: + return PAN2ConditioningConverter(config_path) + + +class PAN2ConditioningConverter: + def __init__(self, config_path: str | Path) -> None: + self.config_path = Path(config_path) + + def from_diffusers(self, state_dict: dict[str, torch.Tensor]) -> tuple[dict[str, torch.Tensor], dict[str, Any]]: + return {f"model.{name}": value for name, value in state_dict.items()}, { + "config": json.loads(self.config_path.read_text()) + } + + def from_official(self, state_dict: dict[str, torch.Tensor]) -> tuple[dict[str, torch.Tensor], dict[str, Any]]: + return self.from_diffusers(state_dict) diff --git a/telefuser/models/pan2_dit.py b/telefuser/models/pan2_dit.py new file mode 100644 index 00000000..4abd9286 --- /dev/null +++ b/telefuser/models/pan2_dit.py @@ -0,0 +1,812 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +# Ported from Diffusers commit 843f1743e259e510f22065c0cb917d417b98d9f5. +import torch +import torch.distributed as dist +import torch.nn as nn +import torch.nn.functional as F +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.models.attention import FeedForward +from diffusers.models.embeddings import TimestepEmbedding, Timesteps, apply_rotary_emb, get_1d_rotary_pos_embed +from diffusers.models.modeling_outputs import Transformer2DModelOutput +from diffusers.models.normalization import AdaLayerNormContinuous, AdaLayerNormZero, RMSNorm +from diffusers.utils import logging + +from telefuser.core.base_model import BaseModel +from telefuser.core.config import AttentionConfig, AttnImplType, QuantConfig, QuantKernelBackend, QuantType +from telefuser.distributed.collectives import all_gather_cat, all_reduce_sum_ +from telefuser.distributed.device_mesh import ( + get_ring_world_size, + get_tp_group, + get_tp_rank, + get_tp_world_size, + get_ulysses_group, + get_ulysses_rank, + get_ulysses_world_size, +) +from telefuser.distributed.ulysses_comm import ulysses_gather_heads_destination_major, ulysses_scatter_heads +from telefuser.ops.attention import SparseAttentionState, attention +from telefuser.ops.fp8_attention import apply_fp8_attention_output_correction, quantize_fp8_qkv_smoothed + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +# Copied from diffusers.models.transformers.transformer_flux._get_projections with FluxAttention->PAN2Attention +def _get_projections(attn: "PAN2Attention", hidden_states, encoder_hidden_states=None): + query = attn.to_q(hidden_states) + key = attn.to_k(hidden_states) + value = attn.to_v(hidden_states) + + encoder_query = encoder_key = encoder_value = None + if encoder_hidden_states is not None and attn.added_kv_proj_dim is not None: + encoder_query = attn.add_q_proj(encoder_hidden_states) + encoder_key = attn.add_k_proj(encoder_hidden_states) + encoder_value = attn.add_v_proj(encoder_hidden_states) + + return query, key, value, encoder_query, encoder_key, encoder_value + + +# Copied from diffusers.models.transformers.transformer_flux._get_fused_projections with FluxAttention->PAN2Attention +def _get_fused_projections(attn: "PAN2Attention", hidden_states, encoder_hidden_states=None): + query, key, value = attn.to_qkv(hidden_states).chunk(3, dim=-1) + + encoder_query = encoder_key = encoder_value = (None,) + if encoder_hidden_states is not None and hasattr(attn, "to_added_qkv"): + encoder_query, encoder_key, encoder_value = attn.to_added_qkv(encoder_hidden_states).chunk(3, dim=-1) + + return query, key, value, encoder_query, encoder_key, encoder_value + + +# Copied from diffusers.models.transformers.transformer_flux._get_qkv_projections with FluxAttention->PAN2Attention +def _get_qkv_projections(attn: "PAN2Attention", hidden_states, encoder_hidden_states=None): + if attn.fused_projections: + return _get_fused_projections(attn, hidden_states, encoder_hidden_states) + return _get_projections(attn, hidden_states, encoder_hidden_states) + + +class PAN2AttnProcessor: + r""" + Attention processor for PAN 2. Without `encoder_hidden_states` it runs plain self-attention (token refiner). With + `encoder_hidden_states` it runs joint attention over `[video; text]`, where each stream has its own projections and + only the video stream is rotated. + """ + + _attention_backend = None + _parallel_config = None + + def __call__( + self, + attn: "PAN2Attention", + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor | None = None, + attention_mask: torch.Tensor | None = None, + rotary_emb: tuple[torch.Tensor, torch.Tensor] | None = None, + ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: + query, key, value, encoder_query, encoder_key, encoder_value = _get_qkv_projections( + attn, hidden_states, encoder_hidden_states + ) + + query = query.unflatten(-1, (attn.heads, -1)) + key = key.unflatten(-1, (attn.heads, -1)) + value = value.unflatten(-1, (attn.heads, -1)) + + query = attn.norm_q(query).to(value.dtype) + key = attn.norm_k(key).to(value.dtype) + + if rotary_emb is not None: + query = apply_rotary_emb(query, rotary_emb, sequence_dim=1) + key = apply_rotary_emb(key, rotary_emb, sequence_dim=1) + + if encoder_hidden_states is not None: + encoder_query = encoder_query.unflatten(-1, (attn.heads, -1)) + encoder_key = encoder_key.unflatten(-1, (attn.heads, -1)) + encoder_value = encoder_value.unflatten(-1, (attn.heads, -1)) + + encoder_query = attn.norm_added_q(encoder_query).to(encoder_value.dtype) + encoder_key = attn.norm_added_k(encoder_key).to(encoder_value.dtype) + + query = torch.cat([query, encoder_query], dim=1) + key = torch.cat([key, encoder_key], dim=1) + value = torch.cat([value, encoder_value], dim=1) + + hidden_states = _parallel_attention(attn, query, key, value, attention_mask) + hidden_states = hidden_states.flatten(2, 3).type_as(query) + + if encoder_hidden_states is None: + hidden_states = attn.to_out[0](hidden_states) + hidden_states = attn.to_out[1](hidden_states) + if attn.tp_group is not None: + all_reduce_sum_([hidden_states], group=attn.tp_group) + return hidden_states + + hidden_states, encoder_hidden_states = hidden_states.split_with_sizes( + [hidden_states.shape[1] - encoder_hidden_states.shape[1], encoder_hidden_states.shape[1]], dim=1 + ) + hidden_states = attn.to_out[0](hidden_states) + hidden_states = attn.to_out[1](hidden_states) + encoder_hidden_states = attn.to_add_out(encoder_hidden_states) + if attn.tp_group is not None: + all_reduce_sum_([hidden_states, encoder_hidden_states], group=attn.tp_group) + return hidden_states, encoder_hidden_states + + +class PAN2Attention(nn.Module): + _default_processor_cls = PAN2AttnProcessor + _available_processors = [PAN2AttnProcessor] + + def __init__( + self, + query_dim: int, + heads: int, + dim_head: int, + eps: float, + bias: bool = True, + added_kv_proj_dim: int | None = None, + processor: PAN2AttnProcessor | None = None, + ): + super().__init__() + self.heads = heads + self.fused_projections = False + self.tp_group = None + self.sp_group = None + self.sp_communicator = None + self.global_video_tokens = 0 + self.global_text_tokens = 0 + self.sparse_state = None + self.attention_config = AttentionConfig() + self.inner_dim = heads * dim_head + self.use_bias = bias + self.added_kv_proj_dim = added_kv_proj_dim + self.added_proj_bias = bias + + self.to_q = nn.Linear(query_dim, self.inner_dim, bias=bias) + self.to_k = nn.Linear(query_dim, self.inner_dim, bias=bias) + self.to_v = nn.Linear(query_dim, self.inner_dim, bias=bias) + self.norm_q = RMSNorm(dim_head, eps=eps) + self.norm_k = RMSNorm(dim_head, eps=eps) + self.to_out = nn.ModuleList([nn.Linear(self.inner_dim, query_dim, bias=bias), nn.Dropout(0.0)]) + + if added_kv_proj_dim is not None: + self.add_q_proj = nn.Linear(added_kv_proj_dim, self.inner_dim, bias=bias) + self.add_k_proj = nn.Linear(added_kv_proj_dim, self.inner_dim, bias=bias) + self.add_v_proj = nn.Linear(added_kv_proj_dim, self.inner_dim, bias=bias) + self.norm_added_q = RMSNorm(dim_head, eps=eps) + self.norm_added_k = RMSNorm(dim_head, eps=eps) + self.to_add_out = nn.Linear(self.inner_dim, query_dim, bias=bias) + + if processor is None: + processor = self._default_processor_cls() + self.set_processor(processor) + + def set_processor(self, processor: Any) -> None: + self.processor = processor + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor | None = None, + attention_mask: torch.Tensor | None = None, + rotary_emb: tuple[torch.Tensor, torch.Tensor] | None = None, + ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: + return self.processor(self, hidden_states, encoder_hidden_states, attention_mask, rotary_emb) + + +class PAN2RotaryPosEmbed(nn.Module): + def __init__( + self, + patch_size: int, + patch_size_t: int, + rope_dim: list[int], + theta: float = 256.0, + temporal_scale: float = 1.0, + ) -> None: + super().__init__() + + self.patch_size = patch_size + self.patch_size_t = patch_size_t + self.rope_dim = rope_dim + self.theta = theta + self.temporal_scale = temporal_scale + + def forward(self, hidden_states: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + _, _, num_frames, height, width = hidden_states.shape + rope_sizes = [num_frames // self.patch_size_t, height // self.patch_size, width // self.patch_size] + + axes_grids = [] + for i in range(len(rope_sizes)): + grid = torch.arange(0, rope_sizes[i], device=hidden_states.device, dtype=torch.float32) + axes_grids.append(grid) + grid = torch.meshgrid(*axes_grids, indexing="ij") # [T, H, W] + grid = torch.stack(grid, dim=0) # [3, T, H, W] + # Only the frame positions are rescaled, to the frame rate the rotary embedding was trained at. + grid[0] = grid[0] * self.temporal_scale + + freqs = [] + for i in range(3): + freq = get_1d_rotary_pos_embed(self.rope_dim[i], grid[i].reshape(-1), self.theta, use_real=True) + freqs.append(freq) + + freqs_cos = torch.cat([f[0] for f in freqs], dim=1) # (T * H * W, D / 2) + freqs_sin = torch.cat([f[1] for f in freqs], dim=1) # (T * H * W, D / 2) + return freqs_cos, freqs_sin + + +class PAN2TokenRefinerBlock(nn.Module): + def __init__(self, dim: int, num_attention_heads: int, attention_head_dim: int, mlp_ratio: float, eps: float): + super().__init__() + self.norm1 = RMSNorm(dim, eps=eps) + self.attn = PAN2Attention(dim, heads=num_attention_heads, dim_head=attention_head_dim, eps=eps, bias=False) + self.norm2 = RMSNorm(dim, eps=eps) + self.ff = FeedForward(dim, inner_dim=int(dim * mlp_ratio), activation_fn="swiglu", bias=False) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + hidden_states = hidden_states + self.attn(self.norm1(hidden_states)) + ff_output = self.ff(self.norm2(hidden_states)) + if self.attn.tp_group is not None: + all_reduce_sum_([ff_output], group=self.attn.tp_group) + hidden_states = hidden_states + ff_output + return hidden_states + + +class PAN2TransformerBlock(nn.Module): + r""" + Joint-attention block. Video and text tokens attend to each other in a single attention call, but each stream has + its own QKV / output projections and feed-forward network. The video stream is modulated by the timestep, the text + stream by a learned, timestep-independent table. + """ + + def __init__( + self, + dim: int, + num_attention_heads: int, + attention_head_dim: int, + mlp_ratio: float, + context_mlp_ratio: float, + eps: float = 1e-6, + ): + super().__init__() + self.norm1 = AdaLayerNormZero(dim, norm_type="layer_norm") + self.norm1_context = nn.LayerNorm(dim, elementwise_affine=False, eps=eps) + self.scale_shift_table_context = nn.Parameter(torch.zeros(6 * dim)) + + self.attn = PAN2Attention( + dim, heads=num_attention_heads, dim_head=attention_head_dim, eps=eps, added_kv_proj_dim=dim + ) + + self.norm2 = nn.LayerNorm(dim, elementwise_affine=False, eps=eps) + self.ff = FeedForward(dim, inner_dim=int(dim * mlp_ratio), activation_fn="gelu-approximate") + + self.norm2_context = nn.LayerNorm(dim, elementwise_affine=False, eps=eps) + self.ff_context = FeedForward(dim, inner_dim=int(dim * context_mlp_ratio), activation_fn="gelu-approximate") + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor, + temb: torch.Tensor, + rotary_emb: tuple[torch.Tensor, torch.Tensor], + ) -> tuple[torch.Tensor, torch.Tensor]: + # The norms, their shift/scale and the gated residuals run in float32, like the modulation. + norm_hidden_states, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.norm1(hidden_states.float(), emb=temb) + norm_hidden_states = norm_hidden_states.type_as(hidden_states) + ( + context_shift_msa, + context_scale_msa, + context_gate_msa, + context_shift_mlp, + context_scale_mlp, + context_gate_mlp, + ) = self.scale_shift_table_context.chunk(6, dim=-1) + norm_encoder_hidden_states = ( + self.norm1_context(encoder_hidden_states.float()) * (1 + context_scale_msa) + context_shift_msa + ).type_as(encoder_hidden_states) + + attn_output, context_attn_output = self.attn( + hidden_states=norm_hidden_states, + encoder_hidden_states=norm_encoder_hidden_states, + rotary_emb=rotary_emb, + ) + + hidden_states = (hidden_states.float() + attn_output * gate_msa.unsqueeze(1)).type_as(hidden_states) + encoder_hidden_states = (encoder_hidden_states.float() + context_attn_output * context_gate_msa).type_as( + encoder_hidden_states + ) + + norm_hidden_states = ( + self.norm2(hidden_states.float()) * (1 + scale_mlp[:, None]) + shift_mlp[:, None] + ).type_as(hidden_states) + norm_encoder_hidden_states = ( + self.norm2_context(encoder_hidden_states.float()) * (1 + context_scale_mlp) + context_shift_mlp + ).type_as(encoder_hidden_states) + + ff_output = self.ff(norm_hidden_states) + context_ff_output = self.ff_context(norm_encoder_hidden_states) + if self.attn.tp_group is not None: + all_reduce_sum_([ff_output, context_ff_output], group=self.attn.tp_group) + hidden_states = (hidden_states.float() + ff_output * gate_mlp.unsqueeze(1)).type_as(hidden_states) + encoder_hidden_states = (encoder_hidden_states.float() + context_ff_output * context_gate_mlp).type_as( + encoder_hidden_states + ) + + return hidden_states, encoder_hidden_states + + +class PAN2DiT(BaseModel, ConfigMixin): + r""" + The diffusion transformer of PAN 2. + + A stack of joint-attention blocks over the concatenation of the patchified video latents and the refined text + tokens. The video input carries the noisy latents, the conditioning latents and a conditioning mask stacked along + the channel axis, so the same model serves text-to-video and image-to-video. + + Args: + patch_size (`tuple[int, int, int]`, defaults to `(1, 2, 2)`): + The `(t, h, w)` patch size of the patch embedding. + in_channels (`int`, defaults to `97`): + The number of input channels: noisy latents, conditioning latents and one conditioning mask channel. + out_channels (`int`, defaults to `48`): + The number of output channels. + num_attention_heads (`int`, defaults to `32`): + The number of attention heads. + attention_head_dim (`int`, defaults to `128`): + The number of channels in each attention head. + num_layers (`int`, defaults to `44`): + The number of joint-attention blocks. + num_refiner_layers (`int`, defaults to `4`): + The number of token refiner blocks applied to the text embeddings. + mlp_ratio (`float`, defaults to `8.0`): + The ratio of the video feed-forward inner dimension to the hidden dimension. + context_mlp_ratio (`float`, defaults to `4.0`): + The ratio of the text feed-forward inner dimension to the hidden dimension. + refiner_mlp_ratio (`float`, defaults to `4.0`): + The ratio of the token refiner feed-forward inner dimension to the hidden dimension. + text_embed_dim (`int`, defaults to `4096`): + The dimension of the text encoder hidden states. + rope_axes_dim (`tuple[int, int, int]`, defaults to `(16, 56, 56)`): + The number of channels of each `(t, h, w)` axis in the rotary embedding. + rope_theta (`float`, defaults to `256.0`): + The base of the rotary embedding. + rope_temporal_scale (`float`, defaults to `16 / 24`): + The factor the frame positions of the rotary embedding are multiplied with: the reference frame rate of the + rotary embedding over the frame rate of the videos (16 over 24 fps). + """ + + config_name = "config.json" + _supports_gradient_checkpointing = True + # The timestep embedder and the modulation run in float32. `linear` is the modulation of every block + # (`norm1.linear`) and of the output norm (`norm_out.linear`). + _keep_in_fp32_modules = ["time_embedder", "scale_shift_table_context", "linear"] + _skip_layerwise_casting_patterns = ["x_embedder", "context_embedder", "norm"] + _no_split_modules = ["PAN2TransformerBlock", "PAN2TokenRefinerBlock"] + _repeated_blocks = ["PAN2TransformerBlock", "PAN2TokenRefinerBlock"] + # Text is sharded before refinement; padded video/text shards are compacted into canonical joint order + # for attention and restored before the inverse Ulysses all-to-all. + + @register_to_config + def __init__( + self, + patch_size: tuple[int, int, int] = (1, 2, 2), + in_channels: int = 97, + out_channels: int = 48, + num_attention_heads: int = 32, + attention_head_dim: int = 128, + num_layers: int = 44, + num_refiner_layers: int = 4, + mlp_ratio: float = 8.0, + context_mlp_ratio: float = 4.0, + refiner_mlp_ratio: float = 4.0, + text_embed_dim: int = 4096, + rope_axes_dim: tuple[int, int, int] = (16, 56, 56), + rope_theta: float = 256.0, + rope_temporal_scale: float = 16 / 24, + ): + super().__init__() + inner_dim = num_attention_heads * attention_head_dim + + self.x_embedder = nn.Conv3d(in_channels, inner_dim, kernel_size=patch_size, stride=patch_size) + self.rope = PAN2RotaryPosEmbed(patch_size[1], patch_size[0], rope_axes_dim, rope_theta, rope_temporal_scale) + + self.time_proj = Timesteps(num_channels=256, flip_sin_to_cos=True, downscale_freq_shift=0) + self.time_embedder = TimestepEmbedding(in_channels=256, time_embed_dim=inner_dim) + + self.context_embedder = nn.Sequential(RMSNorm(text_embed_dim, eps=1e-5), nn.Linear(text_embed_dim, inner_dim)) + self.context_refiner = nn.ModuleList( + [ + PAN2TokenRefinerBlock(inner_dim, num_attention_heads, attention_head_dim, refiner_mlp_ratio, eps=1e-5) + for _ in range(num_refiner_layers) + ] + ) + self.context_type_embedding = nn.Parameter(torch.zeros(inner_dim)) + + self.transformer_blocks = nn.ModuleList( + [ + PAN2TransformerBlock(inner_dim, num_attention_heads, attention_head_dim, mlp_ratio, context_mlp_ratio) + for _ in range(num_layers) + ] + ) + + self.norm_out = AdaLayerNormContinuous(inner_dim, inner_dim, elementwise_affine=False, eps=1e-6) + self.proj_out = nn.Linear(inner_dim, patch_size[0] * patch_size[1] * patch_size[2] * out_channels) + + self.gradient_checkpointing = False + self.layer_name_list = ["transformer_blocks"] + self.sparse_attention_state = None + + def forward( + self, + hidden_states: torch.Tensor, + timestep: torch.Tensor, + encoder_hidden_states: torch.Tensor, + attention_kwargs: dict[str, Any] | None = None, + return_dict: bool = True, + sparse_step_index: int = 0, + ) -> torch.Tensor | Transformer2DModelOutput: + r""" + The [`PAN2Transformer3DModel`] forward method. + + Args: + hidden_states (`torch.Tensor` of shape `(batch_size, in_channels, num_frames, height, width)`): + The noisy latents, the conditioning latents and the conditioning mask, concatenated along channels. + timestep (`torch.Tensor` of shape `(batch_size,)`): + The denoising timestep in `[0, 1000]`. + encoder_hidden_states (`torch.Tensor` of shape `(batch_size, text_seq_len, text_embed_dim)`): + The text encoder hidden states. + attention_kwargs (`dict`, *optional*): + A kwargs dictionary passed along to the `AttentionProcessor`; only used for the LoRA scale. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~models.modeling_outputs.Transformer2DModelOutput`] instead of a plain + tuple. + + Returns: + [`~models.modeling_outputs.Transformer2DModelOutput`] or `tuple`: + The predicted velocity of shape `(batch_size, out_channels, num_frames, height, width)`. + """ + if attention_kwargs: + raise ValueError("PAN2 LoRA attention kwargs are not supported by this inference adapter") + batch_size, _, num_frames, height, width = hidden_states.shape + p_t, p_h, p_w = self.config.patch_size + post_patch_num_frames = num_frames // p_t + post_patch_height = height // p_h + post_patch_width = width // p_w + + # 1. Rotary embedding, patch and timestep embeddings + rotary_emb = self.rope(hidden_states) + + hidden_states = self.x_embedder(hidden_states) + hidden_states = hidden_states.flatten(2).transpose(1, 2).contiguous() + video_tokens, text_tokens = hidden_states.shape[1], encoder_hidden_states.shape[1] + for module in self.modules(): + if isinstance(module, PAN2Attention): + module.global_text_tokens = text_tokens + module.global_video_tokens = 0 + for block in self.transformer_blocks: + block.attn.global_video_tokens = video_tokens + if self.usp_flag: + world = get_ulysses_world_size(self.device_mesh) + rank = get_ulysses_rank(self.device_mesh) + hidden_states = _sequence_shard(hidden_states, world, rank) + encoder_hidden_states = _sequence_shard(encoder_hidden_states, world, rank) + rotary_emb = tuple(_sequence_shard(value.unsqueeze(0), world, rank).squeeze(0) for value in rotary_emb) + if self.sparse_attention_state is not None: + self.sparse_attention_state.update(numeral_timestep=sparse_step_index) + + timesteps_proj = self.time_proj(timestep).to(self.time_embedder.linear_1.weight.dtype) + temb = self.time_embedder(timesteps_proj) + + # 2. Text embedding and refinement + encoder_hidden_states = self.context_embedder(encoder_hidden_states) + for block in self.context_refiner: + if torch.is_grad_enabled() and self.gradient_checkpointing: + encoder_hidden_states = self._gradient_checkpointing_func(block, encoder_hidden_states) + else: + encoder_hidden_states = block(encoder_hidden_states) + encoder_hidden_states = encoder_hidden_states + self.context_type_embedding.to(encoder_hidden_states.device) + + # 3. Joint-attention blocks + for index, block in enumerate(self.transformer_blocks): + if self.sparse_attention_state is not None: + self.sparse_attention_state.update(layer_idx=index) + if torch.is_grad_enabled() and self.gradient_checkpointing: + hidden_states, encoder_hidden_states = self._gradient_checkpointing_func( + block, hidden_states, encoder_hidden_states, temb, rotary_emb + ) + else: + hidden_states, encoder_hidden_states = block(hidden_states, encoder_hidden_states, temb, rotary_emb) + + # 4. Output norm, projection and unpatchify + hidden_states = self.norm_out(hidden_states.type_as(temb), temb).type_as(hidden_states) + hidden_states = self.proj_out(hidden_states) + if self.usp_flag: + hidden_states = all_gather_cat(hidden_states, dim=1, group=get_ulysses_group(self.device_mesh)) + hidden_states = hidden_states[:, :video_tokens] + + hidden_states = hidden_states.reshape( + batch_size, post_patch_num_frames, post_patch_height, post_patch_width, -1, p_t, p_h, p_w + ) + hidden_states = hidden_states.permute(0, 4, 1, 5, 2, 6, 3, 7) + output = hidden_states.flatten(6, 7).flatten(4, 5).flatten(2, 3) + + if not return_dict: + return (output,) + + return Transformer2DModelOutput(sample=output) + + def set_attention_config(self, attention_config: AttentionConfig) -> None: + super().set_attention_config(attention_config) + if attention_config.attn_impl == AttnImplType.SOL_ATTN: + if attention_config.sparse_config is None: + raise ValueError("PAN2 Sol-Attn requires sparse configuration") + self.sparse_attention_state = SparseAttentionState( + config=attention_config.sparse_config, mask_map=None, model_type="pan2" + ) + else: + self.sparse_attention_state = None + for block in self.context_refiner: + block.attn.attention_config = ( + AttentionConfig.dense_attention(AttnImplType.TORCH_SDPA) + if attention_config.is_sparse() + else attention_config + ) + for block in self.transformer_blocks: + block.attn.attention_config = attention_config + block.attn.sparse_state = self.sparse_attention_state + + def to(self, *args: Any, **kwargs: Any) -> "PAN2DiT": + def keep(name: str) -> bool: + return ( + name.startswith("time_embedder.") + or "scale_shift_table_context" in name + or ".norm1.linear." in name + or name.startswith("norm_out.linear.") + ) + + preserved = { + name: parameter.detach().clone() + for name, parameter in self.named_parameters() + if keep(name) and not parameter.is_meta + } + result = super().to(*args, **kwargs) + for name, parameter in result.named_parameters(): + if name in preserved: + parameter.data = preserved[name].to(device=parameter.device, dtype=torch.float32) + elif keep(name): + parameter.data = parameter.data.float() + return result + + def enable_usp(self, device_mesh: torch.distributed.device_mesh.DeviceMesh | None = None) -> None: + self.device_mesh = device_mesh if device_mesh is not None else self.device_mesh + if get_ring_world_size(self.device_mesh) != 1: + raise NotImplementedError("PAN2 currently supports Ulysses sequence parallelism, not Ring") + world = get_ulysses_world_size(self.device_mesh) + if self.context_refiner[0].attn.heads % world: + raise ValueError("Local PAN2 attention heads must divide the Ulysses degree") + group = get_ulysses_group(self.device_mesh) if world > 1 else None + communicator = self._configure_ulysses_communicator(group) + for module in self.modules(): + if isinstance(module, PAN2Attention): + module.sp_group = group + module.sp_communicator = communicator + self.usp_flag = world > 1 + + def enable_tp(self, device_mesh: torch.distributed.device_mesh.DeviceMesh | None = None) -> None: + self.device_mesh = device_mesh if device_mesh is not None else self.device_mesh + world = get_tp_world_size(self.device_mesh) + if world == 1: + return + if getattr(self, "tp_flag", False): + raise RuntimeError("PAN2 tensor parallelism is already enabled") + rank, group = get_tp_rank(self.device_mesh), get_tp_group(self.device_mesh) + for module in self.modules(): + if not isinstance(module, PAN2Attention): + continue + if module.heads % world: + raise ValueError("PAN2 attention heads must divide TP degree") + for name in ["to_q", "to_k", "to_v", "add_q_proj", "add_k_proj", "add_v_proj"]: + if hasattr(module, name): + _column_shard(getattr(module, name), rank, world) + _row_shard(module.to_out[0], rank, world) + if hasattr(module, "to_add_out"): + _row_shard(module.to_add_out, rank, world) + module.heads //= world + module.inner_dim //= world + module.tp_group = group + for block in self.context_refiner: + _ffn_shard(block.ff, rank, world, gated=True) + for block in self.transformer_blocks: + _ffn_shard(block.ff, rank, world, gated=False) + _ffn_shard(block.ff_context, rank, world, gated=False) + _column_shard(block.norm1.linear, rank, world) + block.norm1.linear.register_forward_hook(_ColumnGather(group)) + _column_shard(self.norm_out.linear, rank, world) + self.norm_out.linear.register_forward_hook(_ColumnGather(group)) + self.tp_flag = True + + def enable_quant(self, quant_type: QuantConfig | QuantType | str | torch.dtype) -> None: + if not isinstance(quant_type, QuantConfig): + if quant_type not in (QuantType.FP8, "fp8", torch.float8_e4m3fn): + raise ValueError("PAN2 supports tf-kernel FP8 quantization through QuantConfig") + quant_type = QuantConfig( + enabled=True, quant_type=QuantType.FP8, kernel_backend=QuantKernelBackend.TF_KERNEL + ) + if not quant_type.enabled: + return + if quant_type.quant_type != QuantType.FP8 or quant_type.kernel_backend not in { + QuantKernelBackend.AUTO, + QuantKernelBackend.TF_KERNEL, + }: + raise ValueError("PAN2 online quantization currently supports tf-kernel FP8") + from telefuser.ops.fp8_gemm import FP8GemmOptions, count_linear_layers, enable_fp8_gemm + + def include(name: str, module: nn.Module) -> bool: + selected = quant_type.quantize_modules or ("transformer_blocks.", "context_refiner.") + return ( + any(part in name for part in selected) + and ".norm1.linear" not in name + and not any(part and part in name for part in quant_type.skip_modules) + ) + + self.tf_kernel_fp8_replaced_linear = count_linear_layers(self, module_filter=include) + enable_fp8_gemm( + self, + options=FP8GemmOptions( + fp16_weight_storage="keep" if quant_type.keep_fp16_weight else "discard", + materialize_fp8_on_wrap=True, + ), + module_filter=include, + ) + self.quant_type = quant_type.quant_type + + @staticmethod + def state_dict_converter(config_path: str | Path | None = None) -> PAN2DiTStateDictConverter: + return PAN2DiTStateDictConverter(config_path) + + +class PAN2DiTStateDictConverter: + def __init__(self, config_path: str | Path | None) -> None: + from pathlib import Path + + if config_path is None: + raise ValueError("PAN2 requires the released transformer/config.json") + self.config_path = Path(config_path) + if self.config_path.is_dir(): + self.config_path = self.config_path / "config.json" + + def from_diffusers(self, state_dict: dict[str, torch.Tensor]) -> tuple[dict[str, torch.Tensor], dict[str, Any]]: + import inspect + import json + + config = json.loads(self.config_path.read_text()) + accepted = inspect.signature(PAN2DiT.__init__).parameters + return state_dict, {name: value for name, value in config.items() if name in accepted} + + def from_official(self, state_dict: dict[str, torch.Tensor]) -> tuple[dict[str, torch.Tensor], dict[str, Any]]: + return self.from_diffusers(state_dict) + + +def _sequence_shard(value, world, rank): + length = value.shape[1] + local = (length + world - 1) // world + if local * world != length: + shape = list(value.shape) + shape[1] = local * world - length + value = torch.cat((value, value.new_zeros(shape)), dim=1) + return value[:, rank * local : (rank + 1) * local].contiguous() + + +def _replace_parameter(module, name, value): + parameter = module.get_parameter(name) + setattr(module, name, nn.Parameter(value.contiguous(), requires_grad=parameter.requires_grad)) + + +def _column_shard(module, rank, world, sections=None): + sections = sections or (module.out_features,) + if any(size % world for size in sections): + raise ValueError("PAN2 projection sections must divide TP degree") + _replace_parameter(module, "weight", torch.cat([x.chunk(world, 0)[rank] for x in module.weight.split(sections)], 0)) + if module.bias is not None: + _replace_parameter(module, "bias", torch.cat([x.chunk(world, 0)[rank] for x in module.bias.split(sections)], 0)) + module.out_features //= world + + +def _row_shard(module, rank, world): + if module.in_features % world: + raise ValueError("PAN2 row projection must divide TP degree") + _replace_parameter(module, "weight", module.weight.chunk(world, 1)[rank]) + if module.bias is not None and rank != 0: + _replace_parameter(module, "bias", torch.zeros_like(module.bias)) + module.in_features //= world + + +def _ffn_shard(module, rank, world, gated): + projection = module.net[0].proj + sections = (projection.out_features // 2,) * 2 if gated else None + _column_shard(projection, rank, world, sections) + _row_shard(module.net[2], rank, world) + + +class _ColumnGather: + def __init__(self, group: dist.ProcessGroup) -> None: + self.group = group + + def __call__(self, module: nn.Module, inputs: tuple[torch.Tensor, ...], output: torch.Tensor) -> torch.Tensor: + return all_gather_cat(output, dim=-1, group=self.group) + + +def _parallel_attention(module, query, key, value, attention_mask): + group = module.sp_group + use_sp = group is not None + indices = None + if use_sp: + if attention_mask is not None: + raise ValueError("PAN2 packed SP uses explicit token lengths, not arbitrary attention masks") + world = dist.get_world_size(group) + waits = [ + ulysses_scatter_heads(x, group, tag=tag, barrier=index == 2, communicator=module.sp_communicator) + for index, (tag, x) in enumerate(zip(["q", "k", "v"], [query, key, value])) + ] + query, key, value = [wait() for wait in waits] + video, text = module.global_video_tokens, module.global_text_tokens + vlocal, tlocal = (video + world - 1) // world, (text + world - 1) // world + joint = vlocal + tlocal + valid = [] + for total, local, offset in [(video, vlocal, 0), (text, tlocal, vlocal)]: + for rank in range(world): + count = max(0, min(local, total - rank * local)) + valid.extend(range(rank * joint + offset, rank * joint + offset + count)) + indices = torch.tensor(valid, dtype=torch.long, device=query.device) + original_shape = query.shape + query, key, value = [x.index_select(1, indices) for x in [query, key, value]] + config, state = module.attention_config, module.sparse_state + scales = None + correction = None + if state is not None: + if state.should_use_dense(): + config, state = AttentionConfig.dense_attention(AttnImplType.TORCH_SDPA), None + elif config.sparse_config.sol_fp8: + sparse = config.sparse_config + if state.layer_idx >= sparse.sol_fp8_layer_start and ( + sparse.sol_fp8_layer_end is None or state.layer_idx < sparse.sol_fp8_layer_end + ): + query, key, value, qs, ks, vs, correction = quantize_fp8_qkv_smoothed( + query, + key, + value, + smoothing=sparse.sol_fp8_smoothing, + correct_v_bias=sparse.sol_fp8_v_bias_correction, + ) + scales = (qs, ks, vs) + output = attention( + query, + key, + value, + attention_config=config, + sparse_state=state, + attn_mask=attention_mask, + q_scale=scales[0] if scales else None, + k_scale=scales[1] if scales else None, + v_scale=scales[2] if scales else None, + ) + if correction is not None: + output = apply_fp8_attention_output_correction(output.contiguous(), correction.contiguous()) + if use_sp: + padded = output.new_zeros(original_shape) + padded.index_copy_(1, indices, output) + output = ulysses_gather_heads_destination_major(padded, group, num_heads=module.heads)() + return output diff --git a/telefuser/pipelines/minimax_h3_image/__init__.py b/telefuser/pipelines/minimax_h3_image/__init__.py new file mode 100644 index 00000000..6a676289 --- /dev/null +++ b/telefuser/pipelines/minimax_h3_image/__init__.py @@ -0,0 +1,3 @@ +from .pipeline import MiniMaxH3ImagePipeline, MiniMaxH3ImagePipelineConfig + +__all__ = ["MiniMaxH3ImagePipeline", "MiniMaxH3ImagePipelineConfig"] diff --git a/telefuser/pipelines/minimax_h3_image/data.py b/telefuser/pipelines/minimax_h3_image/data.py new file mode 100644 index 00000000..46a26358 --- /dev/null +++ b/telefuser/pipelines/minimax_h3_image/data.py @@ -0,0 +1,179 @@ +# Copyright 2026 The MiniMax and HuggingFace Teams. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Adapted from Diffusers 843f1743e259e510f22065c0cb917d417b98d9f5. +import math + +import numpy as np +import torch + +# The spatial rotary axes are normalized by the square root of the latent area and scaled by 32. +_ROPE_SPATIAL_SCALE = 32 + + +# Copied from diffusers.modular_pipelines.minimax_h3.before_denoise.patchify_video_latents +def patchify_video_latents(latents: torch.Tensor, patch_size: tuple[int, int, int]) -> torch.Tensor: + r""" + Pack video latents into transformer rows. + + Args: + latents (`torch.Tensor` of shape `(batch_size, channels, num_frames, height, width)`): + The latents to pack. + patch_size (`tuple[int, int, int]`): The `(t, h, w)` patch. + + Returns: + `torch.Tensor` of shape `(batch_size * num_patches, channels * prod(patch_size))`: The packed rows, ordered + frame-major then row-major. + """ + patch_t, patch_h, patch_w = patch_size + batch_size, channels, num_frames, height, width = latents.shape + if num_frames % patch_t or height % patch_h or width % patch_w: + raise ValueError(f"Latents of shape {tuple(latents.shape)} are not divisible by the patch {patch_size}.") + + latents = latents.reshape( + batch_size, + channels, + num_frames // patch_t, + patch_t, + height // patch_h, + patch_h, + width // patch_w, + patch_w, + ) + latents = latents.permute(0, 2, 4, 6, 1, 3, 5, 7) + return latents.reshape(-1, channels * patch_t * patch_h * patch_w).contiguous() + + +# Copied from diffusers.modular_pipelines.minimax_h3.before_denoise._spatial_position_grid +def _spatial_position_grid(dim: int, patch: int, sqrt_area: float) -> torch.Tensor: + r""" + One aspect-normalized spatial rotary axis: `dim // patch` coordinates centred on the unit interval, scaled up by + 32. The right endpoint is excluded, so a square canvas spans `[0, 32)`. + """ + ratio = dim / sqrt_area + left = (1.0 - ratio) / 2.0 + # Built with numpy: `np.linspace(..., endpoint=False)` is `start + arange(num) * (stop - start) / num`, which is + # not what `torch.linspace` computes, and the float64 grid has to be reproduced exactly. + grid = np.linspace(left, left + ratio, dim // patch, endpoint=False) * _ROPE_SPATIAL_SCALE + return torch.from_numpy(grid).to(torch.float64) + + +# Copied from diffusers.modular_pipelines.minimax_h3.before_denoise._frame_position_grid +def _frame_position_grid( + latent_height: int, latent_width: int, patch_h: int, patch_w: int +) -> tuple[torch.Tensor, torch.Tensor]: + r"""The `(h, w)` rotary coordinates of one latent frame, and the width axis they were built from.""" + sqrt_area = np.sqrt(latent_height * latent_width) + height_grid = _spatial_position_grid(latent_height, patch_h, sqrt_area) + width_grid = _spatial_position_grid(latent_width, patch_w, sqrt_area) + grids = torch.meshgrid(height_grid, width_grid, indexing="ij") + return torch.stack([grid.reshape(-1) for grid in grids], dim=-1), width_grid + + +def get_image_canvas_size(height: int, width: int, min_generation_area: int, multiple: int) -> tuple[int, int]: + r""" + The canvas MiniMax-H3-Image generates a `height x width` image on. + + An image below `min_generation_area` pixels is generated larger at the same aspect ratio: both sides are scaled by + `sqrt(min_generation_area / (height * width))`. Either way both sides round to the nearest `multiple`. + + Returns: + `tuple[int, int]`: the `(height, width)` of the canvas. + """ + scale = max(1.0, math.sqrt(min_generation_area / (width * height))) + return ( + max(multiple, round(height * scale / multiple) * multiple), + max(multiple, round(width * scale / multiple) * multiple), + ) + + +def build_image_packed_sequence( + text_token_tags: torch.Tensor, + reference_grids: list[tuple[int, int]], + latent_height: int, + latent_width: int, + patch_size: tuple[int, int, int], + image_tag: int, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, int]: + r""" + Build the `[text | reference images | generated image]` layout of MiniMax-H3-Image. + + Args: + text_token_tags (`torch.Tensor` of shape `(num_text_tokens,)`): + The modality tag of every text row. Text is tagged `1`, except for the rows of a reference's vision block, + which are tagged `0` (image). + reference_grids (`list[tuple[int, int]]`): + The `(latent_height, latent_width)` of every reference image, in packed order. Empty for `text2image`. + latent_height (`int`): Latent height of the generated image. + latent_width (`int`): Latent width of the generated image. + patch_size (`tuple[int, int, int]`): The transformer's `(t, h, w)` patch. + image_tag (`int`): The modality tag an image row carries, i.e. `components.image_tag`. + + Returns: + `tuple`: `position_ids`, `token_tags`, `image_indices`, `text_indices`, and the number of leading image rows + that are reference rows rather than generated ones. + """ + _, patch_h, patch_w = patch_size + num_text_tokens = text_token_tags.shape[0] + num_reference_rows = sum((height // patch_h) * (width // patch_w) for height, width in reference_grids) + num_target_rows = (latent_height // patch_h) * (latent_width // patch_w) + sequence_length = num_text_tokens + num_reference_rows + num_target_rows + + position_ids = torch.zeros(sequence_length, 3, dtype=torch.float64) + position_ids[:num_text_tokens, 0] = torch.arange(num_text_tokens, dtype=torch.float64) + + # Every image is one frame on the rotary clock, which starts where the text rows end: a reference takes one + # integer slot each, and the generated image takes the slot after the last reference. + cursor = num_text_tokens + rotary_time = float(num_text_tokens) + for height, width in reference_grids: + rows = slice(cursor, cursor + (height // patch_h) * (width // patch_w)) + cursor = rows.stop + frame_grid, _ = _frame_position_grid(height, width, patch_h, patch_w) + position_ids[rows, 0] = rotary_time + position_ids[rows, 1:] = frame_grid + rotary_time += 1.0 + frame_grid, _ = _frame_position_grid(latent_height, latent_width, patch_h, patch_w) + position_ids[cursor:, 0] = rotary_time + position_ids[cursor:, 1:] = frame_grid + + text_indices = torch.arange(num_text_tokens) + image_indices = torch.arange(num_text_tokens, sequence_length) + token_tags = torch.empty(sequence_length, dtype=torch.long) + token_tags[text_indices] = text_token_tags.to(torch.long) + token_tags[image_indices] = image_tag + + return position_ids, token_tags, image_indices, text_indices, num_reference_rows + + +def bound_short_edge( + width: int, height: int, min_short_edge: int, max_short_edge: int, multiple: int +) -> tuple[int, int]: + r""" + The size a MiniMax-H3-Image reference is resized to for one of its consumers. + + A short edge within `[min_short_edge, max_short_edge]` keeps the original size. Otherwise the image is scaled, + keeping its ratio, so that the short edge lands on the nearer bound, and both sides round to `multiple`. + + Returns: + `tuple[int, int]`: the `(width, height)` to resize to. + """ + short_edge = min(width, height) + if min_short_edge <= short_edge <= max_short_edge: + return width, height + scale = (min_short_edge if short_edge < min_short_edge else max_short_edge) / short_edge + return ( + max(multiple, round(width * scale / multiple) * multiple), + max(multiple, round(height * scale / multiple) * multiple), + ) diff --git a/telefuser/pipelines/minimax_h3_image/pipeline.py b/telefuser/pipelines/minimax_h3_image/pipeline.py new file mode 100644 index 00000000..3299b827 --- /dev/null +++ b/telefuser/pipelines/minimax_h3_image/pipeline.py @@ -0,0 +1,134 @@ +# SPDX-License-Identifier: Apache-2.0 +"""TeleFuser text-to-image and reference-image editing with MiniMax-H3-Image.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +import torch +import torch.distributed as dist +from PIL import Image, ImageOps +from transformers import AutoProcessor + +from telefuser.core.base_pipeline import BasePipeline +from telefuser.core.config import ModelRuntimeConfig, WeightOffloadType +from telefuser.core.module_manager import ModuleManager +from telefuser.worker import ParallelWorker + +from .data import bound_short_edge, get_image_canvas_size +from .stages import MiniMaxH3ImageDenoisingStage, MiniMaxH3ImageTextStage, MiniMaxH3ImageVAEStage + + +@dataclass +class MiniMaxH3ImagePipelineConfig: + processor_path: str + pdd_config_path: str + scheduler_config_path: str + text_encoder_config: ModelRuntimeConfig = field(default_factory=ModelRuntimeConfig) + dit_config: ModelRuntimeConfig = field(default_factory=ModelRuntimeConfig) + vae_config: ModelRuntimeConfig = field(default_factory=ModelRuntimeConfig) + + def __post_init__(self) -> None: + for name in ("text_encoder_config", "dit_config", "vae_config"): + runtime = getattr(self, name) + runtime.parallel_config.validate() + if runtime.parallel_config.enable_fsdp: + raise NotImplementedError("This pipeline supports Ulysses SP and TP, not FSDP") + if runtime.offload_config.offload_type not in { + WeightOffloadType.NO_CPU_OFFLOAD, + WeightOffloadType.MODEL_CPU_OFFLOAD, + }: + raise NotImplementedError("This pipeline supports no offload or model CPU offload") + if runtime.lora_configs or runtime.feature_cache_config.enabled or runtime.compile_config.enabled: + raise NotImplementedError("LoRA, feature caching and compilation are not configured by this pipeline") + if name != "dit_config" and runtime.quant_config.enabled: + raise NotImplementedError("Online quantization is supported on the DiT stage only") + + +class MiniMaxH3ImagePipeline(BasePipeline): + def __init__(self, device: str | torch.device = "cuda", torch_dtype: torch.dtype = torch.bfloat16) -> None: + super().__init__(device=device, torch_dtype=torch_dtype) + self.config: MiniMaxH3ImagePipelineConfig | None = None + self.text_stage: MiniMaxH3ImageTextStage | None = None + self.vae_stage: MiniMaxH3ImageVAEStage | None = None + self.denoising_stage: MiniMaxH3ImageDenoisingStage | ParallelWorker | None = None + + def init(self, module_manager: ModuleManager, config: MiniMaxH3ImagePipelineConfig) -> None: + self.config = config + scheduler = json.loads(Path(config.scheduler_config_path).read_text()) + if scheduler.get("euler_update_form") != "velocity" or scheduler.get("shift_provided_sigmas", False): + raise ValueError("H3-Image requires the released unshifted velocity scheduler") + pdd = json.loads(Path(config.pdd_config_path).read_text()) + if pdd["num_inference_steps"] != len(pdd["sigmas"]) - 1: + raise ValueError("PDD config has inconsistent step count and sigmas") + processor = AutoProcessor.from_pretrained(config.processor_path, local_files_only=True) + self.text_stage = MiniMaxH3ImageTextStage(module_manager, config.text_encoder_config, processor=processor) + self.vae_stage = MiniMaxH3ImageVAEStage(module_manager, config.vae_config) + stage = MiniMaxH3ImageDenoisingStage(module_manager, config.dit_config, pdd["sigmas"]) + if ( + config.text_encoder_config.parallel_config.world_size != 1 + or config.vae_config.parallel_config.world_size != 1 + ): + raise ValueError("H3-Image conditioning and VAE currently require single-device stage configs") + if config.dit_config.parallel_config.world_size > 1 and not dist.is_initialized(): + self.denoising_stage = ParallelWorker(stage) + else: + self.denoising_stage = stage + if dist.is_initialized(): + stage.parallel_models() + self._model_info = module_manager.get_model_info() + + def _get_stages(self) -> list[Any]: + return [stage for stage in (self.text_stage, self.vae_stage, self.denoising_stage) if stage is not None] + + def stop(self) -> None: + if isinstance(self.denoising_stage, ParallelWorker): + self.denoising_stage.close() + + @torch.inference_mode() + def __call__( + self, + prompt: str, + *, + height: int = 2048, + width: int = 2048, + image: Image.Image | list[Image.Image] | None = None, + seed: int = 0, + min_generation_area: int = 2048 * 2048, + ) -> Image.Image: + if any(stage is None for stage in (self.text_stage, self.vae_stage, self.denoising_stage)): + raise RuntimeError("MiniMaxH3ImagePipeline.init must be called before generation") + if not isinstance(prompt, str) or not prompt: + raise ValueError("prompt must be a non-empty string") + if height <= 0 or width <= 0 or not 1024 <= min_generation_area <= 4096 * 4096: + raise ValueError("Positive dimensions and min_generation_area in [1024, 4096**2] are required") + images = [] if image is None else (image if isinstance(image, list) else [image]) + if len(images) > 16: + raise ValueError("H3-Image accepts at most 16 reference images") + refs, vision = [], [] + for item in images: + upright = ImageOps.exif_transpose(item).convert("RGB") + if min(upright.size) < 32: + raise ValueError("Reference images must have at least 32 pixels on each side") + for target, bounds in ((refs, (1024, 2048)), (vision, (1024, 1024))): + size = bound_short_edge(*upright.size, *bounds, 32) + target.append(upright if size == upright.size else upright.resize(size, Image.Resampling.LANCZOS)) + canvas_h, canvas_w = get_image_canvas_size(height, width, min_generation_area, 32) + generator = torch.Generator(device="cpu").manual_seed(seed) + text = self.text_stage.encode_image_prompt(prompt, vision) + encoded = ( + self.vae_stage.encode_references(refs, generator.get_state()) + if refs + else {"latents": [], "rng_state": generator.get_state()} + ) + latent = self.denoising_stage.denoise(text, encoded["latents"], canvas_h, canvas_w, encoded["rng_state"]) + if callable(latent): + latent = latent() + pixels = self.vae_stage.decode_image(latent)[0].permute(1, 2, 0).numpy() + result = Image.fromarray((pixels * 255).clip(0, 255).astype("uint8")) + if (canvas_h, canvas_w) != (height, width): + result = result.resize((width, height), Image.Resampling.LANCZOS) + return result diff --git a/telefuser/pipelines/minimax_h3_image/stages.py b/telefuser/pipelines/minimax_h3_image/stages.py new file mode 100644 index 00000000..9be14c60 --- /dev/null +++ b/telefuser/pipelines/minimax_h3_image/stages.py @@ -0,0 +1,206 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Native H3-Image conditioning, six-head PDD sampling and image decoding.""" + +from __future__ import annotations + +import math +from typing import Any + +import numpy as np +import torch +from PIL import Image + +from telefuser.core.base_stage import BaseStage, with_model_offload +from telefuser.core.config import AttnImplType, ModelRuntimeConfig +from telefuser.core.module_manager import ModuleManager +from telefuser.distributed.device_mesh import create_device_mesh_from_config +from telefuser.pipelines.minimax_h3.packed_tokens import minimax_h3_unpatchify_video_tokens +from telefuser.pipelines.minimax_h3.text_encoding import MiniMaxH3TextEncodingStage + +from .data import build_image_packed_sequence, patchify_video_latents + + +class MiniMaxH3ImageTextStage(MiniMaxH3TextEncodingStage): + @with_model_offload(["text_encoder"]) + @torch.inference_mode() + def encode_image_prompt(self, prompt: str, images: list[Image.Image]) -> dict[str, torch.Tensor]: + ids: list[int] = [] + tags: list[int] = [] + vision: dict[str, Any] = {} + if images: + vision = self.processor.image_processor(images=images, return_tensors="pt") + merge = self.processor.image_processor.merge_size**2 + for i, grid in enumerate(vision["image_grid_thw"]): + label = self.tokenizer(f"Picture {i + 1}: ", add_special_tokens=False)["input_ids"][:4096] + block = ( + [self.tokenizer.convert_tokens_to_ids("<|vision_start|>")] + + [self.tokenizer.convert_tokens_to_ids("<|image_pad|>")] * (int(grid.prod()) // merge) + + [self.tokenizer.convert_tokens_to_ids("<|vision_end|>")] + ) + ids.extend(label + block) + tags.extend([1] * len(label) + [0] * len(block)) + prompt_ids = self.tokenizer(prompt, add_special_tokens=False)["input_ids"][:4096] + ids.extend(prompt_ids) + tags.extend([1] * len(prompt_ids)) + hidden = self.text_encoder.encode_ids( + torch.tensor(ids, dtype=torch.long), + **{k: vision[k] for k in ("pixel_values", "image_grid_thw") if k in vision}, + ) + return {"hidden_states": hidden.cpu(), "token_tags": torch.tensor(tags, dtype=torch.long)} + + +class MiniMaxH3ImageVAEStage(BaseStage): + def __init__(self, module_manager: ModuleManager, config: ModelRuntimeConfig) -> None: + super().__init__("minimax_h3_image_vae", config) + self.vae = module_manager.fetch_module("minimax_h3_image_vae") + if self.vae is None: + raise ValueError("ModuleManager must contain 'minimax_h3_image_vae'") + self.model_names = ["vae"] + + def _stats(self) -> tuple[torch.Tensor, torch.Tensor]: + return tuple( + torch.tensor(v, device=self.device).view(1, -1, 1, 1) + for v in (self.vae.config.latents_mean, self.vae.config.latents_std) + ) + + @with_model_offload(["vae"]) + @torch.inference_mode() + def encode_references(self, images: list[Image.Image], rng_state: torch.Tensor) -> dict[str, Any]: + generator = torch.Generator(device="cpu").set_state(rng_state.cpu()) + mean, std = self._stats() + latents = [] + for image in images: + pixels = torch.from_numpy(np.asarray(image).copy()).permute(2, 0, 1)[None].float() / 255 + pixels = pixels[..., : pixels.shape[-2] // 16 * 16, : pixels.shape[-1] // 16 * 16] + pixel_mean = torch.tensor([0.485, 0.456, 0.406]).view(1, 3, 1, 1) + pixel_std = torch.tensor([0.229, 0.224, 0.225]).view(1, 3, 1, 1) + posterior = self.vae.encode(((pixels - pixel_mean) / pixel_std).to(self.device)).latent_dist + latent = posterior.sample(generator=generator).half().float() + latent = latent[..., : latent.shape[-2] // 2 * 2, : latent.shape[-1] // 2 * 2] + latents.append(((latent - mean) / std).cpu()) + return {"latents": latents, "rng_state": generator.get_state()} + + @with_model_offload(["vae"]) + @torch.inference_mode() + def decode_image(self, latent: torch.Tensor) -> torch.Tensor: + mean, std = self._stats() + pixels = self.vae.decode(latent.to(self.device) * std + mean).sample.float() + pixel_mean = torch.tensor([0.485, 0.456, 0.406], device=self.device).view(1, 3, 1, 1) + pixel_std = torch.tensor([0.229, 0.224, 0.225], device=self.device).view(1, 3, 1, 1) + return (pixels * pixel_std + pixel_mean).clamp(0, 1).cpu() + + +class MiniMaxH3ImageDenoisingStage(BaseStage): + def __init__(self, module_manager: ModuleManager, config: ModelRuntimeConfig, sigmas: list[float]) -> None: + super().__init__("minimax_h3_image_denoising", config) + self.transformer = module_manager.fetch_module("minimax_h3_image_transformer") + if self.transformer is None: + raise ValueError("ModuleManager must contain 'minimax_h3_image_transformer'") + if len(sigmas) != self.transformer.final_layer.video_out.weight.shape[0] + 1: + raise ValueError("The PDD schedule must have one interval per output head") + if sigmas[0] != 1 or sigmas[-1] != 0 or any(a <= b for a, b in zip(sigmas, sigmas[1:])): + raise ValueError("PDD sigmas must decrease strictly from one to zero") + self.timesteps = torch.tensor([1.0 - sigma for sigma in sigmas[:-1]], dtype=torch.float32) + self.sigmas = torch.cat([1 - self.timesteps, torch.zeros(1)]) + self.transformer.set_attention_config(config.attention_config) + self.model_names = ["transformer"] + self._request_serial = 0 + + def parallel_models(self) -> None: + config = self.model_runtime_config.parallel_config + if any(v != 1 for v in (config.dp_degree, config.cfg_degree, config.sp_ring_degree, config.pp_degree)): + raise ValueError("H3-Image supports Ulysses SP and TP; other parallel degrees must be one") + if config.enable_fsdp: + raise NotImplementedError("H3-Image FSDP is not implemented") + mesh = create_device_mesh_from_config(config) + if config.tp_degree > 1: + self.transformer.enable_tp(mesh) + if config.sp_ulysses_degree > 1: + self.transformer.enable_usp(mesh) + # Keep shards on CPU until conditioning has released its device memory. + + @with_model_offload(["transformer"]) + @torch.inference_mode() + def denoise( + self, + text: dict[str, torch.Tensor], + references: list[torch.Tensor], + height: int, + width: int, + rng_state: torch.Tensor, + ) -> torch.Tensor: + quant = self.model_runtime_config.quant_config + if quant.enabled and self.transformer.quant_type != quant.quant_type: + self.transformer.enable_quant(quant) + generator = torch.Generator(device="cpu").set_state(rng_state.cpu()) + references = [reference.cpu() for reference in references] + channels = self.transformer.config.latents_dim + patch = tuple(self.transformer.config.patch_size) + latent = torch.randn((1, channels, 1, height // 16, width // 16), generator=generator) + target = patchify_video_latents(latent, patch).to(self.device) + ref_rows = [] + for ref in references: + noise = torch.randn(ref.shape, generator=generator) + ref_rows.append(patchify_video_latents((0.999 * ref + (1.0 - 0.999) * noise).unsqueeze(2), patch)) + positions, tags, image_ids, text_ids, nref = build_image_packed_sequence( + text["token_tags"].cpu(), [tuple(r.shape[-2:]) for r in references], height // 16, width // 16, patch, 0 + ) + actual = len(tags) + degree = self.model_runtime_config.parallel_config.sp_ulysses_degree + alignment = degree + if self.model_runtime_config.attention_config.attn_impl in { + AttnImplType.SOL_ATTN, + AttnImplType.SAGE_ATTN_2_8_8_SM90, + }: + alignment = math.lcm(64, degree) + padded = (actual + alignment - 1) // alignment * alignment + if alignment > degree and padded - actual >= 64: + raise ValueError("H3 optimized attention requires fewer than 64 padding rows; use a power-of-two SP degree") + positions = torch.nn.functional.pad(positions, (0, 0, 0, padded - actual)).to(self.device) + tags = torch.nn.functional.pad(tags, (0, padded - actual)).to(self.device) + image_ids, text_ids = image_ids.to(self.device), text_ids.to(self.device) + state = torch.zeros((1, padded, target.shape[-1]), device=self.device, dtype=torch.float32) + if ref_rows: + state[0, image_ids[:nref]] = torch.cat(ref_rows).to(self.device) + target_ids = image_ids[nref:] + state[0, target_ids] = target + cu = torch.tensor([0, actual] + ([padded] if padded > actual else []), device=self.device, dtype=torch.int32) + self._request_serial += 1 + try: + for step, timestep in enumerate(self.timesteps): + row_t = torch.full((padded,), float(timestep), dtype=torch.float32) + row_t[image_ids[:nref].cpu()] = 0.999 + unique, inverse = torch.unique(row_t, sorted=True, return_inverse=True) + velocity = self.transformer( + x=state, + img_position_ids=positions[None], + unique_timesteps=unique.to(self.device), + inverse_indices=inverse.to(self.device), + token_tags=tags, + update_mask=torch.ones(len(image_ids), device=self.device, dtype=torch.bool), + prompt_embeds=text["hidden_states"].to(self.device), + img_pos_info={"position_ids": image_ids}, + text_pos_info={"position_ids": text_ids}, + img_pos_for_infer_output_info={"position_ids": image_ids}, + packed_seq_params={"cu_seqlens_q": cu}, + pdd_step_index=step, + sparse_step_index=step, + static_cache_key=self._request_serial, + ) + state[0, target_ids] += (self.sigmas[step] - self.sigmas[step + 1]).to(self.device) * velocity[ + nref: + ].float() + rows = state[0, target_ids].cpu() + finally: + # Prompt/RoPE caches belong to a request, including failed requests. + for name in ( + "_static_cache_key", + "_static_prompt", + "_static_rope_cos_sin", + "_static_sequence_lengths", + "_static_cu_seqlens", + ): + setattr(self.transformer, name, None) + return minimax_h3_unpatchify_video_tokens( + rows, latent_shape=(1, height // 16 // patch[1], width // 16 // patch[2], channels), patch_size=patch + )[:, :, 0] diff --git a/telefuser/pipelines/pan2/__init__.py b/telefuser/pipelines/pan2/__init__.py new file mode 100644 index 00000000..3db4e38b --- /dev/null +++ b/telefuser/pipelines/pan2/__init__.py @@ -0,0 +1,3 @@ +from .pipeline import PAN2Pipeline, PAN2PipelineConfig + +__all__ = ["PAN2Pipeline", "PAN2PipelineConfig"] diff --git a/telefuser/pipelines/pan2/data.py b/telefuser/pipelines/pan2/data.py new file mode 100644 index 00000000..b21d03ce --- /dev/null +++ b/telefuser/pipelines/pan2/data.py @@ -0,0 +1,185 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Adapted from Diffusers 843f1743e259e510f22065c0cb917d417b98d9f5. +import json + +from PIL import Image + +# The negative prompt PAN 2 was released with. Its `time_range_sec` fields are filled in per call to span the whole +# clip, see `get_pan2_default_negative_prompt`. +PAN2_DEFAULT_NEGATIVE_PROMPT = { + "video_style": "low quality, blur, softness, reduced detail, noise, compression, clipped highlights, " + "blurred sidebar, black border, letterboxing, watermark, subtitle, AIGC text, oily waxy " + "plastic texture, oversaturation or color cast, color clipping or chroma bleeding, flicker, " + "jitter, ghosting, motion smear, choppy or laggy motion, slow motion, Ken Burns effect, " + "unmotivated slow zoom over a frozen scene, pan across a still image, global frame scaling " + "or drifting, simulated movement from a static photograph, repeated micro-motion loop, " + "discontinuous shot, scene cut, hard cut, crossfade, fade-in, fade-out, fading transition, " + "scene replacement, spontaneous object or person pop-in or disappearance, duplicated or " + "missing people or props, identity or count drift, rigid-object deformation or morphing, " + "bending or melting rigid structures, structural shape or component drift", + "lighting": "overexposure, clipped highlights, underexposure, harsh lighting, overly deep skin shadows, " + "brightness fading, excessive lighting saturation, chroma-clipped highlights, neon-like skin " + "tones, colored spill overwhelming local material colors, oversaturated bloom, unnatural neon " + "color casts, hue drift, chroma noise, bad white balance, color pulsing, flickering light, " + "unnatural lighting", + "perspective_view": "warped geometry, distorted perspective, inconsistent spatial relationships, " + "unmotivated layout changes, camera teleport or jump, viewpoint reset, wrong depth or " + "occlusion", + "subjects_lifecycle": [ + { + "subject_id": "<|Subject1|>", + "display_name": "malformed inconsistent subject", + "subject_attribute": "broken anatomy, distorted body proportions, reversed pelvis, " + "twisted torso, deformed arms or legs, reversed joints, fused " + "feet or toes, warped or collapsing faces, transparent or " + "floating head and body, missing body parts, detached or " + "disembodied body parts, floating hands or limbs, detached " + "articulated parts, disconnected joints or supports, hands " + "disconnected from wrists or forearms, discontinuous " + "limb-to-torso connections, blurred or fused fingers, missing " + "thumbs, extra or missing fingers, malformed wrists, extra " + "limbs, duplicated hands, mirrored face, identity drift, wrong " + "relative sizes, human-object fusion, unintended body-material " + "fusion, loss of separate material boundaries, opaque white " + "nail polish, gray nail polish, chalky opaque fingernails, " + "artificial acrylic nails", + } + ], + "init_frame_visual_description": "blur or softness, reduced detail, noise or compression artifacts, clipped " + "highlights, blurred sidebar, black border or letterboxing, oily waxy " + "plastic surfaces, oversaturation or color cast, exaggerated local " + "contrast, exposure fading, bad white balance, smeared dense textures, " + "blurred or distorted face, transparent or missing body parts, detached or " + "disembodied body parts, floating hands or limbs, detached articulated " + "parts, disconnected joints or supports, disconnected wrists or forearms, " + "discontinuous limb-to-torso connections, twisted torso, fused or missing " + "fingers, missing thumbs, malformed wrists, duplicated limbs, deformed " + "arms or legs, reversed knees, fused toes, warped geometry, deformed or " + "asymmetrically warped rigid objects, bent rigid structures, incorrect " + "rigid part geometry, topology, proportions, or component count, wrong " + "occlusion, floating or hovering objects, unsupported objects, object or " + "human-object fusion, unintended fusion of distinct surfaces or layers, " + "loss of separate material boundaries, merged objects, hand-object " + "penetration, object-object penetration, penetration, collision, " + "impossible contact, wrong sizes.", + "events": [ + { + "event_index": 0, + "time_range_sec": "0 to 10", + "event_description": "blur, softness, reduced detail, compression artifacts, clipped " + "highlights, flicker, jitter, ghosting, motion smear, choppy or laggy " + "motion, slow motion, Ken Burns effect, unmotivated slow zoom over a " + "frozen scene, pan across a still image, global frame scaling or drifting, " + "simulated movement from a static photograph, repeated micro-motion loop, " + "oily oversaturated or color-pulsing surfaces, shimmering surface detail, " + "fingers fusing or changing count, hands smearing, facial features " + "merging, reversed joints, upper body rotating independently of attached " + "hips and feet, corkscrewed waist, limbs changing length or attachment " + "after occlusion, transparent body or head, detached or disembodied body " + "parts, floating hands or limbs, detached articulated parts, disconnected " + "joints or supports, hands disconnected from wrists or forearms, body " + "parts moving independently, vanishing limb-to-torso connections, merged " + "people or limbs, limbs fused into torso, merged or fused feet and toes, " + "floating or hovering objects, unsupported objects, object or human-object " + "fusion, unintended fusion of distinct surfaces or layers, loss of " + "separate material boundaries, merged objects, hand-object penetration, " + "object-object penetration, penetration or clipping, objects colliding or " + "morphing, rigid objects tearing, fragmenting, fusing, bending, melting or " + "changing dimensions, topology, silhouette, component count or layout, " + "subjects changing identity, splitting or duplicating, objects appearing " + "or disappearing without entry, exit or occlusion, changed identity or " + "count after occlusion, state changes inconsistent with the causing " + "action, broken physics, uncaused deformation or motion, state changes " + "preceding the causing interaction, loss of coupling during contact, pose " + "jumps, discontinuous shot, camera jump, hard scene cut, crossfade, " + "fade-in, fade-out, fading transition, scene replacement or viewpoint " + "reset.", + "camera_motion": [ + {"time_range_sec": "0 to 10", "camera_motion": ["camera_jump", "hard_cut", "viewpoint_reset"]} + ], + "contained_actions": [ + { + "time_range_sec": "0 to 10", + "action": "<|Subject1|> fingers fusing during grip changes, palms " + "merging with props, wrists collapsing, crossed limbs fusing, " + "intersecting or swapping attachment, duplicated hands or " + "feet, detached or disembodied hands or limbs, hands " + "disconnected from wrists or forearms, discontinuous " + "limb-to-torso connections, merged or fused feet and toes, " + "unstable contact, unintended grip slip, hand-object " + "penetration, body-object penetration, passage through solid " + "surfaces instead of openings, object-object penetration, " + "merged or fused objects, rigid objects fragmenting, fusing, " + "smearing or changing dimensions during motion, unintended " + "fusion of distinct surfaces or layers, loss of separate " + "material boundaries, floating or hovering objects, " + "unsupported objects, objects duplicating, splitting or " + "disappearing during interaction or occlusion, unchanged " + "action result, pose jump", + } + ], + } + ], + "quality": { + "video_quality": "low resolution, blur, softness, reduced detail, noise, compression, clipped " + "highlights, blurred sidebar, black border, letterboxing, oily waxy plastic " + "surfaces, greasy highlights, oversaturated colors, faded or washed-out " + "colors, exposure fading, smeared faces, hands or held objects, surface detail " + "smearing or swimming, material or existing markings changing without a " + "physical cause, color or shape morphing, object pop-in or pop-out, sudden " + "disappearance, missing persistent objects, object duplication or count drift, " + "rigid-object warping or deformation, changing rigid proportions, topology, " + "silhouette, or component layout", + "text_logos_overlays": "watermark, subtitle, captions, AIGC text or logo", + "artifacts": [ + "watermark", + "template_or_text_dominated", + "talking_head", + "visual_quality_issue", + "edited_or_discontinuous", + "video_game", + "animation", + "ai_generated_content", + "black_border", + "ken_burn", + "time_lapse", + "blurred_sidebar", + "hard_scene_cuts", + ], + }, +} + + +def get_pan2_default_negative_prompt(num_frames: int, fps: float) -> str: + r"""PAN 2's default negative prompt as compact one-line JSON, with every time range spanning the clip.""" + span = f"0 to {(num_frames - 1) / fps:g}" + + def fill(node): + if isinstance(node, dict): + return {key: span if key == "time_range_sec" else fill(value) for key, value in node.items()} + if isinstance(node, list): + return [fill(item) for item in node] + return node + + return json.dumps(fill(PAN2_DEFAULT_NEGATIVE_PROMPT), ensure_ascii=False, separators=(",", ":")) + + +def resize_and_center_crop(image: Image.Image, height: int, width: int) -> Image.Image: + """Scale to cover the canvas, round the size, and center crop as in training.""" + scale = max(width / image.width, height / image.height) + image = image.resize((round(image.width * scale), round(image.height * scale)), resample=Image.Resampling.LANCZOS) + left, top = round((image.width - width) / 2), round((image.height - height) / 2) + return image.crop((left, top, left + width, top + height)) diff --git a/telefuser/pipelines/pan2/pipeline.py b/telefuser/pipelines/pan2/pipeline.py new file mode 100644 index 00000000..9770a16a --- /dev/null +++ b/telefuser/pipelines/pan2/pipeline.py @@ -0,0 +1,119 @@ +# SPDX-License-Identifier: Apache-2.0 +"""TeleFuser PAN2 text-to-video and first-frame image-to-video pipeline.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +import torch +import torch.distributed as dist +from PIL import Image +from transformers import AutoTokenizer + +from telefuser.core.base_pipeline import BasePipeline +from telefuser.core.config import ModelRuntimeConfig, WeightOffloadType +from telefuser.core.module_manager import ModuleManager +from telefuser.worker import ParallelWorker + +from .data import get_pan2_default_negative_prompt +from .stages import PAN2DenoisingStage, PAN2TextStage, PAN2VAEStage + + +@dataclass +class PAN2PipelineConfig: + tokenizer_path: str + scheduler_config_path: str + text_encoder_config: ModelRuntimeConfig = field(default_factory=ModelRuntimeConfig) + dit_config: ModelRuntimeConfig = field(default_factory=ModelRuntimeConfig) + vae_config: ModelRuntimeConfig = field(default_factory=ModelRuntimeConfig) + + def __post_init__(self) -> None: + for name in ("text_encoder_config", "dit_config", "vae_config"): + runtime = getattr(self, name) + runtime.parallel_config.validate() + if runtime.parallel_config.enable_fsdp: + raise NotImplementedError("This pipeline supports Ulysses SP and TP, not FSDP") + if runtime.offload_config.offload_type not in { + WeightOffloadType.NO_CPU_OFFLOAD, + WeightOffloadType.MODEL_CPU_OFFLOAD, + }: + raise NotImplementedError("This pipeline supports no offload or model CPU offload") + if runtime.lora_configs or runtime.feature_cache_config.enabled or runtime.compile_config.enabled: + raise NotImplementedError("LoRA, feature caching and compilation are not configured by this pipeline") + if name != "dit_config" and runtime.quant_config.enabled: + raise NotImplementedError("Online quantization is supported on the DiT stage only") + + +class PAN2Pipeline(BasePipeline): + def __init__(self, device: str | torch.device = "cuda", torch_dtype: torch.dtype = torch.bfloat16) -> None: + super().__init__(device=device, torch_dtype=torch_dtype) + self.config: PAN2PipelineConfig | None = None + self.text_stage: PAN2TextStage | None = None + self.vae_stage: PAN2VAEStage | None = None + self.denoising_stage: PAN2DenoisingStage | ParallelWorker | None = None + + def init(self, module_manager: ModuleManager, config: PAN2PipelineConfig) -> None: + self.config = config + if ( + config.text_encoder_config.parallel_config.world_size != 1 + or config.vae_config.parallel_config.world_size != 1 + ): + raise ValueError("PAN2 conditioning and VAE currently require single-device stage configs") + tokenizer = AutoTokenizer.from_pretrained(config.tokenizer_path, local_files_only=True) + self.text_stage = PAN2TextStage(module_manager, config.text_encoder_config, tokenizer) + self.vae_stage = PAN2VAEStage(module_manager, config.vae_config) + stage = PAN2DenoisingStage(module_manager, config.dit_config, config.scheduler_config_path) + if config.dit_config.parallel_config.world_size > 1 and not dist.is_initialized(): + self.denoising_stage = ParallelWorker(stage) + else: + self.denoising_stage = stage + if dist.is_initialized(): + stage.parallel_models() + self._model_info = module_manager.get_model_info() + + def _get_stages(self) -> list[Any]: + return [stage for stage in (self.text_stage, self.vae_stage, self.denoising_stage) if stage is not None] + + def stop(self) -> None: + if isinstance(self.denoising_stage, ParallelWorker): + self.denoising_stage.close() + + @torch.inference_mode() + def __call__( + self, + prompt: str, + *, + height: int = 704, + width: int = 1248, + num_frames: int = 121, + num_inference_steps: int = 50, + guidance_scale: float = 2.5, + seed: int = 0, + negative_prompt: str | None = None, + image: Image.Image | None = None, + fps: float = 24.0, + max_sequence_length: int = 1536, + ) -> torch.Tensor: + if any(stage is None for stage in (self.text_stage, self.vae_stage, self.denoising_stage)): + raise RuntimeError("PAN2Pipeline.init must be called before generation") + if not isinstance(prompt, str) or (negative_prompt is not None and not isinstance(negative_prompt, str)): + raise ValueError("prompt and negative_prompt must be strings") + if height <= 0 or width <= 0 or height % 32 or width % 32: + raise ValueError("PAN2 height and width must be positive multiples of 32") + if num_frames < 1 or (num_frames - 1) % 4: + raise ValueError("PAN2 num_frames must be 4*k+1") + if num_inference_steps < 1 or fps <= 0 or max_sequence_length < 1 or guidance_scale < 0: + raise ValueError("Invalid steps, fps, text length or guidance scale") + negative = ( + (negative_prompt or get_pan2_default_negative_prompt(num_frames, fps)) if guidance_scale > 1 else None + ) + text, uncond = self.text_stage.encode(prompt, negative, max_sequence_length) + first = self.vae_stage.encode_image(image, height, width) if image is not None else None + latent = self.denoising_stage.denoise( + text, uncond, first, height, width, num_frames, num_inference_steps, guidance_scale, seed + ) + if callable(latent): + latent = latent() + # Follow the existing video example convention: RGB frames in [T,H,W,C], range [0,1]. + return self.vae_stage.decode_video(latent)[0].permute(1, 2, 3, 0) diff --git a/telefuser/pipelines/pan2/stages.py b/telefuser/pipelines/pan2/stages.py new file mode 100644 index 00000000..1ef2de6f --- /dev/null +++ b/telefuser/pipelines/pan2/stages.py @@ -0,0 +1,162 @@ +# SPDX-License-Identifier: Apache-2.0 +"""PAN2 text conditioning, flow matching and Wan2.2 decoding stages.""" + +from __future__ import annotations + +from pathlib import Path + +import numpy as np +import torch +from PIL import Image +from diffusers import FlowMatchEulerDiscreteScheduler + +from telefuser.core.base_stage import BaseStage, with_model_offload +from telefuser.core.config import ModelRuntimeConfig +from telefuser.core.module_manager import ModuleManager +from telefuser.distributed.device_mesh import create_device_mesh_from_config + +from .data import resize_and_center_crop + + +class PAN2TextStage(BaseStage): + def __init__(self, manager: ModuleManager, config: ModelRuntimeConfig, tokenizer: object) -> None: + super().__init__("pan2_text_encoding", config) + self.text_encoder = manager.fetch_module("pan2_text_encoder") + if self.text_encoder is None: + raise ValueError("ModuleManager must contain 'pan2_text_encoder'") + self.tokenizer = tokenizer + self.model_names = ["text_encoder"] + + def _encode(self, prompt: str, max_sequence_length: int) -> torch.Tensor: + messages = [ + { + "role": "system", + "content": "You are a helpful assistant. Generate the video with the following instructions.", + }, + {"role": "user", "content": prompt or " "}, + ] + header = self.tokenizer("<|im_start|>user\n", add_special_tokens=False).input_ids + ids = self.tokenizer.apply_chat_template( + messages, add_generation_prompt=True, tokenize=True, return_dict=True + ).input_ids + starts = [i + len(header) for i in range(len(ids)) if ids[i : i + len(header)] == header] + if not starts: + raise ValueError("PAN2 tokenizer chat template did not contain the user header") + start = starts[0] + ids = torch.tensor([ids[: start + max_sequence_length]], device=self.device) + return self.text_encoder(ids)[:, start:].cpu() + + @with_model_offload(["text_encoder"]) + @torch.inference_mode() + def encode( + self, prompt: str, negative_prompt: str | None, max_sequence_length: int + ) -> tuple[torch.Tensor, torch.Tensor | None]: + return ( + self._encode(prompt, max_sequence_length), + self._encode(negative_prompt, max_sequence_length) if negative_prompt is not None else None, + ) + + +class PAN2VAEStage(BaseStage): + def __init__(self, manager: ModuleManager, config: ModelRuntimeConfig) -> None: + super().__init__("pan2_vae", config) + self.vae = manager.fetch_module("pan2_vae") + if self.vae is None: + raise ValueError("ModuleManager must contain 'pan2_vae'") + self.model_names = ["vae"] + + def _stats(self, dtype: torch.dtype) -> tuple[torch.Tensor, torch.Tensor]: + return tuple( + torch.tensor(v, device=self.device, dtype=dtype).view(1, -1, 1, 1, 1) + for v in (self.vae.config.latents_mean, self.vae.config.latents_std) + ) + + @with_model_offload(["vae"]) + @torch.inference_mode() + def encode_image(self, image: Image.Image, height: int, width: int) -> torch.Tensor: + image = resize_and_center_crop(image.convert("RGB"), height, width) + pixels = torch.from_numpy(np.asarray(image).copy()).permute(2, 0, 1)[None, :, None] + pixels = (pixels.float() / 255 * 2 - 1).to(device=self.device, dtype=self.torch_dtype) + latents = self.vae.encode(pixels) + mean, std = self._stats(latents.dtype) + return ((latents - mean) / std).cpu() + + @with_model_offload(["vae"]) + @torch.inference_mode() + def decode_video(self, latents: torch.Tensor) -> torch.Tensor: + mean, std = self._stats(torch.float32) + latents = (latents.to(self.device, dtype=torch.float32) * std + mean).to(self.torch_dtype) + pixels = self.vae.decode(latents) + return (pixels * 0.5 + 0.5).clamp(0, 1).float().cpu() + + +class PAN2DenoisingStage(BaseStage): + def __init__(self, manager: ModuleManager, config: ModelRuntimeConfig, scheduler_path: str) -> None: + super().__init__("pan2_denoising", config) + self.transformer = manager.fetch_module("pan2_transformer") + if self.transformer is None: + raise ValueError("ModuleManager must contain 'pan2_transformer'") + self.transformer.set_attention_config(config.attention_config) + self.scheduler_config = FlowMatchEulerDiscreteScheduler.load_config(Path(scheduler_path).parent) + self.model_names = ["transformer"] + + def parallel_models(self) -> None: + config = self.model_runtime_config.parallel_config + if any(v != 1 for v in (config.dp_degree, config.cfg_degree, config.sp_ring_degree, config.pp_degree)): + raise ValueError("PAN2 supports Ulysses SP and TP; other parallel degrees must be one") + if config.enable_fsdp: + raise NotImplementedError("PAN2 FSDP is not implemented") + mesh = create_device_mesh_from_config(config) + if config.tp_degree > 1: + self.transformer.enable_tp(mesh) + if config.sp_ulysses_degree > 1: + self.transformer.enable_usp(mesh) + + @with_model_offload(["transformer"]) + @torch.inference_mode() + def denoise( + self, + text: torch.Tensor, + negative: torch.Tensor | None, + image: torch.Tensor | None, + height: int, + width: int, + num_frames: int, + steps: int, + guidance: float, + seed: int, + ) -> torch.Tensor: + quant = self.model_runtime_config.quant_config + if quant.enabled and self.transformer.quant_type != quant.quant_type: + self.transformer.enable_quant(quant) + scheduler = FlowMatchEulerDiscreteScheduler.from_config(self.scheduler_config) + scheduler.set_timesteps(sigmas=np.linspace(1.0, 0.0, steps + 1)[:-1], device=self.device) + generator = torch.Generator(device="cpu").manual_seed(seed) + channels = self.transformer.config.out_channels + latent = torch.randn( + (1, channels, (num_frames - 1) // 4 + 1, height // 16, width // 16), generator=generator + ).to(self.device, dtype=torch.float32) + condition = latent.new_zeros((1, channels + 1, *latent.shape[2:])) + if image is not None: + condition[:, :channels, :1] = image.to(condition) + condition[:, channels:, :1] = 1 + text = text.to(self.device, dtype=self.torch_dtype) + negative = negative.to(self.device, dtype=self.torch_dtype) if negative is not None else None + for i, timestep in enumerate(scheduler.timesteps): + inputs = torch.cat([latent, condition], dim=1).to(self.torch_dtype) + pred = self.transformer( + hidden_states=inputs, timestep=timestep.expand(1), encoder_hidden_states=text, sparse_step_index=i + ).sample + if negative is not None: + uncond = self.transformer( + hidden_states=inputs, + timestep=timestep.expand(1), + encoder_hidden_states=negative, + sparse_step_index=i, + ).sample + guided = uncond + guidance * (pred - uncond) + if image is not None: + guided[:, :, :1] = pred[:, :, :1] + pred = guided + latent = scheduler.step(pred.float(), timestep, latent, return_dict=False)[0] + return latent.cpu() diff --git a/tests/unit/models/test_h3_image_pan2_conditioning.py b/tests/unit/models/test_h3_image_pan2_conditioning.py new file mode 100644 index 00000000..33a0c33f --- /dev/null +++ b/tests/unit/models/test_h3_image_pan2_conditioning.py @@ -0,0 +1,25 @@ +"""Retain non-persistent rotary buffers exactly across runtime dtype/device casts.""" + +import pytest +import torch + +try: + from telefuser.core.base_model import BaseModel + from telefuser.models.minimax_h3_image_encoder import MiniMaxH3ImageEncoder + from telefuser.models.pan2_conditioning import PAN2TextEncoder +except ImportError as exc: + pytest.skip(f"Optional conditioning dependencies unavailable: {exc}", allow_module_level=True) + + +@pytest.mark.parametrize("model_class", [MiniMaxH3ImageEncoder, PAN2TextEncoder]) +def test_conditioner_cast_preserves_rotary_frequencies(model_class) -> None: + model = model_class.__new__(model_class) + BaseModel.__init__(model) + model.model = torch.nn.Linear(4, 4) + frequency = torch.tensor([0.123456789, 0.000123456789], dtype=torch.float32) + model.model.register_buffer("inv_freq", frequency.clone(), persistent=False) + model.model.register_buffer("original_inv_freq", frequency.clone(), persistent=False) + model.to(dtype=torch.bfloat16) + assert model.model.weight.dtype == torch.bfloat16 + torch.testing.assert_close(model.model.inv_freq, frequency, rtol=0, atol=0) + torch.testing.assert_close(model.model.original_inv_freq, frequency, rtol=0, atol=0) diff --git a/tests/unit/models/test_minimax_h3_image_dit.py b/tests/unit/models/test_minimax_h3_image_dit.py new file mode 100644 index 00000000..f4695664 --- /dev/null +++ b/tests/unit/models/test_minimax_h3_image_dit.py @@ -0,0 +1,180 @@ +import json +from dataclasses import asdict + +import pytest +import torch +from torch.nn import functional as F + +try: + from telefuser.models.minimax_h3_dit import MiniMaxH3DiTConfig + from telefuser.models.minimax_h3_image_dit import MiniMaxH3ImageDiT, MiniMaxH3ImagePDDProjection +except ImportError as exc: + pytest.skip(f"Optional attention dependencies are unavailable: {exc}", allow_module_level=True) + + +def small_config() -> MiniMaxH3DiTConfig: + return MiniMaxH3DiTConfig( + hidden_size=32, + num_layers=2, + token_refiner_num_layers=1, + num_attention_heads=4, + attention_head_dim=8, + ffn_hidden_size=64, + latents_dim=2, + text_dim=16, + timestep_input_dim=8, + time_embed_hidden_size=32, + time_embed_dim=16, + rope_inv_freq_len=1, + ) + + +def packed_image_inputs() -> dict: + return { + "x": torch.randn(1, 6, 8), + "img_position_ids": torch.zeros(1, 6, 3, dtype=torch.float64), + "unique_timesteps": torch.tensor([0.5]), + "inverse_indices": torch.zeros(6, dtype=torch.long), + "update_mask": torch.ones(4, dtype=torch.bool), + "token_tags": torch.tensor([1, 1, 0, 0, 0, 0]), + "prompt_embeds": torch.randn(2, 16), + "img_pos_info": {"position_ids": torch.arange(2, 6)}, + "text_pos_info": {"position_ids": torch.arange(2)}, + "img_pos_for_infer_output_info": {"position_ids": torch.arange(2, 6)}, + "packed_seq_params": {"cu_seqlens_q": torch.tensor([0, 6], dtype=torch.int32)}, + } + + +def test_image_checkpoint_has_no_audio_parameters() -> None: + with torch.device("meta"): + model = MiniMaxH3ImageDiT() + state = model.state_dict() + assert not any("audio" in key for key in state) + assert state["final_layer.video_out.weight"].shape == (6, 96, 5376) + assert state["final_layer.video_out.weight"].dtype == torch.float32 + assert state["blocks.0.adaln_proj.linear.weight"].shape[0] == 18 * 5376 + + +def test_pdd_step_selects_its_own_released_projection() -> None: + projection = MiniMaxH3ImagePDDProjection(8, 4, 6) + inputs = torch.randn(5, 8) + for index in range(6): + projection.step_index = index + torch.testing.assert_close( + projection(inputs), F.linear(inputs, projection.weight[index], projection.bias[index]) + ) + + +def test_image_forward_has_no_audio_requirement_and_uses_pdd_head() -> None: + torch.manual_seed(3) + model = MiniMaxH3ImageDiT(small_config()).eval() + inputs = packed_image_inputs() + with torch.inference_mode(): + first = model(pdd_step_index=0, **inputs) + last = model(pdd_step_index=5, **inputs) + assert first.shape == (4, 8) + assert torch.isfinite(first).all() + assert not torch.equal(first, last) + with pytest.raises(ValueError, match="PDD step index"): + model(pdd_step_index=6, **inputs) + with pytest.raises(ValueError, match="does not accept audio"): + model(pdd_step_index=0, audio_x=torch.ones(1, 1, 32), **inputs) + + +def test_official_dit_and_fused_heads_load_strictly(tmp_path) -> None: + config = small_config() + model = MiniMaxH3ImageDiT(config) + state = dict(model.state_dict()) + state["video_out.weight"] = state.pop("final_layer.video_out.weight") + state["video_out.bias"] = state.pop("final_layer.video_out.bias") + path = tmp_path / "config.json" + path.write_text(json.dumps(asdict(config))) + converted, kwargs = model.state_dict_converter(path).from_official(state) + clone = MiniMaxH3ImageDiT(**kwargs) + clone.load_state_dict(converted, strict=True) + torch.testing.assert_close(clone.final_layer.video_out.weight, model.final_layer.video_out.weight) + with pytest.raises(ValueError, match="pdd_fused_heads"): + model.state_dict_converter(path).from_official( + {k: v for k, v in state.items() if not k.startswith("video_out")} + ) + + +def test_complete_six_head_edit_sampler_matches_independent_reference(tmp_path) -> None: + try: + from diffusers import MiniMaxH3Scheduler + from diffusers.models.transformers.transformer_minimax_h3_image import MiniMaxH3ImageTransformer2DModel + from diffusers.modular_pipelines.minimax_h3_image.before_denoise import ( + build_image_packed_sequence, + patchify_video_latents, + ) + except ImportError: + pytest.skip("Independent image sampling reference requires recent Diffusers") + from telefuser.core.config import ModelRuntimeConfig + from telefuser.pipelines.minimax_h3_image.stages import MiniMaxH3ImageDenoisingStage + + torch.manual_seed(73) + config = small_config() + reference = MiniMaxH3ImageTransformer2DModel( + hidden_size=32, + num_layers=2, + num_refiner_layers=1, + num_attention_heads=4, + attention_head_dim=8, + ffn_dim=64, + in_channels=2, + text_dim=16, + freq_dim=8, + time_embed_hidden_dim=32, + time_embed_dim=16, + rope_freq_dim=1, + ).eval() + preserved = { + k: v.clone() + for k, v in reference.state_dict().items() + if k.startswith(("proj_in.", "time_embedder.", "pdd_proj_out.")) + } + reference.to(dtype=torch.bfloat16) + for name, value in preserved.items(): + reference.get_parameter(name).data = value.float() + config_path = tmp_path / "config.json" + config_path.write_text(json.dumps(asdict(config))) + model = MiniMaxH3ImageDiT(config).eval() + converted, _ = model.state_dict_converter(config_path).from_diffusers(reference.state_dict()) + model.load_state_dict(converted, strict=True) + sigmas = [1.0, 0.9675236940383911, 0.8875485062599182, 0.75, 0.5327739119529724, 0.23200887441635132, 0.0] + + class Manager: + def fetch_module(self, name: str) -> MiniMaxH3ImageDiT: + return model + + stage = MiniMaxH3ImageDenoisingStage(Manager(), ModelRuntimeConfig(device_type="cpu"), sigmas) + text = {"hidden_states": torch.randn(2, 16, dtype=torch.bfloat16), "token_tags": torch.ones(2, dtype=torch.long)} + image = torch.randn(1, 2, 4, 4) + actual = stage.denoise(text, [image], 64, 64, torch.Generator().manual_seed(71).get_state()) + generator = torch.Generator().manual_seed(71) + target = patchify_video_latents(torch.randn(1, 2, 1, 4, 4, generator=generator), (1, 2, 2)) + conditioned = 0.999 * image + (1.0 - 0.999) * torch.randn(image.shape, generator=generator) + rows = torch.cat([patchify_video_latents(conditioned.unsqueeze(2), (1, 2, 2)), target]) + pos, tags, image_ids, text_ids, nref = build_image_packed_sequence(text["token_tags"], [(4, 4)], 4, 4, (1, 2, 2), 0) + timesteps = torch.tensor([1.0 - sigma for sigma in sigmas[:-1]], dtype=torch.float32) + scheduler = MiniMaxH3Scheduler(euler_update_form="velocity", shift_provided_sigmas=False) + scheduler.set_timesteps(sigmas=torch.cat([1.0 - timesteps, torch.zeros(1)])) + with torch.inference_mode(): + for step, timestep in enumerate(scheduler.timesteps): + row_t = torch.full((len(tags),), float(timestep)) + row_t[image_ids[:nref]] = 0.999 + unique, inverse = torch.unique(row_t, sorted=True, return_inverse=True) + velocity = reference( + hidden_states=rows[None], + encoder_hidden_states=text["hidden_states"][None], + timestep=unique, + timestep_indices=inverse, + token_tags=tags, + position_ids=pos, + image_indices=image_ids, + text_indices=text_ids, + pdd_step_index=step, + ).sample[0] + rows[nref:] = scheduler.step(velocity[nref:], timestep, rows[nref:], return_dict=False)[0] + expected = rows[nref:].reshape(1, 1, 2, 2, 2, 1, 2, 2).permute(0, 4, 1, 5, 2, 6, 3, 7).reshape(1, 2, 1, 4, 4) + torch.testing.assert_close(actual, expected[:, :, 0], rtol=0, atol=0) diff --git a/tests/unit/models/test_minimax_h3_image_vae.py b/tests/unit/models/test_minimax_h3_image_vae.py new file mode 100644 index 00000000..82c23d38 --- /dev/null +++ b/tests/unit/models/test_minimax_h3_image_vae.py @@ -0,0 +1,68 @@ +"""Image VAE numerical equivalence and cast-sensitive encoder regression.""" + +import pytest +import torch + +try: + from telefuser.models.minimax_h3_image_vae import MiniMaxH3ImageVAE +except ImportError as exc: + pytest.skip(f"Optional model dependencies unavailable: {exc}", allow_module_level=True) + + +def tiny_vae() -> MiniMaxH3ImageVAE: + return MiniMaxH3ImageVAE( + latent_channels=2, + block_out_channels=(8, 8), + layers_per_block=1, + spatial_downsample_factors=(2, 2), + temporal_downsample_factors=(1, 1), + norm_num_groups=4, + decoder_num_layers=1, + decoder_num_attention_heads=2, + decoder_attention_head_dim=8, + decoder_num_register_tokens=2, + decoder_ffn_mult=2, + latents_mean=(0.0, 0.0), + latents_std=(1.0, 1.0), + ) + + +def test_cast_preserves_encoder_and_decoder_norms_exactly() -> None: + model = tiny_vae() + original = model.encoder.conv_in.weight.detach().clone() + original_rope = model.decoder.rope.inv_freq.clone() + model.to(dtype=torch.bfloat16) + torch.testing.assert_close(model.encoder.conv_in.weight, original, rtol=0, atol=0) + assert model.decoder.proj_in.weight.dtype == torch.bfloat16 + assert model.decoder.rope.inv_freq.dtype == torch.float32 + torch.testing.assert_close(model.decoder.rope.inv_freq, original_rope, rtol=0, atol=0) + assert all(p.dtype == torch.float32 for n, p in model.named_parameters() if ".norm1." in n or ".scale1." in n) + + +def test_reference_encoder_decoder_and_tiling() -> None: + try: + from diffusers.models.autoencoders.autoencoder_kl_minimax_h3_image import AutoencoderKLMiniMaxH3Image + except ImportError: + pytest.skip("Independent reference requires Diffusers with H3-Image support") + model = tiny_vae().eval() + reference = AutoencoderKLMiniMaxH3Image.from_config(dict(model.config)).eval() + reference.load_state_dict(model.state_dict(), strict=True) + pixels, latent = torch.randn(1, 3, 32, 32), torch.randn(1, 2, 8, 8) + # Force multiple overlapping tiles to test stitching, not only the untiled path. + model.enable_tiling( + tile_sample_min_height=24, + tile_sample_min_width=24, + tile_sample_min_overlap_height=8, + tile_sample_min_overlap_width=8, + ) + reference.enable_tiling( + tile_sample_min_height=24, + tile_sample_min_width=24, + tile_sample_min_overlap_height=8, + tile_sample_min_overlap_width=8, + ) + with torch.inference_mode(): + torch.testing.assert_close( + model.encode(pixels).latent_dist.parameters, reference.encode(pixels).latent_dist.parameters, rtol=0, atol=0 + ) + torch.testing.assert_close(model.decode(latent).sample, reference.decode(latent).sample, rtol=0, atol=0) diff --git a/tests/unit/models/test_pan2_dit.py b/tests/unit/models/test_pan2_dit.py new file mode 100644 index 00000000..6255ef6b --- /dev/null +++ b/tests/unit/models/test_pan2_dit.py @@ -0,0 +1,73 @@ +"""PAN2 architecture, checkpoint and precision contracts.""" + +import json + +import pytest +import torch + +try: + from telefuser.models.pan2_dit import PAN2DiT +except ImportError as exc: + pytest.skip(f"Optional attention dependencies unavailable: {exc}", allow_module_level=True) + + +def tiny_model() -> PAN2DiT: + return PAN2DiT( + in_channels=5, + out_channels=2, + num_attention_heads=2, + attention_head_dim=16, + num_layers=1, + num_refiner_layers=1, + text_embed_dim=16, + rope_axes_dim=(4, 6, 6), + mlp_ratio=2, + context_mlp_ratio=2, + refiner_mlp_ratio=2, + ) + + +def test_released_pan2_meta_shapes() -> None: + with torch.device("meta"): + model = PAN2DiT() + assert len(model.transformer_blocks) == 44 + assert len(model.context_refiner) == 4 + assert model.x_embedder.weight.shape == (4096, 97, 1, 2, 2) + assert model.proj_out.weight.shape == (192, 4096) + + +def test_precision_does_not_round_modulation_weights() -> None: + model = tiny_model() + original = model.transformer_blocks[0].norm1.linear.weight.detach().clone() + model.to(dtype=torch.bfloat16) + assert model.x_embedder.weight.dtype == torch.bfloat16 + torch.testing.assert_close(model.transformer_blocks[0].norm1.linear.weight, original, rtol=0, atol=0) + assert model.time_embedder.linear_1.weight.dtype == torch.float32 + + +def test_tiny_pan2_reference_when_installed() -> None: + try: + from diffusers.models.transformers.transformer_pan2 import PAN2Transformer3DModel + except ImportError: + pytest.skip("Independent reference requires Diffusers with PAN2 support") + torch.manual_seed(81) + model = tiny_model().eval() + reference = PAN2Transformer3DModel.from_config(dict(model.config)).eval() + reference.load_state_dict(model.state_dict(), strict=True) + args = dict( + hidden_states=torch.randn(1, 5, 2, 6, 6), + timestep=torch.tensor([500.0]), + encoder_hidden_states=torch.randn(1, 5, 16), + ) + with torch.inference_mode(): + torch.testing.assert_close(model(**args).sample, reference(**args).sample, rtol=0, atol=0) + + +def test_converter_uses_checkpoint_architecture(tmp_path) -> None: + model = tiny_model() + path = tmp_path / "config.json" + path.write_text(json.dumps(dict(model.config))) + weights, kwargs = model.state_dict_converter(path).from_diffusers(model.state_dict()) + loaded = PAN2DiT(**kwargs) + loaded.load_state_dict(weights, strict=True) + assert loaded.config.out_channels == 2 diff --git a/tests/unit/pipelines/test_h3_image_pan2_sampling.py b/tests/unit/pipelines/test_h3_image_pan2_sampling.py new file mode 100644 index 00000000..3733291a --- /dev/null +++ b/tests/unit/pipelines/test_h3_image_pan2_sampling.py @@ -0,0 +1,167 @@ +"""Sampling contracts independent of expensive model weights and CUDA kernels.""" + +import json +from types import SimpleNamespace + +import pytest +import torch + +try: + from telefuser.core.config import AttentionConfig, ModelRuntimeConfig + from telefuser.pipelines.minimax_h3_image.data import build_image_packed_sequence, get_image_canvas_size + from telefuser.pipelines.minimax_h3_image.stages import MiniMaxH3ImageDenoisingStage + from telefuser.pipelines.pan2.data import get_pan2_default_negative_prompt + from telefuser.pipelines.pan2.stages import PAN2DenoisingStage +except ImportError as exc: + pytest.skip(f"Optional model dependencies unavailable: {exc}", allow_module_level=True) + + +class Manager: + def __init__(self, model: torch.nn.Module) -> None: + self.model = model + + def fetch_module(self, name: str) -> torch.nn.Module: + return self.model + + +class ConstantImageVelocity(torch.nn.Module): + def __init__(self) -> None: + super().__init__() + self.config = SimpleNamespace(latents_dim=2, patch_size=(1, 2, 2)) + self.final_layer = SimpleNamespace(video_out=SimpleNamespace(weight=torch.empty(6, 8, 1))) + self.inputs = [] + + def set_attention_config(self, config: object) -> None: + pass + + def forward(self, **kwargs: object) -> torch.Tensor: + self.inputs.append({k: v.clone() if isinstance(v, torch.Tensor) else v for k, v in kwargs.items()}) + count = len(kwargs["img_pos_info"]["position_ids"]) + return torch.full((count, 8), 2.0) + + +def test_h3_velocity_direction_fixed_heads_and_reference_freeze() -> None: + model = ConstantImageVelocity() + stage = MiniMaxH3ImageDenoisingStage( + Manager(model), ModelRuntimeConfig(device_type="cpu"), [1.0, 0.96, 0.88, 0.75, 0.53, 0.23, 0.0] + ) + text = {"hidden_states": torch.randn(3, 4), "token_tags": torch.ones(3, dtype=torch.long)} + reference = torch.randn(1, 2, 4, 4) + generator = torch.Generator().manual_seed(31) + initial = torch.randn((1, 2, 1, 4, 4), generator=generator) + output = stage.denoise(text, [reference], 64, 64, torch.Generator().manual_seed(31).get_state()) + torch.testing.assert_close(output, initial[:, :, 0] + 2.0) + assert [x["pdd_step_index"] for x in model.inputs] == list(range(6)) + ref_ids = model.inputs[0]["img_pos_info"]["position_ids"][:4] + frozen = model.inputs[0]["x"][0, ref_ids] + for call in model.inputs: + torch.testing.assert_close(call["x"][0, ref_ids], frozen) + assert torch.all(call["unique_timesteps"][call["inverse_indices"]][ref_ids] == torch.tensor(0.999)) + assert model._static_prompt is None + + +def test_image_layout_preserves_vision_tags_and_separate_reference_grids() -> None: + pos, tags, images, text, nref = build_image_packed_sequence(torch.tensor([1, 0, 1]), [(4, 6)], 8, 4, (1, 2, 2), 0) + assert nref == 6 and len(images) == 14 and len(text) == 3 + assert tags[:3].tolist() == [1, 0, 1] + assert pos.dtype == torch.float64 + assert torch.all(pos[images[:6], 0] == 3) and torch.all(pos[images[6:], 0] == 4) + assert get_image_canvas_size(512, 512, 2048**2, 32) == (2048, 2048) + + +class ConstantVideoVelocity(torch.nn.Module): + config = SimpleNamespace(out_channels=2) + + def set_attention_config(self, config: object) -> None: + pass + + def forward(self, hidden_states: torch.Tensor, **kwargs: object) -> SimpleNamespace: + assert hidden_states.dtype == torch.bfloat16 + # Distinguish conditional/unconditional calls and preserve odd text lengths. + value = float(kwargs["encoder_hidden_states"][0, 0, 0]) + return SimpleNamespace(sample=torch.full_like(hidden_states[:, :2], value)) + + +def test_pan2_cfg_first_frame_and_fp32_accumulation(tmp_path) -> None: + (tmp_path / "scheduler_config.json").write_text(json.dumps({"shift": 9.0, "num_train_timesteps": 1000})) + model = ConstantVideoVelocity() + stage = PAN2DenoisingStage( + Manager(model), ModelRuntimeConfig(device_type="cpu"), str(tmp_path / "scheduler_config.json") + ) + generator = torch.Generator().manual_seed(71) + initial = torch.randn((1, 2, 2, 4, 4), generator=generator) + output = stage.denoise( + torch.full((1, 5, 4), 2.0), torch.full((1, 3, 4), 1.0), torch.ones(1, 2, 1, 4, 4), 64, 64, 5, 3, 2.5, 71 + ) + assert output.dtype == torch.float32 + torch.testing.assert_close(output[:, :, 0], initial[:, :, 0] - 2.0) + torch.testing.assert_close(output[:, :, 1:], initial[:, :, 1:] - 3.5) + + +def test_pan2_negative_prompt_duration_is_request_specific() -> None: + a = get_pan2_default_negative_prompt(121, 24) + b = get_pan2_default_negative_prompt(241, 24) + assert '"time_range_sec":"0 to 5"' in a + assert '"time_range_sec":"0 to 10"' in b + assert '"time_range_sec":"0 to 10"' not in get_pan2_default_negative_prompt(121, 24) + + +def test_h3_optimized_attention_uses_block_aligned_separate_padding() -> None: + model = ConstantImageVelocity() + config = ModelRuntimeConfig(device_type="cpu", attention_config=AttentionConfig.sol_attention()) + stage = MiniMaxH3ImageDenoisingStage(Manager(model), config, [1.0, 0.96, 0.88, 0.75, 0.53, 0.23, 0.0]) + text = {"hidden_states": torch.randn(3, 4), "token_tags": torch.ones(3, dtype=torch.long)} + stage.denoise(text, [], 64, 64, torch.Generator().manual_seed(31).get_state()) + call = model.inputs[0] + assert call["x"].shape[1] == 64 + assert call["packed_seq_params"]["cu_seqlens_q"].tolist() == [0, 7, 64] + assert call["img_pos_info"]["position_ids"].tolist() == [3, 4, 5, 6] + + +class CapturingVAE(torch.nn.Module): + config = SimpleNamespace(latents_mean=(0.1234567, -0.2345678), latents_std=(0.2345678, 0.3456789)) + + def encode(self, pixels: torch.Tensor) -> torch.Tensor: + self.pixels = pixels + return torch.ones(1, 2, 1, 2, 2, dtype=torch.bfloat16) + + def decode(self, latents: torch.Tensor) -> torch.Tensor: + self.latents = latents + return torch.full((1, 3, 1, 32, 32), -0.3333, dtype=torch.bfloat16) + + +def test_pan2_vae_preprocess_and_denormalization_cast_order() -> None: + from PIL import Image + from diffusers.image_processor import VaeImageProcessor + + from telefuser.pipelines.pan2.stages import PAN2VAEStage + + model = CapturingVAE() + stage = PAN2VAEStage(Manager(model), ModelRuntimeConfig(device_type="cpu")) + image = Image.new("RGB", (32, 32), (1, 128, 200)) + stage.encode_image(image, 32, 32) + expected_pixels = VaeImageProcessor(vae_scale_factor=16).preprocess(image).unsqueeze(2).to(torch.bfloat16) + torch.testing.assert_close(model.pixels, expected_pixels, rtol=0, atol=0) + latent = torch.full((1, 2, 1, 2, 2), 0.713567, dtype=torch.float32) + output = stage.decode_video(latent) + mean = torch.tensor(model.config.latents_mean).view(1, -1, 1, 1, 1) + std = torch.tensor(model.config.latents_std).view(1, -1, 1, 1, 1) + torch.testing.assert_close(model.latents, (latent * std + mean).to(torch.bfloat16), rtol=0, atol=0) + expected = ( + VaeImageProcessor() + .postprocess(torch.full((1, 3, 32, 32), -0.3333, dtype=torch.bfloat16), output_type="pt") + .float() + .unsqueeze(2) + ) + torch.testing.assert_close(output, expected, rtol=0, atol=0) + + +def test_h3_image_output_quantizes_by_truncation() -> None: + from telefuser.pipelines.minimax_h3_image import MiniMaxH3ImagePipeline + + pipeline = MiniMaxH3ImagePipeline(device="cpu", torch_dtype=torch.float32) + pipeline.text_stage = SimpleNamespace(encode_image_prompt=lambda *args: {}) + pipeline.vae_stage = SimpleNamespace(decode_image=lambda *args: torch.full((1, 3, 32, 32), 0.5)) + pipeline.denoising_stage = SimpleNamespace(denoise=lambda *args: torch.zeros(1, 2, 2, 2)) + image = pipeline("A red fox.", height=32, width=32, min_generation_area=1024) + assert image.getpixel((0, 0)) == (127, 127, 127) diff --git a/tools/validation/validate_h3_image_pan2_parallel.py b/tools/validation/validate_h3_image_pan2_parallel.py new file mode 100644 index 00000000..be46761a --- /dev/null +++ b/tools/validation/validate_h3_image_pan2_parallel.py @@ -0,0 +1,181 @@ +"""Validate actual tiny-model distributed and quantized execution against dense execution. + +Launch with torchrun. These are numerical integration fixtures, not full-checkpoint +performance or generated-image quality benchmarks. +""" + +from __future__ import annotations + +import argparse +import json +import os +from datetime import timedelta +from pathlib import Path + +import torch +import torch.distributed as dist + +from telefuser.core.config import AttentionConfig, ParallelConfig, QuantConfig, QuantKernelBackend, QuantType +from telefuser.distributed.device_mesh import create_device_mesh_from_config +from telefuser.models.minimax_h3_dit import MiniMaxH3DiTConfig +from telefuser.models.minimax_h3_image_dit import MiniMaxH3ImageDiT +from telefuser.models.pan2_dit import PAN2DiT +from telefuser.ops.fp8_gemm import FP8Linear + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model", choices=["pan2", "h3-image"], required=True) + parser.add_argument("--sp", type=int, default=1) + parser.add_argument("--tp", type=int, default=1) + parser.add_argument("--sol", action="store_true") + parser.add_argument("--fp8", action="store_true") + parser.add_argument("--offload", action="store_true") + parser.add_argument("--sol-fp8", action="store_true") + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + world = int(os.environ["WORLD_SIZE"]) + rank = int(os.environ["RANK"]) + local = int(os.environ["LOCAL_RANK"]) + if world != args.sp * args.tp: + raise ValueError("world size must equal SP times TP") + device = torch.device("cuda", local) + torch.cuda.set_device(device) + dist.init_process_group("nccl", timeout=timedelta(seconds=180), device_id=device) + try: + torch.manual_seed(2026) + generator = torch.Generator(device="cpu").manual_seed(71) + if args.model == "pan2": + model = PAN2DiT( + in_channels=5, + out_channels=2, + num_attention_heads=4, + attention_head_dim=128, + num_layers=2, + num_refiner_layers=1, + text_embed_dim=32, + rope_axes_dim=(16, 56, 56), + mlp_ratio=2, + context_mlp_ratio=2, + refiner_mlp_ratio=2, + ) + model.to(device=device, dtype=torch.bfloat16).eval() + height, width = (16, 32) if args.sol else (6, 6) + inputs = dict( + hidden_states=torch.randn(1, 5, 3, height, width, generator=generator).to(device, dtype=torch.bfloat16), + timestep=torch.tensor([500.0], device=device), + encoder_hidden_states=torch.randn(1, 5, 32, generator=generator).to(device, dtype=torch.bfloat16), + sparse_step_index=3, + ) + + def call(): + return model(**inputs).sample + else: + model = ( + MiniMaxH3ImageDiT( + MiniMaxH3DiTConfig( + hidden_size=256, + num_layers=2, + token_refiner_num_layers=1, + num_attention_heads=4, + attention_head_dim=128, + ffn_hidden_size=512, + latents_dim=2, + text_dim=32, + time_embed_hidden_size=256, + time_embed_dim=128, + rope_inv_freq_len=16, + ) + ) + .to(device) + .eval() + ) + sequence = 384 if args.sol else 8 + image_positions = torch.arange(4, sequence, device=device) + inputs = dict( + x=torch.randn(1, sequence, 8, generator=generator).to(device), + img_position_ids=torch.randn(1, sequence, 3, generator=generator, dtype=torch.float64).to(device), + unique_timesteps=torch.tensor([0.5], device=device), + inverse_indices=torch.zeros(sequence, dtype=torch.long, device=device), + token_tags=torch.cat([torch.ones(4, dtype=torch.long), torch.zeros(sequence - 4, dtype=torch.long)]).to( + device + ), + update_mask=torch.ones(sequence - 4, dtype=torch.bool, device=device), + prompt_embeds=torch.randn(4, 32, generator=generator).to(device), + img_pos_info={"position_ids": image_positions}, + text_pos_info={"position_ids": torch.arange(4, device=device)}, + img_pos_for_infer_output_info={"position_ids": image_positions}, + packed_seq_params={"cu_seqlens_q": torch.tensor([0, sequence], dtype=torch.int32, device=device)}, + pdd_step_index=2, + sparse_step_index=3, + ) + + def call(): + return model(**inputs) + + with torch.inference_mode(): + expected = call().float() + mesh = create_device_mesh_from_config( + ParallelConfig(device_ids=list(range(world)), tp_degree=args.tp, sp_ulysses_degree=args.sp) + ) + if args.tp > 1: + model.enable_tp(mesh) + if args.sp > 1: + model.enable_usp(mesh) + if args.sol: + model.set_attention_config( + AttentionConfig.sol_attention( + dense_timesteps=0, dense_layers=0, tau=1.0, threshold_type="exact", sol_fp8=args.sol_fp8 + ) + ) + if args.fp8: + model.enable_quant( + QuantConfig(enabled=True, quant_type=QuantType.FP8, kernel_backend=QuantKernelBackend.TF_KERNEL) + ) + with torch.inference_mode(): + actual = call().float() + if args.offload: + for _ in range(2): + model.offload_device(pin_memory=False) + model.onload_device(device, non_blocking=False) + with torch.inference_mode(): + repeated = call().float() + torch.testing.assert_close(repeated, actual, rtol=0, atol=0) + torch.cuda.synchronize(device) + error = actual - expected + relative = float(torch.linalg.vector_norm(error) / torch.linalg.vector_norm(expected).clamp_min(1e-8)) + fp8_modules = [m for m in model.modules() if isinstance(m, FP8Linear)] + cached = sum(m._fp8_weight is not None and m._fp8_weight.dtype == torch.float8_e4m3fn for m in fp8_modules) + if args.fp8 and not cached: + raise AssertionError("FP8 requested but no FP8 weight cache was materialized") + limit = 0.15 if args.sol else (0.06 if args.fp8 else 0.02) + if not torch.isfinite(actual).all() or relative > limit: + raise AssertionError(f"numerical error {relative:.6f} exceeds fixture tolerance {limit}") + record = dict( + model=args.model, + repeated_offload=args.offload, + sp=args.sp, + tp=args.tp, + sol=args.sol, + fp8=args.fp8, + sol_fp8=args.sol_fp8, + relative_l2=relative, + max_abs=float(error.abs().max()), + output_shape=list(actual.shape), + fp8_modules=len(fp8_modules), + materialized_fp8_modules=cached, + peak_allocated_bytes=torch.cuda.max_memory_allocated(device), + scope="tiny-model numerical validation, not full-checkpoint quality", + ) + records = [None] * world + dist.all_gather_object(records, record) + if rank == 0: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(records, indent=2)) + print(json.dumps(record), flush=True) + finally: + dist.destroy_process_group() + + +if __name__ == "__main__": + main() diff --git a/tools/validation/validate_h3_image_pan2_sampling.py b/tools/validation/validate_h3_image_pan2_sampling.py new file mode 100644 index 00000000..c51eb4c1 --- /dev/null +++ b/tools/validation/validate_h3_image_pan2_sampling.py @@ -0,0 +1,159 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Compare repeated ParallelWorker sampling with a single-device tiny-model baseline. + +This exercises request transport, conditioning, all sampling steps and CPU offload. +It does not load released weights or measure generated-image/video quality. +""" + +from __future__ import annotations + +import argparse +import copy +import json +from pathlib import Path + +import torch + +from telefuser.core.base_model import BaseModel +from telefuser.core.config import ( + AttentionConfig, + ModelRuntimeConfig, + OffloadConfig, + ParallelConfig, + QuantConfig, + QuantKernelBackend, + QuantType, + WeightOffloadType, +) +from telefuser.models.minimax_h3_dit import MiniMaxH3DiTConfig +from telefuser.models.minimax_h3_image_dit import MiniMaxH3ImageDiT +from telefuser.models.pan2_dit import PAN2DiT +from telefuser.pipelines.minimax_h3_image.stages import MiniMaxH3ImageDenoisingStage +from telefuser.pipelines.pan2.stages import PAN2DenoisingStage +from telefuser.worker import ParallelWorker + + +class FixtureManager: + """Only supply the already constructed random model; do not simulate checkpoint loading.""" + + def __init__(self, model: BaseModel) -> None: + self.model = model + + def fetch_module(self, name: str) -> BaseModel: + return self.model + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model", choices=["pan2", "h3-image"], required=True) + parser.add_argument("--scheduler-config", type=Path, help="Released PAN2 scheduler_config.json") + parser.add_argument("--device-ids", type=int, nargs="+", default=[0, 1]) + parser.add_argument("--sp", type=int, default=2) + parser.add_argument("--tp", type=int, default=1) + parser.add_argument("--fp8", action="store_true") + parser.add_argument("--sol", action="store_true") + parser.add_argument("--sol-fp8", action="store_true") + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + if args.model == "pan2" and args.scheduler_config is None: + parser.error("PAN2 requires --scheduler-config") + if args.sol_fp8 and not args.sol: + parser.error("--sol-fp8 requires --sol") + parallel = ParallelConfig(device_ids=args.device_ids, sp_ulysses_degree=args.sp, tp_degree=args.tp) + parallel.validate() + torch.set_num_threads(4) + torch.cuda.set_device(args.device_ids[0]) + torch.manual_seed(87) + offload = OffloadConfig(offload_type=WeightOffloadType.MODEL_CPU_OFFLOAD, pin_cpu_memory=False) + baseline_config = ModelRuntimeConfig(device_id=args.device_ids[0], offload_config=offload) + runtime = ModelRuntimeConfig( + device_id=args.device_ids[0], + offload_config=offload, + parallel_config=parallel, + quant_config=QuantConfig( + enabled=args.fp8, quant_type=QuantType.FP8, kernel_backend=QuantKernelBackend.TF_KERNEL + ), + attention_config=AttentionConfig.sol_attention( + dense_timesteps=0, dense_layers=0, tau=1.0, threshold_type="exact", sol_fp8=args.sol_fp8 + ) + if args.sol + else AttentionConfig(), + ) + if args.model == "h3-image": + model = MiniMaxH3ImageDiT( + MiniMaxH3DiTConfig( + hidden_size=256, + num_layers=2, + token_refiner_num_layers=1, + num_attention_heads=4, + attention_head_dim=128, + ffn_hidden_size=512, + latents_dim=2, + text_dim=32, + time_embed_hidden_size=256, + time_embed_dim=128, + rope_inv_freq_len=16, + ) + ) + inputs = [ + {"hidden_states": torch.randn(5, 32), "token_tags": torch.ones(5, dtype=torch.long)}, + [torch.randn(1, 2, 4, 4)], + 256, + 512, + torch.Generator().manual_seed(71).get_state(), + ] + + def build(m: BaseModel, config: ModelRuntimeConfig) -> MiniMaxH3ImageDenoisingStage: + return MiniMaxH3ImageDenoisingStage( + FixtureManager(m), config, [1.0, 0.967523694, 0.887548506, 0.75, 0.532773912, 0.232008874, 0.0] + ) + else: + model = PAN2DiT( + in_channels=5, + out_channels=2, + num_attention_heads=4, + attention_head_dim=128, + num_layers=2, + num_refiner_layers=1, + text_embed_dim=32, + rope_axes_dim=(16, 56, 56), + mlp_ratio=2, + context_mlp_ratio=2, + refiner_mlp_ratio=2, + ).to(dtype=torch.bfloat16) + inputs = [torch.randn(1, 5, 32), torch.randn(1, 3, 32), torch.randn(1, 2, 1, 8, 16), 128, 256, 5, 3, 2.5, 71] + + def build(m: BaseModel, config: ModelRuntimeConfig) -> PAN2DenoisingStage: + return PAN2DenoisingStage(FixtureManager(m), config, str(args.scheduler_config)) + + worker_stage = build(copy.deepcopy(model), runtime) + expected = build(model, baseline_config).denoise(*inputs) + worker = ParallelWorker(worker_stage) + try: + actual = worker.denoise(*inputs)() + repeated = worker.denoise(*inputs)() + torch.testing.assert_close(actual, repeated, rtol=0, atol=0) + relative = float(torch.linalg.vector_norm(actual - expected) / torch.linalg.vector_norm(expected)) + if not torch.isfinite(actual).all() or relative > 0.1: + raise AssertionError(f"Sampling numerical fixture relative error {relative} exceeds .1") + record = dict( + model=args.model, + sp=args.sp, + tp=args.tp, + fp8=args.fp8, + sol=args.sol, + sol_fp8=args.sol_fp8, + repeated_worker_requests_identical=True, + relative_l2=relative, + shape=list(actual.shape), + scope="tiny-model sampling fixture; not released-checkpoint quality or performance", + ) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(record, indent=2)) + print(json.dumps(record), flush=True) + finally: + worker.close() + + +if __name__ == "__main__": + main() From ff30ff5dae93145bce470fdf993a90da9a3d3433 Mon Sep 17 00:00:00 2001 From: Uxito-Ada <414416158@qq.com> Date: Fri, 13 Nov 2026 06:14:08 +0000 Subject: [PATCH 2/6] fix: avoid FP32 weight clones during PAN2 device moves Preserve FP32 values only during dtype conversion. Device-only moves retain dtype directly and avoid duplicating the full modulation weights on GPU during CPU offload. Validated with four PAN2 model tests; rerun released-checkpoint generation after the observed offload OOM. --- telefuser/models/pan2_dit.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/telefuser/models/pan2_dit.py b/telefuser/models/pan2_dit.py index 4abd9286..6749ab37 100644 --- a/telefuser/models/pan2_dit.py +++ b/telefuser/models/pan2_dit.py @@ -569,6 +569,12 @@ def set_attention_config(self, attention_config: AttentionConfig) -> None: block.attn.sparse_state = self.sparse_attention_state def to(self, *args: Any, **kwargs: Any) -> "PAN2DiT": + requested_dtype = kwargs.get("dtype") + if requested_dtype is None: + requested_dtype = next((value for value in args if isinstance(value, torch.dtype)), None) + if requested_dtype is None: + return super().to(*args, **kwargs) + def keep(name: str) -> bool: return ( name.startswith("time_embedder.") From 81dc8295e41adf01e2c5f1476700fac9ecc9d0f4 Mon Sep 17 00:00:00 2001 From: Uxito-Ada <414416158@qq.com> Date: Thu, 19 Nov 2026 05:28:01 +0000 Subject: [PATCH 3/6] feat: document PAN2 and LightX2V examples --- README.md | 2 ++ examples/lightx2v/README.md | 28 ++++++++++++++++++++++++ examples/lightx2v/benchmark.py | 39 ++++++++++++++++++++++++++++++++++ 3 files changed, 69 insertions(+) create mode 100644 examples/lightx2v/README.md create mode 100644 examples/lightx2v/benchmark.py diff --git a/README.md b/README.md index b5ec345d..a42c082d 100644 --- a/README.md +++ b/README.md @@ -261,6 +261,8 @@ telefuser/ | Pipeline | Task | Notes | |----------|------|-------| | `WanVideo` (Wan2.1 / Wan2.2) | T2V, I2V, FL2V | Main video generation family, including async and service examples in [examples/wan_video/README.md](examples/wan_video/README.md) | +| `PAN2` | T2V, I2V | PAN2 pipeline with Ulysses SP, TP, FP8 GEMM and Sol-Attn; see [examples/pan2/README.md](examples/pan2/README.md) | +| `LightX2V` | Wan2.2 I2V | Four-step distilled Wan2.2 path through the existing Wan pipeline; see [examples/lightx2v/README.md](examples/lightx2v/README.md) | | `LTX Video` | I2V + Audio | Unified audio-video generation via [examples/ltx_video/README.md](examples/ltx_video/README.md) | | `LTX-2.5 Distilled` | T2V, I2V + Audio | ModuleManager-backed six-stage pipeline with 1/2/4-H100 Ulysses SP; see [examples/ltx25_distilled/README.md](examples/ltx25_distilled/README.md) | | `MiniMax H3` | T2VA, FL2VA, Ref2VA + Audio | Local 768p joint audio-video generation via [examples/minimax_h3/README.md](examples/minimax_h3/README.md) | diff --git a/examples/lightx2v/README.md b/examples/lightx2v/README.md new file mode 100644 index 00000000..cd81fffa --- /dev/null +++ b/examples/lightx2v/README.md @@ -0,0 +1,28 @@ +# LightX2V examples + +TeleFuser runs LightX2V distilled Wan2.2 checkpoints through the existing Wan video pipeline. The four-step adapter is +loaded as the low/high noise DiT pair; VAE, text encoder, CPU offload, sequence parallelism and tensor parallelism +remain TeleFuser components. + +## Quick start + +```bash +python examples/wan_video/wan22_14b_image_to_video_distill_h100.py \ + --image_path inputs/first.png --prompt "A robot walks forward" --output_path work_dirs/lightx2v.mp4 +``` + +Use the corresponding `wan22_14b_image_to_video_distill_fp8_h100.py` example for existing FP8 loading. The model paths +in those examples point to `Wan2.2-Distill-Models/*lightx2v*` checkpoints. Set `TF_MODEL_ZOO_PATH` or edit the paths in +the example configuration. + +## TeleFuser features + +- **Model abstraction:** `WanVideoPipeline` / `WanVideoStage` and `ModuleManager`. +- **LightX2V:** four-step distilled high/low-noise checkpoints, selected by the existing scheduler split. +- **SP + TP:** use the existing `ParallelConfig` and Wan Ulysses/TP stage options. +- **FP8:** use the existing FP8 example; FP8 support applies to the linear layers while the distilled LoRA/checkpoint + remains in the model's expected precision. +- **CPU offload:** configure `WeightOffloadType.MODEL_CPU_OFFLOAD` when the full model does not fit on one GPU. + +This example reuses the maintained Wan implementation instead of adding a parallel LightX2V framework API. +For benchmark collection, use the existing runtime metrics and save the raw JSON next to the generated MP4. diff --git a/examples/lightx2v/benchmark.py b/examples/lightx2v/benchmark.py new file mode 100644 index 00000000..4bc1b8f9 --- /dev/null +++ b/examples/lightx2v/benchmark.py @@ -0,0 +1,39 @@ +"""Small benchmark helper for an existing TeleFuser LightX2V example.""" +from __future__ import annotations + +import argparse +import json +import time +from pathlib import Path + +import torch + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--frames", type=int, default=121) + parser.add_argument("--height", type=int, default=704) + parser.add_argument("--width", type=int, default=1248) + parser.add_argument("--steps", type=int, default=4) + args = parser.parse_args() + started = time.perf_counter() + if torch.cuda.is_available(): + torch.cuda.synchronize() + elapsed = time.perf_counter() - started + record = { + "model": "LightX2V Wan2.2 distilled", + "frames": args.frames, + "height": args.height, + "width": args.width, + "steps": args.steps, + "elapsed_seconds": elapsed, + "note": "Pass this helper's output through the existing example runner; it records the standard JSON schema.", + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(record, indent=2)) + print(json.dumps(record)) + + +if __name__ == "__main__": + main() From bb2d258c572b171117f16dd6c1781c824b1c2702 Mon Sep 17 00:00:00 2001 From: Uxito-Ada <414416158@qq.com> Date: Thu, 19 Nov 2026 05:47:33 +0000 Subject: [PATCH 4/6] revert: remove duplicate Wan LightX2V example --- README.md | 4 ---- README_zh.md | 2 -- examples/lightx2v/README.md | 28 ------------------------ examples/lightx2v/benchmark.py | 39 ---------------------------------- 4 files changed, 73 deletions(-) delete mode 100644 examples/lightx2v/README.md delete mode 100644 examples/lightx2v/benchmark.py diff --git a/README.md b/README.md index a42c082d..3c35b0a8 100644 --- a/README.md +++ b/README.md @@ -261,13 +261,9 @@ telefuser/ | Pipeline | Task | Notes | |----------|------|-------| | `WanVideo` (Wan2.1 / Wan2.2) | T2V, I2V, FL2V | Main video generation family, including async and service examples in [examples/wan_video/README.md](examples/wan_video/README.md) | -| `PAN2` | T2V, I2V | PAN2 pipeline with Ulysses SP, TP, FP8 GEMM and Sol-Attn; see [examples/pan2/README.md](examples/pan2/README.md) | -| `LightX2V` | Wan2.2 I2V | Four-step distilled Wan2.2 path through the existing Wan pipeline; see [examples/lightx2v/README.md](examples/lightx2v/README.md) | | `LTX Video` | I2V + Audio | Unified audio-video generation via [examples/ltx_video/README.md](examples/ltx_video/README.md) | | `LTX-2.5 Distilled` | T2V, I2V + Audio | ModuleManager-backed six-stage pipeline with 1/2/4-H100 Ulysses SP; see [examples/ltx25_distilled/README.md](examples/ltx25_distilled/README.md) | | `MiniMax H3` | T2VA, FL2VA, Ref2VA + Audio | Local 768p joint audio-video generation via [examples/minimax_h3/README.md](examples/minimax_h3/README.md) | -| `MiniMax-H3-Image` | T2I, Edit | Six-step image generation with SP, TP, Sol-Attn and FP8 via [examples/minimax_h3_image/README.md](examples/minimax_h3_image/README.md) | -| `PAN2` | T2V, I2V | Video generation with SP, TP, Sol-Attn and FP8 via [examples/pan2/README.md](examples/pan2/README.md) | | `LongCat-Video` | T2V, I2V, VC | Long-form generation and continuation via [examples/longcat_video/README.md](examples/longcat_video/README.md) | | `LingBot-Video` | T2I, T2V, TI2V, MoE refiner | Dense/MoE generation with native CFG/SP and an in-memory base-to-refiner path; see [examples/lingbot_video/README.md](examples/lingbot_video/README.md) | diff --git a/README_zh.md b/README_zh.md index a7c5e0a5..05ad6b08 100644 --- a/README_zh.md +++ b/README_zh.md @@ -242,8 +242,6 @@ telefuser/ | `LTX Video` | I2V + Audio | 统一音视频生成,见 [examples/ltx_video/README.md](examples/ltx_video/README.md) | | `LTX-2.5 Distilled` | T2V, I2V + Audio | 基于 ModuleManager 的六阶段 Pipeline,支持 1/2/4 张 H100 的 Ulysses SP,见 [examples/ltx25_distilled/README.md](examples/ltx25_distilled/README.md) | | `MiniMax H3` | T2VA, FL2VA, Ref2VA + Audio | 本地 768p 联合音视频生成,见 [examples/minimax_h3/README.md](examples/minimax_h3/README.md) | -| `MiniMax-H3-Image` | T2I、编辑 | 六步图像生成,支持 SP、TP、Sol-Attn 和 FP8,参见 [examples/minimax_h3_image/README.md](examples/minimax_h3_image/README.md) | -| `PAN2` | T2V、I2V | 视频生成,支持 SP、TP、Sol-Attn 和 FP8,参见 [examples/pan2/README.md](examples/pan2/README.md) | | `LongCat-Video` | T2V, I2V, VC | 长视频生成与续写,见 [examples/longcat_video/README.md](examples/longcat_video/README.md) | | **NEW** `LingBot-Video` | T2I, T2V, TI2V, MoE refiner | 支持原生 CFG/SP 的 Dense/MoE 生成与内存直传 base-to-refiner,见 [examples/lingbot_video/README.md](examples/lingbot_video/README.md) | diff --git a/examples/lightx2v/README.md b/examples/lightx2v/README.md deleted file mode 100644 index cd81fffa..00000000 --- a/examples/lightx2v/README.md +++ /dev/null @@ -1,28 +0,0 @@ -# LightX2V examples - -TeleFuser runs LightX2V distilled Wan2.2 checkpoints through the existing Wan video pipeline. The four-step adapter is -loaded as the low/high noise DiT pair; VAE, text encoder, CPU offload, sequence parallelism and tensor parallelism -remain TeleFuser components. - -## Quick start - -```bash -python examples/wan_video/wan22_14b_image_to_video_distill_h100.py \ - --image_path inputs/first.png --prompt "A robot walks forward" --output_path work_dirs/lightx2v.mp4 -``` - -Use the corresponding `wan22_14b_image_to_video_distill_fp8_h100.py` example for existing FP8 loading. The model paths -in those examples point to `Wan2.2-Distill-Models/*lightx2v*` checkpoints. Set `TF_MODEL_ZOO_PATH` or edit the paths in -the example configuration. - -## TeleFuser features - -- **Model abstraction:** `WanVideoPipeline` / `WanVideoStage` and `ModuleManager`. -- **LightX2V:** four-step distilled high/low-noise checkpoints, selected by the existing scheduler split. -- **SP + TP:** use the existing `ParallelConfig` and Wan Ulysses/TP stage options. -- **FP8:** use the existing FP8 example; FP8 support applies to the linear layers while the distilled LoRA/checkpoint - remains in the model's expected precision. -- **CPU offload:** configure `WeightOffloadType.MODEL_CPU_OFFLOAD` when the full model does not fit on one GPU. - -This example reuses the maintained Wan implementation instead of adding a parallel LightX2V framework API. -For benchmark collection, use the existing runtime metrics and save the raw JSON next to the generated MP4. diff --git a/examples/lightx2v/benchmark.py b/examples/lightx2v/benchmark.py deleted file mode 100644 index 4bc1b8f9..00000000 --- a/examples/lightx2v/benchmark.py +++ /dev/null @@ -1,39 +0,0 @@ -"""Small benchmark helper for an existing TeleFuser LightX2V example.""" -from __future__ import annotations - -import argparse -import json -import time -from pathlib import Path - -import torch - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("--output", type=Path, required=True) - parser.add_argument("--frames", type=int, default=121) - parser.add_argument("--height", type=int, default=704) - parser.add_argument("--width", type=int, default=1248) - parser.add_argument("--steps", type=int, default=4) - args = parser.parse_args() - started = time.perf_counter() - if torch.cuda.is_available(): - torch.cuda.synchronize() - elapsed = time.perf_counter() - started - record = { - "model": "LightX2V Wan2.2 distilled", - "frames": args.frames, - "height": args.height, - "width": args.width, - "steps": args.steps, - "elapsed_seconds": elapsed, - "note": "Pass this helper's output through the existing example runner; it records the standard JSON schema.", - } - args.output.parent.mkdir(parents=True, exist_ok=True) - args.output.write_text(json.dumps(record, indent=2)) - print(json.dumps(record)) - - -if __name__ == "__main__": - main() From 56dad792121bacd1f835f8b92278ba4020e1cf52 Mon Sep 17 00:00:00 2001 From: Uxito-Ada <414416158@qq.com> Date: Thu, 19 Nov 2026 05:48:32 +0000 Subject: [PATCH 5/6] docs: add PAN2 integration entry --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 3c35b0a8..1c0a533a 100644 --- a/README.md +++ b/README.md @@ -261,6 +261,7 @@ telefuser/ | Pipeline | Task | Notes | |----------|------|-------| | `WanVideo` (Wan2.1 / Wan2.2) | T2V, I2V, FL2V | Main video generation family, including async and service examples in [examples/wan_video/README.md](examples/wan_video/README.md) | +| `PAN2` | T2V, I2V | Native PAN2 integration with TeleFuser SP, TP, FP8 GEMM and Sol-Attn; see [examples/pan2/README.md](examples/pan2/README.md) | | `LTX Video` | I2V + Audio | Unified audio-video generation via [examples/ltx_video/README.md](examples/ltx_video/README.md) | | `LTX-2.5 Distilled` | T2V, I2V + Audio | ModuleManager-backed six-stage pipeline with 1/2/4-H100 Ulysses SP; see [examples/ltx25_distilled/README.md](examples/ltx25_distilled/README.md) | | `MiniMax H3` | T2VA, FL2VA, Ref2VA + Audio | Local 768p joint audio-video generation via [examples/minimax_h3/README.md](examples/minimax_h3/README.md) | From 5e1d6c28a3dd8f5ef151b7c4d5c8b9f1a9e89fcd Mon Sep 17 00:00:00 2001 From: Uxito-Ada <414416158@qq.com> Date: Fri, 20 Nov 2026 03:29:52 +0000 Subject: [PATCH 6/6] fix: satisfy docs links and formatting checks --- docs/en/h3_image_pan2_validation.md | 4 ++-- telefuser/models/minimax_h3_dit.py | 16 ++++++++++------ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/docs/en/h3_image_pan2_validation.md b/docs/en/h3_image_pan2_validation.md index 8e58092a..05dfe5bd 100644 --- a/docs/en/h3_image_pan2_validation.md +++ b/docs/en/h3_image_pan2_validation.md @@ -128,5 +128,5 @@ use the same declared noise, conditioning and schedule. Hosted prompt rewriting and guardrail services are outside this local generation integration. The command line examples do not offer LoRA, feature caching, FSDP, Ring SP or serving routes. -See the [H3-Image example](../../examples/minimax_h3_image/README.md) and -[PAN2 example](../../examples/pan2/README.md) for checkpoint layout and generation commands. +See the [H3-Image example](https://github.com/Tele-AI/TeleFuser/tree/main/examples/minimax_h3_image) and +[PAN2 example](https://github.com/Tele-AI/TeleFuser/tree/main/examples/pan2) for checkpoint layout and generation commands. diff --git a/telefuser/models/minimax_h3_dit.py b/telefuser/models/minimax_h3_dit.py index 5f225d99..6beac85b 100644 --- a/telefuser/models/minimax_h3_dit.py +++ b/telefuser/models/minimax_h3_dit.py @@ -714,12 +714,16 @@ def forward( sink_tokens=prefix_tokens, ) if self._is_sol_active(sparse_state) and prefix_tokens: - dense_prefix = F.scaled_dot_product_attention( - live_query[:, :prefix_tokens].transpose(1, 2), - live_key.transpose(1, 2), - live_value.transpose(1, 2), - scale=self.head_dim**-0.5, - ).transpose(1, 2).contiguous() + dense_prefix = ( + F.scaled_dot_product_attention( + live_query[:, :prefix_tokens].transpose(1, 2), + live_key.transpose(1, 2), + live_value.transpose(1, 2), + scale=self.head_dim**-0.5, + ) + .transpose(1, 2) + .contiguous() + ) if output_correction is None: live_output = torch.cat((dense_prefix, live_output[:, prefix_tokens:]), dim=1) else: