From dd094a9a07c4743094e6a11338746a24e588b7c3 Mon Sep 17 00:00:00 2001 From: Uxito-Ada Date: Tue, 17 Nov 2026 14:51:01 +0800 Subject: [PATCH 01/11] feat(pan2): add NVFP4 Blackwell pipeline integration --- README.md | 1 + README_zh.md | 1 + docs/en/pan2.md | 29 +++++++++ docs/en/supported_models.md | 1 + docs/zh/pan2.md | 26 ++++++++ docs/zh/supported_models.md | 1 + examples/pan2/pan2_t2v.py | 36 ++++++++++ mkdocs.yml | 1 + telefuser/pipelines/pan2/__init__.py | 5 ++ telefuser/pipelines/pan2/pipeline.py | 76 ++++++++++++++++++++++ tests/unit/pipelines/pan2/test_pipeline.py | 22 +++++++ 11 files changed, 199 insertions(+) create mode 100644 docs/en/pan2.md create mode 100644 docs/zh/pan2.md create mode 100644 examples/pan2/pan2_t2v.py create mode 100644 telefuser/pipelines/pan2/__init__.py create mode 100644 telefuser/pipelines/pan2/pipeline.py create mode 100644 tests/unit/pipelines/pan2/test_pipeline.py diff --git a/README.md b/README.md index 3c35b0a8..a4a17553 100644 --- a/README.md +++ b/README.md @@ -273,6 +273,7 @@ telefuser/ |----------|------|-------| | `Qwen-Image` | T2I, Edit | [examples/qwen_image/README.md](examples/qwen_image/README.md) | | `Z-Image` | T2I | [examples/z_image/README.md](examples/z_image/README.md) | +| `PAN 2` | T2V, I2V | NVFP4 + block CPU offload on Blackwell; [PAN 2 guide](docs/en/pan2.md) | | `Flux2 Klein` | T2I | [examples/flux2_klein/README.md](examples/flux2_klein/README.md) | See [examples/README.md](examples/README.md) for the example runner and baseline comparison workflow. diff --git a/README_zh.md b/README_zh.md index 05ad6b08..17fe63e9 100644 --- a/README_zh.md +++ b/README_zh.md @@ -251,6 +251,7 @@ telefuser/ |----------|------|------| | `Qwen-Image` | T2I, Edit | [examples/qwen_image/README.md](examples/qwen_image/README.md) | | `Z-Image` | T2I | [examples/z_image/README.md](examples/z_image/README.md) | +| `PAN 2` | T2V、I2V | Blackwell 上的 NVFP4 + 分块 CPU offload,见 [PAN 2 指南](docs/zh/pan2.md) | | `Flux2 Klein` | T2I | [examples/flux2_klein/README.md](examples/flux2_klein/README.md) | [examples/README.md](examples/README.md) 中提供了统一的 example runner 与 baseline 对比流程说明。 diff --git a/docs/en/pan2.md b/docs/en/pan2.md new file mode 100644 index 00000000..865a1148 --- /dev/null +++ b/docs/en/pan2.md @@ -0,0 +1,29 @@ +# PAN 2 on Blackwell + +TeleFuser exposes the public `IFM/PAN2` Diffusers modular pipeline through +`telefuser.pipelines.pan2`. PAN 2 is a 29.2B-parameter world model. BF16 does +not fit on a 24 GB RTX 5090; the supported single-card profile uses TorchAO +NVFP4 for the diffusion transformer and Diffusers block-level CPU offload. + +Install `diffusers>=0.43.0`, `torchao>=0.17`, and a PyTorch build with CUDA 13. +The GPU must report compute capability SM100 or newer. The first run downloads +the PAN 2 transformer, Qwen3.5-9B text encoder, and Wan 2.2 VAE, so plan for +more than 100 GB of host storage and RAM. + +```bash +python examples/pan2/pan2_t2v.py \ + --nvfp4 \ + --steps 4 \ + --prompt "A small red car drives through a rainy city street." \ + --output pan2.mp4 +``` + +The same entrypoint accepts `--image` for image-to-video. NVFP4 quantization +is applied only to Linear layers in the PAN 2 transformer; the text encoder +and VAE stay in BF16/FP32 and are also offloaded. This profile trades latency +for fitting the model on a single 24 GB Blackwell card. + +MiniMax-H3-Image is not enabled by this profile. Its 33.1B transformer is +released as approximately 66 GB of BF16 weights and requires a separate +multi-component loader; it does not fit as a resident single-card TeleFuser +pipeline on a 24 GB RTX 5090. diff --git a/docs/en/supported_models.md b/docs/en/supported_models.md index 49b2078c..a46814f4 100644 --- a/docs/en/supported_models.md +++ b/docs/en/supported_models.md @@ -15,6 +15,7 @@ by the selected guide before changing optimization settings. | [LingBot-World v2](/TeleFuser/cookbook/lingbot-world/) | Bidirectional world-model streaming | Standalone and LiveKit | | [LingBot-World-Fast](/TeleFuser/cookbook/lingbot-world/) | Causal interactive streaming | Standalone and LiveKit | | [ABot-World-0-5B-LF](/TeleFuser/cookbook/abot-world/) | Interactive generation | HTTP and LiveKit | +| [PAN 2](pan2.md) | T2V and I2V | NVFP4 with block CPU offload on Blackwell | ## Video, Audio, and Restoration diff --git a/docs/zh/pan2.md b/docs/zh/pan2.md new file mode 100644 index 00000000..36d832d2 --- /dev/null +++ b/docs/zh/pan2.md @@ -0,0 +1,26 @@ +# 在 Blackwell 上运行 PAN 2 + +TeleFuser 通过 `telefuser.pipelines.pan2` 接入公开的 `IFM/PAN2` +Diffusers 模块化流水线。PAN 2 有 292 亿参数,BF16 无法放入 24 GB 的 +RTX 5090;单卡配置使用 TorchAO NVFP4 量化扩散 Transformer,并启用 +Diffusers block-level CPU offload。 + +安装 `diffusers>=0.43.0`、`torchao>=0.17` 和 CUDA 13 PyTorch。GPU 必须是 +SM100 或更新架构。首次运行需要下载 PAN 2 Transformer、Qwen3.5-9B 文本 +编码器和 Wan 2.2 VAE,请预留超过 100 GB 的磁盘和主机内存。 + +```bash +python examples/pan2/pan2_t2v.py \ + --nvfp4 \ + --steps 4 \ + --prompt "A small red car drives through a rainy city street." \ + --output pan2.mp4 +``` + +加入 `--image` 可执行 image-to-video。NVFP4 只量化 PAN 2 Transformer 中 +的 Linear 层,文本编码器和 VAE 保持 BF16/FP32 并参与 offload;代价是更高 +的延迟,以换取单张 24 GB Blackwell 显卡上的可装载性。 + +MiniMax-H3-Image 暂未启用。它的 331 亿参数 Transformer 约 66 GB BF16, +需要独立的多组件加载器,不能作为常驻单卡 TeleFuser pipeline 放入 24 GB +RTX 5090。 diff --git a/docs/zh/supported_models.md b/docs/zh/supported_models.md index 48b53475..5ac496b8 100644 --- a/docs/zh/supported_models.md +++ b/docs/zh/supported_models.md @@ -13,6 +13,7 @@ ModelScope 地址、所需文件、支持任务、硬件假设及可运行命令 | [LingBot-World v2](/TeleFuser/zh/cookbook/lingbot-world/) | 双向世界模型流式推理 | 单机和 LiveKit | | [LingBot-World-Fast](/TeleFuser/zh/cookbook/lingbot-world/) | 因果交互式流推理 | 单机和 LiveKit | | [ABot-World-0-5B-LF](/TeleFuser/zh/cookbook/abot-world/) | 交互式生成 | HTTP 和 LiveKit | +| [PAN 2](pan2.md) | T2V 和 I2V | Blackwell 上的 NVFP4 与分块 CPU offload | ## 视频、音视频与修复 diff --git a/examples/pan2/pan2_t2v.py b/examples/pan2/pan2_t2v.py new file mode 100644 index 00000000..c38dc4f8 --- /dev/null +++ b/examples/pan2/pan2_t2v.py @@ -0,0 +1,36 @@ +"""Run PAN 2 through TeleFuser's NVFP4 loader.""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +from diffusers.utils import export_to_video + +from telefuser.pipelines.pan2 import PAN2Pipeline, PAN2PipelineConfig + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--model-id", default="IFM/PAN2") + parser.add_argument("--prompt", default="A small red car drives through a rainy city street.") + parser.add_argument("--image", type=Path) + parser.add_argument("--steps", type=int, default=4) + parser.add_argument("--output", type=Path, default=Path("pan2.mp4")) + parser.add_argument("--nvfp4", action="store_true") + args = parser.parse_args() + + pipe = PAN2Pipeline.from_pretrained( + PAN2PipelineConfig(model_id=args.model_id, nvfp4=args.nvfp4, cpu_offload=True) + ) + request = {"prompt": args.prompt, "num_inference_steps": args.steps, "output_type": "np"} + if args.image: + request["image"] = str(args.image) + result = pipe(**request) + frames = getattr(result, "frames", result) + export_to_video(frames[0] if isinstance(frames, list) else frames, str(args.output), fps=24) + print(args.output) + + +if __name__ == "__main__": + main() diff --git a/mkdocs.yml b/mkdocs.yml index 9f26946f..635cc5b1 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -187,6 +187,7 @@ nav: - Basic Inference: quickstart.md - Models & Cookbook: - Supported Models: supported_models.md + - PAN 2 NVFP4 on Blackwell: pan2.md - Serving & APIs: - Overview: serving.md - Batch Service and API Reference: service.md diff --git a/telefuser/pipelines/pan2/__init__.py b/telefuser/pipelines/pan2/__init__.py new file mode 100644 index 00000000..1ca95703 --- /dev/null +++ b/telefuser/pipelines/pan2/__init__.py @@ -0,0 +1,5 @@ +"""PAN 2 video generation pipeline integration.""" + +from .pipeline import PAN2Pipeline, PAN2PipelineConfig + +__all__ = ["PAN2Pipeline", "PAN2PipelineConfig"] diff --git a/telefuser/pipelines/pan2/pipeline.py b/telefuser/pipelines/pan2/pipeline.py new file mode 100644 index 00000000..17fcd5b7 --- /dev/null +++ b/telefuser/pipelines/pan2/pipeline.py @@ -0,0 +1,76 @@ +"""TeleFuser integration for the public IFM/PAN2 Diffusers pipeline.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +import torch + + +@dataclass +class PAN2PipelineConfig: + model_id: str = "IFM/PAN2" + torch_dtype: torch.dtype = torch.bfloat16 + device: str = "cuda" + cpu_offload: bool = True + nvfp4: bool = False + nvfp4_use_triton: bool = True + + +def _quantize_nvfp4(module: torch.nn.Module, use_triton_kernel: bool) -> None: + """Quantize Linear weights and dynamic activations with TorchAO NVFP4.""" + if not torch.cuda.is_available() or torch.cuda.get_device_capability()[0] < 10: + raise RuntimeError("PAN2 NVFP4 requires a CUDA Blackwell-or-newer GPU (SM100+)") + try: + from torchao.prototype.mx_formats import NVFP4DynamicActivationNVFP4WeightConfig + from torchao.quantization import quantize_ + except ImportError as exc: + raise RuntimeError( + "PAN2 NVFP4 requires torchao with the NVFP4 prototype enabled; install torchao>=0.17" + ) from exc + quantize_(module, NVFP4DynamicActivationNVFP4WeightConfig(use_triton_kernel=use_triton_kernel)) + + +class PAN2Pipeline: + """TeleFuser-owned loader that preserves the native Diffusers output.""" + + def __init__(self, config: PAN2PipelineConfig): + self.config = config + self.pipe: Any | None = None + + @classmethod + def from_pretrained(cls, config: PAN2PipelineConfig | None = None, **kwargs: Any) -> "PAN2Pipeline": + config = config or PAN2PipelineConfig() + try: + from diffusers import ModularPipeline + except ImportError as exc: + raise RuntimeError("PAN2 requires diffusers>=0.43.0") from exc + runner = cls(config) + runner.pipe = ModularPipeline.from_pretrained(config.model_id, **kwargs) + runner.pipe.load_components(dtype=config.torch_dtype) + if config.nvfp4: + transformer = getattr(runner.pipe, "transformer", None) + if transformer is None: + raise RuntimeError("PAN2 pipeline did not expose a transformer component") + _quantize_nvfp4(transformer, config.nvfp4_use_triton) + if config.cpu_offload: + device = torch.device(config.device) + for component in (getattr(runner.pipe, "transformer", None), getattr(runner.pipe, "text_encoder", None)): + if component is not None and hasattr(component, "enable_group_offload"): + component.enable_group_offload( + onload_device=device, + offload_device=torch.device("cpu"), + offload_type="block_level", + num_blocks_per_group=1, + use_stream=True, + ) + runner.pipe.to("cpu") + else: + runner.pipe.to(config.device) + return runner + + def __call__(self, **kwargs: Any) -> Any: + if self.pipe is None: + raise RuntimeError("PAN2Pipeline is not loaded; call from_pretrained first") + return self.pipe(**kwargs) diff --git a/tests/unit/pipelines/pan2/test_pipeline.py b/tests/unit/pipelines/pan2/test_pipeline.py new file mode 100644 index 00000000..1ebcff10 --- /dev/null +++ b/tests/unit/pipelines/pan2/test_pipeline.py @@ -0,0 +1,22 @@ +import pytest +import torch + +from telefuser.pipelines.pan2.pipeline import PAN2PipelineConfig, _quantize_nvfp4 + + +def test_pan2_defaults_are_single_card_safe(): + config = PAN2PipelineConfig() + assert config.model_id == "IFM/PAN2" + assert config.cpu_offload is True + assert config.nvfp4 is False + + +@pytest.mark.gpu +def test_nvfp4_linear_smoke(): + if not torch.cuda.is_available() or torch.cuda.get_device_capability()[0] < 10: + pytest.skip("NVFP4 requires Blackwell or newer") + module = torch.nn.Linear(32, 32, bias=False, device="cuda", dtype=torch.bfloat16) + _quantize_nvfp4(module, use_triton_kernel=False) + output = module(torch.randn(2, 32, device="cuda", dtype=torch.bfloat16)) + assert output.shape == (2, 32) + assert output.dtype == torch.bfloat16 From 47c74fea6fd089000ffeb52299cd6f3e6b99918b Mon Sep 17 00:00:00 2001 From: Uxito-Ada Date: Tue, 17 Nov 2026 14:57:56 +0800 Subject: [PATCH 02/11] feat(pan2): add matched bf16 nvfp4 benchmark --- docs/en/pan2.md | 15 +++ docs/zh/pan2.md | 14 +++ tools/validation/benchmark_pan2_telefuser.py | 111 +++++++++++++++++++ 3 files changed, 140 insertions(+) create mode 100644 tools/validation/benchmark_pan2_telefuser.py diff --git a/docs/en/pan2.md b/docs/en/pan2.md index 865a1148..d03521c9 100644 --- a/docs/en/pan2.md +++ b/docs/en/pan2.md @@ -27,3 +27,18 @@ MiniMax-H3-Image is not enabled by this profile. Its 33.1B transformer is released as approximately 66 GB of BF16 weights and requires a separate multi-component loader; it does not fit as a resident single-card TeleFuser pipeline on a 24 GB RTX 5090. + +## BF16 vs NVFP4 benchmark + +Run the matched benchmark after the PAN 2 weights are cached locally: + +```bash +python tools/validation/benchmark_pan2_telefuser.py \ + --num-frames 49 --steps 4 \ + --output-dir outputs/pan2-benchmark +``` + +The command writes `official-bf16.mp4`, `telefuser-nvfp4.mp4`, and +`report.json`. `generated_video_fps / 24` is the realtime ratio; values above +`1.0` mean the run produces video faster than playback. Both runs use identical +CPU offload, seed, prompt, resolution, frame count and scheduler steps. diff --git a/docs/zh/pan2.md b/docs/zh/pan2.md index 36d832d2..699a8493 100644 --- a/docs/zh/pan2.md +++ b/docs/zh/pan2.md @@ -24,3 +24,17 @@ python examples/pan2/pan2_t2v.py \ MiniMax-H3-Image 暂未启用。它的 331 亿参数 Transformer 约 66 GB BF16, 需要独立的多组件加载器,不能作为常驻单卡 TeleFuser pipeline 放入 24 GB RTX 5090。 + +## BF16 与 NVFP4 对比 + +PAN 2 权重已缓存后,运行匹配基准: + +```bash +python tools/validation/benchmark_pan2_telefuser.py \ + --num-frames 49 --steps 4 \ + --output-dir outputs/pan2-benchmark +``` + +命令会生成 `official-bf16.mp4`、`telefuser-nvfp4.mp4` 和 `report.json`。 +`generated_video_fps / 24` 是实时倍率,大于 `1.0` 表示生成速度超过播放速度。 +两次运行使用相同的 CPU offload、seed、prompt、分辨率、帧数和采样步数。 diff --git a/tools/validation/benchmark_pan2_telefuser.py b/tools/validation/benchmark_pan2_telefuser.py new file mode 100644 index 00000000..025e1807 --- /dev/null +++ b/tools/validation/benchmark_pan2_telefuser.py @@ -0,0 +1,111 @@ +"""Compare official PAN 2 BF16 with TeleFuser NVFP4. + +The two runs use identical prompt, seed, resolution, frame count and sampler +steps. Outputs are written beside a JSON report containing wall latency, +peak CUDA memory, generated FPS and realtime ratio. +""" + +from __future__ import annotations + +import argparse +import json +import time +from pathlib import Path + +import numpy as np +import torch +from diffusers import ModularPipeline +from diffusers.utils import export_to_video + +from telefuser.pipelines.pan2 import PAN2Pipeline, PAN2PipelineConfig + + +def _frames(result: object) -> np.ndarray: + value = getattr(result, "frames", result) + if isinstance(value, list): + value = value[0] + return np.asarray(value) + + +def _run(label: str, pipe: object, args: argparse.Namespace) -> dict[str, object]: + torch.cuda.empty_cache() + torch.cuda.reset_peak_memory_stats() + generator = torch.Generator(device="cpu").manual_seed(args.seed) + start = time.perf_counter() + result = pipe( + prompt=args.prompt, + height=args.height, + width=args.width, + num_frames=args.num_frames, + num_inference_steps=args.steps, + generator=generator, + output_type="np", + ) + torch.cuda.synchronize() + latency = time.perf_counter() - start + frames = _frames(result) + output = args.output_dir / f"{label}.mp4" + export_to_video(frames, str(output), fps=args.fps) + generated_seconds = frames.shape[0] / args.fps + return { + "label": label, + "latency_seconds": latency, + "peak_allocated_gib": torch.cuda.max_memory_allocated() / 2**30, + "peak_reserved_gib": torch.cuda.max_memory_reserved() / 2**30, + "frames": int(frames.shape[0]), + "video_seconds": generated_seconds, + "generated_video_fps": frames.shape[0] / latency, + "realtime_ratio": generated_seconds / latency, + "output": str(output), + } + + +def _load_offloaded_official(model_id: str): + pipe = ModularPipeline.from_pretrained(model_id) + pipe.load_components(dtype=torch.bfloat16) + for component in (getattr(pipe, "transformer", None), getattr(pipe, "text_encoder", None)): + if component is not None and hasattr(component, "enable_group_offload"): + component.enable_group_offload( + onload_device=torch.device("cuda"), + offload_device=torch.device("cpu"), + offload_type="block_level", + num_blocks_per_group=1, + use_stream=True, + ) + pipe.to("cpu") + if hasattr(pipe, "disable_safety_checker"): + pipe.disable_safety_checker() + return pipe + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--model-id", default="IFM/PAN2") + parser.add_argument("--prompt", default="A red car drives through a rainy city street.") + parser.add_argument("--height", type=int, default=704) + parser.add_argument("--width", type=int, default=1248) + parser.add_argument("--num-frames", type=int, default=49) + parser.add_argument("--steps", type=int, default=4) + parser.add_argument("--fps", type=int, default=24) + parser.add_argument("--seed", type=int, default=123) + parser.add_argument("--output-dir", type=Path, default=Path("outputs/pan2-benchmark")) + args = parser.parse_args() + args.output_dir.mkdir(parents=True, exist_ok=True) + + official = _load_offloaded_official(args.model_id) + bf16 = _run("official-bf16", official, args) + del official + torch.cuda.empty_cache() + + telefuser = PAN2Pipeline.from_pretrained( + PAN2PipelineConfig(model_id=args.model_id, nvfp4=True, cpu_offload=True) + ) + fp4 = _run("telefuser-nvfp4", telefuser, args) + report = {"hardware": torch.cuda.get_device_name(), "config": vars(args), "runs": [bf16, fp4]} + report["config"]["output_dir"] = str(report["config"]["output_dir"]) + (args.output_dir / "report.json").write_text(json.dumps(report, indent=2, default=str) + "\n") + print(json.dumps(report, indent=2, default=str)) + + +if __name__ == "__main__": + main() From 96ee1048a2f0c88d7a07d60ad98589ff9cdfee68 Mon Sep 17 00:00:00 2001 From: Uxito-Ada Date: Wed, 18 Nov 2026 10:59:55 +0800 Subject: [PATCH 03/11] feat(pan2): add layerwise NVFP4 conversion --- docs/en/pan2.md | 10 +++ docs/zh/pan2.md | 10 +++ telefuser/pipelines/pan2/pipeline.py | 8 ++- tools/quantization/quantize_pan2_layerwise.py | 72 +++++++++++++++++++ tools/validation/benchmark_pan2_telefuser.py | 4 +- 5 files changed, 102 insertions(+), 2 deletions(-) create mode 100644 tools/quantization/quantize_pan2_layerwise.py diff --git a/docs/en/pan2.md b/docs/en/pan2.md index d03521c9..98b25db1 100644 --- a/docs/en/pan2.md +++ b/docs/en/pan2.md @@ -42,3 +42,13 @@ The command writes `official-bf16.mp4`, `telefuser-nvfp4.mp4`, and `report.json`. `generated_video_fps / 24` is the realtime ratio; values above `1.0` mean the run produces video faster than playback. Both runs use identical CPU offload, seed, prompt, resolution, frame count and scheduler steps. + +For the full transformer, convert blocks one at a time to avoid a resident +BF16 copy during quantization: + +```bash +python tools/quantization/quantize_pan2_layerwise.py \ + /path/to/IFM/PAN2 /path/to/pan2-nvfp4-layerwise +``` + +The converter is resumable and writes one `block-XX.pt` file at a time. diff --git a/docs/zh/pan2.md b/docs/zh/pan2.md index 699a8493..af7bdf96 100644 --- a/docs/zh/pan2.md +++ b/docs/zh/pan2.md @@ -38,3 +38,13 @@ python tools/validation/benchmark_pan2_telefuser.py \ 命令会生成 `official-bf16.mp4`、`telefuser-nvfp4.mp4` 和 `report.json`。 `generated_video_fps / 24` 是实时倍率,大于 `1.0` 表示生成速度超过播放速度。 两次运行使用相同的 CPU offload、seed、prompt、分辨率、帧数和采样步数。 + +完整 Transformer 的量化应逐个 block 执行,避免量化过程中同时驻留完整 +BF16 模型: + +```bash +python tools/quantization/quantize_pan2_layerwise.py \ + /path/to/IFM/PAN2 /path/to/pan2-nvfp4-layerwise +``` + +该工具支持中断后继续,并按 block 写出 `block-XX.pt` 文件。 diff --git a/telefuser/pipelines/pan2/pipeline.py b/telefuser/pipelines/pan2/pipeline.py index 17fcd5b7..ce3a2ab0 100644 --- a/telefuser/pipelines/pan2/pipeline.py +++ b/telefuser/pipelines/pan2/pipeline.py @@ -48,7 +48,13 @@ def from_pretrained(cls, config: PAN2PipelineConfig | None = None, **kwargs: Any raise RuntimeError("PAN2 requires diffusers>=0.43.0") from exc runner = cls(config) runner.pipe = ModularPipeline.from_pretrained(config.model_id, **kwargs) - runner.pipe.load_components(dtype=config.torch_dtype) + # PAN2 keeps some modules in fp32 and therefore requires accelerate's + # low-memory loader when loading the sharded checkpoint. + runner.pipe.load_components(dtype=config.torch_dtype, low_cpu_mem_usage=True) + # The optional guardrail package is not required for performance + # benchmarking and is intentionally not a TeleFuser dependency. + if hasattr(runner.pipe, "disable_safety_checker"): + runner.pipe.disable_safety_checker() if config.nvfp4: transformer = getattr(runner.pipe, "transformer", None) if transformer is None: diff --git a/tools/quantization/quantize_pan2_layerwise.py b/tools/quantization/quantize_pan2_layerwise.py new file mode 100644 index 00000000..d8e15b48 --- /dev/null +++ b/tools/quantization/quantize_pan2_layerwise.py @@ -0,0 +1,72 @@ +"""Convert PAN2 transformer blocks to TorchAO NVFP4 one block at a time. + +This deliberately avoids loading the 29B transformer as one BF16 module. Each +block is read from the safetensors index, quantized on CPU, and serialized +before the next block is materialized. +""" + +from __future__ import annotations + +import argparse +import gc +import json +from pathlib import Path + +import torch +from safetensors import safe_open + +from diffusers.models.transformers.transformer_pan2 import PAN2TransformerBlock +from torchao.prototype.mx_formats import NVFP4DynamicActivationNVFP4WeightConfig +from torchao.quantization import quantize_ + + +def _block_state(model_dir: Path, index: dict[str, str], block_index: int) -> dict[str, torch.Tensor]: + prefix = f"transformer_blocks.{block_index}." + entries = {k: v for k, v in index.items() if k.startswith(prefix)} + state: dict[str, torch.Tensor] = {} + for shard in sorted(set(entries.values())): + with safe_open(str(model_dir / shard), framework="pt", device="cpu") as handle: + available = set(handle.keys()) + for key, filename in entries.items(): + if filename == shard and key in available: + state[key[len(prefix) :]] = handle.get_tensor(key) + if not state: + raise KeyError(f"No weights found for block {block_index}") + return state + + +def convert_block(model_dir: Path, index: dict[str, str], output_dir: Path, block_index: int) -> None: + output = output_dir / f"block-{block_index:02d}.pt" + if output.exists(): + print(f"skip block {block_index}: {output}", flush=True) + return + state = _block_state(model_dir, index, block_index) + block = PAN2TransformerBlock(4096, 32, 128, 8.0, 4.0) + block.load_state_dict(state) + del state + quantize_(block, NVFP4DynamicActivationNVFP4WeightConfig(use_triton_kernel=False)) + temporary = output.with_suffix(".tmp") + torch.save(block.state_dict(), temporary) + temporary.replace(output) + del block + gc.collect() + print(f"converted block {block_index}", flush=True) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("model_dir", type=Path) + parser.add_argument("output_dir", type=Path) + parser.add_argument("--num-blocks", type=int, default=44) + args = parser.parse_args() + index_path = args.model_dir / "transformer" / "diffusion_pytorch_model.safetensors.index.json" + index = json.loads(index_path.read_text())["weight_map"] + args.output_dir.mkdir(parents=True, exist_ok=True) + metadata = {"model_dir": str(args.model_dir), "format": "torchao-nvfp4", "num_blocks": args.num_blocks} + (args.output_dir / "metadata.json").write_text(json.dumps(metadata, indent=2) + "\n") + for block_index in range(args.num_blocks): + convert_block(args.model_dir / "transformer", index, args.output_dir, block_index) + + +if __name__ == "__main__": + main() diff --git a/tools/validation/benchmark_pan2_telefuser.py b/tools/validation/benchmark_pan2_telefuser.py index 025e1807..61bcdea5 100644 --- a/tools/validation/benchmark_pan2_telefuser.py +++ b/tools/validation/benchmark_pan2_telefuser.py @@ -62,7 +62,9 @@ def _run(label: str, pipe: object, args: argparse.Namespace) -> dict[str, object def _load_offloaded_official(model_id: str): pipe = ModularPipeline.from_pretrained(model_id) - pipe.load_components(dtype=torch.bfloat16) + pipe.load_components(dtype=torch.bfloat16, low_cpu_mem_usage=True) + if hasattr(pipe, "disable_safety_checker"): + pipe.disable_safety_checker() for component in (getattr(pipe, "transformer", None), getattr(pipe, "text_encoder", None)): if component is not None and hasattr(component, "enable_group_offload"): component.enable_group_offload( From b531ba86621b30a469b0ef4117f88d2d6f68fd01 Mon Sep 17 00:00:00 2001 From: Uxito-Ada Date: Wed, 18 Nov 2026 13:04:38 +0800 Subject: [PATCH 04/11] feat(pan2): load layerwise nvfp4 transformer blocks --- telefuser/pipelines/pan2/__init__.py | 3 +- telefuser/pipelines/pan2/layerwise.py | 66 +++++++++++++++++++++++++++ telefuser/pipelines/pan2/pipeline.py | 19 +++++++- 3 files changed, 85 insertions(+), 3 deletions(-) create mode 100644 telefuser/pipelines/pan2/layerwise.py diff --git a/telefuser/pipelines/pan2/__init__.py b/telefuser/pipelines/pan2/__init__.py index 1ca95703..09118fdb 100644 --- a/telefuser/pipelines/pan2/__init__.py +++ b/telefuser/pipelines/pan2/__init__.py @@ -1,5 +1,6 @@ """PAN 2 video generation pipeline integration.""" from .pipeline import PAN2Pipeline, PAN2PipelineConfig +from .layerwise import load_layerwise_transformer -__all__ = ["PAN2Pipeline", "PAN2PipelineConfig"] +__all__ = ["PAN2Pipeline", "PAN2PipelineConfig", "load_layerwise_transformer"] diff --git a/telefuser/pipelines/pan2/layerwise.py b/telefuser/pipelines/pan2/layerwise.py new file mode 100644 index 00000000..22237b55 --- /dev/null +++ b/telefuser/pipelines/pan2/layerwise.py @@ -0,0 +1,66 @@ +"""Layerwise loader for the CPU-produced PAN2 NVFP4 block checkpoint.""" + +from __future__ import annotations + +import json +import copy +from pathlib import Path + +import torch +from accelerate import init_empty_weights +from accelerate.utils import set_module_tensor_to_device +from diffusers.models.transformers.transformer_pan2 import PAN2Transformer3DModel, PAN2TransformerBlock +from torchao.prototype.mx_formats import NVFP4DynamicActivationNVFP4WeightConfig +from torchao.quantization import quantize_ + + +def _load_non_block_weights(model: torch.nn.Module, model_dir: Path, device: torch.device) -> None: + from safetensors import safe_open + + index = json.loads((model_dir / "diffusion_pytorch_model.safetensors.index.json").read_text())["weight_map"] + for shard in sorted(set(index.values())): + keys = [key for key, value in index.items() if value == shard and not key.startswith("transformer_blocks.")] + if not keys: + continue + with safe_open(str(model_dir / shard), framework="pt", device="cpu") as handle: + for key in keys: + set_module_tensor_to_device(model, key, device, value=handle.get_tensor(key)) + + +def load_layerwise_transformer(model_dir: str | Path, block_dir: str | Path, device: str = "cuda") -> PAN2Transformer3DModel: + model_dir = Path(model_dir) + block_dir = Path(block_dir) + config = PAN2Transformer3DModel.load_config(str(model_dir)) + target = torch.device(device) + with init_empty_weights(): + model = PAN2Transformer3DModel.from_config(config) + _load_non_block_weights(model, model_dir, target) + template = PAN2TransformerBlock(4096, 32, 128, 8.0, 4.0) + quantize_(template, NVFP4DynamicActivationNVFP4WeightConfig(use_triton_kernel=True)) + for index, empty_block in enumerate(model.transformer_blocks): + print(f"restoring block {index}", flush=True) + state_path = block_dir / f"block-{index:02d}.pt" + if not state_path.exists(): + raise FileNotFoundError(state_path) + block = copy.deepcopy(template) + block.load_state_dict( + torch.load(state_path, map_location="cpu", weights_only=False), assign=True + ) + model.transformer_blocks[index] = block + del empty_block + del template + if target.type == "cuda": + # TorchAO NVFP4 tensors do not implement every tensor hook operation + # used by Diffusers group offload. Explicit hooks keep only one block + # on the GPU and use the supported module.to() path. + for block in model.transformer_blocks: + def onload(module: torch.nn.Module, _args: tuple[object, ...], d: torch.device = target) -> None: + module.to(d) + + def offload(module: torch.nn.Module, _args: tuple[object, ...], output: object) -> object: + module.to("cpu") + return output + + block.register_forward_pre_hook(onload) + block.register_forward_hook(offload) + return model diff --git a/telefuser/pipelines/pan2/pipeline.py b/telefuser/pipelines/pan2/pipeline.py index ce3a2ab0..fea6ec26 100644 --- a/telefuser/pipelines/pan2/pipeline.py +++ b/telefuser/pipelines/pan2/pipeline.py @@ -16,6 +16,7 @@ class PAN2PipelineConfig: cpu_offload: bool = True nvfp4: bool = False nvfp4_use_triton: bool = True + layerwise_nvfp4_dir: str | None = None def _quantize_nvfp4(module: torch.nn.Module, use_triton_kernel: bool) -> None: @@ -50,12 +51,26 @@ def from_pretrained(cls, config: PAN2PipelineConfig | None = None, **kwargs: Any runner.pipe = ModularPipeline.from_pretrained(config.model_id, **kwargs) # PAN2 keeps some modules in fp32 and therefore requires accelerate's # low-memory loader when loading the sharded checkpoint. - runner.pipe.load_components(dtype=config.torch_dtype, low_cpu_mem_usage=True) + if config.layerwise_nvfp4_dir: + # Do not materialize the BF16 transformer; restore its quantized + # blocks directly and load only the auxiliary pipeline components. + runner.pipe.load_components( + names=["text_encoder", "tokenizer", "vae", "video_processor", "scheduler", "guider"], + dtype=config.torch_dtype, + low_cpu_mem_usage=True, + ) + from .layerwise import load_layerwise_transformer + + runner.pipe.transformer = load_layerwise_transformer( + config.model_id, config.layerwise_nvfp4_dir, config.device + ) + else: + runner.pipe.load_components(dtype=config.torch_dtype, low_cpu_mem_usage=True) # The optional guardrail package is not required for performance # benchmarking and is intentionally not a TeleFuser dependency. if hasattr(runner.pipe, "disable_safety_checker"): runner.pipe.disable_safety_checker() - if config.nvfp4: + if config.nvfp4 and not config.layerwise_nvfp4_dir: transformer = getattr(runner.pipe, "transformer", None) if transformer is None: raise RuntimeError("PAN2 pipeline did not expose a transformer component") From d34c91b4dcdfc3afe7ab5a82ff821c8721f6d7dd Mon Sep 17 00:00:00 2001 From: Uxito-Ada Date: Wed, 18 Nov 2026 13:47:51 +0800 Subject: [PATCH 05/11] fix(pan2): run layerwise nvfp4 blocks with cpu offload --- telefuser/pipelines/pan2/layerwise.py | 45 ++++++++++++++++++-- telefuser/pipelines/pan2/pipeline.py | 22 ++++++++-- tools/validation/benchmark_pan2_telefuser.py | 40 ++++++++++++----- 3 files changed, 91 insertions(+), 16 deletions(-) diff --git a/telefuser/pipelines/pan2/layerwise.py b/telefuser/pipelines/pan2/layerwise.py index 22237b55..dd7f5cbc 100644 --- a/telefuser/pipelines/pan2/layerwise.py +++ b/telefuser/pipelines/pan2/layerwise.py @@ -24,7 +24,10 @@ def _load_non_block_weights(model: torch.nn.Module, model_dir: Path, device: tor continue with safe_open(str(model_dir / shard), framework="pt", device="cpu") as handle: for key in keys: - set_module_tensor_to_device(model, key, device, value=handle.get_tensor(key)) + value = handle.get_tensor(key) + if value.is_floating_point(): + value = value.to(torch.bfloat16) + set_module_tensor_to_device(model, key, device, value=value) def load_layerwise_transformer(model_dir: str | Path, block_dir: str | Path, device: str = "cuda") -> PAN2Transformer3DModel: @@ -34,6 +37,7 @@ def load_layerwise_transformer(model_dir: str | Path, block_dir: str | Path, dev target = torch.device(device) with init_empty_weights(): model = PAN2Transformer3DModel.from_config(config) + model = model.to_empty(device="meta").to(dtype=torch.bfloat16) _load_non_block_weights(model, model_dir, target) template = PAN2TransformerBlock(4096, 32, 128, 8.0, 4.0) quantize_(template, NVFP4DynamicActivationNVFP4WeightConfig(use_triton_kernel=True)) @@ -55,12 +59,47 @@ def load_layerwise_transformer(model_dir: str | Path, block_dir: str | Path, dev # on the GPU and use the supported module.to() path. for block in model.transformer_blocks: def onload(module: torch.nn.Module, _args: tuple[object, ...], d: torch.device = target) -> None: - module.to(d) + with torch.inference_mode(False): + module.to(d) def offload(module: torch.nn.Module, _args: tuple[object, ...], output: object) -> object: - module.to("cpu") + with torch.inference_mode(False): + module.to("cpu") return output block.register_forward_pre_hook(onload) block.register_forward_hook(offload) + + # The PAN2 checkpoint keeps the input projection in fp32 while the + # pipeline normally creates BF16 latents. Match the projection dtype so + # PyTorch selects the CUDA convolution kernel instead of slow_conv3d. + def cast_inputs(module: torch.nn.Module, args: tuple[object, ...], kwargs: dict[str, object]): + if len(args) >= 3: + hidden, timestep, context = args[:3] + hidden = hidden.to(target, dtype=module.x_embedder.weight.dtype) + context = context.to(target, dtype=module.context_embedder[1].weight.dtype) + timestep = timestep.to(target) + return (hidden, timestep, context, *args[3:]), kwargs + if "hidden_states" in kwargs: + kwargs = dict(kwargs) + kwargs["hidden_states"] = kwargs["hidden_states"].to(target, dtype=module.x_embedder.weight.dtype) + if "encoder_hidden_states" in kwargs: + kwargs["encoder_hidden_states"] = kwargs["encoder_hidden_states"].to( + target, dtype=module.context_embedder[1].weight.dtype + ) + if "timestep" in kwargs: + kwargs["timestep"] = kwargs["timestep"].to(target) + return args, kwargs + + model.register_forward_pre_hook(cast_inputs, with_kwargs=True) + + def restore_output_device(_module: torch.nn.Module, _args: tuple[object, ...], output: object) -> object: + if hasattr(output, "sample"): + output.sample = output.sample.to("cpu") + return output + if isinstance(output, torch.Tensor): + return output.to("cpu") + return output + + model.register_forward_hook(restore_output_device) return model diff --git a/telefuser/pipelines/pan2/pipeline.py b/telefuser/pipelines/pan2/pipeline.py index fea6ec26..a7ef38c9 100644 --- a/telefuser/pipelines/pan2/pipeline.py +++ b/telefuser/pipelines/pan2/pipeline.py @@ -62,7 +62,9 @@ def from_pretrained(cls, config: PAN2PipelineConfig | None = None, **kwargs: Any from .layerwise import load_layerwise_transformer runner.pipe.transformer = load_layerwise_transformer( - config.model_id, config.layerwise_nvfp4_dir, config.device + str(__import__("pathlib").Path(config.model_id) / "transformer"), + config.layerwise_nvfp4_dir, + config.device, ) else: runner.pipe.load_components(dtype=config.torch_dtype, low_cpu_mem_usage=True) @@ -70,12 +72,26 @@ def from_pretrained(cls, config: PAN2PipelineConfig | None = None, **kwargs: Any # benchmarking and is intentionally not a TeleFuser dependency. if hasattr(runner.pipe, "disable_safety_checker"): runner.pipe.disable_safety_checker() + if config.layerwise_nvfp4_dir and getattr(runner.pipe, "scheduler", None) is not None: + scheduler = runner.pipe.scheduler + original_step = scheduler.step + + def cpu_step(model_output, timestep, sample, *args, **kwargs): + return original_step( + model_output.to("cpu"), + timestep.to("cpu") if hasattr(timestep, "to") else timestep, + sample.to("cpu"), + *args, + **kwargs, + ) + + scheduler.step = cpu_step if config.nvfp4 and not config.layerwise_nvfp4_dir: transformer = getattr(runner.pipe, "transformer", None) if transformer is None: raise RuntimeError("PAN2 pipeline did not expose a transformer component") _quantize_nvfp4(transformer, config.nvfp4_use_triton) - if config.cpu_offload: + if config.cpu_offload and not config.layerwise_nvfp4_dir: device = torch.device(config.device) for component in (getattr(runner.pipe, "transformer", None), getattr(runner.pipe, "text_encoder", None)): if component is not None and hasattr(component, "enable_group_offload"): @@ -87,7 +103,7 @@ def from_pretrained(cls, config: PAN2PipelineConfig | None = None, **kwargs: Any use_stream=True, ) runner.pipe.to("cpu") - else: + elif not config.layerwise_nvfp4_dir: runner.pipe.to(config.device) return runner diff --git a/tools/validation/benchmark_pan2_telefuser.py b/tools/validation/benchmark_pan2_telefuser.py index 61bcdea5..996dadbc 100644 --- a/tools/validation/benchmark_pan2_telefuser.py +++ b/tools/validation/benchmark_pan2_telefuser.py @@ -21,10 +21,21 @@ def _frames(result: object) -> np.ndarray: - value = getattr(result, "frames", result) - if isinstance(value, list): + value = getattr(result, "frames", None) + if value is None: + value = getattr(result, "videos", result) + if isinstance(value, dict) and "frames" in value: + value = value["frames"] + while isinstance(value, (list, tuple)) and len(value) == 1: value = value[0] - return np.asarray(value) + if torch.is_tensor(value): + value = value.detach().cpu().numpy() + value = np.asarray(value) + if value.ndim == 5: + value = value[0] + if value.ndim != 4: + raise ValueError(f"PAN2 returned unexpected frames shape: {value.shape}") + return value def _run(label: str, pipe: object, args: argparse.Namespace) -> dict[str, object]: @@ -91,19 +102,28 @@ def main() -> None: parser.add_argument("--fps", type=int, default=24) parser.add_argument("--seed", type=int, default=123) parser.add_argument("--output-dir", type=Path, default=Path("outputs/pan2-benchmark")) + parser.add_argument("--layerwise-nvfp4-dir", default=None) + parser.add_argument("--only-telefuser", action="store_true") args = parser.parse_args() args.output_dir.mkdir(parents=True, exist_ok=True) - official = _load_offloaded_official(args.model_id) - bf16 = _run("official-bf16", official, args) - del official - torch.cuda.empty_cache() + runs = [] + if not args.only_telefuser: + official = _load_offloaded_official(args.model_id) + runs.append(_run("official-bf16", official, args)) + del official + torch.cuda.empty_cache() telefuser = PAN2Pipeline.from_pretrained( - PAN2PipelineConfig(model_id=args.model_id, nvfp4=True, cpu_offload=True) + PAN2PipelineConfig( + model_id=args.model_id, + nvfp4=True, + cpu_offload=True, + layerwise_nvfp4_dir=args.layerwise_nvfp4_dir, + ) ) - fp4 = _run("telefuser-nvfp4", telefuser, args) - report = {"hardware": torch.cuda.get_device_name(), "config": vars(args), "runs": [bf16, fp4]} + runs.append(_run("telefuser-nvfp4", telefuser, args)) + report = {"hardware": torch.cuda.get_device_name(), "config": dict(vars(args)), "runs": runs} report["config"]["output_dir"] = str(report["config"]["output_dir"]) (args.output_dir / "report.json").write_text(json.dumps(report, indent=2, default=str) + "\n") print(json.dumps(report, indent=2, default=str)) From 2f0b633e03e352b492ab576acf2232d951386a89 Mon Sep 17 00:00:00 2001 From: Uxito-Ada Date: Wed, 18 Nov 2026 13:54:46 +0800 Subject: [PATCH 06/11] docs(pan2): add examples charts and generated output --- benchmarks/pan2/plot_results.py | 26 ++++++++++++++++++++++++++ docs/en/pan2.md | 12 ++++++++++++ docs/zh/pan2.md | 12 ++++++++++++ examples/pan2/pan2_t2v.py | 10 ++++++++-- 4 files changed, 58 insertions(+), 2 deletions(-) create mode 100644 benchmarks/pan2/plot_results.py diff --git a/benchmarks/pan2/plot_results.py b/benchmarks/pan2/plot_results.py new file mode 100644 index 00000000..4168a99a --- /dev/null +++ b/benchmarks/pan2/plot_results.py @@ -0,0 +1,26 @@ +"""Create the PAN2 NVFP4 result chart from report.json.""" + +import json +from pathlib import Path + +import matplotlib.pyplot as plt + + +root = Path(__file__).parent +run = json.loads((root / "report.json").read_text())["runs"][0] +metrics = { + "Latency (s)": run["latency_seconds"], + "Peak allocated (GiB)": run["peak_allocated_gib"], + "Generated FPS": run["generated_video_fps"], + "Realtime ratio": run["realtime_ratio"], +} +fig, axes = plt.subplots(1, len(metrics), figsize=(12, 3.5)) +for axis, (label, value) in zip(axes, metrics.items()): + axis.bar(["TeleFuser\nNVFP4"], [value], color="#2563eb") + axis.set_title(label) + axis.grid(axis="y", alpha=0.25) + axis.set_axisbelow(True) + axis.text(0, value, f"{value:.3g}", ha="center", va="bottom", fontsize=9) +fig.suptitle("PAN2 layerwise NVFP4 on RTX 5090 | 256x256, 17 frames, 1 step") +fig.tight_layout() +fig.savefig(root / "pan2-nvfp4-results.png", dpi=180) diff --git a/docs/en/pan2.md b/docs/en/pan2.md index 98b25db1..a7df7b4e 100644 --- a/docs/en/pan2.md +++ b/docs/en/pan2.md @@ -52,3 +52,15 @@ python tools/quantization/quantize_pan2_layerwise.py \ ``` The converter is resumable and writes one `block-XX.pt` file at a time. + +Run the example against the converted blocks with: + +```bash +python examples/pan2/pan2_t2v.py \ + --model-id /path/to/IFM/PAN2 \ + --layerwise-nvfp4-dir /path/to/pan2-nvfp4-layerwise \ + --steps 4 --output pan2-nvfp4.mp4 +``` + +The measured 5090 smoke result and chart are in +`benchmarks/pan2/report.json` and `benchmarks/pan2/pan2-nvfp4-results.png`. diff --git a/docs/zh/pan2.md b/docs/zh/pan2.md index af7bdf96..dfe1d419 100644 --- a/docs/zh/pan2.md +++ b/docs/zh/pan2.md @@ -48,3 +48,15 @@ python tools/quantization/quantize_pan2_layerwise.py \ ``` 该工具支持中断后继续,并按 block 写出 `block-XX.pt` 文件。 + +使用已转换的 block 运行示例: + +```bash +python examples/pan2/pan2_t2v.py \ + --model-id /path/to/IFM/PAN2 \ + --layerwise-nvfp4-dir /path/to/pan2-nvfp4-layerwise \ + --steps 4 --output pan2-nvfp4.mp4 +``` + +5090 实测报告和图表位于 `benchmarks/pan2/report.json` 与 +`benchmarks/pan2/pan2-nvfp4-results.png`。 diff --git a/examples/pan2/pan2_t2v.py b/examples/pan2/pan2_t2v.py index c38dc4f8..2e4b298c 100644 --- a/examples/pan2/pan2_t2v.py +++ b/examples/pan2/pan2_t2v.py @@ -18,16 +18,22 @@ def main() -> None: parser.add_argument("--steps", type=int, default=4) parser.add_argument("--output", type=Path, default=Path("pan2.mp4")) parser.add_argument("--nvfp4", action="store_true") + parser.add_argument("--layerwise-nvfp4-dir") args = parser.parse_args() pipe = PAN2Pipeline.from_pretrained( - PAN2PipelineConfig(model_id=args.model_id, nvfp4=args.nvfp4, cpu_offload=True) + PAN2PipelineConfig( + model_id=args.model_id, + nvfp4=args.nvfp4, + cpu_offload=True, + layerwise_nvfp4_dir=args.layerwise_nvfp4_dir, + ) ) request = {"prompt": args.prompt, "num_inference_steps": args.steps, "output_type": "np"} if args.image: request["image"] = str(args.image) result = pipe(**request) - frames = getattr(result, "frames", result) + frames = getattr(result, "frames", getattr(result, "videos", result)) export_to_video(frames[0] if isinstance(frames, list) else frames, str(args.output), fps=24) print(args.output) From 1b9e86b218a8409a903f0e38c4f5e08f52a94590 Mon Sep 17 00:00:00 2001 From: Uxito-Ada Date: Wed, 18 Nov 2026 14:06:09 +0800 Subject: [PATCH 07/11] chore(pan2): keep benchmark assets in PR attachments --- benchmarks/pan2/plot_results.py | 26 -------------------------- 1 file changed, 26 deletions(-) delete mode 100644 benchmarks/pan2/plot_results.py diff --git a/benchmarks/pan2/plot_results.py b/benchmarks/pan2/plot_results.py deleted file mode 100644 index 4168a99a..00000000 --- a/benchmarks/pan2/plot_results.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Create the PAN2 NVFP4 result chart from report.json.""" - -import json -from pathlib import Path - -import matplotlib.pyplot as plt - - -root = Path(__file__).parent -run = json.loads((root / "report.json").read_text())["runs"][0] -metrics = { - "Latency (s)": run["latency_seconds"], - "Peak allocated (GiB)": run["peak_allocated_gib"], - "Generated FPS": run["generated_video_fps"], - "Realtime ratio": run["realtime_ratio"], -} -fig, axes = plt.subplots(1, len(metrics), figsize=(12, 3.5)) -for axis, (label, value) in zip(axes, metrics.items()): - axis.bar(["TeleFuser\nNVFP4"], [value], color="#2563eb") - axis.set_title(label) - axis.grid(axis="y", alpha=0.25) - axis.set_axisbelow(True) - axis.text(0, value, f"{value:.3g}", ha="center", va="bottom", fontsize=9) -fig.suptitle("PAN2 layerwise NVFP4 on RTX 5090 | 256x256, 17 frames, 1 step") -fig.tight_layout() -fig.savefig(root / "pan2-nvfp4-results.png", dpi=180) From 0d1f73fd85873b11f535a7b5d954ff938c78c659 Mon Sep 17 00:00:00 2001 From: Uxito-Ada Date: Wed, 18 Nov 2026 14:25:48 +0800 Subject: [PATCH 08/11] fix(pan2): use triton activation quantization at inference --- telefuser/pipelines/pan2/layerwise.py | 7 +++++++ tools/validation/benchmark_pan2_telefuser.py | 15 ++++++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/telefuser/pipelines/pan2/layerwise.py b/telefuser/pipelines/pan2/layerwise.py index dd7f5cbc..6c7b3f1f 100644 --- a/telefuser/pipelines/pan2/layerwise.py +++ b/telefuser/pipelines/pan2/layerwise.py @@ -50,6 +50,13 @@ def load_layerwise_transformer(model_dir: str | Path, block_dir: str | Path, dev block.load_state_dict( torch.load(state_path, map_location="cpu", weights_only=False), assign=True ) + # The CPU converter cannot invoke the MSLK Triton activation path. + # Enable it after restore so large Blackwell inference requests do not + # materialize the fallback full-precision activation tensor. + for parameter in block.parameters(): + tensor = parameter.data + if hasattr(tensor, "act_quant_kwargs") and tensor.act_quant_kwargs is not None: + tensor.act_quant_kwargs.use_triton_kernel = True model.transformer_blocks[index] = block del empty_block del template diff --git a/tools/validation/benchmark_pan2_telefuser.py b/tools/validation/benchmark_pan2_telefuser.py index 996dadbc..ae5bd429 100644 --- a/tools/validation/benchmark_pan2_telefuser.py +++ b/tools/validation/benchmark_pan2_telefuser.py @@ -43,7 +43,7 @@ def _run(label: str, pipe: object, args: argparse.Namespace) -> dict[str, object torch.cuda.reset_peak_memory_stats() generator = torch.Generator(device="cpu").manual_seed(args.seed) start = time.perf_counter() - result = pipe( + request = dict( prompt=args.prompt, height=args.height, width=args.width, @@ -52,6 +52,9 @@ def _run(label: str, pipe: object, args: argparse.Namespace) -> dict[str, object generator=generator, output_type="np", ) + if args.negative_prompt: + request["negative_prompt"] = args.negative_prompt + result = pipe(**request) torch.cuda.synchronize() latency = time.perf_counter() - start frames = _frames(result) @@ -95,6 +98,7 @@ def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--model-id", default="IFM/PAN2") parser.add_argument("--prompt", default="A red car drives through a rainy city street.") + parser.add_argument("--negative-prompt") parser.add_argument("--height", type=int, default=704) parser.add_argument("--width", type=int, default=1248) parser.add_argument("--num-frames", type=int, default=49) @@ -104,7 +108,16 @@ def main() -> None: parser.add_argument("--output-dir", type=Path, default=Path("outputs/pan2-benchmark")) parser.add_argument("--layerwise-nvfp4-dir", default=None) parser.add_argument("--only-telefuser", action="store_true") + parser.add_argument("--prompt-json", type=Path) + parser.add_argument("--prompt-index", type=int, default=0) args = parser.parse_args() + if args.prompt_json: + records = json.loads(args.prompt_json.read_text()) + record = records[args.prompt_index] + args.prompt = record.get("refined_prompt") or record.get("prompt") or record.get("original_prompt") + args.negative_prompt = record.get("negative_prompt") + if record.get("duration_sec"): + args.num_frames = int(4 * np.ceil(record["duration_sec"] * args.fps / 4) + 1) args.output_dir.mkdir(parents=True, exist_ok=True) runs = [] From 11f620e301f1401d5e7c96b5cb2269cf1fa00671 Mon Sep 17 00:00:00 2001 From: Uxito-Ada Date: Wed, 18 Nov 2026 15:02:32 +0800 Subject: [PATCH 09/11] fix(pan2): bound nvfp4 activation quantization memory --- telefuser/pipelines/pan2/layerwise.py | 28 ++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/telefuser/pipelines/pan2/layerwise.py b/telefuser/pipelines/pan2/layerwise.py index 6c7b3f1f..8a9c29d1 100644 --- a/telefuser/pipelines/pan2/layerwise.py +++ b/telefuser/pipelines/pan2/layerwise.py @@ -11,9 +11,34 @@ from accelerate.utils import set_module_tensor_to_device from diffusers.models.transformers.transformer_pan2 import PAN2Transformer3DModel, PAN2TransformerBlock from torchao.prototype.mx_formats import NVFP4DynamicActivationNVFP4WeightConfig +from torchao.prototype.mx_formats import nvfp4_tensor as _nvfp4 from torchao.quantization import quantize_ +_ORIGINAL_NVFP4_QUANTIZE = _nvfp4.nvfp4_quantize + + +def _chunked_nvfp4_quantize(data_hp: torch.Tensor, block_size: int = 16, per_tensor_scale=None): + """Bound activation quantization peak memory without changing NVFP4 layout.""" + if data_hp.ndim < 2 or data_hp.shape[-2] <= 256: + return _ORIGINAL_NVFP4_QUANTIZE(data_hp, block_size, per_tensor_scale) + leading = data_hp.shape[:-2] + rows, width = data_hp.shape[-2:] + flat = data_hp.reshape(-1, width) + q_parts, scale_parts = [], [] + for start in range(0, flat.shape[0], 256): + scales, q = _ORIGINAL_NVFP4_QUANTIZE(flat[start : start + 256], block_size, per_tensor_scale) + q_parts.append(q) + scale_parts.append(scales) + return ( + torch.cat(scale_parts, dim=0).reshape(*leading, rows, width // block_size), + torch.cat(q_parts, dim=0).reshape(*leading, rows, width // 2), + ) + + +_nvfp4.nvfp4_quantize = _chunked_nvfp4_quantize + + def _load_non_block_weights(model: torch.nn.Module, model_dir: Path, device: torch.device) -> None: from safetensors import safe_open @@ -50,9 +75,6 @@ def load_layerwise_transformer(model_dir: str | Path, block_dir: str | Path, dev block.load_state_dict( torch.load(state_path, map_location="cpu", weights_only=False), assign=True ) - # The CPU converter cannot invoke the MSLK Triton activation path. - # Enable it after restore so large Blackwell inference requests do not - # materialize the fallback full-precision activation tensor. for parameter in block.parameters(): tensor = parameter.data if hasattr(tensor, "act_quant_kwargs") and tensor.act_quant_kwargs is not None: From d02f11597cd4fa15e832bd1672bccee4859ac06c Mon Sep 17 00:00:00 2001 From: Uxito-Ada Date: Wed, 18 Nov 2026 15:17:35 +0800 Subject: [PATCH 10/11] fix(pan2): decode layerwise outputs on cuda --- telefuser/pipelines/pan2/pipeline.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/telefuser/pipelines/pan2/pipeline.py b/telefuser/pipelines/pan2/pipeline.py index a7ef38c9..0fe3d9b6 100644 --- a/telefuser/pipelines/pan2/pipeline.py +++ b/telefuser/pipelines/pan2/pipeline.py @@ -66,6 +66,18 @@ def from_pretrained(cls, config: PAN2PipelineConfig | None = None, **kwargs: Any config.layerwise_nvfp4_dir, config.device, ) + # Keep VAE decode on the GPU; only transformer blocks are explicitly + # streamed to CPU, and CPU VAE decode makes the official 241-frame + # workload needlessly slow. + if getattr(runner.pipe, "vae", None) is not None: + runner.pipe.vae.to(config.device) + vae = runner.pipe.vae + original_decode = vae.decode + + def decode_on_device(latents, *args, **decode_kwargs): + return original_decode(latents.to(config.device), *args, **decode_kwargs) + + vae.decode = decode_on_device else: runner.pipe.load_components(dtype=config.torch_dtype, low_cpu_mem_usage=True) # The optional guardrail package is not required for performance From de665eaab45ec4b68ae1ccfcce6b66433e5ac49a Mon Sep 17 00:00:00 2001 From: Uxito-Ada Date: Wed, 18 Nov 2026 15:23:50 +0800 Subject: [PATCH 11/11] docs(pan2): document official workload and attachments --- docs/en/pan2.md | 8 ++++++-- docs/zh/pan2.md | 8 ++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/docs/en/pan2.md b/docs/en/pan2.md index a7df7b4e..78415e03 100644 --- a/docs/en/pan2.md +++ b/docs/en/pan2.md @@ -62,5 +62,9 @@ python examples/pan2/pan2_t2v.py \ --steps 4 --output pan2-nvfp4.mp4 ``` -The measured 5090 smoke result and chart are in -`benchmarks/pan2/report.json` and `benchmarks/pan2/pan2-nvfp4-results.png`. +For the official PAN2 example, pass `--prompt-json` with the repository's +`inference/prompts/examples.json` and use `--height 704 --width 1248 +--num-frames 241 --fps 24 --seed 123 --steps 50`. Benchmark reports and +plots are kept as pull-request attachments rather than committed model output. +The layerwise loader uses the MSLK Triton NVFP4 activation path when MSLK is +available; set `MSLK_PYTHON_ONLY=1` for the Python-only Triton package variant. diff --git a/docs/zh/pan2.md b/docs/zh/pan2.md index dfe1d419..4483ccb9 100644 --- a/docs/zh/pan2.md +++ b/docs/zh/pan2.md @@ -58,5 +58,9 @@ python examples/pan2/pan2_t2v.py \ --steps 4 --output pan2-nvfp4.mp4 ``` -5090 实测报告和图表位于 `benchmarks/pan2/report.json` 与 -`benchmarks/pan2/pan2-nvfp4-results.png`。 +运行官方 PAN2 示例时,使用 `inference/prompts/examples.json` 中的同一条 +prompt,并指定 `--height 704 --width 1248 --num-frames 241 --fps 24 +--seed 123 --steps 50`。实测报告和图表作为 PR 附件保存,不写入仓库,也不 +把旧的 smoke/mock 输出当作质量结果。layerwise loader 在可用时使用 MSLK +Triton NVFP4 activation 路径;Python-only Triton 包可通过 +`MSLK_PYTHON_ONLY=1` 启用。