From cf3248c63fa93f3d5753b9eb06add4e7947c5889 Mon Sep 17 00:00:00 2001 From: jinyx5 Date: Tue, 1 Sep 2026 00:45:14 +0800 Subject: [PATCH 01/21] fix(models): gate channels_last_3d activations to CUDA devices The VAE weight-side conversion to channels_last_3d is already guarded by cuDNN availability, but the activation-side casts in CausalConv3d.forward and the spatial-parallel halo conv were unconditional. NPU and CPU reject channels_last_3d activations, so Wan2.2 VAE forward failed on Ascend with ERR01007 ("NPU contiguous operator only supported contiguous memory format"). Route non-CUDA tensors through standard contiguous instead. Verified: Wan2.2-TI2V-5B text-to-video smoke on Ascend 910B2 (CANN 8.2, torch_npu 2.9.0) completes end to end; CUDA path unchanged. --- telefuser/distributed/vae_spatial.py | 6 +++++- telefuser/models/wan_video_vae.py | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/telefuser/distributed/vae_spatial.py b/telefuser/distributed/vae_spatial.py index 85f2f410..ebeb30fe 100644 --- a/telefuser/distributed/vae_spatial.py +++ b/telefuser/distributed/vae_spatial.py @@ -128,7 +128,11 @@ def _spatial_causal_conv3d_forward( if any(padding): tensor = F.pad(tensor, padding) tensor = _exchange_height_halo(module, tensor, module._height_halo_size) - tensor = tensor.contiguous(memory_format=torch.channels_last_3d) + if tensor.device.type == "cuda": + tensor = tensor.contiguous(memory_format=torch.channels_last_3d) + else: + # channels_last_3d activations are only supported by cuDNN; NPU/CPU require standard contiguous. + tensor = tensor.contiguous() return F.conv3d( tensor, module.weight, diff --git a/telefuser/models/wan_video_vae.py b/telefuser/models/wan_video_vae.py index 22b4bc7c..941f8056 100644 --- a/telefuser/models/wan_video_vae.py +++ b/telefuser/models/wan_video_vae.py @@ -119,7 +119,11 @@ def forward(self, x: torch.Tensor, cache_x: torch.Tensor | None = None) -> torch x = torch.cat([cache_x, x], dim=2) padding[4] -= cache_x.shape[2] x = F.pad(x, padding) - x = x.contiguous(memory_format=torch.channels_last_3d) + if x.device.type == "cuda": + x = x.contiguous(memory_format=torch.channels_last_3d) + else: + # channels_last_3d activations are only supported by cuDNN; NPU/CPU require standard contiguous. + x = x.contiguous() return super().forward(x) From e9fd11636d80022837fe4b4797e9d5bf5579c0d7 Mon Sep 17 00:00:00 2001 From: jinyx5 Date: Tue, 1 Sep 2026 00:45:14 +0800 Subject: [PATCH 02/21] fix(distributed): default device mesh to the detected platform create_device_mesh_from_config defaulted device_type to "cuda" and 12 of 14 call sites rely on the default, so parallel denoising on NPU crashed inside torch DeviceMesh with "integer modulo by zero" (zero visible CUDA devices). Resolve the default from current_platform; explicit arguments keep working. Verified: 4-card (cfg=2 x ulysses=2) Wan2.2-TI2V-5B run on Ascend 910B2 builds the [cfg, ulysses] mesh over hccl; on CUDA the default resolves to "cuda" as before. --- telefuser/distributed/device_mesh.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/telefuser/distributed/device_mesh.py b/telefuser/distributed/device_mesh.py index 8281327e..ce3e69ff 100644 --- a/telefuser/distributed/device_mesh.py +++ b/telefuser/distributed/device_mesh.py @@ -13,10 +13,11 @@ from torch.distributed.device_mesh import DeviceMesh from telefuser.core.config import ParallelConfig +from telefuser.platforms import current_platform from telefuser.utils.logging import logger -def create_device_mesh_from_config(parallel_config: ParallelConfig, device_type: str = "cuda") -> DeviceMesh: +def create_device_mesh_from_config(parallel_config: ParallelConfig, device_type: str | None = None) -> DeviceMesh: """Create PyTorch DeviceMesh from ParallelConfig. Mesh dimensions are built in order: DP -> CFG -> SP (ring, ulysses) -> PP -> TP @@ -24,11 +25,13 @@ def create_device_mesh_from_config(parallel_config: ParallelConfig, device_type: Args: parallel_config: Parallel configuration with degrees for each dimension - device_type: Device type ("cuda" or "cpu") + device_type: Device type ("cuda", "npu", or "cpu"); defaults to the current platform's device type Returns: PyTorch DeviceMesh instance with named dimensions """ + if device_type is None: + device_type = current_platform.device_type _validate_parallel_config(parallel_config) sp_degree = parallel_config.sp_ulysses_degree * parallel_config.sp_ring_degree From 7d5c8f457b3e2f3aff758ddbe76cc9181c600078 Mon Sep 17 00:00:00 2001 From: jinyx5 Date: Tue, 1 Sep 2026 00:45:14 +0800 Subject: [PATCH 03/21] fix(worker): marshal parallel-worker queue tensors through CPU off CUDA Parallel workers exchange request and result tensors over torch.multiprocessing queues, which relies on device IPC. torch_npu cross-process sharing raised "devptr INTERNAL ASSERT FAILED ... entry in cache has missing shared_ptr" when rebuilding queued NPU tensors. Enable the existing queue_with_cpu marshalling by default on non-CUDA platforms, mirror it on the result path, and move results back to the stage device in the main process. Worker dispatch already moves inputs to the local device, so the contract is unchanged and CUDA keeps direct device-queue transport. Verified: 4-card Wan2.2-TI2V-5B denoising on Ascend 910B2 completes with workers exiting cleanly. --- telefuser/worker/parallel_worker.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/telefuser/worker/parallel_worker.py b/telefuser/worker/parallel_worker.py index 2c160544..9adb8450 100644 --- a/telefuser/worker/parallel_worker.py +++ b/telefuser/worker/parallel_worker.py @@ -158,6 +158,9 @@ def _worker_loop( y = tensor_output_channel.send(y) # Always output results when world_size=1 if world_size == 1 or rank == 0: + if current_platform.device_type != "cuda": + # Queue transport without reliable device IPC: marshal results through CPU. + y = to_device(y, "cpu") queue_out.put(y) except Exception as e: import traceback @@ -206,7 +209,8 @@ def __init__( self.device_ids = list(range(self.world_size)) self.name: str = f"Parallel Worker {stage.name}" - self.queue_with_cpu: bool = parallel_config.queue_with_cpu + # Queue transport requires CPU marshalling on platforms without reliable device IPC (e.g. NPU). + self.queue_with_cpu: bool = parallel_config.queue_with_cpu or current_platform.device_type != "cuda" self.timeout: int = parallel_config.timeout self._lifecycle_lock = threading.Lock() self._failed = False @@ -319,6 +323,9 @@ def _wait_result(self, method_name: str) -> Any: reason = f"{method_name} failed: {result}" self._mark_failed(reason) raise RuntimeError(f"ParallelWorker:{self.name} {reason}") from result + if current_platform.device_type != "cuda": + # CPU-marshalled queue results move back to the stage device for downstream consumers. + result = to_device(result, self._stage.device) return result def enable_metrics(self, registry: Any | None = None) -> None: From 74dd3abd613804509849884ecc243a9d61205abf Mon Sep 17 00:00:00 2001 From: jinyx5 Date: Tue, 1 Sep 2026 00:45:22 +0800 Subject: [PATCH 04/21] refactor(ops): move fused FP8 QKV Triton kernels into kernel.triton ops/fp8_attention.py defined @triton.jit kernels at module scope, making "import triton" unconditional for every consumer of the wan_video and minimax DiT import chains even though the fused path is CUDA-only by contract. Move the two kernels and their launcher to telefuser/kernel/triton/fp8_attention.py and import them lazily inside quantize_fp8_qkv after its existing CUDA validation, matching the established ops -> kernel.triton dispatch pattern (see ops/rotary.py and kernel/__init__.py). The pure-torch quantize/dequantize helpers are unchanged. Verified: wan22 pipeline import and Wan2.2-TI2V-5B smoke succeed on an NPU host without triton installed; tests/unit/models/test_wan_video_sol_attention.py passes with CUDA-only cases skipped. --- telefuser/kernel/triton/fp8_attention.py | 119 +++++++++++++++++++++++ telefuser/ops/fp8_attention.py | 110 +-------------------- 2 files changed, 123 insertions(+), 106 deletions(-) create mode 100644 telefuser/kernel/triton/fp8_attention.py diff --git a/telefuser/kernel/triton/fp8_attention.py b/telefuser/kernel/triton/fp8_attention.py new file mode 100644 index 00000000..a620f374 --- /dev/null +++ b/telefuser/kernel/triton/fp8_attention.py @@ -0,0 +1,119 @@ +"""Fused block-scaled FP8 Q/K/V quantization Triton kernels.""" + +from __future__ import annotations + +import torch +import triton +import triton.language as tl + + +@triton.jit +def _quantize_qkv_fp8_stage1( + q, + k, + v, + q_out, + k_out, + q_scale, + k_scale, + v_scale, + tokens: tl.constexpr, + heads: tl.constexpr, + head_dim: tl.constexpr, + block: tl.constexpr, +): + block_idx = tl.program_id(0) + batch_head = tl.program_id(1) + batch = batch_head // heads + head = batch_head % heads + token_offsets = block_idx * block + tl.arange(0, block) + dim_offsets = tl.arange(0, head_dim) + valid = token_offsets < tokens + offsets = ((batch * tokens + token_offsets[:, None]) * heads + head) * head_dim + dim_offsets[None, :] + q_values = tl.load(q + offsets, mask=valid[:, None], other=0.0).to(tl.float32) + k_values = tl.load(k + offsets, mask=valid[:, None], other=0.0).to(tl.float32) + v_values = tl.load(v + offsets, mask=valid[:, None], other=0.0).to(tl.float32) + + q_s = tl.maximum(tl.max(tl.max(tl.abs(q_values), axis=1), axis=0), 1.0e-6) / 448.0 + k_s = tl.maximum(tl.max(tl.max(tl.abs(k_values), axis=1), axis=0), 1.0e-6) / 448.0 + scale_offset = (batch * tl.cdiv(tokens, block) + block_idx) * heads + head + tl.store(q_scale + scale_offset, q_s) + tl.store(k_scale + scale_offset, k_s) + tl.store(q_out + offsets, q_values / q_s, mask=valid[:, None]) + tl.store(k_out + offsets, k_values / k_s, mask=valid[:, None]) + + v_s = tl.max(tl.abs(v_values), axis=0) / 448.0 + v_scale_offsets = (batch * heads + head) * head_dim + dim_offsets + tl.atomic_max(v_scale + v_scale_offsets, v_s) + + +@triton.jit +def _quantize_qkv_fp8_stage2_v( + v, + v_out, + v_scale, + tokens: tl.constexpr, + heads: tl.constexpr, + head_dim: tl.constexpr, + block: tl.constexpr, +): + block_idx = tl.program_id(0) + batch_head = tl.program_id(1) + batch = batch_head // heads + head = batch_head % heads + token_offsets = block_idx * block + tl.arange(0, block) + dim_offsets = tl.arange(0, head_dim) + valid = token_offsets < tokens + input_offsets = ((batch * tokens + token_offsets[:, None]) * heads + head) * head_dim + dim_offsets[None, :] + output_offsets = ((batch * heads + head) * head_dim + dim_offsets[None, :]) * tokens + token_offsets[:, None] + scale_offsets = (batch * heads + head) * head_dim + dim_offsets + scale = tl.maximum(tl.load(v_scale + scale_offsets), 1.0e-6 / 448.0) + values = tl.load(v + input_offsets, mask=valid[:, None], other=0.0).to(tl.float32) + tl.store(v_out + output_offsets, values / scale[None, :], mask=valid[:, None]) + + +def quantize_fp8_qkv_triton( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + block_size: int, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """Launch the fused Q/K/V quantization kernels on validated CUDA inputs.""" + batch, tokens, heads, head_dim = q.shape + blocks = triton.cdiv(tokens, block_size) + q_out = torch.empty(q.shape, device=q.device, dtype=torch.float8_e4m3fn) + k_out = torch.empty_like(q_out) + v_storage = torch.empty((batch, heads, head_dim, tokens), device=q.device, dtype=torch.float8_e4m3fn) + q_scale = torch.empty((batch, blocks, heads), device=q.device, dtype=torch.float32) + k_scale = torch.ones_like(q_scale) + v_scale = torch.zeros((batch, heads, head_dim), device=q.device, dtype=torch.float32) + grid = (blocks, batch * heads) + _quantize_qkv_fp8_stage1[grid]( + q, + k, + v, + q_out, + k_out, + q_scale, + k_scale, + v_scale, + tokens, + heads, + head_dim, + block_size, + num_warps=8, + num_stages=1, + ) + _quantize_qkv_fp8_stage2_v[grid]( + v, + v_storage, + v_scale, + tokens, + heads, + head_dim, + block_size, + num_warps=8, + num_stages=1, + ) + v_out = v_storage.permute(0, 3, 1, 2) + return q_out, k_out, v_out, q_scale, k_scale, v_scale diff --git a/telefuser/ops/fp8_attention.py b/telefuser/ops/fp8_attention.py index 7a2f36d5..eeec0f20 100644 --- a/telefuser/ops/fp8_attention.py +++ b/telefuser/ops/fp8_attention.py @@ -4,77 +4,10 @@ import torch import torch.nn.functional as F -import triton -import triton.language as tl FP8_ATTENTION_BLOCK_SIZE = 64 -@triton.jit -def _quantize_qkv_fp8_stage1( - q, - k, - v, - q_out, - k_out, - q_scale, - k_scale, - v_scale, - tokens: tl.constexpr, - heads: tl.constexpr, - head_dim: tl.constexpr, - block: tl.constexpr, -): - block_idx = tl.program_id(0) - batch_head = tl.program_id(1) - batch = batch_head // heads - head = batch_head % heads - token_offsets = block_idx * block + tl.arange(0, block) - dim_offsets = tl.arange(0, head_dim) - valid = token_offsets < tokens - offsets = ((batch * tokens + token_offsets[:, None]) * heads + head) * head_dim + dim_offsets[None, :] - q_values = tl.load(q + offsets, mask=valid[:, None], other=0.0).to(tl.float32) - k_values = tl.load(k + offsets, mask=valid[:, None], other=0.0).to(tl.float32) - v_values = tl.load(v + offsets, mask=valid[:, None], other=0.0).to(tl.float32) - - q_s = tl.maximum(tl.max(tl.max(tl.abs(q_values), axis=1), axis=0), 1.0e-6) / 448.0 - k_s = tl.maximum(tl.max(tl.max(tl.abs(k_values), axis=1), axis=0), 1.0e-6) / 448.0 - scale_offset = (batch * tl.cdiv(tokens, block) + block_idx) * heads + head - tl.store(q_scale + scale_offset, q_s) - tl.store(k_scale + scale_offset, k_s) - tl.store(q_out + offsets, q_values / q_s, mask=valid[:, None]) - tl.store(k_out + offsets, k_values / k_s, mask=valid[:, None]) - - v_s = tl.max(tl.abs(v_values), axis=0) / 448.0 - v_scale_offsets = (batch * heads + head) * head_dim + dim_offsets - tl.atomic_max(v_scale + v_scale_offsets, v_s) - - -@triton.jit -def _quantize_qkv_fp8_stage2_v( - v, - v_out, - v_scale, - tokens: tl.constexpr, - heads: tl.constexpr, - head_dim: tl.constexpr, - block: tl.constexpr, -): - block_idx = tl.program_id(0) - batch_head = tl.program_id(1) - batch = batch_head // heads - head = batch_head % heads - token_offsets = block_idx * block + tl.arange(0, block) - dim_offsets = tl.arange(0, head_dim) - valid = token_offsets < tokens - input_offsets = ((batch * tokens + token_offsets[:, None]) * heads + head) * head_dim + dim_offsets[None, :] - output_offsets = ((batch * heads + head) * head_dim + dim_offsets[None, :]) * tokens + token_offsets[:, None] - scale_offsets = (batch * heads + head) * head_dim + dim_offsets - scale = tl.maximum(tl.load(v_scale + scale_offsets), 1.0e-6 / 448.0) - values = tl.load(v + input_offsets, mask=valid[:, None], other=0.0).to(tl.float32) - tl.store(v_out + output_offsets, values / scale[None, :], mask=valid[:, None]) - - def quantize_fp8_qkv( q: torch.Tensor, k: torch.Tensor, @@ -86,46 +19,11 @@ def quantize_fp8_qkv( raise ValueError("q, k, and v must share shape [B, T, H, D]") if not (q.is_cuda and q.is_contiguous() and k.is_contiguous() and v.is_contiguous()): raise ValueError("fused FP8 QKV quantization requires contiguous CUDA tensors") - batch, tokens, heads, head_dim = q.shape - if head_dim != 128: + if q.shape[-1] != 128: raise ValueError("fused FP8 QKV quantization requires head dimension 128") - blocks = triton.cdiv(tokens, FP8_ATTENTION_BLOCK_SIZE) - q_out = torch.empty(q.shape, device=q.device, dtype=torch.float8_e4m3fn) - k_out = torch.empty_like(q_out) - v_storage = torch.empty((batch, heads, head_dim, tokens), device=q.device, dtype=torch.float8_e4m3fn) - q_scale = torch.empty((batch, blocks, heads), device=q.device, dtype=torch.float32) - k_scale = torch.ones_like(q_scale) - v_scale = torch.zeros((batch, heads, head_dim), device=q.device, dtype=torch.float32) - grid = (blocks, batch * heads) - _quantize_qkv_fp8_stage1[grid]( - q, - k, - v, - q_out, - k_out, - q_scale, - k_scale, - v_scale, - tokens, - heads, - head_dim, - FP8_ATTENTION_BLOCK_SIZE, - num_warps=8, - num_stages=1, - ) - _quantize_qkv_fp8_stage2_v[grid]( - v, - v_storage, - v_scale, - tokens, - heads, - head_dim, - FP8_ATTENTION_BLOCK_SIZE, - num_warps=8, - num_stages=1, - ) - v_out = v_storage.permute(0, 3, 1, 2) - return q_out, k_out, v_out, q_scale, k_scale, v_scale + from telefuser.kernel.triton.fp8_attention import quantize_fp8_qkv_triton + + return quantize_fp8_qkv_triton(q, k, v, FP8_ATTENTION_BLOCK_SIZE) def quantize_fp8_per_block( From b6dea52ab1338f9078cb95d42f83b115e0df841d Mon Sep 17 00:00:00 2001 From: jinyx5 Date: Tue, 1 Sep 2026 00:45:22 +0800 Subject: [PATCH 05/21] fix(distributed): allocate pipeline P2P buffers on the platform device recv, recv_latent, and recv_latent_async allocated receive buffers with device="cuda", which fails on NPU-only hosts before irecv can run. Allocate on current_platform.device_type (identical behavior on CUDA) and parameterize the test tensor device the same way so the suite runs on CUDA, NPU, and CPU hosts. Verified: tests/unit/distributed/test_pp_comm.py 12 passed on Ascend 910B2 (previously 6 device-related failures). --- telefuser/distributed/pp_comm.py | 7 ++++--- tests/unit/distributed/test_pp_comm.py | 18 +++++++++++------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/telefuser/distributed/pp_comm.py b/telefuser/distributed/pp_comm.py index 9c3b9085..5a4a2de7 100644 --- a/telefuser/distributed/pp_comm.py +++ b/telefuser/distributed/pp_comm.py @@ -24,6 +24,7 @@ import torch import torch.distributed as dist +from telefuser.platforms import current_platform from telefuser.utils.logging import logger @@ -107,7 +108,7 @@ def recv( if buffer is None: if shape is None: raise ValueError("Either buffer or shape must be provided") - buffer = torch.empty(shape, dtype=torch.float16, device="cuda") + buffer = torch.empty(shape, dtype=torch.float16, device=current_platform.device_type) buffer = buffer.contiguous() if async_op: @@ -266,7 +267,7 @@ def recv_latent(self, shape: tuple | None = None, dtype: torch.dtype = torch.bfl if shape is None: raise ValueError("recv_latent: shape must be provided") - buffer = torch.empty(shape, dtype=dtype, device="cuda") + buffer = torch.empty(shape, dtype=dtype, device=current_platform.device_type) buffer = buffer.contiguous() work = dist.irecv(buffer, self.recv_src, group=self._process_group) work.wait() @@ -300,7 +301,7 @@ def recv_latent_async(self, shape: tuple, dtype: torch.dtype = torch.bfloat16) - if self.is_first_stage: raise RuntimeError("recv_latent_async: First stage has no previous stage to receive from") - buffer = torch.empty(shape, dtype=dtype, device="cuda") + buffer = torch.empty(shape, dtype=dtype, device=current_platform.device_type) buffer = buffer.contiguous() work = dist.irecv(buffer, self.recv_src, group=self._process_group) return buffer, work diff --git a/tests/unit/distributed/test_pp_comm.py b/tests/unit/distributed/test_pp_comm.py index f247d001..99b52a7d 100644 --- a/tests/unit/distributed/test_pp_comm.py +++ b/tests/unit/distributed/test_pp_comm.py @@ -7,10 +7,14 @@ import torch.distributed as dist from telefuser.distributed.pp_comm import PipelineP2PComm +from telefuser.platforms import current_platform # Skip if distributed not available HAS_DISTRIBUTED = dist.is_available() +# Buffers follow the detected platform so the suite runs on CUDA, NPU, and CPU hosts alike. +DEVICE = current_platform.device_type + pytestmark = [ pytest.mark.skipif(not HAS_DISTRIBUTED, reason="Distributed not available"), pytest.mark.distributed, @@ -56,7 +60,7 @@ def test_send_on_last_stage_logs_warning(self): """Test that send on last stage logs warning and returns None.""" comm = PipelineP2PComm(None) # Single GPU, is_last_stage=True - tensor = torch.randn(1, 10, 512, device="cuda") + tensor = torch.randn(1, 10, 512, device=DEVICE) result = comm.send(tensor) assert result is None @@ -72,7 +76,7 @@ def test_send_recv_single_gpu(self): """Test send_recv on single GPU (no-op).""" comm = PipelineP2PComm(None) # Single GPU - send_tensor = torch.randn(1, 10, 512, device="cuda") + send_tensor = torch.randn(1, 10, 512, device=DEVICE) result = comm.send_recv(send_tensor) # On single GPU, recv_buffer is None since there's no previous stage @@ -114,7 +118,7 @@ def test_queue_send_on_last_stage(self): """Test queue_send on last stage does nothing.""" comm = PipelineP2PComm(None) # is_last_stage=True - tensor = torch.randn(1, 10, 512, device="cuda") + tensor = torch.randn(1, 10, 512, device=DEVICE) comm.queue_send(tensor) assert len(comm._ops) == 0 @@ -123,7 +127,7 @@ def test_queue_recv_on_first_stage(self): """Test queue_recv on first stage does nothing.""" comm = PipelineP2PComm(None) # is_first_stage=True - buffer = torch.randn(1, 10, 512, device="cuda") + buffer = torch.randn(1, 10, 512, device=DEVICE) comm.queue_recv(buffer) assert len(comm._ops) == 0 @@ -152,7 +156,7 @@ def test_send_latent_on_last_stage(self): """Test send_latent on last stage returns early.""" comm = PipelineP2PComm(None) # is_last_stage=True - tensor = torch.randn(1, 10, 512, device="cuda") + tensor = torch.randn(1, 10, 512, device=DEVICE) # Should not raise, just return comm.send_latent(tensor) @@ -182,7 +186,7 @@ def test_send_latent_async_on_last_stage(self): """Test send_latent_async on last stage returns None.""" comm = PipelineP2PComm(None) # is_last_stage=True - tensor = torch.randn(1, 10, 512, device="cuda") + tensor = torch.randn(1, 10, 512, device=DEVICE) result = comm.send_latent_async(tensor) assert result is None @@ -248,7 +252,7 @@ def test_actual_p2p_communication(self): if comm.is_first_stage: # Send tensor to next stage - send_tensor = torch.ones(1, 10, 512, device="cuda") * comm.rank + send_tensor = torch.ones(1, 10, 512, device=DEVICE) * comm.rank comm.send_latent(send_tensor) elif comm.is_last_stage: # Receive tensor from previous stage From 00a2124a9bc59b640d3fb0988006ae3030e00104 Mon Sep 17 00:00:00 2001 From: jinyx5 Date: Tue, 1 Sep 2026 09:11:53 +0800 Subject: [PATCH 06/21] fix(worker): keep marshalled queue results device-agnostic The non-CUDA result path introduced for CPU-marshalled queues moved results back to self._stage.device inside _wait_result. That breaks worker unit tests on non-CUDA hosts (mocked stages resolve .device through the method proxy or to a MagicMock), and it is unnecessary: stage entry points already place their inputs, and the Wan VAE decode paths move latents themselves. Drop the move so _wait_result returns results untouched; workers still marshal results through CPU when device IPC is unavailable. Verified: tests/unit/worker passes on an Ascend host (same non-CUDA branch as CPU CI) and the 4-card Wan2.2-TI2V-5B smoke still completes; ruff clean. --- telefuser/worker/parallel_worker.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/telefuser/worker/parallel_worker.py b/telefuser/worker/parallel_worker.py index 9adb8450..01727299 100644 --- a/telefuser/worker/parallel_worker.py +++ b/telefuser/worker/parallel_worker.py @@ -323,9 +323,6 @@ def _wait_result(self, method_name: str) -> Any: reason = f"{method_name} failed: {result}" self._mark_failed(reason) raise RuntimeError(f"ParallelWorker:{self.name} {reason}") from result - if current_platform.device_type != "cuda": - # CPU-marshalled queue results move back to the stage device for downstream consumers. - result = to_device(result, self._stage.device) return result def enable_metrics(self, registry: Any | None = None) -> None: From 1b9642584ce689aab39bcb3d26f98c3ad126bc44 Mon Sep 17 00:00:00 2001 From: jinyx5 Date: Tue, 1 Sep 2026 11:30:50 +0800 Subject: [PATCH 07/21] feat(examples): auto-detect device in the wan22 5B T2V example The example hardcoded device="cuda", so running it on an NPU host required editing the script. Default the pipeline device to current_platform.device_type so the same command runs unmodified on CUDA and NPU; on CUDA hosts this resolves to "cuda" as before. Verified: python examples/wan_video/wan22_t2v_5b.py --gpu_num 1 and --gpu_num 4 run unmodified on Ascend 910B (50 steps, 121 frames, default prompt). --- examples/wan_video/wan22_t2v_5b.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/wan_video/wan22_t2v_5b.py b/examples/wan_video/wan22_t2v_5b.py index 76cc1a56..573f1eb5 100644 --- a/examples/wan_video/wan22_t2v_5b.py +++ b/examples/wan_video/wan22_t2v_5b.py @@ -17,6 +17,7 @@ Wan22TI2VPipeline, Wan22TI2VPipelineConfig, ) +from telefuser.platforms import current_platform from telefuser.utils.utils import get_example_name from telefuser.utils.video import get_target_video_size_from_ratio, save_video @@ -80,7 +81,7 @@ def get_pipeline(parallelism: int = 1, model_root: str = PPL_CONFIG["model_root" ) # Create pipeline - pipe = Wan22TI2VPipeline(device="cuda", torch_dtype=torch.bfloat16) + pipe = Wan22TI2VPipeline(device=current_platform.device_type, torch_dtype=torch.bfloat16) # Configure pipeline pipe_config = Wan22TI2VPipelineConfig() From edf966dbbcceb18482f8f0a4d2a5201d498f03ad Mon Sep 17 00:00:00 2001 From: jinyx5 Date: Tue, 1 Sep 2026 02:09:18 +0800 Subject: [PATCH 08/21] fix(models): accept batched tensors in wan22 VAE parallel decode Wan22VideoVAE.decode's parallel branch called torch.stack on its input, assuming a list of per-video tensors, but VAEStage.decode_video passes a batched [B, C, T, H, W] tensor; the serial branch only worked because iterating a tensor yields its batch slices. Any enable_vae_parallel run of the Wan2.2 48-channel VAE therefore failed with TypeError regardless of platform. Normalize tensor inputs before stacking. Found while enabling spatially parallel VAE decode on Ascend 910B2; the parallel path now proceeds to communicator setup (further multi-communicator progress on that host is limited by its CANN driver, see branch notes). --- telefuser/models/wan22_video_vae.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/telefuser/models/wan22_video_vae.py b/telefuser/models/wan22_video_vae.py index 8ca32a35..6e70f21f 100644 --- a/telefuser/models/wan22_video_vae.py +++ b/telefuser/models/wan22_video_vae.py @@ -1403,7 +1403,11 @@ def decode( if self.parallelism > 1 and dist.is_initialized(): # tiled=True → tile_dist, tiled=False → 2d_split method = "tile_dist" if tiled else "2d_split" - hidden_states_tensor = torch.stack(hidden_states) + # The stage passes a batched [B, C, T, H, W] tensor; lists of per-video tensors are stacked. + if isinstance(hidden_states, torch.Tensor): + hidden_states_tensor = hidden_states + else: + hidden_states_tensor = torch.stack(hidden_states) return self.decode_parallel(hidden_states_tensor, device, method=method) # Single GPU processing From d34c842713880a6af8b8cb7a5a8614664ed633cf Mon Sep 17 00:00:00 2001 From: jinyx5 Date: Tue, 1 Sep 2026 02:09:18 +0800 Subject: [PATCH 09/21] fix(worker): isolate HCCL socket ranges per parallel worker group Concurrent worker groups sharing the same NPU devices (e.g. denoising plus VAE workers) collide on HCCL's default data-plane socket range and fail comm init with EJ0003 ("IP address and port have been bound already"). Allocate a distinct HCCL_IF_BASE_PORT per spawned group on NPU platforms, mirroring the existing per-group MASTER_PORT allocation; other platforms are untouched. Verified: single-group 4-card Wan2.2-TI2V-5B regression on Ascend 910B2 passes with the env applied (33.4s generate, parity with baseline). --- telefuser/worker/parallel_worker.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/telefuser/worker/parallel_worker.py b/telefuser/worker/parallel_worker.py index 01727299..b025d5ae 100644 --- a/telefuser/worker/parallel_worker.py +++ b/telefuser/worker/parallel_worker.py @@ -7,6 +7,7 @@ from __future__ import annotations import gc +import itertools import os import signal import threading @@ -35,6 +36,10 @@ _DISCARD_TENSOR_REFS = "__telefuser_discard_tensor_refs__" +# Each concurrent worker group needs a distinct HCCL socket range on NPU hosts; +# overlapping ranges fail comm init with EJ0003 (port already bound). +_hccl_group_counter = itertools.count() + def to_device(data: Any, device: str | torch.device) -> Any: """Recursively move data to target device.""" @@ -65,6 +70,7 @@ def _worker_loop( master_port: int, tensor_output_channel: WorkerTensorChannel | None = None, tensor_input_channels: tuple[WorkerTensorChannel, ...] = (), + hccl_if_base_port: int | None = None, ) -> None: """Worker process main loop. @@ -89,6 +95,11 @@ def _worker_loop( os.environ["WORLD_SIZE"] = str(world_size) os.environ["MASTER_ADDR"] = "localhost" os.environ["MASTER_PORT"] = str(master_port) + if hccl_if_base_port is not None: + # Give each worker group a distinct host-side HCCL socket base; + # overlapping ranges fail comm init with EJ0003 when several + # groups share the same devices (e.g. denoise + VAE workers). + os.environ["HCCL_IF_BASE_PORT"] = str(hccl_if_base_port) device_ids = parallel_config.device_ids device_id = rank if device_ids is not None: @@ -257,6 +268,7 @@ def __init__( master_port, self.tensor_output_channel, self.tensor_input_channels, + 60000 + 1024 * next(_hccl_group_counter) if current_platform.device_type == "npu" else None, ), nprocs=self.world_size, join=False, From 54a2b943fc07e80b0d372782bd9791dbcd15edb5 Mon Sep 17 00:00:00 2001 From: jinyx5 Date: Fri, 11 Sep 2026 16:02:51 +0800 Subject: [PATCH 10/21] docs: register Ascend NPU in the platform, installation, and wan_video guides Mirror the ROCm registration pattern for the NPU support validated by this branch, updating the guides introduced on main: - docs/en|zh/platforms.md: expand the "NPU and CPU" section with the native-fallback attention path (TORCH_SDPA, no triton/tf-kernel needed), forward_npu dispatch, HCCL with CPU-marshalled worker queues and per-group HCCL_IF_BASE_PORT isolation, and the validated wan22_t2v_5b.py entry point on Atlas 910B (CANN 8.2); note that Wan2.2 A14B is not yet exercised. - docs/en|zh/installation.md: add an Ascend NPU requirements row and a verification note (torch_npu selection, ASCEND_RT_VISIBLE_DEVICES, torch.cuda.is_available() False is expected). - examples/wan_video/README.md: add the Ascend requirements bullet and platform auto-detection / validated-hardware bullets to wan22_t2v_5b.py. Verification: python scripts/docs/prepare_cookbook.py build passes with rendered pages, assets, and anchors validated; the tests/docs unittest failures on this host are macOS /var tempdir symlink artifacts unrelated to these files. English additions stay within the 120-column style. --- docs/en/installation.md | 6 ++++++ docs/en/platforms.md | 23 +++++++++++++++++++---- docs/zh/installation.md | 5 +++++ docs/zh/platforms.md | 19 ++++++++++++++++--- examples/wan_video/README.md | 6 ++++++ 5 files changed, 52 insertions(+), 7 deletions(-) diff --git a/docs/en/installation.md b/docs/en/installation.md index a13c2e10..7bcefbed 100644 --- a/docs/en/installation.md +++ b/docs/en/installation.md @@ -12,6 +12,7 @@ distribution are installed separately. | PyTorch | 2.6 or newer | | CUDA toolkit | 12.8 or newer for the maintained CUDA development path | | ROCm | 7.x with a PyTorch `+rocm` build for AMD GPUs; see the ROCm note under verification | +| Ascend NPU | CANN 8.2 with matching `torch` and `torch_npu` builds; see the NPU note under verification | | GPU | Depends on the selected model; check its Cookbook guide | An example may impose stricter versions or GPU architecture requirements. In particular, locally built `tf-kernel` @@ -62,6 +63,11 @@ ROCm before CUDA. Examples ending in `_rocm.py` (for example `examples/wan_video are the validated entry points; see [Hardware Platforms](platforms.md) for per-platform capabilities and backend availability. +On Huawei Ascend hosts, install a `torch_npu` build matching your PyTorch version instead of the CUDA toolkit +path. The platform layer selects NPU when `torch_npu` imports and an Ascend device is visible +(`ASCEND_RT_VISIBLE_DEVICES` controls visibility); `torch.cuda.is_available()` printing `False` is expected there. +`examples/wan_video/wan22_t2v_5b.py` is the validated NPU entry point. + ## Model Checkpoints TeleFuser does not bundle model weights. The [Supported Models](supported_models.md) page links to each Cookbook diff --git a/docs/en/platforms.md b/docs/en/platforms.md index 50de62b7..353950e6 100644 --- a/docs/en/platforms.md +++ b/docs/en/platforms.md @@ -65,7 +65,22 @@ setup path. ## NPU and CPU -The NPU platform targets Huawei Ascend devices through `torch_npu` with the HCCL distributed backend. It is wired -into the platform and ops dispatch layers, but the maintained examples are validated on CUDA and, for select -examples, ROCm — validate on your target NPU before production use. The CPU platform is the fallback when no -accelerator is detected; it is intended for tests and for pipelines that explicitly request CPU execution. +The NPU platform targets Huawei Ascend devices through `torch_npu` with the HCCL distributed backend. + +- Attention uses `TORCH_SDPA` through the native fallback paths; no `tf-kernel`, `flash_attn`, `sageattention`, or + `triton` installation is required. +- The ops layer selects `forward_npu` where an op defines one and otherwise falls back to native PyTorch. No + NPU-optimized kernels are integrated yet, so pipelines run entirely on the native paths. +- Multi-card inference uses HCCL through the `hccl` backend string. Parallel-worker queues marshal tensors through + CPU because `torch_npu` has no reliable cross-process device IPC, and each spawned worker group receives a + distinct `HCCL_IF_BASE_PORT` so concurrent groups do not collide on HCCL's data-plane socket range. +- `torch.compile` is not validated on NPU; NPU examples run eager. +- The validated entry point is + [Wan2.2 TI2V-5B text-to-video](https://github.com/Tele-AI/TeleFuser/tree/main/examples/wan_video) + (`wan22_t2v_5b.py`), which auto-detects the platform and runs unmodified on an Atlas 910B (CANN 8.2, torch 2.9 + with a matching `torch_npu`): single-card, and four-card CFG × Ulysses parallelism over HCCL. Wan2.2 A14B shares + these code paths but has not been exercised on NPU hardware; validate other examples on your target NPU before + production use. + +The CPU platform is the fallback when no accelerator is detected; it is intended for tests and for pipelines that +explicitly request CPU execution. diff --git a/docs/zh/installation.md b/docs/zh/installation.md index a4df9c7e..2ec15661 100644 --- a/docs/zh/installation.md +++ b/docs/zh/installation.md @@ -11,6 +11,7 @@ | PyTorch | 2.6 或更高版本 | | CUDA Toolkit | 当前 CUDA 开发路径要求 12.8 或更高版本 | | ROCm | AMD GPU 使用 ROCm 7.x 与 PyTorch `+rocm` 构建,详见验证安装一节的说明 | +| 昇腾 NPU | CANN 8.2 搭配版本匹配的 `torch` 与 `torch_npu` 构建,详见验证安装一节的说明 | | GPU | 取决于所选模型,以对应 Cookbook 为准 | 具体示例可能要求更严格的软件版本或 GPU 架构。特别是本地构建的 `tf-kernel` 产物与其记录的 PyTorch、 @@ -61,6 +62,10 @@ AMD ROCm 主机应安装 PyTorch `+rocm` 构建,而非 CUDA Toolkit 路径。H `examples/wan_video/wan21_1_3b_text_to_video_rocm.py`)是已验证的入口;各平台能力与后端可用性见 [硬件平台](platforms.md)。 +华为昇腾主机应安装与 PyTorch 版本匹配的 `torch_npu`,而非 CUDA Toolkit 路径。当 `torch_npu` 可导入且存在 +可见昇腾设备(由 `ASCEND_RT_VISIBLE_DEVICES` 控制)时,平台层会选择 NPU;此时 `torch.cuda.is_available()` +输出 `False` 属预期行为。`examples/wan_video/wan22_t2v_5b.py` 是已验证的 NPU 入口。 + ## 模型权重 TeleFuser 不随软件包分发模型权重。[支持的模型](supported_models.md)页面会链接到各模型的 Cookbook, diff --git a/docs/zh/platforms.md b/docs/zh/platforms.md index 28d27424..b2986a7a 100644 --- a/docs/zh/platforms.md +++ b/docs/zh/platforms.md @@ -61,6 +61,19 @@ ROCm 支持面向使用 ROCm 7.x 与 PyTorch `+rocm` 构建的 AMD GPU;安装 ## NPU 与 CPU -NPU 平台通过 `torch_npu` 与 HCCL 分布式后端支持华为昇腾设备。平台层与算子分发层均已接入 NPU,但现有 -示例在 CUDA 上验证、部分示例在 ROCm 上验证 —— 生产使用前请先在目标 NPU 上完成验证。CPU 平台是未检测到 -加速器时的回退,面向测试以及显式请求 CPU 执行的 Pipeline。 +NPU 平台通过 `torch_npu` 与 HCCL 分布式后端支持华为昇腾设备。 + +- 注意力经原生回退路径使用 `TORCH_SDPA`;无需安装 `tf-kernel`、`flash_attn`、`sageattention` 或 `triton`。 +- 算子层在算子定义了 `forward_npu` 时优先选择,否则回退到 PyTorch 原生实现。目前尚未集成 NPU 优化内核, + Pipeline 完全运行在原生路径上。 +- 多卡推理通过 `hccl` 后端字符串使用 HCCL。并行 worker 队列经 CPU 中转张量(`torch_npu` 不提供可靠的 + 跨进程设备 IPC);每个新起的 worker 组会分配独立的 `HCCL_IF_BASE_PORT`,避免并发组在 HCCL 数据面端口段 + 上冲突。 +- `torch.compile` 在 NPU 上未验证;NPU 示例以 eager 模式运行。 +- 已验证入口为 + [Wan2.2 TI2V-5B 文生视频](https://github.com/Tele-AI/TeleFuser/tree/main/examples/wan_video) + (`wan22_t2v_5b.py`):示例自动检测平台、免修改运行,已在 Atlas 910B(CANN 8.2,torch 2.9 搭配版本匹配的 + `torch_npu`)上验证单卡以及 4 卡 CFG × Ulysses 并行(HCCL)。Wan2.2 A14B 复用同一代码路径,但尚未在 + NPU 硬件上运行;其他示例在生产使用前请先在目标 NPU 上完成验证。 + +CPU 平台是未检测到加速器时的回退,面向测试以及显式请求 CPU 执行的 Pipeline。 diff --git a/examples/wan_video/README.md b/examples/wan_video/README.md index e46cb48e..d497146b 100644 --- a/examples/wan_video/README.md +++ b/examples/wan_video/README.md @@ -35,6 +35,9 @@ Video generation using Wan2.1 and Wan2.2 models for Text-to-Video and Image-to-V - GPU: AMD ROCm GPUs for scripts ending in `_rocm.py`; validated on a Radeon RX 9070 (ROCm 7.2, `torch` built with `+rocm`). These examples use the PyTorch SDPA attention backend and need no tf-kernel, flash-attn, or SageAttention installation +- GPU: Huawei Ascend NPUs for `wan22_t2v_5b.py`, which auto-detects the platform; validated on an Atlas 910B + (CANN 8.2, torch 2.9 with a matching `torch_npu`). NPU execution uses the PyTorch SDPA attention backend and + needs no tf-kernel, flash-attn, SageAttention, or triton installation - Software: the standard TeleFuser installation; optional attention, FP8, Ray, and RIFE paths require their respective dependencies - Input assets: a readable image for I2V/FL2V and optional LoRA, distillation, cache, or RIFE weights for those variants @@ -463,6 +466,9 @@ python examples/wan_video/wan22_t2v_5b.py --resolution 480p --aspect_ratio 16:9 - CFG parallel enabled by default (cfg_scale=5.0) - Ulysses sequence parallelism for multi-GPU - 50-step UNPC sampling with sigma_shift=5.0 +- Platform auto-detection via `current_platform`: the script runs unmodified on CUDA and Ascend NPU hosts +- Validated on an Ascend Atlas 910B (CANN 8.2, torch 2.9 with a matching `torch_npu`): single-card and 4-card + CFG × Ulysses over HCCL, PyTorch SDPA attention, eager execution #### `wan22_14b_text_to_video_h100.py` From e451e7a921685c48cdae483319292a59186aa0cf Mon Sep 17 00:00:00 2001 From: jinyx5 Date: Fri, 20 Nov 2026 17:24:19 +0800 Subject: [PATCH 11/21] fix(ops): restore lazy FP8 attention kernels and native helpers Restore the mainline smoothing, scale and correction paths for Wan and MiniMax when the optional Triton implementation is split out. The CPU paths import without Triton. All 18 FP8 function definitions match main apart from import placement. Four CPU smoothing modes matched bitwise. H100 execution remains a separate qualification. --- telefuser/kernel/triton/fp8_attention.py | 212 ++++++++++++++---- telefuser/ops/fp8_attention.py | 130 ++++++++++- .../unit/ops/test_fp8_attention_smoothing.py | 14 ++ 3 files changed, 302 insertions(+), 54 deletions(-) diff --git a/telefuser/kernel/triton/fp8_attention.py b/telefuser/kernel/triton/fp8_attention.py index a620f374..6e906e08 100644 --- a/telefuser/kernel/triton/fp8_attention.py +++ b/telefuser/kernel/triton/fp8_attention.py @@ -1,17 +1,159 @@ -"""Fused block-scaled FP8 Q/K/V quantization Triton kernels.""" +"""Triton kernels for FP8 attention quantization and output correction.""" from __future__ import annotations -import torch import triton import triton.language as tl +@triton.jit +def _sequence_mean_kernel( + x, + output, + tokens: tl.constexpr, + heads: tl.constexpr, + head_dim: tl.constexpr, + stride_batch, + stride_token, + stride_head, + stride_dim, + block_tokens: tl.constexpr, + block_dim: tl.constexpr, +): + batch_head = tl.program_id(0) + dim_block = tl.program_id(1) + batch = batch_head // heads + head = batch_head % heads + dims = dim_block * block_dim + tl.arange(0, block_dim) + valid_dims = dims < head_dim + total = tl.zeros((block_dim,), dtype=tl.float32) + for start in range(0, tokens, block_tokens): + token_offsets = start + tl.arange(0, block_tokens) + valid_tokens = token_offsets < tokens + offsets = ( + batch * stride_batch + + token_offsets[:, None] * stride_token + + head * stride_head + + dims[None, :] * stride_dim + ) + values = tl.load( + x + offsets, + mask=valid_tokens[:, None] & valid_dims[None, :], + other=0.0, + ).to(tl.float32) + total += tl.sum(values, axis=0) + tl.store(output + batch_head * head_dim + dims, total / tokens, mask=valid_dims) + + +@triton.jit +def _sequence_mean_kv_kernel( + k, + v, + k_output, + v_output, + tokens: tl.constexpr, + heads: tl.constexpr, + head_dim: tl.constexpr, + k_stride_batch, + k_stride_token, + k_stride_head, + k_stride_dim, + v_stride_batch, + v_stride_token, + v_stride_head, + v_stride_dim, + block_tokens: tl.constexpr, + block_dim: tl.constexpr, +): + batch_head = tl.program_id(0) + dim_block = tl.program_id(1) + batch = batch_head // heads + head = batch_head % heads + dims = dim_block * block_dim + tl.arange(0, block_dim) + valid_dims = dims < head_dim + k_total = tl.zeros((block_dim,), dtype=tl.float32) + v_total = tl.zeros((block_dim,), dtype=tl.float32) + for start in range(0, tokens, block_tokens): + token_offsets = start + tl.arange(0, block_tokens) + valid_tokens = token_offsets < tokens + k_offsets = ( + batch * k_stride_batch + + token_offsets[:, None] * k_stride_token + + head * k_stride_head + + dims[None, :] * k_stride_dim + ) + v_offsets = ( + batch * v_stride_batch + + token_offsets[:, None] * v_stride_token + + head * v_stride_head + + dims[None, :] * v_stride_dim + ) + mask = valid_tokens[:, None] & valid_dims[None, :] + k_values = tl.load(k + k_offsets, mask=mask, other=0.0).to(tl.float32) + v_values = tl.load(v + v_offsets, mask=mask, other=0.0).to(tl.float32) + k_total += tl.sum(k_values, axis=0) + v_total += tl.sum(v_values, axis=0) + output_offsets = batch_head * head_dim + dims + tl.store(k_output + output_offsets, k_total / tokens, mask=valid_dims) + tl.store(v_output + output_offsets, v_total / tokens, mask=valid_dims) + + +@triton.jit +def _add_output_correction_kernel( + output, + correction, + elements, + tokens: tl.constexpr, + heads: tl.constexpr, + head_dim: tl.constexpr, + block: tl.constexpr, +): + offsets = tl.program_id(0) * block + tl.arange(0, block) + valid = offsets < elements + batch = offsets // (tokens * heads * head_dim) + head = (offsets // head_dim) % heads + dim = offsets % head_dim + correction_offsets = (batch * heads + head) * head_dim + dim + values = tl.load(output + offsets, mask=valid).to(tl.float32) + shift = tl.load(correction + correction_offsets, mask=valid) + tl.store(output + offsets, values + shift, mask=valid) + + +@triton.jit +def _merge_prefix_output_correction_kernel( + prefix, + output, + correction, + merged, + elements, + prefix_tokens: tl.constexpr, + tokens: tl.constexpr, + heads: tl.constexpr, + head_dim: tl.constexpr, + block: tl.constexpr, +): + offsets = tl.program_id(0) * block + tl.arange(0, block) + valid = offsets < elements + batch = offsets // (tokens * heads * head_dim) + token = (offsets // (heads * head_dim)) % tokens + head = (offsets // head_dim) % heads + dim = offsets % head_dim + prefix_offsets = ((batch * prefix_tokens + token) * heads + head) * head_dim + dim + correction_offsets = (batch * heads + head) * head_dim + dim + prefix_values = tl.load(prefix + prefix_offsets, mask=valid & (token < prefix_tokens), other=0.0) + output_values = tl.load(output + offsets, mask=valid & (token >= prefix_tokens), other=0.0).to(tl.float32) + shift = tl.load(correction + correction_offsets, mask=valid & (token >= prefix_tokens), other=0.0) + values = tl.where(token < prefix_tokens, prefix_values, output_values + shift) + tl.store(merged + offsets, values, mask=valid) + + @triton.jit def _quantize_qkv_fp8_stage1( q, k, v, + k_mean, + v_mean, q_out, k_out, q_scale, @@ -21,6 +163,8 @@ def _quantize_qkv_fp8_stage1( heads: tl.constexpr, head_dim: tl.constexpr, block: tl.constexpr, + smooth_k: tl.constexpr, + smooth_v: tl.constexpr, ): block_idx = tl.program_id(0) batch_head = tl.program_id(1) @@ -33,6 +177,19 @@ def _quantize_qkv_fp8_stage1( q_values = tl.load(q + offsets, mask=valid[:, None], other=0.0).to(tl.float32) k_values = tl.load(k + offsets, mask=valid[:, None], other=0.0).to(tl.float32) v_values = tl.load(v + offsets, mask=valid[:, None], other=0.0).to(tl.float32) + mean_offsets = batch_head * head_dim + dim_offsets + if smooth_k: + k_values = tl.where( + valid[:, None], + k_values - tl.load(k_mean + mean_offsets)[None, :], + 0.0, + ) + if smooth_v: + v_values = tl.where( + valid[:, None], + v_values - tl.load(v_mean + mean_offsets)[None, :], + 0.0, + ) q_s = tl.maximum(tl.max(tl.max(tl.abs(q_values), axis=1), axis=0), 1.0e-6) / 448.0 k_s = tl.maximum(tl.max(tl.max(tl.abs(k_values), axis=1), axis=0), 1.0e-6) / 448.0 @@ -50,12 +207,14 @@ def _quantize_qkv_fp8_stage1( @triton.jit def _quantize_qkv_fp8_stage2_v( v, + v_mean, v_out, v_scale, tokens: tl.constexpr, heads: tl.constexpr, head_dim: tl.constexpr, block: tl.constexpr, + smooth_v: tl.constexpr, ): block_idx = tl.program_id(0) batch_head = tl.program_id(1) @@ -69,51 +228,6 @@ def _quantize_qkv_fp8_stage2_v( scale_offsets = (batch * heads + head) * head_dim + dim_offsets scale = tl.maximum(tl.load(v_scale + scale_offsets), 1.0e-6 / 448.0) values = tl.load(v + input_offsets, mask=valid[:, None], other=0.0).to(tl.float32) + if smooth_v: + values = values - tl.load(v_mean + scale_offsets)[None, :] tl.store(v_out + output_offsets, values / scale[None, :], mask=valid[:, None]) - - -def quantize_fp8_qkv_triton( - q: torch.Tensor, - k: torch.Tensor, - v: torch.Tensor, - block_size: int, -) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: - """Launch the fused Q/K/V quantization kernels on validated CUDA inputs.""" - batch, tokens, heads, head_dim = q.shape - blocks = triton.cdiv(tokens, block_size) - q_out = torch.empty(q.shape, device=q.device, dtype=torch.float8_e4m3fn) - k_out = torch.empty_like(q_out) - v_storage = torch.empty((batch, heads, head_dim, tokens), device=q.device, dtype=torch.float8_e4m3fn) - q_scale = torch.empty((batch, blocks, heads), device=q.device, dtype=torch.float32) - k_scale = torch.ones_like(q_scale) - v_scale = torch.zeros((batch, heads, head_dim), device=q.device, dtype=torch.float32) - grid = (blocks, batch * heads) - _quantize_qkv_fp8_stage1[grid]( - q, - k, - v, - q_out, - k_out, - q_scale, - k_scale, - v_scale, - tokens, - heads, - head_dim, - block_size, - num_warps=8, - num_stages=1, - ) - _quantize_qkv_fp8_stage2_v[grid]( - v, - v_storage, - v_scale, - tokens, - heads, - head_dim, - block_size, - num_warps=8, - num_stages=1, - ) - v_out = v_storage.permute(0, 3, 1, 2) - return q_out, k_out, v_out, q_scale, k_scale, v_scale diff --git a/telefuser/ops/fp8_attention.py b/telefuser/ops/fp8_attention.py index 85d3cc96..92c7cde7 100644 --- a/telefuser/ops/fp8_attention.py +++ b/telefuser/ops/fp8_attention.py @@ -10,7 +10,73 @@ FP8_ATTENTION_BLOCK_SIZE = 64 -def quantize_fp8_qkv( +def _sequence_mean(x: torch.Tensor) -> torch.Tensor: + """Reduce BTHD sequence means in FP32 without materializing an FP32 copy.""" + + if x.ndim != 4 or not x.is_floating_point(): + raise ValueError("attention smoothing expects a floating-point [B, T, H, D] tensor") + if not x.is_cuda: + return x.float().mean(dim=1) + import triton + + from telefuser.kernel.triton.fp8_attention import _sequence_mean_kernel + + batch, tokens, heads, head_dim = x.shape + output = torch.empty((batch, heads, head_dim), device=x.device, dtype=torch.float32) + block_tokens = 128 + block_dim = 32 + _sequence_mean_kernel[(batch * heads, triton.cdiv(head_dim, block_dim))]( + x, + output, + tokens, + heads, + head_dim, + *x.stride(), + block_tokens, + block_dim, + num_warps=4, + num_stages=1, + ) + return output + + +def _sequence_means(k: torch.Tensor, v: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + """Reduce matching K/V sequence means in one GPU kernel launch.""" + + if k.shape != v.shape or k.ndim != 4 or not (k.is_floating_point() and v.is_floating_point()): + raise ValueError("K/V attention smoothing expects matching floating-point [B, T, H, D] tensors") + if k.device != v.device: + raise ValueError("K and V must be on the same device") + if not k.is_cuda: + return k.float().mean(dim=1), v.float().mean(dim=1) + import triton + + from telefuser.kernel.triton.fp8_attention import _sequence_mean_kv_kernel + + batch, tokens, heads, head_dim = k.shape + k_output = torch.empty((batch, heads, head_dim), device=k.device, dtype=torch.float32) + v_output = torch.empty_like(k_output) + block_tokens = 128 + block_dim = 32 + _sequence_mean_kv_kernel[(batch * heads, triton.cdiv(head_dim, block_dim))]( + k, + v, + k_output, + v_output, + tokens, + heads, + head_dim, + *k.stride(), + *v.stride(), + block_tokens, + block_dim, + num_warps=4, + num_stages=1, + ) + return k_output, v_output + + +def _quantize_fp8_qkv_sm90( q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, @@ -24,11 +90,57 @@ def quantize_fp8_qkv( raise ValueError("q, k, and v must share shape [B, T, H, D]") if not (q.is_cuda and q.is_contiguous() and k.is_contiguous() and v.is_contiguous()): raise ValueError("fused FP8 QKV quantization requires contiguous CUDA tensors") - if q.shape[-1] != 128: + batch, tokens, heads, head_dim = q.shape + if head_dim != 128: raise ValueError("fused FP8 QKV quantization requires head dimension 128") - from telefuser.kernel.triton.fp8_attention import quantize_fp8_qkv_triton - - return quantize_fp8_qkv_triton(q, k, v, FP8_ATTENTION_BLOCK_SIZE) + import triton + + from telefuser.kernel.triton.fp8_attention import _quantize_qkv_fp8_stage1, _quantize_qkv_fp8_stage2_v + + blocks = triton.cdiv(tokens, FP8_ATTENTION_BLOCK_SIZE) + q_out = torch.empty(q.shape, device=q.device, dtype=torch.float8_e4m3fn) + k_out = torch.empty_like(q_out) + v_storage = torch.empty((batch, heads, head_dim, tokens), device=q.device, dtype=torch.float8_e4m3fn) + q_scale = torch.empty((batch, blocks, heads), device=q.device, dtype=torch.float32) + k_scale = torch.ones_like(q_scale) + v_scale = torch.zeros((batch, heads, head_dim), device=q.device, dtype=torch.float32) + dummy_mean = torch.empty((1,), device=q.device, dtype=torch.float32) + grid = (blocks, batch * heads) + _quantize_qkv_fp8_stage1[grid]( + q, + k, + v, + dummy_mean if k_mean is None else k_mean, + dummy_mean if v_mean is None else v_mean, + q_out, + k_out, + q_scale, + k_scale, + v_scale, + tokens, + heads, + head_dim, + FP8_ATTENTION_BLOCK_SIZE, + smooth_k=k_mean is not None, + smooth_v=v_mean is not None, + num_warps=8, + num_stages=1, + ) + _quantize_qkv_fp8_stage2_v[grid]( + v, + dummy_mean if v_mean is None else v_mean, + v_storage, + v_scale, + tokens, + heads, + head_dim, + FP8_ATTENTION_BLOCK_SIZE, + smooth_v=v_mean is not None, + num_warps=8, + num_stages=1, + ) + v_out = v_storage.permute(0, 3, 1, 2) + return q_out, k_out, v_out, q_scale, k_scale, v_scale def quantize_fp8_qkv( @@ -192,6 +304,10 @@ def apply_fp8_attention_output_correction(output: torch.Tensor, correction: torc return (output.float() + correction).to(output.dtype) if not output.is_contiguous() or not correction.is_contiguous(): raise ValueError("FP8 attention output and correction must be contiguous") + import triton + + from telefuser.kernel.triton.fp8_attention import _add_output_correction_kernel + batch, tokens, heads, head_dim = output.shape elements = output.numel() block = 256 @@ -236,6 +352,10 @@ def merge_fp8_attention_prefix( if not (dense_prefix.is_contiguous() and output.is_contiguous() and correction.is_contiguous()): raise ValueError("FP8 attention prefix merge inputs must be contiguous") + import triton + + from telefuser.kernel.triton.fp8_attention import _merge_prefix_output_correction_kernel + merged = torch.empty_like(output) batch, tokens, heads, head_dim = output.shape elements = output.numel() diff --git a/tests/unit/ops/test_fp8_attention_smoothing.py b/tests/unit/ops/test_fp8_attention_smoothing.py index 01e95bdb..5f6d24c5 100644 --- a/tests/unit/ops/test_fp8_attention_smoothing.py +++ b/tests/unit/ops/test_fp8_attention_smoothing.py @@ -8,6 +8,7 @@ apply_fp8_attention_output_correction, dequantize_fp8_per_channel, merge_fp8_attention_prefix, + quantize_fp8_qkv, quantize_fp8_qkv_smoothed, ) @@ -26,6 +27,19 @@ def test_kv_sequence_mean_smoothing_is_attention_equivalent() -> None: torch.testing.assert_close(smoothed, reference, atol=2e-6, rtol=2e-6) +def test_eager_cpu_helpers_work_without_triton() -> None: + q = torch.randn(1, 5, 2, 8) + with pytest.raises(ValueError, match="contiguous CUDA tensors"): + quantize_fp8_qkv(q, q, q) + *_, correction = quantize_fp8_qkv_smoothed(q, q, q, smoothing="kv", correct_v_bias=False) + torch.testing.assert_close(correction, q.mean(dim=1, keepdim=True)) + corrected = apply_fp8_attention_output_correction(q, correction) + torch.testing.assert_close(corrected, q + correction) + torch.testing.assert_close( + merge_fp8_attention_prefix(q[:, :2], q, correction), torch.cat([q[:, :2], corrected[:, 2:]], dim=1) + ) + + @pytest.mark.gpu def test_fp8_value_bias_correction_restores_original_sequence_mean_on_h100() -> None: if not torch.cuda.is_available() or torch.cuda.get_device_capability() != (9, 0): From cff81ccd94cc0f94b3d7ef2c98659fe50ffecc9c Mon Sep 17 00:00:00 2001 From: jinyx5 Date: Fri, 20 Nov 2026 17:24:20 +0800 Subject: [PATCH 12/21] fix(worker): isolate Ascend rank placement, queues and HCCL ports Bind visible accelerator IDs before initializing the process group. Stage NPU tensors on the host for both multiprocessing transports, reserve and recycle aligned port blocks, and clean up a partially spawned HCCL group. Restore the CPU and GPU Conv3d activation layout while using NCDHW on Ascend. Exercise independent-port owners, shutdown, source metadata and uneven shards on CPU. --- examples/wan_video/wan22_t2v_5b.py | 13 +- telefuser/core/base_stage.py | 7 +- telefuser/distributed/vae_spatial.py | 12 +- telefuser/models/wan_video_vae.py | 7 +- telefuser/platforms/__init__.py | 41 ++++ telefuser/platforms/hccl.py | 53 +++++ telefuser/utils/system.py | 12 ++ telefuser/worker/parallel_worker.py | 191 +++++++++++------- telefuser/worker/tensor_channel.py | 28 ++- tests/unit/platforms/test_ascend.py | 67 ++++++ .../worker/test_parallel_worker_lifecycle.py | 23 +++ tests/unit/worker/test_tensor_channel.py | 27 +++ 12 files changed, 383 insertions(+), 98 deletions(-) create mode 100644 telefuser/platforms/hccl.py create mode 100644 tests/unit/platforms/test_ascend.py diff --git a/examples/wan_video/wan22_t2v_5b.py b/examples/wan_video/wan22_t2v_5b.py index 573f1eb5..7fb3097e 100644 --- a/examples/wan_video/wan22_t2v_5b.py +++ b/examples/wan_video/wan22_t2v_5b.py @@ -17,7 +17,7 @@ Wan22TI2VPipeline, Wan22TI2VPipelineConfig, ) -from telefuser.platforms import current_platform +from telefuser.platforms import current_platform, resolve_device from telefuser.utils.utils import get_example_name from telefuser.utils.video import get_target_video_size_from_ratio, save_video @@ -59,6 +59,15 @@ def get_pipeline(parallelism: int = 1, model_root: str = PPL_CONFIG["model_root" """ ppl_config = PPL_CONFIG + if current_platform.device_type == "npu": + if parallelism not in (1, 2, 4): + raise ValueError("The Ascend 910B recipe supports 1, 2 or 4 visible NPUs") + resolve_device(f"npu:{parallelism - 1}") + print( + f"{current_platform.get_device_name()}: {parallelism} NPU(s), " + f"CFG={min(2, parallelism)}, Ulysses={max(1, parallelism // 2)}, BF16 TORCH_SDPA" + ) + # Load models using ModuleManager module_manager = ModuleManager(device="cpu") @@ -112,7 +121,7 @@ def get_pipeline(parallelism: int = 1, model_root: str = PPL_CONFIG["model_root" @click.command() -@click.option("--gpu_num", default=1, help="Number of GPUs to use, default is 1") +@click.option("--gpu_num", default=1, help="Number of local accelerator devices (default: 1)") @click.option( "--prompt", default="A stylish little girl gently caressing her dog while they relax in a sunny, beautiful backyard. Perfect for pet and family content, or videos aiming to showcase love, style, and the bond between kids and their pets.", diff --git a/telefuser/core/base_stage.py b/telefuser/core/base_stage.py index 53d0a110..a314d34f 100644 --- a/telefuser/core/base_stage.py +++ b/telefuser/core/base_stage.py @@ -10,7 +10,7 @@ import torch from telefuser.cache.session_memory import DeviceMemorySnapshot, capture_device_memory_snapshot -from telefuser.platforms import current_platform +from telefuser.platforms import current_platform, resolve_device from telefuser.utils.logging import logger from .config import FeatureCacheConfig, ModelRuntimeConfig, WeightOffloadType @@ -89,7 +89,10 @@ def __init__(self, name: str, model_runtime_config: ModelRuntimeConfig) -> None: self.model_runtime_config = model_runtime_config self.torch_dtype = model_runtime_config.torch_dtype self.device_type = model_runtime_config.device_type or current_platform.device_type - self.device = torch.device(type=self.device_type, index=model_runtime_config.device_id) + if self.device_type == "npu": + self.device = resolve_device(f"npu:{model_runtime_config.device_id}") + else: + self.device = torch.device(type=self.device_type, index=model_runtime_config.device_id) self.model_names: list[str] = [] self.onload_models_flag = False self._metrics_hook: StageMetricContext | None = None diff --git a/telefuser/distributed/vae_spatial.py b/telefuser/distributed/vae_spatial.py index ebeb30fe..cb68f23c 100644 --- a/telefuser/distributed/vae_spatial.py +++ b/telefuser/distributed/vae_spatial.py @@ -10,6 +10,12 @@ from .collectives import all_gather_stacked +def conv3d_activation_layout(tensor: torch.Tensor) -> torch.Tensor: + """Keep NCDHW on Ascend and use the existing channels-last layout elsewhere.""" + layout = torch.contiguous_format if tensor.device.type == "npu" else torch.channels_last_3d + return tensor.contiguous(memory_format=layout) + + def _spatial_rank() -> int: return dist.get_rank() if dist.is_initialized() else 0 @@ -128,11 +134,7 @@ def _spatial_causal_conv3d_forward( if any(padding): tensor = F.pad(tensor, padding) tensor = _exchange_height_halo(module, tensor, module._height_halo_size) - if tensor.device.type == "cuda": - tensor = tensor.contiguous(memory_format=torch.channels_last_3d) - else: - # channels_last_3d activations are only supported by cuDNN; NPU/CPU require standard contiguous. - tensor = tensor.contiguous() + tensor = conv3d_activation_layout(tensor) return F.conv3d( tensor, module.weight, diff --git a/telefuser/models/wan_video_vae.py b/telefuser/models/wan_video_vae.py index 941f8056..5389ce9d 100644 --- a/telefuser/models/wan_video_vae.py +++ b/telefuser/models/wan_video_vae.py @@ -17,6 +17,7 @@ _spatial_causal_conv3d_forward, _spatial_world_size, _split_height, + conv3d_activation_layout, ) from telefuser.utils.logging import logger from telefuser.utils.model_weight import hash_state_dict_keys @@ -119,11 +120,7 @@ def forward(self, x: torch.Tensor, cache_x: torch.Tensor | None = None) -> torch x = torch.cat([cache_x, x], dim=2) padding[4] -= cache_x.shape[2] x = F.pad(x, padding) - if x.device.type == "cuda": - x = x.contiguous(memory_format=torch.channels_last_3d) - else: - # channels_last_3d activations are only supported by cuDNN; NPU/CPU require standard contiguous. - x = x.contiguous() + x = conv3d_activation_layout(x) return super().forward(x) diff --git a/telefuser/platforms/__init__.py b/telefuser/platforms/__init__.py index 7d517faa..c324d077 100644 --- a/telefuser/platforms/__init__.py +++ b/telefuser/platforms/__init__.py @@ -6,6 +6,7 @@ from __future__ import annotations import logging +import re import torch @@ -73,9 +74,49 @@ def _resolve_current_platform() -> BasePlatform: current_platform: BasePlatform = _resolve_current_platform() +def normalize_device_name(device: str | torch.device) -> str: + """Parse a TeleFuser device without importing its vendor runtime.""" + match = re.fullmatch(r"(cpu|cuda|rocm|npu)(?::([0-9]+))?", str(device)) + if match is None: + raise ValueError(f"Expected cpu, cuda, rocm or npu with an optional index, got {device!r}") + kind, index = match.groups() + if kind == "cpu": + return "cpu" + return f"{'cuda' if kind == 'rocm' else kind}:{int(index or 0)}" + + +def resolve_device(device: str | torch.device) -> torch.device: + """Validate a compute device against this process before starting workers.""" + name = normalize_device_name(device) + if name == "cpu": + return torch.device(name) + kind, index = name.split(":") + if kind == "npu": + try: + import torch_npu # noqa: F401 + except (ImportError, OSError, RuntimeError) as exc: + raise RuntimeError(f"Cannot initialize torch_npu for {name}: {exc}") from exc + if kind != current_platform.device_type or ( + str(device).split(":")[0] == "rocm" and current_platform.device_name != "rocm" + ): + raise RuntimeError(f"Requested {device} but detected {current_platform.device_name}; check device visibility") + count = current_platform.device_count() + if int(index) >= count: + raise ValueError(f"{name} is outside the {count} visible {current_platform.device_name} devices") + return torch.device(name) + + +def requires_host_tensor_queue(device_type: str) -> bool: + """Ascend tensors cross multiprocessing queues through host memory.""" + return device_type == "npu" + + __all__ = [ "BasePlatform", "current_platform", + "normalize_device_name", + "resolve_device", + "requires_host_tensor_queue", "CudaPlatform", "RocmPlatform", "NPUPlatform", diff --git a/telefuser/platforms/hccl.py b/telefuser/platforms/hccl.py new file mode 100644 index 00000000..99de0389 --- /dev/null +++ b/telefuser/platforms/hccl.py @@ -0,0 +1,53 @@ +"""Reserve distinct HCCL TCP port blocks for local multiprocessing groups.""" + +from __future__ import annotations + +import errno +import os +import socket +import tempfile +from contextlib import ExitStack +from pathlib import Path +from typing import BinaryIO + +_BLOCK_SIZE = 16 +_LAST_BASE = 65536 - _BLOCK_SIZE + + +def reserve_hccl_ports() -> tuple[int, BinaryIO]: + """Hold the returned lock until all ranks exit. Kernel locks survive parent concurrency.""" + import fcntl + + value = os.environ.get("HCCL_IF_BASE_PORT", "60000") + try: + start = int(value) + if not 1024 <= start <= _LAST_BASE: + raise ValueError + except ValueError: + raise ValueError(f"HCCL_IF_BASE_PORT must be between 1024 and {_LAST_BASE}, got {value!r}") from None + + # Use common block boundaries even when two launchers specify different starting ports. + start += -start % _BLOCK_SIZE + for base in range(start, _LAST_BASE + 1, _BLOCK_SIZE): + path = Path(tempfile.gettempdir()) / f"telefuser-hccl-{base}.lock" + # Read-only descriptors allow launchers with different UIDs to lock an existing file. + try: + fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDONLY, 0o644) + except FileExistsError: + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW) + lock = os.fdopen(fd, "rb") + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + with ExitStack() as probes: + for port in range(base, base + _BLOCK_SIZE): + probe = probes.enter_context(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) + probe.bind(("", port)) + return base, lock + except OSError as exc: + lock.close() + if exc.errno not in (errno.EAGAIN, errno.EWOULDBLOCK, errno.EADDRINUSE): + raise + except BaseException: + lock.close() + raise + raise RuntimeError(f"No free {_BLOCK_SIZE}-port HCCL block at or above {start} (last base {_LAST_BASE})") diff --git a/telefuser/utils/system.py b/telefuser/utils/system.py index 5569f071..82649d6b 100644 --- a/telefuser/utils/system.py +++ b/telefuser/utils/system.py @@ -10,6 +10,7 @@ class PortAllocator: """Thread-safe port allocator for managing non-overlapping port interval allocation.""" + _port_intervals: dict[int, tuple[int, int]] = {} _allocated_intervals: set[tuple[int, int]] = set() _lock = threading.RLock() # Use reentrant lock to allow the same thread to acquire multiple times @@ -57,8 +58,17 @@ def get_free_port_in_interval( # Randomly select a port within the chosen interval port = random.randint(chosen_interval[0], chosen_interval[1]) + cls._port_intervals[port] = chosen_interval return port + @classmethod + def release_port(cls, port: int) -> None: + """Return the interval owned by a port from get_free_port_in_interval.""" + with cls._lock: + interval = cls._port_intervals.pop(port, None) + if interval is not None: + cls._allocated_intervals.discard(interval) + @classmethod def release_interval(cls, start_port: int, end_port: int): """Release an interval to make it available for reuse (thread-safe). @@ -71,6 +81,7 @@ def release_interval(cls, start_port: int, end_port: int): interval = (start_port, end_port) if interval in cls._allocated_intervals: cls._allocated_intervals.remove(interval) + cls._port_intervals = {p: v for p, v in cls._port_intervals.items() if v != interval} @classmethod def get_specific_interval(cls, start_port: int, end_port: int) -> int: @@ -99,6 +110,7 @@ def reset_allocator(cls): """Reset the allocator, releasing all allocated intervals (thread-safe).""" with cls._lock: cls._allocated_intervals.clear() + cls._port_intervals.clear() @classmethod def get_allocated_intervals(cls) -> set[tuple[int, int]]: diff --git a/telefuser/worker/parallel_worker.py b/telefuser/worker/parallel_worker.py index b025d5ae..058f411b 100644 --- a/telefuser/worker/parallel_worker.py +++ b/telefuser/worker/parallel_worker.py @@ -7,24 +7,25 @@ from __future__ import annotations import gc -import itertools import os import signal import threading import time from collections.abc import Callable, Collection from datetime import timedelta +from functools import partial from multiprocessing.queues import SimpleQueue from queue import Empty -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, BinaryIO import torch import torch.distributed as dist import torch.multiprocessing as mp -import torch.multiprocessing.spawn +from torch.multiprocessing.spawn import ProcessContext from telefuser.core.base_stage import BaseStage -from telefuser.platforms import current_platform +from telefuser.platforms import BasePlatform, CpuPlatform, current_platform, requires_host_tensor_queue, resolve_device +from telefuser.platforms.hccl import reserve_hccl_ports from telefuser.utils.logging import logger from telefuser.utils.system import PortAllocator @@ -36,29 +37,26 @@ _DISCARD_TENSOR_REFS = "__telefuser_discard_tensor_refs__" -# Each concurrent worker group needs a distinct HCCL socket range on NPU hosts; -# overlapping ranges fail comm init with EJ0003 (port already bound). -_hccl_group_counter = itertools.count() +def _worker_platform(stage: BaseStage) -> BasePlatform: + if stage.device_type == "cpu" and current_platform.device_type != "cpu": + return CpuPlatform() + return current_platform -def to_device(data: Any, device: str | torch.device) -> Any: - """Recursively move data to target device.""" + +def to_device(data: Any, device: str | torch.device, *, preserve_tuples: bool = False) -> Any: + """Move nested inputs; legacy callers receive lists for positional argument tuples.""" if isinstance(data, dict): - return {k: to_device(v, device) for k, v in data.items()} - if isinstance(data, tuple | list): - return [to_device(t, device) for t in data] - elif isinstance(data, torch.Tensor): - if data.device == device: - # Already on target device - ensure sharing for CPU tensors - if device == "cpu" and not data.is_shared(): - data.share_memory_() - return data + return {k: to_device(v, device, preserve_tuples=preserve_tuples) for k, v in data.items()} + if isinstance(data, (tuple, list)): + items = [to_device(v, device, preserve_tuples=preserve_tuples) for v in data] + return tuple(items) if preserve_tuples and isinstance(data, tuple) else items + if isinstance(data, torch.Tensor): tensor = data.to(device) - if device == "cpu" and not tensor.is_shared(): + if tensor.device.type == "cpu" and not tensor.is_shared(): tensor.share_memory_() return tensor - else: - return data + return data def _worker_loop( @@ -71,6 +69,8 @@ def _worker_loop( tensor_output_channel: WorkerTensorChannel | None = None, tensor_input_channels: tuple[WorkerTensorChannel, ...] = (), hccl_if_base_port: int | None = None, + *, + rank_offset: int = 0, ) -> None: """Worker process main loop. @@ -78,8 +78,10 @@ def _worker_loop( """ from telefuser.utils.profiler import mark_as_worker_process + rank += rank_offset mark_as_worker_process() previous_sigint_handler = signal.signal(signal.SIGINT, signal.SIG_IGN) + platform = _worker_platform(stage) args = None kwargs = None stage_inputs = None @@ -90,24 +92,28 @@ def _worker_loop( # single-rank workers such as the LingBot condition encoder. torch.set_num_threads(parallel_config.worker_intra_op_threads) device = stage.device + host_queue = requires_host_tensor_queue(platform.device_type) if world_size > 1: os.environ["RANK"] = str(rank) os.environ["WORLD_SIZE"] = str(world_size) os.environ["MASTER_ADDR"] = "localhost" os.environ["MASTER_PORT"] = str(master_port) if hccl_if_base_port is not None: - # Give each worker group a distinct host-side HCCL socket base; - # overlapping ranges fail comm init with EJ0003 when several - # groups share the same devices (e.g. denoise + VAE workers). os.environ["HCCL_IF_BASE_PORT"] = str(hccl_if_base_port) - device_ids = parallel_config.device_ids - device_id = rank - if device_ids is not None: - device_id = device_ids[rank] - device = torch.device(type=current_platform.device_type, index=device_id) + + if host_queue or world_size > 1: + ids = parallel_config.device_ids + device_id = rank if world_size > 1 else device.index or 0 + if ids is not None: + device_id = ids[rank] + device = torch.device(platform.device_type, device_id) stage.model_runtime_config.device_id = device_id stage.device = device - if current_platform.device_type == "cuda": + if platform.device_type != "cpu": + platform.set_device(device) + logger.info(f"Worker {stage.name} rank {rank}/{world_size} bound to {device}") + if world_size > 1: + if platform.device_type == "cuda": # torch.distributed.all_reduce does not free the input tensor until # the synchronization point. This causes the memory usage to grow # as the number of all_reduce calls increases. This env var disables @@ -118,14 +124,13 @@ def _worker_loop( timeout = timedelta(seconds=600) dist.init_process_group( - backend=current_platform.dist_backend, + backend=platform.dist_backend, init_method="env://", timeout=timeout, world_size=world_size, rank=rank, - device_id=device, + device_id=device if device.type != "cpu" else None, ) - current_platform.set_device(device) stage.parallel_models() while True: @@ -156,22 +161,21 @@ def _worker_loop( if channel.contains_ref(stage_inputs): stage_inputs = channel.receive(stage_inputs, rank=rank, device=device) args, kwargs = stage_inputs - kwargs = to_device(kwargs, device) - args = to_device(args, device) + kwargs = to_device(kwargs, device, preserve_tuples=host_queue) + args = to_device(args, device, preserve_tuples=host_queue) with torch.no_grad(): - if not isinstance(args, list): + if not isinstance(args, (list, tuple)): args = [args] y = getattr(stage, name)(*args, **kwargs) del kwargs, args if getattr(stage, "empty_cache_after_call", True): - current_platform.empty_cache() + platform.empty_cache() if transport_output and tensor_output_channel is not None and (world_size == 1 or rank == 0): y = tensor_output_channel.send(y) # Always output results when world_size=1 if world_size == 1 or rank == 0: - if current_platform.device_type != "cuda": - # Queue transport without reliable device IPC: marshal results through CPU. - y = to_device(y, "cpu") + if host_queue: + y = to_device(y, "cpu", preserve_tuples=True) queue_out.put(y) except Exception as e: import traceback @@ -189,18 +193,43 @@ def _worker_loop( kwargs = None stage_inputs = None y = None - current_platform.synchronize() + if platform.device_type != "cpu": + platform.synchronize() + platform.ipc_collect() gc.collect() - current_platform.empty_cache() - current_platform.ipc_collect() - if world_size > 1: + platform.empty_cache() + if world_size > 1 and dist.is_initialized(): dist.destroy_process_group() signal.signal(signal.SIGINT, previous_sigint_handler) +def _spawn_hccl_group(world_size: int, arguments: tuple) -> ProcessContext: + """Keep handles to earlier ranks if spawning a later rank fails in the parent.""" + contexts = [] + try: + for rank in range(world_size): + context = mp.spawn(partial(_worker_loop, rank_offset=rank), args=arguments, nprocs=1, join=False) + contexts.append(context) + return ProcessContext( + [context.processes[0] for context in contexts], + [context.error_files[0] for context in contexts], + ) + except BaseException: + for context in contexts: + process = context.processes[0] + if process.is_alive(): + process.kill() + process.join() + raise + + class ParallelWorker: """Multi-process worker for distributed execution across multiple GPUs.""" + _hccl_lock: BinaryIO | None = None + _master_port: int | None = None + ctx = None + def __init__( self, stage: BaseStage, @@ -214,14 +243,25 @@ def __init__( self._stage: BaseStage = stage # Keep reference to wrapped stage for metrics proxying self.world_size: int = parallel_config.world_size - self.device: str = current_platform.device_type + platform = _worker_platform(stage) + self.device: str = platform.device_type self.device_ids: list[int] = parallel_config.device_ids if self.device_ids is None: self.device_ids = list(range(self.world_size)) + if stage.device_type in ("npu", "cuda") and stage.device_type != platform.device_type: + resolve_device(stage.device) + if self.device == "npu": + if parallel_config.device_ids is None: + self.device_ids = [stage.device.index or 0] + if len(set(self.device_ids)) != len(self.device_ids): + raise ValueError(f"NPU worker ranks require distinct device IDs, got {self.device_ids}") + for idx in self.device_ids: + resolve_device(f"npu:{idx}") + self.name: str = f"Parallel Worker {stage.name}" # Queue transport requires CPU marshalling on platforms without reliable device IPC (e.g. NPU). - self.queue_with_cpu: bool = parallel_config.queue_with_cpu or current_platform.device_type != "cuda" + self.queue_with_cpu: bool = parallel_config.queue_with_cpu or requires_host_tensor_queue(self.device) self.timeout: int = parallel_config.timeout self._lifecycle_lock = threading.Lock() self._failed = False @@ -254,25 +294,38 @@ def __init__( self.queue_in: list[SimpleQueue] = [spawn_ctx.SimpleQueue() for _ in range(self.world_size)] self.queue_out: SimpleQueue = spawn_ctx.SimpleQueue() - master_port = PortAllocator().get_free_port_in_interval() - logger.info(f"parallel worker {self.name} with port {master_port}, world_size={self.world_size}") - - # For world_size=1 case, still use spawn to start one process - self.ctx = mp.spawn( - _worker_loop, - args=( + try: + self._master_port = PortAllocator().get_free_port_in_interval() + hccl_port = None + if self.world_size > 1 and platform.dist_backend == "hccl": + hccl_port, self._hccl_lock = reserve_hccl_ports() + logger.info(f"{self.name}: world_size={self.world_size}, master_port={self._master_port}, hccl={hccl_port}") + arguments = ( self.world_size, self.queue_in, self.queue_out, stage, - master_port, + self._master_port, self.tensor_output_channel, self.tensor_input_channels, - 60000 + 1024 * next(_hccl_group_counter) if current_platform.device_type == "npu" else None, - ), - nprocs=self.world_size, - join=False, - ) + hccl_port, + ) + if hccl_port is not None: + self.ctx = _spawn_hccl_group(self.world_size, arguments) + else: + self.ctx = mp.spawn(_worker_loop, args=arguments, nprocs=self.world_size, join=False) + except BaseException: + self.close() + raise + + def _release_ports(self) -> None: + with self._lifecycle_lock: + if self._hccl_lock is not None: + self._hccl_lock.close() + self._hccl_lock = None + if self._master_port is not None: + PortAllocator.release_port(self._master_port) + self._master_port = None @property def failed(self) -> bool: @@ -296,7 +349,7 @@ def _ensure_usable(self) -> None: raise RuntimeError(f"ParallelWorker:{self.name} has failed: {self._failure_reason}") def _terminate_processes(self) -> None: - if not hasattr(self, "ctx"): + if self.ctx is None: return for process in self.ctx.processes: try: @@ -314,6 +367,7 @@ def _mark_failed(self, reason: str) -> None: self._failure_reason = reason logger.error(f"ParallelWorker:{self.name} marked failed: {reason}") self._terminate_processes() + self._release_ports() def _wait_result(self, method_name: str) -> Any: try: @@ -364,7 +418,7 @@ def put_data(self, data: Any) -> None: """Send data to all worker processes.""" self._ensure_usable() if self.queue_with_cpu: - data = to_device(data, "cpu") + data = to_device(data, "cpu", preserve_tuples=requires_host_tensor_queue(self.device)) for q in self.queue_in: q.put(data) @@ -436,7 +490,7 @@ def close(self) -> None: return self._closed = True - if hasattr(self, "ctx"): + if self.ctx is not None: if not self._failed: for queue in self.queue_in: try: @@ -451,15 +505,16 @@ def close(self) -> None: process.join(timeout=2) except Exception as exc: logger.warning(f"Failed to join {self.name} process: {exc}") - for queue in self.queue_in: - try: - queue.close() - except Exception: - pass + for queue in self.queue_in: try: - self.queue_out.close() + queue.close() except Exception: pass + try: + self.queue_out.close() + except Exception: + pass + self._release_ports() def __del__(self) -> None: """Best-effort fallback; callers should use close explicitly.""" diff --git a/telefuser/worker/tensor_channel.py b/telefuser/worker/tensor_channel.py index 7243e58f..2370123f 100644 --- a/telefuser/worker/tensor_channel.py +++ b/telefuser/worker/tensor_channel.py @@ -13,6 +13,8 @@ import torch import torch.multiprocessing as mp +from telefuser.platforms import requires_host_tensor_queue + _MAX_CUDA_IPC_POOL_PROFILES = 8 @@ -75,7 +77,7 @@ class WorkerTensorChannel: """Point-to-point tensor path that bypasses the parent process. CUDA tensors use persistent IPC pools whose rank-local metadata travels in - :class:`WorkerTensorRef`. CPU tensors and dynamic-profile CUDA fallbacks use + :class:`WorkerTensorRef`. Ascend tensors are staged on CPU. Host tensors and dynamic-profile CUDA fallbacks use one multiprocessing queue per consumer rank. The parent never materializes tensor contents. """ @@ -166,23 +168,15 @@ def replace(item: Any) -> Any: shard_dim=shard_dim, ) tensor_index += 1 - if item.device.type == "cuda": - rank_payloads = self._stage_cuda_tensor(ref, item) - if rank_payloads is not None: - ref = dataclass_replace(ref, _cuda_payloads=rank_payloads) - else: - rank_tensors = ( - (item,) * self.consumer_world_size - if shard_dim is None - else torch.tensor_split(item, self.consumer_world_size, dim=shard_dim) - ) - for queue, rank_tensor in zip(self._queues, rank_tensors): - queue.put((ref, rank_tensor)) + rank_payloads = self._stage_cuda_tensor(ref, item) if item.device.type == "cuda" else None + if rank_payloads is not None: + ref = dataclass_replace(ref, _cuda_payloads=rank_payloads) else: + payload = item.cpu() if requires_host_tensor_queue(item.device.type) else item rank_tensors = ( - (item,) * self.consumer_world_size + (payload,) * self.consumer_world_size if shard_dim is None - else torch.tensor_split(item, self.consumer_world_size, dim=shard_dim) + else torch.tensor_split(payload, self.consumer_world_size, dim=shard_dim) ) for queue, rank_tensor in zip(self._queues, rank_tensors): queue.put((ref, rank_tensor)) @@ -231,7 +225,9 @@ def replace(item: Any) -> Any: raise RuntimeError( f"Tensor channel {self.channel_id} received incompatible tensor metadata for {item}" ) - if not isinstance(received, _CudaIpcPayload) and str(tensor.device) != item.source_device: + source_type = item.source_device.partition(":")[0] + queue_device = "cpu" if requires_host_tensor_queue(source_type) else item.source_device + if not isinstance(received, _CudaIpcPayload) and str(tensor.device) != queue_device: raise RuntimeError( f"Tensor channel {self.channel_id} received incompatible tensor metadata for {item}" ) diff --git a/tests/unit/platforms/test_ascend.py b/tests/unit/platforms/test_ascend.py new file mode 100644 index 00000000..89d8fc0e --- /dev/null +++ b/tests/unit/platforms/test_ascend.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +import os +import socket +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest +import torch + +from telefuser import platforms +from telefuser.platforms.hccl import reserve_hccl_ports + + +@pytest.mark.parametrize(("given", "expected"), [("npu", "npu:0"), ("npu:07", "npu:7"), ("rocm:1", "cuda:1")]) +def test_normalize_without_vendor_module(given: str, expected: str) -> None: + assert platforms.normalize_device_name(given) == expected + + +def test_explicit_devices_fail_before_starting_workers(monkeypatch) -> None: + with pytest.raises(ValueError): + platforms.normalize_device_name("npu:-1") + monkeypatch.setitem(sys.modules, "torch_npu", None) + with pytest.raises(RuntimeError, match="torch_npu") as failure: + platforms.resolve_device("npu:1") + assert isinstance(failure.value.__cause__, ImportError) + monkeypatch.setitem(sys.modules, "torch_npu", SimpleNamespace()) + monkeypatch.setattr(platforms, "current_platform", platforms.CpuPlatform()) + with pytest.raises(RuntimeError, match="detected cpu"): + platforms.resolve_device("npu:1") + monkeypatch.setattr( + platforms, "current_platform", SimpleNamespace(device_type="npu", device_name="npu", device_count=lambda: 2) + ) + with pytest.raises(ValueError, match="2 visible"): + platforms.resolve_device("npu:2") + assert platforms.resolve_device("cpu") == torch.device("cpu") + + +@pytest.mark.parametrize("value", ["0", "65521", "bad"]) +def test_invalid_hccl_base(monkeypatch, value: str) -> None: + monkeypatch.setenv("HCCL_IF_BASE_PORT", value) + with pytest.raises(ValueError, match="HCCL_IF_BASE_PORT"): + reserve_hccl_ports() + + +def test_hccl_lease_across_parents_and_shutdown(tmp_path: Path, monkeypatch) -> None: + monkeypatch.setattr("tempfile.tempdir", str(tmp_path)) + monkeypatch.setenv("HCCL_IF_BASE_PORT", "61001") + base, lock = reserve_hccl_ports() + with lock: + assert base % 16 == 0 and base >= 61001 + code = "from telefuser.platforms.hccl import reserve_hccl_ports; base, lock = reserve_hccl_ports(); print(base)" + other = subprocess.check_output( + [sys.executable, "-c", code], text=True, env={**os.environ, "TMPDIR": str(tmp_path)} + ) + assert int(other.strip()) >= base + 16 + for _ in range(8): + recycled, lock = reserve_hccl_ports() + with lock: + assert recycled == base + with socket.socket() as occupied: + occupied.bind(("", base + 7)) + different, lock = reserve_hccl_ports() + with lock: + assert different >= base + 16 diff --git a/tests/unit/worker/test_parallel_worker_lifecycle.py b/tests/unit/worker/test_parallel_worker_lifecycle.py index 284b90eb..1663d044 100644 --- a/tests/unit/worker/test_parallel_worker_lifecycle.py +++ b/tests/unit/worker/test_parallel_worker_lifecycle.py @@ -99,3 +99,26 @@ def test_close_after_failure_does_not_submit_more_worker_commands() -> None: for queue in worker.queue_in: queue.put.assert_not_called() queue.close.assert_called_once_with() + + +def test_hccl_spawn_failure_returns_lease_and_closes_queues(monkeypatch) -> None: + from telefuser.worker import parallel_worker as module + + lock = MagicMock() + mp = MagicMock() + first_rank = MagicMock() + mp.spawn.side_effect = [SimpleNamespace(processes=[first_rank]), RuntimeError("failed spawn")] + stage = MagicMock() + config = stage.model_runtime_config.parallel_config + config.world_size, config.device_ids = 2, [1, 2] + monkeypatch.setattr(module, "current_platform", SimpleNamespace(device_type="npu", dist_backend="hccl")) + monkeypatch.setattr(module, "resolve_device", MagicMock()) + monkeypatch.setattr(module, "reserve_hccl_ports", lambda: (60000, lock)) + monkeypatch.setattr(module, "mp", mp) + with pytest.raises(RuntimeError, match="failed spawn"): + ParallelWorker(stage) + assert mp.spawn.call_args.kwargs["args"][-1] == 60000 + first_rank.kill.assert_called_once_with() + first_rank.join.assert_called_once_with() + lock.close.assert_called_once() + assert mp.get_context.return_value.SimpleQueue.return_value.close.call_count == 3 diff --git a/tests/unit/worker/test_tensor_channel.py b/tests/unit/worker/test_tensor_channel.py index 510ff5e8..9568324a 100644 --- a/tests/unit/worker/test_tensor_channel.py +++ b/tests/unit/worker/test_tensor_channel.py @@ -299,3 +299,30 @@ def test_tensor_channel_uses_cuda_ipc_and_cross_gpu_peer_copy() -> None: metadata_queue.close() result_queue.close() channel.close() + + +def test_ascend_payload_stages_once_before_sharding_and_keeps_source_metadata() -> None: + class NpuDevice: + type = "npu" + + def __str__(self): + return "npu:1" + + class NpuTensor(torch.Tensor): + device = NpuDevice() + copies = 0 + + def cpu(self): + self.copies += 1 + return self.as_subclass(torch.Tensor) + + channel = WorkerTensorChannel(consumer_world_size=2, shard_dim=0, timeout=1) + original = torch.arange(5) + source = original.as_subclass(NpuTensor) + try: + ref, duplicate = channel.send((source, [source])) + assert ref is duplicate[0] and ref.source_device == "npu:1" and source.copies == 1 + parts = [channel.receive(ref, rank=i, device="cpu") for i in range(2)] + torch.testing.assert_close(torch.cat(parts), original) + finally: + channel.close() From 2770453dfa2f596fed44d875d7d18452fdfc092b Mon Sep 17 00:00:00 2001 From: jinyx5 Date: Fri, 20 Nov 2026 17:24:20 +0800 Subject: [PATCH 13/21] feat(docker): add the CANN 9.3 arm64 image and Wan runbook Pin the 910B CANN 9.3.0-beta.1 arm64 manifest, CPU PyTorch 2.11.0 and the Ascend 2.11.0.post4 wheel, including hashes for the runtime closure. Move GPU-specific packages into the cuda extra. Document source builds, release digests, device mapping and the 1/2/4-card offline T2V recipe in both languages. The documentation build and shell examples were checked. Hardware qualification is reported per card count. --- README.md | 2 + README_zh.md | 2 + docker/Dockerfile.ascend | 30 +++++ docker/Dockerfile.ascend.dockerignore | 15 +++ docker/ascend/README.md | 61 +++++++++ docker/ascend/requirements.in | 12 ++ docker/ascend/requirements.lock | 114 +++++++++++++++++ docs/en/ascend_910b.md | 149 ++++++++++++++++++++++ docs/en/installation.md | 13 +- docs/en/platforms.md | 18 ++- docs/zh/ascend_910b.md | 173 ++++++++++++++++++++++++++ docs/zh/installation.md | 12 +- docs/zh/platforms.md | 17 ++- examples/wan_video/README.md | 3 + mkdocs.yml | 2 + pyproject.toml | 9 +- 16 files changed, 597 insertions(+), 35 deletions(-) create mode 100644 docker/Dockerfile.ascend create mode 100644 docker/Dockerfile.ascend.dockerignore create mode 100644 docker/ascend/README.md create mode 100644 docker/ascend/requirements.in create mode 100644 docker/ascend/requirements.lock create mode 100644 docs/en/ascend_910b.md create mode 100644 docs/zh/ascend_910b.md diff --git a/README.md b/README.md index 3c35b0a8..27a97742 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,8 @@ The project treats a world model as more than a function that returns a single c pip install -e . ``` +For NVIDIA Triton and quantization support, install `pip install -e ".[cuda]"`. Ascend 910B2 users can build the pinned container and run the Wan example. [Ascend installation](docs/en/ascend_910b.md) + For development: ```bash diff --git a/README_zh.md b/README_zh.md index 05ad6b08..694aaa72 100644 --- a/README_zh.md +++ b/README_zh.md @@ -79,6 +79,8 @@ TeleFuser 是一个开源的多模态生成与世界模型流式推理和服务 pip install -e . ``` +NVIDIA 的 Triton 与量化功能使用 `pip install -e ".[cuda]"`。Atlas 910B2 可从固定版本镜像运行 Wan 示例。[昇腾镜像安装](docs/zh/ascend_910b.md) + 开发环境安装: ```bash diff --git a/docker/Dockerfile.ascend b/docker/Dockerfile.ascend new file mode 100644 index 00000000..bf0dff21 --- /dev/null +++ b/docker/Dockerfile.ascend @@ -0,0 +1,30 @@ +ARG CANN_REPOSITORY=swr.cn-south-1.myhuaweicloud.com/ascendhub/cann +FROM ${CANN_REPOSITORY}:9.3.0-beta.1-910b-ubuntu22.04-py3.11@sha256:cf6ab60eefb56b8d0e3ecad6d03c30c457b4dff86bcbf5dd7ff1a90427348b7b + +ARG TELEFUSER_VERSION=0.0.0+local +ARG SOURCE_REVISION=local +LABEL org.opencontainers.image.source="https://github.com/Tele-AI/TeleFuser" \ + org.opencontainers.image.revision="${SOURCE_REVISION}" \ + org.opencontainers.image.version="${TELEFUSER_VERSION}" + +USER root +RUN test "$(uname -m)" = aarch64 && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends ffmpeg \ + && rm -rf /var/lib/apt/lists/* +RUN python3.11 -m venv /opt/telefuser/venv +ENV PATH="/opt/telefuser/venv/bin:${PATH}" +COPY docker/ascend/requirements.lock /opt/telefuser/requirements.lock +RUN python -m pip install --no-cache-dir --only-binary=:all: --require-hashes \ + --index-url https://pypi.org/simple -r /opt/telefuser/requirements.lock + +WORKDIR /opt/telefuser +COPY pyproject.toml README.md LICENSE /tmp/telefuser-src/ +COPY telefuser /tmp/telefuser-src/telefuser +RUN SETUPTOOLS_SCM_PRETEND_VERSION_FOR_TELEFUSER="${TELEFUSER_VERSION}" \ + python -m pip wheel --no-build-isolation --no-deps -w /tmp/telefuser-dist /tmp/telefuser-src \ + && python -m pip install --no-deps /tmp/telefuser-dist/telefuser-*.whl \ + && python -m pip check && rm -rf /tmp/telefuser-src /tmp/telefuser-dist \ + && ffprobe -version > /dev/null && hf --version +COPY examples/wan_video/wan22_t2v_5b.py ./examples/wan_video/wan22_t2v_5b.py +# Inherit the pinned CANN image's entrypoint to initialize the toolkit, AscendNPU IR and ATB. +CMD ["telefuser", "--help"] diff --git a/docker/Dockerfile.ascend.dockerignore b/docker/Dockerfile.ascend.dockerignore new file mode 100644 index 00000000..cc01d391 --- /dev/null +++ b/docker/Dockerfile.ascend.dockerignore @@ -0,0 +1,15 @@ +** +!pyproject.toml +!README.md +!LICENSE +!telefuser/ +!telefuser/** +**/__pycache__ +**/*.pyc +telefuser/_version.py +!docker/ +!docker/ascend/ +!docker/ascend/requirements.lock +!examples/ +!examples/wan_video/ +!examples/wan_video/wan22_t2v_5b.py diff --git a/docker/ascend/README.md b/docker/ascend/README.md new file mode 100644 index 00000000..7054255d --- /dev/null +++ b/docker/ascend/README.md @@ -0,0 +1,61 @@ +# Ascend image maintenance + +The 910B image targets Linux arm64 and CPython 3.11. The Dockerfile pins the architecture-specific CANN manifest. The runtime lock combines the base dependencies from `pyproject.toml` and the stack from `requirements.in`. Every entry names one compatible wheel and its SHA-256. The Ascend release provides the wheel for torch_npu 2.11.0.post4. [Ascend release artifacts](https://api.gitcode.com/api/v5/repos/Ascend/pytorch/releases/tags/v26.3.0-beta.1-pytorch2.11.0) + +The default registry is AscendHub in China. For Docker Hub, pass `--build-arg CANN_REPOSITORY=docker.io/ascendai/cann`. Both registries serve the same pinned arm64 manifest. [Ascend image repositories](https://github.com/Ascend/cann-container-image/blob/main/OVERVIEW.zh.md) + +## Update the dependency lock + +Run from the repository root in a clean CPython 3.11 environment on Ubuntu 22.04 arm64. Match each of the four direct wheel URLs and hashes in `requirements.in` to their publisher before resolving. The pip report includes the source checkout as a separate local distribution. + +```bash +python -m pip install --dry-run --ignore-installed --only-binary=:all: \ + --index-url https://pypi.org/simple --report /tmp/telefuser-ascend-resolution.json \ + -r docker/ascend/requirements.in . +python - <<'PY' +import json +from pathlib import Path + +from packaging.utils import canonicalize_name + +directory = Path("docker/ascend") +direct = { + line.split(" @ ")[0]: line.split(" @ ")[1].partition("#")[0] + for line in (directory / "requirements.in").read_text().splitlines() if " @ " in line +} +entries = [] +for record in json.loads(Path("/tmp/telefuser-ascend-resolution.json").read_text())["install"]: + name = canonicalize_name(record["metadata"]["name"]) + if name == "telefuser": + continue + digest = record["download_info"]["archive_info"]["hashes"]["sha256"] + requirement = f"{name} @ {direct[name]}" if name in direct else f"{name}=={record['metadata']['version']}" + entries.append((name, f"{requirement} --hash=sha256:{digest}")) +header = "# Linux arm64, Ubuntu 22.04, CPython 3.11. One wheel hash per distribution." +(directory / "requirements.lock").write_text("\n".join([header, *(line for _, line in sorted(entries))]) + "\n") +PY +``` + +## Build, qualify and publish + +Build from a clean release checkout. Replace the version below with its PEP 440 version. Tag the candidate with the commit. Record the digest printed by `docker push`. Run the hardware checks on that digest, including separate 1-, 2- and 4-card Wan runs, before promoting a release. [Qualification procedure](../../docs/en/ascend_910b.md) + +```bash +read -r -p 'TeleFuser package version > ' TF_VERSION +TF_REV=$(git rev-parse HEAD) +TF_REPO=ghcr.io/tele-ai/telefuser +TF_CANDIDATE="$TF_REPO:sha-$TF_REV-ascend910b" +docker build --platform linux/arm64 -f docker/Dockerfile.ascend \ + --build-arg "TELEFUSER_VERSION=$TF_VERSION" --build-arg "SOURCE_REVISION=$TF_REV" \ + -t "$TF_CANDIDATE" . +docker push "$TF_CANDIDATE" +``` + +After qualification, point the public release tag at the tested manifest. Confirm anonymous access and publish the digest, driver and firmware versions and validated card counts alongside the release. + +```bash +read -r -p 'Tested TeleFuser sha256 digest > ' TF_DIGEST +docker buildx imagetools create --prefer-index=false \ + --tag "$TF_REPO:$TF_VERSION-ascend910b" "$TF_REPO@$TF_DIGEST" +docker buildx imagetools inspect "$TF_REPO:$TF_VERSION-ascend910b" +``` diff --git a/docker/ascend/requirements.in b/docker/ascend/requirements.in new file mode 100644 index 00000000..c9723165 --- /dev/null +++ b/docker/ascend/requirements.in @@ -0,0 +1,12 @@ +# Ubuntu 22.04 (glibc 2.35), CPython 3.11, Linux arm64. +# Match Ascend Extension for PyTorch v26.3.0-beta.1-pytorch2.11.0. +torch @ https://download.pytorch.org/whl/cpu/torch-2.11.0%2Bcpu-cp311-cp311-manylinux_2_28_aarch64.whl#sha256=46fbb0aa257bb781efbfad648f5b045c0e232573b661f1461593db61342e9096 +torchvision @ https://download.pytorch.org/whl/cpu/torchvision-0.26.0%2Bcpu-cp311-cp311-manylinux_2_28_aarch64.whl#sha256=c11e55041f6b84a6c4fb28981b901475aa81c38695ccec6ddfcc54c3fa9fac4f +torchaudio @ https://download.pytorch.org/whl/cpu/torchaudio-2.11.0%2Bcpu-cp311-cp311-manylinux_2_28_aarch64.whl#sha256=7a5ecdd20fb26d8eb1d62135ddc6db0f806b17667c10e82df913fd509d1f5cfe +torch-npu @ https://gitcode.com/Ascend/pytorch/releases/download/v26.3.0-beta.1-pytorch2.11.0/torch_npu-2.11.0.post4-cp311-cp311-manylinux_2_28_aarch64.whl#sha256=10b3f58c8b9396b4be4027c35cf0e08ef6223699b9cf186c1d89fd5b75652536 +torchcodec==0.11.0 + +# Build the TeleFuser wheel with the same pinned tools in the image. +setuptools==80.9.0 +setuptools-scm==8.3.1 +wheel==0.45.1 diff --git a/docker/ascend/requirements.lock b/docker/ascend/requirements.lock new file mode 100644 index 00000000..4b3c94e9 --- /dev/null +++ b/docker/ascend/requirements.lock @@ -0,0 +1,114 @@ +# Linux arm64, Ubuntu 22.04, CPython 3.11. One wheel hash per distribution. +agent-detector==2.0.0 --hash=sha256:22c6a1c9c23894a12f55aea625f060b8d3330fba774de72d02d5442bb580417f +aiofiles==25.1.0 --hash=sha256:abe311e527c862958650f9438e859c1fa7568a141b22abcd015e120e86a85695 +aiohappyeyeballs==2.7.1 --hash=sha256:9243213661e29250eb41368e5daa826fc017156c3b8a11440826b2e3ed376472 +aiohttp==3.14.5 --hash=sha256:32e8fa6644e541fcd7e02430588c7fc93b602c1778ea0bc345505db76b61cfb4 +aiosignal==1.4.0 --hash=sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e +annotated-doc==0.0.7 --hash=sha256:6bc31bbbec6dfaaedc1676dbc11dbda96fe22a2b5c400e9ebad113fec44a9dc6 +annotated-types==0.8.0 --hash=sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0 +anyio==4.16.0 --hash=sha256:c66ae634e0b8228396b801741beb3294326d4b6102f7bba4041c0dfb58c29026 +attrs==26.1.0 --hash=sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309 +av==16.1.0 --hash=sha256:720edd4d25aa73723c1532bb0597806d7b9af5ee34fc02358782c358cfe2f879 +certifi==2026.10.23 --hash=sha256:e9b54de76e530bb6cf0ccd036787103acb58fd42ba930ea8bf948ddbaa2f61d1 +cffi==2.1.1 --hash=sha256:3311ed60d36f83378794e1009ac6258bafbf81f7888b4caa7b35a521e3f95813 +charset-normalizer==3.5.2 --hash=sha256:d760fe2a4d7c3b226cb9026d6a842868d52a7901bd98420e1baf14e80da85cf5 +click==8.5.1 --hash=sha256:8bb5376d2006950386f8c6427613334a2acadc5f559cb80e638d00b4df957e37 +detect-installer==0.2.1 --hash=sha256:87e2ce7d05cb108b178f8e45bb80d3d88c0cb4b397983345c18ad6562e02e620 +diffusers==0.43.0 --hash=sha256:8afb8ff1b09d693854de0412d8bdc94fc9e6e6e4efefb2386c616f64a4dc9ac6 +dnspython==2.9.2 --hash=sha256:af15f7d385a6ae2fb35a796fde2138004c8c42c7389090e2addc83e3b3340fd8 +einops==0.8.2 --hash=sha256:54058201ac7087911181bfec4af6091bb59380360f069276601256a76af08193 +email-validator==2.4.0 --hash=sha256:4454e41c4e1c6b6caee34a1dc71d5e736791562be8fd179b02e298f796522684 +fastapi==0.165.0 --hash=sha256:896f48462042bb4f162147eb8fac7e858753dc1f486765709baf105d70e4fc5b +fastapi-cli==0.0.34 --hash=sha256:4a1185b66308c7543e98957426a8ecd205d58853514cb9a68c78066f2e6e467b +fastapi-cloud-cli==0.28.0 --hash=sha256:e1409bc42017a86f8092fdf1dd64ead129698936cfeb151a6c3e023f611fd620 +fastar==0.12.0 --hash=sha256:1f0432d1138e7f3b65a3afc324ac373706751d1b8554541fe33d5e62ab08ff11 +filelock==4.2.12 --hash=sha256:21decfec02f116ed4c954433cdb31f28c3b346b784543177b6d7059da2702ee9 +frozenlist==1.8.0 --hash=sha256:957e7c38f250991e48a9a73e6423db1bb9dd14e722a10f6b8bb8e16a0f55f695 +fsspec==2026.9.0 --hash=sha256:8dd6e646e99ea382bd85f97a45e6b526a442d79423a7dc673f1e2756d05fcb5f +ftfy==6.3.1 --hash=sha256:7c70eb532015cd2f9adb53f101fb6c7945988d023a085d127d1573dc49dd0083 +googleapis-common-protos==1.76.0 --hash=sha256:59e13344a9f3651aa201a03e2bee815a3bf43a4baf1f9ff8a9b62a840ce25fac +h11==0.16.0 --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 +hf-xet==1.8.0 --hash=sha256:8800e3444770f2c630b361bc6d5c6ebb770b7830646cf632fbb79d0e5bbc9e7f +httpcore==1.0.9 --hash=sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55 +httpcore2==2.16.2 --hash=sha256:16051ae2a42f776007543f32de9349a317b23d2cbb4e1456b4bac0e88b87419a +httptools==0.9.0 --hash=sha256:6e2780e33a58a93f27cc3bb74a55bae6f9a8278a1dbabdff392940d30d381671 +httpx==0.28.1 --hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad +httpx2==2.16.2 --hash=sha256:f05c13aade4c45dbf1c9060647a1058f0acad62398fad96e741269edb33d46f5 +huggingface-hub==1.33.0 --hash=sha256:04e434b06e100eddbce9a6e817d72693a7884b10a79bd67ab48080d5c07eb899 +idna==3.20 --hash=sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c +imageio==2.38.2 --hash=sha256:d71174e7af8369a4f73525c5b1c0d2621eb2827ed3cfd7327560be02ac1b4084 +imageio-ffmpeg==0.8.1 --hash=sha256:fad3176f859b25869e60a8ed79957d4e3ed74a97a95e5f0cdcb91455f6c6d00e +importlib-metadata==10.2.0 --hash=sha256:c7a4bf1c7479601f6b8e437cfc14f4f77028ba62a8f34ae95cd93d9774dfba51 +jinja2==3.1.6 --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 +lazy-importer==0.1.0 --hash=sha256:bd93fab7bef18e842eea02b839410954d37889140dc64ca4d9680aec6880c739 +livekit==1.3.0 --hash=sha256:e8e60f88925b812adb06adb73e5e3b1c667c401b019b6213a9bac81e0328b3b3 +livekit-api==1.2.3 --hash=sha256:9f832cf1ad2b6b41e4ef183cd8d6cfdc97967e0ef82d4e7c982db87ed86fc9da +livekit-protocol==1.1.32 --hash=sha256:f86e104b7a84e743f7e382892d781df4f103eefdbfb24520359158f9c427e032 +loguru==0.8.0 --hash=sha256:e0fba09d6f76ba2d2f4bd1757122942f5a670c702e988b5efaeab99dacba0069 +markdown-it-py==4.3.0 --hash=sha256:7aa1511a46c39138de78d9eba92ce47e63124d8003fc98b3a351909e04239efa +markupsafe==3.0.4 --hash=sha256:849dd2bb0e5e4ab2b71c7191726a4a8d5aa8a610daa584728cbee0b710ddc4ef +mdurl==0.1.2 --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 +mpmath==1.3.0 --hash=sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c +multidict==7.2.1 --hash=sha256:295a15ccb8cfde9351dd6062b80af4e839460636b3c46813b4aa443905ded86a +networkx==3.6.1 --hash=sha256:d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762 +numpy==2.4.6 --hash=sha256:0ab0a9c4ffb1a6d95ef519fe4247dba8eb6b18ad93999f76b7f657039acabd47 +opencv-python-headless==5.1.0.95 --hash=sha256:11d9753e15b8b3c56e6aa765139f722ca19ff26016d43bc81035f5d122731a81 +opentelemetry-api==1.45.2 --hash=sha256:2cbd737377e710ee740d261c94e93fa0f08e6d037c18f17513cc0a2d94479a34 +opentelemetry-exporter-http-transport==0.66b2 --hash=sha256:fd5f5b490c099acda1cd7b909d5e8981436bf18cc8c6098d302d114c271720e9 +opentelemetry-exporter-otlp-common==0.66b2 --hash=sha256:ace86efb75489f925f1ef809856ae5f0c7db534e4afdc624530222228b1236a5 +opentelemetry-exporter-otlp-proto-common==1.45.2 --hash=sha256:af483b24217028892f017096731b794b196b785025d2e6808874720fd0b05a80 +opentelemetry-exporter-otlp-proto-http==1.45.2 --hash=sha256:6e4d025138829d0ee57946930c2ccad6491ad4d92a958e9c7eea69278fe60cfc +opentelemetry-proto==1.45.2 --hash=sha256:03a7ca1b43f89fd5b965d5975902bb2515c0c86e2428b456836e9a6149c6235a +opentelemetry-sdk==1.45.2 --hash=sha256:ff974ea70ee4f1d51e2ec152db37c2b626c7bf82bb3f7d86114fd8704e7199c6 +opentelemetry-semantic-conventions==0.66b2 --hash=sha256:4cbe61e5f9275d57ab31708446e7938ef4291ebbe6a7fca6f7a629c5acae87d8 +packaging==26.3 --hash=sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c +pillow==13.0.0 --hash=sha256:7c88c10dd432f3b39ccdafc9ed81d0c21b8fd634502574da76d1bfa4fe2c76d0 +propcache==0.5.4 --hash=sha256:62530ca89187827e4a4fe733f971abe81a7542eeea48ff61995f19b64d7199c8 +protobuf==7.36.2 --hash=sha256:e11e1f0180583a2af89db6a2ecd9e8dc40aa6d2988ca175bfd0e6d12ea72d74e +psutil==7.2.2 --hash=sha256:b0726cecd84f9474419d67252add4ac0cd9811b04d61123054b9fb6f57df6e9e +pycparser==3.12 --hash=sha256:a076f3f489b39b43ed13563a1dafe42447445bec2e294154ec5c57fcdda4c85e +pydantic==2.14.4 --hash=sha256:653a2ad9f6d8858529c77c72717d0ea991d6c6837fe683246b2f4b2bbfc3fadc +pydantic-core==2.50.5 --hash=sha256:ba2b34529984a8e3cb1bc62a7687617722695a2cb4e57958de32a0eba128a07a +pydantic-extra-types==2.11.1 --hash=sha256:1722ea2bddae5628ace25f2aa685b69978ef533123e5638cfbddb999e0100ec1 +pydantic-settings==2.16.3 --hash=sha256:d1db16a2d34006323a1b42c0640898da58adfbad7ec30baaa9e2b18aafc08999 +pygments==2.21.0 --hash=sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9 +pyjwt==2.15.1 --hash=sha256:42d59d631f7768a1028a64c7ff581a9bf7519804daf91fc5b6c56e30eec5e193 +python-dotenv==1.2.4 --hash=sha256:42269a8a5b3fd54ffa6f3d84b18abed50064717576b4ecf03dc4a55d8aa04fdc +python-multipart==0.0.34 --hash=sha256:ebacd224bd686eee2f9aa1066a657affe07435a185834e05e5235e2e59173e91 +pyyaml==6.0.3 --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c +regex==2026.11.19 --hash=sha256:3bd48fcc3222607d2b0e260be6e1a957b874f852b13af8c8643c8fbd8acf2425 +requests==2.34.2 --hash=sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0 +rich==15.0.0 --hash=sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb +rich-toolkit==0.20.6 --hash=sha256:465453ba6f94f99401f59f79fb562827ba5f6b4d4759015d40019f5c67750262 +rignore==0.8.1 --hash=sha256:2aa76742832db1fdac070fa8e694dda0daf6f2f7191832bd555d1e80f8c9c42b +safetensors==0.8.0 --hash=sha256:7a46e5ff292c356d6991e60942ba7f79817682d3a2cef0702136448cb9c4d235 +scipy==1.17.1 --hash=sha256:744b2bf3640d907b79f3fd7874efe432d1cf171ee721243e350f55234b4cec4c +sentry-sdk==2.75.0 --hash=sha256:9a10e97de2cbd42ad7e4a6e02a0d5af19de9811eb02445077d41ba14c08e170c +setuptools==80.9.0 --hash=sha256:062d34222ad13e0cc312a4c02d73f059e86a4acbfbdea8f8f76b28c99f306922 +setuptools-scm==8.3.1 --hash=sha256:332ca0d43791b818b841213e76b1971b7711a960761c5bea5fc5cdb5196fbce3 +shellingham==1.5.4 --hash=sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686 +soundfile==0.14.0 --hash=sha256:e85724a90bc99a6e8062c0b4ddf725f53b2a3b70afd4da875e9d2cfc4e92f377 +starlette==1.10.0 --hash=sha256:478c0ab3b4ced6e82719849c118fa406f0278a3499c1a898cd15cd31bdc38ce4 +sympy==1.14.0 --hash=sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5 +tokenizers==0.22.2 --hash=sha256:2249487018adec45d6e3554c71d46eb39fa8ea67156c640f7513eb26f318cec7 +torch @ https://download.pytorch.org/whl/cpu/torch-2.11.0%2Bcpu-cp311-cp311-manylinux_2_28_aarch64.whl --hash=sha256:46fbb0aa257bb781efbfad648f5b045c0e232573b661f1461593db61342e9096 +torch-npu @ https://gitcode.com/Ascend/pytorch/releases/download/v26.3.0-beta.1-pytorch2.11.0/torch_npu-2.11.0.post4-cp311-cp311-manylinux_2_28_aarch64.whl --hash=sha256:10b3f58c8b9396b4be4027c35cf0e08ef6223699b9cf186c1d89fd5b75652536 +torchaudio @ https://download.pytorch.org/whl/cpu/torchaudio-2.11.0%2Bcpu-cp311-cp311-manylinux_2_28_aarch64.whl --hash=sha256:7a5ecdd20fb26d8eb1d62135ddc6db0f806b17667c10e82df913fd509d1f5cfe +torchcodec==0.11.0 --hash=sha256:5562c28dce8114597116f0884908745c29ce401e4a9d0ae46853cbcd858d4f52 +torchvision @ https://download.pytorch.org/whl/cpu/torchvision-0.26.0%2Bcpu-cp311-cp311-manylinux_2_28_aarch64.whl --hash=sha256:c11e55041f6b84a6c4fb28981b901475aa81c38695ccec6ddfcc54c3fa9fac4f +tqdm==4.70.1 --hash=sha256:c293e525e6fef9c20e8728fd4612df02a0aa31bb5fe91ecd93e123b1b7bffa73 +transformers==5.14.1 --hash=sha256:9db974c4079ede2d1a3ea7ca5a240df33f2cc26fc2b36ba64c5f2a4f43b6e725 +truststore==0.10.5 --hash=sha256:9aaaedaefaf06d8b206278cf8b5012bc897f485a874503501e12d776df78951c +typer==0.29.2 --hash=sha256:f30731f8ca758368b836b3714872e58e395cef8fcaff918c853b1969ed5274e7 +types-protobuf==7.35.1.20260906 --hash=sha256:5155e48569e0dabff303fdf578db96cd31ea9a4a63b18018a4ceac6b0ae17462 +typing-extensions==4.16.0 --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 +typing-inspection==0.4.5 --hash=sha256:bc210535442b39e1bf2805e0c7a17dfd6a4ace760c84c7f7d16245d23a6b1c39 +urllib3==2.8.0 --hash=sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3 +uvicorn==0.54.0 --hash=sha256:505bdb0f318731d45f1f712071fc781a8981f6847a31c902c9f5e652d4f67faf +uvloop==0.23.0 --hash=sha256:8fcd721113260ffb5e38bf14a8725b17d431f34209f7d1c7005b667946e630b3 +watchfiles==1.3.0 --hash=sha256:4af9c464d410e8c44b58ddf1cbec7c3a02c1cb2c6c80df9c74659ba2fbda0ab5 +wcwidth==0.9.4 --hash=sha256:093469e01d881bff3db93ff3f56c364dc87c441ea01f9f41c22de0c0bdf5c6ef +websockets==17.4 --hash=sha256:954d43574f72be244b65f0fa61e2f814214b024f2ec849f487aa2f553f7c6e71 +wheel==0.45.1 --hash=sha256:708e7481cc80179af0e556bbf0cc00b8444c7321e2700b8d8580231d13017248 +yarl==1.25.1 --hash=sha256:acae6b45d1ace09b6ba3876da43b88366ef368f73b988c7f57e14231753d4420 +zipp==4.1.2 --hash=sha256:b30cb6cd253122169c47de245ca24a86bac2a5d3802ba15bf7ba6414eb9e144d diff --git a/docs/en/ascend_910b.md b/docs/en/ascend_910b.md new file mode 100644 index 00000000..ec599268 --- /dev/null +++ b/docs/en/ascend_910b.md @@ -0,0 +1,149 @@ +# Run Wan2.2 on Ascend 910B with Docker + +## Build the image + +From the TeleFuser checkout containing `docker/Dockerfile.ascend`, run the following commands in one Bash session on a Linux arm64 Atlas 910B2 host. Check that the assigned cards are idle and that the host driver and firmware match CANN. The container supplies Ubuntu 22.04, Python 3.11, CANN 9.3.0-beta.1, PyTorch 2.11.0+cpu and torch_npu 2.11.0.post4. [Ascend release notes and wheels](https://api.gitcode.com/api/v5/repos/Ascend/pytorch/releases/tags/v26.3.0-beta.1-pytorch2.11.0) + +```bash +uname -m +npu-smi info +docker version +TF_IMAGE=telefuser:ascend910b-local +docker build --platform linux/arm64 -f docker/Dockerfile.ascend \ + --build-arg "SOURCE_REVISION=$(git rev-parse HEAD)" \ + --build-arg "TELEFUSER_VERSION=0.0.0+g$(git rev-parse --short HEAD).local" \ + -t "$TF_IMAGE" . +``` + +The build defaults to AscendHub in China. For Docker Hub add `--build-arg CANN_REPOSITORY=docker.io/ascendai/cann`. Both registries serve the same arm64 manifest. + +After a qualified image is published, you can install it directly using the TeleFuser digest from its release page. The intended release tag is `ghcr.io/tele-ai/telefuser:-ascend910b`. + +```bash +read -r -p 'TeleFuser image digest from release page > ' TF_IMAGE +docker pull "$TF_IMAGE" +``` + +## Prepare checkpoints and outputs + +Allow about 34 GB for the model in addition to the image. Model-level CPU offload and host-staged worker queues also use host memory. + +```bash +TF_MODEL_DIR="$PWD/models/Wan2.2-TI2V-5B" +TF_OUTPUT_DIR=$(mktemp -d "$PWD/telefuser-results.XXXXXX") +mkdir -p "$TF_MODEL_DIR" "$TF_OUTPUT_DIR"/{1,2,4}card +``` + +Download the pinned Wan weights once, or set `TF_MODEL_DIR` to the absolute path of an existing complete checkpoint. + +```bash +docker run --rm --entrypoint hf \ + --user "$(id -u):$(id -g)" -e HF_HOME=/tmp/hf \ + --mount "type=bind,src=$TF_MODEL_DIR,dst=/download" \ + "$TF_IMAGE" download Wan-AI/Wan2.2-TI2V-5B \ + --revision 921dbaf3f1674a56f47e83fb80a34bac8a8f203e \ + --local-dir /download +``` + +## Check the devices + +The example below maps physical devices starting at 0. Set `TF_DEVICE_IDS` to your assigned physical IDs. For example, use `TF_DEVICE_IDS=(5)` for physical card 5. Workers use container-local indices starting at 0. Confirm each visible device can execute a tensor operation. The weight mount is read-only. + +```bash +TF_DEVICE_IDS=(0 1 2 3) + +run_910b() { + local count=$1 i + shift + case "$count" in + 1|2|4) ;; + *) echo 'Choose 1, 2 or 4 cards' >&2; return 2 ;; + esac + (( count <= ${#TF_DEVICE_IDS[@]} )) || { echo "Assigned devices: ${TF_DEVICE_IDS[*]}" >&2; return 2; } + local -a devices=() ids=() + for ((i=0; i-ascend910b`,按 digest 拉取可复现同一批验收结果。 + +```bash +read -r -p '粘贴发布页的 TeleFuser 镜像完整引用 > ' TF_IMAGE +docker pull "$TF_IMAGE" +``` + +```bash +TF_MODEL_DIR="$PWD/models/Wan2.2-TI2V-5B" +TF_OUTPUT_DIR=$(mktemp -d "$PWD/telefuser-results.XXXXXX") +mkdir -p "$TF_MODEL_DIR" "$TF_OUTPUT_DIR"/{1,2,4}card +``` + +已有完整权重时,将 `TF_MODEL_DIR` 设为已有目录的绝对路径。首次下载可用镜像内的 `hf` 命令,下载这一步无需挂卡。 + +```bash +docker run --rm --entrypoint hf \ + --user "$(id -u):$(id -g)" -e HF_HOME=/tmp/hf \ + --mount "type=bind,src=$TF_MODEL_DIR,dst=/download" \ + "$TF_IMAGE" download Wan-AI/Wan2.2-TI2V-5B \ + --revision 921dbaf3f1674a56f47e83fb80a34bac8a8f203e \ + --local-dir /download +``` + +`--model_root` 指向同时含有三个 DiT `.safetensors` 分片、`Wan2.2_VAE.pth`、`models_t5_umt5-xxl-enc-bf16.pth` 和 `google/umt5-xxl/` tokenizer 的目录。权重就绪后,后续步骤使用本地文件。[固定版本的官方模型目录](https://huggingface.co/Wan-AI/Wan2.2-TI2V-5B/tree/921dbaf3f1674a56f47e83fb80a34bac8a8f203e) + +## 挂载设备并自检 + +以下步骤共用同一个 Bash 会话。下面的 `run_910b` 是当前 shell 的辅助函数。单卡传入 `davinci0`,双卡传入 `davinci0–1`,四卡传入 `davinci0–3`。把 `TF_DEVICE_IDS` 改成宿主分配的物理卡号,例如单用物理 5 号卡时设为 `TF_DEVICE_IDS=(5)`。示例在容器内使用从 0 起的逻辑编号。`--shm-size=64g` 是此多卡配方的共享内存上限,供 worker 经主存交换张量。 + +```bash +TF_DEVICE_IDS=(0 1 2 3) + +run_910b() { + local count=$1 i + shift + case "$count" in + 1|2|4) ;; + *) echo '卡数取 1、2 或 4' >&2; return 2 ;; + esac + (( count <= ${#TF_DEVICE_IDS[@]} )) || { echo "Assigned devices: ${TF_DEVICE_IDS[*]}" >&2; return 2; } + local -a devices=() ids=() + for ((i=0; i=0.21.0", "tqdm>=4.66.0", "transformers==5.14.1", - "triton", "uvicorn>=0.30,<1.0", - "torchao==0.17.0", - "bitsandbytes==0.48.0", ] [project.optional-dependencies] +cuda = [ + "triton", + "torchao==0.17.0", + "bitsandbytes==0.48.0", +] + ui = [ "gradio==5.50", ] From 515c39e1b9b713537ce43201b87cd5427d30431f Mon Sep 17 00:00:00 2001 From: jinyx5 Date: Fri, 20 Nov 2026 17:33:38 +0800 Subject: [PATCH 14/21] fix(docker): isolate TeleFuser dependency validation CANN exports development tools through PYTHONPATH, including an optional profiler with absent dependencies and a separate NumPy constraint. Resolve and check the application venv with Python isolated mode while preserving the CANN entrypoint and its runtime library paths. Keep source labels after the locked dependency layer for cached application rebuilds. --- docker/Dockerfile.ascend | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/docker/Dockerfile.ascend b/docker/Dockerfile.ascend index bf0dff21..a871a3c5 100644 --- a/docker/Dockerfile.ascend +++ b/docker/Dockerfile.ascend @@ -1,12 +1,6 @@ ARG CANN_REPOSITORY=swr.cn-south-1.myhuaweicloud.com/ascendhub/cann FROM ${CANN_REPOSITORY}:9.3.0-beta.1-910b-ubuntu22.04-py3.11@sha256:cf6ab60eefb56b8d0e3ecad6d03c30c457b4dff86bcbf5dd7ff1a90427348b7b -ARG TELEFUSER_VERSION=0.0.0+local -ARG SOURCE_REVISION=local -LABEL org.opencontainers.image.source="https://github.com/Tele-AI/TeleFuser" \ - org.opencontainers.image.revision="${SOURCE_REVISION}" \ - org.opencontainers.image.version="${TELEFUSER_VERSION}" - USER root RUN test "$(uname -m)" = aarch64 && apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends ffmpeg \ @@ -14,16 +8,24 @@ RUN test "$(uname -m)" = aarch64 && apt-get update \ RUN python3.11 -m venv /opt/telefuser/venv ENV PATH="/opt/telefuser/venv/bin:${PATH}" COPY docker/ascend/requirements.lock /opt/telefuser/requirements.lock -RUN python -m pip install --no-cache-dir --only-binary=:all: --require-hashes \ +# The upstream PYTHONPATH also includes optional CANN development and profiling tools. +# Resolve and validate the TeleFuser venv in isolation; retain the upstream runtime paths at launch. +RUN python -I -m pip install --no-cache-dir --only-binary=:all: --require-hashes \ --index-url https://pypi.org/simple -r /opt/telefuser/requirements.lock +ARG TELEFUSER_VERSION=0.0.0+local +ARG SOURCE_REVISION=local +LABEL org.opencontainers.image.source="https://github.com/Tele-AI/TeleFuser" \ + org.opencontainers.image.revision="${SOURCE_REVISION}" \ + org.opencontainers.image.version="${TELEFUSER_VERSION}" + WORKDIR /opt/telefuser COPY pyproject.toml README.md LICENSE /tmp/telefuser-src/ COPY telefuser /tmp/telefuser-src/telefuser RUN SETUPTOOLS_SCM_PRETEND_VERSION_FOR_TELEFUSER="${TELEFUSER_VERSION}" \ - python -m pip wheel --no-build-isolation --no-deps -w /tmp/telefuser-dist /tmp/telefuser-src \ - && python -m pip install --no-deps /tmp/telefuser-dist/telefuser-*.whl \ - && python -m pip check && rm -rf /tmp/telefuser-src /tmp/telefuser-dist \ + python -I -m pip wheel --no-build-isolation --no-deps -w /tmp/telefuser-dist /tmp/telefuser-src \ + && python -I -m pip install --no-deps /tmp/telefuser-dist/telefuser-*.whl \ + && python -I -m pip check && rm -rf /tmp/telefuser-src /tmp/telefuser-dist \ && ffprobe -version > /dev/null && hf --version COPY examples/wan_video/wan22_t2v_5b.py ./examples/wan_video/wan22_t2v_5b.py # Inherit the pinned CANN image's entrypoint to initialize the toolkit, AscendNPU IR and ATB. From 7428f04f476dc544b04029350cf12dddb5b74f6e Mon Sep 17 00:00:00 2001 From: jinyx5 Date: Fri, 20 Nov 2026 17:42:37 +0800 Subject: [PATCH 15/21] build(ascend): reuse verified base and wheel archives on restricted hosts --- docker/Dockerfile.ascend | 23 +++++++++++++++++------ docker/Dockerfile.ascend.dockerignore | 2 ++ docker/ascend/README.md | 10 ++++++++++ 3 files changed, 29 insertions(+), 6 deletions(-) diff --git a/docker/Dockerfile.ascend b/docker/Dockerfile.ascend index a871a3c5..2f22c538 100644 --- a/docker/Dockerfile.ascend +++ b/docker/Dockerfile.ascend @@ -1,18 +1,29 @@ ARG CANN_REPOSITORY=swr.cn-south-1.myhuaweicloud.com/ascendhub/cann -FROM ${CANN_REPOSITORY}:9.3.0-beta.1-910b-ubuntu22.04-py3.11@sha256:cf6ab60eefb56b8d0e3ecad6d03c30c457b4dff86bcbf5dd7ff1a90427348b7b +ARG CANN_IMAGE=${CANN_REPOSITORY}:9.3.0-beta.1-910b-ubuntu22.04-py3.11@sha256:cf6ab60eefb56b8d0e3ecad6d03c30c457b4dff86bcbf5dd7ff1a90427348b7b +FROM ${CANN_IMAGE} AS system USER root RUN test "$(uname -m)" = aarch64 && apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends ffmpeg \ && rm -rf /var/lib/apt/lists/* -RUN python3.11 -m venv /opt/telefuser/venv ENV PATH="/opt/telefuser/venv/bin:${PATH}" -COPY docker/ascend/requirements.lock /opt/telefuser/requirements.lock + +FROM system AS python-runtime +RUN python3.11 -m venv /opt/telefuser/venv +COPY docker/ascend/ /tmp/ascend/ # The upstream PYTHONPATH also includes optional CANN development and profiling tools. -# Resolve and validate the TeleFuser venv in isolation; retain the upstream runtime paths at launch. -RUN python -I -m pip install --no-cache-dir --only-binary=:all: --require-hashes \ - --index-url https://pypi.org/simple -r /opt/telefuser/requirements.lock +# Keep hashes in both modes, including the four wheels with explicit publisher URLs. +RUN if [ -d /tmp/ascend/wheelhouse ]; then \ + sed -E 's# @ https://[^ ]*/([^ ]+\.whl) # @ file:///tmp/ascend/wheelhouse/\1 #' \ + /tmp/ascend/requirements.lock > /tmp/offline.lock && \ + set -- --no-index --find-links=/tmp/ascend/wheelhouse -r /tmp/offline.lock; \ + else \ + set -- --index-url https://pypi.org/simple -r /tmp/ascend/requirements.lock; \ + fi && python -I -m pip install --no-cache-dir --only-binary=:all: --require-hashes "$@" +FROM system +COPY --from=python-runtime /opt/telefuser/venv /opt/telefuser/venv +COPY docker/ascend/requirements.lock /opt/telefuser/requirements.lock ARG TELEFUSER_VERSION=0.0.0+local ARG SOURCE_REVISION=local LABEL org.opencontainers.image.source="https://github.com/Tele-AI/TeleFuser" \ diff --git a/docker/Dockerfile.ascend.dockerignore b/docker/Dockerfile.ascend.dockerignore index cc01d391..6b8f12d7 100644 --- a/docker/Dockerfile.ascend.dockerignore +++ b/docker/Dockerfile.ascend.dockerignore @@ -10,6 +10,8 @@ telefuser/_version.py !docker/ !docker/ascend/ !docker/ascend/requirements.lock +!docker/ascend/wheelhouse/ +!docker/ascend/wheelhouse/*.whl !examples/ !examples/wan_video/ !examples/wan_video/wan22_t2v_5b.py diff --git a/docker/ascend/README.md b/docker/ascend/README.md index 7054255d..ad4aaaf8 100644 --- a/docker/ascend/README.md +++ b/docker/ascend/README.md @@ -4,6 +4,16 @@ The 910B image targets Linux arm64 and CPython 3.11. The Dockerfile pins the arc The default registry is AscendHub in China. For Docker Hub, pass `--build-arg CANN_REPOSITORY=docker.io/ascendai/cann`. Both registries serve the same pinned arm64 manifest. [Ascend image repositories](https://github.com/Ascend/cann-container-image/blob/main/OVERVIEW.zh.md) +If the upstream base was transferred by `docker save` / `docker load`, Docker may retain only its local tag. Verify the loaded arm64 image ID against `sha256:92ceaadada7011d9179f5854b08fc66981d3d604cfd72445bcbae756c777dee1`, the config referenced by the pinned manifest. Pass that verified local tag via `--build-arg CANN_IMAGE=` to reuse the cached layers. + +For a build host with limited access to Python package servers, prepare the wheels on a connected Linux arm64 machine with Python 3.11. Transfer the directory to the same path in the build checkout. The Dockerfile detects it, installs from disk, checks every hash in the release lock and leaves the wheel archives in the build stage. The system stage still needs Ubuntu packages for FFmpeg. + +```bash +python3.11 -m pip download --only-binary=:all: --require-hashes \ + --index-url https://pypi.org/simple -r docker/ascend/requirements.lock \ + --dest docker/ascend/wheelhouse +``` + ## Update the dependency lock Run from the repository root in a clean CPython 3.11 environment on Ubuntu 22.04 arm64. Match each of the four direct wheel URLs and hashes in `requirements.in` to their publisher before resolving. The pip report includes the source checkout as a separate local distribution. From c27fabdcdf6f1156126e1bc22699f1af4580c97e Mon Sep 17 00:00:00 2001 From: jinyx5 Date: Fri, 20 Nov 2026 17:51:45 +0800 Subject: [PATCH 16/21] fix(docker): pin the arm64 CPU TorchCodec wheel and load its decoder --- docker/Dockerfile.ascend | 5 +++-- docker/ascend/README.md | 2 +- docker/ascend/requirements.in | 2 +- docker/ascend/requirements.lock | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/docker/Dockerfile.ascend b/docker/Dockerfile.ascend index 2f22c538..86ee0060 100644 --- a/docker/Dockerfile.ascend +++ b/docker/Dockerfile.ascend @@ -12,7 +12,7 @@ FROM system AS python-runtime RUN python3.11 -m venv /opt/telefuser/venv COPY docker/ascend/ /tmp/ascend/ # The upstream PYTHONPATH also includes optional CANN development and profiling tools. -# Keep hashes in both modes, including the four wheels with explicit publisher URLs. +# Keep hashes in both modes, including wheels with explicit publisher URLs. RUN if [ -d /tmp/ascend/wheelhouse ]; then \ sed -E 's# @ https://[^ ]*/([^ ]+\.whl) # @ file:///tmp/ascend/wheelhouse/\1 #' \ /tmp/ascend/requirements.lock > /tmp/offline.lock && \ @@ -37,7 +37,8 @@ RUN SETUPTOOLS_SCM_PRETEND_VERSION_FOR_TELEFUSER="${TELEFUSER_VERSION}" \ python -I -m pip wheel --no-build-isolation --no-deps -w /tmp/telefuser-dist /tmp/telefuser-src \ && python -I -m pip install --no-deps /tmp/telefuser-dist/telefuser-*.whl \ && python -I -m pip check && rm -rf /tmp/telefuser-src /tmp/telefuser-dist \ - && ffprobe -version > /dev/null && hf --version + && ffprobe -version > /dev/null && hf --version \ + && TORCH_DEVICE_BACKEND_AUTOLOAD=0 python -I -c 'from torchcodec.decoders import VideoDecoder' COPY examples/wan_video/wan22_t2v_5b.py ./examples/wan_video/wan22_t2v_5b.py # Inherit the pinned CANN image's entrypoint to initialize the toolkit, AscendNPU IR and ATB. CMD ["telefuser", "--help"] diff --git a/docker/ascend/README.md b/docker/ascend/README.md index ad4aaaf8..3792ee45 100644 --- a/docker/ascend/README.md +++ b/docker/ascend/README.md @@ -16,7 +16,7 @@ python3.11 -m pip download --only-binary=:all: --require-hashes \ ## Update the dependency lock -Run from the repository root in a clean CPython 3.11 environment on Ubuntu 22.04 arm64. Match each of the four direct wheel URLs and hashes in `requirements.in` to their publisher before resolving. The pip report includes the source checkout as a separate local distribution. +Run from the repository root in a clean CPython 3.11 environment on Ubuntu 22.04 arm64. Match each direct wheel URL and hash in `requirements.in` to its publisher before resolving. For TorchCodec, use the pinned arm64 CPU wheel. Its decoder must load against the system FFmpeg libraries during the image build. The pip report includes the source checkout as a separate local distribution. [TorchCodec CPU releases](https://download.pytorch.org/whl/cpu/torchcodec/) ```bash python -m pip install --dry-run --ignore-installed --only-binary=:all: \ diff --git a/docker/ascend/requirements.in b/docker/ascend/requirements.in index c9723165..575a17d8 100644 --- a/docker/ascend/requirements.in +++ b/docker/ascend/requirements.in @@ -4,7 +4,7 @@ torch @ https://download.pytorch.org/whl/cpu/torch-2.11.0%2Bcpu-cp311-cp311-many torchvision @ https://download.pytorch.org/whl/cpu/torchvision-0.26.0%2Bcpu-cp311-cp311-manylinux_2_28_aarch64.whl#sha256=c11e55041f6b84a6c4fb28981b901475aa81c38695ccec6ddfcc54c3fa9fac4f torchaudio @ https://download.pytorch.org/whl/cpu/torchaudio-2.11.0%2Bcpu-cp311-cp311-manylinux_2_28_aarch64.whl#sha256=7a5ecdd20fb26d8eb1d62135ddc6db0f806b17667c10e82df913fd509d1f5cfe torch-npu @ https://gitcode.com/Ascend/pytorch/releases/download/v26.3.0-beta.1-pytorch2.11.0/torch_npu-2.11.0.post4-cp311-cp311-manylinux_2_28_aarch64.whl#sha256=10b3f58c8b9396b4be4027c35cf0e08ef6223699b9cf186c1d89fd5b75652536 -torchcodec==0.11.0 +torchcodec @ https://download.pytorch.org/whl/cpu/torchcodec-0.13.0%2Bcpu-cp311-cp311-manylinux_2_28_aarch64.whl#sha256=960577cfc46cafef1553dffcc529d19a0fcdb9423d374f504875c16dd5facfe5 # Build the TeleFuser wheel with the same pinned tools in the image. setuptools==80.9.0 diff --git a/docker/ascend/requirements.lock b/docker/ascend/requirements.lock index 4b3c94e9..0197699d 100644 --- a/docker/ascend/requirements.lock +++ b/docker/ascend/requirements.lock @@ -94,7 +94,7 @@ tokenizers==0.22.2 --hash=sha256:2249487018adec45d6e3554c71d46eb39fa8ea67156c640 torch @ https://download.pytorch.org/whl/cpu/torch-2.11.0%2Bcpu-cp311-cp311-manylinux_2_28_aarch64.whl --hash=sha256:46fbb0aa257bb781efbfad648f5b045c0e232573b661f1461593db61342e9096 torch-npu @ https://gitcode.com/Ascend/pytorch/releases/download/v26.3.0-beta.1-pytorch2.11.0/torch_npu-2.11.0.post4-cp311-cp311-manylinux_2_28_aarch64.whl --hash=sha256:10b3f58c8b9396b4be4027c35cf0e08ef6223699b9cf186c1d89fd5b75652536 torchaudio @ https://download.pytorch.org/whl/cpu/torchaudio-2.11.0%2Bcpu-cp311-cp311-manylinux_2_28_aarch64.whl --hash=sha256:7a5ecdd20fb26d8eb1d62135ddc6db0f806b17667c10e82df913fd509d1f5cfe -torchcodec==0.11.0 --hash=sha256:5562c28dce8114597116f0884908745c29ce401e4a9d0ae46853cbcd858d4f52 +torchcodec @ https://download.pytorch.org/whl/cpu/torchcodec-0.13.0%2Bcpu-cp311-cp311-manylinux_2_28_aarch64.whl --hash=sha256:960577cfc46cafef1553dffcc529d19a0fcdb9423d374f504875c16dd5facfe5 torchvision @ https://download.pytorch.org/whl/cpu/torchvision-0.26.0%2Bcpu-cp311-cp311-manylinux_2_28_aarch64.whl --hash=sha256:c11e55041f6b84a6c4fb28981b901475aa81c38695ccec6ddfcc54c3fa9fac4f tqdm==4.70.1 --hash=sha256:c293e525e6fef9c20e8728fd4612df02a0aa31bb5fe91ecd93e123b1b7bffa73 transformers==5.14.1 --hash=sha256:9db974c4079ede2d1a3ea7ca5a240df33f2cc26fc2b36ba64c5f2a4f43b6e725 From 682992b1fa35ebca3041d78bf240ba06bd9709a2 Mon Sep 17 00:00:00 2001 From: jinyx5 Date: Fri, 20 Nov 2026 18:22:58 +0800 Subject: [PATCH 17/21] docs(ascend): record 910B2 hardware results and tested offline installation --- docker/ascend/README.md | 14 +++++++++++++ docs/en/ascend_910b.md | 42 +++++++++++++++++++++++++++++++-------- docs/en/installation.md | 2 +- docs/en/platforms.md | 2 +- docs/zh/ascend_910b.md | 44 ++++++++++++++++++++++++++++++----------- docs/zh/installation.md | 2 +- docs/zh/platforms.md | 2 +- 7 files changed, 85 insertions(+), 23 deletions(-) diff --git a/docker/ascend/README.md b/docker/ascend/README.md index 3792ee45..a90a329d 100644 --- a/docker/ascend/README.md +++ b/docker/ascend/README.md @@ -69,3 +69,17 @@ docker buildx imagetools create --prefer-index=false \ --tag "$TF_REPO:$TF_VERSION-ascend910b" "$TF_REPO@$TF_DIGEST" docker buildx imagetools inspect "$TF_REPO:$TF_VERSION-ascend910b" ``` + +## Transfer a tested candidate without a registry + +Use the tag on the qualification host so the recipient loads the exact image that ran the tests. Transfer both files together. The first build qualified on one and two 910B2 devices used the tag below. + +```bash +set -o pipefail +TF_IMAGE=telefuser:ascend910b-c27fabd +TF_ARCHIVE=telefuser-ascend910b-c27fabd.tar.gz +docker save "$TF_IMAGE" | gzip -1 > "$TF_ARCHIVE" +sha256sum "$TF_ARCHIVE" > "$TF_ARCHIVE.sha256" +``` + +On the target host, verify the archive, load it with `docker load -i` and compare the resulting image ID with the hardware qualification record. [Tested image and run instructions](../../docs/en/ascend_910b.md) diff --git a/docs/en/ascend_910b.md b/docs/en/ascend_910b.md index ec599268..53973be7 100644 --- a/docs/en/ascend_910b.md +++ b/docs/en/ascend_910b.md @@ -1,8 +1,23 @@ # Run Wan2.2 on Ascend 910B with Docker -## Build the image +## Install the image -From the TeleFuser checkout containing `docker/Dockerfile.ascend`, run the following commands in one Bash session on a Linux arm64 Atlas 910B2 host. Check that the assigned cards are idle and that the host driver and firmware match CANN. The container supplies Ubuntu 22.04, Python 3.11, CANN 9.3.0-beta.1, PyTorch 2.11.0+cpu and torch_npu 2.11.0.post4. [Ascend release notes and wheels](https://api.gitcode.com/api/v5/repos/Ascend/pytorch/releases/tags/v26.3.0-beta.1-pytorch2.11.0) +The candidate passed Wan generation on one and two Atlas 910B2 cards. Choose one installation method, then keep the same Bash session for checkpoint setup and generation. + +### Load the offline archive + +To install the tested image on a Linux arm64 host, place the archive and its checksum in one directory, run the commands below and proceed to the checkpoints. Compare the Docker image ID with the hardware record at the end of this page. The tested archive is 6,137,716,301 bytes with SHA-256 `8a85baafb991f8640af6f3dba9f38ac8a76376ca669e7e64363c7eee8e9f2369`. + +```bash +sha256sum -c telefuser-ascend910b-c27fabd.tar.gz.sha256 +docker load -i telefuser-ascend910b-c27fabd.tar.gz +TF_IMAGE=telefuser:ascend910b-c27fabd +docker image inspect --format '{{.Id}}' "$TF_IMAGE" +``` + +### Build from source + +Run the following commands in one Bash session from a checkout containing `docker/Dockerfile.ascend`. Check that the assigned cards are idle and that the host driver and firmware match CANN. The container supplies Ubuntu 22.04, Python 3.11, CANN 9.3.0-beta.1, PyTorch 2.11.0+cpu and torch_npu 2.11.0.post4. [Ascend release notes and wheels](https://api.gitcode.com/api/v5/repos/Ascend/pytorch/releases/tags/v26.3.0-beta.1-pytorch2.11.0) ```bash uname -m @@ -17,6 +32,8 @@ docker build --platform linux/arm64 -f docker/Dockerfile.ascend \ The build defaults to AscendHub in China. For Docker Hub add `--build-arg CANN_REPOSITORY=docker.io/ascendai/cann`. Both registries serve the same arm64 manifest. +### Pull a published release + After a qualified image is published, you can install it directly using the TeleFuser digest from its release page. The intended release tag is `ghcr.io/tele-ai/telefuser:-ascend910b`. ```bash @@ -47,7 +64,7 @@ docker run --rm --entrypoint hf \ ## Check the devices -The example below maps physical devices starting at 0. Set `TF_DEVICE_IDS` to your assigned physical IDs. For example, use `TF_DEVICE_IDS=(5)` for physical card 5. Workers use container-local indices starting at 0. Confirm each visible device can execute a tensor operation. The weight mount is read-only. +The example below maps physical devices starting at 0. Set `TF_DEVICE_IDS` to your assigned physical IDs. For example, use `TF_DEVICE_IDS=(5)` for physical card 5. Docker restricts visibility through the mounted device nodes. CANN numbers this set starting at 0. With only physical card 5 attached, the application uses `npu:0`. Passing host index 5 again in `ASCEND_RT_VISIBLE_DEVICES` causes ACL error 207004. Confirm each visible device can execute a tensor operation. The weight mount is read-only. ```bash TF_DEVICE_IDS=(0 1 2 3) @@ -60,12 +77,10 @@ run_910b() { *) echo 'Choose 1, 2 or 4 cards' >&2; return 2 ;; esac (( count <= ${#TF_DEVICE_IDS[@]} )) || { echo "Assigned devices: ${TF_DEVICE_IDS[*]}" >&2; return 2; } - local -a devices=() ids=() + local -a devices=() for ((i=0; i-ascend910b`,按 digest 拉取可复现同一批验收结果。 +### 从发布仓拉取 + +发布后,可使用发布页给出的完整 TeleFuser 镜像引用。镜像将使用 `ghcr.io/tele-ai/telefuser:-ascend910b`,按 digest 拉取可复现同一批验收结果。 ```bash read -r -p '粘贴发布页的 TeleFuser 镜像完整引用 > ' TF_IMAGE docker pull "$TF_IMAGE" ``` +### 准备权重与输出目录 + ```bash TF_MODEL_DIR="$PWD/models/Wan2.2-TI2V-5B" TF_OUTPUT_DIR=$(mktemp -d "$PWD/telefuser-results.XXXXXX") @@ -58,7 +75,7 @@ docker run --rm --entrypoint hf \ ## 挂载设备并自检 -以下步骤共用同一个 Bash 会话。下面的 `run_910b` 是当前 shell 的辅助函数。单卡传入 `davinci0`,双卡传入 `davinci0–1`,四卡传入 `davinci0–3`。把 `TF_DEVICE_IDS` 改成宿主分配的物理卡号,例如单用物理 5 号卡时设为 `TF_DEVICE_IDS=(5)`。示例在容器内使用从 0 起的逻辑编号。`--shm-size=64g` 是此多卡配方的共享内存上限,供 worker 经主存交换张量。 +以下步骤共用同一个 Bash 会话。下面的 `run_910b` 是当前 shell 的辅助函数。单卡传入 `davinci0`,双卡传入 `davinci0–1`,四卡传入 `davinci0–3`。把 `TF_DEVICE_IDS` 改成宿主分配的物理卡号,例如单用物理 5 号卡时设为 `TF_DEVICE_IDS=(5)`。Docker 通过设备节点限制可见范围,CANN 再从 0 编号。只挂载物理 5 号卡时,应用使用 `npu:0`。此时把宿主编号 5 写入 `ASCEND_RT_VISIBLE_DEVICES` 会触发 `207004`。`--shm-size=64g` 是此多卡配方的共享内存上限,供 worker 经主存交换张量。 ```bash TF_DEVICE_IDS=(0 1 2 3) @@ -71,12 +88,10 @@ run_910b() { *) echo '卡数取 1、2 或 4' >&2; return 2 ;; esac (( count <= ${#TF_DEVICE_IDS[@]} )) || { echo "Assigned devices: ${TF_DEVICE_IDS[*]}" >&2; return 2; } - local -a devices=() ids=() + local -a devices=() for ((i=0; i Date: Fri, 20 Nov 2026 18:39:12 +0800 Subject: [PATCH 18/21] test(ascend): collect mesh and pipeline tests without Triton Remove the stale Triton exclusions for the shared device mesh and pipeline communication suites. Exercise both list and batched Wan2.2 VAE inputs with tiled and spatial parallel dispatch. Validated the focused CPU suite without Triton: 141 passed, 8 skipped. Ruff and formatting checks passed. --- tests/conftest.py | 6 ++---- tests/unit/models/test_wan22_video_vae.py | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 4c56747b..7c1e8bb9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,7 +12,7 @@ # Collection Ignore - Skip modules that require unavailable dependencies # ============================================================================ -# Check if triton is available (required by telefuser.distributed and telefuser.kernel.triton) +# Only skip suites that import Triton-dependent operations. try: import triton # noqa: F401 @@ -21,10 +21,8 @@ HAS_TRITON = False TRITON_REQUIRED_TESTS = [ - # Distributed tests require triton - "unit/distributed/test_device_mesh.py", + # Distributed attention and sharding tests "unit/distributed/test_parallel_shard.py", - "unit/distributed/test_pp_comm.py", "unit/distributed/test_ulysses_comm.py", # Kernel tests require triton "unit/kernel/test_rmsnorm.py", diff --git a/tests/unit/models/test_wan22_video_vae.py b/tests/unit/models/test_wan22_video_vae.py index 3e2b1d7b..9a5ec988 100644 --- a/tests/unit/models/test_wan22_video_vae.py +++ b/tests/unit/models/test_wan22_video_vae.py @@ -2,12 +2,30 @@ from types import SimpleNamespace +import pytest import torch import torch.nn as nn from telefuser.models import wan22_video_vae +@pytest.mark.parametrize("tiled, expected_method", [(False, "2d_split"), (True, "tile_dist")]) +def test_parallel_decode_accepts_batched_latents_and_lists(monkeypatch, tiled, expected_method) -> None: + monkeypatch.setattr(torch.distributed, "is_initialized", lambda: True) + device = torch.device("cpu") + batch = torch.arange(48).reshape(2, 3, 2, 2, 2) + + def decode_parallel(latents, target_device, method): + assert target_device == device + assert method == expected_method + return latents + + vae = SimpleNamespace(parallelism=2, decode_parallel=decode_parallel) + decode = wan22_video_vae.Wan22VideoVAE.decode + assert decode(vae, batch, device, tiled=tiled) is batch + torch.testing.assert_close(decode(vae, list(batch.unbind()), device, tiled=tiled), batch) + + class _RecordingDecoder(nn.Module): def __init__(self) -> None: super().__init__() From 3ebeec8b2dcf93c0f5791a442c0ccaf58094f11e Mon Sep 17 00:00:00 2001 From: jinyx5 Date: Fri, 20 Nov 2026 18:46:30 +0800 Subject: [PATCH 19/21] fix(ops): defer sparse attention Triton imports until use Import the sparse SageAttention kernels only when that backend executes. LongCat and FlashVSR can now load their normal service entry points without the optional CUDA extra. Keep the same Triton functions and arguments for sparse attention. Skip the MiniMax Ulysses CUDA kernel tests when Triton itself is absent. Other missing dependencies still fail collection. Verified the CPU service parity tests and both model imports without Triton. --- telefuser/ops/attention/sparse_sage.py | 6 +++--- .../kernel/test_minimax_h3_ulysses_kernels.py | 16 +++++++++++----- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/telefuser/ops/attention/sparse_sage.py b/telefuser/ops/attention/sparse_sage.py index 8332ce77..bc784183 100644 --- a/telefuser/ops/attention/sparse_sage.py +++ b/telefuser/ops/attention/sparse_sage.py @@ -18,9 +18,6 @@ import torch -from telefuser.kernel.triton.quant_per_block import per_block_int8 -from telefuser.kernel.triton.sparse_int8_attn import forward as sparse_sageattn_fwd - def sparse_sageattn( q: torch.Tensor, @@ -43,6 +40,9 @@ def sparse_sageattn( Returns: Output tensor with same shape and layout as input. """ + from telefuser.kernel.triton.quant_per_block import per_block_int8 + from telefuser.kernel.triton.sparse_int8_attn import forward as sparse_sageattn_fwd + seq_dim = 2 if tensor_layout == "HND" else 1 if mask_id is None: diff --git a/tests/unit/kernel/test_minimax_h3_ulysses_kernels.py b/tests/unit/kernel/test_minimax_h3_ulysses_kernels.py index 6046350d..5650206b 100644 --- a/tests/unit/kernel/test_minimax_h3_ulysses_kernels.py +++ b/tests/unit/kernel/test_minimax_h3_ulysses_kernels.py @@ -1,13 +1,19 @@ import pytest import torch -from telefuser.kernel.triton.ulysses_relayout import ( - merge_ulysses_head_chunk, - pack_qkv_destination_major, - pack_qkv_qknorm_rope_destination_major, -) from telefuser.ops.rotary import apply_qk_norm_rope_neox +try: + from telefuser.kernel.triton.ulysses_relayout import ( + merge_ulysses_head_chunk, + pack_qkv_destination_major, + pack_qkv_qknorm_rope_destination_major, + ) +except ModuleNotFoundError as error: + if error.name != "triton": + raise + pytest.skip("Ulysses CUDA kernels require Triton", allow_module_level=True) + pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA Triton kernels") From aa45a1334e99be0f9090bc1700ba0351cba86def Mon Sep 17 00:00:00 2001 From: jinyx5 Date: Fri, 20 Nov 2026 18:48:31 +0800 Subject: [PATCH 20/21] docs(ascend): clarify the four-card qualification status --- docs/zh/ascend_910b.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/ascend_910b.md b/docs/zh/ascend_910b.md index 26a7e2f3..3d23c8c8 100644 --- a/docs/zh/ascend_910b.md +++ b/docs/zh/ascend_910b.md @@ -8,7 +8,7 @@ | --- | --- | | 宿主机 | Linux arm64,Atlas 910B2 64 GB,已安装该 CANN 版本配套的昇腾驱动、固件及 Docker。`npu-smi info` 能看到分配给作业的空闲卡。 | | 容器 | Ubuntu 22.04、Python 3.11、CANN 9.3.0-beta.1、Torch 2.11.0+cpu、torch_npu 2.11.0.post4。候选镜像支持本地构建。 | -| 工作负载 | `Wan-AI/Wan2.2-TI2V-5B` 的离线 T2V,BF16、PyTorch SDPA、UniPC、模型级 CPU offload。首轮验收覆盖单机 1、2、4 卡。 | +| 工作负载 | `Wan-AI/Wan2.2-TI2V-5B` 的离线 T2V,BF16、PyTorch SDPA、UniPC、模型级 CPU offload。单机 1 卡和 2 卡已通过,4 卡待验收。 | | 本地文件 | 官方模型目录约 34 GB,另预留镜像和输出空间。主存需容纳 offload 权重与 worker 队列。模型目录以只读方式传入,输出目录应对容器运行用户可写。 | 以下安装方式任选其一。准备权重、设备自检与生成使用同一个 **Bash** 会话。设备映射示例假定宿主分配的物理设备从 0 开始。设备节点、驱动库和 `npu-smi` 的宿主路径参照昇腾标准安装。[昇腾软件与固件下载](https://www.hiascend.com/developer/download) · [上游容器挂载示例](https://docs.vllm.ai/projects/vllm-omni/en/latest/getting_started/installation/npu/) From a9a3495fa9aa9ae4b29fba610247c2f07a7db9a6 Mon Sep 17 00:00:00 2001 From: jinyx5 Date: Fri, 20 Nov 2026 19:25:14 +0800 Subject: [PATCH 21/21] build(ascend): ship a prebuilt TeleFuser application image Export the locked Python environment and Wan example as an arm64 application image. Assemble it with the official CANN image and Ubuntu FFmpeg on the user host. Document the public GitHub Release download, checksum verification and Docker commands in both runbooks. Verified the original source-build path, the application archive and no-context assembly, the TeleFuser and Wan CLI entry points, a TorchCodec and FFmpeg round trip, both docs builds, and Bash syntax. --- docker/Dockerfile.ascend | 13 ++++++++++++- docker/Dockerfile.ascend.install | 19 +++++++++++++++++++ docker/ascend/README.md | 25 ++++++++----------------- docs/en/ascend_910b.md | 24 +++++++++++++++--------- docs/en/installation.md | 2 +- docs/zh/ascend_910b.md | 26 ++++++++++++++++---------- docs/zh/installation.md | 2 +- 7 files changed, 72 insertions(+), 39 deletions(-) create mode 100644 docker/Dockerfile.ascend.install diff --git a/docker/Dockerfile.ascend b/docker/Dockerfile.ascend index 86ee0060..c63d2bba 100644 --- a/docker/Dockerfile.ascend +++ b/docker/Dockerfile.ascend @@ -21,7 +21,7 @@ RUN if [ -d /tmp/ascend/wheelhouse ]; then \ set -- --index-url https://pypi.org/simple -r /tmp/ascend/requirements.lock; \ fi && python -I -m pip install --no-cache-dir --only-binary=:all: --require-hashes "$@" -FROM system +FROM system AS installed COPY --from=python-runtime /opt/telefuser/venv /opt/telefuser/venv COPY docker/ascend/requirements.lock /opt/telefuser/requirements.lock ARG TELEFUSER_VERSION=0.0.0+local @@ -40,5 +40,16 @@ RUN SETUPTOOLS_SCM_PRETEND_VERSION_FOR_TELEFUSER="${TELEFUSER_VERSION}" \ && ffprobe -version > /dev/null && hf --version \ && TORCH_DEVICE_BACKEND_AUTOLOAD=0 python -I -c 'from torchcodec.decoders import VideoDecoder' COPY examples/wan_video/wan22_t2v_5b.py ./examples/wan_video/wan22_t2v_5b.py + +# Publish the application files separately from the vendor runtime. +FROM scratch AS runtime-payload +ARG SOURCE_REVISION=local +ARG TELEFUSER_VERSION=0.0.0+local +LABEL org.opencontainers.image.source="https://github.com/Tele-AI/TeleFuser" \ + org.opencontainers.image.revision="${SOURCE_REVISION}" \ + org.opencontainers.image.version="${TELEFUSER_VERSION}" +COPY --from=installed /opt/telefuser/ /opt/telefuser/ + +FROM installed # Inherit the pinned CANN image's entrypoint to initialize the toolkit, AscendNPU IR and ATB. CMD ["telefuser", "--help"] diff --git a/docker/Dockerfile.ascend.install b/docker/Dockerfile.ascend.install new file mode 100644 index 00000000..42ea33d8 --- /dev/null +++ b/docker/Dockerfile.ascend.install @@ -0,0 +1,19 @@ +ARG TELEFUSER_PAYLOAD=telefuser:ascend910b-c27fabd-payload +ARG CANN_IMAGE=quay.io/ascend/cann:9.3.0-beta.1-910b-ubuntu22.04-py3.11@sha256:cf6ab60eefb56b8d0e3ecad6d03c30c457b4dff86bcbf5dd7ff1a90427348b7b +FROM ${TELEFUSER_PAYLOAD} AS payload +FROM ${CANN_IMAGE} + +USER root +RUN test "$(uname -m)" = aarch64 && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends ffmpeg \ + && rm -rf /var/lib/apt/lists/* +ENV PATH="/opt/telefuser/venv/bin:${PATH}" + +ARG SOURCE_REVISION=c27fabdcdf6f1156126e1bc22699f1af4580c97e +ARG TELEFUSER_VERSION=0.0.0+gc27fabd.local +LABEL org.opencontainers.image.source="https://github.com/Tele-AI/TeleFuser" \ + org.opencontainers.image.revision="${SOURCE_REVISION}" \ + org.opencontainers.image.version="${TELEFUSER_VERSION}" +WORKDIR /opt/telefuser +COPY --from=payload /opt/telefuser/ /opt/telefuser/ +CMD ["telefuser", "--help"] diff --git a/docker/ascend/README.md b/docker/ascend/README.md index a90a329d..cc3eb62e 100644 --- a/docker/ascend/README.md +++ b/docker/ascend/README.md @@ -2,7 +2,7 @@ The 910B image targets Linux arm64 and CPython 3.11. The Dockerfile pins the architecture-specific CANN manifest. The runtime lock combines the base dependencies from `pyproject.toml` and the stack from `requirements.in`. Every entry names one compatible wheel and its SHA-256. The Ascend release provides the wheel for torch_npu 2.11.0.post4. [Ascend release artifacts](https://api.gitcode.com/api/v5/repos/Ascend/pytorch/releases/tags/v26.3.0-beta.1-pytorch2.11.0) -The default registry is AscendHub in China. For Docker Hub, pass `--build-arg CANN_REPOSITORY=docker.io/ascendai/cann`. Both registries serve the same pinned arm64 manifest. [Ascend image repositories](https://github.com/Ascend/cann-container-image/blob/main/OVERVIEW.zh.md) +The default registry is AscendHub in China. For public pulls use `--build-arg CANN_REPOSITORY=quay.io/ascend/cann` or `--build-arg CANN_REPOSITORY=docker.io/ascendai/cann`. All three registries serve the same pinned arm64 manifest. [Ascend image repositories](https://github.com/Ascend/cann-container-image/blob/main/OVERVIEW.zh.md) If the upstream base was transferred by `docker save` / `docker load`, Docker may retain only its local tag. Verify the loaded arm64 image ID against `sha256:92ceaadada7011d9179f5854b08fc66981d3d604cfd72445bcbae756c777dee1`, the config referenced by the pinned manifest. Pass that verified local tag via `--build-arg CANN_IMAGE=` to reuse the cached layers. @@ -46,31 +46,22 @@ header = "# Linux arm64, Ubuntu 22.04, CPython 3.11. One wheel hash per distribu PY ``` -## Build, qualify and publish +## Publish the TeleFuser application image -Build from a clean release checkout. Replace the version below with its PEP 440 version. Tag the candidate with the commit. Record the digest printed by `docker push`. Run the hardware checks on that digest, including separate 1-, 2- and 4-card Wan runs, before promoting a release. [Qualification procedure](../../docs/en/ascend_910b.md) +Build the `runtime-payload` target from a clean release checkout. This target packages the Python environment and Wan example in a scratch image. The runnable image uses CANN layers pulled from the official Ascend registry. Record the source revision, checksums, tested driver and firmware, and validated card counts in the release. [CANN software license](https://www.hiascend.com/en/legal/cannua-download?isNewCon=true) · [Qualification procedure](../../docs/en/ascend_910b.md) ```bash read -r -p 'TeleFuser package version > ' TF_VERSION TF_REV=$(git rev-parse HEAD) -TF_REPO=ghcr.io/tele-ai/telefuser -TF_CANDIDATE="$TF_REPO:sha-$TF_REV-ascend910b" -docker build --platform linux/arm64 -f docker/Dockerfile.ascend \ +TF_PAYLOAD="telefuser:ascend910b-${TF_REV:0:7}-payload" +docker build --platform linux/arm64 --target runtime-payload -f docker/Dockerfile.ascend \ --build-arg "TELEFUSER_VERSION=$TF_VERSION" --build-arg "SOURCE_REVISION=$TF_REV" \ - -t "$TF_CANDIDATE" . -docker push "$TF_CANDIDATE" + -t "$TF_PAYLOAD" . ``` -After qualification, point the public release tag at the tested manifest. Confirm anonymous access and publish the digest, driver and firmware versions and validated card counts alongside the release. +Assemble the runnable image with `Dockerfile.ascend.install`. Pass `--build-arg TELEFUSER_PAYLOAD="$TF_PAYLOAD"` and the same version and revision arguments. Run the hardware checks on that image. For a GitHub Release, upload the compressed `docker save` archive, `Dockerfile.ascend.install` and their SHA-256 checksums. Verify anonymous download and repeat the documented install from those assets. -```bash -read -r -p 'Tested TeleFuser sha256 digest > ' TF_DIGEST -docker buildx imagetools create --prefer-index=false \ - --tag "$TF_REPO:$TF_VERSION-ascend910b" "$TF_REPO@$TF_DIGEST" -docker buildx imagetools inspect "$TF_REPO:$TF_VERSION-ascend910b" -``` - -## Transfer a tested candidate without a registry +## Save a complete image for an existing installation Use the tag on the qualification host so the recipient loads the exact image that ran the tests. Transfer both files together. The first build qualified on one and two 910B2 devices used the tag below. diff --git a/docs/en/ascend_910b.md b/docs/en/ascend_910b.md index 53973be7..6e71c6da 100644 --- a/docs/en/ascend_910b.md +++ b/docs/en/ascend_910b.md @@ -4,6 +4,21 @@ The candidate passed Wan generation on one and two Atlas 910B2 cards. Choose one installation method, then keep the same Bash session for checkpoint setup and generation. +### Install the public Docker package + +On a Linux arm64 host, download the prebuilt TeleFuser application image from the public candidate release. Docker loads the frozen Python environment and assembles the runnable image with the official CANN base and Ubuntu FFmpeg packages. The commands require access to GitHub Releases, Quay and the Ubuntu package mirrors. [Public Docker package](https://github.com/yx0716/TeleFuser-upstream/releases/tag/ascend910b-c27fabd) + +```bash +TF_RELEASE=https://github.com/yx0716/TeleFuser-upstream/releases/download/ascend910b-c27fabd +TF_IMAGE=telefuser:ascend910b-c27fabd-public +curl -fsSLO "$TF_RELEASE/telefuser-ascend910b-c27fabd-runtime.tar.gz" +curl -fsSLO "$TF_RELEASE/Dockerfile.ascend.install" +curl -fsSLO "$TF_RELEASE/SHA256SUMS" +sha256sum -c SHA256SUMS && + docker load -i telefuser-ascend910b-c27fabd-runtime.tar.gz && + docker build --platform linux/arm64 -t "$TF_IMAGE" - < Dockerfile.ascend.install +``` + ### Load the offline archive To install the tested image on a Linux arm64 host, place the archive and its checksum in one directory, run the commands below and proceed to the checkpoints. Compare the Docker image ID with the hardware record at the end of this page. The tested archive is 6,137,716,301 bytes with SHA-256 `8a85baafb991f8640af6f3dba9f38ac8a76376ca669e7e64363c7eee8e9f2369`. @@ -32,15 +47,6 @@ docker build --platform linux/arm64 -f docker/Dockerfile.ascend \ The build defaults to AscendHub in China. For Docker Hub add `--build-arg CANN_REPOSITORY=docker.io/ascendai/cann`. Both registries serve the same arm64 manifest. -### Pull a published release - -After a qualified image is published, you can install it directly using the TeleFuser digest from its release page. The intended release tag is `ghcr.io/tele-ai/telefuser:-ascend910b`. - -```bash -read -r -p 'TeleFuser image digest from release page > ' TF_IMAGE -docker pull "$TF_IMAGE" -``` - ## Prepare checkpoints and outputs Allow about 34 GB for the model in addition to the image. Model-level CPU offload and host-staged worker queues also use host memory. diff --git a/docs/en/installation.md b/docs/en/installation.md index 062cca53..82760724 100644 --- a/docs/en/installation.md +++ b/docs/en/installation.md @@ -65,7 +65,7 @@ ROCm before CUDA. Examples ending in `_rocm.py` (for example `examples/wan_video are the validated entry points; see [Hardware Platforms](platforms.md) for per-platform capabilities and backend availability. -For Atlas 910B2, the image includes CANN 9.3.0-beta.1, PyTorch 2.11.0+cpu, torch_npu 2.11.0.post4 and the Wan2.2 5B example. The runbook covers offline image loading, source builds, digest-based pulls after release and device checks. One- and two-card generation passed on hardware. [Ascend 910B image and runbook](ascend_910b.md) +For Atlas 910B2, the public Docker package supplies PyTorch 2.11.0+cpu, torch_npu 2.11.0.post4 and the Wan2.2 5B example. Docker assembles it with the official CANN 9.3.0-beta.1 base. The runbook covers installation, device checks and generation. One- and two-card generation passed on hardware. [Ascend 910B image and runbook](ascend_910b.md) ## Model Checkpoints diff --git a/docs/zh/ascend_910b.md b/docs/zh/ascend_910b.md index 3d23c8c8..e589e25a 100644 --- a/docs/zh/ascend_910b.md +++ b/docs/zh/ascend_910b.md @@ -1,6 +1,6 @@ # 使用镜像在 Ascend 910B 上运行 Wan2.2 文生视频 -本手册面向 Linux arm64 的 Atlas 910B2 64 GB。镜像将 CANN 9.3.0-beta.1 与昇腾同次发布的 TorchNPU 配套。该候选镜像已通过单卡与双卡 Wan 实机验收。可加载候选镜像归档,或从包含 `docker/Dockerfile.ascend` 的 checkout 构建。四卡仍待验收。[昇腾 CANN](https://www.hiascend.com/cann/download?versionId=822&ids=d806%2Ch0501%2Ch0601%2Ch0703¤tTab=0) · [TorchNPU 发布与附件](https://api.gitcode.com/api/v5/repos/Ascend/pytorch/releases/tags/v26.3.0-beta.1-pytorch2.11.0) +本手册面向 Linux arm64 的 Atlas 910B2 64 GB。镜像将 CANN 9.3.0-beta.1 与昇腾同次发布的 TorchNPU 配套。该候选版本已通过单卡与双卡 Wan 实机验收,四卡仍待验收。[昇腾 CANN](https://www.hiascend.com/cann/download?versionId=822&ids=d806%2Ch0501%2Ch0601%2Ch0703¤tTab=0) · [TorchNPU 发布与附件](https://api.gitcode.com/api/v5/repos/Ascend/pytorch/releases/tags/v26.3.0-beta.1-pytorch2.11.0) ## 运行条件 @@ -15,6 +15,21 @@ ## 安装镜像与准备权重 +### 从公开 Docker 包安装 + +在 Linux arm64 宿主机上下载公开候选版本。Docker 加载预装好的 TeleFuser 和 Python 依赖,从昇腾官方仓拉取固定版本的 CANN 底座,并安装 Ubuntu 的 FFmpeg 包。安装需要访问 GitHub Releases、Quay 和 Ubuntu 软件源。[公开 Docker 包](https://github.com/yx0716/TeleFuser-upstream/releases/tag/ascend910b-c27fabd) + +```bash +TF_RELEASE=https://github.com/yx0716/TeleFuser-upstream/releases/download/ascend910b-c27fabd +TF_IMAGE=telefuser:ascend910b-c27fabd-public +curl -fsSLO "$TF_RELEASE/telefuser-ascend910b-c27fabd-runtime.tar.gz" +curl -fsSLO "$TF_RELEASE/Dockerfile.ascend.install" +curl -fsSLO "$TF_RELEASE/SHA256SUMS" +sha256sum -c SHA256SUMS && + docker load -i telefuser-ascend910b-c27fabd-runtime.tar.gz && + docker build --platform linux/arm64 -t "$TF_IMAGE" - < Dockerfile.ascend.install +``` + ### 从离线归档安装 拿到已验收的镜像归档和同名校验文件时,在目标机的同一个目录执行下列命令,然后直接准备权重。Docker image ID 应与本页末尾的实机记录一致。该归档为 6,137,716,301 字节,SHA-256 为 `8a85baafb991f8640af6f3dba9f38ac8a76376ca669e7e64363c7eee8e9f2369`。 @@ -43,15 +58,6 @@ docker build --platform linux/arm64 -f docker/Dockerfile.ascend \ 默认从昇腾境内仓下载 CANN 底座。使用 Docker Hub 时,在构建命令中加 `--build-arg CANN_REPOSITORY=docker.io/ascendai/cann`,两处清单的 arm64 摘要相同。 -### 从发布仓拉取 - -发布后,可使用发布页给出的完整 TeleFuser 镜像引用。镜像将使用 `ghcr.io/tele-ai/telefuser:-ascend910b`,按 digest 拉取可复现同一批验收结果。 - -```bash -read -r -p '粘贴发布页的 TeleFuser 镜像完整引用 > ' TF_IMAGE -docker pull "$TF_IMAGE" -``` - ### 准备权重与输出目录 ```bash diff --git a/docs/zh/installation.md b/docs/zh/installation.md index 94d0bb80..48df19a9 100644 --- a/docs/zh/installation.md +++ b/docs/zh/installation.md @@ -64,7 +64,7 @@ AMD ROCm 主机应安装 PyTorch `+rocm` 构建,而非 CUDA Toolkit 路径。H `examples/wan_video/wan21_1_3b_text_to_video_rocm.py`)是已验证的入口;各平台能力与后端可用性见 [硬件平台](platforms.md)。 -Atlas 910B2 镜像包含 CANN 9.3.0-beta.1、PyTorch 2.11.0+cpu、torch_npu 2.11.0.post4 和 Wan2.2 5B 示例。运行手册给出离线镜像加载、源码构建、发布后按摘要拉取、设备自检和单机生成命令。1、2 卡已在实机上通过。[910B 镜像安装与运行](ascend_910b.md) +Atlas 910B2 的公开 Docker 包预装 PyTorch 2.11.0+cpu、torch_npu 2.11.0.post4 和 Wan2.2 5B 示例。Docker 将它与官方 CANN 9.3.0-beta.1 底座装配成可运行镜像。运行手册给出安装、设备自检和生成命令。单卡和双卡已在实机上通过。[910B 镜像安装与运行](ascend_910b.md) ## 模型权重