diff --git a/README.md b/README.md index 3c35b0a8..27a97742 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,8 @@ The project treats a world model as more than a function that returns a single c pip install -e . ``` +For NVIDIA Triton and quantization support, install `pip install -e ".[cuda]"`. Ascend 910B2 users can build the pinned container and run the Wan example. [Ascend installation](docs/en/ascend_910b.md) + For development: ```bash diff --git a/README_zh.md b/README_zh.md index 05ad6b08..694aaa72 100644 --- a/README_zh.md +++ b/README_zh.md @@ -79,6 +79,8 @@ TeleFuser 是一个开源的多模态生成与世界模型流式推理和服务 pip install -e . ``` +NVIDIA 的 Triton 与量化功能使用 `pip install -e ".[cuda]"`。Atlas 910B2 可从固定版本镜像运行 Wan 示例。[昇腾镜像安装](docs/zh/ascend_910b.md) + 开发环境安装: ```bash diff --git a/docker/Dockerfile.ascend b/docker/Dockerfile.ascend new file mode 100644 index 00000000..c63d2bba --- /dev/null +++ b/docker/Dockerfile.ascend @@ -0,0 +1,55 @@ +ARG CANN_REPOSITORY=swr.cn-south-1.myhuaweicloud.com/ascendhub/cann +ARG CANN_IMAGE=${CANN_REPOSITORY}:9.3.0-beta.1-910b-ubuntu22.04-py3.11@sha256:cf6ab60eefb56b8d0e3ecad6d03c30c457b4dff86bcbf5dd7ff1a90427348b7b +FROM ${CANN_IMAGE} AS system + +USER root +RUN test "$(uname -m)" = aarch64 && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends ffmpeg \ + && rm -rf /var/lib/apt/lists/* +ENV PATH="/opt/telefuser/venv/bin:${PATH}" + +FROM system AS python-runtime +RUN python3.11 -m venv /opt/telefuser/venv +COPY docker/ascend/ /tmp/ascend/ +# The upstream PYTHONPATH also includes optional CANN development and profiling tools. +# Keep hashes in both modes, including wheels with explicit publisher URLs. +RUN if [ -d /tmp/ascend/wheelhouse ]; then \ + sed -E 's# @ https://[^ ]*/([^ ]+\.whl) # @ file:///tmp/ascend/wheelhouse/\1 #' \ + /tmp/ascend/requirements.lock > /tmp/offline.lock && \ + set -- --no-index --find-links=/tmp/ascend/wheelhouse -r /tmp/offline.lock; \ + else \ + set -- --index-url https://pypi.org/simple -r /tmp/ascend/requirements.lock; \ + fi && python -I -m pip install --no-cache-dir --only-binary=:all: --require-hashes "$@" + +FROM system AS installed +COPY --from=python-runtime /opt/telefuser/venv /opt/telefuser/venv +COPY docker/ascend/requirements.lock /opt/telefuser/requirements.lock +ARG TELEFUSER_VERSION=0.0.0+local +ARG SOURCE_REVISION=local +LABEL org.opencontainers.image.source="https://github.com/Tele-AI/TeleFuser" \ + org.opencontainers.image.revision="${SOURCE_REVISION}" \ + org.opencontainers.image.version="${TELEFUSER_VERSION}" + +WORKDIR /opt/telefuser +COPY pyproject.toml README.md LICENSE /tmp/telefuser-src/ +COPY telefuser /tmp/telefuser-src/telefuser +RUN SETUPTOOLS_SCM_PRETEND_VERSION_FOR_TELEFUSER="${TELEFUSER_VERSION}" \ + python -I -m pip wheel --no-build-isolation --no-deps -w /tmp/telefuser-dist /tmp/telefuser-src \ + && python -I -m pip install --no-deps /tmp/telefuser-dist/telefuser-*.whl \ + && python -I -m pip check && rm -rf /tmp/telefuser-src /tmp/telefuser-dist \ + && ffprobe -version > /dev/null && hf --version \ + && TORCH_DEVICE_BACKEND_AUTOLOAD=0 python -I -c 'from torchcodec.decoders import VideoDecoder' +COPY examples/wan_video/wan22_t2v_5b.py ./examples/wan_video/wan22_t2v_5b.py + +# Publish the application files separately from the vendor runtime. +FROM scratch AS runtime-payload +ARG SOURCE_REVISION=local +ARG TELEFUSER_VERSION=0.0.0+local +LABEL org.opencontainers.image.source="https://github.com/Tele-AI/TeleFuser" \ + org.opencontainers.image.revision="${SOURCE_REVISION}" \ + org.opencontainers.image.version="${TELEFUSER_VERSION}" +COPY --from=installed /opt/telefuser/ /opt/telefuser/ + +FROM installed +# Inherit the pinned CANN image's entrypoint to initialize the toolkit, AscendNPU IR and ATB. +CMD ["telefuser", "--help"] diff --git a/docker/Dockerfile.ascend.dockerignore b/docker/Dockerfile.ascend.dockerignore new file mode 100644 index 00000000..6b8f12d7 --- /dev/null +++ b/docker/Dockerfile.ascend.dockerignore @@ -0,0 +1,17 @@ +** +!pyproject.toml +!README.md +!LICENSE +!telefuser/ +!telefuser/** +**/__pycache__ +**/*.pyc +telefuser/_version.py +!docker/ +!docker/ascend/ +!docker/ascend/requirements.lock +!docker/ascend/wheelhouse/ +!docker/ascend/wheelhouse/*.whl +!examples/ +!examples/wan_video/ +!examples/wan_video/wan22_t2v_5b.py diff --git a/docker/Dockerfile.ascend.install b/docker/Dockerfile.ascend.install new file mode 100644 index 00000000..42ea33d8 --- /dev/null +++ b/docker/Dockerfile.ascend.install @@ -0,0 +1,19 @@ +ARG TELEFUSER_PAYLOAD=telefuser:ascend910b-c27fabd-payload +ARG CANN_IMAGE=quay.io/ascend/cann:9.3.0-beta.1-910b-ubuntu22.04-py3.11@sha256:cf6ab60eefb56b8d0e3ecad6d03c30c457b4dff86bcbf5dd7ff1a90427348b7b +FROM ${TELEFUSER_PAYLOAD} AS payload +FROM ${CANN_IMAGE} + +USER root +RUN test "$(uname -m)" = aarch64 && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends ffmpeg \ + && rm -rf /var/lib/apt/lists/* +ENV PATH="/opt/telefuser/venv/bin:${PATH}" + +ARG SOURCE_REVISION=c27fabdcdf6f1156126e1bc22699f1af4580c97e +ARG TELEFUSER_VERSION=0.0.0+gc27fabd.local +LABEL org.opencontainers.image.source="https://github.com/Tele-AI/TeleFuser" \ + org.opencontainers.image.revision="${SOURCE_REVISION}" \ + org.opencontainers.image.version="${TELEFUSER_VERSION}" +WORKDIR /opt/telefuser +COPY --from=payload /opt/telefuser/ /opt/telefuser/ +CMD ["telefuser", "--help"] diff --git a/docker/ascend/README.md b/docker/ascend/README.md new file mode 100644 index 00000000..cc3eb62e --- /dev/null +++ b/docker/ascend/README.md @@ -0,0 +1,76 @@ +# Ascend image maintenance + +The 910B image targets Linux arm64 and CPython 3.11. The Dockerfile pins the architecture-specific CANN manifest. The runtime lock combines the base dependencies from `pyproject.toml` and the stack from `requirements.in`. Every entry names one compatible wheel and its SHA-256. The Ascend release provides the wheel for torch_npu 2.11.0.post4. [Ascend release artifacts](https://api.gitcode.com/api/v5/repos/Ascend/pytorch/releases/tags/v26.3.0-beta.1-pytorch2.11.0) + +The default registry is AscendHub in China. For public pulls use `--build-arg CANN_REPOSITORY=quay.io/ascend/cann` or `--build-arg CANN_REPOSITORY=docker.io/ascendai/cann`. All three registries serve the same pinned arm64 manifest. [Ascend image repositories](https://github.com/Ascend/cann-container-image/blob/main/OVERVIEW.zh.md) + +If the upstream base was transferred by `docker save` / `docker load`, Docker may retain only its local tag. Verify the loaded arm64 image ID against `sha256:92ceaadada7011d9179f5854b08fc66981d3d604cfd72445bcbae756c777dee1`, the config referenced by the pinned manifest. Pass that verified local tag via `--build-arg CANN_IMAGE=` to reuse the cached layers. + +For a build host with limited access to Python package servers, prepare the wheels on a connected Linux arm64 machine with Python 3.11. Transfer the directory to the same path in the build checkout. The Dockerfile detects it, installs from disk, checks every hash in the release lock and leaves the wheel archives in the build stage. The system stage still needs Ubuntu packages for FFmpeg. + +```bash +python3.11 -m pip download --only-binary=:all: --require-hashes \ + --index-url https://pypi.org/simple -r docker/ascend/requirements.lock \ + --dest docker/ascend/wheelhouse +``` + +## Update the dependency lock + +Run from the repository root in a clean CPython 3.11 environment on Ubuntu 22.04 arm64. Match each direct wheel URL and hash in `requirements.in` to its publisher before resolving. For TorchCodec, use the pinned arm64 CPU wheel. Its decoder must load against the system FFmpeg libraries during the image build. The pip report includes the source checkout as a separate local distribution. [TorchCodec CPU releases](https://download.pytorch.org/whl/cpu/torchcodec/) + +```bash +python -m pip install --dry-run --ignore-installed --only-binary=:all: \ + --index-url https://pypi.org/simple --report /tmp/telefuser-ascend-resolution.json \ + -r docker/ascend/requirements.in . +python - <<'PY' +import json +from pathlib import Path + +from packaging.utils import canonicalize_name + +directory = Path("docker/ascend") +direct = { + line.split(" @ ")[0]: line.split(" @ ")[1].partition("#")[0] + for line in (directory / "requirements.in").read_text().splitlines() if " @ " in line +} +entries = [] +for record in json.loads(Path("/tmp/telefuser-ascend-resolution.json").read_text())["install"]: + name = canonicalize_name(record["metadata"]["name"]) + if name == "telefuser": + continue + digest = record["download_info"]["archive_info"]["hashes"]["sha256"] + requirement = f"{name} @ {direct[name]}" if name in direct else f"{name}=={record['metadata']['version']}" + entries.append((name, f"{requirement} --hash=sha256:{digest}")) +header = "# Linux arm64, Ubuntu 22.04, CPython 3.11. One wheel hash per distribution." +(directory / "requirements.lock").write_text("\n".join([header, *(line for _, line in sorted(entries))]) + "\n") +PY +``` + +## Publish the TeleFuser application image + +Build the `runtime-payload` target from a clean release checkout. This target packages the Python environment and Wan example in a scratch image. The runnable image uses CANN layers pulled from the official Ascend registry. Record the source revision, checksums, tested driver and firmware, and validated card counts in the release. [CANN software license](https://www.hiascend.com/en/legal/cannua-download?isNewCon=true) · [Qualification procedure](../../docs/en/ascend_910b.md) + +```bash +read -r -p 'TeleFuser package version > ' TF_VERSION +TF_REV=$(git rev-parse HEAD) +TF_PAYLOAD="telefuser:ascend910b-${TF_REV:0:7}-payload" +docker build --platform linux/arm64 --target runtime-payload -f docker/Dockerfile.ascend \ + --build-arg "TELEFUSER_VERSION=$TF_VERSION" --build-arg "SOURCE_REVISION=$TF_REV" \ + -t "$TF_PAYLOAD" . +``` + +Assemble the runnable image with `Dockerfile.ascend.install`. Pass `--build-arg TELEFUSER_PAYLOAD="$TF_PAYLOAD"` and the same version and revision arguments. Run the hardware checks on that image. For a GitHub Release, upload the compressed `docker save` archive, `Dockerfile.ascend.install` and their SHA-256 checksums. Verify anonymous download and repeat the documented install from those assets. + +## Save a complete image for an existing installation + +Use the tag on the qualification host so the recipient loads the exact image that ran the tests. Transfer both files together. The first build qualified on one and two 910B2 devices used the tag below. + +```bash +set -o pipefail +TF_IMAGE=telefuser:ascend910b-c27fabd +TF_ARCHIVE=telefuser-ascend910b-c27fabd.tar.gz +docker save "$TF_IMAGE" | gzip -1 > "$TF_ARCHIVE" +sha256sum "$TF_ARCHIVE" > "$TF_ARCHIVE.sha256" +``` + +On the target host, verify the archive, load it with `docker load -i` and compare the resulting image ID with the hardware qualification record. [Tested image and run instructions](../../docs/en/ascend_910b.md) diff --git a/docker/ascend/requirements.in b/docker/ascend/requirements.in new file mode 100644 index 00000000..575a17d8 --- /dev/null +++ b/docker/ascend/requirements.in @@ -0,0 +1,12 @@ +# Ubuntu 22.04 (glibc 2.35), CPython 3.11, Linux arm64. +# Match Ascend Extension for PyTorch v26.3.0-beta.1-pytorch2.11.0. +torch @ https://download.pytorch.org/whl/cpu/torch-2.11.0%2Bcpu-cp311-cp311-manylinux_2_28_aarch64.whl#sha256=46fbb0aa257bb781efbfad648f5b045c0e232573b661f1461593db61342e9096 +torchvision @ https://download.pytorch.org/whl/cpu/torchvision-0.26.0%2Bcpu-cp311-cp311-manylinux_2_28_aarch64.whl#sha256=c11e55041f6b84a6c4fb28981b901475aa81c38695ccec6ddfcc54c3fa9fac4f +torchaudio @ https://download.pytorch.org/whl/cpu/torchaudio-2.11.0%2Bcpu-cp311-cp311-manylinux_2_28_aarch64.whl#sha256=7a5ecdd20fb26d8eb1d62135ddc6db0f806b17667c10e82df913fd509d1f5cfe +torch-npu @ https://gitcode.com/Ascend/pytorch/releases/download/v26.3.0-beta.1-pytorch2.11.0/torch_npu-2.11.0.post4-cp311-cp311-manylinux_2_28_aarch64.whl#sha256=10b3f58c8b9396b4be4027c35cf0e08ef6223699b9cf186c1d89fd5b75652536 +torchcodec @ https://download.pytorch.org/whl/cpu/torchcodec-0.13.0%2Bcpu-cp311-cp311-manylinux_2_28_aarch64.whl#sha256=960577cfc46cafef1553dffcc529d19a0fcdb9423d374f504875c16dd5facfe5 + +# Build the TeleFuser wheel with the same pinned tools in the image. +setuptools==80.9.0 +setuptools-scm==8.3.1 +wheel==0.45.1 diff --git a/docker/ascend/requirements.lock b/docker/ascend/requirements.lock new file mode 100644 index 00000000..0197699d --- /dev/null +++ b/docker/ascend/requirements.lock @@ -0,0 +1,114 @@ +# Linux arm64, Ubuntu 22.04, CPython 3.11. One wheel hash per distribution. +agent-detector==2.0.0 --hash=sha256:22c6a1c9c23894a12f55aea625f060b8d3330fba774de72d02d5442bb580417f +aiofiles==25.1.0 --hash=sha256:abe311e527c862958650f9438e859c1fa7568a141b22abcd015e120e86a85695 +aiohappyeyeballs==2.7.1 --hash=sha256:9243213661e29250eb41368e5daa826fc017156c3b8a11440826b2e3ed376472 +aiohttp==3.14.5 --hash=sha256:32e8fa6644e541fcd7e02430588c7fc93b602c1778ea0bc345505db76b61cfb4 +aiosignal==1.4.0 --hash=sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e +annotated-doc==0.0.7 --hash=sha256:6bc31bbbec6dfaaedc1676dbc11dbda96fe22a2b5c400e9ebad113fec44a9dc6 +annotated-types==0.8.0 --hash=sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0 +anyio==4.16.0 --hash=sha256:c66ae634e0b8228396b801741beb3294326d4b6102f7bba4041c0dfb58c29026 +attrs==26.1.0 --hash=sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309 +av==16.1.0 --hash=sha256:720edd4d25aa73723c1532bb0597806d7b9af5ee34fc02358782c358cfe2f879 +certifi==2026.10.23 --hash=sha256:e9b54de76e530bb6cf0ccd036787103acb58fd42ba930ea8bf948ddbaa2f61d1 +cffi==2.1.1 --hash=sha256:3311ed60d36f83378794e1009ac6258bafbf81f7888b4caa7b35a521e3f95813 +charset-normalizer==3.5.2 --hash=sha256:d760fe2a4d7c3b226cb9026d6a842868d52a7901bd98420e1baf14e80da85cf5 +click==8.5.1 --hash=sha256:8bb5376d2006950386f8c6427613334a2acadc5f559cb80e638d00b4df957e37 +detect-installer==0.2.1 --hash=sha256:87e2ce7d05cb108b178f8e45bb80d3d88c0cb4b397983345c18ad6562e02e620 +diffusers==0.43.0 --hash=sha256:8afb8ff1b09d693854de0412d8bdc94fc9e6e6e4efefb2386c616f64a4dc9ac6 +dnspython==2.9.2 --hash=sha256:af15f7d385a6ae2fb35a796fde2138004c8c42c7389090e2addc83e3b3340fd8 +einops==0.8.2 --hash=sha256:54058201ac7087911181bfec4af6091bb59380360f069276601256a76af08193 +email-validator==2.4.0 --hash=sha256:4454e41c4e1c6b6caee34a1dc71d5e736791562be8fd179b02e298f796522684 +fastapi==0.165.0 --hash=sha256:896f48462042bb4f162147eb8fac7e858753dc1f486765709baf105d70e4fc5b +fastapi-cli==0.0.34 --hash=sha256:4a1185b66308c7543e98957426a8ecd205d58853514cb9a68c78066f2e6e467b +fastapi-cloud-cli==0.28.0 --hash=sha256:e1409bc42017a86f8092fdf1dd64ead129698936cfeb151a6c3e023f611fd620 +fastar==0.12.0 --hash=sha256:1f0432d1138e7f3b65a3afc324ac373706751d1b8554541fe33d5e62ab08ff11 +filelock==4.2.12 --hash=sha256:21decfec02f116ed4c954433cdb31f28c3b346b784543177b6d7059da2702ee9 +frozenlist==1.8.0 --hash=sha256:957e7c38f250991e48a9a73e6423db1bb9dd14e722a10f6b8bb8e16a0f55f695 +fsspec==2026.9.0 --hash=sha256:8dd6e646e99ea382bd85f97a45e6b526a442d79423a7dc673f1e2756d05fcb5f +ftfy==6.3.1 --hash=sha256:7c70eb532015cd2f9adb53f101fb6c7945988d023a085d127d1573dc49dd0083 +googleapis-common-protos==1.76.0 --hash=sha256:59e13344a9f3651aa201a03e2bee815a3bf43a4baf1f9ff8a9b62a840ce25fac +h11==0.16.0 --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 +hf-xet==1.8.0 --hash=sha256:8800e3444770f2c630b361bc6d5c6ebb770b7830646cf632fbb79d0e5bbc9e7f +httpcore==1.0.9 --hash=sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55 +httpcore2==2.16.2 --hash=sha256:16051ae2a42f776007543f32de9349a317b23d2cbb4e1456b4bac0e88b87419a +httptools==0.9.0 --hash=sha256:6e2780e33a58a93f27cc3bb74a55bae6f9a8278a1dbabdff392940d30d381671 +httpx==0.28.1 --hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad +httpx2==2.16.2 --hash=sha256:f05c13aade4c45dbf1c9060647a1058f0acad62398fad96e741269edb33d46f5 +huggingface-hub==1.33.0 --hash=sha256:04e434b06e100eddbce9a6e817d72693a7884b10a79bd67ab48080d5c07eb899 +idna==3.20 --hash=sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c +imageio==2.38.2 --hash=sha256:d71174e7af8369a4f73525c5b1c0d2621eb2827ed3cfd7327560be02ac1b4084 +imageio-ffmpeg==0.8.1 --hash=sha256:fad3176f859b25869e60a8ed79957d4e3ed74a97a95e5f0cdcb91455f6c6d00e +importlib-metadata==10.2.0 --hash=sha256:c7a4bf1c7479601f6b8e437cfc14f4f77028ba62a8f34ae95cd93d9774dfba51 +jinja2==3.1.6 --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 +lazy-importer==0.1.0 --hash=sha256:bd93fab7bef18e842eea02b839410954d37889140dc64ca4d9680aec6880c739 +livekit==1.3.0 --hash=sha256:e8e60f88925b812adb06adb73e5e3b1c667c401b019b6213a9bac81e0328b3b3 +livekit-api==1.2.3 --hash=sha256:9f832cf1ad2b6b41e4ef183cd8d6cfdc97967e0ef82d4e7c982db87ed86fc9da +livekit-protocol==1.1.32 --hash=sha256:f86e104b7a84e743f7e382892d781df4f103eefdbfb24520359158f9c427e032 +loguru==0.8.0 --hash=sha256:e0fba09d6f76ba2d2f4bd1757122942f5a670c702e988b5efaeab99dacba0069 +markdown-it-py==4.3.0 --hash=sha256:7aa1511a46c39138de78d9eba92ce47e63124d8003fc98b3a351909e04239efa +markupsafe==3.0.4 --hash=sha256:849dd2bb0e5e4ab2b71c7191726a4a8d5aa8a610daa584728cbee0b710ddc4ef +mdurl==0.1.2 --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 +mpmath==1.3.0 --hash=sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c +multidict==7.2.1 --hash=sha256:295a15ccb8cfde9351dd6062b80af4e839460636b3c46813b4aa443905ded86a +networkx==3.6.1 --hash=sha256:d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762 +numpy==2.4.6 --hash=sha256:0ab0a9c4ffb1a6d95ef519fe4247dba8eb6b18ad93999f76b7f657039acabd47 +opencv-python-headless==5.1.0.95 --hash=sha256:11d9753e15b8b3c56e6aa765139f722ca19ff26016d43bc81035f5d122731a81 +opentelemetry-api==1.45.2 --hash=sha256:2cbd737377e710ee740d261c94e93fa0f08e6d037c18f17513cc0a2d94479a34 +opentelemetry-exporter-http-transport==0.66b2 --hash=sha256:fd5f5b490c099acda1cd7b909d5e8981436bf18cc8c6098d302d114c271720e9 +opentelemetry-exporter-otlp-common==0.66b2 --hash=sha256:ace86efb75489f925f1ef809856ae5f0c7db534e4afdc624530222228b1236a5 +opentelemetry-exporter-otlp-proto-common==1.45.2 --hash=sha256:af483b24217028892f017096731b794b196b785025d2e6808874720fd0b05a80 +opentelemetry-exporter-otlp-proto-http==1.45.2 --hash=sha256:6e4d025138829d0ee57946930c2ccad6491ad4d92a958e9c7eea69278fe60cfc +opentelemetry-proto==1.45.2 --hash=sha256:03a7ca1b43f89fd5b965d5975902bb2515c0c86e2428b456836e9a6149c6235a +opentelemetry-sdk==1.45.2 --hash=sha256:ff974ea70ee4f1d51e2ec152db37c2b626c7bf82bb3f7d86114fd8704e7199c6 +opentelemetry-semantic-conventions==0.66b2 --hash=sha256:4cbe61e5f9275d57ab31708446e7938ef4291ebbe6a7fca6f7a629c5acae87d8 +packaging==26.3 --hash=sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c +pillow==13.0.0 --hash=sha256:7c88c10dd432f3b39ccdafc9ed81d0c21b8fd634502574da76d1bfa4fe2c76d0 +propcache==0.5.4 --hash=sha256:62530ca89187827e4a4fe733f971abe81a7542eeea48ff61995f19b64d7199c8 +protobuf==7.36.2 --hash=sha256:e11e1f0180583a2af89db6a2ecd9e8dc40aa6d2988ca175bfd0e6d12ea72d74e +psutil==7.2.2 --hash=sha256:b0726cecd84f9474419d67252add4ac0cd9811b04d61123054b9fb6f57df6e9e +pycparser==3.12 --hash=sha256:a076f3f489b39b43ed13563a1dafe42447445bec2e294154ec5c57fcdda4c85e +pydantic==2.14.4 --hash=sha256:653a2ad9f6d8858529c77c72717d0ea991d6c6837fe683246b2f4b2bbfc3fadc +pydantic-core==2.50.5 --hash=sha256:ba2b34529984a8e3cb1bc62a7687617722695a2cb4e57958de32a0eba128a07a +pydantic-extra-types==2.11.1 --hash=sha256:1722ea2bddae5628ace25f2aa685b69978ef533123e5638cfbddb999e0100ec1 +pydantic-settings==2.16.3 --hash=sha256:d1db16a2d34006323a1b42c0640898da58adfbad7ec30baaa9e2b18aafc08999 +pygments==2.21.0 --hash=sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9 +pyjwt==2.15.1 --hash=sha256:42d59d631f7768a1028a64c7ff581a9bf7519804daf91fc5b6c56e30eec5e193 +python-dotenv==1.2.4 --hash=sha256:42269a8a5b3fd54ffa6f3d84b18abed50064717576b4ecf03dc4a55d8aa04fdc +python-multipart==0.0.34 --hash=sha256:ebacd224bd686eee2f9aa1066a657affe07435a185834e05e5235e2e59173e91 +pyyaml==6.0.3 --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c +regex==2026.11.19 --hash=sha256:3bd48fcc3222607d2b0e260be6e1a957b874f852b13af8c8643c8fbd8acf2425 +requests==2.34.2 --hash=sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0 +rich==15.0.0 --hash=sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb +rich-toolkit==0.20.6 --hash=sha256:465453ba6f94f99401f59f79fb562827ba5f6b4d4759015d40019f5c67750262 +rignore==0.8.1 --hash=sha256:2aa76742832db1fdac070fa8e694dda0daf6f2f7191832bd555d1e80f8c9c42b +safetensors==0.8.0 --hash=sha256:7a46e5ff292c356d6991e60942ba7f79817682d3a2cef0702136448cb9c4d235 +scipy==1.17.1 --hash=sha256:744b2bf3640d907b79f3fd7874efe432d1cf171ee721243e350f55234b4cec4c +sentry-sdk==2.75.0 --hash=sha256:9a10e97de2cbd42ad7e4a6e02a0d5af19de9811eb02445077d41ba14c08e170c +setuptools==80.9.0 --hash=sha256:062d34222ad13e0cc312a4c02d73f059e86a4acbfbdea8f8f76b28c99f306922 +setuptools-scm==8.3.1 --hash=sha256:332ca0d43791b818b841213e76b1971b7711a960761c5bea5fc5cdb5196fbce3 +shellingham==1.5.4 --hash=sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686 +soundfile==0.14.0 --hash=sha256:e85724a90bc99a6e8062c0b4ddf725f53b2a3b70afd4da875e9d2cfc4e92f377 +starlette==1.10.0 --hash=sha256:478c0ab3b4ced6e82719849c118fa406f0278a3499c1a898cd15cd31bdc38ce4 +sympy==1.14.0 --hash=sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5 +tokenizers==0.22.2 --hash=sha256:2249487018adec45d6e3554c71d46eb39fa8ea67156c640f7513eb26f318cec7 +torch @ https://download.pytorch.org/whl/cpu/torch-2.11.0%2Bcpu-cp311-cp311-manylinux_2_28_aarch64.whl --hash=sha256:46fbb0aa257bb781efbfad648f5b045c0e232573b661f1461593db61342e9096 +torch-npu @ https://gitcode.com/Ascend/pytorch/releases/download/v26.3.0-beta.1-pytorch2.11.0/torch_npu-2.11.0.post4-cp311-cp311-manylinux_2_28_aarch64.whl --hash=sha256:10b3f58c8b9396b4be4027c35cf0e08ef6223699b9cf186c1d89fd5b75652536 +torchaudio @ https://download.pytorch.org/whl/cpu/torchaudio-2.11.0%2Bcpu-cp311-cp311-manylinux_2_28_aarch64.whl --hash=sha256:7a5ecdd20fb26d8eb1d62135ddc6db0f806b17667c10e82df913fd509d1f5cfe +torchcodec @ https://download.pytorch.org/whl/cpu/torchcodec-0.13.0%2Bcpu-cp311-cp311-manylinux_2_28_aarch64.whl --hash=sha256:960577cfc46cafef1553dffcc529d19a0fcdb9423d374f504875c16dd5facfe5 +torchvision @ https://download.pytorch.org/whl/cpu/torchvision-0.26.0%2Bcpu-cp311-cp311-manylinux_2_28_aarch64.whl --hash=sha256:c11e55041f6b84a6c4fb28981b901475aa81c38695ccec6ddfcc54c3fa9fac4f +tqdm==4.70.1 --hash=sha256:c293e525e6fef9c20e8728fd4612df02a0aa31bb5fe91ecd93e123b1b7bffa73 +transformers==5.14.1 --hash=sha256:9db974c4079ede2d1a3ea7ca5a240df33f2cc26fc2b36ba64c5f2a4f43b6e725 +truststore==0.10.5 --hash=sha256:9aaaedaefaf06d8b206278cf8b5012bc897f485a874503501e12d776df78951c +typer==0.29.2 --hash=sha256:f30731f8ca758368b836b3714872e58e395cef8fcaff918c853b1969ed5274e7 +types-protobuf==7.35.1.20260906 --hash=sha256:5155e48569e0dabff303fdf578db96cd31ea9a4a63b18018a4ceac6b0ae17462 +typing-extensions==4.16.0 --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 +typing-inspection==0.4.5 --hash=sha256:bc210535442b39e1bf2805e0c7a17dfd6a4ace760c84c7f7d16245d23a6b1c39 +urllib3==2.8.0 --hash=sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3 +uvicorn==0.54.0 --hash=sha256:505bdb0f318731d45f1f712071fc781a8981f6847a31c902c9f5e652d4f67faf +uvloop==0.23.0 --hash=sha256:8fcd721113260ffb5e38bf14a8725b17d431f34209f7d1c7005b667946e630b3 +watchfiles==1.3.0 --hash=sha256:4af9c464d410e8c44b58ddf1cbec7c3a02c1cb2c6c80df9c74659ba2fbda0ab5 +wcwidth==0.9.4 --hash=sha256:093469e01d881bff3db93ff3f56c364dc87c441ea01f9f41c22de0c0bdf5c6ef +websockets==17.4 --hash=sha256:954d43574f72be244b65f0fa61e2f814214b024f2ec849f487aa2f553f7c6e71 +wheel==0.45.1 --hash=sha256:708e7481cc80179af0e556bbf0cc00b8444c7321e2700b8d8580231d13017248 +yarl==1.25.1 --hash=sha256:acae6b45d1ace09b6ba3876da43b88366ef368f73b988c7f57e14231753d4420 +zipp==4.1.2 --hash=sha256:b30cb6cd253122169c47de245ca24a86bac2a5d3802ba15bf7ba6414eb9e144d diff --git a/docs/en/ascend_910b.md b/docs/en/ascend_910b.md new file mode 100644 index 00000000..6e71c6da --- /dev/null +++ b/docs/en/ascend_910b.md @@ -0,0 +1,181 @@ +# Run Wan2.2 on Ascend 910B with Docker + +## Install the image + +The candidate passed Wan generation on one and two Atlas 910B2 cards. Choose one installation method, then keep the same Bash session for checkpoint setup and generation. + +### Install the public Docker package + +On a Linux arm64 host, download the prebuilt TeleFuser application image from the public candidate release. Docker loads the frozen Python environment and assembles the runnable image with the official CANN base and Ubuntu FFmpeg packages. The commands require access to GitHub Releases, Quay and the Ubuntu package mirrors. [Public Docker package](https://github.com/yx0716/TeleFuser-upstream/releases/tag/ascend910b-c27fabd) + +```bash +TF_RELEASE=https://github.com/yx0716/TeleFuser-upstream/releases/download/ascend910b-c27fabd +TF_IMAGE=telefuser:ascend910b-c27fabd-public +curl -fsSLO "$TF_RELEASE/telefuser-ascend910b-c27fabd-runtime.tar.gz" +curl -fsSLO "$TF_RELEASE/Dockerfile.ascend.install" +curl -fsSLO "$TF_RELEASE/SHA256SUMS" +sha256sum -c SHA256SUMS && + docker load -i telefuser-ascend910b-c27fabd-runtime.tar.gz && + docker build --platform linux/arm64 -t "$TF_IMAGE" - < Dockerfile.ascend.install +``` + +### Load the offline archive + +To install the tested image on a Linux arm64 host, place the archive and its checksum in one directory, run the commands below and proceed to the checkpoints. Compare the Docker image ID with the hardware record at the end of this page. The tested archive is 6,137,716,301 bytes with SHA-256 `8a85baafb991f8640af6f3dba9f38ac8a76376ca669e7e64363c7eee8e9f2369`. + +```bash +sha256sum -c telefuser-ascend910b-c27fabd.tar.gz.sha256 +docker load -i telefuser-ascend910b-c27fabd.tar.gz +TF_IMAGE=telefuser:ascend910b-c27fabd +docker image inspect --format '{{.Id}}' "$TF_IMAGE" +``` + +### Build from source + +Run the following commands in one Bash session from a checkout containing `docker/Dockerfile.ascend`. Check that the assigned cards are idle and that the host driver and firmware match CANN. The container supplies Ubuntu 22.04, Python 3.11, CANN 9.3.0-beta.1, PyTorch 2.11.0+cpu and torch_npu 2.11.0.post4. [Ascend release notes and wheels](https://api.gitcode.com/api/v5/repos/Ascend/pytorch/releases/tags/v26.3.0-beta.1-pytorch2.11.0) + +```bash +uname -m +npu-smi info +docker version +TF_IMAGE=telefuser:ascend910b-local +docker build --platform linux/arm64 -f docker/Dockerfile.ascend \ + --build-arg "SOURCE_REVISION=$(git rev-parse HEAD)" \ + --build-arg "TELEFUSER_VERSION=0.0.0+g$(git rev-parse --short HEAD).local" \ + -t "$TF_IMAGE" . +``` + +The build defaults to AscendHub in China. For Docker Hub add `--build-arg CANN_REPOSITORY=docker.io/ascendai/cann`. Both registries serve the same arm64 manifest. + +## Prepare checkpoints and outputs + +Allow about 34 GB for the model in addition to the image. Model-level CPU offload and host-staged worker queues also use host memory. + +```bash +TF_MODEL_DIR="$PWD/models/Wan2.2-TI2V-5B" +TF_OUTPUT_DIR=$(mktemp -d "$PWD/telefuser-results.XXXXXX") +mkdir -p "$TF_MODEL_DIR" "$TF_OUTPUT_DIR"/{1,2,4}card +``` + +Download the pinned Wan weights once, or set `TF_MODEL_DIR` to the absolute path of an existing complete checkpoint. + +```bash +docker run --rm --entrypoint hf \ + --user "$(id -u):$(id -g)" -e HF_HOME=/tmp/hf \ + --mount "type=bind,src=$TF_MODEL_DIR,dst=/download" \ + "$TF_IMAGE" download Wan-AI/Wan2.2-TI2V-5B \ + --revision 921dbaf3f1674a56f47e83fb80a34bac8a8f203e \ + --local-dir /download +``` + +## Check the devices + +The example below maps physical devices starting at 0. Set `TF_DEVICE_IDS` to your assigned physical IDs. For example, use `TF_DEVICE_IDS=(5)` for physical card 5. Docker restricts visibility through the mounted device nodes. CANN numbers this set starting at 0. With only physical card 5 attached, the application uses `npu:0`. Passing host index 5 again in `ASCEND_RT_VISIBLE_DEVICES` causes ACL error 207004. Confirm each visible device can execute a tensor operation. The weight mount is read-only. + +```bash +TF_DEVICE_IDS=(0 1 2 3) + +run_910b() { + local count=$1 i + shift + case "$count" in + 1|2|4) ;; + *) echo 'Choose 1, 2 or 4 cards' >&2; return 2 ;; + esac + (( count <= ${#TF_DEVICE_IDS[@]} )) || { echo "Assigned devices: ${TF_DEVICE_IDS[*]}" >&2; return 2; } + local -a devices=() + for ((i=0; i&2; return 2 ;; + esac + (( count <= ${#TF_DEVICE_IDS[@]} )) || { echo "Assigned devices: ${TF_DEVICE_IDS[*]}" >&2; return 2; } + local -a devices=() + for ((i=0; i=0.21.0", "tqdm>=4.66.0", "transformers==5.14.1", - "triton", "uvicorn>=0.30,<1.0", - "torchao==0.17.0", - "bitsandbytes==0.48.0", ] [project.optional-dependencies] +cuda = [ + "triton", + "torchao==0.17.0", + "bitsandbytes==0.48.0", +] + ui = [ "gradio==5.50", ] diff --git a/telefuser/core/base_stage.py b/telefuser/core/base_stage.py index 53d0a110..a314d34f 100644 --- a/telefuser/core/base_stage.py +++ b/telefuser/core/base_stage.py @@ -10,7 +10,7 @@ import torch from telefuser.cache.session_memory import DeviceMemorySnapshot, capture_device_memory_snapshot -from telefuser.platforms import current_platform +from telefuser.platforms import current_platform, resolve_device from telefuser.utils.logging import logger from .config import FeatureCacheConfig, ModelRuntimeConfig, WeightOffloadType @@ -89,7 +89,10 @@ def __init__(self, name: str, model_runtime_config: ModelRuntimeConfig) -> None: self.model_runtime_config = model_runtime_config self.torch_dtype = model_runtime_config.torch_dtype self.device_type = model_runtime_config.device_type or current_platform.device_type - self.device = torch.device(type=self.device_type, index=model_runtime_config.device_id) + if self.device_type == "npu": + self.device = resolve_device(f"npu:{model_runtime_config.device_id}") + else: + self.device = torch.device(type=self.device_type, index=model_runtime_config.device_id) self.model_names: list[str] = [] self.onload_models_flag = False self._metrics_hook: StageMetricContext | None = None diff --git a/telefuser/distributed/device_mesh.py b/telefuser/distributed/device_mesh.py index 8281327e..ce3e69ff 100644 --- a/telefuser/distributed/device_mesh.py +++ b/telefuser/distributed/device_mesh.py @@ -13,10 +13,11 @@ from torch.distributed.device_mesh import DeviceMesh from telefuser.core.config import ParallelConfig +from telefuser.platforms import current_platform from telefuser.utils.logging import logger -def create_device_mesh_from_config(parallel_config: ParallelConfig, device_type: str = "cuda") -> DeviceMesh: +def create_device_mesh_from_config(parallel_config: ParallelConfig, device_type: str | None = None) -> DeviceMesh: """Create PyTorch DeviceMesh from ParallelConfig. Mesh dimensions are built in order: DP -> CFG -> SP (ring, ulysses) -> PP -> TP @@ -24,11 +25,13 @@ def create_device_mesh_from_config(parallel_config: ParallelConfig, device_type: Args: parallel_config: Parallel configuration with degrees for each dimension - device_type: Device type ("cuda" or "cpu") + device_type: Device type ("cuda", "npu", or "cpu"); defaults to the current platform's device type Returns: PyTorch DeviceMesh instance with named dimensions """ + if device_type is None: + device_type = current_platform.device_type _validate_parallel_config(parallel_config) sp_degree = parallel_config.sp_ulysses_degree * parallel_config.sp_ring_degree diff --git a/telefuser/distributed/pp_comm.py b/telefuser/distributed/pp_comm.py index 9c3b9085..5a4a2de7 100644 --- a/telefuser/distributed/pp_comm.py +++ b/telefuser/distributed/pp_comm.py @@ -24,6 +24,7 @@ import torch import torch.distributed as dist +from telefuser.platforms import current_platform from telefuser.utils.logging import logger @@ -107,7 +108,7 @@ def recv( if buffer is None: if shape is None: raise ValueError("Either buffer or shape must be provided") - buffer = torch.empty(shape, dtype=torch.float16, device="cuda") + buffer = torch.empty(shape, dtype=torch.float16, device=current_platform.device_type) buffer = buffer.contiguous() if async_op: @@ -266,7 +267,7 @@ def recv_latent(self, shape: tuple | None = None, dtype: torch.dtype = torch.bfl if shape is None: raise ValueError("recv_latent: shape must be provided") - buffer = torch.empty(shape, dtype=dtype, device="cuda") + buffer = torch.empty(shape, dtype=dtype, device=current_platform.device_type) buffer = buffer.contiguous() work = dist.irecv(buffer, self.recv_src, group=self._process_group) work.wait() @@ -300,7 +301,7 @@ def recv_latent_async(self, shape: tuple, dtype: torch.dtype = torch.bfloat16) - if self.is_first_stage: raise RuntimeError("recv_latent_async: First stage has no previous stage to receive from") - buffer = torch.empty(shape, dtype=dtype, device="cuda") + buffer = torch.empty(shape, dtype=dtype, device=current_platform.device_type) buffer = buffer.contiguous() work = dist.irecv(buffer, self.recv_src, group=self._process_group) return buffer, work diff --git a/telefuser/distributed/vae_spatial.py b/telefuser/distributed/vae_spatial.py index 85f2f410..cb68f23c 100644 --- a/telefuser/distributed/vae_spatial.py +++ b/telefuser/distributed/vae_spatial.py @@ -10,6 +10,12 @@ from .collectives import all_gather_stacked +def conv3d_activation_layout(tensor: torch.Tensor) -> torch.Tensor: + """Keep NCDHW on Ascend and use the existing channels-last layout elsewhere.""" + layout = torch.contiguous_format if tensor.device.type == "npu" else torch.channels_last_3d + return tensor.contiguous(memory_format=layout) + + def _spatial_rank() -> int: return dist.get_rank() if dist.is_initialized() else 0 @@ -128,7 +134,7 @@ def _spatial_causal_conv3d_forward( if any(padding): tensor = F.pad(tensor, padding) tensor = _exchange_height_halo(module, tensor, module._height_halo_size) - tensor = tensor.contiguous(memory_format=torch.channels_last_3d) + tensor = conv3d_activation_layout(tensor) return F.conv3d( tensor, module.weight, diff --git a/telefuser/kernel/triton/fp8_attention.py b/telefuser/kernel/triton/fp8_attention.py new file mode 100644 index 00000000..6e906e08 --- /dev/null +++ b/telefuser/kernel/triton/fp8_attention.py @@ -0,0 +1,233 @@ +"""Triton kernels for FP8 attention quantization and output correction.""" + +from __future__ import annotations + +import triton +import triton.language as tl + + +@triton.jit +def _sequence_mean_kernel( + x, + output, + tokens: tl.constexpr, + heads: tl.constexpr, + head_dim: tl.constexpr, + stride_batch, + stride_token, + stride_head, + stride_dim, + block_tokens: tl.constexpr, + block_dim: tl.constexpr, +): + batch_head = tl.program_id(0) + dim_block = tl.program_id(1) + batch = batch_head // heads + head = batch_head % heads + dims = dim_block * block_dim + tl.arange(0, block_dim) + valid_dims = dims < head_dim + total = tl.zeros((block_dim,), dtype=tl.float32) + for start in range(0, tokens, block_tokens): + token_offsets = start + tl.arange(0, block_tokens) + valid_tokens = token_offsets < tokens + offsets = ( + batch * stride_batch + + token_offsets[:, None] * stride_token + + head * stride_head + + dims[None, :] * stride_dim + ) + values = tl.load( + x + offsets, + mask=valid_tokens[:, None] & valid_dims[None, :], + other=0.0, + ).to(tl.float32) + total += tl.sum(values, axis=0) + tl.store(output + batch_head * head_dim + dims, total / tokens, mask=valid_dims) + + +@triton.jit +def _sequence_mean_kv_kernel( + k, + v, + k_output, + v_output, + tokens: tl.constexpr, + heads: tl.constexpr, + head_dim: tl.constexpr, + k_stride_batch, + k_stride_token, + k_stride_head, + k_stride_dim, + v_stride_batch, + v_stride_token, + v_stride_head, + v_stride_dim, + block_tokens: tl.constexpr, + block_dim: tl.constexpr, +): + batch_head = tl.program_id(0) + dim_block = tl.program_id(1) + batch = batch_head // heads + head = batch_head % heads + dims = dim_block * block_dim + tl.arange(0, block_dim) + valid_dims = dims < head_dim + k_total = tl.zeros((block_dim,), dtype=tl.float32) + v_total = tl.zeros((block_dim,), dtype=tl.float32) + for start in range(0, tokens, block_tokens): + token_offsets = start + tl.arange(0, block_tokens) + valid_tokens = token_offsets < tokens + k_offsets = ( + batch * k_stride_batch + + token_offsets[:, None] * k_stride_token + + head * k_stride_head + + dims[None, :] * k_stride_dim + ) + v_offsets = ( + batch * v_stride_batch + + token_offsets[:, None] * v_stride_token + + head * v_stride_head + + dims[None, :] * v_stride_dim + ) + mask = valid_tokens[:, None] & valid_dims[None, :] + k_values = tl.load(k + k_offsets, mask=mask, other=0.0).to(tl.float32) + v_values = tl.load(v + v_offsets, mask=mask, other=0.0).to(tl.float32) + k_total += tl.sum(k_values, axis=0) + v_total += tl.sum(v_values, axis=0) + output_offsets = batch_head * head_dim + dims + tl.store(k_output + output_offsets, k_total / tokens, mask=valid_dims) + tl.store(v_output + output_offsets, v_total / tokens, mask=valid_dims) + + +@triton.jit +def _add_output_correction_kernel( + output, + correction, + elements, + tokens: tl.constexpr, + heads: tl.constexpr, + head_dim: tl.constexpr, + block: tl.constexpr, +): + offsets = tl.program_id(0) * block + tl.arange(0, block) + valid = offsets < elements + batch = offsets // (tokens * heads * head_dim) + head = (offsets // head_dim) % heads + dim = offsets % head_dim + correction_offsets = (batch * heads + head) * head_dim + dim + values = tl.load(output + offsets, mask=valid).to(tl.float32) + shift = tl.load(correction + correction_offsets, mask=valid) + tl.store(output + offsets, values + shift, mask=valid) + + +@triton.jit +def _merge_prefix_output_correction_kernel( + prefix, + output, + correction, + merged, + elements, + prefix_tokens: tl.constexpr, + tokens: tl.constexpr, + heads: tl.constexpr, + head_dim: tl.constexpr, + block: tl.constexpr, +): + offsets = tl.program_id(0) * block + tl.arange(0, block) + valid = offsets < elements + batch = offsets // (tokens * heads * head_dim) + token = (offsets // (heads * head_dim)) % tokens + head = (offsets // head_dim) % heads + dim = offsets % head_dim + prefix_offsets = ((batch * prefix_tokens + token) * heads + head) * head_dim + dim + correction_offsets = (batch * heads + head) * head_dim + dim + prefix_values = tl.load(prefix + prefix_offsets, mask=valid & (token < prefix_tokens), other=0.0) + output_values = tl.load(output + offsets, mask=valid & (token >= prefix_tokens), other=0.0).to(tl.float32) + shift = tl.load(correction + correction_offsets, mask=valid & (token >= prefix_tokens), other=0.0) + values = tl.where(token < prefix_tokens, prefix_values, output_values + shift) + tl.store(merged + offsets, values, mask=valid) + + +@triton.jit +def _quantize_qkv_fp8_stage1( + q, + k, + v, + k_mean, + v_mean, + q_out, + k_out, + q_scale, + k_scale, + v_scale, + tokens: tl.constexpr, + heads: tl.constexpr, + head_dim: tl.constexpr, + block: tl.constexpr, + smooth_k: tl.constexpr, + smooth_v: tl.constexpr, +): + block_idx = tl.program_id(0) + batch_head = tl.program_id(1) + batch = batch_head // heads + head = batch_head % heads + token_offsets = block_idx * block + tl.arange(0, block) + dim_offsets = tl.arange(0, head_dim) + valid = token_offsets < tokens + offsets = ((batch * tokens + token_offsets[:, None]) * heads + head) * head_dim + dim_offsets[None, :] + q_values = tl.load(q + offsets, mask=valid[:, None], other=0.0).to(tl.float32) + k_values = tl.load(k + offsets, mask=valid[:, None], other=0.0).to(tl.float32) + v_values = tl.load(v + offsets, mask=valid[:, None], other=0.0).to(tl.float32) + mean_offsets = batch_head * head_dim + dim_offsets + if smooth_k: + k_values = tl.where( + valid[:, None], + k_values - tl.load(k_mean + mean_offsets)[None, :], + 0.0, + ) + if smooth_v: + v_values = tl.where( + valid[:, None], + v_values - tl.load(v_mean + mean_offsets)[None, :], + 0.0, + ) + + q_s = tl.maximum(tl.max(tl.max(tl.abs(q_values), axis=1), axis=0), 1.0e-6) / 448.0 + k_s = tl.maximum(tl.max(tl.max(tl.abs(k_values), axis=1), axis=0), 1.0e-6) / 448.0 + scale_offset = (batch * tl.cdiv(tokens, block) + block_idx) * heads + head + tl.store(q_scale + scale_offset, q_s) + tl.store(k_scale + scale_offset, k_s) + tl.store(q_out + offsets, q_values / q_s, mask=valid[:, None]) + tl.store(k_out + offsets, k_values / k_s, mask=valid[:, None]) + + v_s = tl.max(tl.abs(v_values), axis=0) / 448.0 + v_scale_offsets = (batch * heads + head) * head_dim + dim_offsets + tl.atomic_max(v_scale + v_scale_offsets, v_s) + + +@triton.jit +def _quantize_qkv_fp8_stage2_v( + v, + v_mean, + v_out, + v_scale, + tokens: tl.constexpr, + heads: tl.constexpr, + head_dim: tl.constexpr, + block: tl.constexpr, + smooth_v: tl.constexpr, +): + block_idx = tl.program_id(0) + batch_head = tl.program_id(1) + batch = batch_head // heads + head = batch_head % heads + token_offsets = block_idx * block + tl.arange(0, block) + dim_offsets = tl.arange(0, head_dim) + valid = token_offsets < tokens + input_offsets = ((batch * tokens + token_offsets[:, None]) * heads + head) * head_dim + dim_offsets[None, :] + output_offsets = ((batch * heads + head) * head_dim + dim_offsets[None, :]) * tokens + token_offsets[:, None] + scale_offsets = (batch * heads + head) * head_dim + dim_offsets + scale = tl.maximum(tl.load(v_scale + scale_offsets), 1.0e-6 / 448.0) + values = tl.load(v + input_offsets, mask=valid[:, None], other=0.0).to(tl.float32) + if smooth_v: + values = values - tl.load(v_mean + scale_offsets)[None, :] + tl.store(v_out + output_offsets, values / scale[None, :], mask=valid[:, None]) diff --git a/telefuser/models/wan22_video_vae.py b/telefuser/models/wan22_video_vae.py index 8ca32a35..6e70f21f 100644 --- a/telefuser/models/wan22_video_vae.py +++ b/telefuser/models/wan22_video_vae.py @@ -1403,7 +1403,11 @@ def decode( if self.parallelism > 1 and dist.is_initialized(): # tiled=True → tile_dist, tiled=False → 2d_split method = "tile_dist" if tiled else "2d_split" - hidden_states_tensor = torch.stack(hidden_states) + # The stage passes a batched [B, C, T, H, W] tensor; lists of per-video tensors are stacked. + if isinstance(hidden_states, torch.Tensor): + hidden_states_tensor = hidden_states + else: + hidden_states_tensor = torch.stack(hidden_states) return self.decode_parallel(hidden_states_tensor, device, method=method) # Single GPU processing diff --git a/telefuser/models/wan_video_vae.py b/telefuser/models/wan_video_vae.py index 22b4bc7c..5389ce9d 100644 --- a/telefuser/models/wan_video_vae.py +++ b/telefuser/models/wan_video_vae.py @@ -17,6 +17,7 @@ _spatial_causal_conv3d_forward, _spatial_world_size, _split_height, + conv3d_activation_layout, ) from telefuser.utils.logging import logger from telefuser.utils.model_weight import hash_state_dict_keys @@ -119,7 +120,7 @@ def forward(self, x: torch.Tensor, cache_x: torch.Tensor | None = None) -> torch x = torch.cat([cache_x, x], dim=2) padding[4] -= cache_x.shape[2] x = F.pad(x, padding) - x = x.contiguous(memory_format=torch.channels_last_3d) + x = conv3d_activation_layout(x) return super().forward(x) diff --git a/telefuser/ops/attention/sparse_sage.py b/telefuser/ops/attention/sparse_sage.py index 8332ce77..bc784183 100644 --- a/telefuser/ops/attention/sparse_sage.py +++ b/telefuser/ops/attention/sparse_sage.py @@ -18,9 +18,6 @@ import torch -from telefuser.kernel.triton.quant_per_block import per_block_int8 -from telefuser.kernel.triton.sparse_int8_attn import forward as sparse_sageattn_fwd - def sparse_sageattn( q: torch.Tensor, @@ -43,6 +40,9 @@ def sparse_sageattn( Returns: Output tensor with same shape and layout as input. """ + from telefuser.kernel.triton.quant_per_block import per_block_int8 + from telefuser.kernel.triton.sparse_int8_attn import forward as sparse_sageattn_fwd + seq_dim = 2 if tensor_layout == "HND" else 1 if mask_id is None: diff --git a/telefuser/ops/fp8_attention.py b/telefuser/ops/fp8_attention.py index 02b0eff1..92c7cde7 100644 --- a/telefuser/ops/fp8_attention.py +++ b/telefuser/ops/fp8_attention.py @@ -6,239 +6,10 @@ import torch import torch.nn.functional as F -import triton -import triton.language as tl FP8_ATTENTION_BLOCK_SIZE = 64 -@triton.jit -def _sequence_mean_kernel( - x, - output, - tokens: tl.constexpr, - heads: tl.constexpr, - head_dim: tl.constexpr, - stride_batch, - stride_token, - stride_head, - stride_dim, - block_tokens: tl.constexpr, - block_dim: tl.constexpr, -): - batch_head = tl.program_id(0) - dim_block = tl.program_id(1) - batch = batch_head // heads - head = batch_head % heads - dims = dim_block * block_dim + tl.arange(0, block_dim) - valid_dims = dims < head_dim - total = tl.zeros((block_dim,), dtype=tl.float32) - for start in range(0, tokens, block_tokens): - token_offsets = start + tl.arange(0, block_tokens) - valid_tokens = token_offsets < tokens - offsets = ( - batch * stride_batch - + token_offsets[:, None] * stride_token - + head * stride_head - + dims[None, :] * stride_dim - ) - values = tl.load( - x + offsets, - mask=valid_tokens[:, None] & valid_dims[None, :], - other=0.0, - ).to(tl.float32) - total += tl.sum(values, axis=0) - tl.store(output + batch_head * head_dim + dims, total / tokens, mask=valid_dims) - - -@triton.jit -def _sequence_mean_kv_kernel( - k, - v, - k_output, - v_output, - tokens: tl.constexpr, - heads: tl.constexpr, - head_dim: tl.constexpr, - k_stride_batch, - k_stride_token, - k_stride_head, - k_stride_dim, - v_stride_batch, - v_stride_token, - v_stride_head, - v_stride_dim, - block_tokens: tl.constexpr, - block_dim: tl.constexpr, -): - batch_head = tl.program_id(0) - dim_block = tl.program_id(1) - batch = batch_head // heads - head = batch_head % heads - dims = dim_block * block_dim + tl.arange(0, block_dim) - valid_dims = dims < head_dim - k_total = tl.zeros((block_dim,), dtype=tl.float32) - v_total = tl.zeros((block_dim,), dtype=tl.float32) - for start in range(0, tokens, block_tokens): - token_offsets = start + tl.arange(0, block_tokens) - valid_tokens = token_offsets < tokens - k_offsets = ( - batch * k_stride_batch - + token_offsets[:, None] * k_stride_token - + head * k_stride_head - + dims[None, :] * k_stride_dim - ) - v_offsets = ( - batch * v_stride_batch - + token_offsets[:, None] * v_stride_token - + head * v_stride_head - + dims[None, :] * v_stride_dim - ) - mask = valid_tokens[:, None] & valid_dims[None, :] - k_values = tl.load(k + k_offsets, mask=mask, other=0.0).to(tl.float32) - v_values = tl.load(v + v_offsets, mask=mask, other=0.0).to(tl.float32) - k_total += tl.sum(k_values, axis=0) - v_total += tl.sum(v_values, axis=0) - output_offsets = batch_head * head_dim + dims - tl.store(k_output + output_offsets, k_total / tokens, mask=valid_dims) - tl.store(v_output + output_offsets, v_total / tokens, mask=valid_dims) - - -@triton.jit -def _add_output_correction_kernel( - output, - correction, - elements, - tokens: tl.constexpr, - heads: tl.constexpr, - head_dim: tl.constexpr, - block: tl.constexpr, -): - offsets = tl.program_id(0) * block + tl.arange(0, block) - valid = offsets < elements - batch = offsets // (tokens * heads * head_dim) - head = (offsets // head_dim) % heads - dim = offsets % head_dim - correction_offsets = (batch * heads + head) * head_dim + dim - values = tl.load(output + offsets, mask=valid).to(tl.float32) - shift = tl.load(correction + correction_offsets, mask=valid) - tl.store(output + offsets, values + shift, mask=valid) - - -@triton.jit -def _merge_prefix_output_correction_kernel( - prefix, - output, - correction, - merged, - elements, - prefix_tokens: tl.constexpr, - tokens: tl.constexpr, - heads: tl.constexpr, - head_dim: tl.constexpr, - block: tl.constexpr, -): - offsets = tl.program_id(0) * block + tl.arange(0, block) - valid = offsets < elements - batch = offsets // (tokens * heads * head_dim) - token = (offsets // (heads * head_dim)) % tokens - head = (offsets // head_dim) % heads - dim = offsets % head_dim - prefix_offsets = ((batch * prefix_tokens + token) * heads + head) * head_dim + dim - correction_offsets = (batch * heads + head) * head_dim + dim - prefix_values = tl.load(prefix + prefix_offsets, mask=valid & (token < prefix_tokens), other=0.0) - output_values = tl.load(output + offsets, mask=valid & (token >= prefix_tokens), other=0.0).to(tl.float32) - shift = tl.load(correction + correction_offsets, mask=valid & (token >= prefix_tokens), other=0.0) - values = tl.where(token < prefix_tokens, prefix_values, output_values + shift) - tl.store(merged + offsets, values, mask=valid) - - -@triton.jit -def _quantize_qkv_fp8_stage1( - q, - k, - v, - k_mean, - v_mean, - q_out, - k_out, - q_scale, - k_scale, - v_scale, - tokens: tl.constexpr, - heads: tl.constexpr, - head_dim: tl.constexpr, - block: tl.constexpr, - smooth_k: tl.constexpr, - smooth_v: tl.constexpr, -): - block_idx = tl.program_id(0) - batch_head = tl.program_id(1) - batch = batch_head // heads - head = batch_head % heads - token_offsets = block_idx * block + tl.arange(0, block) - dim_offsets = tl.arange(0, head_dim) - valid = token_offsets < tokens - offsets = ((batch * tokens + token_offsets[:, None]) * heads + head) * head_dim + dim_offsets[None, :] - q_values = tl.load(q + offsets, mask=valid[:, None], other=0.0).to(tl.float32) - k_values = tl.load(k + offsets, mask=valid[:, None], other=0.0).to(tl.float32) - v_values = tl.load(v + offsets, mask=valid[:, None], other=0.0).to(tl.float32) - mean_offsets = batch_head * head_dim + dim_offsets - if smooth_k: - k_values = tl.where( - valid[:, None], - k_values - tl.load(k_mean + mean_offsets)[None, :], - 0.0, - ) - if smooth_v: - v_values = tl.where( - valid[:, None], - v_values - tl.load(v_mean + mean_offsets)[None, :], - 0.0, - ) - - q_s = tl.maximum(tl.max(tl.max(tl.abs(q_values), axis=1), axis=0), 1.0e-6) / 448.0 - k_s = tl.maximum(tl.max(tl.max(tl.abs(k_values), axis=1), axis=0), 1.0e-6) / 448.0 - scale_offset = (batch * tl.cdiv(tokens, block) + block_idx) * heads + head - tl.store(q_scale + scale_offset, q_s) - tl.store(k_scale + scale_offset, k_s) - tl.store(q_out + offsets, q_values / q_s, mask=valid[:, None]) - tl.store(k_out + offsets, k_values / k_s, mask=valid[:, None]) - - v_s = tl.max(tl.abs(v_values), axis=0) / 448.0 - v_scale_offsets = (batch * heads + head) * head_dim + dim_offsets - tl.atomic_max(v_scale + v_scale_offsets, v_s) - - -@triton.jit -def _quantize_qkv_fp8_stage2_v( - v, - v_mean, - v_out, - v_scale, - tokens: tl.constexpr, - heads: tl.constexpr, - head_dim: tl.constexpr, - block: tl.constexpr, - smooth_v: tl.constexpr, -): - block_idx = tl.program_id(0) - batch_head = tl.program_id(1) - batch = batch_head // heads - head = batch_head % heads - token_offsets = block_idx * block + tl.arange(0, block) - dim_offsets = tl.arange(0, head_dim) - valid = token_offsets < tokens - input_offsets = ((batch * tokens + token_offsets[:, None]) * heads + head) * head_dim + dim_offsets[None, :] - output_offsets = ((batch * heads + head) * head_dim + dim_offsets[None, :]) * tokens + token_offsets[:, None] - scale_offsets = (batch * heads + head) * head_dim + dim_offsets - scale = tl.maximum(tl.load(v_scale + scale_offsets), 1.0e-6 / 448.0) - values = tl.load(v + input_offsets, mask=valid[:, None], other=0.0).to(tl.float32) - if smooth_v: - values = values - tl.load(v_mean + scale_offsets)[None, :] - tl.store(v_out + output_offsets, values / scale[None, :], mask=valid[:, None]) - - def _sequence_mean(x: torch.Tensor) -> torch.Tensor: """Reduce BTHD sequence means in FP32 without materializing an FP32 copy.""" @@ -246,6 +17,10 @@ def _sequence_mean(x: torch.Tensor) -> torch.Tensor: raise ValueError("attention smoothing expects a floating-point [B, T, H, D] tensor") if not x.is_cuda: return x.float().mean(dim=1) + import triton + + from telefuser.kernel.triton.fp8_attention import _sequence_mean_kernel + batch, tokens, heads, head_dim = x.shape output = torch.empty((batch, heads, head_dim), device=x.device, dtype=torch.float32) block_tokens = 128 @@ -274,6 +49,10 @@ def _sequence_means(k: torch.Tensor, v: torch.Tensor) -> tuple[torch.Tensor, tor raise ValueError("K and V must be on the same device") if not k.is_cuda: return k.float().mean(dim=1), v.float().mean(dim=1) + import triton + + from telefuser.kernel.triton.fp8_attention import _sequence_mean_kv_kernel + batch, tokens, heads, head_dim = k.shape k_output = torch.empty((batch, heads, head_dim), device=k.device, dtype=torch.float32) v_output = torch.empty_like(k_output) @@ -314,6 +93,10 @@ def _quantize_fp8_qkv_sm90( batch, tokens, heads, head_dim = q.shape if head_dim != 128: raise ValueError("fused FP8 QKV quantization requires head dimension 128") + import triton + + from telefuser.kernel.triton.fp8_attention import _quantize_qkv_fp8_stage1, _quantize_qkv_fp8_stage2_v + blocks = triton.cdiv(tokens, FP8_ATTENTION_BLOCK_SIZE) q_out = torch.empty(q.shape, device=q.device, dtype=torch.float8_e4m3fn) k_out = torch.empty_like(q_out) @@ -521,6 +304,10 @@ def apply_fp8_attention_output_correction(output: torch.Tensor, correction: torc return (output.float() + correction).to(output.dtype) if not output.is_contiguous() or not correction.is_contiguous(): raise ValueError("FP8 attention output and correction must be contiguous") + import triton + + from telefuser.kernel.triton.fp8_attention import _add_output_correction_kernel + batch, tokens, heads, head_dim = output.shape elements = output.numel() block = 256 @@ -565,6 +352,10 @@ def merge_fp8_attention_prefix( if not (dense_prefix.is_contiguous() and output.is_contiguous() and correction.is_contiguous()): raise ValueError("FP8 attention prefix merge inputs must be contiguous") + import triton + + from telefuser.kernel.triton.fp8_attention import _merge_prefix_output_correction_kernel + merged = torch.empty_like(output) batch, tokens, heads, head_dim = output.shape elements = output.numel() diff --git a/telefuser/platforms/__init__.py b/telefuser/platforms/__init__.py index 7d517faa..c324d077 100644 --- a/telefuser/platforms/__init__.py +++ b/telefuser/platforms/__init__.py @@ -6,6 +6,7 @@ from __future__ import annotations import logging +import re import torch @@ -73,9 +74,49 @@ def _resolve_current_platform() -> BasePlatform: current_platform: BasePlatform = _resolve_current_platform() +def normalize_device_name(device: str | torch.device) -> str: + """Parse a TeleFuser device without importing its vendor runtime.""" + match = re.fullmatch(r"(cpu|cuda|rocm|npu)(?::([0-9]+))?", str(device)) + if match is None: + raise ValueError(f"Expected cpu, cuda, rocm or npu with an optional index, got {device!r}") + kind, index = match.groups() + if kind == "cpu": + return "cpu" + return f"{'cuda' if kind == 'rocm' else kind}:{int(index or 0)}" + + +def resolve_device(device: str | torch.device) -> torch.device: + """Validate a compute device against this process before starting workers.""" + name = normalize_device_name(device) + if name == "cpu": + return torch.device(name) + kind, index = name.split(":") + if kind == "npu": + try: + import torch_npu # noqa: F401 + except (ImportError, OSError, RuntimeError) as exc: + raise RuntimeError(f"Cannot initialize torch_npu for {name}: {exc}") from exc + if kind != current_platform.device_type or ( + str(device).split(":")[0] == "rocm" and current_platform.device_name != "rocm" + ): + raise RuntimeError(f"Requested {device} but detected {current_platform.device_name}; check device visibility") + count = current_platform.device_count() + if int(index) >= count: + raise ValueError(f"{name} is outside the {count} visible {current_platform.device_name} devices") + return torch.device(name) + + +def requires_host_tensor_queue(device_type: str) -> bool: + """Ascend tensors cross multiprocessing queues through host memory.""" + return device_type == "npu" + + __all__ = [ "BasePlatform", "current_platform", + "normalize_device_name", + "resolve_device", + "requires_host_tensor_queue", "CudaPlatform", "RocmPlatform", "NPUPlatform", diff --git a/telefuser/platforms/hccl.py b/telefuser/platforms/hccl.py new file mode 100644 index 00000000..99de0389 --- /dev/null +++ b/telefuser/platforms/hccl.py @@ -0,0 +1,53 @@ +"""Reserve distinct HCCL TCP port blocks for local multiprocessing groups.""" + +from __future__ import annotations + +import errno +import os +import socket +import tempfile +from contextlib import ExitStack +from pathlib import Path +from typing import BinaryIO + +_BLOCK_SIZE = 16 +_LAST_BASE = 65536 - _BLOCK_SIZE + + +def reserve_hccl_ports() -> tuple[int, BinaryIO]: + """Hold the returned lock until all ranks exit. Kernel locks survive parent concurrency.""" + import fcntl + + value = os.environ.get("HCCL_IF_BASE_PORT", "60000") + try: + start = int(value) + if not 1024 <= start <= _LAST_BASE: + raise ValueError + except ValueError: + raise ValueError(f"HCCL_IF_BASE_PORT must be between 1024 and {_LAST_BASE}, got {value!r}") from None + + # Use common block boundaries even when two launchers specify different starting ports. + start += -start % _BLOCK_SIZE + for base in range(start, _LAST_BASE + 1, _BLOCK_SIZE): + path = Path(tempfile.gettempdir()) / f"telefuser-hccl-{base}.lock" + # Read-only descriptors allow launchers with different UIDs to lock an existing file. + try: + fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDONLY, 0o644) + except FileExistsError: + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW) + lock = os.fdopen(fd, "rb") + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + with ExitStack() as probes: + for port in range(base, base + _BLOCK_SIZE): + probe = probes.enter_context(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) + probe.bind(("", port)) + return base, lock + except OSError as exc: + lock.close() + if exc.errno not in (errno.EAGAIN, errno.EWOULDBLOCK, errno.EADDRINUSE): + raise + except BaseException: + lock.close() + raise + raise RuntimeError(f"No free {_BLOCK_SIZE}-port HCCL block at or above {start} (last base {_LAST_BASE})") diff --git a/telefuser/utils/system.py b/telefuser/utils/system.py index 5569f071..82649d6b 100644 --- a/telefuser/utils/system.py +++ b/telefuser/utils/system.py @@ -10,6 +10,7 @@ class PortAllocator: """Thread-safe port allocator for managing non-overlapping port interval allocation.""" + _port_intervals: dict[int, tuple[int, int]] = {} _allocated_intervals: set[tuple[int, int]] = set() _lock = threading.RLock() # Use reentrant lock to allow the same thread to acquire multiple times @@ -57,8 +58,17 @@ def get_free_port_in_interval( # Randomly select a port within the chosen interval port = random.randint(chosen_interval[0], chosen_interval[1]) + cls._port_intervals[port] = chosen_interval return port + @classmethod + def release_port(cls, port: int) -> None: + """Return the interval owned by a port from get_free_port_in_interval.""" + with cls._lock: + interval = cls._port_intervals.pop(port, None) + if interval is not None: + cls._allocated_intervals.discard(interval) + @classmethod def release_interval(cls, start_port: int, end_port: int): """Release an interval to make it available for reuse (thread-safe). @@ -71,6 +81,7 @@ def release_interval(cls, start_port: int, end_port: int): interval = (start_port, end_port) if interval in cls._allocated_intervals: cls._allocated_intervals.remove(interval) + cls._port_intervals = {p: v for p, v in cls._port_intervals.items() if v != interval} @classmethod def get_specific_interval(cls, start_port: int, end_port: int) -> int: @@ -99,6 +110,7 @@ def reset_allocator(cls): """Reset the allocator, releasing all allocated intervals (thread-safe).""" with cls._lock: cls._allocated_intervals.clear() + cls._port_intervals.clear() @classmethod def get_allocated_intervals(cls) -> set[tuple[int, int]]: diff --git a/telefuser/worker/parallel_worker.py b/telefuser/worker/parallel_worker.py index 2c160544..058f411b 100644 --- a/telefuser/worker/parallel_worker.py +++ b/telefuser/worker/parallel_worker.py @@ -13,17 +13,19 @@ import time from collections.abc import Callable, Collection from datetime import timedelta +from functools import partial from multiprocessing.queues import SimpleQueue from queue import Empty -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, BinaryIO import torch import torch.distributed as dist import torch.multiprocessing as mp -import torch.multiprocessing.spawn +from torch.multiprocessing.spawn import ProcessContext from telefuser.core.base_stage import BaseStage -from telefuser.platforms import current_platform +from telefuser.platforms import BasePlatform, CpuPlatform, current_platform, requires_host_tensor_queue, resolve_device +from telefuser.platforms.hccl import reserve_hccl_ports from telefuser.utils.logging import logger from telefuser.utils.system import PortAllocator @@ -36,24 +38,25 @@ _DISCARD_TENSOR_REFS = "__telefuser_discard_tensor_refs__" -def to_device(data: Any, device: str | torch.device) -> Any: - """Recursively move data to target device.""" +def _worker_platform(stage: BaseStage) -> BasePlatform: + if stage.device_type == "cpu" and current_platform.device_type != "cpu": + return CpuPlatform() + return current_platform + + +def to_device(data: Any, device: str | torch.device, *, preserve_tuples: bool = False) -> Any: + """Move nested inputs; legacy callers receive lists for positional argument tuples.""" if isinstance(data, dict): - return {k: to_device(v, device) for k, v in data.items()} - if isinstance(data, tuple | list): - return [to_device(t, device) for t in data] - elif isinstance(data, torch.Tensor): - if data.device == device: - # Already on target device - ensure sharing for CPU tensors - if device == "cpu" and not data.is_shared(): - data.share_memory_() - return data + return {k: to_device(v, device, preserve_tuples=preserve_tuples) for k, v in data.items()} + if isinstance(data, (tuple, list)): + items = [to_device(v, device, preserve_tuples=preserve_tuples) for v in data] + return tuple(items) if preserve_tuples and isinstance(data, tuple) else items + if isinstance(data, torch.Tensor): tensor = data.to(device) - if device == "cpu" and not tensor.is_shared(): + if tensor.device.type == "cpu" and not tensor.is_shared(): tensor.share_memory_() return tensor - else: - return data + return data def _worker_loop( @@ -65,6 +68,9 @@ def _worker_loop( master_port: int, tensor_output_channel: WorkerTensorChannel | None = None, tensor_input_channels: tuple[WorkerTensorChannel, ...] = (), + hccl_if_base_port: int | None = None, + *, + rank_offset: int = 0, ) -> None: """Worker process main loop. @@ -72,8 +78,10 @@ def _worker_loop( """ from telefuser.utils.profiler import mark_as_worker_process + rank += rank_offset mark_as_worker_process() previous_sigint_handler = signal.signal(signal.SIGINT, signal.SIG_IGN) + platform = _worker_platform(stage) args = None kwargs = None stage_inputs = None @@ -84,19 +92,28 @@ def _worker_loop( # single-rank workers such as the LingBot condition encoder. torch.set_num_threads(parallel_config.worker_intra_op_threads) device = stage.device + host_queue = requires_host_tensor_queue(platform.device_type) if world_size > 1: os.environ["RANK"] = str(rank) os.environ["WORLD_SIZE"] = str(world_size) os.environ["MASTER_ADDR"] = "localhost" os.environ["MASTER_PORT"] = str(master_port) - device_ids = parallel_config.device_ids - device_id = rank - if device_ids is not None: - device_id = device_ids[rank] - device = torch.device(type=current_platform.device_type, index=device_id) + if hccl_if_base_port is not None: + os.environ["HCCL_IF_BASE_PORT"] = str(hccl_if_base_port) + + if host_queue or world_size > 1: + ids = parallel_config.device_ids + device_id = rank if world_size > 1 else device.index or 0 + if ids is not None: + device_id = ids[rank] + device = torch.device(platform.device_type, device_id) stage.model_runtime_config.device_id = device_id stage.device = device - if current_platform.device_type == "cuda": + if platform.device_type != "cpu": + platform.set_device(device) + logger.info(f"Worker {stage.name} rank {rank}/{world_size} bound to {device}") + if world_size > 1: + if platform.device_type == "cuda": # torch.distributed.all_reduce does not free the input tensor until # the synchronization point. This causes the memory usage to grow # as the number of all_reduce calls increases. This env var disables @@ -107,14 +124,13 @@ def _worker_loop( timeout = timedelta(seconds=600) dist.init_process_group( - backend=current_platform.dist_backend, + backend=platform.dist_backend, init_method="env://", timeout=timeout, world_size=world_size, rank=rank, - device_id=device, + device_id=device if device.type != "cpu" else None, ) - current_platform.set_device(device) stage.parallel_models() while True: @@ -145,19 +161,21 @@ def _worker_loop( if channel.contains_ref(stage_inputs): stage_inputs = channel.receive(stage_inputs, rank=rank, device=device) args, kwargs = stage_inputs - kwargs = to_device(kwargs, device) - args = to_device(args, device) + kwargs = to_device(kwargs, device, preserve_tuples=host_queue) + args = to_device(args, device, preserve_tuples=host_queue) with torch.no_grad(): - if not isinstance(args, list): + if not isinstance(args, (list, tuple)): args = [args] y = getattr(stage, name)(*args, **kwargs) del kwargs, args if getattr(stage, "empty_cache_after_call", True): - current_platform.empty_cache() + platform.empty_cache() if transport_output and tensor_output_channel is not None and (world_size == 1 or rank == 0): y = tensor_output_channel.send(y) # Always output results when world_size=1 if world_size == 1 or rank == 0: + if host_queue: + y = to_device(y, "cpu", preserve_tuples=True) queue_out.put(y) except Exception as e: import traceback @@ -175,18 +193,43 @@ def _worker_loop( kwargs = None stage_inputs = None y = None - current_platform.synchronize() + if platform.device_type != "cpu": + platform.synchronize() + platform.ipc_collect() gc.collect() - current_platform.empty_cache() - current_platform.ipc_collect() - if world_size > 1: + platform.empty_cache() + if world_size > 1 and dist.is_initialized(): dist.destroy_process_group() signal.signal(signal.SIGINT, previous_sigint_handler) +def _spawn_hccl_group(world_size: int, arguments: tuple) -> ProcessContext: + """Keep handles to earlier ranks if spawning a later rank fails in the parent.""" + contexts = [] + try: + for rank in range(world_size): + context = mp.spawn(partial(_worker_loop, rank_offset=rank), args=arguments, nprocs=1, join=False) + contexts.append(context) + return ProcessContext( + [context.processes[0] for context in contexts], + [context.error_files[0] for context in contexts], + ) + except BaseException: + for context in contexts: + process = context.processes[0] + if process.is_alive(): + process.kill() + process.join() + raise + + class ParallelWorker: """Multi-process worker for distributed execution across multiple GPUs.""" + _hccl_lock: BinaryIO | None = None + _master_port: int | None = None + ctx = None + def __init__( self, stage: BaseStage, @@ -200,13 +243,25 @@ def __init__( self._stage: BaseStage = stage # Keep reference to wrapped stage for metrics proxying self.world_size: int = parallel_config.world_size - self.device: str = current_platform.device_type + platform = _worker_platform(stage) + self.device: str = platform.device_type self.device_ids: list[int] = parallel_config.device_ids if self.device_ids is None: self.device_ids = list(range(self.world_size)) + if stage.device_type in ("npu", "cuda") and stage.device_type != platform.device_type: + resolve_device(stage.device) + if self.device == "npu": + if parallel_config.device_ids is None: + self.device_ids = [stage.device.index or 0] + if len(set(self.device_ids)) != len(self.device_ids): + raise ValueError(f"NPU worker ranks require distinct device IDs, got {self.device_ids}") + for idx in self.device_ids: + resolve_device(f"npu:{idx}") + self.name: str = f"Parallel Worker {stage.name}" - self.queue_with_cpu: bool = parallel_config.queue_with_cpu + # Queue transport requires CPU marshalling on platforms without reliable device IPC (e.g. NPU). + self.queue_with_cpu: bool = parallel_config.queue_with_cpu or requires_host_tensor_queue(self.device) self.timeout: int = parallel_config.timeout self._lifecycle_lock = threading.Lock() self._failed = False @@ -239,24 +294,38 @@ def __init__( self.queue_in: list[SimpleQueue] = [spawn_ctx.SimpleQueue() for _ in range(self.world_size)] self.queue_out: SimpleQueue = spawn_ctx.SimpleQueue() - master_port = PortAllocator().get_free_port_in_interval() - logger.info(f"parallel worker {self.name} with port {master_port}, world_size={self.world_size}") - - # For world_size=1 case, still use spawn to start one process - self.ctx = mp.spawn( - _worker_loop, - args=( + try: + self._master_port = PortAllocator().get_free_port_in_interval() + hccl_port = None + if self.world_size > 1 and platform.dist_backend == "hccl": + hccl_port, self._hccl_lock = reserve_hccl_ports() + logger.info(f"{self.name}: world_size={self.world_size}, master_port={self._master_port}, hccl={hccl_port}") + arguments = ( self.world_size, self.queue_in, self.queue_out, stage, - master_port, + self._master_port, self.tensor_output_channel, self.tensor_input_channels, - ), - nprocs=self.world_size, - join=False, - ) + hccl_port, + ) + if hccl_port is not None: + self.ctx = _spawn_hccl_group(self.world_size, arguments) + else: + self.ctx = mp.spawn(_worker_loop, args=arguments, nprocs=self.world_size, join=False) + except BaseException: + self.close() + raise + + def _release_ports(self) -> None: + with self._lifecycle_lock: + if self._hccl_lock is not None: + self._hccl_lock.close() + self._hccl_lock = None + if self._master_port is not None: + PortAllocator.release_port(self._master_port) + self._master_port = None @property def failed(self) -> bool: @@ -280,7 +349,7 @@ def _ensure_usable(self) -> None: raise RuntimeError(f"ParallelWorker:{self.name} has failed: {self._failure_reason}") def _terminate_processes(self) -> None: - if not hasattr(self, "ctx"): + if self.ctx is None: return for process in self.ctx.processes: try: @@ -298,6 +367,7 @@ def _mark_failed(self, reason: str) -> None: self._failure_reason = reason logger.error(f"ParallelWorker:{self.name} marked failed: {reason}") self._terminate_processes() + self._release_ports() def _wait_result(self, method_name: str) -> Any: try: @@ -348,7 +418,7 @@ def put_data(self, data: Any) -> None: """Send data to all worker processes.""" self._ensure_usable() if self.queue_with_cpu: - data = to_device(data, "cpu") + data = to_device(data, "cpu", preserve_tuples=requires_host_tensor_queue(self.device)) for q in self.queue_in: q.put(data) @@ -420,7 +490,7 @@ def close(self) -> None: return self._closed = True - if hasattr(self, "ctx"): + if self.ctx is not None: if not self._failed: for queue in self.queue_in: try: @@ -435,15 +505,16 @@ def close(self) -> None: process.join(timeout=2) except Exception as exc: logger.warning(f"Failed to join {self.name} process: {exc}") - for queue in self.queue_in: - try: - queue.close() - except Exception: - pass + for queue in self.queue_in: try: - self.queue_out.close() + queue.close() except Exception: pass + try: + self.queue_out.close() + except Exception: + pass + self._release_ports() def __del__(self) -> None: """Best-effort fallback; callers should use close explicitly.""" diff --git a/telefuser/worker/tensor_channel.py b/telefuser/worker/tensor_channel.py index 7243e58f..2370123f 100644 --- a/telefuser/worker/tensor_channel.py +++ b/telefuser/worker/tensor_channel.py @@ -13,6 +13,8 @@ import torch import torch.multiprocessing as mp +from telefuser.platforms import requires_host_tensor_queue + _MAX_CUDA_IPC_POOL_PROFILES = 8 @@ -75,7 +77,7 @@ class WorkerTensorChannel: """Point-to-point tensor path that bypasses the parent process. CUDA tensors use persistent IPC pools whose rank-local metadata travels in - :class:`WorkerTensorRef`. CPU tensors and dynamic-profile CUDA fallbacks use + :class:`WorkerTensorRef`. Ascend tensors are staged on CPU. Host tensors and dynamic-profile CUDA fallbacks use one multiprocessing queue per consumer rank. The parent never materializes tensor contents. """ @@ -166,23 +168,15 @@ def replace(item: Any) -> Any: shard_dim=shard_dim, ) tensor_index += 1 - if item.device.type == "cuda": - rank_payloads = self._stage_cuda_tensor(ref, item) - if rank_payloads is not None: - ref = dataclass_replace(ref, _cuda_payloads=rank_payloads) - else: - rank_tensors = ( - (item,) * self.consumer_world_size - if shard_dim is None - else torch.tensor_split(item, self.consumer_world_size, dim=shard_dim) - ) - for queue, rank_tensor in zip(self._queues, rank_tensors): - queue.put((ref, rank_tensor)) + rank_payloads = self._stage_cuda_tensor(ref, item) if item.device.type == "cuda" else None + if rank_payloads is not None: + ref = dataclass_replace(ref, _cuda_payloads=rank_payloads) else: + payload = item.cpu() if requires_host_tensor_queue(item.device.type) else item rank_tensors = ( - (item,) * self.consumer_world_size + (payload,) * self.consumer_world_size if shard_dim is None - else torch.tensor_split(item, self.consumer_world_size, dim=shard_dim) + else torch.tensor_split(payload, self.consumer_world_size, dim=shard_dim) ) for queue, rank_tensor in zip(self._queues, rank_tensors): queue.put((ref, rank_tensor)) @@ -231,7 +225,9 @@ def replace(item: Any) -> Any: raise RuntimeError( f"Tensor channel {self.channel_id} received incompatible tensor metadata for {item}" ) - if not isinstance(received, _CudaIpcPayload) and str(tensor.device) != item.source_device: + source_type = item.source_device.partition(":")[0] + queue_device = "cpu" if requires_host_tensor_queue(source_type) else item.source_device + if not isinstance(received, _CudaIpcPayload) and str(tensor.device) != queue_device: raise RuntimeError( f"Tensor channel {self.channel_id} received incompatible tensor metadata for {item}" ) diff --git a/tests/conftest.py b/tests/conftest.py index 4c56747b..7c1e8bb9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,7 +12,7 @@ # Collection Ignore - Skip modules that require unavailable dependencies # ============================================================================ -# Check if triton is available (required by telefuser.distributed and telefuser.kernel.triton) +# Only skip suites that import Triton-dependent operations. try: import triton # noqa: F401 @@ -21,10 +21,8 @@ HAS_TRITON = False TRITON_REQUIRED_TESTS = [ - # Distributed tests require triton - "unit/distributed/test_device_mesh.py", + # Distributed attention and sharding tests "unit/distributed/test_parallel_shard.py", - "unit/distributed/test_pp_comm.py", "unit/distributed/test_ulysses_comm.py", # Kernel tests require triton "unit/kernel/test_rmsnorm.py", diff --git a/tests/unit/distributed/test_pp_comm.py b/tests/unit/distributed/test_pp_comm.py index f247d001..99b52a7d 100644 --- a/tests/unit/distributed/test_pp_comm.py +++ b/tests/unit/distributed/test_pp_comm.py @@ -7,10 +7,14 @@ import torch.distributed as dist from telefuser.distributed.pp_comm import PipelineP2PComm +from telefuser.platforms import current_platform # Skip if distributed not available HAS_DISTRIBUTED = dist.is_available() +# Buffers follow the detected platform so the suite runs on CUDA, NPU, and CPU hosts alike. +DEVICE = current_platform.device_type + pytestmark = [ pytest.mark.skipif(not HAS_DISTRIBUTED, reason="Distributed not available"), pytest.mark.distributed, @@ -56,7 +60,7 @@ def test_send_on_last_stage_logs_warning(self): """Test that send on last stage logs warning and returns None.""" comm = PipelineP2PComm(None) # Single GPU, is_last_stage=True - tensor = torch.randn(1, 10, 512, device="cuda") + tensor = torch.randn(1, 10, 512, device=DEVICE) result = comm.send(tensor) assert result is None @@ -72,7 +76,7 @@ def test_send_recv_single_gpu(self): """Test send_recv on single GPU (no-op).""" comm = PipelineP2PComm(None) # Single GPU - send_tensor = torch.randn(1, 10, 512, device="cuda") + send_tensor = torch.randn(1, 10, 512, device=DEVICE) result = comm.send_recv(send_tensor) # On single GPU, recv_buffer is None since there's no previous stage @@ -114,7 +118,7 @@ def test_queue_send_on_last_stage(self): """Test queue_send on last stage does nothing.""" comm = PipelineP2PComm(None) # is_last_stage=True - tensor = torch.randn(1, 10, 512, device="cuda") + tensor = torch.randn(1, 10, 512, device=DEVICE) comm.queue_send(tensor) assert len(comm._ops) == 0 @@ -123,7 +127,7 @@ def test_queue_recv_on_first_stage(self): """Test queue_recv on first stage does nothing.""" comm = PipelineP2PComm(None) # is_first_stage=True - buffer = torch.randn(1, 10, 512, device="cuda") + buffer = torch.randn(1, 10, 512, device=DEVICE) comm.queue_recv(buffer) assert len(comm._ops) == 0 @@ -152,7 +156,7 @@ def test_send_latent_on_last_stage(self): """Test send_latent on last stage returns early.""" comm = PipelineP2PComm(None) # is_last_stage=True - tensor = torch.randn(1, 10, 512, device="cuda") + tensor = torch.randn(1, 10, 512, device=DEVICE) # Should not raise, just return comm.send_latent(tensor) @@ -182,7 +186,7 @@ def test_send_latent_async_on_last_stage(self): """Test send_latent_async on last stage returns None.""" comm = PipelineP2PComm(None) # is_last_stage=True - tensor = torch.randn(1, 10, 512, device="cuda") + tensor = torch.randn(1, 10, 512, device=DEVICE) result = comm.send_latent_async(tensor) assert result is None @@ -248,7 +252,7 @@ def test_actual_p2p_communication(self): if comm.is_first_stage: # Send tensor to next stage - send_tensor = torch.ones(1, 10, 512, device="cuda") * comm.rank + send_tensor = torch.ones(1, 10, 512, device=DEVICE) * comm.rank comm.send_latent(send_tensor) elif comm.is_last_stage: # Receive tensor from previous stage diff --git a/tests/unit/kernel/test_minimax_h3_ulysses_kernels.py b/tests/unit/kernel/test_minimax_h3_ulysses_kernels.py index 6046350d..5650206b 100644 --- a/tests/unit/kernel/test_minimax_h3_ulysses_kernels.py +++ b/tests/unit/kernel/test_minimax_h3_ulysses_kernels.py @@ -1,13 +1,19 @@ import pytest import torch -from telefuser.kernel.triton.ulysses_relayout import ( - merge_ulysses_head_chunk, - pack_qkv_destination_major, - pack_qkv_qknorm_rope_destination_major, -) from telefuser.ops.rotary import apply_qk_norm_rope_neox +try: + from telefuser.kernel.triton.ulysses_relayout import ( + merge_ulysses_head_chunk, + pack_qkv_destination_major, + pack_qkv_qknorm_rope_destination_major, + ) +except ModuleNotFoundError as error: + if error.name != "triton": + raise + pytest.skip("Ulysses CUDA kernels require Triton", allow_module_level=True) + pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA Triton kernels") diff --git a/tests/unit/models/test_wan22_video_vae.py b/tests/unit/models/test_wan22_video_vae.py index 3e2b1d7b..9a5ec988 100644 --- a/tests/unit/models/test_wan22_video_vae.py +++ b/tests/unit/models/test_wan22_video_vae.py @@ -2,12 +2,30 @@ from types import SimpleNamespace +import pytest import torch import torch.nn as nn from telefuser.models import wan22_video_vae +@pytest.mark.parametrize("tiled, expected_method", [(False, "2d_split"), (True, "tile_dist")]) +def test_parallel_decode_accepts_batched_latents_and_lists(monkeypatch, tiled, expected_method) -> None: + monkeypatch.setattr(torch.distributed, "is_initialized", lambda: True) + device = torch.device("cpu") + batch = torch.arange(48).reshape(2, 3, 2, 2, 2) + + def decode_parallel(latents, target_device, method): + assert target_device == device + assert method == expected_method + return latents + + vae = SimpleNamespace(parallelism=2, decode_parallel=decode_parallel) + decode = wan22_video_vae.Wan22VideoVAE.decode + assert decode(vae, batch, device, tiled=tiled) is batch + torch.testing.assert_close(decode(vae, list(batch.unbind()), device, tiled=tiled), batch) + + class _RecordingDecoder(nn.Module): def __init__(self) -> None: super().__init__() diff --git a/tests/unit/ops/test_fp8_attention_smoothing.py b/tests/unit/ops/test_fp8_attention_smoothing.py index 01e95bdb..5f6d24c5 100644 --- a/tests/unit/ops/test_fp8_attention_smoothing.py +++ b/tests/unit/ops/test_fp8_attention_smoothing.py @@ -8,6 +8,7 @@ apply_fp8_attention_output_correction, dequantize_fp8_per_channel, merge_fp8_attention_prefix, + quantize_fp8_qkv, quantize_fp8_qkv_smoothed, ) @@ -26,6 +27,19 @@ def test_kv_sequence_mean_smoothing_is_attention_equivalent() -> None: torch.testing.assert_close(smoothed, reference, atol=2e-6, rtol=2e-6) +def test_eager_cpu_helpers_work_without_triton() -> None: + q = torch.randn(1, 5, 2, 8) + with pytest.raises(ValueError, match="contiguous CUDA tensors"): + quantize_fp8_qkv(q, q, q) + *_, correction = quantize_fp8_qkv_smoothed(q, q, q, smoothing="kv", correct_v_bias=False) + torch.testing.assert_close(correction, q.mean(dim=1, keepdim=True)) + corrected = apply_fp8_attention_output_correction(q, correction) + torch.testing.assert_close(corrected, q + correction) + torch.testing.assert_close( + merge_fp8_attention_prefix(q[:, :2], q, correction), torch.cat([q[:, :2], corrected[:, 2:]], dim=1) + ) + + @pytest.mark.gpu def test_fp8_value_bias_correction_restores_original_sequence_mean_on_h100() -> None: if not torch.cuda.is_available() or torch.cuda.get_device_capability() != (9, 0): diff --git a/tests/unit/platforms/test_ascend.py b/tests/unit/platforms/test_ascend.py new file mode 100644 index 00000000..89d8fc0e --- /dev/null +++ b/tests/unit/platforms/test_ascend.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +import os +import socket +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest +import torch + +from telefuser import platforms +from telefuser.platforms.hccl import reserve_hccl_ports + + +@pytest.mark.parametrize(("given", "expected"), [("npu", "npu:0"), ("npu:07", "npu:7"), ("rocm:1", "cuda:1")]) +def test_normalize_without_vendor_module(given: str, expected: str) -> None: + assert platforms.normalize_device_name(given) == expected + + +def test_explicit_devices_fail_before_starting_workers(monkeypatch) -> None: + with pytest.raises(ValueError): + platforms.normalize_device_name("npu:-1") + monkeypatch.setitem(sys.modules, "torch_npu", None) + with pytest.raises(RuntimeError, match="torch_npu") as failure: + platforms.resolve_device("npu:1") + assert isinstance(failure.value.__cause__, ImportError) + monkeypatch.setitem(sys.modules, "torch_npu", SimpleNamespace()) + monkeypatch.setattr(platforms, "current_platform", platforms.CpuPlatform()) + with pytest.raises(RuntimeError, match="detected cpu"): + platforms.resolve_device("npu:1") + monkeypatch.setattr( + platforms, "current_platform", SimpleNamespace(device_type="npu", device_name="npu", device_count=lambda: 2) + ) + with pytest.raises(ValueError, match="2 visible"): + platforms.resolve_device("npu:2") + assert platforms.resolve_device("cpu") == torch.device("cpu") + + +@pytest.mark.parametrize("value", ["0", "65521", "bad"]) +def test_invalid_hccl_base(monkeypatch, value: str) -> None: + monkeypatch.setenv("HCCL_IF_BASE_PORT", value) + with pytest.raises(ValueError, match="HCCL_IF_BASE_PORT"): + reserve_hccl_ports() + + +def test_hccl_lease_across_parents_and_shutdown(tmp_path: Path, monkeypatch) -> None: + monkeypatch.setattr("tempfile.tempdir", str(tmp_path)) + monkeypatch.setenv("HCCL_IF_BASE_PORT", "61001") + base, lock = reserve_hccl_ports() + with lock: + assert base % 16 == 0 and base >= 61001 + code = "from telefuser.platforms.hccl import reserve_hccl_ports; base, lock = reserve_hccl_ports(); print(base)" + other = subprocess.check_output( + [sys.executable, "-c", code], text=True, env={**os.environ, "TMPDIR": str(tmp_path)} + ) + assert int(other.strip()) >= base + 16 + for _ in range(8): + recycled, lock = reserve_hccl_ports() + with lock: + assert recycled == base + with socket.socket() as occupied: + occupied.bind(("", base + 7)) + different, lock = reserve_hccl_ports() + with lock: + assert different >= base + 16 diff --git a/tests/unit/worker/test_parallel_worker_lifecycle.py b/tests/unit/worker/test_parallel_worker_lifecycle.py index 284b90eb..1663d044 100644 --- a/tests/unit/worker/test_parallel_worker_lifecycle.py +++ b/tests/unit/worker/test_parallel_worker_lifecycle.py @@ -99,3 +99,26 @@ def test_close_after_failure_does_not_submit_more_worker_commands() -> None: for queue in worker.queue_in: queue.put.assert_not_called() queue.close.assert_called_once_with() + + +def test_hccl_spawn_failure_returns_lease_and_closes_queues(monkeypatch) -> None: + from telefuser.worker import parallel_worker as module + + lock = MagicMock() + mp = MagicMock() + first_rank = MagicMock() + mp.spawn.side_effect = [SimpleNamespace(processes=[first_rank]), RuntimeError("failed spawn")] + stage = MagicMock() + config = stage.model_runtime_config.parallel_config + config.world_size, config.device_ids = 2, [1, 2] + monkeypatch.setattr(module, "current_platform", SimpleNamespace(device_type="npu", dist_backend="hccl")) + monkeypatch.setattr(module, "resolve_device", MagicMock()) + monkeypatch.setattr(module, "reserve_hccl_ports", lambda: (60000, lock)) + monkeypatch.setattr(module, "mp", mp) + with pytest.raises(RuntimeError, match="failed spawn"): + ParallelWorker(stage) + assert mp.spawn.call_args.kwargs["args"][-1] == 60000 + first_rank.kill.assert_called_once_with() + first_rank.join.assert_called_once_with() + lock.close.assert_called_once() + assert mp.get_context.return_value.SimpleQueue.return_value.close.call_count == 3 diff --git a/tests/unit/worker/test_tensor_channel.py b/tests/unit/worker/test_tensor_channel.py index 510ff5e8..9568324a 100644 --- a/tests/unit/worker/test_tensor_channel.py +++ b/tests/unit/worker/test_tensor_channel.py @@ -299,3 +299,30 @@ def test_tensor_channel_uses_cuda_ipc_and_cross_gpu_peer_copy() -> None: metadata_queue.close() result_queue.close() channel.close() + + +def test_ascend_payload_stages_once_before_sharding_and_keeps_source_metadata() -> None: + class NpuDevice: + type = "npu" + + def __str__(self): + return "npu:1" + + class NpuTensor(torch.Tensor): + device = NpuDevice() + copies = 0 + + def cpu(self): + self.copies += 1 + return self.as_subclass(torch.Tensor) + + channel = WorkerTensorChannel(consumer_world_size=2, shard_dim=0, timeout=1) + original = torch.arange(5) + source = original.as_subclass(NpuTensor) + try: + ref, duplicate = channel.send((source, [source])) + assert ref is duplicate[0] and ref.source_device == "npu:1" and source.copies == 1 + parts = [channel.receive(ref, rank=i, device="cpu") for i in range(2)] + torch.testing.assert_close(torch.cat(parts), original) + finally: + channel.close()