feat(router): route Anthropic Messages through virtual models #10702

Workflow file for this run

.github/workflows/python.yaml at e59d7c3

	name: Python CI

	on:
	push:
	branches:
	- main
	pull_request:
	types: ['opened', 'reopened', 'synchronize', 'ready_for_review']

	concurrency:
	group: ${{ github.workflow }}-${{ github.ref }}
	cancel-in-progress: true

	jobs:
	lint:
	# A push event cannot tell whether its branch belongs to a draft PR, so
	# branch CI is driven by pull_request events and push is limited to main.
	if: ${{ github.event_name != 'pull_request' \|\| github.event.pull_request.draft == false }}
	runs-on: ${{ matrix.os }}
	strategy:
	fail-fast: false
	matrix:
	os: [ "ubuntu-latest" ]
	python-version: [ "3.10" ]
	steps:
	- name: Check out code
	uses: actions/checkout@v3
	with:
	fetch-depth: 0
	submodules: recursive
	- name: Set up Python environment
	uses: actions/setup-python@v4
	with:
	python-version: "3.10"
	- name: Install pre-commit
	run: pip install pre-commit
	- name: Run pre-commit
	run: pre-commit run --all-files
	- name: Set up Node.js
	uses: actions/setup-node@v1
	with:
	node-version: 20.19.0
	# ESLint and Prettier must be in `package.json`
	- name: Install Node.js dependencies
	run: cd frontend && npm ci
	- name: ESLint Check
	run: cd frontend && npx eslint .
	- name: Build frontend (static export)
	run: cd frontend && npm run build
	- name: Verify static export output
	run: \|
	test -f frontend/out/index.html
	test -f frontend/out/404.html
	# Dynamic routes must emit __shell__ placeholder pages.
	find frontend/out -name '__shell__*' \| grep -q .
	- name: Serve static export from backend
	run: \|
	pip install fastapi httpx pytest xoscar
	pytest --noconftest -vv \
	xinference/api/tests/test_frontend_static.py \
	xinference/api/tests/test_frontend_static_real_export.py


	changes:
	if: ${{ github.event_name != 'pull_request' \|\| github.event.pull_request.draft == false }}
	runs-on: ubuntu-latest
	outputs:
	gpu: ${{ steps.gpu.outputs.gpu }}
	groups: ${{ steps.gpu.outputs.groups }}
	steps:
	- name: Check out code
	uses: actions/checkout@v3
	with:
	fetch-depth: 0
	submodules: recursive

	- name: Detect GPU CI changes
	id: gpu
	shell: bash
	run: \|
	set -uo pipefail

	if [ "${{ github.event_name }}" = "pull_request" ]; then
	base_sha="${{ github.event.pull_request.base.sha }}"
	head_sha="${{ github.event.pull_request.head.sha }}"
	# Match GitHub's PR diff: exclude commits added to the base branch
	# after the PR branch diverged.
	diff_range="$base_sha...$head_sha"
	else
	base_sha="${{ github.event.before }}"
	head_sha="${{ github.sha }}"
	diff_range="$base_sha..$head_sha"
	fi

	full=false
	llm=false; embedding=false; image=false; audio=false

	# Fall back to a full run when there is no usable diff base (new
	# branch, or a force push whose previous head no longer exists).
	if [ -z "$base_sha" ] \|\| [[ "$base_sha" =~ ^0+$ ]] \
	\|\| ! git diff --name-only "$diff_range" > changed-files.txt 2>/dev/null; then
	full=true
	else
	# Map each changed path to the GPU test group(s) it affects.
	# Paths matching no pattern (docs, examples, web UI, other
	# workflows, ...) do not need GPU CI at all.
	while IFS= read -r file; do
	case "$file" in
	xinference/model/audio/*) audio=true ;;
	xinference/model/image/*) image=true ;;
	xinference/model/embedding/\|xinference/model/rerank/) embedding=true ;;
	xinference/model/llm/\|xinference/model/scheduler/\|xinference/core/tests/test_continuous_batching.py) llm=true ;;
	# Vendored model code is audio-only today (cosyvoice, f5_tts,
	# matcha, fish_speech, ...).
	xinference/thirdparty/*) audio=true ;;
	# The web UI never affects GPU tests.
	xinference/ui/*) ;;
	# Tests under the shared runtime packages exercise CPU logic
	# that the ubuntu/macos/windows CI already covers; editing a
	# test file does not change the GPU launch/inference path, so
	# it needs no GPU CI. (test_continuous_batching is matched by
	# the llm rule above and still triggers the llm group.)
	xinference/core/tests/\|xinference/api/tests/\|xinference/client/tests/*) ;;
	# OAuth2 / auth and the admin & launch-history routers are pure
	# HTTP/auth logic with no model inference path; the non-GPU CI
	# covers them. Keep them out of GPU CI entirely.
	xinference/api/oauth2/*\|xinference/api/routers/admin.py\|xinference/api/routers/launch_history.py) ;;
	# Per-family HTTP routers map to a single GPU group instead of
	# forcing a full run.
	xinference/api/routers/llm.py) llm=true ;;
	xinference/api/routers/embeddings.py\|xinference/api/routers/rerank.py) embedding=true ;;
	xinference/api/routers/images.py) image=true ;;
	xinference/api/routers/audio.py) audio=true ;;
	# Every GPU test launches models through a real RESTful API
	# server subprocess and the xinference/client/ package (see
	# conftest.py's `setup` fixture), so changes to either affect
	# every family. The same fixture starts the test cluster through
	# xinference/deploy/, and xinference/core/ is the actor runtime
	# shared by every model family. Any of these affects every group.
	# This also covers restful_api.py (the launch/inference
	# endpoints), dependencies.py, and the models/videos routers.
	xinference/core/\|xinference/api/\|xinference/client/\|xinference/deploy/) full=true; break ;;
	# video/ and flexible/ have no GPU test group at all: no
	# group below runs a single test from either package, so a full
	# run costs every GPU minute while covering none of the change.
	# Their tests (video/tests/, flexible/tests/) run in the
	# ubuntu/macos/windows CI, which is where they are covered.
	xinference/model/video/\|xinference/model/flexible/) ;;
	# Remaining model families with no dedicated group, plus
	# top-level files directly under xinference/model/ (batch.py,
	# cache_manager.py, core.py, custom.py, utils.py) which are
	# shared by every family; play it safe and run every group.
	xinference/model/*) full=true; break ;;
	# The rest of the package (deploy/, docs, examples, other
	# top-level modules) is not exercised by any GPU test.
	.github/workflows/python.yaml\|pyproject.toml\|build_backend.py\|build_web.py) full=true; break ;;
	esac
	done < changed-files.txt
	fi

	groups=""
	if [ "$full" = true ]; then
	groups="llm embedding image audio"
	else
	[ "$llm" = true ] && groups="$groups llm"
	[ "$embedding" = true ] && groups="$groups embedding"
	[ "$image" = true ] && groups="$groups image"
	[ "$audio" = true ] && groups="$groups audio"
	groups="${groups# }"
	fi

	gpu=true
	[ -z "$groups" ] && gpu=false
	echo "gpu=$gpu" >> "$GITHUB_OUTPUT"
	echo "groups=$groups" >> "$GITHUB_OUTPUT"
	echo "GPU CI required: $gpu (groups: ${groups:-none})"

	build_test_job:
	runs-on: ${{ matrix.os }}
	needs: lint
	env:
	CONDA_ENV: test
	defaults:
	run:
	shell: bash -l {0}
	strategy:
	fail-fast: false
	matrix:
	os: [ "ubuntu-latest", "macos-latest", "windows-latest" ]
	python-version: [ "3.10", "3.11", "3.12", "3.13" ]
	module: [ "xinference" ]
	exclude:
	- { os: macos-latest, python-version: 3.11 }
	- { os: macos-latest, python-version: 3.12 }
	- { os: windows-latest, python-version: 3.11 }
	- { os: windows-latest, python-version: 3.12 }
	include:
	- { os: macos-latest, module: metal, python-version: "3.10" }

	steps:
	- name: Check out code
	uses: actions/checkout@v3
	with:
	fetch-depth: 0
	submodules: recursive

	- name: Set up conda ${{ matrix.python-version }}
	uses: conda-incubator/setup-miniconda@v3
	with:
	python-version: ${{ matrix.python-version }}
	activate-environment: ${{ env.CONDA_ENV }}

	# Important for python == 3.12 and 3.13
	- name: Update pip and setuptools
	if: ${{ matrix.python-version == '3.12' \|\| matrix.python-version == '3.13' }}
	run: \|
	python -m pip install -U pip "setuptools<82"

	# Install torch for Python 3.13 using nightly builds
	- name: Install torch for Python 3.13
	if: ${{ matrix.python-version == '3.13'}}
	run: \|
	python -m pip install torch torchvision torchaudio

	- name: Install numpy
	if: \|
	(startsWith(matrix.os, 'macos') && (matrix.python-version == '3.13')) \|\|
	(startsWith(matrix.os, 'windows'))
	run: \|
	python -m pip install "numpy<2"


	- name: Install dependencies
	env:
	MODULE: ${{ matrix.module }}
	OS: ${{ matrix.os }}
	# Tests never touch the web UI; skip the npm build that the build
	# backend would otherwise run on editable installs.
	NO_WEB_UI: 1
	run: \|
	if [ "$OS" == "ubuntu-latest" ]; then
	sudo rm -rf /usr/share/dotnet
	sudo rm -rf /opt/ghc
	sudo rm -rf "/usr/local/share/boost"
	sudo rm -rf "$AGENT_TOOLSDIRECTORY"
	fi
	pip install -e ".[dev]"
	pip install "xllamacpp>=0.2.0"
	if [ "$MODULE" == "metal" ]; then
	conda install -c conda-forge "ffmpeg<7"
	pip install "mlx>=0.22.0"
	pip install mlx-lm
	pip install "mlx-vlm>=0.3.4"
	pip install mlx-whisper
	pip install f5-tts-mlx
	pip install qwen-vl-utils!=0.0.9
	pip install tomli
	else
	pip install "transformers<4.49"
	pip install attrdict
	pip install "timm>=0.9.16"
	if [ "${{ matrix.python-version }}" != "3.13" ]; then
	pip install torch torchvision
	fi
	pip install accelerate
	pip install sentencepiece
	pip install transformers_stream_generator
	pip install bitsandbytes
	pip install "sentence-transformers>=5.1.1,<6"
	pip install modelscope
	pip install diffusers
	pip install protobuf
	pip install "FlagEmbedding==1.3.5"
	pip install "tenacity>=8.2.0,<8.4.0"
	pip install "jinja2==3.1.2"
	pip install jj-pytorchvideo
	pip install qwen-vl-utils!=0.0.9
	pip install datamodel_code_generator
	pip install jsonschema
	fi
	working-directory: .

	- name: Clean up disk
	if: \|
	(startsWith(matrix.os, 'ubuntu'))
	run: \|
	sudo rm -rf /usr/share/dotnet
	sudo rm -rf /usr/local/lib/android
	sudo rm -rf /opt/ghc
	sudo apt-get clean
	sudo rm -rf /var/lib/apt/lists/*
	df -h

	- name: Fix SSL on Windows
	if: startsWith(matrix.os, 'windows')
	shell: bash
	run: \|
	echo "activate conda env"

	source $CONDA/etc/profile.d/conda.sh \|\| true
	conda activate $CONDA_ENV \|\| true

	python -V
	which python

	echo "before: $SSL_CERT_FILE"

	python -m pip install --quiet certifi \|\| true

	SSL_CERT_FILE=$(python -c "import certifi,os;print(os.path.normpath(certifi.where()))")

	export SSL_CERT_FILE
	export REQUESTS_CA_BUNDLE=$SSL_CERT_FILE
	export CURL_CA_BUNDLE=$SSL_CERT_FILE

	echo "after: $SSL_CERT_FILE"
	echo "SSL_CERT_FILE=$(python -c 'import certifi;print(certifi.where())')" >> $GITHUB_ENV

	- name: Test with pytest
	env:
	MODULE: ${{ matrix.module }}
	PYTORCH_MPS_HIGH_WATERMARK_RATIO: 1.0
	PYTORCH_MPS_LOW_WATERMARK_RATIO: 0.2
	XFORMERS_FORCE_DISABLE_TRITON: 1
	TORCH_DISABLE_FLASH_ATTENTION: 1
	run: \|
	set -e

	run_pytest() {
	pytest --timeout=3000 \
	-W ignore::PendingDeprecationWarning \
	--cov-config=pyproject.toml \
	--cov-report=xml \
	--cov=xinference \
	"$@"
	}

	if [ "$MODULE" == "metal" ]; then
	run_pytest xinference/model/llm/mlx/tests/test_mlx.py && \
	run_pytest xinference/model/audio/tests/test_whisper_mlx.py && \
	run_pytest xinference/model/audio/tests/test_f5tts_mlx.py && \
	run_pytest xinference/model/llm/mlx/tests/test_distributed_model.py
	else
	run_pytest \
	-vv \
	--ignore xinference/core/tests/test_continuous_batching.py \
	--ignore xinference/model/image/tests/test_stable_diffusion.py \
	--ignore xinference/model/image/tests/test_got_ocr2.py \
	--ignore xinference/model/audio/tests \
	--ignore xinference/model/embedding/tests/test_integrated_embedding.py \
	--ignore xinference/model/llm/transformers/tests/test_tensorizer.py \
	--ignore xinference/model/llm/tests/test_llm_model.py \
	--ignore xinference/model/llm/vllm \
	--ignore xinference/model/llm/sglang \
	--ignore xinference/client/tests/test_client.py \
	--ignore xinference/client/tests/test_async_client.py \
	--ignore xinference/model/llm/mlx \
	xinference
	fi
	working-directory: .

	gpu_test_job:
	name: build_test_job (gpu-t4, gpu, 3.12)
	runs-on: gpu-t4
	needs: [lint, changes]
	# Runners are billed per-minute: skip when no GPU-relevant path changed,
	# and only run for PRs and pushes to main (a push to a PR branch already
	# triggers the pull_request run).
	if: ${{ needs.changes.outputs.gpu == 'true' && (github.event_name == 'pull_request' \|\| github.ref == 'refs/heads/main') }}
	timeout-minutes: 90
	env:
	CONDA_ENV: test
	# GPU tests never touch the web UI; skipping the npm build saves
	# minutes. The API server only warns when the UI bundle is absent.
	NO_WEB_UI: 1
	defaults:
	run:
	shell: bash -l {0}
	steps:
	- name: Check out code
	uses: actions/checkout@v3
	with:
	fetch-depth: 0
	submodules: recursive

	- name: Set up conda 3.12
	uses: conda-incubator/setup-miniconda@v3
	with:
	python-version: "3.12"
	activate-environment: ${{ env.CONDA_ENV }}

	- name: Restore pip package cache
	uses: actions/cache@v4
	with:
	path: ~/.cache/pip
	key: gpu-pip-${{ hashFiles('.github/workflows/python.yaml', 'pyproject.toml') }}
	restore-keys: gpu-pip-

	# funasr downloads its auxiliary models (fsmn-vad, ct-punc, cam++) from
	# ModelScope, whose CDN is very slow from GitHub's US runners (~15min
	# per run). They are only ~1.5GB in total, so cache them. Main models
	# come from HuggingFace (fast) and are intentionally not cached — all
	# of them together would blow the 10GB per-repo cache limit.
	# NOTE: xinference redirects MODELSCOPE_CACHE to ~/.xinference/modelscope
	# (see get_xinference_home in xinference/constants.py), so that is the
	# directory to cache — ~/.cache/modelscope is never created.
	- name: Restore ModelScope model cache
	uses: actions/cache@v4
	with:
	path: ~/.xinference/modelscope
	key: gpu-modelscope-${{ hashFiles('xinference/model/audio/model_spec.json') }}
	restore-keys: gpu-modelscope-

	- name: Update pip and setuptools
	run: \|
	python -m pip install -U pip "setuptools<82"

	- name: Install dependencies
	run: \|
	pip install -U -e ".[audio,dev]"
	pip install -U "openai>1"
	pip install -U modelscope
	pip install -U gguf
	pip install -U uv
	pip install -U sse_starlette
	pip install -U xoscar
	pip install -U "python-jose[cryptography]"
	pip install -U "passlib[bcrypt]"
	pip install -U "aioprometheus[starlette]"
	pip install -U "pynvml"
	pip install "transformers==4.53.2"
	pip install "funasr==1.2.7"
	conda install -y -c conda-forge "pynini==2.1.5" "ffmpeg<7"
	pip install -U "nemo_text_processing<1.1.0"
	pip install -U omegaconf~=2.3.0
	pip install -U "WeTextProcessing<1.0.4"
	pip install -U librosa
	pip install -U xxhash
	pip install -U "ChatTTS==0.2.4"
	pip install -U HyperPyYAML
	pip uninstall -y matcha-tts
	pip install -U 'onnxruntime-gpu==1.22.0; sys_platform == "linux"'
	pip install -U openai-whisper
	pip install -U "torch==2.7.0" "torchaudio==2.7.0" "torchvision==0.22.0"
	pip install -U "loguru"
	pip install -U "natsort"
	pip install -U "loralib"
	pip install -U "ormsgpack"
	pip uninstall -y opencc
	pip uninstall -y "faster_whisper"
	pip install -U accelerate
	pip install -U verovio
	pip install -U cachetools
	pip install -U silero-vad
	pip install -U pydantic
	pip install -U "diffusers==0.35.1"
	pip install -U onnx
	pip install -U onnxconverter_common
	pip install -U torchdiffeq
	pip install -U "x_transformers>=1.31.14"
	pip install -U pypinyin
	pip install -U tomli
	pip install -U vocos
	pip install -U jieba
	pip install -U soundfile
	pip install tensorizer
	pip install -U "sentence-transformers==5.1.1"
	pip install -U "FlagEmbedding==1.3.5"
	pip install -U "peft>=0.18.0"
	pip install "xllamacpp>=0.2.0" --index-url https://xorbitsai.github.io/xllamacpp/whl/cu124 --extra-index-url https://pypi.org/simple
	pip install hf_transfer
	# espeak-ng (TTS phonemization) is not packaged on conda-forge;
	# install it from apt.
	sudo apt-get update -qq
	sudo apt-get install -y -qq espeak-ng
	# Final guard pins. The serial `pip install -U` calls above ignore
	# the upper bounds of already-installed packages, letting shared
	# deps drift past what the stack supports at runtime: numba needs
	# numpy<2.5, and diffusers 0.35 / tensorizer still import
	# is_offline_mode, which huggingface_hub 1.0 removed
	# (transformers==4.53.2 declares the same <1.0 cap).
	pip install "numpy>=2,<2.5" "huggingface_hub<1.0"
	working-directory: .

	- name: Test with pytest
	env:
	XFORMERS_FORCE_DISABLE_TRITON: 1
	TORCH_DISABLE_FLASH_ATTENTION: 1
	# Speed up model downloads from the HuggingFace hub; runners are
	# ephemeral so every run downloads the main models from scratch.
	HF_HUB_ENABLE_HF_TRANSFER: 1
	# Pin the model source instead of relying on locale auto-detection;
	# ModelScope is very slow from US-hosted runners.
	XINFERENCE_MODEL_SRC: huggingface
	run: \|
	# One pytest process per selected group: one cold start per family
	# instead of one per file, while an import error or crash in one
	# family cannot take down the others. No --cov: nothing in this
	# repo consumes the coverage report. A failing group does not stop
	# the remaining groups, so one run reports every failure.
	#
	# On a failure, retry only the failed tests once via --last-failed.
	# test_continuous_batching hits an intermittent KV-cache batching
	# bug (~1 in 4 runs, tracked separately) that is a same-process
	# race, not environment flakiness; each pytest invocation still
	# gets fresh model/server subprocesses from the `setup` fixture, so
	# a rerun is not just re-hitting stale state.
	status=0
	run_pytest() {
	local rc=0
	pytest --timeout=3000 \
	-W ignore::PendingDeprecationWarning \
	"$@" \|\| rc=$?
	if [ "$rc" -ne 0 ]; then
	echo "::warning::pytest exited with code $rc for group '$group'; retrying failed tests once"
	rc=0
	pytest --timeout=3000 \
	-W ignore::PendingDeprecationWarning \
	--last-failed \
	"$@" \|\| rc=$?
	fi
	echo "group '$group' pytest exit code: $rc"
	if [ "$rc" -ne 0 ]; then
	echo "::warning::pytest exited with code $rc for group '$group' after retry"
	status=1
	fi
	}

	for group in ${{ needs.changes.outputs.groups }}; do
	echo "::group::GPU tests: $group"
	case "$group" in
	llm)
	run_pytest \
	xinference/core/tests/test_continuous_batching.py \
	xinference/model/llm/tests/test_llm_model.py \
	xinference/model/llm/transformers/tests/test_tensorizer.py
	;;
	embedding)
	# test_integrated_embedding must run first: the other three
	# tests launch models in per-model virtualenvs, which leak
	# their site-packages into the process environment, and models
	# launched afterwards in the same pytest process import
	# packages from the wrong env (seen with both the
	# Qwen3-VL-Embedding and Qwen3-VL-Reranker virtualenvs).
	run_pytest \
	xinference/model/embedding/tests/test_integrated_embedding.py \
	xinference/model/embedding/tests/test_qwen3_vl_engine_params.py \
	xinference/model/embedding/vllm/tests/test_vllm_embedding.py \
	xinference/model/rerank/tests/test_qwen3_vl_reranker_virtualenv.py
	;;
	image)
	# test_got_ocr2 is intentionally omitted: it self-skips on
	# transformers >= 4.48.0 and this job pins 4.53.2, so it never
	# runs a single case here.
	run_pytest \
	xinference/model/image/tests/test_stable_diffusion.py
	;;
	audio)
	# test_cosyvoice is intentionally omitted: all of its cases
	# are @pytest.mark.skip'd (diffusers on this image is
	# incompatible), so it never runs anything here.
	run_pytest \
	xinference/model/audio/tests/test_whisper.py \
	xinference/model/audio/tests/test_funasr.py \
	xinference/model/audio/tests/test_chattts.py \
	xinference/model/audio/tests/test_f5tts.py \
	xinference/model/audio/tests/test_melotts.py \
	xinference/model/audio/tests/test_kokoro.py \
	xinference/model/audio/tests/test_fish_speech.py \
	xinference/model/audio/tests/test_megatts.py
	;;
	esac
	echo "::endgroup::"
	done
	echo "aggregate pytest status: $status"
	# Finish with a plain test instead of an explicit `exit`: two runs
	# with every pytest group green still ended in exit code 1 through
	# the login shell's explicit-exit path on this runner image.
	[ "$status" -eq 0 ]
	working-directory: .

Navigation Menu

Uh oh!

feat(router): route Anthropic Messages through virtual models #10702

Workflow file

feat(router): route Anthropic Messages through virtual models #10702

Uh oh!

Workflow file for this run