diff --git a/.github/workflows/local-gateway-build.yml b/.github/workflows/local-gateway-build.yml index ce45d37..c33885a 100644 --- a/.github/workflows/local-gateway-build.yml +++ b/.github/workflows/local-gateway-build.yml @@ -14,7 +14,7 @@ jobs: fail-fast: false matrix: include: - - os: macos-14 + - os: macos-15 target: darwin-arm64 - os: ubuntu-24.04 target: linux-amd64 diff --git a/CHANGELOG.md b/CHANGELOG.md index 9dccdb5..db27322 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,22 @@ All notable changes to this project are listed here. The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and versions follow [Semantic Versioning](https://semver.org/). +## Unreleased + +### Changed + +- `route()` and LiteLLM's `Router` no longer raise when `AGENTROUTER_BASE_URL` is set. They set up nothing on this computer; calls send the route's name to that gateway, whose route of the same name applies, and the SDK warns once. A `Router` sends the group name, such as `chat`. Code tested against the local gateway now runs unchanged against a shared one. +- `local_gateway.export_config()` names each LiteLLM `Router` route after its group, `chat` instead of `router-1a2b3c4d:chat`, so a gateway made from it answers to the names code calls. Two routes that would get the same name raise `BadRequestError`. + +### Fixed + +- LiteLLM's `Router` shares a group's traffic by `weight`, `rpm` or `tpm` in `litellm_params`, as LiteLLM's default `simple-shuffle` does, instead of equally. A deployment without the setting, beside one with it, gets no traffic, as in LiteLLM. Weights that are not whole numbers keep their ratio: `0.01` and `100` become 1 and 10000. +- LiteLLM's `Router` warns about what it does not use instead of dropping it without a word: a `routing_strategy` other than `simple-shuffle` (LiteLLM's usage-based strategies hold deployments to `rpm` and `tpm`), and `litellm_params` such as `api_base` and `timeout`. +- A call with `api_base=` warns once per address that it goes there in place of the gateway, so routes, fallbacks and limits apply only if that address is an Agent Router gateway that has them. +- A LiteLLM `Router` chooses its route name at each call, so one made before `AGENTROUTER_BASE_URL` was set sends the group name to that gateway, and one made with it set sets up its route on this computer once it is unset. Before, it kept the name it was made with and the gateway answered `NotFoundError`. +- When a gateway set with `AGENTROUTER_BASE_URL` answers 403 in plain text, the `PermissionDeniedError` message names `AGENTROUTER_API_KEY` as a possible cause, which an exported gateway needs set to the same value on its server. It says "possible" because the gateway gives the same answer for other access rules, such as an address allowlist. Before, it carried only the gateway's own refusal text. The message no longer starts with the provider's name, since the gateway refused it, not the provider. +- A call to a model without a known price, such as a route name on a gateway you set, reports `response_cost` as `None` and warns nothing, as LiteLLM does. Before, every such call warned "No price is known" and reported $0. `completion_cost()` called directly still warns and returns 0. + ## [0.1.0a11] - 2026-11-03 ### Added diff --git a/README.md b/README.md index c8922d4..c199073 100644 --- a/README.md +++ b/README.md @@ -96,7 +96,7 @@ Here a call to `chat` goes to `openai/gpt-5.4`. If it cannot be reached or answe - A route can also limit requests: `limit=ar.Limit(60, per="minute")` lets 60 requests a minute through and answers the rest with a `RateLimitError`. - Routes are saved in `~/.agentrouter/routes.json`, so every program on this computer can use them. -- Routes work with the local gateway only. With `AGENTROUTER_BASE_URL` set, set up fallbacks on that gateway instead. +- Routes defined in code set up the local gateway. With `AGENTROUTER_BASE_URL` set, the same code runs, and that gateway's route of the same name applies; `local_gateway.export_config()` gives it this computer's routes. - `Router` from `theagentrouter.compat.litellm`, with LiteLLM's `fallbacks` and `num_retries`, is turned into routes. Details: [Routes](docs/usage.md#routes-fallbacks-retries-and-names-of-your-own). diff --git a/apps/litellm-assistant/tests/test_litellm_behaviour.py b/apps/litellm-assistant/tests/test_litellm_behaviour.py index 43a5a99..465c6ea 100644 --- a/apps/litellm-assistant/tests/test_litellm_behaviour.py +++ b/apps/litellm-assistant/tests/test_litellm_behaviour.py @@ -456,6 +456,13 @@ def test_success_callback_runs_after_a_stream(base): assert [type(r).__name__ for r in seen if type(r).__name__ == "ModelResponse"] == ["ModelResponse"] +def test_router_does_not_hold_a_deployment_to_rpm_by_default(): + # LiteLLM's default simple-shuffle uses rpm as a traffic share; only its usage-based strategies limit by it. + router = litellm.Router(model_list=[{"model_name": "chat", "litellm_params": {"model": OPENAI, "rpm": 1}}]) + replies = [router.completion(model="chat", messages=HELLO, mock_response="ok") for _ in range(3)] + assert [r.choices[0].message.content for r in replies] == ["ok"] * 3 + + def test_mock_response(): resp = litellm.completion(model=OPENAI, messages=HELLO, mock_response="Offline reply") assert resp.choices[0].message.content == "Offline reply" diff --git a/docs/coming-from-litellm.md b/docs/coming-from-litellm.md index 84a0d28..453df12 100644 --- a/docs/coming-from-litellm.md +++ b/docs/coming-from-litellm.md @@ -32,7 +32,8 @@ A full example: [`examples/from_litellm/change_the_import.py`](../examples/from_ | `stream=True` | Same chunks, with `usage` on the last one | | `tools`, `tool_choice`, `parallel_tool_calls` | OpenAI-shaped inputs, translated for Anthropic too; `parallel_tool_calls=False` disables parallel Anthropic tool calls | | `response_format`, `temperature`, `max_tokens` and the other OpenAI parameters | Passed through for OpenAI Chat Completions. Unsupported Anthropic options raise `UnsupportedParamsError` ([the full list](usage.md#parameters)) | -| `api_base`, `timeout`, `extra_headers`, `extra_body` | Same | +| `api_base` | Same: the call goes to that address in place of the gateway, so routes, fallbacks and limits apply only if it is an Agent Router gateway that has them. Warns once per address. Remove an `api_base` that points at a provider to call through the gateway | +| `timeout`, `extra_headers`, `extra_body` | Same | | `num_retries`, `max_retries` | Bounded request retries before response/stream output; a partially consumed completion stream is never replayed. Default is zero. Route retries remain gateway settings | | Module `api_base`, `api_key`, `request_timeout`, `num_retries` | Defaults for completion, embeddings and Responses calls; per-call values win. `modify_params=True` warns because message rewriting is unsupported | | `api_key` | Used with `api_base`, or as the key of a gateway you set. With the local gateway it is ignored with a warning: the gateway reads provider keys from the environment ([Models and keys](usage.md#models-and-keys)) | @@ -48,7 +49,7 @@ A full example: [`examples/from_litellm/change_the_import.py`](../examples/from_ | `drop_params`, `additional_drop_params` | Same, on the call or on the module, off by default: with it, Claude drops the parameters it does not take (`seed`, `presence_penalty` and others, and on newer models `temperature` and `top_p`) instead of raising, and a forced tool call becomes `auto` on the models that refuse one ([the full list](usage.md#parameters)). `n=1` is always fine | | `set_verbose`, `suppress_debug_info` | Accepted | | Response types under `litellm.types.utils` (`ModelResponse`, `Choices`, `Message`, `Usage` and others) | Same names | -| `Router` | For `model_list`, `fallbacks` and `num_retries`: the groups become routes in the local gateway, which does the retries and fallbacks. Each route is named after its group and its definition (`router-1a2b3c4d:chat`), so programs with a group of the same name keep their own routes, and a group named after a model does not take over that model in other programs. Call the Router's methods with the group name. A new definition of a group removes the route an earlier one left, and a program still using that route adds it back on its next call. Any change restarts the shared gateway on the next call. Deployments with one `model_name` share its traffic; keys come from the environment. Other `Router` options warn and are ignored. See [Routes](usage.md#routes-fallbacks-retries-and-names-of-your-own) | +| `Router` | For `model_list`, `fallbacks` and `num_retries`: the groups become routes in the local gateway, which does the retries and fallbacks. Each route is named after its group and its definition (`router-1a2b3c4d:chat`), so programs with a group of the same name keep their own routes, and a group named after a model does not take over that model in other programs. Call the Router's methods with the group name. A new definition of a group removes the route an earlier one left, and a program still using that route adds it back on its next call. Any change restarts the shared gateway on the next call. Deployments with one `model_name` share its traffic as LiteLLM's default `simple-shuffle` shares it: by `weight`, `rpm` or `tpm` in `litellm_params`, the first that any deployment sets, and equally when none does. Keys come from the environment. Other `Router` options warn and are ignored, and so do other `litellm_params`, such as `api_base` (see below). With `AGENTROUTER_BASE_URL` set, nothing is set up on this computer: calls send the group name, `chat`, and that gateway's route of that name applies. `export_config()` names each group's route `chat` too. See [Routes](usage.md#routes-fallbacks-retries-and-names-of-your-own) | ## What is missing @@ -73,6 +74,14 @@ These `completion` arguments work in LiteLLM but do nothing in theagentrouter. C | `custom_llm_provider` | Name the provider in the model, for example `openai/gpt-5.4-mini` | | `api_version` | Set `AZURE_API_VERSION` where the gateway starts | +`Router` warns in the same way, once for each `Router`, for what it does not use: + +| `Router` setting | Instead | +|---|---| +| `routing_strategy` other than `simple-shuffle` | The gateway shares a group's traffic as `simple-shuffle` does. LiteLLM's usage-based strategies also hold each deployment to its `rpm` and `tpm`; with the default strategy LiteLLM does not, and neither does the SDK. For a request limit, define a route with `limit=Limit(...)`; it counts requests per route, not tokens | +| `api_base`, `api_version` in `litellm_params` | Set the provider's address where the gateway starts, such as `AZURE_API_BASE`. `os.environ/AZURE_API_BASE` and `os.environ/AZURE_API_VERSION` do not warn | +| Other `litellm_params`, such as `timeout` | Not used | + A value that asks for nothing does not warn. Other unknown completion arguments warn instead of disappearing silently. ## Other differences diff --git a/docs/local-gateway.md b/docs/local-gateway.md index 8d15b1a..1cf9ab5 100644 --- a/docs/local-gateway.md +++ b/docs/local-gateway.md @@ -22,7 +22,7 @@ The macOS arm64 wheel is about 80 MB compressed (the program itself is 281 MB). `routing.resolve`, first match wins: -1. `api_base` on the call: that endpoint, in the provider's own format (LiteLLM's argument). +1. `api_base` on the call: that endpoint, in the provider's own format (LiteLLM's argument). It skips the gateway, so it warns once per address. 2. A gateway set with `AGENTROUTER_BASE_URL` (or `config.json`). 3. A route name: the local gateway's route for it. The running gateway is used only if it has this computer's current routes (`routes_fingerprint` in `state.json`); otherwise it restarts. With `AGENTROUTER_AUTOSTART=0`, a running gateway that already has the route is still used. 4. Anything else: the running local gateway, if it has a rule for this name and the name was not a route when it started (`local_gateway.serves`). Otherwise, if the package is installed and `AGENTROUTER_AUTOSTART` is not `0`, the gateway is started with this model, or restarted with a rule for it, as LiteLLM sends any name to the provider. With no package or no autostart, an error says to set `AGENTROUTER_BASE_URL`. An `api_key=` here cannot reach the provider (the gateway holds the keys), so it warns once. diff --git a/docs/usage.md b/docs/usage.md index 495f566..abfc920 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -51,7 +51,7 @@ For Azure OpenAI, the name after `azure/` is your deployment's name: `azure/gpt- `api_key=` on a call is used in two cases: with `api_base=`, as the key for that endpoint, and with a gateway you set, as that gateway's key. With the local gateway it has no effect and warns once, because the local gateway reads provider keys from the environment when it starts. -To send one call to an endpoint you name, in the provider's own format, use `api_base=` (LiteLLM's argument), for example a local test server. +To send one call to an endpoint you name, in the provider's own format, use `api_base=` (LiteLLM's argument), for example a local test server. The call goes there in place of the gateway, as in LiteLLM, so routes, fallbacks and limits apply only if that address is an Agent Router gateway that has them; the SDK warns once for each address. ## Parameters @@ -93,7 +93,7 @@ These control the call itself: | Parameter | What it does | |---|---| | `api_key` | The key for `api_base`, or for a gateway you set. Ignored, with a warning, for the local gateway | -| `api_base` | Send this call to this address instead, in the provider's own format | +| `api_base` | Send this call to this address instead of the gateway, in the provider's own format. Warns once per address | | `timeout` | Seconds before giving up; the default is 600, as in LiteLLM | | `extra_headers` | More HTTP headers on the request | | `extra_body` | More fields in the request body, for provider-specific options | @@ -213,7 +213,7 @@ resp.to_dict() | `sent_to` | `http://127.0.0.1:52413/anthropic/v1/messages`: the address the SDK sent the request to, here the local gateway | | `llm_provider` | `anthropic` | | `model_sent` | `anthropic/claude-sonnet-5-5`: the model name in the request, as you wrote it. The gateway passes the provider its own name | -| `response_cost` | `0.000123`, in US dollars, from the model list. `completion_cost(resp)` gives the same | +| `response_cost` | `0.000123`, in US dollars, from the model list. `completion_cost(resp)` gives the same. `None` when the model has no known price, such as a route name on a gateway you set, as in LiteLLM; `completion_cost(resp)` then warns and returns 0 | | `served_by`, `route_name` | For a route: the model that answered, and the route's name | ## Errors @@ -350,7 +350,7 @@ export AGENTROUTER_BASE_URL=https://gateway.acme.example/v1 export AGENTROUTER_API_KEY=... # the gateway's key, not a provider's ``` -Set the gateway's address with or without the `/v1` that OpenAI clients add; the SDK adds the paths above. Every call goes there and the local gateway never starts. Provider keys stay on the gateway, so this computer needs none. The key goes as `Authorization: Bearer` on OpenAI-format calls and as `x-api-key` on Claude calls, as Anthropic's own clients send it. Routes belong to the local gateway: see [Routes and a gateway you set](#routes-and-a-gateway-you-set). +Set the gateway's address with or without the `/v1` that OpenAI clients add; the SDK adds the paths above. Every call goes there and the local gateway never starts. Provider keys stay on the gateway, so this computer needs none. The key goes as `Authorization: Bearer` on OpenAI-format calls and as `x-api-key` on Claude calls, as Anthropic's own clients send it. Routes defined in code set up the local gateway; that gateway uses its own routes of the same name: see [Routes and a gateway you set](#routes-and-a-gateway-you-set). ### The local gateway @@ -402,7 +402,7 @@ The local gateway does the retries and the fallbacks, as Agent Router Enterprise - **Retry:** the gateway tries each tier `attempts` more times on the listed status codes and on connection failures, with `backoff` seconds before the first retry, doubling up to 10 seconds, and an optional per-try `timeout`. - **Fallback:** when a tier still fails with one of those codes, or cannot be reached, the gateway moves to the next tier. A stream falls back only before its reply starts. -- **Sharing:** `primary=["openai/gpt-5.4", "openai/gpt-5.4-mini"]` puts two models in one tier, which share the traffic. A fallback can be a list too. +- **Sharing:** `primary=["openai/gpt-5.4", "openai/gpt-5.4-mini"]` puts two models in one tier, which share the traffic equally. A fallback can be a list too. LiteLLM's `Router` shares a group by `weight`, `rpm` or `tpm`, as LiteLLM does: see [Coming from LiteLLM](coming-from-litellm.md). - `served_by` is read from the model name in the reply, so it is missing when a provider answers with a name that does not start with the model's own. - Routes are kept in `~/.agentrouter/routes.json` and shared by every program on the computer, like the local gateway. Defining a route again replaces it; `ar.list_routes()` and `ar.remove_route(name)` list and remove them. - A route can take a model's name, such as `openai/gpt-5.4` or `gpt-5.4`, as a group in the LiteLLM proxy's model list can. Every program's calls to that model then go through the route. @@ -437,17 +437,18 @@ What the file holds: - **Providers:** those whose keys are set on your computer, or the ones you name: `export_config(providers=["openai", "anthropic"])`. - **Models:** a rule for every model in the SDK's bundled model list for those providers, with and without the provider in the name, plus any you added with `register_model`. **The server serves only these.** The bundled list has no Bedrock, Mistral, DeepSeek or Together models, so for those providers name the models (`export_config(models=["mistral/mistral-large-latest"])`) or register LiteLLM's list first (see [Adding models](#adding-models)). Export again when the models change. -- **Routes:** the routes defined on your computer, with their fallbacks, retries and limits. A client cannot see which models are behind a route's name, so `max_tokens` goes as written. If an OpenAI model refuses it, the SDK makes the call once more with `max_completion_tokens`. Pass `max_completion_tokens` yourself to save that extra request. A route name shaped like a Claude model, such as `claude-chat-5`, goes to the gateway as Claude, so use another name for a route. +- **Routes:** the routes defined on your computer, with their fallbacks, retries and limits, under the names code calls them by: a LiteLLM `Router` group's route is named after the group, `chat`, not `router-1a2b3c4d:chat`. Two routes that would get the same name raise `BadRequestError`; remove one with `ar.remove_route()`. A client cannot see which models are behind a route's name, so `max_tokens` goes as written. If an OpenAI model refuses it, the SDK makes the call once more with `max_completion_tokens`. Pass `max_completion_tokens` yourself to save that extra request. A route name shaped like a Claude model, such as `claude-chat-5`, goes to the gateway as Claude, so use another name for a route. - **No key values:** only references such as `${OPENAI_API_KEY}`, which `aigw` fills in from the server's environment. -Exported configurations require a client key by default. Set `AGENTROUTER_API_KEY` on the server and on each client. The gateway accepts `Authorization: Bearer ` or `x-api-key: `, refuses a missing or wrong key with 403, and removes the client key before forwarding. A server started without `AGENTROUTER_API_KEY` refuses every request. `export_config(authentication=False)` explicitly exports an open endpoint. Terminate TLS in front of the server when callers use an untrusted network. +Exported configurations require a client key by default. Set `AGENTROUTER_API_KEY` on the server and on each client. The gateway accepts `Authorization: Bearer ` or `x-api-key: `, refuses a missing or wrong key with 403, and removes the client key before forwarding. A server started without `AGENTROUTER_API_KEY` refuses every request. The SDK raises that refusal as `PermissionDeniedError`, and its message names `AGENTROUTER_API_KEY` on both sides. `export_config(authentication=False)` explicitly exports an open endpoint. Terminate TLS in front of the server when callers use an untrusted network. #### Routes and a gateway you set -Routes program the local gateway only. With `AGENTROUTER_BASE_URL` set: +Routes defined in code set up the local gateway only. With `AGENTROUTER_BASE_URL` set, the same code runs unchanged: -- `route()` refuses to define a route, and says to set up fallbacks on that gateway instead. -- A route defined earlier is not used: its name goes to that gateway as a plain model name, and the SDK warns once. That gateway's own routes and fallbacks apply. +- `route()` and LiteLLM's `Router` set up nothing on this computer and write nothing to `routes.json`. +- A call to a route sends its name to that gateway, and the SDK warns once. That gateway's own route of that name, with its fallbacks, retries and limit, applies. A `Router` sends the group name, `chat`. +- A gateway made with `export_config()` has this computer's routes under those names, so code tested against the local gateway calls the same routes there. ## Settings diff --git a/skills/migrate-from-litellm/SKILL.md b/skills/migrate-from-litellm/SKILL.md index a569e85..5cdc01d 100644 --- a/skills/migrate-from-litellm/SKILL.md +++ b/skills/migrate-from-litellm/SKILL.md @@ -54,11 +54,13 @@ from google.adk.models.lite_llm import LiteLlm # now gets theagentrouter OpenAI and Anthropic names without a provider (`gpt-5.4-mini`, `claude-sonnet-5-5`) work, as in LiteLLM. A bare `gemini-…` name meant Vertex AI in LiteLLM, which the local gateway does not serve yet: tell the user, and if they use the Gemini API, change it to `gemini/…`. With `AGENTROUTER_BASE_URL` set, the gateway takes its own names. +An `api_base=` that points at a provider (common in Azure OpenAI code) sends the call there in place of the gateway, as in LiteLLM. Remove it, and tell the user the provider address now belongs where the gateway starts (`AZURE_API_BASE` for Azure). + ## 5. Move retries and fallbacks `num_retries=`, `fallbacks=` and `max_retries=` on a single `completion` call are ignored with a warning. Two ways to keep them: -- Keep LiteLLM's `Router` as it is: `model_list`, `fallbacks` and `num_retries` work, served by the local gateway. +- Keep LiteLLM's `Router` as it is: `model_list`, `fallbacks` and `num_retries` work, served by the local gateway, or with `AGENTROUTER_BASE_URL` set by that gateway's route of the group's name. `weight`, `rpm` and `tpm` share a group's traffic, as in LiteLLM's default strategy. A `routing_strategy` other than `simple-shuffle` warns, because the SDK does not hold deployments to `rpm` or `tpm`: tell the user, and offer a route with `limit=ar.Limit(...)` (requests, not tokens). - Or define a route once and call it by name: ```python diff --git a/skills/theagentrouter/SKILL.md b/skills/theagentrouter/SKILL.md index 2a1143e..0dd5eb8 100644 --- a/skills/theagentrouter/SKILL.md +++ b/skills/theagentrouter/SKILL.md @@ -55,7 +55,7 @@ resp = ar.completion(model="chat", messages=messages) resp._hidden_params["served_by"] # which model answered ``` -Routes are served by the local gateway and shared by every program on the computer. LiteLLM's `Router` (`from theagentrouter.compat.litellm import Router`) works too. +Routes are served by the local gateway and shared by every program on the computer. With `AGENTROUTER_BASE_URL` set, the same code runs and that gateway's route of the same name applies. LiteLLM's `Router` (`from theagentrouter.compat.litellm import Router`) works too. ## Agent frameworks diff --git a/src/theagentrouter/anthropic_messages.py b/src/theagentrouter/anthropic_messages.py index 8fb678f..38ce382 100644 --- a/src/theagentrouter/anthropic_messages.py +++ b/src/theagentrouter/anthropic_messages.py @@ -116,8 +116,8 @@ def _hidden(target: routing.Target, url: str, usage: dict[str, Any] | None) -> d } if usage is not None: u = anthropic_format._usage(usage) # the same token counts completion() reports for Claude - hidden["response_cost"] = models.completion_cost( - model=key, prompt_tokens=u.prompt_tokens, completion_tokens=u.completion_tokens + hidden["response_cost"] = models._response_cost( + key, prompt_tokens=u.prompt_tokens, completion_tokens=u.completion_tokens ) return hidden diff --git a/src/theagentrouter/compat/litellm.py b/src/theagentrouter/compat/litellm.py index ac20d4b..35e475b 100644 --- a/src/theagentrouter/compat/litellm.py +++ b/src/theagentrouter/compat/litellm.py @@ -265,16 +265,66 @@ def get_llm_provider( return name, provider, api_key, api_base -def _route_name(group: str, models: list[str], fallbacks: list[list[str]], retry: Any) -> str: +def _route_name(group: str, models: list[str], fallbacks: list[list[str]], retry: Any, weights: Any = ()) -> str: """The route a Router group becomes: the same definition gets the same name in every program.""" import hashlib import json from dataclasses import asdict - definition = json.dumps([group, models, fallbacks, asdict(retry)], sort_keys=True) + parts = [group, models, fallbacks, asdict(retry)] + if weights: # only when set, so a group without weights keeps the name it had before weights existed + parts.append([list(w) for w in weights]) + definition = json.dumps(parts, sort_keys=True) return f"router-{hashlib.sha256(definition.encode()).hexdigest()[:8]}:{group}" +# Settings a deployment's litellm_params can hold that the Router uses: the model, and weight, rpm or +# tpm for its share of the group's traffic. +_USED = ("model", "api_key", "weight", "rpm", "tpm") +# A setting the gateway reads from where it starts, so os.environ/ asks for nothing new. +_GATEWAY_VARIABLES = {"api_base": "AZURE_API_BASE", "api_version": "AZURE_API_VERSION"} +_WEIGHT_MAX = 1_000_000 # Gateway API's largest backend weight + + +def _weights(deployments: list[dict[str, Any]]) -> tuple[int, ...]: + """A group's traffic shares, by LiteLLM's simple-shuffle rule: the first of weight, rpm and tpm that any + deployment sets; a deployment without it gets none. Empty when the shares are equal.""" + for metric in ("weight", "rpm", "tpm"): + values = [] + for d in deployments: + value = (d.get("litellm_params") or {}).get(metric) + if value is not None and (isinstance(value, bool) or not isinstance(value, (int, float))): + raise TypeError(f"Deployment {metric} must be numeric") # LiteLLM's error + values.append(max(float(value or 0), 0.0)) + largest = max(values, default=0.0) + if largest <= 0: + continue + if len(set(values)) == 1: + return () + if all(v.is_integer() for v in values) and largest <= _WEIGHT_MAX: + return tuple(int(v) for v in values) + return _whole_weights(values) + return () + + +def _whole_weights(values: list[float]) -> tuple[int, ...]: + """Whole-number weights in the same ratio, for the gateway, which takes 0 to 1,000,000. + + Exact when the ratio fits: 0.01 and 100 become 1 and 10000, 0.25 and 1 become 1 and 4. When it does + not, the largest becomes 1,000,000 and the others are rounded to it, a positive one to at least 1. + """ + from fractions import Fraction + from math import lcm + + exact = [Fraction(str(v)) for v in values] # 0.1 as 1/10, not as its nearest binary fraction + scale = lcm(*(f.denominator for f in exact)) + whole = [int(f * scale) for f in exact] + largest = max(whole) + if largest <= _WEIGHT_MAX: + return tuple(whole) + return tuple(max(1, round(w * _WEIGHT_MAX / largest)) if w > 0 else 0 for w in whole) + + def _remove_earlier(group: str, keep: str) -> None: """Remove the routes earlier definitions of a group left, so their providers' keys stop being needed.""" import re @@ -304,9 +354,14 @@ class Router: such as ``router-1a2b3c4d:chat``, so two programs with a group of the same name keep separate routes, and a group named after a model does not take over that model's calls in other programs. A new definition of a group removes the routes earlier ones left; a Router still using one adds it - back on its next call. - The Router's methods take the group name. Several deployments with one model_name share its traffic. Keys come from the environment: litellm_params api_key values - other than os.environ/ are not used, and say so once. + back on its next call. ``export_config()`` names each of these routes after its group, ``chat``. + With AGENTROUTER_BASE_URL set, nothing is set up here: calls send the group name to that gateway, + and its route of that name applies. + The Router's methods take the group name. Several deployments with one model_name share its traffic, + by weight, rpm or tpm in litellm_params as LiteLLM's default simple-shuffle shares it. Keys come from + the environment: litellm_params api_key values other than os.environ/ + are not used, and say so once. Other litellm_params, such as api_base, are not used either, and say + so; so does a routing_strategy other than simple-shuffle. """ def __init__( @@ -320,14 +375,22 @@ def __init__( from .. import routes from ..local_gateway import PROVIDERS + from ..routing import settings self.model_list = list(model_list or []) groups: dict[str, list[str]] = {} + deployments: dict[str, list[dict[str, Any]]] = {} + unused: dict[str, list[str]] = {} # a litellm_params setting the gateway does not take -> its groups for d in self.model_list: params = d.get("litellm_params") or {} model = params.get("model") if not d.get("model_name") or not model: raise exceptions.BadRequestError("each model_list entry needs model_name and litellm_params.model") + for setting, value in sorted(params.items()): + if value is None or setting in _USED or value == f"os.environ/{_GATEWAY_VARIABLES.get(setting)}": + continue + if d["model_name"] not in unused.setdefault(setting, []): + unused[setting].append(d["model_name"]) key = params.get("api_key") provider = model.split("/", 1)[0] usual = (PROVIDERS.get(provider) or {}).get("keys", [None])[0] @@ -339,6 +402,7 @@ def __init__( stacklevel=2, ) groups.setdefault(d["model_name"], []).append(model) + deployments.setdefault(d["model_name"], []).append(d) chains: dict[str, list[str]] = {} for item in fallbacks or []: for name, rest in item.items(): @@ -350,24 +414,64 @@ def __init__( ignored = sorted(k for k, v in kwargs.items() if v and k not in ("routing_strategy", "set_verbose")) if ignored: warnings.warn(f"theagentrouter's Router ignores {', '.join(ignored)}.", UserWarning, stacklevel=2) + strategy = kwargs.get("routing_strategy") + strategy = getattr(strategy, "value", strategy) + if strategy and strategy != "simple-shuffle": + # LiteLLM's other strategies choose a deployment in the program; usage-based ones also hold + # each deployment to its rpm and tpm. The gateway shares traffic as simple-shuffle does. + warnings.warn( + f"theagentrouter's Router ignores routing_strategy={strategy!r}: the gateway shares a group's " + "traffic by weight, rpm or tpm, as with 'simple-shuffle', and does not hold deployments to rpm or " + "tpm. For a request limit, define a route with limit=Limit(...).", + UserWarning, + stacklevel=2, + ) + if unused: + listed = "; ".join(f"{k} (for {', '.join(groups_)})" for k, groups_ in sorted(unused.items())) + warnings.warn( + f"theagentrouter's Router does not use these litellm_params: {listed}. " + "The gateway takes provider addresses and settings from where it starts.", + UserWarning, + stacklevel=2, + ) # LiteLLM's Router falls back on nearly any error, including a wrong key or an unknown model. on = (401, 403, 404, 408, *routes.RETRY_ON) retry = routes.Retry(attempts=num_retries if num_retries is not None else 1, on=on) self.routes = {} + gateway = settings().get("base_url") for name, models in groups.items(): - later = [groups[f] for f in chains.get(name, [])] - self.routes[name] = routes.route(_route_name(name, models, later, retry), models, later, retry) - _remove_earlier(name, keep=self.routes[name].name) + order = [name, *chains.get(name, [])] + later = [groups[f] for f in order[1:]] + shares = [_weights(deployments[g]) for g in order] + weights = ( + tuple(w or (1,) * len(groups[g]) for w, g in zip(shares, order, strict=True)) if any(shares) else () + ) + tiers = tuple(tuple(groups[g]) for g in order) + # The gateway you set has its own route by the group's name, as export_config() names it. + route_name = name if gateway else _route_name(name, models, later, retry, weights) + self.routes[name] = routes._define(routes.Route(route_name, tiers, retry, None, weights)) + if not gateway: + _remove_earlier(name, keep=route_name) def _target(self, model: str) -> str: + from dataclasses import replace + from .. import routes + from ..routing import settings r = self.routes.get(model) if r is None: return model - if routes.get(r.name) is None: # a newer definition of this group removed it; this Router still uses it - routes.route(r.name, list(r.tiers[0]), [list(t) for t in r.tiers[1:]], r.retry) - return r.name + # Decided at each call, not when the Router was made: AGENTROUTER_BASE_URL can be set or unset in between. + if settings().get("base_url"): + # The gateway you set has its own route by the group's name, as export_config() names it. + if not routes.defined_for_gateway(model): + routes._define(replace(r, name=model)) + return model + local = _route_name(model, list(r.tiers[0]), [list(t) for t in r.tiers[1:]], r.retry, r.weights) + if routes.get(local) is None: # made with a gateway set, or a newer definition of this group removed it + routes._define(replace(r, name=local)) + return local def completion(self, model: str, messages: list[Any], **kwargs: Any): return completion(model=self._target(model), messages=messages, **kwargs) diff --git a/src/theagentrouter/embeddings.py b/src/theagentrouter/embeddings.py index 3250667..241820a 100644 --- a/src/theagentrouter/embeddings.py +++ b/src/theagentrouter/embeddings.py @@ -86,7 +86,7 @@ def _parse(target: routing.Target, data: dict[str, Any], url: str) -> EmbeddingR if served: resp._hidden_params["served_by"] = served resp._hidden_params["model_key"] = key = served - resp._hidden_params["response_cost"] = models.completion_cost(model=key, prompt_tokens=prompt) + resp._hidden_params["response_cost"] = models._response_cost(key, prompt_tokens=prompt) return resp diff --git a/src/theagentrouter/local_gateway.py b/src/theagentrouter/local_gateway.py index db6651b..ea28392 100644 --- a/src/theagentrouter/local_gateway.py +++ b/src/theagentrouter/local_gateway.py @@ -551,7 +551,7 @@ def export_config( bare = sorted(m for m in extra if "/" not in m) if bare: raise exceptions.BadRequestError(f"models need their provider, for example openai/{bare[0]}") - current = routes_mod.list_routes() + current = _as_called(routes_mod.list_routes()) used = {m.split("/", 1)[0] for r in current.values() for m in r.models} | {m.split("/", 1)[0] for m in extra} outside = sorted(used - set(chosen)) if outside: @@ -568,6 +568,25 @@ def export_config( return config +def _as_called(current: dict) -> dict: + """Routes under the names code calls them by: a LiteLLM Router's ``router-1a2b3c4d:chat`` becomes ``chat``.""" + import dataclasses + + out: dict = {} + origin: dict[str, str] = {} + for name, r in sorted(current.items()): + group = re.fullmatch(r"router-[0-9a-f]{8}:(.+)", name) + called = group.group(1) if group else name + if called in out: + raise exceptions.BadRequestError( + f"routes {origin[called]!r} and {name!r} would both be {called!r} in the config; " + "remove one with theagentrouter.remove_route()" + ) + out[called] = dataclasses.replace(r, name=called) + origin[called] = name + return out + + def _api_key_config(variable: str) -> str: # aigw fills in ${VAR} across the whole file. Missing or empty, the expected value becomes a line break, # which no header can carry, so a server started without the variable refuses every request. @@ -618,7 +637,8 @@ def _route_docs(r) -> list[str]: name = _k8s_name(r.name) refs = [] for priority, tier in enumerate(r.tiers): - for model in tier: + for index, model in enumerate(tier): + weight = r.weights[priority][index] if r.weights else 1 backend, own = _route_backend(model) removed = [] if model.startswith("anthropic/") and _refuses_sampling(own): @@ -633,7 +653,7 @@ def _route_docs(r) -> list[str]: namespace: default modelNameOverride: {json.dumps(own)} priority: {priority} - weight: 1{mutation}""" + weight: {weight}{mutation}""" ) docs = [ f"""apiVersion: aigateway.envoyproxy.io/v1beta1 diff --git a/src/theagentrouter/mock.py b/src/theagentrouter/mock.py index 4886c3f..87cf2f8 100644 --- a/src/theagentrouter/mock.py +++ b/src/theagentrouter/mock.py @@ -68,8 +68,8 @@ def _hidden(model: str, usage: Usage) -> dict[str, Any]: "llm_provider": _provider(model), "model_sent": model, "model_key": key, - "response_cost": models.completion_cost( - model=key, prompt_tokens=usage.prompt_tokens, completion_tokens=usage.completion_tokens + "response_cost": models._response_cost( + key, prompt_tokens=usage.prompt_tokens, completion_tokens=usage.completion_tokens ), } diff --git a/src/theagentrouter/models.py b/src/theagentrouter/models.py index bc51fd8..c0be9b1 100644 --- a/src/theagentrouter/models.py +++ b/src/theagentrouter/models.py @@ -281,6 +281,17 @@ def cost_per_token( ) +def _response_cost(model: str | None, **kwargs: Any) -> float | None: + """The cost a call reports in _hidden_params["response_cost"], or None when the model has no known price. + + As in LiteLLM: a call to a model without a price, such as a route name on a gateway you set, leaves + the cost empty and warns nothing. completion_cost() called directly still says so and returns 0. + """ + if not model or _key(model, kwargs.get("custom_llm_provider")) is None: + return None + return completion_cost(model=model, **kwargs) + + def completion_cost( completion_response: Any = None, model: str | None = None, diff --git a/src/theagentrouter/responses.py b/src/theagentrouter/responses.py index ea75b2a..0aa3b2d 100644 --- a/src/theagentrouter/responses.py +++ b/src/theagentrouter/responses.py @@ -130,8 +130,8 @@ def _parse(target: routing.Target, data: dict[str, Any], url: str, model: str) - "llm_provider": target.provider, "model_sent": target.model, "model_key": key, - "response_cost": models.completion_cost( - model=key, + "response_cost": models._response_cost( + key, prompt_tokens=int(usage.get("input_tokens") or 0), completion_tokens=int(usage.get("output_tokens") or 0), ), diff --git a/src/theagentrouter/routes.py b/src/theagentrouter/routes.py index c4cd780..be91405 100644 --- a/src/theagentrouter/routes.py +++ b/src/theagentrouter/routes.py @@ -10,6 +10,11 @@ kept in ``AGENTROUTER_HOME/routes.json`` because the local gateway is shared by every program on the computer, so its routes are too. Defining a route again replaces it, and the gateway restarts with the change on the next call that needs it. + +With a gateway set (``AGENTROUTER_BASE_URL``), defining a route sets up nothing: calls send the +route's name to that gateway, and its route of the same name applies. ``export_config()`` writes a +gateway config with this computer's routes under the names code calls them by, so the same code runs +against both. """ from __future__ import annotations @@ -111,12 +116,16 @@ class Route: """A name of your own, and the models behind it in order: tiers[0] first, then each fallback. Models in one tier share the traffic between them; the next tier is used when they fail. + weights: one whole number per model, in the shape of tiers, for how a tier's traffic is shared + (LiteLLM's Router sets them from weight, rpm or tpm); empty means an equal share. A model with 0 + gets no traffic. """ name: str tiers: tuple[tuple[str, ...], ...] retry: Retry = field(default_factory=Retry) limit: Limit | None = None + weights: tuple[tuple[int, ...], ...] = () @property def primary(self) -> str: @@ -147,6 +156,7 @@ def _load() -> dict[str, Route]: tuple(tuple(t) for t in r["tiers"]), Retry(**{**retry, "on": tuple(retry.get("on", RETRY_ON))}), Limit(**limit) if limit else None, + tuple(tuple(t) for t in r.get("weights") or ()), ) except (ValueError, TypeError, KeyError): warnings.warn( @@ -158,14 +168,16 @@ def _load() -> dict[str, Route]: def _dump(routes: dict[str, Route]) -> dict: - return { - n: { + out = {} + for n, r in sorted(routes.items()): + out[n] = { "tiers": [list(t) for t in r.tiers], "retry": asdict(r.retry), "limit": asdict(r.limit) if r.limit else None, } - for n, r in sorted(routes.items()) - } + if r.weights: # only when set, so routes saved without weights read the same + out[n]["weights"] = [list(t) for t in r.weights] + return out def _save(routes: dict[str, Route]) -> None: @@ -214,19 +226,24 @@ def _tier(value: str | list[str] | tuple[str, ...]) -> tuple[str, ...]: return models -def _check(name: str, tiers: tuple[tuple[str, ...], ...]) -> None: +# Routes defined while a gateway is set. Nothing is set up for them on this computer; they are kept so +# a call to one can say that the gateway's route of that name applies. +_for_gateway: dict[str, Route] = {} + + +def defined_for_gateway(name: str) -> bool: + """Whether this program defined a route by this name while a gateway was set.""" + return name in _for_gateway + + +def _check(name: str, tiers: tuple[tuple[str, ...], ...], *, local: bool = True) -> None: """Fail now, with the fix in the message, rather than on the first call.""" from . import local_gateway - from .routing import settings if not name or name != name.strip(): raise exceptions.BadRequestError(f"route name {name!r} must be a non-empty name without spaces at the ends") - gateway = settings().get("base_url") - if gateway: - raise exceptions.BadRequestError( - f"routes are served by the local gateway, and AGENTROUTER_BASE_URL sends calls to {gateway}. " - "Set up fallbacks in that gateway instead, or unset AGENTROUTER_BASE_URL." - ) + if not local: # the gateway you set owns its model names and providers + return for model in (m for tier in tiers for m in tier): provider, sep, own = model.partition("/") if not sep or not own: @@ -248,16 +265,36 @@ def route( ) -> Route: """Define (or replace) a route: call it with completion(model=name, ...). + With AGENTROUTER_BASE_URL set, nothing is set up here: calls send the name to that gateway, and + its route of the same name applies. + primary: a provider/model name, or a list of them that share the traffic. fallbacks: tried in order when the primary fails; each item a model or a list of them. retry: how many times each model is retried first, and on which errors. Default: Retry(). limit: at most this many requests to the route in a period, for example Limit(60, per="minute"). """ tiers = (_tier(primary), *(_tier(f) for f in fallbacks or [])) - _check(name, tiers) - new = Route(name, tiers, retry or Retry(), limit) + return _define(Route(name, tiers, retry or Retry(), limit)) + + +def _define(new: Route) -> Route: + """Check and save a route; LiteLLM's Router uses this too, for routes with weights.""" + from .routing import settings + + name, tiers = new.name, new.tiers + gateway = settings().get("base_url") + _check(name, tiers, local=not gateway) + if new.weights and ( + len(new.weights) != len(tiers) + or any(len(w) != len(t) or any(x < 0 for x in w) for w, t in zip(new.weights, tiers, strict=False)) + ): + raise ValueError("route weights need one whole number of 0 or more for each model, tier by tier") if (new.retry.attempts + 1) * len(tiers) - 1 >= 2**32: raise ValueError("the total retries across route tiers must fit an unsigned 32-bit integer") + if gateway: + # The same code runs on a laptop and against a shared gateway: here the gateway's route applies. + _for_gateway[name] = new + return new with _editing(): routes = _load() old = routes.get(name) diff --git a/src/theagentrouter/routing.py b/src/theagentrouter/routing.py index 7fd5f92..50afdc7 100644 --- a/src/theagentrouter/routing.py +++ b/src/theagentrouter/routing.py @@ -21,6 +21,7 @@ from __future__ import annotations +import json import os import warnings from dataclasses import dataclass, field @@ -178,6 +179,7 @@ def resolve( ) -> Target: headers = dict(extra_headers or {}) if api_base: + _warn_api_base(api_base) return _named_endpoint(model, api_base, api_key, headers) configured = settings() @@ -214,6 +216,55 @@ def resolve( _key_warned = False _route_warned: set[str] = set() +_api_base_warned: set[str] = set() + + +def _warn_api_base(api_base: str) -> None: + """LiteLLM's api_base keeps its meaning, but code moved from LiteLLM may not expect it to replace the gateway.""" + gateway = settings().get("base_url") + if gateway and api_base.rstrip("/") == gateway.rstrip("/"): + return # the gateway already set: nothing changes + address = _safe_url(api_base) + if address in _api_base_warned: + return + _api_base_warned.add(address) + instead = ( + f"in place of the gateway in AGENTROUTER_BASE_URL ({_safe_url(gateway)})" + if gateway + else "in place of the local gateway, which is not started for them" + ) + warnings.warn( + f"api_base= sends these calls to {address} {instead}, as in LiteLLM. Routes, fallbacks and limits " + f"apply only if {address} is an Agent Router gateway that has them.", + UserWarning, + stacklevel=5, + ) + + +def _explain_refusal(status: int, text: str, target: Target) -> str: + """A 403 that a gateway you set gives itself, not a provider's JSON error, means it refused the client key. + + Envoy answers it in plain text, so the error alone does not say which setting to look at. + """ + if status != 403 or target.route != "gateway": + return text + try: + json.loads(text) + return text # a provider's error, passed on by the gateway + except ValueError: + pass + # Envoy gives the same plain-text 403 for any access rule: a client key, but also an address allowlist or + # a firewall in front. So the key is named as a possible cause, not as the cause. + where = _safe_url(target.base_url) + cause = ( + "AGENTROUTER_API_KEY may not be the key it expects" + if target.api_key + else "no client key was sent: set AGENTROUTER_API_KEY" + ) + return ( + f"{text.strip() or 'HTTP 403'} ({where} refused the request. If that gateway checks a client key, {cause}; " + "a gateway made with export_config() expects the same value as AGENTROUTER_API_KEY on its server.)" + ) def _warn_route_unused(model: str, gateway: str) -> None: @@ -222,7 +273,7 @@ def _warn_route_unused(model: str, gateway: str) -> None: return from . import routes - if routes.get(model) is not None: + if routes.defined_for_gateway(model) or routes.get(model) is not None: _route_warned.add(model) warnings.warn( f"route {model!r} is defined in Python, but AGENTROUTER_BASE_URL is set: routes apply to the local " diff --git a/src/theagentrouter/tokens.py b/src/theagentrouter/tokens.py index c018571..c84fd3e 100644 --- a/src/theagentrouter/tokens.py +++ b/src/theagentrouter/tokens.py @@ -107,7 +107,11 @@ def _gateway_count(model: str, text: Any, messages: list[dict[str, Any]] | None, with httpx.Client(timeout=_TIMEOUT) as client: r = client.post(url, json=body, headers=headers) if r.status_code >= 400: - raise exceptions.from_http(r.status_code, r.text, model, target.provider) + explained = routing._explain_refusal(r.status_code, r.text, target) + # The gateway's own refusal is not the provider's, so no provider name goes in front of the message. + error = exceptions.from_http(r.status_code, explained, model, None if explained != r.text else target.provider) + error.llm_provider = target.provider + raise error return int(r.json()[field]) diff --git a/src/theagentrouter/transport.py b/src/theagentrouter/transport.py index 5634516..4330cef 100644 --- a/src/theagentrouter/transport.py +++ b/src/theagentrouter/transport.py @@ -201,9 +201,9 @@ def _parse(target: routing.Target, data: dict[str, Any], model: str) -> ModelRes resp._hidden_params["served_by"] = served resp._hidden_params["model_key"] = served # From the bundled table; a gateway that reports cost would override this. - resp._hidden_params["response_cost"] = models.completion_cost( + resp._hidden_params["response_cost"] = models._response_cost( + resp._hidden_params["model_key"], completion_response=resp, - model=resp._hidden_params["model_key"], prompt_tokens=resp.usage.prompt_tokens, completion_tokens=resp.usage.completion_tokens, ) @@ -212,7 +212,11 @@ def _parse(target: routing.Target, data: dict[str, Any], model: str) -> ModelRes def _raise_for(response: httpx.Response, text: str, target: routing.Target, model: str) -> None: if response.status_code >= 300: - error = exceptions.from_http(response.status_code, text, model, target.provider) + explained = routing._explain_refusal(response.status_code, text, target) + # The gateway's own refusal is not the provider's, so no provider name goes in front of the message. + prefix = None if explained != text else target.provider + error = exceptions.from_http(response.status_code, explained, model, prefix) + error.llm_provider = target.provider error.response = response error.headers = response.headers error.request = response.request diff --git a/tests/test_routes.py b/tests/test_routes.py index affa6cd..764e959 100644 --- a/tests/test_routes.py +++ b/tests/test_routes.py @@ -85,10 +85,85 @@ def test_route_can_be_defined_for_export_without_a_local_gateway_or_keys(env, mo ar.completion(model="chat", messages=HELLO) -def test_route_refuses_when_a_gateway_is_set(env, installed, monkeypatch): - monkeypatch.setenv("AGENTROUTER_BASE_URL", "https://gw.acme.example/v1") - with pytest.raises(exceptions.BadRequestError, match="Set up fallbacks in that gateway"): - ar.route("chat", primary="openai/gpt-4o") +@pytest.fixture +def gateway_set(env, monkeypatch): + """A gateway set with AGENTROUTER_BASE_URL, at the fake provider; nothing remembered from other tests.""" + from theagentrouter import routing + + from .fake_provider import FakeProvider + + server = FakeProvider() + monkeypatch.setenv("AGENTROUTER_BASE_URL", server.url) + monkeypatch.setenv("AGENTROUTER_AUTOSTART", "0") + monkeypatch.setattr(routing, "_route_warned", set()) + monkeypatch.setattr(routes, "_for_gateway", {}) + yield server + server.close() + + +def test_a_route_defined_with_a_gateway_set_is_called_by_name_there(env, gateway_set): + # The same code as on a laptop: no error, nothing set up here, and the gateway's route of that name applies. + r = ar.route("chat", primary="openai/gpt-4o", fallbacks=["anthropic/claude-sonnet-4-5"]) + assert r.name == "chat" and r.tiers == (("openai/gpt-4o",), ("anthropic/claude-sonnet-4-5",)) + assert ar.list_routes() == {} and not (env / "routes.json").exists() + with pytest.warns(UserWarning, match="route 'chat' is defined in Python.*that gateway's own routes"): + resp = ar.completion(model="chat", messages=HELLO) + assert resp._hidden_params["route"] == "gateway" and gateway_set.requests[-1][2]["model"] == "chat" + assert local_gateway.running() is None + + +def test_a_router_with_a_gateway_set_calls_its_groups_by_name(env, gateway_set): + router = litellm.Router( + model_list=[ + {"model_name": "chat", "litellm_params": {"model": "openai/gpt-4o"}}, + {"model_name": "backup", "litellm_params": {"model": "anthropic/claude-sonnet-4-5"}}, + ], + fallbacks=[{"chat": ["backup"]}], + ) + assert router.routes["chat"].name == "chat" and ar.list_routes() == {} + with pytest.warns(UserWarning, match="route 'chat' is defined in Python"): + router.completion(model="chat", messages=HELLO) + assert gateway_set.requests[-1][2]["model"] == "chat" + + +def test_a_router_made_here_sends_its_group_name_once_a_gateway_is_set(env, installed, monkeypatch): + # Made on a laptop, then pointed at a gateway built with export_config(): that gateway knows the group's + # name, not the hashed one this computer uses, so the name is chosen at each call. + from theagentrouter import routing + + from .fake_provider import FakeProvider + + router = litellm.Router(model_list=[{"model_name": "chat", "litellm_params": {"model": "openai/gpt-4o"}}]) + assert router.routes["chat"].name.startswith("router-") and router.routes["chat"].name in ar.list_routes() + server = FakeProvider() + try: + monkeypatch.setenv("AGENTROUTER_BASE_URL", server.url) + monkeypatch.setenv("AGENTROUTER_AUTOSTART", "0") + monkeypatch.setattr(routing, "_route_warned", set()) + monkeypatch.setattr(routes, "_for_gateway", {}) + with pytest.warns(UserWarning, match="route 'chat' is defined in Python.*that gateway's own routes"): + router.completion(model="chat", messages=HELLO) + assert server.requests[-1][2]["model"] == "chat" + finally: + server.close() + + +def test_a_router_made_with_a_gateway_set_sets_up_its_route_here_once_unset(env, gateway_set, monkeypatch): + router = litellm.Router(model_list=[{"model_name": "chat", "litellm_params": {"model": "openai/gpt-4o"}}]) + assert router.routes["chat"].name == "chat" and ar.list_routes() == {} + monkeypatch.delenv("AGENTROUTER_BASE_URL") + name = router._target("chat") + assert name.startswith("router-") and name.endswith(":chat") and name in ar.list_routes() + + +def test_export_names_router_routes_after_their_group(env, installed): + router = litellm.Router(model_list=[{"model_name": "chat", "litellm_params": {"model": "openai/gpt-4o"}}]) + assert router.routes["chat"].name.startswith("router-") # on this computer, the hashed name + config = local_gateway.export_config(providers=["openai"]) + assert 'value: "chat"' in config and router.routes["chat"].name not in config # the name code calls + ar.route("chat", primary="openai/gpt-4o-mini") + with pytest.raises(exceptions.BadRequestError, match="would both be 'chat'"): + local_gateway.export_config(providers=["openai"]) # --- the gateway config --------------------------------------------------------------------------- @@ -432,6 +507,106 @@ def test_router_takes_keys_from_the_environment(env, installed): assert not [w for w in seen if "api_key" in str(w.message)] +def test_router_shares_a_groups_traffic_by_weight_rpm_or_tpm(env, installed): + # LiteLLM's default simple-shuffle: the first of weight, rpm and tpm in litellm_params that any + # deployment sets; a deployment without it gets no traffic. rpm and tpm are not limits there. + with warnings.catch_warnings(): + warnings.simplefilter("error") + router = litellm.Router( + model_list=[ + {"model_name": "chat", "litellm_params": {"model": "openai/gpt-4o", "rpm": 10, "tpm": 50000}}, + {"model_name": "chat", "litellm_params": {"model": "openai/gpt-4o-mini", "rpm": 30}}, + {"model_name": "chat", "litellm_params": {"model": "openai/gpt-4.1", "tpm": 9}}, + {"model_name": "backup", "litellm_params": {"model": "anthropic/claude-sonnet-4-5"}, "rpm": 5}, + ], + fallbacks=[{"chat": ["backup"]}], + ) + chat = router.routes["chat"] + assert chat.weights == ((10, 30, 0), (1,)) # rpm decides; a top-level rpm is not a weight + assert ar.list_routes()[chat.name].weights == chat.weights # saved + config = local_gateway.config_yaml( + ["openai", "anthropic"], dict(os.environ), port=1975, models=[], routes={"chat": chat} + ) + assert [line.strip() for line in config.splitlines() if line.strip().startswith("weight:")] == [ + "weight: 10", + "weight: 30", + "weight: 0", + "weight: 1", + ] + weight = litellm.Router( + model_list=[ + {"model_name": "chat", "litellm_params": {"model": "openai/gpt-4o", "weight": 0.25, "rpm": 9}}, + {"model_name": "chat", "litellm_params": {"model": "openai/gpt-4o-mini", "weight": 1}}, + ] + ) + assert weight.routes["chat"].weights == ((1, 4),) # weight wins over rpm; a fraction keeps its ratio + equal = litellm.Router( + model_list=[ + {"model_name": "chat", "litellm_params": {"model": "openai/gpt-4o", "rpm": 5}}, + {"model_name": "chat", "litellm_params": {"model": "openai/gpt-4o-mini", "rpm": 5}}, + ] + ) + assert equal.routes["chat"].weights == () # equal shares need no weights + with pytest.raises(TypeError, match="Deployment rpm must be numeric"): + litellm.Router(model_list=[{"model_name": "chat", "litellm_params": {"model": "openai/gpt-4o", "rpm": "10"}}]) + + +def _shares(*weights): + return litellm._weights([{"litellm_params": {"weight": w}} for w in weights]) + + +def test_router_weights_keep_their_ratio(): + # The gateway takes whole numbers up to 1,000,000; LiteLLM's weights can be any number. + assert _shares(0.01, 100) == (1, 10000) # 1 in 10,001 of the traffic, as in LiteLLM, not 1 in 1,001 + assert _shares(0.1, 0.2, 0.7) == (1, 2, 7) # 0.1 as one tenth, not its nearest binary fraction + assert _shares(10, 30, 0) == (10, 30, 0) # whole numbers that fit are kept as written + # A ratio too wide to fit: the largest becomes 1,000,000 and a positive weight keeps at least 1. + assert _shares(1e-9, 1) == (1, 1_000_000) + assert _shares(3_000_000, 1, 0) == (1_000_000, 1, 0) + + +def test_router_says_what_it_does_not_use(env, installed): + with pytest.warns( + UserWarning, match=r"ignores routing_strategy='usage-based-routing-v2'.*does not hold deployments to rpm" + ): + litellm.Router( + model_list=[{"model_name": "chat", "litellm_params": {"model": "openai/gpt-4o", "rpm": 10}}], + routing_strategy="usage-based-routing-v2", + ) + with pytest.warns( + UserWarning, match=r"does not use these litellm_params: api_base \(for chat\); timeout \(for chat\)" + ): + litellm.Router( + model_list=[ + { + "model_name": "chat", + "litellm_params": { + "model": "azure/gpt-4o", + "api_base": "https://acme.openai.azure.com", + "timeout": 5, + }, + } + ] + ) + with warnings.catch_warnings(): + warnings.simplefilter("error") # what the gateway reads where it starts, and the default strategy + litellm.Router( + model_list=[ + { + "model_name": "chat", + "litellm_params": { + "model": "azure/gpt-4o", + "api_key": "os.environ/AZURE_API_KEY", + "api_base": "os.environ/AZURE_API_BASE", + "api_version": "os.environ/AZURE_API_VERSION", + }, + "model_info": {"id": "a"}, + } + ], + routing_strategy="simple-shuffle", + ) + + def test_router_rejects_unknown_fallback_groups(env, installed): with pytest.raises(exceptions.BadRequestError, match="not in model_list"): litellm.Router( @@ -591,6 +766,82 @@ def test_exported_config_serves_a_team_gateway(real_gateway, second_fake, env, m proc.wait(timeout=30) +@contextlib.contextmanager +def _team_gateway(folder, config: str, port: int): + """aigw run on an exported config, as a team runs it on a server.""" + folder.mkdir() + (folder / "config.yaml").write_text(config) + aigw_env = dict( + os.environ, + ENVOY_VERSION=local_gateway.ENVOY_VERSION, + AIGW_CONFIG_HOME=str(folder / "config-home"), + AIGW_STATE_HOME=str(folder / "state"), + AIGW_RUNTIME_DIR=str(local_gateway._runtime_dir()), + ) + with open(folder / "aigw.log", "wb") as log: + proc = subprocess.Popen( + [local_gateway._binary(), "run", str(folder / "config.yaml"), f"--admin-port={_free_port()}"], + stdout=log, + stderr=subprocess.STDOUT, + env=aigw_env, + start_new_session=True, + ) + try: + deadline = time.monotonic() + 120 + while not ( + "listening on" in (folder / "aigw.log").read_text(errors="replace") and local_gateway._listening(port) + ): + assert proc.poll() is None and time.monotonic() < deadline, (folder / "aigw.log").read_text()[-2000:] + time.sleep(0.2) + yield + finally: + os.killpg(proc.pid, signal.SIGTERM) + proc.wait(timeout=30) + + +@needs_aigw +@needs_xds_port +def test_router_code_runs_unchanged_against_an_exported_gateway(real_gateway, second_fake, env, monkeypatch): + """The same LiteLLM Router code on a laptop, then against a team gateway made with export_config().""" + + def build(): + return litellm.Router( + model_list=[ + {"model_name": "main", "litellm_params": {"model": "acme/err-500"}}, + {"model_name": "spare", "litellm_params": {"model": "acme2/acme-small"}}, + ], + fallbacks=[{"main": ["spare"]}], + num_retries=0, + ) + + laptop = build() + monkeypatch.setenv("AGENTROUTER_API_KEY", "acme-server-secret") + port = _free_port() + config = local_gateway.export_config(port=port, providers=["acme", "acme2"]) + with _team_gateway(env.parent / "server", config, port): + monkeypatch.setenv("AGENTROUTER_BASE_URL", f"http://127.0.0.1:{port}/v1") + monkeypatch.setenv("AGENTROUTER_AUTOSTART", "0") + monkeypatch.setattr(routes, "_for_gateway", {}) + warnings.simplefilter("ignore", UserWarning) # "the gateway's route of that name applies" + router = build() + # Once Envoy has looked up the backends' hostnames, the server's "main" falls back to acme2. + deadline = time.monotonic() + 60 + while not second_fake.requests: + assert time.monotonic() < deadline + with contextlib.suppress(exceptions.APIError): + r = router.completion(model="main", messages=HELLO) + time.sleep(0.5) + assert r.choices[0].message.content == "echo: refunds yesterday" + assert second_fake.requests[-1][2]["model"] == "acme-small" + # The Router made on the laptop, before AGENTROUTER_BASE_URL was set, sends the group's name too. + assert laptop.completion(model="main", messages=HELLO).choices[0].message.content == "echo: refunds yesterday" + assert local_gateway.running() is None # the SDK started nothing on this computer + # A client key the server does not have: Envoy refuses it in plain text, and the error names the setting. + monkeypatch.setenv("AGENTROUTER_API_KEY", "not-the-server-key") + with pytest.raises(exceptions.PermissionDeniedError, match="AGENTROUTER_API_KEY on its server"): + router.completion(model="main", messages=HELLO) + + @needs_aigw @needs_xds_port def test_real_gateway_tries_the_primary_first_every_time(real_gateway, second_fake): @@ -637,6 +888,23 @@ def test_real_gateway_serves_litellm_router(real_gateway, second_fake): assert [b["model"] for _, _, b in second_fake.requests] == ["acme-small"] +@needs_aigw +@needs_xds_port +def test_real_gateway_shares_a_router_group_by_rpm(real_gateway, second_fake): + # As in LiteLLM's simple-shuffle, a deployment with no rpm, beside one with rpm, gets no traffic. + router = litellm.Router( + model_list=[ + {"model_name": "chat", "litellm_params": {"model": "acme/acme-large", "rpm": 5}}, + {"model_name": "chat", "litellm_params": {"model": "acme2/acme-small"}}, + ], + num_retries=0, + ) + for _ in range(6): + router.completion(model="chat", messages=HELLO) + assert [b["model"] for _, _, b in real_gateway.requests] == ["acme-large"] * 6 + assert second_fake.requests == [] + + @needs_aigw @needs_xds_port def test_real_gateway_enforces_a_request_limit(real_gateway): diff --git a/tests/test_sdk.py b/tests/test_sdk.py index 3a4f354..7e4af12 100644 --- a/tests/test_sdk.py +++ b/tests/test_sdk.py @@ -358,6 +358,107 @@ def test_litellm_only_arguments_warn_once_and_are_not_sent(fake, monkeypatch): theagentrouter.completion(model="openai/gpt-4o-mini", messages=HELLO, api_base=fake.url, fallbacks=[]) +def test_api_base_says_once_that_the_call_skips_the_gateway(fake, monkeypatch): + from theagentrouter import routing + + monkeypatch.setattr(routing, "_api_base_warned", set()) + monkeypatch.setenv("AGENTROUTER_BASE_URL", "https://gw.acme.example/v1") # api_base still comes first + with pytest.warns( + UserWarning, match=r"api_base= sends these calls to .* in place of the gateway in AGENTROUTER_BASE_URL" + ): + r = theagentrouter.completion(model="openai/gpt-4o-mini", messages=HELLO, api_base=fake.url) + assert r._hidden_params["route"] == "api_base" + with warnings.catch_warnings(): + warnings.simplefilter("error") # the same address again: said once + theagentrouter.completion(model="openai/gpt-4o-mini", messages=HELLO, api_base=fake.url) + + +def test_api_base_naming_the_gateway_already_set_says_nothing(fake, monkeypatch): + from theagentrouter import routing + + monkeypatch.setattr(routing, "_api_base_warned", set()) + monkeypatch.setenv("AGENTROUTER_BASE_URL", fake.url) + with warnings.catch_warnings(): + warnings.simplefilter("error") + theagentrouter.completion(model="openai/gpt-4o-mini", messages=HELLO, api_base=fake.url + "/") + + +def test_a_gateway_refusing_the_client_key_names_the_setting(): + from dataclasses import replace + + from theagentrouter import routing + + target = routing.Target( + route="gateway", + format=routing.OPENAI, + base_url="https://gw.acme.example/v1", + api_key=None, + model="chat", + provider="openai", + ) + # Envoy refuses in plain text; the error says which setting to look at, and whether a key was sent. + unkeyed = routing._explain_refusal(403, "RBAC: access denied", target) + assert unkeyed.startswith("RBAC: access denied") and "no client key was sent: set AGENTROUTER_API_KEY" in unkeyed + keyed = replace(target, api_key="wrong") + said = routing._explain_refusal(403, "RBAC: access denied", keyed) + # Envoy gives the same 403 for other access rules, so the key is a possible cause, not a verdict. + assert "If that gateway checks a client key, AGENTROUTER_API_KEY may not be the key it expects" in said + assert "AGENTROUTER_API_KEY on its server" in said + error = exceptions.from_http(403, said, "chat", "openai") + assert isinstance(error, exceptions.PermissionDeniedError) and "AGENTROUTER_API_KEY" in str(error) + # A provider's own 403 passed on as JSON, another status, or another route: left as it is. + body = '{"error": {"message": "Project does not have access to model"}}' + assert routing._explain_refusal(403, body, keyed) == body + assert routing._explain_refusal(401, "nope", keyed) == "nope" + assert routing._explain_refusal(403, "nope", replace(target, route="local gateway")) == "nope" + + +def test_a_gateway_refusal_does_not_carry_the_provider_name(): + import httpx + + from theagentrouter import routing, transport + + target = routing.Target( + route="gateway", + format=routing.ANTHROPIC, + base_url="https://gw.acme.example/anthropic/v1", + api_key="wrong", + model="claude-sonnet-5-5", + provider="anthropic", + ) + request = httpx.Request("POST", "https://gw.acme.example/anthropic/v1/messages") + with pytest.raises(exceptions.PermissionDeniedError) as refused: + transport._raise_for( + httpx.Response(403, text="RBAC: access denied", request=request), + "RBAC: access denied", + target, + "claude-sonnet-5-5", + ) + # The gateway refused the key, not Anthropic: no "anthropic:" in front, but the error still says which provider. + assert refused.value.message.startswith("RBAC: access denied (") and refused.value.llm_provider == "anthropic" + # Anthropic's own 403, passed on by the gateway as JSON, keeps the provider name. + body = '{"type": "error", "error": {"type": "permission_error", "message": "Not allowed"}}' + with pytest.raises(exceptions.PermissionDeniedError) as provider_said: + transport._raise_for(httpx.Response(403, text=body, request=request), body, target, "claude-sonnet-5-5") + assert provider_said.value.message.startswith("anthropic: ") + + +def test_a_call_to_a_model_without_a_price_reports_no_cost_and_says_nothing(fake, monkeypatch): + # As in LiteLLM: a call's cost is left empty, with no warning, when the model has no known price, + # such as a route name on a gateway you set. + from theagentrouter import models + + monkeypatch.setattr(models, "_unknown_prices", set()) + monkeypatch.setenv("AGENTROUTER_BASE_URL", fake.url) + with warnings.catch_warnings(): + warnings.simplefilter("error") + r = theagentrouter.completion(model="chat", messages=HELLO) + assert r._hidden_params["response_cost"] is None + # Asked directly, completion_cost() still says so and reports 0. + with pytest.warns(UserWarning, match="No price is known"): + assert theagentrouter.completion_cost(r) == 0 + + def test_a_set_gateway_takes_its_own_model_names(fake, monkeypatch): monkeypatch.setenv("AGENTROUTER_BASE_URL", fake.url) r = theagentrouter.completion(model="claude-sonnet-5", messages=HELLO)