From ca772f4314df0851e1c90b57a6517393fb028a1a Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Thu, 12 Nov 2026 13:25:46 +0000 Subject: [PATCH 01/22] Log AitoolsMcpEvent from aitools mcp and the serve proxy The MCP commands and the proxy now buffer an AitoolsMcpEvent for the telemetry upload at exit: one result per (service, agent) for add and remove, one per service for login and logout, and PROXY_STARTED and SIGN_IN_PROMPTED from serve. Events carry enums and counts only; no host, profile, workspace ID, custom service or schema name, path or error text. The event mirrors the lumberjack schema for aitools MCP, which is not merged yet. The enum values for the MCP error categories are added next to the install ones, and the commands use them in place of local constants. Profile, sign-in and list failures that had no typed category now report NO_PROFILE, PROFILE_REQUIRED, MCP_PERMISSION_DENIED, MCP_LOCATION_NOT_FOUND and MCP_NAME_CONFLICT. The fake /telemetry-ext now reports the batch size, as the server does. Before, a batch of two or more events was resent by the CLI's retry. Co-authored-by: Isaac --- .../aitools/telemetry/mcp-add/.databrickscfg | 3 + .../aitools/telemetry/mcp-add/out.test.toml | 1 + .../cmd/aitools/telemetry/mcp-add/output.txt | 90 +++++++ .../cmd/aitools/telemetry/mcp-add/script | 29 +++ .../telemetry/mcp-remove/.databrickscfg | 7 + .../telemetry/mcp-remove/out.test.toml | 1 + .../aitools/telemetry/mcp-remove/output.txt | 70 ++++++ .../cmd/aitools/telemetry/mcp-remove/script | 34 +++ .../telemetry/mcp-serve/.databrickscfg | 3 + .../aitools/telemetry/mcp-serve/out.test.toml | 1 + .../aitools/telemetry/mcp-serve/output.txt | 38 +++ .../telemetry/mcp-serve/requests-after.ndjson | 2 + .../mcp-serve/requests-authenticate.ndjson | 1 + .../telemetry/mcp-serve/requests-list.ndjson | 3 + .../cmd/aitools/telemetry/mcp-serve/script | 40 +++ .../telemetry/mcp-sign-in/.databrickscfg | 3 + .../telemetry/mcp-sign-in/out.test.toml | 1 + .../aitools/telemetry/mcp-sign-in/output.txt | 98 ++++++++ .../cmd/aitools/telemetry/mcp-sign-in/script | 51 ++++ cmd/aitools/mcp.go | 21 +- cmd/aitools/mcp_add.go | 31 ++- cmd/aitools/mcp_add_test.go | 28 +-- cmd/aitools/mcp_login.go | 40 ++- cmd/aitools/mcp_login_test.go | 4 +- cmd/aitools/mcp_remove.go | 18 +- cmd/aitools/mcp_remove_test.go | 10 +- cmd/aitools/mcp_serve.go | 10 + cmd/aitools/mcp_telemetry.go | 228 +++++++++++++++++ cmd/aitools/mcp_telemetry_test.go | 235 ++++++++++++++++++ cmd/aitools/mcp_test.go | 10 +- libs/aitools/mcp/proxy/bridge_test.go | 2 +- libs/aitools/mcp/proxy/challenge.go | 17 +- libs/aitools/mcp/proxy/challenge_test.go | 52 +++- libs/telemetry/protos/aitools_install.go | 17 ++ libs/telemetry/protos/aitools_mcp.go | 94 +++++++ libs/telemetry/protos/aitools_mcp_test.go | 57 +++++ libs/telemetry/protos/frontend_log.go | 1 + 37 files changed, 1287 insertions(+), 64 deletions(-) create mode 100644 acceptance/cmd/aitools/telemetry/mcp-add/.databrickscfg create mode 100644 acceptance/cmd/aitools/telemetry/mcp-add/out.test.toml create mode 100644 acceptance/cmd/aitools/telemetry/mcp-add/output.txt create mode 100644 acceptance/cmd/aitools/telemetry/mcp-add/script create mode 100644 acceptance/cmd/aitools/telemetry/mcp-remove/.databrickscfg create mode 100644 acceptance/cmd/aitools/telemetry/mcp-remove/out.test.toml create mode 100644 acceptance/cmd/aitools/telemetry/mcp-remove/output.txt create mode 100644 acceptance/cmd/aitools/telemetry/mcp-remove/script create mode 100644 acceptance/cmd/aitools/telemetry/mcp-serve/.databrickscfg create mode 100644 acceptance/cmd/aitools/telemetry/mcp-serve/out.test.toml create mode 100644 acceptance/cmd/aitools/telemetry/mcp-serve/output.txt create mode 100644 acceptance/cmd/aitools/telemetry/mcp-serve/requests-after.ndjson create mode 100644 acceptance/cmd/aitools/telemetry/mcp-serve/requests-authenticate.ndjson create mode 100644 acceptance/cmd/aitools/telemetry/mcp-serve/requests-list.ndjson create mode 100644 acceptance/cmd/aitools/telemetry/mcp-serve/script create mode 100644 acceptance/cmd/aitools/telemetry/mcp-sign-in/.databrickscfg create mode 100644 acceptance/cmd/aitools/telemetry/mcp-sign-in/out.test.toml create mode 100644 acceptance/cmd/aitools/telemetry/mcp-sign-in/output.txt create mode 100644 acceptance/cmd/aitools/telemetry/mcp-sign-in/script create mode 100644 cmd/aitools/mcp_telemetry.go create mode 100644 cmd/aitools/mcp_telemetry_test.go create mode 100644 libs/telemetry/protos/aitools_mcp.go create mode 100644 libs/telemetry/protos/aitools_mcp_test.go diff --git a/acceptance/cmd/aitools/telemetry/mcp-add/.databrickscfg b/acceptance/cmd/aitools/telemetry/mcp-add/.databrickscfg new file mode 100644 index 00000000000..0369be633c5 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-add/.databrickscfg @@ -0,0 +1,3 @@ +[logfood] +host = $DATABRICKS_HOST +token = test-token-logfood diff --git a/acceptance/cmd/aitools/telemetry/mcp-add/out.test.toml b/acceptance/cmd/aitools/telemetry/mcp-add/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-add/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/cmd/aitools/telemetry/mcp-add/output.txt b/acceptance/cmd/aitools/telemetry/mcp-add/output.txt new file mode 100644 index 00000000000..b7ea889e8b8 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-add/output.txt @@ -0,0 +1,90 @@ + +=== A new install for two agents: CHANGED per (service, agent); github needs sign-in, so its sign-in state is PENDING +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "ADD", + "servers": [ + { + "service": "GENIE_ONE", + "agent": "CLAUDE_CODE", + "outcome": "CHANGED", + "sign_in_state": "NOT_REQUIRED" + }, + { + "service": "GENIE_ONE", + "agent": "CODEX", + "outcome": "CHANGED", + "sign_in_state": "NOT_REQUIRED" + }, + { + "service": "GITHUB", + "agent": "CLAUDE_CODE", + "outcome": "CHANGED", + "sign_in_state": "PENDING" + }, + { + "service": "GITHUB", + "agent": "CODEX", + "outcome": "CHANGED", + "sign_in_state": "PENDING" + } + ], + "bound_profile_count": 1 +} + +=== The same servers again are re-asserted: UNCHANGED +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "ADD", + "servers": [ + { + "service": "GENIE_ONE", + "agent": "CLAUDE_CODE", + "outcome": "UNCHANGED", + "sign_in_state": "NOT_REQUIRED" + }, + { + "service": "GENIE_ONE", + "agent": "CODEX", + "outcome": "UNCHANGED", + "sign_in_state": "NOT_REQUIRED" + }, + { + "service": "GITHUB", + "agent": "CLAUDE_CODE", + "outcome": "UNCHANGED", + "sign_in_state": "PENDING" + }, + { + "service": "GITHUB", + "agent": "CODEX", + "outcome": "UNCHANGED", + "sign_in_state": "PENDING" + } + ], + "bound_profile_count": 1 +} + +=== An agent config that does not parse fails that agent only: FAILED with AGENT_CONFIG_INVALID + +Exit code: 1 +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "ADD", + "servers": [ + { + "service": "GENIE_ONE", + "agent": "CLAUDE_CODE", + "outcome": "UNCHANGED", + "sign_in_state": "NOT_REQUIRED" + }, + { + "service": "GENIE_ONE", + "agent": "CODEX", + "outcome": "FAILED", + "error_category": "AGENT_CONFIG_INVALID", + "sign_in_state": "NOT_REQUIRED" + } + ], + "bound_profile_count": 1 +} diff --git a/acceptance/cmd/aitools/telemetry/mcp-add/script b/acceptance/cmd/aitools/telemetry/mcp-add/script new file mode 100644 index 00000000000..83d6b223112 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-add/script @@ -0,0 +1,29 @@ +sethome home +envsubst < .databrickscfg > out && mv out .databrickscfg +export DATABRICKS_CONFIG_FILE=.databrickscfg + +# Prints the aitools_mcp_event payloads uploaded since the last call, then forgets the recorded uploads. +mcp_events() { + if [ -f "$OUT_REQUESTS" ]; then + jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.aitools_mcp_event | select(. != null)' "$OUT_REQUESTS" + rm -f "$OUT_REQUESTS" + fi +} + +# Seeding the fake is not an MCP command, so its uploads (if any) are dropped here. +$CLI api post "/api/2.1/unity-catalog/mcp-services?parent=schemas/system.ai&mcp_service_id=genie_one_mcp" --profile logfood --json '{"comment":"Ask questions about your data"}' > /dev/null +$CLI api post "/api/2.1/unity-catalog/mcp-services?parent=schemas/system.ai&mcp_service_id=github" --profile logfood --json '{"comment":"Repositories, issues, pull requests","config":{"source_connection":{"name":"connections/github","options":{"oauth_provider":"github"}}}}' > /dev/null +rm -f "$OUT_REQUESTS" + +title "A new install for two agents: CHANGED per (service, agent); github needs sign-in, so its sign-in state is PENDING" +$CLI aitools mcp add --profile logfood --names system.ai.genie_one_mcp,system.ai.github --agents claude-code,codex --output json > /dev/null +mcp_events + +title "The same servers again are re-asserted: UNCHANGED" +$CLI aitools mcp add --profile logfood --names system.ai.genie_one_mcp,system.ai.github --agents claude-code,codex --output json > /dev/null +mcp_events + +title "An agent config that does not parse fails that agent only: FAILED with AGENT_CONFIG_INVALID" +printf 'not = [valid\n' > home/.codex/config.toml +errcode $CLI aitools mcp add --profile logfood --names system.ai.genie_one_mcp --agents claude-code,codex --output json > /dev/null +mcp_events diff --git a/acceptance/cmd/aitools/telemetry/mcp-remove/.databrickscfg b/acceptance/cmd/aitools/telemetry/mcp-remove/.databrickscfg new file mode 100644 index 00000000000..d64cd1da28f --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-remove/.databrickscfg @@ -0,0 +1,7 @@ +[logfood] +host = $DATABRICKS_HOST +token = test-token-logfood + +[dogfood] +host = $DATABRICKS_HOST +token = test-token-logfood diff --git a/acceptance/cmd/aitools/telemetry/mcp-remove/out.test.toml b/acceptance/cmd/aitools/telemetry/mcp-remove/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-remove/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/cmd/aitools/telemetry/mcp-remove/output.txt b/acceptance/cmd/aitools/telemetry/mcp-remove/output.txt new file mode 100644 index 00000000000..6ab420b2562 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-remove/output.txt @@ -0,0 +1,70 @@ + +=== Installed for two profiles: the bound profile count is 2 after the second add +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "ADD", + "servers": [ + { + "service": "GENIE_ONE", + "agent": "CLAUDE_CODE", + "outcome": "CHANGED", + "sign_in_state": "NOT_REQUIRED" + } + ], + "bound_profile_count": 1 +} +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "ADD", + "servers": [ + { + "service": "GENIE_ONE", + "agent": "CLAUDE_CODE", + "outcome": "CHANGED", + "sign_in_state": "NOT_REQUIRED" + } + ], + "bound_profile_count": 2 +} + +=== remove --profile dogfood: CHANGED for its agent entry, and one profile stays bound +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "REMOVE", + "servers": [ + { + "service": "GENIE_ONE", + "agent": "CLAUDE_CODE", + "outcome": "CHANGED" + } + ], + "bound_profile_count": 1 +} + +=== An entry changed outside this CLI is left in place: SKIPPED +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "REMOVE", + "servers": [ + { + "service": "GENIE_ONE", + "agent": "CLAUDE_CODE", + "outcome": "SKIPPED" + } + ], + "bound_profile_count": 1 +} + +=== An entry already gone from the agent config is SKIPPED, and the state record goes +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "REMOVE", + "servers": [ + { + "service": "GENIE_ONE", + "agent": "CLAUDE_CODE", + "outcome": "SKIPPED" + } + ], + "bound_profile_count": 0 +} diff --git a/acceptance/cmd/aitools/telemetry/mcp-remove/script b/acceptance/cmd/aitools/telemetry/mcp-remove/script new file mode 100644 index 00000000000..b25e4b4a31a --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-remove/script @@ -0,0 +1,34 @@ +sethome home +envsubst < .databrickscfg > out && mv out .databrickscfg +export DATABRICKS_CONFIG_FILE=.databrickscfg + +# Prints the aitools_mcp_event payloads uploaded since the last call, then forgets the recorded uploads. +mcp_events() { + if [ -f "$OUT_REQUESTS" ]; then + jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.aitools_mcp_event | select(. != null)' "$OUT_REQUESTS" + rm -f "$OUT_REQUESTS" + fi +} + +$CLI api post "/api/2.1/unity-catalog/mcp-services?parent=schemas/system.ai&mcp_service_id=genie_one_mcp" --profile logfood --json '{"comment":"Ask questions about your data"}' > /dev/null +rm -f "$OUT_REQUESTS" + +title "Installed for two profiles: the bound profile count is 2 after the second add" +$CLI aitools mcp add --profile logfood --names system.ai.genie_one_mcp --agents claude-code --output json > /dev/null +mcp_events +$CLI aitools mcp add --profile dogfood --names system.ai.genie_one_mcp --agents claude-code --output json > /dev/null +mcp_events + +title "remove --profile dogfood: CHANGED for its agent entry, and one profile stays bound" +$CLI aitools mcp remove --profile dogfood --output json > /dev/null +mcp_events + +title "An entry changed outside this CLI is left in place: SKIPPED" +jq '.mcpServers["genie_one_mcp-logfood"].command = "/usr/bin/other"' home/.claude.json > home/claude.tmp && mv home/claude.tmp home/.claude.json +$CLI aitools mcp remove --names genie_one_mcp-logfood --profile logfood --output json > /dev/null +mcp_events + +title "An entry already gone from the agent config is SKIPPED, and the state record goes" +jq 'del(.mcpServers["genie_one_mcp-logfood"])' home/.claude.json > home/claude.tmp && mv home/claude.tmp home/.claude.json +$CLI aitools mcp remove --names genie_one_mcp-logfood --profile logfood --output json > /dev/null +mcp_events diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve/.databrickscfg b/acceptance/cmd/aitools/telemetry/mcp-serve/.databrickscfg new file mode 100644 index 00000000000..0369be633c5 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-serve/.databrickscfg @@ -0,0 +1,3 @@ +[logfood] +host = $DATABRICKS_HOST +token = test-token-logfood diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve/out.test.toml b/acceptance/cmd/aitools/telemetry/mcp-serve/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-serve/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve/output.txt b/acceptance/cmd/aitools/telemetry/mcp-serve/output.txt new file mode 100644 index 00000000000..08586c94def --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-serve/output.txt @@ -0,0 +1,38 @@ + +=== serve logs PROXY_STARTED after the binding checks; the upload is sent when stdin closes +{ + "calling_agent": "CLAUDE_CODE", + "operation": "PROXY_STARTED", + "servers": [ + { + "service": "GENIE_ONE" + } + ], + "bound_profile_count": 0 +} + +=== A binding that fails logs PROXY_STARTED with its category, but no upload follows: there is no config to upload with + +=== authenticate opens the sign-in page: PROXY_STARTED, then SIGN_IN_PROMPTED with the sign-in state after the wait +{ + "calling_agent": "CLAUDE_CODE", + "operation": "PROXY_STARTED", + "servers": [ + { + "service": "GITHUB" + } + ], + "bound_profile_count": 0 +} +{ + "calling_agent": "CLAUDE_CODE", + "operation": "SIGN_IN_PROMPTED", + "servers": [ + { + "service": "GITHUB", + "error_category": "SIGN_IN_REQUIRED", + "sign_in_state": "ACTIVE" + } + ], + "bound_profile_count": 0 +} diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve/requests-after.ndjson b/acceptance/cmd/aitools/telemetry/mcp-serve/requests-after.ndjson new file mode 100644 index 00000000000..57d5e4a4066 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-serve/requests-after.ndjson @@ -0,0 +1,2 @@ +{"jsonrpc":"2.0","id":4,"method":"tools/list"} +{"jsonrpc":"2.0","id":5,"method":"tools/call","params":{"name":"echo","arguments":{"text":"hello"}}} diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve/requests-authenticate.ndjson b/acceptance/cmd/aitools/telemetry/mcp-serve/requests-authenticate.ndjson new file mode 100644 index 00000000000..cc792544a79 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-serve/requests-authenticate.ndjson @@ -0,0 +1 @@ +{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"authenticate","arguments":{}}} diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve/requests-list.ndjson b/acceptance/cmd/aitools/telemetry/mcp-serve/requests-list.ndjson new file mode 100644 index 00000000000..84c14278376 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-serve/requests-list.ndjson @@ -0,0 +1,3 @@ +{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-06-18","capabilities":{},"clientInfo":{"name":"test-client","version":"0.0.1"}}} +{"jsonrpc":"2.0","method":"notifications/initialized"} +{"jsonrpc":"2.0","id":2,"method":"tools/list"} diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve/script b/acceptance/cmd/aitools/telemetry/mcp-serve/script new file mode 100644 index 00000000000..fdceb646167 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-serve/script @@ -0,0 +1,40 @@ +sethome home +envsubst < .databrickscfg > out && mv out .databrickscfg +export DATABRICKS_CONFIG_FILE=.databrickscfg + +# No browser: the proxy prints the sign-in URL and does not open anything. +export BROWSER=none +export RETRY_MAX_ATTEMPTS=60 +# Prints the aitools_mcp_event payloads uploaded since the last call, then forgets the recorded uploads. +mcp_events() { + if [ -f "$OUT_REQUESTS" ]; then + jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.aitools_mcp_event | select(. != null)' "$OUT_REQUESTS" + rm -f "$OUT_REQUESTS" + fi +} + +# Seeding the fake is not an MCP command, so its uploads (if any) are dropped here. +$CLI api post "/api/2.1/unity-catalog/mcp-services?parent=schemas/system.ai&mcp_service_id=genie_one_mcp" --profile logfood --json '{"comment":"Ask questions about your data"}' > /dev/null +$CLI api post "/api/2.1/unity-catalog/mcp-services?parent=schemas/system.ai&mcp_service_id=github" --profile logfood --json '{"comment":"Repositories, issues, pull requests","config":{"source_connection":{"name":"connections/github","options":{"oauth_provider":"github"}}}}' > /dev/null +rm -f "$OUT_REQUESTS" + +title "serve logs PROXY_STARTED after the binding checks; the upload is sent when stdin closes" +$CLI aitools mcp serve --profile logfood --service system.ai.genie_one_mcp --agent claude-code < requests-list.ndjson > /dev/null 2> /dev/null +mcp_events + +title "A binding that fails logs PROXY_STARTED with its category, but no upload follows: there is no config to upload with" +errcode $CLI aitools mcp serve --profile gone --service system.ai.genie_one_mcp --agent codex < /dev/null > /dev/null 2> /dev/null +mcp_events + +title "authenticate opens the sign-in page: PROXY_STARTED, then SIGN_IN_PROMPTED with the sign-in state after the wait" +{ + cat requests-list.ndjson + retry --until '"id":2' cat serve.stdout.txt > /dev/null + cat requests-authenticate.ndjson + retry --until "Waiting for sign-in" cat serve.stderr.txt > /dev/null + $CLI api post /api/2.1/unity-catalog/mcp-services/system.ai.github/user-credentials --profile logfood --json '{}' > /dev/null + retry --until '"id":3' cat serve.stdout.txt > /dev/null + cat requests-after.ndjson +} | $CLI aitools mcp serve --profile logfood --service system.ai.github --agent claude-code > serve.stdout.txt 2> serve.stderr.txt +mcp_events +rm serve.stdout.txt serve.stderr.txt diff --git a/acceptance/cmd/aitools/telemetry/mcp-sign-in/.databrickscfg b/acceptance/cmd/aitools/telemetry/mcp-sign-in/.databrickscfg new file mode 100644 index 00000000000..0369be633c5 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-sign-in/.databrickscfg @@ -0,0 +1,3 @@ +[logfood] +host = $DATABRICKS_HOST +token = test-token-logfood diff --git a/acceptance/cmd/aitools/telemetry/mcp-sign-in/out.test.toml b/acceptance/cmd/aitools/telemetry/mcp-sign-in/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-sign-in/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/cmd/aitools/telemetry/mcp-sign-in/output.txt b/acceptance/cmd/aitools/telemetry/mcp-sign-in/output.txt new file mode 100644 index 00000000000..d2b56d441f2 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-sign-in/output.txt @@ -0,0 +1,98 @@ + +=== login to a service that needs no sign-in: UNCHANGED, sign-in state NOT_REQUIRED +system.ai.genie_one_mcp does not need sign-in. +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "SIGN_IN", + "servers": [ + { + "service": "GENIE_ONE", + "outcome": "UNCHANGED", + "sign_in_state": "NOT_REQUIRED" + } + ], + "bound_profile_count": 0 +} + +=== login completes once the credential is ACTIVE: CHANGED, sign-in state ACTIVE +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "SIGN_IN", + "servers": [ + { + "service": "GITHUB", + "outcome": "CHANGED", + "sign_in_state": "ACTIVE" + } + ], + "bound_profile_count": 0 +} + +=== login while already signed in: UNCHANGED +Already signed in to system.ai.github on profile logfood. +Public repositories only until a repository owner installs https://github.com/apps/databricks-github-connector. +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "SIGN_IN", + "servers": [ + { + "service": "GITHUB", + "outcome": "UNCHANGED", + "sign_in_state": "ACTIVE" + } + ], + "bound_profile_count": 0 +} + +=== logout removes the sign-in: CHANGED; a second logout finds nothing: UNCHANGED +Signed out of system.ai.github on profile logfood. +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "SIGN_OUT", + "servers": [ + { + "service": "GITHUB", + "outcome": "CHANGED" + } + ], + "bound_profile_count": 0 +} +Not signed in to system.ai.github on profile logfood. +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "SIGN_OUT", + "servers": [ + { + "service": "GITHUB", + "outcome": "UNCHANGED" + } + ], + "bound_profile_count": 0 +} + +=== add --login signs in during the same run: SIGN_IN first, then ADD with sign-in state ACTIVE +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "SIGN_IN", + "servers": [ + { + "service": "GITHUB", + "outcome": "CHANGED", + "sign_in_state": "ACTIVE" + } + ], + "bound_profile_count": 1 +} +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "ADD", + "servers": [ + { + "service": "GITHUB", + "agent": "CLAUDE_CODE", + "outcome": "CHANGED", + "sign_in_state": "ACTIVE" + } + ], + "bound_profile_count": 1 +} diff --git a/acceptance/cmd/aitools/telemetry/mcp-sign-in/script b/acceptance/cmd/aitools/telemetry/mcp-sign-in/script new file mode 100644 index 00000000000..943af911b86 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-sign-in/script @@ -0,0 +1,51 @@ +sethome home +envsubst < .databrickscfg > out && mv out .databrickscfg +export DATABRICKS_CONFIG_FILE=.databrickscfg + +# No browser: the CLI prints the sign-in URL and polls. +export BROWSER=none +export RETRY_MAX_ATTEMPTS=60 +# Prints the aitools_mcp_event payloads uploaded since the last call, then forgets the recorded uploads. +mcp_events() { + if [ -f "$OUT_REQUESTS" ]; then + jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.aitools_mcp_event | select(. != null)' "$OUT_REQUESTS" + rm -f "$OUT_REQUESTS" + fi +} + +# Seeding the fake is not an MCP command, so its uploads (if any) are dropped here. +$CLI api post "/api/2.1/unity-catalog/mcp-services?parent=schemas/system.ai&mcp_service_id=genie_one_mcp" --profile logfood --json '{"comment":"Ask questions about your data"}' > /dev/null +$CLI api post "/api/2.1/unity-catalog/mcp-services?parent=schemas/system.ai&mcp_service_id=github" --profile logfood --json '{"comment":"Repositories, issues, pull requests","config":{"source_connection":{"name":"connections/github","options":{"oauth_provider":"github"}}}}' > /dev/null +rm -f "$OUT_REQUESTS" + +title "login to a service that needs no sign-in: UNCHANGED, sign-in state NOT_REQUIRED" +$CLI aitools mcp login system.ai.genie_one_mcp --profile logfood > /dev/null +mcp_events + +title "login completes once the credential is ACTIVE: CHANGED, sign-in state ACTIVE" +$CLI aitools mcp login system.ai.github --profile logfood > login.txt 2>&1 & +PID=$! +retry --until "Waiting for sign-in" cat login.txt > /dev/null +$CLI api post /api/2.1/unity-catalog/mcp-services/system.ai.github/user-credentials --profile logfood --json '{}' > /dev/null +wait $PID +mcp_events +rm login.txt + +title "login while already signed in: UNCHANGED" +$CLI aitools mcp login system.ai.github --profile logfood > /dev/null +mcp_events + +title "logout removes the sign-in: CHANGED; a second logout finds nothing: UNCHANGED" +$CLI aitools mcp logout system.ai.github --profile logfood > /dev/null +mcp_events +$CLI aitools mcp logout system.ai.github --profile logfood > /dev/null +mcp_events + +title "add --login signs in during the same run: SIGN_IN first, then ADD with sign-in state ACTIVE" +$CLI aitools mcp add --profile logfood --names system.ai.github --agents claude-code --login > add.txt 2>&1 & +PID=$! +retry --until "Waiting for sign-in" cat add.txt > /dev/null +$CLI api post /api/2.1/unity-catalog/mcp-services/system.ai.github/user-credentials --profile logfood --json '{}' > /dev/null +wait $PID +mcp_events +rm add.txt diff --git a/cmd/aitools/mcp.go b/cmd/aitools/mcp.go index 1f81d5fb0d0..7552c8be62f 100644 --- a/cmd/aitools/mcp.go +++ b/cmd/aitools/mcp.go @@ -16,19 +16,6 @@ import ( "github.com/spf13/cobra" ) -// Error categories that the telemetry enum does not define yet. The JSON output reports them by name. The -// binding categories come from mcp.BindCategory, which has the same names. -const ( - categoryAgentConfigInvalid protos.AitoolsErrorCategory = "AGENT_CONFIG_INVALID" - categoryNameConflict protos.AitoolsErrorCategory = "MCP_NAME_CONFLICT" - categoryServiceNotFound protos.AitoolsErrorCategory = "MCP_SERVICE_NOT_FOUND" - categoryNoProfile protos.AitoolsErrorCategory = "NO_PROFILE" - categoryProfileRequired protos.AitoolsErrorCategory = "PROFILE_REQUIRED" - categoryPermissionDenied protos.AitoolsErrorCategory = "MCP_PERMISSION_DENIED" - categoryLocationNotFound protos.AitoolsErrorCategory = "MCP_LOCATION_NOT_FOUND" - categorySignInTimeout protos.AitoolsErrorCategory = "SIGN_IN_TIMEOUT" -) - // NewMcpCmd returns the `aitools mcp` group. It stays hidden until the whole MCP // stack has landed, so a release never ships `add` without `serve`. func NewMcpCmd() *cobra.Command { @@ -75,9 +62,9 @@ func errorCategory(err error) protos.AitoolsErrorCategory { } switch { case errors.Is(err, agentconfig.ErrConfigInvalid): - return categoryAgentConfigInvalid + return protos.AitoolsErrorCategoryAgentConfigInvalid case errors.Is(err, agentconfig.ErrNotOwned): - return categoryNameConflict + return protos.AitoolsErrorCategoryMCPNameConflict } return classifyInstallError(err) } @@ -107,11 +94,11 @@ func resolveProfile(ctx context.Context, flagValue string) (string, error) { } switch len(profiles) { case 0: - return "", failure(categoryNoProfile, nil, "no workspace profiles in %s. Run: databricks auth login --host --profile ", filepath.ToSlash(path)) + return "", failure(protos.AitoolsErrorCategoryNoProfile, nil, "no workspace profiles in %s. Run: databricks auth login --host --profile ", filepath.ToSlash(path)) case 1: return profiles[0].Name, nil } - return "", failure(categoryProfileRequired, &profileChoice{profiles: profiles}, "several workspace profiles exist; pass --profile. Found: %s", strings.Join(profiles.Names(), ", ")) + return "", failure(protos.AitoolsErrorCategoryProfileRequired, &profileChoice{profiles: profiles}, "several workspace profiles exist; pass --profile. Found: %s", strings.Join(profiles.Names(), ", ")) } // profileFlag returns the root --profile value, or "" when it is not set. diff --git a/cmd/aitools/mcp_add.go b/cmd/aitools/mcp_add.go index b97c9e3eaca..9cb504bb423 100644 --- a/cmd/aitools/mcp_add.go +++ b/cmd/aitools/mcp_add.go @@ -19,6 +19,7 @@ import ( "github.com/databricks/cli/libs/aitools/mcpconfig" "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/browser" + "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/telemetry/protos" @@ -177,6 +178,11 @@ Supported agents: ` + strings.Join(mcpAgentNames(), ", "), ErrorCategory: protos.AitoolsErrorCategoryUnspecified, } runErr := runMcpAdd(ctx, cmd, f, jsonMode, out) + logMcpEvent(ctx, protos.AitoolsMcpEvent{ + CallingAgent: callingAgentType(), + Operation: protos.AitoolsMcpOperationAdd, + Servers: addServerResults(out.Servers), + }) if !jsonMode { return runErr } @@ -342,6 +348,8 @@ func runMcpAdd(ctx context.Context, cmd *cobra.Command, f mcpAddFlags, jsonMode return err } } + // The bound profile authenticates the exit-time telemetry upload. + cmd.SetContext(cmdctx.SetConfigUsed(cmd.Context(), w.Config)) for i := range plans { plans[i].service = picked[i] } @@ -542,9 +550,9 @@ func describeListError(profileName string, err error) error { var category protos.AitoolsErrorCategory switch listErr.Kind { case mcp.ListErrLocationNotFound: - category = categoryLocationNotFound + category = protos.AitoolsErrorCategoryMCPLocationNotFound case mcp.ListErrPermissionDenied: - category = categoryPermissionDenied + category = protos.AitoolsErrorCategoryMCPPermissionDenied case mcp.ListErrRateLimited: category = protos.AitoolsErrorCategoryRateLimited default: @@ -705,13 +713,13 @@ func pickServices(location string, names []string, services []mcp.Service) ([]mc i := slices.IndexFunc(services, func(s mcp.Service) bool { return strings.EqualFold(s.FullName, name) }) if i < 0 { if len(services) == 0 { - return nil, failure(categoryServiceNotFound, nil, "%s is not an MCP service in %s, which has none", name, location) + return nil, failure(protos.AitoolsErrorCategoryMCPServiceNotFound, nil, "%s is not an MCP service in %s, which has none", name, location) } available := make([]string, 0, len(services)) for _, s := range services { available = append(available, s.FullName) } - return nil, failure(categoryServiceNotFound, nil, "%s is not an MCP service in %s. Available: %s", name, location, strings.Join(available, ", ")) + return nil, failure(protos.AitoolsErrorCategoryMCPServiceNotFound, nil, "%s is not an MCP service in %s. Available: %s", name, location, strings.Join(available, ", ")) } picked = append(picked, services[i]) } @@ -846,7 +854,7 @@ func chooseName(ctx context.Context, in planInput, state *mcp.State, fullName st case 1: stored = owned[0] default: - return "", nil, failure(categoryNameConflict, nil, "%s is installed on profile %s under more than one name in the agents (%s); remove the extra entries", fullName, b.profile, strings.Join(owned, ", ")) + return "", nil, failure(protos.AitoolsErrorCategoryMCPNameConflict, nil, "%s is installed on profile %s under more than one name in the agents (%s); remove the extra entries", fullName, b.profile, strings.Join(owned, ", ")) } } @@ -855,7 +863,7 @@ func chooseName(ctx context.Context, in planInput, state *mcp.State, fullName st return "", nil, failure(protos.AitoolsErrorCategoryInvalidFlags, nil, "%s is already installed on profile %s as %s; remove it before using --name %s", fullName, b.profile, stored, in.nameFlag) } if stateHolds(state, stored, entry) { - return "", nil, failure(categoryNameConflict, nil, "%s is installed as %s, but another server in the state holds %s; remove that entry first", fullName, stored, stored) + return "", nil, failure(protos.AitoolsErrorCategoryMCPNameConflict, nil, "%s is installed as %s, but another server in the state holds %s; remove that entry first", fullName, stored, stored) } failed, err := nameFailures(ctx, in, fullName, stored, true) return stored, failed, err @@ -943,7 +951,7 @@ func holderConflict(ctx context.Context, in planInput, a *agents.Agent, fullName if in.force { return nil, nil } - return failure(categoryNameConflict, agentconfig.ErrNotOwned, "%s already has a server named %s that this CLI did not write; pass --force to replace it", a.DisplayName, name), nil + return failure(protos.AitoolsErrorCategoryMCPNameConflict, agentconfig.ErrNotOwned, "%s already has a server named %s that this CLI did not write; pass --force to replace it", a.DisplayName, name), nil } serves, err := mcp.EntryServes(ctx, entry, fullName, in.binding.profile, in.binding.configFile) if err != nil { @@ -953,7 +961,7 @@ func holderConflict(ctx context.Context, in planInput, a *agents.Agent, fullName case serves: return nil, nil case stored: - return failure(categoryNameConflict, nil, "%s is installed as %s, but %s holds %s for another server; remove that entry first", fullName, name, a.Name, name), nil + return failure(protos.AitoolsErrorCategoryMCPNameConflict, nil, "%s is installed as %s, but %s holds %s for another server; remove that entry first", fullName, name, a.Name, name), nil default: return failure(protos.AitoolsErrorCategoryInvalidFlags, nil, "--name %s is used by another server in %s; pick a different name", name, a.Name), nil } @@ -1090,8 +1098,13 @@ func signInInstalled(ctx context.Context, errOut io.Writer, w *databricks.Worksp } } url := mcp.SignInURL(w.Config.Host, plans[i].service.FullName, req.workspaceID) - state, _, _, err := signIn(ctx, errOut, w, req.pinned, url, plans[i].service, signInOptions{Profile: req.profile, Open: browser.OpenOutputToStderr}) + state, alreadySignedIn, _, err := signIn(ctx, errOut, w, req.pinned, url, plans[i].service, signInOptions{Profile: req.profile, Open: browser.OpenOutputToStderr}) server.SignIn = &addSignInOutput{State: state} + logMcpEvent(ctx, protos.AitoolsMcpEvent{ + CallingAgent: callingAgentType(), + Operation: protos.AitoolsMcpOperationSignIn, + Servers: []protos.AitoolsMcpServerResult{signInServerResult(server.Service, state, alreadySignedIn, err)}, + }) if err != nil { server.SignIn.ErrorCategory = errorCategory(err) server.SignIn.Error = err.Error() diff --git a/cmd/aitools/mcp_add_test.go b/cmd/aitools/mcp_add_test.go index 05c89ca7509..312d2dd5fc4 100644 --- a/cmd/aitools/mcp_add_test.go +++ b/cmd/aitools/mcp_add_test.go @@ -201,7 +201,7 @@ func TestPickServices(t *testing.T) { _, err = pickServices("system.ai", []string{"system.ai.foo"}, services) require.Error(t, err) assert.Equal(t, "system.ai.foo is not an MCP service in system.ai. Available: system.ai.genie_one_mcp, system.ai.github", err.Error()) - assert.Equal(t, categoryServiceNotFound, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPServiceNotFound, errorCategory(err)) } func TestResolveMCPAgents(t *testing.T) { @@ -227,7 +227,7 @@ func TestDescribeListErrorNamesProfileAndKeepsCause(t *testing.T) { err := describeListError("logfood", listErr) assert.Equal(t, "profile logfood: "+listErr.Error(), err.Error()) - assert.Equal(t, categoryPermissionDenied, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPPermissionDenied, errorCategory(err)) got, ok := errors.AsType[*mcp.ListError](err) require.True(t, ok) @@ -238,7 +238,7 @@ func TestDescribeListErrorNamesProfileAndKeepsCause(t *testing.T) { assert.Equal(t, protos.AitoolsErrorCategoryRateLimited, errorCategory(rateLimited)) notFound := describeListError("logfood", &mcp.ListError{Kind: mcp.ListErrLocationNotFound, Location: "main.tools", Err: cause}) - assert.Equal(t, categoryLocationNotFound, errorCategory(notFound)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPLocationNotFound, errorCategory(notFound)) plain := errors.New("network down") assert.Equal(t, plain, describeListError("logfood", plain)) @@ -517,7 +517,7 @@ func TestPlanAddsNaming(t *testing.T) { require.NoError(t, os.WriteFile(filepath.Join(openCode, "opencode.json"), []byte(`{"mcp":{"servers":{"two":{"type":"local","command":["databricks","aitools","mcp","serve","--profile","logfood","--service","system.ai.github","--agent","opencode"]}}}}`), 0o600)) _, err := planNames(t, "", defaultBinding(t, "logfood"), &mcp.State{}, []string{"system.ai.github"}, []*agents.Agent{claude, agents.ByName("opencode")}) require.Error(t, err) - assert.Equal(t, categoryNameConflict, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPNameConflict, errorCategory(err)) }) t.Run("OpenCode entry with its variables in environment is reused", func(t *testing.T) { @@ -560,7 +560,7 @@ func TestPlanAddsNaming(t *testing.T) { require.Len(t, plans, 1) assert.Equal(t, "github-logfood", plans[0].name) require.Contains(t, plans[0].failed, claude) - assert.Equal(t, categoryNameConflict, errorCategory(plans[0].failed[claude])) + assert.Equal(t, protos.AitoolsErrorCategoryMCPNameConflict, errorCategory(plans[0].failed[claude])) }) t.Run("planning reads the agent entries it is given, not the files", func(t *testing.T) { @@ -794,7 +794,7 @@ func TestAgentFailureErrorWhenAServiceReachedNoAgent(t *testing.T) { } func TestErrorCategoryMapsTypedFailures(t *testing.T) { - assert.Equal(t, categoryNameConflict, + assert.Equal(t, protos.AitoolsErrorCategoryMCPNameConflict, errorCategory(fmt.Errorf("claude-code: server %q: %w", "github", agentconfig.ErrNotOwned))) assert.Equal(t, protos.AitoolsErrorCategoryTimeout, errorCategory(fmt.Errorf("list: %w", context.DeadlineExceeded))) assert.Equal(t, protos.AitoolsErrorCategoryPermissionDenied, errorCategory(&fs.PathError{Op: "open", Path: "x", Err: fs.ErrPermission})) @@ -952,7 +952,7 @@ func TestMcpAddRefusesStoredNameHeldByOtherBinding(t *testing.T) { err := runAdd(t, "--profile", "logfood", "--names", "github", "--agents", "claude-code") require.Error(t, err) - assert.Equal(t, categoryNameConflict, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPNameConflict, errorCategory(err)) assert.ErrorContains(t, err, "github-logfood") got, err := os.ReadFile(claudeFile) require.NoError(t, err) @@ -1137,7 +1137,7 @@ func TestPlanAddsRefusesRecoveredNameHeldByAnotherServer(t *testing.T) { _, err := planNames(t, "", defaultBinding(t, "logfood"), state, []string{"system.ai.github"}, []*agents.Agent{agents.ByName("claude-code")}) require.Error(t, err) - assert.Equal(t, categoryNameConflict, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPNameConflict, errorCategory(err)) assert.Equal(t, "system.ai.github is installed as github-logfood, but another server in the state holds github-logfood; remove that entry first", err.Error()) } @@ -1156,7 +1156,7 @@ func TestMcpAddRefusesUnmanagedEntryBeforeAnyRequest(t *testing.T) { err := runAdd(t, "--profile", "logfood", "--names", "github", "--agents", "claude-code") require.Error(t, err) - assert.Equal(t, categoryNameConflict, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPNameConflict, errorCategory(err)) assert.ErrorContains(t, err, "Claude Code already has a server named github-logfood that this CLI did not write; pass --force to replace it") got, err := os.ReadFile(claudeFile) require.NoError(t, err) @@ -1280,14 +1280,14 @@ func TestPlanAddsDetectedAgentConflictFailsOnlyThatAgent(t *testing.T) { require.Len(t, plans, 1) assert.Equal(t, "github-logfood", plans[0].name) require.Contains(t, plans[0].failed, claude) - assert.Equal(t, categoryNameConflict, errorCategory(plans[0].failed[claude])) + assert.Equal(t, protos.AitoolsErrorCategoryMCPNameConflict, errorCategory(plans[0].failed[claude])) assert.NotContains(t, plans[0].failed, codex) assert.NoError(t, planningRefusals(plans, targets)) in.explicit = true _, err = planAdds(ctx, in, state, []string{"system.ai.github"}) require.Error(t, err) - assert.Equal(t, categoryNameConflict, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPNameConflict, errorCategory(err)) onlyClaude := []*agents.Agent{claude} in = planInput{binding: defaultBinding(t, "logfood"), targets: onlyClaude, held: readHeld(ctx, onlyClaude)} @@ -1295,7 +1295,7 @@ func TestPlanAddsDetectedAgentConflictFailsOnlyThatAgent(t *testing.T) { require.NoError(t, err) refusal := planningRefusals(plans, onlyClaude) require.Error(t, refusal) - assert.Equal(t, categoryNameConflict, errorCategory(refusal)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPNameConflict, errorCategory(refusal)) } // TestPlanAddsUnmanagedEntryIsReplacedOnlyWithForce checks that a stored name held by an entry this CLI did not write is refused at planning, @@ -1310,7 +1310,7 @@ func TestPlanAddsUnmanagedEntryIsReplacedOnlyWithForce(t *testing.T) { in := planInput{binding: defaultBinding(t, "logfood"), targets: targets, held: held, explicit: true} _, err := planAdds(ctx, in, state, []string{"system.ai.github"}) require.Error(t, err) - assert.Equal(t, categoryNameConflict, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPNameConflict, errorCategory(err)) in.force = true plans, err := planAdds(ctx, in, state, []string{"system.ai.github"}) @@ -1323,7 +1323,7 @@ func TestPickServicesNamesSchemaWithNoServices(t *testing.T) { _, err := pickServices("system.ai", []string{"system.ai.github"}, nil) require.Error(t, err) assert.Equal(t, "system.ai.github is not an MCP service in system.ai, which has none", err.Error()) - assert.Equal(t, categoryServiceNotFound, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPServiceNotFound, errorCategory(err)) } // TestInstallServerWritesOnlyAgentsInstalledForMCP checks that an agent named with --agents is written when it counts as installed for MCP, and that diff --git a/cmd/aitools/mcp_login.go b/cmd/aitools/mcp_login.go index 26b222a4479..63aa724003e 100644 --- a/cmd/aitools/mcp_login.go +++ b/cmd/aitools/mcp_login.go @@ -12,10 +12,12 @@ import ( "github.com/databricks/cli/libs/aitools/mcpconfig" "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/browser" + "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/shellquote" "github.com/databricks/cli/libs/telemetry/protos" "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/config" "github.com/spf13/cobra" ) @@ -45,6 +47,11 @@ type mcpSignInOutput struct { URL string `json:"url,omitempty"` ErrorCategory protos.AitoolsErrorCategory `json:"error_category,omitempty"` Error string `json:"error,omitempty"` + + // cfg is the bound profile's config, which authenticates the exit-time telemetry upload. + cfg *config.Config + // alreadySignedIn is set by login when the caller was signed in before it ran. + alreadySignedIn bool } // mcpSignInAction runs login or logout against the profile's client and records the result in out. pinnedFile is the @@ -63,7 +70,7 @@ sign-in to complete. Without a browser, open the printed URL yourself. is a full name such as system.ai.github. The profile is the one whose user signs in.`, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - return runMcpSignInCommand(cmd, args[0], mcpLogin) + return runMcpSignInCommand(cmd, args[0], mcpLogin, protos.AitoolsMcpOperationSignIn) }, } } @@ -79,19 +86,23 @@ stays in place. Running it when you are not signed in is not an error. is a full name such as system.ai.github.`, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - return runMcpSignInCommand(cmd, args[0], mcpLogout) + return runMcpSignInCommand(cmd, args[0], mcpLogout, protos.AitoolsMcpOperationSignOut) }, } } // runMcpSignInCommand resolves the profile for service and runs action. In JSON mode the document is // the only stdout output; progress goes to stderr. -func runMcpSignInCommand(cmd *cobra.Command, service string, action mcpSignInAction) error { +func runMcpSignInCommand(cmd *cobra.Command, service string, action mcpSignInAction, operation protos.AitoolsMcpOperation) error { ctx := cmd.Context() jsonMode := installOutputIsJSON(cmd) out := &mcpSignInOutput{Service: service} runErr := runMcpSignIn(ctx, cmd.ErrOrStderr(), profileFlag(cmd), service, action, out) + if out.cfg != nil { + cmd.SetContext(cmdctx.SetConfigUsed(cmd.Context(), out.cfg)) + } + logMcpEvent(ctx, signInEvent(operation, out, runErr)) if !jsonMode { return runErr } @@ -147,15 +158,31 @@ func runMcpSignIn(ctx context.Context, errOut io.Writer, flagValue, service stri return err } out.Host = w.Config.Host + out.cfg = w.Config return action(ctx, errOut, w, out, pinnedFile) } +// signInEvent is the SIGN_IN or SIGN_OUT event of one login or logout. +func signInEvent(operation protos.AitoolsMcpOperation, out *mcpSignInOutput, err error) protos.AitoolsMcpEvent { + var r protos.AitoolsMcpServerResult + if operation == protos.AitoolsMcpOperationSignIn { + r = signInServerResult(out.Service, out.State, out.alreadySignedIn, err) + } else { + r = signOutServerResult(out.Service, out.State, err) + } + return protos.AitoolsMcpEvent{ + CallingAgent: callingAgentType(), + Operation: operation, + Servers: []protos.AitoolsMcpServerResult{r}, + } +} + // serviceFor returns the service with its sign-in requirement, as the profile sees it. A service the profile cannot see // is an error, so that a misspelled name is reported rather than treated as a service with no sign-in. func serviceFor(ctx context.Context, w *databricks.WorkspaceClient, profileName, pinnedFile, service string) (mcp.Service, error) { svc, err := mcp.GetService(ctx, w, service) if mcp.IsNotFound(err) { - return mcp.Service{}, failure(categoryServiceNotFound, err, "%s is not an MCP service visible to profile %s", service, profileName) + return mcp.Service{}, failure(protos.AitoolsErrorCategoryMCPServiceNotFound, err, "%s is not an MCP service visible to profile %s", service, profileName) } if err != nil { return mcp.Service{}, describeRequestError(profileName, pinnedFile, err) @@ -183,8 +210,9 @@ func mcpLogin(ctx context.Context, errOut io.Writer, w *databricks.WorkspaceClie return describeRequestError(out.Profile, pinnedFile, fmt.Errorf("cannot read the workspace ID of %s: %w", w.Config.Host, err)) } url := mcp.SignInURL(out.Host, svc.FullName, workspaceID) - state, _, urlShown, err := signIn(ctx, errOut, w, pinnedFile, url, svc, signInOptions{Profile: out.Profile, Open: browser.OpenOutputToStderr}) + state, alreadySignedIn, urlShown, err := signIn(ctx, errOut, w, pinnedFile, url, svc, signInOptions{Profile: out.Profile, Open: browser.OpenOutputToStderr}) out.State = state + out.alreadySignedIn = alreadySignedIn // The URL is named when it was printed. A sign-in that failed before its page is printed names none. if urlShown { out.URL = url @@ -264,7 +292,7 @@ func signIn(ctx context.Context, errOut io.Writer, w *databricks.WorkspaceClient timeout, timedOut := errors.AsType[*mcp.SignInTimeoutError](err) switch { case timedOut: - return signInStateTimeout, false, urlShown, failure(categorySignInTimeout, err, + return signInStateTimeout, false, urlShown, failure(protos.AitoolsErrorCategorySignInTimeout, err, "sign-in to %s did not complete in %g minutes%s. Finish at %s then retry: %s", svc.FullName, signInWaitTimeout.Minutes(), lastCheckText(timeout), url, mcpLoginHint(svc.FullName, profileName, pinnedFile)) case errors.Is(err, mcp.ErrSignInFailed): diff --git a/cmd/aitools/mcp_login_test.go b/cmd/aitools/mcp_login_test.go index 33658f88cef..fc25c25c2ab 100644 --- a/cmd/aitools/mcp_login_test.go +++ b/cmd/aitools/mcp_login_test.go @@ -116,7 +116,7 @@ func TestMcpLogoutUnknownServiceIsAnError(t *testing.T) { out := mcpSignInOutput{Service: "system.ai.gihtub"} err := runMcpSignIn(t.Context(), io.Discard, "logfood", "system.ai.gihtub", mcpLogout, &out) require.Error(t, err) - assert.Equal(t, categoryServiceNotFound, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPServiceNotFound, errorCategory(err)) assert.Contains(t, err.Error(), "system.ai.gihtub is not an MCP service visible to profile logfood") assert.Empty(t, out.State) } @@ -164,7 +164,7 @@ func TestMcpLogoutEndpointNotFoundIsNotAServiceError(t *testing.T) { out := mcpSignInOutput{Service: "system.ai.github"} err := runMcpSignIn(t.Context(), io.Discard, "logfood", "system.ai.github", mcpLogout, &out) require.Error(t, err) - assert.NotEqual(t, categoryServiceNotFound, errorCategory(err)) + assert.NotEqual(t, protos.AitoolsErrorCategoryMCPServiceNotFound, errorCategory(err)) assert.NotContains(t, err.Error(), "is not an MCP service visible") } diff --git a/cmd/aitools/mcp_remove.go b/cmd/aitools/mcp_remove.go index b4f2f254c93..6ae241880f7 100644 --- a/cmd/aitools/mcp_remove.go +++ b/cmd/aitools/mcp_remove.go @@ -32,6 +32,9 @@ type removeOutput struct { Skipped []removeSkipped `json:"skipped"` Error string `json:"error,omitempty"` ErrorCategory protos.AitoolsErrorCategory `json:"error_category,omitempty"` + + // results is one telemetry result per (service, agent) attempted. It is not in the JSON. + results []protos.AitoolsMcpServerResult } type removeRemoved struct { @@ -91,12 +94,19 @@ and reported. When an agent's config moved, the entry is removed from the file i Entries that run 'aitools mcp serve' without an install record are not chosen by any flag; 'aitools mcp list' shows them as unmanaged. Provider credentials are not deleted.`, Args: cobra.NoArgs, + // Resolve auth best-effort so telemetry can upload; see tryConfigureAuth. + PreRunE: tryConfigureAuth, RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() jsonMode := installOutputIsJSON(cmd) out := &removeOutput{Removed: []removeRemoved{}, Skipped: []removeSkipped{}} runErr := runMcpRemove(ctx, cmd, f, jsonMode, out) + logMcpEvent(ctx, protos.AitoolsMcpEvent{ + CallingAgent: callingAgentType(), + Operation: protos.AitoolsMcpOperationRemove, + Servers: out.results, + }) if !jsonMode { return runErr } @@ -205,10 +215,12 @@ func runMcpRemove(ctx context.Context, cmd *cobra.Command, f mcpRemoveFlags, jso // There is no config this CLI can edit for the agent, so only the record can go. removed.Agents = append(removed.Agents, agentName) removed.RecordOnly = append(removed.RecordOnly, agentName) + out.results = append(out.results, removeServerResult(e.Service, agentName, removeDeleted, nil)) drops = append(drops, recordDrop{name: e.Name, agent: agentName}) continue } outcome, reason, err := removeFromAgent(ctx, files, e, a, pin) + out.results = append(out.results, removeServerResult(e.Service, agentName, outcome, err)) if err != nil { errs = append(errs, err) out.Skipped = append(out.Skipped, removeSkipped{Name: e.Name, Agent: agentName, Reason: fmt.Sprintf("could not update %s: %s", agentDisplayName(agentName), err)}) @@ -282,14 +294,14 @@ func selectInstalls(state *mcp.State, names []string, profileName string) ([]mcp return nil, err } if len(matched) == 0 { - return nil, failure(categoryServiceNotFound, nil, "no installed MCP server named %s%s", name, where) + return nil, failure(protos.AitoolsErrorCategoryMCPServiceNotFound, nil, "no installed MCP server named %s%s", name, where) } for _, i := range matched { chosen[i] = true } } if profileName != "" && len(scope) == 0 { - return nil, failure(categoryServiceNotFound, nil, "no installed MCP servers on profile %s", profileName) + return nil, failure(protos.AitoolsErrorCategoryMCPServiceNotFound, nil, "no installed MCP servers on profile %s", profileName) } if len(names) == 0 { return scope, nil @@ -365,7 +377,7 @@ func checkTargetsInstalled(selected []mcp.Entry, targets []*agents.Agent) error _, ok := e.Agents[a.Name] return ok }) { - return failure(categoryServiceNotFound, nil, "no MCP server is installed for %s", agentDisplayName(a.Name)) + return failure(protos.AitoolsErrorCategoryMCPServiceNotFound, nil, "no MCP server is installed for %s", agentDisplayName(a.Name)) } } return nil diff --git a/cmd/aitools/mcp_remove_test.go b/cmd/aitools/mcp_remove_test.go index d97ed4a6034..6a62975cf83 100644 --- a/cmd/aitools/mcp_remove_test.go +++ b/cmd/aitools/mcp_remove_test.go @@ -82,13 +82,13 @@ func TestSelectInstalls(t *testing.T) { _, err = selectInstalls(state, []string{"github-dogfood"}, "logfood") require.Error(t, err) assert.Equal(t, "no installed MCP server named github-dogfood on profile logfood", err.Error()) - assert.Equal(t, categoryServiceNotFound, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPServiceNotFound, errorCategory(err)) // A profile with no installed server is a typo or an empty profile, not an empty removal. _, err = selectInstalls(state, nil, "typo") require.Error(t, err) assert.Equal(t, "no installed MCP servers on profile typo", err.Error()) - assert.Equal(t, categoryServiceNotFound, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPServiceNotFound, errorCategory(err)) none, err := selectInstalls(&mcp.State{}, nil, "") require.NoError(t, err, "--all on an empty state is a no-op") @@ -400,13 +400,13 @@ func TestMcpRemoveRefusesProfileAndAgentWithNothingInstalled(t *testing.T) { require.ErrorIs(t, err, root.ErrAlreadyPrinted) out := readRemoveOutput(t, stdout.String()) assert.Equal(t, "no installed MCP servers on profile typo", out.Error) - assert.Equal(t, categoryServiceNotFound, out.ErrorCategory) + assert.Equal(t, protos.AitoolsErrorCategoryMCPServiceNotFound, out.ErrorCategory) stdout, _, err = runMcpCmd(t, "remove", "--names", "genie_one_mcp-logfood", "--agents", "cursor", "--output", "json") require.ErrorIs(t, err, root.ErrAlreadyPrinted) out = readRemoveOutput(t, stdout.String()) assert.Equal(t, "no MCP server is installed for Cursor", out.Error) - assert.Equal(t, categoryServiceNotFound, out.ErrorCategory) + assert.Equal(t, protos.AitoolsErrorCategoryMCPServiceNotFound, out.ErrorCategory) // Nothing was removed by either refusal. state, err := mcp.LoadState(t.Context()) @@ -696,7 +696,7 @@ func TestMcpRemovePickerRefusesAgentWithNoServer(t *testing.T) { err := runInteractive(t, "mcp", "remove", "--agents", "codex") require.Error(t, err) assert.Equal(t, "no MCP server is installed for "+agents.ByName(agents.NameCodex).DisplayName, err.Error()) - assert.Equal(t, categoryServiceNotFound, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPServiceNotFound, errorCategory(err)) state, err := mcp.LoadState(t.Context()) require.NoError(t, err) assert.Len(t, state.Servers, 1) diff --git a/cmd/aitools/mcp_serve.go b/cmd/aitools/mcp_serve.go index f1531de44ca..5bcee77cf1c 100644 --- a/cmd/aitools/mcp_serve.go +++ b/cmd/aitools/mcp_serve.go @@ -24,6 +24,7 @@ import ( "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/auth/u2m" "github.com/databricks/cli/libs/browser" + "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/config" "github.com/databricks/databricks-sdk-go/config/credentials" @@ -104,6 +105,7 @@ func runMcpServe(cmd *cobra.Command, f mcpServeFlags) error { if err != nil { return serveFailure(cmd, err, serveBindStatus) } + agent := proxyAgentType(f.agent) cfg, entry, err := mcp.Bind(cmd.Context(), mcp.BindOptions{ Profile: profileName, Service: f.service, @@ -111,9 +113,14 @@ func runMcpServe(cmd *cobra.Command, f mcpServeFlags) error { Stderr: cmd.ErrOrStderr(), }) if err != nil { + logMcpEvent(cmd.Context(), proxyStartedEvent(agent, f.service, err)) return serveFailure(cmd, err, serveBindStatus) } + // The bound profile authenticates the exit-time telemetry upload. + cmd.SetContext(cmdctx.SetConfigUsed(cmd.Context(), cfg)) + logMcpEvent(cmd.Context(), proxyStartedEvent(agent, f.service, nil)) + // A server pinned to a config file names that file in its hints, as add and login do. pinnedFile := mcp.PinnedFile(entry, cfg.ConfigFile) ctx, stop := serveSignalContext(cmd.Context()) @@ -131,6 +138,9 @@ func runMcpServe(cmd *cobra.Command, f mcpServeFlags) error { } } challenge, err := serveChallenge(ctx, serveTarget{w: w, entry: entry, profile: profileName, service: f.service, pinnedFile: pinnedFile}, cmd.ErrOrStderr()) + if challenge != nil { + challenge.Recorder = mcpProxyTelemetry{agent: agent, service: f.service} + } if err != nil { // A rejected token stops serve by cancelling ctx, so the refresh failure is checked before the signal, and its // message is the one that names the login. diff --git a/cmd/aitools/mcp_telemetry.go b/cmd/aitools/mcp_telemetry.go new file mode 100644 index 00000000000..ad05d2f6646 --- /dev/null +++ b/cmd/aitools/mcp_telemetry.go @@ -0,0 +1,228 @@ +package aitools + +import ( + "context" + "strings" + + "github.com/databricks/cli/libs/aitools/agents" + "github.com/databricks/cli/libs/aitools/mcp" + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/telemetry" + "github.com/databricks/cli/libs/telemetry/protos" +) + +// mcpServiceTypes maps the system.ai leaves in the CLI's service table to their telemetry enum. +var mcpServiceTypes = map[string]protos.AitoolsMcpService{ + "genie_one_mcp": protos.AitoolsMcpServiceGenieOne, + "dbsql": protos.AitoolsMcpServiceDBSQL, + "web_search": protos.AitoolsMcpServiceWebSearch, + "sandbox": protos.AitoolsMcpServiceSandbox, + "slack": protos.AitoolsMcpServiceSlack, + "github": protos.AitoolsMcpServiceGitHub, + "atlassian": protos.AitoolsMcpServiceAtlassian, + "google_drive": protos.AitoolsMcpServiceGoogleDrive, + "google_calendar": protos.AitoolsMcpServiceGoogleCalendar, + "gmail": protos.AitoolsMcpServiceGmail, + "microsoft_365": protos.AitoolsMcpServiceMicrosoft365, +} + +// mcpServiceType maps a service full name to its telemetry enum. The name itself is never sent: +// a system.ai leaf in the table has its own value, other system.ai leaves are OTHER_SYSTEM, and +// services outside system.ai are CUSTOM. +func mcpServiceType(fullName string) protos.AitoolsMcpService { + parts := strings.Split(fullName, ".") + if len(parts) != 3 || !strings.EqualFold(parts[0]+"."+parts[1], "system.ai") { + return protos.AitoolsMcpServiceCustom + } + if t, ok := mcpServiceTypes[strings.ToLower(parts[2])]; ok { + return t + } + return protos.AitoolsMcpServiceOtherSystem +} + +// callingAgentType returns the agent that drives the CLI, Unspecified when none is detected. +func callingAgentType() protos.AitoolsAgentType { + a := agents.CallingAgent() + if a == nil { + return protos.AitoolsAgentTypeUnspecified + } + return agentType(a.Name) +} + +// proxyAgentType returns the agent that started the proxy: the --agent flag when given, else the +// detected agent. +func proxyAgentType(flag string) protos.AitoolsAgentType { + if flag == "" { + return callingAgentType() + } + return agentType(flag) +} + +// logMcpEvent buffers an MCP event; cmd/root uploads it at exit. An event without server results +// is not logged: a run that stopped before any server was reached has nothing to report. The bound +// profile count is read from the state file as it is after the operation. A state file that cannot +// be read drops the event; telemetry never fails a command. +func logMcpEvent(ctx context.Context, event protos.AitoolsMcpEvent) { + if len(event.Servers) == 0 { + return + } + state, err := mcp.LoadState(ctx) + if err != nil { + log.Debugf(ctx, "telemetry: cannot read the MCP state; dropping the event: %s", err) + return + } + event.BoundProfileCount = boundProfileCount(state) + telemetry.Log(ctx, protos.DatabricksCliLog{AitoolsMcpEvent: &event}) +} + +// boundProfileCount returns the number of distinct profiles that have servers installed. +func boundProfileCount(state *mcp.State) int64 { + profiles := map[string]bool{} + for _, e := range state.Servers { + profiles[e.Profile] = true + } + return int64(len(profiles)) +} + +// addSignInState is the sign-in state of a server after an add: the state that --login reports, else PENDING for a +// server that needs sign-in, else NOT_REQUIRED. +func addSignInState(s addServerOutput) protos.AitoolsMcpSignInState { + if s.SignIn != nil { + return signInStateOf(s.SignIn.State) + } + if s.SignInRequired { + return protos.AitoolsMcpSignInStatePending + } + return protos.AitoolsMcpSignInStateNotRequired +} + +// addServerResults returns one result per (service, agent) of an add run. Installed entries are +// CHANGED, re-asserted ones UNCHANGED, and failed ones FAILED with their category. +func addServerResults(servers []addServerOutput) []protos.AitoolsMcpServerResult { + var results []protos.AitoolsMcpServerResult + for _, s := range servers { + for _, a := range s.Agents { + r := protos.AitoolsMcpServerResult{ + Service: mcpServiceType(s.Service), + Agent: agentType(a.Name), + SignInState: addSignInState(s), + } + switch a.Status { + case statusInstalled: + r.Outcome = protos.AitoolsTargetOutcomeChanged + case statusReasserted: + r.Outcome = protos.AitoolsTargetOutcomeUnchanged + default: + r.Outcome = protos.AitoolsTargetOutcomeFailed + r.ErrorCategory = a.ErrorCategory + } + results = append(results, r) + } + } + return results +} + +// removeServerResult returns the result of removing one server from one agent config. An entry that +// is gone already, or that was changed outside this CLI, is SKIPPED: nothing was changed there. +func removeServerResult(service, agent string, outcome removeOutcome, err error) protos.AitoolsMcpServerResult { + r := protos.AitoolsMcpServerResult{Service: mcpServiceType(service), Agent: agentType(agent)} + switch { + case err != nil: + r.Outcome = protos.AitoolsTargetOutcomeFailed + r.ErrorCategory = errorCategory(err) + case outcome == removeDeleted: + r.Outcome = protos.AitoolsTargetOutcomeChanged + default: + r.Outcome = protos.AitoolsTargetOutcomeSkipped + } + return r +} + +// signInStateOf maps a sign-in state name of login to its telemetry state. Anything but ACTIVE or +// NOT_REQUIRED is still PENDING. +func signInStateOf(state string) protos.AitoolsMcpSignInState { + switch state { + case signInStateActive: + return protos.AitoolsMcpSignInStateActive + case signInStateNotRequired: + return protos.AitoolsMcpSignInStateNotRequired + } + return protos.AitoolsMcpSignInStatePending +} + +// signInServerResult returns the result of one provider sign-in. The sign-in was already active +// (alreadySignedIn) or the service needed none, so nothing changed: UNCHANGED. +func signInServerResult(service, state string, alreadySignedIn bool, err error) protos.AitoolsMcpServerResult { + r := protos.AitoolsMcpServerResult{Service: mcpServiceType(service), SignInState: signInStateOf(state)} + switch { + case err != nil: + r.Outcome = protos.AitoolsTargetOutcomeFailed + r.ErrorCategory = errorCategory(err) + case alreadySignedIn || state == signInStateNotRequired: + r.Outcome = protos.AitoolsTargetOutcomeUnchanged + default: + r.Outcome = protos.AitoolsTargetOutcomeChanged + } + return r +} + +// signOutServerResult returns the result of a logout. Logout does not read whether the service needs +// sign-in, so the sign-in state is not set. +func signOutServerResult(service, state string, err error) protos.AitoolsMcpServerResult { + r := protos.AitoolsMcpServerResult{Service: mcpServiceType(service)} + switch { + case err != nil: + r.Outcome = protos.AitoolsTargetOutcomeFailed + r.ErrorCategory = errorCategory(err) + case state == signOutStateDeleted: + r.Outcome = protos.AitoolsTargetOutcomeChanged + default: + r.Outcome = protos.AitoolsTargetOutcomeUnchanged + } + return r +} + +// proxyStartedEvent is the PROXY_STARTED event of serve. err is the binding failure, or nil when the +// proxy is about to serve. The event only uploads when the bound profile resolved, so a failed +// binding usually leaves it unsent. +func proxyStartedEvent(agent protos.AitoolsAgentType, service string, err error) protos.AitoolsMcpEvent { + r := protos.AitoolsMcpServerResult{Service: mcpServiceType(service)} + if err != nil { + r.ErrorCategory = errorCategory(err) + } + return protos.AitoolsMcpEvent{ + CallingAgent: agent, + Operation: protos.AitoolsMcpOperationProxyStarted, + Servers: []protos.AitoolsMcpServerResult{r}, + } +} + +// mcpProxyTelemetry is the proxy's telemetry for one serve session. It implements proxy.Recorder. +type mcpProxyTelemetry struct { + agent protos.AitoolsAgentType + service string +} + +// SignInPrompted logs SIGN_IN_PROMPTED once the sign-in page the authenticate tool opened has been +// waited on. signedIn reports whether the credential became ACTIVE. +func (t mcpProxyTelemetry) SignInPrompted(ctx context.Context, signedIn bool) { + logMcpEvent(ctx, signInPromptedEvent(t.agent, t.service, signedIn)) +} + +// signInPromptedEvent is the SIGN_IN_PROMPTED event of serve: the sign-in is required, and the +// sign-in state is ACTIVE when the wait ended with the credential, PENDING otherwise. +func signInPromptedEvent(agent protos.AitoolsAgentType, service string, signedIn bool) protos.AitoolsMcpEvent { + state := protos.AitoolsMcpSignInStatePending + if signedIn { + state = protos.AitoolsMcpSignInStateActive + } + return protos.AitoolsMcpEvent{ + CallingAgent: agent, + Operation: protos.AitoolsMcpOperationSignInPrompted, + Servers: []protos.AitoolsMcpServerResult{{ + Service: mcpServiceType(service), + ErrorCategory: protos.AitoolsErrorCategorySignInRequired, + SignInState: state, + }}, + } +} diff --git a/cmd/aitools/mcp_telemetry_test.go b/cmd/aitools/mcp_telemetry_test.go new file mode 100644 index 00000000000..11cb852b8ef --- /dev/null +++ b/cmd/aitools/mcp_telemetry_test.go @@ -0,0 +1,235 @@ +package aitools + +import ( + "encoding/json" + "errors" + "testing" + + "github.com/databricks/cli/libs/aitools/mcp" + "github.com/databricks/cli/libs/telemetry/protos" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMcpServiceType(t *testing.T) { + // Every leaf in the CLI's known table has its own enum, so a known service never reports OTHER_SYSTEM. + cases := map[string]protos.AitoolsMcpService{ + "system.ai.genie_one_mcp": protos.AitoolsMcpServiceGenieOne, + "system.ai.dbsql": protos.AitoolsMcpServiceDBSQL, + "system.ai.web_search": protos.AitoolsMcpServiceWebSearch, + "system.ai.sandbox": protos.AitoolsMcpServiceSandbox, + "system.ai.slack": protos.AitoolsMcpServiceSlack, + "system.ai.github": protos.AitoolsMcpServiceGitHub, + "system.ai.atlassian": protos.AitoolsMcpServiceAtlassian, + "system.ai.google_drive": protos.AitoolsMcpServiceGoogleDrive, + "system.ai.google_calendar": protos.AitoolsMcpServiceGoogleCalendar, + "system.ai.gmail": protos.AitoolsMcpServiceGmail, + "system.ai.microsoft_365": protos.AitoolsMcpServiceMicrosoft365, + // The schema name is case-insensitive in Unity Catalog. + "System.AI.GitHub": protos.AitoolsMcpServiceGitHub, + // A system.ai leaf the CLI does not know yet. + "system.ai.future_tool": protos.AitoolsMcpServiceOtherSystem, + // Anything outside system.ai, whatever its name. + "main.tools.github": protos.AitoolsMcpServiceCustom, + "system.ai": protos.AitoolsMcpServiceCustom, + "system.ai.a.github": protos.AitoolsMcpServiceCustom, + } + for name, want := range cases { + assert.Equal(t, want, mcpServiceType(name), name) + } +} + +func TestAddServerResults(t *testing.T) { + servers := []addServerOutput{ + { + Service: "system.ai.github", + SignInRequired: true, + SignIn: &addSignInOutput{State: signInStateActive}, + Agents: []addAgentOutput{ + {Name: "claude-code", Status: statusInstalled}, + {Name: "codex", Status: statusReasserted}, + {Name: "cursor", Status: statusFailed, ErrorCategory: protos.AitoolsErrorCategoryAgentConfigInvalid}, + }, + }, + { + Service: "main.tools.custom", + Agents: []addAgentOutput{{Name: "claude-code", Status: statusInstalled}}, + }, + } + + assert.Equal(t, []protos.AitoolsMcpServerResult{ + { + Service: protos.AitoolsMcpServiceGitHub, + Agent: protos.AitoolsAgentTypeClaudeCode, + Outcome: protos.AitoolsTargetOutcomeChanged, + SignInState: protos.AitoolsMcpSignInStateActive, + }, + { + Service: protos.AitoolsMcpServiceGitHub, + Agent: protos.AitoolsAgentTypeCodex, + Outcome: protos.AitoolsTargetOutcomeUnchanged, + SignInState: protos.AitoolsMcpSignInStateActive, + }, + { + Service: protos.AitoolsMcpServiceGitHub, + Agent: protos.AitoolsAgentTypeCursor, + Outcome: protos.AitoolsTargetOutcomeFailed, + ErrorCategory: protos.AitoolsErrorCategoryAgentConfigInvalid, + SignInState: protos.AitoolsMcpSignInStateActive, + }, + { + Service: protos.AitoolsMcpServiceCustom, + Agent: protos.AitoolsAgentTypeClaudeCode, + Outcome: protos.AitoolsTargetOutcomeChanged, + SignInState: protos.AitoolsMcpSignInStateNotRequired, + }, + }, addServerResults(servers)) +} + +func TestRemoveServerResult(t *testing.T) { + assert.Equal(t, protos.AitoolsTargetOutcomeChanged, + removeServerResult("system.ai.github", "claude-code", removeDeleted, nil).Outcome) + // An entry already gone, or changed outside this CLI, is left as it is. + assert.Equal(t, protos.AitoolsTargetOutcomeSkipped, + removeServerResult("system.ai.github", "claude-code", removePruned, nil).Outcome) + assert.Equal(t, protos.AitoolsTargetOutcomeSkipped, + removeServerResult("system.ai.github", "claude-code", removeKept, nil).Outcome) + + failed := removeServerResult("system.ai.github", "codex", removeDeleted, errors.New("config is not JSON")) + assert.Equal(t, protos.AitoolsMcpServerResult{ + Service: protos.AitoolsMcpServiceGitHub, + Agent: protos.AitoolsAgentTypeCodex, + Outcome: protos.AitoolsTargetOutcomeFailed, + ErrorCategory: protos.AitoolsErrorCategoryUncategorized, + }, failed) +} + +func TestSignInServerResult(t *testing.T) { + cases := []struct { + name string + state string + alreadySignedIn bool + err error + want protos.AitoolsMcpServerResult + }{ + { + name: "signed in now", + state: signInStateActive, + want: protos.AitoolsMcpServerResult{Service: protos.AitoolsMcpServiceGitHub, Outcome: protos.AitoolsTargetOutcomeChanged, SignInState: protos.AitoolsMcpSignInStateActive}, + }, + { + name: "already active", + state: signInStateActive, + alreadySignedIn: true, + want: protos.AitoolsMcpServerResult{Service: protos.AitoolsMcpServiceGitHub, Outcome: protos.AitoolsTargetOutcomeUnchanged, SignInState: protos.AitoolsMcpSignInStateActive}, + }, + { + name: "no sign-in needed", + state: signInStateNotRequired, + want: protos.AitoolsMcpServerResult{Service: protos.AitoolsMcpServiceGitHub, Outcome: protos.AitoolsTargetOutcomeUnchanged, SignInState: protos.AitoolsMcpSignInStateNotRequired}, + }, + { + name: "timed out", + state: signInStateTimeout, + err: failure(protos.AitoolsErrorCategorySignInTimeout, nil, "timed out"), + want: protos.AitoolsMcpServerResult{ + Service: protos.AitoolsMcpServiceGitHub, Outcome: protos.AitoolsTargetOutcomeFailed, + ErrorCategory: protos.AitoolsErrorCategorySignInTimeout, SignInState: protos.AitoolsMcpSignInStatePending, + }, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, signInServerResult("system.ai.github", tc.state, tc.alreadySignedIn, tc.err)) + }) + } +} + +func TestSignOutServerResult(t *testing.T) { + // Logout does not read the sign-in state, so the result has none. + assert.Equal(t, protos.AitoolsMcpServerResult{Service: protos.AitoolsMcpServiceGitHub, Outcome: protos.AitoolsTargetOutcomeChanged}, + signOutServerResult("system.ai.github", signOutStateDeleted, nil)) + assert.Equal(t, protos.AitoolsMcpServerResult{Service: protos.AitoolsMcpServiceGitHub, Outcome: protos.AitoolsTargetOutcomeUnchanged}, + signOutServerResult("system.ai.github", signOutStateNotSignedIn, nil)) +} + +func TestProxyStartedEvent(t *testing.T) { + ok := proxyStartedEvent(protos.AitoolsAgentTypeClaudeCode, "system.ai.genie_one_mcp", nil) + assert.Equal(t, protos.AitoolsMcpEvent{ + CallingAgent: protos.AitoolsAgentTypeClaudeCode, + Operation: protos.AitoolsMcpOperationProxyStarted, + Servers: []protos.AitoolsMcpServerResult{{Service: protos.AitoolsMcpServiceGenieOne}}, + }, ok) + + // Each binding failure has its category; the error text never reaches the event. + for category, bind := range map[protos.AitoolsErrorCategory]mcp.BindCategory{ + protos.AitoolsErrorCategoryProfileNotFound: mcp.BindProfileNotFound, + protos.AitoolsErrorCategoryProfileHostChanged: mcp.BindProfileHostChanged, + protos.AitoolsErrorCategoryAuthFailed: mcp.BindAuthFailed, + } { + event := proxyStartedEvent(protos.AitoolsAgentTypeUnspecified, "main.tools.custom", &mcp.BindError{Category: bind}) + require.Len(t, event.Servers, 1) + assert.Equal(t, category, event.Servers[0].ErrorCategory) + assert.Equal(t, protos.AitoolsMcpServiceCustom, event.Servers[0].Service) + } +} + +func TestSignInPromptedEvent(t *testing.T) { + for signedIn, state := range map[bool]protos.AitoolsMcpSignInState{ + true: protos.AitoolsMcpSignInStateActive, + false: protos.AitoolsMcpSignInStatePending, + } { + event := signInPromptedEvent(protos.AitoolsAgentTypeCursor, "system.ai.github", signedIn) + assert.Equal(t, protos.AitoolsMcpEvent{ + CallingAgent: protos.AitoolsAgentTypeCursor, + Operation: protos.AitoolsMcpOperationSignInPrompted, + Servers: []protos.AitoolsMcpServerResult{{ + Service: protos.AitoolsMcpServiceGitHub, + ErrorCategory: protos.AitoolsErrorCategorySignInRequired, + SignInState: state, + }}, + }, event) + } +} + +func TestBoundProfileCount(t *testing.T) { + state := &mcp.State{Servers: []mcp.Entry{ + {Profile: "logfood", Service: "system.ai.github"}, + {Profile: "logfood", Service: "system.ai.genie_one_mcp"}, + {Profile: "dogfood", Service: "system.ai.github"}, + }} + assert.Equal(t, int64(2), boundProfileCount(state)) + assert.Equal(t, int64(0), boundProfileCount(&mcp.State{})) +} + +// The event carries enums and counts only. A custom service and the host of its profile must not +// appear anywhere in the payload. +func TestMcpEventOmitsUserContent(t *testing.T) { + event := protos.AitoolsMcpEvent{ + CallingAgent: protos.AitoolsAgentTypeClaudeCode, + Operation: protos.AitoolsMcpOperationAdd, + BoundProfileCount: 1, + Servers: addServerResults([]addServerOutput{{ + Service: "acme_catalog.secret_schema.internal_tool", + Agents: []addAgentOutput{{Name: "claude-code", Status: statusInstalled}}, + }}), + } + b, err := json.Marshal(event) + require.NoError(t, err) + + var got map[string]any + require.NoError(t, json.Unmarshal(b, &got)) + assert.ElementsMatch(t, []string{"calling_agent", "operation", "servers", "bound_profile_count"}, keys(got)) + assert.NotContains(t, string(b), "acme_catalog") + assert.NotContains(t, string(b), "secret_schema") + assert.NotContains(t, string(b), "internal_tool") + assert.Contains(t, string(b), `"service":"CUSTOM"`) +} + +func keys(m map[string]any) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + return out +} diff --git a/cmd/aitools/mcp_test.go b/cmd/aitools/mcp_test.go index 8a70c18bbad..5217647400d 100644 --- a/cmd/aitools/mcp_test.go +++ b/cmd/aitools/mcp_test.go @@ -96,7 +96,7 @@ host = https://b.test _, err := resolveProfile(ctx, "") require.Error(t, err) assert.Equal(t, "several workspace profiles exist; pass --profile. Found: a, b", err.Error()) - assert.Equal(t, categoryProfileRequired, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryProfileRequired, errorCategory(err)) }) t.Run("no workspace profiles is an error naming the file", func(t *testing.T) { @@ -104,7 +104,7 @@ host = https://b.test _, err := resolveProfile(ctx, "") require.Error(t, err) assert.Contains(t, err.Error(), "no workspace profiles in ") - assert.Equal(t, categoryNoProfile, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryNoProfile, errorCategory(err)) assert.Contains(t, err.Error(), "databricks auth login --host --profile ") }) } @@ -114,11 +114,11 @@ func TestErrorCategoryUsesTypedErrorsOnly(t *testing.T) { assert.Equal(t, protos.AitoolsErrorCategoryUnknownAgent, errorCategory(typed)) assert.Equal(t, "unknown agent \"pi\"", typed.Error()) - wrapped := failure(categoryServiceNotFound, errors.New("boom"), "%s", "boom") - assert.Equal(t, categoryServiceNotFound, errorCategory(wrapped)) + wrapped := failure(protos.AitoolsErrorCategoryMCPServiceNotFound, errors.New("boom"), "%s", "boom") + assert.Equal(t, protos.AitoolsErrorCategoryMCPServiceNotFound, errorCategory(wrapped)) configInvalid := &os.PathError{Op: "parse", Path: "x", Err: agentconfig.ErrConfigInvalid} - assert.Equal(t, categoryAgentConfigInvalid, errorCategory(configInvalid)) + assert.Equal(t, protos.AitoolsErrorCategoryAgentConfigInvalid, errorCategory(configInvalid)) assert.Equal(t, protos.AitoolsErrorCategoryUncategorized, errorCategory(errors.New("some failure"))) } diff --git a/libs/aitools/mcp/proxy/bridge_test.go b/libs/aitools/mcp/proxy/bridge_test.go index 11c60b775c4..3cc4fd531d3 100644 --- a/libs/aitools/mcp/proxy/bridge_test.go +++ b/libs/aitools/mcp/proxy/bridge_test.go @@ -2319,7 +2319,7 @@ func TestSignInWritesShareTheBridgeStderr(t *testing.T) { Config: &config.Config{Host: testHost}, Authenticate: func(*http.Request) error { return nil }, Profile: "logfood", - Challenge: &Challenge{Service: testService, Profile: "logfood", Host: testHost, Label: "GitHub", LoginCommand: loginCommand, SignIn: signIn}, + Challenge: &Challenge{Service: testService, Profile: "logfood", Host: testHost, Label: "GitHub", LoginCommand: loginCommand, SignIn: signIn, Recorder: &promptRecorder{}}, }) }() diff --git a/libs/aitools/mcp/proxy/challenge.go b/libs/aitools/mcp/proxy/challenge.go index a1959a18e49..3ee395f8c48 100644 --- a/libs/aitools/mcp/proxy/challenge.go +++ b/libs/aitools/mcp/proxy/challenge.go @@ -50,6 +50,14 @@ type SignInRequest struct { // cancelled. Its error is the answer's text. type SignInFunc func(ctx context.Context, req SignInRequest) (alreadySignedIn, urlShown bool, err error) +// Recorder receives the proxy's telemetry events. The command implements it, so the proxy does not +// build telemetry events itself. +type Recorder interface { + // SignInPrompted runs after the wait of an authenticate call that opened the sign-in page. + // signedIn reports whether the credential became ACTIVE. + SignInPrompted(ctx context.Context, signedIn bool) +} + // Challenge turns on sign-in handling for a service that needs a provider sign-in. type Challenge struct { // Service is the full name of the MCP service, for example system.ai.github. @@ -70,6 +78,8 @@ type Challenge struct { // SignIn runs the sign-in. The command provides it, so the proxy never polls or opens a browser itself. Its error is // the answer's text, so it names the service, the profile and the fix. SignIn SignInFunc + // Recorder logs the sign-in prompt. The command provides it. + Recorder Recorder } // signInHooks applies the sign-in rules to the bridge. Services without a challenge get no hooks. @@ -445,7 +455,12 @@ func (h *signInHooks) authenticate(ctx context.Context, id, progressToken json.R return [][]byte{toolResult(id, h.stillChallengedText(), true)} } - alreadySignedIn, urlShown, err := h.c.SignIn(ctx, SignInRequest{Open: !h.pageCancelled, Progress: report, Stderr: h.stderr}) + open := !h.pageCancelled + alreadySignedIn, urlShown, err := h.c.SignIn(ctx, SignInRequest{Open: open, Progress: report, Stderr: h.stderr}) + // A caller that was already signed in opened no page, so there is no prompt to record. + if open && !alreadySignedIn { + h.c.Recorder.SignInPrompted(ctx, err == nil) + } h.pageCancelled = urlShown && err != nil && ctx.Err() != nil h.stateMu.Lock() h.alreadySignedIn = err == nil && alreadySignedIn diff --git a/libs/aitools/mcp/proxy/challenge_test.go b/libs/aitools/mcp/proxy/challenge_test.go index 0a399b0e1bb..0b212946abc 100644 --- a/libs/aitools/mcp/proxy/challenge_test.go +++ b/libs/aitools/mcp/proxy/challenge_test.go @@ -68,6 +68,24 @@ func (s *signInStub) requests() []SignInRequest { // loginCommand is the hint that the command builds for a pinned config file and a profile name with a space. const loginCommand = `DATABRICKS_CONFIG_FILE='/work/cfg/my file.cfg' databricks aitools mcp login system.ai.github --profile 'my profile'` +// promptRecorder records the SIGN_IN_PROMPTED events: one per sign-in page that authenticate opened. +type promptRecorder struct { + mu sync.Mutex + signedIn []bool +} + +func (r *promptRecorder) SignInPrompted(_ context.Context, signedIn bool) { + r.mu.Lock() + defer r.mu.Unlock() + r.signedIn = append(r.signedIn, signedIn) +} + +func (r *promptRecorder) prompts() []bool { + r.mu.Lock() + defer r.mu.Unlock() + return append([]bool(nil), r.signedIn...) +} + // challengedBridge returns a bridge to url with the sign-in hooks of a service that needs sign-in. func challengedBridge(url string, signIn SignInFunc) *Bridge { b := bridgeFor(url) @@ -255,6 +273,37 @@ func TestAuthenticateSaysAlreadySignedInAfterSignIn(t *testing.T) { require.NoError(t, p.finish(t)) } +// The prompt is recorded once per page that was opened, after its wait, with whether the credential +// became ACTIVE. A call that opens no page, or finds the caller already signed in, records nothing. +func TestAuthenticateRecordsPromptWhenPageOpens(t *testing.T) { + srv, _ := challengedService(t) + signIn := &signInStub{reply: func(call int) (bool, error) { + if call == 1 { + return false, errors.New("sign-in did not complete") + } + return call == 3, nil + }} + recorder := &promptRecorder{} + c := testChallenge(signIn.SignIn) + c.Recorder = recorder + b := bridgeFor(srv.URL) + b.Hooks = newSignInHooks(c, io.Discard) + p := startPiped(t, t.Context(), b) + authenticate := `{"jsonrpc":"2.0","id":%d,"method":"tools/call","params":{"name":"authenticate","arguments":{}}}` + challenge(t, p) + + p.send(t, fmt.Sprintf(authenticate, 1)) + p.waitLines(t, 2) + p.send(t, fmt.Sprintf(authenticate, 2)) + p.waitLines(t, 4) + p.send(t, fmt.Sprintf(authenticate, 3)) + p.waitLines(t, 6) + + assert.Equal(t, []bool{true, true, true}, signIn.calls(), "every call opens the page") + assert.Equal(t, []bool{false, true}, recorder.prompts(), "call 3 was already signed in and records nothing") + require.NoError(t, p.finish(t)) +} + func TestAuthenticateReopensPageAfterTimeout(t *testing.T) { srv, _ := challengedService(t) signIn := &signInStub{reply: func(call int) (bool, error) { @@ -343,7 +392,7 @@ func TestChallengeHintsPrintTheLoginCommand(t *testing.T) { return false, true, errors.New("sign-in to system.ai.github did not complete in 5 minutes. Finish at " + testURL + " then retry: " + loginCommand) } b := bridgeFor(srv.URL) - b.Hooks = newSignInHooks(&Challenge{Service: testService, Profile: "my profile", Host: testHost, Label: "GitHub", LoginCommand: loginCommand, SignIn: signIn}, io.Discard) + b.Hooks = newSignInHooks(&Challenge{Service: testService, Profile: "my profile", Host: testHost, Label: "GitHub", LoginCommand: loginCommand, SignIn: signIn, Recorder: &promptRecorder{}}, io.Discard) p := startPiped(t, t.Context(), b) p.send(t, `{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"echo","arguments":{}}}`) @@ -965,6 +1014,7 @@ func testChallenge(signIn SignInFunc) *Challenge { LoginCommand: "databricks aitools mcp login system.ai.github --profile logfood", LogoutCommand: "databricks aitools mcp logout system.ai.github --profile logfood", SignIn: signIn, + Recorder: &promptRecorder{}, } } diff --git a/libs/telemetry/protos/aitools_install.go b/libs/telemetry/protos/aitools_install.go index 406dde89a58..ead0a7b76c8 100644 --- a/libs/telemetry/protos/aitools_install.go +++ b/libs/telemetry/protos/aitools_install.go @@ -74,6 +74,23 @@ const ( // The command's context was cancelled. SIGINT outside a prompt is not // counted here: it ends the process before telemetry is uploaded. AitoolsErrorCategoryCommandInterrupted AitoolsErrorCategory = "COMMAND_INTERRUPTED" + + // `aitools mcp`: profiles and sign-in. + AitoolsErrorCategoryNoProfile AitoolsErrorCategory = "NO_PROFILE" + AitoolsErrorCategoryProfileRequired AitoolsErrorCategory = "PROFILE_REQUIRED" + AitoolsErrorCategoryProfileNotFound AitoolsErrorCategory = "PROFILE_NOT_FOUND" + AitoolsErrorCategoryProfileHostChanged AitoolsErrorCategory = "PROFILE_HOST_CHANGED" + AitoolsErrorCategoryAuthFailed AitoolsErrorCategory = "AUTH_FAILED" + AitoolsErrorCategorySignInRequired AitoolsErrorCategory = "SIGN_IN_REQUIRED" + AitoolsErrorCategorySignInTimeout AitoolsErrorCategory = "SIGN_IN_TIMEOUT" + + // `aitools mcp`: the service list, agent configs and the proxy. + AitoolsErrorCategoryMCPServiceNotFound AitoolsErrorCategory = "MCP_SERVICE_NOT_FOUND" + AitoolsErrorCategoryMCPPermissionDenied AitoolsErrorCategory = "MCP_PERMISSION_DENIED" + AitoolsErrorCategoryMCPLocationNotFound AitoolsErrorCategory = "MCP_LOCATION_NOT_FOUND" + AitoolsErrorCategoryMCPNameConflict AitoolsErrorCategory = "MCP_NAME_CONFLICT" + AitoolsErrorCategoryMCPUpstreamError AitoolsErrorCategory = "MCP_UPSTREAM_ERROR" + AitoolsErrorCategoryAgentConfigInvalid AitoolsErrorCategory = "AGENT_CONFIG_INVALID" ) // AitoolsAgentResult records one agent's failed or skipped outcome within an diff --git a/libs/telemetry/protos/aitools_mcp.go b/libs/telemetry/protos/aitools_mcp.go new file mode 100644 index 00000000000..c42dbc20285 --- /dev/null +++ b/libs/telemetry/protos/aitools_mcp.go @@ -0,0 +1,94 @@ +package protos + +// The types in this file mirror the aitools MCP messages of the databricks_cli +// lumberjack protos: AitoolsMcpEvent and its server result in +// proto/logs/frontend/databricks_cli/aitools_mcp.proto, and the enums in +// proto/logs/frontend/databricks_cli/enum.proto. + +// AitoolsMcpOperation mirrors AitoolsMcpOperation.Type in aitools_mcp.proto. +type AitoolsMcpOperation string + +const ( + AitoolsMcpOperationAdd AitoolsMcpOperation = "ADD" + AitoolsMcpOperationRemove AitoolsMcpOperation = "REMOVE" + AitoolsMcpOperationSignIn AitoolsMcpOperation = "SIGN_IN" + AitoolsMcpOperationSignOut AitoolsMcpOperation = "SIGN_OUT" + AitoolsMcpOperationProxyStarted AitoolsMcpOperation = "PROXY_STARTED" + AitoolsMcpOperationSignInPrompted AitoolsMcpOperation = "SIGN_IN_PROMPTED" +) + +// AitoolsMcpService mirrors AitoolsMcpService.Type in enum.proto. The CLI +// sends only these values, never a service name: the system.ai leaves in the +// table map to their own value, other system.ai leaves to OtherSystem, and any +// service outside system.ai to Custom. +type AitoolsMcpService string + +const ( + AitoolsMcpServiceOtherSystem AitoolsMcpService = "OTHER_SYSTEM" + AitoolsMcpServiceCustom AitoolsMcpService = "CUSTOM" + AitoolsMcpServiceGenieOne AitoolsMcpService = "GENIE_ONE" + AitoolsMcpServiceDBSQL AitoolsMcpService = "DBSQL" + AitoolsMcpServiceWebSearch AitoolsMcpService = "WEB_SEARCH" + AitoolsMcpServiceSandbox AitoolsMcpService = "SANDBOX" + AitoolsMcpServiceSlack AitoolsMcpService = "SLACK" + AitoolsMcpServiceGitHub AitoolsMcpService = "GITHUB" + AitoolsMcpServiceAtlassian AitoolsMcpService = "ATLASSIAN" + AitoolsMcpServiceGoogleDrive AitoolsMcpService = "GOOGLE_DRIVE" + AitoolsMcpServiceGoogleCalendar AitoolsMcpService = "GOOGLE_CALENDAR" + AitoolsMcpServiceGmail AitoolsMcpService = "GMAIL" + AitoolsMcpServiceMicrosoft365 AitoolsMcpService = "MICROSOFT_365" +) + +// AitoolsMcpSignInState mirrors AitoolsMcpSignInState.Type in enum.proto. It is +// the sign-in state of a service after the operation. +type AitoolsMcpSignInState string + +const ( + AitoolsMcpSignInStateNotRequired AitoolsMcpSignInState = "NOT_REQUIRED" + AitoolsMcpSignInStateActive AitoolsMcpSignInState = "ACTIVE" + AitoolsMcpSignInStatePending AitoolsMcpSignInState = "PENDING" +) + +// AitoolsTargetOutcome mirrors AitoolsTargetOutcome.Type in enum.proto. It is +// what an operation did to one target: an agent config entry, or a sign-in. +type AitoolsTargetOutcome string + +const ( + AitoolsTargetOutcomeChanged AitoolsTargetOutcome = "CHANGED" + AitoolsTargetOutcomeUnchanged AitoolsTargetOutcome = "UNCHANGED" + AitoolsTargetOutcomeUpdateAvailable AitoolsTargetOutcome = "UPDATE_AVAILABLE" + AitoolsTargetOutcomeSkipped AitoolsTargetOutcome = "SKIPPED" + AitoolsTargetOutcomeFailed AitoolsTargetOutcome = "FAILED" +) + +// AitoolsMcpServerResult is the result for one service, for one agent when the +// operation writes agent config. Unset fields are omitted: an agent is not set +// for sign-in, an outcome is not set for proxy events, and an error category is +// set only when the target failed or was skipped. +type AitoolsMcpServerResult struct { + Service AitoolsMcpService `json:"service"` + Agent AitoolsAgentType `json:"agent,omitempty"` + Outcome AitoolsTargetOutcome `json:"outcome,omitempty"` + ErrorCategory AitoolsErrorCategory `json:"error_category,omitempty"` + SignInState AitoolsMcpSignInState `json:"sign_in_state,omitempty"` +} + +// AitoolsMcpEvent is emitted by `databricks aitools mcp` commands and by the +// proxy (`serve`). It carries enums and counts only: no host, profile, workspace +// ID, custom service or schema name, path, error text or tool name. +type AitoolsMcpEvent struct { + // CallingAgent is the coding agent that ran the command or started the proxy. + // AitoolsAgentTypeUnspecified when no agent is detected. + CallingAgent AitoolsAgentType `json:"calling_agent"` + + Operation AitoolsMcpOperation `json:"operation"` + + // Servers holds one entry per (service, agent) for ADD and REMOVE, one per + // service for SIGN_IN, SIGN_OUT, PROXY_STARTED and SIGN_IN_PROMPTED. + Servers []AitoolsMcpServerResult `json:"servers,omitempty"` + + // BoundProfileCount is the number of distinct profiles with MCP servers + // installed, read from the state file after the operation. Zero is a real + // value (every server removed), so it is not omitted. + BoundProfileCount int64 `json:"bound_profile_count"` +} diff --git a/libs/telemetry/protos/aitools_mcp_test.go b/libs/telemetry/protos/aitools_mcp_test.go new file mode 100644 index 00000000000..3b6ef512e3a --- /dev/null +++ b/libs/telemetry/protos/aitools_mcp_test.go @@ -0,0 +1,57 @@ +package protos + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The lumberjack proto keys these fields in snake_case. This pins the wire keys of an event +// and of one server result, and that unset enums are omitted rather than sent empty. +func TestAitoolsMcpEventEncodesSnakeCaseKeys(t *testing.T) { + b, err := json.Marshal(DatabricksCliLog{AitoolsMcpEvent: &AitoolsMcpEvent{ + CallingAgent: AitoolsAgentTypeClaudeCode, + Operation: AitoolsMcpOperationAdd, + BoundProfileCount: 0, + Servers: []AitoolsMcpServerResult{ + { + Service: AitoolsMcpServiceGitHub, + Agent: AitoolsAgentTypeCodex, + Outcome: AitoolsTargetOutcomeFailed, + ErrorCategory: AitoolsErrorCategoryAgentConfigInvalid, + SignInState: AitoolsMcpSignInStatePending, + }, + }, + }}) + require.NoError(t, err) + + var got map[string]any + require.NoError(t, json.Unmarshal(b, &got)) + event, ok := got["aitools_mcp_event"].(map[string]any) + require.True(t, ok, "aitools_mcp_event must be the event key") + + assert.Equal(t, "CLAUDE_CODE", event["calling_agent"]) + assert.Equal(t, "ADD", event["operation"]) + // Zero is a real count and is sent. + assert.EqualValues(t, 0, event["bound_profile_count"]) + + servers := event["servers"].([]any) + require.Len(t, servers, 1) + server := servers[0].(map[string]any) + assert.Equal(t, map[string]any{ + "service": "GITHUB", + "agent": "CODEX", + "outcome": "FAILED", + "error_category": "AGENT_CONFIG_INVALID", + "sign_in_state": "PENDING", + }, server) +} + +// A proxy event has no agent on the result, no outcome and no error category when it succeeded. +func TestAitoolsMcpServerResultOmitsUnsetFields(t *testing.T) { + b, err := json.Marshal(AitoolsMcpServerResult{Service: AitoolsMcpServiceOtherSystem}) + require.NoError(t, err) + assert.JSONEq(t, `{"service":"OTHER_SYSTEM"}`, string(b)) +} diff --git a/libs/telemetry/protos/frontend_log.go b/libs/telemetry/protos/frontend_log.go index f3c37736cc1..dcaf17d8e92 100644 --- a/libs/telemetry/protos/frontend_log.go +++ b/libs/telemetry/protos/frontend_log.go @@ -24,6 +24,7 @@ type DatabricksCliLog struct { SshAgentShimEvent *SshAgentShimEvent `json:"ssh_agent_shim_event,omitempty"` BundleConfigRemoteSyncEvent *BundleConfigRemoteSyncEvent `json:"bundle_config_remote_sync_event,omitempty"` AitoolsInstallEvent *AitoolsInstallEvent `json:"aitools_install_event,omitempty"` + AitoolsMcpEvent *AitoolsMcpEvent `json:"aitools_mcp_event,omitempty"` SetupLocalEvent *SetupLocalEvent `json:"setup_local_event,omitempty"` AirRunEvent *AirRunEvent `json:"air_run_event,omitempty"` AitoolsHintEvent *AitoolsHintEvent `json:"aitools_hint_event,omitempty"` From 635ce6dd14d26291271483a248b151a0164e167d Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Thu, 12 Nov 2026 14:46:41 +0000 Subject: [PATCH 02/22] Create the agent config dirs in the telemetry acceptance cases mcp add now refuses an agent named with --agents whose config directory is missing. These cases add for claude-code (and codex in mcp-add) in a fresh HOME, so they create the directories first. The telemetry events do not change. Co-authored-by: Isaac --- acceptance/cmd/aitools/telemetry/mcp-add/script | 1 + acceptance/cmd/aitools/telemetry/mcp-remove/script | 1 + acceptance/cmd/aitools/telemetry/mcp-sign-in/script | 1 + 3 files changed, 3 insertions(+) diff --git a/acceptance/cmd/aitools/telemetry/mcp-add/script b/acceptance/cmd/aitools/telemetry/mcp-add/script index 83d6b223112..2280d098810 100644 --- a/acceptance/cmd/aitools/telemetry/mcp-add/script +++ b/acceptance/cmd/aitools/telemetry/mcp-add/script @@ -1,4 +1,5 @@ sethome home +mkdir -p home/.claude home/.codex envsubst < .databrickscfg > out && mv out .databrickscfg export DATABRICKS_CONFIG_FILE=.databrickscfg diff --git a/acceptance/cmd/aitools/telemetry/mcp-remove/script b/acceptance/cmd/aitools/telemetry/mcp-remove/script index b25e4b4a31a..3d2481d9f43 100644 --- a/acceptance/cmd/aitools/telemetry/mcp-remove/script +++ b/acceptance/cmd/aitools/telemetry/mcp-remove/script @@ -1,4 +1,5 @@ sethome home +mkdir -p home/.claude envsubst < .databrickscfg > out && mv out .databrickscfg export DATABRICKS_CONFIG_FILE=.databrickscfg diff --git a/acceptance/cmd/aitools/telemetry/mcp-sign-in/script b/acceptance/cmd/aitools/telemetry/mcp-sign-in/script index 943af911b86..72025e3cde6 100644 --- a/acceptance/cmd/aitools/telemetry/mcp-sign-in/script +++ b/acceptance/cmd/aitools/telemetry/mcp-sign-in/script @@ -1,4 +1,5 @@ sethome home +mkdir -p home/.claude envsubst < .databrickscfg > out && mv out .databrickscfg export DATABRICKS_CONFIG_FILE=.databrickscfg From b6f82a6f9f7906825184d52ecd113a1e088d1a09 Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Fri, 13 Nov 2026 14:46:54 +0000 Subject: [PATCH 03/22] Record the sign-in prompt only when a page URL was shown A sign-in that fails before its URL is printed, such as a read of the workspace ID, prompted nobody, so it no longer logs SIGN_IN_PROMPTED. The serve command sets the proxy's recorder where it builds the challenge, so the challenge is never handed over without one. Co-authored-by: Isaac --- cmd/aitools/mcp_serve.go | 12 ++++++------ libs/aitools/mcp/proxy/challenge.go | 7 ++++--- libs/aitools/mcp/proxy/challenge_test.go | 24 ++++++++++++++++++++++++ 3 files changed, 34 insertions(+), 9 deletions(-) diff --git a/cmd/aitools/mcp_serve.go b/cmd/aitools/mcp_serve.go index 5bcee77cf1c..0af3cd04017 100644 --- a/cmd/aitools/mcp_serve.go +++ b/cmd/aitools/mcp_serve.go @@ -25,6 +25,7 @@ import ( "github.com/databricks/cli/libs/auth/u2m" "github.com/databricks/cli/libs/browser" "github.com/databricks/cli/libs/cmdctx" + "github.com/databricks/cli/libs/telemetry/protos" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/config" "github.com/databricks/databricks-sdk-go/config/credentials" @@ -137,10 +138,7 @@ func runMcpServe(cmd *cobra.Command, f mcpServeFlags) error { return serveFailure(cmd, err, 1) } } - challenge, err := serveChallenge(ctx, serveTarget{w: w, entry: entry, profile: profileName, service: f.service, pinnedFile: pinnedFile}, cmd.ErrOrStderr()) - if challenge != nil { - challenge.Recorder = mcpProxyTelemetry{agent: agent, service: f.service} - } + challenge, err := serveChallenge(ctx, serveTarget{w: w, entry: entry, profile: profileName, service: f.service, pinnedFile: pinnedFile, agent: agent}, cmd.ErrOrStderr()) if err != nil { // A rejected token stops serve by cancelling ctx, so the refresh failure is checked before the signal, and its // message is the one that names the login. @@ -196,14 +194,15 @@ func (g guardCredentials) Configure(context.Context, *config.Config) (credential return credentials.CredentialsProviderFn(g.refresh.authenticateCall), nil } -// serveTarget is the server that a serve session bridges: its service, the bound profile, the install record, and the -// config file it is pinned to. w is the profile's client, which must be guardedClient's. +// serveTarget is the server that a serve session bridges: its service, the bound profile, the install record, the config +// file it is pinned to, and the agent that started it. w is the profile's client, which must be guardedClient's. type serveTarget struct { w *databricks.WorkspaceClient entry *mcp.Entry profile string service string pinnedFile string + agent protos.AitoolsAgentType } // serveChallenge returns the sign-in handling of a service that needs provider sign-in, or nil for @@ -239,6 +238,7 @@ func serveChallenge(ctx context.Context, t serveTarget, errOut io.Writer) (*prox LogoutCommand: mcpLogoutHint(t.service, t.profile, t.pinnedFile), PinnedFile: t.pinnedFile, SignIn: serveSignIn(t), + Recorder: mcpProxyTelemetry{agent: t.agent, service: t.service}, }, nil } diff --git a/libs/aitools/mcp/proxy/challenge.go b/libs/aitools/mcp/proxy/challenge.go index 3ee395f8c48..889d9bbd4a4 100644 --- a/libs/aitools/mcp/proxy/challenge.go +++ b/libs/aitools/mcp/proxy/challenge.go @@ -78,7 +78,7 @@ type Challenge struct { // SignIn runs the sign-in. The command provides it, so the proxy never polls or opens a browser itself. Its error is // the answer's text, so it names the service, the profile and the fix. SignIn SignInFunc - // Recorder logs the sign-in prompt. The command provides it. + // Recorder logs the sign-in prompt. The command provides it; it must not be nil. Recorder Recorder } @@ -457,8 +457,9 @@ func (h *signInHooks) authenticate(ctx context.Context, id, progressToken json.R open := !h.pageCancelled alreadySignedIn, urlShown, err := h.c.SignIn(ctx, SignInRequest{Open: open, Progress: report, Stderr: h.stderr}) - // A caller that was already signed in opened no page, so there is no prompt to record. - if open && !alreadySignedIn { + // A prompt is a page the user was shown. A caller that was already signed in, or a sign-in that failed before its + // URL was printed, prompted nobody. + if open && !alreadySignedIn && urlShown { h.c.Recorder.SignInPrompted(ctx, err == nil) } h.pageCancelled = urlShown && err != nil && ctx.Err() != nil diff --git a/libs/aitools/mcp/proxy/challenge_test.go b/libs/aitools/mcp/proxy/challenge_test.go index 0b212946abc..cad710c7c9a 100644 --- a/libs/aitools/mcp/proxy/challenge_test.go +++ b/libs/aitools/mcp/proxy/challenge_test.go @@ -304,6 +304,30 @@ func TestAuthenticateRecordsPromptWhenPageOpens(t *testing.T) { require.NoError(t, p.finish(t)) } +// A sign-in that fails before its URL is printed, such as a read of the workspace ID, shows no page: nothing is recorded. +func TestAuthenticateRecordsNoPromptWhenNoURLShown(t *testing.T) { + srv, _ := challengedService(t) + var calls int + signIn := func(context.Context, SignInRequest) (bool, bool, error) { + calls++ + return false, false, errors.New("cannot read the workspace ID") + } + recorder := &promptRecorder{} + c := testChallenge(signIn) + c.Recorder = recorder + b := bridgeFor(srv.URL) + b.Hooks = newSignInHooks(c, io.Discard) + p := startPiped(t, t.Context(), b) + challenge(t, p) + + p.send(t, `{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"authenticate","arguments":{}}}`) + p.waitLines(t, 2) + + assert.Equal(t, 1, calls, "the sign-in ran") + assert.Empty(t, recorder.prompts(), "no page was shown, so no prompt is recorded") + require.NoError(t, p.finish(t)) +} + func TestAuthenticateReopensPageAfterTimeout(t *testing.T) { srv, _ := challengedService(t) signIn := &signInStub{reply: func(call int) (bool, error) { From 231e0b4a322f61f3084a0fd4ed8eb5b2f1838b0b Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Fri, 13 Nov 2026 14:48:27 +0000 Subject: [PATCH 04/22] Keep one set of login steps on an MCP authentication failure The MCP sign-in commands set their config for the telemetry upload, which also turns on root's auth enrichment. An AUTH_FAILED error already names the login command for its profile, so enrichment now leaves such an error as it is, and the user gets one set of next steps. Co-authored-by: Isaac --- cmd/aitools/mcp_login_test.go | 17 +++++++++++++++++ libs/aitools/mcp/binding.go | 6 ++++++ libs/auth/error.go | 10 ++++++++++ libs/auth/error_test.go | 27 +++++++++++++++++++++++++++ 4 files changed, 60 insertions(+) diff --git a/cmd/aitools/mcp_login_test.go b/cmd/aitools/mcp_login_test.go index fc25c25c2ab..8e8da4fe637 100644 --- a/cmd/aitools/mcp_login_test.go +++ b/cmd/aitools/mcp_login_test.go @@ -20,6 +20,7 @@ import ( "github.com/databricks/cli/libs/aitools/agents" "github.com/databricks/cli/libs/aitools/installer" "github.com/databricks/cli/libs/aitools/mcp" + "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/telemetry/protos" "github.com/databricks/databricks-sdk-go" @@ -106,6 +107,22 @@ func TestMcpLoginRejectedTokenIsAuthFailed(t *testing.T) { assert.Contains(t, err.Error(), "Run: databricks auth login --profile logfood") } +// TestMcpLoginRejectedTokenHasOneSetOfNextSteps checks the error that root prints for a rejected token after login has +// its client: the MCP login command is the only set of next steps, since root's auth enrichment leaves it alone. +func TestMcpLoginRejectedTokenHasOneSetOfNextSteps(t *testing.T) { + home := withHome(t) + server := loginTestServer(t, http.StatusUnauthorized, `{"error_code":"UNAUTHENTICATED","message":"Invalid access token."}`) + writeHomeConfig(t, home, "[logfood]\nhost = "+server.URL+"\nauth_type = pat\ntoken = dapi-expired\n") + + out := mcpSignInOutput{Service: "system.ai.github"} + err := runMcpSignIn(t.Context(), io.Discard, "logfood", "system.ai.github", mcpLogin, &out) + require.Error(t, err) + require.NotNil(t, out.cfg, "the client exists, so root enriches errors") + enriched := auth.EnrichAuthError(t.Context(), out.cfg, err) + assert.Equal(t, err.Error(), enriched.Error()) + assert.Equal(t, 0, strings.Count(enriched.Error(), "Next steps:")) +} + // TestMcpLogoutUnknownServiceIsAnError checks that logout of a service the profile cannot see fails, rather than // reporting that the caller is not signed in, so that a typo in the name is seen. func TestMcpLogoutUnknownServiceIsAnError(t *testing.T) { diff --git a/libs/aitools/mcp/binding.go b/libs/aitools/mcp/binding.go index 1a530a73efa..297d820fef5 100644 --- a/libs/aitools/mcp/binding.go +++ b/libs/aitools/mcp/binding.go @@ -59,6 +59,12 @@ func (e *BindError) Unwrap() error { return e.cause } +// NextStepsIncluded reports whether the message already names the next steps: an AUTH_FAILED message names the login +// command for the profile. The root command does not add a second set of steps to it. +func (e *BindError) NextStepsIncluded() bool { + return e.Category == BindAuthFailed +} + // Bind checks that the profile still names the host and workspace recorded for the service, // then authenticates with the profile's own credentials. It never reads stdin, prompts, or // loads a bundle. It returns the resolved config and the matching state entry, which is nil diff --git a/libs/auth/error.go b/libs/auth/error.go index fdb5f256f43..5e5b6369c24 100644 --- a/libs/auth/error.go +++ b/libs/auth/error.go @@ -82,6 +82,13 @@ func RewriteAuthError(ctx context.Context, host, accountId, profile string, err return false, err } +// nextStepsIncluded is implemented by an error whose message already gives the next steps for the failure, such as the +// login command of an MCP server. EnrichAuthError leaves such an error as it is, so the user is given one set of steps. +type nextStepsIncluded interface { + error + NextStepsIncluded() bool +} + // EnrichAuthError appends identity context and remediation steps to 401/403 API errors. // For non-API errors or other status codes, the original error is returned unchanged. func EnrichAuthError(ctx context.Context, cfg *config.Config, err error) error { @@ -92,6 +99,9 @@ func EnrichAuthError(ctx context.Context, cfg *config.Config, err error) error { if apiErr.StatusCode != http.StatusUnauthorized && apiErr.StatusCode != http.StatusForbidden { return err } + if steps, ok := errors.AsType[nextStepsIncluded](err); ok && steps.NextStepsIncluded() { + return err + } var b strings.Builder diff --git a/libs/auth/error_test.go b/libs/auth/error_test.go index ce8f0df1b63..b96a0386405 100644 --- a/libs/auth/error_test.go +++ b/libs/auth/error_test.go @@ -63,6 +63,33 @@ func TestEnrichAuthError_PreservesOriginalError(t *testing.T) { assert.Equal(t, "PERMISSION_DENIED", unwrapped.ErrorCode) } +// stepsIncludedError is an authentication failure whose message already gives the next steps. +type stepsIncludedError struct { + cause error +} + +func (e *stepsIncludedError) Error() string { + return "credential rejected. Run: databricks auth login --profile test" +} + +func (e *stepsIncludedError) Unwrap() error { + return e.cause +} + +func (e *stepsIncludedError) NextStepsIncluded() bool { + return true +} + +func TestEnrichAuthErrorLeavesErrorWithNextStepsAsIs(t *testing.T) { + cfg := &config.Config{Profile: "test", Host: "https://example.com", AuthType: "pat"} + original := &stepsIncludedError{cause: &apierr.APIError{StatusCode: 401, Message: "Unauthorized"}} + assert.Equal(t, original, EnrichAuthError(t.Context(), cfg, original)) + + // An API error without a message of its own is still enriched. + plain := &apierr.APIError{StatusCode: 401, Message: "Unauthorized"} + assert.Contains(t, EnrichAuthError(t.Context(), cfg, plain).Error(), "Next steps:") +} + func TestEnrichAuthError(t *testing.T) { tests := []struct { name string From a87a80d43dac21b5eca108cd1b2023d49c5685f7 Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Fri, 13 Nov 2026 14:50:20 +0000 Subject: [PATCH 05/22] Resolve the remove upload's auth after the local edit remove resolved the profile before it touched any file, so an offline run could wait on the network first, even with telemetry disabled. The edit now runs first; the upload's profile is resolved afterwards, with a three second budget, and only when telemetry is on. Co-authored-by: Isaac --- cmd/aitools/mcp_remove.go | 3 +-- cmd/aitools/mcp_remove_test.go | 17 +++++++++++++++++ cmd/aitools/mcp_telemetry.go | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 2 deletions(-) diff --git a/cmd/aitools/mcp_remove.go b/cmd/aitools/mcp_remove.go index 6ae241880f7..ecf7d7fc89f 100644 --- a/cmd/aitools/mcp_remove.go +++ b/cmd/aitools/mcp_remove.go @@ -94,14 +94,13 @@ and reported. When an agent's config moved, the entry is removed from the file i Entries that run 'aitools mcp serve' without an install record are not chosen by any flag; 'aitools mcp list' shows them as unmanaged. Provider credentials are not deleted.`, Args: cobra.NoArgs, - // Resolve auth best-effort so telemetry can upload; see tryConfigureAuth. - PreRunE: tryConfigureAuth, RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() jsonMode := installOutputIsJSON(cmd) out := &removeOutput{Removed: []removeRemoved{}, Skipped: []removeSkipped{}} runErr := runMcpRemove(ctx, cmd, f, jsonMode, out) + configureUploadAuth(cmd) logMcpEvent(ctx, protos.AitoolsMcpEvent{ CallingAgent: callingAgentType(), Operation: protos.AitoolsMcpOperationRemove, diff --git a/cmd/aitools/mcp_remove_test.go b/cmd/aitools/mcp_remove_test.go index 6a62975cf83..2f2c01c2c97 100644 --- a/cmd/aitools/mcp_remove_test.go +++ b/cmd/aitools/mcp_remove_test.go @@ -180,6 +180,23 @@ func TestMcpRemoveDeletesOwnedEntriesAndPrunesMissing(t *testing.T) { assert.Equal(t, "github-dogfood", state.Servers[0].Name) } +// TestMcpRemoveMakesNoRequestWhenTelemetryIsOff checks that remove edits its agent config without a request to the +// profile's host. Nothing is uploaded when telemetry is off, so no auth is resolved for an upload. +func TestMcpRemoveMakesNoRequestWhenTelemetryIsOff(t *testing.T) { + home := withHome(t) + t.Setenv("DATABRICKS_CLI_DISABLE_TELEMETRY", "1") + server := noRequestServer(t) + writeHomeConfig(t, home, "[logfood]\nhost = "+server.URL+"\nauth_type = oauth-m2m\nclient_id = id\nclient_secret = secret\n") + claude := agents.ByName("claude-code") + claudePath := filepath.Join(home, ".claude.json") + genie := seedInstall(t, claude, "genie_one_mcp-logfood", "system.ai.genie_one_mcp", "logfood", server.URL) + saveState(t, genie) + + _, _, err := runMcpCmd(t, "remove", "--names", "genie_one_mcp-logfood", "--output", "json") + require.NoError(t, err) + assert.NotContains(t, readJSONFile(t, claudePath)["mcpServers"], "genie_one_mcp-logfood") +} + func TestMcpRemoveSkipsEntryChangedOutsideCLI(t *testing.T) { home := withHome(t) claude := agents.ByName("claude-code") diff --git a/cmd/aitools/mcp_telemetry.go b/cmd/aitools/mcp_telemetry.go index ad05d2f6646..f23497a01d7 100644 --- a/cmd/aitools/mcp_telemetry.go +++ b/cmd/aitools/mcp_telemetry.go @@ -3,14 +3,47 @@ package aitools import ( "context" "strings" + "time" "github.com/databricks/cli/libs/aitools/agents" "github.com/databricks/cli/libs/aitools/mcp" + "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/telemetry" "github.com/databricks/cli/libs/telemetry/protos" + "github.com/spf13/cobra" ) +// uploadAuthBudget bounds the resolution of the profile that authenticates a telemetry upload. +const uploadAuthBudget = 3 * time.Second + +// configureUploadAuth sets the config that authenticates the exit-time upload of a command that edits local files +// only. It runs after the edit, so an offline run does not wait on the network before it edits a file, and it does +// nothing when telemetry is disabled. The profile is the one the command would use without a flag; without one +// the upload is skipped. +func configureUploadAuth(cmd *cobra.Command) { + ctx := cmd.Context() + if telemetry.Disabled(ctx) { + return + } + configFile, _, err := addConfigFile(ctx) + if err != nil { + log.Debugf(ctx, "telemetry: no config file to upload with: %s", err) + return + } + profileName, err := resolveProfile(ctx, profileFlag(cmd)) + if err != nil { + log.Debugf(ctx, "telemetry: no profile to upload with: %s", err) + return + } + cfg, err := mcp.BoundConfigWithin(profileName, configFile, uploadAuthBudget) + if err != nil { + log.Debugf(ctx, "telemetry: cannot resolve profile %s to upload with: %s", profileName, err) + return + } + cmd.SetContext(cmdctx.SetConfigUsed(ctx, cfg)) +} + // mcpServiceTypes maps the system.ai leaves in the CLI's service table to their telemetry enum. var mcpServiceTypes = map[string]protos.AitoolsMcpService{ "genie_one_mcp": protos.AitoolsMcpServiceGenieOne, From 3452750d45fb09581cbd0f71eef48f44b87c478b Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Fri, 13 Nov 2026 14:55:08 +0000 Subject: [PATCH 06/22] Log serve's start after its checks, and keep the failed start's upload PROXY_STARTED was logged before the startup checks, so a start the lookup rejected uploaded as a clean start. It is now logged once the checks have run, as AUTH_FAILED when they fail. A binding that fails AUTH_FAILED uploads with its profile, whose host the binding has already checked. A profile whose host changed still uploads nothing, since the binding never contacts a host the record does not name. A rejected refresh token clears the upload config, so the exit upload does not ask the token endpoint again. Co-authored-by: Isaac --- .../mcp-serve-rejected/.databrickscfg | 3 +++ .../telemetry/mcp-serve-rejected/out.test.toml | 1 + .../telemetry/mcp-serve-rejected/output.txt | 18 ++++++++++++++++++ .../telemetry/mcp-serve-rejected/script | 8 ++++++++ .../telemetry/mcp-serve-rejected/test.toml | 6 ++++++ .../cmd/aitools/telemetry/mcp-serve/output.txt | 3 +++ .../cmd/aitools/telemetry/mcp-serve/script | 2 +- cmd/aitools/mcp_serve.go | 13 ++++++++++++- cmd/aitools/mcp_serve_test.go | 4 ++++ cmd/aitools/mcp_telemetry.go | 13 ++++++++++--- libs/cmdctx/config_used.go | 6 ++++++ libs/cmdctx/config_used_test.go | 4 ++++ 12 files changed, 76 insertions(+), 5 deletions(-) create mode 100644 acceptance/cmd/aitools/telemetry/mcp-serve-rejected/.databrickscfg create mode 100644 acceptance/cmd/aitools/telemetry/mcp-serve-rejected/out.test.toml create mode 100644 acceptance/cmd/aitools/telemetry/mcp-serve-rejected/output.txt create mode 100644 acceptance/cmd/aitools/telemetry/mcp-serve-rejected/script create mode 100644 acceptance/cmd/aitools/telemetry/mcp-serve-rejected/test.toml diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/.databrickscfg b/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/.databrickscfg new file mode 100644 index 00000000000..0369be633c5 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/.databrickscfg @@ -0,0 +1,3 @@ +[logfood] +host = $DATABRICKS_HOST +token = test-token-logfood diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/out.test.toml b/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/output.txt b/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/output.txt new file mode 100644 index 00000000000..18e5860dd73 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/output.txt @@ -0,0 +1,18 @@ + +=== serve whose startup check is rejected: exit 2, and PROXY_STARTED is uploaded with AUTH_FAILED + +>>> musterr [CLI] aitools mcp serve --profile logfood --service system.ai.genie_one_mcp --agent claude-code +No install record for system.ai.genie_one_mcp on profile "logfood"; using its current host [DATABRICKS_URL] +cannot read MCP service system.ai.genie_one_mcp: Invalid access token. +Run: databricks auth login --profile logfood +{ + "calling_agent": "CLAUDE_CODE", + "operation": "PROXY_STARTED", + "servers": [ + { + "service": "GENIE_ONE", + "error_category": "AUTH_FAILED" + } + ], + "bound_profile_count": 0 +} diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/script b/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/script new file mode 100644 index 00000000000..0de703dfbc0 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/script @@ -0,0 +1,8 @@ +sethome home +envsubst < .databrickscfg > out && mv out .databrickscfg +export DATABRICKS_CONFIG_FILE=.databrickscfg +export BROWSER=none + +title "serve whose startup check is rejected: exit 2, and PROXY_STARTED is uploaded with AUTH_FAILED" +trace musterr $CLI aitools mcp serve --profile logfood --service system.ai.genie_one_mcp --agent claude-code < /dev/null > /dev/null +print_requests.py //telemetry-ext | jq '.body.protoLogs[] | fromjson | .entry.databricks_cli_log.aitools_mcp_event | select(. != null)' diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/test.toml b/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/test.toml new file mode 100644 index 00000000000..c081d32cf7a --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/test.toml @@ -0,0 +1,6 @@ +# The startup lookup of the service is rejected, so serve exits 2 before it serves. Its PROXY_STARTED event records the +# failure as AUTH_FAILED, and the upload that follows carries it. +[[Server]] +Pattern = "GET /api/2.1/unity-catalog/mcp-services/{name}" +Response.StatusCode = 401 +Response.Body = '{"error_code": "UNAUTHENTICATED", "message": "Invalid access token."}' diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve/output.txt b/acceptance/cmd/aitools/telemetry/mcp-serve/output.txt index 08586c94def..5ce1d986506 100644 --- a/acceptance/cmd/aitools/telemetry/mcp-serve/output.txt +++ b/acceptance/cmd/aitools/telemetry/mcp-serve/output.txt @@ -13,6 +13,9 @@ === A binding that fails logs PROXY_STARTED with its category, but no upload follows: there is no config to upload with +>>> musterr [CLI] aitools mcp serve --profile gone --service system.ai.genie_one_mcp --agent codex +MCP server system.ai.genie_one_mcp needs Databricks profile "gone", which does not exist in [TEST_TMP_DIR]/.databrickscfg. Set it up with: databricks auth login --profile gone + === authenticate opens the sign-in page: PROXY_STARTED, then SIGN_IN_PROMPTED with the sign-in state after the wait { "calling_agent": "CLAUDE_CODE", diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve/script b/acceptance/cmd/aitools/telemetry/mcp-serve/script index fdceb646167..5b30a8e7ddc 100644 --- a/acceptance/cmd/aitools/telemetry/mcp-serve/script +++ b/acceptance/cmd/aitools/telemetry/mcp-serve/script @@ -23,7 +23,7 @@ $CLI aitools mcp serve --profile logfood --service system.ai.genie_one_mcp --age mcp_events title "A binding that fails logs PROXY_STARTED with its category, but no upload follows: there is no config to upload with" -errcode $CLI aitools mcp serve --profile gone --service system.ai.genie_one_mcp --agent codex < /dev/null > /dev/null 2> /dev/null +trace musterr $CLI aitools mcp serve --profile gone --service system.ai.genie_one_mcp --agent codex < /dev/null > /dev/null mcp_events title "authenticate opens the sign-in page: PROXY_STARTED, then SIGN_IN_PROMPTED with the sign-in state after the wait" diff --git a/cmd/aitools/mcp_serve.go b/cmd/aitools/mcp_serve.go index 0af3cd04017..876212ffffc 100644 --- a/cmd/aitools/mcp_serve.go +++ b/cmd/aitools/mcp_serve.go @@ -114,13 +114,19 @@ func runMcpServe(cmd *cobra.Command, f mcpServeFlags) error { Stderr: cmd.ErrOrStderr(), }) if err != nil { + // An AUTH_FAILED binding has passed the host check, so its profile may authenticate the upload. A profile whose + // host changed is not used: Bind never contacts a host the record does not name. + if errorCategory(err) == protos.AitoolsErrorCategoryAuthFailed { + if configFile, _, cerr := addConfigFile(cmd.Context()); cerr == nil { + setUploadConfig(cmd, profileName, configFile) + } + } logMcpEvent(cmd.Context(), proxyStartedEvent(agent, f.service, err)) return serveFailure(cmd, err, serveBindStatus) } // The bound profile authenticates the exit-time telemetry upload. cmd.SetContext(cmdctx.SetConfigUsed(cmd.Context(), cfg)) - logMcpEvent(cmd.Context(), proxyStartedEvent(agent, f.service, nil)) // A server pinned to a config file names that file in its hints, as add and login do. pinnedFile := mcp.PinnedFile(entry, cfg.ConfigFile) @@ -143,6 +149,7 @@ func runMcpServe(cmd *cobra.Command, f mcpServeFlags) error { // A rejected token stops serve by cancelling ctx, so the refresh failure is checked before the signal, and its // message is the one that names the login. if refreshErr := refresh.failure(); refreshErr != nil { + cmd.SetContext(cmdctx.ClearConfigUsed(cmd.Context())) return serveFailure(cmd, refreshErr, serveBindStatus) } if ctx.Err() != nil { @@ -150,8 +157,10 @@ func runMcpServe(cmd *cobra.Command, f mcpServeFlags) error { return nil } // serveChallenge fails only on an auth failure (see lookupAtStart), which exits serveBindStatus. + logMcpEvent(cmd.Context(), proxyStartedEvent(agent, f.service, err)) return serveFailure(cmd, err, serveBindStatus) } + logMcpEvent(cmd.Context(), proxyStartedEvent(agent, f.service, nil)) return serveProxy(ctx, cmd, refresh, proxy.Options{ In: cmd.InOrStdin(), @@ -301,6 +310,8 @@ func serveProxy(ctx context.Context, cmd *cobra.Command, refresh *refreshGuard, // stdin error after it is a consequence. The final line always names the rejection, as the bridge may not have logged it. func proxyResult(cmd *cobra.Command, refresh *refreshGuard, runErr error) error { if refreshErr := refresh.failure(); refreshErr != nil { + // The token was rejected, so the upload at exit would only ask the token endpoint again. + cmd.SetContext(cmdctx.ClearConfigUsed(cmd.Context())) fmt.Fprintf(cmd.ErrOrStderr(), "stopped after a token refresh failure for profile %s\n", refresh.profile) return exitAlreadyPrinted(refreshErr, serveBindStatus) } diff --git a/cmd/aitools/mcp_serve_test.go b/cmd/aitools/mcp_serve_test.go index 17262c8ba2e..a35064107e5 100644 --- a/cmd/aitools/mcp_serve_test.go +++ b/cmd/aitools/mcp_serve_test.go @@ -25,6 +25,7 @@ import ( "github.com/databricks/cli/libs/aitools/mcp" "github.com/databricks/cli/libs/aitools/mcp/proxy" "github.com/databricks/cli/libs/aitools/mcpconfig" + "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/config" @@ -384,11 +385,14 @@ func TestProxyResultRefreshFailureWinsOverStdinError(t *testing.T) { } require.Error(t, g.authenticateRequest(&http.Request{Header: http.Header{}})) cmd := NewMcpServeCmd() + cmd.SetContext(cmdctx.SetConfigUsed(t.Context(), &config.Config{Host: "https://adb-1.test"})) var stderr bytes.Buffer cmd.SetErr(&stderr) err := proxyResult(cmd, g, errors.New("read stdin: input/output error")) + // The exit-time upload would ask the rejected token's endpoint again, so it has no config. + assert.False(t, cmdctx.HasConfigUsed(cmd.Context())) assert.ErrorIs(t, err, root.ErrAlreadyPrinted) assert.Equal(t, serveBindStatus, root.ExitStatus(err)) assert.ErrorIs(t, err, cause) diff --git a/cmd/aitools/mcp_telemetry.go b/cmd/aitools/mcp_telemetry.go index f23497a01d7..5db01724e1c 100644 --- a/cmd/aitools/mcp_telemetry.go +++ b/cmd/aitools/mcp_telemetry.go @@ -36,6 +36,13 @@ func configureUploadAuth(cmd *cobra.Command) { log.Debugf(ctx, "telemetry: no profile to upload with: %s", err) return } + setUploadConfig(cmd, profileName, configFile) +} + +// setUploadConfig sets the config of profileName in configFile for the exit-time upload, resolved within +// uploadAuthBudget. A profile that does not resolve uploads nothing. +func setUploadConfig(cmd *cobra.Command, profileName, configFile string) { + ctx := cmd.Context() cfg, err := mcp.BoundConfigWithin(profileName, configFile, uploadAuthBudget) if err != nil { log.Debugf(ctx, "telemetry: cannot resolve profile %s to upload with: %s", profileName, err) @@ -215,9 +222,9 @@ func signOutServerResult(service, state string, err error) protos.AitoolsMcpServ return r } -// proxyStartedEvent is the PROXY_STARTED event of serve. err is the binding failure, or nil when the -// proxy is about to serve. The event only uploads when the bound profile resolved, so a failed -// binding usually leaves it unsent. +// proxyStartedEvent is the PROXY_STARTED event of serve, logged once the startup checks have run. err is the failure +// of a check, or nil when the proxy is about to serve. The event uploads only when its profile resolved: a failed +// binding with no profile to upload with leaves it unsent. func proxyStartedEvent(agent protos.AitoolsAgentType, service string, err error) protos.AitoolsMcpEvent { r := protos.AitoolsMcpServerResult{Service: mcpServiceType(service)} if err != nil { diff --git a/libs/cmdctx/config_used.go b/libs/cmdctx/config_used.go index 40595ef8aa9..ed20650aea1 100644 --- a/libs/cmdctx/config_used.go +++ b/libs/cmdctx/config_used.go @@ -18,6 +18,12 @@ func ConfigUsed(ctx context.Context) *config.Config { return cfg } +// ClearConfigUsed returns a context with no config used, as if SetConfigUsed had not run: the telemetry upload is +// skipped and auth errors are not enriched. +func ClearConfigUsed(ctx context.Context) context.Context { + return context.WithValue(ctx, configUsedKey, nil) +} + func HasConfigUsed(ctx context.Context) bool { _, ok := ctx.Value(configUsedKey).(*config.Config) return ok diff --git a/libs/cmdctx/config_used_test.go b/libs/cmdctx/config_used_test.go index f99febf8046..8a0724ceb76 100644 --- a/libs/cmdctx/config_used_test.go +++ b/libs/cmdctx/config_used_test.go @@ -27,6 +27,10 @@ func TestCommandConfigUsed(t *testing.T) { // The config should have the correct configuration. assert.Equal(t, "https://test.test", ConfigUsed(ctx).Host) + // Clearing the config leaves none, as if it had never been set. + assert.False(t, HasConfigUsed(ClearConfigUsed(ctx))) + assert.True(t, HasConfigUsed(ctx)) + // Second call should update the config used. cfg2 := &config.Config{ Host: "https://test2.test", From 2bd5d176c1a36586e57c3713a89b32a430e3d584 Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Fri, 13 Nov 2026 14:57:39 +0000 Subject: [PATCH 07/22] Log only events with a reached server, and count profiles by config file Login and logout log nothing when no server was reached, since their upload has no config. A sign-in state that was never learned is left unset rather than PENDING. A remove that only drops records logs its REMOVE, since the state changes. add --login logs SIGN_IN UNCHANGED when the credential was already active and the question was skipped. The bound profile count keys profiles by config file as well as name, as list does. Co-authored-by: Isaac --- .../aitools/telemetry/mcp-remove/output.txt | 15 ++++++++ .../mcp-remove/record-only.state.json | 20 ++++++++++ .../cmd/aitools/telemetry/mcp-remove/script | 6 +++ cmd/aitools/mcp_add.go | 7 +--- cmd/aitools/mcp_login.go | 5 ++- cmd/aitools/mcp_remove.go | 6 ++- cmd/aitools/mcp_telemetry.go | 31 +++++++++++---- cmd/aitools/mcp_telemetry_test.go | 38 ++++++++++++++++--- 8 files changed, 109 insertions(+), 19 deletions(-) create mode 100644 acceptance/cmd/aitools/telemetry/mcp-remove/record-only.state.json diff --git a/acceptance/cmd/aitools/telemetry/mcp-remove/output.txt b/acceptance/cmd/aitools/telemetry/mcp-remove/output.txt index 6ab420b2562..d470471158e 100644 --- a/acceptance/cmd/aitools/telemetry/mcp-remove/output.txt +++ b/acceptance/cmd/aitools/telemetry/mcp-remove/output.txt @@ -68,3 +68,18 @@ ], "bound_profile_count": 0 } + +=== A record with no agent left goes with a REMOVE of its own: CHANGED with no agent, and no profile bound + +>>> [CLI] aitools mcp remove --names dbsql-logfood --profile logfood --output json +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "REMOVE", + "servers": [ + { + "service": "DBSQL", + "outcome": "CHANGED" + } + ], + "bound_profile_count": 0 +} diff --git a/acceptance/cmd/aitools/telemetry/mcp-remove/record-only.state.json b/acceptance/cmd/aitools/telemetry/mcp-remove/record-only.state.json new file mode 100644 index 00000000000..e069ce0fcac --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-remove/record-only.state.json @@ -0,0 +1,20 @@ +{ + "schema_version": 3, + "mcp": { + "servers": [ + { + "name": "dbsql-logfood", + "service": "system.ai.dbsql", + "profile": "logfood", + "host": "https://adb-1.test", + "workspace_id": "", + "sign_in_required": false, + "command": "", + "config_file": "", + "agents": {}, + "installed_at": "2026-01-01T00:00:00Z", + "cli_version": "0.0.0" + } + ] + } +} diff --git a/acceptance/cmd/aitools/telemetry/mcp-remove/script b/acceptance/cmd/aitools/telemetry/mcp-remove/script index 3d2481d9f43..fe054c38ce8 100644 --- a/acceptance/cmd/aitools/telemetry/mcp-remove/script +++ b/acceptance/cmd/aitools/telemetry/mcp-remove/script @@ -33,3 +33,9 @@ title "An entry already gone from the agent config is SKIPPED, and the state rec jq 'del(.mcpServers["genie_one_mcp-logfood"])' home/.claude.json > home/claude.tmp && mv home/claude.tmp home/.claude.json $CLI aitools mcp remove --names genie_one_mcp-logfood --profile logfood --output json > /dev/null mcp_events + +title "A record with no agent left goes with a REMOVE of its own: CHANGED with no agent, and no profile bound" +mkdir -p home/.databricks/aitools/skills +cp record-only.state.json home/.databricks/aitools/skills/.state.json +trace $CLI aitools mcp remove --names dbsql-logfood --profile logfood --output json > /dev/null +mcp_events diff --git a/cmd/aitools/mcp_add.go b/cmd/aitools/mcp_add.go index 9cb504bb423..dd0b83014c0 100644 --- a/cmd/aitools/mcp_add.go +++ b/cmd/aitools/mcp_add.go @@ -1084,6 +1084,7 @@ func signInInstalled(ctx context.Context, errOut io.Writer, w *databricks.Worksp if signedIn(ctx, w, plans[i].service.FullName) { server.SignIn = &addSignInOutput{State: signInStateActive} reportSignedIn(errOut, plans[i].service, req.profile, true) + logMcpEvent(ctx, addSignInEvent(server.Service, signInStateActive, true, nil)) continue } yes, err := promptSignInNow(server.Service) @@ -1100,11 +1101,7 @@ func signInInstalled(ctx context.Context, errOut io.Writer, w *databricks.Worksp url := mcp.SignInURL(w.Config.Host, plans[i].service.FullName, req.workspaceID) state, alreadySignedIn, _, err := signIn(ctx, errOut, w, req.pinned, url, plans[i].service, signInOptions{Profile: req.profile, Open: browser.OpenOutputToStderr}) server.SignIn = &addSignInOutput{State: state} - logMcpEvent(ctx, protos.AitoolsMcpEvent{ - CallingAgent: callingAgentType(), - Operation: protos.AitoolsMcpOperationSignIn, - Servers: []protos.AitoolsMcpServerResult{signInServerResult(server.Service, state, alreadySignedIn, err)}, - }) + logMcpEvent(ctx, addSignInEvent(server.Service, state, alreadySignedIn, err)) if err != nil { server.SignIn.ErrorCategory = errorCategory(err) server.SignIn.Error = err.Error() diff --git a/cmd/aitools/mcp_login.go b/cmd/aitools/mcp_login.go index 63aa724003e..b5cafb58f8d 100644 --- a/cmd/aitools/mcp_login.go +++ b/cmd/aitools/mcp_login.go @@ -102,7 +102,10 @@ func runMcpSignInCommand(cmd *cobra.Command, service string, action mcpSignInAct if out.cfg != nil { cmd.SetContext(cmdctx.SetConfigUsed(cmd.Context(), out.cfg)) } - logMcpEvent(ctx, signInEvent(operation, out, runErr)) + // The event uploads with the profile's config, so a run that reached no server logs nothing. + if out.cfg != nil { + logMcpEvent(ctx, signInEvent(operation, out, runErr)) + } if !jsonMode { return runErr } diff --git a/cmd/aitools/mcp_remove.go b/cmd/aitools/mcp_remove.go index ecf7d7fc89f..a36530eb203 100644 --- a/cmd/aitools/mcp_remove.go +++ b/cmd/aitools/mcp_remove.go @@ -232,7 +232,11 @@ func runMcpRemove(ctx context.Context, cmd *cobra.Command, f mcpRemoveFlags, jso removed.Agents = append(removed.Agents, agentName) drops = append(drops, recordDrop{name: e.Name, agent: agentName}) } - // A server with no agent left in state is removed from state too, so it is reported as removed. + // A server with no agent left in state is removed from state too, so it is reported as removed. Only its + // record goes, which is a change to the state. + if len(e.Agents) == 0 { + out.results = append(out.results, protos.AitoolsMcpServerResult{Service: mcpServiceType(e.Service), Outcome: protos.AitoolsTargetOutcomeChanged}) + } if len(removed.Agents) > 0 || len(e.Agents) == 0 { out.Removed = append(out.Removed, removed) } diff --git a/cmd/aitools/mcp_telemetry.go b/cmd/aitools/mcp_telemetry.go index 5db01724e1c..e12c166ac39 100644 --- a/cmd/aitools/mcp_telemetry.go +++ b/cmd/aitools/mcp_telemetry.go @@ -111,15 +111,20 @@ func logMcpEvent(ctx context.Context, event protos.AitoolsMcpEvent) { log.Debugf(ctx, "telemetry: cannot read the MCP state; dropping the event: %s", err) return } - event.BoundProfileCount = boundProfileCount(state) + event.BoundProfileCount = boundProfileCount(ctx, state) telemetry.Log(ctx, protos.DatabricksCliLog{AitoolsMcpEvent: &event}) } -// boundProfileCount returns the number of distinct profiles that have servers installed. -func boundProfileCount(state *mcp.State) int64 { - profiles := map[string]bool{} +// boundProfileCount returns the number of distinct profiles that have servers installed. A profile is its name in its +// config file, as list groups them, so DEFAULT in two config files counts twice. +func boundProfileCount(ctx context.Context, state *mcp.State) int64 { + profiles := map[profileKey]bool{} for _, e := range state.Servers { - profiles[e.Profile] = true + path, err := mcp.ResolveConfigFile(ctx, e.ConfigFile) + if err != nil { + path = e.ConfigFile + } + profiles[profileKey{profile: e.Profile, path: path}] = true } return int64(len(profiles)) } @@ -178,10 +183,12 @@ func removeServerResult(service, agent string, outcome removeOutcome, err error) return r } -// signInStateOf maps a sign-in state name of login to its telemetry state. Anything but ACTIVE or -// NOT_REQUIRED is still PENDING. +// signInStateOf maps a sign-in state name of login to its telemetry state. A run that never learned the state leaves it +// unset. Anything but ACTIVE or NOT_REQUIRED is still PENDING. func signInStateOf(state string) protos.AitoolsMcpSignInState { switch state { + case "": + return "" case signInStateActive: return protos.AitoolsMcpSignInStateActive case signInStateNotRequired: @@ -206,6 +213,16 @@ func signInServerResult(service, state string, alreadySignedIn bool, err error) return r } +// addSignInEvent is the SIGN_IN event of one provider sign-in under add --login. A sign-in that found the caller +// already signed in is UNCHANGED. +func addSignInEvent(service, state string, alreadySignedIn bool, err error) protos.AitoolsMcpEvent { + return protos.AitoolsMcpEvent{ + CallingAgent: callingAgentType(), + Operation: protos.AitoolsMcpOperationSignIn, + Servers: []protos.AitoolsMcpServerResult{signInServerResult(service, state, alreadySignedIn, err)}, + } +} + // signOutServerResult returns the result of a logout. Logout does not read whether the service needs // sign-in, so the sign-in state is not set. func signOutServerResult(service, state string, err error) protos.AitoolsMcpServerResult { diff --git a/cmd/aitools/mcp_telemetry_test.go b/cmd/aitools/mcp_telemetry_test.go index 11cb852b8ef..046f7083ea4 100644 --- a/cmd/aitools/mcp_telemetry_test.go +++ b/cmd/aitools/mcp_telemetry_test.go @@ -3,6 +3,7 @@ package aitools import ( "encoding/json" "errors" + "path/filepath" "testing" "github.com/databricks/cli/libs/aitools/mcp" @@ -128,6 +129,15 @@ func TestSignInServerResult(t *testing.T) { state: signInStateNotRequired, want: protos.AitoolsMcpServerResult{Service: protos.AitoolsMcpServiceGitHub, Outcome: protos.AitoolsTargetOutcomeUnchanged, SignInState: protos.AitoolsMcpSignInStateNotRequired}, }, + { + // A run that never learned the state, such as a read that failed, reports none. + name: "state not learned", + err: failure(protos.AitoolsErrorCategoryMCPServiceNotFound, nil, "not found"), + want: protos.AitoolsMcpServerResult{ + Service: protos.AitoolsMcpServiceGitHub, Outcome: protos.AitoolsTargetOutcomeFailed, + ErrorCategory: protos.AitoolsErrorCategoryMCPServiceNotFound, + }, + }, { name: "timed out", state: signInStateTimeout, @@ -193,13 +203,31 @@ func TestSignInPromptedEvent(t *testing.T) { } func TestBoundProfileCount(t *testing.T) { + dir := t.TempDir() + first := filepath.Join(dir, "first.cfg") + second := filepath.Join(dir, "second.cfg") state := &mcp.State{Servers: []mcp.Entry{ - {Profile: "logfood", Service: "system.ai.github"}, - {Profile: "logfood", Service: "system.ai.genie_one_mcp"}, - {Profile: "dogfood", Service: "system.ai.github"}, + {Profile: "logfood", Service: "system.ai.github", ConfigFile: first}, + {Profile: "logfood", Service: "system.ai.genie_one_mcp", ConfigFile: first}, + // DEFAULT in two config files is two profiles, as list groups them. + {Profile: "DEFAULT", Service: "system.ai.github", ConfigFile: first}, + {Profile: "DEFAULT", Service: "system.ai.github", ConfigFile: second}, }} - assert.Equal(t, int64(2), boundProfileCount(state)) - assert.Equal(t, int64(0), boundProfileCount(&mcp.State{})) + assert.Equal(t, int64(3), boundProfileCount(t.Context(), state)) + assert.Equal(t, int64(0), boundProfileCount(t.Context(), &mcp.State{})) +} + +func TestAddSignInEvent(t *testing.T) { + // An ACTIVE credential that the add found signed in is UNCHANGED, as add --login reports it. + assert.Equal(t, protos.AitoolsMcpEvent{ + CallingAgent: callingAgentType(), + Operation: protos.AitoolsMcpOperationSignIn, + Servers: []protos.AitoolsMcpServerResult{{ + Service: protos.AitoolsMcpServiceGitHub, + Outcome: protos.AitoolsTargetOutcomeUnchanged, + SignInState: protos.AitoolsMcpSignInStateActive, + }}, + }, addSignInEvent("system.ai.github", signInStateActive, true, nil)) } // The event carries enums and counts only. A custom service and the host of its profile must not From 0f3e651ca6733e90e03ddb806ab8f0b8dbf2be5f Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Fri, 13 Nov 2026 14:58:17 +0000 Subject: [PATCH 08/22] Derive the telemetry service enum from the known system.ai table mcpServiceType parsed the system.ai schema name itself and repeated the table's leaves. It now uses the shared schema constant and the known table, so an unknown system.ai leaf is OTHER_SYSTEM and a service outside system.ai is CUSTOM. A test checks that every known leaf has its own enum. Co-authored-by: Isaac --- cmd/aitools/mcp_telemetry.go | 8 ++++---- cmd/aitools/mcp_telemetry_test.go | 10 ++++++++++ libs/aitools/mcp/known.go | 11 ++++++++++- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/cmd/aitools/mcp_telemetry.go b/cmd/aitools/mcp_telemetry.go index e12c166ac39..c3c7b32ecca 100644 --- a/cmd/aitools/mcp_telemetry.go +++ b/cmd/aitools/mcp_telemetry.go @@ -71,13 +71,13 @@ var mcpServiceTypes = map[string]protos.AitoolsMcpService{ // services outside system.ai are CUSTOM. func mcpServiceType(fullName string) protos.AitoolsMcpService { parts := strings.Split(fullName, ".") - if len(parts) != 3 || !strings.EqualFold(parts[0]+"."+parts[1], "system.ai") { + if len(parts) != 3 || !strings.EqualFold(parts[0]+"."+parts[1], mcp.SystemAILocation) { return protos.AitoolsMcpServiceCustom } - if t, ok := mcpServiceTypes[strings.ToLower(parts[2])]; ok { - return t + if mcp.KnownOf(fullName) == nil { + return protos.AitoolsMcpServiceOtherSystem } - return protos.AitoolsMcpServiceOtherSystem + return mcpServiceTypes[strings.ToLower(parts[2])] } // callingAgentType returns the agent that drives the CLI, Unspecified when none is detected. diff --git a/cmd/aitools/mcp_telemetry_test.go b/cmd/aitools/mcp_telemetry_test.go index 046f7083ea4..df0d31c894e 100644 --- a/cmd/aitools/mcp_telemetry_test.go +++ b/cmd/aitools/mcp_telemetry_test.go @@ -40,6 +40,16 @@ func TestMcpServiceType(t *testing.T) { } } +// Every service in the CLI's known table has its own enum, so a known service never reports OTHER_SYSTEM. A new table +// entry needs a new value in mcpServiceTypes. +func TestMcpServiceTypeCoversKnownServices(t *testing.T) { + for _, leaf := range mcp.KnownLeaves() { + assert.Contains(t, mcpServiceTypes, leaf, "system.ai.%s has no telemetry enum: add it to mcpServiceTypes", leaf) + assert.NotEqualf(t, protos.AitoolsMcpServiceOtherSystem, mcpServiceType(mcp.SystemAILocation+"."+leaf), + "system.ai.%s has no telemetry enum", leaf) + } +} + func TestAddServerResults(t *testing.T) { servers := []addServerOutput{ { diff --git a/libs/aitools/mcp/known.go b/libs/aitools/mcp/known.go index 17883fe1a0b..01c944fb90c 100644 --- a/libs/aitools/mcp/known.go +++ b/libs/aitools/mcp/known.go @@ -1,6 +1,10 @@ package mcp -import "strings" +import ( + "maps" + "slices" + "strings" +) // SystemAILocation is the schema that ships the Databricks managed MCP services. const SystemAILocation = "system.ai" @@ -61,6 +65,11 @@ var knownSystemAI = map[string]KnownService{ }, } +// KnownLeaves returns the leaf names of the system.ai services in the CLI's table, sorted. +func KnownLeaves() []string { + return slices.Sorted(maps.Keys(knownSystemAI)) +} + // KnownOf returns the CLI's description of a service, or nil when it is not a system.ai service in the table. The leaf // is compared without case, as Unity Catalog names are. func KnownOf(fullName string) *KnownService { From 61fdc2ca5b60c0ed7a07ef3a4b29f9c4e80bdc6d Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Fri, 13 Nov 2026 15:00:18 +0000 Subject: [PATCH 09/22] Tidy the telemetry acceptance scripts and the fake telemetry endpoint The four telemetry scripts share one mcp_events helper that filters the uploads with print_requests.py. Their API posts carry MSYS_NO_PATHCONV so Windows keeps the path, the required failures use musterr, the invalid Codex config is a committed input, and the CLI calls are traced. The fake /telemetry-ext reports an unreadable batch as an error, as its neighbors do. Co-authored-by: Isaac --- .../telemetry/mcp-add/codex-config.toml | 1 + .../cmd/aitools/telemetry/mcp-add/output.txt | 6 ++++- .../cmd/aitools/telemetry/mcp-add/script | 20 +++++----------- .../aitools/telemetry/mcp-remove/output.txt | 10 ++++++++ .../cmd/aitools/telemetry/mcp-remove/script | 20 +++++----------- .../telemetry/mcp-serve-rejected/script | 2 +- .../aitools/telemetry/mcp-serve/output.txt | 3 +++ .../cmd/aitools/telemetry/mcp-serve/script | 16 ++++--------- .../aitools/telemetry/mcp-sign-in/output.txt | 8 +++++++ .../cmd/aitools/telemetry/mcp-sign-in/script | 24 +++++++------------ .../cmd/aitools/telemetry/script.prepare | 8 +++++++ 11 files changed, 60 insertions(+), 58 deletions(-) create mode 100644 acceptance/cmd/aitools/telemetry/mcp-add/codex-config.toml diff --git a/acceptance/cmd/aitools/telemetry/mcp-add/codex-config.toml b/acceptance/cmd/aitools/telemetry/mcp-add/codex-config.toml new file mode 100644 index 00000000000..8758ff120ff --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-add/codex-config.toml @@ -0,0 +1 @@ +not = [valid diff --git a/acceptance/cmd/aitools/telemetry/mcp-add/output.txt b/acceptance/cmd/aitools/telemetry/mcp-add/output.txt index b7ea889e8b8..132249dbdea 100644 --- a/acceptance/cmd/aitools/telemetry/mcp-add/output.txt +++ b/acceptance/cmd/aitools/telemetry/mcp-add/output.txt @@ -1,5 +1,7 @@ === A new install for two agents: CHANGED per (service, agent); github needs sign-in, so its sign-in state is PENDING + +>>> [CLI] aitools mcp add --profile logfood --names system.ai.genie_one_mcp,system.ai.github --agents claude-code,codex --output json { "calling_agent": "TYPE_UNSPECIFIED", "operation": "ADD", @@ -33,6 +35,8 @@ } === The same servers again are re-asserted: UNCHANGED + +>>> [CLI] aitools mcp add --profile logfood --names system.ai.genie_one_mcp,system.ai.github --agents claude-code,codex --output json { "calling_agent": "TYPE_UNSPECIFIED", "operation": "ADD", @@ -67,7 +71,7 @@ === An agent config that does not parse fails that agent only: FAILED with AGENT_CONFIG_INVALID -Exit code: 1 +>>> musterr [CLI] aitools mcp add --profile logfood --names system.ai.genie_one_mcp --agents claude-code,codex --output json { "calling_agent": "TYPE_UNSPECIFIED", "operation": "ADD", diff --git a/acceptance/cmd/aitools/telemetry/mcp-add/script b/acceptance/cmd/aitools/telemetry/mcp-add/script index 2280d098810..6cf6f91e92a 100644 --- a/acceptance/cmd/aitools/telemetry/mcp-add/script +++ b/acceptance/cmd/aitools/telemetry/mcp-add/script @@ -3,28 +3,20 @@ mkdir -p home/.claude home/.codex envsubst < .databrickscfg > out && mv out .databrickscfg export DATABRICKS_CONFIG_FILE=.databrickscfg -# Prints the aitools_mcp_event payloads uploaded since the last call, then forgets the recorded uploads. -mcp_events() { - if [ -f "$OUT_REQUESTS" ]; then - jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.aitools_mcp_event | select(. != null)' "$OUT_REQUESTS" - rm -f "$OUT_REQUESTS" - fi -} - # Seeding the fake is not an MCP command, so its uploads (if any) are dropped here. -$CLI api post "/api/2.1/unity-catalog/mcp-services?parent=schemas/system.ai&mcp_service_id=genie_one_mcp" --profile logfood --json '{"comment":"Ask questions about your data"}' > /dev/null -$CLI api post "/api/2.1/unity-catalog/mcp-services?parent=schemas/system.ai&mcp_service_id=github" --profile logfood --json '{"comment":"Repositories, issues, pull requests","config":{"source_connection":{"name":"connections/github","options":{"oauth_provider":"github"}}}}' > /dev/null +MSYS_NO_PATHCONV=1 $CLI api post "/api/2.1/unity-catalog/mcp-services?parent=schemas/system.ai&mcp_service_id=genie_one_mcp" --profile logfood --json '{"comment":"Ask questions about your data"}' > /dev/null +MSYS_NO_PATHCONV=1 $CLI api post "/api/2.1/unity-catalog/mcp-services?parent=schemas/system.ai&mcp_service_id=github" --profile logfood --json '{"comment":"Repositories, issues, pull requests","config":{"source_connection":{"name":"connections/github","options":{"oauth_provider":"github"}}}}' > /dev/null rm -f "$OUT_REQUESTS" title "A new install for two agents: CHANGED per (service, agent); github needs sign-in, so its sign-in state is PENDING" -$CLI aitools mcp add --profile logfood --names system.ai.genie_one_mcp,system.ai.github --agents claude-code,codex --output json > /dev/null +trace $CLI aitools mcp add --profile logfood --names system.ai.genie_one_mcp,system.ai.github --agents claude-code,codex --output json > /dev/null mcp_events title "The same servers again are re-asserted: UNCHANGED" -$CLI aitools mcp add --profile logfood --names system.ai.genie_one_mcp,system.ai.github --agents claude-code,codex --output json > /dev/null +trace $CLI aitools mcp add --profile logfood --names system.ai.genie_one_mcp,system.ai.github --agents claude-code,codex --output json > /dev/null mcp_events title "An agent config that does not parse fails that agent only: FAILED with AGENT_CONFIG_INVALID" -printf 'not = [valid\n' > home/.codex/config.toml -errcode $CLI aitools mcp add --profile logfood --names system.ai.genie_one_mcp --agents claude-code,codex --output json > /dev/null +cp codex-config.toml home/.codex/config.toml +trace musterr $CLI aitools mcp add --profile logfood --names system.ai.genie_one_mcp --agents claude-code,codex --output json > /dev/null mcp_events diff --git a/acceptance/cmd/aitools/telemetry/mcp-remove/output.txt b/acceptance/cmd/aitools/telemetry/mcp-remove/output.txt index d470471158e..4f85fef980b 100644 --- a/acceptance/cmd/aitools/telemetry/mcp-remove/output.txt +++ b/acceptance/cmd/aitools/telemetry/mcp-remove/output.txt @@ -1,5 +1,7 @@ === Installed for two profiles: the bound profile count is 2 after the second add + +>>> [CLI] aitools mcp add --profile logfood --names system.ai.genie_one_mcp --agents claude-code --output json { "calling_agent": "TYPE_UNSPECIFIED", "operation": "ADD", @@ -13,6 +15,8 @@ ], "bound_profile_count": 1 } + +>>> [CLI] aitools mcp add --profile dogfood --names system.ai.genie_one_mcp --agents claude-code --output json { "calling_agent": "TYPE_UNSPECIFIED", "operation": "ADD", @@ -28,6 +32,8 @@ } === remove --profile dogfood: CHANGED for its agent entry, and one profile stays bound + +>>> [CLI] aitools mcp remove --profile dogfood --output json { "calling_agent": "TYPE_UNSPECIFIED", "operation": "REMOVE", @@ -42,6 +48,8 @@ } === An entry changed outside this CLI is left in place: SKIPPED + +>>> [CLI] aitools mcp remove --names genie_one_mcp-logfood --profile logfood --output json { "calling_agent": "TYPE_UNSPECIFIED", "operation": "REMOVE", @@ -56,6 +64,8 @@ } === An entry already gone from the agent config is SKIPPED, and the state record goes + +>>> [CLI] aitools mcp remove --names genie_one_mcp-logfood --profile logfood --output json { "calling_agent": "TYPE_UNSPECIFIED", "operation": "REMOVE", diff --git a/acceptance/cmd/aitools/telemetry/mcp-remove/script b/acceptance/cmd/aitools/telemetry/mcp-remove/script index fe054c38ce8..ad266636d8a 100644 --- a/acceptance/cmd/aitools/telemetry/mcp-remove/script +++ b/acceptance/cmd/aitools/telemetry/mcp-remove/script @@ -3,35 +3,27 @@ mkdir -p home/.claude envsubst < .databrickscfg > out && mv out .databrickscfg export DATABRICKS_CONFIG_FILE=.databrickscfg -# Prints the aitools_mcp_event payloads uploaded since the last call, then forgets the recorded uploads. -mcp_events() { - if [ -f "$OUT_REQUESTS" ]; then - jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.aitools_mcp_event | select(. != null)' "$OUT_REQUESTS" - rm -f "$OUT_REQUESTS" - fi -} - -$CLI api post "/api/2.1/unity-catalog/mcp-services?parent=schemas/system.ai&mcp_service_id=genie_one_mcp" --profile logfood --json '{"comment":"Ask questions about your data"}' > /dev/null +MSYS_NO_PATHCONV=1 $CLI api post "/api/2.1/unity-catalog/mcp-services?parent=schemas/system.ai&mcp_service_id=genie_one_mcp" --profile logfood --json '{"comment":"Ask questions about your data"}' > /dev/null rm -f "$OUT_REQUESTS" title "Installed for two profiles: the bound profile count is 2 after the second add" -$CLI aitools mcp add --profile logfood --names system.ai.genie_one_mcp --agents claude-code --output json > /dev/null +trace $CLI aitools mcp add --profile logfood --names system.ai.genie_one_mcp --agents claude-code --output json > /dev/null mcp_events -$CLI aitools mcp add --profile dogfood --names system.ai.genie_one_mcp --agents claude-code --output json > /dev/null +trace $CLI aitools mcp add --profile dogfood --names system.ai.genie_one_mcp --agents claude-code --output json > /dev/null mcp_events title "remove --profile dogfood: CHANGED for its agent entry, and one profile stays bound" -$CLI aitools mcp remove --profile dogfood --output json > /dev/null +trace $CLI aitools mcp remove --profile dogfood --output json > /dev/null mcp_events title "An entry changed outside this CLI is left in place: SKIPPED" jq '.mcpServers["genie_one_mcp-logfood"].command = "/usr/bin/other"' home/.claude.json > home/claude.tmp && mv home/claude.tmp home/.claude.json -$CLI aitools mcp remove --names genie_one_mcp-logfood --profile logfood --output json > /dev/null +trace $CLI aitools mcp remove --names genie_one_mcp-logfood --profile logfood --output json > /dev/null mcp_events title "An entry already gone from the agent config is SKIPPED, and the state record goes" jq 'del(.mcpServers["genie_one_mcp-logfood"])' home/.claude.json > home/claude.tmp && mv home/claude.tmp home/.claude.json -$CLI aitools mcp remove --names genie_one_mcp-logfood --profile logfood --output json > /dev/null +trace $CLI aitools mcp remove --names genie_one_mcp-logfood --profile logfood --output json > /dev/null mcp_events title "A record with no agent left goes with a REMOVE of its own: CHANGED with no agent, and no profile bound" diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/script b/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/script index 0de703dfbc0..25fb6292ef9 100644 --- a/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/script +++ b/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/script @@ -5,4 +5,4 @@ export BROWSER=none title "serve whose startup check is rejected: exit 2, and PROXY_STARTED is uploaded with AUTH_FAILED" trace musterr $CLI aitools mcp serve --profile logfood --service system.ai.genie_one_mcp --agent claude-code < /dev/null > /dev/null -print_requests.py //telemetry-ext | jq '.body.protoLogs[] | fromjson | .entry.databricks_cli_log.aitools_mcp_event | select(. != null)' +mcp_events diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve/output.txt b/acceptance/cmd/aitools/telemetry/mcp-serve/output.txt index 5ce1d986506..1bcd45d41b1 100644 --- a/acceptance/cmd/aitools/telemetry/mcp-serve/output.txt +++ b/acceptance/cmd/aitools/telemetry/mcp-serve/output.txt @@ -1,5 +1,8 @@ === serve logs PROXY_STARTED after the binding checks; the upload is sent when stdin closes + +>>> [CLI] aitools mcp serve --profile logfood --service system.ai.genie_one_mcp --agent claude-code +No install record for system.ai.genie_one_mcp on profile "logfood"; using its current host [DATABRICKS_URL] { "calling_agent": "CLAUDE_CODE", "operation": "PROXY_STARTED", diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve/script b/acceptance/cmd/aitools/telemetry/mcp-serve/script index 5b30a8e7ddc..2e865c20274 100644 --- a/acceptance/cmd/aitools/telemetry/mcp-serve/script +++ b/acceptance/cmd/aitools/telemetry/mcp-serve/script @@ -5,21 +5,13 @@ export DATABRICKS_CONFIG_FILE=.databrickscfg # No browser: the proxy prints the sign-in URL and does not open anything. export BROWSER=none export RETRY_MAX_ATTEMPTS=60 -# Prints the aitools_mcp_event payloads uploaded since the last call, then forgets the recorded uploads. -mcp_events() { - if [ -f "$OUT_REQUESTS" ]; then - jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.aitools_mcp_event | select(. != null)' "$OUT_REQUESTS" - rm -f "$OUT_REQUESTS" - fi -} - # Seeding the fake is not an MCP command, so its uploads (if any) are dropped here. -$CLI api post "/api/2.1/unity-catalog/mcp-services?parent=schemas/system.ai&mcp_service_id=genie_one_mcp" --profile logfood --json '{"comment":"Ask questions about your data"}' > /dev/null -$CLI api post "/api/2.1/unity-catalog/mcp-services?parent=schemas/system.ai&mcp_service_id=github" --profile logfood --json '{"comment":"Repositories, issues, pull requests","config":{"source_connection":{"name":"connections/github","options":{"oauth_provider":"github"}}}}' > /dev/null +MSYS_NO_PATHCONV=1 $CLI api post "/api/2.1/unity-catalog/mcp-services?parent=schemas/system.ai&mcp_service_id=genie_one_mcp" --profile logfood --json '{"comment":"Ask questions about your data"}' > /dev/null +MSYS_NO_PATHCONV=1 $CLI api post "/api/2.1/unity-catalog/mcp-services?parent=schemas/system.ai&mcp_service_id=github" --profile logfood --json '{"comment":"Repositories, issues, pull requests","config":{"source_connection":{"name":"connections/github","options":{"oauth_provider":"github"}}}}' > /dev/null rm -f "$OUT_REQUESTS" title "serve logs PROXY_STARTED after the binding checks; the upload is sent when stdin closes" -$CLI aitools mcp serve --profile logfood --service system.ai.genie_one_mcp --agent claude-code < requests-list.ndjson > /dev/null 2> /dev/null +trace $CLI aitools mcp serve --profile logfood --service system.ai.genie_one_mcp --agent claude-code < requests-list.ndjson > /dev/null mcp_events title "A binding that fails logs PROXY_STARTED with its category, but no upload follows: there is no config to upload with" @@ -32,7 +24,7 @@ title "authenticate opens the sign-in page: PROXY_STARTED, then SIGN_IN_PROMPTED retry --until '"id":2' cat serve.stdout.txt > /dev/null cat requests-authenticate.ndjson retry --until "Waiting for sign-in" cat serve.stderr.txt > /dev/null - $CLI api post /api/2.1/unity-catalog/mcp-services/system.ai.github/user-credentials --profile logfood --json '{}' > /dev/null + MSYS_NO_PATHCONV=1 $CLI api post /api/2.1/unity-catalog/mcp-services/system.ai.github/user-credentials --profile logfood --json '{}' > /dev/null retry --until '"id":3' cat serve.stdout.txt > /dev/null cat requests-after.ndjson } | $CLI aitools mcp serve --profile logfood --service system.ai.github --agent claude-code > serve.stdout.txt 2> serve.stderr.txt diff --git a/acceptance/cmd/aitools/telemetry/mcp-sign-in/output.txt b/acceptance/cmd/aitools/telemetry/mcp-sign-in/output.txt index d2b56d441f2..46253de58e8 100644 --- a/acceptance/cmd/aitools/telemetry/mcp-sign-in/output.txt +++ b/acceptance/cmd/aitools/telemetry/mcp-sign-in/output.txt @@ -1,5 +1,7 @@ === login to a service that needs no sign-in: UNCHANGED, sign-in state NOT_REQUIRED + +>>> [CLI] aitools mcp login system.ai.genie_one_mcp --profile logfood system.ai.genie_one_mcp does not need sign-in. { "calling_agent": "TYPE_UNSPECIFIED", @@ -29,6 +31,8 @@ system.ai.genie_one_mcp does not need sign-in. } === login while already signed in: UNCHANGED + +>>> [CLI] aitools mcp login system.ai.github --profile logfood Already signed in to system.ai.github on profile logfood. Public repositories only until a repository owner installs https://github.com/apps/databricks-github-connector. { @@ -45,6 +49,8 @@ Public repositories only until a repository owner installs https://github.com/ap } === logout removes the sign-in: CHANGED; a second logout finds nothing: UNCHANGED + +>>> [CLI] aitools mcp logout system.ai.github --profile logfood Signed out of system.ai.github on profile logfood. { "calling_agent": "TYPE_UNSPECIFIED", @@ -57,6 +63,8 @@ Signed out of system.ai.github on profile logfood. ], "bound_profile_count": 0 } + +>>> [CLI] aitools mcp logout system.ai.github --profile logfood Not signed in to system.ai.github on profile logfood. { "calling_agent": "TYPE_UNSPECIFIED", diff --git a/acceptance/cmd/aitools/telemetry/mcp-sign-in/script b/acceptance/cmd/aitools/telemetry/mcp-sign-in/script index 72025e3cde6..6ddd1bce499 100644 --- a/acceptance/cmd/aitools/telemetry/mcp-sign-in/script +++ b/acceptance/cmd/aitools/telemetry/mcp-sign-in/script @@ -6,47 +6,39 @@ export DATABRICKS_CONFIG_FILE=.databrickscfg # No browser: the CLI prints the sign-in URL and polls. export BROWSER=none export RETRY_MAX_ATTEMPTS=60 -# Prints the aitools_mcp_event payloads uploaded since the last call, then forgets the recorded uploads. -mcp_events() { - if [ -f "$OUT_REQUESTS" ]; then - jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.aitools_mcp_event | select(. != null)' "$OUT_REQUESTS" - rm -f "$OUT_REQUESTS" - fi -} - # Seeding the fake is not an MCP command, so its uploads (if any) are dropped here. -$CLI api post "/api/2.1/unity-catalog/mcp-services?parent=schemas/system.ai&mcp_service_id=genie_one_mcp" --profile logfood --json '{"comment":"Ask questions about your data"}' > /dev/null -$CLI api post "/api/2.1/unity-catalog/mcp-services?parent=schemas/system.ai&mcp_service_id=github" --profile logfood --json '{"comment":"Repositories, issues, pull requests","config":{"source_connection":{"name":"connections/github","options":{"oauth_provider":"github"}}}}' > /dev/null +MSYS_NO_PATHCONV=1 $CLI api post "/api/2.1/unity-catalog/mcp-services?parent=schemas/system.ai&mcp_service_id=genie_one_mcp" --profile logfood --json '{"comment":"Ask questions about your data"}' > /dev/null +MSYS_NO_PATHCONV=1 $CLI api post "/api/2.1/unity-catalog/mcp-services?parent=schemas/system.ai&mcp_service_id=github" --profile logfood --json '{"comment":"Repositories, issues, pull requests","config":{"source_connection":{"name":"connections/github","options":{"oauth_provider":"github"}}}}' > /dev/null rm -f "$OUT_REQUESTS" title "login to a service that needs no sign-in: UNCHANGED, sign-in state NOT_REQUIRED" -$CLI aitools mcp login system.ai.genie_one_mcp --profile logfood > /dev/null +trace $CLI aitools mcp login system.ai.genie_one_mcp --profile logfood > /dev/null mcp_events title "login completes once the credential is ACTIVE: CHANGED, sign-in state ACTIVE" $CLI aitools mcp login system.ai.github --profile logfood > login.txt 2>&1 & PID=$! retry --until "Waiting for sign-in" cat login.txt > /dev/null -$CLI api post /api/2.1/unity-catalog/mcp-services/system.ai.github/user-credentials --profile logfood --json '{}' > /dev/null +MSYS_NO_PATHCONV=1 $CLI api post /api/2.1/unity-catalog/mcp-services/system.ai.github/user-credentials --profile logfood --json '{}' > /dev/null wait $PID mcp_events rm login.txt title "login while already signed in: UNCHANGED" -$CLI aitools mcp login system.ai.github --profile logfood > /dev/null +trace $CLI aitools mcp login system.ai.github --profile logfood > /dev/null mcp_events title "logout removes the sign-in: CHANGED; a second logout finds nothing: UNCHANGED" -$CLI aitools mcp logout system.ai.github --profile logfood > /dev/null +trace $CLI aitools mcp logout system.ai.github --profile logfood > /dev/null mcp_events -$CLI aitools mcp logout system.ai.github --profile logfood > /dev/null +trace $CLI aitools mcp logout system.ai.github --profile logfood > /dev/null mcp_events title "add --login signs in during the same run: SIGN_IN first, then ADD with sign-in state ACTIVE" $CLI aitools mcp add --profile logfood --names system.ai.github --agents claude-code --login > add.txt 2>&1 & PID=$! retry --until "Waiting for sign-in" cat add.txt > /dev/null -$CLI api post /api/2.1/unity-catalog/mcp-services/system.ai.github/user-credentials --profile logfood --json '{}' > /dev/null +MSYS_NO_PATHCONV=1 $CLI api post /api/2.1/unity-catalog/mcp-services/system.ai.github/user-credentials --profile logfood --json '{}' > /dev/null wait $PID mcp_events rm add.txt diff --git a/acceptance/cmd/aitools/telemetry/script.prepare b/acceptance/cmd/aitools/telemetry/script.prepare index f55e1cef710..6a9112e4947 100644 --- a/acceptance/cmd/aitools/telemetry/script.prepare +++ b/acceptance/cmd/aitools/telemetry/script.prepare @@ -15,3 +15,11 @@ print_telemetry() { + (del(.execution_context))' fi } + +# Prints the aitools_mcp_event payloads uploaded since the last call, then forgets the recorded uploads. The uploads are +# filtered by path with print_requests.py; jq reads only the body. +mcp_events() { + if [ -f "$OUT_REQUESTS" ]; then + print_requests.py //telemetry-ext | jq '.body.protoLogs[] | fromjson | .entry.databricks_cli_log.aitools_mcp_event | select(. != null)' + fi +} From 48880f0e488ac092785071cf14efd4ffb1d8ec68 Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Fri, 13 Nov 2026 15:01:31 +0000 Subject: [PATCH 10/22] Document the error categories of the server result and tidy its test The server result's doc now names the categories a proxy start and a SIGN_IN_PROMPTED carry. The user-content test uses slices and maps from the standard library, and its comment no longer claims a host check the test does not make. Co-authored-by: Isaac --- cmd/aitools/mcp_telemetry_test.go | 15 ++++----------- libs/telemetry/protos/aitools_mcp.go | 5 +++-- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/cmd/aitools/mcp_telemetry_test.go b/cmd/aitools/mcp_telemetry_test.go index df0d31c894e..b78201e3e29 100644 --- a/cmd/aitools/mcp_telemetry_test.go +++ b/cmd/aitools/mcp_telemetry_test.go @@ -3,7 +3,9 @@ package aitools import ( "encoding/json" "errors" + "maps" "path/filepath" + "slices" "testing" "github.com/databricks/cli/libs/aitools/mcp" @@ -240,8 +242,7 @@ func TestAddSignInEvent(t *testing.T) { }, addSignInEvent("system.ai.github", signInStateActive, true, nil)) } -// The event carries enums and counts only. A custom service and the host of its profile must not -// appear anywhere in the payload. +// The event carries enums and counts only. A custom service's name must not appear anywhere in the payload. func TestMcpEventOmitsUserContent(t *testing.T) { event := protos.AitoolsMcpEvent{ CallingAgent: protos.AitoolsAgentTypeClaudeCode, @@ -257,17 +258,9 @@ func TestMcpEventOmitsUserContent(t *testing.T) { var got map[string]any require.NoError(t, json.Unmarshal(b, &got)) - assert.ElementsMatch(t, []string{"calling_agent", "operation", "servers", "bound_profile_count"}, keys(got)) + assert.ElementsMatch(t, []string{"calling_agent", "operation", "servers", "bound_profile_count"}, slices.Collect(maps.Keys(got))) assert.NotContains(t, string(b), "acme_catalog") assert.NotContains(t, string(b), "secret_schema") assert.NotContains(t, string(b), "internal_tool") assert.Contains(t, string(b), `"service":"CUSTOM"`) } - -func keys(m map[string]any) []string { - out := make([]string, 0, len(m)) - for k := range m { - out = append(out, k) - } - return out -} diff --git a/libs/telemetry/protos/aitools_mcp.go b/libs/telemetry/protos/aitools_mcp.go index c42dbc20285..f6a8d2f677b 100644 --- a/libs/telemetry/protos/aitools_mcp.go +++ b/libs/telemetry/protos/aitools_mcp.go @@ -63,8 +63,9 @@ const ( // AitoolsMcpServerResult is the result for one service, for one agent when the // operation writes agent config. Unset fields are omitted: an agent is not set -// for sign-in, an outcome is not set for proxy events, and an error category is -// set only when the target failed or was skipped. +// for sign-in, and an outcome is not set for proxy events. An error category is +// set when the target failed or was skipped, when a proxy start failed its +// checks (PROXY_STARTED), and on SIGN_IN_PROMPTED, where it is SIGN_IN_REQUIRED. type AitoolsMcpServerResult struct { Service AitoolsMcpService `json:"service"` Agent AitoolsAgentType `json:"agent,omitempty"` From b50fd132f91e4a195bae821728089b3766d85a23 Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Fri, 13 Nov 2026 15:33:25 +0000 Subject: [PATCH 11/22] Authenticate telemetry uploads without refreshing a token The config of a telemetry upload that the command does not otherwise use never refreshes its token: the command may exit before a refresh finishes, and a rotated single-use refresh token would be lost. serve's session keeps its refreshing config, and its upload gets a config of its own. Co-authored-by: Isaac --- cmd/aitools/mcp_serve.go | 5 +++-- cmd/aitools/mcp_telemetry.go | 9 +++------ libs/aitools/mcp/binding.go | 19 +++++++++++++++++-- libs/aitools/mcp/binding_test.go | 15 +++++++++++++++ 4 files changed, 38 insertions(+), 10 deletions(-) diff --git a/cmd/aitools/mcp_serve.go b/cmd/aitools/mcp_serve.go index 876212ffffc..bf1ce9bbc03 100644 --- a/cmd/aitools/mcp_serve.go +++ b/cmd/aitools/mcp_serve.go @@ -125,8 +125,9 @@ func runMcpServe(cmd *cobra.Command, f mcpServeFlags) error { return serveFailure(cmd, err, serveBindStatus) } - // The bound profile authenticates the exit-time telemetry upload. - cmd.SetContext(cmdctx.SetConfigUsed(cmd.Context(), cfg)) + // The bound profile authenticates the exit-time telemetry upload. The session's own config refreshes its token, which + // the upload must not do, so the upload gets a config of its own. + setUploadConfig(cmd, profileName, cfg.ConfigFile) // A server pinned to a config file names that file in its hints, as add and login do. pinnedFile := mcp.PinnedFile(entry, cfg.ConfigFile) diff --git a/cmd/aitools/mcp_telemetry.go b/cmd/aitools/mcp_telemetry.go index c3c7b32ecca..0f609ba3a42 100644 --- a/cmd/aitools/mcp_telemetry.go +++ b/cmd/aitools/mcp_telemetry.go @@ -3,7 +3,6 @@ package aitools import ( "context" "strings" - "time" "github.com/databricks/cli/libs/aitools/agents" "github.com/databricks/cli/libs/aitools/mcp" @@ -14,9 +13,6 @@ import ( "github.com/spf13/cobra" ) -// uploadAuthBudget bounds the resolution of the profile that authenticates a telemetry upload. -const uploadAuthBudget = 3 * time.Second - // configureUploadAuth sets the config that authenticates the exit-time upload of a command that edits local files // only. It runs after the edit, so an offline run does not wait on the network before it edits a file, and it does // nothing when telemetry is disabled. The profile is the one the command would use without a flag; without one @@ -40,10 +36,11 @@ func configureUploadAuth(cmd *cobra.Command) { } // setUploadConfig sets the config of profileName in configFile for the exit-time upload, resolved within -// uploadAuthBudget. A profile that does not resolve uploads nothing. +// telemetryAuthBudget, with credentials that never refresh (mcp.BoundUploadConfig). A profile that does not resolve +// uploads nothing. func setUploadConfig(cmd *cobra.Command, profileName, configFile string) { ctx := cmd.Context() - cfg, err := mcp.BoundConfigWithin(profileName, configFile, uploadAuthBudget) + cfg, err := mcp.BoundUploadConfig(profileName, configFile, telemetryAuthBudget) if err != nil { log.Debugf(ctx, "telemetry: cannot resolve profile %s to upload with: %s", profileName, err) return diff --git a/libs/aitools/mcp/binding.go b/libs/aitools/mcp/binding.go index 297d820fef5..6d4f49c9f2d 100644 --- a/libs/aitools/mcp/binding.go +++ b/libs/aitools/mcp/binding.go @@ -11,6 +11,7 @@ import ( "time" "github.com/databricks/cli/libs/aitools/mcpconfig" + "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/databrickscfg" "github.com/databricks/cli/libs/databrickscfg/profile" "github.com/databricks/cli/libs/env" @@ -115,14 +116,28 @@ func BoundConfig(profileName, configFile string) (*config.Config, error) { // profile runs on a context of its own, so a context deadline does not reach it, and these budgets are what bound it (see // profileMetadata.Load in cmd/auth). func BoundConfigWithin(profileName, configFile string, budget time.Duration) (*config.Config, error) { + return resolveBound(withinBudget(profileName, configFile, budget)) +} + +// BoundUploadConfig is BoundConfigWithin for a config that only a telemetry upload uses. Its credentials never refresh a +// token, since the command may exit before a refresh finishes, and a rotated single-use refresh token would be lost (see +// auth.CredentialsWithoutRefresh). It does not authenticate. +func BoundUploadConfig(profileName, configFile string, budget time.Duration) (*config.Config, error) { + cfg := withinBudget(profileName, configFile, budget) + cfg.Credentials = auth.CredentialsWithoutRefresh() + return resolveBound(cfg) +} + +// withinBudget is the config of profileName in configFile, with the SDK's request and retry budgets set to budget. +func withinBudget(profileName, configFile string, budget time.Duration) *config.Config { seconds := int(budget / time.Second) - return resolveBound(&config.Config{ + return &config.Config{ Profile: profileName, ConfigFile: configFile, Loaders: []config.Loader{databrickscfg.ResolveNonAuthFromEnv, config.ConfigFile}, HTTPTimeoutSeconds: seconds, RetryTimeoutSeconds: seconds, - }) + } } // resolveBound resolves cfg, as BoundConfig does. diff --git a/libs/aitools/mcp/binding_test.go b/libs/aitools/mcp/binding_test.go index 372c141cae3..d9642d04716 100644 --- a/libs/aitools/mcp/binding_test.go +++ b/libs/aitools/mcp/binding_test.go @@ -10,8 +10,10 @@ import ( "os" "path/filepath" "testing" + "time" "github.com/databricks/cli/libs/aitools/mcp" + "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/shellquote" "github.com/stretchr/testify/assert" @@ -935,3 +937,16 @@ func TestConfigFileCommandUsesForwardSlashes(t *testing.T) { assert.Equal(t, "DATABRICKS_CONFIG_FILE="+filepath.ToSlash(path)+" databricks auth login --profile logfood", mcp.ConfigFileCommand(path, "databricks auth login --profile logfood")) } + +// The config of a telemetry upload never refreshes a token; the config a serve session binds keeps the default chain. +func TestBoundUploadConfigNeverRefreshes(t *testing.T) { + cfgPath := bindingTestConfig(t, bindingTestProfileContents(bindingTestToken)) + + upload, err := mcp.BoundUploadConfig(bindingTestProfile, cfgPath, time.Second) + require.NoError(t, err) + assert.Equal(t, auth.CredentialsWithoutRefresh(), upload.Credentials) + + session, err := mcp.BoundConfigWithin(bindingTestProfile, cfgPath, time.Second) + require.NoError(t, err) + assert.NotEqual(t, auth.CredentialsWithoutRefresh(), session.Credentials) +} From b5918d030e1d29343cea4eecd0b1a347cd6cf7ce Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Fri, 13 Nov 2026 15:36:29 +0000 Subject: [PATCH 12/22] Resolve remove's telemetry auth only after it logs an event remove writes its output first, then resolves the auth of its upload, only when a server was touched. The upload is skipped when a server recorded for its profile no longer matches the profile's host, workspace or config file, so the token is not sent to a host the server was not set up against. The auth resolution is the one install uses, logged at debug level and bounded by the same budget. Co-authored-by: Isaac --- .../mcp-remove/record-only.state.json | 4 +- .../cmd/aitools/telemetry/mcp-remove/script | 2 +- cmd/aitools/mcp_remove.go | 76 ++++++++---- cmd/aitools/mcp_remove_test.go | 110 ++++++++++++++++++ cmd/aitools/mcp_telemetry.go | 38 +++--- 5 files changed, 184 insertions(+), 46 deletions(-) diff --git a/acceptance/cmd/aitools/telemetry/mcp-remove/record-only.state.json b/acceptance/cmd/aitools/telemetry/mcp-remove/record-only.state.json index e069ce0fcac..e40e76fff80 100644 --- a/acceptance/cmd/aitools/telemetry/mcp-remove/record-only.state.json +++ b/acceptance/cmd/aitools/telemetry/mcp-remove/record-only.state.json @@ -6,11 +6,11 @@ "name": "dbsql-logfood", "service": "system.ai.dbsql", "profile": "logfood", - "host": "https://adb-1.test", + "host": "$DATABRICKS_HOST", "workspace_id": "", "sign_in_required": false, "command": "", - "config_file": "", + "config_file": "$PWD/.databrickscfg", "agents": {}, "installed_at": "2026-01-01T00:00:00Z", "cli_version": "0.0.0" diff --git a/acceptance/cmd/aitools/telemetry/mcp-remove/script b/acceptance/cmd/aitools/telemetry/mcp-remove/script index ad266636d8a..ef98d35f619 100644 --- a/acceptance/cmd/aitools/telemetry/mcp-remove/script +++ b/acceptance/cmd/aitools/telemetry/mcp-remove/script @@ -28,6 +28,6 @@ mcp_events title "A record with no agent left goes with a REMOVE of its own: CHANGED with no agent, and no profile bound" mkdir -p home/.databricks/aitools/skills -cp record-only.state.json home/.databricks/aitools/skills/.state.json +envsubst < record-only.state.json > home/.databricks/aitools/skills/.state.json trace $CLI aitools mcp remove --names dbsql-logfood --profile logfood --output json > /dev/null mcp_events diff --git a/cmd/aitools/mcp_remove.go b/cmd/aitools/mcp_remove.go index a36530eb203..33347f72430 100644 --- a/cmd/aitools/mcp_remove.go +++ b/cmd/aitools/mcp_remove.go @@ -13,6 +13,8 @@ import ( "github.com/databricks/cli/libs/aitools/mcp" "github.com/databricks/cli/libs/aitools/mcp/agentconfig" "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/telemetry" "github.com/databricks/cli/libs/telemetry/protos" "github.com/spf13/cobra" ) @@ -35,6 +37,8 @@ type removeOutput struct { // results is one telemetry result per (service, agent) attempted. It is not in the JSON. results []protos.AitoolsMcpServerResult + // stateBefore is the state as it was read before the removal. It is not in the JSON. + stateBefore *mcp.State } type removeRemoved struct { @@ -100,28 +104,12 @@ Entries that run 'aitools mcp serve' without an install record are not chosen by out := &removeOutput{Removed: []removeRemoved{}, Skipped: []removeSkipped{}} runErr := runMcpRemove(ctx, cmd, f, jsonMode, out) - configureUploadAuth(cmd) - logMcpEvent(ctx, protos.AitoolsMcpEvent{ - CallingAgent: callingAgentType(), - Operation: protos.AitoolsMcpOperationRemove, - Servers: out.results, - }) - if !jsonMode { - return runErr + err := runErr + if jsonMode { + err = writeRemoveJSON(cmd, out, runErr) } - - // The document carries a top-level failure; per-agent failures are already in skipped. - if top := topLevelFailure(runErr); top != nil { - out.Error = top.Error() - out.ErrorCategory = errorCategory(top) - } - if err := renderJSON(cmd.OutOrStdout(), out); err != nil { - return errors.Join(runErr, err) - } - if runErr != nil { - return root.ErrAlreadyPrinted - } - return nil + logRemoveEvent(cmd, args, out) + return err }, } @@ -131,6 +119,51 @@ Entries that run 'aitools mcp serve' without an install record are not chosen by return cmd } +// writeRemoveJSON writes the JSON document of a run that ended with runErr and returns the command's error. +func writeRemoveJSON(cmd *cobra.Command, out *removeOutput, runErr error) error { + // The document carries a top-level failure; per-agent failures are already in skipped. + if top := topLevelFailure(runErr); top != nil { + out.Error = top.Error() + out.ErrorCategory = errorCategory(top) + } + if err := renderJSON(cmd.OutOrStdout(), out); err != nil { + return errors.Join(runErr, err) + } + if runErr != nil { + return root.ErrAlreadyPrinted + } + return nil +} + +// logRemoveEvent buffers the REMOVE event and sets the config its exit-time upload authenticates with. It runs after +// the output is written, and only when the run touched a server: a run that touched none has nothing to upload, so it +// resolves no auth. The auth resolves as install's does (startTelemetryAuth), within telemetryAuthBudget, and the upload +// is skipped when a server recorded for the profile no longer matches it (bindingDrifted). +func logRemoveEvent(cmd *cobra.Command, args []string, out *removeOutput) { + ctx := cmd.Context() + logMcpEvent(ctx, protos.AitoolsMcpEvent{ + CallingAgent: callingAgentType(), + Operation: protos.AitoolsMcpOperationRemove, + Servers: out.results, + }) + if len(telemetry.BufferedEvents(ctx)) == 0 { + return + } + auth := startTelemetryAuth(cmd, args) + if auth == nil { + return + } + if !auth.wait() { + log.Debugf(ctx, "auth for telemetry not resolved within %s; not uploading telemetry", telemetryAuthBudget) + return + } + if auth.cfg != nil && bindingDrifted(ctx, out.stateBefore, auth.cfg) { + log.Debugf(ctx, "telemetry: profile %s no longer matches an MCP server recorded for it; not uploading telemetry", auth.cfg.Profile) + return + } + auth.apply(cmd, false) +} + // runMcpRemove removes the chosen servers from the agent configs and records the result. Results are printed // before a state failure is returned. An agent failure fails the command when --agents named the agent, or when a // selected server was removed from no agent; otherwise it is reported in skipped. @@ -157,6 +190,7 @@ func runMcpRemove(ctx context.Context, cmd *cobra.Command, f mcpRemoveFlags, jso if err != nil { return err } + out.stateBefore = state var selected []mcp.Entry if interactive && !f.all && len(names) == 0 && profileName == "" { // An agent with no installed server is refused as it is without a terminal, before the picker offers nothing for it. diff --git a/cmd/aitools/mcp_remove_test.go b/cmd/aitools/mcp_remove_test.go index 2f2c01c2c97..57d9442bdec 100644 --- a/cmd/aitools/mcp_remove_test.go +++ b/cmd/aitools/mcp_remove_test.go @@ -1,19 +1,30 @@ package aitools import ( + "bytes" "context" "encoding/json" "errors" + "log/slog" "maps" "os" "path/filepath" "slices" "testing" + "time" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/aitools/agents" "github.com/databricks/cli/libs/aitools/mcp" + "github.com/databricks/cli/libs/cmdctx" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/log/handler" + "github.com/databricks/cli/libs/telemetry" "github.com/databricks/cli/libs/telemetry/protos" + "github.com/databricks/databricks-sdk-go/config" + "github.com/spf13/cobra" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -789,3 +800,102 @@ func claudeServerEntry(claudePath, name, service string) mcp.Entry { Agents: map[string]mcp.AgentRecord{agents.NameClaudeCode: {Path: claudePath, Scope: "user"}}, } } + +// runRemoveWithTelemetry runs `mcp remove` with args on a context that buffers telemetry, and returns the remove command, +// so a test can read the context it left, and its stdout. +// The logs are those of the command's logger at logLevel. +func runRemoveWithTelemetry(t *testing.T, stdout, logs *bytes.Buffer, logLevel slog.Level, args ...string) (*cobra.Command, error) { + t.Helper() + parent := newMCPTestRoot() + ctx := env.Set(telemetry.WithNewLogger(t.Context()), "DATABRICKS_CLI_DISABLE_TELEMETRY", "") + ctx = log.NewContext(ctx, slog.New(handler.NewFriendlyHandler(logs, &handler.Options{Level: logLevel}))) + ctx, _ = cmdio.NewTestContextWithStderr(ctx) + parent.SetOut(stdout) + parent.SetErr(&bytes.Buffer{}) + parent.SetArgs(append([]string{"mcp", "remove"}, args...)) + err := parent.ExecuteContext(ctx) + remove, _, findErr := parent.Find([]string{"mcp", "remove"}) + require.NoError(t, findErr) + return remove, err +} + +// A run that removes nothing logs no event, so it resolves no auth for an upload. +func TestMcpRemoveResolvesNothingWithoutAnEvent(t *testing.T) { + withHome(t) + resolves := 0 + setTelemetryAuth(t, time.Second, func(*cobra.Command, []string) error { + resolves++ + return nil + }) + + var stdout, logs bytes.Buffer + _, err := runRemoveWithTelemetry(t, &stdout, &logs, log.LevelDebug, "--all", "--output", "json") + require.NoError(t, err) + assert.Zero(t, resolves) +} + +// The upload is authenticated with the profile only when no server recorded for it has drifted: its host here is the one +// the server was set up against. The removal is done either way, and the output is written before the auth resolves. +func TestMcpRemoveUploadSkipsDriftedProfile(t *testing.T) { + home := withHome(t) + writeHomeConfig(t, home, "[logfood]\nhost = https://adb-2.test\nauth_type = pat\ntoken = t\n") + claude := agents.ByName("claude-code") + claudePath := filepath.Join(home, ".claude.json") + // Recorded against another host than the profile names now. + entry := seedInstall(t, claude, "genie_one_mcp-logfood", "system.ai.genie_one_mcp", "logfood", "https://adb-1.test") + saveState(t, entry) + + var stdout, logs bytes.Buffer + var outputWrittenFirst bool + setTelemetryAuth(t, time.Second, func(cmd *cobra.Command, _ []string) error { + outputWrittenFirst = stdout.Len() > 0 + cmd.SetContext(cmdctx.SetConfigUsed(cmd.Context(), &config.Config{Profile: "logfood", Host: "https://adb-2.test", ConfigFile: filepath.Join(home, ".databrickscfg")})) + return nil + }) + + remove, err := runRemoveWithTelemetry(t, &stdout, &logs, log.LevelDebug, "--names", "genie_one_mcp-logfood", "--output", "json") + require.NoError(t, err) + assert.True(t, outputWrittenFirst, "the auth resolves after the JSON is written") + assert.False(t, cmdctx.HasConfigUsed(remove.Context()), "a drifted profile is not used to upload") + assert.NotContains(t, readJSONFile(t, claudePath)["mcpServers"], "genie_one_mcp-logfood", "the removal is done") +} + +// A profile whose recorded servers still match is used for the upload. +func TestMcpRemoveUploadUsesMatchingProfile(t *testing.T) { + home := withHome(t) + writeHomeConfig(t, home, "[logfood]\nhost = https://adb-1.test\nauth_type = pat\ntoken = t\n") + claude := agents.ByName("claude-code") + entry := seedInstall(t, claude, "genie_one_mcp-logfood", "system.ai.genie_one_mcp", "logfood", "https://adb-1.test") + saveState(t, entry) + + setTelemetryAuth(t, time.Second, func(cmd *cobra.Command, _ []string) error { + cmd.SetContext(cmdctx.SetConfigUsed(cmd.Context(), &config.Config{Profile: "logfood", Host: "https://adb-1.test", ConfigFile: filepath.Join(home, ".databrickscfg")})) + return nil + }) + + var stdout, logs bytes.Buffer + remove, err := runRemoveWithTelemetry(t, &stdout, &logs, log.LevelDebug, "--names", "genie_one_mcp-logfood", "--output", "json") + require.NoError(t, err) + require.True(t, cmdctx.HasConfigUsed(remove.Context())) + assert.Equal(t, "https://adb-1.test", cmdctx.ConfigUsed(remove.Context()).Host) +} + +// An auth resolution that fails, as it does offline, leaves nothing in the output: its logs are debug only. +func TestMcpRemoveOfflineResolutionWarnsNothing(t *testing.T) { + home := withHome(t) + writeHomeConfig(t, home, "[logfood]\nhost = https://adb-1.test\nauth_type = pat\ntoken = t\n") + claude := agents.ByName("claude-code") + saveState(t, seedInstall(t, claude, "genie_one_mcp-logfood", "system.ai.genie_one_mcp", "logfood", "https://adb-1.test")) + + setTelemetryAuth(t, time.Second, func(cmd *cobra.Command, _ []string) error { + log.Warnf(cmd.Context(), "[hostmetadata] failed to fetch host metadata for https://adb-1.test") + return errors.New("cannot reach https://adb-1.test") + }) + + var stdout, logs bytes.Buffer + remove, err := runRemoveWithTelemetry(t, &stdout, &logs, log.LevelWarn, "--names", "genie_one_mcp-logfood", "--output", "json") + require.NoError(t, err) + assert.False(t, cmdctx.HasConfigUsed(remove.Context())) + assert.Empty(t, logs.String()) + assert.Contains(t, stdout.String(), `"removed"`) +} diff --git a/cmd/aitools/mcp_telemetry.go b/cmd/aitools/mcp_telemetry.go index 0f609ba3a42..e3b623f8282 100644 --- a/cmd/aitools/mcp_telemetry.go +++ b/cmd/aitools/mcp_telemetry.go @@ -10,31 +10,10 @@ import ( "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/telemetry" "github.com/databricks/cli/libs/telemetry/protos" + "github.com/databricks/databricks-sdk-go/config" "github.com/spf13/cobra" ) -// configureUploadAuth sets the config that authenticates the exit-time upload of a command that edits local files -// only. It runs after the edit, so an offline run does not wait on the network before it edits a file, and it does -// nothing when telemetry is disabled. The profile is the one the command would use without a flag; without one -// the upload is skipped. -func configureUploadAuth(cmd *cobra.Command) { - ctx := cmd.Context() - if telemetry.Disabled(ctx) { - return - } - configFile, _, err := addConfigFile(ctx) - if err != nil { - log.Debugf(ctx, "telemetry: no config file to upload with: %s", err) - return - } - profileName, err := resolveProfile(ctx, profileFlag(cmd)) - if err != nil { - log.Debugf(ctx, "telemetry: no profile to upload with: %s", err) - return - } - setUploadConfig(cmd, profileName, configFile) -} - // setUploadConfig sets the config of profileName in configFile for the exit-time upload, resolved within // telemetryAuthBudget, with credentials that never refresh (mcp.BoundUploadConfig). A profile that does not resolve // uploads nothing. @@ -280,3 +259,18 @@ func signInPromptedEvent(agent protos.AitoolsAgentType, service string, signedIn }}, } } + +// bindingDrifted reports whether a server recorded for cfg's profile in state no longer matches the profile: its host or +// workspace changed, or its config file moved. The upload would send the profile's token to the host the profile names +// now, which the server was not set up against. It reads the config files only. +func bindingDrifted(ctx context.Context, state *mcp.State, cfg *config.Config) bool { + for _, e := range state.Servers { + if e.Profile != cfg.Profile { + continue + } + if _, err := mcp.CheckBinding(ctx, state, e.Service, cfg.Profile, cfg.ConfigFile); err != nil { + return true + } + } + return false +} From 8c93399627b82dc45f250671a05e30cd5ed5bda4 Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Fri, 13 Nov 2026 15:38:08 +0000 Subject: [PATCH 13/22] Resolve add, login and logout telemetry auth as install does The three commands no longer set their profile's config for the upload themselves. withTelemetryAuth resolves it in the background, as install does, and the run waits for it once it has logged an event. The events upload with the auth root resolves, so root's auth enrichment still applies to their errors; the MCP login hint stays the only set of next steps. Co-authored-by: Isaac --- cmd/aitools/mcp_add.go | 8 +++++--- cmd/aitools/mcp_login.go | 24 ++++++++++++------------ cmd/aitools/mcp_login_test.go | 8 +++++--- 3 files changed, 22 insertions(+), 18 deletions(-) diff --git a/cmd/aitools/mcp_add.go b/cmd/aitools/mcp_add.go index dd0b83014c0..58a346b3504 100644 --- a/cmd/aitools/mcp_add.go +++ b/cmd/aitools/mcp_add.go @@ -19,7 +19,6 @@ import ( "github.com/databricks/cli/libs/aitools/mcpconfig" "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/browser" - "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/telemetry/protos" @@ -151,6 +150,7 @@ func (o *addOutput) slashPaths() { // NewMcpAddCmd returns `aitools mcp add`. func NewMcpAddCmd() *cobra.Command { var f mcpAddFlags + var auth *telemetryAuth cmd := &cobra.Command{ Use: "add", Short: "Install Databricks managed MCP servers into coding agents", @@ -171,6 +171,8 @@ Supported agents: ` + strings.Join(mcpAgentNames(), ", "), RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() jsonMode := installOutputIsJSON(cmd) + // The events upload with the auth resolved for the profile, and a run that logs one waits for it. + auth.waitAtExit = true out := &addOutput{ Scope: installer.ScopeGlobal, @@ -210,6 +212,8 @@ Supported agents: ` + strings.Join(mcpAgentNames(), ", "), cmd.Flags().BoolVar(&f.force, "force", false, "Overwrite an existing server with the same name that this CLI did not write") cmd.Flags().BoolVar(&f.login, "login", false, "Sign in to each sign-in-required server that was installed, after the installs") cmd.Flags().BoolVar(&f.noLogin, "no-login", false, "Do not sign in; print the login commands for sign-in-required servers (default without a terminal; in a terminal, the default asks for each one)") + // Resolve auth best-effort so the telemetry can upload, as install does. + auth = withTelemetryAuth(cmd) return cmd } @@ -348,8 +352,6 @@ func runMcpAdd(ctx context.Context, cmd *cobra.Command, f mcpAddFlags, jsonMode return err } } - // The bound profile authenticates the exit-time telemetry upload. - cmd.SetContext(cmdctx.SetConfigUsed(cmd.Context(), w.Config)) for i := range plans { plans[i].service = picked[i] } diff --git a/cmd/aitools/mcp_login.go b/cmd/aitools/mcp_login.go index b5cafb58f8d..cd5c9168fc6 100644 --- a/cmd/aitools/mcp_login.go +++ b/cmd/aitools/mcp_login.go @@ -12,12 +12,10 @@ import ( "github.com/databricks/cli/libs/aitools/mcpconfig" "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/browser" - "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/shellquote" "github.com/databricks/cli/libs/telemetry/protos" "github.com/databricks/databricks-sdk-go" - "github.com/databricks/databricks-sdk-go/config" "github.com/spf13/cobra" ) @@ -48,8 +46,6 @@ type mcpSignInOutput struct { ErrorCategory protos.AitoolsErrorCategory `json:"error_category,omitempty"` Error string `json:"error,omitempty"` - // cfg is the bound profile's config, which authenticates the exit-time telemetry upload. - cfg *config.Config // alreadySignedIn is set by login when the caller was signed in before it ran. alreadySignedIn bool } @@ -60,7 +56,8 @@ type mcpSignInAction func(ctx context.Context, errOut io.Writer, w *databricks.W // NewMcpLoginCmd returns `aitools mcp login`. func NewMcpLoginCmd() *cobra.Command { - return &cobra.Command{ + var auth *telemetryAuth + cmd := &cobra.Command{ Use: "login ", Short: "Sign in to a Databricks managed MCP service with your account", Long: `Sign in to a Databricks managed MCP service that uses your account with an external provider, such @@ -70,14 +67,18 @@ sign-in to complete. Without a browser, open the printed URL yourself. is a full name such as system.ai.github. The profile is the one whose user signs in.`, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { + auth.waitAtExit = true return runMcpSignInCommand(cmd, args[0], mcpLogin, protos.AitoolsMcpOperationSignIn) }, } + auth = withTelemetryAuth(cmd) + return cmd } // NewMcpLogoutCmd returns `aitools mcp logout`. func NewMcpLogoutCmd() *cobra.Command { - return &cobra.Command{ + var auth *telemetryAuth + cmd := &cobra.Command{ Use: "logout ", Short: "Remove your sign-in to a Databricks managed MCP service", Long: `Remove your sign-in to a Databricks managed MCP service on a profile. The install of the server @@ -86,9 +87,12 @@ stays in place. Running it when you are not signed in is not an error. is a full name such as system.ai.github.`, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { + auth.waitAtExit = true return runMcpSignInCommand(cmd, args[0], mcpLogout, protos.AitoolsMcpOperationSignOut) }, } + auth = withTelemetryAuth(cmd) + return cmd } // runMcpSignInCommand resolves the profile for service and runs action. In JSON mode the document is @@ -99,11 +103,8 @@ func runMcpSignInCommand(cmd *cobra.Command, service string, action mcpSignInAct out := &mcpSignInOutput{Service: service} runErr := runMcpSignIn(ctx, cmd.ErrOrStderr(), profileFlag(cmd), service, action, out) - if out.cfg != nil { - cmd.SetContext(cmdctx.SetConfigUsed(cmd.Context(), out.cfg)) - } - // The event uploads with the profile's config, so a run that reached no server logs nothing. - if out.cfg != nil { + // The host is set once the profile's binding has passed and its client exists, so a run that reached no server logs nothing. + if out.Host != "" { logMcpEvent(ctx, signInEvent(operation, out, runErr)) } if !jsonMode { @@ -161,7 +162,6 @@ func runMcpSignIn(ctx context.Context, errOut io.Writer, flagValue, service stri return err } out.Host = w.Config.Host - out.cfg = w.Config return action(ctx, errOut, w, out, pinnedFile) } diff --git a/cmd/aitools/mcp_login_test.go b/cmd/aitools/mcp_login_test.go index 8e8da4fe637..a04399f2f08 100644 --- a/cmd/aitools/mcp_login_test.go +++ b/cmd/aitools/mcp_login_test.go @@ -25,6 +25,7 @@ import ( "github.com/databricks/cli/libs/telemetry/protos" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/config" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -108,7 +109,8 @@ func TestMcpLoginRejectedTokenIsAuthFailed(t *testing.T) { } // TestMcpLoginRejectedTokenHasOneSetOfNextSteps checks the error that root prints for a rejected token after login has -// its client: the MCP login command is the only set of next steps, since root's auth enrichment leaves it alone. +// its client: the MCP login command is the only set of next steps, since root's auth enrichment leaves it alone. Root +// enriches the errors of a command whose upload config is set, which is the profile's config. func TestMcpLoginRejectedTokenHasOneSetOfNextSteps(t *testing.T) { home := withHome(t) server := loginTestServer(t, http.StatusUnauthorized, `{"error_code":"UNAUTHENTICATED","message":"Invalid access token."}`) @@ -117,8 +119,8 @@ func TestMcpLoginRejectedTokenHasOneSetOfNextSteps(t *testing.T) { out := mcpSignInOutput{Service: "system.ai.github"} err := runMcpSignIn(t.Context(), io.Discard, "logfood", "system.ai.github", mcpLogin, &out) require.Error(t, err) - require.NotNil(t, out.cfg, "the client exists, so root enriches errors") - enriched := auth.EnrichAuthError(t.Context(), out.cfg, err) + require.NotEmpty(t, out.Host, "the client exists, so the profile's config is known") + enriched := auth.EnrichAuthError(t.Context(), &config.Config{Host: out.Host, Profile: "logfood"}, err) assert.Equal(t, err.Error(), enriched.Error()) assert.Equal(t, 0, strings.Count(enriched.Error(), "Next steps:")) } From 76a784ded8811dca95c85e51a5a5e2d1e52a1060 Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Fri, 13 Nov 2026 15:40:35 +0000 Subject: [PATCH 14/22] Log PROXY_STARTED only for a serving start A start that fails its checks has no upload to make: a binding that failed has no profile the upload may authenticate with, and a rejected credential would only repeat its rejected request. serve logs its start once the checks have passed, and an event that names a rejected credential is not logged. Co-authored-by: Isaac --- .../telemetry/mcp-serve-rejected/output.txt | 13 +----- .../telemetry/mcp-serve-rejected/script | 2 +- .../telemetry/mcp-serve-rejected/test.toml | 4 +- .../aitools/telemetry/mcp-serve/output.txt | 2 +- .../cmd/aitools/telemetry/mcp-serve/script | 2 +- cmd/aitools/mcp_serve.go | 19 ++------- cmd/aitools/mcp_serve_test.go | 17 ++++++++ cmd/aitools/mcp_telemetry.go | 29 ++++++++------ cmd/aitools/mcp_telemetry_test.go | 40 +++++++++++++------ libs/telemetry/protos/aitools_mcp.go | 4 +- 10 files changed, 73 insertions(+), 59 deletions(-) diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/output.txt b/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/output.txt index 18e5860dd73..89d332ab32f 100644 --- a/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/output.txt +++ b/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/output.txt @@ -1,18 +1,7 @@ -=== serve whose startup check is rejected: exit 2, and PROXY_STARTED is uploaded with AUTH_FAILED +=== serve whose startup check is rejected: exit 2, and nothing is uploaded: a failed start logs no event >>> musterr [CLI] aitools mcp serve --profile logfood --service system.ai.genie_one_mcp --agent claude-code No install record for system.ai.genie_one_mcp on profile "logfood"; using its current host [DATABRICKS_URL] cannot read MCP service system.ai.genie_one_mcp: Invalid access token. Run: databricks auth login --profile logfood -{ - "calling_agent": "CLAUDE_CODE", - "operation": "PROXY_STARTED", - "servers": [ - { - "service": "GENIE_ONE", - "error_category": "AUTH_FAILED" - } - ], - "bound_profile_count": 0 -} diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/script b/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/script index 25fb6292ef9..cf5022fbc50 100644 --- a/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/script +++ b/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/script @@ -3,6 +3,6 @@ envsubst < .databrickscfg > out && mv out .databrickscfg export DATABRICKS_CONFIG_FILE=.databrickscfg export BROWSER=none -title "serve whose startup check is rejected: exit 2, and PROXY_STARTED is uploaded with AUTH_FAILED" +title "serve whose startup check is rejected: exit 2, and nothing is uploaded: a failed start logs no event" trace musterr $CLI aitools mcp serve --profile logfood --service system.ai.genie_one_mcp --agent claude-code < /dev/null > /dev/null mcp_events diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/test.toml b/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/test.toml index c081d32cf7a..d60a4a6d774 100644 --- a/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/test.toml +++ b/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/test.toml @@ -1,5 +1,5 @@ -# The startup lookup of the service is rejected, so serve exits 2 before it serves. Its PROXY_STARTED event records the -# failure as AUTH_FAILED, and the upload that follows carries it. +# The startup lookup of the service is rejected, so serve exits 2 before it serves. A start that fails its checks logs no +# event, so no upload follows: the rejected token is not sent again. [[Server]] Pattern = "GET /api/2.1/unity-catalog/mcp-services/{name}" Response.StatusCode = 401 diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve/output.txt b/acceptance/cmd/aitools/telemetry/mcp-serve/output.txt index 1bcd45d41b1..d8fec5e3dcf 100644 --- a/acceptance/cmd/aitools/telemetry/mcp-serve/output.txt +++ b/acceptance/cmd/aitools/telemetry/mcp-serve/output.txt @@ -14,7 +14,7 @@ No install record for system.ai.genie_one_mcp on profile "logfood"; using its cu "bound_profile_count": 0 } -=== A binding that fails logs PROXY_STARTED with its category, but no upload follows: there is no config to upload with +=== A binding that fails logs no PROXY_STARTED, so no upload follows >>> musterr [CLI] aitools mcp serve --profile gone --service system.ai.genie_one_mcp --agent codex MCP server system.ai.genie_one_mcp needs Databricks profile "gone", which does not exist in [TEST_TMP_DIR]/.databrickscfg. Set it up with: databricks auth login --profile gone diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve/script b/acceptance/cmd/aitools/telemetry/mcp-serve/script index 2e865c20274..0a5108cdb15 100644 --- a/acceptance/cmd/aitools/telemetry/mcp-serve/script +++ b/acceptance/cmd/aitools/telemetry/mcp-serve/script @@ -14,7 +14,7 @@ title "serve logs PROXY_STARTED after the binding checks; the upload is sent whe trace $CLI aitools mcp serve --profile logfood --service system.ai.genie_one_mcp --agent claude-code < requests-list.ndjson > /dev/null mcp_events -title "A binding that fails logs PROXY_STARTED with its category, but no upload follows: there is no config to upload with" +title "A binding that fails logs no PROXY_STARTED, so no upload follows" trace musterr $CLI aitools mcp serve --profile gone --service system.ai.genie_one_mcp --agent codex < /dev/null > /dev/null mcp_events diff --git a/cmd/aitools/mcp_serve.go b/cmd/aitools/mcp_serve.go index bf1ce9bbc03..71c53112b07 100644 --- a/cmd/aitools/mcp_serve.go +++ b/cmd/aitools/mcp_serve.go @@ -114,21 +114,9 @@ func runMcpServe(cmd *cobra.Command, f mcpServeFlags) error { Stderr: cmd.ErrOrStderr(), }) if err != nil { - // An AUTH_FAILED binding has passed the host check, so its profile may authenticate the upload. A profile whose - // host changed is not used: Bind never contacts a host the record does not name. - if errorCategory(err) == protos.AitoolsErrorCategoryAuthFailed { - if configFile, _, cerr := addConfigFile(cmd.Context()); cerr == nil { - setUploadConfig(cmd, profileName, configFile) - } - } - logMcpEvent(cmd.Context(), proxyStartedEvent(agent, f.service, err)) return serveFailure(cmd, err, serveBindStatus) } - // The bound profile authenticates the exit-time telemetry upload. The session's own config refreshes its token, which - // the upload must not do, so the upload gets a config of its own. - setUploadConfig(cmd, profileName, cfg.ConfigFile) - // A server pinned to a config file names that file in its hints, as add and login do. pinnedFile := mcp.PinnedFile(entry, cfg.ConfigFile) ctx, stop := serveSignalContext(cmd.Context()) @@ -150,7 +138,6 @@ func runMcpServe(cmd *cobra.Command, f mcpServeFlags) error { // A rejected token stops serve by cancelling ctx, so the refresh failure is checked before the signal, and its // message is the one that names the login. if refreshErr := refresh.failure(); refreshErr != nil { - cmd.SetContext(cmdctx.ClearConfigUsed(cmd.Context())) return serveFailure(cmd, refreshErr, serveBindStatus) } if ctx.Err() != nil { @@ -158,10 +145,12 @@ func runMcpServe(cmd *cobra.Command, f mcpServeFlags) error { return nil } // serveChallenge fails only on an auth failure (see lookupAtStart), which exits serveBindStatus. - logMcpEvent(cmd.Context(), proxyStartedEvent(agent, f.service, err)) return serveFailure(cmd, err, serveBindStatus) } - logMcpEvent(cmd.Context(), proxyStartedEvent(agent, f.service, nil)) + // Only a serving start is logged. The upload authenticates with a config of its own that never refreshes a token + // (see setUploadConfig), since the session's config refreshes its token. + setUploadConfig(cmd, profileName, cfg.ConfigFile) + logMcpEvent(cmd.Context(), proxyStartedEvent(agent, f.service)) return serveProxy(ctx, cmd, refresh, proxy.Options{ In: cmd.InOrStdin(), diff --git a/cmd/aitools/mcp_serve_test.go b/cmd/aitools/mcp_serve_test.go index a35064107e5..4d12afe636b 100644 --- a/cmd/aitools/mcp_serve_test.go +++ b/cmd/aitools/mcp_serve_test.go @@ -27,6 +27,8 @@ import ( "github.com/databricks/cli/libs/aitools/mcpconfig" "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/libs/telemetry" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/config" "github.com/databricks/databricks-sdk-go/httpclient" @@ -335,6 +337,21 @@ func TestMcpServeRejectsServiceWithoutThreeParts(t *testing.T) { // TestMcpServeStateReadFailureExitsTwo checks that a state file that cannot be read stops the binding checks, so it // exits 2 like the other binding failures. +// A start that fails its binding checks logs no PROXY_STARTED, so nothing uploads. +func TestMcpServeFailedStartLogsNothing(t *testing.T) { + withHome(t) + parent := newMCPTestRoot() + parent.SetErr(&bytes.Buffer{}) + parent.SetArgs([]string{"mcp", "serve", "--profile", testProfile, "--service", testService}) + ctx := env.Set(telemetry.WithNewLogger(cmdio.MockDiscard(t.Context())), "DATABRICKS_CLI_DISABLE_TELEMETRY", "") + err := parent.ExecuteContext(ctx) + require.ErrorIs(t, err, root.ErrAlreadyPrinted) + assert.Equal(t, serveBindStatus, root.ExitStatus(err)) + serve, _, findErr := parent.Find([]string{"mcp", "serve"}) + require.NoError(t, findErr) + assert.Empty(t, telemetry.BufferedEvents(serve.Context())) +} + func TestMcpServeStateReadFailureExitsTwo(t *testing.T) { withHome(t) dir, err := installer.GlobalSkillsDir(t.Context()) diff --git a/cmd/aitools/mcp_telemetry.go b/cmd/aitools/mcp_telemetry.go index e3b623f8282..72266d9d7a4 100644 --- a/cmd/aitools/mcp_telemetry.go +++ b/cmd/aitools/mcp_telemetry.go @@ -2,6 +2,7 @@ package aitools import ( "context" + "slices" "strings" "github.com/databricks/cli/libs/aitools/agents" @@ -75,13 +76,20 @@ func proxyAgentType(flag string) protos.AitoolsAgentType { } // logMcpEvent buffers an MCP event; cmd/root uploads it at exit. An event without server results -// is not logged: a run that stopped before any server was reached has nothing to report. The bound -// profile count is read from the state file as it is after the operation. A state file that cannot -// be read drops the event; telemetry never fails a command. +// is not logged: a run that stopped before any server was reached has nothing to report. An event +// that names a rejected credential is not logged either: its upload would authenticate with that +// credential and be rejected the same way. The bound profile count is read from the state file as +// it is after the operation. A state file that cannot be read drops the event; telemetry never +// fails a command. func logMcpEvent(ctx context.Context, event protos.AitoolsMcpEvent) { if len(event.Servers) == 0 { return } + if slices.ContainsFunc(event.Servers, func(r protos.AitoolsMcpServerResult) bool { + return r.ErrorCategory == protos.AitoolsErrorCategoryAuthFailed + }) { + return + } state, err := mcp.LoadState(ctx) if err != nil { log.Debugf(ctx, "telemetry: cannot read the MCP state; dropping the event: %s", err) @@ -215,18 +223,15 @@ func signOutServerResult(service, state string, err error) protos.AitoolsMcpServ return r } -// proxyStartedEvent is the PROXY_STARTED event of serve, logged once the startup checks have run. err is the failure -// of a check, or nil when the proxy is about to serve. The event uploads only when its profile resolved: a failed -// binding with no profile to upload with leaves it unsent. -func proxyStartedEvent(agent protos.AitoolsAgentType, service string, err error) protos.AitoolsMcpEvent { - r := protos.AitoolsMcpServerResult{Service: mcpServiceType(service)} - if err != nil { - r.ErrorCategory = errorCategory(err) - } +// proxyStartedEvent is the PROXY_STARTED event of serve, logged once the startup checks have passed and the proxy serves. +// A start that fails its checks logs nothing. A binding that failed has no profile the upload may authenticate with +// (PROFILE_NOT_FOUND and PROFILE_HOST_CHANGED are never sent), and a rejected credential would only repeat its rejected +// request, so none of them uploads. +func proxyStartedEvent(agent protos.AitoolsAgentType, service string) protos.AitoolsMcpEvent { return protos.AitoolsMcpEvent{ CallingAgent: agent, Operation: protos.AitoolsMcpOperationProxyStarted, - Servers: []protos.AitoolsMcpServerResult{r}, + Servers: []protos.AitoolsMcpServerResult{{Service: mcpServiceType(service)}}, } } diff --git a/cmd/aitools/mcp_telemetry_test.go b/cmd/aitools/mcp_telemetry_test.go index b78201e3e29..8c2b8649ac0 100644 --- a/cmd/aitools/mcp_telemetry_test.go +++ b/cmd/aitools/mcp_telemetry_test.go @@ -9,6 +9,8 @@ import ( "testing" "github.com/databricks/cli/libs/aitools/mcp" + "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/libs/telemetry" "github.com/databricks/cli/libs/telemetry/protos" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -176,24 +178,36 @@ func TestSignOutServerResult(t *testing.T) { } func TestProxyStartedEvent(t *testing.T) { - ok := proxyStartedEvent(protos.AitoolsAgentTypeClaudeCode, "system.ai.genie_one_mcp", nil) assert.Equal(t, protos.AitoolsMcpEvent{ CallingAgent: protos.AitoolsAgentTypeClaudeCode, Operation: protos.AitoolsMcpOperationProxyStarted, Servers: []protos.AitoolsMcpServerResult{{Service: protos.AitoolsMcpServiceGenieOne}}, - }, ok) + }, proxyStartedEvent(protos.AitoolsAgentTypeClaudeCode, "system.ai.genie_one_mcp")) +} - // Each binding failure has its category; the error text never reaches the event. - for category, bind := range map[protos.AitoolsErrorCategory]mcp.BindCategory{ - protos.AitoolsErrorCategoryProfileNotFound: mcp.BindProfileNotFound, - protos.AitoolsErrorCategoryProfileHostChanged: mcp.BindProfileHostChanged, - protos.AitoolsErrorCategoryAuthFailed: mcp.BindAuthFailed, - } { - event := proxyStartedEvent(protos.AitoolsAgentTypeUnspecified, "main.tools.custom", &mcp.BindError{Category: bind}) - require.Len(t, event.Servers, 1) - assert.Equal(t, category, event.Servers[0].ErrorCategory) - assert.Equal(t, protos.AitoolsMcpServiceCustom, event.Servers[0].Service) - } +// An event that names a rejected credential is not logged: its upload would authenticate with that credential again. +func TestLogMcpEventDropsRejectedCredential(t *testing.T) { + withHome(t) + ctx := env.Set(telemetry.WithNewLogger(t.Context()), "DATABRICKS_CLI_DISABLE_TELEMETRY", "") + logMcpEvent(ctx, protos.AitoolsMcpEvent{ + Operation: protos.AitoolsMcpOperationSignIn, + Servers: []protos.AitoolsMcpServerResult{{ + Service: protos.AitoolsMcpServiceGitHub, + Outcome: protos.AitoolsTargetOutcomeFailed, + ErrorCategory: protos.AitoolsErrorCategoryAuthFailed, + }}, + }) + assert.Empty(t, telemetry.BufferedEvents(ctx)) + + logMcpEvent(ctx, protos.AitoolsMcpEvent{ + Operation: protos.AitoolsMcpOperationSignIn, + Servers: []protos.AitoolsMcpServerResult{{ + Service: protos.AitoolsMcpServiceGitHub, + Outcome: protos.AitoolsTargetOutcomeFailed, + ErrorCategory: protos.AitoolsErrorCategoryMCPPermissionDenied, + }}, + }) + assert.Len(t, telemetry.BufferedEvents(ctx), 1) } func TestSignInPromptedEvent(t *testing.T) { diff --git a/libs/telemetry/protos/aitools_mcp.go b/libs/telemetry/protos/aitools_mcp.go index f6a8d2f677b..0d7af69e138 100644 --- a/libs/telemetry/protos/aitools_mcp.go +++ b/libs/telemetry/protos/aitools_mcp.go @@ -64,8 +64,8 @@ const ( // AitoolsMcpServerResult is the result for one service, for one agent when the // operation writes agent config. Unset fields are omitted: an agent is not set // for sign-in, and an outcome is not set for proxy events. An error category is -// set when the target failed or was skipped, when a proxy start failed its -// checks (PROXY_STARTED), and on SIGN_IN_PROMPTED, where it is SIGN_IN_REQUIRED. +// set when the target failed or was skipped, and on SIGN_IN_PROMPTED, where it is +// SIGN_IN_REQUIRED. type AitoolsMcpServerResult struct { Service AitoolsMcpService `json:"service"` Agent AitoolsAgentType `json:"agent,omitempty"` From e98a36adf2892ad94e4cfa226e5ea1ea0a513f27 Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Fri, 13 Nov 2026 15:45:49 +0000 Subject: [PATCH 15/22] Tidy the MCP telemetry: sign-in and record-only states, service enum, docs An add whose sign-in failed before reading a state reports PENDING. A record-only removal that fails to save is reported FAILED. The service enum is one lookup that falls back to OTHER_SYSTEM. The server result doc says which outcomes carry an error category. Test comments name their test. Co-authored-by: Isaac --- cmd/aitools/mcp_add_test.go | 4 ++- cmd/aitools/mcp_login_test.go | 2 +- cmd/aitools/mcp_remove.go | 11 +++++++ cmd/aitools/mcp_remove_test.go | 37 ++++++++++++++++++++--- cmd/aitools/mcp_serve.go | 2 +- cmd/aitools/mcp_serve_test.go | 3 +- cmd/aitools/mcp_telemetry.go | 10 +++--- cmd/aitools/mcp_telemetry_test.go | 18 ++++++++--- libs/aitools/mcp/binding_test.go | 12 +++++++- libs/aitools/mcp/proxy/challenge_test.go | 8 +++-- libs/auth/error_test.go | 2 +- libs/telemetry/protos/aitools_mcp.go | 5 +-- libs/telemetry/protos/aitools_mcp_test.go | 7 +++-- 13 files changed, 93 insertions(+), 28 deletions(-) diff --git a/cmd/aitools/mcp_add_test.go b/cmd/aitools/mcp_add_test.go index 312d2dd5fc4..78944886c3b 100644 --- a/cmd/aitools/mcp_add_test.go +++ b/cmd/aitools/mcp_add_test.go @@ -277,9 +277,11 @@ func withDatabricksBinary(t *testing.T) { } // noRequestServer returns a server that fails the test on any request. A command that refuses before -// it contacts the profile's host never reaches it. +// it contacts the profile's host never reaches it. Telemetry is off: its auth resolution, which starts with the command, +// reads the host's metadata, a request the refusal does not make. func noRequestServer(t *testing.T) *httptest.Server { t.Helper() + t.Setenv("DATABRICKS_CLI_DISABLE_TELEMETRY", "1") server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { t.Errorf("unexpected request before the refusal: %s %s", r.Method, r.URL) w.WriteHeader(http.StatusInternalServerError) diff --git a/cmd/aitools/mcp_login_test.go b/cmd/aitools/mcp_login_test.go index a04399f2f08..338651ee04b 100644 --- a/cmd/aitools/mcp_login_test.go +++ b/cmd/aitools/mcp_login_test.go @@ -110,7 +110,7 @@ func TestMcpLoginRejectedTokenIsAuthFailed(t *testing.T) { // TestMcpLoginRejectedTokenHasOneSetOfNextSteps checks the error that root prints for a rejected token after login has // its client: the MCP login command is the only set of next steps, since root's auth enrichment leaves it alone. Root -// enriches the errors of a command whose upload config is set, which is the profile's config. +// enriches the errors of a command whose telemetry upload is authenticated with the profile. func TestMcpLoginRejectedTokenHasOneSetOfNextSteps(t *testing.T) { home := withHome(t) server := loginTestServer(t, http.StatusUnauthorized, `{"error_code":"UNAUTHENTICATED","message":"Invalid access token."}`) diff --git a/cmd/aitools/mcp_remove.go b/cmd/aitools/mcp_remove.go index 33347f72430..3de8322b403 100644 --- a/cmd/aitools/mcp_remove.go +++ b/cmd/aitools/mcp_remove.go @@ -235,6 +235,8 @@ func runMcpRemove(ctx context.Context, cmd *cobra.Command, f mcpRemoveFlags, jso files := newAgentFiles() var drops []recordDrop var agentErrs []error + // recordOnly are the indexes of out.results whose removal only drops a state record. + var recordOnly []int for _, e := range selected { removed := removeRemoved{Name: e.Name, Service: e.Service, Profile: e.Profile, Agents: []string{}} pin := resolveBinding(ctx, e) @@ -248,6 +250,7 @@ func runMcpRemove(ctx context.Context, cmd *cobra.Command, f mcpRemoveFlags, jso // There is no config this CLI can edit for the agent, so only the record can go. removed.Agents = append(removed.Agents, agentName) removed.RecordOnly = append(removed.RecordOnly, agentName) + recordOnly = append(recordOnly, len(out.results)) out.results = append(out.results, removeServerResult(e.Service, agentName, removeDeleted, nil)) drops = append(drops, recordDrop{name: e.Name, agent: agentName}) continue @@ -269,6 +272,7 @@ func runMcpRemove(ctx context.Context, cmd *cobra.Command, f mcpRemoveFlags, jso // A server with no agent left in state is removed from state too, so it is reported as removed. Only its // record goes, which is a change to the state. if len(e.Agents) == 0 { + recordOnly = append(recordOnly, len(out.results)) out.results = append(out.results, protos.AitoolsMcpServerResult{Service: mcpServiceType(e.Service), Outcome: protos.AitoolsTargetOutcomeChanged}) } if len(removed.Agents) > 0 || len(e.Agents) == 0 { @@ -286,6 +290,13 @@ func runMcpRemove(ctx context.Context, cmd *cobra.Command, f mcpRemoveFlags, jso if len(drops) > 0 || slices.ContainsFunc(selected, func(e mcp.Entry) bool { return len(e.Agents) == 0 }) { saveErr = dropRecords(ctx, drops, selected) } + if saveErr != nil { + // A record-only removal changes nothing but the state, so the save that failed undid it. + for _, i := range recordOnly { + out.results[i].Outcome = protos.AitoolsTargetOutcomeFailed + out.results[i].ErrorCategory = errorCategory(saveErr) + } + } if !jsonMode { printRemoveResults(ctx, out) } diff --git a/cmd/aitools/mcp_remove_test.go b/cmd/aitools/mcp_remove_test.go index 57d9442bdec..180ec8cacb8 100644 --- a/cmd/aitools/mcp_remove_test.go +++ b/cmd/aitools/mcp_remove_test.go @@ -819,7 +819,8 @@ func runRemoveWithTelemetry(t *testing.T, stdout, logs *bytes.Buffer, logLevel s return remove, err } -// A run that removes nothing logs no event, so it resolves no auth for an upload. +// TestMcpRemoveResolvesNothingWithoutAnEvent checks that a run that removes nothing logs no event, so it resolves no auth +// for an upload. func TestMcpRemoveResolvesNothingWithoutAnEvent(t *testing.T) { withHome(t) resolves := 0 @@ -834,8 +835,9 @@ func TestMcpRemoveResolvesNothingWithoutAnEvent(t *testing.T) { assert.Zero(t, resolves) } -// The upload is authenticated with the profile only when no server recorded for it has drifted: its host here is the one -// the server was set up against. The removal is done either way, and the output is written before the auth resolves. +// TestMcpRemoveUploadSkipsDriftedProfile checks that the upload is not authenticated with a profile that a server recorded +// for it no longer matches, as its host here is not the one the server was set up against. The removal is done either +// way, and the output is written before the auth resolves. func TestMcpRemoveUploadSkipsDriftedProfile(t *testing.T) { home := withHome(t) writeHomeConfig(t, home, "[logfood]\nhost = https://adb-2.test\nauth_type = pat\ntoken = t\n") @@ -860,7 +862,7 @@ func TestMcpRemoveUploadSkipsDriftedProfile(t *testing.T) { assert.NotContains(t, readJSONFile(t, claudePath)["mcpServers"], "genie_one_mcp-logfood", "the removal is done") } -// A profile whose recorded servers still match is used for the upload. +// TestMcpRemoveUploadUsesMatchingProfile checks that a profile whose recorded servers still match is used for the upload. func TestMcpRemoveUploadUsesMatchingProfile(t *testing.T) { home := withHome(t) writeHomeConfig(t, home, "[logfood]\nhost = https://adb-1.test\nauth_type = pat\ntoken = t\n") @@ -880,7 +882,8 @@ func TestMcpRemoveUploadUsesMatchingProfile(t *testing.T) { assert.Equal(t, "https://adb-1.test", cmdctx.ConfigUsed(remove.Context()).Host) } -// An auth resolution that fails, as it does offline, leaves nothing in the output: its logs are debug only. +// TestMcpRemoveOfflineResolutionWarnsNothing checks that an auth resolution that fails, as it does offline, leaves nothing +// in the output: its logs are debug only. func TestMcpRemoveOfflineResolutionWarnsNothing(t *testing.T) { home := withHome(t) writeHomeConfig(t, home, "[logfood]\nhost = https://adb-1.test\nauth_type = pat\ntoken = t\n") @@ -899,3 +902,27 @@ func TestMcpRemoveOfflineResolutionWarnsNothing(t *testing.T) { assert.Empty(t, logs.String()) assert.Contains(t, stdout.String(), `"removed"`) } + +// TestMcpRemoveRecordOnlyReportsFailedWhenStateIsNotSaved checks that a removal that only drops state records is reported +// FAILED when the state cannot be saved, since nothing was removed. +func TestMcpRemoveRecordOnlyReportsFailedWhenStateIsNotSaved(t *testing.T) { + withHome(t) + retired := mcp.Entry{ + Name: "dbsql-logfood", Service: "system.ai.dbsql", Profile: "logfood", Host: "https://adb-1.test", Command: testDatabricksCommand, + Agents: map[string]mcp.AgentRecord{"retired-agent": {Path: "/x/config.json", Scope: "user"}}, + } + saveState(t, retired) + saveMCPState = func(context.Context, *mcp.State) error { return errors.New("state file is read-only") } + t.Cleanup(func() { saveMCPState = mcp.SaveState }) + setTelemetryAuth(t, time.Second, func(*cobra.Command, []string) error { return nil }) + + var stdout, logs bytes.Buffer + remove, err := runRemoveWithTelemetry(t, &stdout, &logs, log.LevelDebug, "--names", "dbsql-logfood", "--output", "json") + require.ErrorIs(t, err, root.ErrAlreadyPrinted) + + events := telemetry.BufferedEvents(remove.Context()) + require.Len(t, events, 1) + require.Len(t, events[0].AitoolsMcpEvent.Servers, 1) + assert.Equal(t, protos.AitoolsTargetOutcomeFailed, events[0].AitoolsMcpEvent.Servers[0].Outcome) + assert.Equal(t, protos.AitoolsErrorCategoryUncategorized, events[0].AitoolsMcpEvent.Servers[0].ErrorCategory) +} diff --git a/cmd/aitools/mcp_serve.go b/cmd/aitools/mcp_serve.go index 71c53112b07..eda41fbb6a0 100644 --- a/cmd/aitools/mcp_serve.go +++ b/cmd/aitools/mcp_serve.go @@ -81,7 +81,7 @@ func NewMcpServeCmd() *cobra.Command { cmd.Flags().StringVar(&f.service, strings.TrimPrefix(mcpconfig.ServiceFlag, "--"), "", "Full name of the MCP service, for example system.ai.github") cmd.Flags().StringVar(&f.agent, strings.TrimPrefix(mcpconfig.AgentFlag, "--"), "", - "Registry name of the calling agent, used only for the User-Agent") + "Registry name of the calling agent, for the User-Agent and telemetry") return cmd } diff --git a/cmd/aitools/mcp_serve_test.go b/cmd/aitools/mcp_serve_test.go index 4d12afe636b..2ecedd90aa1 100644 --- a/cmd/aitools/mcp_serve_test.go +++ b/cmd/aitools/mcp_serve_test.go @@ -337,7 +337,8 @@ func TestMcpServeRejectsServiceWithoutThreeParts(t *testing.T) { // TestMcpServeStateReadFailureExitsTwo checks that a state file that cannot be read stops the binding checks, so it // exits 2 like the other binding failures. -// A start that fails its binding checks logs no PROXY_STARTED, so nothing uploads. +// TestMcpServeFailedStartLogsNothing checks that a start that fails its binding checks logs no PROXY_STARTED, so nothing +// uploads. func TestMcpServeFailedStartLogsNothing(t *testing.T) { withHome(t) parent := newMCPTestRoot() diff --git a/cmd/aitools/mcp_telemetry.go b/cmd/aitools/mcp_telemetry.go index 72266d9d7a4..1ccd734dacb 100644 --- a/cmd/aitools/mcp_telemetry.go +++ b/cmd/aitools/mcp_telemetry.go @@ -51,10 +51,10 @@ func mcpServiceType(fullName string) protos.AitoolsMcpService { if len(parts) != 3 || !strings.EqualFold(parts[0]+"."+parts[1], mcp.SystemAILocation) { return protos.AitoolsMcpServiceCustom } - if mcp.KnownOf(fullName) == nil { - return protos.AitoolsMcpServiceOtherSystem + if t, ok := mcpServiceTypes[strings.ToLower(parts[2])]; ok { + return t } - return mcpServiceTypes[strings.ToLower(parts[2])] + return protos.AitoolsMcpServiceOtherSystem } // callingAgentType returns the agent that drives the CLI, Unspecified when none is detected. @@ -114,9 +114,9 @@ func boundProfileCount(ctx context.Context, state *mcp.State) int64 { } // addSignInState is the sign-in state of a server after an add: the state that --login reports, else PENDING for a -// server that needs sign-in, else NOT_REQUIRED. +// server that needs sign-in, else NOT_REQUIRED. A sign-in that failed before it read a state is PENDING. func addSignInState(s addServerOutput) protos.AitoolsMcpSignInState { - if s.SignIn != nil { + if s.SignIn != nil && s.SignIn.State != "" { return signInStateOf(s.SignIn.State) } if s.SignInRequired { diff --git a/cmd/aitools/mcp_telemetry_test.go b/cmd/aitools/mcp_telemetry_test.go index 8c2b8649ac0..deef70fd421 100644 --- a/cmd/aitools/mcp_telemetry_test.go +++ b/cmd/aitools/mcp_telemetry_test.go @@ -44,8 +44,8 @@ func TestMcpServiceType(t *testing.T) { } } -// Every service in the CLI's known table has its own enum, so a known service never reports OTHER_SYSTEM. A new table -// entry needs a new value in mcpServiceTypes. +// TestMcpServiceTypeCoversKnownServices checks that every service in the CLI's known table has its own enum, so a known +// service never reports OTHER_SYSTEM. A new table entry needs a new value in mcpServiceTypes. func TestMcpServiceTypeCoversKnownServices(t *testing.T) { for _, leaf := range mcp.KnownLeaves() { assert.Contains(t, mcpServiceTypes, leaf, "system.ai.%s has no telemetry enum: add it to mcpServiceTypes", leaf) @@ -185,7 +185,8 @@ func TestProxyStartedEvent(t *testing.T) { }, proxyStartedEvent(protos.AitoolsAgentTypeClaudeCode, "system.ai.genie_one_mcp")) } -// An event that names a rejected credential is not logged: its upload would authenticate with that credential again. +// TestLogMcpEventDropsRejectedCredential checks that an event that names a rejected credential is not logged: its upload +// would authenticate with that credential again. func TestLogMcpEventDropsRejectedCredential(t *testing.T) { withHome(t) ctx := env.Set(telemetry.WithNewLogger(t.Context()), "DATABRICKS_CLI_DISABLE_TELEMETRY", "") @@ -256,7 +257,8 @@ func TestAddSignInEvent(t *testing.T) { }, addSignInEvent("system.ai.github", signInStateActive, true, nil)) } -// The event carries enums and counts only. A custom service's name must not appear anywhere in the payload. +// TestMcpEventOmitsUserContent checks that the event carries enums and counts only: a custom service's name must not +// appear anywhere in the payload. func TestMcpEventOmitsUserContent(t *testing.T) { event := protos.AitoolsMcpEvent{ CallingAgent: protos.AitoolsAgentTypeClaudeCode, @@ -278,3 +280,11 @@ func TestMcpEventOmitsUserContent(t *testing.T) { assert.NotContains(t, string(b), "internal_tool") assert.Contains(t, string(b), `"service":"CUSTOM"`) } + +// TestAddSignInStateWithoutAState checks that a sign-in that failed before it read a state is PENDING, as a server that +// needs sign-in is until its state is known. A server that needs no sign-in is NOT_REQUIRED. +func TestAddSignInStateWithoutAState(t *testing.T) { + failed := addServerOutput{Service: "system.ai.github", SignInRequired: true, SignIn: &addSignInOutput{ErrorCategory: protos.AitoolsErrorCategoryUncategorized}} + assert.Equal(t, protos.AitoolsMcpSignInStatePending, addSignInState(failed)) + assert.Equal(t, protos.AitoolsMcpSignInStateNotRequired, addSignInState(addServerOutput{Service: "system.ai.dbsql"})) +} diff --git a/libs/aitools/mcp/binding_test.go b/libs/aitools/mcp/binding_test.go index d9642d04716..3f834451c17 100644 --- a/libs/aitools/mcp/binding_test.go +++ b/libs/aitools/mcp/binding_test.go @@ -938,7 +938,8 @@ func TestConfigFileCommandUsesForwardSlashes(t *testing.T) { mcp.ConfigFileCommand(path, "databricks auth login --profile logfood")) } -// The config of a telemetry upload never refreshes a token; the config a serve session binds keeps the default chain. +// TestBoundUploadConfigNeverRefreshes checks that the config of a telemetry upload never refreshes a token, and that the +// config a serve session binds keeps the default chain. func TestBoundUploadConfigNeverRefreshes(t *testing.T) { cfgPath := bindingTestConfig(t, bindingTestProfileContents(bindingTestToken)) @@ -950,3 +951,12 @@ func TestBoundUploadConfigNeverRefreshes(t *testing.T) { require.NoError(t, err) assert.NotEqual(t, auth.CredentialsWithoutRefresh(), session.Credentials) } + +// TestBindErrorNextStepsIncludedOnlyForAuthFailed checks that only an AUTH_FAILED message names the next steps, so root +// adds no second set of steps to the others. +func TestBindErrorNextStepsIncludedOnlyForAuthFailed(t *testing.T) { + assert.True(t, (&mcp.BindError{Category: mcp.BindAuthFailed}).NextStepsIncluded()) + for _, category := range []mcp.BindCategory{mcp.BindProfileNotFound, mcp.BindProfileHostChanged, mcp.BindUncategorized} { + assert.False(t, (&mcp.BindError{Category: category}).NextStepsIncluded(), category) + } +} diff --git a/libs/aitools/mcp/proxy/challenge_test.go b/libs/aitools/mcp/proxy/challenge_test.go index cad710c7c9a..6b3ea77cac7 100644 --- a/libs/aitools/mcp/proxy/challenge_test.go +++ b/libs/aitools/mcp/proxy/challenge_test.go @@ -273,8 +273,9 @@ func TestAuthenticateSaysAlreadySignedInAfterSignIn(t *testing.T) { require.NoError(t, p.finish(t)) } -// The prompt is recorded once per page that was opened, after its wait, with whether the credential -// became ACTIVE. A call that opens no page, or finds the caller already signed in, records nothing. +// TestAuthenticateRecordsPromptWhenPageOpens checks that the prompt is recorded once per page that was opened, after its +// wait, with whether the credential became ACTIVE. A call that opens no page, or finds the caller already signed in, +// records nothing. func TestAuthenticateRecordsPromptWhenPageOpens(t *testing.T) { srv, _ := challengedService(t) signIn := &signInStub{reply: func(call int) (bool, error) { @@ -304,7 +305,8 @@ func TestAuthenticateRecordsPromptWhenPageOpens(t *testing.T) { require.NoError(t, p.finish(t)) } -// A sign-in that fails before its URL is printed, such as a read of the workspace ID, shows no page: nothing is recorded. +// TestAuthenticateRecordsNoPromptWhenNoURLShown checks that a sign-in that fails before its URL is printed, such as a read +// of the workspace ID, shows no page, so nothing is recorded. func TestAuthenticateRecordsNoPromptWhenNoURLShown(t *testing.T) { srv, _ := challengedService(t) var calls int diff --git a/libs/auth/error_test.go b/libs/auth/error_test.go index b96a0386405..2ef1ff08ff5 100644 --- a/libs/auth/error_test.go +++ b/libs/auth/error_test.go @@ -81,7 +81,7 @@ func (e *stepsIncludedError) NextStepsIncluded() bool { } func TestEnrichAuthErrorLeavesErrorWithNextStepsAsIs(t *testing.T) { - cfg := &config.Config{Profile: "test", Host: "https://example.com", AuthType: "pat"} + cfg := &config.Config{Profile: "test", Host: "https://ws.test", AuthType: "pat"} original := &stepsIncludedError{cause: &apierr.APIError{StatusCode: 401, Message: "Unauthorized"}} assert.Equal(t, original, EnrichAuthError(t.Context(), cfg, original)) diff --git a/libs/telemetry/protos/aitools_mcp.go b/libs/telemetry/protos/aitools_mcp.go index 0d7af69e138..4e50bd67d46 100644 --- a/libs/telemetry/protos/aitools_mcp.go +++ b/libs/telemetry/protos/aitools_mcp.go @@ -64,8 +64,9 @@ const ( // AitoolsMcpServerResult is the result for one service, for one agent when the // operation writes agent config. Unset fields are omitted: an agent is not set // for sign-in, and an outcome is not set for proxy events. An error category is -// set when the target failed or was skipped, and on SIGN_IN_PROMPTED, where it is -// SIGN_IN_REQUIRED. +// set on a FAILED target and on SIGN_IN_PROMPTED, where it is SIGN_IN_REQUIRED. A +// SKIPPED target carries none: remove skips an entry it left in place, or one +// already gone, without a failure. type AitoolsMcpServerResult struct { Service AitoolsMcpService `json:"service"` Agent AitoolsAgentType `json:"agent,omitempty"` diff --git a/libs/telemetry/protos/aitools_mcp_test.go b/libs/telemetry/protos/aitools_mcp_test.go index 3b6ef512e3a..715a196103a 100644 --- a/libs/telemetry/protos/aitools_mcp_test.go +++ b/libs/telemetry/protos/aitools_mcp_test.go @@ -8,8 +8,8 @@ import ( "github.com/stretchr/testify/require" ) -// The lumberjack proto keys these fields in snake_case. This pins the wire keys of an event -// and of one server result, and that unset enums are omitted rather than sent empty. +// TestAitoolsMcpEventEncodesSnakeCaseKeys checks that the lumberjack proto keys these fields in snake_case. It pins the +// wire keys of an event and of one server result, and that unset enums are omitted rather than sent empty. func TestAitoolsMcpEventEncodesSnakeCaseKeys(t *testing.T) { b, err := json.Marshal(DatabricksCliLog{AitoolsMcpEvent: &AitoolsMcpEvent{ CallingAgent: AitoolsAgentTypeClaudeCode, @@ -49,7 +49,8 @@ func TestAitoolsMcpEventEncodesSnakeCaseKeys(t *testing.T) { }, server) } -// A proxy event has no agent on the result, no outcome and no error category when it succeeded. +// TestAitoolsMcpServerResultOmitsUnsetFields checks that a proxy event has no agent on the result, no outcome and no error +// category when it succeeded. func TestAitoolsMcpServerResultOmitsUnsetFields(t *testing.T) { b, err := json.Marshal(AitoolsMcpServerResult{Service: AitoolsMcpServiceOtherSystem}) require.NoError(t, err) From 2ab75ded17cf9b520fdfc5972ddd91bfd073a508 Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Fri, 13 Nov 2026 16:13:14 +0000 Subject: [PATCH 16/22] Upload telemetry with a config the command sets The upload authenticates with a config a command sets once its binding checks have passed, in place of the one cmdctx.ConfigUsed holds, and a run can refuse its upload after a rejected credential. Commands that bind a profile use this instead of resolving the profile's auth in the background. Co-authored-by: Isaac --- libs/telemetry/logger.go | 45 +++++++++++++++++++++++-- libs/telemetry/logger_test.go | 63 +++++++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+), 3 deletions(-) diff --git a/libs/telemetry/logger.go b/libs/telemetry/logger.go index aac2063fbe5..97ffad8cd30 100644 --- a/libs/telemetry/logger.go +++ b/libs/telemetry/logger.go @@ -37,6 +37,39 @@ func Disabled(ctx context.Context) bool { return env.Get(ctx, disableEnvVar) != "" } +// SetUploadConfig sets the config that Upload authenticates with, in place of the one cmdctx.ConfigUsed holds. A command +// sets it once its own binding checks have passed, so that the upload authenticates with the profile the command bound. +// Like Log, it is a no-op without a logger. +func SetUploadConfig(ctx context.Context, cfg *config.Config) { + if l, ok := loggerFromContext(ctx); ok { + l.upload = cfg + } +} + +// RefuseUpload makes Upload send nothing for the run, whatever config is set. A run calls it once a credential was +// rejected, since the upload would present that credential again. Like Log, it is a no-op without a logger. +func RefuseUpload(ctx context.Context) { + if l, ok := loggerFromContext(ctx); ok { + l.refused = true + } +} + +// UploadConfig returns the config that Upload authenticates with: the one SetUploadConfig set, else the one +// cmdctx.ConfigUsed holds. It returns false when the run refused its upload or has no config. +func UploadConfig(ctx context.Context) (*config.Config, bool) { + l, ok := loggerFromContext(ctx) + if !ok || l.refused { + return nil, false + } + if l.upload != nil { + return l.upload, true + } + if cmdctx.HasConfigUsed(ctx) { + return cmdctx.ConfigUsed(ctx), true + } + return nil, false +} + func Log(ctx context.Context, event protos.DatabricksCliLog) { // A missing logger means telemetry was never initialized on this context // (e.g. a command invoked outside the normal cmd/root setup). Dropping the @@ -54,6 +87,11 @@ type logger struct { logs []protos.FrontendLog stateLineageHash string + // upload is the config Upload authenticates with, as SetUploadConfig set it. Nil leaves cmdctx.ConfigUsed to it. + upload *config.Config + // refused means the run refused its upload (RefuseUpload), so Upload sends nothing. + refused bool + // running counts the goroutines that Upload started and stopped waiting // for, which may still be using the config (see within). running atomic.Int32 @@ -123,8 +161,9 @@ func Upload(ctx context.Context, ec protos.ExecutionContext) error { protoLogs[i] = string(b) } - if !cmdctx.HasConfigUsed(ctx) { - log.Debugf(ctx, "no auth config available; skipping telemetry upload") + cfg, ok := UploadConfig(ctx) + if !ok { + log.Debugf(ctx, "no auth config available, or the run refused its telemetry upload; skipping telemetry upload") return nil } @@ -150,7 +189,7 @@ func Upload(ctx context.Context, ec protos.ExecutionContext) error { } apiClient, err := within(ctx, &l.running, func() (*client.DatabricksClient, error) { - return newAPIClient(cmdctx.ConfigUsed(ctx), &l.running) + return newAPIClient(cfg, &l.running) }) if err != nil { return fmt.Errorf("failed to create API client: %w", err) diff --git a/libs/telemetry/logger_test.go b/libs/telemetry/logger_test.go index 94673a9dd1a..a6a01c9a72e 100644 --- a/libs/telemetry/logger_test.go +++ b/libs/telemetry/logger_test.go @@ -465,3 +465,66 @@ func TestWithinReturnsPanicAsError(t *testing.T) { assert.EqualError(t, err, "panic: boom") assert.Eventually(t, func() bool { return running.Load() == 0 }, time.Second, time.Millisecond) } + +// TestUploadUsesConfigSetByCommand checks that the upload authenticates with the config the command set, in place of the +// one cmdctx.ConfigUsed holds. +func TestUploadUsesConfigSetByCommand(t *testing.T) { + bound := testserver.New(t) + t.Cleanup(bound.Close) + other := testserver.New(t) + t.Cleanup(other.Close) + var boundCalls, otherCalls int + bound.Handle("POST", "/telemetry-ext", func(req testserver.Request) any { + boundCalls++ + return ResponseBody{NumProtoSuccess: 1} + }) + other.Handle("POST", "/telemetry-ext", func(req testserver.Request) any { + otherCalls++ + return ResponseBody{NumProtoSuccess: 1} + }) + + ctx := WithNewLogger(t.Context()) + Log(ctx, protos.DatabricksCliLog{CliTestEvent: &protos.CliTestEvent{Name: protos.DummyCliEnumValue1}}) + ctx = cmdctx.SetConfigUsed(ctx, &config.Config{Host: other.URL, Token: "other"}) + SetUploadConfig(ctx, &config.Config{Host: bound.URL, Token: "bound"}) + + cfg, ok := UploadConfig(ctx) + require.True(t, ok) + assert.Equal(t, bound.URL, cfg.Host) + + require.NoError(t, Upload(ctx, protos.ExecutionContext{})) + assert.Equal(t, 1, boundCalls) + assert.Zero(t, otherCalls) +} + +// TestUploadRefusedSendsNothing checks that a run which refused its upload sends nothing, even with a config set by the +// command, and that a config set after the refusal is not used. +func TestUploadRefusedSendsNothing(t *testing.T) { + server := testserver.New(t) + t.Cleanup(server.Close) + var calls int + server.Handle("POST", "/telemetry-ext", func(req testserver.Request) any { + calls++ + return ResponseBody{NumProtoSuccess: 1} + }) + + ctx := WithNewLogger(t.Context()) + Log(ctx, protos.DatabricksCliLog{CliTestEvent: &protos.CliTestEvent{Name: protos.DummyCliEnumValue1}}) + SetUploadConfig(ctx, &config.Config{Host: server.URL, Token: "token"}) + RefuseUpload(ctx) + SetUploadConfig(ctx, &config.Config{Host: server.URL, Token: "token"}) + + _, ok := UploadConfig(ctx) + assert.False(t, ok) + require.NoError(t, Upload(ctx, protos.ExecutionContext{})) + assert.Zero(t, calls) +} + +// TestUploadConfigWithoutLogger checks that the upload slot is a no-op without a logger, as Log is. +func TestUploadConfigWithoutLogger(t *testing.T) { + ctx := t.Context() + SetUploadConfig(ctx, &config.Config{Host: "https://ws.test"}) + RefuseUpload(ctx) + _, ok := UploadConfig(ctx) + assert.False(t, ok) +} From 8a5baa29e545d1b15674155d9bc452e5dd5c6313 Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Fri, 13 Nov 2026 16:19:23 +0000 Subject: [PATCH 17/22] Authenticate MCP telemetry uploads with the profile each command binds add, login, logout, remove and serve no longer resolve the upload's auth in the background, as install does. Each one sets the config of the profile it bound once its binding checks have passed, and an upload never refreshes a token. remove sets it only when the removed servers share one profile and none of the servers recorded for it has drifted. serve sets it at exit, after its refresh guard has run, and a rejected credential refuses the upload. Errors of these commands are no longer enriched by the root command, since their upload no longer sets the command's config. The auth error hint that skipped that enrichment, and cmdctx.ClearConfigUsed, are removed. A removal that prunes an entry is reported CHANGED, and a failed state save reports it FAILED. Co-authored-by: Isaac --- .../aitools/telemetry/mcp-remove/output.txt | 2 +- cmd/aitools/mcp_add.go | 8 +- cmd/aitools/mcp_add_test.go | 5 +- cmd/aitools/mcp_login.go | 7 +- cmd/aitools/mcp_login_test.go | 37 +++++--- cmd/aitools/mcp_remove.go | 43 +++++---- cmd/aitools/mcp_remove_test.go | 92 ++++++++++--------- cmd/aitools/mcp_serve.go | 14 +-- cmd/aitools/mcp_serve_test.go | 9 +- cmd/aitools/mcp_telemetry.go | 74 +++++++++------ cmd/aitools/mcp_telemetry_test.go | 4 +- libs/aitools/mcp/binding.go | 6 -- libs/aitools/mcp/binding_test.go | 9 -- libs/auth/error.go | 10 -- libs/auth/error_test.go | 27 ------ libs/cmdctx/config_used.go | 6 -- libs/cmdctx/config_used_test.go | 4 - 17 files changed, 164 insertions(+), 193 deletions(-) diff --git a/acceptance/cmd/aitools/telemetry/mcp-remove/output.txt b/acceptance/cmd/aitools/telemetry/mcp-remove/output.txt index 4f85fef980b..1d732f75094 100644 --- a/acceptance/cmd/aitools/telemetry/mcp-remove/output.txt +++ b/acceptance/cmd/aitools/telemetry/mcp-remove/output.txt @@ -73,7 +73,7 @@ { "service": "GENIE_ONE", "agent": "CLAUDE_CODE", - "outcome": "SKIPPED" + "outcome": "CHANGED" } ], "bound_profile_count": 0 diff --git a/cmd/aitools/mcp_add.go b/cmd/aitools/mcp_add.go index 58a346b3504..6aeba213cbd 100644 --- a/cmd/aitools/mcp_add.go +++ b/cmd/aitools/mcp_add.go @@ -150,7 +150,6 @@ func (o *addOutput) slashPaths() { // NewMcpAddCmd returns `aitools mcp add`. func NewMcpAddCmd() *cobra.Command { var f mcpAddFlags - var auth *telemetryAuth cmd := &cobra.Command{ Use: "add", Short: "Install Databricks managed MCP servers into coding agents", @@ -171,8 +170,6 @@ Supported agents: ` + strings.Join(mcpAgentNames(), ", "), RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() jsonMode := installOutputIsJSON(cmd) - // The events upload with the auth resolved for the profile, and a run that logs one waits for it. - auth.waitAtExit = true out := &addOutput{ Scope: installer.ScopeGlobal, @@ -212,8 +209,6 @@ Supported agents: ` + strings.Join(mcpAgentNames(), ", "), cmd.Flags().BoolVar(&f.force, "force", false, "Overwrite an existing server with the same name that this CLI did not write") cmd.Flags().BoolVar(&f.login, "login", false, "Sign in to each sign-in-required server that was installed, after the installs") cmd.Flags().BoolVar(&f.noLogin, "no-login", false, "Do not sign in; print the login commands for sign-in-required servers (default without a terminal; in a terminal, the default asks for each one)") - // Resolve auth best-effort so the telemetry can upload, as install does. - auth = withTelemetryAuth(cmd) return cmd } @@ -338,6 +333,9 @@ func runMcpAdd(ctx context.Context, cmd *cobra.Command, f mcpAddFlags, jsonMode if err := planningRefusals(plans, targets); err != nil { return err } + // Every refusal that needs no request has come, so the upload authenticates with the profile this run binds. Resolving + // its config makes the host's metadata request, which is why it waits for the refusals. + setUploadConfig(ctx, profileName, configFile) if !pickingServices { w, workspaceID, err = connectWorkspace(ctx, out, profileName, configFile, pinned) diff --git a/cmd/aitools/mcp_add_test.go b/cmd/aitools/mcp_add_test.go index 78944886c3b..3b2dd9879ee 100644 --- a/cmd/aitools/mcp_add_test.go +++ b/cmd/aitools/mcp_add_test.go @@ -277,11 +277,10 @@ func withDatabricksBinary(t *testing.T) { } // noRequestServer returns a server that fails the test on any request. A command that refuses before -// it contacts the profile's host never reaches it. Telemetry is off: its auth resolution, which starts with the command, -// reads the host's metadata, a request the refusal does not make. +// it contacts the profile's host never reaches it, with telemetry on or off: the upload's config is resolved only after +// the binding checks have passed, and that resolution makes no request. func noRequestServer(t *testing.T) *httptest.Server { t.Helper() - t.Setenv("DATABRICKS_CLI_DISABLE_TELEMETRY", "1") server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { t.Errorf("unexpected request before the refusal: %s %s", r.Method, r.URL) w.WriteHeader(http.StatusInternalServerError) diff --git a/cmd/aitools/mcp_login.go b/cmd/aitools/mcp_login.go index cd5c9168fc6..7e577d6d11b 100644 --- a/cmd/aitools/mcp_login.go +++ b/cmd/aitools/mcp_login.go @@ -56,7 +56,6 @@ type mcpSignInAction func(ctx context.Context, errOut io.Writer, w *databricks.W // NewMcpLoginCmd returns `aitools mcp login`. func NewMcpLoginCmd() *cobra.Command { - var auth *telemetryAuth cmd := &cobra.Command{ Use: "login ", Short: "Sign in to a Databricks managed MCP service with your account", @@ -67,17 +66,14 @@ sign-in to complete. Without a browser, open the printed URL yourself. is a full name such as system.ai.github. The profile is the one whose user signs in.`, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - auth.waitAtExit = true return runMcpSignInCommand(cmd, args[0], mcpLogin, protos.AitoolsMcpOperationSignIn) }, } - auth = withTelemetryAuth(cmd) return cmd } // NewMcpLogoutCmd returns `aitools mcp logout`. func NewMcpLogoutCmd() *cobra.Command { - var auth *telemetryAuth cmd := &cobra.Command{ Use: "logout ", Short: "Remove your sign-in to a Databricks managed MCP service", @@ -87,11 +83,9 @@ stays in place. Running it when you are not signed in is not an error. is a full name such as system.ai.github.`, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - auth.waitAtExit = true return runMcpSignInCommand(cmd, args[0], mcpLogout, protos.AitoolsMcpOperationSignOut) }, } - auth = withTelemetryAuth(cmd) return cmd } @@ -155,6 +149,7 @@ func runMcpSignIn(ctx context.Context, errOut io.Writer, flagValue, service stri if err != nil { return err } + setUploadConfig(ctx, profileName, configFile) // A server pinned to a config file names that file in the hints, as add does. pinnedFile := mcp.PinnedFile(entry, configFile) w, err := profileClient(ctx, profileName, configFile, pinnedFile) diff --git a/cmd/aitools/mcp_login_test.go b/cmd/aitools/mcp_login_test.go index 338651ee04b..a8f35097a9e 100644 --- a/cmd/aitools/mcp_login_test.go +++ b/cmd/aitools/mcp_login_test.go @@ -20,12 +20,12 @@ import ( "github.com/databricks/cli/libs/aitools/agents" "github.com/databricks/cli/libs/aitools/installer" "github.com/databricks/cli/libs/aitools/mcp" - "github.com/databricks/cli/libs/auth" + "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/telemetry" "github.com/databricks/cli/libs/telemetry/protos" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" - "github.com/databricks/databricks-sdk-go/config" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -108,21 +108,30 @@ func TestMcpLoginRejectedTokenIsAuthFailed(t *testing.T) { assert.Contains(t, err.Error(), "Run: databricks auth login --profile logfood") } -// TestMcpLoginRejectedTokenHasOneSetOfNextSteps checks the error that root prints for a rejected token after login has -// its client: the MCP login command is the only set of next steps, since root's auth enrichment leaves it alone. Root -// enriches the errors of a command whose telemetry upload is authenticated with the profile. -func TestMcpLoginRejectedTokenHasOneSetOfNextSteps(t *testing.T) { +// TestMcpLogoutForbiddenKeepsItsText checks that a 403 from logout is reported as the API said it, with no next steps +// added: root enriches a command's auth errors only when the command sets its profile's config for the context, and +// logout's upload config is not that config. The run still uploads with the profile it bound. +func TestMcpLogoutForbiddenKeepsItsText(t *testing.T) { home := withHome(t) - server := loginTestServer(t, http.StatusUnauthorized, `{"error_code":"UNAUTHENTICATED","message":"Invalid access token."}`) - writeHomeConfig(t, home, "[logfood]\nhost = "+server.URL+"\nauth_type = pat\ntoken = dapi-expired\n") + server := loginTestServer(t, http.StatusForbidden, `{"error_code":"PERMISSION_DENIED","message":"no access"}`) + writeHomeConfig(t, home, "[logfood]\nhost = "+server.URL+"\nauth_type = pat\ntoken = dapi-logfood\n") + + parent := newMCPTestRoot() + parent.SetOut(&bytes.Buffer{}) + parent.SetErr(&bytes.Buffer{}) + parent.SetArgs([]string{"mcp", "logout", "system.ai.github", "--profile", "logfood"}) + ctx, _ := cmdio.NewTestContextWithStderr(telemetry.WithNewLogger(t.Context())) + err := parent.ExecuteContext(ctx) + logout, _, findErr := parent.Find([]string{"mcp", "logout"}) + require.NoError(t, findErr) - out := mcpSignInOutput{Service: "system.ai.github"} - err := runMcpSignIn(t.Context(), io.Discard, "logfood", "system.ai.github", mcpLogin, &out) require.Error(t, err) - require.NotEmpty(t, out.Host, "the client exists, so the profile's config is known") - enriched := auth.EnrichAuthError(t.Context(), &config.Config{Host: out.Host, Profile: "logfood"}, err) - assert.Equal(t, err.Error(), enriched.Error()) - assert.Equal(t, 0, strings.Count(enriched.Error(), "Next steps:")) + assert.Contains(t, err.Error(), "no access") + assert.NotContains(t, err.Error(), "Next steps") + assert.False(t, cmdctx.HasConfigUsed(logout.Context()), "root does not enrich the error") + cfg, uploads := telemetry.UploadConfig(logout.Context()) + require.True(t, uploads) + assert.Equal(t, "logfood", cfg.Profile) } // TestMcpLogoutUnknownServiceIsAnError checks that logout of a service the profile cannot see fails, rather than diff --git a/cmd/aitools/mcp_remove.go b/cmd/aitools/mcp_remove.go index 3de8322b403..2af23712d6e 100644 --- a/cmd/aitools/mcp_remove.go +++ b/cmd/aitools/mcp_remove.go @@ -12,9 +12,10 @@ import ( "github.com/databricks/cli/libs/aitools/agents" "github.com/databricks/cli/libs/aitools/mcp" "github.com/databricks/cli/libs/aitools/mcp/agentconfig" + "github.com/databricks/cli/libs/aitools/mcpconfig" "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/log" - "github.com/databricks/cli/libs/telemetry" "github.com/databricks/cli/libs/telemetry/protos" "github.com/spf13/cobra" ) @@ -39,6 +40,9 @@ type removeOutput struct { results []protos.AitoolsMcpServerResult // stateBefore is the state as it was read before the removal. It is not in the JSON. stateBefore *mcp.State + // profile is the profile that the selected servers are bound to, when they all are bound to one. It is "" otherwise. + // It is not in the JSON. + profile string } type removeRemoved struct { @@ -108,7 +112,7 @@ Entries that run 'aitools mcp serve' without an install record are not chosen by if jsonMode { err = writeRemoveJSON(cmd, out, runErr) } - logRemoveEvent(cmd, args, out) + logRemoveEvent(ctx, out) return err }, } @@ -135,33 +139,31 @@ func writeRemoveJSON(cmd *cobra.Command, out *removeOutput, runErr error) error return nil } -// logRemoveEvent buffers the REMOVE event and sets the config its exit-time upload authenticates with. It runs after -// the output is written, and only when the run touched a server: a run that touched none has nothing to upload, so it -// resolves no auth. The auth resolves as install's does (startTelemetryAuth), within telemetryAuthBudget, and the upload -// is skipped when a server recorded for the profile no longer matches it (bindingDrifted). -func logRemoveEvent(cmd *cobra.Command, args []string, out *removeOutput) { - ctx := cmd.Context() - logMcpEvent(ctx, protos.AitoolsMcpEvent{ +// logRemoveEvent buffers the REMOVE event and sets the config its exit-time upload authenticates with. It runs after the +// output is written. A run that logged no event touched no server, so it has nothing to upload and sets no config. +func logRemoveEvent(ctx context.Context, out *removeOutput) { + if !logMcpEvent(ctx, protos.AitoolsMcpEvent{ CallingAgent: callingAgentType(), Operation: protos.AitoolsMcpOperationRemove, Servers: out.results, - }) - if len(telemetry.BufferedEvents(ctx)) == 0 { + }) { return } - auth := startTelemetryAuth(cmd, args) - if auth == nil { + if out.profile == "" { return } - if !auth.wait() { - log.Debugf(ctx, "auth for telemetry not resolved within %s; not uploading telemetry", telemetryAuthBudget) + configFile, err := mcp.ResolveConfigFile(ctx, env.Get(ctx, mcpconfig.ConfigFileEnv)) + if err != nil { + log.Debugf(ctx, "telemetry: cannot resolve the config file to upload with: %s", err) return } - if auth.cfg != nil && bindingDrifted(ctx, out.stateBefore, auth.cfg) { - log.Debugf(ctx, "telemetry: profile %s no longer matches an MCP server recorded for it; not uploading telemetry", auth.cfg.Profile) + // The upload sends the profile's token to the profile's host, so a profile whose recorded servers no longer match it + // uploads nothing. The records are the ones read before the removal. + if bindingDrifted(ctx, out.stateBefore, out.profile, configFile) { + log.Debugf(ctx, "telemetry: profile %s no longer matches an MCP server recorded for it; not uploading telemetry", out.profile) return } - auth.apply(cmd, false) + setUploadConfig(ctx, out.profile, configFile) } // runMcpRemove removes the chosen servers from the agent configs and records the result. Results are printed @@ -230,6 +232,7 @@ func runMcpRemove(ctx context.Context, cmd *cobra.Command, f mcpRemoveFlags, jso // --agents chooses the servers installed for those agents, so a server with no agent left is not chosen. selected = slices.DeleteFunc(selected, func(e mcp.Entry) bool { return len(e.Agents) == 0 }) } + out.profile = sharedProfile(selected) // Each agent config file is read once per command, and each server's config file is resolved once. files := newAgentFiles() @@ -256,6 +259,10 @@ func runMcpRemove(ctx context.Context, cmd *cobra.Command, f mcpRemoveFlags, jso continue } outcome, reason, err := removeFromAgent(ctx, files, e, a, pin) + // A pruned entry changes the state only, so a save that fails undoes it, as for a record-only removal. + if outcome == removePruned && err == nil { + recordOnly = append(recordOnly, len(out.results)) + } out.results = append(out.results, removeServerResult(e.Service, agentName, outcome, err)) if err != nil { errs = append(errs, err) diff --git a/cmd/aitools/mcp_remove_test.go b/cmd/aitools/mcp_remove_test.go index 180ec8cacb8..89e3e9a6234 100644 --- a/cmd/aitools/mcp_remove_test.go +++ b/cmd/aitools/mcp_remove_test.go @@ -11,19 +11,16 @@ import ( "path/filepath" "slices" "testing" - "time" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/aitools/agents" "github.com/databricks/cli/libs/aitools/mcp" - "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/log/handler" "github.com/databricks/cli/libs/telemetry" "github.com/databricks/cli/libs/telemetry/protos" - "github.com/databricks/databricks-sdk-go/config" "github.com/spf13/cobra" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -807,7 +804,8 @@ func claudeServerEntry(claudePath, name, service string) mcp.Entry { func runRemoveWithTelemetry(t *testing.T, stdout, logs *bytes.Buffer, logLevel slog.Level, args ...string) (*cobra.Command, error) { t.Helper() parent := newMCPTestRoot() - ctx := env.Set(telemetry.WithNewLogger(t.Context()), "DATABRICKS_CLI_DISABLE_TELEMETRY", "") + ctx := telemetry.WithNewLogger(t.Context()) + ctx = env.Set(ctx, "DATABRICKS_CLI_DISABLE_TELEMETRY", "") ctx = log.NewContext(ctx, slog.New(handler.NewFriendlyHandler(logs, &handler.Options{Level: logLevel}))) ctx, _ = cmdio.NewTestContextWithStderr(ctx) parent.SetOut(stdout) @@ -819,25 +817,27 @@ func runRemoveWithTelemetry(t *testing.T, stdout, logs *bytes.Buffer, logLevel s return remove, err } -// TestMcpRemoveResolvesNothingWithoutAnEvent checks that a run that removes nothing logs no event, so it resolves no auth -// for an upload. -func TestMcpRemoveResolvesNothingWithoutAnEvent(t *testing.T) { - withHome(t) - resolves := 0 - setTelemetryAuth(t, time.Second, func(*cobra.Command, []string) error { - resolves++ - return nil - }) +// unsetConfigFile makes DATABRICKS_CONFIG_FILE unset for the test, as opposed to set to "". +func unsetConfigFile(t *testing.T) { + t.Setenv("DATABRICKS_CONFIG_FILE", "") + require.NoError(t, os.Unsetenv("DATABRICKS_CONFIG_FILE")) +} +// TestMcpRemoveSetsNoUploadWithoutAnEvent checks that a run that removes nothing logs no event, so it sets no config for +// an upload. +func TestMcpRemoveSetsNoUploadWithoutAnEvent(t *testing.T) { + withHome(t) var stdout, logs bytes.Buffer - _, err := runRemoveWithTelemetry(t, &stdout, &logs, log.LevelDebug, "--all", "--output", "json") + remove, err := runRemoveWithTelemetry(t, &stdout, &logs, log.LevelDebug, "--all", "--output", "json") require.NoError(t, err) - assert.Zero(t, resolves) + _, uploads := telemetry.UploadConfig(remove.Context()) + assert.False(t, uploads) + assert.Empty(t, telemetry.BufferedEvents(remove.Context())) } // TestMcpRemoveUploadSkipsDriftedProfile checks that the upload is not authenticated with a profile that a server recorded // for it no longer matches, as its host here is not the one the server was set up against. The removal is done either -// way, and the output is written before the auth resolves. +// way. func TestMcpRemoveUploadSkipsDriftedProfile(t *testing.T) { home := withHome(t) writeHomeConfig(t, home, "[logfood]\nhost = https://adb-2.test\nauth_type = pat\ntoken = t\n") @@ -848,57 +848,65 @@ func TestMcpRemoveUploadSkipsDriftedProfile(t *testing.T) { saveState(t, entry) var stdout, logs bytes.Buffer - var outputWrittenFirst bool - setTelemetryAuth(t, time.Second, func(cmd *cobra.Command, _ []string) error { - outputWrittenFirst = stdout.Len() > 0 - cmd.SetContext(cmdctx.SetConfigUsed(cmd.Context(), &config.Config{Profile: "logfood", Host: "https://adb-2.test", ConfigFile: filepath.Join(home, ".databrickscfg")})) - return nil - }) - remove, err := runRemoveWithTelemetry(t, &stdout, &logs, log.LevelDebug, "--names", "genie_one_mcp-logfood", "--output", "json") require.NoError(t, err) - assert.True(t, outputWrittenFirst, "the auth resolves after the JSON is written") - assert.False(t, cmdctx.HasConfigUsed(remove.Context()), "a drifted profile is not used to upload") + require.Len(t, telemetry.BufferedEvents(remove.Context()), 1, "the removal is logged") + _, uploads := telemetry.UploadConfig(remove.Context()) + assert.False(t, uploads, "a drifted profile is not used to upload") assert.NotContains(t, readJSONFile(t, claudePath)["mcpServers"], "genie_one_mcp-logfood", "the removal is done") } -// TestMcpRemoveUploadUsesMatchingProfile checks that a profile whose recorded servers still match is used for the upload. +// TestMcpRemoveUploadUsesMatchingProfile checks that a profile whose recorded servers still match is used for the upload, +// read from the default config file when DATABRICKS_CONFIG_FILE is unset. func TestMcpRemoveUploadUsesMatchingProfile(t *testing.T) { home := withHome(t) + unsetConfigFile(t) writeHomeConfig(t, home, "[logfood]\nhost = https://adb-1.test\nauth_type = pat\ntoken = t\n") claude := agents.ByName("claude-code") entry := seedInstall(t, claude, "genie_one_mcp-logfood", "system.ai.genie_one_mcp", "logfood", "https://adb-1.test") saveState(t, entry) - setTelemetryAuth(t, time.Second, func(cmd *cobra.Command, _ []string) error { - cmd.SetContext(cmdctx.SetConfigUsed(cmd.Context(), &config.Config{Profile: "logfood", Host: "https://adb-1.test", ConfigFile: filepath.Join(home, ".databrickscfg")})) - return nil - }) - var stdout, logs bytes.Buffer remove, err := runRemoveWithTelemetry(t, &stdout, &logs, log.LevelDebug, "--names", "genie_one_mcp-logfood", "--output", "json") require.NoError(t, err) - require.True(t, cmdctx.HasConfigUsed(remove.Context())) - assert.Equal(t, "https://adb-1.test", cmdctx.ConfigUsed(remove.Context()).Host) + cfg, uploads := telemetry.UploadConfig(remove.Context()) + require.True(t, uploads) + assert.Equal(t, "logfood", cfg.Profile) + assert.Equal(t, "https://adb-1.test", cfg.Host) + assert.Equal(t, "t", cfg.Token) } -// TestMcpRemoveOfflineResolutionWarnsNothing checks that an auth resolution that fails, as it does offline, leaves nothing -// in the output: its logs are debug only. -func TestMcpRemoveOfflineResolutionWarnsNothing(t *testing.T) { +// TestMcpRemoveUploadIgnoresEnvironmentToken checks that the upload authenticates with the profile's own token, never with +// a DATABRICKS_TOKEN of another workspace in the environment. +func TestMcpRemoveUploadIgnoresEnvironmentToken(t *testing.T) { home := withHome(t) + unsetConfigFile(t) + t.Setenv("DATABRICKS_TOKEN", "token-of-another-workspace") writeHomeConfig(t, home, "[logfood]\nhost = https://adb-1.test\nauth_type = pat\ntoken = t\n") claude := agents.ByName("claude-code") saveState(t, seedInstall(t, claude, "genie_one_mcp-logfood", "system.ai.genie_one_mcp", "logfood", "https://adb-1.test")) - setTelemetryAuth(t, time.Second, func(cmd *cobra.Command, _ []string) error { - log.Warnf(cmd.Context(), "[hostmetadata] failed to fetch host metadata for https://adb-1.test") - return errors.New("cannot reach https://adb-1.test") - }) + var stdout, logs bytes.Buffer + remove, err := runRemoveWithTelemetry(t, &stdout, &logs, log.LevelDebug, "--names", "genie_one_mcp-logfood", "--output", "json") + require.NoError(t, err) + cfg, uploads := telemetry.UploadConfig(remove.Context()) + require.True(t, uploads) + assert.Equal(t, "t", cfg.Token) +} + +// TestMcpRemoveOfflineSetsUploadWithoutWarning checks that a remove that cannot reach the workspace warns nothing: the +// upload's config is resolved without a request, so there is no failure to report. +func TestMcpRemoveOfflineSetsUploadWithoutWarning(t *testing.T) { + home := withHome(t) + writeHomeConfig(t, home, "[logfood]\nhost = https://adb-1.test\nauth_type = pat\ntoken = t\n") + claude := agents.ByName("claude-code") + saveState(t, seedInstall(t, claude, "genie_one_mcp-logfood", "system.ai.genie_one_mcp", "logfood", "https://adb-1.test")) var stdout, logs bytes.Buffer remove, err := runRemoveWithTelemetry(t, &stdout, &logs, log.LevelWarn, "--names", "genie_one_mcp-logfood", "--output", "json") require.NoError(t, err) - assert.False(t, cmdctx.HasConfigUsed(remove.Context())) + _, uploads := telemetry.UploadConfig(remove.Context()) + assert.True(t, uploads) assert.Empty(t, logs.String()) assert.Contains(t, stdout.String(), `"removed"`) } @@ -914,7 +922,6 @@ func TestMcpRemoveRecordOnlyReportsFailedWhenStateIsNotSaved(t *testing.T) { saveState(t, retired) saveMCPState = func(context.Context, *mcp.State) error { return errors.New("state file is read-only") } t.Cleanup(func() { saveMCPState = mcp.SaveState }) - setTelemetryAuth(t, time.Second, func(*cobra.Command, []string) error { return nil }) var stdout, logs bytes.Buffer remove, err := runRemoveWithTelemetry(t, &stdout, &logs, log.LevelDebug, "--names", "dbsql-logfood", "--output", "json") @@ -922,6 +929,7 @@ func TestMcpRemoveRecordOnlyReportsFailedWhenStateIsNotSaved(t *testing.T) { events := telemetry.BufferedEvents(remove.Context()) require.Len(t, events, 1) + require.NotNil(t, events[0].AitoolsMcpEvent) require.Len(t, events[0].AitoolsMcpEvent.Servers, 1) assert.Equal(t, protos.AitoolsTargetOutcomeFailed, events[0].AitoolsMcpEvent.Servers[0].Outcome) assert.Equal(t, protos.AitoolsErrorCategoryUncategorized, events[0].AitoolsMcpEvent.Servers[0].ErrorCategory) diff --git a/cmd/aitools/mcp_serve.go b/cmd/aitools/mcp_serve.go index eda41fbb6a0..8d83de2613b 100644 --- a/cmd/aitools/mcp_serve.go +++ b/cmd/aitools/mcp_serve.go @@ -24,7 +24,7 @@ import ( "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/auth/u2m" "github.com/databricks/cli/libs/browser" - "github.com/databricks/cli/libs/cmdctx" + "github.com/databricks/cli/libs/telemetry" "github.com/databricks/cli/libs/telemetry/protos" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/config" @@ -147,12 +147,10 @@ func runMcpServe(cmd *cobra.Command, f mcpServeFlags) error { // serveChallenge fails only on an auth failure (see lookupAtStart), which exits serveBindStatus. return serveFailure(cmd, err, serveBindStatus) } - // Only a serving start is logged. The upload authenticates with a config of its own that never refreshes a token - // (see setUploadConfig), since the session's config refreshes its token. - setUploadConfig(cmd, profileName, cfg.ConfigFile) + // Only a serving start is logged. logMcpEvent(cmd.Context(), proxyStartedEvent(agent, f.service)) - return serveProxy(ctx, cmd, refresh, proxy.Options{ + err = serveProxy(ctx, cmd, refresh, proxy.Options{ In: cmd.InOrStdin(), Out: cmd.OutOrStdout(), Err: cmd.ErrOrStderr(), @@ -163,6 +161,10 @@ func runMcpServe(cmd *cobra.Command, f mcpServeFlags) error { Transport: serveTransport(cfg), Challenge: challenge, }) + // The upload authenticates with the profile at exit, with a config of its own that never refreshes a token, since the + // session's config refreshes its token. A refresh failure has refused the upload (proxyResult), so nothing is set then. + setUploadConfig(cmd.Context(), profileName, cfg.ConfigFile) + return err } // guardedClient returns the client of the bound profile whose requests the refresh guard authenticates. The SDK calls @@ -301,7 +303,7 @@ func serveProxy(ctx context.Context, cmd *cobra.Command, refresh *refreshGuard, func proxyResult(cmd *cobra.Command, refresh *refreshGuard, runErr error) error { if refreshErr := refresh.failure(); refreshErr != nil { // The token was rejected, so the upload at exit would only ask the token endpoint again. - cmd.SetContext(cmdctx.ClearConfigUsed(cmd.Context())) + telemetry.RefuseUpload(cmd.Context()) fmt.Fprintf(cmd.ErrOrStderr(), "stopped after a token refresh failure for profile %s\n", refresh.profile) return exitAlreadyPrinted(refreshErr, serveBindStatus) } diff --git a/cmd/aitools/mcp_serve_test.go b/cmd/aitools/mcp_serve_test.go index 2ecedd90aa1..1c2a14dd6d9 100644 --- a/cmd/aitools/mcp_serve_test.go +++ b/cmd/aitools/mcp_serve_test.go @@ -25,7 +25,6 @@ import ( "github.com/databricks/cli/libs/aitools/mcp" "github.com/databricks/cli/libs/aitools/mcp/proxy" "github.com/databricks/cli/libs/aitools/mcpconfig" - "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/telemetry" @@ -403,14 +402,16 @@ func TestProxyResultRefreshFailureWinsOverStdinError(t *testing.T) { } require.Error(t, g.authenticateRequest(&http.Request{Header: http.Header{}})) cmd := NewMcpServeCmd() - cmd.SetContext(cmdctx.SetConfigUsed(t.Context(), &config.Config{Host: "https://adb-1.test"})) + cmd.SetContext(telemetry.WithNewLogger(t.Context())) + telemetry.SetUploadConfig(cmd.Context(), &config.Config{Host: "https://adb-1.test"}) var stderr bytes.Buffer cmd.SetErr(&stderr) err := proxyResult(cmd, g, errors.New("read stdin: input/output error")) - // The exit-time upload would ask the rejected token's endpoint again, so it has no config. - assert.False(t, cmdctx.HasConfigUsed(cmd.Context())) + // The exit-time upload would ask the rejected token's endpoint again, so the run uploads nothing. + _, uploads := telemetry.UploadConfig(cmd.Context()) + assert.False(t, uploads) assert.ErrorIs(t, err, root.ErrAlreadyPrinted) assert.Equal(t, serveBindStatus, root.ExitStatus(err)) assert.ErrorIs(t, err, cause) diff --git a/cmd/aitools/mcp_telemetry.go b/cmd/aitools/mcp_telemetry.go index 1ccd734dacb..9e0492aed1e 100644 --- a/cmd/aitools/mcp_telemetry.go +++ b/cmd/aitools/mcp_telemetry.go @@ -7,25 +7,25 @@ import ( "github.com/databricks/cli/libs/aitools/agents" "github.com/databricks/cli/libs/aitools/mcp" - "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/telemetry" "github.com/databricks/cli/libs/telemetry/protos" - "github.com/databricks/databricks-sdk-go/config" - "github.com/spf13/cobra" ) -// setUploadConfig sets the config of profileName in configFile for the exit-time upload, resolved within -// telemetryAuthBudget, with credentials that never refresh (mcp.BoundUploadConfig). A profile that does not resolve -// uploads nothing. -func setUploadConfig(cmd *cobra.Command, profileName, configFile string) { - ctx := cmd.Context() +// setUploadConfig sets the config of profileName in configFile for the run's telemetry upload: resolved within +// telemetryAuthBudget, with credentials that never refresh (mcp.BoundUploadConfig). A command calls it once its binding +// checks have passed, so the upload authenticates with the profile the command bound. Nothing is resolved when telemetry +// is disabled, and a profile that does not resolve uploads nothing. +func setUploadConfig(ctx context.Context, profileName, configFile string) { + if telemetry.Disabled(ctx) { + return + } cfg, err := mcp.BoundUploadConfig(profileName, configFile, telemetryAuthBudget) if err != nil { log.Debugf(ctx, "telemetry: cannot resolve profile %s to upload with: %s", profileName, err) return } - cmd.SetContext(cmdctx.SetConfigUsed(ctx, cfg)) + telemetry.SetUploadConfig(ctx, cfg) } // mcpServiceTypes maps the system.ai leaves in the CLI's service table to their telemetry enum. @@ -75,28 +75,29 @@ func proxyAgentType(flag string) protos.AitoolsAgentType { return agentType(flag) } -// logMcpEvent buffers an MCP event; cmd/root uploads it at exit. An event without server results -// is not logged: a run that stopped before any server was reached has nothing to report. An event -// that names a rejected credential is not logged either: its upload would authenticate with that -// credential and be rejected the same way. The bound profile count is read from the state file as -// it is after the operation. A state file that cannot be read drops the event; telemetry never -// fails a command. -func logMcpEvent(ctx context.Context, event protos.AitoolsMcpEvent) { +// logMcpEvent buffers an MCP event, and reports whether it did; cmd/root uploads it at exit. An event without server +// results is not logged: a run that stopped before any server was reached has nothing to report. An event that names a +// rejected credential is not logged either, and the run refuses its upload, since the upload would present that +// credential again. The bound profile count is read from the state file as it is after the operation. A state file that +// cannot be read drops the event; telemetry never fails a command. +func logMcpEvent(ctx context.Context, event protos.AitoolsMcpEvent) bool { if len(event.Servers) == 0 { - return + return false } if slices.ContainsFunc(event.Servers, func(r protos.AitoolsMcpServerResult) bool { return r.ErrorCategory == protos.AitoolsErrorCategoryAuthFailed }) { - return + telemetry.RefuseUpload(ctx) + return false } state, err := mcp.LoadState(ctx) if err != nil { log.Debugf(ctx, "telemetry: cannot read the MCP state; dropping the event: %s", err) - return + return false } event.BoundProfileCount = boundProfileCount(ctx, state) telemetry.Log(ctx, protos.DatabricksCliLog{AitoolsMcpEvent: &event}) + return true } // boundProfileCount returns the number of distinct profiles that have servers installed. A profile is its name in its @@ -151,18 +152,18 @@ func addServerResults(servers []addServerOutput) []protos.AitoolsMcpServerResult return results } -// removeServerResult returns the result of removing one server from one agent config. An entry that -// is gone already, or that was changed outside this CLI, is SKIPPED: nothing was changed there. +// removeServerResult returns the result of removing one server from one agent config. An entry that was deleted, or +// pruned (its record dropped), is CHANGED. An entry that was changed outside this CLI is SKIPPED: it is left in place. func removeServerResult(service, agent string, outcome removeOutcome, err error) protos.AitoolsMcpServerResult { r := protos.AitoolsMcpServerResult{Service: mcpServiceType(service), Agent: agentType(agent)} switch { case err != nil: r.Outcome = protos.AitoolsTargetOutcomeFailed r.ErrorCategory = errorCategory(err) - case outcome == removeDeleted: - r.Outcome = protos.AitoolsTargetOutcomeChanged - default: + case outcome == removeKept: r.Outcome = protos.AitoolsTargetOutcomeSkipped + default: + r.Outcome = protos.AitoolsTargetOutcomeChanged } return r } @@ -265,17 +266,30 @@ func signInPromptedEvent(agent protos.AitoolsAgentType, service string, signedIn } } -// bindingDrifted reports whether a server recorded for cfg's profile in state no longer matches the profile: its host or -// workspace changed, or its config file moved. The upload would send the profile's token to the host the profile names -// now, which the server was not set up against. It reads the config files only. -func bindingDrifted(ctx context.Context, state *mcp.State, cfg *config.Config) bool { +// bindingDrifted reports whether a server recorded for profileName in state no longer matches the profile in configFile: +// its host or workspace changed, or its config file moved. The upload would send the profile's token to the host the +// profile names now, which the server was not set up against. It reads the config files only. +func bindingDrifted(ctx context.Context, state *mcp.State, profileName, configFile string) bool { for _, e := range state.Servers { - if e.Profile != cfg.Profile { + if e.Profile != profileName { continue } - if _, err := mcp.CheckBinding(ctx, state, e.Service, cfg.Profile, cfg.ConfigFile); err != nil { + if _, err := mcp.CheckBinding(ctx, state, e.Service, profileName, configFile); err != nil { return true } } return false } + +// sharedProfile returns the profile that all entries are bound to, or "" when they are bound to none or to several. +func sharedProfile(entries []mcp.Entry) string { + if len(entries) == 0 { + return "" + } + for _, e := range entries[1:] { + if e.Profile != entries[0].Profile { + return "" + } + } + return entries[0].Profile +} diff --git a/cmd/aitools/mcp_telemetry_test.go b/cmd/aitools/mcp_telemetry_test.go index deef70fd421..742a7971348 100644 --- a/cmd/aitools/mcp_telemetry_test.go +++ b/cmd/aitools/mcp_telemetry_test.go @@ -104,8 +104,8 @@ func TestAddServerResults(t *testing.T) { func TestRemoveServerResult(t *testing.T) { assert.Equal(t, protos.AitoolsTargetOutcomeChanged, removeServerResult("system.ai.github", "claude-code", removeDeleted, nil).Outcome) - // An entry already gone, or changed outside this CLI, is left as it is. - assert.Equal(t, protos.AitoolsTargetOutcomeSkipped, + // A pruned entry changes the state, so it is CHANGED too. An entry changed outside this CLI is left in place: SKIPPED. + assert.Equal(t, protos.AitoolsTargetOutcomeChanged, removeServerResult("system.ai.github", "claude-code", removePruned, nil).Outcome) assert.Equal(t, protos.AitoolsTargetOutcomeSkipped, removeServerResult("system.ai.github", "claude-code", removeKept, nil).Outcome) diff --git a/libs/aitools/mcp/binding.go b/libs/aitools/mcp/binding.go index 6d4f49c9f2d..1eb8f06db1c 100644 --- a/libs/aitools/mcp/binding.go +++ b/libs/aitools/mcp/binding.go @@ -60,12 +60,6 @@ func (e *BindError) Unwrap() error { return e.cause } -// NextStepsIncluded reports whether the message already names the next steps: an AUTH_FAILED message names the login -// command for the profile. The root command does not add a second set of steps to it. -func (e *BindError) NextStepsIncluded() bool { - return e.Category == BindAuthFailed -} - // Bind checks that the profile still names the host and workspace recorded for the service, // then authenticates with the profile's own credentials. It never reads stdin, prompts, or // loads a bundle. It returns the resolved config and the matching state entry, which is nil diff --git a/libs/aitools/mcp/binding_test.go b/libs/aitools/mcp/binding_test.go index 3f834451c17..5be3fd88932 100644 --- a/libs/aitools/mcp/binding_test.go +++ b/libs/aitools/mcp/binding_test.go @@ -951,12 +951,3 @@ func TestBoundUploadConfigNeverRefreshes(t *testing.T) { require.NoError(t, err) assert.NotEqual(t, auth.CredentialsWithoutRefresh(), session.Credentials) } - -// TestBindErrorNextStepsIncludedOnlyForAuthFailed checks that only an AUTH_FAILED message names the next steps, so root -// adds no second set of steps to the others. -func TestBindErrorNextStepsIncludedOnlyForAuthFailed(t *testing.T) { - assert.True(t, (&mcp.BindError{Category: mcp.BindAuthFailed}).NextStepsIncluded()) - for _, category := range []mcp.BindCategory{mcp.BindProfileNotFound, mcp.BindProfileHostChanged, mcp.BindUncategorized} { - assert.False(t, (&mcp.BindError{Category: category}).NextStepsIncluded(), category) - } -} diff --git a/libs/auth/error.go b/libs/auth/error.go index 5e5b6369c24..fdb5f256f43 100644 --- a/libs/auth/error.go +++ b/libs/auth/error.go @@ -82,13 +82,6 @@ func RewriteAuthError(ctx context.Context, host, accountId, profile string, err return false, err } -// nextStepsIncluded is implemented by an error whose message already gives the next steps for the failure, such as the -// login command of an MCP server. EnrichAuthError leaves such an error as it is, so the user is given one set of steps. -type nextStepsIncluded interface { - error - NextStepsIncluded() bool -} - // EnrichAuthError appends identity context and remediation steps to 401/403 API errors. // For non-API errors or other status codes, the original error is returned unchanged. func EnrichAuthError(ctx context.Context, cfg *config.Config, err error) error { @@ -99,9 +92,6 @@ func EnrichAuthError(ctx context.Context, cfg *config.Config, err error) error { if apiErr.StatusCode != http.StatusUnauthorized && apiErr.StatusCode != http.StatusForbidden { return err } - if steps, ok := errors.AsType[nextStepsIncluded](err); ok && steps.NextStepsIncluded() { - return err - } var b strings.Builder diff --git a/libs/auth/error_test.go b/libs/auth/error_test.go index 2ef1ff08ff5..ce8f0df1b63 100644 --- a/libs/auth/error_test.go +++ b/libs/auth/error_test.go @@ -63,33 +63,6 @@ func TestEnrichAuthError_PreservesOriginalError(t *testing.T) { assert.Equal(t, "PERMISSION_DENIED", unwrapped.ErrorCode) } -// stepsIncludedError is an authentication failure whose message already gives the next steps. -type stepsIncludedError struct { - cause error -} - -func (e *stepsIncludedError) Error() string { - return "credential rejected. Run: databricks auth login --profile test" -} - -func (e *stepsIncludedError) Unwrap() error { - return e.cause -} - -func (e *stepsIncludedError) NextStepsIncluded() bool { - return true -} - -func TestEnrichAuthErrorLeavesErrorWithNextStepsAsIs(t *testing.T) { - cfg := &config.Config{Profile: "test", Host: "https://ws.test", AuthType: "pat"} - original := &stepsIncludedError{cause: &apierr.APIError{StatusCode: 401, Message: "Unauthorized"}} - assert.Equal(t, original, EnrichAuthError(t.Context(), cfg, original)) - - // An API error without a message of its own is still enriched. - plain := &apierr.APIError{StatusCode: 401, Message: "Unauthorized"} - assert.Contains(t, EnrichAuthError(t.Context(), cfg, plain).Error(), "Next steps:") -} - func TestEnrichAuthError(t *testing.T) { tests := []struct { name string diff --git a/libs/cmdctx/config_used.go b/libs/cmdctx/config_used.go index ed20650aea1..40595ef8aa9 100644 --- a/libs/cmdctx/config_used.go +++ b/libs/cmdctx/config_used.go @@ -18,12 +18,6 @@ func ConfigUsed(ctx context.Context) *config.Config { return cfg } -// ClearConfigUsed returns a context with no config used, as if SetConfigUsed had not run: the telemetry upload is -// skipped and auth errors are not enriched. -func ClearConfigUsed(ctx context.Context) context.Context { - return context.WithValue(ctx, configUsedKey, nil) -} - func HasConfigUsed(ctx context.Context) bool { _, ok := ctx.Value(configUsedKey).(*config.Config) return ok diff --git a/libs/cmdctx/config_used_test.go b/libs/cmdctx/config_used_test.go index 8a0724ceb76..f99febf8046 100644 --- a/libs/cmdctx/config_used_test.go +++ b/libs/cmdctx/config_used_test.go @@ -27,10 +27,6 @@ func TestCommandConfigUsed(t *testing.T) { // The config should have the correct configuration. assert.Equal(t, "https://test.test", ConfigUsed(ctx).Host) - // Clearing the config leaves none, as if it had never been set. - assert.False(t, HasConfigUsed(ClearConfigUsed(ctx))) - assert.True(t, HasConfigUsed(ctx)) - // Second call should update the config used. cfg2 := &config.Config{ Host: "https://test2.test", From a40ef5cf34065ce9d2ec49cea9632e224aa3599a Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Fri, 13 Nov 2026 16:21:25 +0000 Subject: [PATCH 18/22] Record sign-in prompts of waits on a cancelled page, and test the refusal and removal outcomes An authenticate call that waits on the page an earlier cancelled call showed records its outcome as a prompt too, so a sign-in that completes there is ACTIVE rather than PENDING. The proto mirror says a retry reports its own outcome. Tests cover the upload refusal after a rejected credential, a pruned entry reported CHANGED (FAILED when the state is not saved), and a failed state read after another event. Co-authored-by: Isaac --- cmd/aitools/mcp_remove_test.go | 58 ++++++++++++++++++++++++ cmd/aitools/mcp_telemetry_test.go | 27 +++++++---- libs/aitools/mcp/proxy/challenge.go | 7 +-- libs/aitools/mcp/proxy/challenge_test.go | 52 +++++++++++++++++++-- libs/telemetry/protos/aitools_mcp.go | 12 +++-- 5 files changed, 135 insertions(+), 21 deletions(-) diff --git a/cmd/aitools/mcp_remove_test.go b/cmd/aitools/mcp_remove_test.go index 89e3e9a6234..f3b94c05af7 100644 --- a/cmd/aitools/mcp_remove_test.go +++ b/cmd/aitools/mcp_remove_test.go @@ -14,6 +14,7 @@ import ( "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/aitools/agents" + "github.com/databricks/cli/libs/aitools/installer" "github.com/databricks/cli/libs/aitools/mcp" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/env" @@ -911,6 +912,63 @@ func TestMcpRemoveOfflineSetsUploadWithoutWarning(t *testing.T) { assert.Contains(t, stdout.String(), `"removed"`) } +// TestMcpRemovePrunedEntryReportsChanged checks that an entry the agent config no longer holds is reported CHANGED, since +// its record is dropped, and FAILED when the state cannot be saved, as a record-only removal is. +func TestMcpRemovePrunedEntryReportsChanged(t *testing.T) { + home := withHome(t) + claude := agents.ByName("claude-code") + claudePath := filepath.Join(home, ".claude.json") + seedPruned := func() { + saveState(t, seedInstall(t, claude, "genie_one_mcp-logfood", "system.ai.genie_one_mcp", "logfood", "https://adb-1.test")) + require.NoError(t, os.WriteFile(claudePath, []byte(`{"mcpServers":{}}`), 0o600)) + } + + seedPruned() + var stdout, logs bytes.Buffer + remove, err := runRemoveWithTelemetry(t, &stdout, &logs, log.LevelDebug, "--names", "genie_one_mcp-logfood", "--output", "json") + require.NoError(t, err) + events := telemetry.BufferedEvents(remove.Context()) + require.Len(t, events, 1) + require.NotNil(t, events[0].AitoolsMcpEvent) + require.Len(t, events[0].AitoolsMcpEvent.Servers, 1) + assert.Equal(t, protos.AitoolsTargetOutcomeChanged, events[0].AitoolsMcpEvent.Servers[0].Outcome) + + seedPruned() + saveMCPState = func(context.Context, *mcp.State) error { return errors.New("state file is read-only") } + t.Cleanup(func() { saveMCPState = mcp.SaveState }) + stdout.Reset() + remove, err = runRemoveWithTelemetry(t, &stdout, &logs, log.LevelDebug, "--names", "genie_one_mcp-logfood", "--output", "json") + require.ErrorIs(t, err, root.ErrAlreadyPrinted) + events = telemetry.BufferedEvents(remove.Context()) + require.Len(t, events, 1) + require.NotNil(t, events[0].AitoolsMcpEvent) + require.Len(t, events[0].AitoolsMcpEvent.Servers, 1) + assert.Equal(t, protos.AitoolsTargetOutcomeFailed, events[0].AitoolsMcpEvent.Servers[0].Outcome) +} + +// TestMcpRemoveFailedStateReadAfterAnotherEventLogsNothing checks that a remove that cannot read the state logs nothing +// of its own, even when the run already buffered another event, and does not read the state it never read. +func TestMcpRemoveFailedStateReadAfterAnotherEventLogsNothing(t *testing.T) { + withHome(t) + dir, err := installer.GlobalSkillsDir(t.Context()) + require.NoError(t, err) + require.NoError(t, os.MkdirAll(filepath.Join(dir, ".state.json"), 0o700)) + + parent := newMCPTestRoot() + parent.SetOut(&bytes.Buffer{}) + parent.SetErr(&bytes.Buffer{}) + parent.SetArgs([]string{"mcp", "remove", "--all", "--output", "json"}) + ctx := telemetry.WithNewLogger(t.Context()) + telemetry.Log(ctx, protos.DatabricksCliLog{CliTestEvent: &protos.CliTestEvent{Name: protos.DummyCliEnumValue1}}) + ctx, _ = cmdio.NewTestContextWithStderr(ctx) + require.Error(t, parent.ExecuteContext(ctx)) + remove, _, findErr := parent.Find([]string{"mcp", "remove"}) + require.NoError(t, findErr) + events := telemetry.BufferedEvents(remove.Context()) + require.Len(t, events, 1) + assert.NotNil(t, events[0].CliTestEvent) +} + // TestMcpRemoveRecordOnlyReportsFailedWhenStateIsNotSaved checks that a removal that only drops state records is reported // FAILED when the state cannot be saved, since nothing was removed. func TestMcpRemoveRecordOnlyReportsFailedWhenStateIsNotSaved(t *testing.T) { diff --git a/cmd/aitools/mcp_telemetry_test.go b/cmd/aitools/mcp_telemetry_test.go index 742a7971348..df29d6c9237 100644 --- a/cmd/aitools/mcp_telemetry_test.go +++ b/cmd/aitools/mcp_telemetry_test.go @@ -12,6 +12,7 @@ import ( "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/telemetry" "github.com/databricks/cli/libs/telemetry/protos" + "github.com/databricks/databricks-sdk-go/config" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -185,30 +186,36 @@ func TestProxyStartedEvent(t *testing.T) { }, proxyStartedEvent(protos.AitoolsAgentTypeClaudeCode, "system.ai.genie_one_mcp")) } -// TestLogMcpEventDropsRejectedCredential checks that an event that names a rejected credential is not logged: its upload -// would authenticate with that credential again. +// TestLogMcpEventDropsRejectedCredential checks that an event that names a rejected credential is not logged, and that the +// run then uploads nothing: its upload would authenticate with that credential again. The events of the run that were +// logged before stay buffered, and a later event of the run is logged, but not uploaded. func TestLogMcpEventDropsRejectedCredential(t *testing.T) { withHome(t) ctx := env.Set(telemetry.WithNewLogger(t.Context()), "DATABRICKS_CLI_DISABLE_TELEMETRY", "") - logMcpEvent(ctx, protos.AitoolsMcpEvent{ + telemetry.SetUploadConfig(ctx, &config.Config{Profile: "logfood", Host: "https://adb-1.test"}) + assert.False(t, logMcpEvent(ctx, protos.AitoolsMcpEvent{ Operation: protos.AitoolsMcpOperationSignIn, Servers: []protos.AitoolsMcpServerResult{{ Service: protos.AitoolsMcpServiceGitHub, Outcome: protos.AitoolsTargetOutcomeFailed, ErrorCategory: protos.AitoolsErrorCategoryAuthFailed, }}, - }) + })) assert.Empty(t, telemetry.BufferedEvents(ctx)) + _, uploads := telemetry.UploadConfig(ctx) + assert.False(t, uploads) - logMcpEvent(ctx, protos.AitoolsMcpEvent{ - Operation: protos.AitoolsMcpOperationSignIn, + assert.True(t, logMcpEvent(ctx, protos.AitoolsMcpEvent{ + Operation: protos.AitoolsMcpOperationAdd, Servers: []protos.AitoolsMcpServerResult{{ - Service: protos.AitoolsMcpServiceGitHub, - Outcome: protos.AitoolsTargetOutcomeFailed, - ErrorCategory: protos.AitoolsErrorCategoryMCPPermissionDenied, + Service: protos.AitoolsMcpServiceGitHub, + Outcome: protos.AitoolsTargetOutcomeChanged, + SignInState: protos.AitoolsMcpSignInStatePending, }}, - }) + })) assert.Len(t, telemetry.BufferedEvents(ctx), 1) + _, uploads = telemetry.UploadConfig(ctx) + assert.False(t, uploads, "the add event is logged, and it is not uploaded with the rejected run") } func TestSignInPromptedEvent(t *testing.T) { diff --git a/libs/aitools/mcp/proxy/challenge.go b/libs/aitools/mcp/proxy/challenge.go index 889d9bbd4a4..d3cf58ba1c6 100644 --- a/libs/aitools/mcp/proxy/challenge.go +++ b/libs/aitools/mcp/proxy/challenge.go @@ -53,8 +53,9 @@ type SignInFunc func(ctx context.Context, req SignInRequest) (alreadySignedIn, u // Recorder receives the proxy's telemetry events. The command implements it, so the proxy does not // build telemetry events itself. type Recorder interface { - // SignInPrompted runs after the wait of an authenticate call that opened the sign-in page. - // signedIn reports whether the credential became ACTIVE. + // SignInPrompted runs after the wait of an authenticate call that showed the sign-in page: one it opened, or one an + // earlier cancelled call opened, which the user may still have open. signedIn reports whether the credential became + // ACTIVE. A retry after a cancelled call reports its own outcome. SignInPrompted(ctx context.Context, signedIn bool) } @@ -459,7 +460,7 @@ func (h *signInHooks) authenticate(ctx context.Context, id, progressToken json.R alreadySignedIn, urlShown, err := h.c.SignIn(ctx, SignInRequest{Open: open, Progress: report, Stderr: h.stderr}) // A prompt is a page the user was shown. A caller that was already signed in, or a sign-in that failed before its // URL was printed, prompted nobody. - if open && !alreadySignedIn && urlShown { + if !alreadySignedIn && urlShown { h.c.Recorder.SignInPrompted(ctx, err == nil) } h.pageCancelled = urlShown && err != nil && ctx.Err() != nil diff --git a/libs/aitools/mcp/proxy/challenge_test.go b/libs/aitools/mcp/proxy/challenge_test.go index 6b3ea77cac7..4af63aeeddd 100644 --- a/libs/aitools/mcp/proxy/challenge_test.go +++ b/libs/aitools/mcp/proxy/challenge_test.go @@ -273,9 +273,8 @@ func TestAuthenticateSaysAlreadySignedInAfterSignIn(t *testing.T) { require.NoError(t, p.finish(t)) } -// TestAuthenticateRecordsPromptWhenPageOpens checks that the prompt is recorded once per page that was opened, after its -// wait, with whether the credential became ACTIVE. A call that opens no page, or finds the caller already signed in, -// records nothing. +// TestAuthenticateRecordsPromptWhenPageOpens checks that the prompt is recorded once per page that was shown, after its +// wait, with whether the credential became ACTIVE. A call that finds the caller already signed in records nothing. func TestAuthenticateRecordsPromptWhenPageOpens(t *testing.T) { srv, _ := challengedService(t) signIn := &signInStub{reply: func(call int) (bool, error) { @@ -330,6 +329,53 @@ func TestAuthenticateRecordsNoPromptWhenNoURLShown(t *testing.T) { require.NoError(t, p.finish(t)) } +// TestAuthenticateRecordsPromptOfCallThatWaitsOnCancelledPage checks that a call which waits on the page an earlier, +// cancelled call showed records its own outcome: the page was shown for it too, so a sign-in that completes there is +// recorded as ACTIVE, not left PENDING from the cancelled call. +func TestAuthenticateRecordsPromptOfCallThatWaitsOnCancelledPage(t *testing.T) { + srv, _ := newFakeEndpoint(t, answerWith(http.StatusOK, listChallenge)) + entered, done := make(chan struct{}), make(chan struct{}) + var mu sync.Mutex + var opens []bool + signIn := func(ctx context.Context, req SignInRequest) (bool, bool, error) { + mu.Lock() + opens = append(opens, req.Open) + first := len(opens) == 1 + mu.Unlock() + if !first { + return false, true, nil + } + close(entered) + <-ctx.Done() + close(done) + return false, true, ctx.Err() + } + recorder := &promptRecorder{} + c := testChallenge(signIn) + c.Recorder = recorder + b := bridgeFor(srv.URL) + b.Hooks = newSignInHooks(c, io.Discard) + p := startPiped(t, t.Context(), b) + authenticate := `{"jsonrpc":"2.0","id":%d,"method":"tools/call","params":{"name":"authenticate","arguments":{}}}` + + p.send(t, `{"jsonrpc":"2.0","id":2,"method":"tools/list"}`) + p.waitLines(t, 1) + p.send(t, fmt.Sprintf(authenticate, 5)) + waitClosed(t, entered, "the first sign-in to start") + p.send(t, `{"jsonrpc":"2.0","method":"notifications/cancelled","params":{"requestId":5}}`) + waitClosed(t, done, "the first sign-in to return") + p.send(t, fmt.Sprintf(authenticate, 6)) + require.Eventually(t, func() bool { + return len(recorder.prompts()) == 2 + }, 10*time.Second, time.Millisecond) + require.NoError(t, p.finish(t)) + + mu.Lock() + defer mu.Unlock() + assert.Equal(t, []bool{true, false}, opens, "the second call does not open the page again") + assert.Equal(t, []bool{false, true}, recorder.prompts(), "the cancelled call is PENDING, and the second call is ACTIVE") +} + func TestAuthenticateReopensPageAfterTimeout(t *testing.T) { srv, _ := challengedService(t) signIn := &signInStub{reply: func(call int) (bool, error) { diff --git a/libs/telemetry/protos/aitools_mcp.go b/libs/telemetry/protos/aitools_mcp.go index 4e50bd67d46..5546025d526 100644 --- a/libs/telemetry/protos/aitools_mcp.go +++ b/libs/telemetry/protos/aitools_mcp.go @@ -9,11 +9,13 @@ package protos type AitoolsMcpOperation string const ( - AitoolsMcpOperationAdd AitoolsMcpOperation = "ADD" - AitoolsMcpOperationRemove AitoolsMcpOperation = "REMOVE" - AitoolsMcpOperationSignIn AitoolsMcpOperation = "SIGN_IN" - AitoolsMcpOperationSignOut AitoolsMcpOperation = "SIGN_OUT" - AitoolsMcpOperationProxyStarted AitoolsMcpOperation = "PROXY_STARTED" + AitoolsMcpOperationAdd AitoolsMcpOperation = "ADD" + AitoolsMcpOperationRemove AitoolsMcpOperation = "REMOVE" + AitoolsMcpOperationSignIn AitoolsMcpOperation = "SIGN_IN" + AitoolsMcpOperationSignOut AitoolsMcpOperation = "SIGN_OUT" + AitoolsMcpOperationProxyStarted AitoolsMcpOperation = "PROXY_STARTED" + // AitoolsMcpOperationSignInPrompted is logged for each authenticate call that showed the sign-in page, with the + // state of the sign-in when that call's wait ended. A retry after a cancelled call logs its own event. AitoolsMcpOperationSignInPrompted AitoolsMcpOperation = "SIGN_IN_PROMPTED" ) From be0e3bac3f6ba3d8bb4bd74a81ecb7e77a8d772e Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Fri, 13 Nov 2026 16:23:43 +0000 Subject: [PATCH 19/22] Tidy the MCP telemetry: one SIGN_IN builder, rejected-start test, documented enum mirrors signInEventOf is the one SIGN_IN event builder, used by add --login and login. A misplaced test doc comment moves to its test, and remove's doc says the edit needs no network. The rejected-start case of the telemetry acceptance tests is replaced by a unit test that a rejected startup read logs nothing. KnownLeaves has its own test, and the enum values that no code names have comments that they mirror the schema. Co-authored-by: Isaac --- .../mcp-serve-rejected/.databrickscfg | 3 --- .../mcp-serve-rejected/out.test.toml | 1 - .../telemetry/mcp-serve-rejected/output.txt | 7 ------ .../telemetry/mcp-serve-rejected/script | 8 ------ .../telemetry/mcp-serve-rejected/test.toml | 6 ----- cmd/aitools/mcp_add.go | 4 +-- cmd/aitools/mcp_login.go | 7 ++---- cmd/aitools/mcp_remove.go | 2 +- cmd/aitools/mcp_serve_test.go | 25 +++++++++++++++++-- cmd/aitools/mcp_telemetry.go | 4 +-- cmd/aitools/mcp_telemetry_test.go | 4 +-- libs/aitools/mcp/known_test.go | 13 ++++++++++ libs/telemetry/protos/aitools_install.go | 6 +++-- libs/telemetry/protos/aitools_mcp.go | 5 ++-- 14 files changed, 52 insertions(+), 43 deletions(-) delete mode 100644 acceptance/cmd/aitools/telemetry/mcp-serve-rejected/.databrickscfg delete mode 100644 acceptance/cmd/aitools/telemetry/mcp-serve-rejected/out.test.toml delete mode 100644 acceptance/cmd/aitools/telemetry/mcp-serve-rejected/output.txt delete mode 100644 acceptance/cmd/aitools/telemetry/mcp-serve-rejected/script delete mode 100644 acceptance/cmd/aitools/telemetry/mcp-serve-rejected/test.toml diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/.databrickscfg b/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/.databrickscfg deleted file mode 100644 index 0369be633c5..00000000000 --- a/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/.databrickscfg +++ /dev/null @@ -1,3 +0,0 @@ -[logfood] -host = $DATABRICKS_HOST -token = test-token-logfood diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/out.test.toml b/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/out.test.toml deleted file mode 100644 index 18b1a88417e..00000000000 --- a/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/out.test.toml +++ /dev/null @@ -1 +0,0 @@ -Cloud = false diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/output.txt b/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/output.txt deleted file mode 100644 index 89d332ab32f..00000000000 --- a/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/output.txt +++ /dev/null @@ -1,7 +0,0 @@ - -=== serve whose startup check is rejected: exit 2, and nothing is uploaded: a failed start logs no event - ->>> musterr [CLI] aitools mcp serve --profile logfood --service system.ai.genie_one_mcp --agent claude-code -No install record for system.ai.genie_one_mcp on profile "logfood"; using its current host [DATABRICKS_URL] -cannot read MCP service system.ai.genie_one_mcp: Invalid access token. -Run: databricks auth login --profile logfood diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/script b/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/script deleted file mode 100644 index cf5022fbc50..00000000000 --- a/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/script +++ /dev/null @@ -1,8 +0,0 @@ -sethome home -envsubst < .databrickscfg > out && mv out .databrickscfg -export DATABRICKS_CONFIG_FILE=.databrickscfg -export BROWSER=none - -title "serve whose startup check is rejected: exit 2, and nothing is uploaded: a failed start logs no event" -trace musterr $CLI aitools mcp serve --profile logfood --service system.ai.genie_one_mcp --agent claude-code < /dev/null > /dev/null -mcp_events diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/test.toml b/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/test.toml deleted file mode 100644 index d60a4a6d774..00000000000 --- a/acceptance/cmd/aitools/telemetry/mcp-serve-rejected/test.toml +++ /dev/null @@ -1,6 +0,0 @@ -# The startup lookup of the service is rejected, so serve exits 2 before it serves. A start that fails its checks logs no -# event, so no upload follows: the rejected token is not sent again. -[[Server]] -Pattern = "GET /api/2.1/unity-catalog/mcp-services/{name}" -Response.StatusCode = 401 -Response.Body = '{"error_code": "UNAUTHENTICATED", "message": "Invalid access token."}' diff --git a/cmd/aitools/mcp_add.go b/cmd/aitools/mcp_add.go index 6aeba213cbd..8ee0949aaad 100644 --- a/cmd/aitools/mcp_add.go +++ b/cmd/aitools/mcp_add.go @@ -1084,7 +1084,7 @@ func signInInstalled(ctx context.Context, errOut io.Writer, w *databricks.Worksp if signedIn(ctx, w, plans[i].service.FullName) { server.SignIn = &addSignInOutput{State: signInStateActive} reportSignedIn(errOut, plans[i].service, req.profile, true) - logMcpEvent(ctx, addSignInEvent(server.Service, signInStateActive, true, nil)) + logMcpEvent(ctx, signInEventOf(server.Service, signInStateActive, true, nil)) continue } yes, err := promptSignInNow(server.Service) @@ -1101,7 +1101,7 @@ func signInInstalled(ctx context.Context, errOut io.Writer, w *databricks.Worksp url := mcp.SignInURL(w.Config.Host, plans[i].service.FullName, req.workspaceID) state, alreadySignedIn, _, err := signIn(ctx, errOut, w, req.pinned, url, plans[i].service, signInOptions{Profile: req.profile, Open: browser.OpenOutputToStderr}) server.SignIn = &addSignInOutput{State: state} - logMcpEvent(ctx, addSignInEvent(server.Service, state, alreadySignedIn, err)) + logMcpEvent(ctx, signInEventOf(server.Service, state, alreadySignedIn, err)) if err != nil { server.SignIn.ErrorCategory = errorCategory(err) server.SignIn.Error = err.Error() diff --git a/cmd/aitools/mcp_login.go b/cmd/aitools/mcp_login.go index 7e577d6d11b..fb54ccee626 100644 --- a/cmd/aitools/mcp_login.go +++ b/cmd/aitools/mcp_login.go @@ -162,16 +162,13 @@ func runMcpSignIn(ctx context.Context, errOut io.Writer, flagValue, service stri // signInEvent is the SIGN_IN or SIGN_OUT event of one login or logout. func signInEvent(operation protos.AitoolsMcpOperation, out *mcpSignInOutput, err error) protos.AitoolsMcpEvent { - var r protos.AitoolsMcpServerResult if operation == protos.AitoolsMcpOperationSignIn { - r = signInServerResult(out.Service, out.State, out.alreadySignedIn, err) - } else { - r = signOutServerResult(out.Service, out.State, err) + return signInEventOf(out.Service, out.State, out.alreadySignedIn, err) } return protos.AitoolsMcpEvent{ CallingAgent: callingAgentType(), Operation: operation, - Servers: []protos.AitoolsMcpServerResult{r}, + Servers: []protos.AitoolsMcpServerResult{signOutServerResult(out.Service, out.State, err)}, } } diff --git a/cmd/aitools/mcp_remove.go b/cmd/aitools/mcp_remove.go index 2af23712d6e..b3b662a7387 100644 --- a/cmd/aitools/mcp_remove.go +++ b/cmd/aitools/mcp_remove.go @@ -83,7 +83,7 @@ type recordDrop struct { agent string } -// NewMcpRemoveCmd returns `aitools mcp remove`. It needs no network. +// NewMcpRemoveCmd returns `aitools mcp remove`. The edit needs no network; the telemetry upload after it is best-effort. func NewMcpRemoveCmd() *cobra.Command { var f mcpRemoveFlags cmd := &cobra.Command{ diff --git a/cmd/aitools/mcp_serve_test.go b/cmd/aitools/mcp_serve_test.go index 1c2a14dd6d9..91964dd6ac8 100644 --- a/cmd/aitools/mcp_serve_test.go +++ b/cmd/aitools/mcp_serve_test.go @@ -334,8 +334,6 @@ func TestMcpServeRejectsServiceWithoutThreeParts(t *testing.T) { } } -// TestMcpServeStateReadFailureExitsTwo checks that a state file that cannot be read stops the binding checks, so it -// exits 2 like the other binding failures. // TestMcpServeFailedStartLogsNothing checks that a start that fails its binding checks logs no PROXY_STARTED, so nothing // uploads. func TestMcpServeFailedStartLogsNothing(t *testing.T) { @@ -352,6 +350,29 @@ func TestMcpServeFailedStartLogsNothing(t *testing.T) { assert.Empty(t, telemetry.BufferedEvents(serve.Context())) } +// TestMcpServeRejectedStartLogsNothing checks that a start whose startup read is rejected by the profile's credential logs +// no PROXY_STARTED, so the run uploads nothing: the rejected token is not sent again. +func TestMcpServeRejectedStartLogsNothing(t *testing.T) { + home := withHome(t) + server := serviceServer(t, http.StatusUnauthorized, `{"error_code":"UNAUTHENTICATED","message":"Invalid access token."}`) + writeHomeConfig(t, home, "[logfood]\nhost = "+server.URL+"\nauth_type = pat\ntoken = dapi-expired\n") + + parent := newMCPTestRoot() + parent.SetErr(&bytes.Buffer{}) + parent.SetArgs([]string{"mcp", "serve", "--profile", testProfile, "--service", testService}) + ctx := telemetry.WithNewLogger(cmdio.MockDiscard(t.Context())) + err := parent.ExecuteContext(ctx) + require.ErrorIs(t, err, root.ErrAlreadyPrinted) + assert.Equal(t, serveBindStatus, root.ExitStatus(err)) + serve, _, findErr := parent.Find([]string{"mcp", "serve"}) + require.NoError(t, findErr) + assert.Empty(t, telemetry.BufferedEvents(serve.Context())) + _, uploads := telemetry.UploadConfig(serve.Context()) + assert.False(t, uploads) +} + +// TestMcpServeStateReadFailureExitsTwo checks that a state file that cannot be read stops the binding checks, so it +// exits 2 like the other binding failures. func TestMcpServeStateReadFailureExitsTwo(t *testing.T) { withHome(t) dir, err := installer.GlobalSkillsDir(t.Context()) diff --git a/cmd/aitools/mcp_telemetry.go b/cmd/aitools/mcp_telemetry.go index 9e0492aed1e..ca94f89e33c 100644 --- a/cmd/aitools/mcp_telemetry.go +++ b/cmd/aitools/mcp_telemetry.go @@ -198,9 +198,9 @@ func signInServerResult(service, state string, alreadySignedIn bool, err error) return r } -// addSignInEvent is the SIGN_IN event of one provider sign-in under add --login. A sign-in that found the caller +// signInEventOf is the SIGN_IN event of one provider sign-in, under add --login or login. A sign-in that found the caller // already signed in is UNCHANGED. -func addSignInEvent(service, state string, alreadySignedIn bool, err error) protos.AitoolsMcpEvent { +func signInEventOf(service, state string, alreadySignedIn bool, err error) protos.AitoolsMcpEvent { return protos.AitoolsMcpEvent{ CallingAgent: callingAgentType(), Operation: protos.AitoolsMcpOperationSignIn, diff --git a/cmd/aitools/mcp_telemetry_test.go b/cmd/aitools/mcp_telemetry_test.go index df29d6c9237..ccd28d3e870 100644 --- a/cmd/aitools/mcp_telemetry_test.go +++ b/cmd/aitools/mcp_telemetry_test.go @@ -251,7 +251,7 @@ func TestBoundProfileCount(t *testing.T) { assert.Equal(t, int64(0), boundProfileCount(t.Context(), &mcp.State{})) } -func TestAddSignInEvent(t *testing.T) { +func TestSignInEventOf(t *testing.T) { // An ACTIVE credential that the add found signed in is UNCHANGED, as add --login reports it. assert.Equal(t, protos.AitoolsMcpEvent{ CallingAgent: callingAgentType(), @@ -261,7 +261,7 @@ func TestAddSignInEvent(t *testing.T) { Outcome: protos.AitoolsTargetOutcomeUnchanged, SignInState: protos.AitoolsMcpSignInStateActive, }}, - }, addSignInEvent("system.ai.github", signInStateActive, true, nil)) + }, signInEventOf("system.ai.github", signInStateActive, true, nil)) } // TestMcpEventOmitsUserContent checks that the event carries enums and counts only: a custom service's name must not diff --git a/libs/aitools/mcp/known_test.go b/libs/aitools/mcp/known_test.go index b401338982d..4b8c89a58d3 100644 --- a/libs/aitools/mcp/known_test.go +++ b/libs/aitools/mcp/known_test.go @@ -1,6 +1,7 @@ package mcp import ( + "slices" "testing" "github.com/stretchr/testify/assert" @@ -38,6 +39,18 @@ func TestKnownSystemAITable(t *testing.T) { } } +// TestKnownLeavesAreTheSystemAITable checks that KnownLeaves lists the leaves of the system.ai table, sorted, with each +// leaf known as a system.ai service. +func TestKnownLeavesAreTheSystemAITable(t *testing.T) { + leaves := KnownLeaves() + assert.Contains(t, leaves, "genie_one_mcp") + assert.True(t, slices.IsSorted(leaves)) + assert.Len(t, leaves, len(knownSystemAI)) + for _, leaf := range leaves { + assert.NotNil(t, KnownOf(SystemAILocation+"."+leaf), leaf) + } +} + func TestProviderLabel(t *testing.T) { assert.Equal(t, "GitHub", ProviderLabel("system.ai.github")) assert.Equal(t, "Genie One", ProviderLabel("SYSTEM.AI.genie_one_mcp")) diff --git a/libs/telemetry/protos/aitools_install.go b/libs/telemetry/protos/aitools_install.go index ead0a7b76c8..6c69b38f1ab 100644 --- a/libs/telemetry/protos/aitools_install.go +++ b/libs/telemetry/protos/aitools_install.go @@ -75,7 +75,8 @@ const ( // counted here: it ends the process before telemetry is uploaded. AitoolsErrorCategoryCommandInterrupted AitoolsErrorCategory = "COMMAND_INTERRUPTED" - // `aitools mcp`: profiles and sign-in. + // `aitools mcp`: profiles and sign-in. Bind failures send their category by value (mcp.BindCategory), so + // PROFILE_NOT_FOUND and PROFILE_HOST_CHANGED have no reference by name. AitoolsErrorCategoryNoProfile AitoolsErrorCategory = "NO_PROFILE" AitoolsErrorCategoryProfileRequired AitoolsErrorCategory = "PROFILE_REQUIRED" AitoolsErrorCategoryProfileNotFound AitoolsErrorCategory = "PROFILE_NOT_FOUND" @@ -84,7 +85,8 @@ const ( AitoolsErrorCategorySignInRequired AitoolsErrorCategory = "SIGN_IN_REQUIRED" AitoolsErrorCategorySignInTimeout AitoolsErrorCategory = "SIGN_IN_TIMEOUT" - // `aitools mcp`: the service list, agent configs and the proxy. + // `aitools mcp`: the service list, agent configs and the proxy. MCP_UPSTREAM_ERROR is in the schema but not sent yet: + // the proxy does not log upstream errors in v1. AitoolsErrorCategoryMCPServiceNotFound AitoolsErrorCategory = "MCP_SERVICE_NOT_FOUND" AitoolsErrorCategoryMCPPermissionDenied AitoolsErrorCategory = "MCP_PERMISSION_DENIED" AitoolsErrorCategoryMCPLocationNotFound AitoolsErrorCategory = "MCP_LOCATION_NOT_FOUND" diff --git a/libs/telemetry/protos/aitools_mcp.go b/libs/telemetry/protos/aitools_mcp.go index 5546025d526..aae953d4ede 100644 --- a/libs/telemetry/protos/aitools_mcp.go +++ b/libs/telemetry/protos/aitools_mcp.go @@ -56,8 +56,9 @@ const ( type AitoolsTargetOutcome string const ( - AitoolsTargetOutcomeChanged AitoolsTargetOutcome = "CHANGED" - AitoolsTargetOutcomeUnchanged AitoolsTargetOutcome = "UNCHANGED" + AitoolsTargetOutcomeChanged AitoolsTargetOutcome = "CHANGED" + AitoolsTargetOutcomeUnchanged AitoolsTargetOutcome = "UNCHANGED" + // AitoolsTargetOutcomeUpdateAvailable mirrors the schema. No MCP command sends it yet. AitoolsTargetOutcomeUpdateAvailable AitoolsTargetOutcome = "UPDATE_AVAILABLE" AitoolsTargetOutcomeSkipped AitoolsTargetOutcome = "SKIPPED" AitoolsTargetOutcomeFailed AitoolsTargetOutcome = "FAILED" From 1174ea72030231a07376e26736c04203c9cb8d26 Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Fri, 13 Nov 2026 16:44:52 +0000 Subject: [PATCH 20/22] Refuse the MCP telemetry upload after the service rejects the credential A 401 that does not ask for a sign-in is answered with the auth failure, but the run's exit-time upload still authenticated with the credential the workspace had just rejected. The proxy now reports that answer to the command, which refuses the upload as a refresh failure does. Co-authored-by: Isaac --- cmd/aitools/mcp_telemetry.go | 5 +++++ cmd/aitools/mcp_telemetry_test.go | 10 +++++++++ libs/aitools/mcp/proxy/challenge.go | 6 +++++- libs/aitools/mcp/proxy/challenge_test.go | 27 +++++++++++++++++++++++- 4 files changed, 46 insertions(+), 2 deletions(-) diff --git a/cmd/aitools/mcp_telemetry.go b/cmd/aitools/mcp_telemetry.go index ca94f89e33c..db8fc52b85b 100644 --- a/cmd/aitools/mcp_telemetry.go +++ b/cmd/aitools/mcp_telemetry.go @@ -248,6 +248,11 @@ func (t mcpProxyTelemetry) SignInPrompted(ctx context.Context, signedIn bool) { logMcpEvent(ctx, signInPromptedEvent(t.agent, t.service, signedIn)) } +// AuthRejected refuses the run's upload, since the upload would present the rejected credential again. +func (t mcpProxyTelemetry) AuthRejected(ctx context.Context) { + telemetry.RefuseUpload(ctx) +} + // signInPromptedEvent is the SIGN_IN_PROMPTED event of serve: the sign-in is required, and the // sign-in state is ACTIVE when the wait ended with the credential, PENDING otherwise. func signInPromptedEvent(agent protos.AitoolsAgentType, service string, signedIn bool) protos.AitoolsMcpEvent { diff --git a/cmd/aitools/mcp_telemetry_test.go b/cmd/aitools/mcp_telemetry_test.go index ccd28d3e870..4133aa14c64 100644 --- a/cmd/aitools/mcp_telemetry_test.go +++ b/cmd/aitools/mcp_telemetry_test.go @@ -218,6 +218,16 @@ func TestLogMcpEventDropsRejectedCredential(t *testing.T) { assert.False(t, uploads, "the add event is logged, and it is not uploaded with the rejected run") } +// TestMcpProxyAuthRejectedRefusesUpload checks that a 401 the proxy answers refuses the run's upload, even though the +// serve command sets its upload config at exit. +func TestMcpProxyAuthRejectedRefusesUpload(t *testing.T) { + ctx := telemetry.WithNewLogger(t.Context()) + telemetry.SetUploadConfig(ctx, &config.Config{Profile: "logfood", Host: "https://adb-1.test"}) + mcpProxyTelemetry{}.AuthRejected(ctx) + _, uploads := telemetry.UploadConfig(ctx) + assert.False(t, uploads) +} + func TestSignInPromptedEvent(t *testing.T) { for signedIn, state := range map[bool]protos.AitoolsMcpSignInState{ true: protos.AitoolsMcpSignInStateActive, diff --git a/libs/aitools/mcp/proxy/challenge.go b/libs/aitools/mcp/proxy/challenge.go index d3cf58ba1c6..9109db7906c 100644 --- a/libs/aitools/mcp/proxy/challenge.go +++ b/libs/aitools/mcp/proxy/challenge.go @@ -57,6 +57,9 @@ type Recorder interface { // earlier cancelled call opened, which the user may still have open. signedIn reports whether the credential became // ACTIVE. A retry after a cancelled call reports its own outcome. SignInPrompted(ctx context.Context, signedIn bool) + // AuthRejected runs when the service answers 401 without asking for a sign-in: the Databricks credential was + // rejected, and the answer is the auth failure. + AuthRejected(ctx context.Context) } // Challenge turns on sign-in handling for a service that needs a provider sign-in. @@ -156,7 +159,7 @@ func (h *signInHooks) Local(ctx context.Context, request []byte, notify func([]b // answer becomes a sign-in message, followed by tools/list_changed so an agent that cached its list re-lists. A 401 that // is not a challenge becomes the auth failure (see authFailure). Only the response to the request is read; any other // message passes through untouched. -func (h *signInHooks) Result(_ context.Context, request []byte, status int, message []byte) ([][]byte, error) { +func (h *signInHooks) Result(ctx context.Context, request []byte, status int, message []byte) ([][]byte, error) { req, err := parseMessage(request) if err != nil { return nil, err @@ -174,6 +177,7 @@ func (h *signInHooks) Result(_ context.Context, request []byte, status int, mess } challenged := h.challenged(&reply) if status == http.StatusUnauthorized && !challenged { + h.c.Recorder.AuthRejected(ctx) return [][]byte{errorResponse(req.ID, codeUpstream, h.authFailure())}, nil } if req.Method != methodToolsList && req.Method != methodToolsCall { diff --git a/libs/aitools/mcp/proxy/challenge_test.go b/libs/aitools/mcp/proxy/challenge_test.go index 4af63aeeddd..34ae73b06a5 100644 --- a/libs/aitools/mcp/proxy/challenge_test.go +++ b/libs/aitools/mcp/proxy/challenge_test.go @@ -68,10 +68,12 @@ func (s *signInStub) requests() []SignInRequest { // loginCommand is the hint that the command builds for a pinned config file and a profile name with a space. const loginCommand = `DATABRICKS_CONFIG_FILE='/work/cfg/my file.cfg' databricks aitools mcp login system.ai.github --profile 'my profile'` -// promptRecorder records the SIGN_IN_PROMPTED events: one per sign-in page that authenticate opened. +// promptRecorder records the SIGN_IN_PROMPTED events: one per sign-in page that authenticate opened. It counts the +// rejections of the Databricks credential too. type promptRecorder struct { mu sync.Mutex signedIn []bool + rejected int } func (r *promptRecorder) SignInPrompted(_ context.Context, signedIn bool) { @@ -80,6 +82,12 @@ func (r *promptRecorder) SignInPrompted(_ context.Context, signedIn bool) { r.signedIn = append(r.signedIn, signedIn) } +func (r *promptRecorder) AuthRejected(_ context.Context) { + r.mu.Lock() + defer r.mu.Unlock() + r.rejected++ +} + func (r *promptRecorder) prompts() []bool { r.mu.Lock() defer r.mu.Unlock() @@ -986,6 +994,23 @@ func TestUnauthorizedIsAuthFailureUnlessItAsksForSignIn(t *testing.T) { } } +// TestUnauthorizedRecordsRejectedCredential checks that a 401 without a sign-in challenge is recorded as a rejected +// credential, once per answer, so that the command refuses its telemetry upload. +func TestUnauthorizedRecordsRejectedCredential(t *testing.T) { + srv, _ := newFakeEndpoint(t, func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, "unauthorized", http.StatusUnauthorized) + }) + signIn := &signInStub{reply: func(int) (bool, error) { return false, nil }} + recorder := &promptRecorder{} + challenge := testChallenge(signIn.SignIn) + challenge.Recorder = recorder + b := bridgeFor(srv.URL) + b.Hooks = newSignInHooks(challenge, io.Discard) + runAll(t, b, `{"jsonrpc":"2.0","id":2,"method":"tools/list"}`+"\n") + + assert.Equal(t, 1, recorder.rejected) +} + // TestRejectedCredentialDoesNotAskForLogout checks a rejected Databricks token after "Already signed in": the credential reads // as signed in, and the service rejects the token with a 401. The 401 is an auth failure, not a challenge, so the next // authenticate runs the sign-in as usual, and the logout answer is not given. From a1d30e0b0942e2e97374421b6e208b716604abae Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Fri, 13 Nov 2026 16:46:10 +0000 Subject: [PATCH 21/22] Keep the telemetry upload's host metadata failures out of the command output The SDK logs a failed host metadata lookup as a warning through its global logger, and the upload's resolution runs on a background context that reaches it. A command that succeeded then printed a warning that only its telemetry upload caused. The upload's resolver now logs the failure at debug and falls back to the user config, as the SDK does. Co-authored-by: Isaac --- cmd/aitools/mcp_telemetry_test.go | 26 ++++++++++++++++++++++++++ libs/aitools/mcp/binding.go | 23 +++++++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/cmd/aitools/mcp_telemetry_test.go b/cmd/aitools/mcp_telemetry_test.go index 4133aa14c64..e388ca81568 100644 --- a/cmd/aitools/mcp_telemetry_test.go +++ b/cmd/aitools/mcp_telemetry_test.go @@ -1,8 +1,11 @@ package aitools import ( + "bytes" + "context" "encoding/json" "errors" + "log/slog" "maps" "path/filepath" "slices" @@ -218,6 +221,29 @@ func TestLogMcpEventDropsRejectedCredential(t *testing.T) { assert.False(t, uploads, "the add event is logged, and it is not uploaded with the rejected run") } +// TestSetUploadConfigLogsNoWarningForFailedHostMetadata checks that an upload whose host metadata lookup fails warns +// nothing through the global logger, since the SDK would log that failure there. The command has succeeded, and the +// upload is best-effort. +func TestSetUploadConfigLogsNoWarningForFailedHostMetadata(t *testing.T) { + home := withHome(t) + writeHomeConfig(t, home, "[logfood]\nhost = https://adb-1.test\nauth_type = pat\ntoken = t\n") + factory := config.DefaultHostMetadataResolverFactory + config.DefaultHostMetadataResolverFactory = func(*config.Config) config.HostMetadataResolver { + return func(context.Context, string) (*config.HostMetadata, error) { return nil, errors.New("no such host") } + } + t.Cleanup(func() { config.DefaultHostMetadataResolverFactory = factory }) + var warnings bytes.Buffer + defaultLogger := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(&warnings, &slog.HandlerOptions{Level: slog.LevelWarn}))) + t.Cleanup(func() { slog.SetDefault(defaultLogger) }) + + ctx := env.Set(telemetry.WithNewLogger(t.Context()), "DATABRICKS_CLI_DISABLE_TELEMETRY", "") + setUploadConfig(ctx, "logfood", filepath.Join(home, ".databrickscfg")) + _, uploads := telemetry.UploadConfig(ctx) + assert.True(t, uploads) + assert.Empty(t, warnings.String()) +} + // TestMcpProxyAuthRejectedRefusesUpload checks that a 401 the proxy answers refuses the run's upload, even though the // serve command sets its upload config at exit. func TestMcpProxyAuthRejectedRefusesUpload(t *testing.T) { diff --git a/libs/aitools/mcp/binding.go b/libs/aitools/mcp/binding.go index 1eb8f06db1c..1c8c4afc9cb 100644 --- a/libs/aitools/mcp/binding.go +++ b/libs/aitools/mcp/binding.go @@ -15,6 +15,7 @@ import ( "github.com/databricks/cli/libs/databrickscfg" "github.com/databricks/cli/libs/databrickscfg/profile" "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/shellquote" "github.com/databricks/databricks-sdk-go/config" ) @@ -119,9 +120,31 @@ func BoundConfigWithin(profileName, configFile string, budget time.Duration) (*c func BoundUploadConfig(profileName, configFile string, budget time.Duration) (*config.Config, error) { cfg := withinBudget(profileName, configFile, budget) cfg.Credentials = auth.CredentialsWithoutRefresh() + quietHostMetadata(cfg) return resolveBound(cfg) } +// quietHostMetadata makes a failed host metadata lookup of cfg a debug message. The SDK logs that failure as a warning +// through its global logger, since the resolution runs on a background context, so a command that succeeded would print a +// warning that only its telemetry upload caused. The SDK falls back to the user config when the lookup fails, and so does +// this one, with a nil result. +func quietHostMetadata(cfg *config.Config) { + resolve := cfg.DefaultHostMetadataResolver() + if factory := config.DefaultHostMetadataResolverFactory; factory != nil { + if r := factory(cfg); r != nil { + resolve = r + } + } + cfg.HostMetadataResolver = func(ctx context.Context, host string) (*config.HostMetadata, error) { + meta, err := resolve(ctx, host) + if err != nil { + log.Debugf(ctx, "telemetry: cannot resolve host metadata for %s: %s", host, err) + return nil, nil + } + return meta, nil + } +} + // withinBudget is the config of profileName in configFile, with the SDK's request and retry budgets set to budget. func withinBudget(profileName, configFile string, budget time.Duration) *config.Config { seconds := int(budget / time.Second) From f20df6b2655354ee72d0bc6f49422be0bfa87f3c Mon Sep 17 00:00:00 2001 From: Simon Faltum Date: Fri, 13 Nov 2026 16:49:25 +0000 Subject: [PATCH 22/22] Group the MCP telemetry profile count as mcp list does, and tidy the MCP helpers boundProfileCount now groups servers through the same helper mcp list uses, so the count matches the listing in every case, including a config file that does not resolve. KnownLeaves is marked as kept for its tests, the sign-in constructors return their command directly, the pruned-install test setup is a named helper, and the enum comments say that the schema mirrors are not sent by the CLI. Co-authored-by: Isaac --- cmd/aitools/mcp_list.go | 18 ++++++++++++------ cmd/aitools/mcp_login.go | 6 ++---- cmd/aitools/mcp_remove_test.go | 17 ++++++++++------- cmd/aitools/mcp_telemetry.go | 13 +++++-------- libs/aitools/mcp/known.go | 2 ++ libs/aitools/mcp/proxy/challenge_test.go | 4 ++-- libs/aitools/mcp/proxy/run_test.go | 2 +- libs/telemetry/protos/aitools_install.go | 8 ++++---- 8 files changed, 38 insertions(+), 32 deletions(-) diff --git a/cmd/aitools/mcp_list.go b/cmd/aitools/mcp_list.go index 61872786fe6..f109fb0be14 100644 --- a/cmd/aitools/mcp_list.go +++ b/cmd/aitools/mcp_list.go @@ -128,6 +128,16 @@ type profileKey struct { path string } +// profileGroup returns the group of a server entry in list: its profile in the config file that check resolved, or in the +// file as written when that file does not resolve. +func profileGroup(e mcp.Entry, check mcp.EntryCheck) profileKey { + path := check.Path + if path == "" { + path = e.ConfigFile + } + return profileKey{profile: e.Profile, path: path} +} + // NewMcpListCmd returns `aitools mcp list`. Without --check it reads the state file and the agent configs only, and // makes no network calls. Drift and unreadable entries are reported, not failed: the command exits 0. func NewMcpListCmd() *cobra.Command { @@ -229,12 +239,8 @@ func buildMcpList(ctx context.Context, profileName string) (mcpListOutput, error warnedFiles := map[string]bool{} for i, e := range entries { check := checks[i] - path := check.Path - if path == "" { - // The file cannot be resolved, so the entry is grouped under its file as written. - path = e.ConfigFile - } - key := profileKey{profile: e.Profile, path: path} + key := profileGroup(e, check) + path := key.path p, ok := groups[key] if !ok { p = &mcpListProfile{Profile: e.Profile, Host: e.Host, Status: statusOK, Servers: []mcpListServer{}, configPath: path} diff --git a/cmd/aitools/mcp_login.go b/cmd/aitools/mcp_login.go index fb54ccee626..06f909eb38d 100644 --- a/cmd/aitools/mcp_login.go +++ b/cmd/aitools/mcp_login.go @@ -56,7 +56,7 @@ type mcpSignInAction func(ctx context.Context, errOut io.Writer, w *databricks.W // NewMcpLoginCmd returns `aitools mcp login`. func NewMcpLoginCmd() *cobra.Command { - cmd := &cobra.Command{ + return &cobra.Command{ Use: "login ", Short: "Sign in to a Databricks managed MCP service with your account", Long: `Sign in to a Databricks managed MCP service that uses your account with an external provider, such @@ -69,12 +69,11 @@ sign-in to complete. Without a browser, open the printed URL yourself. return runMcpSignInCommand(cmd, args[0], mcpLogin, protos.AitoolsMcpOperationSignIn) }, } - return cmd } // NewMcpLogoutCmd returns `aitools mcp logout`. func NewMcpLogoutCmd() *cobra.Command { - cmd := &cobra.Command{ + return &cobra.Command{ Use: "logout ", Short: "Remove your sign-in to a Databricks managed MCP service", Long: `Remove your sign-in to a Databricks managed MCP service on a profile. The install of the server @@ -86,7 +85,6 @@ stays in place. Running it when you are not signed in is not an error. return runMcpSignInCommand(cmd, args[0], mcpLogout, protos.AitoolsMcpOperationSignOut) }, } - return cmd } // runMcpSignInCommand resolves the profile for service and runs action. In JSON mode the document is diff --git a/cmd/aitools/mcp_remove_test.go b/cmd/aitools/mcp_remove_test.go index f3b94c05af7..8deb0e92988 100644 --- a/cmd/aitools/mcp_remove_test.go +++ b/cmd/aitools/mcp_remove_test.go @@ -912,18 +912,21 @@ func TestMcpRemoveOfflineSetsUploadWithoutWarning(t *testing.T) { assert.Contains(t, stdout.String(), `"removed"`) } +// seedPrunedInstall records an install of genie_one_mcp for logfood in claude-code whose entry the agent config no longer holds. +func seedPrunedInstall(t *testing.T, claudePath string) { + t.Helper() + claude := agents.ByName("claude-code") + saveState(t, seedInstall(t, claude, "genie_one_mcp-logfood", "system.ai.genie_one_mcp", "logfood", "https://adb-1.test")) + require.NoError(t, os.WriteFile(claudePath, []byte(`{"mcpServers":{}}`), 0o600)) +} + // TestMcpRemovePrunedEntryReportsChanged checks that an entry the agent config no longer holds is reported CHANGED, since // its record is dropped, and FAILED when the state cannot be saved, as a record-only removal is. func TestMcpRemovePrunedEntryReportsChanged(t *testing.T) { home := withHome(t) - claude := agents.ByName("claude-code") claudePath := filepath.Join(home, ".claude.json") - seedPruned := func() { - saveState(t, seedInstall(t, claude, "genie_one_mcp-logfood", "system.ai.genie_one_mcp", "logfood", "https://adb-1.test")) - require.NoError(t, os.WriteFile(claudePath, []byte(`{"mcpServers":{}}`), 0o600)) - } - seedPruned() + seedPrunedInstall(t, claudePath) var stdout, logs bytes.Buffer remove, err := runRemoveWithTelemetry(t, &stdout, &logs, log.LevelDebug, "--names", "genie_one_mcp-logfood", "--output", "json") require.NoError(t, err) @@ -933,7 +936,7 @@ func TestMcpRemovePrunedEntryReportsChanged(t *testing.T) { require.Len(t, events[0].AitoolsMcpEvent.Servers, 1) assert.Equal(t, protos.AitoolsTargetOutcomeChanged, events[0].AitoolsMcpEvent.Servers[0].Outcome) - seedPruned() + seedPrunedInstall(t, claudePath) saveMCPState = func(context.Context, *mcp.State) error { return errors.New("state file is read-only") } t.Cleanup(func() { saveMCPState = mcp.SaveState }) stdout.Reset() diff --git a/cmd/aitools/mcp_telemetry.go b/cmd/aitools/mcp_telemetry.go index db8fc52b85b..35f88327a0d 100644 --- a/cmd/aitools/mcp_telemetry.go +++ b/cmd/aitools/mcp_telemetry.go @@ -100,16 +100,13 @@ func logMcpEvent(ctx context.Context, event protos.AitoolsMcpEvent) bool { return true } -// boundProfileCount returns the number of distinct profiles that have servers installed. A profile is its name in its -// config file, as list groups them, so DEFAULT in two config files counts twice. +// boundProfileCount returns the number of distinct profiles that have servers installed, grouped as list groups them +// (profileGroup), so DEFAULT in two config files counts twice. func boundProfileCount(ctx context.Context, state *mcp.State) int64 { profiles := map[profileKey]bool{} - for _, e := range state.Servers { - path, err := mcp.ResolveConfigFile(ctx, e.ConfigFile) - if err != nil { - path = e.ConfigFile - } - profiles[profileKey{profile: e.Profile, path: path}] = true + checks := mcp.CheckEntries(ctx, state) + for i, e := range state.Servers { + profiles[profileGroup(e, checks[i])] = true } return int64(len(profiles)) } diff --git a/libs/aitools/mcp/known.go b/libs/aitools/mcp/known.go index 01c944fb90c..47dab54d23e 100644 --- a/libs/aitools/mcp/known.go +++ b/libs/aitools/mcp/known.go @@ -66,6 +66,8 @@ var knownSystemAI = map[string]KnownService{ } // KnownLeaves returns the leaf names of the system.ai services in the CLI's table, sorted. +// +//deadcode:allow used by the service-mapping tests func KnownLeaves() []string { return slices.Sorted(maps.Keys(knownSystemAI)) } diff --git a/libs/aitools/mcp/proxy/challenge_test.go b/libs/aitools/mcp/proxy/challenge_test.go index 34ae73b06a5..7c5bc6d60bb 100644 --- a/libs/aitools/mcp/proxy/challenge_test.go +++ b/libs/aitools/mcp/proxy/challenge_test.go @@ -616,7 +616,7 @@ func TestChallengeMatchesServiceNameWithoutCase(t *testing.T) { `{"mode":"url","url":"` + testURL + `"}]}}}` srv, _ := newFakeEndpoint(t, answerWith(http.StatusOK, body)) b := bridgeFor(srv.URL) - b.Hooks = newSignInHooks(&Challenge{Service: "SYSTEM.AI.GITHUB", Profile: "logfood", Host: testHost, Label: "GitHub", LoginCommand: loginCommand}, io.Discard) + b.Hooks = newSignInHooks(&Challenge{Service: "SYSTEM.AI.GITHUB", Profile: "logfood", Host: testHost, Label: "GitHub", LoginCommand: loginCommand, Recorder: &promptRecorder{}}, io.Discard) out, _ := runAll(t, b, `{"jsonrpc":"2.0","id":2,"method":"tools/list"}`+"\n") require.Len(t, out, 1) @@ -1051,7 +1051,7 @@ func TestAuthFailureNamesPinnedConfigFile(t *testing.T) { http.Error(w, "unauthorized", http.StatusUnauthorized) }) b := bridgeFor(srv.URL) - b.Hooks = newSignInHooks(&Challenge{Service: testService, Profile: "logfood", Host: testHost, Label: "GitHub", LoginCommand: loginCommand, PinnedFile: "/work/cfg/my file.cfg"}, io.Discard) + b.Hooks = newSignInHooks(&Challenge{Service: testService, Profile: "logfood", Host: testHost, Label: "GitHub", LoginCommand: loginCommand, PinnedFile: "/work/cfg/my file.cfg", Recorder: &promptRecorder{}}, io.Discard) out, _ := runAll(t, b, `{"jsonrpc":"2.0","id":2,"method":"tools/list"}`+"\n") require.Len(t, out, 1) diff --git a/libs/aitools/mcp/proxy/run_test.go b/libs/aitools/mcp/proxy/run_test.go index 8a8f195027d..f4020a9d3a1 100644 --- a/libs/aitools/mcp/proxy/run_test.go +++ b/libs/aitools/mcp/proxy/run_test.go @@ -100,7 +100,7 @@ func TestRunAdvertisesListChangedOnlyForSignInServices(t *testing.T) { assert.Contains(t, out.String(), `"listChanged":false`) opts, out, _ = runOptions(t, srv.URL, cfg, initialize) - opts.Challenge = &Challenge{Service: testService, Profile: "logfood", Host: testHost, Label: "GitHub"} + opts.Challenge = &Challenge{Service: testService, Profile: "logfood", Host: testHost, Label: "GitHub", Recorder: &promptRecorder{}} require.NoError(t, Run(t.Context(), opts)) assert.Contains(t, out.String(), `"listChanged":true`) } diff --git a/libs/telemetry/protos/aitools_install.go b/libs/telemetry/protos/aitools_install.go index 6c69b38f1ab..13c700bf24d 100644 --- a/libs/telemetry/protos/aitools_install.go +++ b/libs/telemetry/protos/aitools_install.go @@ -75,8 +75,8 @@ const ( // counted here: it ends the process before telemetry is uploaded. AitoolsErrorCategoryCommandInterrupted AitoolsErrorCategory = "COMMAND_INTERRUPTED" - // `aitools mcp`: profiles and sign-in. Bind failures send their category by value (mcp.BindCategory), so - // PROFILE_NOT_FOUND and PROFILE_HOST_CHANGED have no reference by name. + // `aitools mcp`: profiles and sign-in. Bind failures send their category by value (mcp.BindCategory). PROFILE_NOT_FOUND + // and PROFILE_HOST_CHANGED mirror the schema, and no CLI command names them. AitoolsErrorCategoryNoProfile AitoolsErrorCategory = "NO_PROFILE" AitoolsErrorCategoryProfileRequired AitoolsErrorCategory = "PROFILE_REQUIRED" AitoolsErrorCategoryProfileNotFound AitoolsErrorCategory = "PROFILE_NOT_FOUND" @@ -85,8 +85,8 @@ const ( AitoolsErrorCategorySignInRequired AitoolsErrorCategory = "SIGN_IN_REQUIRED" AitoolsErrorCategorySignInTimeout AitoolsErrorCategory = "SIGN_IN_TIMEOUT" - // `aitools mcp`: the service list, agent configs and the proxy. MCP_UPSTREAM_ERROR is in the schema but not sent yet: - // the proxy does not log upstream errors in v1. + // `aitools mcp`: the service list, agent configs and the proxy. MCP_UPSTREAM_ERROR mirrors the schema, and no CLI + // command sends it yet: the proxy does not log upstream errors in v1. AitoolsErrorCategoryMCPServiceNotFound AitoolsErrorCategory = "MCP_SERVICE_NOT_FOUND" AitoolsErrorCategoryMCPPermissionDenied AitoolsErrorCategory = "MCP_PERMISSION_DENIED" AitoolsErrorCategoryMCPLocationNotFound AitoolsErrorCategory = "MCP_LOCATION_NOT_FOUND"