diff --git a/acceptance/cmd/aitools/telemetry/mcp-add/.databrickscfg b/acceptance/cmd/aitools/telemetry/mcp-add/.databrickscfg new file mode 100644 index 00000000000..0369be633c5 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-add/.databrickscfg @@ -0,0 +1,3 @@ +[logfood] +host = $DATABRICKS_HOST +token = test-token-logfood diff --git a/acceptance/cmd/aitools/telemetry/mcp-add/codex-config.toml b/acceptance/cmd/aitools/telemetry/mcp-add/codex-config.toml new file mode 100644 index 00000000000..8758ff120ff --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-add/codex-config.toml @@ -0,0 +1 @@ +not = [valid diff --git a/acceptance/cmd/aitools/telemetry/mcp-add/out.test.toml b/acceptance/cmd/aitools/telemetry/mcp-add/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-add/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/cmd/aitools/telemetry/mcp-add/output.txt b/acceptance/cmd/aitools/telemetry/mcp-add/output.txt new file mode 100644 index 00000000000..132249dbdea --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-add/output.txt @@ -0,0 +1,94 @@ + +=== A new install for two agents: CHANGED per (service, agent); github needs sign-in, so its sign-in state is PENDING + +>>> [CLI] aitools mcp add --profile logfood --names system.ai.genie_one_mcp,system.ai.github --agents claude-code,codex --output json +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "ADD", + "servers": [ + { + "service": "GENIE_ONE", + "agent": "CLAUDE_CODE", + "outcome": "CHANGED", + "sign_in_state": "NOT_REQUIRED" + }, + { + "service": "GENIE_ONE", + "agent": "CODEX", + "outcome": "CHANGED", + "sign_in_state": "NOT_REQUIRED" + }, + { + "service": "GITHUB", + "agent": "CLAUDE_CODE", + "outcome": "CHANGED", + "sign_in_state": "PENDING" + }, + { + "service": "GITHUB", + "agent": "CODEX", + "outcome": "CHANGED", + "sign_in_state": "PENDING" + } + ], + "bound_profile_count": 1 +} + +=== The same servers again are re-asserted: UNCHANGED + +>>> [CLI] aitools mcp add --profile logfood --names system.ai.genie_one_mcp,system.ai.github --agents claude-code,codex --output json +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "ADD", + "servers": [ + { + "service": "GENIE_ONE", + "agent": "CLAUDE_CODE", + "outcome": "UNCHANGED", + "sign_in_state": "NOT_REQUIRED" + }, + { + "service": "GENIE_ONE", + "agent": "CODEX", + "outcome": "UNCHANGED", + "sign_in_state": "NOT_REQUIRED" + }, + { + "service": "GITHUB", + "agent": "CLAUDE_CODE", + "outcome": "UNCHANGED", + "sign_in_state": "PENDING" + }, + { + "service": "GITHUB", + "agent": "CODEX", + "outcome": "UNCHANGED", + "sign_in_state": "PENDING" + } + ], + "bound_profile_count": 1 +} + +=== An agent config that does not parse fails that agent only: FAILED with AGENT_CONFIG_INVALID + +>>> musterr [CLI] aitools mcp add --profile logfood --names system.ai.genie_one_mcp --agents claude-code,codex --output json +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "ADD", + "servers": [ + { + "service": "GENIE_ONE", + "agent": "CLAUDE_CODE", + "outcome": "UNCHANGED", + "sign_in_state": "NOT_REQUIRED" + }, + { + "service": "GENIE_ONE", + "agent": "CODEX", + "outcome": "FAILED", + "error_category": "AGENT_CONFIG_INVALID", + "sign_in_state": "NOT_REQUIRED" + } + ], + "bound_profile_count": 1 +} diff --git a/acceptance/cmd/aitools/telemetry/mcp-add/script b/acceptance/cmd/aitools/telemetry/mcp-add/script new file mode 100644 index 00000000000..6cf6f91e92a --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-add/script @@ -0,0 +1,22 @@ +sethome home +mkdir -p home/.claude home/.codex +envsubst < .databrickscfg > out && mv out .databrickscfg +export DATABRICKS_CONFIG_FILE=.databrickscfg + +# Seeding the fake is not an MCP command, so its uploads (if any) are dropped here. +MSYS_NO_PATHCONV=1 $CLI api post "/api/2.1/unity-catalog/mcp-services?parent=schemas/system.ai&mcp_service_id=genie_one_mcp" --profile logfood --json '{"comment":"Ask questions about your data"}' > /dev/null +MSYS_NO_PATHCONV=1 $CLI api post "/api/2.1/unity-catalog/mcp-services?parent=schemas/system.ai&mcp_service_id=github" --profile logfood --json '{"comment":"Repositories, issues, pull requests","config":{"source_connection":{"name":"connections/github","options":{"oauth_provider":"github"}}}}' > /dev/null +rm -f "$OUT_REQUESTS" + +title "A new install for two agents: CHANGED per (service, agent); github needs sign-in, so its sign-in state is PENDING" +trace $CLI aitools mcp add --profile logfood --names system.ai.genie_one_mcp,system.ai.github --agents claude-code,codex --output json > /dev/null +mcp_events + +title "The same servers again are re-asserted: UNCHANGED" +trace $CLI aitools mcp add --profile logfood --names system.ai.genie_one_mcp,system.ai.github --agents claude-code,codex --output json > /dev/null +mcp_events + +title "An agent config that does not parse fails that agent only: FAILED with AGENT_CONFIG_INVALID" +cp codex-config.toml home/.codex/config.toml +trace musterr $CLI aitools mcp add --profile logfood --names system.ai.genie_one_mcp --agents claude-code,codex --output json > /dev/null +mcp_events diff --git a/acceptance/cmd/aitools/telemetry/mcp-remove/.databrickscfg b/acceptance/cmd/aitools/telemetry/mcp-remove/.databrickscfg new file mode 100644 index 00000000000..d64cd1da28f --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-remove/.databrickscfg @@ -0,0 +1,7 @@ +[logfood] +host = $DATABRICKS_HOST +token = test-token-logfood + +[dogfood] +host = $DATABRICKS_HOST +token = test-token-logfood diff --git a/acceptance/cmd/aitools/telemetry/mcp-remove/out.test.toml b/acceptance/cmd/aitools/telemetry/mcp-remove/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-remove/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/cmd/aitools/telemetry/mcp-remove/output.txt b/acceptance/cmd/aitools/telemetry/mcp-remove/output.txt new file mode 100644 index 00000000000..1d732f75094 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-remove/output.txt @@ -0,0 +1,95 @@ + +=== Installed for two profiles: the bound profile count is 2 after the second add + +>>> [CLI] aitools mcp add --profile logfood --names system.ai.genie_one_mcp --agents claude-code --output json +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "ADD", + "servers": [ + { + "service": "GENIE_ONE", + "agent": "CLAUDE_CODE", + "outcome": "CHANGED", + "sign_in_state": "NOT_REQUIRED" + } + ], + "bound_profile_count": 1 +} + +>>> [CLI] aitools mcp add --profile dogfood --names system.ai.genie_one_mcp --agents claude-code --output json +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "ADD", + "servers": [ + { + "service": "GENIE_ONE", + "agent": "CLAUDE_CODE", + "outcome": "CHANGED", + "sign_in_state": "NOT_REQUIRED" + } + ], + "bound_profile_count": 2 +} + +=== remove --profile dogfood: CHANGED for its agent entry, and one profile stays bound + +>>> [CLI] aitools mcp remove --profile dogfood --output json +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "REMOVE", + "servers": [ + { + "service": "GENIE_ONE", + "agent": "CLAUDE_CODE", + "outcome": "CHANGED" + } + ], + "bound_profile_count": 1 +} + +=== An entry changed outside this CLI is left in place: SKIPPED + +>>> [CLI] aitools mcp remove --names genie_one_mcp-logfood --profile logfood --output json +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "REMOVE", + "servers": [ + { + "service": "GENIE_ONE", + "agent": "CLAUDE_CODE", + "outcome": "SKIPPED" + } + ], + "bound_profile_count": 1 +} + +=== An entry already gone from the agent config is SKIPPED, and the state record goes + +>>> [CLI] aitools mcp remove --names genie_one_mcp-logfood --profile logfood --output json +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "REMOVE", + "servers": [ + { + "service": "GENIE_ONE", + "agent": "CLAUDE_CODE", + "outcome": "CHANGED" + } + ], + "bound_profile_count": 0 +} + +=== A record with no agent left goes with a REMOVE of its own: CHANGED with no agent, and no profile bound + +>>> [CLI] aitools mcp remove --names dbsql-logfood --profile logfood --output json +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "REMOVE", + "servers": [ + { + "service": "DBSQL", + "outcome": "CHANGED" + } + ], + "bound_profile_count": 0 +} diff --git a/acceptance/cmd/aitools/telemetry/mcp-remove/record-only.state.json b/acceptance/cmd/aitools/telemetry/mcp-remove/record-only.state.json new file mode 100644 index 00000000000..e40e76fff80 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-remove/record-only.state.json @@ -0,0 +1,20 @@ +{ + "schema_version": 3, + "mcp": { + "servers": [ + { + "name": "dbsql-logfood", + "service": "system.ai.dbsql", + "profile": "logfood", + "host": "$DATABRICKS_HOST", + "workspace_id": "", + "sign_in_required": false, + "command": "", + "config_file": "$PWD/.databrickscfg", + "agents": {}, + "installed_at": "2026-01-01T00:00:00Z", + "cli_version": "0.0.0" + } + ] + } +} diff --git a/acceptance/cmd/aitools/telemetry/mcp-remove/script b/acceptance/cmd/aitools/telemetry/mcp-remove/script new file mode 100644 index 00000000000..ef98d35f619 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-remove/script @@ -0,0 +1,33 @@ +sethome home +mkdir -p home/.claude +envsubst < .databrickscfg > out && mv out .databrickscfg +export DATABRICKS_CONFIG_FILE=.databrickscfg + +MSYS_NO_PATHCONV=1 $CLI api post "/api/2.1/unity-catalog/mcp-services?parent=schemas/system.ai&mcp_service_id=genie_one_mcp" --profile logfood --json '{"comment":"Ask questions about your data"}' > /dev/null +rm -f "$OUT_REQUESTS" + +title "Installed for two profiles: the bound profile count is 2 after the second add" +trace $CLI aitools mcp add --profile logfood --names system.ai.genie_one_mcp --agents claude-code --output json > /dev/null +mcp_events +trace $CLI aitools mcp add --profile dogfood --names system.ai.genie_one_mcp --agents claude-code --output json > /dev/null +mcp_events + +title "remove --profile dogfood: CHANGED for its agent entry, and one profile stays bound" +trace $CLI aitools mcp remove --profile dogfood --output json > /dev/null +mcp_events + +title "An entry changed outside this CLI is left in place: SKIPPED" +jq '.mcpServers["genie_one_mcp-logfood"].command = "/usr/bin/other"' home/.claude.json > home/claude.tmp && mv home/claude.tmp home/.claude.json +trace $CLI aitools mcp remove --names genie_one_mcp-logfood --profile logfood --output json > /dev/null +mcp_events + +title "An entry already gone from the agent config is SKIPPED, and the state record goes" +jq 'del(.mcpServers["genie_one_mcp-logfood"])' home/.claude.json > home/claude.tmp && mv home/claude.tmp home/.claude.json +trace $CLI aitools mcp remove --names genie_one_mcp-logfood --profile logfood --output json > /dev/null +mcp_events + +title "A record with no agent left goes with a REMOVE of its own: CHANGED with no agent, and no profile bound" +mkdir -p home/.databricks/aitools/skills +envsubst < record-only.state.json > home/.databricks/aitools/skills/.state.json +trace $CLI aitools mcp remove --names dbsql-logfood --profile logfood --output json > /dev/null +mcp_events diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve/.databrickscfg b/acceptance/cmd/aitools/telemetry/mcp-serve/.databrickscfg new file mode 100644 index 00000000000..0369be633c5 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-serve/.databrickscfg @@ -0,0 +1,3 @@ +[logfood] +host = $DATABRICKS_HOST +token = test-token-logfood diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve/out.test.toml b/acceptance/cmd/aitools/telemetry/mcp-serve/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-serve/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve/output.txt b/acceptance/cmd/aitools/telemetry/mcp-serve/output.txt new file mode 100644 index 00000000000..d8fec5e3dcf --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-serve/output.txt @@ -0,0 +1,44 @@ + +=== serve logs PROXY_STARTED after the binding checks; the upload is sent when stdin closes + +>>> [CLI] aitools mcp serve --profile logfood --service system.ai.genie_one_mcp --agent claude-code +No install record for system.ai.genie_one_mcp on profile "logfood"; using its current host [DATABRICKS_URL] +{ + "calling_agent": "CLAUDE_CODE", + "operation": "PROXY_STARTED", + "servers": [ + { + "service": "GENIE_ONE" + } + ], + "bound_profile_count": 0 +} + +=== A binding that fails logs no PROXY_STARTED, so no upload follows + +>>> musterr [CLI] aitools mcp serve --profile gone --service system.ai.genie_one_mcp --agent codex +MCP server system.ai.genie_one_mcp needs Databricks profile "gone", which does not exist in [TEST_TMP_DIR]/.databrickscfg. Set it up with: databricks auth login --profile gone + +=== authenticate opens the sign-in page: PROXY_STARTED, then SIGN_IN_PROMPTED with the sign-in state after the wait +{ + "calling_agent": "CLAUDE_CODE", + "operation": "PROXY_STARTED", + "servers": [ + { + "service": "GITHUB" + } + ], + "bound_profile_count": 0 +} +{ + "calling_agent": "CLAUDE_CODE", + "operation": "SIGN_IN_PROMPTED", + "servers": [ + { + "service": "GITHUB", + "error_category": "SIGN_IN_REQUIRED", + "sign_in_state": "ACTIVE" + } + ], + "bound_profile_count": 0 +} diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve/requests-after.ndjson b/acceptance/cmd/aitools/telemetry/mcp-serve/requests-after.ndjson new file mode 100644 index 00000000000..57d5e4a4066 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-serve/requests-after.ndjson @@ -0,0 +1,2 @@ +{"jsonrpc":"2.0","id":4,"method":"tools/list"} +{"jsonrpc":"2.0","id":5,"method":"tools/call","params":{"name":"echo","arguments":{"text":"hello"}}} diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve/requests-authenticate.ndjson b/acceptance/cmd/aitools/telemetry/mcp-serve/requests-authenticate.ndjson new file mode 100644 index 00000000000..cc792544a79 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-serve/requests-authenticate.ndjson @@ -0,0 +1 @@ +{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"authenticate","arguments":{}}} diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve/requests-list.ndjson b/acceptance/cmd/aitools/telemetry/mcp-serve/requests-list.ndjson new file mode 100644 index 00000000000..84c14278376 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-serve/requests-list.ndjson @@ -0,0 +1,3 @@ +{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-06-18","capabilities":{},"clientInfo":{"name":"test-client","version":"0.0.1"}}} +{"jsonrpc":"2.0","method":"notifications/initialized"} +{"jsonrpc":"2.0","id":2,"method":"tools/list"} diff --git a/acceptance/cmd/aitools/telemetry/mcp-serve/script b/acceptance/cmd/aitools/telemetry/mcp-serve/script new file mode 100644 index 00000000000..0a5108cdb15 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-serve/script @@ -0,0 +1,32 @@ +sethome home +envsubst < .databrickscfg > out && mv out .databrickscfg +export DATABRICKS_CONFIG_FILE=.databrickscfg + +# No browser: the proxy prints the sign-in URL and does not open anything. +export BROWSER=none +export RETRY_MAX_ATTEMPTS=60 +# Seeding the fake is not an MCP command, so its uploads (if any) are dropped here. +MSYS_NO_PATHCONV=1 $CLI api post "/api/2.1/unity-catalog/mcp-services?parent=schemas/system.ai&mcp_service_id=genie_one_mcp" --profile logfood --json '{"comment":"Ask questions about your data"}' > /dev/null +MSYS_NO_PATHCONV=1 $CLI api post "/api/2.1/unity-catalog/mcp-services?parent=schemas/system.ai&mcp_service_id=github" --profile logfood --json '{"comment":"Repositories, issues, pull requests","config":{"source_connection":{"name":"connections/github","options":{"oauth_provider":"github"}}}}' > /dev/null +rm -f "$OUT_REQUESTS" + +title "serve logs PROXY_STARTED after the binding checks; the upload is sent when stdin closes" +trace $CLI aitools mcp serve --profile logfood --service system.ai.genie_one_mcp --agent claude-code < requests-list.ndjson > /dev/null +mcp_events + +title "A binding that fails logs no PROXY_STARTED, so no upload follows" +trace musterr $CLI aitools mcp serve --profile gone --service system.ai.genie_one_mcp --agent codex < /dev/null > /dev/null +mcp_events + +title "authenticate opens the sign-in page: PROXY_STARTED, then SIGN_IN_PROMPTED with the sign-in state after the wait" +{ + cat requests-list.ndjson + retry --until '"id":2' cat serve.stdout.txt > /dev/null + cat requests-authenticate.ndjson + retry --until "Waiting for sign-in" cat serve.stderr.txt > /dev/null + MSYS_NO_PATHCONV=1 $CLI api post /api/2.1/unity-catalog/mcp-services/system.ai.github/user-credentials --profile logfood --json '{}' > /dev/null + retry --until '"id":3' cat serve.stdout.txt > /dev/null + cat requests-after.ndjson +} | $CLI aitools mcp serve --profile logfood --service system.ai.github --agent claude-code > serve.stdout.txt 2> serve.stderr.txt +mcp_events +rm serve.stdout.txt serve.stderr.txt diff --git a/acceptance/cmd/aitools/telemetry/mcp-sign-in/.databrickscfg b/acceptance/cmd/aitools/telemetry/mcp-sign-in/.databrickscfg new file mode 100644 index 00000000000..0369be633c5 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-sign-in/.databrickscfg @@ -0,0 +1,3 @@ +[logfood] +host = $DATABRICKS_HOST +token = test-token-logfood diff --git a/acceptance/cmd/aitools/telemetry/mcp-sign-in/out.test.toml b/acceptance/cmd/aitools/telemetry/mcp-sign-in/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-sign-in/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/cmd/aitools/telemetry/mcp-sign-in/output.txt b/acceptance/cmd/aitools/telemetry/mcp-sign-in/output.txt new file mode 100644 index 00000000000..46253de58e8 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-sign-in/output.txt @@ -0,0 +1,106 @@ + +=== login to a service that needs no sign-in: UNCHANGED, sign-in state NOT_REQUIRED + +>>> [CLI] aitools mcp login system.ai.genie_one_mcp --profile logfood +system.ai.genie_one_mcp does not need sign-in. +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "SIGN_IN", + "servers": [ + { + "service": "GENIE_ONE", + "outcome": "UNCHANGED", + "sign_in_state": "NOT_REQUIRED" + } + ], + "bound_profile_count": 0 +} + +=== login completes once the credential is ACTIVE: CHANGED, sign-in state ACTIVE +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "SIGN_IN", + "servers": [ + { + "service": "GITHUB", + "outcome": "CHANGED", + "sign_in_state": "ACTIVE" + } + ], + "bound_profile_count": 0 +} + +=== login while already signed in: UNCHANGED + +>>> [CLI] aitools mcp login system.ai.github --profile logfood +Already signed in to system.ai.github on profile logfood. +Public repositories only until a repository owner installs https://github.com/apps/databricks-github-connector. +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "SIGN_IN", + "servers": [ + { + "service": "GITHUB", + "outcome": "UNCHANGED", + "sign_in_state": "ACTIVE" + } + ], + "bound_profile_count": 0 +} + +=== logout removes the sign-in: CHANGED; a second logout finds nothing: UNCHANGED + +>>> [CLI] aitools mcp logout system.ai.github --profile logfood +Signed out of system.ai.github on profile logfood. +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "SIGN_OUT", + "servers": [ + { + "service": "GITHUB", + "outcome": "CHANGED" + } + ], + "bound_profile_count": 0 +} + +>>> [CLI] aitools mcp logout system.ai.github --profile logfood +Not signed in to system.ai.github on profile logfood. +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "SIGN_OUT", + "servers": [ + { + "service": "GITHUB", + "outcome": "UNCHANGED" + } + ], + "bound_profile_count": 0 +} + +=== add --login signs in during the same run: SIGN_IN first, then ADD with sign-in state ACTIVE +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "SIGN_IN", + "servers": [ + { + "service": "GITHUB", + "outcome": "CHANGED", + "sign_in_state": "ACTIVE" + } + ], + "bound_profile_count": 1 +} +{ + "calling_agent": "TYPE_UNSPECIFIED", + "operation": "ADD", + "servers": [ + { + "service": "GITHUB", + "agent": "CLAUDE_CODE", + "outcome": "CHANGED", + "sign_in_state": "ACTIVE" + } + ], + "bound_profile_count": 1 +} diff --git a/acceptance/cmd/aitools/telemetry/mcp-sign-in/script b/acceptance/cmd/aitools/telemetry/mcp-sign-in/script new file mode 100644 index 00000000000..6ddd1bce499 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/mcp-sign-in/script @@ -0,0 +1,44 @@ +sethome home +mkdir -p home/.claude +envsubst < .databrickscfg > out && mv out .databrickscfg +export DATABRICKS_CONFIG_FILE=.databrickscfg + +# No browser: the CLI prints the sign-in URL and polls. +export BROWSER=none +export RETRY_MAX_ATTEMPTS=60 +# Seeding the fake is not an MCP command, so its uploads (if any) are dropped here. +MSYS_NO_PATHCONV=1 $CLI api post "/api/2.1/unity-catalog/mcp-services?parent=schemas/system.ai&mcp_service_id=genie_one_mcp" --profile logfood --json '{"comment":"Ask questions about your data"}' > /dev/null +MSYS_NO_PATHCONV=1 $CLI api post "/api/2.1/unity-catalog/mcp-services?parent=schemas/system.ai&mcp_service_id=github" --profile logfood --json '{"comment":"Repositories, issues, pull requests","config":{"source_connection":{"name":"connections/github","options":{"oauth_provider":"github"}}}}' > /dev/null +rm -f "$OUT_REQUESTS" + +title "login to a service that needs no sign-in: UNCHANGED, sign-in state NOT_REQUIRED" +trace $CLI aitools mcp login system.ai.genie_one_mcp --profile logfood > /dev/null +mcp_events + +title "login completes once the credential is ACTIVE: CHANGED, sign-in state ACTIVE" +$CLI aitools mcp login system.ai.github --profile logfood > login.txt 2>&1 & +PID=$! +retry --until "Waiting for sign-in" cat login.txt > /dev/null +MSYS_NO_PATHCONV=1 $CLI api post /api/2.1/unity-catalog/mcp-services/system.ai.github/user-credentials --profile logfood --json '{}' > /dev/null +wait $PID +mcp_events +rm login.txt + +title "login while already signed in: UNCHANGED" +trace $CLI aitools mcp login system.ai.github --profile logfood > /dev/null +mcp_events + +title "logout removes the sign-in: CHANGED; a second logout finds nothing: UNCHANGED" +trace $CLI aitools mcp logout system.ai.github --profile logfood > /dev/null +mcp_events +trace $CLI aitools mcp logout system.ai.github --profile logfood > /dev/null +mcp_events + +title "add --login signs in during the same run: SIGN_IN first, then ADD with sign-in state ACTIVE" +$CLI aitools mcp add --profile logfood --names system.ai.github --agents claude-code --login > add.txt 2>&1 & +PID=$! +retry --until "Waiting for sign-in" cat add.txt > /dev/null +MSYS_NO_PATHCONV=1 $CLI api post /api/2.1/unity-catalog/mcp-services/system.ai.github/user-credentials --profile logfood --json '{}' > /dev/null +wait $PID +mcp_events +rm add.txt diff --git a/acceptance/cmd/aitools/telemetry/script.prepare b/acceptance/cmd/aitools/telemetry/script.prepare index f55e1cef710..6a9112e4947 100644 --- a/acceptance/cmd/aitools/telemetry/script.prepare +++ b/acceptance/cmd/aitools/telemetry/script.prepare @@ -15,3 +15,11 @@ print_telemetry() { + (del(.execution_context))' fi } + +# Prints the aitools_mcp_event payloads uploaded since the last call, then forgets the recorded uploads. The uploads are +# filtered by path with print_requests.py; jq reads only the body. +mcp_events() { + if [ -f "$OUT_REQUESTS" ]; then + print_requests.py //telemetry-ext | jq '.body.protoLogs[] | fromjson | .entry.databricks_cli_log.aitools_mcp_event | select(. != null)' + fi +} diff --git a/cmd/aitools/mcp.go b/cmd/aitools/mcp.go index 1f81d5fb0d0..7552c8be62f 100644 --- a/cmd/aitools/mcp.go +++ b/cmd/aitools/mcp.go @@ -16,19 +16,6 @@ import ( "github.com/spf13/cobra" ) -// Error categories that the telemetry enum does not define yet. The JSON output reports them by name. The -// binding categories come from mcp.BindCategory, which has the same names. -const ( - categoryAgentConfigInvalid protos.AitoolsErrorCategory = "AGENT_CONFIG_INVALID" - categoryNameConflict protos.AitoolsErrorCategory = "MCP_NAME_CONFLICT" - categoryServiceNotFound protos.AitoolsErrorCategory = "MCP_SERVICE_NOT_FOUND" - categoryNoProfile protos.AitoolsErrorCategory = "NO_PROFILE" - categoryProfileRequired protos.AitoolsErrorCategory = "PROFILE_REQUIRED" - categoryPermissionDenied protos.AitoolsErrorCategory = "MCP_PERMISSION_DENIED" - categoryLocationNotFound protos.AitoolsErrorCategory = "MCP_LOCATION_NOT_FOUND" - categorySignInTimeout protos.AitoolsErrorCategory = "SIGN_IN_TIMEOUT" -) - // NewMcpCmd returns the `aitools mcp` group. It stays hidden until the whole MCP // stack has landed, so a release never ships `add` without `serve`. func NewMcpCmd() *cobra.Command { @@ -75,9 +62,9 @@ func errorCategory(err error) protos.AitoolsErrorCategory { } switch { case errors.Is(err, agentconfig.ErrConfigInvalid): - return categoryAgentConfigInvalid + return protos.AitoolsErrorCategoryAgentConfigInvalid case errors.Is(err, agentconfig.ErrNotOwned): - return categoryNameConflict + return protos.AitoolsErrorCategoryMCPNameConflict } return classifyInstallError(err) } @@ -107,11 +94,11 @@ func resolveProfile(ctx context.Context, flagValue string) (string, error) { } switch len(profiles) { case 0: - return "", failure(categoryNoProfile, nil, "no workspace profiles in %s. Run: databricks auth login --host --profile ", filepath.ToSlash(path)) + return "", failure(protos.AitoolsErrorCategoryNoProfile, nil, "no workspace profiles in %s. Run: databricks auth login --host --profile ", filepath.ToSlash(path)) case 1: return profiles[0].Name, nil } - return "", failure(categoryProfileRequired, &profileChoice{profiles: profiles}, "several workspace profiles exist; pass --profile. Found: %s", strings.Join(profiles.Names(), ", ")) + return "", failure(protos.AitoolsErrorCategoryProfileRequired, &profileChoice{profiles: profiles}, "several workspace profiles exist; pass --profile. Found: %s", strings.Join(profiles.Names(), ", ")) } // profileFlag returns the root --profile value, or "" when it is not set. diff --git a/cmd/aitools/mcp_add.go b/cmd/aitools/mcp_add.go index b97c9e3eaca..8ee0949aaad 100644 --- a/cmd/aitools/mcp_add.go +++ b/cmd/aitools/mcp_add.go @@ -177,6 +177,11 @@ Supported agents: ` + strings.Join(mcpAgentNames(), ", "), ErrorCategory: protos.AitoolsErrorCategoryUnspecified, } runErr := runMcpAdd(ctx, cmd, f, jsonMode, out) + logMcpEvent(ctx, protos.AitoolsMcpEvent{ + CallingAgent: callingAgentType(), + Operation: protos.AitoolsMcpOperationAdd, + Servers: addServerResults(out.Servers), + }) if !jsonMode { return runErr } @@ -328,6 +333,9 @@ func runMcpAdd(ctx context.Context, cmd *cobra.Command, f mcpAddFlags, jsonMode if err := planningRefusals(plans, targets); err != nil { return err } + // Every refusal that needs no request has come, so the upload authenticates with the profile this run binds. Resolving + // its config makes the host's metadata request, which is why it waits for the refusals. + setUploadConfig(ctx, profileName, configFile) if !pickingServices { w, workspaceID, err = connectWorkspace(ctx, out, profileName, configFile, pinned) @@ -542,9 +550,9 @@ func describeListError(profileName string, err error) error { var category protos.AitoolsErrorCategory switch listErr.Kind { case mcp.ListErrLocationNotFound: - category = categoryLocationNotFound + category = protos.AitoolsErrorCategoryMCPLocationNotFound case mcp.ListErrPermissionDenied: - category = categoryPermissionDenied + category = protos.AitoolsErrorCategoryMCPPermissionDenied case mcp.ListErrRateLimited: category = protos.AitoolsErrorCategoryRateLimited default: @@ -705,13 +713,13 @@ func pickServices(location string, names []string, services []mcp.Service) ([]mc i := slices.IndexFunc(services, func(s mcp.Service) bool { return strings.EqualFold(s.FullName, name) }) if i < 0 { if len(services) == 0 { - return nil, failure(categoryServiceNotFound, nil, "%s is not an MCP service in %s, which has none", name, location) + return nil, failure(protos.AitoolsErrorCategoryMCPServiceNotFound, nil, "%s is not an MCP service in %s, which has none", name, location) } available := make([]string, 0, len(services)) for _, s := range services { available = append(available, s.FullName) } - return nil, failure(categoryServiceNotFound, nil, "%s is not an MCP service in %s. Available: %s", name, location, strings.Join(available, ", ")) + return nil, failure(protos.AitoolsErrorCategoryMCPServiceNotFound, nil, "%s is not an MCP service in %s. Available: %s", name, location, strings.Join(available, ", ")) } picked = append(picked, services[i]) } @@ -846,7 +854,7 @@ func chooseName(ctx context.Context, in planInput, state *mcp.State, fullName st case 1: stored = owned[0] default: - return "", nil, failure(categoryNameConflict, nil, "%s is installed on profile %s under more than one name in the agents (%s); remove the extra entries", fullName, b.profile, strings.Join(owned, ", ")) + return "", nil, failure(protos.AitoolsErrorCategoryMCPNameConflict, nil, "%s is installed on profile %s under more than one name in the agents (%s); remove the extra entries", fullName, b.profile, strings.Join(owned, ", ")) } } @@ -855,7 +863,7 @@ func chooseName(ctx context.Context, in planInput, state *mcp.State, fullName st return "", nil, failure(protos.AitoolsErrorCategoryInvalidFlags, nil, "%s is already installed on profile %s as %s; remove it before using --name %s", fullName, b.profile, stored, in.nameFlag) } if stateHolds(state, stored, entry) { - return "", nil, failure(categoryNameConflict, nil, "%s is installed as %s, but another server in the state holds %s; remove that entry first", fullName, stored, stored) + return "", nil, failure(protos.AitoolsErrorCategoryMCPNameConflict, nil, "%s is installed as %s, but another server in the state holds %s; remove that entry first", fullName, stored, stored) } failed, err := nameFailures(ctx, in, fullName, stored, true) return stored, failed, err @@ -943,7 +951,7 @@ func holderConflict(ctx context.Context, in planInput, a *agents.Agent, fullName if in.force { return nil, nil } - return failure(categoryNameConflict, agentconfig.ErrNotOwned, "%s already has a server named %s that this CLI did not write; pass --force to replace it", a.DisplayName, name), nil + return failure(protos.AitoolsErrorCategoryMCPNameConflict, agentconfig.ErrNotOwned, "%s already has a server named %s that this CLI did not write; pass --force to replace it", a.DisplayName, name), nil } serves, err := mcp.EntryServes(ctx, entry, fullName, in.binding.profile, in.binding.configFile) if err != nil { @@ -953,7 +961,7 @@ func holderConflict(ctx context.Context, in planInput, a *agents.Agent, fullName case serves: return nil, nil case stored: - return failure(categoryNameConflict, nil, "%s is installed as %s, but %s holds %s for another server; remove that entry first", fullName, name, a.Name, name), nil + return failure(protos.AitoolsErrorCategoryMCPNameConflict, nil, "%s is installed as %s, but %s holds %s for another server; remove that entry first", fullName, name, a.Name, name), nil default: return failure(protos.AitoolsErrorCategoryInvalidFlags, nil, "--name %s is used by another server in %s; pick a different name", name, a.Name), nil } @@ -1076,6 +1084,7 @@ func signInInstalled(ctx context.Context, errOut io.Writer, w *databricks.Worksp if signedIn(ctx, w, plans[i].service.FullName) { server.SignIn = &addSignInOutput{State: signInStateActive} reportSignedIn(errOut, plans[i].service, req.profile, true) + logMcpEvent(ctx, signInEventOf(server.Service, signInStateActive, true, nil)) continue } yes, err := promptSignInNow(server.Service) @@ -1090,8 +1099,9 @@ func signInInstalled(ctx context.Context, errOut io.Writer, w *databricks.Worksp } } url := mcp.SignInURL(w.Config.Host, plans[i].service.FullName, req.workspaceID) - state, _, _, err := signIn(ctx, errOut, w, req.pinned, url, plans[i].service, signInOptions{Profile: req.profile, Open: browser.OpenOutputToStderr}) + state, alreadySignedIn, _, err := signIn(ctx, errOut, w, req.pinned, url, plans[i].service, signInOptions{Profile: req.profile, Open: browser.OpenOutputToStderr}) server.SignIn = &addSignInOutput{State: state} + logMcpEvent(ctx, signInEventOf(server.Service, state, alreadySignedIn, err)) if err != nil { server.SignIn.ErrorCategory = errorCategory(err) server.SignIn.Error = err.Error() diff --git a/cmd/aitools/mcp_add_test.go b/cmd/aitools/mcp_add_test.go index 05c89ca7509..3b2dd9879ee 100644 --- a/cmd/aitools/mcp_add_test.go +++ b/cmd/aitools/mcp_add_test.go @@ -201,7 +201,7 @@ func TestPickServices(t *testing.T) { _, err = pickServices("system.ai", []string{"system.ai.foo"}, services) require.Error(t, err) assert.Equal(t, "system.ai.foo is not an MCP service in system.ai. Available: system.ai.genie_one_mcp, system.ai.github", err.Error()) - assert.Equal(t, categoryServiceNotFound, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPServiceNotFound, errorCategory(err)) } func TestResolveMCPAgents(t *testing.T) { @@ -227,7 +227,7 @@ func TestDescribeListErrorNamesProfileAndKeepsCause(t *testing.T) { err := describeListError("logfood", listErr) assert.Equal(t, "profile logfood: "+listErr.Error(), err.Error()) - assert.Equal(t, categoryPermissionDenied, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPPermissionDenied, errorCategory(err)) got, ok := errors.AsType[*mcp.ListError](err) require.True(t, ok) @@ -238,7 +238,7 @@ func TestDescribeListErrorNamesProfileAndKeepsCause(t *testing.T) { assert.Equal(t, protos.AitoolsErrorCategoryRateLimited, errorCategory(rateLimited)) notFound := describeListError("logfood", &mcp.ListError{Kind: mcp.ListErrLocationNotFound, Location: "main.tools", Err: cause}) - assert.Equal(t, categoryLocationNotFound, errorCategory(notFound)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPLocationNotFound, errorCategory(notFound)) plain := errors.New("network down") assert.Equal(t, plain, describeListError("logfood", plain)) @@ -277,7 +277,8 @@ func withDatabricksBinary(t *testing.T) { } // noRequestServer returns a server that fails the test on any request. A command that refuses before -// it contacts the profile's host never reaches it. +// it contacts the profile's host never reaches it, with telemetry on or off: the upload's config is resolved only after +// the binding checks have passed, and that resolution makes no request. func noRequestServer(t *testing.T) *httptest.Server { t.Helper() server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -517,7 +518,7 @@ func TestPlanAddsNaming(t *testing.T) { require.NoError(t, os.WriteFile(filepath.Join(openCode, "opencode.json"), []byte(`{"mcp":{"servers":{"two":{"type":"local","command":["databricks","aitools","mcp","serve","--profile","logfood","--service","system.ai.github","--agent","opencode"]}}}}`), 0o600)) _, err := planNames(t, "", defaultBinding(t, "logfood"), &mcp.State{}, []string{"system.ai.github"}, []*agents.Agent{claude, agents.ByName("opencode")}) require.Error(t, err) - assert.Equal(t, categoryNameConflict, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPNameConflict, errorCategory(err)) }) t.Run("OpenCode entry with its variables in environment is reused", func(t *testing.T) { @@ -560,7 +561,7 @@ func TestPlanAddsNaming(t *testing.T) { require.Len(t, plans, 1) assert.Equal(t, "github-logfood", plans[0].name) require.Contains(t, plans[0].failed, claude) - assert.Equal(t, categoryNameConflict, errorCategory(plans[0].failed[claude])) + assert.Equal(t, protos.AitoolsErrorCategoryMCPNameConflict, errorCategory(plans[0].failed[claude])) }) t.Run("planning reads the agent entries it is given, not the files", func(t *testing.T) { @@ -794,7 +795,7 @@ func TestAgentFailureErrorWhenAServiceReachedNoAgent(t *testing.T) { } func TestErrorCategoryMapsTypedFailures(t *testing.T) { - assert.Equal(t, categoryNameConflict, + assert.Equal(t, protos.AitoolsErrorCategoryMCPNameConflict, errorCategory(fmt.Errorf("claude-code: server %q: %w", "github", agentconfig.ErrNotOwned))) assert.Equal(t, protos.AitoolsErrorCategoryTimeout, errorCategory(fmt.Errorf("list: %w", context.DeadlineExceeded))) assert.Equal(t, protos.AitoolsErrorCategoryPermissionDenied, errorCategory(&fs.PathError{Op: "open", Path: "x", Err: fs.ErrPermission})) @@ -952,7 +953,7 @@ func TestMcpAddRefusesStoredNameHeldByOtherBinding(t *testing.T) { err := runAdd(t, "--profile", "logfood", "--names", "github", "--agents", "claude-code") require.Error(t, err) - assert.Equal(t, categoryNameConflict, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPNameConflict, errorCategory(err)) assert.ErrorContains(t, err, "github-logfood") got, err := os.ReadFile(claudeFile) require.NoError(t, err) @@ -1137,7 +1138,7 @@ func TestPlanAddsRefusesRecoveredNameHeldByAnotherServer(t *testing.T) { _, err := planNames(t, "", defaultBinding(t, "logfood"), state, []string{"system.ai.github"}, []*agents.Agent{agents.ByName("claude-code")}) require.Error(t, err) - assert.Equal(t, categoryNameConflict, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPNameConflict, errorCategory(err)) assert.Equal(t, "system.ai.github is installed as github-logfood, but another server in the state holds github-logfood; remove that entry first", err.Error()) } @@ -1156,7 +1157,7 @@ func TestMcpAddRefusesUnmanagedEntryBeforeAnyRequest(t *testing.T) { err := runAdd(t, "--profile", "logfood", "--names", "github", "--agents", "claude-code") require.Error(t, err) - assert.Equal(t, categoryNameConflict, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPNameConflict, errorCategory(err)) assert.ErrorContains(t, err, "Claude Code already has a server named github-logfood that this CLI did not write; pass --force to replace it") got, err := os.ReadFile(claudeFile) require.NoError(t, err) @@ -1280,14 +1281,14 @@ func TestPlanAddsDetectedAgentConflictFailsOnlyThatAgent(t *testing.T) { require.Len(t, plans, 1) assert.Equal(t, "github-logfood", plans[0].name) require.Contains(t, plans[0].failed, claude) - assert.Equal(t, categoryNameConflict, errorCategory(plans[0].failed[claude])) + assert.Equal(t, protos.AitoolsErrorCategoryMCPNameConflict, errorCategory(plans[0].failed[claude])) assert.NotContains(t, plans[0].failed, codex) assert.NoError(t, planningRefusals(plans, targets)) in.explicit = true _, err = planAdds(ctx, in, state, []string{"system.ai.github"}) require.Error(t, err) - assert.Equal(t, categoryNameConflict, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPNameConflict, errorCategory(err)) onlyClaude := []*agents.Agent{claude} in = planInput{binding: defaultBinding(t, "logfood"), targets: onlyClaude, held: readHeld(ctx, onlyClaude)} @@ -1295,7 +1296,7 @@ func TestPlanAddsDetectedAgentConflictFailsOnlyThatAgent(t *testing.T) { require.NoError(t, err) refusal := planningRefusals(plans, onlyClaude) require.Error(t, refusal) - assert.Equal(t, categoryNameConflict, errorCategory(refusal)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPNameConflict, errorCategory(refusal)) } // TestPlanAddsUnmanagedEntryIsReplacedOnlyWithForce checks that a stored name held by an entry this CLI did not write is refused at planning, @@ -1310,7 +1311,7 @@ func TestPlanAddsUnmanagedEntryIsReplacedOnlyWithForce(t *testing.T) { in := planInput{binding: defaultBinding(t, "logfood"), targets: targets, held: held, explicit: true} _, err := planAdds(ctx, in, state, []string{"system.ai.github"}) require.Error(t, err) - assert.Equal(t, categoryNameConflict, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPNameConflict, errorCategory(err)) in.force = true plans, err := planAdds(ctx, in, state, []string{"system.ai.github"}) @@ -1323,7 +1324,7 @@ func TestPickServicesNamesSchemaWithNoServices(t *testing.T) { _, err := pickServices("system.ai", []string{"system.ai.github"}, nil) require.Error(t, err) assert.Equal(t, "system.ai.github is not an MCP service in system.ai, which has none", err.Error()) - assert.Equal(t, categoryServiceNotFound, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPServiceNotFound, errorCategory(err)) } // TestInstallServerWritesOnlyAgentsInstalledForMCP checks that an agent named with --agents is written when it counts as installed for MCP, and that diff --git a/cmd/aitools/mcp_list.go b/cmd/aitools/mcp_list.go index 61872786fe6..f109fb0be14 100644 --- a/cmd/aitools/mcp_list.go +++ b/cmd/aitools/mcp_list.go @@ -128,6 +128,16 @@ type profileKey struct { path string } +// profileGroup returns the group of a server entry in list: its profile in the config file that check resolved, or in the +// file as written when that file does not resolve. +func profileGroup(e mcp.Entry, check mcp.EntryCheck) profileKey { + path := check.Path + if path == "" { + path = e.ConfigFile + } + return profileKey{profile: e.Profile, path: path} +} + // NewMcpListCmd returns `aitools mcp list`. Without --check it reads the state file and the agent configs only, and // makes no network calls. Drift and unreadable entries are reported, not failed: the command exits 0. func NewMcpListCmd() *cobra.Command { @@ -229,12 +239,8 @@ func buildMcpList(ctx context.Context, profileName string) (mcpListOutput, error warnedFiles := map[string]bool{} for i, e := range entries { check := checks[i] - path := check.Path - if path == "" { - // The file cannot be resolved, so the entry is grouped under its file as written. - path = e.ConfigFile - } - key := profileKey{profile: e.Profile, path: path} + key := profileGroup(e, check) + path := key.path p, ok := groups[key] if !ok { p = &mcpListProfile{Profile: e.Profile, Host: e.Host, Status: statusOK, Servers: []mcpListServer{}, configPath: path} diff --git a/cmd/aitools/mcp_login.go b/cmd/aitools/mcp_login.go index 26b222a4479..06f909eb38d 100644 --- a/cmd/aitools/mcp_login.go +++ b/cmd/aitools/mcp_login.go @@ -45,6 +45,9 @@ type mcpSignInOutput struct { URL string `json:"url,omitempty"` ErrorCategory protos.AitoolsErrorCategory `json:"error_category,omitempty"` Error string `json:"error,omitempty"` + + // alreadySignedIn is set by login when the caller was signed in before it ran. + alreadySignedIn bool } // mcpSignInAction runs login or logout against the profile's client and records the result in out. pinnedFile is the @@ -63,7 +66,7 @@ sign-in to complete. Without a browser, open the printed URL yourself. is a full name such as system.ai.github. The profile is the one whose user signs in.`, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - return runMcpSignInCommand(cmd, args[0], mcpLogin) + return runMcpSignInCommand(cmd, args[0], mcpLogin, protos.AitoolsMcpOperationSignIn) }, } } @@ -79,19 +82,23 @@ stays in place. Running it when you are not signed in is not an error. is a full name such as system.ai.github.`, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - return runMcpSignInCommand(cmd, args[0], mcpLogout) + return runMcpSignInCommand(cmd, args[0], mcpLogout, protos.AitoolsMcpOperationSignOut) }, } } // runMcpSignInCommand resolves the profile for service and runs action. In JSON mode the document is // the only stdout output; progress goes to stderr. -func runMcpSignInCommand(cmd *cobra.Command, service string, action mcpSignInAction) error { +func runMcpSignInCommand(cmd *cobra.Command, service string, action mcpSignInAction, operation protos.AitoolsMcpOperation) error { ctx := cmd.Context() jsonMode := installOutputIsJSON(cmd) out := &mcpSignInOutput{Service: service} runErr := runMcpSignIn(ctx, cmd.ErrOrStderr(), profileFlag(cmd), service, action, out) + // The host is set once the profile's binding has passed and its client exists, so a run that reached no server logs nothing. + if out.Host != "" { + logMcpEvent(ctx, signInEvent(operation, out, runErr)) + } if !jsonMode { return runErr } @@ -140,6 +147,7 @@ func runMcpSignIn(ctx context.Context, errOut io.Writer, flagValue, service stri if err != nil { return err } + setUploadConfig(ctx, profileName, configFile) // A server pinned to a config file names that file in the hints, as add does. pinnedFile := mcp.PinnedFile(entry, configFile) w, err := profileClient(ctx, profileName, configFile, pinnedFile) @@ -150,12 +158,24 @@ func runMcpSignIn(ctx context.Context, errOut io.Writer, flagValue, service stri return action(ctx, errOut, w, out, pinnedFile) } +// signInEvent is the SIGN_IN or SIGN_OUT event of one login or logout. +func signInEvent(operation protos.AitoolsMcpOperation, out *mcpSignInOutput, err error) protos.AitoolsMcpEvent { + if operation == protos.AitoolsMcpOperationSignIn { + return signInEventOf(out.Service, out.State, out.alreadySignedIn, err) + } + return protos.AitoolsMcpEvent{ + CallingAgent: callingAgentType(), + Operation: operation, + Servers: []protos.AitoolsMcpServerResult{signOutServerResult(out.Service, out.State, err)}, + } +} + // serviceFor returns the service with its sign-in requirement, as the profile sees it. A service the profile cannot see // is an error, so that a misspelled name is reported rather than treated as a service with no sign-in. func serviceFor(ctx context.Context, w *databricks.WorkspaceClient, profileName, pinnedFile, service string) (mcp.Service, error) { svc, err := mcp.GetService(ctx, w, service) if mcp.IsNotFound(err) { - return mcp.Service{}, failure(categoryServiceNotFound, err, "%s is not an MCP service visible to profile %s", service, profileName) + return mcp.Service{}, failure(protos.AitoolsErrorCategoryMCPServiceNotFound, err, "%s is not an MCP service visible to profile %s", service, profileName) } if err != nil { return mcp.Service{}, describeRequestError(profileName, pinnedFile, err) @@ -183,8 +203,9 @@ func mcpLogin(ctx context.Context, errOut io.Writer, w *databricks.WorkspaceClie return describeRequestError(out.Profile, pinnedFile, fmt.Errorf("cannot read the workspace ID of %s: %w", w.Config.Host, err)) } url := mcp.SignInURL(out.Host, svc.FullName, workspaceID) - state, _, urlShown, err := signIn(ctx, errOut, w, pinnedFile, url, svc, signInOptions{Profile: out.Profile, Open: browser.OpenOutputToStderr}) + state, alreadySignedIn, urlShown, err := signIn(ctx, errOut, w, pinnedFile, url, svc, signInOptions{Profile: out.Profile, Open: browser.OpenOutputToStderr}) out.State = state + out.alreadySignedIn = alreadySignedIn // The URL is named when it was printed. A sign-in that failed before its page is printed names none. if urlShown { out.URL = url @@ -264,7 +285,7 @@ func signIn(ctx context.Context, errOut io.Writer, w *databricks.WorkspaceClient timeout, timedOut := errors.AsType[*mcp.SignInTimeoutError](err) switch { case timedOut: - return signInStateTimeout, false, urlShown, failure(categorySignInTimeout, err, + return signInStateTimeout, false, urlShown, failure(protos.AitoolsErrorCategorySignInTimeout, err, "sign-in to %s did not complete in %g minutes%s. Finish at %s then retry: %s", svc.FullName, signInWaitTimeout.Minutes(), lastCheckText(timeout), url, mcpLoginHint(svc.FullName, profileName, pinnedFile)) case errors.Is(err, mcp.ErrSignInFailed): diff --git a/cmd/aitools/mcp_login_test.go b/cmd/aitools/mcp_login_test.go index 33658f88cef..a8f35097a9e 100644 --- a/cmd/aitools/mcp_login_test.go +++ b/cmd/aitools/mcp_login_test.go @@ -20,7 +20,9 @@ import ( "github.com/databricks/cli/libs/aitools/agents" "github.com/databricks/cli/libs/aitools/installer" "github.com/databricks/cli/libs/aitools/mcp" + "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/telemetry" "github.com/databricks/cli/libs/telemetry/protos" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" @@ -106,6 +108,32 @@ func TestMcpLoginRejectedTokenIsAuthFailed(t *testing.T) { assert.Contains(t, err.Error(), "Run: databricks auth login --profile logfood") } +// TestMcpLogoutForbiddenKeepsItsText checks that a 403 from logout is reported as the API said it, with no next steps +// added: root enriches a command's auth errors only when the command sets its profile's config for the context, and +// logout's upload config is not that config. The run still uploads with the profile it bound. +func TestMcpLogoutForbiddenKeepsItsText(t *testing.T) { + home := withHome(t) + server := loginTestServer(t, http.StatusForbidden, `{"error_code":"PERMISSION_DENIED","message":"no access"}`) + writeHomeConfig(t, home, "[logfood]\nhost = "+server.URL+"\nauth_type = pat\ntoken = dapi-logfood\n") + + parent := newMCPTestRoot() + parent.SetOut(&bytes.Buffer{}) + parent.SetErr(&bytes.Buffer{}) + parent.SetArgs([]string{"mcp", "logout", "system.ai.github", "--profile", "logfood"}) + ctx, _ := cmdio.NewTestContextWithStderr(telemetry.WithNewLogger(t.Context())) + err := parent.ExecuteContext(ctx) + logout, _, findErr := parent.Find([]string{"mcp", "logout"}) + require.NoError(t, findErr) + + require.Error(t, err) + assert.Contains(t, err.Error(), "no access") + assert.NotContains(t, err.Error(), "Next steps") + assert.False(t, cmdctx.HasConfigUsed(logout.Context()), "root does not enrich the error") + cfg, uploads := telemetry.UploadConfig(logout.Context()) + require.True(t, uploads) + assert.Equal(t, "logfood", cfg.Profile) +} + // TestMcpLogoutUnknownServiceIsAnError checks that logout of a service the profile cannot see fails, rather than // reporting that the caller is not signed in, so that a typo in the name is seen. func TestMcpLogoutUnknownServiceIsAnError(t *testing.T) { @@ -116,7 +144,7 @@ func TestMcpLogoutUnknownServiceIsAnError(t *testing.T) { out := mcpSignInOutput{Service: "system.ai.gihtub"} err := runMcpSignIn(t.Context(), io.Discard, "logfood", "system.ai.gihtub", mcpLogout, &out) require.Error(t, err) - assert.Equal(t, categoryServiceNotFound, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPServiceNotFound, errorCategory(err)) assert.Contains(t, err.Error(), "system.ai.gihtub is not an MCP service visible to profile logfood") assert.Empty(t, out.State) } @@ -164,7 +192,7 @@ func TestMcpLogoutEndpointNotFoundIsNotAServiceError(t *testing.T) { out := mcpSignInOutput{Service: "system.ai.github"} err := runMcpSignIn(t.Context(), io.Discard, "logfood", "system.ai.github", mcpLogout, &out) require.Error(t, err) - assert.NotEqual(t, categoryServiceNotFound, errorCategory(err)) + assert.NotEqual(t, protos.AitoolsErrorCategoryMCPServiceNotFound, errorCategory(err)) assert.NotContains(t, err.Error(), "is not an MCP service visible") } diff --git a/cmd/aitools/mcp_remove.go b/cmd/aitools/mcp_remove.go index b4f2f254c93..b3b662a7387 100644 --- a/cmd/aitools/mcp_remove.go +++ b/cmd/aitools/mcp_remove.go @@ -12,7 +12,10 @@ import ( "github.com/databricks/cli/libs/aitools/agents" "github.com/databricks/cli/libs/aitools/mcp" "github.com/databricks/cli/libs/aitools/mcp/agentconfig" + "github.com/databricks/cli/libs/aitools/mcpconfig" "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/telemetry/protos" "github.com/spf13/cobra" ) @@ -32,6 +35,14 @@ type removeOutput struct { Skipped []removeSkipped `json:"skipped"` Error string `json:"error,omitempty"` ErrorCategory protos.AitoolsErrorCategory `json:"error_category,omitempty"` + + // results is one telemetry result per (service, agent) attempted. It is not in the JSON. + results []protos.AitoolsMcpServerResult + // stateBefore is the state as it was read before the removal. It is not in the JSON. + stateBefore *mcp.State + // profile is the profile that the selected servers are bound to, when they all are bound to one. It is "" otherwise. + // It is not in the JSON. + profile string } type removeRemoved struct { @@ -72,7 +83,7 @@ type recordDrop struct { agent string } -// NewMcpRemoveCmd returns `aitools mcp remove`. It needs no network. +// NewMcpRemoveCmd returns `aitools mcp remove`. The edit needs no network; the telemetry upload after it is best-effort. func NewMcpRemoveCmd() *cobra.Command { var f mcpRemoveFlags cmd := &cobra.Command{ @@ -97,22 +108,12 @@ Entries that run 'aitools mcp serve' without an install record are not chosen by out := &removeOutput{Removed: []removeRemoved{}, Skipped: []removeSkipped{}} runErr := runMcpRemove(ctx, cmd, f, jsonMode, out) - if !jsonMode { - return runErr - } - - // The document carries a top-level failure; per-agent failures are already in skipped. - if top := topLevelFailure(runErr); top != nil { - out.Error = top.Error() - out.ErrorCategory = errorCategory(top) - } - if err := renderJSON(cmd.OutOrStdout(), out); err != nil { - return errors.Join(runErr, err) + err := runErr + if jsonMode { + err = writeRemoveJSON(cmd, out, runErr) } - if runErr != nil { - return root.ErrAlreadyPrinted - } - return nil + logRemoveEvent(ctx, out) + return err }, } @@ -122,6 +123,49 @@ Entries that run 'aitools mcp serve' without an install record are not chosen by return cmd } +// writeRemoveJSON writes the JSON document of a run that ended with runErr and returns the command's error. +func writeRemoveJSON(cmd *cobra.Command, out *removeOutput, runErr error) error { + // The document carries a top-level failure; per-agent failures are already in skipped. + if top := topLevelFailure(runErr); top != nil { + out.Error = top.Error() + out.ErrorCategory = errorCategory(top) + } + if err := renderJSON(cmd.OutOrStdout(), out); err != nil { + return errors.Join(runErr, err) + } + if runErr != nil { + return root.ErrAlreadyPrinted + } + return nil +} + +// logRemoveEvent buffers the REMOVE event and sets the config its exit-time upload authenticates with. It runs after the +// output is written. A run that logged no event touched no server, so it has nothing to upload and sets no config. +func logRemoveEvent(ctx context.Context, out *removeOutput) { + if !logMcpEvent(ctx, protos.AitoolsMcpEvent{ + CallingAgent: callingAgentType(), + Operation: protos.AitoolsMcpOperationRemove, + Servers: out.results, + }) { + return + } + if out.profile == "" { + return + } + configFile, err := mcp.ResolveConfigFile(ctx, env.Get(ctx, mcpconfig.ConfigFileEnv)) + if err != nil { + log.Debugf(ctx, "telemetry: cannot resolve the config file to upload with: %s", err) + return + } + // The upload sends the profile's token to the profile's host, so a profile whose recorded servers no longer match it + // uploads nothing. The records are the ones read before the removal. + if bindingDrifted(ctx, out.stateBefore, out.profile, configFile) { + log.Debugf(ctx, "telemetry: profile %s no longer matches an MCP server recorded for it; not uploading telemetry", out.profile) + return + } + setUploadConfig(ctx, out.profile, configFile) +} + // runMcpRemove removes the chosen servers from the agent configs and records the result. Results are printed // before a state failure is returned. An agent failure fails the command when --agents named the agent, or when a // selected server was removed from no agent; otherwise it is reported in skipped. @@ -148,6 +192,7 @@ func runMcpRemove(ctx context.Context, cmd *cobra.Command, f mcpRemoveFlags, jso if err != nil { return err } + out.stateBefore = state var selected []mcp.Entry if interactive && !f.all && len(names) == 0 && profileName == "" { // An agent with no installed server is refused as it is without a terminal, before the picker offers nothing for it. @@ -187,11 +232,14 @@ func runMcpRemove(ctx context.Context, cmd *cobra.Command, f mcpRemoveFlags, jso // --agents chooses the servers installed for those agents, so a server with no agent left is not chosen. selected = slices.DeleteFunc(selected, func(e mcp.Entry) bool { return len(e.Agents) == 0 }) } + out.profile = sharedProfile(selected) // Each agent config file is read once per command, and each server's config file is resolved once. files := newAgentFiles() var drops []recordDrop var agentErrs []error + // recordOnly are the indexes of out.results whose removal only drops a state record. + var recordOnly []int for _, e := range selected { removed := removeRemoved{Name: e.Name, Service: e.Service, Profile: e.Profile, Agents: []string{}} pin := resolveBinding(ctx, e) @@ -205,10 +253,17 @@ func runMcpRemove(ctx context.Context, cmd *cobra.Command, f mcpRemoveFlags, jso // There is no config this CLI can edit for the agent, so only the record can go. removed.Agents = append(removed.Agents, agentName) removed.RecordOnly = append(removed.RecordOnly, agentName) + recordOnly = append(recordOnly, len(out.results)) + out.results = append(out.results, removeServerResult(e.Service, agentName, removeDeleted, nil)) drops = append(drops, recordDrop{name: e.Name, agent: agentName}) continue } outcome, reason, err := removeFromAgent(ctx, files, e, a, pin) + // A pruned entry changes the state only, so a save that fails undoes it, as for a record-only removal. + if outcome == removePruned && err == nil { + recordOnly = append(recordOnly, len(out.results)) + } + out.results = append(out.results, removeServerResult(e.Service, agentName, outcome, err)) if err != nil { errs = append(errs, err) out.Skipped = append(out.Skipped, removeSkipped{Name: e.Name, Agent: agentName, Reason: fmt.Sprintf("could not update %s: %s", agentDisplayName(agentName), err)}) @@ -221,7 +276,12 @@ func runMcpRemove(ctx context.Context, cmd *cobra.Command, f mcpRemoveFlags, jso removed.Agents = append(removed.Agents, agentName) drops = append(drops, recordDrop{name: e.Name, agent: agentName}) } - // A server with no agent left in state is removed from state too, so it is reported as removed. + // A server with no agent left in state is removed from state too, so it is reported as removed. Only its + // record goes, which is a change to the state. + if len(e.Agents) == 0 { + recordOnly = append(recordOnly, len(out.results)) + out.results = append(out.results, protos.AitoolsMcpServerResult{Service: mcpServiceType(e.Service), Outcome: protos.AitoolsTargetOutcomeChanged}) + } if len(removed.Agents) > 0 || len(e.Agents) == 0 { out.Removed = append(out.Removed, removed) } @@ -237,6 +297,13 @@ func runMcpRemove(ctx context.Context, cmd *cobra.Command, f mcpRemoveFlags, jso if len(drops) > 0 || slices.ContainsFunc(selected, func(e mcp.Entry) bool { return len(e.Agents) == 0 }) { saveErr = dropRecords(ctx, drops, selected) } + if saveErr != nil { + // A record-only removal changes nothing but the state, so the save that failed undid it. + for _, i := range recordOnly { + out.results[i].Outcome = protos.AitoolsTargetOutcomeFailed + out.results[i].ErrorCategory = errorCategory(saveErr) + } + } if !jsonMode { printRemoveResults(ctx, out) } @@ -282,14 +349,14 @@ func selectInstalls(state *mcp.State, names []string, profileName string) ([]mcp return nil, err } if len(matched) == 0 { - return nil, failure(categoryServiceNotFound, nil, "no installed MCP server named %s%s", name, where) + return nil, failure(protos.AitoolsErrorCategoryMCPServiceNotFound, nil, "no installed MCP server named %s%s", name, where) } for _, i := range matched { chosen[i] = true } } if profileName != "" && len(scope) == 0 { - return nil, failure(categoryServiceNotFound, nil, "no installed MCP servers on profile %s", profileName) + return nil, failure(protos.AitoolsErrorCategoryMCPServiceNotFound, nil, "no installed MCP servers on profile %s", profileName) } if len(names) == 0 { return scope, nil @@ -365,7 +432,7 @@ func checkTargetsInstalled(selected []mcp.Entry, targets []*agents.Agent) error _, ok := e.Agents[a.Name] return ok }) { - return failure(categoryServiceNotFound, nil, "no MCP server is installed for %s", agentDisplayName(a.Name)) + return failure(protos.AitoolsErrorCategoryMCPServiceNotFound, nil, "no MCP server is installed for %s", agentDisplayName(a.Name)) } } return nil diff --git a/cmd/aitools/mcp_remove_test.go b/cmd/aitools/mcp_remove_test.go index d97ed4a6034..8deb0e92988 100644 --- a/cmd/aitools/mcp_remove_test.go +++ b/cmd/aitools/mcp_remove_test.go @@ -1,9 +1,11 @@ package aitools import ( + "bytes" "context" "encoding/json" "errors" + "log/slog" "maps" "os" "path/filepath" @@ -12,8 +14,15 @@ import ( "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/aitools/agents" + "github.com/databricks/cli/libs/aitools/installer" "github.com/databricks/cli/libs/aitools/mcp" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/log/handler" + "github.com/databricks/cli/libs/telemetry" "github.com/databricks/cli/libs/telemetry/protos" + "github.com/spf13/cobra" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -82,13 +91,13 @@ func TestSelectInstalls(t *testing.T) { _, err = selectInstalls(state, []string{"github-dogfood"}, "logfood") require.Error(t, err) assert.Equal(t, "no installed MCP server named github-dogfood on profile logfood", err.Error()) - assert.Equal(t, categoryServiceNotFound, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPServiceNotFound, errorCategory(err)) // A profile with no installed server is a typo or an empty profile, not an empty removal. _, err = selectInstalls(state, nil, "typo") require.Error(t, err) assert.Equal(t, "no installed MCP servers on profile typo", err.Error()) - assert.Equal(t, categoryServiceNotFound, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPServiceNotFound, errorCategory(err)) none, err := selectInstalls(&mcp.State{}, nil, "") require.NoError(t, err, "--all on an empty state is a no-op") @@ -180,6 +189,23 @@ func TestMcpRemoveDeletesOwnedEntriesAndPrunesMissing(t *testing.T) { assert.Equal(t, "github-dogfood", state.Servers[0].Name) } +// TestMcpRemoveMakesNoRequestWhenTelemetryIsOff checks that remove edits its agent config without a request to the +// profile's host. Nothing is uploaded when telemetry is off, so no auth is resolved for an upload. +func TestMcpRemoveMakesNoRequestWhenTelemetryIsOff(t *testing.T) { + home := withHome(t) + t.Setenv("DATABRICKS_CLI_DISABLE_TELEMETRY", "1") + server := noRequestServer(t) + writeHomeConfig(t, home, "[logfood]\nhost = "+server.URL+"\nauth_type = oauth-m2m\nclient_id = id\nclient_secret = secret\n") + claude := agents.ByName("claude-code") + claudePath := filepath.Join(home, ".claude.json") + genie := seedInstall(t, claude, "genie_one_mcp-logfood", "system.ai.genie_one_mcp", "logfood", server.URL) + saveState(t, genie) + + _, _, err := runMcpCmd(t, "remove", "--names", "genie_one_mcp-logfood", "--output", "json") + require.NoError(t, err) + assert.NotContains(t, readJSONFile(t, claudePath)["mcpServers"], "genie_one_mcp-logfood") +} + func TestMcpRemoveSkipsEntryChangedOutsideCLI(t *testing.T) { home := withHome(t) claude := agents.ByName("claude-code") @@ -400,13 +426,13 @@ func TestMcpRemoveRefusesProfileAndAgentWithNothingInstalled(t *testing.T) { require.ErrorIs(t, err, root.ErrAlreadyPrinted) out := readRemoveOutput(t, stdout.String()) assert.Equal(t, "no installed MCP servers on profile typo", out.Error) - assert.Equal(t, categoryServiceNotFound, out.ErrorCategory) + assert.Equal(t, protos.AitoolsErrorCategoryMCPServiceNotFound, out.ErrorCategory) stdout, _, err = runMcpCmd(t, "remove", "--names", "genie_one_mcp-logfood", "--agents", "cursor", "--output", "json") require.ErrorIs(t, err, root.ErrAlreadyPrinted) out = readRemoveOutput(t, stdout.String()) assert.Equal(t, "no MCP server is installed for Cursor", out.Error) - assert.Equal(t, categoryServiceNotFound, out.ErrorCategory) + assert.Equal(t, protos.AitoolsErrorCategoryMCPServiceNotFound, out.ErrorCategory) // Nothing was removed by either refusal. state, err := mcp.LoadState(t.Context()) @@ -696,7 +722,7 @@ func TestMcpRemovePickerRefusesAgentWithNoServer(t *testing.T) { err := runInteractive(t, "mcp", "remove", "--agents", "codex") require.Error(t, err) assert.Equal(t, "no MCP server is installed for "+agents.ByName(agents.NameCodex).DisplayName, err.Error()) - assert.Equal(t, categoryServiceNotFound, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryMCPServiceNotFound, errorCategory(err)) state, err := mcp.LoadState(t.Context()) require.NoError(t, err) assert.Len(t, state.Servers, 1) @@ -772,3 +798,200 @@ func claudeServerEntry(claudePath, name, service string) mcp.Entry { Agents: map[string]mcp.AgentRecord{agents.NameClaudeCode: {Path: claudePath, Scope: "user"}}, } } + +// runRemoveWithTelemetry runs `mcp remove` with args on a context that buffers telemetry, and returns the remove command, +// so a test can read the context it left, and its stdout. +// The logs are those of the command's logger at logLevel. +func runRemoveWithTelemetry(t *testing.T, stdout, logs *bytes.Buffer, logLevel slog.Level, args ...string) (*cobra.Command, error) { + t.Helper() + parent := newMCPTestRoot() + ctx := telemetry.WithNewLogger(t.Context()) + ctx = env.Set(ctx, "DATABRICKS_CLI_DISABLE_TELEMETRY", "") + ctx = log.NewContext(ctx, slog.New(handler.NewFriendlyHandler(logs, &handler.Options{Level: logLevel}))) + ctx, _ = cmdio.NewTestContextWithStderr(ctx) + parent.SetOut(stdout) + parent.SetErr(&bytes.Buffer{}) + parent.SetArgs(append([]string{"mcp", "remove"}, args...)) + err := parent.ExecuteContext(ctx) + remove, _, findErr := parent.Find([]string{"mcp", "remove"}) + require.NoError(t, findErr) + return remove, err +} + +// unsetConfigFile makes DATABRICKS_CONFIG_FILE unset for the test, as opposed to set to "". +func unsetConfigFile(t *testing.T) { + t.Setenv("DATABRICKS_CONFIG_FILE", "") + require.NoError(t, os.Unsetenv("DATABRICKS_CONFIG_FILE")) +} + +// TestMcpRemoveSetsNoUploadWithoutAnEvent checks that a run that removes nothing logs no event, so it sets no config for +// an upload. +func TestMcpRemoveSetsNoUploadWithoutAnEvent(t *testing.T) { + withHome(t) + var stdout, logs bytes.Buffer + remove, err := runRemoveWithTelemetry(t, &stdout, &logs, log.LevelDebug, "--all", "--output", "json") + require.NoError(t, err) + _, uploads := telemetry.UploadConfig(remove.Context()) + assert.False(t, uploads) + assert.Empty(t, telemetry.BufferedEvents(remove.Context())) +} + +// TestMcpRemoveUploadSkipsDriftedProfile checks that the upload is not authenticated with a profile that a server recorded +// for it no longer matches, as its host here is not the one the server was set up against. The removal is done either +// way. +func TestMcpRemoveUploadSkipsDriftedProfile(t *testing.T) { + home := withHome(t) + writeHomeConfig(t, home, "[logfood]\nhost = https://adb-2.test\nauth_type = pat\ntoken = t\n") + claude := agents.ByName("claude-code") + claudePath := filepath.Join(home, ".claude.json") + // Recorded against another host than the profile names now. + entry := seedInstall(t, claude, "genie_one_mcp-logfood", "system.ai.genie_one_mcp", "logfood", "https://adb-1.test") + saveState(t, entry) + + var stdout, logs bytes.Buffer + remove, err := runRemoveWithTelemetry(t, &stdout, &logs, log.LevelDebug, "--names", "genie_one_mcp-logfood", "--output", "json") + require.NoError(t, err) + require.Len(t, telemetry.BufferedEvents(remove.Context()), 1, "the removal is logged") + _, uploads := telemetry.UploadConfig(remove.Context()) + assert.False(t, uploads, "a drifted profile is not used to upload") + assert.NotContains(t, readJSONFile(t, claudePath)["mcpServers"], "genie_one_mcp-logfood", "the removal is done") +} + +// TestMcpRemoveUploadUsesMatchingProfile checks that a profile whose recorded servers still match is used for the upload, +// read from the default config file when DATABRICKS_CONFIG_FILE is unset. +func TestMcpRemoveUploadUsesMatchingProfile(t *testing.T) { + home := withHome(t) + unsetConfigFile(t) + writeHomeConfig(t, home, "[logfood]\nhost = https://adb-1.test\nauth_type = pat\ntoken = t\n") + claude := agents.ByName("claude-code") + entry := seedInstall(t, claude, "genie_one_mcp-logfood", "system.ai.genie_one_mcp", "logfood", "https://adb-1.test") + saveState(t, entry) + + var stdout, logs bytes.Buffer + remove, err := runRemoveWithTelemetry(t, &stdout, &logs, log.LevelDebug, "--names", "genie_one_mcp-logfood", "--output", "json") + require.NoError(t, err) + cfg, uploads := telemetry.UploadConfig(remove.Context()) + require.True(t, uploads) + assert.Equal(t, "logfood", cfg.Profile) + assert.Equal(t, "https://adb-1.test", cfg.Host) + assert.Equal(t, "t", cfg.Token) +} + +// TestMcpRemoveUploadIgnoresEnvironmentToken checks that the upload authenticates with the profile's own token, never with +// a DATABRICKS_TOKEN of another workspace in the environment. +func TestMcpRemoveUploadIgnoresEnvironmentToken(t *testing.T) { + home := withHome(t) + unsetConfigFile(t) + t.Setenv("DATABRICKS_TOKEN", "token-of-another-workspace") + writeHomeConfig(t, home, "[logfood]\nhost = https://adb-1.test\nauth_type = pat\ntoken = t\n") + claude := agents.ByName("claude-code") + saveState(t, seedInstall(t, claude, "genie_one_mcp-logfood", "system.ai.genie_one_mcp", "logfood", "https://adb-1.test")) + + var stdout, logs bytes.Buffer + remove, err := runRemoveWithTelemetry(t, &stdout, &logs, log.LevelDebug, "--names", "genie_one_mcp-logfood", "--output", "json") + require.NoError(t, err) + cfg, uploads := telemetry.UploadConfig(remove.Context()) + require.True(t, uploads) + assert.Equal(t, "t", cfg.Token) +} + +// TestMcpRemoveOfflineSetsUploadWithoutWarning checks that a remove that cannot reach the workspace warns nothing: the +// upload's config is resolved without a request, so there is no failure to report. +func TestMcpRemoveOfflineSetsUploadWithoutWarning(t *testing.T) { + home := withHome(t) + writeHomeConfig(t, home, "[logfood]\nhost = https://adb-1.test\nauth_type = pat\ntoken = t\n") + claude := agents.ByName("claude-code") + saveState(t, seedInstall(t, claude, "genie_one_mcp-logfood", "system.ai.genie_one_mcp", "logfood", "https://adb-1.test")) + + var stdout, logs bytes.Buffer + remove, err := runRemoveWithTelemetry(t, &stdout, &logs, log.LevelWarn, "--names", "genie_one_mcp-logfood", "--output", "json") + require.NoError(t, err) + _, uploads := telemetry.UploadConfig(remove.Context()) + assert.True(t, uploads) + assert.Empty(t, logs.String()) + assert.Contains(t, stdout.String(), `"removed"`) +} + +// seedPrunedInstall records an install of genie_one_mcp for logfood in claude-code whose entry the agent config no longer holds. +func seedPrunedInstall(t *testing.T, claudePath string) { + t.Helper() + claude := agents.ByName("claude-code") + saveState(t, seedInstall(t, claude, "genie_one_mcp-logfood", "system.ai.genie_one_mcp", "logfood", "https://adb-1.test")) + require.NoError(t, os.WriteFile(claudePath, []byte(`{"mcpServers":{}}`), 0o600)) +} + +// TestMcpRemovePrunedEntryReportsChanged checks that an entry the agent config no longer holds is reported CHANGED, since +// its record is dropped, and FAILED when the state cannot be saved, as a record-only removal is. +func TestMcpRemovePrunedEntryReportsChanged(t *testing.T) { + home := withHome(t) + claudePath := filepath.Join(home, ".claude.json") + + seedPrunedInstall(t, claudePath) + var stdout, logs bytes.Buffer + remove, err := runRemoveWithTelemetry(t, &stdout, &logs, log.LevelDebug, "--names", "genie_one_mcp-logfood", "--output", "json") + require.NoError(t, err) + events := telemetry.BufferedEvents(remove.Context()) + require.Len(t, events, 1) + require.NotNil(t, events[0].AitoolsMcpEvent) + require.Len(t, events[0].AitoolsMcpEvent.Servers, 1) + assert.Equal(t, protos.AitoolsTargetOutcomeChanged, events[0].AitoolsMcpEvent.Servers[0].Outcome) + + seedPrunedInstall(t, claudePath) + saveMCPState = func(context.Context, *mcp.State) error { return errors.New("state file is read-only") } + t.Cleanup(func() { saveMCPState = mcp.SaveState }) + stdout.Reset() + remove, err = runRemoveWithTelemetry(t, &stdout, &logs, log.LevelDebug, "--names", "genie_one_mcp-logfood", "--output", "json") + require.ErrorIs(t, err, root.ErrAlreadyPrinted) + events = telemetry.BufferedEvents(remove.Context()) + require.Len(t, events, 1) + require.NotNil(t, events[0].AitoolsMcpEvent) + require.Len(t, events[0].AitoolsMcpEvent.Servers, 1) + assert.Equal(t, protos.AitoolsTargetOutcomeFailed, events[0].AitoolsMcpEvent.Servers[0].Outcome) +} + +// TestMcpRemoveFailedStateReadAfterAnotherEventLogsNothing checks that a remove that cannot read the state logs nothing +// of its own, even when the run already buffered another event, and does not read the state it never read. +func TestMcpRemoveFailedStateReadAfterAnotherEventLogsNothing(t *testing.T) { + withHome(t) + dir, err := installer.GlobalSkillsDir(t.Context()) + require.NoError(t, err) + require.NoError(t, os.MkdirAll(filepath.Join(dir, ".state.json"), 0o700)) + + parent := newMCPTestRoot() + parent.SetOut(&bytes.Buffer{}) + parent.SetErr(&bytes.Buffer{}) + parent.SetArgs([]string{"mcp", "remove", "--all", "--output", "json"}) + ctx := telemetry.WithNewLogger(t.Context()) + telemetry.Log(ctx, protos.DatabricksCliLog{CliTestEvent: &protos.CliTestEvent{Name: protos.DummyCliEnumValue1}}) + ctx, _ = cmdio.NewTestContextWithStderr(ctx) + require.Error(t, parent.ExecuteContext(ctx)) + remove, _, findErr := parent.Find([]string{"mcp", "remove"}) + require.NoError(t, findErr) + events := telemetry.BufferedEvents(remove.Context()) + require.Len(t, events, 1) + assert.NotNil(t, events[0].CliTestEvent) +} + +// TestMcpRemoveRecordOnlyReportsFailedWhenStateIsNotSaved checks that a removal that only drops state records is reported +// FAILED when the state cannot be saved, since nothing was removed. +func TestMcpRemoveRecordOnlyReportsFailedWhenStateIsNotSaved(t *testing.T) { + withHome(t) + retired := mcp.Entry{ + Name: "dbsql-logfood", Service: "system.ai.dbsql", Profile: "logfood", Host: "https://adb-1.test", Command: testDatabricksCommand, + Agents: map[string]mcp.AgentRecord{"retired-agent": {Path: "/x/config.json", Scope: "user"}}, + } + saveState(t, retired) + saveMCPState = func(context.Context, *mcp.State) error { return errors.New("state file is read-only") } + t.Cleanup(func() { saveMCPState = mcp.SaveState }) + + var stdout, logs bytes.Buffer + remove, err := runRemoveWithTelemetry(t, &stdout, &logs, log.LevelDebug, "--names", "dbsql-logfood", "--output", "json") + require.ErrorIs(t, err, root.ErrAlreadyPrinted) + + events := telemetry.BufferedEvents(remove.Context()) + require.Len(t, events, 1) + require.NotNil(t, events[0].AitoolsMcpEvent) + require.Len(t, events[0].AitoolsMcpEvent.Servers, 1) + assert.Equal(t, protos.AitoolsTargetOutcomeFailed, events[0].AitoolsMcpEvent.Servers[0].Outcome) + assert.Equal(t, protos.AitoolsErrorCategoryUncategorized, events[0].AitoolsMcpEvent.Servers[0].ErrorCategory) +} diff --git a/cmd/aitools/mcp_serve.go b/cmd/aitools/mcp_serve.go index f1531de44ca..8d83de2613b 100644 --- a/cmd/aitools/mcp_serve.go +++ b/cmd/aitools/mcp_serve.go @@ -24,6 +24,8 @@ import ( "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/auth/u2m" "github.com/databricks/cli/libs/browser" + "github.com/databricks/cli/libs/telemetry" + "github.com/databricks/cli/libs/telemetry/protos" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/config" "github.com/databricks/databricks-sdk-go/config/credentials" @@ -79,7 +81,7 @@ func NewMcpServeCmd() *cobra.Command { cmd.Flags().StringVar(&f.service, strings.TrimPrefix(mcpconfig.ServiceFlag, "--"), "", "Full name of the MCP service, for example system.ai.github") cmd.Flags().StringVar(&f.agent, strings.TrimPrefix(mcpconfig.AgentFlag, "--"), "", - "Registry name of the calling agent, used only for the User-Agent") + "Registry name of the calling agent, for the User-Agent and telemetry") return cmd } @@ -104,6 +106,7 @@ func runMcpServe(cmd *cobra.Command, f mcpServeFlags) error { if err != nil { return serveFailure(cmd, err, serveBindStatus) } + agent := proxyAgentType(f.agent) cfg, entry, err := mcp.Bind(cmd.Context(), mcp.BindOptions{ Profile: profileName, Service: f.service, @@ -130,7 +133,7 @@ func runMcpServe(cmd *cobra.Command, f mcpServeFlags) error { return serveFailure(cmd, err, 1) } } - challenge, err := serveChallenge(ctx, serveTarget{w: w, entry: entry, profile: profileName, service: f.service, pinnedFile: pinnedFile}, cmd.ErrOrStderr()) + challenge, err := serveChallenge(ctx, serveTarget{w: w, entry: entry, profile: profileName, service: f.service, pinnedFile: pinnedFile, agent: agent}, cmd.ErrOrStderr()) if err != nil { // A rejected token stops serve by cancelling ctx, so the refresh failure is checked before the signal, and its // message is the one that names the login. @@ -144,8 +147,10 @@ func runMcpServe(cmd *cobra.Command, f mcpServeFlags) error { // serveChallenge fails only on an auth failure (see lookupAtStart), which exits serveBindStatus. return serveFailure(cmd, err, serveBindStatus) } + // Only a serving start is logged. + logMcpEvent(cmd.Context(), proxyStartedEvent(agent, f.service)) - return serveProxy(ctx, cmd, refresh, proxy.Options{ + err = serveProxy(ctx, cmd, refresh, proxy.Options{ In: cmd.InOrStdin(), Out: cmd.OutOrStdout(), Err: cmd.ErrOrStderr(), @@ -156,6 +161,10 @@ func runMcpServe(cmd *cobra.Command, f mcpServeFlags) error { Transport: serveTransport(cfg), Challenge: challenge, }) + // The upload authenticates with the profile at exit, with a config of its own that never refreshes a token, since the + // session's config refreshes its token. A refresh failure has refused the upload (proxyResult), so nothing is set then. + setUploadConfig(cmd.Context(), profileName, cfg.ConfigFile) + return err } // guardedClient returns the client of the bound profile whose requests the refresh guard authenticates. The SDK calls @@ -186,14 +195,15 @@ func (g guardCredentials) Configure(context.Context, *config.Config) (credential return credentials.CredentialsProviderFn(g.refresh.authenticateCall), nil } -// serveTarget is the server that a serve session bridges: its service, the bound profile, the install record, and the -// config file it is pinned to. w is the profile's client, which must be guardedClient's. +// serveTarget is the server that a serve session bridges: its service, the bound profile, the install record, the config +// file it is pinned to, and the agent that started it. w is the profile's client, which must be guardedClient's. type serveTarget struct { w *databricks.WorkspaceClient entry *mcp.Entry profile string service string pinnedFile string + agent protos.AitoolsAgentType } // serveChallenge returns the sign-in handling of a service that needs provider sign-in, or nil for @@ -229,6 +239,7 @@ func serveChallenge(ctx context.Context, t serveTarget, errOut io.Writer) (*prox LogoutCommand: mcpLogoutHint(t.service, t.profile, t.pinnedFile), PinnedFile: t.pinnedFile, SignIn: serveSignIn(t), + Recorder: mcpProxyTelemetry{agent: t.agent, service: t.service}, }, nil } @@ -291,6 +302,8 @@ func serveProxy(ctx context.Context, cmd *cobra.Command, refresh *refreshGuard, // stdin error after it is a consequence. The final line always names the rejection, as the bridge may not have logged it. func proxyResult(cmd *cobra.Command, refresh *refreshGuard, runErr error) error { if refreshErr := refresh.failure(); refreshErr != nil { + // The token was rejected, so the upload at exit would only ask the token endpoint again. + telemetry.RefuseUpload(cmd.Context()) fmt.Fprintf(cmd.ErrOrStderr(), "stopped after a token refresh failure for profile %s\n", refresh.profile) return exitAlreadyPrinted(refreshErr, serveBindStatus) } diff --git a/cmd/aitools/mcp_serve_test.go b/cmd/aitools/mcp_serve_test.go index 17262c8ba2e..91964dd6ac8 100644 --- a/cmd/aitools/mcp_serve_test.go +++ b/cmd/aitools/mcp_serve_test.go @@ -26,6 +26,8 @@ import ( "github.com/databricks/cli/libs/aitools/mcp/proxy" "github.com/databricks/cli/libs/aitools/mcpconfig" "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/libs/telemetry" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/config" "github.com/databricks/databricks-sdk-go/httpclient" @@ -332,6 +334,43 @@ func TestMcpServeRejectsServiceWithoutThreeParts(t *testing.T) { } } +// TestMcpServeFailedStartLogsNothing checks that a start that fails its binding checks logs no PROXY_STARTED, so nothing +// uploads. +func TestMcpServeFailedStartLogsNothing(t *testing.T) { + withHome(t) + parent := newMCPTestRoot() + parent.SetErr(&bytes.Buffer{}) + parent.SetArgs([]string{"mcp", "serve", "--profile", testProfile, "--service", testService}) + ctx := env.Set(telemetry.WithNewLogger(cmdio.MockDiscard(t.Context())), "DATABRICKS_CLI_DISABLE_TELEMETRY", "") + err := parent.ExecuteContext(ctx) + require.ErrorIs(t, err, root.ErrAlreadyPrinted) + assert.Equal(t, serveBindStatus, root.ExitStatus(err)) + serve, _, findErr := parent.Find([]string{"mcp", "serve"}) + require.NoError(t, findErr) + assert.Empty(t, telemetry.BufferedEvents(serve.Context())) +} + +// TestMcpServeRejectedStartLogsNothing checks that a start whose startup read is rejected by the profile's credential logs +// no PROXY_STARTED, so the run uploads nothing: the rejected token is not sent again. +func TestMcpServeRejectedStartLogsNothing(t *testing.T) { + home := withHome(t) + server := serviceServer(t, http.StatusUnauthorized, `{"error_code":"UNAUTHENTICATED","message":"Invalid access token."}`) + writeHomeConfig(t, home, "[logfood]\nhost = "+server.URL+"\nauth_type = pat\ntoken = dapi-expired\n") + + parent := newMCPTestRoot() + parent.SetErr(&bytes.Buffer{}) + parent.SetArgs([]string{"mcp", "serve", "--profile", testProfile, "--service", testService}) + ctx := telemetry.WithNewLogger(cmdio.MockDiscard(t.Context())) + err := parent.ExecuteContext(ctx) + require.ErrorIs(t, err, root.ErrAlreadyPrinted) + assert.Equal(t, serveBindStatus, root.ExitStatus(err)) + serve, _, findErr := parent.Find([]string{"mcp", "serve"}) + require.NoError(t, findErr) + assert.Empty(t, telemetry.BufferedEvents(serve.Context())) + _, uploads := telemetry.UploadConfig(serve.Context()) + assert.False(t, uploads) +} + // TestMcpServeStateReadFailureExitsTwo checks that a state file that cannot be read stops the binding checks, so it // exits 2 like the other binding failures. func TestMcpServeStateReadFailureExitsTwo(t *testing.T) { @@ -384,11 +423,16 @@ func TestProxyResultRefreshFailureWinsOverStdinError(t *testing.T) { } require.Error(t, g.authenticateRequest(&http.Request{Header: http.Header{}})) cmd := NewMcpServeCmd() + cmd.SetContext(telemetry.WithNewLogger(t.Context())) + telemetry.SetUploadConfig(cmd.Context(), &config.Config{Host: "https://adb-1.test"}) var stderr bytes.Buffer cmd.SetErr(&stderr) err := proxyResult(cmd, g, errors.New("read stdin: input/output error")) + // The exit-time upload would ask the rejected token's endpoint again, so the run uploads nothing. + _, uploads := telemetry.UploadConfig(cmd.Context()) + assert.False(t, uploads) assert.ErrorIs(t, err, root.ErrAlreadyPrinted) assert.Equal(t, serveBindStatus, root.ExitStatus(err)) assert.ErrorIs(t, err, cause) diff --git a/cmd/aitools/mcp_telemetry.go b/cmd/aitools/mcp_telemetry.go new file mode 100644 index 00000000000..35f88327a0d --- /dev/null +++ b/cmd/aitools/mcp_telemetry.go @@ -0,0 +1,297 @@ +package aitools + +import ( + "context" + "slices" + "strings" + + "github.com/databricks/cli/libs/aitools/agents" + "github.com/databricks/cli/libs/aitools/mcp" + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/telemetry" + "github.com/databricks/cli/libs/telemetry/protos" +) + +// setUploadConfig sets the config of profileName in configFile for the run's telemetry upload: resolved within +// telemetryAuthBudget, with credentials that never refresh (mcp.BoundUploadConfig). A command calls it once its binding +// checks have passed, so the upload authenticates with the profile the command bound. Nothing is resolved when telemetry +// is disabled, and a profile that does not resolve uploads nothing. +func setUploadConfig(ctx context.Context, profileName, configFile string) { + if telemetry.Disabled(ctx) { + return + } + cfg, err := mcp.BoundUploadConfig(profileName, configFile, telemetryAuthBudget) + if err != nil { + log.Debugf(ctx, "telemetry: cannot resolve profile %s to upload with: %s", profileName, err) + return + } + telemetry.SetUploadConfig(ctx, cfg) +} + +// mcpServiceTypes maps the system.ai leaves in the CLI's service table to their telemetry enum. +var mcpServiceTypes = map[string]protos.AitoolsMcpService{ + "genie_one_mcp": protos.AitoolsMcpServiceGenieOne, + "dbsql": protos.AitoolsMcpServiceDBSQL, + "web_search": protos.AitoolsMcpServiceWebSearch, + "sandbox": protos.AitoolsMcpServiceSandbox, + "slack": protos.AitoolsMcpServiceSlack, + "github": protos.AitoolsMcpServiceGitHub, + "atlassian": protos.AitoolsMcpServiceAtlassian, + "google_drive": protos.AitoolsMcpServiceGoogleDrive, + "google_calendar": protos.AitoolsMcpServiceGoogleCalendar, + "gmail": protos.AitoolsMcpServiceGmail, + "microsoft_365": protos.AitoolsMcpServiceMicrosoft365, +} + +// mcpServiceType maps a service full name to its telemetry enum. The name itself is never sent: +// a system.ai leaf in the table has its own value, other system.ai leaves are OTHER_SYSTEM, and +// services outside system.ai are CUSTOM. +func mcpServiceType(fullName string) protos.AitoolsMcpService { + parts := strings.Split(fullName, ".") + if len(parts) != 3 || !strings.EqualFold(parts[0]+"."+parts[1], mcp.SystemAILocation) { + return protos.AitoolsMcpServiceCustom + } + if t, ok := mcpServiceTypes[strings.ToLower(parts[2])]; ok { + return t + } + return protos.AitoolsMcpServiceOtherSystem +} + +// callingAgentType returns the agent that drives the CLI, Unspecified when none is detected. +func callingAgentType() protos.AitoolsAgentType { + a := agents.CallingAgent() + if a == nil { + return protos.AitoolsAgentTypeUnspecified + } + return agentType(a.Name) +} + +// proxyAgentType returns the agent that started the proxy: the --agent flag when given, else the +// detected agent. +func proxyAgentType(flag string) protos.AitoolsAgentType { + if flag == "" { + return callingAgentType() + } + return agentType(flag) +} + +// logMcpEvent buffers an MCP event, and reports whether it did; cmd/root uploads it at exit. An event without server +// results is not logged: a run that stopped before any server was reached has nothing to report. An event that names a +// rejected credential is not logged either, and the run refuses its upload, since the upload would present that +// credential again. The bound profile count is read from the state file as it is after the operation. A state file that +// cannot be read drops the event; telemetry never fails a command. +func logMcpEvent(ctx context.Context, event protos.AitoolsMcpEvent) bool { + if len(event.Servers) == 0 { + return false + } + if slices.ContainsFunc(event.Servers, func(r protos.AitoolsMcpServerResult) bool { + return r.ErrorCategory == protos.AitoolsErrorCategoryAuthFailed + }) { + telemetry.RefuseUpload(ctx) + return false + } + state, err := mcp.LoadState(ctx) + if err != nil { + log.Debugf(ctx, "telemetry: cannot read the MCP state; dropping the event: %s", err) + return false + } + event.BoundProfileCount = boundProfileCount(ctx, state) + telemetry.Log(ctx, protos.DatabricksCliLog{AitoolsMcpEvent: &event}) + return true +} + +// boundProfileCount returns the number of distinct profiles that have servers installed, grouped as list groups them +// (profileGroup), so DEFAULT in two config files counts twice. +func boundProfileCount(ctx context.Context, state *mcp.State) int64 { + profiles := map[profileKey]bool{} + checks := mcp.CheckEntries(ctx, state) + for i, e := range state.Servers { + profiles[profileGroup(e, checks[i])] = true + } + return int64(len(profiles)) +} + +// addSignInState is the sign-in state of a server after an add: the state that --login reports, else PENDING for a +// server that needs sign-in, else NOT_REQUIRED. A sign-in that failed before it read a state is PENDING. +func addSignInState(s addServerOutput) protos.AitoolsMcpSignInState { + if s.SignIn != nil && s.SignIn.State != "" { + return signInStateOf(s.SignIn.State) + } + if s.SignInRequired { + return protos.AitoolsMcpSignInStatePending + } + return protos.AitoolsMcpSignInStateNotRequired +} + +// addServerResults returns one result per (service, agent) of an add run. Installed entries are +// CHANGED, re-asserted ones UNCHANGED, and failed ones FAILED with their category. +func addServerResults(servers []addServerOutput) []protos.AitoolsMcpServerResult { + var results []protos.AitoolsMcpServerResult + for _, s := range servers { + for _, a := range s.Agents { + r := protos.AitoolsMcpServerResult{ + Service: mcpServiceType(s.Service), + Agent: agentType(a.Name), + SignInState: addSignInState(s), + } + switch a.Status { + case statusInstalled: + r.Outcome = protos.AitoolsTargetOutcomeChanged + case statusReasserted: + r.Outcome = protos.AitoolsTargetOutcomeUnchanged + default: + r.Outcome = protos.AitoolsTargetOutcomeFailed + r.ErrorCategory = a.ErrorCategory + } + results = append(results, r) + } + } + return results +} + +// removeServerResult returns the result of removing one server from one agent config. An entry that was deleted, or +// pruned (its record dropped), is CHANGED. An entry that was changed outside this CLI is SKIPPED: it is left in place. +func removeServerResult(service, agent string, outcome removeOutcome, err error) protos.AitoolsMcpServerResult { + r := protos.AitoolsMcpServerResult{Service: mcpServiceType(service), Agent: agentType(agent)} + switch { + case err != nil: + r.Outcome = protos.AitoolsTargetOutcomeFailed + r.ErrorCategory = errorCategory(err) + case outcome == removeKept: + r.Outcome = protos.AitoolsTargetOutcomeSkipped + default: + r.Outcome = protos.AitoolsTargetOutcomeChanged + } + return r +} + +// signInStateOf maps a sign-in state name of login to its telemetry state. A run that never learned the state leaves it +// unset. Anything but ACTIVE or NOT_REQUIRED is still PENDING. +func signInStateOf(state string) protos.AitoolsMcpSignInState { + switch state { + case "": + return "" + case signInStateActive: + return protos.AitoolsMcpSignInStateActive + case signInStateNotRequired: + return protos.AitoolsMcpSignInStateNotRequired + } + return protos.AitoolsMcpSignInStatePending +} + +// signInServerResult returns the result of one provider sign-in. The sign-in was already active +// (alreadySignedIn) or the service needed none, so nothing changed: UNCHANGED. +func signInServerResult(service, state string, alreadySignedIn bool, err error) protos.AitoolsMcpServerResult { + r := protos.AitoolsMcpServerResult{Service: mcpServiceType(service), SignInState: signInStateOf(state)} + switch { + case err != nil: + r.Outcome = protos.AitoolsTargetOutcomeFailed + r.ErrorCategory = errorCategory(err) + case alreadySignedIn || state == signInStateNotRequired: + r.Outcome = protos.AitoolsTargetOutcomeUnchanged + default: + r.Outcome = protos.AitoolsTargetOutcomeChanged + } + return r +} + +// signInEventOf is the SIGN_IN event of one provider sign-in, under add --login or login. A sign-in that found the caller +// already signed in is UNCHANGED. +func signInEventOf(service, state string, alreadySignedIn bool, err error) protos.AitoolsMcpEvent { + return protos.AitoolsMcpEvent{ + CallingAgent: callingAgentType(), + Operation: protos.AitoolsMcpOperationSignIn, + Servers: []protos.AitoolsMcpServerResult{signInServerResult(service, state, alreadySignedIn, err)}, + } +} + +// signOutServerResult returns the result of a logout. Logout does not read whether the service needs +// sign-in, so the sign-in state is not set. +func signOutServerResult(service, state string, err error) protos.AitoolsMcpServerResult { + r := protos.AitoolsMcpServerResult{Service: mcpServiceType(service)} + switch { + case err != nil: + r.Outcome = protos.AitoolsTargetOutcomeFailed + r.ErrorCategory = errorCategory(err) + case state == signOutStateDeleted: + r.Outcome = protos.AitoolsTargetOutcomeChanged + default: + r.Outcome = protos.AitoolsTargetOutcomeUnchanged + } + return r +} + +// proxyStartedEvent is the PROXY_STARTED event of serve, logged once the startup checks have passed and the proxy serves. +// A start that fails its checks logs nothing. A binding that failed has no profile the upload may authenticate with +// (PROFILE_NOT_FOUND and PROFILE_HOST_CHANGED are never sent), and a rejected credential would only repeat its rejected +// request, so none of them uploads. +func proxyStartedEvent(agent protos.AitoolsAgentType, service string) protos.AitoolsMcpEvent { + return protos.AitoolsMcpEvent{ + CallingAgent: agent, + Operation: protos.AitoolsMcpOperationProxyStarted, + Servers: []protos.AitoolsMcpServerResult{{Service: mcpServiceType(service)}}, + } +} + +// mcpProxyTelemetry is the proxy's telemetry for one serve session. It implements proxy.Recorder. +type mcpProxyTelemetry struct { + agent protos.AitoolsAgentType + service string +} + +// SignInPrompted logs SIGN_IN_PROMPTED once the sign-in page the authenticate tool opened has been +// waited on. signedIn reports whether the credential became ACTIVE. +func (t mcpProxyTelemetry) SignInPrompted(ctx context.Context, signedIn bool) { + logMcpEvent(ctx, signInPromptedEvent(t.agent, t.service, signedIn)) +} + +// AuthRejected refuses the run's upload, since the upload would present the rejected credential again. +func (t mcpProxyTelemetry) AuthRejected(ctx context.Context) { + telemetry.RefuseUpload(ctx) +} + +// signInPromptedEvent is the SIGN_IN_PROMPTED event of serve: the sign-in is required, and the +// sign-in state is ACTIVE when the wait ended with the credential, PENDING otherwise. +func signInPromptedEvent(agent protos.AitoolsAgentType, service string, signedIn bool) protos.AitoolsMcpEvent { + state := protos.AitoolsMcpSignInStatePending + if signedIn { + state = protos.AitoolsMcpSignInStateActive + } + return protos.AitoolsMcpEvent{ + CallingAgent: agent, + Operation: protos.AitoolsMcpOperationSignInPrompted, + Servers: []protos.AitoolsMcpServerResult{{ + Service: mcpServiceType(service), + ErrorCategory: protos.AitoolsErrorCategorySignInRequired, + SignInState: state, + }}, + } +} + +// bindingDrifted reports whether a server recorded for profileName in state no longer matches the profile in configFile: +// its host or workspace changed, or its config file moved. The upload would send the profile's token to the host the +// profile names now, which the server was not set up against. It reads the config files only. +func bindingDrifted(ctx context.Context, state *mcp.State, profileName, configFile string) bool { + for _, e := range state.Servers { + if e.Profile != profileName { + continue + } + if _, err := mcp.CheckBinding(ctx, state, e.Service, profileName, configFile); err != nil { + return true + } + } + return false +} + +// sharedProfile returns the profile that all entries are bound to, or "" when they are bound to none or to several. +func sharedProfile(entries []mcp.Entry) string { + if len(entries) == 0 { + return "" + } + for _, e := range entries[1:] { + if e.Profile != entries[0].Profile { + return "" + } + } + return entries[0].Profile +} diff --git a/cmd/aitools/mcp_telemetry_test.go b/cmd/aitools/mcp_telemetry_test.go new file mode 100644 index 00000000000..e388ca81568 --- /dev/null +++ b/cmd/aitools/mcp_telemetry_test.go @@ -0,0 +1,333 @@ +package aitools + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "log/slog" + "maps" + "path/filepath" + "slices" + "testing" + + "github.com/databricks/cli/libs/aitools/mcp" + "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/libs/telemetry" + "github.com/databricks/cli/libs/telemetry/protos" + "github.com/databricks/databricks-sdk-go/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMcpServiceType(t *testing.T) { + // Every leaf in the CLI's known table has its own enum, so a known service never reports OTHER_SYSTEM. + cases := map[string]protos.AitoolsMcpService{ + "system.ai.genie_one_mcp": protos.AitoolsMcpServiceGenieOne, + "system.ai.dbsql": protos.AitoolsMcpServiceDBSQL, + "system.ai.web_search": protos.AitoolsMcpServiceWebSearch, + "system.ai.sandbox": protos.AitoolsMcpServiceSandbox, + "system.ai.slack": protos.AitoolsMcpServiceSlack, + "system.ai.github": protos.AitoolsMcpServiceGitHub, + "system.ai.atlassian": protos.AitoolsMcpServiceAtlassian, + "system.ai.google_drive": protos.AitoolsMcpServiceGoogleDrive, + "system.ai.google_calendar": protos.AitoolsMcpServiceGoogleCalendar, + "system.ai.gmail": protos.AitoolsMcpServiceGmail, + "system.ai.microsoft_365": protos.AitoolsMcpServiceMicrosoft365, + // The schema name is case-insensitive in Unity Catalog. + "System.AI.GitHub": protos.AitoolsMcpServiceGitHub, + // A system.ai leaf the CLI does not know yet. + "system.ai.future_tool": protos.AitoolsMcpServiceOtherSystem, + // Anything outside system.ai, whatever its name. + "main.tools.github": protos.AitoolsMcpServiceCustom, + "system.ai": protos.AitoolsMcpServiceCustom, + "system.ai.a.github": protos.AitoolsMcpServiceCustom, + } + for name, want := range cases { + assert.Equal(t, want, mcpServiceType(name), name) + } +} + +// TestMcpServiceTypeCoversKnownServices checks that every service in the CLI's known table has its own enum, so a known +// service never reports OTHER_SYSTEM. A new table entry needs a new value in mcpServiceTypes. +func TestMcpServiceTypeCoversKnownServices(t *testing.T) { + for _, leaf := range mcp.KnownLeaves() { + assert.Contains(t, mcpServiceTypes, leaf, "system.ai.%s has no telemetry enum: add it to mcpServiceTypes", leaf) + assert.NotEqualf(t, protos.AitoolsMcpServiceOtherSystem, mcpServiceType(mcp.SystemAILocation+"."+leaf), + "system.ai.%s has no telemetry enum", leaf) + } +} + +func TestAddServerResults(t *testing.T) { + servers := []addServerOutput{ + { + Service: "system.ai.github", + SignInRequired: true, + SignIn: &addSignInOutput{State: signInStateActive}, + Agents: []addAgentOutput{ + {Name: "claude-code", Status: statusInstalled}, + {Name: "codex", Status: statusReasserted}, + {Name: "cursor", Status: statusFailed, ErrorCategory: protos.AitoolsErrorCategoryAgentConfigInvalid}, + }, + }, + { + Service: "main.tools.custom", + Agents: []addAgentOutput{{Name: "claude-code", Status: statusInstalled}}, + }, + } + + assert.Equal(t, []protos.AitoolsMcpServerResult{ + { + Service: protos.AitoolsMcpServiceGitHub, + Agent: protos.AitoolsAgentTypeClaudeCode, + Outcome: protos.AitoolsTargetOutcomeChanged, + SignInState: protos.AitoolsMcpSignInStateActive, + }, + { + Service: protos.AitoolsMcpServiceGitHub, + Agent: protos.AitoolsAgentTypeCodex, + Outcome: protos.AitoolsTargetOutcomeUnchanged, + SignInState: protos.AitoolsMcpSignInStateActive, + }, + { + Service: protos.AitoolsMcpServiceGitHub, + Agent: protos.AitoolsAgentTypeCursor, + Outcome: protos.AitoolsTargetOutcomeFailed, + ErrorCategory: protos.AitoolsErrorCategoryAgentConfigInvalid, + SignInState: protos.AitoolsMcpSignInStateActive, + }, + { + Service: protos.AitoolsMcpServiceCustom, + Agent: protos.AitoolsAgentTypeClaudeCode, + Outcome: protos.AitoolsTargetOutcomeChanged, + SignInState: protos.AitoolsMcpSignInStateNotRequired, + }, + }, addServerResults(servers)) +} + +func TestRemoveServerResult(t *testing.T) { + assert.Equal(t, protos.AitoolsTargetOutcomeChanged, + removeServerResult("system.ai.github", "claude-code", removeDeleted, nil).Outcome) + // A pruned entry changes the state, so it is CHANGED too. An entry changed outside this CLI is left in place: SKIPPED. + assert.Equal(t, protos.AitoolsTargetOutcomeChanged, + removeServerResult("system.ai.github", "claude-code", removePruned, nil).Outcome) + assert.Equal(t, protos.AitoolsTargetOutcomeSkipped, + removeServerResult("system.ai.github", "claude-code", removeKept, nil).Outcome) + + failed := removeServerResult("system.ai.github", "codex", removeDeleted, errors.New("config is not JSON")) + assert.Equal(t, protos.AitoolsMcpServerResult{ + Service: protos.AitoolsMcpServiceGitHub, + Agent: protos.AitoolsAgentTypeCodex, + Outcome: protos.AitoolsTargetOutcomeFailed, + ErrorCategory: protos.AitoolsErrorCategoryUncategorized, + }, failed) +} + +func TestSignInServerResult(t *testing.T) { + cases := []struct { + name string + state string + alreadySignedIn bool + err error + want protos.AitoolsMcpServerResult + }{ + { + name: "signed in now", + state: signInStateActive, + want: protos.AitoolsMcpServerResult{Service: protos.AitoolsMcpServiceGitHub, Outcome: protos.AitoolsTargetOutcomeChanged, SignInState: protos.AitoolsMcpSignInStateActive}, + }, + { + name: "already active", + state: signInStateActive, + alreadySignedIn: true, + want: protos.AitoolsMcpServerResult{Service: protos.AitoolsMcpServiceGitHub, Outcome: protos.AitoolsTargetOutcomeUnchanged, SignInState: protos.AitoolsMcpSignInStateActive}, + }, + { + name: "no sign-in needed", + state: signInStateNotRequired, + want: protos.AitoolsMcpServerResult{Service: protos.AitoolsMcpServiceGitHub, Outcome: protos.AitoolsTargetOutcomeUnchanged, SignInState: protos.AitoolsMcpSignInStateNotRequired}, + }, + { + // A run that never learned the state, such as a read that failed, reports none. + name: "state not learned", + err: failure(protos.AitoolsErrorCategoryMCPServiceNotFound, nil, "not found"), + want: protos.AitoolsMcpServerResult{ + Service: protos.AitoolsMcpServiceGitHub, Outcome: protos.AitoolsTargetOutcomeFailed, + ErrorCategory: protos.AitoolsErrorCategoryMCPServiceNotFound, + }, + }, + { + name: "timed out", + state: signInStateTimeout, + err: failure(protos.AitoolsErrorCategorySignInTimeout, nil, "timed out"), + want: protos.AitoolsMcpServerResult{ + Service: protos.AitoolsMcpServiceGitHub, Outcome: protos.AitoolsTargetOutcomeFailed, + ErrorCategory: protos.AitoolsErrorCategorySignInTimeout, SignInState: protos.AitoolsMcpSignInStatePending, + }, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, signInServerResult("system.ai.github", tc.state, tc.alreadySignedIn, tc.err)) + }) + } +} + +func TestSignOutServerResult(t *testing.T) { + // Logout does not read the sign-in state, so the result has none. + assert.Equal(t, protos.AitoolsMcpServerResult{Service: protos.AitoolsMcpServiceGitHub, Outcome: protos.AitoolsTargetOutcomeChanged}, + signOutServerResult("system.ai.github", signOutStateDeleted, nil)) + assert.Equal(t, protos.AitoolsMcpServerResult{Service: protos.AitoolsMcpServiceGitHub, Outcome: protos.AitoolsTargetOutcomeUnchanged}, + signOutServerResult("system.ai.github", signOutStateNotSignedIn, nil)) +} + +func TestProxyStartedEvent(t *testing.T) { + assert.Equal(t, protos.AitoolsMcpEvent{ + CallingAgent: protos.AitoolsAgentTypeClaudeCode, + Operation: protos.AitoolsMcpOperationProxyStarted, + Servers: []protos.AitoolsMcpServerResult{{Service: protos.AitoolsMcpServiceGenieOne}}, + }, proxyStartedEvent(protos.AitoolsAgentTypeClaudeCode, "system.ai.genie_one_mcp")) +} + +// TestLogMcpEventDropsRejectedCredential checks that an event that names a rejected credential is not logged, and that the +// run then uploads nothing: its upload would authenticate with that credential again. The events of the run that were +// logged before stay buffered, and a later event of the run is logged, but not uploaded. +func TestLogMcpEventDropsRejectedCredential(t *testing.T) { + withHome(t) + ctx := env.Set(telemetry.WithNewLogger(t.Context()), "DATABRICKS_CLI_DISABLE_TELEMETRY", "") + telemetry.SetUploadConfig(ctx, &config.Config{Profile: "logfood", Host: "https://adb-1.test"}) + assert.False(t, logMcpEvent(ctx, protos.AitoolsMcpEvent{ + Operation: protos.AitoolsMcpOperationSignIn, + Servers: []protos.AitoolsMcpServerResult{{ + Service: protos.AitoolsMcpServiceGitHub, + Outcome: protos.AitoolsTargetOutcomeFailed, + ErrorCategory: protos.AitoolsErrorCategoryAuthFailed, + }}, + })) + assert.Empty(t, telemetry.BufferedEvents(ctx)) + _, uploads := telemetry.UploadConfig(ctx) + assert.False(t, uploads) + + assert.True(t, logMcpEvent(ctx, protos.AitoolsMcpEvent{ + Operation: protos.AitoolsMcpOperationAdd, + Servers: []protos.AitoolsMcpServerResult{{ + Service: protos.AitoolsMcpServiceGitHub, + Outcome: protos.AitoolsTargetOutcomeChanged, + SignInState: protos.AitoolsMcpSignInStatePending, + }}, + })) + assert.Len(t, telemetry.BufferedEvents(ctx), 1) + _, uploads = telemetry.UploadConfig(ctx) + assert.False(t, uploads, "the add event is logged, and it is not uploaded with the rejected run") +} + +// TestSetUploadConfigLogsNoWarningForFailedHostMetadata checks that an upload whose host metadata lookup fails warns +// nothing through the global logger, since the SDK would log that failure there. The command has succeeded, and the +// upload is best-effort. +func TestSetUploadConfigLogsNoWarningForFailedHostMetadata(t *testing.T) { + home := withHome(t) + writeHomeConfig(t, home, "[logfood]\nhost = https://adb-1.test\nauth_type = pat\ntoken = t\n") + factory := config.DefaultHostMetadataResolverFactory + config.DefaultHostMetadataResolverFactory = func(*config.Config) config.HostMetadataResolver { + return func(context.Context, string) (*config.HostMetadata, error) { return nil, errors.New("no such host") } + } + t.Cleanup(func() { config.DefaultHostMetadataResolverFactory = factory }) + var warnings bytes.Buffer + defaultLogger := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(&warnings, &slog.HandlerOptions{Level: slog.LevelWarn}))) + t.Cleanup(func() { slog.SetDefault(defaultLogger) }) + + ctx := env.Set(telemetry.WithNewLogger(t.Context()), "DATABRICKS_CLI_DISABLE_TELEMETRY", "") + setUploadConfig(ctx, "logfood", filepath.Join(home, ".databrickscfg")) + _, uploads := telemetry.UploadConfig(ctx) + assert.True(t, uploads) + assert.Empty(t, warnings.String()) +} + +// TestMcpProxyAuthRejectedRefusesUpload checks that a 401 the proxy answers refuses the run's upload, even though the +// serve command sets its upload config at exit. +func TestMcpProxyAuthRejectedRefusesUpload(t *testing.T) { + ctx := telemetry.WithNewLogger(t.Context()) + telemetry.SetUploadConfig(ctx, &config.Config{Profile: "logfood", Host: "https://adb-1.test"}) + mcpProxyTelemetry{}.AuthRejected(ctx) + _, uploads := telemetry.UploadConfig(ctx) + assert.False(t, uploads) +} + +func TestSignInPromptedEvent(t *testing.T) { + for signedIn, state := range map[bool]protos.AitoolsMcpSignInState{ + true: protos.AitoolsMcpSignInStateActive, + false: protos.AitoolsMcpSignInStatePending, + } { + event := signInPromptedEvent(protos.AitoolsAgentTypeCursor, "system.ai.github", signedIn) + assert.Equal(t, protos.AitoolsMcpEvent{ + CallingAgent: protos.AitoolsAgentTypeCursor, + Operation: protos.AitoolsMcpOperationSignInPrompted, + Servers: []protos.AitoolsMcpServerResult{{ + Service: protos.AitoolsMcpServiceGitHub, + ErrorCategory: protos.AitoolsErrorCategorySignInRequired, + SignInState: state, + }}, + }, event) + } +} + +func TestBoundProfileCount(t *testing.T) { + dir := t.TempDir() + first := filepath.Join(dir, "first.cfg") + second := filepath.Join(dir, "second.cfg") + state := &mcp.State{Servers: []mcp.Entry{ + {Profile: "logfood", Service: "system.ai.github", ConfigFile: first}, + {Profile: "logfood", Service: "system.ai.genie_one_mcp", ConfigFile: first}, + // DEFAULT in two config files is two profiles, as list groups them. + {Profile: "DEFAULT", Service: "system.ai.github", ConfigFile: first}, + {Profile: "DEFAULT", Service: "system.ai.github", ConfigFile: second}, + }} + assert.Equal(t, int64(3), boundProfileCount(t.Context(), state)) + assert.Equal(t, int64(0), boundProfileCount(t.Context(), &mcp.State{})) +} + +func TestSignInEventOf(t *testing.T) { + // An ACTIVE credential that the add found signed in is UNCHANGED, as add --login reports it. + assert.Equal(t, protos.AitoolsMcpEvent{ + CallingAgent: callingAgentType(), + Operation: protos.AitoolsMcpOperationSignIn, + Servers: []protos.AitoolsMcpServerResult{{ + Service: protos.AitoolsMcpServiceGitHub, + Outcome: protos.AitoolsTargetOutcomeUnchanged, + SignInState: protos.AitoolsMcpSignInStateActive, + }}, + }, signInEventOf("system.ai.github", signInStateActive, true, nil)) +} + +// TestMcpEventOmitsUserContent checks that the event carries enums and counts only: a custom service's name must not +// appear anywhere in the payload. +func TestMcpEventOmitsUserContent(t *testing.T) { + event := protos.AitoolsMcpEvent{ + CallingAgent: protos.AitoolsAgentTypeClaudeCode, + Operation: protos.AitoolsMcpOperationAdd, + BoundProfileCount: 1, + Servers: addServerResults([]addServerOutput{{ + Service: "acme_catalog.secret_schema.internal_tool", + Agents: []addAgentOutput{{Name: "claude-code", Status: statusInstalled}}, + }}), + } + b, err := json.Marshal(event) + require.NoError(t, err) + + var got map[string]any + require.NoError(t, json.Unmarshal(b, &got)) + assert.ElementsMatch(t, []string{"calling_agent", "operation", "servers", "bound_profile_count"}, slices.Collect(maps.Keys(got))) + assert.NotContains(t, string(b), "acme_catalog") + assert.NotContains(t, string(b), "secret_schema") + assert.NotContains(t, string(b), "internal_tool") + assert.Contains(t, string(b), `"service":"CUSTOM"`) +} + +// TestAddSignInStateWithoutAState checks that a sign-in that failed before it read a state is PENDING, as a server that +// needs sign-in is until its state is known. A server that needs no sign-in is NOT_REQUIRED. +func TestAddSignInStateWithoutAState(t *testing.T) { + failed := addServerOutput{Service: "system.ai.github", SignInRequired: true, SignIn: &addSignInOutput{ErrorCategory: protos.AitoolsErrorCategoryUncategorized}} + assert.Equal(t, protos.AitoolsMcpSignInStatePending, addSignInState(failed)) + assert.Equal(t, protos.AitoolsMcpSignInStateNotRequired, addSignInState(addServerOutput{Service: "system.ai.dbsql"})) +} diff --git a/cmd/aitools/mcp_test.go b/cmd/aitools/mcp_test.go index 8a70c18bbad..5217647400d 100644 --- a/cmd/aitools/mcp_test.go +++ b/cmd/aitools/mcp_test.go @@ -96,7 +96,7 @@ host = https://b.test _, err := resolveProfile(ctx, "") require.Error(t, err) assert.Equal(t, "several workspace profiles exist; pass --profile. Found: a, b", err.Error()) - assert.Equal(t, categoryProfileRequired, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryProfileRequired, errorCategory(err)) }) t.Run("no workspace profiles is an error naming the file", func(t *testing.T) { @@ -104,7 +104,7 @@ host = https://b.test _, err := resolveProfile(ctx, "") require.Error(t, err) assert.Contains(t, err.Error(), "no workspace profiles in ") - assert.Equal(t, categoryNoProfile, errorCategory(err)) + assert.Equal(t, protos.AitoolsErrorCategoryNoProfile, errorCategory(err)) assert.Contains(t, err.Error(), "databricks auth login --host --profile ") }) } @@ -114,11 +114,11 @@ func TestErrorCategoryUsesTypedErrorsOnly(t *testing.T) { assert.Equal(t, protos.AitoolsErrorCategoryUnknownAgent, errorCategory(typed)) assert.Equal(t, "unknown agent \"pi\"", typed.Error()) - wrapped := failure(categoryServiceNotFound, errors.New("boom"), "%s", "boom") - assert.Equal(t, categoryServiceNotFound, errorCategory(wrapped)) + wrapped := failure(protos.AitoolsErrorCategoryMCPServiceNotFound, errors.New("boom"), "%s", "boom") + assert.Equal(t, protos.AitoolsErrorCategoryMCPServiceNotFound, errorCategory(wrapped)) configInvalid := &os.PathError{Op: "parse", Path: "x", Err: agentconfig.ErrConfigInvalid} - assert.Equal(t, categoryAgentConfigInvalid, errorCategory(configInvalid)) + assert.Equal(t, protos.AitoolsErrorCategoryAgentConfigInvalid, errorCategory(configInvalid)) assert.Equal(t, protos.AitoolsErrorCategoryUncategorized, errorCategory(errors.New("some failure"))) } diff --git a/libs/aitools/mcp/binding.go b/libs/aitools/mcp/binding.go index 1a530a73efa..1c8c4afc9cb 100644 --- a/libs/aitools/mcp/binding.go +++ b/libs/aitools/mcp/binding.go @@ -11,9 +11,11 @@ import ( "time" "github.com/databricks/cli/libs/aitools/mcpconfig" + "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/databrickscfg" "github.com/databricks/cli/libs/databrickscfg/profile" "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/shellquote" "github.com/databricks/databricks-sdk-go/config" ) @@ -109,14 +111,50 @@ func BoundConfig(profileName, configFile string) (*config.Config, error) { // profile runs on a context of its own, so a context deadline does not reach it, and these budgets are what bound it (see // profileMetadata.Load in cmd/auth). func BoundConfigWithin(profileName, configFile string, budget time.Duration) (*config.Config, error) { + return resolveBound(withinBudget(profileName, configFile, budget)) +} + +// BoundUploadConfig is BoundConfigWithin for a config that only a telemetry upload uses. Its credentials never refresh a +// token, since the command may exit before a refresh finishes, and a rotated single-use refresh token would be lost (see +// auth.CredentialsWithoutRefresh). It does not authenticate. +func BoundUploadConfig(profileName, configFile string, budget time.Duration) (*config.Config, error) { + cfg := withinBudget(profileName, configFile, budget) + cfg.Credentials = auth.CredentialsWithoutRefresh() + quietHostMetadata(cfg) + return resolveBound(cfg) +} + +// quietHostMetadata makes a failed host metadata lookup of cfg a debug message. The SDK logs that failure as a warning +// through its global logger, since the resolution runs on a background context, so a command that succeeded would print a +// warning that only its telemetry upload caused. The SDK falls back to the user config when the lookup fails, and so does +// this one, with a nil result. +func quietHostMetadata(cfg *config.Config) { + resolve := cfg.DefaultHostMetadataResolver() + if factory := config.DefaultHostMetadataResolverFactory; factory != nil { + if r := factory(cfg); r != nil { + resolve = r + } + } + cfg.HostMetadataResolver = func(ctx context.Context, host string) (*config.HostMetadata, error) { + meta, err := resolve(ctx, host) + if err != nil { + log.Debugf(ctx, "telemetry: cannot resolve host metadata for %s: %s", host, err) + return nil, nil + } + return meta, nil + } +} + +// withinBudget is the config of profileName in configFile, with the SDK's request and retry budgets set to budget. +func withinBudget(profileName, configFile string, budget time.Duration) *config.Config { seconds := int(budget / time.Second) - return resolveBound(&config.Config{ + return &config.Config{ Profile: profileName, ConfigFile: configFile, Loaders: []config.Loader{databrickscfg.ResolveNonAuthFromEnv, config.ConfigFile}, HTTPTimeoutSeconds: seconds, RetryTimeoutSeconds: seconds, - }) + } } // resolveBound resolves cfg, as BoundConfig does. diff --git a/libs/aitools/mcp/binding_test.go b/libs/aitools/mcp/binding_test.go index 372c141cae3..5be3fd88932 100644 --- a/libs/aitools/mcp/binding_test.go +++ b/libs/aitools/mcp/binding_test.go @@ -10,8 +10,10 @@ import ( "os" "path/filepath" "testing" + "time" "github.com/databricks/cli/libs/aitools/mcp" + "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/shellquote" "github.com/stretchr/testify/assert" @@ -935,3 +937,17 @@ func TestConfigFileCommandUsesForwardSlashes(t *testing.T) { assert.Equal(t, "DATABRICKS_CONFIG_FILE="+filepath.ToSlash(path)+" databricks auth login --profile logfood", mcp.ConfigFileCommand(path, "databricks auth login --profile logfood")) } + +// TestBoundUploadConfigNeverRefreshes checks that the config of a telemetry upload never refreshes a token, and that the +// config a serve session binds keeps the default chain. +func TestBoundUploadConfigNeverRefreshes(t *testing.T) { + cfgPath := bindingTestConfig(t, bindingTestProfileContents(bindingTestToken)) + + upload, err := mcp.BoundUploadConfig(bindingTestProfile, cfgPath, time.Second) + require.NoError(t, err) + assert.Equal(t, auth.CredentialsWithoutRefresh(), upload.Credentials) + + session, err := mcp.BoundConfigWithin(bindingTestProfile, cfgPath, time.Second) + require.NoError(t, err) + assert.NotEqual(t, auth.CredentialsWithoutRefresh(), session.Credentials) +} diff --git a/libs/aitools/mcp/known.go b/libs/aitools/mcp/known.go index 17883fe1a0b..47dab54d23e 100644 --- a/libs/aitools/mcp/known.go +++ b/libs/aitools/mcp/known.go @@ -1,6 +1,10 @@ package mcp -import "strings" +import ( + "maps" + "slices" + "strings" +) // SystemAILocation is the schema that ships the Databricks managed MCP services. const SystemAILocation = "system.ai" @@ -61,6 +65,13 @@ var knownSystemAI = map[string]KnownService{ }, } +// KnownLeaves returns the leaf names of the system.ai services in the CLI's table, sorted. +// +//deadcode:allow used by the service-mapping tests +func KnownLeaves() []string { + return slices.Sorted(maps.Keys(knownSystemAI)) +} + // KnownOf returns the CLI's description of a service, or nil when it is not a system.ai service in the table. The leaf // is compared without case, as Unity Catalog names are. func KnownOf(fullName string) *KnownService { diff --git a/libs/aitools/mcp/known_test.go b/libs/aitools/mcp/known_test.go index b401338982d..4b8c89a58d3 100644 --- a/libs/aitools/mcp/known_test.go +++ b/libs/aitools/mcp/known_test.go @@ -1,6 +1,7 @@ package mcp import ( + "slices" "testing" "github.com/stretchr/testify/assert" @@ -38,6 +39,18 @@ func TestKnownSystemAITable(t *testing.T) { } } +// TestKnownLeavesAreTheSystemAITable checks that KnownLeaves lists the leaves of the system.ai table, sorted, with each +// leaf known as a system.ai service. +func TestKnownLeavesAreTheSystemAITable(t *testing.T) { + leaves := KnownLeaves() + assert.Contains(t, leaves, "genie_one_mcp") + assert.True(t, slices.IsSorted(leaves)) + assert.Len(t, leaves, len(knownSystemAI)) + for _, leaf := range leaves { + assert.NotNil(t, KnownOf(SystemAILocation+"."+leaf), leaf) + } +} + func TestProviderLabel(t *testing.T) { assert.Equal(t, "GitHub", ProviderLabel("system.ai.github")) assert.Equal(t, "Genie One", ProviderLabel("SYSTEM.AI.genie_one_mcp")) diff --git a/libs/aitools/mcp/proxy/bridge_test.go b/libs/aitools/mcp/proxy/bridge_test.go index 11c60b775c4..3cc4fd531d3 100644 --- a/libs/aitools/mcp/proxy/bridge_test.go +++ b/libs/aitools/mcp/proxy/bridge_test.go @@ -2319,7 +2319,7 @@ func TestSignInWritesShareTheBridgeStderr(t *testing.T) { Config: &config.Config{Host: testHost}, Authenticate: func(*http.Request) error { return nil }, Profile: "logfood", - Challenge: &Challenge{Service: testService, Profile: "logfood", Host: testHost, Label: "GitHub", LoginCommand: loginCommand, SignIn: signIn}, + Challenge: &Challenge{Service: testService, Profile: "logfood", Host: testHost, Label: "GitHub", LoginCommand: loginCommand, SignIn: signIn, Recorder: &promptRecorder{}}, }) }() diff --git a/libs/aitools/mcp/proxy/challenge.go b/libs/aitools/mcp/proxy/challenge.go index a1959a18e49..9109db7906c 100644 --- a/libs/aitools/mcp/proxy/challenge.go +++ b/libs/aitools/mcp/proxy/challenge.go @@ -50,6 +50,18 @@ type SignInRequest struct { // cancelled. Its error is the answer's text. type SignInFunc func(ctx context.Context, req SignInRequest) (alreadySignedIn, urlShown bool, err error) +// Recorder receives the proxy's telemetry events. The command implements it, so the proxy does not +// build telemetry events itself. +type Recorder interface { + // SignInPrompted runs after the wait of an authenticate call that showed the sign-in page: one it opened, or one an + // earlier cancelled call opened, which the user may still have open. signedIn reports whether the credential became + // ACTIVE. A retry after a cancelled call reports its own outcome. + SignInPrompted(ctx context.Context, signedIn bool) + // AuthRejected runs when the service answers 401 without asking for a sign-in: the Databricks credential was + // rejected, and the answer is the auth failure. + AuthRejected(ctx context.Context) +} + // Challenge turns on sign-in handling for a service that needs a provider sign-in. type Challenge struct { // Service is the full name of the MCP service, for example system.ai.github. @@ -70,6 +82,8 @@ type Challenge struct { // SignIn runs the sign-in. The command provides it, so the proxy never polls or opens a browser itself. Its error is // the answer's text, so it names the service, the profile and the fix. SignIn SignInFunc + // Recorder logs the sign-in prompt. The command provides it; it must not be nil. + Recorder Recorder } // signInHooks applies the sign-in rules to the bridge. Services without a challenge get no hooks. @@ -145,7 +159,7 @@ func (h *signInHooks) Local(ctx context.Context, request []byte, notify func([]b // answer becomes a sign-in message, followed by tools/list_changed so an agent that cached its list re-lists. A 401 that // is not a challenge becomes the auth failure (see authFailure). Only the response to the request is read; any other // message passes through untouched. -func (h *signInHooks) Result(_ context.Context, request []byte, status int, message []byte) ([][]byte, error) { +func (h *signInHooks) Result(ctx context.Context, request []byte, status int, message []byte) ([][]byte, error) { req, err := parseMessage(request) if err != nil { return nil, err @@ -163,6 +177,7 @@ func (h *signInHooks) Result(_ context.Context, request []byte, status int, mess } challenged := h.challenged(&reply) if status == http.StatusUnauthorized && !challenged { + h.c.Recorder.AuthRejected(ctx) return [][]byte{errorResponse(req.ID, codeUpstream, h.authFailure())}, nil } if req.Method != methodToolsList && req.Method != methodToolsCall { @@ -445,7 +460,13 @@ func (h *signInHooks) authenticate(ctx context.Context, id, progressToken json.R return [][]byte{toolResult(id, h.stillChallengedText(), true)} } - alreadySignedIn, urlShown, err := h.c.SignIn(ctx, SignInRequest{Open: !h.pageCancelled, Progress: report, Stderr: h.stderr}) + open := !h.pageCancelled + alreadySignedIn, urlShown, err := h.c.SignIn(ctx, SignInRequest{Open: open, Progress: report, Stderr: h.stderr}) + // A prompt is a page the user was shown. A caller that was already signed in, or a sign-in that failed before its + // URL was printed, prompted nobody. + if !alreadySignedIn && urlShown { + h.c.Recorder.SignInPrompted(ctx, err == nil) + } h.pageCancelled = urlShown && err != nil && ctx.Err() != nil h.stateMu.Lock() h.alreadySignedIn = err == nil && alreadySignedIn diff --git a/libs/aitools/mcp/proxy/challenge_test.go b/libs/aitools/mcp/proxy/challenge_test.go index 0a399b0e1bb..7c5bc6d60bb 100644 --- a/libs/aitools/mcp/proxy/challenge_test.go +++ b/libs/aitools/mcp/proxy/challenge_test.go @@ -68,6 +68,32 @@ func (s *signInStub) requests() []SignInRequest { // loginCommand is the hint that the command builds for a pinned config file and a profile name with a space. const loginCommand = `DATABRICKS_CONFIG_FILE='/work/cfg/my file.cfg' databricks aitools mcp login system.ai.github --profile 'my profile'` +// promptRecorder records the SIGN_IN_PROMPTED events: one per sign-in page that authenticate opened. It counts the +// rejections of the Databricks credential too. +type promptRecorder struct { + mu sync.Mutex + signedIn []bool + rejected int +} + +func (r *promptRecorder) SignInPrompted(_ context.Context, signedIn bool) { + r.mu.Lock() + defer r.mu.Unlock() + r.signedIn = append(r.signedIn, signedIn) +} + +func (r *promptRecorder) AuthRejected(_ context.Context) { + r.mu.Lock() + defer r.mu.Unlock() + r.rejected++ +} + +func (r *promptRecorder) prompts() []bool { + r.mu.Lock() + defer r.mu.Unlock() + return append([]bool(nil), r.signedIn...) +} + // challengedBridge returns a bridge to url with the sign-in hooks of a service that needs sign-in. func challengedBridge(url string, signIn SignInFunc) *Bridge { b := bridgeFor(url) @@ -255,6 +281,109 @@ func TestAuthenticateSaysAlreadySignedInAfterSignIn(t *testing.T) { require.NoError(t, p.finish(t)) } +// TestAuthenticateRecordsPromptWhenPageOpens checks that the prompt is recorded once per page that was shown, after its +// wait, with whether the credential became ACTIVE. A call that finds the caller already signed in records nothing. +func TestAuthenticateRecordsPromptWhenPageOpens(t *testing.T) { + srv, _ := challengedService(t) + signIn := &signInStub{reply: func(call int) (bool, error) { + if call == 1 { + return false, errors.New("sign-in did not complete") + } + return call == 3, nil + }} + recorder := &promptRecorder{} + c := testChallenge(signIn.SignIn) + c.Recorder = recorder + b := bridgeFor(srv.URL) + b.Hooks = newSignInHooks(c, io.Discard) + p := startPiped(t, t.Context(), b) + authenticate := `{"jsonrpc":"2.0","id":%d,"method":"tools/call","params":{"name":"authenticate","arguments":{}}}` + challenge(t, p) + + p.send(t, fmt.Sprintf(authenticate, 1)) + p.waitLines(t, 2) + p.send(t, fmt.Sprintf(authenticate, 2)) + p.waitLines(t, 4) + p.send(t, fmt.Sprintf(authenticate, 3)) + p.waitLines(t, 6) + + assert.Equal(t, []bool{true, true, true}, signIn.calls(), "every call opens the page") + assert.Equal(t, []bool{false, true}, recorder.prompts(), "call 3 was already signed in and records nothing") + require.NoError(t, p.finish(t)) +} + +// TestAuthenticateRecordsNoPromptWhenNoURLShown checks that a sign-in that fails before its URL is printed, such as a read +// of the workspace ID, shows no page, so nothing is recorded. +func TestAuthenticateRecordsNoPromptWhenNoURLShown(t *testing.T) { + srv, _ := challengedService(t) + var calls int + signIn := func(context.Context, SignInRequest) (bool, bool, error) { + calls++ + return false, false, errors.New("cannot read the workspace ID") + } + recorder := &promptRecorder{} + c := testChallenge(signIn) + c.Recorder = recorder + b := bridgeFor(srv.URL) + b.Hooks = newSignInHooks(c, io.Discard) + p := startPiped(t, t.Context(), b) + challenge(t, p) + + p.send(t, `{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"authenticate","arguments":{}}}`) + p.waitLines(t, 2) + + assert.Equal(t, 1, calls, "the sign-in ran") + assert.Empty(t, recorder.prompts(), "no page was shown, so no prompt is recorded") + require.NoError(t, p.finish(t)) +} + +// TestAuthenticateRecordsPromptOfCallThatWaitsOnCancelledPage checks that a call which waits on the page an earlier, +// cancelled call showed records its own outcome: the page was shown for it too, so a sign-in that completes there is +// recorded as ACTIVE, not left PENDING from the cancelled call. +func TestAuthenticateRecordsPromptOfCallThatWaitsOnCancelledPage(t *testing.T) { + srv, _ := newFakeEndpoint(t, answerWith(http.StatusOK, listChallenge)) + entered, done := make(chan struct{}), make(chan struct{}) + var mu sync.Mutex + var opens []bool + signIn := func(ctx context.Context, req SignInRequest) (bool, bool, error) { + mu.Lock() + opens = append(opens, req.Open) + first := len(opens) == 1 + mu.Unlock() + if !first { + return false, true, nil + } + close(entered) + <-ctx.Done() + close(done) + return false, true, ctx.Err() + } + recorder := &promptRecorder{} + c := testChallenge(signIn) + c.Recorder = recorder + b := bridgeFor(srv.URL) + b.Hooks = newSignInHooks(c, io.Discard) + p := startPiped(t, t.Context(), b) + authenticate := `{"jsonrpc":"2.0","id":%d,"method":"tools/call","params":{"name":"authenticate","arguments":{}}}` + + p.send(t, `{"jsonrpc":"2.0","id":2,"method":"tools/list"}`) + p.waitLines(t, 1) + p.send(t, fmt.Sprintf(authenticate, 5)) + waitClosed(t, entered, "the first sign-in to start") + p.send(t, `{"jsonrpc":"2.0","method":"notifications/cancelled","params":{"requestId":5}}`) + waitClosed(t, done, "the first sign-in to return") + p.send(t, fmt.Sprintf(authenticate, 6)) + require.Eventually(t, func() bool { + return len(recorder.prompts()) == 2 + }, 10*time.Second, time.Millisecond) + require.NoError(t, p.finish(t)) + + mu.Lock() + defer mu.Unlock() + assert.Equal(t, []bool{true, false}, opens, "the second call does not open the page again") + assert.Equal(t, []bool{false, true}, recorder.prompts(), "the cancelled call is PENDING, and the second call is ACTIVE") +} + func TestAuthenticateReopensPageAfterTimeout(t *testing.T) { srv, _ := challengedService(t) signIn := &signInStub{reply: func(call int) (bool, error) { @@ -343,7 +472,7 @@ func TestChallengeHintsPrintTheLoginCommand(t *testing.T) { return false, true, errors.New("sign-in to system.ai.github did not complete in 5 minutes. Finish at " + testURL + " then retry: " + loginCommand) } b := bridgeFor(srv.URL) - b.Hooks = newSignInHooks(&Challenge{Service: testService, Profile: "my profile", Host: testHost, Label: "GitHub", LoginCommand: loginCommand, SignIn: signIn}, io.Discard) + b.Hooks = newSignInHooks(&Challenge{Service: testService, Profile: "my profile", Host: testHost, Label: "GitHub", LoginCommand: loginCommand, SignIn: signIn, Recorder: &promptRecorder{}}, io.Discard) p := startPiped(t, t.Context(), b) p.send(t, `{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"echo","arguments":{}}}`) @@ -487,7 +616,7 @@ func TestChallengeMatchesServiceNameWithoutCase(t *testing.T) { `{"mode":"url","url":"` + testURL + `"}]}}}` srv, _ := newFakeEndpoint(t, answerWith(http.StatusOK, body)) b := bridgeFor(srv.URL) - b.Hooks = newSignInHooks(&Challenge{Service: "SYSTEM.AI.GITHUB", Profile: "logfood", Host: testHost, Label: "GitHub", LoginCommand: loginCommand}, io.Discard) + b.Hooks = newSignInHooks(&Challenge{Service: "SYSTEM.AI.GITHUB", Profile: "logfood", Host: testHost, Label: "GitHub", LoginCommand: loginCommand, Recorder: &promptRecorder{}}, io.Discard) out, _ := runAll(t, b, `{"jsonrpc":"2.0","id":2,"method":"tools/list"}`+"\n") require.Len(t, out, 1) @@ -865,6 +994,23 @@ func TestUnauthorizedIsAuthFailureUnlessItAsksForSignIn(t *testing.T) { } } +// TestUnauthorizedRecordsRejectedCredential checks that a 401 without a sign-in challenge is recorded as a rejected +// credential, once per answer, so that the command refuses its telemetry upload. +func TestUnauthorizedRecordsRejectedCredential(t *testing.T) { + srv, _ := newFakeEndpoint(t, func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, "unauthorized", http.StatusUnauthorized) + }) + signIn := &signInStub{reply: func(int) (bool, error) { return false, nil }} + recorder := &promptRecorder{} + challenge := testChallenge(signIn.SignIn) + challenge.Recorder = recorder + b := bridgeFor(srv.URL) + b.Hooks = newSignInHooks(challenge, io.Discard) + runAll(t, b, `{"jsonrpc":"2.0","id":2,"method":"tools/list"}`+"\n") + + assert.Equal(t, 1, recorder.rejected) +} + // TestRejectedCredentialDoesNotAskForLogout checks a rejected Databricks token after "Already signed in": the credential reads // as signed in, and the service rejects the token with a 401. The 401 is an auth failure, not a challenge, so the next // authenticate runs the sign-in as usual, and the logout answer is not given. @@ -905,7 +1051,7 @@ func TestAuthFailureNamesPinnedConfigFile(t *testing.T) { http.Error(w, "unauthorized", http.StatusUnauthorized) }) b := bridgeFor(srv.URL) - b.Hooks = newSignInHooks(&Challenge{Service: testService, Profile: "logfood", Host: testHost, Label: "GitHub", LoginCommand: loginCommand, PinnedFile: "/work/cfg/my file.cfg"}, io.Discard) + b.Hooks = newSignInHooks(&Challenge{Service: testService, Profile: "logfood", Host: testHost, Label: "GitHub", LoginCommand: loginCommand, PinnedFile: "/work/cfg/my file.cfg", Recorder: &promptRecorder{}}, io.Discard) out, _ := runAll(t, b, `{"jsonrpc":"2.0","id":2,"method":"tools/list"}`+"\n") require.Len(t, out, 1) @@ -965,6 +1111,7 @@ func testChallenge(signIn SignInFunc) *Challenge { LoginCommand: "databricks aitools mcp login system.ai.github --profile logfood", LogoutCommand: "databricks aitools mcp logout system.ai.github --profile logfood", SignIn: signIn, + Recorder: &promptRecorder{}, } } diff --git a/libs/aitools/mcp/proxy/run_test.go b/libs/aitools/mcp/proxy/run_test.go index 8a8f195027d..f4020a9d3a1 100644 --- a/libs/aitools/mcp/proxy/run_test.go +++ b/libs/aitools/mcp/proxy/run_test.go @@ -100,7 +100,7 @@ func TestRunAdvertisesListChangedOnlyForSignInServices(t *testing.T) { assert.Contains(t, out.String(), `"listChanged":false`) opts, out, _ = runOptions(t, srv.URL, cfg, initialize) - opts.Challenge = &Challenge{Service: testService, Profile: "logfood", Host: testHost, Label: "GitHub"} + opts.Challenge = &Challenge{Service: testService, Profile: "logfood", Host: testHost, Label: "GitHub", Recorder: &promptRecorder{}} require.NoError(t, Run(t.Context(), opts)) assert.Contains(t, out.String(), `"listChanged":true`) } diff --git a/libs/telemetry/logger.go b/libs/telemetry/logger.go index aac2063fbe5..97ffad8cd30 100644 --- a/libs/telemetry/logger.go +++ b/libs/telemetry/logger.go @@ -37,6 +37,39 @@ func Disabled(ctx context.Context) bool { return env.Get(ctx, disableEnvVar) != "" } +// SetUploadConfig sets the config that Upload authenticates with, in place of the one cmdctx.ConfigUsed holds. A command +// sets it once its own binding checks have passed, so that the upload authenticates with the profile the command bound. +// Like Log, it is a no-op without a logger. +func SetUploadConfig(ctx context.Context, cfg *config.Config) { + if l, ok := loggerFromContext(ctx); ok { + l.upload = cfg + } +} + +// RefuseUpload makes Upload send nothing for the run, whatever config is set. A run calls it once a credential was +// rejected, since the upload would present that credential again. Like Log, it is a no-op without a logger. +func RefuseUpload(ctx context.Context) { + if l, ok := loggerFromContext(ctx); ok { + l.refused = true + } +} + +// UploadConfig returns the config that Upload authenticates with: the one SetUploadConfig set, else the one +// cmdctx.ConfigUsed holds. It returns false when the run refused its upload or has no config. +func UploadConfig(ctx context.Context) (*config.Config, bool) { + l, ok := loggerFromContext(ctx) + if !ok || l.refused { + return nil, false + } + if l.upload != nil { + return l.upload, true + } + if cmdctx.HasConfigUsed(ctx) { + return cmdctx.ConfigUsed(ctx), true + } + return nil, false +} + func Log(ctx context.Context, event protos.DatabricksCliLog) { // A missing logger means telemetry was never initialized on this context // (e.g. a command invoked outside the normal cmd/root setup). Dropping the @@ -54,6 +87,11 @@ type logger struct { logs []protos.FrontendLog stateLineageHash string + // upload is the config Upload authenticates with, as SetUploadConfig set it. Nil leaves cmdctx.ConfigUsed to it. + upload *config.Config + // refused means the run refused its upload (RefuseUpload), so Upload sends nothing. + refused bool + // running counts the goroutines that Upload started and stopped waiting // for, which may still be using the config (see within). running atomic.Int32 @@ -123,8 +161,9 @@ func Upload(ctx context.Context, ec protos.ExecutionContext) error { protoLogs[i] = string(b) } - if !cmdctx.HasConfigUsed(ctx) { - log.Debugf(ctx, "no auth config available; skipping telemetry upload") + cfg, ok := UploadConfig(ctx) + if !ok { + log.Debugf(ctx, "no auth config available, or the run refused its telemetry upload; skipping telemetry upload") return nil } @@ -150,7 +189,7 @@ func Upload(ctx context.Context, ec protos.ExecutionContext) error { } apiClient, err := within(ctx, &l.running, func() (*client.DatabricksClient, error) { - return newAPIClient(cmdctx.ConfigUsed(ctx), &l.running) + return newAPIClient(cfg, &l.running) }) if err != nil { return fmt.Errorf("failed to create API client: %w", err) diff --git a/libs/telemetry/logger_test.go b/libs/telemetry/logger_test.go index 94673a9dd1a..a6a01c9a72e 100644 --- a/libs/telemetry/logger_test.go +++ b/libs/telemetry/logger_test.go @@ -465,3 +465,66 @@ func TestWithinReturnsPanicAsError(t *testing.T) { assert.EqualError(t, err, "panic: boom") assert.Eventually(t, func() bool { return running.Load() == 0 }, time.Second, time.Millisecond) } + +// TestUploadUsesConfigSetByCommand checks that the upload authenticates with the config the command set, in place of the +// one cmdctx.ConfigUsed holds. +func TestUploadUsesConfigSetByCommand(t *testing.T) { + bound := testserver.New(t) + t.Cleanup(bound.Close) + other := testserver.New(t) + t.Cleanup(other.Close) + var boundCalls, otherCalls int + bound.Handle("POST", "/telemetry-ext", func(req testserver.Request) any { + boundCalls++ + return ResponseBody{NumProtoSuccess: 1} + }) + other.Handle("POST", "/telemetry-ext", func(req testserver.Request) any { + otherCalls++ + return ResponseBody{NumProtoSuccess: 1} + }) + + ctx := WithNewLogger(t.Context()) + Log(ctx, protos.DatabricksCliLog{CliTestEvent: &protos.CliTestEvent{Name: protos.DummyCliEnumValue1}}) + ctx = cmdctx.SetConfigUsed(ctx, &config.Config{Host: other.URL, Token: "other"}) + SetUploadConfig(ctx, &config.Config{Host: bound.URL, Token: "bound"}) + + cfg, ok := UploadConfig(ctx) + require.True(t, ok) + assert.Equal(t, bound.URL, cfg.Host) + + require.NoError(t, Upload(ctx, protos.ExecutionContext{})) + assert.Equal(t, 1, boundCalls) + assert.Zero(t, otherCalls) +} + +// TestUploadRefusedSendsNothing checks that a run which refused its upload sends nothing, even with a config set by the +// command, and that a config set after the refusal is not used. +func TestUploadRefusedSendsNothing(t *testing.T) { + server := testserver.New(t) + t.Cleanup(server.Close) + var calls int + server.Handle("POST", "/telemetry-ext", func(req testserver.Request) any { + calls++ + return ResponseBody{NumProtoSuccess: 1} + }) + + ctx := WithNewLogger(t.Context()) + Log(ctx, protos.DatabricksCliLog{CliTestEvent: &protos.CliTestEvent{Name: protos.DummyCliEnumValue1}}) + SetUploadConfig(ctx, &config.Config{Host: server.URL, Token: "token"}) + RefuseUpload(ctx) + SetUploadConfig(ctx, &config.Config{Host: server.URL, Token: "token"}) + + _, ok := UploadConfig(ctx) + assert.False(t, ok) + require.NoError(t, Upload(ctx, protos.ExecutionContext{})) + assert.Zero(t, calls) +} + +// TestUploadConfigWithoutLogger checks that the upload slot is a no-op without a logger, as Log is. +func TestUploadConfigWithoutLogger(t *testing.T) { + ctx := t.Context() + SetUploadConfig(ctx, &config.Config{Host: "https://ws.test"}) + RefuseUpload(ctx) + _, ok := UploadConfig(ctx) + assert.False(t, ok) +} diff --git a/libs/telemetry/protos/aitools_install.go b/libs/telemetry/protos/aitools_install.go index 406dde89a58..13c700bf24d 100644 --- a/libs/telemetry/protos/aitools_install.go +++ b/libs/telemetry/protos/aitools_install.go @@ -74,6 +74,25 @@ const ( // The command's context was cancelled. SIGINT outside a prompt is not // counted here: it ends the process before telemetry is uploaded. AitoolsErrorCategoryCommandInterrupted AitoolsErrorCategory = "COMMAND_INTERRUPTED" + + // `aitools mcp`: profiles and sign-in. Bind failures send their category by value (mcp.BindCategory). PROFILE_NOT_FOUND + // and PROFILE_HOST_CHANGED mirror the schema, and no CLI command names them. + AitoolsErrorCategoryNoProfile AitoolsErrorCategory = "NO_PROFILE" + AitoolsErrorCategoryProfileRequired AitoolsErrorCategory = "PROFILE_REQUIRED" + AitoolsErrorCategoryProfileNotFound AitoolsErrorCategory = "PROFILE_NOT_FOUND" + AitoolsErrorCategoryProfileHostChanged AitoolsErrorCategory = "PROFILE_HOST_CHANGED" + AitoolsErrorCategoryAuthFailed AitoolsErrorCategory = "AUTH_FAILED" + AitoolsErrorCategorySignInRequired AitoolsErrorCategory = "SIGN_IN_REQUIRED" + AitoolsErrorCategorySignInTimeout AitoolsErrorCategory = "SIGN_IN_TIMEOUT" + + // `aitools mcp`: the service list, agent configs and the proxy. MCP_UPSTREAM_ERROR mirrors the schema, and no CLI + // command sends it yet: the proxy does not log upstream errors in v1. + AitoolsErrorCategoryMCPServiceNotFound AitoolsErrorCategory = "MCP_SERVICE_NOT_FOUND" + AitoolsErrorCategoryMCPPermissionDenied AitoolsErrorCategory = "MCP_PERMISSION_DENIED" + AitoolsErrorCategoryMCPLocationNotFound AitoolsErrorCategory = "MCP_LOCATION_NOT_FOUND" + AitoolsErrorCategoryMCPNameConflict AitoolsErrorCategory = "MCP_NAME_CONFLICT" + AitoolsErrorCategoryMCPUpstreamError AitoolsErrorCategory = "MCP_UPSTREAM_ERROR" + AitoolsErrorCategoryAgentConfigInvalid AitoolsErrorCategory = "AGENT_CONFIG_INVALID" ) // AitoolsAgentResult records one agent's failed or skipped outcome within an diff --git a/libs/telemetry/protos/aitools_mcp.go b/libs/telemetry/protos/aitools_mcp.go new file mode 100644 index 00000000000..aae953d4ede --- /dev/null +++ b/libs/telemetry/protos/aitools_mcp.go @@ -0,0 +1,99 @@ +package protos + +// The types in this file mirror the aitools MCP messages of the databricks_cli +// lumberjack protos: AitoolsMcpEvent and its server result in +// proto/logs/frontend/databricks_cli/aitools_mcp.proto, and the enums in +// proto/logs/frontend/databricks_cli/enum.proto. + +// AitoolsMcpOperation mirrors AitoolsMcpOperation.Type in aitools_mcp.proto. +type AitoolsMcpOperation string + +const ( + AitoolsMcpOperationAdd AitoolsMcpOperation = "ADD" + AitoolsMcpOperationRemove AitoolsMcpOperation = "REMOVE" + AitoolsMcpOperationSignIn AitoolsMcpOperation = "SIGN_IN" + AitoolsMcpOperationSignOut AitoolsMcpOperation = "SIGN_OUT" + AitoolsMcpOperationProxyStarted AitoolsMcpOperation = "PROXY_STARTED" + // AitoolsMcpOperationSignInPrompted is logged for each authenticate call that showed the sign-in page, with the + // state of the sign-in when that call's wait ended. A retry after a cancelled call logs its own event. + AitoolsMcpOperationSignInPrompted AitoolsMcpOperation = "SIGN_IN_PROMPTED" +) + +// AitoolsMcpService mirrors AitoolsMcpService.Type in enum.proto. The CLI +// sends only these values, never a service name: the system.ai leaves in the +// table map to their own value, other system.ai leaves to OtherSystem, and any +// service outside system.ai to Custom. +type AitoolsMcpService string + +const ( + AitoolsMcpServiceOtherSystem AitoolsMcpService = "OTHER_SYSTEM" + AitoolsMcpServiceCustom AitoolsMcpService = "CUSTOM" + AitoolsMcpServiceGenieOne AitoolsMcpService = "GENIE_ONE" + AitoolsMcpServiceDBSQL AitoolsMcpService = "DBSQL" + AitoolsMcpServiceWebSearch AitoolsMcpService = "WEB_SEARCH" + AitoolsMcpServiceSandbox AitoolsMcpService = "SANDBOX" + AitoolsMcpServiceSlack AitoolsMcpService = "SLACK" + AitoolsMcpServiceGitHub AitoolsMcpService = "GITHUB" + AitoolsMcpServiceAtlassian AitoolsMcpService = "ATLASSIAN" + AitoolsMcpServiceGoogleDrive AitoolsMcpService = "GOOGLE_DRIVE" + AitoolsMcpServiceGoogleCalendar AitoolsMcpService = "GOOGLE_CALENDAR" + AitoolsMcpServiceGmail AitoolsMcpService = "GMAIL" + AitoolsMcpServiceMicrosoft365 AitoolsMcpService = "MICROSOFT_365" +) + +// AitoolsMcpSignInState mirrors AitoolsMcpSignInState.Type in enum.proto. It is +// the sign-in state of a service after the operation. +type AitoolsMcpSignInState string + +const ( + AitoolsMcpSignInStateNotRequired AitoolsMcpSignInState = "NOT_REQUIRED" + AitoolsMcpSignInStateActive AitoolsMcpSignInState = "ACTIVE" + AitoolsMcpSignInStatePending AitoolsMcpSignInState = "PENDING" +) + +// AitoolsTargetOutcome mirrors AitoolsTargetOutcome.Type in enum.proto. It is +// what an operation did to one target: an agent config entry, or a sign-in. +type AitoolsTargetOutcome string + +const ( + AitoolsTargetOutcomeChanged AitoolsTargetOutcome = "CHANGED" + AitoolsTargetOutcomeUnchanged AitoolsTargetOutcome = "UNCHANGED" + // AitoolsTargetOutcomeUpdateAvailable mirrors the schema. No MCP command sends it yet. + AitoolsTargetOutcomeUpdateAvailable AitoolsTargetOutcome = "UPDATE_AVAILABLE" + AitoolsTargetOutcomeSkipped AitoolsTargetOutcome = "SKIPPED" + AitoolsTargetOutcomeFailed AitoolsTargetOutcome = "FAILED" +) + +// AitoolsMcpServerResult is the result for one service, for one agent when the +// operation writes agent config. Unset fields are omitted: an agent is not set +// for sign-in, and an outcome is not set for proxy events. An error category is +// set on a FAILED target and on SIGN_IN_PROMPTED, where it is SIGN_IN_REQUIRED. A +// SKIPPED target carries none: remove skips an entry it left in place, or one +// already gone, without a failure. +type AitoolsMcpServerResult struct { + Service AitoolsMcpService `json:"service"` + Agent AitoolsAgentType `json:"agent,omitempty"` + Outcome AitoolsTargetOutcome `json:"outcome,omitempty"` + ErrorCategory AitoolsErrorCategory `json:"error_category,omitempty"` + SignInState AitoolsMcpSignInState `json:"sign_in_state,omitempty"` +} + +// AitoolsMcpEvent is emitted by `databricks aitools mcp` commands and by the +// proxy (`serve`). It carries enums and counts only: no host, profile, workspace +// ID, custom service or schema name, path, error text or tool name. +type AitoolsMcpEvent struct { + // CallingAgent is the coding agent that ran the command or started the proxy. + // AitoolsAgentTypeUnspecified when no agent is detected. + CallingAgent AitoolsAgentType `json:"calling_agent"` + + Operation AitoolsMcpOperation `json:"operation"` + + // Servers holds one entry per (service, agent) for ADD and REMOVE, one per + // service for SIGN_IN, SIGN_OUT, PROXY_STARTED and SIGN_IN_PROMPTED. + Servers []AitoolsMcpServerResult `json:"servers,omitempty"` + + // BoundProfileCount is the number of distinct profiles with MCP servers + // installed, read from the state file after the operation. Zero is a real + // value (every server removed), so it is not omitted. + BoundProfileCount int64 `json:"bound_profile_count"` +} diff --git a/libs/telemetry/protos/aitools_mcp_test.go b/libs/telemetry/protos/aitools_mcp_test.go new file mode 100644 index 00000000000..715a196103a --- /dev/null +++ b/libs/telemetry/protos/aitools_mcp_test.go @@ -0,0 +1,58 @@ +package protos + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestAitoolsMcpEventEncodesSnakeCaseKeys checks that the lumberjack proto keys these fields in snake_case. It pins the +// wire keys of an event and of one server result, and that unset enums are omitted rather than sent empty. +func TestAitoolsMcpEventEncodesSnakeCaseKeys(t *testing.T) { + b, err := json.Marshal(DatabricksCliLog{AitoolsMcpEvent: &AitoolsMcpEvent{ + CallingAgent: AitoolsAgentTypeClaudeCode, + Operation: AitoolsMcpOperationAdd, + BoundProfileCount: 0, + Servers: []AitoolsMcpServerResult{ + { + Service: AitoolsMcpServiceGitHub, + Agent: AitoolsAgentTypeCodex, + Outcome: AitoolsTargetOutcomeFailed, + ErrorCategory: AitoolsErrorCategoryAgentConfigInvalid, + SignInState: AitoolsMcpSignInStatePending, + }, + }, + }}) + require.NoError(t, err) + + var got map[string]any + require.NoError(t, json.Unmarshal(b, &got)) + event, ok := got["aitools_mcp_event"].(map[string]any) + require.True(t, ok, "aitools_mcp_event must be the event key") + + assert.Equal(t, "CLAUDE_CODE", event["calling_agent"]) + assert.Equal(t, "ADD", event["operation"]) + // Zero is a real count and is sent. + assert.EqualValues(t, 0, event["bound_profile_count"]) + + servers := event["servers"].([]any) + require.Len(t, servers, 1) + server := servers[0].(map[string]any) + assert.Equal(t, map[string]any{ + "service": "GITHUB", + "agent": "CODEX", + "outcome": "FAILED", + "error_category": "AGENT_CONFIG_INVALID", + "sign_in_state": "PENDING", + }, server) +} + +// TestAitoolsMcpServerResultOmitsUnsetFields checks that a proxy event has no agent on the result, no outcome and no error +// category when it succeeded. +func TestAitoolsMcpServerResultOmitsUnsetFields(t *testing.T) { + b, err := json.Marshal(AitoolsMcpServerResult{Service: AitoolsMcpServiceOtherSystem}) + require.NoError(t, err) + assert.JSONEq(t, `{"service":"OTHER_SYSTEM"}`, string(b)) +} diff --git a/libs/telemetry/protos/frontend_log.go b/libs/telemetry/protos/frontend_log.go index f3c37736cc1..dcaf17d8e92 100644 --- a/libs/telemetry/protos/frontend_log.go +++ b/libs/telemetry/protos/frontend_log.go @@ -24,6 +24,7 @@ type DatabricksCliLog struct { SshAgentShimEvent *SshAgentShimEvent `json:"ssh_agent_shim_event,omitempty"` BundleConfigRemoteSyncEvent *BundleConfigRemoteSyncEvent `json:"bundle_config_remote_sync_event,omitempty"` AitoolsInstallEvent *AitoolsInstallEvent `json:"aitools_install_event,omitempty"` + AitoolsMcpEvent *AitoolsMcpEvent `json:"aitools_mcp_event,omitempty"` SetupLocalEvent *SetupLocalEvent `json:"setup_local_event,omitempty"` AirRunEvent *AirRunEvent `json:"air_run_event,omitempty"` AitoolsHintEvent *AitoolsHintEvent `json:"aitools_hint_event,omitempty"`