From 5a3cc2592ac8b844d06d52d7b23725f5bc30f051 Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Wed, 11 Nov 2026 13:52:58 +0000 Subject: [PATCH] aitools: add install funnel telemetry Every `databricks aitools install` run now logs an `aitools_command_event` funnel next to the existing `aitools_install_event`, so the telemetry shows where runs stop: ```text install RunE STARTED (every run past flag parsing) validate flags, --agents names --> FINISHED FAILURE INVALID_FLAGS | UNKNOWN_AGENT STEP FLAGS_VALIDATED --path dump --> STEP SKILLS_DUMPED, FINISHED SUCCESS (0 skills written: FINISHED NOTHING_TO_DO) scope prompt --> FINISHED CANCELLED USER_ABORTED | FAILURE PROMPT_FAILED STEP SCOPE_RESOLVED picker --> FINISHED CANCELLED (nothing checked) no agent detected: FINISHED NOTHING_TO_DO STEP AGENTS_SELECTED confirm prompt --> FINISHED CANCELLED (declined) STEP PLAN_CONFIRMED executePlan STEP SKILLS_INSTALLED (skills group installed) STEP PLUGINS_INSTALLED (at least one plugin installed) FINISHED SUCCESS | FAILURE ``` Each event carries `command`, `stage`, `interactive`, `install_mode` (DEFAULT / SKILLS_ONLY / PATH) and, once known, `scope`. FINISHED also carries `outcome`, `error_category` and the targeted `agents`. Every event of a run shares the entry's `cmd_exec_id`. The CLI sets no calling-agent field on these events: the upload's User-Agent already has `agent/`, the calling agent's Databricks tooling (`aitools-agent/...`) and `cmd-exec-id`. `error_category` comes from the existing `classifyInstallError`. Failures in flag validation, agent lookup and prompts are tagged with a category where they happen (a prompt's own failure is PROMPT_FAILED whatever its type, except a Ctrl-C abort; an error before the prompt opens keeps its own category). `Error()` / `Unwrap()` pass through, so the user still sees the same message: ```diff classifyInstallError(err) *installer.BlockedError / *installer.SkillError -> their category + *categorizedError (withCategory, promptError) -> the tagged category stdlibErrorCategory(err) ``` Some runs get an explicit outcome instead of the one their return value implies: - Returns nil but installed nothing (e.g. the only detected agent's plugin install failed, which is a warning): FAILURE with the first failing agent's category. - Fails only because a named agent failed or was skipped (`--agents`): the category comes from that agent's outcome (e.g. UNSUPPORTED_SCOPE), not UNCATEGORIZED. - `--output json` failure: classifies the real error, not the `ErrAlreadyPrinted` sentinel. - `--project --global` and `--agents ","`: INVALID_FLAGS, not PROMPT_FAILED / UNKNOWN_AGENT. Runs that end before PLAN_CONFIRMED (a flag error, a cancelled prompt, no agent detected, a `--path` dump) upload their events only if the background telemetry auth has already resolved; they never wait for it, so they stay instant. The funnel therefore under-counts the fastest early exits relative to slower ones. Runs past PLAN_CONFIRMED keep the bounded wait. An unknown `--agents` name now fails before the scope prompt instead of after it (changelog fragment `.nextchanges/cli/aitools-install-agents-before-scope-prompt.md`). Otherwise no change to output, prompts, exit codes or error text. Co-authored-by: Isaac --- ...ools-install-agents-before-scope-prompt.md | 1 + .../telemetry/install-failure/out.test.toml | 1 + .../telemetry/install-failure/output.txt | 66 +++ .../aitools/telemetry/install-failure/script | 24 + .../telemetry/install-failure/test.toml | 7 + .../cmd/aitools/telemetry/install/output.txt | 14 + .../cmd/aitools/telemetry/install/test.toml | 29 - acceptance/cmd/aitools/telemetry/test.toml | 32 +- cmd/aitools/categorize.go | 36 +- cmd/aitools/categorize_test.go | 11 + cmd/aitools/funnel.go | 173 ++++++ cmd/aitools/funnel_test.go | 510 ++++++++++++++++++ cmd/aitools/install.go | 107 +++- cmd/aitools/scope.go | 13 +- cmd/aitools/telemetry.go | 85 ++- cmd/aitools/telemetry_auth_test.go | 82 ++- libs/telemetry/protos/README.md | 8 + libs/telemetry/protos/aitools_events.go | 116 ++++ libs/telemetry/protos/aitools_events_test.go | 26 +- libs/telemetry/protos/frontend_log.go | 1 + 20 files changed, 1253 insertions(+), 89 deletions(-) create mode 100644 .nextchanges/cli/aitools-install-agents-before-scope-prompt.md create mode 100644 acceptance/cmd/aitools/telemetry/install-failure/out.test.toml create mode 100644 acceptance/cmd/aitools/telemetry/install-failure/output.txt create mode 100644 acceptance/cmd/aitools/telemetry/install-failure/script create mode 100644 acceptance/cmd/aitools/telemetry/install-failure/test.toml delete mode 100644 acceptance/cmd/aitools/telemetry/install/test.toml create mode 100644 cmd/aitools/funnel.go create mode 100644 cmd/aitools/funnel_test.go diff --git a/.nextchanges/cli/aitools-install-agents-before-scope-prompt.md b/.nextchanges/cli/aitools-install-agents-before-scope-prompt.md new file mode 100644 index 00000000000..5ce3cc9cb1d --- /dev/null +++ b/.nextchanges/cli/aitools-install-agents-before-scope-prompt.md @@ -0,0 +1 @@ +* `databricks aitools install` now rejects an unknown `--agents` name before it shows the scope prompt, instead of after. ([#7469](https://github.com/databricks/cli/pull/7469)) diff --git a/acceptance/cmd/aitools/telemetry/install-failure/out.test.toml b/acceptance/cmd/aitools/telemetry/install-failure/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/install-failure/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/cmd/aitools/telemetry/install-failure/output.txt b/acceptance/cmd/aitools/telemetry/install-failure/output.txt new file mode 100644 index 00000000000..cb4c408db1f --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/install-failure/output.txt @@ -0,0 +1,66 @@ + +=== a skill missing from the manifest fails after the plan is confirmed + +>>> [CLI] aitools install --skills-only --scope=global --agents=cursor --skills=nonexistent +Installing Databricks skills for Cursor... +Using skills version test-ref +Fetching skills manifest... +Error: skill "nonexistent" not found + +>>> print_telemetry +{"command":"aitools_install","exit_code":1,"aitools_command_event":{"command":"INSTALL","stage":"STARTED","interactive":false,"install_mode":"SKILLS_ONLY"}} +{"command":"aitools_install","exit_code":1,"aitools_command_event":{"command":"INSTALL","stage":"STEP_COMPLETED","step":"FLAGS_VALIDATED","interactive":false,"install_mode":"SKILLS_ONLY"}} +{"command":"aitools_install","exit_code":1,"aitools_command_event":{"command":"INSTALL","stage":"STEP_COMPLETED","step":"SCOPE_RESOLVED","interactive":false,"install_mode":"SKILLS_ONLY","scope":"GLOBAL"}} +{"command":"aitools_install","exit_code":1,"aitools_command_event":{"command":"INSTALL","stage":"STEP_COMPLETED","step":"AGENTS_SELECTED","interactive":false,"install_mode":"SKILLS_ONLY","scope":"GLOBAL"}} +{"command":"aitools_install","exit_code":1,"aitools_command_event":{"command":"INSTALL","stage":"STEP_COMPLETED","step":"PLAN_CONFIRMED","interactive":false,"install_mode":"SKILLS_ONLY","scope":"GLOBAL"}} +{"command":"aitools_install","exit_code":1,"aitools_command_event":{"command":"INSTALL","stage":"FINISHED","outcome":"FAILURE","error_category":"SKILL_NOT_FOUND","interactive":false,"install_mode":"SKILLS_ONLY","agents":["CURSOR"],"scope":"GLOBAL"}} +{"command":"aitools_install","exit_code":1,"aitools_install_event":{"agents":["CURSOR"],"scope":"GLOBAL","error_category":"SKILL_NOT_FOUND"}} + +=== a manifest the server does not have is MANIFEST_NOT_FOUND + +>>> DATABRICKS_SKILLS_REF=missing-ref [CLI] aitools install --skills-only --scope=global --agents=cursor +Installing Databricks skills for Cursor... +Using skills version missing-ref +Fetching skills manifest... +Error: skills manifest at databricks-agent-skills@missing-ref: not found + +>>> print_telemetry +{"command":"aitools_install","exit_code":1,"aitools_command_event":{"command":"INSTALL","stage":"STARTED","interactive":false,"install_mode":"SKILLS_ONLY"}} +{"command":"aitools_install","exit_code":1,"aitools_command_event":{"command":"INSTALL","stage":"STEP_COMPLETED","step":"FLAGS_VALIDATED","interactive":false,"install_mode":"SKILLS_ONLY"}} +{"command":"aitools_install","exit_code":1,"aitools_command_event":{"command":"INSTALL","stage":"STEP_COMPLETED","step":"SCOPE_RESOLVED","interactive":false,"install_mode":"SKILLS_ONLY","scope":"GLOBAL"}} +{"command":"aitools_install","exit_code":1,"aitools_command_event":{"command":"INSTALL","stage":"STEP_COMPLETED","step":"AGENTS_SELECTED","interactive":false,"install_mode":"SKILLS_ONLY","scope":"GLOBAL"}} +{"command":"aitools_install","exit_code":1,"aitools_command_event":{"command":"INSTALL","stage":"STEP_COMPLETED","step":"PLAN_CONFIRMED","interactive":false,"install_mode":"SKILLS_ONLY","scope":"GLOBAL"}} +{"command":"aitools_install","exit_code":1,"aitools_command_event":{"command":"INSTALL","stage":"FINISHED","outcome":"FAILURE","error_category":"MANIFEST_NOT_FOUND","interactive":false,"install_mode":"SKILLS_ONLY","agents":["CURSOR"],"scope":"GLOBAL"}} +{"command":"aitools_install","exit_code":1,"aitools_install_event":{"agents":["CURSOR"],"scope":"GLOBAL","error_category":"MANIFEST_NOT_FOUND"}} + +=== any other HTTP error status from the manifest fetch is not classified yet (the error is untyped) + +>>> DATABRICKS_SKILLS_REF=broken-ref [CLI] aitools install --skills-only --scope=global --agents=cursor +Installing Databricks skills for Cursor... +Using skills version broken-ref +Fetching skills manifest... +Error: failed to fetch manifest: HTTP 500 + +>>> print_telemetry +{"command":"aitools_install","exit_code":1,"aitools_command_event":{"command":"INSTALL","stage":"STARTED","interactive":false,"install_mode":"SKILLS_ONLY"}} +{"command":"aitools_install","exit_code":1,"aitools_command_event":{"command":"INSTALL","stage":"STEP_COMPLETED","step":"FLAGS_VALIDATED","interactive":false,"install_mode":"SKILLS_ONLY"}} +{"command":"aitools_install","exit_code":1,"aitools_command_event":{"command":"INSTALL","stage":"STEP_COMPLETED","step":"SCOPE_RESOLVED","interactive":false,"install_mode":"SKILLS_ONLY","scope":"GLOBAL"}} +{"command":"aitools_install","exit_code":1,"aitools_command_event":{"command":"INSTALL","stage":"STEP_COMPLETED","step":"AGENTS_SELECTED","interactive":false,"install_mode":"SKILLS_ONLY","scope":"GLOBAL"}} +{"command":"aitools_install","exit_code":1,"aitools_command_event":{"command":"INSTALL","stage":"STEP_COMPLETED","step":"PLAN_CONFIRMED","interactive":false,"install_mode":"SKILLS_ONLY","scope":"GLOBAL"}} +{"command":"aitools_install","exit_code":1,"aitools_command_event":{"command":"INSTALL","stage":"FINISHED","outcome":"FAILURE","error_category":"UNCATEGORIZED","interactive":false,"install_mode":"SKILLS_ONLY","agents":["CURSOR"],"scope":"GLOBAL"}} +{"command":"aitools_install","exit_code":1,"aitools_install_event":{"agents":["CURSOR"],"scope":"GLOBAL","error_category":"UNCATEGORIZED"}} + +=== a named agent skipped for the scope takes its category from the agent's outcome + +>>> [CLI] aitools install --skills-only --scope=project --agents=cursor +Skipped Cursor: does not support project-scoped skills +Error: Cursor: does not support project-scoped skills + +>>> print_telemetry +{"command":"aitools_install","exit_code":1,"aitools_command_event":{"command":"INSTALL","stage":"STARTED","interactive":false,"install_mode":"SKILLS_ONLY"}} +{"command":"aitools_install","exit_code":1,"aitools_command_event":{"command":"INSTALL","stage":"STEP_COMPLETED","step":"FLAGS_VALIDATED","interactive":false,"install_mode":"SKILLS_ONLY"}} +{"command":"aitools_install","exit_code":1,"aitools_command_event":{"command":"INSTALL","stage":"STEP_COMPLETED","step":"SCOPE_RESOLVED","interactive":false,"install_mode":"SKILLS_ONLY","scope":"PROJECT"}} +{"command":"aitools_install","exit_code":1,"aitools_command_event":{"command":"INSTALL","stage":"STEP_COMPLETED","step":"AGENTS_SELECTED","interactive":false,"install_mode":"SKILLS_ONLY","scope":"PROJECT"}} +{"command":"aitools_install","exit_code":1,"aitools_command_event":{"command":"INSTALL","stage":"STEP_COMPLETED","step":"PLAN_CONFIRMED","interactive":false,"install_mode":"SKILLS_ONLY","scope":"PROJECT"}} +{"command":"aitools_install","exit_code":1,"aitools_command_event":{"command":"INSTALL","stage":"FINISHED","outcome":"FAILURE","error_category":"UNSUPPORTED_SCOPE","interactive":false,"install_mode":"SKILLS_ONLY","agents":["CURSOR"],"scope":"PROJECT"}} +{"command":"aitools_install","exit_code":1,"aitools_install_event":{"agents":["CURSOR"],"scope":"PROJECT","error_category":"TYPE_UNSPECIFIED","agent_results":[{"agent":"CURSOR","error_category":"UNSUPPORTED_SCOPE"}]}} diff --git a/acceptance/cmd/aitools/telemetry/install-failure/script b/acceptance/cmd/aitools/telemetry/install-failure/script new file mode 100644 index 00000000000..6b9bcf51b11 --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/install-failure/script @@ -0,0 +1,24 @@ +# Isolate HOME so parallel aitools tests don't race on a shared ~/.databricks. +sethome home +mkdir -p "${USERPROFILE:-$HOME}/.cursor" + +# Only runs past the plan confirmation are here: a run that ends earlier (a +# flag error, an unknown agent, a --path dump) uploads only if auth has already +# resolved, so whether its events show up depends on timing. The funnel unit +# tests in cmd/aitools cover those runs. + +title "a skill missing from the manifest fails after the plan is confirmed" +musterr trace $CLI aitools install --skills-only --scope=global --agents=cursor --skills=nonexistent +trace print_telemetry + +title "a manifest the server does not have is MANIFEST_NOT_FOUND" +musterr trace DATABRICKS_SKILLS_REF=missing-ref $CLI aitools install --skills-only --scope=global --agents=cursor +trace print_telemetry + +title "any other HTTP error status from the manifest fetch is not classified yet (the error is untyped)" +musterr trace DATABRICKS_SKILLS_REF=broken-ref $CLI aitools install --skills-only --scope=global --agents=cursor +trace print_telemetry + +title "a named agent skipped for the scope takes its category from the agent's outcome" +musterr trace $CLI aitools install --skills-only --scope=project --agents=cursor +trace print_telemetry diff --git a/acceptance/cmd/aitools/telemetry/install-failure/test.toml b/acceptance/cmd/aitools/telemetry/install-failure/test.toml new file mode 100644 index 00000000000..f35d2d4ed2e --- /dev/null +++ b/acceptance/cmd/aitools/telemetry/install-failure/test.toml @@ -0,0 +1,7 @@ +[[Server]] +Pattern = "GET /missing-ref/manifest.json" +Response.StatusCode = 404 + +[[Server]] +Pattern = "GET /broken-ref/manifest.json" +Response.StatusCode = 500 diff --git a/acceptance/cmd/aitools/telemetry/install/output.txt b/acceptance/cmd/aitools/telemetry/install/output.txt index 2aea215a26c..dd509504411 100644 --- a/acceptance/cmd/aitools/telemetry/install/output.txt +++ b/acceptance/cmd/aitools/telemetry/install/output.txt @@ -8,7 +8,14 @@ Fetching skills manifest... Installed 1 skill. >>> print_telemetry +{"command":"aitools_install","exit_code":0,"aitools_command_event":{"command":"INSTALL","stage":"STARTED","interactive":false,"install_mode":"SKILLS_ONLY"}} +{"command":"aitools_install","exit_code":0,"aitools_command_event":{"command":"INSTALL","stage":"STEP_COMPLETED","step":"FLAGS_VALIDATED","interactive":false,"install_mode":"SKILLS_ONLY"}} +{"command":"aitools_install","exit_code":0,"aitools_command_event":{"command":"INSTALL","stage":"STEP_COMPLETED","step":"SCOPE_RESOLVED","interactive":false,"install_mode":"SKILLS_ONLY","scope":"GLOBAL"}} +{"command":"aitools_install","exit_code":0,"aitools_command_event":{"command":"INSTALL","stage":"STEP_COMPLETED","step":"AGENTS_SELECTED","interactive":false,"install_mode":"SKILLS_ONLY","scope":"GLOBAL"}} +{"command":"aitools_install","exit_code":0,"aitools_command_event":{"command":"INSTALL","stage":"STEP_COMPLETED","step":"PLAN_CONFIRMED","interactive":false,"install_mode":"SKILLS_ONLY","scope":"GLOBAL"}} +{"command":"aitools_install","exit_code":0,"aitools_command_event":{"command":"INSTALL","stage":"STEP_COMPLETED","step":"SKILLS_INSTALLED","interactive":false,"install_mode":"SKILLS_ONLY","scope":"GLOBAL"}} {"command":"aitools_install","exit_code":0,"aitools_install_event":{"agents":["CLAUDE_CODE"],"scope":"GLOBAL","error_category":"TYPE_UNSPECIFIED"}} +{"command":"aitools_install","exit_code":0,"aitools_command_event":{"command":"INSTALL","stage":"FINISHED","outcome":"SUCCESS","interactive":false,"install_mode":"SKILLS_ONLY","agents":["CLAUDE_CODE"],"scope":"GLOBAL"}} === databricks-cli auth with a valid cached token: the event is uploaded @@ -19,7 +26,14 @@ Fetching skills manifest... Installed 1 skill. >>> print_telemetry +{"command":"aitools_install","exit_code":0,"aitools_command_event":{"command":"INSTALL","stage":"STARTED","interactive":false,"install_mode":"SKILLS_ONLY"}} +{"command":"aitools_install","exit_code":0,"aitools_command_event":{"command":"INSTALL","stage":"STEP_COMPLETED","step":"FLAGS_VALIDATED","interactive":false,"install_mode":"SKILLS_ONLY"}} +{"command":"aitools_install","exit_code":0,"aitools_command_event":{"command":"INSTALL","stage":"STEP_COMPLETED","step":"SCOPE_RESOLVED","interactive":false,"install_mode":"SKILLS_ONLY","scope":"GLOBAL"}} +{"command":"aitools_install","exit_code":0,"aitools_command_event":{"command":"INSTALL","stage":"STEP_COMPLETED","step":"AGENTS_SELECTED","interactive":false,"install_mode":"SKILLS_ONLY","scope":"GLOBAL"}} +{"command":"aitools_install","exit_code":0,"aitools_command_event":{"command":"INSTALL","stage":"STEP_COMPLETED","step":"PLAN_CONFIRMED","interactive":false,"install_mode":"SKILLS_ONLY","scope":"GLOBAL"}} +{"command":"aitools_install","exit_code":0,"aitools_command_event":{"command":"INSTALL","stage":"STEP_COMPLETED","step":"SKILLS_INSTALLED","interactive":false,"install_mode":"SKILLS_ONLY","scope":"GLOBAL"}} {"command":"aitools_install","exit_code":0,"aitools_install_event":{"agents":["CLAUDE_CODE"],"scope":"GLOBAL","error_category":"TYPE_UNSPECIFIED"}} +{"command":"aitools_install","exit_code":0,"aitools_command_event":{"command":"INSTALL","stage":"FINISHED","outcome":"SUCCESS","interactive":false,"install_mode":"SKILLS_ONLY","agents":["CLAUDE_CODE"],"scope":"GLOBAL"}} === databricks-cli auth with an expired cached token: no refresh, nothing is uploaded diff --git a/acceptance/cmd/aitools/telemetry/install/test.toml b/acceptance/cmd/aitools/telemetry/install/test.toml deleted file mode 100644 index 07ef4a0af15..00000000000 --- a/acceptance/cmd/aitools/telemetry/install/test.toml +++ /dev/null @@ -1,29 +0,0 @@ -# Mock server replaces raw.githubusercontent.com for the skills manifest and files. -Env.DATABRICKS_SKILLS_BASE_URL = "$DATABRICKS_HOST" -Env.DATABRICKS_SKILLS_REF = "test-ref" - -[[Server]] -Pattern = "GET /test-ref/manifest.json" -Response.Body = ''' -{ - "version": "2", - "updated_at": "2026-01-01T00:00:00Z", - "skills": { - "test-stable": { - "version": "1.0.0", - "description": "Stable test skill", - "files": ["SKILL.md"], - "repo_dir": "skills" - } - } -} -''' - -[[Server]] -Pattern = "GET /test-ref/skills/test-stable/SKILL.md" -Response.Body = '''--- -name: test-stable ---- - -# Test stable skill -''' diff --git a/acceptance/cmd/aitools/telemetry/test.toml b/acceptance/cmd/aitools/telemetry/test.toml index ad4d6d69eec..45f00a1b4a2 100644 --- a/acceptance/cmd/aitools/telemetry/test.toml +++ b/acceptance/cmd/aitools/telemetry/test.toml @@ -1,6 +1,36 @@ -# Telemetry payloads of the install hint. +# Telemetry payloads of the aitools commands and of the install hint. +# The mock server replaces raw.githubusercontent.com (skills manifest and files). +Env.DATABRICKS_SKILLS_BASE_URL = "$DATABRICKS_HOST" +Env.DATABRICKS_CLI_GITHUB_API_URL = "$DATABRICKS_HOST" +Env.DATABRICKS_SKILLS_REF = "test-ref" RecordRequests = true Ignore = [ "home", ] + +[[Server]] +Pattern = "GET /test-ref/manifest.json" +Response.Body = ''' +{ + "version": "2", + "updated_at": "2026-01-01T00:00:00Z", + "skills": { + "databricks-test": { + "version": "1.0.0", + "description": "Test skill", + "files": ["SKILL.md"], + "repo_dir": "skills" + } + } +} +''' + +[[Server]] +Pattern = "GET /test-ref/skills/databricks-test/SKILL.md" +Response.Body = '''--- +name: databricks-test +--- + +# Test skill +''' diff --git a/cmd/aitools/categorize.go b/cmd/aitools/categorize.go index fe5fd42f5a7..7f6f8f42dba 100644 --- a/cmd/aitools/categorize.go +++ b/cmd/aitools/categorize.go @@ -14,9 +14,38 @@ import ( "github.com/databricks/cli/libs/telemetry/protos" ) +// categorizedError tags an error with a telemetry category at the site that +// knows what failed (flag validation, agent lookup, a prompt). The tag wins +// over the standard library types. Error() and Unwrap() pass through, so the +// user-visible message is unchanged. +type categorizedError struct { + err error + category protos.AitoolsErrorCategory +} + +func (e *categorizedError) Error() string { return e.err.Error() } +func (e *categorizedError) Unwrap() error { return e.err } + +// withCategory tags a non-nil err with category. +func withCategory(err error, category protos.AitoolsErrorCategory) error { + return &categorizedError{err: err, category: category} +} + +// promptError tags a non-nil prompt failure. A user abort (Ctrl-C) keeps its own +// classification so the funnel reports it as CANCELLED, not as a failure. +// Any other error of a prompt step is PROMPT_FAILED, even one with an OS +// error type: the prompt library wraps a failure to open the terminal as an +// *fs.PathError, and FILESYSTEM would point at the skill files instead. +func promptError(err error) error { + if errors.Is(err, huh.ErrUserAborted) { + return err + } + return withCategory(err, protos.AitoolsErrorCategoryPromptFailed) +} + // classifyInstallError maps an aitools error to its telemetry category without -// reading any error text. Typed installer errors win, then standard library -// error types. +// reading any error text. Typed installer errors win, then categories tagged at +// the failure site, then standard library error types. func classifyInstallError(err error) protos.AitoolsErrorCategory { if err == nil { return protos.AitoolsErrorCategoryUnspecified @@ -28,6 +57,9 @@ func classifyInstallError(err error) protos.AitoolsErrorCategory { if skill, ok := errors.AsType[*installer.SkillError](err); ok { return skillErrorCategory(skill) } + if tagged, ok := errors.AsType[*categorizedError](err); ok { + return tagged.category + } return stdlibErrorCategory(err) } diff --git a/cmd/aitools/categorize_test.go b/cmd/aitools/categorize_test.go index 0926697474b..4491fa3a2d2 100644 --- a/cmd/aitools/categorize_test.go +++ b/cmd/aitools/categorize_test.go @@ -114,6 +114,10 @@ func TestClassifyInstallErrorGenericCauses(t *testing.T) { {"filesystem on rename", fmt.Errorf("replace skill: %w", &os.LinkError{Op: "rename", Old: "/a", New: "/b", Err: syscall.EBUSY}), protos.AitoolsErrorCategoryFilesystem}, {"filesystem syscall", fmt.Errorf("resolve cwd: %w", os.NewSyscallError("getwd", syscall.ENOENT)), protos.AitoolsErrorCategoryFilesystem}, {"user abort", huh.ErrUserAborted, protos.AitoolsErrorCategoryUserAborted}, + {"tagged", withCategory(errors.New("bad flags"), protos.AitoolsErrorCategoryInvalidFlags), protos.AitoolsErrorCategoryInvalidFlags}, + {"prompt failure", promptError(errors.New("no tty")), protos.AitoolsErrorCategoryPromptFailed}, + {"prompt abort keeps abort", promptError(huh.ErrUserAborted), protos.AitoolsErrorCategoryUserAborted}, + {"typed error beats tag", withCategory(&installer.BlockedError{Reason: installer.ReasonCLINotOnPath}, protos.AitoolsErrorCategoryInvalidFlags), protos.AitoolsErrorCategoryCLINotOnPath}, {"plain text stays uncategorized", errors.New("something"), protos.AitoolsErrorCategoryUncategorized}, } for _, tc := range cases { @@ -123,6 +127,13 @@ func TestClassifyInstallErrorGenericCauses(t *testing.T) { } } +func TestWithCategoryKeepsMessage(t *testing.T) { + cause := errors.New("cannot use --skills-only with --path") + err := withCategory(cause, protos.AitoolsErrorCategoryInvalidFlags) + assert.Equal(t, cause.Error(), err.Error()) + assert.ErrorIs(t, err, cause) +} + // A corrupt install state must not be blamed on the remote manifest. It stays // UNCATEGORIZED until the installer returns a typed state error, which should // change this to STATE_INVALID. diff --git a/cmd/aitools/funnel.go b/cmd/aitools/funnel.go new file mode 100644 index 00000000000..eeb579ac8e2 --- /dev/null +++ b/cmd/aitools/funnel.go @@ -0,0 +1,173 @@ +package aitools + +import ( + "context" + "errors" + + "github.com/charmbracelet/huh" + "github.com/databricks/cli/libs/aitools/agents" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/telemetry" + "github.com/databricks/cli/libs/telemetry/protos" +) + +// commandFunnel emits the AitoolsCommandEvent funnel of one aitools subcommand +// run: STARTED when created, STEP_COMPLETED per step, and exactly one FINISHED. +// The scope from setScope is attached to every later event; the agents from +// setAgents only to FINISHED. +type commandFunnel struct { + command protos.AitoolsCommand + interactive bool + installMode protos.AitoolsInstallMode + scope protos.AitoolsInstallScope + agents []protos.AitoolsAgentType + finished bool +} + +// startFunnel logs the STARTED event for command and returns the funnel to +// report later steps and the outcome on. +// installMode is set for install only, so it is on the STARTED event too. +func startFunnel(ctx context.Context, command protos.AitoolsCommand, installMode protos.AitoolsInstallMode) *commandFunnel { + f := &commandFunnel{ + command: command, + interactive: cmdio.IsPromptSupported(ctx), + installMode: installMode, + } + f.log(ctx, protos.AitoolsCommandEvent{Stage: protos.AitoolsCommandStageStarted}) + return f +} + +// setScope records the resolved scope (installer.ScopeGlobal/ScopeProject). +func (f *commandFunnel) setScope(scope string) { f.scope = scopeType(scope) } + +// setAgents records the plan's agents, reported on the FINISHED event. +func (f *commandFunnel) setAgents(plan []agentPlanItem) { f.agents = agentsField(plan) } + +// setTargets records agents known before the plan is built (named via +// --agents), so a run that stops at the scope prompt still reports them. +func (f *commandFunnel) setTargets(targets []*agents.Agent) { f.agents = agentTypes(targets) } + +// step logs a STEP_COMPLETED event. +func (f *commandFunnel) step(ctx context.Context, s protos.AitoolsCommandStep) { + f.log(ctx, protos.AitoolsCommandEvent{ + Stage: protos.AitoolsCommandStageStepCompleted, + Step: s, + }) +} + +// finish logs the FINISHED event for a run that returned err: SUCCESS for nil, +// CANCELLED when the user aborted a prompt or picked no agents, FAILURE +// otherwise. No-op once the run has finished, so it is safe to defer after an +// explicit finishWith. +func (f *commandFunnel) finish(ctx context.Context, err error) { + switch { + case err == nil: + f.finishWith(ctx, protos.AitoolsCommandOutcomeSuccess, protos.AitoolsErrorCategoryUnspecified) + case errors.Is(err, huh.ErrUserAborted): + f.finishWith(ctx, protos.AitoolsCommandOutcomeCancelled, protos.AitoolsErrorCategoryUserAborted) + case errors.Is(err, errNoAgentsSelected): + f.finishWith(ctx, protos.AitoolsCommandOutcomeCancelled, protos.AitoolsErrorCategoryUnspecified) + default: + f.finishWith(ctx, protos.AitoolsCommandOutcomeFailure, classifyInstallError(err)) + } +} + +// finishWith logs the FINISHED event with an explicit outcome, for runs whose +// returned error does not tell it: a nil return without success (declined +// confirm, nothing to act on, nothing installed) or a named agent's failure. +func (f *commandFunnel) finishWith(ctx context.Context, outcome protos.AitoolsCommandOutcome, category protos.AitoolsErrorCategory) { + if f.finished { + return + } + f.finished = true + // Unspecified means "no error"; omit it rather than send the sentinel. + if category == protos.AitoolsErrorCategoryUnspecified { + category = "" + } + f.log(ctx, protos.AitoolsCommandEvent{ + Stage: protos.AitoolsCommandStageFinished, + Outcome: outcome, + ErrorCategory: category, + Agents: f.agents, + }) +} + +// log stamps e with the run's command, interactivity, install mode and scope, +// and buffers it for upload. +func (f *commandFunnel) log(ctx context.Context, e protos.AitoolsCommandEvent) { + e.Command = f.command + e.Interactive = f.interactive + e.InstallMode = f.installMode + e.Scope = f.scope + telemetry.Log(ctx, protos.DatabricksCliLog{AitoolsCommandEvent: &e}) +} + +// installModeType maps the install delivery flags to the telemetry enum. +func installModeType(skillsOnly bool, path string) protos.AitoolsInstallMode { + switch { + case path != "": + return protos.AitoolsInstallModePath + case skillsOnly: + return protos.AitoolsInstallModeSkillsOnly + default: + return protos.AitoolsInstallModeDefault + } +} + +// deliverySteps returns the delivery steps executePlan got through, derived +// from its outcomes: SKILLS_INSTALLED when the skills group installed, and +// PLUGINS_INSTALLED when at least one plugin installed. Events are +// uploaded together at exit, so logging them after executePlan returns yields +// the same payload as logging them inside it. +func deliverySteps(outcomes []agentOutcome) []protos.AitoolsCommandStep { + var skills, plugins bool + for _, o := range outcomes { + switch o.delivery { + case deliverySkills: + skills = skills || o.status == outcomeInstalled + case deliveryPlugin: + plugins = plugins || o.status == outcomeInstalled + case deliverySkip: + } + } + var steps []protos.AitoolsCommandStep + if skills { + steps = append(steps, protos.AitoolsCommandStepSkillsInstalled) + } + if plugins { + steps = append(steps, protos.AitoolsCommandStepPluginsInstalled) + } + return steps +} + +// nothingInstalledCategory reports whether a run that returned no error still +// installed nothing for any agent (e.g. the only detected agent's plugin +// install failed, which is a warning, not an error), and if so the category of +// the first agent that failed or was skipped. The funnel records such a run as +// FAILURE rather than SUCCESS. That includes a non-interactive run whose +// detected agents are all skipped for the scope (UNSUPPORTED_SCOPE): it set +// up nothing it was asked to. An interactive run never gets here for them, +// because the picker offers no skipped agent and the run finishes NOTHING_TO_DO +// before the plan. +func nothingInstalledCategory(outcomes []agentOutcome) (protos.AitoolsErrorCategory, bool) { + if len(outcomes) == 0 { + return "", false + } + for _, o := range outcomes { + if o.status == outcomeInstalled { + return "", false + } + } + return firstFailureCategory(outcomes), true +} + +// firstFailureCategory returns the category of the first agent that failed or +// was skipped, or UNCATEGORIZED when none carries one. +func firstFailureCategory(outcomes []agentOutcome) protos.AitoolsErrorCategory { + for _, o := range outcomes { + if o.status != outcomeInstalled && o.errorCategory != "" && o.errorCategory != protos.AitoolsErrorCategoryUnspecified { + return o.errorCategory + } + } + return protos.AitoolsErrorCategoryUncategorized +} diff --git a/cmd/aitools/funnel_test.go b/cmd/aitools/funnel_test.go new file mode 100644 index 00000000000..e29156d08a1 --- /dev/null +++ b/cmd/aitools/funnel_test.go @@ -0,0 +1,510 @@ +package aitools + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "path/filepath" + "testing" + + "github.com/charmbracelet/huh" + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/internal/testutil" + "github.com/databricks/cli/libs/aitools/agents" + "github.com/databricks/cli/libs/aitools/installer" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/telemetry" + "github.com/databricks/cli/libs/telemetry/protos" + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// isolateAgents starts the test from an empty environment with only kv set +// (testutil.SetAgentEnv), so a host CLAUDE_CONFIG_DIR, GEMINI_CLI_HOME, +// APPDATA (Windows) etc. cannot add agents to a test that builds its own temp +// HOME. +func isolateAgents(t *testing.T, kv ...string) { + t.Helper() + testutil.SetAgentEnv(t, kv...) +} + +// forbidSkillsInstall fails the test if the run reaches the real skills +// installer, which would fetch skills and write into agent config dirs. +func forbidSkillsInstall(t *testing.T) { + t.Helper() + orig := installSkillsForAgentsFn + t.Cleanup(func() { installSkillsForAgentsFn = orig }) + installSkillsForAgentsFn = func(_ context.Context, _ installer.ManifestSource, targets []*agents.Agent, _ installer.InstallOptions) error { + t.Errorf("unexpected skills install for %d agents", len(targets)) + return assert.AnError + } +} + +// funnelEvents returns the AitoolsCommandEvents buffered on ctx, in order. +func funnelEvents(ctx context.Context) []protos.AitoolsCommandEvent { + var out []protos.AitoolsCommandEvent + for _, e := range telemetry.BufferedEvents(ctx) { + if e.AitoolsCommandEvent != nil { + out = append(out, *e.AitoolsCommandEvent) + } + } + return out +} + +// funnelShape renders each event as its stage plus step or outcome, so a test +// can assert the whole sequence in one line. +func funnelShape(events []protos.AitoolsCommandEvent) []string { + out := make([]string, 0, len(events)) + for _, e := range events { + switch e.Stage { + case protos.AitoolsCommandStageStepCompleted: + out = append(out, string(e.Step)) + case protos.AitoolsCommandStageFinished: + out = append(out, string(e.Stage)+":"+string(e.Outcome)+":"+string(e.ErrorCategory)) + default: + out = append(out, string(e.Stage)) + } + } + return out +} + +func runInstall(t *testing.T, ctx context.Context, args ...string) (context.Context, error) { + t.Helper() + ctx = telemetry.WithNewLogger(ctx) + cmd := newTestInstallCmd() + cmd.SetContext(ctx) + cmd.SetOut(io.Discard) + cmd.SetArgs(args) + return ctx, cmd.Execute() +} + +func TestInstallFunnelSkillsOnlySuccess(t *testing.T) { + isolateAgents(t, "CURSOR_AGENT", "1") + setupTestAgents(t) + setupInstallMock(t) + + ctx, err := runInstall(t, cmdio.MockDiscard(t.Context()), "--skills-only") + require.NoError(t, err) + + events := funnelEvents(ctx) + require.Equal(t, []string{ + "STARTED", + "FLAGS_VALIDATED", + "SCOPE_RESOLVED", + "AGENTS_SELECTED", + "PLAN_CONFIRMED", + "SKILLS_INSTALLED", + "FINISHED:SUCCESS:", + }, funnelShape(events)) + + for _, e := range events { + assert.Equal(t, protos.AitoolsCommandInstall, e.Command) + assert.Equal(t, protos.AitoolsInstallModeSkillsOnly, e.InstallMode) + assert.False(t, e.Interactive) + } + finished := events[len(events)-1] + assert.Equal(t, protos.AitoolsInstallScopeGlobal, finished.Scope) + assert.Equal(t, []protos.AitoolsAgentType{protos.AitoolsAgentTypeClaudeCode, protos.AitoolsAgentTypeCursor}, finished.Agents) + + // The install event is still emitted next to the funnel. + var installEvents int + for _, e := range telemetry.BufferedEvents(ctx) { + if e.AitoolsInstallEvent != nil { + installEvents++ + } + } + assert.Equal(t, 1, installEvents) +} + +func TestInstallFunnelEarlyExits(t *testing.T) { + cases := []struct { + name string + args []string + want []string + }{ + { + name: "conflicting delivery flags", + args: []string{"--skills-only", "--path", "out"}, + want: []string{"STARTED", "FINISHED:FAILURE:INVALID_FLAGS"}, + }, + { + name: "skills without skills-only", + args: []string{"--skills", "databricks"}, + want: []string{"STARTED", "FINISHED:FAILURE:INVALID_FLAGS"}, + }, + { + name: "bad scope", + args: []string{"--scope", "nowhere"}, + want: []string{"STARTED", "FINISHED:FAILURE:INVALID_FLAGS"}, + }, + { + name: "json without agents", + args: []string{"--output", "json", "--scope", "global"}, + want: []string{"STARTED", "FINISHED:FAILURE:INVALID_FLAGS"}, + }, + { + name: "unknown agent", + args: []string{"--agents", "no-such-agent"}, + want: []string{"STARTED", "FINISHED:FAILURE:UNKNOWN_AGENT"}, + }, + { + name: "project and global", + args: []string{"--project", "--global"}, + want: []string{"STARTED", "FINISHED:FAILURE:INVALID_FLAGS"}, + }, + { + name: "empty agents list", + args: []string{"--agents", ","}, + want: []string{"STARTED", "FINISHED:FAILURE:INVALID_FLAGS"}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + isolateAgents(t) + setupTestAgents(t) + ctx, err := runInstall(t, cmdio.MockDiscard(t.Context()), tc.args...) + require.Error(t, err) + assert.Equal(t, tc.want, funnelShape(funnelEvents(ctx))) + }) + } +} + +func TestInstallFunnelNoAgentsDetected(t *testing.T) { + isolateAgents(t) + tmp := t.TempDir() + t.Setenv("HOME", tmp) + t.Setenv("USERPROFILE", tmp) + t.Setenv("XDG_CONFIG_HOME", filepath.Join(tmp, ".config")) + fakeBinsOnPath(t) + forbidSkillsInstall(t) + + ctx, err := runInstall(t, cmdio.MockDiscard(t.Context())) + require.NoError(t, err) + assert.Equal(t, []string{ + "STARTED", "FLAGS_VALIDATED", "SCOPE_RESOLVED", "FINISHED:NOTHING_TO_DO:", + }, funnelShape(funnelEvents(ctx))) +} + +func TestInstallFunnelPluginFailureThroughJSON(t *testing.T) { + isolateAgents(t, "CODEX_CI", "1") + tmp := t.TempDir() + t.Setenv("HOME", tmp) + t.Setenv("USERPROFILE", tmp) + fakeBinsOnPath(t, "codex") + t.Setenv("DATABRICKS_SKILLS_REF", "v0.2.6") + + origInstall := installPluginForAgentFn + t.Cleanup(func() { installPluginForAgentFn = origInstall }) + installPluginForAgentFn = func(_ context.Context, a *agents.Agent, _, _ string) (installer.PluginRecord, error) { + return installer.PluginRecord{}, &installer.BlockedError{Agent: a.Name, Reason: installer.ReasonCLINotOnPath} + } + + ctx, err := runInstall(t, cmdio.MockDiscard(t.Context()), "--agents", "codex", "--scope", "global", "--output", "json") + require.ErrorIs(t, err, root.ErrAlreadyPrinted) + + // A named agent's failure finishes with that agent's category, not the + // ErrAlreadyPrinted sentinel the command returns in JSON mode. + assert.Equal(t, []string{ + "STARTED", "FLAGS_VALIDATED", "SCOPE_RESOLVED", "AGENTS_SELECTED", "PLAN_CONFIRMED", + "FINISHED:FAILURE:CLI_NOT_ON_PATH", + }, funnelShape(funnelEvents(ctx))) +} + +// A failed skills install is a top-level failure, so in JSON mode it reaches +// the ErrAlreadyPrinted return. FINISHED must classify the install error, not +// the sentinel (which would be UNCATEGORIZED). +func TestInstallFunnelSkillsFailureThroughJSON(t *testing.T) { + isolateAgents(t) + setupTestAgents(t) + orig := installSkillsForAgentsFn + t.Cleanup(func() { installSkillsForAgentsFn = orig }) + installSkillsForAgentsFn = func(context.Context, installer.ManifestSource, []*agents.Agent, installer.InstallOptions) error { + return fmt.Errorf("fetch skills: %w", context.DeadlineExceeded) + } + + ctx, err := runInstall(t, cmdio.MockDiscard(t.Context()), "--agents", "cursor", "--skills-only", "--scope", "global", "--output", "json") + require.ErrorIs(t, err, root.ErrAlreadyPrinted) + assert.Equal(t, []string{ + "STARTED", "FLAGS_VALIDATED", "SCOPE_RESOLVED", "AGENTS_SELECTED", "PLAN_CONFIRMED", + "FINISHED:FAILURE:TIMEOUT", + }, funnelShape(funnelEvents(ctx))) +} + +func TestInstallFunnelInteractiveConfirm(t *testing.T) { + cases := []struct { + name string + proceed func() (bool, error) + wantErr bool + want []string + }{ + {name: "confirmed", proceed: func() (bool, error) { return true, nil }, want: []string{"PLAN_CONFIRMED", "SKILLS_INSTALLED", "FINISHED:SUCCESS:"}}, + {name: "declined", proceed: func() (bool, error) { return false, nil }, want: []string{"FINISHED:CANCELLED:"}}, + {name: "aborted", proceed: func() (bool, error) { return false, huh.ErrUserAborted }, wantErr: true, want: []string{"FINISHED:CANCELLED:USER_ABORTED"}}, + // The real prompt tags its own failure (defaultPromptProceed). + {name: "prompt failed", proceed: func() (bool, error) { return false, promptError(assert.AnError) }, wantErr: true, want: []string{"FINISHED:FAILURE:PROMPT_FAILED"}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + isolateAgents(t) + setupTestAgents(t) + setupScopeMock(t, installer.ScopeGlobal) + setupInstallMock(t) + origPicker, origProceed := promptAgentSelection, promptProceed + t.Cleanup(func() { promptAgentSelection, promptProceed = origPicker, origProceed }) + // Cursor gets raw skills, which setupInstallMock stands in for. + promptAgentSelection = func(context.Context, []agentChoice) ([]*agents.Agent, error) { + return []*agents.Agent{agents.ByName(agents.NameCursor)}, nil + } + promptProceed = tc.proceed + + ioCtx, test := cmdio.SetupTest(t.Context(), cmdio.TestOptions{PromptSupported: true}) + defer test.Done() + go drainReader(test.Stdout) + go drainReader(test.Stderr) + + ctx := telemetry.WithNewLogger(ioCtx) + cmd := newTestInstallCmd() + cmd.SetContext(ctx) + err := cmd.RunE(cmd, nil) + if tc.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + + events := funnelEvents(ctx) + want := append([]string{"STARTED", "FLAGS_VALIDATED", "SCOPE_RESOLVED", "AGENTS_SELECTED"}, tc.want...) + require.Equal(t, want, funnelShape(events)) + assert.True(t, events[0].Interactive) + }) + } +} + +// An unknown --agents name fails before the scope prompt is shown. +func TestInstallUnknownAgentFailsBeforeScopePrompt(t *testing.T) { + isolateAgents(t) + setupTestAgents(t) + orig := promptScopeSelection + t.Cleanup(func() { promptScopeSelection = orig }) + promptScopeSelection = func(context.Context) (string, error) { + t.Error("the scope prompt must not be shown for an unknown --agents name") + return installer.ScopeGlobal, nil + } + + ioCtx, test := cmdio.SetupTest(t.Context(), cmdio.TestOptions{PromptSupported: true}) + defer test.Done() + go drainReader(test.Stdout) + go drainReader(test.Stderr) + + ctx := telemetry.WithNewLogger(ioCtx) + cmd := newTestInstallCmd() + cmd.SetContext(ctx) + cmd.SetArgs([]string{"--agents", "bogus"}) + require.ErrorContains(t, cmd.Execute(), `unknown agent "bogus"`) + assert.Equal(t, []string{"STARTED", "FINISHED:FAILURE:UNKNOWN_AGENT"}, funnelShape(funnelEvents(ctx))) +} + +// Agents named via --agents are known before the scope prompt, so a run that +// is aborted there still reports them. +func TestInstallFunnelAbortedScopePromptKeepsNamedAgents(t *testing.T) { + isolateAgents(t) + setupTestAgents(t) + orig := promptScopeSelection + t.Cleanup(func() { promptScopeSelection = orig }) + promptScopeSelection = func(context.Context) (string, error) { return "", huh.ErrUserAborted } + + ioCtx, test := cmdio.SetupTest(t.Context(), cmdio.TestOptions{PromptSupported: true}) + defer test.Done() + go drainReader(test.Stdout) + go drainReader(test.Stderr) + + ctx := telemetry.WithNewLogger(ioCtx) + cmd := newTestInstallCmd() + cmd.SetContext(ctx) + cmd.SetArgs([]string{"--agents", "cursor"}) + require.ErrorIs(t, cmd.Execute(), huh.ErrUserAborted) + + events := funnelEvents(ctx) + require.Equal(t, []string{"STARTED", "FLAGS_VALIDATED", "FINISHED:CANCELLED:USER_ABORTED"}, funnelShape(events)) + assert.Equal(t, []protos.AitoolsAgentType{protos.AitoolsAgentTypeCursor}, events[len(events)-1].Agents) +} + +// Confirming the picker with nothing checked is the user opting out, so it +// finishes CANCELLED like a declined confirm, though the command still errors. +func TestInstallFunnelEmptyPickerSelectionIsCancelled(t *testing.T) { + isolateAgents(t) + setupTestAgents(t) + setupScopeMock(t, installer.ScopeGlobal) + origPicker := promptAgentSelection + t.Cleanup(func() { promptAgentSelection = origPicker }) + promptAgentSelection = func(context.Context, []agentChoice) ([]*agents.Agent, error) { + return nil, errNoAgentsSelected + } + + ioCtx, test := cmdio.SetupTest(t.Context(), cmdio.TestOptions{PromptSupported: true}) + defer test.Done() + go drainReader(test.Stdout) + go drainReader(test.Stderr) + + ctx := telemetry.WithNewLogger(ioCtx) + cmd := newTestInstallCmd() + cmd.SetContext(ctx) + err := cmd.RunE(cmd, nil) + require.ErrorIs(t, err, errNoAgentsSelected) + assert.Equal(t, "at least one agent must be selected", err.Error()) + assert.Equal(t, []string{"STARTED", "FLAGS_VALIDATED", "SCOPE_RESOLVED", "FINISHED:CANCELLED:"}, funnelShape(funnelEvents(ctx))) +} + +// A named agent skipped for the scope fails the run with an error that has no +// type of its own; the funnel takes the category from the agent's outcome. +func TestInstallFunnelNamedAgentSkippedForScope(t *testing.T) { + isolateAgents(t) + setupTestAgents(t) + setupInstallMock(t) + + ctx, err := runInstall(t, cmdio.MockDiscard(t.Context()), "--agents", "cursor", "--skills-only", "--scope", "project") + require.ErrorContains(t, err, "does not support project-scoped skills") + assert.Equal(t, []string{ + "STARTED", "FLAGS_VALIDATED", "SCOPE_RESOLVED", "AGENTS_SELECTED", "PLAN_CONFIRMED", + "FINISHED:FAILURE:UNSUPPORTED_SCOPE", + }, funnelShape(funnelEvents(ctx))) +} + +// setupPathDump prepares a --path dump that fetches skills from baseURL. It +// does not clear the whole environment (isolateAgents): Windows sockets need +// SystemRoot, and a dump detects no agents, so a temp HOME is enough. The +// stubbed telemetry auth keeps a developer's profile out of the run. +func setupPathDump(t *testing.T, baseURL string) { + t.Helper() + setupTestAgents(t) + setTelemetryAuth(t, 0, func(*cobra.Command, []string) error { return errors.New("no auth in this test") }) + t.Setenv("DATABRICKS_SKILLS_REF", "test-ref") + t.Setenv("DATABRICKS_SKILLS_BASE_URL", baseURL) +} + +// serveSkills serves a skills manifest and every file under it from a local +// server, standing in for raw.githubusercontent.com. +func serveSkills(t *testing.T, manifest string, files map[string]string) string { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/test-ref/manifest.json" { + _, _ = io.WriteString(w, manifest) + return + } + body, ok := files[r.URL.Path] + if !ok { + http.NotFound(w, r) + return + } + _, _ = io.WriteString(w, body) + })) + t.Cleanup(srv.Close) + return srv.URL +} + +func TestInstallFunnelPathDump(t *testing.T) { + // An unreachable manifest makes the dump fail after flag validation. + setupPathDump(t, "http://127.0.0.1:1") + + ctx, err := runInstall(t, cmdio.MockDiscard(t.Context()), "--path", t.TempDir()) + require.Error(t, err) + events := funnelEvents(ctx) + require.Equal(t, []string{"STARTED", "FLAGS_VALIDATED", "FINISHED:FAILURE:NETWORK"}, funnelShape(events)) + assert.Equal(t, protos.AitoolsInstallModePath, events[0].InstallMode) +} + +// A --path dump that writes skills: SKILLS_DUMPED, then SUCCESS. +func TestInstallFunnelPathDumpSuccess(t *testing.T) { + setupPathDump(t, serveSkills(t, + `{"version": "2", "skills": {"databricks-test": {"version": "1.0.0", "files": ["SKILL.md"], "repo_dir": "skills"}}}`, + map[string]string{"/test-ref/skills/databricks-test/SKILL.md": "# Test skill\n"})) + + ctx, err := runInstall(t, cmdio.MockDiscard(t.Context()), "--path", t.TempDir()) + require.NoError(t, err) + assert.Equal(t, []string{"STARTED", "FLAGS_VALIDATED", "SKILLS_DUMPED", "FINISHED:SUCCESS:"}, funnelShape(funnelEvents(ctx))) +} + +// A --path dump where every skill is filtered out (here: the only skill is +// experimental) writes nothing: NOTHING_TO_DO, with no SKILLS_DUMPED step. +func TestInstallFunnelPathDumpWithNoSkillToWrite(t *testing.T) { + setupPathDump(t, serveSkills(t, + `{"version": "2", "skills": {"databricks-preview": {"version": "1.0.0", "files": ["SKILL.md"], "repo_dir": "experimental"}}}`, + nil)) + + ctx, err := runInstall(t, cmdio.MockDiscard(t.Context()), "--path", t.TempDir()) + require.NoError(t, err) + assert.Equal(t, []string{"STARTED", "FLAGS_VALIDATED", "FINISHED:NOTHING_TO_DO:"}, funnelShape(funnelEvents(ctx))) +} + +// A plugin installed for a named agent: PLUGINS_INSTALLED, then SUCCESS. +func TestInstallFunnelPluginSuccess(t *testing.T) { + isolateAgents(t, "CODEX_CI", "1") + tmp := t.TempDir() + t.Setenv("HOME", tmp) + t.Setenv("USERPROFILE", tmp) + fakeBinsOnPath(t, "codex") + t.Setenv("DATABRICKS_SKILLS_REF", "v0.2.6") + setupPluginMock(t) + forbidSkillsInstall(t) + + ctx, err := runInstall(t, cmdio.MockDiscard(t.Context()), "--agents", "codex", "--scope", "global") + require.NoError(t, err) + events := funnelEvents(ctx) + require.Equal(t, []string{ + "STARTED", "FLAGS_VALIDATED", "SCOPE_RESOLVED", "AGENTS_SELECTED", "PLAN_CONFIRMED", + "PLUGINS_INSTALLED", "FINISHED:SUCCESS:", + }, funnelShape(events)) + assert.Equal(t, []protos.AitoolsAgentType{protos.AitoolsAgentTypeCodex}, events[len(events)-1].Agents) +} + +// A plugin failure for an auto-detected agent is only a warning (exit 0), but +// when nothing got installed the funnel must not report SUCCESS. +func TestInstallFunnelDetectedPluginFailureIsNotSuccess(t *testing.T) { + isolateAgents(t) + tmp := t.TempDir() + t.Setenv("HOME", tmp) + t.Setenv("USERPROFILE", tmp) + t.Setenv("XDG_CONFIG_HOME", filepath.Join(tmp, ".config")) + fakeBinsOnPath(t, "codex") + t.Setenv("DATABRICKS_SKILLS_REF", "v0.2.6") + setupPluginMock(t) + forbidSkillsInstall(t) + installPluginForAgentFn = func(_ context.Context, a *agents.Agent, _, _ string) (installer.PluginRecord, error) { + return installer.PluginRecord{}, &installer.BlockedError{Agent: a.Name, Reason: installer.ReasonInstallFailed} + } + + ctx, err := runInstall(t, cmdio.MockDiscard(t.Context()), "--scope", "global") + require.NoError(t, err) + assert.Equal(t, []string{ + "STARTED", "FLAGS_VALIDATED", "SCOPE_RESOLVED", "AGENTS_SELECTED", "PLAN_CONFIRMED", + "FINISHED:FAILURE:PLUGIN_INSTALL_FAILED", + }, funnelShape(funnelEvents(ctx))) +} + +func TestNothingInstalledCategory(t *testing.T) { + codex := &agents.Agent{Name: agents.NameCodex} + cursor := &agents.Agent{Name: agents.NameCursor} + failed := agentOutcome{agent: codex, delivery: deliveryPlugin, status: outcomeFailed, errorCategory: protos.AitoolsErrorCategoryPluginInstallFailed} + skipped := agentOutcome{agent: cursor, delivery: deliverySkip, status: outcomeSkipped, errorCategory: protos.AitoolsErrorCategoryUnsupportedScope} + installed := agentOutcome{agent: cursor, delivery: deliverySkills, status: outcomeInstalled} + + _, ok := nothingInstalledCategory(nil) + assert.False(t, ok) + _, ok = nothingInstalledCategory([]agentOutcome{failed, installed}) + assert.False(t, ok, "one installed agent makes the run a success") + got, ok := nothingInstalledCategory([]agentOutcome{failed, skipped}) + assert.True(t, ok) + assert.Equal(t, protos.AitoolsErrorCategoryPluginInstallFailed, got) + + uncategorized := agentOutcome{agent: codex, delivery: deliveryPlugin, status: outcomeFailed} + assert.Equal(t, protos.AitoolsErrorCategoryUnsupportedScope, firstFailureCategory([]agentOutcome{installed, uncategorized, skipped})) + assert.Equal(t, protos.AitoolsErrorCategoryUncategorized, firstFailureCategory([]agentOutcome{installed, uncategorized})) + + assert.Equal(t, []protos.AitoolsCommandStep{protos.AitoolsCommandStepSkillsInstalled}, deliverySteps([]agentOutcome{failed, installed})) + assert.Equal(t, []protos.AitoolsCommandStep{protos.AitoolsCommandStepPluginsInstalled}, + deliverySteps([]agentOutcome{failed, {agent: codex, delivery: deliveryPlugin, status: outcomeInstalled}})) +} diff --git a/cmd/aitools/install.go b/cmd/aitools/install.go index d154a22ca85..b050ec7bac0 100644 --- a/cmd/aitools/install.go +++ b/cmd/aitools/install.go @@ -28,6 +28,10 @@ var ( cleanupLegacyFn = installer.RemoveLegacyRawSkills ) +// errNoAgentsSelected is returned when the picker is confirmed with nothing +// checked. The funnel reports it as CANCELLED: the user chose to act on nothing. +var errNoAgentsSelected = errors.New("at least one agent must be selected") + // delivery is how the databricks tools are delivered to one agent. type delivery int @@ -77,6 +81,7 @@ func NewInstallCmd() *cobra.Command { var skillsFlag, agentsFlag, scopeFlag, pathFlag string var includeExperimental, skillsOnly bool var projectFlag, globalFlag bool + var auth *telemetryAuth cmd := &cobra.Command{ Use: "install", @@ -103,7 +108,7 @@ Output: Supported agents: ` + strings.Join(agents.SupportedNames(), ", "), Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, args []string) (err error) { ctx := cmd.Context() jsonMode := installOutputIsJSON(cmd) if jsonMode { @@ -113,15 +118,18 @@ Supported agents: ` + strings.Join(agents.SupportedNames(), ", "), ctx = cmdio.WithQuiet(ctx) } + funnel := startFunnel(ctx, protos.AitoolsCommandInstall, installModeType(skillsOnly, pathFlag)) + defer func() { funnel.finish(ctx, err) }() + if skillsOnly && pathFlag != "" { - return errors.New("cannot use --skills-only with --path; --path always writes raw skill files") + return withCategory(errors.New("cannot use --skills-only with --path; --path always writes raw skill files"), protos.AitoolsErrorCategoryInvalidFlags) } // --path is a plain file dump with no agents or install state, so there // is no per-agent result to report. Reject --output json here rather than // letting the dump run and silently emit no JSON. if jsonMode && pathFlag != "" { - return errors.New("cannot use --output json with --path; --path writes raw skill files and produces no JSON result") + return withCategory(errors.New("cannot use --output json with --path; --path writes raw skill files and produces no JSON result"), protos.AitoolsErrorCategoryInvalidFlags) } opts := installer.InstallOptions{ @@ -133,20 +141,31 @@ Supported agents: ` + strings.Join(agents.SupportedNames(), ", "), // raw-skills delivery. The plugin is installed in full, so reject // --skills unless raw skills were requested via --skills-only or --path. if len(opts.SpecificSkills) > 0 && !skillsOnly && pathFlag == "" { - return errors.New("--skills requires --skills-only or --path; the databricks plugin is installed in full") + return withCategory(errors.New("--skills requires --skills-only or --path; the databricks plugin is installed in full"), protos.AitoolsErrorCategoryInvalidFlags) } src := &installer.GitHubManifestSource{} // --path is a dumb dump: no agents, no scope, no state. if pathFlag != "" { - _, err := installer.DumpSkillsToPath(ctx, src, pathFlag, opts) - return err + funnel.step(ctx, protos.AitoolsCommandStepFlagsValidated) + written, err := installer.DumpSkillsToPath(ctx, src, pathFlag, opts) + if err != nil { + return err + } + if written == 0 { + // Every skill was filtered out (e.g. all experimental without + // --experimental): there was nothing to write. + funnel.finishWith(ctx, protos.AitoolsCommandOutcomeNothingToDo, protos.AitoolsErrorCategoryUnspecified) + return nil + } + funnel.step(ctx, protos.AitoolsCommandStepSkillsDumped) + return nil } projectFlag, globalFlag, err := parseScopeFlag(scopeFlag, projectFlag, globalFlag, false) if err != nil { - return err + return withCategory(err, protos.AitoolsErrorCategoryInvalidFlags) } // JSON output must be fully non-interactive: every choice has to come @@ -162,17 +181,20 @@ Supported agents: ` + strings.Join(agents.SupportedNames(), ", "), missing = append(missing, "--agents") } if len(missing) > 0 { - return fmt.Errorf("--output json requires %s so the command runs without interactive prompts", strings.Join(missing, " and ")) + return withCategory(fmt.Errorf("--output json requires %s so the command runs without interactive prompts", strings.Join(missing, " and ")), protos.AitoolsErrorCategoryInvalidFlags) } } - - scope, err := resolveScopeWithPrompt(ctx, projectFlag, globalFlag) - if err != nil { - return err + // A scope flag sets the scope here, so --project with --global is a + // flag error; the scope prompt below runs only when no flag is set. + var scope string + if projectFlag || globalFlag { + scope, err = resolveScope(projectFlag, globalFlag) + if err != nil { + return withCategory(err, protos.AitoolsErrorCategoryInvalidFlags) + } } - opts.Scope = scope - // Resolve the agents to act on. + // Validate --agents with the other flags, before the scope prompt. var targetAgents []*agents.Agent explicit := agentsFlag != "" if explicit { @@ -180,18 +202,35 @@ Supported agents: ` + strings.Join(agents.SupportedNames(), ", "), if err != nil { return err } - } else { + funnel.setTargets(targetAgents) + } + funnel.step(ctx, protos.AitoolsCommandStepFlagsValidated) + + if scope == "" { + scope, err = resolveScopeWithPrompt(ctx) + if err != nil { + return err + } + } + opts.Scope = scope + funnel.setScope(scope) + funnel.step(ctx, protos.AitoolsCommandStepScopeResolved) + + if !explicit { targetAgents, err = selectAgents(ctx, scope, skillsOnly) if err != nil { return err } if len(targetAgents) == 0 { printNoAgentsMessage(ctx) + funnel.finishWith(ctx, protos.AitoolsCommandOutcomeNothingToDo, protos.AitoolsErrorCategoryUnspecified) return nil } } plan := buildPlan(targetAgents, scope, skillsOnly, explicit) + funnel.setAgents(plan) + funnel.step(ctx, protos.AitoolsCommandStepAgentsSelected) // In the interactive picker path, show a plan summary and confirm. if !explicit && cmdio.IsPromptSupported(ctx) { @@ -202,9 +241,18 @@ Supported agents: ` + strings.Join(agents.SupportedNames(), ", "), } if !proceed { cmdio.LogString(ctx, "Cancelled.") + funnel.finishWith(ctx, protos.AitoolsCommandOutcomeCancelled, protos.AitoolsErrorCategoryUnspecified) return nil } } + funnel.step(ctx, protos.AitoolsCommandStepPlanConfirmed) + // Only a run past this point waits for the telemetry auth. A run that + // ended earlier (a flag error, a cancelled prompt, no agent detected, + // a --path dump) uploads its events only if auth had already resolved, + // so it stays instant. The funnel therefore under-counts the fastest + // early exits (which usually end before auth resolves), and their + // STARTED events, relative to slower ones such as a cancelled prompt. + auth.waitAtExit = true var outcomes []agentOutcome var runErr error @@ -216,6 +264,23 @@ Supported agents: ` + strings.Join(agents.SupportedNames(), ", "), }() outcomes, runErr = executePlan(ctx, src, plan, opts, jsonMode) + for _, s := range deliverySteps(outcomes) { + funnel.step(ctx, s) + } + switch { + case runErr == nil: + if category, ok := nothingInstalledCategory(outcomes); ok { + funnel.finishWith(ctx, protos.AitoolsCommandOutcomeFailure, category) + } + case topLevelFailure(runErr) == nil: + // Named agents failed or were skipped. A skip has no error type of + // its own, so take the category from the outcomes, not runErr. + funnel.finishWith(ctx, protos.AitoolsCommandOutcomeFailure, firstFailureCategory(outcomes)) + default: + // Finish on the real failure here: JSON mode returns the + // ErrAlreadyPrinted sentinel below. + funnel.finish(ctx, runErr) + } if jsonMode { if jerr := renderJSON(cmd.OutOrStdout(), buildInstallOutput(opts.Scope, outcomes, runErr)); jerr != nil { @@ -251,7 +316,7 @@ Supported agents: ` + strings.Join(agents.SupportedNames(), ", "), cmd.Flags().BoolVar(&globalFlag, "global", false, "Install globally (default)") markScopeBoolsDeprecated(cmd) // Resolve auth best-effort and time-bounded so telemetry can upload. - withTelemetryAuth(cmd) + auth = withTelemetryAuth(cmd) return cmd } @@ -357,7 +422,7 @@ func defaultPromptProceed() (bool, error) { Value(&proceed). Run() if err != nil { - return false, err + return false, promptError(err) } return proceed, nil } @@ -378,10 +443,10 @@ func defaultPromptAgentSelection(_ context.Context, choices []agentChoice) ([]*a Value(&selected). Run() if err != nil { - return nil, err + return nil, promptError(err) } if len(selected) == 0 { - return nil, errors.New("at least one agent must be selected") + return nil, errNoAgentsSelected } result := make([]*agents.Agent, 0, len(selected)) @@ -669,13 +734,13 @@ func resolveAgentNames(_ context.Context, names string) ([]*agents.Agent, error) seen[name] = true agent, ok := available[name] if !ok { - return nil, fmt.Errorf("unknown agent %q. Available agents: %s", name, strings.Join(availableNames, ", ")) + return nil, withCategory(fmt.Errorf("unknown agent %q. Available agents: %s", name, strings.Join(availableNames, ", ")), protos.AitoolsErrorCategoryUnknownAgent) } result = append(result, agent) } if len(result) == 0 { - return nil, errors.New("no agents specified") + return nil, withCategory(errors.New("no agents specified"), protos.AitoolsErrorCategoryInvalidFlags) } return result, nil } diff --git a/cmd/aitools/scope.go b/cmd/aitools/scope.go index 71c9b5f24f9..da9c7754185 100644 --- a/cmd/aitools/scope.go +++ b/cmd/aitools/scope.go @@ -58,15 +58,10 @@ func resolveScope(project, global bool) (string, error) { return installer.ScopeGlobal, nil } -// resolveScopeWithPrompt resolves scope with optional interactive prompt. -// When neither flag is set: interactive mode shows a prompt (default: global), +// resolveScopeWithPrompt resolves the scope when neither --project nor +// --global is set: interactive mode shows a prompt (default: global), // non-interactive mode uses global. -func resolveScopeWithPrompt(ctx context.Context, project, global bool) (string, error) { - if project || global { - return resolveScope(project, global) - } - - // No flag: prompt if interactive, default to global otherwise. +func resolveScopeWithPrompt(ctx context.Context) (string, error) { if cmdio.IsPromptSupported(ctx) { return promptScopeSelection(ctx) } @@ -99,7 +94,7 @@ func defaultPromptScopeSelection(ctx context.Context) (string, error) { Value(&scope). Run() if err != nil { - return "", err + return "", promptError(err) } return scope, nil diff --git a/cmd/aitools/telemetry.go b/cmd/aitools/telemetry.go index a6eccb1111e..fbab971b8ed 100644 --- a/cmd/aitools/telemetry.go +++ b/cmd/aitools/telemetry.go @@ -21,7 +21,8 @@ import ( // telemetryAuthBudget bounds how long after its start a command waits for the // background auth resolution its telemetry upload needs. A resolution still // unfinished when the run ends and the budget has passed drops the events. -// Matches the upload's own default timeout. +// Matches the upload's own default timeout. A run waits only once its command +// asks for it (telemetryAuth.waitAtExit). var telemetryAuthBudget = 3 * time.Second // resolveTelemetryAuthFn resolves auth onto cmd's context. A seam for tests. @@ -30,22 +31,37 @@ var resolveTelemetryAuthFn = func(cmd *cobra.Command, args []string) error { return err } +// telemetryAuth is the best-effort auth resolution of one command run. +type telemetryAuth struct { + pending *pendingAuth + + // waitAtExit is whether the end of the run waits for an unfinished + // resolution (within telemetryAuthBudget). False by default: the + // telemetry uploads only if the resolution has already finished, so a run + // that ends early stays instant. RunE sets it once the run has gone far + // enough to be worth the wait. + waitAtExit bool +} + // withTelemetryAuth makes cmd resolve auth best-effort so its telemetry can -// upload at exit: PreRunE starts the resolution in the background and RunE's -// wrapper waits for it within telemetryAuthBudget. Nothing is resolved when -// telemetry is disabled. The command never fails or prompts because of it. -func withTelemetryAuth(cmd *cobra.Command) { - var pending *pendingAuth +// upload at exit: PreRunE starts the resolution in the background, and RunE's +// wrapper uses it if it has finished, or waits for it within +// telemetryAuthBudget once RunE sets waitAtExit on the returned value. Nothing +// is resolved when telemetry is disabled. The command never fails or prompts +// because of it. +func withTelemetryAuth(cmd *cobra.Command) *telemetryAuth { + auth := &telemetryAuth{} cmd.PreRunE = func(cmd *cobra.Command, args []string) error { - pending = startTelemetryAuth(cmd, args) + auth.pending = startTelemetryAuth(cmd, args) return nil } runE := cmd.RunE cmd.RunE = func(cmd *cobra.Command, args []string) error { err := runE(cmd, args) - pending.apply(cmd) + auth.pending.apply(cmd, auth.waitAtExit) return err } + return auth } // pendingAuth is a background auth resolution started by startTelemetryAuth. @@ -92,19 +108,23 @@ func startTelemetryAuth(cmd *cobra.Command, args []string) *pendingAuth { return p } -// apply waits for the resolution until the deadline and, if it finished, puts -// the resolved config on cmd's context for the upload. A run that logged no -// event has nothing to upload, so it does not wait. Only the config is -// copied: the upload needs nothing else, and a workspace client on the context -// would change how root decorates the command's own errors. -func (p *pendingAuth) apply(cmd *cobra.Command) { +// apply puts the resolved config on cmd's context for the upload if the +// resolution has finished, waiting for it until the deadline when wait is set. +// A run that logged no event has nothing to upload, so it does not wait. Only +// the config is copied: the upload needs nothing else, and a workspace client +// on the context would change how root decorates the command's own errors. +func (p *pendingAuth) apply(cmd *cobra.Command, wait bool) { if p == nil { return } if len(telemetry.BufferedEvents(cmd.Context())) == 0 { return } - if !p.wait() { + if !wait && !p.finished() { + log.Debugf(cmd.Context(), "auth for telemetry not resolved yet; not uploading telemetry") + return + } + if wait && !p.wait() { log.Debugf(cmd.Context(), "auth for telemetry not resolved within %s; not uploading telemetry", telemetryAuthBudget) return } @@ -117,10 +137,8 @@ func (p *pendingAuth) apply(cmd *cobra.Command) { // that finished before the wait starts always counts: once a command outlasts // the budget, a select on both channels would pick between them at random. func (p *pendingAuth) wait() bool { - select { - case <-p.done: + if p.finished() { return true - default: } timer := time.NewTimer(time.Until(p.deadline)) defer timer.Stop() @@ -132,6 +150,16 @@ func (p *pendingAuth) wait() bool { } } +// finished reports whether the resolution has finished, without waiting. +func (p *pendingAuth) finished() bool { + select { + case <-p.done: + return true + default: + return false + } +} + // debugOnlyHandler logs records above Debug (e.g. hostmetadata's "failed to // fetch host metadata" Warn) at Debug, so they show only with --debug. type debugOnlyHandler struct{ slog.Handler } @@ -207,16 +235,25 @@ func agentResultsField(outcomes []agentOutcome) []protos.AitoolsAgentResult { // agentsField returns the deduped agent enums from the plan, sorted so the // same set of agents always produces the same array on the analytics side. func agentsField(plan []agentPlanItem) []protos.AitoolsAgentType { - if len(plan) == 0 { + targets := make([]*agents.Agent, 0, len(plan)) + for _, it := range plan { + targets = append(targets, it.agent) + } + return agentTypes(targets) +} + +// agentTypes returns the deduped, sorted agent enums of targets. +func agentTypes(targets []*agents.Agent) []protos.AitoolsAgentType { + if len(targets) == 0 { return nil } - out := make([]protos.AitoolsAgentType, 0, len(plan)) - seen := make(map[protos.AitoolsAgentType]struct{}, len(plan)) - for _, it := range plan { - if it.agent == nil { + out := make([]protos.AitoolsAgentType, 0, len(targets)) + seen := make(map[protos.AitoolsAgentType]struct{}, len(targets)) + for _, a := range targets { + if a == nil { continue } - t := agentType(it.agent.Name) + t := agentType(a.Name) if _, ok := seen[t]; ok { continue } diff --git a/cmd/aitools/telemetry_auth_test.go b/cmd/aitools/telemetry_auth_test.go index a44fb8547fc..ea59542176a 100644 --- a/cmd/aitools/telemetry_auth_test.go +++ b/cmd/aitools/telemetry_auth_test.go @@ -2,13 +2,18 @@ package aitools import ( "bytes" + "context" "errors" "log/slog" "strings" + "sync" "testing" "time" + "github.com/databricks/cli/libs/aitools/agents" + "github.com/databricks/cli/libs/aitools/installer" "github.com/databricks/cli/libs/cmdctx" + "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/log/handler" @@ -49,7 +54,9 @@ func authTestCmdLogging(t *testing.T, logEvent bool) (*cobra.Command, *bool) { }, } cmd.Flags().String("profile", "", "") - withTelemetryAuth(cmd) + // These tests exercise the wait, so the command asks for it, as install + // does once the plan is confirmed. + withTelemetryAuth(cmd).waitAtExit = true ctx := telemetry.WithNewLogger(t.Context()) cmd.SetContext(env.Set(ctx, "DATABRICKS_CLI_DISABLE_TELEMETRY", "")) return cmd, &ran @@ -137,7 +144,7 @@ func TestTelemetryAuthFinishedResolutionSurvivesAnExpiredBudget(t *testing.T) { p := startTelemetryAuth(cmd, nil) <-p.done telemetry.Log(cmd.Context(), protos.DatabricksCliLog{AitoolsInstallEvent: &protos.AitoolsInstallEvent{}}) - p.apply(cmd) + p.apply(cmd, true) require.True(t, cmdctx.HasConfigUsed(cmd.Context())) } } @@ -219,3 +226,74 @@ func TestTelemetryAuthRunEWithoutPreRun(t *testing.T) { require.NoError(t, cmd.RunE(cmd, nil)) assert.True(t, *ran) } + +// Without the wait, a resolution that has already finished still counts. The +// repeats make a race between "done" and "not done" fail reliably. +func TestTelemetryAuthNoWaitUsesFinishedResolution(t *testing.T) { + setTelemetryAuth(t, time.Second, func(cmd *cobra.Command, args []string) error { + cmd.SetContext(cmdctx.SetConfigUsed(cmd.Context(), &config.Config{Host: "https://ws.test"})) + return nil + }) + for range 50 { + cmd, _ := authTestCmd(t) + p := startTelemetryAuth(cmd, nil) + <-p.done + telemetry.Log(cmd.Context(), protos.DatabricksCliLog{AitoolsInstallEvent: &protos.AitoolsInstallEvent{}}) + p.apply(cmd, false) + require.True(t, cmdctx.HasConfigUsed(cmd.Context())) + } +} + +// blockTelemetryAuth makes the resolution finish only once the returned func +// is called (or the test ends), with a budget long enough that a run that +// waits for it is easy to tell apart from one that does not. +func blockTelemetryAuth(t *testing.T) (release func()) { + ch := make(chan struct{}) + release = sync.OnceFunc(func() { close(ch) }) + t.Cleanup(release) + setTelemetryAuth(t, 10*time.Second, func(cmd *cobra.Command, args []string) error { + <-ch + cmd.SetContext(cmdctx.SetConfigUsed(cmd.Context(), &config.Config{Host: "https://ws.test"})) + return nil + }) + return release +} + +// An install run that ends before the plan is confirmed does not wait for an +// unfinished resolution, so its events are dropped instead of delaying exit. +func TestInstallEarlyExitDoesNotWaitForTelemetryAuth(t *testing.T) { + isolateAgents(t) + setupTestAgents(t) + blockTelemetryAuth(t) + + cmd := newTestInstallCmd() + cmd.SetContext(telemetry.WithNewLogger(cmdio.MockDiscard(t.Context()))) + cmd.SetArgs([]string{"--skills-only", "--path", "out"}) + + start := time.Now() + require.ErrorContains(t, cmd.Execute(), "cannot use --skills-only with --path") + assert.Less(t, time.Since(start), 5*time.Second) +} + +// An install run past the plan confirmation keeps waiting for the resolution +// within the budget. The resolution can finish only after the install step, +// so the config reaches the context only if the end of the run waits. +func TestInstallPastPlanConfirmedWaitsForTelemetryAuth(t *testing.T) { + isolateAgents(t) + setupTestAgents(t) + release := blockTelemetryAuth(t) + orig := installSkillsForAgentsFn + t.Cleanup(func() { installSkillsForAgentsFn = orig }) + installSkillsForAgentsFn = func(context.Context, installer.ManifestSource, []*agents.Agent, installer.InstallOptions) error { + release() + return nil + } + + cmd := newTestInstallCmd() + cmd.SetContext(telemetry.WithNewLogger(cmdio.MockDiscard(t.Context()))) + cmd.SetArgs([]string{"--skills-only", "--agents", "cursor", "--scope", "global"}) + + require.NoError(t, cmd.Execute()) + require.True(t, cmdctx.HasConfigUsed(cmd.Context())) + assert.Equal(t, "https://ws.test", cmdctx.ConfigUsed(cmd.Context()).Host) +} diff --git a/libs/telemetry/protos/README.md b/libs/telemetry/protos/README.md index c82d4bc5589..232cd12e62a 100644 --- a/libs/telemetry/protos/README.md +++ b/libs/telemetry/protos/README.md @@ -13,3 +13,11 @@ event uploads at exit only when the command resolved auth. The install hint prin Claude Code session per hour, so when the command that printed it resolves no auth (e.g. `databricks version`), that session logs no install hint event for the next hour even though the agent saw the hint. + +`databricks aitools install` logs an `aitools_command_event` funnel per run: STARTED, one +STEP_COMPLETED per install step reached, and FINISHED with the outcome and error category. The CLI +sets no calling-agent field on it: the upload's User-Agent carries the agent (`agent/`), its +Databricks tooling (`aitools-agent/...`) and the `cmd-exec-id` that every event of the run shares. It uploads at exit only when the command resolved +auth. A run that ends before the install plan is confirmed (a flag error, a cancelled prompt, no +agent detected, a `--path` dump) uploads only if auth has already resolved by then; it does not wait +for it. diff --git a/libs/telemetry/protos/aitools_events.go b/libs/telemetry/protos/aitools_events.go index bc4db8f2948..381862be3d3 100644 --- a/libs/telemetry/protos/aitools_events.go +++ b/libs/telemetry/protos/aitools_events.go @@ -32,3 +32,119 @@ const ( type AitoolsHintEvent struct { HintKind AitoolsHintKind `json:"hint_kind"` } + +// AitoolsCommand mirrors AitoolsCommand.Type: the aitools subcommand an +// AitoolsCommandEvent belongs to. +type AitoolsCommand string + +const ( + AitoolsCommandUnspecified AitoolsCommand = "TYPE_UNSPECIFIED" + AitoolsCommandInstall AitoolsCommand = "INSTALL" + AitoolsCommandUpdate AitoolsCommand = "UPDATE" + AitoolsCommandList AitoolsCommand = "LIST" + AitoolsCommandVersion AitoolsCommand = "VERSION" + AitoolsCommandUninstall AitoolsCommand = "UNINSTALL" +) + +// AitoolsCommandStage mirrors AitoolsCommandStage.Type: where in the command's +// funnel an AitoolsCommandEvent was emitted. +type AitoolsCommandStage string + +const ( + AitoolsCommandStageUnspecified AitoolsCommandStage = "TYPE_UNSPECIFIED" + AitoolsCommandStageStarted AitoolsCommandStage = "STARTED" + AitoolsCommandStageStepCompleted AitoolsCommandStage = "STEP_COMPLETED" + AitoolsCommandStageFinished AitoolsCommandStage = "FINISHED" +) + +// AitoolsCommandStep mirrors AitoolsCommandStep.Type: a funnel step of +// `aitools install`, in the order the command reaches them. A run that stops +// early has no events for the later steps, so the last STEP_COMPLETED event +// shows where it stopped. +type AitoolsCommandStep string + +const ( + AitoolsCommandStepUnspecified AitoolsCommandStep = "TYPE_UNSPECIFIED" + // Flag combinations were validated. + AitoolsCommandStepFlagsValidated AitoolsCommandStep = "FLAGS_VALIDATED" + // The install scope was resolved (from flags or the scope prompt). + AitoolsCommandStepScopeResolved AitoolsCommandStep = "SCOPE_RESOLVED" + // The target agents were resolved (from --agents, the picker, or detection). + AitoolsCommandStepAgentsSelected AitoolsCommandStep = "AGENTS_SELECTED" + // The plan was confirmed, by the user or implicitly when no prompt is shown. + AitoolsCommandStepPlanConfirmed AitoolsCommandStep = "PLAN_CONFIRMED" + // Raw skill files were installed for the skills-delivery agents. + AitoolsCommandStepSkillsInstalled AitoolsCommandStep = "SKILLS_INSTALLED" + // At least one plugin-delivery agent got the plugin (others may still have + // failed; the run's AitoolsInstallEvent agent_results has them per agent). + AitoolsCommandStepPluginsInstalled AitoolsCommandStep = "PLUGINS_INSTALLED" + // Skill files were written to --path. + AitoolsCommandStepSkillsDumped AitoolsCommandStep = "SKILLS_DUMPED" +) + +// AitoolsCommandOutcome mirrors AitoolsCommandOutcome.Type: how a command run +// ended. Only set on FINISHED events. +type AitoolsCommandOutcome string + +const ( + AitoolsCommandOutcomeUnspecified AitoolsCommandOutcome = "TYPE_UNSPECIFIED" + AitoolsCommandOutcomeSuccess AitoolsCommandOutcome = "SUCCESS" + AitoolsCommandOutcomeFailure AitoolsCommandOutcome = "FAILURE" + // The user declined the confirm prompt, aborted a prompt, or confirmed the + // agent picker with nothing selected. + AitoolsCommandOutcomeCancelled AitoolsCommandOutcome = "CANCELLED" + // The command had nothing to act on (e.g. no coding agents detected, or a + // --path dump where every skill was filtered out). + AitoolsCommandOutcomeNothingToDo AitoolsCommandOutcome = "NOTHING_TO_DO" +) + +// AitoolsInstallMode mirrors AitoolsInstallMode.Type: how `aitools install` +// delivers the tools. +type AitoolsInstallMode string + +const ( + AitoolsInstallModeUnspecified AitoolsInstallMode = "TYPE_UNSPECIFIED" + // Plugin where the agent supports it, skills otherwise. + AitoolsInstallModeDefault AitoolsInstallMode = "DEFAULT" + // --skills-only: raw skill files for every agent. + AitoolsInstallModeSkillsOnly AitoolsInstallMode = "SKILLS_ONLY" + // --path: skill files written to a directory, no agents. + AitoolsInstallModePath AitoolsInstallMode = "PATH" +) + +// AitoolsCommandEvent is one funnel event of an aitools subcommand. Every +// install run that gets past flag parsing emits a STARTED event, a +// STEP_COMPLETED event per step reached and a FINISHED event. All events of one +// run share ExecutionContext.CmdExecID. +// The CLI sets no calling-agent field on this event: it reports the calling +// agent through the User-Agent of the telemetry upload (`agent/`, and its +// Databricks tooling as `aitools-agent/...`), together with the same +// cmd-exec-id. +type AitoolsCommandEvent struct { + Command AitoolsCommand `json:"command"` + Stage AitoolsCommandStage `json:"stage"` + + // Set on STEP_COMPLETED events only. + Step AitoolsCommandStep `json:"step,omitempty"` + + // Outcome and ErrorCategory are set on FINISHED events only. ErrorCategory + // is set for FAILURE, and for CANCELLED only when a prompt was aborted + // (USER_ABORTED); otherwise it is omitted. + Outcome AitoolsCommandOutcome `json:"outcome,omitempty"` + ErrorCategory AitoolsErrorCategory `json:"error_category,omitempty"` + + // Whether the CLI could show prompts (cmdio.IsPromptSupported). A + // measurable false, so no omitempty. + Interactive bool `json:"interactive"` + + // Install only: delivery mode. Set on every install event once flags parse. + InstallMode AitoolsInstallMode `json:"install_mode,omitempty"` + + // Install only: agents the run targeted, deduplicated and sorted. Set on the + // FINISHED event once the targets are known. + Agents []AitoolsAgentType `json:"agents,omitempty"` + + // Install only: scope the run acted on. Set on events after the scope is + // resolved. + Scope AitoolsInstallScope `json:"scope,omitempty"` +} diff --git a/libs/telemetry/protos/aitools_events_test.go b/libs/telemetry/protos/aitools_events_test.go index 78b3a903707..f043da1b8a5 100644 --- a/libs/telemetry/protos/aitools_events_test.go +++ b/libs/telemetry/protos/aitools_events_test.go @@ -9,11 +9,35 @@ import ( ) // The lumberjack proto keys fields in snake_case; a wrong tag is dropped -// silently on ingestion. This pins the wire keys of the aitools hint event. +// silently on ingestion. This pins the wire keys of the new aitools events, +// including the measurable zeros that must not be omitted. func TestAitoolsEventsWireKeys(t *testing.T) { b, err := json.Marshal(DatabricksCliLog{ AitoolsHintEvent: &AitoolsHintEvent{HintKind: AitoolsHintKindInstallText}, }) require.NoError(t, err) assert.JSONEq(t, `{"aitools_hint_event":{"hint_kind":"INSTALL_TEXT"}}`, string(b)) + + b, err = json.Marshal(DatabricksCliLog{ + AitoolsCommandEvent: &AitoolsCommandEvent{ + Command: AitoolsCommandList, + Stage: AitoolsCommandStageStarted, + }, + }) + require.NoError(t, err) + assert.JSONEq(t, `{"aitools_command_event":{"command":"LIST","stage":"STARTED","interactive":false}}`, string(b)) + + b, err = json.Marshal(AitoolsCommandEvent{ + Command: AitoolsCommandInstall, + Stage: AitoolsCommandStageFinished, + Outcome: AitoolsCommandOutcomeFailure, + ErrorCategory: AitoolsErrorCategoryNetwork, + Interactive: true, + InstallMode: AitoolsInstallModeDefault, + Agents: []AitoolsAgentType{AitoolsAgentTypeCodex}, + Scope: AitoolsInstallScopeGlobal, + }) + require.NoError(t, err) + assert.JSONEq(t, `{"command":"INSTALL","stage":"FINISHED","outcome":"FAILURE","error_category":"NETWORK", + "interactive":true,"install_mode":"DEFAULT","agents":["CODEX"],"scope":"GLOBAL"}`, string(b)) } diff --git a/libs/telemetry/protos/frontend_log.go b/libs/telemetry/protos/frontend_log.go index 355600e1896..f3c37736cc1 100644 --- a/libs/telemetry/protos/frontend_log.go +++ b/libs/telemetry/protos/frontend_log.go @@ -27,4 +27,5 @@ type DatabricksCliLog struct { SetupLocalEvent *SetupLocalEvent `json:"setup_local_event,omitempty"` AirRunEvent *AirRunEvent `json:"air_run_event,omitempty"` AitoolsHintEvent *AitoolsHintEvent `json:"aitools_hint_event,omitempty"` + AitoolsCommandEvent *AitoolsCommandEvent `json:"aitools_command_event,omitempty"` }