Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* `databricks aitools install`, `update` and `uninstall` now exit with code 1 when they do nothing they were asked to. `install` fails when it sets up nothing: no coding agent is detected, every agent is skipped or fails to install, or no skill in the release can be installed (also with `--path`). `update` fails when every update or check it attempts fails, and `uninstall` when every removal it attempts fails. Scripts that ran these commands as a best-effort step and relied on exit code 0 should handle the new exit code. ([#7481](https://github.com/databricks/cli/pull/7481))
16 changes: 16 additions & 0 deletions acceptance/cmd/aitools/telemetry/install-failure/output.txt
Original file line number Diff line number Diff line change
Expand Up @@ -68,3 +68,19 @@ Error: Cursor: does not support project-scoped skills
{"command":"aitools_install","exit_code":1,"aitools_command_event":{"command":"INSTALL","stage":"STEP_COMPLETED","step":"PLAN_CONFIRMED","interactive":false,"install_mode":"SKILLS_ONLY","scope":"PROJECT"}}
{"command":"aitools_install","exit_code":1,"aitools_command_event":{"command":"INSTALL","stage":"FINISHED","outcome":"FAILURE","error_category":"UNSUPPORTED_SCOPE","interactive":false,"install_mode":"SKILLS_ONLY","agents":["CURSOR"],"scope":"PROJECT"}}
{"command":"aitools_install","exit_code":1,"aitools_install_event":{"agents":["CURSOR"],"scope":"PROJECT","error_category":"TYPE_UNSPECIFIED","agent_results":[{"agent":"CURSOR","error_category":"UNSUPPORTED_SCOPE"}]}}

=== a detected agent skipped for the scope prints its recovery step and exits 1 with nothing installed

>>> [CLI] aitools install --skills-only --scope=project
Skipped Cursor: does not support project-scoped skills
To set up Cursor in the global scope, run: databricks aitools install --scope=global --agents=cursor --skills-only
Error: nothing was installed; follow the steps above

>>> print_telemetry
{"command":"aitools_install","exit_code":1,"aitools_command_event":{"command":"INSTALL","stage":"STARTED","interactive":false,"install_mode":"SKILLS_ONLY"}}
{"command":"aitools_install","exit_code":1,"aitools_command_event":{"command":"INSTALL","stage":"STEP_COMPLETED","step":"FLAGS_VALIDATED","interactive":false,"install_mode":"SKILLS_ONLY"}}
{"command":"aitools_install","exit_code":1,"aitools_command_event":{"command":"INSTALL","stage":"STEP_COMPLETED","step":"SCOPE_RESOLVED","interactive":false,"install_mode":"SKILLS_ONLY","scope":"PROJECT"}}
{"command":"aitools_install","exit_code":1,"aitools_command_event":{"command":"INSTALL","stage":"STEP_COMPLETED","step":"AGENTS_SELECTED","interactive":false,"install_mode":"SKILLS_ONLY","scope":"PROJECT"}}
{"command":"aitools_install","exit_code":1,"aitools_command_event":{"command":"INSTALL","stage":"STEP_COMPLETED","step":"PLAN_CONFIRMED","interactive":false,"install_mode":"SKILLS_ONLY","scope":"PROJECT"}}
{"command":"aitools_install","exit_code":1,"aitools_command_event":{"command":"INSTALL","stage":"FINISHED","outcome":"FAILURE","error_category":"UNSUPPORTED_SCOPE","interactive":false,"install_mode":"SKILLS_ONLY","agents":["CURSOR"],"scope":"PROJECT"}}
{"command":"aitools_install","exit_code":1,"aitools_install_event":{"agents":["CURSOR"],"scope":"PROJECT","error_category":"TYPE_UNSPECIFIED","agent_results":[{"agent":"CURSOR","error_category":"UNSUPPORTED_SCOPE"}]}}
9 changes: 9 additions & 0 deletions acceptance/cmd/aitools/telemetry/install-failure/script
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
# Isolate HOME so parallel aitools tests don't race on a shared ~/.databricks.
sethome home
# Agent config overrides from the host would make auto-detection find agents
# outside the isolated HOME; keep every agent's config dir inside it.
unset CLAUDE_CONFIG_DIR PI_CODING_AGENT_DIR GEMINI_CLI_HOME GOOSE_PATH_ROOT
export XDG_CONFIG_HOME="${USERPROFILE:-$HOME}/.config"
export APPDATA="${USERPROFILE:-$HOME}/AppData/Roaming"
mkdir -p "${USERPROFILE:-$HOME}/.cursor"

# Only runs past the plan confirmation are here: a run that ends earlier (a
Expand All @@ -22,3 +27,7 @@ trace print_telemetry
title "a named agent skipped for the scope takes its category from the agent's outcome"
musterr trace $CLI aitools install --skills-only --scope=project --agents=cursor
trace print_telemetry

title "a detected agent skipped for the scope prints its recovery step and exits 1 with nothing installed"
musterr trace $CLI aitools install --skills-only --scope=project
trace print_telemetry
39 changes: 13 additions & 26 deletions cmd/aitools/funnel.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package aitools
import (
"context"
"errors"
"slices"

"github.com/charmbracelet/huh"
"github.com/databricks/cli/libs/aitools/agents"
Expand Down Expand Up @@ -67,15 +68,16 @@ func (f *commandFunnel) step(ctx context.Context, s protos.AitoolsCommandStep) {
}

// finish logs the FINISHED event for a run that returned err: SUCCESS for nil,
// NOTHING_TO_DO when update or uninstall found nothing installed (the command
// still exits non-zero), CANCELLED when the user aborted a prompt or picked no
// agents, FAILURE otherwise. No-op once the run has finished, so it is safe to
// defer after an explicit finishWith.
// NOTHING_TO_DO when update or uninstall found nothing installed, or an install
// found no skill in the release it can install (the command still exits
// non-zero), CANCELLED when the user aborted a prompt or picked no agents,
// FAILURE otherwise. No-op once the run has finished, so it is safe to defer
// after an explicit finishWith.
func (f *commandFunnel) finish(ctx context.Context, err error) {
switch {
case err == nil:
f.finishWith(ctx, protos.AitoolsCommandOutcomeSuccess, protos.AitoolsErrorCategoryUnspecified)
case errors.Is(err, installer.ErrNothingInstalled):
case errors.Is(err, installer.ErrNothingInstalled), errors.Is(err, installer.ErrNoSkillsResolved):

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[nit] When a plugin installs, no skill resolves, and recordPluginInstallsFn fails (e.g. EPERM on the state file), executePlan returns errors.Join(skillsErr, err) (install.go:719). This case matches the sentinel inside the joined error and records NOTHING_TO_DO with no category. installRecovery also matches it first and prints "No skill in this release can be installed... add --experimental" instead of the permission step, while the JSON error_category says PERMISSION_DENIED. Suggest returning the record error without joining ErrNoSkillsResolved once a plugin was installed, or matching the sentinel only when it is the sole error.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in 0e1a08c. When the skills step returned ErrNoSkillsResolved and saving the plugin records fails, executePlan now returns only the record error, so the category and recovery step are the permission ones. The same test covers it.

f.finishWith(ctx, protos.AitoolsCommandOutcomeNothingToDo, protos.AitoolsErrorCategoryUnspecified)
case errors.Is(err, huh.ErrUserAborted):
f.finishWith(ctx, protos.AitoolsCommandOutcomeCancelled, protos.AitoolsErrorCategoryUserAborted)
Expand All @@ -87,8 +89,9 @@ func (f *commandFunnel) finish(ctx context.Context, err error) {
}

// finishWith logs the FINISHED event with an explicit outcome, for runs whose
// returned error does not tell it: a nil return without success (declined
// confirm, nothing to act on, nothing installed) or a named agent's failure.
// returned error does not tell it: a nil return without success (a declined
// confirm, nothing to act on), a failure already reported per agent, or a run
// with no agent detected, which fails but had nothing to act on.
func (f *commandFunnel) finishWith(ctx context.Context, outcome protos.AitoolsCommandOutcome, category protos.AitoolsErrorCategory) {
if f.finished {
return
Expand Down Expand Up @@ -175,25 +178,9 @@ func deliverySteps(outcomes []agentOutcome) []protos.AitoolsCommandStep {
return steps
}

// nothingInstalledCategory reports whether a run that returned no error still
// installed nothing for any agent (e.g. the only detected agent's plugin
// install failed, which is a warning, not an error), and if so the category of
// the first agent that failed or was skipped. The funnel records such a run as
// FAILURE rather than SUCCESS. That includes a non-interactive run whose
// detected agents are all skipped for the scope (UNSUPPORTED_SCOPE): it set
// up nothing it was asked to. An interactive run never gets here for them,
// because the picker offers no skipped agent and the run finishes NOTHING_TO_DO
// before the plan.
func nothingInstalledCategory(outcomes []agentOutcome) (protos.AitoolsErrorCategory, bool) {
if len(outcomes) == 0 {
return "", false
}
for _, o := range outcomes {
if o.status == outcomeInstalled {
return "", false
}
}
return firstFailureCategory(outcomes), true
// anyInstalled reports whether at least one agent was set up.
func anyInstalled(outcomes []agentOutcome) bool {
return slices.ContainsFunc(outcomes, func(o agentOutcome) bool { return o.status == outcomeInstalled })
}

// firstFailureCategory returns the category of the first agent that failed or
Expand Down
94 changes: 73 additions & 21 deletions cmd/aitools/funnel_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"maps"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
Expand Down Expand Up @@ -187,7 +188,7 @@ func TestInstallFunnelNoAgentsDetected(t *testing.T) {
forbidSkillsInstall(t)

ctx, err := runInstall(t, cmdio.MockDiscard(t.Context()))
require.NoError(t, err)
require.ErrorIs(t, err, errNoAgentSetUp)
assert.Equal(t, []string{
"STARTED", "FLAGS_VALIDATED", "SCOPE_RESOLVED", "FINISHED:NOTHING_TO_DO:",
}, funnelShape(funnelEvents(ctx)))
Expand Down Expand Up @@ -458,8 +459,10 @@ func TestInstallFunnelPathDumpWithNoSkillToWrite(t *testing.T) {
`{"version": "2", "skills": {"databricks-preview": {"version": "1.0.0", "files": ["SKILL.md"], "repo_dir": "experimental"}}}`,
nil))

// Nothing was written, so the run exits 1, but the funnel records it as
// NOTHING_TO_DO: no skill in the release could be written.
ctx, err := runInstall(t, cmdio.MockDiscard(t.Context()), "--path", t.TempDir())
require.NoError(t, err)
require.ErrorIs(t, err, installer.ErrNoSkillsResolved)
assert.Equal(t, []string{"STARTED", "FLAGS_VALIDATED", "FINISHED:NOTHING_TO_DO:"}, funnelShape(funnelEvents(ctx)))
}

Expand Down Expand Up @@ -567,8 +570,8 @@ func TestListFunnelFailure(t *testing.T) {
}
}

// A plugin failure for an auto-detected agent is only a warning (exit 0), but
// when nothing got installed the funnel must not report SUCCESS.
// A plugin failure for an auto-detected agent is only a warning, but when
// nothing got installed the run fails and the funnel must not report SUCCESS.
func TestInstallFunnelDetectedPluginFailureIsNotSuccess(t *testing.T) {
isolateAgents(t)
tmp := t.TempDir()
Expand All @@ -583,28 +586,27 @@ func TestInstallFunnelDetectedPluginFailureIsNotSuccess(t *testing.T) {
return installer.PluginRecord{}, &installer.BlockedError{Agent: a.Name, Reason: installer.ReasonInstallFailed}
}

// The run installed nothing, so it exits non-zero; the FINISHED event takes
// the category from the agent's outcome.
ctx, err := runInstall(t, cmdio.MockDiscard(t.Context()), "--scope", "global")
require.NoError(t, err)
require.ErrorIs(t, err, errNoAgentSetUp)
assert.Equal(t, []string{
"STARTED", "FLAGS_VALIDATED", "SCOPE_RESOLVED", "AGENTS_SELECTED", "PLAN_CONFIRMED",
"FINISHED:FAILURE:PLUGIN_INSTALL_FAILED",
}, funnelShape(funnelEvents(ctx)))
}

func TestNothingInstalledCategory(t *testing.T) {
func TestAnyInstalledAndFirstFailureCategory(t *testing.T) {
codex := &agents.Agent{Name: agents.NameCodex}
cursor := &agents.Agent{Name: agents.NameCursor}
failed := agentOutcome{agent: codex, delivery: deliveryPlugin, status: outcomeFailed, errorCategory: protos.AitoolsErrorCategoryPluginInstallFailed}
skipped := agentOutcome{agent: cursor, delivery: deliverySkip, status: outcomeSkipped, errorCategory: protos.AitoolsErrorCategoryUnsupportedScope}
installed := agentOutcome{agent: cursor, delivery: deliverySkills, status: outcomeInstalled}

_, ok := nothingInstalledCategory(nil)
assert.False(t, ok)
_, ok = nothingInstalledCategory([]agentOutcome{failed, installed})
assert.False(t, ok, "one installed agent makes the run a success")
got, ok := nothingInstalledCategory([]agentOutcome{failed, skipped})
assert.True(t, ok)
assert.Equal(t, protos.AitoolsErrorCategoryPluginInstallFailed, got)
assert.False(t, anyInstalled(nil))
assert.True(t, anyInstalled([]agentOutcome{failed, installed}), "one installed agent makes the run a success")
assert.False(t, anyInstalled([]agentOutcome{failed, skipped}))
assert.Equal(t, protos.AitoolsErrorCategoryPluginInstallFailed, firstFailureCategory([]agentOutcome{failed, skipped}))

uncategorized := agentOutcome{agent: codex, delivery: deliveryPlugin, status: outcomeFailed}
assert.Equal(t, protos.AitoolsErrorCategoryUnsupportedScope, firstFailureCategory([]agentOutcome{installed, uncategorized, skipped}))
Expand Down Expand Up @@ -785,8 +787,8 @@ func saveGlobalState(t *testing.T, skills map[string]string) string {
return dir
}

// Plugin failures are warnings (exit 0), but an update in which no plugin got
// updated and nothing else ran did nothing and must not report SUCCESS.
// Plugin failures are warnings, but an update in which no plugin got updated
// and nothing else ran did nothing: it exits 1 and reports FAILURE.
func TestUpdateFunnelWhenPluginsFail(t *testing.T) {
codexFailed := installer.PluginUpdate{Agent: "Codex CLI", Err: errCodexNotOnPath}
cases := []struct {
Expand All @@ -799,7 +801,12 @@ func TestUpdateFunnelWhenPluginsFail(t *testing.T) {
refErr error
checkErr error
want string
// exit0 marks a FAILURE that still exits 0.
exit0 bool
}{
{name: "only an unknown agent's record", results: []installer.PluginUpdate{{Agent: "x", Err: fmt.Errorf("%w %q in state", installer.ErrUnknownAgent, "x")}}, want: "FINISHED:FAILURE:UNKNOWN_AGENT", exit0: true},
// The unknown agent sorts first, but the failed attempt decides the run.
{name: "unknown agent's record and a failed plugin", results: []installer.PluginUpdate{{Agent: "aider", Err: fmt.Errorf("%w %q in state", installer.ErrUnknownAgent, "aider")}, codexFailed}, want: "FINISHED:FAILURE:CLI_NOT_ON_PATH"},
{name: "all plugins failed", results: []installer.PluginUpdate{codexFailed}, want: "FINISHED:FAILURE:CLI_NOT_ON_PATH"},
{name: "one of two plugins failed", results: []installer.PluginUpdate{{Agent: "Claude Code", Version: "0.2.7"}, codexFailed}, want: "FINISHED:SUCCESS:"},
{name: "plugin failed, file skills reconciled", skills: map[string]string{"databricks": "0.2.6"}, results: []installer.PluginUpdate{codexFailed}, want: "FINISHED:SUCCESS:"},
Expand Down Expand Up @@ -851,12 +858,25 @@ func TestUpdateFunnelWhenPluginsFail(t *testing.T) {
ctx := telemetry.WithNewLogger(cmdio.MockDiscard(t.Context()))
cmd := NewUpdateCmd()
cmd.SetContext(ctx)
cmd.SilenceErrors = true
cmd.SilenceUsage = true
args := []string{"--scope=global"}
if tc.check {
args = append(args, "--check")
}
cmd.SetArgs(args)
require.NoError(t, cmd.Execute())
err := cmd.Execute()

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[nit] These tests (here and at line 773) now expect cmd.Execute() to fail. NewUpdateCmd() and NewUninstallCmd() run without SilenceErrors/SilenceUsage, so cobra prints an Error: line and the full usage block for each failing case. Running go test -v -run 'TestUpdateFunnelWhenPluginsFail|TestUninstallFunnelWhenRemovalsFail' prints six usage blocks. Suggest setting both on the command, as newTestInstallCmd does.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in bed3b34. Both tests now set SilenceErrors and SilenceUsage on the update and uninstall commands, as newTestInstallCmd does, so no usage blocks are printed.

// A run where every attempt failed exits 1.
if strings.HasPrefix(tc.want, "FINISHED:FAILURE:") && !tc.exit0 {
require.Error(t, err)
prefix := "nothing was updated: "
if tc.check {
prefix = "nothing was checked: "
}
assert.True(t, strings.HasPrefix(err.Error(), prefix), err.Error())
} else {
require.NoError(t, err)
}
events := funnelEvents(ctx)
assert.Equal(t, []string{"STARTED", tc.want}, funnelShape(events))
assert.Equal(t, protos.AitoolsInstallScopeGlobal, events[len(events)-1].Scope)
Expand All @@ -867,8 +887,8 @@ func TestUpdateFunnelWhenPluginsFail(t *testing.T) {
}
}

// Plugin and skill removal failures are warnings (exit 0), but an uninstall
// that removed nothing because every removal failed must not report SUCCESS.
// Plugin and skill removal failures are warnings, but an uninstall that removed
// nothing because every removal failed exits 1 and reports FAILURE.
func TestUninstallFunnelWhenRemovalsFail(t *testing.T) {
skillFailed := fmt.Errorf("remove skill: %w", fs.ErrPermission)
cases := []struct {
Expand All @@ -877,12 +897,16 @@ func TestUninstallFunnelWhenRemovalsFail(t *testing.T) {
result installer.UninstallResult
want string
wantAgents []protos.AitoolsAgentType
// exit0 marks a FAILURE that still exits 0.
exit0 bool
}{
{name: "only an unknown agent's record", result: installer.UninstallResult{PluginFailures: []error{fmt.Errorf("%w %q in state", installer.ErrUnknownAgent, "x")}}, want: "FINISHED:FAILURE:UNKNOWN_AGENT", exit0: true},
{name: "all plugins failed", result: installer.UninstallResult{PluginFailures: []error{errCodexNotOnPath}}, want: "FINISHED:FAILURE:CLI_NOT_ON_PATH"},
{name: "nothing to remove for the targeted agent", args: []string{"--agents=cursor"}, want: "FINISHED:NOTHING_TO_DO:", wantAgents: []protos.AitoolsAgentType{protos.AitoolsAgentTypeCursor}},
{name: "skill removed for the targeted agent", args: []string{"--agents=cursor"}, result: installer.UninstallResult{SkillsRemoved: 1}, want: "FINISHED:SUCCESS:", wantAgents: []protos.AitoolsAgentType{protos.AitoolsAgentTypeCursor}},
{name: "skill removed for the targeted agent", args: []string{"--agents=cursor"}, result: installer.UninstallResult{SkillsChanged: 1}, want: "FINISHED:SUCCESS:", wantAgents: []protos.AitoolsAgentType{protos.AitoolsAgentTypeCursor}},
{name: "one of two plugins failed", result: installer.UninstallResult{PluginsRemoved: 1, PluginFailures: []error{errCodexNotOnPath}}, want: "FINISHED:SUCCESS:"},
{name: "plugin failed, skills removed", result: installer.UninstallResult{SkillsRemoved: 1, PluginFailures: []error{errCodexNotOnPath}}, want: "FINISHED:SUCCESS:"},
{name: "plugin failed, skills removed", result: installer.UninstallResult{SkillsChanged: 1, PluginFailures: []error{errCodexNotOnPath}}, want: "FINISHED:SUCCESS:"},
{name: "a targeted agent kept its entry, another's copy removed", args: []string{"--agents=claude-code,cursor"}, result: installer.UninstallResult{SkillsChanged: 1, SkillFailures: []error{skillFailed}}, want: "FINISHED:SUCCESS:", wantAgents: []protos.AitoolsAgentType{protos.AitoolsAgentTypeClaudeCode, protos.AitoolsAgentTypeCursor}},
{name: "all skill removals failed", result: installer.UninstallResult{SkillFailures: []error{skillFailed}}, want: "FINISHED:FAILURE:PERMISSION_DENIED"},
{name: "skill removal failed, plugin removed", result: installer.UninstallResult{PluginsRemoved: 1, SkillFailures: []error{skillFailed}}, want: "FINISHED:SUCCESS:"},
}
Expand All @@ -899,8 +923,17 @@ func TestUninstallFunnelWhenRemovalsFail(t *testing.T) {
ctx := telemetry.WithNewLogger(cmdio.MockDiscard(t.Context()))
cmd := NewUninstallCmd()
cmd.SetContext(ctx)
cmd.SilenceErrors = true
cmd.SilenceUsage = true
cmd.SetArgs(append([]string{"--scope=global"}, tc.args...))
require.NoError(t, cmd.Execute())
err := cmd.Execute()
// A run where every attempted removal failed exits 1.
if strings.HasPrefix(tc.want, "FINISHED:FAILURE:") && !tc.exit0 {
require.Error(t, err)
assert.True(t, strings.HasPrefix(err.Error(), "could not remove any skill or plugin: "), err.Error())
} else {
require.NoError(t, err)
}
events := funnelEvents(ctx)
assert.Equal(t, []string{"STARTED", tc.want}, funnelShape(events))
finished := events[len(events)-1]
Expand Down Expand Up @@ -929,3 +962,22 @@ func TestUninstallConfirmPromptFailure(t *testing.T) {
assert.Empty(t, *calls)
assert.Equal(t, []string{"STARTED", "FINISHED:FAILURE:PROMPT_FAILED"}, funnelShape(funnelEvents(ctx)))
}

func TestInstallFunnelNothingInstalledKeepsTheAgentCategory(t *testing.T) {
// Only Cursor is detected and it has no project-scoped skills, so the run
// sets up nothing. The FINISHED event must name why, not UNCATEGORIZED.
isolateAgents(t)
tmp := t.TempDir()
t.Setenv("HOME", tmp)
t.Setenv("USERPROFILE", tmp)
t.Setenv("XDG_CONFIG_HOME", filepath.Join(tmp, ".config"))
require.NoError(t, os.MkdirAll(filepath.Join(tmp, ".cursor"), 0o755))
setupInstallMock(t)

ctx, err := runInstall(t, cmdio.MockDiscard(t.Context()), "--skills-only", "--scope", "project")
require.ErrorIs(t, err, errNoAgentSetUp)
assert.Equal(t, []string{
"STARTED", "FLAGS_VALIDATED", "SCOPE_RESOLVED", "AGENTS_SELECTED", "PLAN_CONFIRMED",
"FINISHED:FAILURE:UNSUPPORTED_SCOPE",
}, funnelShape(funnelEvents(ctx)))
}
Loading
Loading