From 2423089167da2a9e1b1d8f6dd0ec69062d28658b Mon Sep 17 00:00:00 2001 From: Artemiy Kolesnikov Date: Fri, 13 Nov 2026 04:41:39 +0000 Subject: [PATCH 1/3] aitools: Hint the AI Runtime skill only on air subcommands The Claude Code install hint fired on every non-aitools command with a generic line ("Databricks skills are not installed..."). Coding agents treat it as unrelated noise on commands like `auth profiles`, and it never named the skill that helps with the task at hand. MaybeHint now fires only on runnable `databricks air` subcommands and names the databricks-ai-runtime skill, what it covers, the install command, and a no-install preview (`aitools install --path`). Detection looks for that skill by exact name in the home stores and the project store (cwd up to the repository root), and counts the databricks plugin only when it applies to cwd. Bare `air` and `air --help` render help without the pre-run hook, so the air help footer now carries the same benefit line and the preview command. Telemetry keeps INSTALL_TEXT and the per-session throttle is unchanged. Co-authored-by: Isaac --- .nextchanges/cli/air-skill-hint.md | 1 + acceptance/air/help/output.txt | 4 + .../cmd/aitools/claude-config-dir/output.txt | 18 +- .../cmd/aitools/claude-config-dir/script | 10 +- .../telemetry/apps-init-tip/output.txt | 2 - .../cmd/aitools/telemetry/hint/output.txt | 38 ++- acceptance/cmd/aitools/telemetry/hint/script | 19 +- acceptance/cmd/version/output.txt | 3 +- acceptance/cmd/version/script | 5 +- cmd/air/air.go | 4 + cmd/air/air_test.go | 5 + cmd/air/hint_test.go | 89 ++++++ libs/aitools/agents/calling.go | 5 +- libs/aitools/agents/hint.go | 114 ++++--- libs/aitools/agents/hint_test.go | 299 +++++++++++++----- libs/aitools/agents/recommend.go | 4 +- 16 files changed, 452 insertions(+), 168 deletions(-) create mode 100644 .nextchanges/cli/air-skill-hint.md create mode 100644 cmd/air/hint_test.go diff --git a/.nextchanges/cli/air-skill-hint.md b/.nextchanges/cli/air-skill-hint.md new file mode 100644 index 00000000000..65eeb49e63b --- /dev/null +++ b/.nextchanges/cli/air-skill-hint.md @@ -0,0 +1 @@ +* Claude Code gets the Databricks skills install hint only on `databricks air` subcommands, where it names the `databricks-ai-runtime` skill, what it covers, and how to preview it with `databricks aitools install --path`; other commands no longer print the hint, and `databricks air --help` lists the preview too. diff --git a/acceptance/air/help/output.txt b/acceptance/air/help/output.txt index 7b37a4c83ac..f782fa5cf89 100644 --- a/acceptance/air/help/output.txt +++ b/acceptance/air/help/output.txt @@ -43,6 +43,10 @@ Learn more: Quickstart (AWS) https://docs.databricks.com/aws/en/machine-learning/ai-runtime/quickstart Quickstart (Azure) https://learn.microsoft.com/azure/databricks/machine-learning/ai-runtime/quickstart Coding agent skill databricks aitools install --skills-only --skills databricks-ai-runtime + Teaches coding agents the workload YAML, GPU types, secrets, + and how to follow and debug runs. + Preview the skill databricks aitools install --path --skills databricks-ai-runtime + Writes the skill files to without installing anything. === list help diff --git a/acceptance/cmd/aitools/claude-config-dir/output.txt b/acceptance/cmd/aitools/claude-config-dir/output.txt index 508651018c5..32646afffb2 100644 --- a/acceptance/cmd/aitools/claude-config-dir/output.txt +++ b/acceptance/cmd/aitools/claude-config-dir/output.txt @@ -1,21 +1,21 @@ === Plugin installed only in the custom config directory ->>> CLAUDE_CONFIG_DIR=custom [CLI] version -Databricks CLI v[CLI_VERSION] +>>> CLAUDE_CONFIG_DIR=custom [CLI] air list +No active runs found. Use --all-status to include runs that are no longer active. === Custom directory takes precedence over the default installation ->>> CLAUDE_CONFIG_DIR=missing [CLI] version -Databricks skills are not installed. To work with Databricks reliably, first run: databricks aitools install -Databricks CLI v[CLI_VERSION] +>>> CLAUDE_CONFIG_DIR=missing [CLI] air list +Tip: the databricks-ai-runtime skill is not installed. It documents the air workload YAML, GPU types, secrets, and how to follow and debug runs. Install it: databricks aitools install --skills-only --skills databricks-ai-runtime (to read it first without installing: databricks aitools install --path --skills databricks-ai-runtime) +No active runs found. Use --all-status to include runs that are no longer active. === Empty override uses the default installation ->>> CLAUDE_CONFIG_DIR= [CLI] version -Databricks CLI v[CLI_VERSION] +>>> CLAUDE_CONFIG_DIR= [CLI] air list +No active runs found. Use --all-status to include runs that are no longer active. === Unset override uses the default installation ->>> [CLI] version -Databricks CLI v[CLI_VERSION] +>>> [CLI] air list +No active runs found. Use --all-status to include runs that are no longer active. diff --git a/acceptance/cmd/aitools/claude-config-dir/script b/acceptance/cmd/aitools/claude-config-dir/script index 26f51369813..5ab0a15e0b3 100644 --- a/acceptance/cmd/aitools/claude-config-dir/script +++ b/acceptance/cmd/aitools/claude-config-dir/script @@ -1,18 +1,18 @@ export CLAUDECODE=1 unset CLAUDE_CODE_SESSION_ID -# aitools commands skip the install hint; version exercises it without auth or network. +# Only air subcommands print the install hint; air list against the mock server exercises it. sethome ./empty-home title "Plugin installed only in the custom config directory" -trace CLAUDE_CONFIG_DIR=custom $CLI version +trace CLAUDE_CONFIG_DIR=custom $CLI air list sethome ./installed-home title "Custom directory takes precedence over the default installation" -trace CLAUDE_CONFIG_DIR=missing $CLI version +trace CLAUDE_CONFIG_DIR=missing $CLI air list title "Empty override uses the default installation" -trace CLAUDE_CONFIG_DIR= $CLI version +trace CLAUDE_CONFIG_DIR= $CLI air list title "Unset override uses the default installation" unset CLAUDE_CONFIG_DIR -trace $CLI version +trace $CLI air list diff --git a/acceptance/cmd/aitools/telemetry/apps-init-tip/output.txt b/acceptance/cmd/aitools/telemetry/apps-init-tip/output.txt index 335595f4d75..74c334ebe1c 100644 --- a/acceptance/cmd/aitools/telemetry/apps-init-tip/output.txt +++ b/acceptance/cmd/aitools/telemetry/apps-init-tip/output.txt @@ -2,7 +2,6 @@ === apps init with flags prints the tip: one hint-shown event per hint printed >>> [CLI] apps init --template ./template --skip-install --name app-tip -Databricks skills are not installed. To work with Databricks reliably, first run: databricks aitools install ✔ Creating project ✔ Project created successfully! @@ -13,7 +12,6 @@ Databricks skills are not installed. To work with Databricks reliably, first run Tip: coding agents detected without Databricks skills. Run 'databricks aitools install' to install them. >>> print_telemetry -{"command":"apps_init","exit_code":0,"aitools_hint_event":{"hint_kind":"INSTALL_TEXT"}} {"command":"apps_init","exit_code":0,"aitools_hint_event":{"hint_kind":"APPS_INIT_TIP"}} === plugin installed: no hint, no tip, no event diff --git a/acceptance/cmd/aitools/telemetry/hint/output.txt b/acceptance/cmd/aitools/telemetry/hint/output.txt index 1283eb52109..c3831eaa393 100644 --- a/acceptance/cmd/aitools/telemetry/hint/output.txt +++ b/acceptance/cmd/aitools/telemetry/hint/output.txt @@ -1,22 +1,46 @@ -=== hint printed on an authenticated command: one hint-shown event is uploaded +=== hint printed on an air command: one hint-shown event is uploaded ->>> [CLI] jobs list -Databricks skills are not installed. To work with Databricks reliably, first run: databricks aitools install +>>> [CLI] air list +Tip: the databricks-ai-runtime skill is not installed. It documents the air workload YAML, GPU types, secrets, and how to follow and debug runs. Install it: databricks aitools install --skills-only --skills databricks-ai-runtime (to read it first without installing: databricks aitools install --path --skills databricks-ai-runtime) +No active runs found. Use --all-status to include runs that are no longer active. >>> print_telemetry -{"command":"jobs_list","exit_code":0,"aitools_hint_event":{"hint_kind":"INSTALL_TEXT"}} +{"command":"air_list","exit_code":0,"aitools_hint_event":{"hint_kind":"INSTALL_TEXT"}} + +=== commands outside air print no hint and log no event + +>>> [CLI] jobs list -=== hint printed on a command that resolves no auth: nothing is uploaded +>>> print_telemetry >>> [CLI] version -Databricks skills are not installed. To work with Databricks reliably, first run: databricks aitools install Databricks CLI v[CLI_VERSION] +=== previewing a skill with --path installs nothing, so air still hints + +>>> [CLI] aitools install --path ./preview --skills databricks-test +Using skills version test-ref +Wrote 1 skill to ./preview + >>> print_telemetry +{"command":"aitools_install","exit_code":0,"aitools_command_event":{"command":"INSTALL","stage":"STARTED","interactive":false,"install_mode":"PATH"}} +{"command":"aitools_install","exit_code":0,"aitools_command_event":{"command":"INSTALL","stage":"STEP_COMPLETED","step":"FLAGS_VALIDATED","interactive":false,"install_mode":"PATH"}} +{"command":"aitools_install","exit_code":0,"aitools_command_event":{"command":"INSTALL","stage":"STEP_COMPLETED","step":"SKILLS_DUMPED","interactive":false,"install_mode":"PATH"}} +{"command":"aitools_install","exit_code":0,"aitools_command_event":{"command":"INSTALL","stage":"FINISHED","outcome":"SUCCESS","interactive":false,"install_mode":"PATH"}} +preview/databricks-test/SKILL.md +no aitools state in HOME + +>>> [CLI] air list +Tip: the databricks-ai-runtime skill is not installed. It documents the air workload YAML, GPU types, secrets, and how to follow and debug runs. Install it: databricks aitools install --skills-only --skills databricks-ai-runtime (to read it first without installing: databricks aitools install --path --skills databricks-ai-runtime) +No active runs found. Use --all-status to include runs that are no longer active. + +>>> print_telemetry +{"command":"air_list","exit_code":0,"aitools_hint_event":{"hint_kind":"INSTALL_TEXT"}} === plugin installed: no hint, no event ->>> [CLI] jobs list +>>> [CLI] air list +No active runs found. Use --all-status to include runs that are no longer active. >>> print_telemetry diff --git a/acceptance/cmd/aitools/telemetry/hint/script b/acceptance/cmd/aitools/telemetry/hint/script index 1289a90535e..02c87f03d4a 100644 --- a/acceptance/cmd/aitools/telemetry/hint/script +++ b/acceptance/cmd/aitools/telemetry/hint/script @@ -5,15 +5,26 @@ unset CLAUDE_CODE_SESSION_ID # A host CLAUDE_CONFIG_DIR would move the plugin file away from the isolated HOME. unset CLAUDE_CONFIG_DIR -title "hint printed on an authenticated command: one hint-shown event is uploaded" -trace $CLI jobs list +title "hint printed on an air command: one hint-shown event is uploaded" +trace $CLI air list trace print_telemetry -title "hint printed on a command that resolves no auth: nothing is uploaded" +title "commands outside air print no hint and log no event" +trace $CLI jobs list +trace print_telemetry trace $CLI version + +title "previewing a skill with --path installs nothing, so air still hints" +# The mock manifest serves only databricks-test. +trace $CLI aitools install --path ./preview --skills databricks-test +trace print_telemetry +find preview -type f | sort +rm -rf preview +test -e "$HOME/.databricks/aitools" && echo "unexpected aitools state in HOME" || echo "no aitools state in HOME" +trace $CLI air list trace print_telemetry title "plugin installed: no hint, no event" sethome ./installed-home -trace $CLI jobs list +trace $CLI air list trace print_telemetry diff --git a/acceptance/cmd/version/output.txt b/acceptance/cmd/version/output.txt index 56a4d16ed95..cfb4047c4f5 100644 --- a/acceptance/cmd/version/output.txt +++ b/acceptance/cmd/version/output.txt @@ -35,9 +35,8 @@ Tip: coding agents detected without Databricks skills. Run 'databricks aitools i "development_build": true } -=== version --check from Claude Code prints only the root hint +=== version --check from Claude Code prints no hint >>> [CLI] version --check -Databricks skills are not installed. To work with Databricks reliably, first run: databricks aitools install Databricks CLI v[CLI_VERSION] This is a development build; skipping the update check. diff --git a/acceptance/cmd/version/script b/acceptance/cmd/version/script index 3e72d659289..84067aae808 100644 --- a/acceptance/cmd/version/script +++ b/acceptance/cmd/version/script @@ -23,8 +23,9 @@ trace $CLI version --check title "version --check --output json with a coding agent prints no tip" trace $CLI version --check --output json -# Claude Code callers get only the root hint (agents.MaybeHint), not the tip. -title "version --check from Claude Code prints only the root hint" +# Claude Code callers get no tip: agents.MaybeHint owns their hints and only +# hints on air subcommands. +title "version --check from Claude Code prints no hint" sethome ./claude-home export CLAUDECODE=1 unset CLAUDE_CODE_SESSION_ID diff --git a/cmd/air/air.go b/cmd/air/air.go index 4154bab891c..de1b6ac4ec6 100644 --- a/cmd/air/air.go +++ b/cmd/air/air.go @@ -21,6 +21,10 @@ Learn more: Quickstart (AWS) https://docs.databricks.com/aws/en/machine-learning/ai-runtime/quickstart Quickstart (Azure) https://learn.microsoft.com/azure/databricks/machine-learning/ai-runtime/quickstart Coding agent skill databricks aitools install --skills-only --skills databricks-ai-runtime + Teaches coding agents the workload YAML, GPU types, secrets, + and how to follow and debug runs. + Preview the skill databricks aitools install --path --skills databricks-ai-runtime + Writes the skill files to without installing anything. ` // New returns the root command for the AI Runtime CLI. diff --git a/cmd/air/air_test.go b/cmd/air/air_test.go index 11e4546d21d..b038e3de12c 100644 --- a/cmd/air/air_test.go +++ b/cmd/air/air_test.go @@ -54,6 +54,11 @@ func TestHelpFooterOnlyOnAirCommand(t *testing.T) { cmd.HelpFunc()(cmd, nil) assert.Contains(t, airOut.String(), "Learn more:") assert.Contains(t, airOut.String(), "databricks air run -h config") + // The footer is where `air` and `air --help` recommend the agent skill + // (help never reaches the pre-run hook that prints the install hint). + assert.Contains(t, airOut.String(), "databricks aitools install --skills-only --skills databricks-ai-runtime") + assert.Contains(t, airOut.String(), "Teaches coding agents the workload YAML") + assert.Contains(t, airOut.String(), "databricks aitools install --path --skills databricks-ai-runtime") list := findSubcommand(t, cmd, "list") var listOut bytes.Buffer diff --git a/cmd/air/hint_test.go b/cmd/air/hint_test.go new file mode 100644 index 00000000000..cb14b4e5329 --- /dev/null +++ b/cmd/air/hint_test.go @@ -0,0 +1,89 @@ +package aircmd + +import ( + "bytes" + "testing" + + "github.com/databricks/cli/internal/testutil" + "github.com/databricks/cli/libs/aitools/agents" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/env" + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// airUnderRoot mounts the real air tree under a root whose pre-run hook only +// records that it ran, mirroring where cmd/root calls agents.MaybeHint. +func airUnderRoot(preRunCalled *bool) (*cobra.Command, *cobra.Command) { + root := &cobra.Command{Use: "databricks"} + root.AddGroup(&cobra.Group{ID: "development", Title: "Developer Tools"}) + root.PersistentPreRunE = func(cmd *cobra.Command, args []string) error { + *preRunCalled = true + return nil + } + air := New() + root.AddCommand(air) + return root, air +} + +// TestBareAirAndHelpShowFooterWithoutPreRun pins why the footer, not the +// dynamic hint, is the recommendation surface for `air` and `air --help`. +func TestBareAirAndHelpShowFooterWithoutPreRun(t *testing.T) { + for _, args := range [][]string{{"air"}, {"air", "--help"}} { + t.Run(args[len(args)-1], func(t *testing.T) { + preRunCalled := false + root, _ := airUnderRoot(&preRunCalled) + var out bytes.Buffer + root.SetOut(&out) + root.SetErr(&out) + root.SetArgs(args) + require.NoError(t, root.Execute()) + assert.False(t, preRunCalled, "help must not run the pre-run hook") + assert.Contains(t, out.String(), "Coding agent skill databricks aitools install --skills-only --skills databricks-ai-runtime") + assert.Contains(t, out.String(), "Preview the skill databricks aitools install --path --skills databricks-ai-runtime") + }) + } +} + +// TestRunnableAirCommandsGetSkillHint walks the real air tree, so a new +// subcommand is covered without listing it here. +func TestRunnableAirCommandsGetSkillHint(t *testing.T) { + testutil.SetAgentEnv(t, "CLAUDECODE", "1") + preRunCalled := false + _, air := airUnderRoot(&preRunCalled) + assert.False(t, air.Runnable(), "bare air renders help; the footer covers it") + + var runnable []string + var walk func(c *cobra.Command) + walk = func(c *cobra.Command) { + for _, sub := range c.Commands() { + if sub.Runnable() { + runnable = append(runnable, sub.CommandPath()) + } + walk(sub) + } + } + walk(air) + for _, name := range []string{"run", "get", "list", "logs", "cancel", "convert-to-dabs"} { + assert.Contains(t, runnable, "databricks air "+name) + } + + var visit func(c *cobra.Command) + visit = func(c *cobra.Command) { + for _, sub := range c.Commands() { + if sub.Runnable() { + t.Run(sub.CommandPath(), func(t *testing.T) { + t.Chdir(t.TempDir()) + ctx, stderr := cmdio.NewTestContextWithStderr(t.Context()) + ctx = env.WithUserHomeDir(ctx, t.TempDir()) + ctx = env.Set(ctx, "DATABRICKS_CACHE_DIR", t.TempDir()) + agents.MaybeHint(ctx, sub) + assert.Contains(t, stderr.String(), "the databricks-ai-runtime skill is not installed") + }) + } + visit(sub) + } + } + visit(air) +} diff --git a/libs/aitools/agents/calling.go b/libs/aitools/agents/calling.go index 6bb35c3e14b..0cff90deb85 100644 --- a/libs/aitools/agents/calling.go +++ b/libs/aitools/agents/calling.go @@ -62,8 +62,9 @@ func (t Tooling) String() string { // record. For Codex and Copilot, pluginRecorded (the CLI install state) is the // only plugin source, so it misses plugins installed with the agent's own CLI // and includes plugins later removed with it. The shared skills store is not consulted: it cannot attribute skills. -// The install hint's claudeToolingInstalled differs on purpose: it stays quiet -// for any trace of an install, while this reports what the agent loads. +// The install hint's airSkillAvailable differs on purpose: it looks for the AI +// Runtime skill only, in the shared store too, while this reports what the +// agent loads. func (a *Agent) DatabricksTooling(ctx context.Context, pluginRecorded func() bool) Tooling { // An unknown working directory skips the project lookups below. cwd := "" diff --git a/libs/aitools/agents/hint.go b/libs/aitools/agents/hint.go index b37bcd6060c..3c3d6fd3104 100644 --- a/libs/aitools/agents/hint.go +++ b/libs/aitools/agents/hint.go @@ -2,6 +2,8 @@ package agents import ( "context" + "os" + "path/filepath" "time" "github.com/databricks/cli/libs/cache" @@ -15,10 +17,16 @@ import ( ) const ( - // hintText recommends installing the Databricks AI tooling. The wording - // is deliberately chosen to be effective at helping agents; change it only - // after thorough testing. - hintText = "Databricks skills are not installed. To work with Databricks reliably, first run: databricks aitools install" + // airSkillName is the agent skill for `databricks air`. + airSkillName = "databricks-ai-runtime" + + // airHintText recommends the AI Runtime skill on the commands it helps with. + // It names the skill and what it buys the agent, so a coding agent can tell + // the hint is about the task at hand rather than generic setup, and offers a + // --path preview so the agent can read the skill without installing anything. + airHintText = "Tip: the " + airSkillName + " skill is not installed. It documents the air workload YAML, GPU types, secrets, and how to follow and debug runs. " + + "Install it: databricks aitools install --skills-only --skills " + airSkillName + + " (to read it first without installing: databricks aitools install --path --skills " + airSkillName + ")" // claudeAgentProvider is the useragent.AgentProvider() value for Claude Code. // The SDK maps the CLAUDECODE env var to this product name. The hint is @@ -26,10 +34,10 @@ const ( // https://github.com/databricks/databricks-sdk-go/blob/main/useragent/agent.go claudeAgentProvider = "claude-code" - // aiToolsCommandName is the top-level command whose subtree is never hinted: - // telling the agent to run `databricks aitools install` while it is already - // running an aitools command makes no sense. - aiToolsCommandName = "aitools" + // airCommandName is the top-level command whose runnable subcommands get the + // hint. Bare `databricks air` and `--help` render help without running the + // pre-run hook; the air help footer carries the same recommendation there. + airCommandName = "air" // claudeSessionEnv is Claude Code's per-session identifier. It keys the // per-session throttle (at most one hint per hintSessionTTL per session); @@ -46,18 +54,20 @@ const ( // fire again for the same session. const hintSessionTTL = time.Hour -// MaybeHint prints a one-line recommendation to install the Databricks AI -// tooling when Claude Code drives the CLI without it installed. It is a no-op -// for human callers, other agents, aitools commands themselves, when there is -// no output stream, once the tooling is present, and when the session was -// hinted within the last hintSessionTTL. Best-effort: it never fails the -// command. Besides the stderr line, it logs a telemetry event for each hint -// shown, which makes a command that resolved auth upload telemetry at exit. +// MaybeHint prints a one-line recommendation to install the AI Runtime skill +// when Claude Code runs a `databricks air` subcommand without the skill +// available. Other commands are never hinted: an agent running them has no AI +// Runtime task to apply the skill to, so the line is noise it learns to skip. +// It is also a no-op for human callers, other agents, when there is no output +// stream, and when the session was hinted within the last hintSessionTTL. +// Best-effort: it never fails the command. Besides the stderr line, it logs a +// telemetry event for each hint shown, which makes a command that resolved +// auth upload telemetry at exit. func MaybeHint(ctx context.Context, cmd *cobra.Command) { if useragent.AgentProvider() != claudeAgentProvider { return } - if isAIToolsCommand(cmd) { + if !isAIRSubcommand(cmd) { return } if !cmdio.HasIO(ctx) { @@ -70,50 +80,62 @@ func MaybeHint(ctx context.Context, cmd *cobra.Command) { if alreadyHintedThisSession(ctx) { return } - if claudeToolingInstalled(ctx) { + if airSkillAvailable(ctx) { return } - cmdio.LogString(ctx, hintText) + cmdio.LogString(ctx, airHintText) logHintShown(ctx, protos.AitoolsHintKindInstallText) recordSessionHint(ctx) } -// isAIToolsCommand reports whether cmd is the aitools command or one of its -// subcommands, walking the parent chain (mirrors versioncheck.isExemptCommand). -func isAIToolsCommand(cmd *cobra.Command) bool { - for c := cmd; c != nil; c = c.Parent() { - if c.Name() == aiToolsCommandName { - return true +// isAIRSubcommand reports whether cmd sits under the top-level air command +// (`databricks air list`, `databricks air pools list`, ...), excluding air +// itself. Matching on the top-level ancestor keeps hidden aliases such as +// `databricks experimental air` out. +func isAIRSubcommand(cmd *cobra.Command) bool { + for c := cmd; c.HasParent(); c = c.Parent() { + if !c.Parent().HasParent() { + return c != cmd && c.Name() == airCommandName } } return false } -// claudeToolingInstalled reports whether Databricks AI tooling is present for -// Claude Code. It checks two signals: the databricks plugin in Claude's own -// plugin manifest (written both by `databricks aitools install` and by a direct -// `claude plugin install`), and databricks skill files in the shared canonical -// skills dir (written by `aitools install --skills-only`). Any read/parse -// failure reports false, so an unreadable state errs toward hinting. -// (*Agent).DatabricksTooling, which feeds the User-Agent, differs on purpose: -// it ignores the shared store and out-of-project records, and counts -// .claude/skills dirs, to report what the calling agent loads. -func claudeToolingInstalled(ctx context.Context) bool { - claude := ByName(NameClaudeCode) - if claude == nil { - // claude-code is always in the registry; this guard keeps the hint - // best-effort rather than panicking in a persistent pre-run hook. - return false +// airSkillAvailable reports whether Claude Code has the AI Runtime skill in the +// current working directory: the databricks plugin (which bundles every +// Databricks skill) installed for every project or for this one, or a +// databricks-ai-runtime skill dir in the shared canonical or legacy store or in +// the project store of cwd or a parent up to the repository root. Other +// Databricks skills do not count: they don't cover air. Any read/parse failure +// reports false, so an unreadable state errs toward hinting. +func airSkillAvailable(ctx context.Context) bool { + cwd := "" + if wd, err := os.Getwd(); err == nil { + cwd = resolvePath(wd) } - if version, ok := claude.DatabricksPluginVersion(ctx); ok { - log.Debugf(ctx, "Databricks plugin detected for Claude Code (version %q); skipping install hint", version) - return true + if claude := ByName(NameClaudeCode); claude != nil && claude.pluginInstalls != nil { + if active, _ := claude.DatabricksPluginActiveIn(ctx, cwd); active { + log.Debugf(ctx, "Databricks plugin active for Claude Code in %q; skipping AI Runtime skill hint", cwd) + return true + } } - // The canonical dir always holds a real databricks* subdir after a skills - // install, regardless of whether the agent copy is a symlink or a copy. The - // legacy dir is checked too so users of an older install aren't hinted. - return hasSharedSkills(ctx) + var dirs []string + if homeDir, err := env.UserHomeDir(ctx); err == nil { + dirs = append(dirs, filepath.Join(homeDir, CanonicalSkillsDir), filepath.Join(homeDir, LegacySkillsDir)) + } + if cwd != "" { + for _, dir := range projectDirs(cwd) { + dirs = append(dirs, filepath.Join(dir, CanonicalSkillsDir)) + } + } + for _, dir := range dirs { + // Stat follows the symlinks the installer may leave in place of copies. + if info, err := os.Stat(filepath.Join(dir, airSkillName)); err == nil && info.IsDir() { + return true + } + } + return false } // alreadyHintedThisSession reports whether this Claude session was hinted diff --git a/libs/aitools/agents/hint_test.go b/libs/aitools/agents/hint_test.go index 3add0417795..3f77fdc0294 100644 --- a/libs/aitools/agents/hint_test.go +++ b/libs/aitools/agents/hint_test.go @@ -6,6 +6,7 @@ import ( "log/slog" "os" "path/filepath" + "strings" "testing" "github.com/databricks/cli/internal/testutil" @@ -26,17 +27,32 @@ func setClaudeAgent(t *testing.T) { testutil.SetAgentEnv(t, "CLAUDECODE", "1") } -// claudeCtx returns a context with a captured stderr buffer and an isolated -// HOME + cache dir, so detection and the session throttle read only the test's -// temp state. home is returned so the test can drop fixtures under it. +// claudeCtx returns a context with a captured, non-TTY stderr buffer (what a +// coding agent's shell tool gives the CLI) and an isolated HOME, cache dir and +// working directory, so detection and the session throttle read only the +// test's temp state. home is returned so the test can drop fixtures under it. func claudeCtx(t *testing.T, home string) (context.Context, *bytes.Buffer) { + t.Chdir(t.TempDir()) ctx, stderr := cmdio.NewTestContextWithStderr(t.Context()) ctx = env.WithUserHomeDir(ctx, home) ctx = env.Set(ctx, "DATABRICKS_CACHE_DIR", t.TempDir()) - return ctx, stderr + return telemetry.WithNewLogger(ctx), stderr } -func regularCmd() *cobra.Command { return &cobra.Command{Use: "current-user"} } +// commandAt builds the command tree for a space-separated path under a +// "databricks" root and returns the leaf, so ancestry checks see the same +// shape as the real CLI. +func commandAt(path string) *cobra.Command { + cmd := &cobra.Command{Use: "databricks"} + for name := range strings.FieldsSeq(path) { + child := &cobra.Command{Use: name} + cmd.AddCommand(child) + cmd = child + } + return cmd +} + +func airCmd() *cobra.Command { return commandAt("air list") } func writePluginManifest(t *testing.T, home, body string) { dir := filepath.Join(home, ".claude", "plugins") @@ -59,41 +75,88 @@ func hintKinds(t *testing.T, ctx context.Context) []protos.AitoolsHintKind { return out } +// assertHinted checks for exactly one hint line and one hint-shown event of +// kind INSTALL_TEXT (T8). +func assertHinted(t *testing.T, ctx context.Context, stderr *bytes.Buffer) { + t.Helper() + assert.Equal(t, airHintText+"\n", stderr.String()) + assert.Equal(t, []protos.AitoolsHintKind{protos.AitoolsHintKindInstallText}, hintKinds(t, ctx)) +} + +// assertSilent checks for no hint line and no telemetry event (T8). +func assertSilent(t *testing.T, ctx context.Context, stderr *bytes.Buffer) { + t.Helper() + assert.Empty(t, stderr.String()) + assert.Empty(t, hintKinds(t, ctx)) +} + +func TestAIRHintText(t *testing.T) { + // The hint names the skill, says what it is for, and offers both the install + // and a no-install preview, on one line. + assert.Contains(t, airHintText, "databricks-ai-runtime skill is not installed") + assert.Contains(t, airHintText, "workload YAML") + assert.Contains(t, airHintText, "databricks aitools install --skills-only --skills databricks-ai-runtime") + assert.Contains(t, airHintText, "databricks aitools install --path --skills databricks-ai-runtime") + assert.NotContains(t, airHintText, "\n") +} + func TestMaybeHint_HumanCallerSilent(t *testing.T) { // No agent env: AgentProvider() == "", so no hint even with nothing installed. testutil.SetAgentEnv(t) - ctx, stderr := cmdio.NewTestContextWithStderr(t.Context()) - ctx = env.WithUserHomeDir(ctx, t.TempDir()) - ctx = telemetry.WithNewLogger(ctx) - MaybeHint(ctx, regularCmd()) - assert.Empty(t, stderr.String()) - assert.Empty(t, hintKinds(t, ctx)) + ctx, stderr := claudeCtx(t, t.TempDir()) + MaybeHint(ctx, airCmd()) + assertSilent(t, ctx, stderr) } -func TestMaybeHint_ClaudeNotInstalledHints(t *testing.T) { +// T1: every air subcommand, at any depth, gets the skill-specific hint. +func TestMaybeHint_AIRSubcommandsHint(t *testing.T) { setClaudeAgent(t) - ctx, stderr := claudeCtx(t, t.TempDir()) - ctx = telemetry.WithNewLogger(ctx) - MaybeHint(ctx, regularCmd()) - assert.Contains(t, stderr.String(), hintText) - assert.Equal(t, []protos.AitoolsHintKind{protos.AitoolsHintKindInstallText}, hintKinds(t, ctx)) + for _, path := range []string{ + "air run", "air get", "air list", "air logs", "air cancel", + "air convert-to-dabs", "air images", "air images push", "air pools", "air pools list", + } { + t.Run(path, func(t *testing.T) { + ctx, stderr := claudeCtx(t, t.TempDir()) + MaybeHint(ctx, commandAt(path)) + assertHinted(t, ctx, stderr) + }) + } +} + +// T3: commands outside air are never hinted, including the auth and version +// commands the generic hint used to fire on. +func TestMaybeHint_NonAIRCommandsSilent(t *testing.T) { + setClaudeAgent(t) + for _, path := range []string{ + "auth profiles", "current-user me", "clusters list", "jobs list", "bundle deploy", + "workspace list", "version", "aitools install", "experimental aitools install", + // The bare air node only renders help; its footer carries the tip. + "air", + // The hidden alias that points users at `databricks air`. + "experimental air", + } { + t.Run(path, func(t *testing.T) { + ctx, stderr := claudeCtx(t, t.TempDir()) + MaybeHint(ctx, commandAt(path)) + assertSilent(t, ctx, stderr) + }) + } } func TestMaybeHint_PluginInstalledSilent(t *testing.T) { setClaudeAgent(t) home := t.TempDir() ctx, stderr := claudeCtx(t, home) - // The core regression: a plugin install writes only Claude's manifest, no - // skills subdir. Detection must treat it as installed. + // A plugin install writes only Claude's manifest, no skills subdir, and + // bundles every Databricks skill including databricks-ai-runtime. writePluginManifest(t, home, `{"plugins":{"databricks@claude-plugins-official":[{"scope":"user"}]}}`) - ctx = telemetry.WithNewLogger(ctx) - MaybeHint(ctx, regularCmd()) - assert.Empty(t, stderr.String()) - assert.Empty(t, hintKinds(t, ctx)) + MaybeHint(ctx, airCmd()) + assertSilent(t, ctx, stderr) } -func TestClaudeToolingInstalled_LogsPluginVersion(t *testing.T) { +func TestAIRSkillAvailable_LogsPluginDetection(t *testing.T) { + t.Chdir(t.TempDir()) home := t.TempDir() writePluginManifest(t, home, `{"plugins":{"databricks@claude-plugins-official":[{"version":"0.2.6"}]}}`) @@ -103,14 +166,9 @@ func TestClaudeToolingInstalled_LogsPluginVersion(t *testing.T) { ctx := log.NewContext(t.Context(), slog.New(handler)) ctx = env.WithUserHomeDir(ctx, home) - assert.True(t, claudeToolingInstalled(ctx)) - - // The text handler escapes the inner quotes around the version, so assert on - // the stable, unescaped fragments rather than the full quoted message. - out := buf.String() - assert.Contains(t, out, "Databricks plugin detected for Claude Code") - assert.Contains(t, out, "0.2.6") - assert.Contains(t, out, "skipping install hint") + assert.True(t, airSkillAvailable(ctx)) + assert.Contains(t, buf.String(), "Databricks plugin active for Claude Code") + assert.Contains(t, buf.String(), "skipping AI Runtime skill hint") } func TestMaybeHint_CustomClaudeConfigDir(t *testing.T) { @@ -139,58 +197,90 @@ func TestMaybeHint_CustomClaudeConfigDir(t *testing.T) { writePluginManifest(t, home, manifest) } - MaybeHint(ctx, regularCmd()) + MaybeHint(ctx, airCmd()) if tc.wantHint { - assert.Contains(t, stderr.String(), hintText) + assertHinted(t, ctx, stderr) } else { - assert.Empty(t, stderr.String()) + assertSilent(t, ctx, stderr) } }) } } -func TestMaybeHint_SkillsInstalledSilent(t *testing.T) { - setClaudeAgent(t) - home := t.TempDir() - ctx, stderr := claudeCtx(t, home) - writeCanonicalSkill(t, home, "databricks") - - MaybeHint(ctx, regularCmd()) - assert.Empty(t, stderr.String()) -} - -func TestMaybeHint_LegacySkillsInstalledSilent(t *testing.T) { +// T4: only the databricks-ai-runtime skill itself, in a location Claude loads +// for this working directory, silences the hint. +func TestMaybeHint_AIRSkillDetection(t *testing.T) { setClaudeAgent(t) - home := t.TempDir() - ctx, stderr := claudeCtx(t, home) - require.NoError(t, os.MkdirAll(filepath.Join(home, ".databricks", "agent-skills", "databricks"), 0o755)) + for _, tc := range []struct { + name string + setup func(t *testing.T, home, cwd string) + wantHint bool + }{ + {"nothing installed", func(t *testing.T, home, cwd string) {}, true}, + {"canonical store", func(t *testing.T, home, cwd string) { + writeCanonicalSkill(t, home, airSkillName) + }, false}, + {"legacy store", func(t *testing.T, home, cwd string) { + require.NoError(t, os.MkdirAll(filepath.Join(home, LegacySkillsDir, airSkillName), 0o755)) + }, false}, + {"canonical store via symlink", func(t *testing.T, home, cwd string) { + dir := filepath.Join(home, CanonicalSkillsDir) + require.NoError(t, os.MkdirAll(dir, 0o755)) + require.NoError(t, os.Symlink(t.TempDir(), filepath.Join(dir, airSkillName))) + }, false}, + {"project store in cwd", func(t *testing.T, home, cwd string) { + require.NoError(t, os.MkdirAll(filepath.Join(cwd, CanonicalSkillsDir, airSkillName), 0o755)) + }, false}, + {"other databricks skills only", func(t *testing.T, home, cwd string) { + writeCanonicalSkill(t, home, "databricks") + writeCanonicalSkill(t, home, "databricks-jobs") + require.NoError(t, os.MkdirAll(filepath.Join(cwd, CanonicalSkillsDir, "databricks-apps"), 0o755)) + }, true}, + {"a file named like the skill", func(t *testing.T, home, cwd string) { + dir := filepath.Join(home, CanonicalSkillsDir) + require.NoError(t, os.MkdirAll(dir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, airSkillName), nil, 0o600)) + }, true}, + {"plugin for this project", func(t *testing.T, home, cwd string) { + writePluginManifest(t, home, `{"plugins":{"databricks@m":[{"scope":"project","projectPath":"`+cwd+`"}]}}`) + }, false}, + {"plugin for an unrelated project", func(t *testing.T, home, cwd string) { + writePluginManifest(t, home, `{"plugins":{"databricks@m":[{"scope":"project","projectPath":"`+t.TempDir()+`"}]}}`) + }, true}, + } { + t.Run(tc.name, func(t *testing.T) { + home := t.TempDir() + ctx, stderr := claudeCtx(t, home) + cwd, err := os.Getwd() + require.NoError(t, err) + tc.setup(t, home, cwd) - MaybeHint(ctx, regularCmd()) - assert.Empty(t, stderr.String()) + MaybeHint(ctx, airCmd()) + if tc.wantHint { + assertHinted(t, ctx, stderr) + } else { + assertSilent(t, ctx, stderr) + } + }) + } } -func TestMaybeHint_NonDatabricksSkillStillHints(t *testing.T) { +// T4: a project-scoped skill at the repository root still counts when the CLI +// runs in a subdirectory, as Claude Code loads it there. +func TestMaybeHint_AIRSkillInRepositoryRoot(t *testing.T) { setClaudeAgent(t) home := t.TempDir() ctx, stderr := claudeCtx(t, home) - writeCanonicalSkill(t, home, "some-other-skill") - - MaybeHint(ctx, regularCmd()) - assert.Contains(t, stderr.String(), hintText) -} - -func TestMaybeHint_AIToolsCommandSilent(t *testing.T) { - setClaudeAgent(t) - ctx, stderr := claudeCtx(t, t.TempDir()) - ctx = telemetry.WithNewLogger(ctx) - - parent := &cobra.Command{Use: aiToolsCommandName} - child := &cobra.Command{Use: "install"} - parent.AddCommand(child) + root, err := os.Getwd() + require.NoError(t, err) + require.NoError(t, os.Mkdir(filepath.Join(root, ".git"), 0o755)) + require.NoError(t, os.MkdirAll(filepath.Join(root, CanonicalSkillsDir, airSkillName), 0o755)) + sub := filepath.Join(root, "training", "configs") + require.NoError(t, os.MkdirAll(sub, 0o755)) + t.Chdir(sub) - MaybeHint(ctx, child) - assert.Empty(t, stderr.String()) - assert.Empty(t, hintKinds(t, ctx)) + MaybeHint(ctx, airCmd()) + assertSilent(t, ctx, stderr) } func TestMaybeHint_NoIONoPanic(t *testing.T) { @@ -199,30 +289,65 @@ func TestMaybeHint_NoIONoPanic(t *testing.T) { // invocations before PersistentPreRunE runs, so HasIO can be false here. ctx := env.WithUserHomeDir(t.Context(), t.TempDir()) ctx = telemetry.WithNewLogger(ctx) - assert.NotPanics(t, func() { MaybeHint(ctx, regularCmd()) }) + assert.NotPanics(t, func() { MaybeHint(ctx, airCmd()) }) assert.Empty(t, hintKinds(t, ctx)) } -func TestMaybeHint_ThrottledPerSession(t *testing.T) { +// T5 and T6: with a session id the first air command in the session is +// hinted and later ones stay quiet until the marker expires; without one +// (older Claude Code) every air command is hinted. Both cases run with a +// non-TTY stderr, which is how a coding agent's shell tool runs the CLI. +func TestMaybeHint_SessionThrottle(t *testing.T) { + setClaudeAgent(t) + for _, tc := range []struct { + name string + session string + wantSecond bool + }{ + {"session id: first air command only", "session-abc", false}, + {"no session id: every air command", "", true}, + } { + t.Run(tc.name, func(t *testing.T) { + home := t.TempDir() + cacheDir := t.TempDir() + newCtx := func() (context.Context, *bytes.Buffer) { + ctx, stderr := claudeCtx(t, home) + ctx = env.Set(ctx, "DATABRICKS_CACHE_DIR", cacheDir) + if tc.session != "" { + ctx = env.Set(ctx, claudeSessionEnv, tc.session) + } + return ctx, stderr + } + + // A non-air command first must not use up the session's hint. + ctx0, stderr0 := newCtx() + MaybeHint(ctx0, commandAt("auth profiles")) + assertSilent(t, ctx0, stderr0) + + ctx1, stderr1 := newCtx() + MaybeHint(ctx1, commandAt("air list")) + assertHinted(t, ctx1, stderr1) + + ctx2, stderr2 := newCtx() + MaybeHint(ctx2, commandAt("air run")) + if tc.wantSecond { + assertHinted(t, ctx2, stderr2) + } else { + assertSilent(t, ctx2, stderr2) + } + }) + } +} + +func TestMaybeHint_OtherSessionStillHinted(t *testing.T) { setClaudeAgent(t) home := t.TempDir() cacheDir := t.TempDir() - newCtx := func() (context.Context, *bytes.Buffer) { - ctx, stderr := cmdio.NewTestContextWithStderr(t.Context()) - ctx = env.WithUserHomeDir(ctx, home) + for _, session := range []string{"session-a", "session-b"} { + ctx, stderr := claudeCtx(t, home) ctx = env.Set(ctx, "DATABRICKS_CACHE_DIR", cacheDir) - ctx = env.Set(ctx, claudeSessionEnv, "session-abc") - return telemetry.WithNewLogger(ctx), stderr + ctx = env.Set(ctx, claudeSessionEnv, session) + MaybeHint(ctx, airCmd()) + assertHinted(t, ctx, stderr) } - - ctx1, stderr1 := newCtx() - MaybeHint(ctx1, regularCmd()) - assert.Contains(t, stderr1.String(), hintText) - assert.Equal(t, []protos.AitoolsHintKind{protos.AitoolsHintKindInstallText}, hintKinds(t, ctx1)) - - // Same session id, second command: must stay quiet and log nothing. - ctx2, stderr2 := newCtx() - MaybeHint(ctx2, regularCmd()) - assert.Empty(t, stderr2.String()) - assert.Empty(t, hintKinds(t, ctx2)) } diff --git a/libs/aitools/agents/recommend.go b/libs/aitools/agents/recommend.go index 14a4da5b00b..7e34de5b517 100644 --- a/libs/aitools/agents/recommend.go +++ b/libs/aitools/agents/recommend.go @@ -23,8 +23,8 @@ func PrintSkillsTip(ctx context.Context) { // MaybeVersionCheckTip prints skillsTipText after `databricks version --check` // when a coding agent is detected without Databricks tooling. Claude Code -// callers are skipped because MaybeHint already covers them, with its -// per-session throttle. installedPlugins is a parameter because this package +// callers are skipped because MaybeHint owns their hints, and it recommends a +// skill only on the commands that skill helps with. installedPlugins is a parameter because this package // cannot import the installer; callers pass installer.InstalledPluginAgents, // which counts a plugin recorded in the CLI install state with or without a // known version. No telemetry is logged: `databricks version` never resolves From 2f16610aaa3290677302e235b091e2bac9952be4 Mon Sep 17 00:00:00 2001 From: Artemiy Kolesnikov Date: Fri, 13 Nov 2026 23:48:32 +0000 Subject: [PATCH 2/3] aitools: link the air skill hint changelog fragment to the pull request Co-authored-by: Isaac --- .nextchanges/cli/air-skill-hint.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/cli/air-skill-hint.md b/.nextchanges/cli/air-skill-hint.md index 65eeb49e63b..b36abd2362b 100644 --- a/.nextchanges/cli/air-skill-hint.md +++ b/.nextchanges/cli/air-skill-hint.md @@ -1 +1 @@ -* Claude Code gets the Databricks skills install hint only on `databricks air` subcommands, where it names the `databricks-ai-runtime` skill, what it covers, and how to preview it with `databricks aitools install --path`; other commands no longer print the hint, and `databricks air --help` lists the preview too. +* Claude Code gets the Databricks skills install hint only on `databricks air` subcommands, where it names the `databricks-ai-runtime` skill, what it covers, and how to preview it with `databricks aitools install --path`; other commands no longer print the hint, and `databricks air --help` lists the preview too. ([#7622](https://github.com/databricks/cli/pull/7622)) From f38ee96c0111dd6eb74e9feaee29fddb93d8332c Mon Sep 17 00:00:00 2001 From: Artemiy Kolesnikov Date: Sat, 14 Nov 2026 00:09:28 +0000 Subject: [PATCH 3/3] aitools: escape project paths in the hint test plugin manifest Windows paths contain backslashes, so interpolating them into the manifest string produced invalid JSON escapes and the plugin record was silently dropped, hinting when the test expected silence. Marshal the manifest instead. Co-authored-by: Isaac --- libs/aitools/agents/hint_test.go | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/libs/aitools/agents/hint_test.go b/libs/aitools/agents/hint_test.go index 3f77fdc0294..2de2e7d224f 100644 --- a/libs/aitools/agents/hint_test.go +++ b/libs/aitools/agents/hint_test.go @@ -3,6 +3,7 @@ package agents import ( "bytes" "context" + "encoding/json" "log/slog" "os" "path/filepath" @@ -60,6 +61,22 @@ func writePluginManifest(t *testing.T, home, body string) { require.NoError(t, os.WriteFile(filepath.Join(dir, "installed_plugins.json"), []byte(body), 0o600)) } +// writeProjectPluginManifest writes a manifest with one project-scope install +// covering projectPath. Marshaled, not interpolated, so Windows paths with +// backslashes stay valid JSON. +func writeProjectPluginManifest(t *testing.T, home, projectPath string) { + manifest, err := json.Marshal(map[string]any{ + "plugins": map[string]any{ + "databricks@m": []any{map[string]string{ + "scope": "project", + "projectPath": projectPath, + }}, + }, + }) + require.NoError(t, err) + writePluginManifest(t, home, string(manifest)) +} + func writeCanonicalSkill(t *testing.T, home, name string) { require.NoError(t, os.MkdirAll(filepath.Join(home, ".databricks", "aitools", "skills", name), 0o755)) } @@ -242,10 +259,10 @@ func TestMaybeHint_AIRSkillDetection(t *testing.T) { require.NoError(t, os.WriteFile(filepath.Join(dir, airSkillName), nil, 0o600)) }, true}, {"plugin for this project", func(t *testing.T, home, cwd string) { - writePluginManifest(t, home, `{"plugins":{"databricks@m":[{"scope":"project","projectPath":"`+cwd+`"}]}}`) + writeProjectPluginManifest(t, home, cwd) }, false}, {"plugin for an unrelated project", func(t *testing.T, home, cwd string) { - writePluginManifest(t, home, `{"plugins":{"databricks@m":[{"scope":"project","projectPath":"`+t.TempDir()+`"}]}}`) + writeProjectPluginManifest(t, home, t.TempDir()) }, true}, } { t.Run(tc.name, func(t *testing.T) {