diff --git a/.nextchanges/cli/air-skill-hint.md b/.nextchanges/cli/air-skill-hint.md new file mode 100644 index 00000000000..b36abd2362b --- /dev/null +++ b/.nextchanges/cli/air-skill-hint.md @@ -0,0 +1 @@ +* Claude Code gets the Databricks skills install hint only on `databricks air` subcommands, where it names the `databricks-ai-runtime` skill, what it covers, and how to preview it with `databricks aitools install --path`; other commands no longer print the hint, and `databricks air --help` lists the preview too. ([#7622](https://github.com/databricks/cli/pull/7622)) diff --git a/acceptance/air/help/output.txt b/acceptance/air/help/output.txt index 7b37a4c83ac..f782fa5cf89 100644 --- a/acceptance/air/help/output.txt +++ b/acceptance/air/help/output.txt @@ -43,6 +43,10 @@ Learn more: Quickstart (AWS) https://docs.databricks.com/aws/en/machine-learning/ai-runtime/quickstart Quickstart (Azure) https://learn.microsoft.com/azure/databricks/machine-learning/ai-runtime/quickstart Coding agent skill databricks aitools install --skills-only --skills databricks-ai-runtime + Teaches coding agents the workload YAML, GPU types, secrets, + and how to follow and debug runs. + Preview the skill databricks aitools install --path --skills databricks-ai-runtime + Writes the skill files to without installing anything. === list help diff --git a/acceptance/cmd/aitools/claude-config-dir/output.txt b/acceptance/cmd/aitools/claude-config-dir/output.txt index 508651018c5..32646afffb2 100644 --- a/acceptance/cmd/aitools/claude-config-dir/output.txt +++ b/acceptance/cmd/aitools/claude-config-dir/output.txt @@ -1,21 +1,21 @@ === Plugin installed only in the custom config directory ->>> CLAUDE_CONFIG_DIR=custom [CLI] version -Databricks CLI v[CLI_VERSION] +>>> CLAUDE_CONFIG_DIR=custom [CLI] air list +No active runs found. Use --all-status to include runs that are no longer active. === Custom directory takes precedence over the default installation ->>> CLAUDE_CONFIG_DIR=missing [CLI] version -Databricks skills are not installed. To work with Databricks reliably, first run: databricks aitools install -Databricks CLI v[CLI_VERSION] +>>> CLAUDE_CONFIG_DIR=missing [CLI] air list +Tip: the databricks-ai-runtime skill is not installed. It documents the air workload YAML, GPU types, secrets, and how to follow and debug runs. Install it: databricks aitools install --skills-only --skills databricks-ai-runtime (to read it first without installing: databricks aitools install --path --skills databricks-ai-runtime) +No active runs found. Use --all-status to include runs that are no longer active. === Empty override uses the default installation ->>> CLAUDE_CONFIG_DIR= [CLI] version -Databricks CLI v[CLI_VERSION] +>>> CLAUDE_CONFIG_DIR= [CLI] air list +No active runs found. Use --all-status to include runs that are no longer active. === Unset override uses the default installation ->>> [CLI] version -Databricks CLI v[CLI_VERSION] +>>> [CLI] air list +No active runs found. Use --all-status to include runs that are no longer active. diff --git a/acceptance/cmd/aitools/claude-config-dir/script b/acceptance/cmd/aitools/claude-config-dir/script index 26f51369813..5ab0a15e0b3 100644 --- a/acceptance/cmd/aitools/claude-config-dir/script +++ b/acceptance/cmd/aitools/claude-config-dir/script @@ -1,18 +1,18 @@ export CLAUDECODE=1 unset CLAUDE_CODE_SESSION_ID -# aitools commands skip the install hint; version exercises it without auth or network. +# Only air subcommands print the install hint; air list against the mock server exercises it. sethome ./empty-home title "Plugin installed only in the custom config directory" -trace CLAUDE_CONFIG_DIR=custom $CLI version +trace CLAUDE_CONFIG_DIR=custom $CLI air list sethome ./installed-home title "Custom directory takes precedence over the default installation" -trace CLAUDE_CONFIG_DIR=missing $CLI version +trace CLAUDE_CONFIG_DIR=missing $CLI air list title "Empty override uses the default installation" -trace CLAUDE_CONFIG_DIR= $CLI version +trace CLAUDE_CONFIG_DIR= $CLI air list title "Unset override uses the default installation" unset CLAUDE_CONFIG_DIR -trace $CLI version +trace $CLI air list diff --git a/acceptance/cmd/aitools/telemetry/apps-init-tip/output.txt b/acceptance/cmd/aitools/telemetry/apps-init-tip/output.txt index 335595f4d75..74c334ebe1c 100644 --- a/acceptance/cmd/aitools/telemetry/apps-init-tip/output.txt +++ b/acceptance/cmd/aitools/telemetry/apps-init-tip/output.txt @@ -2,7 +2,6 @@ === apps init with flags prints the tip: one hint-shown event per hint printed >>> [CLI] apps init --template ./template --skip-install --name app-tip -Databricks skills are not installed. To work with Databricks reliably, first run: databricks aitools install ✔ Creating project ✔ Project created successfully! @@ -13,7 +12,6 @@ Databricks skills are not installed. To work with Databricks reliably, first run Tip: coding agents detected without Databricks skills. Run 'databricks aitools install' to install them. >>> print_telemetry -{"command":"apps_init","exit_code":0,"aitools_hint_event":{"hint_kind":"INSTALL_TEXT"}} {"command":"apps_init","exit_code":0,"aitools_hint_event":{"hint_kind":"APPS_INIT_TIP"}} === plugin installed: no hint, no tip, no event diff --git a/acceptance/cmd/aitools/telemetry/hint/output.txt b/acceptance/cmd/aitools/telemetry/hint/output.txt index 1283eb52109..c3831eaa393 100644 --- a/acceptance/cmd/aitools/telemetry/hint/output.txt +++ b/acceptance/cmd/aitools/telemetry/hint/output.txt @@ -1,22 +1,46 @@ -=== hint printed on an authenticated command: one hint-shown event is uploaded +=== hint printed on an air command: one hint-shown event is uploaded ->>> [CLI] jobs list -Databricks skills are not installed. To work with Databricks reliably, first run: databricks aitools install +>>> [CLI] air list +Tip: the databricks-ai-runtime skill is not installed. It documents the air workload YAML, GPU types, secrets, and how to follow and debug runs. Install it: databricks aitools install --skills-only --skills databricks-ai-runtime (to read it first without installing: databricks aitools install --path --skills databricks-ai-runtime) +No active runs found. Use --all-status to include runs that are no longer active. >>> print_telemetry -{"command":"jobs_list","exit_code":0,"aitools_hint_event":{"hint_kind":"INSTALL_TEXT"}} +{"command":"air_list","exit_code":0,"aitools_hint_event":{"hint_kind":"INSTALL_TEXT"}} + +=== commands outside air print no hint and log no event + +>>> [CLI] jobs list -=== hint printed on a command that resolves no auth: nothing is uploaded +>>> print_telemetry >>> [CLI] version -Databricks skills are not installed. To work with Databricks reliably, first run: databricks aitools install Databricks CLI v[CLI_VERSION] +=== previewing a skill with --path installs nothing, so air still hints + +>>> [CLI] aitools install --path ./preview --skills databricks-test +Using skills version test-ref +Wrote 1 skill to ./preview + >>> print_telemetry +{"command":"aitools_install","exit_code":0,"aitools_command_event":{"command":"INSTALL","stage":"STARTED","interactive":false,"install_mode":"PATH"}} +{"command":"aitools_install","exit_code":0,"aitools_command_event":{"command":"INSTALL","stage":"STEP_COMPLETED","step":"FLAGS_VALIDATED","interactive":false,"install_mode":"PATH"}} +{"command":"aitools_install","exit_code":0,"aitools_command_event":{"command":"INSTALL","stage":"STEP_COMPLETED","step":"SKILLS_DUMPED","interactive":false,"install_mode":"PATH"}} +{"command":"aitools_install","exit_code":0,"aitools_command_event":{"command":"INSTALL","stage":"FINISHED","outcome":"SUCCESS","interactive":false,"install_mode":"PATH"}} +preview/databricks-test/SKILL.md +no aitools state in HOME + +>>> [CLI] air list +Tip: the databricks-ai-runtime skill is not installed. It documents the air workload YAML, GPU types, secrets, and how to follow and debug runs. Install it: databricks aitools install --skills-only --skills databricks-ai-runtime (to read it first without installing: databricks aitools install --path --skills databricks-ai-runtime) +No active runs found. Use --all-status to include runs that are no longer active. + +>>> print_telemetry +{"command":"air_list","exit_code":0,"aitools_hint_event":{"hint_kind":"INSTALL_TEXT"}} === plugin installed: no hint, no event ->>> [CLI] jobs list +>>> [CLI] air list +No active runs found. Use --all-status to include runs that are no longer active. >>> print_telemetry diff --git a/acceptance/cmd/aitools/telemetry/hint/script b/acceptance/cmd/aitools/telemetry/hint/script index 1289a90535e..02c87f03d4a 100644 --- a/acceptance/cmd/aitools/telemetry/hint/script +++ b/acceptance/cmd/aitools/telemetry/hint/script @@ -5,15 +5,26 @@ unset CLAUDE_CODE_SESSION_ID # A host CLAUDE_CONFIG_DIR would move the plugin file away from the isolated HOME. unset CLAUDE_CONFIG_DIR -title "hint printed on an authenticated command: one hint-shown event is uploaded" -trace $CLI jobs list +title "hint printed on an air command: one hint-shown event is uploaded" +trace $CLI air list trace print_telemetry -title "hint printed on a command that resolves no auth: nothing is uploaded" +title "commands outside air print no hint and log no event" +trace $CLI jobs list +trace print_telemetry trace $CLI version + +title "previewing a skill with --path installs nothing, so air still hints" +# The mock manifest serves only databricks-test. +trace $CLI aitools install --path ./preview --skills databricks-test +trace print_telemetry +find preview -type f | sort +rm -rf preview +test -e "$HOME/.databricks/aitools" && echo "unexpected aitools state in HOME" || echo "no aitools state in HOME" +trace $CLI air list trace print_telemetry title "plugin installed: no hint, no event" sethome ./installed-home -trace $CLI jobs list +trace $CLI air list trace print_telemetry diff --git a/acceptance/cmd/version/output.txt b/acceptance/cmd/version/output.txt index 56a4d16ed95..cfb4047c4f5 100644 --- a/acceptance/cmd/version/output.txt +++ b/acceptance/cmd/version/output.txt @@ -35,9 +35,8 @@ Tip: coding agents detected without Databricks skills. Run 'databricks aitools i "development_build": true } -=== version --check from Claude Code prints only the root hint +=== version --check from Claude Code prints no hint >>> [CLI] version --check -Databricks skills are not installed. To work with Databricks reliably, first run: databricks aitools install Databricks CLI v[CLI_VERSION] This is a development build; skipping the update check. diff --git a/acceptance/cmd/version/script b/acceptance/cmd/version/script index 3e72d659289..84067aae808 100644 --- a/acceptance/cmd/version/script +++ b/acceptance/cmd/version/script @@ -23,8 +23,9 @@ trace $CLI version --check title "version --check --output json with a coding agent prints no tip" trace $CLI version --check --output json -# Claude Code callers get only the root hint (agents.MaybeHint), not the tip. -title "version --check from Claude Code prints only the root hint" +# Claude Code callers get no tip: agents.MaybeHint owns their hints and only +# hints on air subcommands. +title "version --check from Claude Code prints no hint" sethome ./claude-home export CLAUDECODE=1 unset CLAUDE_CODE_SESSION_ID diff --git a/cmd/air/air.go b/cmd/air/air.go index 4154bab891c..de1b6ac4ec6 100644 --- a/cmd/air/air.go +++ b/cmd/air/air.go @@ -21,6 +21,10 @@ Learn more: Quickstart (AWS) https://docs.databricks.com/aws/en/machine-learning/ai-runtime/quickstart Quickstart (Azure) https://learn.microsoft.com/azure/databricks/machine-learning/ai-runtime/quickstart Coding agent skill databricks aitools install --skills-only --skills databricks-ai-runtime + Teaches coding agents the workload YAML, GPU types, secrets, + and how to follow and debug runs. + Preview the skill databricks aitools install --path --skills databricks-ai-runtime + Writes the skill files to without installing anything. ` // New returns the root command for the AI Runtime CLI. diff --git a/cmd/air/air_test.go b/cmd/air/air_test.go index 11e4546d21d..b038e3de12c 100644 --- a/cmd/air/air_test.go +++ b/cmd/air/air_test.go @@ -54,6 +54,11 @@ func TestHelpFooterOnlyOnAirCommand(t *testing.T) { cmd.HelpFunc()(cmd, nil) assert.Contains(t, airOut.String(), "Learn more:") assert.Contains(t, airOut.String(), "databricks air run -h config") + // The footer is where `air` and `air --help` recommend the agent skill + // (help never reaches the pre-run hook that prints the install hint). + assert.Contains(t, airOut.String(), "databricks aitools install --skills-only --skills databricks-ai-runtime") + assert.Contains(t, airOut.String(), "Teaches coding agents the workload YAML") + assert.Contains(t, airOut.String(), "databricks aitools install --path --skills databricks-ai-runtime") list := findSubcommand(t, cmd, "list") var listOut bytes.Buffer diff --git a/cmd/air/hint_test.go b/cmd/air/hint_test.go new file mode 100644 index 00000000000..cb14b4e5329 --- /dev/null +++ b/cmd/air/hint_test.go @@ -0,0 +1,89 @@ +package aircmd + +import ( + "bytes" + "testing" + + "github.com/databricks/cli/internal/testutil" + "github.com/databricks/cli/libs/aitools/agents" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/env" + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// airUnderRoot mounts the real air tree under a root whose pre-run hook only +// records that it ran, mirroring where cmd/root calls agents.MaybeHint. +func airUnderRoot(preRunCalled *bool) (*cobra.Command, *cobra.Command) { + root := &cobra.Command{Use: "databricks"} + root.AddGroup(&cobra.Group{ID: "development", Title: "Developer Tools"}) + root.PersistentPreRunE = func(cmd *cobra.Command, args []string) error { + *preRunCalled = true + return nil + } + air := New() + root.AddCommand(air) + return root, air +} + +// TestBareAirAndHelpShowFooterWithoutPreRun pins why the footer, not the +// dynamic hint, is the recommendation surface for `air` and `air --help`. +func TestBareAirAndHelpShowFooterWithoutPreRun(t *testing.T) { + for _, args := range [][]string{{"air"}, {"air", "--help"}} { + t.Run(args[len(args)-1], func(t *testing.T) { + preRunCalled := false + root, _ := airUnderRoot(&preRunCalled) + var out bytes.Buffer + root.SetOut(&out) + root.SetErr(&out) + root.SetArgs(args) + require.NoError(t, root.Execute()) + assert.False(t, preRunCalled, "help must not run the pre-run hook") + assert.Contains(t, out.String(), "Coding agent skill databricks aitools install --skills-only --skills databricks-ai-runtime") + assert.Contains(t, out.String(), "Preview the skill databricks aitools install --path --skills databricks-ai-runtime") + }) + } +} + +// TestRunnableAirCommandsGetSkillHint walks the real air tree, so a new +// subcommand is covered without listing it here. +func TestRunnableAirCommandsGetSkillHint(t *testing.T) { + testutil.SetAgentEnv(t, "CLAUDECODE", "1") + preRunCalled := false + _, air := airUnderRoot(&preRunCalled) + assert.False(t, air.Runnable(), "bare air renders help; the footer covers it") + + var runnable []string + var walk func(c *cobra.Command) + walk = func(c *cobra.Command) { + for _, sub := range c.Commands() { + if sub.Runnable() { + runnable = append(runnable, sub.CommandPath()) + } + walk(sub) + } + } + walk(air) + for _, name := range []string{"run", "get", "list", "logs", "cancel", "convert-to-dabs"} { + assert.Contains(t, runnable, "databricks air "+name) + } + + var visit func(c *cobra.Command) + visit = func(c *cobra.Command) { + for _, sub := range c.Commands() { + if sub.Runnable() { + t.Run(sub.CommandPath(), func(t *testing.T) { + t.Chdir(t.TempDir()) + ctx, stderr := cmdio.NewTestContextWithStderr(t.Context()) + ctx = env.WithUserHomeDir(ctx, t.TempDir()) + ctx = env.Set(ctx, "DATABRICKS_CACHE_DIR", t.TempDir()) + agents.MaybeHint(ctx, sub) + assert.Contains(t, stderr.String(), "the databricks-ai-runtime skill is not installed") + }) + } + visit(sub) + } + } + visit(air) +} diff --git a/libs/aitools/agents/calling.go b/libs/aitools/agents/calling.go index 6bb35c3e14b..0cff90deb85 100644 --- a/libs/aitools/agents/calling.go +++ b/libs/aitools/agents/calling.go @@ -62,8 +62,9 @@ func (t Tooling) String() string { // record. For Codex and Copilot, pluginRecorded (the CLI install state) is the // only plugin source, so it misses plugins installed with the agent's own CLI // and includes plugins later removed with it. The shared skills store is not consulted: it cannot attribute skills. -// The install hint's claudeToolingInstalled differs on purpose: it stays quiet -// for any trace of an install, while this reports what the agent loads. +// The install hint's airSkillAvailable differs on purpose: it looks for the AI +// Runtime skill only, in the shared store too, while this reports what the +// agent loads. func (a *Agent) DatabricksTooling(ctx context.Context, pluginRecorded func() bool) Tooling { // An unknown working directory skips the project lookups below. cwd := "" diff --git a/libs/aitools/agents/hint.go b/libs/aitools/agents/hint.go index b37bcd6060c..3c3d6fd3104 100644 --- a/libs/aitools/agents/hint.go +++ b/libs/aitools/agents/hint.go @@ -2,6 +2,8 @@ package agents import ( "context" + "os" + "path/filepath" "time" "github.com/databricks/cli/libs/cache" @@ -15,10 +17,16 @@ import ( ) const ( - // hintText recommends installing the Databricks AI tooling. The wording - // is deliberately chosen to be effective at helping agents; change it only - // after thorough testing. - hintText = "Databricks skills are not installed. To work with Databricks reliably, first run: databricks aitools install" + // airSkillName is the agent skill for `databricks air`. + airSkillName = "databricks-ai-runtime" + + // airHintText recommends the AI Runtime skill on the commands it helps with. + // It names the skill and what it buys the agent, so a coding agent can tell + // the hint is about the task at hand rather than generic setup, and offers a + // --path preview so the agent can read the skill without installing anything. + airHintText = "Tip: the " + airSkillName + " skill is not installed. It documents the air workload YAML, GPU types, secrets, and how to follow and debug runs. " + + "Install it: databricks aitools install --skills-only --skills " + airSkillName + + " (to read it first without installing: databricks aitools install --path --skills " + airSkillName + ")" // claudeAgentProvider is the useragent.AgentProvider() value for Claude Code. // The SDK maps the CLAUDECODE env var to this product name. The hint is @@ -26,10 +34,10 @@ const ( // https://github.com/databricks/databricks-sdk-go/blob/main/useragent/agent.go claudeAgentProvider = "claude-code" - // aiToolsCommandName is the top-level command whose subtree is never hinted: - // telling the agent to run `databricks aitools install` while it is already - // running an aitools command makes no sense. - aiToolsCommandName = "aitools" + // airCommandName is the top-level command whose runnable subcommands get the + // hint. Bare `databricks air` and `--help` render help without running the + // pre-run hook; the air help footer carries the same recommendation there. + airCommandName = "air" // claudeSessionEnv is Claude Code's per-session identifier. It keys the // per-session throttle (at most one hint per hintSessionTTL per session); @@ -46,18 +54,20 @@ const ( // fire again for the same session. const hintSessionTTL = time.Hour -// MaybeHint prints a one-line recommendation to install the Databricks AI -// tooling when Claude Code drives the CLI without it installed. It is a no-op -// for human callers, other agents, aitools commands themselves, when there is -// no output stream, once the tooling is present, and when the session was -// hinted within the last hintSessionTTL. Best-effort: it never fails the -// command. Besides the stderr line, it logs a telemetry event for each hint -// shown, which makes a command that resolved auth upload telemetry at exit. +// MaybeHint prints a one-line recommendation to install the AI Runtime skill +// when Claude Code runs a `databricks air` subcommand without the skill +// available. Other commands are never hinted: an agent running them has no AI +// Runtime task to apply the skill to, so the line is noise it learns to skip. +// It is also a no-op for human callers, other agents, when there is no output +// stream, and when the session was hinted within the last hintSessionTTL. +// Best-effort: it never fails the command. Besides the stderr line, it logs a +// telemetry event for each hint shown, which makes a command that resolved +// auth upload telemetry at exit. func MaybeHint(ctx context.Context, cmd *cobra.Command) { if useragent.AgentProvider() != claudeAgentProvider { return } - if isAIToolsCommand(cmd) { + if !isAIRSubcommand(cmd) { return } if !cmdio.HasIO(ctx) { @@ -70,50 +80,62 @@ func MaybeHint(ctx context.Context, cmd *cobra.Command) { if alreadyHintedThisSession(ctx) { return } - if claudeToolingInstalled(ctx) { + if airSkillAvailable(ctx) { return } - cmdio.LogString(ctx, hintText) + cmdio.LogString(ctx, airHintText) logHintShown(ctx, protos.AitoolsHintKindInstallText) recordSessionHint(ctx) } -// isAIToolsCommand reports whether cmd is the aitools command or one of its -// subcommands, walking the parent chain (mirrors versioncheck.isExemptCommand). -func isAIToolsCommand(cmd *cobra.Command) bool { - for c := cmd; c != nil; c = c.Parent() { - if c.Name() == aiToolsCommandName { - return true +// isAIRSubcommand reports whether cmd sits under the top-level air command +// (`databricks air list`, `databricks air pools list`, ...), excluding air +// itself. Matching on the top-level ancestor keeps hidden aliases such as +// `databricks experimental air` out. +func isAIRSubcommand(cmd *cobra.Command) bool { + for c := cmd; c.HasParent(); c = c.Parent() { + if !c.Parent().HasParent() { + return c != cmd && c.Name() == airCommandName } } return false } -// claudeToolingInstalled reports whether Databricks AI tooling is present for -// Claude Code. It checks two signals: the databricks plugin in Claude's own -// plugin manifest (written both by `databricks aitools install` and by a direct -// `claude plugin install`), and databricks skill files in the shared canonical -// skills dir (written by `aitools install --skills-only`). Any read/parse -// failure reports false, so an unreadable state errs toward hinting. -// (*Agent).DatabricksTooling, which feeds the User-Agent, differs on purpose: -// it ignores the shared store and out-of-project records, and counts -// .claude/skills dirs, to report what the calling agent loads. -func claudeToolingInstalled(ctx context.Context) bool { - claude := ByName(NameClaudeCode) - if claude == nil { - // claude-code is always in the registry; this guard keeps the hint - // best-effort rather than panicking in a persistent pre-run hook. - return false +// airSkillAvailable reports whether Claude Code has the AI Runtime skill in the +// current working directory: the databricks plugin (which bundles every +// Databricks skill) installed for every project or for this one, or a +// databricks-ai-runtime skill dir in the shared canonical or legacy store or in +// the project store of cwd or a parent up to the repository root. Other +// Databricks skills do not count: they don't cover air. Any read/parse failure +// reports false, so an unreadable state errs toward hinting. +func airSkillAvailable(ctx context.Context) bool { + cwd := "" + if wd, err := os.Getwd(); err == nil { + cwd = resolvePath(wd) } - if version, ok := claude.DatabricksPluginVersion(ctx); ok { - log.Debugf(ctx, "Databricks plugin detected for Claude Code (version %q); skipping install hint", version) - return true + if claude := ByName(NameClaudeCode); claude != nil && claude.pluginInstalls != nil { + if active, _ := claude.DatabricksPluginActiveIn(ctx, cwd); active { + log.Debugf(ctx, "Databricks plugin active for Claude Code in %q; skipping AI Runtime skill hint", cwd) + return true + } } - // The canonical dir always holds a real databricks* subdir after a skills - // install, regardless of whether the agent copy is a symlink or a copy. The - // legacy dir is checked too so users of an older install aren't hinted. - return hasSharedSkills(ctx) + var dirs []string + if homeDir, err := env.UserHomeDir(ctx); err == nil { + dirs = append(dirs, filepath.Join(homeDir, CanonicalSkillsDir), filepath.Join(homeDir, LegacySkillsDir)) + } + if cwd != "" { + for _, dir := range projectDirs(cwd) { + dirs = append(dirs, filepath.Join(dir, CanonicalSkillsDir)) + } + } + for _, dir := range dirs { + // Stat follows the symlinks the installer may leave in place of copies. + if info, err := os.Stat(filepath.Join(dir, airSkillName)); err == nil && info.IsDir() { + return true + } + } + return false } // alreadyHintedThisSession reports whether this Claude session was hinted diff --git a/libs/aitools/agents/hint_test.go b/libs/aitools/agents/hint_test.go index 3add0417795..2de2e7d224f 100644 --- a/libs/aitools/agents/hint_test.go +++ b/libs/aitools/agents/hint_test.go @@ -3,9 +3,11 @@ package agents import ( "bytes" "context" + "encoding/json" "log/slog" "os" "path/filepath" + "strings" "testing" "github.com/databricks/cli/internal/testutil" @@ -26,17 +28,32 @@ func setClaudeAgent(t *testing.T) { testutil.SetAgentEnv(t, "CLAUDECODE", "1") } -// claudeCtx returns a context with a captured stderr buffer and an isolated -// HOME + cache dir, so detection and the session throttle read only the test's -// temp state. home is returned so the test can drop fixtures under it. +// claudeCtx returns a context with a captured, non-TTY stderr buffer (what a +// coding agent's shell tool gives the CLI) and an isolated HOME, cache dir and +// working directory, so detection and the session throttle read only the +// test's temp state. home is returned so the test can drop fixtures under it. func claudeCtx(t *testing.T, home string) (context.Context, *bytes.Buffer) { + t.Chdir(t.TempDir()) ctx, stderr := cmdio.NewTestContextWithStderr(t.Context()) ctx = env.WithUserHomeDir(ctx, home) ctx = env.Set(ctx, "DATABRICKS_CACHE_DIR", t.TempDir()) - return ctx, stderr + return telemetry.WithNewLogger(ctx), stderr } -func regularCmd() *cobra.Command { return &cobra.Command{Use: "current-user"} } +// commandAt builds the command tree for a space-separated path under a +// "databricks" root and returns the leaf, so ancestry checks see the same +// shape as the real CLI. +func commandAt(path string) *cobra.Command { + cmd := &cobra.Command{Use: "databricks"} + for name := range strings.FieldsSeq(path) { + child := &cobra.Command{Use: name} + cmd.AddCommand(child) + cmd = child + } + return cmd +} + +func airCmd() *cobra.Command { return commandAt("air list") } func writePluginManifest(t *testing.T, home, body string) { dir := filepath.Join(home, ".claude", "plugins") @@ -44,6 +61,22 @@ func writePluginManifest(t *testing.T, home, body string) { require.NoError(t, os.WriteFile(filepath.Join(dir, "installed_plugins.json"), []byte(body), 0o600)) } +// writeProjectPluginManifest writes a manifest with one project-scope install +// covering projectPath. Marshaled, not interpolated, so Windows paths with +// backslashes stay valid JSON. +func writeProjectPluginManifest(t *testing.T, home, projectPath string) { + manifest, err := json.Marshal(map[string]any{ + "plugins": map[string]any{ + "databricks@m": []any{map[string]string{ + "scope": "project", + "projectPath": projectPath, + }}, + }, + }) + require.NoError(t, err) + writePluginManifest(t, home, string(manifest)) +} + func writeCanonicalSkill(t *testing.T, home, name string) { require.NoError(t, os.MkdirAll(filepath.Join(home, ".databricks", "aitools", "skills", name), 0o755)) } @@ -59,41 +92,88 @@ func hintKinds(t *testing.T, ctx context.Context) []protos.AitoolsHintKind { return out } +// assertHinted checks for exactly one hint line and one hint-shown event of +// kind INSTALL_TEXT (T8). +func assertHinted(t *testing.T, ctx context.Context, stderr *bytes.Buffer) { + t.Helper() + assert.Equal(t, airHintText+"\n", stderr.String()) + assert.Equal(t, []protos.AitoolsHintKind{protos.AitoolsHintKindInstallText}, hintKinds(t, ctx)) +} + +// assertSilent checks for no hint line and no telemetry event (T8). +func assertSilent(t *testing.T, ctx context.Context, stderr *bytes.Buffer) { + t.Helper() + assert.Empty(t, stderr.String()) + assert.Empty(t, hintKinds(t, ctx)) +} + +func TestAIRHintText(t *testing.T) { + // The hint names the skill, says what it is for, and offers both the install + // and a no-install preview, on one line. + assert.Contains(t, airHintText, "databricks-ai-runtime skill is not installed") + assert.Contains(t, airHintText, "workload YAML") + assert.Contains(t, airHintText, "databricks aitools install --skills-only --skills databricks-ai-runtime") + assert.Contains(t, airHintText, "databricks aitools install --path --skills databricks-ai-runtime") + assert.NotContains(t, airHintText, "\n") +} + func TestMaybeHint_HumanCallerSilent(t *testing.T) { // No agent env: AgentProvider() == "", so no hint even with nothing installed. testutil.SetAgentEnv(t) - ctx, stderr := cmdio.NewTestContextWithStderr(t.Context()) - ctx = env.WithUserHomeDir(ctx, t.TempDir()) - ctx = telemetry.WithNewLogger(ctx) - MaybeHint(ctx, regularCmd()) - assert.Empty(t, stderr.String()) - assert.Empty(t, hintKinds(t, ctx)) + ctx, stderr := claudeCtx(t, t.TempDir()) + MaybeHint(ctx, airCmd()) + assertSilent(t, ctx, stderr) } -func TestMaybeHint_ClaudeNotInstalledHints(t *testing.T) { +// T1: every air subcommand, at any depth, gets the skill-specific hint. +func TestMaybeHint_AIRSubcommandsHint(t *testing.T) { setClaudeAgent(t) - ctx, stderr := claudeCtx(t, t.TempDir()) - ctx = telemetry.WithNewLogger(ctx) - MaybeHint(ctx, regularCmd()) - assert.Contains(t, stderr.String(), hintText) - assert.Equal(t, []protos.AitoolsHintKind{protos.AitoolsHintKindInstallText}, hintKinds(t, ctx)) + for _, path := range []string{ + "air run", "air get", "air list", "air logs", "air cancel", + "air convert-to-dabs", "air images", "air images push", "air pools", "air pools list", + } { + t.Run(path, func(t *testing.T) { + ctx, stderr := claudeCtx(t, t.TempDir()) + MaybeHint(ctx, commandAt(path)) + assertHinted(t, ctx, stderr) + }) + } +} + +// T3: commands outside air are never hinted, including the auth and version +// commands the generic hint used to fire on. +func TestMaybeHint_NonAIRCommandsSilent(t *testing.T) { + setClaudeAgent(t) + for _, path := range []string{ + "auth profiles", "current-user me", "clusters list", "jobs list", "bundle deploy", + "workspace list", "version", "aitools install", "experimental aitools install", + // The bare air node only renders help; its footer carries the tip. + "air", + // The hidden alias that points users at `databricks air`. + "experimental air", + } { + t.Run(path, func(t *testing.T) { + ctx, stderr := claudeCtx(t, t.TempDir()) + MaybeHint(ctx, commandAt(path)) + assertSilent(t, ctx, stderr) + }) + } } func TestMaybeHint_PluginInstalledSilent(t *testing.T) { setClaudeAgent(t) home := t.TempDir() ctx, stderr := claudeCtx(t, home) - // The core regression: a plugin install writes only Claude's manifest, no - // skills subdir. Detection must treat it as installed. + // A plugin install writes only Claude's manifest, no skills subdir, and + // bundles every Databricks skill including databricks-ai-runtime. writePluginManifest(t, home, `{"plugins":{"databricks@claude-plugins-official":[{"scope":"user"}]}}`) - ctx = telemetry.WithNewLogger(ctx) - MaybeHint(ctx, regularCmd()) - assert.Empty(t, stderr.String()) - assert.Empty(t, hintKinds(t, ctx)) + MaybeHint(ctx, airCmd()) + assertSilent(t, ctx, stderr) } -func TestClaudeToolingInstalled_LogsPluginVersion(t *testing.T) { +func TestAIRSkillAvailable_LogsPluginDetection(t *testing.T) { + t.Chdir(t.TempDir()) home := t.TempDir() writePluginManifest(t, home, `{"plugins":{"databricks@claude-plugins-official":[{"version":"0.2.6"}]}}`) @@ -103,14 +183,9 @@ func TestClaudeToolingInstalled_LogsPluginVersion(t *testing.T) { ctx := log.NewContext(t.Context(), slog.New(handler)) ctx = env.WithUserHomeDir(ctx, home) - assert.True(t, claudeToolingInstalled(ctx)) - - // The text handler escapes the inner quotes around the version, so assert on - // the stable, unescaped fragments rather than the full quoted message. - out := buf.String() - assert.Contains(t, out, "Databricks plugin detected for Claude Code") - assert.Contains(t, out, "0.2.6") - assert.Contains(t, out, "skipping install hint") + assert.True(t, airSkillAvailable(ctx)) + assert.Contains(t, buf.String(), "Databricks plugin active for Claude Code") + assert.Contains(t, buf.String(), "skipping AI Runtime skill hint") } func TestMaybeHint_CustomClaudeConfigDir(t *testing.T) { @@ -139,58 +214,90 @@ func TestMaybeHint_CustomClaudeConfigDir(t *testing.T) { writePluginManifest(t, home, manifest) } - MaybeHint(ctx, regularCmd()) + MaybeHint(ctx, airCmd()) if tc.wantHint { - assert.Contains(t, stderr.String(), hintText) + assertHinted(t, ctx, stderr) } else { - assert.Empty(t, stderr.String()) + assertSilent(t, ctx, stderr) } }) } } -func TestMaybeHint_SkillsInstalledSilent(t *testing.T) { - setClaudeAgent(t) - home := t.TempDir() - ctx, stderr := claudeCtx(t, home) - writeCanonicalSkill(t, home, "databricks") - - MaybeHint(ctx, regularCmd()) - assert.Empty(t, stderr.String()) -} - -func TestMaybeHint_LegacySkillsInstalledSilent(t *testing.T) { +// T4: only the databricks-ai-runtime skill itself, in a location Claude loads +// for this working directory, silences the hint. +func TestMaybeHint_AIRSkillDetection(t *testing.T) { setClaudeAgent(t) - home := t.TempDir() - ctx, stderr := claudeCtx(t, home) - require.NoError(t, os.MkdirAll(filepath.Join(home, ".databricks", "agent-skills", "databricks"), 0o755)) + for _, tc := range []struct { + name string + setup func(t *testing.T, home, cwd string) + wantHint bool + }{ + {"nothing installed", func(t *testing.T, home, cwd string) {}, true}, + {"canonical store", func(t *testing.T, home, cwd string) { + writeCanonicalSkill(t, home, airSkillName) + }, false}, + {"legacy store", func(t *testing.T, home, cwd string) { + require.NoError(t, os.MkdirAll(filepath.Join(home, LegacySkillsDir, airSkillName), 0o755)) + }, false}, + {"canonical store via symlink", func(t *testing.T, home, cwd string) { + dir := filepath.Join(home, CanonicalSkillsDir) + require.NoError(t, os.MkdirAll(dir, 0o755)) + require.NoError(t, os.Symlink(t.TempDir(), filepath.Join(dir, airSkillName))) + }, false}, + {"project store in cwd", func(t *testing.T, home, cwd string) { + require.NoError(t, os.MkdirAll(filepath.Join(cwd, CanonicalSkillsDir, airSkillName), 0o755)) + }, false}, + {"other databricks skills only", func(t *testing.T, home, cwd string) { + writeCanonicalSkill(t, home, "databricks") + writeCanonicalSkill(t, home, "databricks-jobs") + require.NoError(t, os.MkdirAll(filepath.Join(cwd, CanonicalSkillsDir, "databricks-apps"), 0o755)) + }, true}, + {"a file named like the skill", func(t *testing.T, home, cwd string) { + dir := filepath.Join(home, CanonicalSkillsDir) + require.NoError(t, os.MkdirAll(dir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, airSkillName), nil, 0o600)) + }, true}, + {"plugin for this project", func(t *testing.T, home, cwd string) { + writeProjectPluginManifest(t, home, cwd) + }, false}, + {"plugin for an unrelated project", func(t *testing.T, home, cwd string) { + writeProjectPluginManifest(t, home, t.TempDir()) + }, true}, + } { + t.Run(tc.name, func(t *testing.T) { + home := t.TempDir() + ctx, stderr := claudeCtx(t, home) + cwd, err := os.Getwd() + require.NoError(t, err) + tc.setup(t, home, cwd) - MaybeHint(ctx, regularCmd()) - assert.Empty(t, stderr.String()) + MaybeHint(ctx, airCmd()) + if tc.wantHint { + assertHinted(t, ctx, stderr) + } else { + assertSilent(t, ctx, stderr) + } + }) + } } -func TestMaybeHint_NonDatabricksSkillStillHints(t *testing.T) { +// T4: a project-scoped skill at the repository root still counts when the CLI +// runs in a subdirectory, as Claude Code loads it there. +func TestMaybeHint_AIRSkillInRepositoryRoot(t *testing.T) { setClaudeAgent(t) home := t.TempDir() ctx, stderr := claudeCtx(t, home) - writeCanonicalSkill(t, home, "some-other-skill") + root, err := os.Getwd() + require.NoError(t, err) + require.NoError(t, os.Mkdir(filepath.Join(root, ".git"), 0o755)) + require.NoError(t, os.MkdirAll(filepath.Join(root, CanonicalSkillsDir, airSkillName), 0o755)) + sub := filepath.Join(root, "training", "configs") + require.NoError(t, os.MkdirAll(sub, 0o755)) + t.Chdir(sub) - MaybeHint(ctx, regularCmd()) - assert.Contains(t, stderr.String(), hintText) -} - -func TestMaybeHint_AIToolsCommandSilent(t *testing.T) { - setClaudeAgent(t) - ctx, stderr := claudeCtx(t, t.TempDir()) - ctx = telemetry.WithNewLogger(ctx) - - parent := &cobra.Command{Use: aiToolsCommandName} - child := &cobra.Command{Use: "install"} - parent.AddCommand(child) - - MaybeHint(ctx, child) - assert.Empty(t, stderr.String()) - assert.Empty(t, hintKinds(t, ctx)) + MaybeHint(ctx, airCmd()) + assertSilent(t, ctx, stderr) } func TestMaybeHint_NoIONoPanic(t *testing.T) { @@ -199,30 +306,65 @@ func TestMaybeHint_NoIONoPanic(t *testing.T) { // invocations before PersistentPreRunE runs, so HasIO can be false here. ctx := env.WithUserHomeDir(t.Context(), t.TempDir()) ctx = telemetry.WithNewLogger(ctx) - assert.NotPanics(t, func() { MaybeHint(ctx, regularCmd()) }) + assert.NotPanics(t, func() { MaybeHint(ctx, airCmd()) }) assert.Empty(t, hintKinds(t, ctx)) } -func TestMaybeHint_ThrottledPerSession(t *testing.T) { +// T5 and T6: with a session id the first air command in the session is +// hinted and later ones stay quiet until the marker expires; without one +// (older Claude Code) every air command is hinted. Both cases run with a +// non-TTY stderr, which is how a coding agent's shell tool runs the CLI. +func TestMaybeHint_SessionThrottle(t *testing.T) { + setClaudeAgent(t) + for _, tc := range []struct { + name string + session string + wantSecond bool + }{ + {"session id: first air command only", "session-abc", false}, + {"no session id: every air command", "", true}, + } { + t.Run(tc.name, func(t *testing.T) { + home := t.TempDir() + cacheDir := t.TempDir() + newCtx := func() (context.Context, *bytes.Buffer) { + ctx, stderr := claudeCtx(t, home) + ctx = env.Set(ctx, "DATABRICKS_CACHE_DIR", cacheDir) + if tc.session != "" { + ctx = env.Set(ctx, claudeSessionEnv, tc.session) + } + return ctx, stderr + } + + // A non-air command first must not use up the session's hint. + ctx0, stderr0 := newCtx() + MaybeHint(ctx0, commandAt("auth profiles")) + assertSilent(t, ctx0, stderr0) + + ctx1, stderr1 := newCtx() + MaybeHint(ctx1, commandAt("air list")) + assertHinted(t, ctx1, stderr1) + + ctx2, stderr2 := newCtx() + MaybeHint(ctx2, commandAt("air run")) + if tc.wantSecond { + assertHinted(t, ctx2, stderr2) + } else { + assertSilent(t, ctx2, stderr2) + } + }) + } +} + +func TestMaybeHint_OtherSessionStillHinted(t *testing.T) { setClaudeAgent(t) home := t.TempDir() cacheDir := t.TempDir() - newCtx := func() (context.Context, *bytes.Buffer) { - ctx, stderr := cmdio.NewTestContextWithStderr(t.Context()) - ctx = env.WithUserHomeDir(ctx, home) + for _, session := range []string{"session-a", "session-b"} { + ctx, stderr := claudeCtx(t, home) ctx = env.Set(ctx, "DATABRICKS_CACHE_DIR", cacheDir) - ctx = env.Set(ctx, claudeSessionEnv, "session-abc") - return telemetry.WithNewLogger(ctx), stderr + ctx = env.Set(ctx, claudeSessionEnv, session) + MaybeHint(ctx, airCmd()) + assertHinted(t, ctx, stderr) } - - ctx1, stderr1 := newCtx() - MaybeHint(ctx1, regularCmd()) - assert.Contains(t, stderr1.String(), hintText) - assert.Equal(t, []protos.AitoolsHintKind{protos.AitoolsHintKindInstallText}, hintKinds(t, ctx1)) - - // Same session id, second command: must stay quiet and log nothing. - ctx2, stderr2 := newCtx() - MaybeHint(ctx2, regularCmd()) - assert.Empty(t, stderr2.String()) - assert.Empty(t, hintKinds(t, ctx2)) } diff --git a/libs/aitools/agents/recommend.go b/libs/aitools/agents/recommend.go index 14a4da5b00b..7e34de5b517 100644 --- a/libs/aitools/agents/recommend.go +++ b/libs/aitools/agents/recommend.go @@ -23,8 +23,8 @@ func PrintSkillsTip(ctx context.Context) { // MaybeVersionCheckTip prints skillsTipText after `databricks version --check` // when a coding agent is detected without Databricks tooling. Claude Code -// callers are skipped because MaybeHint already covers them, with its -// per-session throttle. installedPlugins is a parameter because this package +// callers are skipped because MaybeHint owns their hints, and it recommends a +// skill only on the commands that skill helps with. installedPlugins is a parameter because this package // cannot import the installer; callers pass installer.InstalledPluginAgents, // which counts a plugin recorded in the CLI install state with or without a // known version. No telemetry is logged: `databricks version` never resolves