diff --git a/.nextchanges/cli/aitools-hint-other-agents.md b/.nextchanges/cli/aitools-hint-other-agents.md new file mode 100644 index 00000000000..4093d961770 --- /dev/null +++ b/.nextchanges/cli/aitools-hint-other-agents.md @@ -0,0 +1 @@ +* When Codex, Cursor, GitHub Copilot CLI, OpenCode, Gemini CLI, Antigravity, Pi, Goose or Kiro run the CLI without Databricks skills installed for that agent, the CLI now prints a one-line hint to install them with `databricks aitools install --agents --scope global`, previously shown only to Claude Code. ([#7586](https://github.com/databricks/cli/pull/7586)) diff --git a/acceptance/cmd/aitools/agent-hints/gemini-home/.gemini/skills/databricks-core/SKILL.md b/acceptance/cmd/aitools/agent-hints/gemini-home/.gemini/skills/databricks-core/SKILL.md new file mode 100644 index 00000000000..4c0f09c4d63 --- /dev/null +++ b/acceptance/cmd/aitools/agent-hints/gemini-home/.gemini/skills/databricks-core/SKILL.md @@ -0,0 +1,4 @@ +--- +name: databricks-core +description: Fixture marking Databricks skills as installed for Gemini CLI. +--- diff --git a/acceptance/cmd/aitools/agent-hints/out.test.toml b/acceptance/cmd/aitools/agent-hints/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/cmd/aitools/agent-hints/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/cmd/aitools/agent-hints/output.txt b/acceptance/cmd/aitools/agent-hints/output.txt new file mode 100644 index 00000000000..ed22089e902 --- /dev/null +++ b/acceptance/cmd/aitools/agent-hints/output.txt @@ -0,0 +1,46 @@ + +=== Codex is hinted once per thread + +>>> CODEX_CI=1 CODEX_THREAD_ID=thread-1 [CLI] version +Databricks skills are not installed. To work with Databricks reliably, first run: databricks aitools install --agents codex --scope global +Databricks CLI v[CLI_VERSION] + +>>> CODEX_CI=1 CODEX_THREAD_ID=thread-1 [CLI] version +Databricks CLI v[CLI_VERSION] + +>>> CODEX_CI=1 CODEX_THREAD_ID=thread-2 [CLI] version +Databricks skills are not installed. To work with Databricks reliably, first run: databricks aitools install --agents codex --scope global +Databricks CLI v[CLI_VERSION] + +=== Cursor exposes no session id, so it is hinted once per machine + +>>> CURSOR_AGENT=1 [CLI] version +Databricks skills are not installed. To work with Databricks reliably, first run: databricks aitools install --agents cursor --scope global +Databricks CLI v[CLI_VERSION] + +>>> CURSOR_AGENT=1 [CLI] version +Databricks CLI v[CLI_VERSION] + +=== Copilot CLI maps to the copilot registry agent + +>>> COPILOT_CLI=1 [CLI] version +Databricks skills are not installed. To work with Databricks reliably, first run: databricks aitools install --agents copilot --scope global +Databricks CLI v[CLI_VERSION] + +=== Gemini CLI with its own Databricks skills is not hinted + +>>> GEMINI_CLI=1 [CLI] version +Databricks CLI v[CLI_VERSION] + +=== Agents the CLI cannot install for are not hinted + +>>> WINDSURF_AGENT=1 [CLI] version +Databricks CLI v[CLI_VERSION] + +>>> VSCODE_AGENT=1 [CLI] version +Databricks CLI v[CLI_VERSION] + +=== Nested agents are not hinted + +>>> CODEX_CI=1 CURSOR_AGENT=1 CODEX_THREAD_ID=thread-3 [CLI] version +Databricks CLI v[CLI_VERSION] diff --git a/acceptance/cmd/aitools/agent-hints/script b/acceptance/cmd/aitools/agent-hints/script new file mode 100644 index 00000000000..405596d991b --- /dev/null +++ b/acceptance/cmd/aitools/agent-hints/script @@ -0,0 +1,27 @@ +# aitools commands skip the install hint; version exercises it without auth or network. +sethome ./home +unset XDG_CONFIG_HOME GEMINI_CLI_HOME CODEX_THREAD_ID + +title "Codex is hinted once per thread" +trace CODEX_CI=1 CODEX_THREAD_ID=thread-1 $CLI version +trace CODEX_CI=1 CODEX_THREAD_ID=thread-1 $CLI version +trace CODEX_CI=1 CODEX_THREAD_ID=thread-2 $CLI version + +title "Cursor exposes no session id, so it is hinted once per machine" +trace CURSOR_AGENT=1 $CLI version +trace CURSOR_AGENT=1 $CLI version + +title "Copilot CLI maps to the copilot registry agent" +trace COPILOT_CLI=1 $CLI version + +title "Gemini CLI with its own Databricks skills is not hinted" +sethome ./gemini-home +trace GEMINI_CLI=1 $CLI version +sethome ./home + +title "Agents the CLI cannot install for are not hinted" +trace WINDSURF_AGENT=1 $CLI version +trace VSCODE_AGENT=1 $CLI version + +title "Nested agents are not hinted" +trace CODEX_CI=1 CURSOR_AGENT=1 CODEX_THREAD_ID=thread-3 $CLI version diff --git a/acceptance/cmd/aitools/agent-hints/test.toml b/acceptance/cmd/aitools/agent-hints/test.toml new file mode 100644 index 00000000000..c6ac528b986 --- /dev/null +++ b/acceptance/cmd/aitools/agent-hints/test.toml @@ -0,0 +1,3 @@ +# The throttle markers live in the file cache, which a host could disable. +Env.DATABRICKS_CACHE_ENABLED = "true" +Ignore = ["home"] diff --git a/cmd/root/root.go b/cmd/root/root.go index 59b9777ef69..967c2afa0c6 100644 --- a/cmd/root/root.go +++ b/cmd/root/root.go @@ -89,10 +89,10 @@ func New(ctx context.Context) *cobra.Command { ctx = InjectTestPidToUserAgent(ctx) cmd.SetContext(ctx) - // Recommend installing Databricks AI tooling to Claude Code when it is - // driving the CLI without the tooling installed (best-effort: a stderr - // line and a telemetry event that is uploaded at exit). - agents.MaybeHint(ctx, cmd) + // Recommend installing Databricks AI tooling to the calling coding agent + // when it drives the CLI without the tooling installed (best-effort: a + // stderr line and a telemetry event that is uploaded at exit). + agents.MaybeHint(ctx, cmd, installer.GlobalPluginRecorded) return nil } diff --git a/libs/aitools/agents/hint.go b/libs/aitools/agents/hint.go index dce48cd4503..fde8b25bb04 100644 --- a/libs/aitools/agents/hint.go +++ b/libs/aitools/agents/hint.go @@ -76,8 +76,8 @@ const ( // degrades to per-command. claudeSessionEnv = "CLAUDE_CODE_SESSION_ID" - // hintCacheComponent names the cache bucket holding per-session "already - // hinted" markers. + // hintCacheComponent names the cache bucket holding "already hinted" + // markers (per session, or per agent on this machine). hintCacheComponent = "aitools-hint" ) @@ -94,17 +94,34 @@ type agentHint struct { installed func(ctx context.Context) bool // text is the model-facing recommendation, printed at most once per - // hintSessionTTL per session (see sessionEnv), together with native. + // hintSessionTTL per session (see sessionEnv), or per agent on this machine + // when there is no session id (see fallbackKey), together with native. text string // textKind tags the hint-shown event logged when text prints. textKind protos.AitoolsHintKind // sessionEnv names the env var carrying the agent's session id, which keys - // the throttle for both lines. Empty, or unset at runtime, means no - // throttle. + // the throttle for both lines. When it is empty or unset at runtime, + // fallbackKey keys the throttle instead. sessionEnv string + // sessionKeyPrefix namespaces the session id in the throttle key, so one + // agent's marker never silences another agent that reuses the same id. + // Empty for Claude Code, whose key is the raw session id. + sessionKeyPrefix string + + // fallbackKey keys the throttle when there is no session id, so the hint + // prints at most once per hintSessionTTL on this machine. Empty means no + // throttle without a session id (Claude Code). + fallbackKey string + + // durableThrottle prints the text only once its throttle marker is + // persisted (see claimHintMarker), so an unwritable cache can't turn the + // hint into a line on every command. False for + // Claude Code, whose marker write stays best-effort. + durableThrottle bool + // native returns a line the agent itself acts on (for Claude Code, its // plugin install prompt), or "" when none applies to this invocation. It // shares the text's per-session throttle. Nil for agents without one. @@ -116,7 +133,7 @@ type agentHint struct { // hintForAgent returns the hint for an agent, keyed by its // useragent.AgentProvider() value, or nil when that agent is not hinted. -func hintForAgent(provider string) *agentHint { +func hintForAgent(provider string, pluginRecorded PluginRecordedFunc) *agentHint { switch provider { case claudeAgentProvider: return &agentHint{ @@ -128,6 +145,9 @@ func hintForAgent(provider string) *agentHint { nativeKind: protos.AitoolsHintKindClaudeCodeNative, } default: + if a := AgentForProvider(provider); a != nil { + return agentInstallHint(a, pluginRecorded) + } return nil } } @@ -146,12 +166,15 @@ func (h *agentHint) nativeLine(ctx context.Context) string { // docker token` credential helpers, a CLI process the CLI started itself, when // there is no output stream, and once the tooling is present. // Both lines, the agent-native one (if any) and the model-facing text, and -// their hint-shown events are throttled together per session. +// their hint-shown events are throttled together per session, or per agent on +// this machine for agents without a session id (fallbackKey). +// pluginRecorded reads the CLI install record for agents whose plugin has no +// manifest reader here (see Agent.DatabricksTooling). // Best-effort: it never fails the command. Besides the stderr lines, it logs a // telemetry event for each hint line shown, which makes a command that resolved // auth upload telemetry at exit. -func MaybeHint(ctx context.Context, cmd *cobra.Command) { - hint := hintForAgent(useragent.AgentProvider()) +func MaybeHint(ctx context.Context, cmd *cobra.Command, pluginRecorded PluginRecordedFunc) { + hint := hintForAgent(useragent.AgentProvider(), pluginRecorded) if hint == nil { return } @@ -168,22 +191,34 @@ func MaybeHint(ctx context.Context, cmd *cobra.Command) { return } // Check the throttle before the filesystem probe, so a throttled session - // skips the plugin/skills detection. The throttle token is written only - // after the hint prints, so an already-installed environment never burns - // it. - if alreadyHintedThisSession(ctx, hint.sessionEnv) { + // skips the plugin/skills detection. The marker is written only once a hint + // is due, so an already-installed environment never burns it. + key := hint.throttleKey(ctx) + if alreadyHintedThisSession(ctx, key) { return } if hint.installed(ctx) { return } + if !hint.claim(ctx, key) { + return + } if native := hint.nativeLine(ctx); native != "" { cmdio.LogString(ctx, native) logHintShown(ctx, hint.nativeKind) } cmdio.LogString(ctx, hint.text) logHintShown(ctx, hint.textKind) - recordSessionHint(ctx, hint.sessionEnv) +} + +// claim records the throttle key and reports whether the hint may print. A +// durable throttle allows it only once the marker has persisted. +func (h *agentHint) claim(ctx context.Context, key string) bool { + if h.durableThrottle { + return claimHintMarker(ctx, key) + } + recordSessionHint(ctx, key) + return true } // claudeNativeHintLine returns claudeNativeHint when Claude Code itself spawned @@ -266,29 +301,36 @@ func claudeToolingInstalled(ctx context.Context) bool { return tooling != ToolingNone } -// alreadyHintedThisSession reports whether the session named by sessionEnv was -// hinted within the last hintSessionTTL. With no session id it returns false -// so the hint fires per command. -func alreadyHintedThisSession(ctx context.Context, sessionEnv string) bool { - session := sessionID(ctx, sessionEnv) - if session == "" { +// throttleKey returns the cache key throttling the hint: the session id behind +// sessionKeyPrefix (Claude's key is the raw id and must not change), else +// fallbackKey. "" means no throttle. +func (h *agentHint) throttleKey(ctx context.Context) string { + if session := sessionID(ctx, h.sessionEnv); session != "" { + return h.sessionKeyPrefix + session + } + return h.fallbackKey +} + +// alreadyHintedThisSession reports whether the throttle key was hinted within +// the last hintSessionTTL. With no key it returns false so the hint fires per +// command. +func alreadyHintedThisSession(ctx context.Context, key string) bool { + if key == "" { return false } c := cache.NewCache(ctx, hintCacheComponent, hintSessionTTL, nil) - _, seen := cache.Get[bool](ctx, c, session) + _, seen := cache.Get[bool](ctx, c, key) return seen } -// recordSessionHint marks the session named by sessionEnv as hinted, so later -// commands in the same session stay quiet until the marker expires -// (hintSessionTTL). No-op when the session id is unset. -func recordSessionHint(ctx context.Context, sessionEnv string) { - session := sessionID(ctx, sessionEnv) - if session == "" { +// recordSessionHint marks the throttle key as hinted, so later commands stay +// quiet until the marker expires (hintSessionTTL). No-op when the key is "". +func recordSessionHint(ctx context.Context, key string) { + if key == "" { return } c := cache.NewCache(ctx, hintCacheComponent, hintSessionTTL, nil) - cache.Put(ctx, c, session, true) + cache.Put(ctx, c, key, true) } // sessionID returns the value of sessionEnv, or "" when the agent has no diff --git a/libs/aitools/agents/hint_agents.go b/libs/aitools/agents/hint_agents.go new file mode 100644 index 00000000000..a566eec993a --- /dev/null +++ b/libs/aitools/agents/hint_agents.go @@ -0,0 +1,60 @@ +package agents + +import ( + "context" + + "github.com/databricks/cli/libs/cache" + "github.com/databricks/cli/libs/telemetry/protos" +) + +// agentSessionEnv names the env var a non-Claude agent sets to a per-session id +// in the shells it spawns. Only verified variables are listed; an agent missing +// here is throttled per machine instead (see agentInstallHint). +var agentSessionEnv = map[string]string{ + // Set by codex-cli (0.166.0) for every shell tool call, next to CODEX_CI. + NameCodex: "CODEX_THREAD_ID", +} + +// PluginRecordedFunc reports whether the CLI install state records a +// databricks plugin install for the named agent. The installer package owns +// that state and provides it (installer.GlobalPluginRecorded). +type PluginRecordedFunc func(ctx context.Context, agentName string) bool + +// agentInstallHint returns the hint for a non-Claude registry agent. The text +// keeps Claude Code's imperative wording (hintText) and names the agent and +// scope, so the install acts on the calling agent and never stops at the agent +// picker or the scope prompt when the agent's shell is a terminal. +// +// The hint is gated on the agent's own tooling (Agent.DatabricksTooling, with +// pluginRecorded as the plugin source for Codex and Copilot). A plugin +// installed with the agent's own CLI rather than `databricks aitools install` +// leaves no trace there for those two, so their users are hinted; the +// throttle bounds that to once per session or per TTL. +func agentInstallHint(a *Agent, pluginRecorded PluginRecordedFunc) *agentHint { + return &agentHint{ + installed: func(ctx context.Context) bool { + recorded := func() bool { return pluginRecorded(ctx, a.Name) } + return a.DatabricksTooling(ctx, recorded) != ToolingNone + }, + text: "Databricks skills are not installed. To work with Databricks reliably, first run: databricks aitools install --agents " + a.Name + " --scope global", + textKind: protos.AitoolsHintKindAgentInstallText, + sessionEnv: agentSessionEnv[a.Name], + sessionKeyPrefix: a.Name + ":session:", + fallbackKey: a.Name + ":machine", + durableThrottle: true, + } +} + +// claimHintMarker writes the throttle marker to the user cache and reports +// whether it reads back. libs/cache silently drops writes it can't make (e.g. +// a read-only home in an agent sandbox), and without a kept marker the hint +// would repeat on every command, so the caller then skips it. A sandbox that +// can't write the user cache can't run the install either. The same holds with +// caching turned off (DATABRICKS_CACHE_ENABLED=false): nothing persists, so +// these agents get no hint. +func claimHintMarker(ctx context.Context, key string) bool { + c := cache.NewCache(ctx, hintCacheComponent, hintSessionTTL, nil) + cache.Put(ctx, c, key, true) + _, ok := cache.Get[bool](ctx, c, key) + return ok +} diff --git a/libs/aitools/agents/hint_agents_test.go b/libs/aitools/agents/hint_agents_test.go new file mode 100644 index 00000000000..e8f954dc1eb --- /dev/null +++ b/libs/aitools/agents/hint_agents_test.go @@ -0,0 +1,309 @@ +package agents + +import ( + "bytes" + "context" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/databricks/cli/internal/testutil" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/libs/telemetry" + "github.com/databricks/cli/libs/telemetry/protos" + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestAgentHintText(t *testing.T) { + const prefix = "Databricks skills are not installed. To work with Databricks reliably, first run: " + cases := map[string]string{ + NameCursor: "databricks aitools install --agents cursor --scope global", + NameCodex: "databricks aitools install --agents codex --scope global", + NameOpenCode: "databricks aitools install --agents opencode --scope global", + NameCopilot: "databricks aitools install --agents copilot --scope global", + NameAntigravity: "databricks aitools install --agents antigravity --scope global", + NamePi: "databricks aitools install --agents pi --scope global", + NameGemini: "databricks aitools install --agents gemini --scope global", + NameGoose: "databricks aitools install --agents goose --scope global", + NameKiro: "databricks aitools install --agents kiro --scope global", + } + for name, command := range cases { + t.Run(name, func(t *testing.T) { + a := ByName(name) + require.NotNil(t, a) + assert.Equal(t, prefix+command, agentInstallHint(a, noPluginRecorded).text) + }) + } +} + +func TestAgentInstallHintThrottleKey(t *testing.T) { + codex := agentInstallHint(ByName(NameCodex), noPluginRecorded) + cursor := agentInstallHint(ByName(NameCursor), noPluginRecorded) + + ctx := env.Set(t.Context(), "CODEX_THREAD_ID", "thread-1") + assert.Equal(t, "codex:session:thread-1", codex.throttleKey(ctx)) + + // No session id: fall back to one key per agent on this machine. + assert.Equal(t, "codex:machine", codex.throttleKey(env.Set(t.Context(), "CODEX_THREAD_ID", ""))) + + // Agents without a verified session variable always use the per-agent key, + // and never pick up another agent's session id. + assert.Equal(t, "cursor:machine", cursor.throttleKey(ctx)) +} + +// agentDetectEnv is the SDK env var that makes AgentProvider() report each +// registry agent. +var agentDetectEnv = map[string]string{ + NameClaudeCode: "CLAUDECODE", + NameCursor: "CURSOR_AGENT", + NameCodex: "CODEX_CI", + NameOpenCode: "OPENCODE", + NameCopilot: "COPILOT_CLI", + NameAntigravity: "ANTIGRAVITY_AGENT", + NamePi: "PI_CODING_AGENT", + NameGemini: "GEMINI_CLI", + NameGoose: "GOOSE_TERMINAL", + NameKiro: "KIRO", +} + +// setAgentEnv makes useragent.AgentProvider() report the given env var's agent, +// with the host's agent and hint env cleared (see testutil.SetAgentEnv). +func setAgentEnv(t *testing.T, envVar string) { + testutil.SetAgentEnv(t, envVar, "1") + unsetEnv(t, append([]string{"CODEX_THREAD_ID"}, hintEnvVars...)) +} + +// agentCtx returns a hint test context whose agent config dirs all resolve +// under home, so fixtures never touch the host's real agent directories. It +// also changes into a repository root under home, so project skills around +// the directory the tests run from are never read. +func agentCtx(t *testing.T, home, cacheDir string) (context.Context, *bytes.Buffer) { + project := filepath.Join(home, "project") + require.NoError(t, os.MkdirAll(filepath.Join(project, ".git"), 0o755)) + t.Chdir(project) + + ctx, stderr := cmdio.NewTestContextWithStderr(t.Context()) + ctx = env.WithUserHomeDir(ctx, home) + ctx = env.Set(ctx, "DATABRICKS_CACHE_DIR", cacheDir) + // The throttle marker lives in the file cache, which a host could disable. + ctx = env.Set(ctx, "DATABRICKS_CACHE_ENABLED", "true") + ctx = env.Set(ctx, "CLAUDE_CONFIG_DIR", "") + ctx = env.Set(ctx, "XDG_CONFIG_HOME", "") + ctx = env.Set(ctx, "APPDATA", "") + ctx = env.Set(ctx, "GEMINI_CLI_HOME", "") + ctx = env.Set(ctx, "PI_CODING_AGENT_DIR", "") + ctx = env.Set(ctx, "GOOSE_PATH_ROOT", "") + ctx = env.Set(ctx, "CODEX_THREAD_ID", "") + return ctx, stderr +} + +func TestAgentCtxIgnoresProjectSkillsAroundTheTest(t *testing.T) { + // Project skills in the directory the tests run from (e.g. the checkout) + // must not count as installed. + gemini := ByName(NameGemini) + outer := t.TempDir() + require.NoError(t, os.Mkdir(filepath.Join(outer, ".git"), 0o755)) + require.NoError(t, os.MkdirAll(filepath.Join(gemini.ProjectSkillsDir(outer), "databricks-core"), 0o755)) + t.Chdir(outer) + + setAgentEnv(t, agentDetectEnv[NameGemini]) + ctx, stderr := agentCtx(t, t.TempDir(), t.TempDir()) + MaybeHint(ctx, regularCmd(), noPluginRecorded) + assert.Equal(t, agentInstallHint(gemini, noPluginRecorded).text+"\n", stderr.String()) +} + +func writeAgentSkill(t *testing.T, ctx context.Context, home string, a *Agent) { + dir, err := a.SkillsDir(ctx) + require.NoError(t, err) + rel, err := filepath.Rel(home, dir) + require.NoError(t, err) + require.False(t, strings.HasPrefix(rel, ".."), "skills dir %s escapes the test home", dir) + require.NoError(t, os.MkdirAll(filepath.Join(dir, "databricks-core"), 0o755)) +} + +func TestMaybeHint_PerAgent(t *testing.T) { + for _, a := range Registry { + if a.Name == NameClaudeCode { + continue // Claude's hint is covered by hint_test.go. + } + t.Run(a.Name, func(t *testing.T) { + setAgentEnv(t, agentDetectEnv[a.Name]) + + t.Run("not installed hints", func(t *testing.T) { + ctx, stderr := agentCtx(t, t.TempDir(), t.TempDir()) + ctx = telemetry.WithNewLogger(ctx) + MaybeHint(ctx, regularCmd(), noPluginRecorded) + assert.Equal(t, agentInstallHint(a, noPluginRecorded).text+"\n", stderr.String()) + + events := telemetry.BufferedEvents(ctx) + require.Len(t, events, 1) + assert.Equal(t, &protos.AitoolsHintEvent{ + HintKind: protos.AitoolsHintKindAgentInstallText, + }, events[0].AitoolsHintEvent) + }) + + t.Run("own skills installed is silent", func(t *testing.T) { + home := t.TempDir() + ctx, stderr := agentCtx(t, home, t.TempDir()) + ctx = telemetry.WithNewLogger(ctx) + writeAgentSkill(t, ctx, home, a) + MaybeHint(ctx, regularCmd(), noPluginRecorded) + assert.Empty(t, stderr.String()) + assert.Empty(t, telemetry.BufferedEvents(ctx)) + }) + + t.Run("canonical skills alone still hint", func(t *testing.T) { + // Skills in the shared store are not loaded by the agent unless + // they are also linked into its own skills dir. + home := t.TempDir() + ctx, stderr := agentCtx(t, home, t.TempDir()) + writeCanonicalSkill(t, home, "databricks") + MaybeHint(ctx, regularCmd(), noPluginRecorded) + assert.Equal(t, agentInstallHint(a, noPluginRecorded).text+"\n", stderr.String()) + }) + + t.Run("throttled per machine without session id", func(t *testing.T) { + home, cacheDir := t.TempDir(), t.TempDir() + ctx1, stderr1 := agentCtx(t, home, cacheDir) + MaybeHint(ctx1, regularCmd(), noPluginRecorded) + assert.NotEmpty(t, stderr1.String()) + + ctx2, stderr2 := agentCtx(t, home, cacheDir) + MaybeHint(ctx2, regularCmd(), noPluginRecorded) + assert.Empty(t, stderr2.String()) + }) + }) + } +} + +// noPluginRecorded is the CLI install record of a machine where `databricks +// aitools install` recorded no plugin. +func noPluginRecorded(context.Context, string) bool { return false } + +func TestMaybeHint_RecordedPluginSilent(t *testing.T) { + // Codex and Copilot have no plugin manifest reader: the CLI install record + // is their plugin source. + for _, name := range []string{NameCodex, NameCopilot} { + t.Run(name, func(t *testing.T) { + setAgentEnv(t, agentDetectEnv[name]) + var asked []string + recorded := func(_ context.Context, agentName string) bool { + asked = append(asked, agentName) + return agentName == name + } + ctx, stderr := agentCtx(t, t.TempDir(), t.TempDir()) + ctx = telemetry.WithNewLogger(ctx) + MaybeHint(ctx, regularCmd(), recorded) + assert.Empty(t, stderr.String()) + assert.Empty(t, telemetry.BufferedEvents(ctx)) + assert.Equal(t, []string{name}, asked) + }) + } +} + +func TestMaybeHint_CodexThrottledPerThread(t *testing.T) { + setAgentEnv(t, "CODEX_CI") + home, cacheDir := t.TempDir(), t.TempDir() + run := func(thread string) string { + ctx, stderr := agentCtx(t, home, cacheDir) + ctx = env.Set(ctx, "CODEX_THREAD_ID", thread) + MaybeHint(ctx, regularCmd(), noPluginRecorded) + return stderr.String() + } + + assert.Contains(t, run("thread-1"), "--agents codex") + assert.Empty(t, run("thread-1")) + // A new Codex session is hinted again even within the TTL. + assert.Contains(t, run("thread-2"), "--agents codex") +} + +func TestMaybeHint_NoInstallTargetSilent(t *testing.T) { + for _, envVar := range []string{"AMP_CURRENT_THREAD_ID", "AUGMENT_AGENT", "CLINE_ACTIVE", "OPENCLAW_SHELL", "VSCODE_AGENT", "WINDSURF_AGENT"} { + t.Run(envVar, func(t *testing.T) { + setAgentEnv(t, envVar) + ctx, stderr := agentCtx(t, t.TempDir(), t.TempDir()) + MaybeHint(ctx, regularCmd(), noPluginRecorded) + assert.Empty(t, stderr.String()) + }) + } +} + +func TestMaybeHint_MultipleAgentsSilent(t *testing.T) { + setAgentEnv(t, "CODEX_CI") + t.Setenv("CURSOR_AGENT", "1") + ctx, stderr := agentCtx(t, t.TempDir(), t.TempDir()) + MaybeHint(ctx, regularCmd(), noPluginRecorded) + assert.Empty(t, stderr.String()) +} + +func TestMaybeHint_AgentAuthTokenSilent(t *testing.T) { + setAgentEnv(t, "CODEX_CI") + ctx, stderr := agentCtx(t, t.TempDir(), t.TempDir()) + parent := &cobra.Command{Use: authCommandName} + child := &cobra.Command{Use: tokenCommandName} + parent.AddCommand(child) + MaybeHint(ctx, child, noPluginRecorded) + assert.Empty(t, stderr.String()) +} + +func TestMaybeHint_AgentAIToolsCommandSilent(t *testing.T) { + setAgentEnv(t, "CODEX_CI") + ctx, stderr := agentCtx(t, t.TempDir(), t.TempDir()) + parent := &cobra.Command{Use: aiToolsCommandName} + child := &cobra.Command{Use: "install"} + parent.AddCommand(child) + MaybeHint(ctx, child, noPluginRecorded) + assert.Empty(t, stderr.String()) +} + +func TestMaybeHint_SessionMarkersAreIsolatedPerAgent(t *testing.T) { + // Claude and Codex both report the same session id. A Claude marker must not + // silence Codex's first hint. + home, cacheDir := t.TempDir(), t.TempDir() + + setClaudeAgent(t) + ctx, stderr := agentCtx(t, home, cacheDir) + ctx = env.Set(ctx, claudeSessionEnv, "shared-id") + MaybeHint(ctx, regularCmd(), noPluginRecorded) + require.Contains(t, stderr.String(), hintText) + + setAgentEnv(t, "CODEX_CI") + ctx, stderr = agentCtx(t, home, cacheDir) + ctx = env.Set(ctx, "CODEX_THREAD_ID", "shared-id") + MaybeHint(ctx, regularCmd(), noPluginRecorded) + assert.Equal(t, agentInstallHint(ByName(NameCodex), noPluginRecorded).text+"\n", stderr.String()) +} + +func TestMaybeHint_UnwritableCacheSkipsHint(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("chmod does not make a directory read-only on Windows") + } + if os.Getuid() == 0 { + t.Skip("root can write to a read-only directory") + } + setAgentEnv(t, "CODEX_CI") + readOnly := t.TempDir() + require.NoError(t, os.Chmod(readOnly, 0o555)) + t.Cleanup(func() { _ = os.Chmod(readOnly, 0o755) }) + tmp := t.TempDir() + t.Setenv("TMPDIR", tmp) + + // Without a kept marker the hint would repeat on every command, so none is + // printed, on every call, and no other store (e.g. under TMPDIR) is used. + for range 2 { + ctx, stderr := agentCtx(t, t.TempDir(), readOnly) + ctx = telemetry.WithNewLogger(ctx) + ctx = env.Set(ctx, "CODEX_THREAD_ID", "thread-1") + MaybeHint(ctx, regularCmd(), noPluginRecorded) + assert.Empty(t, stderr.String()) + assert.Empty(t, telemetry.BufferedEvents(ctx)) + } + entries, err := os.ReadDir(tmp) + require.NoError(t, err) + assert.Empty(t, entries) +} diff --git a/libs/aitools/agents/hint_test.go b/libs/aitools/agents/hint_test.go index e92d02af488..d550afcbeea 100644 --- a/libs/aitools/agents/hint_test.go +++ b/libs/aitools/agents/hint_test.go @@ -64,6 +64,8 @@ func claudeCtx(t *testing.T, home string) (context.Context, *bytes.Buffer) { ctx, stderr := cmdio.NewTestContextWithStderr(t.Context()) ctx = env.WithUserHomeDir(ctx, home) ctx = env.Set(ctx, "DATABRICKS_CACHE_DIR", t.TempDir()) + // Throttle markers live in the file cache, which a host could disable. + ctx = env.Set(ctx, "DATABRICKS_CACHE_ENABLED", "true") return ctx, stderr } @@ -103,7 +105,7 @@ func TestMaybeHint_HumanCallerSilent(t *testing.T) { ctx, stderr := cmdio.NewTestContextWithStderr(t.Context()) ctx = env.WithUserHomeDir(ctx, t.TempDir()) ctx = telemetry.WithNewLogger(ctx) - MaybeHint(ctx, regularCmd()) + MaybeHint(ctx, regularCmd(), noPluginRecorded) assert.Empty(t, stderr.String()) assert.Empty(t, hintKinds(t, ctx)) } @@ -112,7 +114,7 @@ func TestMaybeHint_ClaudeNotInstalledHints(t *testing.T) { setClaudeAgent(t) ctx, stderr := claudeCtx(t, t.TempDir()) ctx = telemetry.WithNewLogger(ctx) - MaybeHint(ctx, regularCmd()) + MaybeHint(ctx, regularCmd(), noPluginRecorded) assert.Contains(t, stderr.String(), hintText) assert.Equal(t, []protos.AitoolsHintKind{protos.AitoolsHintKindInstallText}, hintKinds(t, ctx)) } @@ -126,7 +128,7 @@ func TestMaybeHint_PluginInstalledSilent(t *testing.T) { writePluginManifest(t, home, `{"plugins":{"databricks@claude-plugins-official":[{"scope":"user"}]}}`) ctx = telemetry.WithNewLogger(ctx) - MaybeHint(ctx, regularCmd()) + MaybeHint(ctx, regularCmd(), noPluginRecorded) assert.Empty(t, stderr.String()) assert.Empty(t, hintKinds(t, ctx)) } @@ -154,7 +156,7 @@ func TestMaybeHint_ProjectPluginCountsOnlyInItsProject(t *testing.T) { writePluginManifest(t, home, string(manifest)) t.Chdir(project) - MaybeHint(ctx, regularCmd()) + MaybeHint(ctx, regularCmd(), noPluginRecorded) if tc.wantHint { assert.Contains(t, stderr.String(), hintText) } else { @@ -211,7 +213,7 @@ func TestMaybeHint_CustomClaudeConfigDir(t *testing.T) { writePluginManifest(t, home, manifest) } - MaybeHint(ctx, regularCmd()) + MaybeHint(ctx, regularCmd(), noPluginRecorded) if tc.wantHint { assert.Contains(t, stderr.String(), hintText) } else { @@ -227,7 +229,7 @@ func TestMaybeHint_SkillsInstalledSilent(t *testing.T) { ctx, stderr := claudeCtx(t, home) writeClaudeSkill(t, home, "databricks") - MaybeHint(ctx, regularCmd()) + MaybeHint(ctx, regularCmd(), noPluginRecorded) assert.Empty(t, stderr.String()) } @@ -241,7 +243,7 @@ func TestMaybeHint_SharedStoreOnlyStillHints(t *testing.T) { ctx, stderr := claudeCtx(t, home) require.NoError(t, os.MkdirAll(filepath.Join(home, dir, "databricks"), 0o755)) - MaybeHint(ctx, regularCmd()) + MaybeHint(ctx, regularCmd(), noPluginRecorded) assert.Equal(t, hintText+"\n", stderr.String()) }) } @@ -253,7 +255,7 @@ func TestMaybeHint_NonDatabricksSkillStillHints(t *testing.T) { ctx, stderr := claudeCtx(t, home) writeCanonicalSkill(t, home, "some-other-skill") - MaybeHint(ctx, regularCmd()) + MaybeHint(ctx, regularCmd(), noPluginRecorded) assert.Contains(t, stderr.String(), hintText) } @@ -266,7 +268,7 @@ func TestMaybeHint_AIToolsCommandSilent(t *testing.T) { child := &cobra.Command{Use: "install"} parent.AddCommand(child) - MaybeHint(ctx, child) + MaybeHint(ctx, child, noPluginRecorded) assert.Empty(t, stderr.String()) assert.Empty(t, hintKinds(t, ctx)) } @@ -277,7 +279,7 @@ func TestMaybeHint_NoIONoPanic(t *testing.T) { // invocations before PersistentPreRunE runs, so HasIO can be false here. ctx := env.WithUserHomeDir(t.Context(), t.TempDir()) ctx = telemetry.WithNewLogger(ctx) - assert.NotPanics(t, func() { MaybeHint(ctx, regularCmd()) }) + assert.NotPanics(t, func() { MaybeHint(ctx, regularCmd(), noPluginRecorded) }) assert.Empty(t, hintKinds(t, ctx)) } @@ -294,13 +296,13 @@ func TestMaybeHint_ThrottledPerSession(t *testing.T) { } ctx1, stderr1 := newCtx() - MaybeHint(ctx1, regularCmd()) + MaybeHint(ctx1, regularCmd(), noPluginRecorded) assert.Contains(t, stderr1.String(), hintText) assert.Equal(t, []protos.AitoolsHintKind{protos.AitoolsHintKindInstallText}, hintKinds(t, ctx1)) // Same session id, second command: must stay quiet and log nothing. ctx2, stderr2 := newCtx() - MaybeHint(ctx2, regularCmd()) + MaybeHint(ctx2, regularCmd(), noPluginRecorded) assert.Empty(t, stderr2.String()) assert.Empty(t, hintKinds(t, ctx2)) } @@ -447,7 +449,7 @@ func TestMaybeHint_NativeHint(t *testing.T) { cmd = tc.cmd() } - MaybeHint(ctx, cmd) + MaybeHint(ctx, cmd, noPluginRecorded) assert.Equal(t, tc.wantStderr, stderr.String()) }) } @@ -457,10 +459,12 @@ func TestMaybeHint_NativeHintSilentForOtherCallers(t *testing.T) { for _, tc := range []struct { name string agentEnv []string + wantText string }{ - {"human", nil}, - {"other agent", []string{"CURSOR_AGENT", "1"}}, - {"multiple agents", []string{"CLAUDECODE", "1", "CURSOR_AGENT", "1"}}, + {"human", nil, ""}, + // Other agents get only their own text hint. + {"other agent", []string{"CURSOR_AGENT", "1"}, agentInstallHint(ByName(NameCursor), noPluginRecorded).text + "\n"}, + {"multiple agents", []string{"CLAUDECODE", "1", "CURSOR_AGENT", "1"}, ""}, } { t.Run(tc.name, func(t *testing.T) { testutil.SetAgentEnv(t, tc.agentEnv...) @@ -470,8 +474,8 @@ func TestMaybeHint_NativeHintSilentForOtherCallers(t *testing.T) { // Claude Code caller gets the native hint. ctx = env.Set(ctx, claudeChildSessionEnv, "1") - MaybeHint(ctx, regularCmd()) - assert.Empty(t, stderr.String()) + MaybeHint(ctx, regularCmd(), noPluginRecorded) + assert.Equal(t, tc.wantText, stderr.String()) }) } } @@ -491,19 +495,19 @@ func TestMaybeHint_NativeHintThrottledPerSession(t *testing.T) { } ctx1, stderr1 := newCtx() - MaybeHint(ctx1, regularCmd()) + MaybeHint(ctx1, regularCmd(), noPluginRecorded) assert.Equal(t, wantNativeHint+"\n"+hintText+"\n", stderr1.String()) // Same session, second command: both lines stay quiet, so the tag doesn't // reach a downstream reader such as `2>&1 | jq` on every command. ctx2, stderr2 := newCtx() - MaybeHint(ctx2, regularCmd()) + MaybeHint(ctx2, regularCmd(), noPluginRecorded) assert.Empty(t, stderr2.String()) // Another session: both lines print again. ctx3, stderr3 := newCtx() ctx3 = env.Set(ctx3, claudeSessionEnv, "session-def") - MaybeHint(ctx3, regularCmd()) + MaybeHint(ctx3, regularCmd(), noPluginRecorded) assert.Equal(t, wantNativeHint+"\n"+hintText+"\n", stderr3.String()) } @@ -540,7 +544,7 @@ func TestMaybeHint_NativeHintSilentOnTerminal(t *testing.T) { ctx = env.Set(ctx, "DATABRICKS_CACHE_DIR", t.TempDir()) ctx = env.Set(ctx, claudeChildSessionEnv, "1") - MaybeHint(ctx, regularCmd()) + MaybeHint(ctx, regularCmd(), noPluginRecorded) assert.Equal(t, hintText+"\n", stderr.String()) }) } @@ -552,14 +556,19 @@ func TestAgentHint_NativeLineNilIsEmpty(t *testing.T) { } func TestHintForAgent(t *testing.T) { - claude := hintForAgent(claudeAgentProvider) + claude := hintForAgent(claudeAgentProvider, noPluginRecorded) require.NotNil(t, claude) assert.Equal(t, hintText, claude.text) assert.Equal(t, claudeSessionEnv, claude.sessionEnv) - assert.Nil(t, hintForAgent("")) - assert.Nil(t, hintForAgent("multiple")) - assert.Nil(t, hintForAgent("cursor")) + assert.Nil(t, hintForAgent("", noPluginRecorded)) + assert.Nil(t, hintForAgent("multiple", noPluginRecorded)) + assert.Nil(t, hintForAgent("windsurf", noPluginRecorded)) + + cursor := hintForAgent("cursor", noPluginRecorded) + require.NotNil(t, cursor) + assert.Equal(t, protos.AitoolsHintKindAgentInstallText, cursor.textKind) + assert.Nil(t, cursor.native) } func TestMaybeHint_LogsEventsOncePerSession(t *testing.T) { @@ -580,7 +589,7 @@ func TestMaybeHint_LogsEventsOncePerSession(t *testing.T) { } ctx1, _ := newCtx(true) - MaybeHint(ctx1, regularCmd()) + MaybeHint(ctx1, regularCmd(), noPluginRecorded) assert.Equal(t, []protos.AitoolsHintKind{ protos.AitoolsHintKindClaudeCodeNative, protos.AitoolsHintKindInstallText, @@ -591,13 +600,13 @@ func TestMaybeHint_LogsEventsOncePerSession(t *testing.T) { // Throttled for the session: nothing prints and nothing logs, so the // command doesn't pay for a telemetry upload at exit. ctx2, stderr2 := newCtx(true) - MaybeHint(ctx2, regularCmd()) + MaybeHint(ctx2, regularCmd(), noPluginRecorded) assert.Empty(t, stderr2.String()) assert.Empty(t, telemetry.BufferedEvents(ctx2)) // Throttled and not a Claude Code subprocess: nothing prints, nothing logs. ctx3, stderr3 := newCtx(false) - MaybeHint(ctx3, regularCmd()) + MaybeHint(ctx3, regularCmd(), noPluginRecorded) assert.Empty(t, stderr3.String()) assert.Empty(t, telemetry.BufferedEvents(ctx3)) } @@ -610,6 +619,6 @@ func TestMaybeHint_InstalledLogsNothing(t *testing.T) { ctx = env.Set(ctx, claudeChildSessionEnv, "1") writePluginManifest(t, home, `{"plugins":{"databricks@claude-plugins-official":[{"scope":"user"}]}}`) - MaybeHint(ctx, regularCmd()) + MaybeHint(ctx, regularCmd(), noPluginRecorded) assert.Empty(t, telemetry.BufferedEvents(ctx)) } diff --git a/libs/aitools/installer/useragent.go b/libs/aitools/installer/useragent.go index 95a7cab6492..ff628e1d40d 100644 --- a/libs/aitools/installer/useragent.go +++ b/libs/aitools/installer/useragent.go @@ -63,15 +63,15 @@ func WithCallingAgentToolingInUserAgent(ctx context.Context) context.Context { if a == nil { return ctx } - tooling := a.DatabricksTooling(ctx, func() bool { return globalPluginRecorded(ctx, a.Name) }) + tooling := a.DatabricksTooling(ctx, func() bool { return GlobalPluginRecorded(ctx, a.Name) }) return useragent.InContext(ctx, callingAgentToolingUserAgentKey, tooling.String()) } -// globalPluginRecorded reports whether the global CLI install state records a +// GlobalPluginRecorded reports whether the global CLI install state records a // databricks plugin install for the agent. Only the global state is read: the // CLI installs plugins in project scope only for agents with SupportsProjectScope // (see mapAgentScope in cmd/aitools), and those have a manifest reader. -func globalPluginRecorded(ctx context.Context, agentName string) bool { +func GlobalPluginRecorded(ctx context.Context, agentName string) bool { dir, err := GlobalSkillsDir(ctx) if err != nil { return false diff --git a/libs/telemetry/protos/README.md b/libs/telemetry/protos/README.md index d07f4790963..4186b6614d7 100644 --- a/libs/telemetry/protos/README.md +++ b/libs/telemetry/protos/README.md @@ -6,15 +6,16 @@ which config options were set, the outcome and latency, and how long packaging a uploading the code snapshot took. It never includes code, paths, names, parameters, or image references. Set `DATABRICKS_CLI_DISABLE_TELEMETRY=1` to opt out. -The CLI logs an `aitools_hint_event` for each Claude Code install-hint line it prints (the native -`` line and the text line are one event each) and for the `apps init` skills tip -or install prompt. The install hint is never printed on aitools commands, `auth token`, -`auth docker token` or in CLI processes the CLI started itself. It carries only the kind of hint -(`hint_kind`); the command it was shown on is the entry's `execution_context.command`. The event -uploads at exit only when the command resolved auth. The install hint prints at most once per -Claude Code session per hour, so when the command that printed it resolves no auth (e.g. -`databricks version`), that session logs no install hint event for the next hour even though the -agent saw the hint. +The CLI logs an `aitools_hint_event` for each install-hint line it prints for a coding agent +(Claude Code's native `` line and its text line are one event each; another +agent's text line is one event) and for the `apps init` skills tip or install prompt. The install +hint is never printed on aitools commands, `auth token`, `auth docker token` or in CLI processes the +CLI started itself. It carries only the kind of hint (`hint_kind`); the command it was shown on is +the entry's `execution_context.command`. The event uploads at exit only when the command resolved +auth. The install hint prints at most once per hour per agent session (or per agent on the machine, +for agents without a session id), so when the command that printed it resolves no auth (e.g. +`databricks version`), no install hint event is logged for that hour even though the agent saw the +hint. `databricks aitools` install, update, list, version and uninstall log an `aitools_command_event` funnel per run: STARTED, one STEP_COMPLETED per install step reached (install only), and FINISHED diff --git a/libs/telemetry/protos/aitools_events.go b/libs/telemetry/protos/aitools_events.go index 50d7260aff9..37e0c548f41 100644 --- a/libs/telemetry/protos/aitools_events.go +++ b/libs/telemetry/protos/aitools_events.go @@ -13,8 +13,8 @@ const ( // `` line, which makes Claude Code offer to install the // Databricks plugin. AitoolsHintKindClaudeCodeNative AitoolsHintKind = "CLAUDE_CODE_NATIVE" - // AitoolsHintKindAgentInstallText mirrors an enum value of the proto for a - // hint the CLI does not print. Nothing logs it today. + // AitoolsHintKindAgentInstallText is the stderr line asking a non-Claude + // coding agent to run `databricks aitools install` for itself. AitoolsHintKindAgentInstallText AitoolsHintKind = "AGENT_INSTALL_TEXT" // AitoolsHintKindAppsInitTip is the non-interactive tip printed by // `databricks apps init`. @@ -24,13 +24,14 @@ const ( AitoolsHintKindAppsInitPrompt AitoolsHintKind = "APPS_INIT_PROMPT" ) -// AitoolsHintEvent is emitted once for each Claude Code install-hint line the -// CLI prints (the native line and the text line are one event each) and for the -// `apps init` skills tip or install prompt; other messages that mention -// `databricks aitools install` log nothing. It carries no user-authored strings. -// The command it was shown on is ExecutionContext.Command of the enclosing log -// entry. The CLI sets no calling-agent field on this event: the calling agent -// is reported by the User-Agent of the telemetry upload (`agent/`). +// AitoolsHintEvent is emitted once for each install-hint line the CLI prints +// for a coding agent (Claude Code's native and text lines are one event each) +// and for the `apps init` skills tip or install prompt; other messages that +// mention `databricks aitools install` log nothing. It carries no +// user-authored strings. The command it was shown on is ExecutionContext.Command +// of the enclosing log entry. The CLI sets no calling-agent field on this +// event: the calling agent is reported by the User-Agent of the telemetry +// upload (`agent/`). type AitoolsHintEvent struct { HintKind AitoolsHintKind `json:"hint_kind"` }