diff --git a/cmd/aitools/telemetry.go b/cmd/aitools/telemetry.go index 8ecf0c46542..ad7518c4958 100644 --- a/cmd/aitools/telemetry.go +++ b/cmd/aitools/telemetry.go @@ -43,6 +43,7 @@ func logInstallEvent(ctx context.Context, plan []agentPlanItem, opts installOpts Experimental: opts.Experimental, ErrorCategory: errCategory, AgentResults: agentResultsField(outcomes), + CallingAgent: agents.CallingAgentType(), }, }) } @@ -93,35 +94,10 @@ func agentsField(plan []agentPlanItem) []protos.AitoolsAgentType { return out } -// agentType maps a CLI agent registry name to its telemetry enum. Every agent -// in agents.Registry must have a case here; TestAgentTypeCoversRegistry fails -// if one is missing. The default maps genuinely unknown names to Unspecified -// so an older proto never drops an install. +// agentType maps a CLI agent registry name to its telemetry enum. +// TestAgentTypeCoversRegistry fails if a registry agent maps to Unspecified. func agentType(name string) protos.AitoolsAgentType { - switch name { - case agents.NameClaudeCode: - return protos.AitoolsAgentTypeClaudeCode - case agents.NameCursor: - return protos.AitoolsAgentTypeCursor - case agents.NameCodex: - return protos.AitoolsAgentTypeCodex - case agents.NameOpenCode: - return protos.AitoolsAgentTypeOpenCode - case agents.NameCopilot: - return protos.AitoolsAgentTypeCopilot - case agents.NameAntigravity: - return protos.AitoolsAgentTypeAntigravity - case agents.NamePi: - return protos.AitoolsAgentTypePi - case agents.NameGemini: - return protos.AitoolsAgentTypeGemini - case agents.NameGoose: - return protos.AitoolsAgentTypeGoose - case agents.NameKiro: - return protos.AitoolsAgentTypeKiro - default: - return protos.AitoolsAgentTypeUnspecified - } + return agents.AgentType(name) } // scopeType maps a resolved install scope to its telemetry enum. diff --git a/cmd/aitools/telemetry_test.go b/cmd/aitools/telemetry_test.go index 160bb4fe98e..df024a1b81a 100644 --- a/cmd/aitools/telemetry_test.go +++ b/cmd/aitools/telemetry_test.go @@ -1,14 +1,82 @@ package aitools import ( + "encoding/json" "testing" "github.com/databricks/cli/libs/aitools/agents" + "github.com/databricks/cli/libs/aitools/agents/agenttest" "github.com/databricks/cli/libs/aitools/installer" + "github.com/databricks/cli/libs/cmdctx" + "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/libs/telemetry" "github.com/databricks/cli/libs/telemetry/protos" + "github.com/databricks/cli/libs/testserver" + "github.com/databricks/databricks-sdk-go/config" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) +// uploadInstallEvent runs logInstallEvent, uploads the buffered log to a local +// /telemetry-ext and returns the install event as the server received it. +func uploadInstallEvent(t *testing.T) map[string]any { + server := testserver.New(t) + t.Cleanup(server.Close) + + var protoLogs []string + server.Handle("POST", "/telemetry-ext", func(req testserver.Request) any { + var body telemetry.RequestBody + require.NoError(t, json.Unmarshal(req.Body, &body)) + protoLogs = append(protoLogs, body.ProtoLogs...) + return telemetry.ResponseBody{NumProtoSuccess: int64(len(body.ProtoLogs))} + }) + + // Upload honours these from the host env; an empty value restores the + // defaults (enabled, 3s timeout) so the test does not depend on the runner. + ctx := env.Set(t.Context(), "DATABRICKS_CLI_DISABLE_TELEMETRY", "") + ctx = env.Set(ctx, "DATABRICKS_CLI_TELEMETRY_TIMEOUT", "") + ctx = telemetry.WithNewLogger(ctx) + plan := []agentPlanItem{{agent: &agents.Agent{Name: agents.NameCursor}}} + logInstallEvent(ctx, plan, installOpts{Scope: installer.ScopeGlobal}, protos.AitoolsErrorCategoryUnspecified, nil) + + // AuthType pins PAT so a host DATABRICKS_AUTH_TYPE cannot pick another method. + ctx = cmdctx.SetConfigUsed(ctx, &config.Config{Host: server.URL, Token: "token", AuthType: "pat"}) + require.NoError(t, telemetry.Upload(ctx, protos.ExecutionContext{})) + require.Len(t, protoLogs, 1) + + var frontendLog struct { + Entry struct { + DatabricksCliLog struct { + AitoolsInstallEvent map[string]any `json:"aitools_install_event"` + } `json:"databricks_cli_log"` + } `json:"entry"` + } + require.NoError(t, json.Unmarshal([]byte(protoLogs[0]), &frontendLog)) + event := frontendLog.Entry.DatabricksCliLog.AitoolsInstallEvent + require.NotNil(t, event) + return event +} + +func TestLogInstallEventSendsCallingAgent(t *testing.T) { + cases := []struct { + name string + env []string + want protos.AitoolsAgentType + }{ + {name: "claude code", env: []string{"CLAUDECODE", "1"}, want: protos.AitoolsAgentTypeClaudeCode}, + {name: "no agent", env: nil, want: protos.AitoolsAgentTypeNone}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + agenttest.SetAgentEnv(t, tc.env...) + event := uploadInstallEvent(t) + assert.Equal(t, string(tc.want), event["calling_agent"]) + // The calling agent is not an install target. + assert.Equal(t, []any{string(protos.AitoolsAgentTypeCursor)}, event["agents"]) + }) + } +} + func TestAgentsField(t *testing.T) { cases := []struct { name string @@ -74,7 +142,7 @@ func TestAgentsField(t *testing.T) { func TestAgentTypeCoversRegistry(t *testing.T) { for _, a := range agents.Registry { assert.NotEqualf(t, protos.AitoolsAgentTypeUnspecified, agentType(a.Name), - "agent %q has no telemetry enum: add a case to agentType and a value to "+ + "agent %q has no telemetry enum: add a case to agents.AgentType and a value to "+ "AitoolsAgentType in enum.proto (Universe) + aitools_install.go (CLI)", a.Name) } } diff --git a/libs/aitools/agents/agenttest/agenttest.go b/libs/aitools/agents/agenttest/agenttest.go new file mode 100644 index 00000000000..b988ea199a6 --- /dev/null +++ b/libs/aitools/agents/agenttest/agenttest.go @@ -0,0 +1,43 @@ +// Package agenttest controls which coding agent useragent.AgentProvider() +// reports in tests. +package agenttest + +import ( + "os" + "testing" + + "github.com/databricks/databricks-sdk-go/useragent" +) + +// agentEnvVars are the env vars useragent.AgentProvider() inspects: the SDK's +// listKnownAgents() (useragent/agent.go, unexported, so copied by hand) plus +// the AGENT / AI_AGENT fallbacks. +var agentEnvVars = []string{ + "AGENT", "AI_AGENT", "AMP_CURRENT_THREAD_ID", "ANTIGRAVITY_AGENT", + "AUGMENT_AGENT", "CLAUDECODE", "CLINE_ACTIVE", "CODEX_CI", "COPILOT_CLI", + "CURSOR_AGENT", "GEMINI_CLI", "GOOSE_TERMINAL", "KIRO", "OPENCLAW_SHELL", + "OPENCODE", "PI_CODING_AGENT", "VSCODE_AGENT", "WINDSURF_AGENT", +} + +// SetAgentEnv clears every agent env var (the agent running the test binary +// would otherwise leak in), then sets the given ones, and resets the SDK's +// process-wide AgentProvider cache before and after the test. With no +// arguments AgentProvider() reports no agent. Tests using it must not run in +// parallel. +func SetAgentEnv(t *testing.T, kv ...string) { + t.Helper() + if len(kv)%2 != 0 { + t.Fatalf("SetAgentEnv needs key/value pairs, got %d values", len(kv)) + } + for _, v := range agentEnvVars { + if orig, ok := os.LookupEnv(v); ok { + t.Cleanup(func() { os.Setenv(v, orig) }) //nolint:usetesting // restoring, not setting fresh + os.Unsetenv(v) //nolint:usetesting // t.Setenv cannot unset + } + } + for i := 0; i < len(kv); i += 2 { + t.Setenv(kv[i], kv[i+1]) + } + useragent.ClearCache() + t.Cleanup(useragent.ClearCache) +} diff --git a/libs/aitools/agents/calling.go b/libs/aitools/agents/calling.go new file mode 100644 index 00000000000..dcd35575f1d --- /dev/null +++ b/libs/aitools/agents/calling.go @@ -0,0 +1,113 @@ +package agents + +import ( + "github.com/databricks/cli/libs/telemetry/protos" + "github.com/databricks/databricks-sdk-go/useragent" +) + +// SDK agent products (useragent.AgentProvider values) whose names differ from +// the registry name, plus the SDK's marker for stacked agents. +// https://github.com/databricks/databricks-sdk-go/blob/main/useragent/agent.go +const ( + providerCopilot = "copilot-cli" + providerGemini = "gemini-cli" + providerMultiple = "multiple" +) + +// providerToRegistry maps SDK agent products to registry agent names. Products +// missing here have no registry entry, so the CLI cannot install into them. +var providerToRegistry = map[string]string{ + NameClaudeCode: NameClaudeCode, + NameCursor: NameCursor, + NameCodex: NameCodex, + NameOpenCode: NameOpenCode, + providerCopilot: NameCopilot, + NameAntigravity: NameAntigravity, + NamePi: NamePi, + providerGemini: NameGemini, + NameGoose: NameGoose, + NameKiro: NameKiro, +} + +// providerOnlyTypes are SDK agent products with no registry entry. They are +// reported as a calling agent only. +var providerOnlyTypes = map[string]protos.AitoolsAgentType{ + "amp": protos.AitoolsAgentTypeAmp, + "augment": protos.AitoolsAgentTypeAugment, + "cline": protos.AitoolsAgentTypeCline, + "openclaw": protos.AitoolsAgentTypeOpenClaw, + "vscode-agent": protos.AitoolsAgentTypeVSCodeAgent, + "windsurf": protos.AitoolsAgentTypeWindsurf, +} + +// AgentForProvider returns the registry agent for an SDK agent product +// (useragent.AgentProvider value), or nil when the product has no registry +// entry, is "multiple", or is empty. +func AgentForProvider(provider string) *Agent { + name, ok := providerToRegistry[provider] + if !ok { + return nil + } + return ByName(name) +} + +// CallingAgent returns the registry agent driving the CLI, or nil when no +// agent is detected, several are, or the agent is one the CLI cannot install +// into. +func CallingAgent() *Agent { + return AgentForProvider(useragent.AgentProvider()) +} + +// AgentType maps a registry agent name to its telemetry enum. Every agent in +// Registry has a case; unknown names map to Unspecified so an older proto +// never drops the event. +func AgentType(name string) protos.AitoolsAgentType { + switch name { + case NameClaudeCode: + return protos.AitoolsAgentTypeClaudeCode + case NameCursor: + return protos.AitoolsAgentTypeCursor + case NameCodex: + return protos.AitoolsAgentTypeCodex + case NameOpenCode: + return protos.AitoolsAgentTypeOpenCode + case NameCopilot: + return protos.AitoolsAgentTypeCopilot + case NameAntigravity: + return protos.AitoolsAgentTypeAntigravity + case NamePi: + return protos.AitoolsAgentTypePi + case NameGemini: + return protos.AitoolsAgentTypeGemini + case NameGoose: + return protos.AitoolsAgentTypeGoose + case NameKiro: + return protos.AitoolsAgentTypeKiro + default: + return protos.AitoolsAgentTypeUnspecified + } +} + +// ProviderAgentType maps an SDK agent product to the closed telemetry enum. +// Free text from the AGENT / AI_AGENT env vars, and products a newer SDK +// detects, become UNKNOWN, so the raw value never leaves the machine. +func ProviderAgentType(provider string) protos.AitoolsAgentType { + switch provider { + case "": + return protos.AitoolsAgentTypeNone + case providerMultiple: + return protos.AitoolsAgentTypeMultiple + } + if a := AgentForProvider(provider); a != nil { + return AgentType(a.Name) + } + if t, ok := providerOnlyTypes[provider]; ok { + return t + } + return protos.AitoolsAgentTypeUnknown +} + +// CallingAgentType is the telemetry enum for the agent driving the CLI. +func CallingAgentType() protos.AitoolsAgentType { + return ProviderAgentType(useragent.AgentProvider()) +} diff --git a/libs/aitools/agents/calling_test.go b/libs/aitools/agents/calling_test.go new file mode 100644 index 00000000000..16ac09d310a --- /dev/null +++ b/libs/aitools/agents/calling_test.go @@ -0,0 +1,104 @@ +package agents_test + +import ( + "testing" + + "github.com/databricks/cli/libs/aitools/agents" + "github.com/databricks/cli/libs/aitools/agents/agenttest" + "github.com/databricks/cli/libs/telemetry/protos" + "github.com/stretchr/testify/assert" +) + +func TestProviderAgentType(t *testing.T) { + cases := map[string]protos.AitoolsAgentType{ + // Every product in the SDK's listKnownAgents (v0.195.0). The SDK does + // not export that list, so a product a newer SDK adds maps to UNKNOWN + // without failing here: on every SDK bump, diff useragent/agent.go and + // extend this table, providerToRegistry or providerOnlyTypes, and + // agenttest's agentEnvVars. + "amp": protos.AitoolsAgentTypeAmp, + "antigravity": protos.AitoolsAgentTypeAntigravity, + "augment": protos.AitoolsAgentTypeAugment, + "claude-code": protos.AitoolsAgentTypeClaudeCode, + "cline": protos.AitoolsAgentTypeCline, + "codex": protos.AitoolsAgentTypeCodex, + "copilot-cli": protos.AitoolsAgentTypeCopilot, + "cursor": protos.AitoolsAgentTypeCursor, + "gemini-cli": protos.AitoolsAgentTypeGemini, + "goose": protos.AitoolsAgentTypeGoose, + "kiro": protos.AitoolsAgentTypeKiro, + "openclaw": protos.AitoolsAgentTypeOpenClaw, + "opencode": protos.AitoolsAgentTypeOpenCode, + "pi": protos.AitoolsAgentTypePi, + "vscode-agent": protos.AitoolsAgentTypeVSCodeAgent, + "windsurf": protos.AitoolsAgentTypeWindsurf, + + "multiple": protos.AitoolsAgentTypeMultiple, + "": protos.AitoolsAgentTypeNone, + + // AGENT / AI_AGENT free text never passes through. + "my-private-agent-name": protos.AitoolsAgentTypeUnknown, + // Registry names that differ from the SDK product are not SDK values. + "copilot": protos.AitoolsAgentTypeUnknown, + "gemini": protos.AitoolsAgentTypeUnknown, + } + for provider, want := range cases { + assert.Equal(t, want, agents.ProviderAgentType(provider), "provider %q", provider) + } +} + +func TestAgentForProvider(t *testing.T) { + assert.Equal(t, agents.NameCopilot, agents.AgentForProvider("copilot-cli").Name) + assert.Equal(t, agents.NameGemini, agents.AgentForProvider("gemini-cli").Name) + assert.Equal(t, agents.NameClaudeCode, agents.AgentForProvider("claude-code").Name) + assert.Nil(t, agents.AgentForProvider("windsurf")) + assert.Nil(t, agents.AgentForProvider("multiple")) + assert.Nil(t, agents.AgentForProvider("")) + assert.Nil(t, agents.AgentForProvider("copilot")) +} + +// Every registry agent must be reachable from some SDK product, or it could +// never be the calling agent. +func TestAgentForProviderCoversRegistry(t *testing.T) { + products := []string{"antigravity", "claude-code", "codex", "copilot-cli", "cursor", "gemini-cli", "goose", "kiro", "opencode", "pi"} + reached := map[string]bool{} + for _, p := range products { + if a := agents.AgentForProvider(p); a != nil { + reached[a.Name] = true + } + } + for _, a := range agents.Registry { + assert.True(t, reached[a.Name], "registry agent %q has no SDK product mapping", a.Name) + } +} + +func TestAgentTypeCoversRegistry(t *testing.T) { + for _, a := range agents.Registry { + assert.NotEqual(t, protos.AitoolsAgentTypeUnspecified, agents.AgentType(a.Name), a.Name) + } + assert.Equal(t, protos.AitoolsAgentTypeUnspecified, agents.AgentType("future-agent")) +} + +func TestCallingAgentFromEnv(t *testing.T) { + agenttest.SetAgentEnv(t, "COPILOT_CLI", "1") + assert.Equal(t, agents.NameCopilot, agents.CallingAgent().Name) + assert.Equal(t, protos.AitoolsAgentTypeCopilot, agents.CallingAgentType()) +} + +func TestCallingAgentFreeTextIsUnknown(t *testing.T) { + agenttest.SetAgentEnv(t, "AGENT", "some-internal-tool") + assert.Nil(t, agents.CallingAgent()) + assert.Equal(t, protos.AitoolsAgentTypeUnknown, agents.CallingAgentType()) +} + +func TestCallingAgentNone(t *testing.T) { + agenttest.SetAgentEnv(t) + assert.Nil(t, agents.CallingAgent()) + assert.Equal(t, protos.AitoolsAgentTypeNone, agents.CallingAgentType()) +} + +func TestCallingAgentMultiple(t *testing.T) { + agenttest.SetAgentEnv(t, "CLAUDECODE", "1", "CURSOR_AGENT", "1") + assert.Nil(t, agents.CallingAgent()) + assert.Equal(t, protos.AitoolsAgentTypeMultiple, agents.CallingAgentType()) +} diff --git a/libs/telemetry/protos/aitools_install.go b/libs/telemetry/protos/aitools_install.go index c23503eea09..5f50feefb57 100644 --- a/libs/telemetry/protos/aitools_install.go +++ b/libs/telemetry/protos/aitools_install.go @@ -1,8 +1,10 @@ package protos // AitoolsAgentType mirrors AitoolsAgentType.Type in the databricks_cli -// lumberjack proto. AitoolsAgentTypeUnspecified absorbs agents a newer CLI -// knows about but the server proto does not yet. +// lumberjack proto. It names both the agents `aitools install` targets and the +// agent calling the CLI (the calling_agent field of the aitools events). +// AitoolsAgentTypeUnspecified absorbs agents a newer CLI knows about but the +// server proto does not yet. type AitoolsAgentType string const ( @@ -17,6 +19,25 @@ const ( AitoolsAgentTypeGemini AitoolsAgentType = "GEMINI_CLI" AitoolsAgentTypeGoose AitoolsAgentType = "GOOSE" AitoolsAgentTypeKiro AitoolsAgentType = "KIRO" + + // Agents the SDK detects but the aitools registry cannot install into. + // They only appear as a calling agent. + AitoolsAgentTypeAmp AitoolsAgentType = "AMP" + AitoolsAgentTypeAugment AitoolsAgentType = "AUGMENT" + AitoolsAgentTypeCline AitoolsAgentType = "CLINE" + AitoolsAgentTypeOpenClaw AitoolsAgentType = "OPENCLAW" + AitoolsAgentTypeVSCodeAgent AitoolsAgentType = "VSCODE_AGENT" + AitoolsAgentTypeWindsurf AitoolsAgentType = "WINDSURF" + + // AitoolsAgentTypeUnknown is an agent reported only through the free-text + // AGENT / AI_AGENT env vars, or a product a newer SDK detects. The text + // itself is never sent. + AitoolsAgentTypeUnknown AitoolsAgentType = "UNKNOWN" + // AitoolsAgentTypeMultiple means more than one agent env var matched, + // typically an agent running another agent as a subprocess. + AitoolsAgentTypeMultiple AitoolsAgentType = "MULTIPLE" + // AitoolsAgentTypeNone means no agent was detected: a human or a script. + AitoolsAgentTypeNone AitoolsAgentType = "NONE" ) // AitoolsInstallScope mirrors AitoolsInstallScope.Type in the databricks_cli @@ -104,4 +125,7 @@ type AitoolsInstallEvent struct { // AgentResults records the per-agent failure/skip categories, one entry per // non-successful agent. Empty when every targeted agent succeeded. AgentResults []AitoolsAgentResult `json:"agent_results,omitempty"` + + // Agent calling the CLI. Always set (NONE for humans), so no omitempty. + CallingAgent AitoolsAgentType `json:"calling_agent"` }