diff --git a/.nextchanges/cli/feedback.md b/.nextchanges/cli/feedback.md new file mode 100644 index 00000000000..f8b16deb67e --- /dev/null +++ b/.nextchanges/cli/feedback.md @@ -0,0 +1 @@ +* Add the `databricks feedback` command to send feedback about the CLI to Databricks ([#7567](https://github.com/databricks/cli/pull/7567)). diff --git a/acceptance/cmd/feedback/out.test.toml b/acceptance/cmd/feedback/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/cmd/feedback/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/cmd/feedback/output.txt b/acceptance/cmd/feedback/output.txt new file mode 100644 index 00000000000..e1dcb9f8ed9 --- /dev/null +++ b/acceptance/cmd/feedback/output.txt @@ -0,0 +1,129 @@ + +=== Feedback about a product command is routed to its team + +>>> [CLI] feedback --command air run --run-id 123 --can-be-contacted logs omitted the failure reason +Thank you! Your feedback was sent. Reference: [UUID][0] +{ + "frontend_log_event_id": "[UUID][0]", + "entry": { + "fast_feedback_log": { + "reference_event_id": "[UUID][1]", + "parent_ui": "ModelTraining.AIRuntime", + "origin": "air_run", + "can_be_contacted": true, + "ui_versions": "cli:[CLI_VERSION]", + "user_agent": "cli/[CLI_VERSION] cmd/feedback run-id/123", + "text": "logs omitted the failure reason" + } + } +} + +=== A command group, with a JSON receipt + +>>> [CLI] feedback --command bundle deploy should print which resources changed -o json +{ + "received": true, + "feedback_id": "[UUID][2]" +} +{ + "frontend_log_event_id": "[UUID][2]", + "entry": { + "fast_feedback_log": { + "reference_event_id": "[UUID][3]", + "parent_ui": "DeveloperEcosystem.CoreDabs", + "origin": "bundle", + "can_be_contacted": false, + "ui_versions": "cli:[CLI_VERSION]", + "user_agent": "cli/[CLI_VERSION] cmd/feedback", + "text": "deploy should print which resources changed" + } + } +} + +=== Top-level feedback can name any command + +>>> [CLI] feedback --command bundle deploy deploy is slow +Thank you! Your feedback was sent. Reference: [UUID][4] +{ + "frontend_log_event_id": "[UUID][4]", + "entry": { + "fast_feedback_log": { + "reference_event_id": "[UUID][5]", + "parent_ui": "DeveloperEcosystem.CoreDabs", + "origin": "bundle_deploy", + "can_be_contacted": false, + "ui_versions": "cli:[CLI_VERSION]", + "user_agent": "cli/[CLI_VERSION] cmd/feedback", + "text": "deploy is slow" + } + } +} + +=== General CLI feedback + +>>> [CLI] feedback the CLI should support ... +Thank you! Your feedback was sent. Reference: [UUID][6] +{ + "frontend_log_event_id": "[UUID][6]", + "entry": { + "fast_feedback_log": { + "reference_event_id": "[UUID][7]", + "parent_ui": "DeveloperEcosystem.DatabricksCLI", + "origin": "cli", + "can_be_contacted": false, + "ui_versions": "cli:[CLI_VERSION]", + "user_agent": "cli/[CLI_VERSION] cmd/feedback", + "text": "the CLI should support ..." + } + } +} + +=== An AI agent needs the user's confirmation + +>>> env AGENT=test-agent [CLI] feedback hello +Error: feedback from an AI agent must be confirmed by the user; run with --yes once the user approves the message + +Note for AI agents (test-agent): show the user the exact feedback message and pass --yes only after the user approves sending it. + +Exit code: 1 + +=== An AI agent with --yes + +>>> env AGENT=test-agent [CLI] feedback --yes hello +Thank you! Your feedback was sent. Reference: [UUID][8] +{ + "frontend_log_event_id": "[UUID][8]", + "entry": { + "fast_feedback_log": { + "reference_event_id": "[UUID][9]", + "parent_ui": "DeveloperEcosystem.DatabricksCLI", + "origin": "cli", + "can_be_contacted": false, + "ui_versions": "cli:[CLI_VERSION]", + "user_agent": "cli/[CLI_VERSION] cmd/feedback agent/test-agent", + "text": "hello" + } + } +} + +=== Validation errors + +>>> [CLI] feedback +Error: feedback message must contain 1 to 10000 characters + +Exit code: 1 + +>>> [CLI] feedback --command no-such-command hello +Error: unknown command "no-such-command" + +Exit code: 1 + +>>> [CLI] feedback --command bundle no-such-command hello +Error: unknown command "bundle no-such-command" + +Exit code: 1 + +>>> [CLI] feedback --run-id abc hello +Error: --run-id must be a positive job run ID + +Exit code: 1 diff --git a/acceptance/cmd/feedback/script b/acceptance/cmd/feedback/script new file mode 100644 index 00000000000..83ad87c60dd --- /dev/null +++ b/acceptance/cmd/feedback/script @@ -0,0 +1,36 @@ +# Print the feedback event, keeping only the user agent entries that carry +# feedback context; the rest depend on the runner. +print_feedback() { + jq 'select(.path == "/telemetry-ext") | .body.protoLogs[] | fromjson + | .entry.fast_feedback_log.user_agent |= (split(" ") | map(select(test("^(cli|cmd|run-id|agent)/"))) | join(" "))' < "$OUT_REQUESTS" + rm "$OUT_REQUESTS" +} + +title "Feedback about a product command is routed to its team" +trace $CLI feedback --command "air run" --run-id 123 --can-be-contacted " logs omitted the failure reason " +print_feedback + +title "A command group, with a JSON receipt" +trace $CLI feedback --command bundle "deploy should print which resources changed" -o json +print_feedback + +title "Top-level feedback can name any command" +trace $CLI feedback --command "bundle deploy" "deploy is slow" +print_feedback + +title "General CLI feedback" +trace $CLI feedback "the CLI should support ..." +print_feedback + +title "An AI agent needs the user's confirmation" +errcode trace env AGENT=test-agent $CLI feedback "hello" + +title "An AI agent with --yes" +trace env AGENT=test-agent $CLI feedback --yes "hello" +print_feedback + +title "Validation errors" +errcode trace $CLI feedback " " +errcode trace $CLI feedback --command "no-such-command" "hello" +errcode trace $CLI feedback --command "bundle no-such-command" "hello" +errcode trace $CLI feedback --run-id abc "hello" diff --git a/acceptance/cmd/feedback/test.toml b/acceptance/cmd/feedback/test.toml new file mode 100644 index 00000000000..159efe02696 --- /dev/null +++ b/acceptance/cmd/feedback/test.toml @@ -0,0 +1 @@ +RecordRequests = true diff --git a/acceptance/help/output.txt b/acceptance/help/output.txt index 041a319a3d5..40e1fb4c4a6 100644 --- a/acceptance/help/output.txt +++ b/acceptance/help/output.txt @@ -183,6 +183,7 @@ Domains Developer Tools air Run and manage AI Runtime training workloads bundle Declarative Automation Bundles let you express data/AI/analytics projects as code. + feedback Send feedback about the Databricks CLI to Databricks sandbox *Beta* Manage Databricks Sandbox environments sync Synchronize a local directory to a workspace directory diff --git a/cmd/cmd.go b/cmd/cmd.go index 9bc26de1785..24957e6fc74 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -17,6 +17,7 @@ import ( "github.com/databricks/cli/cmd/completion" "github.com/databricks/cli/cmd/configure" "github.com/databricks/cli/cmd/experimental" + "github.com/databricks/cli/cmd/feedback" "github.com/databricks/cli/cmd/fs" "github.com/databricks/cli/cmd/labs" "github.com/databricks/cli/cmd/pipelines" @@ -126,6 +127,7 @@ func New(ctx context.Context) *cobra.Command { cli.AddCommand(bundle.New()) cli.AddCommand(cache.New()) cli.AddCommand(experimental.New()) + cli.AddCommand(feedback.New()) cli.AddCommand(psql.New()) cli.AddCommand(configure.New()) cli.AddCommand(fs.New()) diff --git a/cmd/feedback/feedback.go b/cmd/feedback/feedback.go new file mode 100644 index 00000000000..37a9cc392e0 --- /dev/null +++ b/cmd/feedback/feedback.go @@ -0,0 +1,185 @@ +package feedback + +import ( + "context" + "errors" + "fmt" + "strconv" + "strings" + "time" + "unicode/utf8" + "uuid" + + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/internal/build" + "github.com/databricks/cli/libs/agent" + "github.com/databricks/cli/libs/cmdctx" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/telemetry" + "github.com/databricks/cli/libs/telemetry/protos" + "github.com/databricks/databricks-sdk-go/client" + "github.com/databricks/databricks-sdk-go/useragent" + "github.com/spf13/cobra" +) + +const ( + messageLimit = 10000 + sendTimeout = 30 * time.Second + + // ES component feedback is filed under unless the command it is about + // belongs to a product listed in productComponents. + defaultComponent = "DeveloperEcosystem.DatabricksCLI" + + // User agent key for context the fast feedback schema has no field for. + // The CLI version and OS are already part of the user agent. + runIDKey = "run-id" +) + +// ES components for top-level commands owned by a product team, so their +// feedback reaches that team's triage queue. +var productComponents = map[string]string{ + "air": "ModelTraining.AIRuntime", + "bundle": "DeveloperEcosystem.CoreDabs", +} + +const receiptTemplate = `Thank you! Your feedback was sent. Reference: {{.FeedbackID}} +` + +type receipt struct { + Received bool `json:"received"` + FeedbackID string `json:"feedback_id"` +} + +func New() *cobra.Command { + cmd := &cobra.Command{ + Use: "feedback MESSAGE", + Short: "Send feedback about the Databricks CLI to Databricks", + GroupID: "development", + Long: `Send feedback about the Databricks CLI to Databricks. + +Describe the command you ran, what happened, what you expected, and any +workaround. Use --command to name the command the feedback is about so it +reaches the team that owns it. + +The message is sent with the workspace and user you are authenticated as and +the CLI user agent, which includes the CLI version, operating system and, if +detected, the AI agent running the CLI. No logs, files or transcripts are +collected. Do not include credentials or sensitive data. Follow-up contact is +opt-in with --can-be-contacted. + +AI agents: submit feedback only with the user's approval. When an agent is +detected, the command asks for confirmation, or requires --yes when no terminal +is attached. Show the user the exact message and pass --yes only after the user +approves it.`, + Example: ` databricks feedback --command "bundle deploy" "deploy should print which resources changed" + databricks feedback --command "air run" --run-id 123 "logs omitted the failure reason"`, + Args: root.ExactArgs(1), + PreRunE: root.MustWorkspaceClient, + } + + var command string + var runID string + var canBeContacted bool + var yes bool + cmd.Flags().StringVar(&command, "command", "", `CLI command the feedback is about, for example "bundle deploy"`) + cmd.Flags().StringVar(&runID, "run-id", "", "ID of the job run the feedback is about") + cmd.Flags().BoolVar(&canBeContacted, "can-be-contacted", false, "Allow Databricks to contact you about this feedback") + cmd.Flags().BoolVarP(&yes, "yes", "y", false, "Confirm sending feedback written by an AI agent") + + cmd.RunE = func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + + text := strings.TrimSpace(args[0]) + if text == "" || utf8.RuneCountInString(text) > messageLimit { + return fmt.Errorf("feedback message must contain 1 to %d characters", messageLimit) + } + + component := defaultComponent + origin := "cli" + if command != "" { + target, rest, err := cmd.Root().Find(strings.Fields(command)) + if err != nil || len(rest) > 0 || target == cmd.Root() { + return fmt.Errorf("unknown command %q", command) + } + origin = root.CommandString(target) + if c, ok := productComponents[topLevelName(target)]; ok { + component = c + } + } + if runID != "" { + if id, err := strconv.ParseInt(runID, 10, 64); err != nil || id <= 0 { + return errors.New("--run-id must be a positive job run ID") + } + ctx = useragent.InContext(ctx, runIDKey, runID) + } + + if provider := useragent.AgentProvider(); provider != "" { + // The SDK adds the agent only to HTTP request headers; the feedback + // record needs it too so triage can tell agent reports apart. + ctx = useragent.InContext(ctx, useragent.AgentKey, provider) + if !yes { + ok, err := confirm(ctx, text) + if err != nil || !ok { + return err + } + } + } + + id := uuid.New().String() + apiClient, err := client.New(cmdctx.WorkspaceClient(ctx).Config) + if err != nil { + return err + } + + sendCtx, cancel := context.WithTimeout(ctx, sendTimeout) + defer cancel() + err = telemetry.Send(sendCtx, apiClient, []protos.FrontendLog{{ + FrontendLogEventID: id, + Entry: protos.FrontendLogEntry{ + FastFeedbackLog: &protos.FastFeedbackLog{ + ReferenceEventID: cmdctx.ExecId(ctx), + ParentUI: component, + Origin: origin, + CanBeContacted: canBeContacted, + UIVersions: "cli:" + build.GetInfo().Version, + UserAgent: useragent.FromContext(ctx), + Text: text, + }, + }, + }}) + if err != nil { + // The upload may have been accepted before the error, so retrying + // can file the same feedback twice. + return fmt.Errorf("feedback %s could not be confirmed as sent: %w", id, err) + } + + return cmdio.RenderWithTemplate(ctx, receipt{Received: true, FeedbackID: id}, "", receiptTemplate) + } + + return cmd +} + +// confirm asks the user to approve feedback an AI agent is about to send. It +// returns false without error if the user declines. +func confirm(ctx context.Context, text string) (bool, error) { + if !cmdio.IsPromptSupported(ctx) { + return false, errors.New("feedback from an AI agent must be confirmed by the user; run with --yes once the user approves the message\n\n" + + agent.AgentNote("show the user the exact feedback message and pass --yes only after the user approves sending it.")) + } + cmdio.LogString(ctx, "The following feedback will be sent to Databricks:\n\n"+text+"\n") + ok, err := cmdio.AskYesOrNo(ctx, "Send this feedback?") + if err != nil { + return false, err + } + if !ok { + cmdio.LogString(ctx, "Feedback not sent.") + } + return ok, nil +} + +func topLevelName(cmd *cobra.Command) string { + for cmd.HasParent() && cmd.Parent().HasParent() { + cmd = cmd.Parent() + } + return cmd.Name() +} diff --git a/cmd/root/root.go b/cmd/root/root.go index 59b9777ef69..e0be3239a60 100644 --- a/cmd/root/root.go +++ b/cmd/root/root.go @@ -194,7 +194,7 @@ Stack Trace: exitCode = 1 } - commandStr := commandString(cmd) + commandStr := CommandString(cmd) ctx = cmd.Context() telemetryErr := telemetry.Upload(cmd.Context(), protos.ExecutionContext{ diff --git a/cmd/root/user_agent_command.go b/cmd/root/user_agent_command.go index c1a9adf27cd..cd5f3316cb8 100644 --- a/cmd/root/user_agent_command.go +++ b/cmd/root/user_agent_command.go @@ -14,9 +14,9 @@ import ( // See unit test [main.TestCommandsDontUseUnderscoreInName]. const commandSeparator = "_" -// commandString walks up the command hierarchy of the specified +// CommandString walks up the command hierarchy of the specified // command to build a string representing this hierarchy. -func commandString(cmd *cobra.Command) string { +func CommandString(cmd *cobra.Command) string { reversed := []string{cmd.Name()} cmd.VisitParents(func(p *cobra.Command) { if !p.HasParent() { @@ -30,5 +30,5 @@ func commandString(cmd *cobra.Command) string { } func withCommandInUserAgent(ctx context.Context, cmd *cobra.Command) context.Context { - return useragent.InContext(ctx, "cmd", commandString(cmd)) + return useragent.InContext(ctx, "cmd", CommandString(cmd)) } diff --git a/cmd/root/user_agent_command_test.go b/cmd/root/user_agent_command_test.go index 9d76d2011cb..7b6907e2dc2 100644 --- a/cmd/root/user_agent_command_test.go +++ b/cmd/root/user_agent_command_test.go @@ -24,9 +24,9 @@ func TestWithCommandInUserAgent(t *testing.T) { root.AddCommand(hello) hello.AddCommand(world) - assert.Equal(t, "root", commandString(root)) - assert.Equal(t, "hello", commandString(hello)) - assert.Equal(t, "hello_world", commandString(world)) + assert.Equal(t, "root", CommandString(root)) + assert.Equal(t, "hello", CommandString(hello)) + assert.Equal(t, "hello_world", CommandString(world)) ctx := withCommandInUserAgent(t.Context(), world) diff --git a/libs/telemetry/logger.go b/libs/telemetry/logger.go index aac2063fbe5..f2da8bc01d8 100644 --- a/libs/telemetry/logger.go +++ b/libs/telemetry/logger.go @@ -114,13 +114,9 @@ func Upload(ctx context.Context, ec protos.ExecutionContext) error { l.logs[i].Entry.DatabricksCliLog.ExecutionContext = &ec } - protoLogs := make([]string, len(l.logs)) - for i, log := range l.logs { - b, err := json.Marshal(log) - if err != nil { - return fmt.Errorf("failed to marshal log: %w", err) - } - protoLogs[i] = string(b) + protoLogs, err := marshalLogs(l.logs) + if err != nil { + return err } if !cmdctx.HasConfigUsed(ctx) { @@ -315,6 +311,38 @@ func authenticate(r *http.Request, cfg *config.Config, running *atomic.Int32) er return nil } +// Send uploads logs in a single request and returns an error unless all of them +// were accepted. It is meant for events the user explicitly asked to send, so +// unlike Upload it does not honor DATABRICKS_CLI_DISABLE_TELEMETRY or run in the +// background after the command. +func Send(ctx context.Context, apiClient *client.DatabricksClient, logs []protos.FrontendLog) error { + protoLogs, err := marshalLogs(logs) + if err != nil { + return err + } + + resp, err := attempt(ctx, apiClient, protoLogs) + if err != nil { + return err + } + if resp.NumProtoSuccess < int64(len(protoLogs)) { + return fmt.Errorf("only %d of %d logs were accepted", resp.NumProtoSuccess, len(protoLogs)) + } + return nil +} + +func marshalLogs(logs []protos.FrontendLog) ([]string, error) { + protoLogs := make([]string, len(logs)) + for i, log := range logs { + b, err := json.Marshal(log) + if err != nil { + return nil, fmt.Errorf("failed to marshal log: %w", err) + } + protoLogs[i] = string(b) + } + return protoLogs, nil +} + func attempt(ctx context.Context, apiClient *client.DatabricksClient, protoLogs []string) (*ResponseBody, error) { // Without the workspace routing header, telemetry on unified hosts is // recorded in a central shard instead of the user's workspace. diff --git a/libs/telemetry/logger_test.go b/libs/telemetry/logger_test.go index 94673a9dd1a..8d2ccc857af 100644 --- a/libs/telemetry/logger_test.go +++ b/libs/telemetry/logger_test.go @@ -2,6 +2,7 @@ package telemetry import ( "context" + "encoding/json" "errors" "sync/atomic" "testing" @@ -12,6 +13,7 @@ import ( "github.com/databricks/cli/libs/telemetry/protos" "github.com/databricks/cli/libs/testserver" "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/client" "github.com/databricks/databricks-sdk-go/config" "github.com/databricks/databricks-sdk-go/config/experimental/auth" "github.com/stretchr/testify/assert" @@ -465,3 +467,43 @@ func TestWithinReturnsPanicAsError(t *testing.T) { assert.EqualError(t, err, "panic: boom") assert.Eventually(t, func() bool { return running.Load() == 0 }, time.Second, time.Millisecond) } + +func TestSend(t *testing.T) { + tests := []struct { + name string + numProtoSuccess int64 + wantErr string + }{ + {name: "all accepted", numProtoSuccess: 1}, + {name: "none accepted", numProtoSuccess: 0, wantErr: "only 0 of 1 logs were accepted"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + server := testserver.New(t) + t.Cleanup(server.Close) + + var body RequestBody + server.Handle("POST", "/telemetry-ext", func(req testserver.Request) any { + require.NoError(t, json.Unmarshal(req.Body, &body)) + return ResponseBody{NumProtoSuccess: tt.numProtoSuccess} + }) + + apiClient, err := client.New(&config.Config{Host: server.URL, Token: "token"}) + require.NoError(t, err) + + err = Send(t.Context(), apiClient, []protos.FrontendLog{{ + FrontendLogEventID: "id", + Entry: protos.FrontendLogEntry{ + FastFeedbackLog: &protos.FastFeedbackLog{Text: "hello"}, + }, + }}) + if tt.wantErr != "" { + assert.EqualError(t, err, tt.wantErr) + } else { + require.NoError(t, err) + } + assert.Equal(t, []string{`{"frontend_log_event_id":"id","entry":{"fast_feedback_log":{"can_be_contacted":false,"text":"hello"}}}`}, body.ProtoLogs) + }) + } +} diff --git a/libs/telemetry/protos/fast_feedback_log.go b/libs/telemetry/protos/fast_feedback_log.go new file mode 100644 index 00000000000..ff51d491ae5 --- /dev/null +++ b/libs/telemetry/protos/fast_feedback_log.go @@ -0,0 +1,26 @@ +package protos + +// FastFeedbackLog corresponds to the FastFeedbackLog lumberjack proto in universe +// (proto/logs/frontend/docs-platform/fast_feedback_log.proto). It is the event the +// in-product feedback widget records, so CLI feedback is triaged the same way. +type FastFeedbackLog struct { + // ID of an event the feedback relates to. The CLI sets the command execution ID so + // triagers can find the matching requests in http_access_log. + ReferenceEventID string `json:"reference_event_id,omitempty"` + + // ES component the feedback is filed under (e.g. "DeveloperEcosystem.DatabricksCLI"). + ParentUI string `json:"parent_ui,omitempty"` + + // Feature within the parent UI the feedback is about. + Origin string `json:"origin,omitempty"` + + CanBeContacted bool `json:"can_be_contacted"` + + // Comma-separated "key:value" pairs of component versions. + UIVersions string `json:"ui_versions,omitempty"` + + UserAgent string `json:"user_agent,omitempty"` + + // Feedback text written by the user. + Text string `json:"text,omitempty"` +} diff --git a/libs/telemetry/protos/frontend_log.go b/libs/telemetry/protos/frontend_log.go index 355600e1896..4f949689dae 100644 --- a/libs/telemetry/protos/frontend_log.go +++ b/libs/telemetry/protos/frontend_log.go @@ -9,8 +9,10 @@ type FrontendLog struct { Entry FrontendLogEntry `json:"entry,omitempty"` } +// FrontendLogEntry mirrors a oneof in the proto, so exactly one field must be set. type FrontendLogEntry struct { - DatabricksCliLog DatabricksCliLog `json:"databricks_cli_log,omitempty"` + DatabricksCliLog DatabricksCliLog `json:"databricks_cli_log,omitzero"` + FastFeedbackLog *FastFeedbackLog `json:"fast_feedback_log,omitempty"` } type DatabricksCliLog struct {