Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/cli/aitools-windows-shim-probe.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* `databricks aitools install` and `databricks aitools update` now report a coding agent CLI shim (`.cmd` or `.bat`) on Windows that prints its plugin help and then exits with an error as `shim-exited-with-error`, with the command to run to see the error, instead of reporting the CLI as not on PATH. ([#7476](https://github.com/databricks/cli/pull/7476))
4 changes: 3 additions & 1 deletion cmd/aitools/categorize.go
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,9 @@ func blockedErrorCategory(e *installer.BlockedError) protos.AitoolsErrorCategory
switch e.Reason {
case installer.ReasonCLINotOnPath:
return protos.AitoolsErrorCategoryCLINotOnPath
case installer.ReasonInstallFailed:
// A shim that exited with an error is a present CLI whose plugin command
// failed, which is closest to a failed plugin install.
case installer.ReasonInstallFailed, installer.ReasonShimExitedWithError:
return protos.AitoolsErrorCategoryPluginInstallFailed
default:
return protos.AitoolsErrorCategoryUncategorized
Expand Down
5 changes: 5 additions & 0 deletions cmd/aitools/categorize_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,11 @@ func TestClassifyInstallError(t *testing.T) {
err: &installer.BlockedError{Agent: "codex", Reason: installer.ReasonInstallFailed},
want: protos.AitoolsErrorCategoryPluginInstallFailed,
},
{
name: "blocked shim exited with error",
err: &installer.BlockedError{Agent: "copilot", Reason: installer.ReasonShimExitedWithError},
want: protos.AitoolsErrorCategoryPluginInstallFailed,
},
{
name: "blocked timeout wins over its reason",
err: &installer.BlockedError{Agent: "claude-code", Reason: installer.ReasonCLINotOnPath, TimedOut: true},
Expand Down
64 changes: 64 additions & 0 deletions libs/aitools/installer/plugin.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,10 @@ import (
"fmt"
"maps"
"os/exec"
"path/filepath"
"regexp"
"strings"
"sync"
"time"

"github.com/databricks/cli/libs/aitools/agents"
Expand Down Expand Up @@ -52,6 +55,10 @@ const (
// ReasonCLINotOnPath: the agent's CLI binary is not on PATH, or its CLI does
// not expose a working `plugin` subcommand.
ReasonCLINotOnPath = "cli-not-on-path"
// ReasonShimExitedWithError: the agent's Windows .cmd/.bat shim printed its
// `plugin --help` but exited with an error. The install stops here rather
// than run plugin commands through a shim that may fail part way.
ReasonShimExitedWithError = "shim-exited-with-error"

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[nice to have] ReasonShimExitedWithError has no case in blockedErrorCategory (cmd/aitools/categorize.go:184), so it falls through to the default case and returns AitoolsErrorCategoryUncategorized. A Windows .cmd/.bat shim that prints its plugin help and exits 1 was categorized as CLI_NOT_ON_PATH before this PR. It is now UNCATEGORIZED in telemetry, in the funnel, and in the error_category field of --output json (cmd/aitools/install.go:699), so it can't be told apart from unknown errors. That also contradicts the Merge Danger note that there are no API changes. Add a case for the new reason, either to an existing category (CLI_NOT_ON_PATH keeps the old bucket; PLUGIN_INSTALL_FAILED is closer to the meaning) or to a new value (which needs the lumberjack proto change first). Add a matching row to TestClassifyInstallError in cmd/aitools/categorize_test.go.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in 38d9620: blockedErrorCategory maps ReasonShimExitedWithError to PLUGIN_INSTALL_FAILED, the closest existing category (the CLI is present and its plugin command failed). No new enum value. Added the blocked shim exited with error row to TestClassifyInstallError, and the Merge Danger note now says --output json reports PLUGIN_INSTALL_FAILED instead of CLI_NOT_ON_PATH for this case.

// ReasonInstallFailed: the agent's plugin CLI ran but returned an error.
ReasonInstallFailed = "install-failed"
// ReasonNoPlugin: the agent has no installable plugin. Callers filter these
Expand Down Expand Up @@ -308,11 +315,68 @@ func probePluginCLI(ctx context.Context, agent *agents.Agent) (string, error) {
return "", &BlockedError{Agent: agent.Name, Reason: ReasonCLINotOnPath, Detail: err.Error()}
}
if _, err := runAgentCmd(ctx, pluginProbeTimeout, []string{bin, "plugin", "--help"}); err != nil {
if perr, ok := windowsShimPrintedHelp(bin, err); ok {
return "", &BlockedError{Agent: agent.Name, Reason: ReasonShimExitedWithError, Detail: shimExitedDetail(bin, perr)}
}
return "", newBlockedError(agent, ReasonCLINotOnPath, err)
}
return bin, nil
}

// shimExitedDetail explains a shim that printed its plugin help but exited with
// an error. It keeps the shim's stderr, since a re-run may show only the help,
// and names how to read the exit code, which neither Windows shell prints. The
// path stays absolute and keeps its backslashes so it can be pasted into
// Command Prompt or PowerShell as is, and it is quoted because a profile path
// can contain spaces. PowerShell needs the & call operator before a quoted path.
// Command Prompt expands %ERRORLEVEL% when it reads a line, so the hint reads it
// in a separate command after the shim has run.
func shimExitedDetail(bin string, perr *process.ProcessError) string {
detail := fmt.Sprintf("printed its plugin help but exited with an error (%s)", perr.Err)
if stderr := strings.TrimSpace(perr.Stderr); stderr != "" {
detail += ": " + stderr
}
return detail + fmt.Sprintf("; to see the error and exit code, run `& \"%[1]s\" plugin --help; $LASTEXITCODE` in PowerShell or `\"%[1]s\" plugin --help` and then `echo %%ERRORLEVEL%%` in Command Prompt, and re-run this command once it exits with 0", bin)
}

// pluginHelpUsage, pluginHelpInstall and pluginHelpMarket match the help that
// `<agent> plugin --help` prints: a "Usage: <bin> plugin" line and an indented
// command list that includes install (add for Codex) and marketplace. An error
// message that merely mentions those words has neither shape.
var (
pluginHelpUsage = sync.OnceValue(func() *regexp.Regexp { return regexp.MustCompile(`(?im)^usage:[ \t]+\S+[ \t]+plugin\b`) })
pluginHelpInstall = sync.OnceValue(func() *regexp.Regexp { return regexp.MustCompile(`(?m)^[ \t]+(install|add)\b`) })
pluginHelpMarket = sync.OnceValue(func() *regexp.Regexp { return regexp.MustCompile(`(?m)^[ \t]+marketplace\b`) })
)

// windowsShimPrintedHelp reports whether a failed probe is a Windows shim that

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[nice to have] .agents/rules/comments.md asks for a source reference when code depends on an external tool's behavior. windowsShimPrintedHelp depends on two outside behaviors with no link to an issue, an upstream report, or an observed repro: Copilot and Codex shims exiting 1 after printing plugin --help, and the commander/clap help layouts. The PR also says it wasn't run on Windows. Every test goes through the process stub, so nothing checks that a real .cmd run through runAgentCmd yields a ProcessError with that stdout and a non-zero exit. Link the observed failure if there is one. Also add a //go:build windows test (as installer_windows_test.go does) that writes a .cmd printing the Copilot help and then exit /b 1, and asserts shim-exited-with-error. CI already has Windows cells.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Partly fixed in 4f48110. There is no public upstream report of this exit code that I can link, so the windowsShimPrintedHelp doc now says that and points at the new test. Added plugin_windows_test.go (//go:build windows): TestProbeRealCmdShimPrintedHelp writes a copilot.cmd that prints the Copilot help and runs exit /b 1, then runs it unstubbed through runAgentCmd and asserts shim-exited-with-error with exit status 1. It passes on both Windows CI cells. This checks how Go and cmd.exe report a .cmd exit; it doesn't prove that the real Copilot or Codex shims behave this way.

// printed the full `plugin --help` and then exited with an error, and returns
// the failed process. Such a CLI is present, so reporting it as not on PATH
// would point the user at the wrong fix. Agent binaries are bare names, so
// LookPath returns a .cmd or .bat path only on Windows. Only stdout is matched:
// the agent CLIs print --help to stdout, and help on stderr usually means the
// CLI rejected its arguments.
//
// TestProbeRealCmdShimPrintedHelp runs a real .cmd that prints the help and
// exits 1, which checks how this path sees a shim's exit code on Windows.
func windowsShimPrintedHelp(bin string, err error) (*process.ProcessError, bool) {
if ext := strings.ToLower(filepath.Ext(bin)); ext != ".cmd" && ext != ".bat" {
return nil, false
}
if _, ok := errors.AsType[*agentCmdTimeoutError](err); ok {
return nil, false
}
perr, ok := errors.AsType[*process.ProcessError](err)
if !ok {
return nil, false
}
out := perr.Stdout
if !pluginHelpUsage().MatchString(out) || !pluginHelpInstall().MatchString(out) || !pluginHelpMarket().MatchString(out) {
return nil, false
}
return perr, true
}

// InstallPluginForAgent registers the databricks marketplace and installs the
// plugin through the agent's own CLI, returning the record to persist in state.
// It never falls back to skills: a blocked install returns a *BlockedError.
Expand Down
75 changes: 75 additions & 0 deletions libs/aitools/installer/plugin_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"log/slog"
"os/exec"
"path/filepath"
"strings"
"testing"
"time"

Expand Down Expand Up @@ -872,3 +873,77 @@ func TestRunAgentCmdReportsTimeout(t *testing.T) {
assert.Equal(t, ReasonCLINotOnPath, blocked.Reason)
assert.Equal(t, "timed out after 1ms", blocked.Detail)
}

// copilotPluginHelp is the commander-style shape of `copilot plugin --help`.
const copilotPluginHelp = "Usage: copilot plugin [options] [command]\n\nManage plugins\n\nCommands:\n install <plugin> Install a plugin\n uninstall <plugin> Uninstall a plugin\n marketplace Manage marketplaces\n"

// codexPluginHelp is the clap-style shape of `codex plugin --help`, which
// lists add rather than install.
const codexPluginHelp = "Manage plugins\n\nUsage: codex plugin [OPTIONS] <COMMAND>\n\nCommands:\n add Install a plugin\n marketplace Manage plugin marketplaces\n help Print this message or the help of the given subcommand(s)\n\nOptions:\n -h, --help Print help\n"

func TestProbeWindowsShimPrintedHelp(t *testing.T) {
copilot := pluginAgent(agents.NameCopilot, "GitHub Copilot", "copilot")
tests := []struct {
name string
agent *agents.Agent
bin string
stdout string
stderr string
shim bool
}{
{name: "windows .bat shim printed help and exited 1", agent: copilot, bin: `C:\npm\copilot.bat`, stdout: copilotPluginHelp, shim: true},
{name: "windows .cmd shim printed help and exited 1", agent: copilot, bin: `C:\npm\copilot.cmd`, stdout: copilotPluginHelp, shim: true},
{name: "windows shim under a path with a space", agent: copilot, bin: `C:\Users\Jane Doe\AppData\Roaming\npm\copilot.cmd`, stdout: copilotPluginHelp, shim: true},
{name: "windows shim that printed an error on stderr", agent: copilot, bin: `C:\npm\copilot.cmd`, stdout: copilotPluginHelp, stderr: " node: bad option\n", shim: true},
{name: "help on stderr is not a shim that printed help", agent: copilot, bin: `C:\npm\copilot.cmd`, stderr: copilotPluginHelp},
{name: "windows codex .cmd shim printed clap help and exited 1", agent: codexAgent(), bin: `C:\npm\codex.cmd`, stdout: codexPluginHelp, shim: true},
{name: "windows shim without a plugin command", agent: codexAgent(), bin: `C:\npm\codex.cmd`, stdout: "error: unknown command 'plugin'\n"},
{name: "unix binary is not a shim", agent: copilot, bin: "/usr/bin/copilot", stdout: copilotPluginHelp},
{name: "windows .exe is not a shim", agent: copilot, bin: `C:\bin\copilot.exe`, stdout: copilotPluginHelp},
{name: "error that mentions both words", agent: copilot, bin: `C:\npm\copilot.bat`, stdout: "Error: cannot install plugins: marketplace disabled by policy\n"},
{name: "usage line without a command list", agent: copilot, bin: `C:\npm\copilot.bat`, stdout: "Usage: copilot plugin\nerror: install and marketplace are not available\n"},
{name: "unindented words after blank lines", agent: copilot, bin: `C:\npm\copilot.bat`, stdout: "Usage: copilot plugin [options]\n\nmarketplace: disabled by policy\n\ninstall is not permitted\n"},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
origLook := lookPath
t.Cleanup(func() { lookPath = origLook })
lookPath = func(string) (string, error) { return tc.bin, nil }

ctx, stub := process.WithStub(t.Context())
stub.WithCallback(func(*exec.Cmd) error { return nil })
stub.WithStdoutFor("plugin --help", tc.stdout)
stub.WithStderrFor("plugin --help", tc.stderr)
stub.WithFailureFor("plugin --help", errors.New("exit status 1"))

_, err := InstallPluginForAgent(ctx, tc.agent, "", "v0.2.6")
var be *BlockedError
require.ErrorAs(t, err, &be)
if tc.shim {
assert.Equal(t, ReasonShimExitedWithError, be.Reason)
assert.False(t, be.TimedOut)
wantDetail := "printed its plugin help but exited with an error (exit status 1)"
if tc.stderr != "" {
wantDetail += ": " + strings.TrimSpace(tc.stderr)
}
wantDetail += "; to see the error and exit code, run `& \"" + tc.bin + "\" plugin --help; $LASTEXITCODE` in PowerShell or `\"" + tc.bin + "\" plugin --help` and then `echo %ERRORLEVEL%` in Command Prompt, and re-run this command once it exits with 0"
assert.Equal(t, wantDetail, be.Detail)
} else {
assert.Equal(t, ReasonCLINotOnPath, be.Reason)
}
assert.Len(t, stub.Commands(), 1, "nothing runs after a failed probe")
})
}
}

func TestWindowsShimPrintedHelpRejectsTimeout(t *testing.T) {
// A shim that printed the help and then hung until its deadline reports the
// timeout, even though the same output without the timeout is a shim error.
perr := &process.ProcessError{Command: "copilot.bat plugin --help", Err: errors.New("exit status 1"), Stdout: copilotPluginHelp}
bin := `C:\npm\copilot.bat`
got, ok := windowsShimPrintedHelp(bin, perr)
assert.True(t, ok)
assert.Same(t, perr, got)
_, ok = windowsShimPrintedHelp(bin, &agentCmdTimeoutError{timeout: time.Second, err: perr})
assert.False(t, ok)
}
34 changes: 34 additions & 0 deletions libs/aitools/installer/plugin_windows_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
//go:build windows

package installer

import (
"os"
"path/filepath"
"strings"
"testing"

"github.com/databricks/cli/libs/aitools/agents"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

// TestProbeRealCmdShimPrintedHelp runs a real .cmd shim that prints the plugin
// help and exits 1, so the probe sees the exit code and stdout that cmd.exe
// actually reports rather than a stub.
func TestProbeRealCmdShimPrintedHelp(t *testing.T) {
dir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(dir, "help.txt"), []byte(copilotPluginHelp), 0o644))
shim := filepath.Join(dir, "copilot.cmd")
require.NoError(t, os.WriteFile(shim, []byte("@type \"%~dp0help.txt\"\r\n@exit /b 1\r\n"), 0o755))

origLook := lookPath
t.Cleanup(func() { lookPath = origLook })
lookPath = func(string) (string, error) { return shim, nil }

_, err := InstallPluginForAgent(t.Context(), pluginAgent(agents.NameCopilot, "GitHub Copilot", "copilot"), "", "v0.2.6")
var be *BlockedError
require.ErrorAs(t, err, &be)
assert.Equal(t, ReasonShimExitedWithError, be.Reason)
assert.True(t, strings.HasPrefix(be.Detail, "printed its plugin help but exited with an error (exit status 1)"), be.Detail)
}
Loading