diff --git a/.nextchanges/cli/aitools-install-fetch-error-categories.md b/.nextchanges/cli/aitools-install-fetch-error-categories.md new file mode 100644 index 00000000000..23790dd1290 --- /dev/null +++ b/.nextchanges/cli/aitools-install-fetch-error-categories.md @@ -0,0 +1 @@ +* `databricks aitools install --output json` reports more specific `error_category` values: `SKILLS_FETCH_FAILED`, `RELEASE_LOOKUP_FAILED`, `RATE_LIMITED`, `MANIFEST_INVALID`, `STATE_INVALID` and `PLUGIN_CLI_TIMEOUT`. A coding agent CLI that does not answer in time is now reported as `timed out after `, followed by any stderr it wrote before it was stopped. ([#7475](https://github.com/databricks/cli/pull/7475)) diff --git a/acceptance/cmd/aitools/telemetry/install-failure/output.txt b/acceptance/cmd/aitools/telemetry/install-failure/output.txt index cb4c408db1f..380eea4597c 100644 --- a/acceptance/cmd/aitools/telemetry/install-failure/output.txt +++ b/acceptance/cmd/aitools/telemetry/install-failure/output.txt @@ -33,7 +33,7 @@ Error: skills manifest at databricks-agent-skills@missing-ref: not found {"command":"aitools_install","exit_code":1,"aitools_command_event":{"command":"INSTALL","stage":"FINISHED","outcome":"FAILURE","error_category":"MANIFEST_NOT_FOUND","interactive":false,"install_mode":"SKILLS_ONLY","agents":["CURSOR"],"scope":"GLOBAL"}} {"command":"aitools_install","exit_code":1,"aitools_install_event":{"agents":["CURSOR"],"scope":"GLOBAL","error_category":"MANIFEST_NOT_FOUND"}} -=== any other HTTP error status from the manifest fetch is not classified yet (the error is untyped) +=== any other HTTP error status from the manifest fetch is SKILLS_FETCH_FAILED >>> DATABRICKS_SKILLS_REF=broken-ref [CLI] aitools install --skills-only --scope=global --agents=cursor Installing Databricks skills for Cursor... @@ -47,8 +47,8 @@ Error: failed to fetch manifest: HTTP 500 {"command":"aitools_install","exit_code":1,"aitools_command_event":{"command":"INSTALL","stage":"STEP_COMPLETED","step":"SCOPE_RESOLVED","interactive":false,"install_mode":"SKILLS_ONLY","scope":"GLOBAL"}} {"command":"aitools_install","exit_code":1,"aitools_command_event":{"command":"INSTALL","stage":"STEP_COMPLETED","step":"AGENTS_SELECTED","interactive":false,"install_mode":"SKILLS_ONLY","scope":"GLOBAL"}} {"command":"aitools_install","exit_code":1,"aitools_command_event":{"command":"INSTALL","stage":"STEP_COMPLETED","step":"PLAN_CONFIRMED","interactive":false,"install_mode":"SKILLS_ONLY","scope":"GLOBAL"}} -{"command":"aitools_install","exit_code":1,"aitools_command_event":{"command":"INSTALL","stage":"FINISHED","outcome":"FAILURE","error_category":"UNCATEGORIZED","interactive":false,"install_mode":"SKILLS_ONLY","agents":["CURSOR"],"scope":"GLOBAL"}} -{"command":"aitools_install","exit_code":1,"aitools_install_event":{"agents":["CURSOR"],"scope":"GLOBAL","error_category":"UNCATEGORIZED"}} +{"command":"aitools_install","exit_code":1,"aitools_command_event":{"command":"INSTALL","stage":"FINISHED","outcome":"FAILURE","error_category":"SKILLS_FETCH_FAILED","interactive":false,"install_mode":"SKILLS_ONLY","agents":["CURSOR"],"scope":"GLOBAL"}} +{"command":"aitools_install","exit_code":1,"aitools_install_event":{"agents":["CURSOR"],"scope":"GLOBAL","error_category":"SKILLS_FETCH_FAILED"}} === a named agent skipped for the scope takes its category from the agent's outcome diff --git a/acceptance/cmd/aitools/telemetry/install-failure/script b/acceptance/cmd/aitools/telemetry/install-failure/script index 6b9bcf51b11..8e36f8eaf0b 100644 --- a/acceptance/cmd/aitools/telemetry/install-failure/script +++ b/acceptance/cmd/aitools/telemetry/install-failure/script @@ -15,7 +15,7 @@ title "a manifest the server does not have is MANIFEST_NOT_FOUND" musterr trace DATABRICKS_SKILLS_REF=missing-ref $CLI aitools install --skills-only --scope=global --agents=cursor trace print_telemetry -title "any other HTTP error status from the manifest fetch is not classified yet (the error is untyped)" +title "any other HTTP error status from the manifest fetch is SKILLS_FETCH_FAILED" musterr trace DATABRICKS_SKILLS_REF=broken-ref $CLI aitools install --skills-only --scope=global --agents=cursor trace print_telemetry diff --git a/cmd/aitools/categorize.go b/cmd/aitools/categorize.go index 46a3785ee0e..60ae1876b11 100644 --- a/cmd/aitools/categorize.go +++ b/cmd/aitools/categorize.go @@ -2,6 +2,7 @@ package aitools import ( "context" + "encoding/json" "errors" "io/fs" "net" @@ -57,6 +58,14 @@ func classifyInstallError(err error) protos.AitoolsErrorCategory { if skill, ok := errors.AsType[*installer.SkillError](err); ok { return skillErrorCategory(skill) } + // FetchError is checked before the standard library types: a response + // that arrived carries a status the net error types know nothing about. + if fetch, ok := errors.AsType[*installer.FetchError](err); ok { + return fetchErrorCategory(fetch) + } + if _, ok := errors.AsType[*installer.StateError](err); ok { + return protos.AitoolsErrorCategoryStateInvalid + } if tagged, ok := errors.AsType[*categorizedError](err); ok { return tagged.category } @@ -71,11 +80,11 @@ func classifyInstallError(err error) protos.AitoolsErrorCategory { } // stdlibErrorCategory classifies errors by their standard library type, which -// the installer wraps with %w: network and timeout errors from fetching the -// manifest or skill files, and filesystem errors from writing skills or the -// install state. JSON decode errors stay UNCATEGORIZED: the manifest, the -// release payload and the install state all decode JSON, so only typed -// installer errors can tell MANIFEST_INVALID from STATE_INVALID. +// the installer wraps with %w: filesystem errors from writing skills or the +// install state, and network and timeout errors, including those of a +// FetchError that got no response. JSON decode errors stay UNCATEGORIZED: the +// manifest, the release payload and the install state all decode JSON, so only +// typed installer errors can tell MANIFEST_INVALID from STATE_INVALID. func stdlibErrorCategory(err error) protos.AitoolsErrorCategory { switch { case errors.Is(err, huh.ErrUserAborted): @@ -133,7 +142,52 @@ func skillErrorCategory(e *installer.SkillError) protos.AitoolsErrorCategory { } } +// fetchErrorCategory maps a FetchError to its category. The first matching +// case wins, so the order matters: an interrupted command is never a fetch +// failure, a transport failure keeps its NETWORK or TIMEOUT category, and the +// release stage is checked before the decode check. +func fetchErrorCategory(e *installer.FetchError) protos.AitoolsErrorCategory { + // The command was interrupted while the request was in flight: the fetch + // did not fail on its own. + if errors.Is(e, context.Canceled) { + return protos.AitoolsErrorCategoryCommandInterrupted + } + // No response arrived (DNS, refused connection, proxy, TLS, client + // timeout), or reading the body failed on the transport (a timeout or a + // reset mid-body): the user cannot reach the host, which NETWORK and + // TIMEOUT already report. + if c := stdlibErrorCategory(e.Err); e.StatusCode == 0 || c == protos.AitoolsErrorCategoryNetwork || c == protos.AitoolsErrorCategoryTimeout { + return c + } + switch { + case e.RateLimited: + return protos.AitoolsErrorCategoryRateLimited + case errors.Is(e, clicompat.ErrNotFound): + return protos.AitoolsErrorCategoryManifestNotFound + // Before the decode check: a release payload that does not decode is a + // failed release lookup, not an invalid manifest. + case e.Stage == installer.FetchStageRelease: + return protos.AitoolsErrorCategoryReleaseLookupFailed + // The response arrived but was not a manifest: content, not transport. + case isManifestDecodeError(e) || errors.Is(e, installer.ErrManifestTooLarge): + return protos.AitoolsErrorCategoryManifestInvalid + default: + return protos.AitoolsErrorCategorySkillsFetchFailed + } +} + +// isManifestDecodeError reports whether the response arrived but was not a +// valid manifest. +func isManifestDecodeError(err error) bool { + _, syntax := errors.AsType[*json.SyntaxError](err) + _, typ := errors.AsType[*json.UnmarshalTypeError](err) + return syntax || typ +} + func blockedErrorCategory(e *installer.BlockedError) protos.AitoolsErrorCategory { + if e.TimedOut { + return protos.AitoolsErrorCategoryPluginCLITimeout + } switch e.Reason { case installer.ReasonCLINotOnPath: return protos.AitoolsErrorCategoryCLINotOnPath diff --git a/cmd/aitools/categorize_test.go b/cmd/aitools/categorize_test.go index 9a6a2956f48..c62a44cf394 100644 --- a/cmd/aitools/categorize_test.go +++ b/cmd/aitools/categorize_test.go @@ -7,11 +7,16 @@ import ( "fmt" "io/fs" "net" + "net/http" + "net/http/httptest" "net/url" "os" "path/filepath" + "strconv" + "strings" "syscall" "testing" + "time" "github.com/charmbracelet/huh" "github.com/databricks/cli/libs/aitools/installer" @@ -42,6 +47,66 @@ func TestClassifyInstallError(t *testing.T) { err: &installer.BlockedError{Agent: "codex", Reason: installer.ReasonInstallFailed}, want: protos.AitoolsErrorCategoryPluginInstallFailed, }, + { + name: "blocked timeout wins over its reason", + err: &installer.BlockedError{Agent: "claude-code", Reason: installer.ReasonCLINotOnPath, TimedOut: true}, + want: protos.AitoolsErrorCategoryPluginCLITimeout, + }, + { + name: "release lookup failed", + err: fmt.Errorf("resolve latest skills release: %w", &installer.FetchError{Stage: installer.FetchStageRelease, StatusCode: 500, Err: errors.New("HTTP 500 fetching latest skills release")}), + want: protos.AitoolsErrorCategoryReleaseLookupFailed, + }, + { + name: "release lookup with no response is NETWORK", + err: fmt.Errorf("resolve latest skills release: %w", &installer.FetchError{Stage: installer.FetchStageRelease, Err: &url.Error{Op: "Get", URL: "https://example.test", Err: &net.DNSError{Err: "no such host"}}}), + want: protos.AitoolsErrorCategoryNetwork, + }, + { + name: "release payload that does not decode is not an invalid manifest", + err: &installer.FetchError{Stage: installer.FetchStageRelease, StatusCode: 200, Err: fmt.Errorf("parse latest skills release: %w", &json.SyntaxError{})}, + want: protos.AitoolsErrorCategoryReleaseLookupFailed, + }, + { + name: "release lookup rate limited", + err: &installer.FetchError{Stage: installer.FetchStageRelease, StatusCode: 403, RateLimited: true, Err: errors.New("HTTP 403")}, + want: protos.AitoolsErrorCategoryRateLimited, + }, + { + name: "release lookup denied without rate-limit evidence", + err: &installer.FetchError{Stage: installer.FetchStageRelease, StatusCode: 403, Err: errors.New("HTTP 403")}, + want: protos.AitoolsErrorCategoryReleaseLookupFailed, + }, + { + name: "manifest fetch rate limited", + err: &installer.FetchError{Stage: installer.FetchStageManifest, StatusCode: 429, RateLimited: true, Err: errors.New("HTTP 429")}, + want: protos.AitoolsErrorCategoryRateLimited, + }, + { + name: "manifest fetch with no response is NETWORK", + err: &installer.FetchError{Stage: installer.FetchStageManifest, Err: fmt.Errorf("failed to fetch manifest: %w", &url.Error{Op: "Get", URL: "https://example.test", Err: &net.OpError{Op: "dial", Net: "tcp", Err: os.NewSyscallError("connect", syscall.ECONNREFUSED)}})}, + want: protos.AitoolsErrorCategoryNetwork, + }, + { + name: "skill file fetch that timed out is TIMEOUT", + err: &installer.FetchError{Stage: installer.FetchStageFile, Err: &url.Error{Op: "Get", URL: "https://example.test", Err: os.ErrDeadlineExceeded}}, + want: protos.AitoolsErrorCategoryTimeout, + }, + { + name: "skill file fetch failed", + err: fmt.Errorf("failed to fetch skills/x/SKILL.md: %w", &installer.FetchError{Stage: installer.FetchStageFile, StatusCode: 500, Err: errors.New("HTTP 500")}), + want: protos.AitoolsErrorCategorySkillsFetchFailed, + }, + { + name: "manifest missing at the pinned ref", + err: &installer.FetchError{Stage: installer.FetchStageManifest, StatusCode: 404, Err: fmt.Errorf("skills manifest at x@v9: %w", clicompat.ErrNotFound)}, + want: protos.AitoolsErrorCategoryManifestNotFound, + }, + { + name: "state file invalid", + err: fmt.Errorf("failed to load install state: %w", &installer.StateError{Path: "/s/.state.json", Err: errors.New("failed to parse state file")}), + want: protos.AitoolsErrorCategoryStateInvalid, + }, { name: "blocked error with unknown reason is uncategorized", err: &installer.BlockedError{Agent: "codex", Reason: "some-future-reason"}, @@ -102,12 +167,12 @@ func TestClassifyInstallErrorGenericCauses(t *testing.T) { err error want protos.AitoolsErrorCategory }{ - {"network", fmt.Errorf("failed to fetch manifest: %w", urlErr), protos.AitoolsErrorCategoryNetwork}, + {"network", fmt.Errorf("fetch: %w", urlErr), protos.AitoolsErrorCategoryNetwork}, // The shape net/http returns for a refused connection: the Errno sits under the network types. - {"network errno", fmt.Errorf("failed to fetch manifest: %w", &url.Error{Op: "Get", URL: "https://example.test", Err: &net.OpError{Op: "dial", Net: "tcp", Err: os.NewSyscallError("connect", syscall.ECONNREFUSED)}}), protos.AitoolsErrorCategoryNetwork}, + {"network errno", fmt.Errorf("fetch: %w", &url.Error{Op: "Get", URL: "https://example.test", Err: &net.OpError{Op: "dial", Net: "tcp", Err: os.NewSyscallError("connect", syscall.ECONNREFUSED)}}), protos.AitoolsErrorCategoryNetwork}, // A sandbox that blocks outbound sockets: a network failure, even though the Errno is EACCES. - {"network permission denied", fmt.Errorf("failed to fetch manifest: %w", &url.Error{Op: "Get", URL: "https://example.test", Err: &net.OpError{Op: "dial", Net: "tcp", Err: os.NewSyscallError("connect", syscall.EACCES)}}), protos.AitoolsErrorCategoryNetwork}, - {"network timeout", fmt.Errorf("failed to fetch manifest: %w", &url.Error{Op: "Get", URL: "https://example.test", Err: os.ErrDeadlineExceeded}), protos.AitoolsErrorCategoryTimeout}, + {"network permission denied", fmt.Errorf("fetch: %w", &url.Error{Op: "Get", URL: "https://example.test", Err: &net.OpError{Op: "dial", Net: "tcp", Err: os.NewSyscallError("connect", syscall.EACCES)}}), protos.AitoolsErrorCategoryNetwork}, + {"network timeout", fmt.Errorf("fetch: %w", &url.Error{Op: "Get", URL: "https://example.test", Err: os.ErrDeadlineExceeded}), protos.AitoolsErrorCategoryTimeout}, {"deadline", fmt.Errorf("fetch: %w", context.DeadlineExceeded), protos.AitoolsErrorCategoryTimeout}, {"cancelled", fmt.Errorf("fetch: %w", context.Canceled), protos.AitoolsErrorCategoryCommandInterrupted}, {"cancelled request", &url.Error{Op: "Get", URL: "https://example.test", Err: context.Canceled}, protos.AitoolsErrorCategoryCommandInterrupted}, @@ -144,15 +209,14 @@ func TestWithCategoryKeepsMessage(t *testing.T) { assert.ErrorIs(t, err, cause) } -// A corrupt install state must not be blamed on the remote manifest. It stays -// UNCATEGORIZED until the installer returns a typed state error, which should -// change this to STATE_INVALID. -func TestClassifyInstallErrorCorruptStateIsUncategorized(t *testing.T) { +// A corrupt install state must not be blamed on the remote manifest: the +// installer's typed StateError makes it STATE_INVALID. +func TestClassifyInstallErrorCorruptStateIsStateInvalid(t *testing.T) { dir := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(dir, ".state.json"), []byte("{not json"), 0o600)) _, err := installer.LoadState(dir) require.Error(t, err) - assert.Equal(t, protos.AitoolsErrorCategoryUncategorized, classifyInstallError(fmt.Errorf("failed to load install state: %w", err))) + assert.Equal(t, protos.AitoolsErrorCategoryStateInvalid, classifyInstallError(fmt.Errorf("failed to load install state: %w", err))) } // readMissingFile returns the error the OS gives for a missing file, so the @@ -163,3 +227,96 @@ func readMissingFile(t *testing.T) error { require.Error(t, err) return err } + +func TestClassifyCancelledManifestFetchIsCommandInterrupted(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + server := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { + // Cancel while the request is in flight, then wait for the client to give up. + cancel() + <-r.Context().Done() + })) + t.Cleanup(server.Close) + t.Setenv("DATABRICKS_SKILLS_BASE_URL", server.URL) + + _, err := (&installer.GitHubManifestSource{}).FetchManifest(ctx, "v1") + require.ErrorIs(t, err, context.Canceled) + assert.Equal(t, protos.AitoolsErrorCategoryCommandInterrupted, classifyInstallError(err)) +} + +// A response whose body stalls or is cut off by a reset is a transport +// failure, not a failed fetch: it keeps TIMEOUT or NETWORK. +func TestClassifyManifestBodyTransportFailure(t *testing.T) { + t.Run("stalled body", func(t *testing.T) { + release := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Length", "100") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{"skills":`)) + w.(http.Flusher).Flush() + select { + case <-release: + case <-r.Context().Done(): + } + })) + t.Cleanup(server.Close) + t.Cleanup(func() { close(release) }) + t.Setenv("DATABRICKS_SKILLS_BASE_URL", server.URL) + + ctx, cancel := context.WithTimeout(t.Context(), 200*time.Millisecond) + defer cancel() + _, err := (&installer.GitHubManifestSource{}).FetchManifest(ctx, "v1") + require.Error(t, err) + assert.Equal(t, protos.AitoolsErrorCategoryTimeout, classifyInstallError(err)) + }) + + t.Run("reset mid-body", func(t *testing.T) { + err := &installer.FetchError{Stage: installer.FetchStageManifest, StatusCode: http.StatusOK, Err: fmt.Errorf("failed to read manifest: %w", &net.OpError{Op: "read", Net: "tcp", Err: os.NewSyscallError("read", syscall.ECONNRESET)})} + assert.Equal(t, protos.AitoolsErrorCategoryNetwork, classifyInstallError(err)) + }) + + t.Run("release body timeout", func(t *testing.T) { + err := &installer.FetchError{Stage: installer.FetchStageRelease, StatusCode: http.StatusOK, Err: fmt.Errorf("parse latest skills release: %w", os.ErrDeadlineExceeded)} + assert.Equal(t, protos.AitoolsErrorCategoryTimeout, classifyInstallError(err)) + }) +} + +func TestClassifyRealManifestFetchErrors(t *testing.T) { + tests := []struct { + name string + status int + body string + // cutOff announces a longer body than is sent, so the connection + // closes mid-body. + cutOff bool + want protos.AitoolsErrorCategory + }{ + // A captive portal or broken mirror answers 200 with something else. + {name: "200 with HTML", status: http.StatusOK, body: "login", want: protos.AitoolsErrorCategoryManifestInvalid}, + {name: "200 with the wrong JSON shape", status: http.StatusOK, body: `{"skills": []}`, want: protos.AitoolsErrorCategoryManifestInvalid}, + {name: "200 with an empty body", status: http.StatusOK, body: "", want: protos.AitoolsErrorCategoryManifestInvalid}, + {name: "200 with incomplete JSON", status: http.StatusOK, body: `{"skills":`, want: protos.AitoolsErrorCategoryManifestInvalid}, + // A body over 1 MiB is not parsed: it is reported as too large, in the + // same category as a manifest that does not parse. + {name: "200 with a body over the size cap", status: http.StatusOK, body: strings.Repeat(" ", 1<<20) + "{}", want: protos.AitoolsErrorCategoryManifestInvalid}, + {name: "connection closed mid-body", status: http.StatusOK, body: `{"skills":`, cutOff: true, want: protos.AitoolsErrorCategorySkillsFetchFailed}, + {name: "404", status: http.StatusNotFound, want: protos.AitoolsErrorCategoryManifestNotFound}, + {name: "503", status: http.StatusServiceUnavailable, want: protos.AitoolsErrorCategorySkillsFetchFailed}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if tc.cutOff { + w.Header().Set("Content-Length", strconv.Itoa(len(tc.body)+100)) + } + w.WriteHeader(tc.status) + _, _ = w.Write([]byte(tc.body)) + })) + t.Cleanup(server.Close) + t.Setenv("DATABRICKS_SKILLS_BASE_URL", server.URL) + + _, err := (&installer.GitHubManifestSource{}).FetchManifest(t.Context(), "v1") + require.Error(t, err) + assert.Equal(t, tc.want, classifyInstallError(err)) + }) + } +} diff --git a/libs/aitools/installer/errors.go b/libs/aitools/installer/errors.go index 7b9991be7f7..b75a007821d 100644 --- a/libs/aitools/installer/errors.go +++ b/libs/aitools/installer/errors.go @@ -3,6 +3,7 @@ package installer import ( "errors" "fmt" + "net/http" ) // ErrNothingInstalled reports that update or uninstall found no install state @@ -57,3 +58,74 @@ func (e *SkillError) Error() string { } return fmt.Sprintf("skill %q %s", e.Skill, e.Reason) } + +// maxFetchBodySize caps how much of a skills manifest or release lookup +// response is read. +const maxFetchBodySize = 1 << 20 + +// ErrManifestTooLarge reports a manifest response larger than maxFetchBodySize. +// The command layer classifies it like a manifest that does not parse. +var ErrManifestTooLarge = fmt.Errorf("skills manifest is larger than %d MiB", maxFetchBodySize>>20) + +// Stages of fetching skills content, recorded on FetchError so a failed +// release lookup can be told apart from a failed download. +const ( + // FetchStageRelease: resolving the latest skills release on api.github.com. + FetchStageRelease = "release" + // FetchStageManifest: downloading the skills manifest. + FetchStageManifest = "manifest" + // FetchStageFile: downloading one skill file. + FetchStageFile = "file" +) + +// FetchError reports that skills content could not be downloaded, or that a +// response arrived but could not be used (an unparsable manifest or release +// payload). It wraps the cause without changing its message; StatusCode is 0 +// when no HTTP response arrived (DNS, proxy, TLS, timeout). A manifest that is +// not at the ref wraps clicompat.ErrNotFound, so clicompat.IsNotFoundError +// recognizes it like any other missing ref; check a 404 that way, not through +// StatusCode. +type FetchError struct { + Stage string + StatusCode int + // RateLimited is set when the server said the request exceeded a rate + // limit, as opposed to an ordinary 403 from an access policy or proxy. + RateLimited bool + Err error +} + +func (e *FetchError) Error() string { return e.Err.Error() } +func (e *FetchError) Unwrap() error { return e.Err } + +// newHTTPFetchError builds the FetchError for a response with a non-OK status. +func newHTTPFetchError(stage string, resp *http.Response, err error) *FetchError { + return &FetchError{Stage: stage, StatusCode: resp.StatusCode, RateLimited: isRateLimited(resp), Err: err} +} + +// isRateLimited reports whether a response is a rate-limit refusal. 429 always +// is; GitHub also answers 403 for exceeded limits, and then says so with +// x-ratelimit-remaining: 0 (primary limit) or retry-after (secondary limit). +// Any other 403 is an access failure. +// See https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api#exceeding-the-rate-limit. +func isRateLimited(resp *http.Response) bool { + switch resp.StatusCode { + case http.StatusTooManyRequests: + return true + case http.StatusForbidden: + return resp.Header.Get("X-RateLimit-Remaining") == "0" || resp.Header.Get("Retry-After") != "" + default: + return false + } +} + +// StateError reports that the install state file exists but its content is +// invalid. Path is the state file that failed to parse. +// A file that cannot be read is not a StateError: that is a filesystem problem +// (usually permissions), and moving the file aside would not fix it. +type StateError struct { + Path string + Err error +} + +func (e *StateError) Error() string { return e.Err.Error() } +func (e *StateError) Unwrap() error { return e.Err } diff --git a/libs/aitools/installer/installer.go b/libs/aitools/installer/installer.go index 537d6c7322d..2699fd4a38a 100644 --- a/libs/aitools/installer/installer.go +++ b/libs/aitools/installer/installer.go @@ -150,29 +150,31 @@ func fetchLatestSkillsReleaseRef(ctx context.Context) (string, error) { resp, err := httpClient().Do(req) if err != nil { - return "", err + return "", &FetchError{Stage: FetchStageRelease, Err: err} } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("HTTP %d fetching latest skills release", resp.StatusCode) + return "", newHTTPFetchError(FetchStageRelease, resp, fmt.Errorf("HTTP %d fetching latest skills release", resp.StatusCode)) } var payload struct { TagName string `json:"tag_name"` } - if err := json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(&payload); err != nil { - return "", fmt.Errorf("parse latest skills release: %w", err) + // A response that arrived but names no usable release is still a failed + // release lookup, so these errors carry the stage too. + if err := json.NewDecoder(io.LimitReader(resp.Body, maxFetchBodySize)).Decode(&payload); err != nil { + return "", &FetchError{Stage: FetchStageRelease, StatusCode: resp.StatusCode, Err: fmt.Errorf("parse latest skills release: %w", err)} } tag := strings.TrimSpace(payload.TagName) if tag == "" { - return "", errors.New("latest skills release has empty tag_name") + return "", &FetchError{Stage: FetchStageRelease, StatusCode: resp.StatusCode, Err: errors.New("latest skills release has empty tag_name")} } if !strings.HasPrefix(tag, "v") { tag = "v" + tag } if !semver.IsValid(tag) { - return "", fmt.Errorf("latest skills release has invalid tag %q", payload.TagName) + return "", &FetchError{Stage: FetchStageRelease, StatusCode: resp.StatusCode, Err: fmt.Errorf("latest skills release has invalid tag %q", payload.TagName)} } return tag, nil } @@ -240,17 +242,17 @@ func fetchSkillFile(ctx context.Context, ref, repoDir, skillName, filePath strin resp, err := httpClient().Do(req) if err != nil { - return nil, err + return nil, &FetchError{Stage: FetchStageFile, Err: err} } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("HTTP %d", resp.StatusCode) + return nil, newHTTPFetchError(FetchStageFile, resp, fmt.Errorf("HTTP %d", resp.StatusCode)) } data, err := io.ReadAll(resp.Body) if err != nil { - return nil, fmt.Errorf("failed to read response: %w", err) + return nil, &FetchError{Stage: FetchStageFile, StatusCode: resp.StatusCode, Err: fmt.Errorf("failed to read response: %w", err)} } return data, nil } diff --git a/libs/aitools/installer/installer_test.go b/libs/aitools/installer/installer_test.go index 781ae86cded..baf02bc3e57 100644 --- a/libs/aitools/installer/installer_test.go +++ b/libs/aitools/installer/installer_test.go @@ -1490,3 +1490,85 @@ func withCachedCompat(t *testing.T, skills string) { manifest := fmt.Sprintf(`{"0.0.0":{"appkit":"0.24.0","skills":%q}}`, skills) require.NoError(t, os.WriteFile(filepath.Join(cacheDir, "compat-manifest.json"), []byte(manifest), 0o644)) } + +func TestFetchLatestSkillsReleaseRefRateLimited(t *testing.T) { + tests := []struct { + name string + header map[string]string + status int + rateLimited bool + }{ + {name: "403 with no remaining requests", status: http.StatusForbidden, header: map[string]string{"X-RateLimit-Remaining": "0"}, rateLimited: true}, + {name: "403 secondary limit", status: http.StatusForbidden, header: map[string]string{"Retry-After": "60"}, rateLimited: true}, + {name: "429", status: http.StatusTooManyRequests, rateLimited: true}, + // A proxy or access policy denying the request is not a rate limit. + {name: "plain 403", status: http.StatusForbidden, rateLimited: false}, + {name: "403 with requests left", status: http.StatusForbidden, header: map[string]string{"X-RateLimit-Remaining": "12"}, rateLimited: false}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + for k, v := range tc.header { + w.Header().Set(k, v) + } + w.WriteHeader(tc.status) + })) + t.Cleanup(server.Close) + t.Setenv("DATABRICKS_CLI_GITHUB_API_URL", server.URL) + + _, err := fetchLatestSkillsReleaseRef(t.Context()) + ferr, ok := errors.AsType[*FetchError](err) + require.True(t, ok, "want *FetchError, got %T", err) + assert.Equal(t, FetchStageRelease, ferr.Stage) + assert.Equal(t, tc.status, ferr.StatusCode) + assert.Equal(t, tc.rateLimited, ferr.RateLimited) + }) + } +} + +func TestFetchLatestSkillsReleaseRefUnusableResponse(t *testing.T) { + tests := []struct { + name string + body string + }{ + {name: "HTML page", body: "login"}, + {name: "empty body", body: ""}, + {name: "empty tag_name", body: `{"tag_name": ""}`}, + {name: "invalid tag", body: `{"tag_name": "not-a-version"}`}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(tc.body)) + })) + t.Cleanup(server.Close) + t.Setenv("DATABRICKS_CLI_GITHUB_API_URL", server.URL) + + _, err := fetchLatestSkillsReleaseRef(t.Context()) + ferr, ok := errors.AsType[*FetchError](err) + require.True(t, ok, "want *FetchError, got %T", err) + assert.Equal(t, FetchStageRelease, ferr.Stage) + assert.Equal(t, http.StatusOK, ferr.StatusCode) + }) + } +} + +func TestFetchErrorsCarryStage(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusServiceUnavailable) + })) + t.Cleanup(server.Close) + t.Setenv("DATABRICKS_SKILLS_BASE_URL", server.URL) + + _, err := (&GitHubManifestSource{}).FetchManifest(t.Context(), "v1") + ferr, ok := errors.AsType[*FetchError](err) + require.True(t, ok, "want *FetchError, got %T", err) + assert.Equal(t, FetchStageManifest, ferr.Stage) + assert.Equal(t, http.StatusServiceUnavailable, ferr.StatusCode) + assert.False(t, ferr.RateLimited) + + _, err = fetchSkillFile(t.Context(), "v1", "skills", "s", "SKILL.md") + ferr, ok = errors.AsType[*FetchError](err) + require.True(t, ok, "want *FetchError, got %T", err) + assert.Equal(t, FetchStageFile, ferr.Stage) +} diff --git a/libs/aitools/installer/plugin.go b/libs/aitools/installer/plugin.go index 4066957527e..34a3edab2b9 100644 --- a/libs/aitools/installer/plugin.go +++ b/libs/aitools/installer/plugin.go @@ -40,6 +40,11 @@ type BlockedError struct { Agent string Reason string Detail string + // TimedOut is set when the agent CLI was killed for exceeding its timeout, + // so the failure can be told apart from a missing or broken CLI. It takes + // precedence over Reason: Reason still names the step that timed out, so + // check TimedOut before switching on Reason. + TimedOut bool } // Reasons a plugin operation can be blocked. @@ -62,20 +67,54 @@ func (e *BlockedError) Error() string { } // runAgentCmd runs an agent CLI command with a timeout, returning stdout and any -// error. Errors are *process.ProcessError, which carries the captured stderr. +// error. Errors wrap a *process.ProcessError, which carries the captured stderr; +// on timeout the error is an *agentCmdTimeoutError that wraps it, so callers +// must use errors.AsType rather than a type assertion. func runAgentCmd(ctx context.Context, timeout time.Duration, argv []string) (string, error) { cctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() - return process.Background(cctx, argv) + out, err := process.Background(cctx, argv) + if err != nil && errors.Is(cctx.Err(), context.DeadlineExceeded) { + return out, &agentCmdTimeoutError{timeout: timeout, err: err} + } + return out, err +} + +// agentCmdTimeoutError reports an agent CLI command killed by its timeout. +type agentCmdTimeoutError struct { + timeout time.Duration + err error +} + +func (e *agentCmdTimeoutError) Error() string { + return fmt.Sprintf("timed out after %s", e.timeout) +} + +func (e *agentCmdTimeoutError) Unwrap() error { return e.err } + +// newBlockedError builds the BlockedError for a failed agent CLI command. +func newBlockedError(agent *agents.Agent, reason string, err error) *BlockedError { + _, timedOut := errors.AsType[*agentCmdTimeoutError](err) + return &BlockedError{Agent: agent.Name, Reason: reason, Detail: stderrOf(err), TimedOut: timedOut} } // stderrOf returns the captured stderr of a failed agent command, falling back // to the error's own message. Callers must not branch on this string. func stderrOf(err error) string { + var stderr string if perr, ok := errors.AsType[*process.ProcessError](err); ok { - if s := strings.TrimSpace(perr.Stderr); s != "" { - return s + stderr = strings.TrimSpace(perr.Stderr) + } + // The timeout is the real cause of a killed process. Whatever it wrote to + // stderr before the kill (a prompt waiting for input, say) follows it. + if terr, ok := errors.AsType[*agentCmdTimeoutError](err); ok { + if stderr != "" { + return terr.Error() + ": " + stderr } + return terr.Error() + } + if stderr != "" { + return stderr } return err.Error() } @@ -269,7 +308,7 @@ func probePluginCLI(ctx context.Context, agent *agents.Agent) (string, error) { return "", &BlockedError{Agent: agent.Name, Reason: ReasonCLINotOnPath, Detail: err.Error()} } if _, err := runAgentCmd(ctx, pluginProbeTimeout, []string{bin, "plugin", "--help"}); err != nil { - return "", &BlockedError{Agent: agent.Name, Reason: ReasonCLINotOnPath, Detail: stderrOf(err)} + return "", newBlockedError(agent, ReasonCLINotOnPath, err) } return bin, nil } @@ -308,7 +347,7 @@ func InstallPluginForAgent(ctx context.Context, agent *agents.Agent, nativeScope log.Warnf(ctx, "%s plugin install failed and the marketplace could not be de-registered: %v", agent.DisplayName, rmErr) } } - return PluginRecord{}, withRefreshError(&BlockedError{Agent: agent.Name, Reason: ReasonInstallFailed, Detail: stderrOf(err)}, refreshErr) + return PluginRecord{}, withRefreshError(newBlockedError(agent, ReasonInstallFailed, err), refreshErr) } // A marketplace we just added is a fresh copy, so a failed refresh right @@ -345,7 +384,7 @@ func UpdatePluginForAgent(ctx context.Context, agent *agents.Agent, rec PluginRe } refreshErr = refreshMarketplace(ctx, bin, pluginRefreshArgs(agent, rec)) if _, err := runAgentCmd(ctx, pluginCmdTimeout, prepend(bin, pluginUpdateArgs(agent, rec))); err != nil { - return nil, withRefreshError(&BlockedError{Agent: agent.Name, Reason: ReasonInstallFailed, Detail: stderrOf(err)}, refreshErr) + return nil, withRefreshError(newBlockedError(agent, ReasonInstallFailed, err), refreshErr) } return refreshErr, nil } @@ -369,11 +408,11 @@ func UninstallPluginForAgent(ctx context.Context, agent *agents.Agent, rec Plugi } if args := pluginDisableArgs(agent, rec); args != nil { if _, err := runAgentCmd(ctx, pluginCmdTimeout, prepend(bin, args)); err != nil && !claudeAlreadyDisabled(err) { - return &BlockedError{Agent: agent.Name, Reason: ReasonInstallFailed, Detail: stderrOf(err)} + return newBlockedError(agent, ReasonInstallFailed, err) } } if _, err := runAgentCmd(ctx, pluginCmdTimeout, prepend(bin, pluginUninstallArgs(agent, rec))); err != nil { - return &BlockedError{Agent: agent.Name, Reason: ReasonInstallFailed, Detail: stderrOf(err)} + return newBlockedError(agent, ReasonInstallFailed, err) } // Never de-register a Shared marketplace, it is shared infrastructure we don't own if rec.InstalledMarketplace && !keepMarketplace && !agent.Plugin.Shared { diff --git a/libs/aitools/installer/plugin_test.go b/libs/aitools/installer/plugin_test.go index a87dfa0a8de..5749329f243 100644 --- a/libs/aitools/installer/plugin_test.go +++ b/libs/aitools/installer/plugin_test.go @@ -8,6 +8,7 @@ import ( "os/exec" "path/filepath" "testing" + "time" "github.com/databricks/cli/libs/aitools/agents" "github.com/databricks/cli/libs/cmdio" @@ -838,3 +839,36 @@ func TestRecordPluginInstallsThenSkillsInstallNoPanic(t *testing.T) { assert.NotEmpty(t, st.Skills) assert.Contains(t, st.Plugins, agents.NameClaudeCode) } + +func TestRunAgentCmdTimeoutKeepsPartialStderr(t *testing.T) { + // What the CLI wrote before it was killed often says why it hung. + ctx, stub := process.WithStub(t.Context()) + stub.WithCallback(func(cmd *exec.Cmd) error { + _, _ = cmd.Stderr.Write([]byte("Waiting for login...\n")) + time.Sleep(50 * time.Millisecond) + return errors.New("signal: killed") + }) + + _, err := runAgentCmd(ctx, time.Millisecond, []string{"claude", "plugin", "--help"}) + require.Error(t, err) + blocked := newBlockedError(claudeAgent(), ReasonCLINotOnPath, err) + assert.True(t, blocked.TimedOut) + assert.Equal(t, "timed out after 1ms: Waiting for login...", blocked.Detail) +} + +func TestRunAgentCmdReportsTimeout(t *testing.T) { + // A slow agent CLI used to be reported as "not on PATH" with whatever the + // killed process left on stderr. + ctx, stub := process.WithStub(t.Context()) + stub.WithCallback(func(*exec.Cmd) error { + time.Sleep(50 * time.Millisecond) + return errors.New("signal: killed") + }) + + _, err := runAgentCmd(ctx, time.Millisecond, []string{"claude", "plugin", "--help"}) + require.Error(t, err) + blocked := newBlockedError(claudeAgent(), ReasonCLINotOnPath, err) + assert.True(t, blocked.TimedOut) + assert.Equal(t, ReasonCLINotOnPath, blocked.Reason) + assert.Equal(t, "timed out after 1ms", blocked.Detail) +} diff --git a/libs/aitools/installer/source.go b/libs/aitools/installer/source.go index f3d9842f876..27721de3f3b 100644 --- a/libs/aitools/installer/source.go +++ b/libs/aitools/installer/source.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "io" "net/http" "time" @@ -33,20 +34,33 @@ func (s *GitHubManifestSource) FetchManifest(ctx context.Context, ref string) (* client := &http.Client{Timeout: 30 * time.Second} resp, err := client.Do(req) if err != nil { - return nil, fmt.Errorf("failed to fetch manifest: %w", err) + return nil, &FetchError{Stage: FetchStageManifest, Err: fmt.Errorf("failed to fetch manifest: %w", err)} } defer resp.Body.Close() if resp.StatusCode == http.StatusNotFound { - return nil, fmt.Errorf("skills manifest at %s@%s: %w", skillsRepoName, ref, clicompat.ErrNotFound) + return nil, &FetchError{Stage: FetchStageManifest, StatusCode: resp.StatusCode, Err: fmt.Errorf("skills manifest at %s@%s: %w", skillsRepoName, ref, clicompat.ErrNotFound)} } if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("failed to fetch manifest: HTTP %d", resp.StatusCode) + return nil, newHTTPFetchError(FetchStageManifest, resp, fmt.Errorf("failed to fetch manifest: HTTP %d", resp.StatusCode)) } + // Read the whole body before parsing it: a connection dropped mid-body is a + // fetch failure, while a body that arrived but is empty or truncated JSON is + // an invalid manifest. A streaming decoder reports both as io.ErrUnexpectedEOF. + // The cap is far above the manifest's size, so a large non-manifest body (a + // captive portal, a wrong mirror) is not read whole. One byte past the cap + // tells an oversized body apart from one that fits exactly. + data, err := io.ReadAll(io.LimitReader(resp.Body, maxFetchBodySize+1)) + if err != nil { + return nil, &FetchError{Stage: FetchStageManifest, StatusCode: resp.StatusCode, Err: fmt.Errorf("failed to read manifest: %w", err)} + } + if len(data) > maxFetchBodySize { + return nil, &FetchError{Stage: FetchStageManifest, StatusCode: resp.StatusCode, Err: ErrManifestTooLarge} + } var manifest Manifest - if err := json.NewDecoder(resp.Body).Decode(&manifest); err != nil { - return nil, fmt.Errorf("failed to parse manifest: %w", err) + if err := json.Unmarshal(data, &manifest); err != nil { + return nil, &FetchError{Stage: FetchStageManifest, StatusCode: resp.StatusCode, Err: fmt.Errorf("failed to parse manifest: %w", err)} } normalizeManifest(&manifest) diff --git a/libs/aitools/installer/source_test.go b/libs/aitools/installer/source_test.go index fb52563b686..a0d38c64459 100644 --- a/libs/aitools/installer/source_test.go +++ b/libs/aitools/installer/source_test.go @@ -1,9 +1,14 @@ package installer import ( + "errors" + "net/http" + "net/http/httptest" + "strings" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestNormalizeManifestStampsSourceNameAndRepoDir(t *testing.T) { @@ -60,3 +65,34 @@ func TestNormalizeManifestOnlyExperimentalSkills(t *testing.T) { assert.Equal(t, experimentalRepoPath, got.RepoDir) assert.Equal(t, "x", got.SourceName) } + +func TestFetchManifestSizeCap(t *testing.T) { + tests := []struct { + name string + body string + wantErr bool + }{ + // Whitespace pads a valid manifest to exactly the cap: it still parses. + {name: "at the cap", body: strings.Repeat(" ", maxFetchBodySize-2) + "{}"}, + {name: "one byte over the cap", body: strings.Repeat(" ", maxFetchBodySize-1) + "{}", wantErr: true}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(tc.body)) + })) + t.Cleanup(server.Close) + t.Setenv("DATABRICKS_SKILLS_BASE_URL", server.URL) + + _, err := (&GitHubManifestSource{}).FetchManifest(t.Context(), "v1") + if !tc.wantErr { + require.NoError(t, err) + return + } + require.ErrorIs(t, err, ErrManifestTooLarge) + _, ok := errors.AsType[*FetchError](err) + assert.True(t, ok) + assert.Equal(t, "skills manifest is larger than 1 MiB", err.Error()) + }) + } +} diff --git a/libs/aitools/installer/state.go b/libs/aitools/installer/state.go index ddd51568bde..eba214eb79e 100644 --- a/libs/aitools/installer/state.go +++ b/libs/aitools/installer/state.go @@ -90,7 +90,8 @@ type FileRecord struct { // LoadState reads install state from the given directory. // Returns (nil, nil) when the state file does not exist. func LoadState(dir string) (*InstallState, error) { - data, err := os.ReadFile(filepath.Join(dir, stateFileName)) + path := filepath.Join(dir, stateFileName) + data, err := os.ReadFile(path) if errors.Is(err, os.ErrNotExist) { return nil, nil } @@ -100,7 +101,7 @@ func LoadState(dir string) (*InstallState, error) { var state InstallState if err := json.Unmarshal(data, &state); err != nil { - return nil, fmt.Errorf("failed to parse state file: %w", err) + return nil, &StateError{Path: path, Err: fmt.Errorf("failed to parse state file: %w", err)} } migrateState(&state) return &state, nil diff --git a/libs/aitools/installer/state_test.go b/libs/aitools/installer/state_test.go index 8650d19d27e..4abee62d8cb 100644 --- a/libs/aitools/installer/state_test.go +++ b/libs/aitools/installer/state_test.go @@ -1,8 +1,11 @@ package installer import ( + "errors" + "io/fs" "os" "path/filepath" + "runtime" "testing" "time" @@ -83,6 +86,34 @@ func TestLoadStateCorruptJSON(t *testing.T) { assert.Contains(t, err.Error(), "failed to parse state file") } +func TestLoadStateCorruptReturnsStateError(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, stateFileName), []byte("{not json"), 0o644)) + + _, err := LoadState(dir) + serr, ok := errors.AsType[*StateError](err) + require.True(t, ok, "want *StateError, got %T", err) + assert.Equal(t, filepath.Join(dir, stateFileName), serr.Path) +} + +func TestLoadStateUnreadableIsNotStateError(t *testing.T) { + if runtime.GOOS == "windows" || os.Geteuid() == 0 { + t.Skip("needs POSIX permissions and a non-root user") + } + dir := t.TempDir() + path := filepath.Join(dir, stateFileName) + require.NoError(t, os.WriteFile(path, []byte("{}"), 0o644)) + require.NoError(t, os.Chmod(path, 0o000)) + t.Cleanup(func() { _ = os.Chmod(path, 0o644) }) + + // An unreadable file is a permission problem, not invalid content: moving + // it aside would not help, so it must not look like a StateError. + _, err := LoadState(dir) + require.ErrorIs(t, err, fs.ErrPermission) + _, isState := errors.AsType[*StateError](err) + assert.False(t, isState) +} + func TestGlobalSkillsDir(t *testing.T) { ctx := env.WithUserHomeDir(t.Context(), "/fake/home") dir, err := GlobalSkillsDir(ctx) diff --git a/libs/telemetry/protos/aitools_install.go b/libs/telemetry/protos/aitools_install.go index c23503eea09..406dde89a58 100644 --- a/libs/telemetry/protos/aitools_install.go +++ b/libs/telemetry/protos/aitools_install.go @@ -53,8 +53,9 @@ const ( // The user aborted a prompt (Ctrl-C / Esc). AitoolsErrorCategoryUserAborted AitoolsErrorCategory = "USER_ABORTED" - // Fetching skills. The typed installer errors (release lookup, skills - // fetch) win over the generic network/timeout classification. + // Fetching skills. A fetch that fails on the transport (no response, or a + // timeout or reset while reading the body) reports NETWORK or TIMEOUT; the + // other categories apply once a response arrived. AitoolsErrorCategoryReleaseLookupFailed AitoolsErrorCategory = "RELEASE_LOOKUP_FAILED" AitoolsErrorCategoryRateLimited AitoolsErrorCategory = "RATE_LIMITED" AitoolsErrorCategorySkillsFetchFailed AitoolsErrorCategory = "SKILLS_FETCH_FAILED"