From f56fb575ee9765c83db0d9bb016e2b5aecdd1032 Mon Sep 17 00:00:00 2001 From: Caroline Chen <324939130+caroline-db@users.noreply.github.com> Date: Wed, 11 Nov 2026 22:02:20 +0000 Subject: [PATCH 1/3] air: validate log node indices --- acceptance/air/logs-download/output.txt | 2 +- cmd/air/logdownload.go | 14 ++++++- cmd/air/logs.go | 14 ++++++- cmd/air/logs_test.go | 55 +++++++++++++++++++++++++ 4 files changed, 81 insertions(+), 4 deletions(-) diff --git a/acceptance/air/logs-download/output.txt b/acceptance/air/logs-download/output.txt index 44bf1ec1e8b..5784d0a6f05 100644 --- a/acceptance/air/logs-download/output.txt +++ b/acceptance/air/logs-download/output.txt @@ -7,7 +7,7 @@ No logs available for run 123. Run terminated in state SUCCESS === download-to with an out-of-range node is rejected >>> [CLI] air logs 123 --download-to dl-logs --node 5 -Error: invalid --node 5: run has 2 node(s), indexed 0 to 1 +Error: invalid --node 5: this run has 2 nodes (valid node indices: 0 through 1) Exit code: 1 diff --git a/cmd/air/logdownload.go b/cmd/air/logdownload.go index a0eeea33a05..54bbc4808e6 100644 --- a/cmd/air/logdownload.go +++ b/cmd/air/logdownload.go @@ -26,6 +26,16 @@ const downloadConcurrency = 8 // input, so the caller reports it as an invalid argument rather than a failure. var errNodeOutOfRange = errors.New("invalid --node") +func validateNodeIndex(node, numNodes int) error { + if node < numNodes { + return nil + } + if numNodes == 1 { + return fmt.Errorf("%w %d: this run has 1 node (valid node index: 0)", errNodeOutOfRange, node) + } + return fmt.Errorf("%w %d: this run has %d nodes (valid node indices: 0 through %d)", errNodeOutOfRange, node, numNodes, numNodes-1) +} + // resolveNodeCount returns how many nodes a run used. func resolveNodeCount(run *jobs.Run) (int, error) { accelType, count := jobCompute(run) @@ -68,8 +78,8 @@ func downloadLogs(ctx context.Context, w *databricks.WorkspaceClient, out io.Wri nodes := make([]int, 0, numNodes) if req.nodeSet { - if req.node >= numNodes { - return false, fmt.Errorf("%w %d: run has %d node(s), indexed 0 to %d", errNodeOutOfRange, req.node, numNodes, numNodes-1) + if err := validateNodeIndex(req.node, numNodes); err != nil { + return false, err } nodes = append(nodes, req.node) } else { diff --git a/cmd/air/logs.go b/cmd/air/logs.go index 949c0e21d36..9d0c31574e8 100644 --- a/cmd/air/logs.go +++ b/cmd/air/logs.go @@ -151,7 +151,7 @@ func runLogs(ctx context.Context, cmd *cobra.Command, req logRequest) error { return authError(ctx, cmd, err) } - status, err := resolveRunStatus(ctx, w, req.runID) + run, err := w.Jobs.GetRun(ctx, jobs.GetRunRequest{RunId: req.runID}) if err != nil { if errors.Is(err, apierr.ErrResourceDoesNotExist) { return renderError(ctx, cmd, "NOT_FOUND", "NOT_FOUND", false, @@ -160,6 +160,18 @@ func runLogs(ctx context.Context, cmd *cobra.Command, req logRequest) error { return renderError(ctx, cmd, "INTERNAL_ERROR", "TRANSIENT", true, fmt.Errorf("failed to get status for run %d: %w", req.runID, err)) } + status := projectRunStatus(run) + + if req.nodeSet { + numNodes, err := resolveNodeCount(run) + if err != nil { + return renderError(ctx, cmd, "INTERNAL_ERROR", "TRANSIENT", true, + fmt.Errorf("failed to resolve node count for run %d: %w", req.runID, err)) + } + if err := validateNodeIndex(req.node, numNodes); err != nil { + return renderError(ctx, cmd, "INVALID_ARGS", "PERMANENT", false, err) + } + } // -1 (default) means latest; reject an attempt past the newest. if req.attempt >= 0 && req.attempt > status.latestAttempt { diff --git a/cmd/air/logs_test.go b/cmd/air/logs_test.go index 857339360f8..ac379611f11 100644 --- a/cmd/air/logs_test.go +++ b/cmd/air/logs_test.go @@ -3,6 +3,7 @@ package aircmd import ( "bytes" "context" + "encoding/json" "net/http" "net/http/httptest" "strings" @@ -103,6 +104,60 @@ func TestLogsFlagValidation(t *testing.T) { } } +func invalidNodeLogsServer(t *testing.T) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == "/api/2.2/jobs/runs/get": + _, _ = w.Write([]byte(`{ + "run_id": 5, + "state": {"life_cycle_state": "TERMINATED", "result_state": "SUCCESS"}, + "tasks": [{ + "run_id": 456, + "ai_runtime_task": {"deployments": [{"compute": {"accelerator_type": "GPU_1xA10", "accelerator_count": 1}}]} + }] + }`)) + case strings.Contains(r.URL.Path, "/logs"): + t.Errorf("logs endpoint called for an invalid node index") + default: + _, _ = w.Write([]byte(`{"userName":"u@example.com"}`)) + } + })) + t.Cleanup(srv.Close) + return srv +} + +func TestLogsRejectsOutOfRangeNode(t *testing.T) { + srv := invalidNodeLogsServer(t) + ctx := cmdctx.SetWorkspaceClient(cmdio.MockDiscard(t.Context()), newTestWorkspaceClient(t, srv.URL)) + cmd := withOutput(&cobra.Command{}, flags.OutputText) + cmd.SetContext(ctx) + + err := runLogs(ctx, cmd, logRequest{runID: 5, node: 1, nodeSet: true, attempt: -1, tailLines: -1}) + require.Error(t, err) + assert.Equal(t, "invalid --node 1: this run has 1 node (valid node index: 0)", err.Error()) +} + +func TestLogsRejectsOutOfRangeNodeJSON(t *testing.T) { + srv := invalidNodeLogsServer(t) + var buf bytes.Buffer + ctx := cmdio.InContext(t.Context(), cmdio.NewIO(t.Context(), flags.OutputJSON, nil, &buf, &buf, "", "")) + ctx = cmdctx.SetWorkspaceClient(ctx, newTestWorkspaceClient(t, srv.URL)) + cmd := withOutput(&cobra.Command{}, flags.OutputJSON) + cmd.SetContext(ctx) + + err := runLogs(ctx, cmd, logRequest{runID: 5, node: 1, nodeSet: true, attempt: -1, tailLines: -1, jsonOutput: true}) + require.ErrorIs(t, err, root.ErrAlreadyPrinted) + + var got errorEnvelope + require.NoError(t, json.Unmarshal(buf.Bytes(), &got)) + assert.Equal(t, jsonError{ + Code: "INVALID_ARGS", + Kind: "PERMANENT", + Message: "invalid --node 1: this run has 1 node (valid node index: 0)", + }, got.Error) +} + // completedRunLogsServer serves the auth probe, a terminal runs/get, and a // single page of Bricklens logs (newest-first, as the tail fetch requests). func completedRunLogsServer(t *testing.T) *httptest.Server { From 2df5eeb1a3cf8b7af412d14327e1379134747046 Mon Sep 17 00:00:00 2001 From: Caroline Chen <324939130+caroline-db@users.noreply.github.com> Date: Wed, 11 Nov 2026 22:03:20 +0000 Subject: [PATCH 2/3] Add changelog entry for log node validation --- .nextchanges/air/7516.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 .nextchanges/air/7516.md diff --git a/.nextchanges/air/7516.md b/.nextchanges/air/7516.md new file mode 100644 index 00000000000..dbc9f1c7150 --- /dev/null +++ b/.nextchanges/air/7516.md @@ -0,0 +1 @@ +* Reject out-of-range node indices in `databricks air logs` with an actionable error. ([#7516](https://github.com/databricks/cli/pull/7516)) From 5249b203f67d1870369f662133a142d170176803 Mon Sep 17 00:00:00 2001 From: Caroline Chen <324939130+caroline-db@users.noreply.github.com> Date: Thu, 12 Nov 2026 21:05:46 +0000 Subject: [PATCH 3/3] air: address log node validation feedback --- .nextchanges/air/7516.md | 2 +- acceptance/air/logs-download/output.txt | 2 +- .../air/logs-mlflow-fallback/output.txt | 10 +++ acceptance/air/logs-mlflow-fallback/script | 6 ++ acceptance/air/logs-mlflow-fallback/test.toml | 9 ++- acceptance/air/logs/output.txt | 5 ++ acceptance/air/logs/script | 3 + acceptance/air/logs/test.toml | 9 ++- cmd/air/logdownload.go | 19 +----- cmd/air/logdownload_test.go | 64 ++++--------------- cmd/air/logs.go | 16 ++--- cmd/air/logs_test.go | 61 +++++++++++++++++- cmd/air/logstream_support.go | 18 ++++-- cmd/air/logstream_test.go | 15 ++++- 14 files changed, 144 insertions(+), 95 deletions(-) diff --git a/.nextchanges/air/7516.md b/.nextchanges/air/7516.md index dbc9f1c7150..0159e492339 100644 --- a/.nextchanges/air/7516.md +++ b/.nextchanges/air/7516.md @@ -1 +1 @@ -* Reject out-of-range node indices in `databricks air logs` with an actionable error. ([#7516](https://github.com/databricks/cli/pull/7516)) +* `databricks air logs --node` now reports an out-of-range node index when it can determine the run's node count, instead of looking like a successful run with no logs. ([#7516](https://github.com/databricks/cli/pull/7516)) diff --git a/acceptance/air/logs-download/output.txt b/acceptance/air/logs-download/output.txt index 5784d0a6f05..44bf1ec1e8b 100644 --- a/acceptance/air/logs-download/output.txt +++ b/acceptance/air/logs-download/output.txt @@ -7,7 +7,7 @@ No logs available for run 123. Run terminated in state SUCCESS === download-to with an out-of-range node is rejected >>> [CLI] air logs 123 --download-to dl-logs --node 5 -Error: invalid --node 5: this run has 2 nodes (valid node indices: 0 through 1) +Error: invalid --node 5: run has 2 node(s), indexed 0 to 1 Exit code: 1 diff --git a/acceptance/air/logs-mlflow-fallback/output.txt b/acceptance/air/logs-mlflow-fallback/output.txt index 75459d38945..34b85a3b214 100644 --- a/acceptance/air/logs-mlflow-fallback/output.txt +++ b/acceptance/air/logs-mlflow-fallback/output.txt @@ -8,3 +8,13 @@ No logs available for run 123. Run terminated in state SUCCESS >>> [CLI] air logs 123 -o json {"type":"ERROR","ts":"[TIMESTAMP]","node":0,"line":"No logs available for run 123. Run terminated in state SUCCESS"} + +=== logs falls back to mlflow for a specific node + +>>> [CLI] air logs 123 --node 0 +No logs available for node 0 in run 123. Run terminated in state SUCCESS + +=== logs falls back to mlflow for a specific node (json) + +>>> [CLI] air logs 123 --node 0 -o json +{"type":"ERROR","ts":"[TIMESTAMP]","node":0,"line":"No logs available for node 0 in run 123. Run terminated in state SUCCESS"} diff --git a/acceptance/air/logs-mlflow-fallback/script b/acceptance/air/logs-mlflow-fallback/script index d8c52588bd2..c0a357c1d88 100644 --- a/acceptance/air/logs-mlflow-fallback/script +++ b/acceptance/air/logs-mlflow-fallback/script @@ -7,3 +7,9 @@ errcode trace $CLI air logs 123 title "logs falls back to mlflow (json)" errcode trace $CLI air logs 123 -o json + +title "logs falls back to mlflow for a specific node" +trace $CLI air logs 123 --node 0 + +title "logs falls back to mlflow for a specific node (json)" +trace $CLI air logs 123 --node 0 -o json diff --git a/acceptance/air/logs-mlflow-fallback/test.toml b/acceptance/air/logs-mlflow-fallback/test.toml index 9b9bcab8e68..1c80590959a 100644 --- a/acceptance/air/logs-mlflow-fallback/test.toml +++ b/acceptance/air/logs-mlflow-fallback/test.toml @@ -13,7 +13,14 @@ Response.Body = ''' "start_time": 1700000000000, "end_time": 1700000012000, "state": {"life_cycle_state": "TERMINATED", "result_state": "SUCCESS"}, - "tasks": [{"task_key": "train", "run_id": 456, "attempt_number": 0}] + "tasks": [{ + "task_key": "train", + "run_id": 456, + "attempt_number": 0, + "ai_runtime_task": { + "deployments": [{"compute": {"accelerator_type": "GPU_1xA10", "accelerator_count": 1}}] + } + }] } ''' diff --git a/acceptance/air/logs/output.txt b/acceptance/air/logs/output.txt index c3d35ed8b3f..7eab317d6c8 100644 --- a/acceptance/air/logs/output.txt +++ b/acceptance/air/logs/output.txt @@ -76,3 +76,8 @@ Exit code: 1 Error: invalid --node -1: must not be negative Exit code: 1 + +=== out-of-range node + +>>> [CLI] air logs 123 --node 1 +Error: invalid --node 1: run has 1 node(s), indexed 0 to 0 diff --git a/acceptance/air/logs/script b/acceptance/air/logs/script index 9060c674a74..a44831a164b 100644 --- a/acceptance/air/logs/script +++ b/acceptance/air/logs/script @@ -27,3 +27,6 @@ errcode trace $CLI air logs notanumber title "negative node" errcode trace $CLI air logs 123 --node -1 + +title "out-of-range node" +musterr trace $CLI air logs 123 --node 1 diff --git a/acceptance/air/logs/test.toml b/acceptance/air/logs/test.toml index 31e95adc8c1..d35dcce7342 100644 --- a/acceptance/air/logs/test.toml +++ b/acceptance/air/logs/test.toml @@ -13,7 +13,14 @@ Response.Body = ''' "start_time": 1700000000000, "end_time": 1700000012000, "state": {"life_cycle_state": "TERMINATED", "result_state": "SUCCESS"}, - "tasks": [{"task_key": "train", "run_id": 456, "attempt_number": 0}] + "tasks": [{ + "task_key": "train", + "run_id": 456, + "attempt_number": 0, + "ai_runtime_task": { + "deployments": [{"compute": {"accelerator_type": "GPU_1xA10", "accelerator_count": 1}}] + } + }] } ''' diff --git a/cmd/air/logdownload.go b/cmd/air/logdownload.go index 54bbc4808e6..a33a0d92ab9 100644 --- a/cmd/air/logdownload.go +++ b/cmd/air/logdownload.go @@ -22,18 +22,11 @@ import ( // the artifact store on a wide run. const downloadConcurrency = 8 -// errNodeOutOfRange marks --node naming a node the run doesn't have. It is user -// input, so the caller reports it as an invalid argument rather than a failure. -var errNodeOutOfRange = errors.New("invalid --node") - func validateNodeIndex(node, numNodes int) error { if node < numNodes { return nil } - if numNodes == 1 { - return fmt.Errorf("%w %d: this run has 1 node (valid node index: 0)", errNodeOutOfRange, node) - } - return fmt.Errorf("%w %d: this run has %d nodes (valid node indices: 0 through %d)", errNodeOutOfRange, node, numNodes, numNodes-1) + return fmt.Errorf("invalid --node %d: run has %d node(s), indexed 0 to %d", node, numNodes, numNodes-1) } // resolveNodeCount returns how many nodes a run used. @@ -65,12 +58,7 @@ func resolveNodeCount(run *jobs.Run) (int, error) { // prints a summary. An explicit --node downloads only that node; otherwise all of // them. Logs come from MLflow artifacts, since Bricklens only streams. The // returned bool is the run's outcome, so the exit code matches the streaming path. -func downloadLogs(ctx context.Context, w *databricks.WorkspaceClient, out io.Writer, req logRequest, status logRunStatus) (bool, error) { - run, err := w.Jobs.GetRun(ctx, jobs.GetRunRequest{RunId: req.runID}) - if err != nil { - return false, err - } - +func downloadLogs(ctx context.Context, w *databricks.WorkspaceClient, out io.Writer, req logRequest, status logRunStatus, run *jobs.Run) (bool, error) { numNodes, err := resolveNodeCount(run) if err != nil { return false, err @@ -78,9 +66,6 @@ func downloadLogs(ctx context.Context, w *databricks.WorkspaceClient, out io.Wri nodes := make([]int, 0, numNodes) if req.nodeSet { - if err := validateNodeIndex(req.node, numNodes); err != nil { - return false, err - } nodes = append(nodes, req.node) } else { for n := range numNodes { diff --git a/cmd/air/logdownload_test.go b/cmd/air/logdownload_test.go index 9a065cbf0b0..63c307b2f58 100644 --- a/cmd/air/logdownload_test.go +++ b/cmd/air/logdownload_test.go @@ -120,20 +120,12 @@ func TestDownloadAllNodeLogs(t *testing.T) { assert.FileExists(t, nodeLogs[1]) } -// fullDownloadServer also serves the run and its output, so downloadLogs can run -// end to end against a 2-node run. +// fullDownloadServer serves the run output and artifact chain for a full download. func fullDownloadServer(t *testing.T) *httptest.Server { t.Helper() var base string - runGet := `{ - "run_id": 123, - "state": {"life_cycle_state": "TERMINATED", "result_state": "SUCCESS"}, - "tasks": [{"run_id": 456, "ai_runtime_task": {"deployments": [{"compute": {"accelerator_type": "GPU_1xA10", "accelerator_count": 2}}]}}] - }` srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { - case "/api/2.2/jobs/runs/get": - _, _ = w.Write([]byte(runGet)) case "/api/2.2/jobs/runs/get-output": _, _ = w.Write([]byte(`{"ai_runtime_task_output": {"mlflow_experiment_id": "exp1", "mlflow_run_id": "run1"}}`)) case "/api/2.0/mlflow/artifacts/list": @@ -160,7 +152,7 @@ func TestDownloadLogsAllNodes(t *testing.T) { w := newTestWorkspaceClient(t, fullDownloadServer(t).URL) dir := t.TempDir() - success, err := downloadLogs(ctx, w, &bytes.Buffer{}, logRequest{runID: 123, attempt: -1, downloadTo: dir}, logRunStatus{resultState: "SUCCESS"}) + success, err := downloadLogs(ctx, w, &bytes.Buffer{}, logRequest{runID: 123, attempt: -1, downloadTo: dir}, logRunStatus{resultState: "SUCCESS"}, airRunWithCompute("GPU_1xA10", 2)) require.NoError(t, err) assert.True(t, success) assert.FileExists(t, filepath.Join(dir, "logs", "node_0.log")) @@ -172,23 +164,13 @@ func TestDownloadLogsSingleNode(t *testing.T) { w := newTestWorkspaceClient(t, fullDownloadServer(t).URL) dir := t.TempDir() - success, err := downloadLogs(ctx, w, &bytes.Buffer{}, logRequest{runID: 123, node: 1, nodeSet: true, attempt: -1, downloadTo: dir}, logRunStatus{resultState: "SUCCESS"}) + success, err := downloadLogs(ctx, w, &bytes.Buffer{}, logRequest{runID: 123, node: 1, nodeSet: true, attempt: -1, downloadTo: dir}, logRunStatus{resultState: "SUCCESS"}, airRunWithCompute("GPU_1xA10", 2)) require.NoError(t, err) assert.True(t, success) assert.FileExists(t, filepath.Join(dir, "logs", "node_1.log")) assert.NoFileExists(t, filepath.Join(dir, "logs", "node_0.log")) } -func TestDownloadLogsOutOfRangeNode(t *testing.T) { - ctx := cmdio.MockDiscard(t.Context()) - w := newTestWorkspaceClient(t, fullDownloadServer(t).URL) - dir := t.TempDir() - - _, err := downloadLogs(ctx, w, &bytes.Buffer{}, logRequest{runID: 123, node: 5, nodeSet: true, attempt: -1, downloadTo: dir}, logRunStatus{resultState: "SUCCESS"}) - require.Error(t, err) - assert.Contains(t, err.Error(), "invalid --node 5") -} - func TestDownloadLogsExplicitNodeZero(t *testing.T) { ctx := cmdio.MockDiscard(t.Context()) w := newTestWorkspaceClient(t, fullDownloadServer(t).URL) @@ -196,37 +178,17 @@ func TestDownloadLogsExplicitNodeZero(t *testing.T) { // An explicit --node 0 must download only node 0, unlike the default which // downloads every node. - _, err := downloadLogs(ctx, w, &bytes.Buffer{}, logRequest{runID: 123, node: 0, nodeSet: true, attempt: -1, downloadTo: dir}, logRunStatus{resultState: "SUCCESS"}) + _, err := downloadLogs(ctx, w, &bytes.Buffer{}, logRequest{runID: 123, node: 0, nodeSet: true, attempt: -1, downloadTo: dir}, logRunStatus{resultState: "SUCCESS"}, airRunWithCompute("GPU_1xA10", 2)) require.NoError(t, err) assert.FileExists(t, filepath.Join(dir, "logs", "node_0.log")) assert.NoFileExists(t, filepath.Join(dir, "logs", "node_1.log")) } -func TestDownloadLogsOutOfRangeNodeIsInvalidArgs(t *testing.T) { - ctx := cmdio.MockDiscard(t.Context()) - w := newTestWorkspaceClient(t, fullDownloadServer(t).URL) - - // The sentinel lets the caller classify this as bad input rather than a - // transient, retryable failure. - _, err := downloadLogs(ctx, w, &bytes.Buffer{}, logRequest{runID: 123, node: 5, nodeSet: true, attempt: -1, downloadTo: t.TempDir()}, logRunStatus{resultState: "SUCCESS"}) - require.ErrorIs(t, err, errNodeOutOfRange) -} - -// noLogsDownloadServer serves a run with no resolvable MLflow run, so there is -// nothing to download. +// noLogsDownloadServer serves no resolvable MLflow run, so there is nothing to download. func noLogsDownloadServer(t *testing.T) *httptest.Server { t.Helper() srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - switch r.URL.Path { - case "/api/2.2/jobs/runs/get": - _, _ = w.Write([]byte(`{ - "run_id": 123, - "state": {"life_cycle_state": "TERMINATED", "result_state": "SUCCESS"}, - "tasks": [{"run_id": 456, "ai_runtime_task": {"deployments": [{"compute": {"accelerator_type": "GPU_1xA10", "accelerator_count": 2}}]}}] - }`)) - default: - _, _ = w.Write([]byte(`{}`)) - } + _, _ = w.Write([]byte(`{}`)) })) t.Cleanup(srv.Close) return srv @@ -239,7 +201,7 @@ func TestDownloadLogsNoLogsMatchesStreamingPath(t *testing.T) { // A SUCCESS run with no logs reports it and still succeeds, exactly as the // streaming path does — no error, no non-zero exit. var buf bytes.Buffer - success, err := downloadLogs(ctx, w, &buf, logRequest{runID: 123, attempt: -1, downloadTo: t.TempDir()}, logRunStatus{lifeCycleState: "TERMINATED", resultState: "SUCCESS"}) + success, err := downloadLogs(ctx, w, &buf, logRequest{runID: 123, attempt: -1, downloadTo: t.TempDir()}, logRunStatus{lifeCycleState: "TERMINATED", resultState: "SUCCESS"}, airRunWithCompute("GPU_1xA10", 2)) require.NoError(t, err) assert.True(t, success) assert.Contains(t, buf.String(), "No logs available for run 123") @@ -506,20 +468,17 @@ func TestDownloadLogsActiveRunExitsZero(t *testing.T) { // Downloading a still-running run's logs is not a failure. success, err := downloadLogs(ctx, w, &bytes.Buffer{}, logRequest{runID: 123, attempt: -1, downloadTo: t.TempDir()}, - logRunStatus{lifeCycleState: "RUNNING"}) + logRunStatus{lifeCycleState: "RUNNING"}, + airRunWithCompute("GPU_1xA10", 2)) require.NoError(t, err) assert.True(t, success) } -// allNodesFailServer serves a FAILED 2-node run whose chunk credentials all 404, -// so every node's download fails. +// allNodesFailServer serves a 2-node log set whose chunk credentials all 404. func allNodesFailServer(t *testing.T) *httptest.Server { t.Helper() srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { - case "/api/2.2/jobs/runs/get": - _, _ = w.Write([]byte(`{"run_id": 123, "state": {"life_cycle_state": "TERMINATED", "result_state": "FAILED"}, - "tasks": [{"run_id": 456, "ai_runtime_task": {"deployments": [{"compute": {"accelerator_type": "GPU_1xA10", "accelerator_count": 2}}]}}]}`)) case "/api/2.2/jobs/runs/get-output": _, _ = w.Write([]byte(`{"ai_runtime_task_output": {"mlflow_experiment_id": "exp1", "mlflow_run_id": "run1"}}`)) case "/api/2.0/mlflow/artifacts/list": @@ -549,7 +508,8 @@ func TestDownloadLogsAllNodesFailedIsNotReportedAsNoLogs(t *testing.T) { // tell a caller the run produced nothing, so this fails instead. _, err := downloadLogs(ctx, w, &stdout, logRequest{runID: 123, attempt: -1, downloadTo: t.TempDir(), jsonOutput: true}, - logRunStatus{lifeCycleState: "TERMINATED", resultState: "FAILED"}) + logRunStatus{lifeCycleState: "TERMINATED", resultState: "FAILED"}, + airRunWithCompute("GPU_1xA10", 2)) require.Error(t, err) assert.Contains(t, err.Error(), "failed to download logs from any of 2 node(s)") assert.NotContains(t, stdout.String(), "No logs available") diff --git a/cmd/air/logs.go b/cmd/air/logs.go index 9d0c31574e8..9d337dd8f18 100644 --- a/cmd/air/logs.go +++ b/cmd/air/logs.go @@ -163,13 +163,10 @@ func runLogs(ctx context.Context, cmd *cobra.Command, req logRequest) error { status := projectRunStatus(run) if req.nodeSet { - numNodes, err := resolveNodeCount(run) - if err != nil { - return renderError(ctx, cmd, "INTERNAL_ERROR", "TRANSIENT", true, - fmt.Errorf("failed to resolve node count for run %d: %w", req.runID, err)) - } - if err := validateNodeIndex(req.node, numNodes); err != nil { - return renderError(ctx, cmd, "INVALID_ARGS", "PERMANENT", false, err) + if numNodes, err := resolveNodeCount(run); err == nil { + if err := validateNodeIndex(req.node, numNodes); err != nil { + return renderError(ctx, cmd, "INVALID_ARGS", "PERMANENT", false, err) + } } } @@ -181,10 +178,7 @@ func runLogs(ctx context.Context, cmd *cobra.Command, req logRequest) error { // --download-to writes each node's logs to disk instead of streaming. if req.downloadTo != "" { - success, err := downloadLogs(ctx, w, cmd.OutOrStdout(), req, status) - if errors.Is(err, errNodeOutOfRange) { - return renderError(ctx, cmd, "INVALID_ARGS", "PERMANENT", false, err) - } + success, err := downloadLogs(ctx, w, cmd.OutOrStdout(), req, status, run) if err != nil { return renderError(ctx, cmd, "INTERNAL_ERROR", "TRANSIENT", true, fmt.Errorf("failed to download logs for run %d: %w", req.runID, err)) diff --git a/cmd/air/logs_test.go b/cmd/air/logs_test.go index ac379611f11..e5b5cbc26c7 100644 --- a/cmd/air/logs_test.go +++ b/cmd/air/logs_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "fmt" "net/http" "net/http/httptest" "strings" @@ -135,7 +136,7 @@ func TestLogsRejectsOutOfRangeNode(t *testing.T) { err := runLogs(ctx, cmd, logRequest{runID: 5, node: 1, nodeSet: true, attempt: -1, tailLines: -1}) require.Error(t, err) - assert.Equal(t, "invalid --node 1: this run has 1 node (valid node index: 0)", err.Error()) + assert.Equal(t, "invalid --node 1: run has 1 node(s), indexed 0 to 0", err.Error()) } func TestLogsRejectsOutOfRangeNodeJSON(t *testing.T) { @@ -154,10 +155,66 @@ func TestLogsRejectsOutOfRangeNodeJSON(t *testing.T) { assert.Equal(t, jsonError{ Code: "INVALID_ARGS", Kind: "PERMANENT", - Message: "invalid --node 1: this run has 1 node (valid node index: 0)", + Message: "invalid --node 1: run has 1 node(s), indexed 0 to 0", }, got.Error) } +func TestLogsSkipsNodeValidationWhenNodeCountCannotBeResolved(t *testing.T) { + tests := []struct { + name string + task string + }{ + { + name: "legacy GenAI accelerator type", + task: `"gen_ai_compute_task": {"compute": {"gpu_type": "h100_80gb", "num_gpus": 8}}`, + }, + { + name: "unknown accelerator type", + task: `"ai_runtime_task": {"deployments": [{"compute": {"accelerator_type": "GPU_8xFuture", "accelerator_count": 8}}]}`, + }, + { + name: "zero accelerator count", + task: `"ai_runtime_task": {"deployments": [{"compute": {"accelerator_type": "GPU_1xA10", "accelerator_count": 0}}]}`, + }, + { + name: "partial node accelerator count", + task: `"ai_runtime_task": {"deployments": [{"compute": {"accelerator_type": "GPU_8xH100", "accelerator_count": 4}}]}`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == "/api/2.2/jobs/runs/get": + _, _ = fmt.Fprintf(w, `{ + "run_id": 5, + "start_time": 1000, + "end_time": 2000, + "state": {"life_cycle_state": "TERMINATED", "result_state": "SUCCESS"}, + "tasks": [{"run_id": 456, %s}] + }`, tt.task) + case strings.HasPrefix(r.URL.Path, "/api/2.0/ai-training/workflows/by-run-id/"): + _, _ = w.Write([]byte(`{"log_records": [{"time_unix_nano": 1000000000, "body": "node log", "node_index": 0}]}`)) + default: + _, _ = w.Write([]byte(`{"userName":"u@example.com"}`)) + } + })) + t.Cleanup(srv.Close) + + var out bytes.Buffer + ctx := cmdctx.SetWorkspaceClient(cmdio.MockDiscard(t.Context()), newTestWorkspaceClient(t, srv.URL)) + cmd := withOutput(&cobra.Command{}, flags.OutputText) + cmd.SetContext(ctx) + cmd.SetOut(&out) + + err := runLogs(ctx, cmd, logRequest{runID: 5, node: 0, nodeSet: true, attempt: -1, tailLines: -1}) + require.NoError(t, err) + assert.Equal(t, "node log\n", out.String()) + }) + } +} + // completedRunLogsServer serves the auth probe, a terminal runs/get, and a // single page of Bricklens logs (newest-first, as the tail fetch requests). func completedRunLogsServer(t *testing.T) *httptest.Server { diff --git a/cmd/air/logstream_support.go b/cmd/air/logstream_support.go index cb7a8d2d6d3..fb6735da646 100644 --- a/cmd/air/logstream_support.go +++ b/cmd/air/logstream_support.go @@ -220,16 +220,22 @@ func emitLogLine(out io.Writer, req logRequest, body string) { printLogEvent(out, "LOG", req.node, body) } -// emitNoLogs reports that a run produced no logs. A terminal run gets its -// termination reason; a still-active run is reported as having no logs yet, -// since the MLflow fallback is a one-shot that does not follow it to completion. -// Under --json it is a JSONL ERROR, so a consumer never sees an empty stream. +// emitNoLogs reports that a run or requested node produced no logs. A terminal +// run gets its termination reason; a still-active run is reported as having no +// logs yet, since the MLflow fallback is a one-shot that does not follow it to +// completion. Under --json it is a JSONL ERROR, so a consumer never sees an empty +// stream. func emitNoLogs(out io.Writer, req logRequest, status logRunStatus) { + subject := fmt.Sprintf("run %d", req.runID) + if req.nodeSet { + subject = fmt.Sprintf("node %d in run %d", req.node, req.runID) + } + var msg string if status.terminal() { - msg = fmt.Sprintf("No logs available for run %d. Run terminated in state %s", req.runID, status.displayState()) + msg = fmt.Sprintf("No logs available for %s. Run terminated in state %s", subject, status.displayState()) } else { - msg = fmt.Sprintf("No logs available yet for run %d, which is still in state %s", req.runID, status.displayState()) + msg = fmt.Sprintf("No logs available yet for %s, which is still in state %s", subject, status.displayState()) } if status.stateMessage != "" { msg = fmt.Sprintf("%s: %s", msg, status.stateMessage) diff --git a/cmd/air/logstream_test.go b/cmd/air/logstream_test.go index b1006d9b49c..37ac5405d4d 100644 --- a/cmd/air/logstream_test.go +++ b/cmd/air/logstream_test.go @@ -577,14 +577,23 @@ func TestEmitNoLogs(t *testing.T) { emitNoLogs(&text, req, tc.status) assert.Equal(t, tc.want, text.String()) + wantNode := strings.Replace(tc.want, "for run 7", "for node 1 in run 7", 1) + nodeReq := req + nodeReq.node = 1 + nodeReq.nodeSet = true + + var nodeText bytes.Buffer + emitNoLogs(&nodeText, nodeReq, tc.status) + assert.Equal(t, wantNode, nodeText.String()) + var jsonBuf bytes.Buffer - req.node, req.jsonOutput = 1, true - emitNoLogs(&jsonBuf, req, tc.status) + nodeReq.jsonOutput = true + emitNoLogs(&jsonBuf, nodeReq, tc.status) var ev logEvent require.NoError(t, json.Unmarshal(jsonBuf.Bytes(), &ev)) assert.Equal(t, "ERROR", ev.Type) assert.Equal(t, 1, ev.Node) - assert.Equal(t, strings.TrimRight(tc.want, "\n"), ev.Line) + assert.Equal(t, strings.TrimRight(wantNode, "\n"), ev.Line) }) } }