Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/air/7516.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* `databricks air logs --node` now reports an out-of-range node index when it can determine the run's node count, instead of looking like a successful run with no logs. ([#7516](https://github.com/databricks/cli/pull/7516))
10 changes: 10 additions & 0 deletions acceptance/air/logs-mlflow-fallback/output.txt
Original file line number Diff line number Diff line change
Expand Up @@ -8,3 +8,13 @@ No logs available for run 123. Run terminated in state SUCCESS

>>> [CLI] air logs 123 -o json
{"type":"ERROR","ts":"[TIMESTAMP]","node":0,"line":"No logs available for run 123. Run terminated in state SUCCESS"}

=== logs falls back to mlflow for a specific node

>>> [CLI] air logs 123 --node 0
No logs available for node 0 in run 123. Run terminated in state SUCCESS

=== logs falls back to mlflow for a specific node (json)

>>> [CLI] air logs 123 --node 0 -o json
{"type":"ERROR","ts":"[TIMESTAMP]","node":0,"line":"No logs available for node 0 in run 123. Run terminated in state SUCCESS"}
6 changes: 6 additions & 0 deletions acceptance/air/logs-mlflow-fallback/script
Original file line number Diff line number Diff line change
Expand Up @@ -7,3 +7,9 @@ errcode trace $CLI air logs 123

title "logs falls back to mlflow (json)"
errcode trace $CLI air logs 123 -o json

title "logs falls back to mlflow for a specific node"
trace $CLI air logs 123 --node 0

title "logs falls back to mlflow for a specific node (json)"
trace $CLI air logs 123 --node 0 -o json
9 changes: 8 additions & 1 deletion acceptance/air/logs-mlflow-fallback/test.toml
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,14 @@ Response.Body = '''
"start_time": 1700000000000,
"end_time": 1700000012000,
"state": {"life_cycle_state": "TERMINATED", "result_state": "SUCCESS"},
"tasks": [{"task_key": "train", "run_id": 456, "attempt_number": 0}]
"tasks": [{
"task_key": "train",
"run_id": 456,
"attempt_number": 0,
"ai_runtime_task": {
"deployments": [{"compute": {"accelerator_type": "GPU_1xA10", "accelerator_count": 1}}]
}
}]
}
'''

Expand Down
5 changes: 5 additions & 0 deletions acceptance/air/logs/output.txt
Original file line number Diff line number Diff line change
Expand Up @@ -76,3 +76,8 @@ Exit code: 1
Error: invalid --node -1: must not be negative

Exit code: 1

=== out-of-range node

>>> [CLI] air logs 123 --node 1
Error: invalid --node 1: run has 1 node(s), indexed 0 to 0
3 changes: 3 additions & 0 deletions acceptance/air/logs/script
Original file line number Diff line number Diff line change
Expand Up @@ -27,3 +27,6 @@ errcode trace $CLI air logs notanumber

title "negative node"
errcode trace $CLI air logs 123 --node -1

title "out-of-range node"
musterr trace $CLI air logs 123 --node 1
9 changes: 8 additions & 1 deletion acceptance/air/logs/test.toml
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,14 @@ Response.Body = '''
"start_time": 1700000000000,
"end_time": 1700000012000,
"state": {"life_cycle_state": "TERMINATED", "result_state": "SUCCESS"},
"tasks": [{"task_key": "train", "run_id": 456, "attempt_number": 0}]
"tasks": [{
"task_key": "train",
"run_id": 456,
"attempt_number": 0,
"ai_runtime_task": {
"deployments": [{"compute": {"accelerator_type": "GPU_1xA10", "accelerator_count": 1}}]
}
}]
}
'''

Expand Down
19 changes: 7 additions & 12 deletions cmd/air/logdownload.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,12 @@ import (
// the artifact store on a wide run.
const downloadConcurrency = 8

// errNodeOutOfRange marks --node naming a node the run doesn't have. It is user
// input, so the caller reports it as an invalid argument rather than a failure.
var errNodeOutOfRange = errors.New("invalid --node")
func validateNodeIndex(node, numNodes int) error {
Comment thread
caroline-db marked this conversation as resolved.
Resolved
if node < numNodes {
return nil
}
return fmt.Errorf("invalid --node %d: run has %d node(s), indexed 0 to %d", node, numNodes, numNodes-1)
}

// resolveNodeCount returns how many nodes a run used.
func resolveNodeCount(run *jobs.Run) (int, error) {
Expand Down Expand Up @@ -55,22 +58,14 @@ func resolveNodeCount(run *jobs.Run) (int, error) {
// prints a summary. An explicit --node downloads only that node; otherwise all of
// them. Logs come from MLflow artifacts, since Bricklens only streams. The
// returned bool is the run's outcome, so the exit code matches the streaming path.
func downloadLogs(ctx context.Context, w *databricks.WorkspaceClient, out io.Writer, req logRequest, status logRunStatus) (bool, error) {
run, err := w.Jobs.GetRun(ctx, jobs.GetRunRequest{RunId: req.runID})
if err != nil {
return false, err
}

func downloadLogs(ctx context.Context, w *databricks.WorkspaceClient, out io.Writer, req logRequest, status logRunStatus, run *jobs.Run) (bool, error) {
numNodes, err := resolveNodeCount(run)
if err != nil {
return false, err
}

nodes := make([]int, 0, numNodes)
if req.nodeSet {
if req.node >= numNodes {
return false, fmt.Errorf("%w %d: run has %d node(s), indexed 0 to %d", errNodeOutOfRange, req.node, numNodes, numNodes-1)
}
nodes = append(nodes, req.node)
} else {
for n := range numNodes {
Expand Down
64 changes: 12 additions & 52 deletions cmd/air/logdownload_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -120,20 +120,12 @@ func TestDownloadAllNodeLogs(t *testing.T) {
assert.FileExists(t, nodeLogs[1])
}

// fullDownloadServer also serves the run and its output, so downloadLogs can run
// end to end against a 2-node run.
// fullDownloadServer serves the run output and artifact chain for a full download.
func fullDownloadServer(t *testing.T) *httptest.Server {
t.Helper()
var base string
runGet := `{
"run_id": 123,
"state": {"life_cycle_state": "TERMINATED", "result_state": "SUCCESS"},
"tasks": [{"run_id": 456, "ai_runtime_task": {"deployments": [{"compute": {"accelerator_type": "GPU_1xA10", "accelerator_count": 2}}]}}]
}`
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/api/2.2/jobs/runs/get":
_, _ = w.Write([]byte(runGet))
case "/api/2.2/jobs/runs/get-output":
_, _ = w.Write([]byte(`{"ai_runtime_task_output": {"mlflow_experiment_id": "exp1", "mlflow_run_id": "run1"}}`))
case "/api/2.0/mlflow/artifacts/list":
Expand All @@ -160,7 +152,7 @@ func TestDownloadLogsAllNodes(t *testing.T) {
w := newTestWorkspaceClient(t, fullDownloadServer(t).URL)
dir := t.TempDir()

success, err := downloadLogs(ctx, w, &bytes.Buffer{}, logRequest{runID: 123, attempt: -1, downloadTo: dir}, logRunStatus{resultState: "SUCCESS"})
success, err := downloadLogs(ctx, w, &bytes.Buffer{}, logRequest{runID: 123, attempt: -1, downloadTo: dir}, logRunStatus{resultState: "SUCCESS"}, airRunWithCompute("GPU_1xA10", 2))
require.NoError(t, err)
assert.True(t, success)
assert.FileExists(t, filepath.Join(dir, "logs", "node_0.log"))
Expand All @@ -172,61 +164,31 @@ func TestDownloadLogsSingleNode(t *testing.T) {
w := newTestWorkspaceClient(t, fullDownloadServer(t).URL)
dir := t.TempDir()

success, err := downloadLogs(ctx, w, &bytes.Buffer{}, logRequest{runID: 123, node: 1, nodeSet: true, attempt: -1, downloadTo: dir}, logRunStatus{resultState: "SUCCESS"})
success, err := downloadLogs(ctx, w, &bytes.Buffer{}, logRequest{runID: 123, node: 1, nodeSet: true, attempt: -1, downloadTo: dir}, logRunStatus{resultState: "SUCCESS"}, airRunWithCompute("GPU_1xA10", 2))
require.NoError(t, err)
assert.True(t, success)
assert.FileExists(t, filepath.Join(dir, "logs", "node_1.log"))
assert.NoFileExists(t, filepath.Join(dir, "logs", "node_0.log"))
}

func TestDownloadLogsOutOfRangeNode(t *testing.T) {
ctx := cmdio.MockDiscard(t.Context())
w := newTestWorkspaceClient(t, fullDownloadServer(t).URL)
dir := t.TempDir()

_, err := downloadLogs(ctx, w, &bytes.Buffer{}, logRequest{runID: 123, node: 5, nodeSet: true, attempt: -1, downloadTo: dir}, logRunStatus{resultState: "SUCCESS"})
require.Error(t, err)
assert.Contains(t, err.Error(), "invalid --node 5")
}

func TestDownloadLogsExplicitNodeZero(t *testing.T) {
ctx := cmdio.MockDiscard(t.Context())
w := newTestWorkspaceClient(t, fullDownloadServer(t).URL)
dir := t.TempDir()

// An explicit --node 0 must download only node 0, unlike the default which
// downloads every node.
_, err := downloadLogs(ctx, w, &bytes.Buffer{}, logRequest{runID: 123, node: 0, nodeSet: true, attempt: -1, downloadTo: dir}, logRunStatus{resultState: "SUCCESS"})
_, err := downloadLogs(ctx, w, &bytes.Buffer{}, logRequest{runID: 123, node: 0, nodeSet: true, attempt: -1, downloadTo: dir}, logRunStatus{resultState: "SUCCESS"}, airRunWithCompute("GPU_1xA10", 2))
require.NoError(t, err)
assert.FileExists(t, filepath.Join(dir, "logs", "node_0.log"))
assert.NoFileExists(t, filepath.Join(dir, "logs", "node_1.log"))
}

func TestDownloadLogsOutOfRangeNodeIsInvalidArgs(t *testing.T) {
ctx := cmdio.MockDiscard(t.Context())
w := newTestWorkspaceClient(t, fullDownloadServer(t).URL)

// The sentinel lets the caller classify this as bad input rather than a
// transient, retryable failure.
_, err := downloadLogs(ctx, w, &bytes.Buffer{}, logRequest{runID: 123, node: 5, nodeSet: true, attempt: -1, downloadTo: t.TempDir()}, logRunStatus{resultState: "SUCCESS"})
require.ErrorIs(t, err, errNodeOutOfRange)
}

// noLogsDownloadServer serves a run with no resolvable MLflow run, so there is
// nothing to download.
// noLogsDownloadServer serves no resolvable MLflow run, so there is nothing to download.
func noLogsDownloadServer(t *testing.T) *httptest.Server {
t.Helper()
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/api/2.2/jobs/runs/get":
_, _ = w.Write([]byte(`{
"run_id": 123,
"state": {"life_cycle_state": "TERMINATED", "result_state": "SUCCESS"},
"tasks": [{"run_id": 456, "ai_runtime_task": {"deployments": [{"compute": {"accelerator_type": "GPU_1xA10", "accelerator_count": 2}}]}}]
}`))
default:
_, _ = w.Write([]byte(`{}`))
}
_, _ = w.Write([]byte(`{}`))
}))
t.Cleanup(srv.Close)
return srv
Expand All @@ -239,7 +201,7 @@ func TestDownloadLogsNoLogsMatchesStreamingPath(t *testing.T) {
// A SUCCESS run with no logs reports it and still succeeds, exactly as the
// streaming path does — no error, no non-zero exit.
var buf bytes.Buffer
success, err := downloadLogs(ctx, w, &buf, logRequest{runID: 123, attempt: -1, downloadTo: t.TempDir()}, logRunStatus{lifeCycleState: "TERMINATED", resultState: "SUCCESS"})
success, err := downloadLogs(ctx, w, &buf, logRequest{runID: 123, attempt: -1, downloadTo: t.TempDir()}, logRunStatus{lifeCycleState: "TERMINATED", resultState: "SUCCESS"}, airRunWithCompute("GPU_1xA10", 2))
require.NoError(t, err)
assert.True(t, success)
assert.Contains(t, buf.String(), "No logs available for run 123")
Expand Down Expand Up @@ -506,20 +468,17 @@ func TestDownloadLogsActiveRunExitsZero(t *testing.T) {
// Downloading a still-running run's logs is not a failure.
success, err := downloadLogs(ctx, w, &bytes.Buffer{},
logRequest{runID: 123, attempt: -1, downloadTo: t.TempDir()},
logRunStatus{lifeCycleState: "RUNNING"})
logRunStatus{lifeCycleState: "RUNNING"},
airRunWithCompute("GPU_1xA10", 2))
require.NoError(t, err)
assert.True(t, success)
}

// allNodesFailServer serves a FAILED 2-node run whose chunk credentials all 404,
// so every node's download fails.
// allNodesFailServer serves a 2-node log set whose chunk credentials all 404.
func allNodesFailServer(t *testing.T) *httptest.Server {
t.Helper()
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/api/2.2/jobs/runs/get":
_, _ = w.Write([]byte(`{"run_id": 123, "state": {"life_cycle_state": "TERMINATED", "result_state": "FAILED"},
"tasks": [{"run_id": 456, "ai_runtime_task": {"deployments": [{"compute": {"accelerator_type": "GPU_1xA10", "accelerator_count": 2}}]}}]}`))
case "/api/2.2/jobs/runs/get-output":
_, _ = w.Write([]byte(`{"ai_runtime_task_output": {"mlflow_experiment_id": "exp1", "mlflow_run_id": "run1"}}`))
case "/api/2.0/mlflow/artifacts/list":
Expand Down Expand Up @@ -549,7 +508,8 @@ func TestDownloadLogsAllNodesFailedIsNotReportedAsNoLogs(t *testing.T) {
// tell a caller the run produced nothing, so this fails instead.
_, err := downloadLogs(ctx, w, &stdout,
logRequest{runID: 123, attempt: -1, downloadTo: t.TempDir(), jsonOutput: true},
logRunStatus{lifeCycleState: "TERMINATED", resultState: "FAILED"})
logRunStatus{lifeCycleState: "TERMINATED", resultState: "FAILED"},
airRunWithCompute("GPU_1xA10", 2))
require.Error(t, err)
assert.Contains(t, err.Error(), "failed to download logs from any of 2 node(s)")
assert.NotContains(t, stdout.String(), "No logs available")
Expand Down
16 changes: 11 additions & 5 deletions cmd/air/logs.go
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,7 @@ func runLogs(ctx context.Context, cmd *cobra.Command, req logRequest) error {
return authError(ctx, cmd, err)
}

status, err := resolveRunStatus(ctx, w, req.runID)
run, err := w.Jobs.GetRun(ctx, jobs.GetRunRequest{RunId: req.runID})
if err != nil {
if errors.Is(err, apierr.ErrResourceDoesNotExist) {
return renderError(ctx, cmd, "NOT_FOUND", "NOT_FOUND", false,
Expand All @@ -160,6 +160,15 @@ func runLogs(ctx context.Context, cmd *cobra.Command, req logRequest) error {
return renderError(ctx, cmd, "INTERNAL_ERROR", "TRANSIENT", true,
fmt.Errorf("failed to get status for run %d: %w", req.runID, err))
}
status := projectRunStatus(run)

if req.nodeSet {
if numNodes, err := resolveNodeCount(run); err == nil {
if err := validateNodeIndex(req.node, numNodes); err != nil {
return renderError(ctx, cmd, "INVALID_ARGS", "PERMANENT", false, err)
}
}
}

// -1 (default) means latest; reject an attempt past the newest.
if req.attempt >= 0 && req.attempt > status.latestAttempt {
Expand All @@ -169,10 +178,7 @@ func runLogs(ctx context.Context, cmd *cobra.Command, req logRequest) error {

// --download-to writes each node's logs to disk instead of streaming.
if req.downloadTo != "" {
success, err := downloadLogs(ctx, w, cmd.OutOrStdout(), req, status)
if errors.Is(err, errNodeOutOfRange) {
return renderError(ctx, cmd, "INVALID_ARGS", "PERMANENT", false, err)
}
success, err := downloadLogs(ctx, w, cmd.OutOrStdout(), req, status, run)
if err != nil {
return renderError(ctx, cmd, "INTERNAL_ERROR", "TRANSIENT", true,
fmt.Errorf("failed to download logs for run %d: %w", req.runID, err))
Expand Down
Loading
Loading