Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/air/air-cancel-verify.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* `air cancel JOB_RUN_ID` now rejects run IDs that are not AI Runtime runs instead of cancelling unrelated Jobs runs. In `-o json` output, the `failed[].error` text for a run that does not exist or that you cannot view now starts with `failed to get run <id>:` instead of the cancel API error. ([#7569](https://github.com/databricks/cli/pull/7569))
12 changes: 12 additions & 0 deletions acceptance/air/cancel/test.toml
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,18 @@
Pattern = "HEAD /"
Response.Body = ''

# Cancel by ID verifies each run is an AIR run via runs/get before cancelling.
# Stubs match on path only, so every ID resolves to this AIR run.
[[Server]]
Pattern = "GET /api/2.2/jobs/runs/get"
Response.Body = '''
{
"run_id": 123,
"state": {"life_cycle_state": "RUNNING"},
"tasks": [{"run_id": 124, "ai_runtime_task": {"experiment": "/Users/tester@databricks.com/exp"}}]
}
'''

# CancelRun accepts the request and returns an empty body.
[[Server]]
Pattern = "POST /api/2.2/jobs/runs/cancel"
Expand Down
20 changes: 16 additions & 4 deletions cmd/air/cancel.go
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,8 @@ all active runs submitted by the current user.
}

for _, rid := range runIDs {
err := cancelRun(ctx, w, rid)
// --all only lists AIR runs, so they don't need re-checking.
err := cancelRun(ctx, w, rid, all)
if err != nil {
data.Failed = append(data.Failed, cancelFailure{RunID: rid, Error: err.Error()})
if !jsonOut {
Expand Down Expand Up @@ -192,13 +193,24 @@ func runNotFound(err error) bool {
return false
}

// cancelRun requests cancellation of a single job run. The cancel is async, so
// the returned waiter is ignored.
func cancelRun(ctx context.Context, w *databricks.WorkspaceClient, rid string) error {
// cancelRun requests cancellation of a single job run. Unless skipAirCheck is
// set, it first verifies the run is an AI Runtime run, so a mistyped but valid
// ID can't cancel an unrelated Jobs run. The cancel is async, so the returned
// waiter is ignored.
func cancelRun(ctx context.Context, w *databricks.WorkspaceClient, rid string, skipAirCheck bool) error {
runID, err := strconv.ParseInt(rid, 10, 64)
if err != nil || runID <= 0 {
return fmt.Errorf("invalid run ID %q: must be a positive integer", rid)
}
if !skipAirCheck {
run, err := fetchJobRun(ctx, w, runID)

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: With this change, the JSON failed[].error text for a run that doesn't exist (or that you can't view) changes. It used to be the cancel API's error. Now it's the error from fetchJobRun, which starts with failed to get run <id>: .... Text output is unaffected because runNotFound still matches the wrapped error. Could you mention this in .nextchanges/air/air-cancel-verify.md in case anyone scripts against the JSON error text?

if err != nil {
return err
}
if !isAirRun(run) {
return fmt.Errorf("run %d is not an AI Runtime run; use \"databricks jobs cancel-run %d\" to cancel other Jobs runs", runID, runID)
}
}
_, err = w.Jobs.CancelRun(ctx, jobs.CancelRun{RunId: runID})
return err
}
Expand Down
55 changes: 52 additions & 3 deletions cmd/air/cancel_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -88,14 +88,22 @@ func TestCancelArgs(t *testing.T) {
func TestCancelRunInvalidID(t *testing.T) {
m := mocks.NewMockWorkspaceClient(t)
for _, id := range []string{"abc", "0", "-1"} {
err := cancelRun(t.Context(), m.WorkspaceClient, id)
err := cancelRun(t.Context(), m.WorkspaceClient, id, false)
require.Error(t, err)
assert.Contains(t, err.Error(), "invalid run ID")
}
}

// expectAirRun stubs runs/get to return an AIR run for id.
func expectAirRun(m *mocks.MockWorkspaceClient, id int64) {
run := airRun(id, "me@example.com", "GPU_1xA10", 1, "/Users/me@example.com/exp")
m.GetMockJobsAPI().EXPECT().GetRun(mock.Anything, jobs.GetRunRequest{RunId: id}).Return(&run, nil)
}

func TestCancelByIDSuccess(t *testing.T) {
m := mocks.NewMockWorkspaceClient(t)
expectAirRun(m, 123)
expectAirRun(m, 456)
m.GetMockJobsAPI().EXPECT().CancelRun(mock.Anything, jobs.CancelRun{RunId: 123}).Return(nil, nil)
m.GetMockJobsAPI().EXPECT().CancelRun(mock.Anything, jobs.CancelRun{RunId: 456}).Return(nil, nil)

Expand All @@ -112,7 +120,7 @@ func TestCancelByIDSuccess(t *testing.T) {

func TestCancelByIDNotFound(t *testing.T) {
m := mocks.NewMockWorkspaceClient(t)
m.GetMockJobsAPI().EXPECT().CancelRun(mock.Anything, jobs.CancelRun{RunId: 5}).Return(nil, apierr.ErrResourceDoesNotExist)
m.GetMockJobsAPI().EXPECT().GetRun(mock.Anything, jobs.GetRunRequest{RunId: 5}).Return(nil, apierr.ErrResourceDoesNotExist)

var buf bytes.Buffer
_, err := runCancel(t, m.WorkspaceClient, flags.OutputText, "", &buf, "5")
Expand All @@ -127,6 +135,7 @@ func TestCancelByIDNotFoundInvalidParam(t *testing.T) {
// The cancel endpoint reports an unknown run as 400 INVALID_PARAMETER_VALUE,
// which the SDK does not remap to ErrResourceDoesNotExist for this path.
m := mocks.NewMockWorkspaceClient(t)
expectAirRun(m, 5)
apiErr := &apierr.APIError{StatusCode: http.StatusBadRequest, ErrorCode: "INVALID_PARAMETER_VALUE", Message: "Run 5 does not exist."}
m.GetMockJobsAPI().EXPECT().CancelRun(mock.Anything, jobs.CancelRun{RunId: 5}).Return(nil, apiErr)

Expand All @@ -138,8 +147,9 @@ func TestCancelByIDNotFoundInvalidParam(t *testing.T) {

func TestCancelPartialFailureJSON(t *testing.T) {
m := mocks.NewMockWorkspaceClient(t)
expectAirRun(m, 123)
m.GetMockJobsAPI().EXPECT().CancelRun(mock.Anything, jobs.CancelRun{RunId: 123}).Return(nil, nil)
m.GetMockJobsAPI().EXPECT().CancelRun(mock.Anything, jobs.CancelRun{RunId: 5}).Return(nil, apierr.ErrResourceDoesNotExist)
m.GetMockJobsAPI().EXPECT().GetRun(mock.Anything, jobs.GetRunRequest{RunId: 5}).Return(nil, apierr.ErrResourceDoesNotExist)

var buf bytes.Buffer
_, err := runCancel(t, m.WorkspaceClient, flags.OutputJSON, "", &buf, "123", "5")
Expand All @@ -156,6 +166,7 @@ func TestCancelPartialFailureJSON(t *testing.T) {

func TestCancelByIDSuccessJSON(t *testing.T) {
m := mocks.NewMockWorkspaceClient(t)
expectAirRun(m, 123)
m.GetMockJobsAPI().EXPECT().CancelRun(mock.Anything, jobs.CancelRun{RunId: 123}).Return(nil, nil)

var buf bytes.Buffer
Expand All @@ -170,6 +181,7 @@ func TestCancelByIDSuccessJSON(t *testing.T) {

func TestCancelByIDGenericFailure(t *testing.T) {
m := mocks.NewMockWorkspaceClient(t)
expectAirRun(m, 7)
m.GetMockJobsAPI().EXPECT().CancelRun(mock.Anything, jobs.CancelRun{RunId: 7}).Return(nil, errors.New("boom"))

var buf bytes.Buffer
Expand All @@ -178,6 +190,43 @@ func TestCancelByIDGenericFailure(t *testing.T) {
assert.Contains(t, buf.String(), "Failed to cancel run 7: boom")
}

func TestCancelByIDRejectsNonAirRun(t *testing.T) {
// A non-AIR run (e.g. a notebook job) is rejected without calling CancelRun;
// the mock fails the test on any unexpected CancelRun call. Other IDs in
// the same invocation are still cancelled.
m := mocks.NewMockWorkspaceClient(t)
notebookRun := &jobs.Run{RunId: 9, Tasks: []jobs.RunTask{{NotebookTask: &jobs.NotebookTask{NotebookPath: "/x"}}}}
m.GetMockJobsAPI().EXPECT().GetRun(mock.Anything, jobs.GetRunRequest{RunId: 9}).Return(notebookRun, nil)
expectAirRun(m, 123)
m.GetMockJobsAPI().EXPECT().CancelRun(mock.Anything, jobs.CancelRun{RunId: 123}).Return(nil, nil)

var buf bytes.Buffer
_, err := runCancel(t, m.WorkspaceClient, flags.OutputText, "", &buf, "9", "123")
require.ErrorIs(t, err, root.ErrAlreadyPrinted)

out := buf.String()
assert.Contains(t, out, "Failed to cancel run 9: run 9 is not an AI Runtime run")
assert.Contains(t, out, "databricks jobs cancel-run 9")
assert.Contains(t, out, "Successfully requested cancellation for run 123")
assert.Contains(t, out, "1 run(s) failed to cancel.")
}

func TestCancelByIDRejectsNonAirRunJSON(t *testing.T) {
m := mocks.NewMockWorkspaceClient(t)
m.GetMockJobsAPI().EXPECT().GetRun(mock.Anything, jobs.GetRunRequest{RunId: 9}).Return(&jobs.Run{RunId: 9, Tasks: []jobs.RunTask{{}}}, nil)

var buf bytes.Buffer
_, err := runCancel(t, m.WorkspaceClient, flags.OutputJSON, "", &buf, "9")
require.ErrorIs(t, err, root.ErrAlreadyPrinted)

var got cancelEnvelope
require.NoError(t, json.Unmarshal(buf.Bytes(), &got))
assert.Empty(t, got.Data.Cancelled)
require.Len(t, got.Data.Failed, 1)
assert.Equal(t, "9", got.Data.Failed[0].RunID)
assert.Contains(t, got.Data.Failed[0].Error, "not an AI Runtime run")
}

func TestCancelAllNoActiveRuns(t *testing.T) {
w := newTestWorkspaceClient(t, runsServer(t, runsListBody(t, "")).URL)
var buf bytes.Buffer
Expand Down
Loading