diff --git a/.nextchanges/air/air-cancel-verify.md b/.nextchanges/air/air-cancel-verify.md new file mode 100644 index 00000000000..36de6759843 --- /dev/null +++ b/.nextchanges/air/air-cancel-verify.md @@ -0,0 +1 @@ +* `air cancel JOB_RUN_ID` now rejects run IDs that are not AI Runtime runs instead of cancelling unrelated Jobs runs. In `-o json` output, the `failed[].error` text for a run that does not exist or that you cannot view now starts with `failed to get run :` instead of the cancel API error. ([#7569](https://github.com/databricks/cli/pull/7569)) diff --git a/acceptance/air/cancel/test.toml b/acceptance/air/cancel/test.toml index 14a54ccb754..ac19a88d6ad 100644 --- a/acceptance/air/cancel/test.toml +++ b/acceptance/air/cancel/test.toml @@ -4,6 +4,18 @@ Pattern = "HEAD /" Response.Body = '' +# Cancel by ID verifies each run is an AIR run via runs/get before cancelling. +# Stubs match on path only, so every ID resolves to this AIR run. +[[Server]] +Pattern = "GET /api/2.2/jobs/runs/get" +Response.Body = ''' +{ + "run_id": 123, + "state": {"life_cycle_state": "RUNNING"}, + "tasks": [{"run_id": 124, "ai_runtime_task": {"experiment": "/Users/tester@databricks.com/exp"}}] +} +''' + # CancelRun accepts the request and returns an empty body. [[Server]] Pattern = "POST /api/2.2/jobs/runs/cancel" diff --git a/cmd/air/cancel.go b/cmd/air/cancel.go index fe7e2cbdbff..0ca93c793cd 100644 --- a/cmd/air/cancel.go +++ b/cmd/air/cancel.go @@ -136,7 +136,8 @@ all active runs submitted by the current user. } for _, rid := range runIDs { - err := cancelRun(ctx, w, rid) + // --all only lists AIR runs, so they don't need re-checking. + err := cancelRun(ctx, w, rid, all) if err != nil { data.Failed = append(data.Failed, cancelFailure{RunID: rid, Error: err.Error()}) if !jsonOut { @@ -192,13 +193,24 @@ func runNotFound(err error) bool { return false } -// cancelRun requests cancellation of a single job run. The cancel is async, so -// the returned waiter is ignored. -func cancelRun(ctx context.Context, w *databricks.WorkspaceClient, rid string) error { +// cancelRun requests cancellation of a single job run. Unless skipAirCheck is +// set, it first verifies the run is an AI Runtime run, so a mistyped but valid +// ID can't cancel an unrelated Jobs run. The cancel is async, so the returned +// waiter is ignored. +func cancelRun(ctx context.Context, w *databricks.WorkspaceClient, rid string, skipAirCheck bool) error { runID, err := strconv.ParseInt(rid, 10, 64) if err != nil || runID <= 0 { return fmt.Errorf("invalid run ID %q: must be a positive integer", rid) } + if !skipAirCheck { + run, err := fetchJobRun(ctx, w, runID) + if err != nil { + return err + } + if !isAirRun(run) { + return fmt.Errorf("run %d is not an AI Runtime run; use \"databricks jobs cancel-run %d\" to cancel other Jobs runs", runID, runID) + } + } _, err = w.Jobs.CancelRun(ctx, jobs.CancelRun{RunId: runID}) return err } diff --git a/cmd/air/cancel_test.go b/cmd/air/cancel_test.go index 5601a99f84a..6f9166d1db3 100644 --- a/cmd/air/cancel_test.go +++ b/cmd/air/cancel_test.go @@ -88,14 +88,22 @@ func TestCancelArgs(t *testing.T) { func TestCancelRunInvalidID(t *testing.T) { m := mocks.NewMockWorkspaceClient(t) for _, id := range []string{"abc", "0", "-1"} { - err := cancelRun(t.Context(), m.WorkspaceClient, id) + err := cancelRun(t.Context(), m.WorkspaceClient, id, false) require.Error(t, err) assert.Contains(t, err.Error(), "invalid run ID") } } +// expectAirRun stubs runs/get to return an AIR run for id. +func expectAirRun(m *mocks.MockWorkspaceClient, id int64) { + run := airRun(id, "me@example.com", "GPU_1xA10", 1, "/Users/me@example.com/exp") + m.GetMockJobsAPI().EXPECT().GetRun(mock.Anything, jobs.GetRunRequest{RunId: id}).Return(&run, nil) +} + func TestCancelByIDSuccess(t *testing.T) { m := mocks.NewMockWorkspaceClient(t) + expectAirRun(m, 123) + expectAirRun(m, 456) m.GetMockJobsAPI().EXPECT().CancelRun(mock.Anything, jobs.CancelRun{RunId: 123}).Return(nil, nil) m.GetMockJobsAPI().EXPECT().CancelRun(mock.Anything, jobs.CancelRun{RunId: 456}).Return(nil, nil) @@ -112,7 +120,7 @@ func TestCancelByIDSuccess(t *testing.T) { func TestCancelByIDNotFound(t *testing.T) { m := mocks.NewMockWorkspaceClient(t) - m.GetMockJobsAPI().EXPECT().CancelRun(mock.Anything, jobs.CancelRun{RunId: 5}).Return(nil, apierr.ErrResourceDoesNotExist) + m.GetMockJobsAPI().EXPECT().GetRun(mock.Anything, jobs.GetRunRequest{RunId: 5}).Return(nil, apierr.ErrResourceDoesNotExist) var buf bytes.Buffer _, err := runCancel(t, m.WorkspaceClient, flags.OutputText, "", &buf, "5") @@ -127,6 +135,7 @@ func TestCancelByIDNotFoundInvalidParam(t *testing.T) { // The cancel endpoint reports an unknown run as 400 INVALID_PARAMETER_VALUE, // which the SDK does not remap to ErrResourceDoesNotExist for this path. m := mocks.NewMockWorkspaceClient(t) + expectAirRun(m, 5) apiErr := &apierr.APIError{StatusCode: http.StatusBadRequest, ErrorCode: "INVALID_PARAMETER_VALUE", Message: "Run 5 does not exist."} m.GetMockJobsAPI().EXPECT().CancelRun(mock.Anything, jobs.CancelRun{RunId: 5}).Return(nil, apiErr) @@ -138,8 +147,9 @@ func TestCancelByIDNotFoundInvalidParam(t *testing.T) { func TestCancelPartialFailureJSON(t *testing.T) { m := mocks.NewMockWorkspaceClient(t) + expectAirRun(m, 123) m.GetMockJobsAPI().EXPECT().CancelRun(mock.Anything, jobs.CancelRun{RunId: 123}).Return(nil, nil) - m.GetMockJobsAPI().EXPECT().CancelRun(mock.Anything, jobs.CancelRun{RunId: 5}).Return(nil, apierr.ErrResourceDoesNotExist) + m.GetMockJobsAPI().EXPECT().GetRun(mock.Anything, jobs.GetRunRequest{RunId: 5}).Return(nil, apierr.ErrResourceDoesNotExist) var buf bytes.Buffer _, err := runCancel(t, m.WorkspaceClient, flags.OutputJSON, "", &buf, "123", "5") @@ -156,6 +166,7 @@ func TestCancelPartialFailureJSON(t *testing.T) { func TestCancelByIDSuccessJSON(t *testing.T) { m := mocks.NewMockWorkspaceClient(t) + expectAirRun(m, 123) m.GetMockJobsAPI().EXPECT().CancelRun(mock.Anything, jobs.CancelRun{RunId: 123}).Return(nil, nil) var buf bytes.Buffer @@ -170,6 +181,7 @@ func TestCancelByIDSuccessJSON(t *testing.T) { func TestCancelByIDGenericFailure(t *testing.T) { m := mocks.NewMockWorkspaceClient(t) + expectAirRun(m, 7) m.GetMockJobsAPI().EXPECT().CancelRun(mock.Anything, jobs.CancelRun{RunId: 7}).Return(nil, errors.New("boom")) var buf bytes.Buffer @@ -178,6 +190,43 @@ func TestCancelByIDGenericFailure(t *testing.T) { assert.Contains(t, buf.String(), "Failed to cancel run 7: boom") } +func TestCancelByIDRejectsNonAirRun(t *testing.T) { + // A non-AIR run (e.g. a notebook job) is rejected without calling CancelRun; + // the mock fails the test on any unexpected CancelRun call. Other IDs in + // the same invocation are still cancelled. + m := mocks.NewMockWorkspaceClient(t) + notebookRun := &jobs.Run{RunId: 9, Tasks: []jobs.RunTask{{NotebookTask: &jobs.NotebookTask{NotebookPath: "/x"}}}} + m.GetMockJobsAPI().EXPECT().GetRun(mock.Anything, jobs.GetRunRequest{RunId: 9}).Return(notebookRun, nil) + expectAirRun(m, 123) + m.GetMockJobsAPI().EXPECT().CancelRun(mock.Anything, jobs.CancelRun{RunId: 123}).Return(nil, nil) + + var buf bytes.Buffer + _, err := runCancel(t, m.WorkspaceClient, flags.OutputText, "", &buf, "9", "123") + require.ErrorIs(t, err, root.ErrAlreadyPrinted) + + out := buf.String() + assert.Contains(t, out, "Failed to cancel run 9: run 9 is not an AI Runtime run") + assert.Contains(t, out, "databricks jobs cancel-run 9") + assert.Contains(t, out, "Successfully requested cancellation for run 123") + assert.Contains(t, out, "1 run(s) failed to cancel.") +} + +func TestCancelByIDRejectsNonAirRunJSON(t *testing.T) { + m := mocks.NewMockWorkspaceClient(t) + m.GetMockJobsAPI().EXPECT().GetRun(mock.Anything, jobs.GetRunRequest{RunId: 9}).Return(&jobs.Run{RunId: 9, Tasks: []jobs.RunTask{{}}}, nil) + + var buf bytes.Buffer + _, err := runCancel(t, m.WorkspaceClient, flags.OutputJSON, "", &buf, "9") + require.ErrorIs(t, err, root.ErrAlreadyPrinted) + + var got cancelEnvelope + require.NoError(t, json.Unmarshal(buf.Bytes(), &got)) + assert.Empty(t, got.Data.Cancelled) + require.Len(t, got.Data.Failed, 1) + assert.Equal(t, "9", got.Data.Failed[0].RunID) + assert.Contains(t, got.Data.Failed[0].Error, "not an AI Runtime run") +} + func TestCancelAllNoActiveRuns(t *testing.T) { w := newTestWorkspaceClient(t, runsServer(t, runsListBody(t, "")).URL) var buf bytes.Buffer