diff --git a/.nextchanges/bundles/7602.md b/.nextchanges/bundles/7602.md new file mode 100644 index 00000000000..9fe798f6fe3 --- /dev/null +++ b/.nextchanges/bundles/7602.md @@ -0,0 +1 @@ +* Use the Lakebase branch when its create request times out but the branch was created, instead of failing the deploy, in the direct deployment engine. ([#7602](https://github.com/databricks/cli/pull/7602)) diff --git a/acceptance/bundle/resources/postgres_branches/create_504/databricks.yml b/acceptance/bundle/resources/postgres_branches/create_504/databricks.yml new file mode 100644 index 00000000000..3a5d296fbfe --- /dev/null +++ b/acceptance/bundle/resources/postgres_branches/create_504/databricks.yml @@ -0,0 +1,18 @@ +bundle: + name: test-bundle + +sync: + paths: [] + +resources: + postgres_projects: + my_project: + project_id: test-pg-proj + display_name: "Test Postgres Project" + pg_version: 16 + + postgres_branches: + dev: + parent: ${resources.postgres_projects.my_project.id} + branch_id: dev + no_expiry: true diff --git a/acceptance/bundle/resources/postgres_branches/create_504/out.test.toml b/acceptance/bundle/resources/postgres_branches/create_504/out.test.toml new file mode 100644 index 00000000000..15c32f324ac --- /dev/null +++ b/acceptance/bundle/resources/postgres_branches/create_504/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] +EnvMatrix.READPLAN = [] diff --git a/acceptance/bundle/resources/postgres_branches/create_504/output.txt b/acceptance/bundle/resources/postgres_branches/create_504/output.txt new file mode 100644 index 00000000000..9efc60f88f6 --- /dev/null +++ b/acceptance/bundle/resources/postgres_branches/create_504/output.txt @@ -0,0 +1,30 @@ + +=== The create request times out but the branch was created: deploy uses the branch + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created postgres_projects.my_project +Warn: deploying resources.postgres_branches.dev: the branch was created despite the failed create request: Fault injected by test. +Created postgres_branches.dev +Files: 0 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --get //postgres/projects/test-pg-proj/branches +POST /api/2.0/postgres/projects/test-pg-proj/branches +GET /api/2.0/postgres/projects/test-pg-proj/branches/dev + +=== The create request times out and the branch was not created: deploy repeats the create + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created postgres_projects.my_project +Warn: deploying resources.postgres_branches.dev: retrying after 504 Gateway Timeout from POST /api/2.0/postgres/projects/test-pg-proj/branches +Created postgres_branches.dev +Files: 0 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --get //postgres/projects/test-pg-proj/branches +POST /api/2.0/postgres/projects/test-pg-proj/branches +GET /api/2.0/postgres/projects/test-pg-proj/branches/dev +POST /api/2.0/postgres/projects/test-pg-proj/branches +GET /api/2.0/postgres/projects/test-pg-proj/branches/dev/operations/[UUID] diff --git a/acceptance/bundle/resources/postgres_branches/create_504/script b/acceptance/bundle/resources/postgres_branches/create_504/script new file mode 100644 index 00000000000..1ea194b4dc4 --- /dev/null +++ b/acceptance/bundle/resources/postgres_branches/create_504/script @@ -0,0 +1,12 @@ +title "The create request times out but the branch was created: deploy uses the branch" +fault.py --after-handler "POST /api/2.0/postgres/projects/*/branches" 504 0 1 DEADLINE_EXCEEDED +trace $CLI bundle deploy +trace print_requests.py --get //postgres/projects/test-pg-proj/branches | jq -r '.method + " " + .path' +$CLI bundle destroy --auto-approve > /dev/null 2>&1 +rm -f "$OUT_REQUESTS" + +title "The create request times out and the branch was not created: deploy repeats the create" +fault.py "POST /api/2.0/postgres/projects/*/branches" 504 0 1 DEADLINE_EXCEEDED +trace $CLI bundle deploy +trace print_requests.py --get //postgres/projects/test-pg-proj/branches | jq -r '.method + " " + .path' +$CLI bundle destroy --auto-approve > /dev/null 2>&1 diff --git a/acceptance/bundle/resources/postgres_branches/create_504/test.toml b/acceptance/bundle/resources/postgres_branches/create_504/test.toml new file mode 100644 index 00000000000..ab3f407bfa7 --- /dev/null +++ b/acceptance/bundle/resources/postgres_branches/create_504/test.toml @@ -0,0 +1,6 @@ +Cloud = false +RecordRequests = true +EnvMatrix.READPLAN = [] +Env.DATABRICKS_BUNDLE_ENGINE = "direct" +Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "100" +SuppressLines.warn_retry = "" diff --git a/bundle/direct/dresources/postgres_branch.go b/bundle/direct/dresources/postgres_branch.go index e134d5d8876..782b92aa59d 100644 --- a/bundle/direct/dresources/postgres_branch.go +++ b/bundle/direct/dresources/postgres_branch.go @@ -2,10 +2,15 @@ package dresources import ( "context" + "fmt" "slices" + "time" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/retry" "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/common/types/fieldmask" sdktime "github.com/databricks/databricks-sdk-go/common/types/time" "github.com/databricks/databricks-sdk-go/marshal" @@ -125,7 +130,7 @@ func (r *ResourcePostgresBranch) DoCreate(ctx context.Context, config *PostgresB ForceSendFields: nil, }) if err != nil { - return "", nil, err + return r.recoverFailedCreate(ctx, config, err) } // Wait for the branch to be ready (long-running operation) @@ -138,6 +143,61 @@ func (r *ResourcePostgresBranch) DoCreate(ctx context.Context, config *PostgresB return remote.Name, remote, nil } +// branchReadyTimeout bounds the wait for a branch that recoverFailedCreate found in INIT. +const branchReadyTimeout = 10 * time.Minute + +// recoverFailedCreate handles a create request that failed with a transient error. The backend +// answers a slow create with 504 DEADLINE_EXCEEDED ("may still be completing"), so the branch may +// exist anyway: use it once it is ready instead of failing, or, if it does not exist, let the +// caller repeat the create. +func (r *ResourcePostgresBranch) recoverFailedCreate(ctx context.Context, config *PostgresBranchState, createErr error) (string, *PostgresBranchRemote, error) { + if !retry.IsTransient(ctx, createErr) { + return "", nil, createErr + } + // With replace_existing, a branch found now may predate the request, and repeating the + // request is safe. + if config.ReplaceExisting != nil && *config.ReplaceExisting { + return "", nil, retrySafe(createErr) + } + + name := config.Parent + "/branches/" + config.BranchId + deadline := time.Now().Add(branchReadyTimeout) + for { + branch, err := retry.With(ctx, retry.IsTransientRead, func() (*postgres.Branch, error) { + return r.client.Postgres.GetBranch(ctx, postgres.GetBranchRequest{Name: name}) + }) + if apierr.IsMissing(err) { + return "", nil, retrySafe(createErr) + } + if err != nil { + log.Warnf(ctx, "checking whether %s was created: %s", name, err) + return "", nil, createErr + } + var state postgres.BranchStatusState + if branch.Status != nil { + state = branch.Status.CurrentState + } + if state == postgres.BranchStatusStateReady { + log.Warnf(ctx, "the branch was created despite the failed create request: %s", createErr) + remote := makePostgresBranchRemote(branch) + return remote.Name, remote, nil + } + // Only a new branch still initializing is worth waiting for; any other state (e.g. DELETED, + // ARCHIVED) will not become READY. + if state != postgres.BranchStatusStateInit { + return "", nil, fmt.Errorf("%w; branch %s exists in state %q", createErr, name, state) + } + if time.Now().After(deadline) { + return "", nil, fmt.Errorf("%w; branch %s exists but did not become ready within %s", createErr, name, branchReadyTimeout) + } + select { + case <-ctx.Done(): + return "", nil, ctx.Err() + case <-time.After(5 * time.Second): + } + } +} + // expire_time, no_expiry and ttl are three sides of one oneof, and the API accepts them // in update_mask only under the group name: masking the field itself is answered with // "Unknown field path in update_mask". Probed against a real workspace on 2026-08-31.