Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/bundles/7602.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* Use the Lakebase branch when its create request times out but the branch was created, instead of failing the deploy, in the direct deployment engine. ([#7602](https://github.com/databricks/cli/pull/7602))
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
bundle:
name: test-bundle

sync:
paths: []

resources:
postgres_projects:
my_project:
project_id: test-pg-proj
display_name: "Test Postgres Project"
pg_version: 16

postgres_branches:
dev:
parent: ${resources.postgres_projects.my_project.id}
branch_id: dev
no_expiry: true
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Cloud = false
EnvMatrix.DMS = ["", "true"]
EnvMatrix.READPLAN = []
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@

=== The create request times out but the branch was created: deploy uses the branch

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
Created postgres_projects.my_project
Warn: deploying resources.postgres_branches.dev: the branch was created despite the failed create request: Fault injected by test.
Created postgres_branches.dev
Files: 0 uploaded, 0 deleted
Resources: 2 created, 0 changed, 0 deleted, 0 unchanged

>>> print_requests.py --get //postgres/projects/test-pg-proj/branches
POST /api/2.0/postgres/projects/test-pg-proj/branches
GET /api/2.0/postgres/projects/test-pg-proj/branches/dev

=== The create request times out and the branch was not created: deploy repeats the create

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
Created postgres_projects.my_project
Warn: deploying resources.postgres_branches.dev: retrying after 504 Gateway Timeout from POST /api/2.0/postgres/projects/test-pg-proj/branches
Created postgres_branches.dev
Files: 0 uploaded, 0 deleted
Resources: 2 created, 0 changed, 0 deleted, 0 unchanged

>>> print_requests.py --get //postgres/projects/test-pg-proj/branches
POST /api/2.0/postgres/projects/test-pg-proj/branches
GET /api/2.0/postgres/projects/test-pg-proj/branches/dev
POST /api/2.0/postgres/projects/test-pg-proj/branches
GET /api/2.0/postgres/projects/test-pg-proj/branches/dev/operations/[UUID]
12 changes: 12 additions & 0 deletions acceptance/bundle/resources/postgres_branches/create_504/script
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
title "The create request times out but the branch was created: deploy uses the branch"
fault.py --after-handler "POST /api/2.0/postgres/projects/*/branches" 504 0 1 DEADLINE_EXCEEDED
trace $CLI bundle deploy
trace print_requests.py --get //postgres/projects/test-pg-proj/branches | jq -r '.method + " " + .path'
$CLI bundle destroy --auto-approve > /dev/null 2>&1
rm -f "$OUT_REQUESTS"

title "The create request times out and the branch was not created: deploy repeats the create"
fault.py "POST /api/2.0/postgres/projects/*/branches" 504 0 1 DEADLINE_EXCEEDED
trace $CLI bundle deploy
trace print_requests.py --get //postgres/projects/test-pg-proj/branches | jq -r '.method + " " + .path'
$CLI bundle destroy --auto-approve > /dev/null 2>&1
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
Cloud = false
RecordRequests = true
EnvMatrix.READPLAN = []
Env.DATABRICKS_BUNDLE_ENGINE = "direct"
Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "100"
SuppressLines.warn_retry = ""
62 changes: 61 additions & 1 deletion bundle/direct/dresources/postgres_branch.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,15 @@ package dresources

import (
"context"
"fmt"
"slices"
"time"

"github.com/databricks/cli/bundle/config/resources"
"github.com/databricks/cli/libs/log"
"github.com/databricks/cli/libs/retry"
"github.com/databricks/databricks-sdk-go"
"github.com/databricks/databricks-sdk-go/apierr"
"github.com/databricks/databricks-sdk-go/common/types/fieldmask"
sdktime "github.com/databricks/databricks-sdk-go/common/types/time"
"github.com/databricks/databricks-sdk-go/marshal"
Expand Down Expand Up @@ -125,7 +130,7 @@ func (r *ResourcePostgresBranch) DoCreate(ctx context.Context, config *PostgresB
ForceSendFields: nil,
})
if err != nil {
return "", nil, err
return r.recoverFailedCreate(ctx, config, err)
}

// Wait for the branch to be ready (long-running operation)
Expand All @@ -138,6 +143,61 @@ func (r *ResourcePostgresBranch) DoCreate(ctx context.Context, config *PostgresB
return remote.Name, remote, nil
}

// branchReadyTimeout bounds the wait for a branch that recoverFailedCreate found in INIT.
const branchReadyTimeout = 10 * time.Minute

// recoverFailedCreate handles a create request that failed with a transient error. The backend
// answers a slow create with 504 DEADLINE_EXCEEDED ("may still be completing"), so the branch may
// exist anyway: use it once it is ready instead of failing, or, if it does not exist, let the
// caller repeat the create.
func (r *ResourcePostgresBranch) recoverFailedCreate(ctx context.Context, config *PostgresBranchState, createErr error) (string, *PostgresBranchRemote, error) {
if !retry.IsTransient(ctx, createErr) {
return "", nil, createErr
}
// With replace_existing, a branch found now may predate the request, and repeating the
// request is safe.
if config.ReplaceExisting != nil && *config.ReplaceExisting {
return "", nil, retrySafe(createErr)
}

name := config.Parent + "/branches/" + config.BranchId
deadline := time.Now().Add(branchReadyTimeout)
for {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

major: Most of recoverFailedCreate is untested: INIT wait, non-INIT state, timeout, read failure, replace_existing

create_504 exercises two outcomes only: the branch is found READY on the first read, and the branch is missing so the create is repeated. These paths of recoverFailedCreate have no coverage:

  • the INIT poll loop (about lines 187-197), which the PR description presents as a main new behavior and which a "may still be completing" 504 is likely to hit;
  • the non-INIT/non-READY state error (line 188) and the 10-minute timeout (line 191);
  • a read that fails with something other than not-found (lines 172-175);
  • replace_existing: true (lines 159-161), which repeats the create without reading and is the documented way to manage the implicit production branch;
  • a non-transient create error (no GET issued).

The PR says the INIT path can't be tested because the testserver always creates branches as READY (libs/testserver/postgres.go:347-349). That rules out an acceptance test for INIT, but not a unit test. Other resources in this package test DoCreate against mocks.NewMockWorkspaceClient(t) (genie_space_test.go:60, feature_spec_test.go:44; the mock exposes GetMockPostgresAPI()), and job_run_test.go:39-66 drives a wait with testserver.New(t) and a custom server.Handle. There is no postgres_branch_test.go. A table-driven test could cover: GET returns INIT then READY; GET returns an unexpected state (the error carries the original create error); GET fails with a non-missing error (the original error is returned); replace_existing: true (no GET, RetrySafeError); a non-transient create error (no GET).

Two literals block such a test: the poll interval time.After(5 * time.Second) (:196) ignores DATABRICKS_BUNDLE_RETRY_INTERVAL_MS, and the deadline is the const branchReadyTimeout (:147). An INIT test would sleep 5s and the timeout path can't be reached. Make them fields or package vars a test can shrink.

Two of the gaps can also be closed with acceptance scenarios and no testserver changes, using the existing fault harness (acceptance/bin/fault.py supports --after-handler and arbitrary method/path patterns):

  • replace_existing: true with fault.py --after-handler "POST /api/2.0/postgres/projects/*/branches" 504 0 1 DEADLINE_EXCEEDED should log POST, POST, operation GET, and no branch GET.
  • A failed read: a GET fault such as fault.py "GET /api/2.0/postgres/projects/*/branches/dev" 500 0 3 after a 504 on the POST should show the checking whether ... was created warning and the original 504 error, with no repeated POST.

These tests don't duplicate create_504: that covers the READY and missing paths, and these cover paths it can't reach.

Agreed by 3 reviewer model(s).

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

minor: Up to 10 minutes of polling inside DoCreate, before state is saved and outside the max-wait cap

bundle/direct/dresources/README.md:44 says to implement WaitAfterCreate instead of polling inline inside DoCreate, because inline polling bypasses state persistence timing (apply.go saves state before WaitAfterCreate runs). The new loop polls for up to branchReadyTimeout (10 minutes) inside DoCreate.

Concrete effects:

  • At the timeout (line 191) and on a cancelled context (line 195) the code already knows the branch exists under name, yet returns an error, so nothing is written to state. If the 10-minute loop times out or the user presses Ctrl-C, the next deploy fails with ALREADY_EXISTS.
  • DATABRICKS_BUNDLE_RESOURCE_MAX_WAIT does not bound this wait, because waitCapped wraps only WaitAfterCreate.

The existing success path also waits inline (waitCreateOperation at :137), so this is not new in kind, and moving it could be a follow-up. But here the ID is known as soon as the read succeeds: returning the branch once it is found and waiting for READY in a WaitAfterCreate would persist state first, respect the cap, and reuse the framework's transient-read retry instead of a second hand-written loop. If you keep it inline, a short comment on why would help the next reader.

Agreed by 3 reviewer model(s).

branch, err := retry.With(ctx, retry.IsTransientRead, func() (*postgres.Branch, error) {
return r.client.Postgres.GetBranch(ctx, postgres.GetBranchRequest{Name: name})
})
if apierr.IsMissing(err) {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

minor: In the "may still be completing" case, the repeated create can end in ALREADY_EXISTS

The backend's 504 says the request "may still be completing", yet the branch is read right away (:166) with no pause. If the backend has not committed the branch yet, the read returns 404. The code then returns retrySafe(createErr), and DeploymentUnit.Create repeats the POST about 15s later. If the first request has landed by then, that POST returns 409 ALREADY_EXISTS, which is not transient, so the deploy fails. That is the failure this PR targets, now with a different message.

The PR lists this as a known gap. But its own argument (the first attempt got a 504, not a 409, so no branch existed beforehand) applies equally to a 409 that follows our own transient failure in the same Create. Options:

  • Re-check after a short pause before returning retrySafe.
  • Have recoverFailedCreate repeat the create itself, so it can treat ALREADY_EXISTS after its own 504 like the READY case.

Without one of these, the fix only helps when the branch already exists when the 504 arrives, which is the case the --after-handler test models.

Agreed by 3 reviewer model(s).

return "", nil, retrySafe(createErr)
}
if err != nil {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: A cancelled context during the recovery read is logged as a failed existence check and replaced by the create error

If ctx is cancelled while the read is inside retry.With, it returns ctx.Err(). That is not apierr.IsMissing, so it falls into the generic branch, logs checking whether ... was created: context canceled at warn level, and returns the original 504 instead of the cancellation. The deploy still stops, but the message is misleading. Check ctx.Err() first, or let that error pass through unwrapped as the select in the INIT loop does.

Agreed by 3 reviewer model(s).

log.Warnf(ctx, "checking whether %s was created: %s", name, err)
return "", nil, createErr
}
var state postgres.BranchStatusState
if branch.Status != nil {
state = branch.Status.CurrentState
}
if state == postgres.BranchStatusStateReady {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

minor: A pre-existing READY branch can be adopted after any transient error, not only a handler timeout

The PR argues that without replace_existing a pre-existing branch would have made the POST fail with ALREADY_EXISTS, so a branch found after the failure must have been created by this request. That holds only if the request reached the backend's existence check. The gate is retry.IsTransient, which is true for any 5xx or 408 the SDK did not retry and for the client-side inactivity timeout. That includes a 502/503 from an edge proxy that never forwarded the request and a backend 500 raised before the ID-uniqueness check.

In those cases a READY branch with the same ID that already existed (created by hand, by another bundle, left from an earlier deploy whose local state was lost, or the implicit production branch every project has) is adopted. The deploy prints Created, the configured spec was never sent, the warning wrongly says the branch "was created despite the failed create request", the branch is recorded in state with the bundle's config, and a later bundle destroy deletes a branch the bundle did not create. replace_existing is the explicit opt-in for taking over an existing branch, and this path bypasses it. Before this change the same sequence failed the deploy.

The window is narrow (a 5xx must coincide with an ID collision), but the outcome is a silent takeover. A cheap guard: record time.Now() before CreateBranch and adopt only when branch.CreateTime is not clearly earlier (allowing a margin for clock skew), otherwise return createErr unchanged. makePostgresBranchRemote (:87-102) shows CreateTime is available on the response. If you keep the current behavior, the warning could name the branch and say it may predate the request.

Agreed by 3 reviewer model(s).

log.Warnf(ctx, "the branch was created despite the failed create request: %s", createErr)

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

major: The new adoption warning is not suppressed, so the cloud tests the PR targets still fail when the branch is adopted

When recovery adopts the branch (found READY, or READY after the INIT wait), it logs Warn: deploying resources.postgres_branches.<key>: the branch was created despite the failed create request: <api message>. Acceptance output only drops warnings matching ^Warn: (.*: )?retrying( after |$), and this line does not match it.

The two tests the description names as affected (postgres_roles/replace_existing, postgres_databases/update) run trace $CLI bundle deploy and pin the result in output.txt. On a real 504 the deploy now succeeds, but the extra Warn: line makes the golden comparison fail, so those tests stay red on exactly the runs this PR targets. Only the missing-branch and replace_existing paths are safe, because their only warning is the suppressed retrying after ....

In postgres_databases/update, the recovery GET .../branches/main would also land in out.requests.create.json, since that recording uses --get '//postgres' and excludes only /operations/.

Suggested fix: either log the adoption below Warn, or add a second root SuppressLines pattern for it and opt create_504 out of it the same way it opts out of warn_retry.

Agreed by 3 reviewer model(s).

remote := makePostgresBranchRemote(branch)
return remote.Name, remote, nil
}
// Only a new branch still initializing is worth waiting for; any other state (e.g. DELETED,
// ARCHIVED) will not become READY.
if state != postgres.BranchStatusStateInit {
return "", nil, fmt.Errorf("%w; branch %s exists in state %q", createErr, name, state)
}
if time.Now().After(deadline) {
return "", nil, fmt.Errorf("%w; branch %s exists but did not become ready within %s", createErr, name, branchReadyTimeout)
}
select {
case <-ctx.Done():
return "", nil, ctx.Err()
case <-time.After(5 * time.Second):
}
}
}

// expire_time, no_expiry and ttl are three sides of one oneof, and the API accepts them
// in update_mask only under the group name: masking the field itself is answered with
// "Unknown field path in update_mask". Probed against a real workspace on 2026-08-31.
Expand Down
Loading