Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/bundles/7557.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* Retry transient server errors when creating Lakebase (`postgres_*`) resources in the direct deployment engine. ([#7557](https://github.com/databricks/cli/pull/7557))
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
bundle:
name: test-bundle

sync:
paths: []

resources:
postgres_projects:
my_project:
project_id: test-pg-proj
display_name: "Test Postgres Project"
pg_version: 16
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Cloud = false
EnvMatrix.DMS = ["", "true"]
EnvMatrix.READPLAN = []
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
Warn: deploying resources.postgres_projects.my_project: retrying after 500 Internal Server Error from POST /api/2.0/postgres/projects
Warn: deploying resources.postgres_projects.my_project: retrying after 500 Internal Server Error from GET /api/2.0/postgres/projects/test-pg-proj/operations/[UUID]
Created postgres_projects.my_project
Files: 0 uploaded, 0 deleted
Resources: 1 created, 0 changed, 0 deleted, 0 unchanged

>>> print_requests.py --get //postgres
POST /api/2.0/postgres/projects
POST /api/2.0/postgres/projects
GET /api/2.0/postgres/projects/test-pg-proj/operations/[UUID]
GET /api/2.0/postgres/projects/test-pg-proj/operations/[UUID]

>>> [CLI] bundle destroy --auto-approve
The following resources will be deleted:
delete resources.postgres_projects.my_project

This action will result in the deletion of the following Lakebase projects along with
all their branches, databases, and endpoints. All data stored in them will be permanently lost:
delete resources.postgres_projects.my_project

All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default

Destroy: 1 deleted
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
# One transient 500 on the create request and one on polling the create operation; both are
# retried and the deploy succeeds.
fault.py "POST /api/2.0/postgres/projects" 500 0 1 INTERNAL_ERROR
fault.py "GET /api/2.0/postgres/projects/*/operations/*" 500 0 1 INTERNAL_ERROR

trace $CLI bundle deploy

# Two create requests and two operation polls: the 500 and the retry of each.
trace print_requests.py --get //postgres | jq -r '.method + " " + .path'

trace $CLI bundle destroy --auto-approve
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
Cloud = false
RecordRequests = true
EnvMatrix.READPLAN = []
Env.DATABRICKS_BUNDLE_ENGINE = "direct"
Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "100"
SuppressLines.warn_retry = ""
2 changes: 1 addition & 1 deletion bundle/direct/dresources/postgres_branch.go
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,7 @@ func (r *ResourcePostgresBranch) DoCreate(ctx context.Context, config *PostgresB
}

// Wait for the branch to be ready (long-running operation)
result, err := waiter.Wait(ctx)
result, err := waitCreateOperation(ctx, waiter.Wait)
if err != nil {
return "", nil, err
}
Expand Down
2 changes: 1 addition & 1 deletion bundle/direct/dresources/postgres_catalog.go
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ func (r *ResourcePostgresCatalog) DoCreate(ctx context.Context, config *Postgres
return "", nil, err
}

result, err := waiter.Wait(ctx)
result, err := waitCreateOperation(ctx, waiter.Wait)
if err != nil {
return "", nil, err
}
Expand Down
2 changes: 1 addition & 1 deletion bundle/direct/dresources/postgres_data_api.go
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,7 @@ func (r *ResourcePostgresDataApi) DoCreate(ctx context.Context, config *Postgres
return "", nil, err
}

result, err := waiter.Wait(ctx)
result, err := waitCreateOperation(ctx, waiter.Wait)
if err != nil {
return "", nil, err
}
Expand Down
2 changes: 1 addition & 1 deletion bundle/direct/dresources/postgres_database.go
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ func (r *ResourcePostgresDatabase) DoCreate(ctx context.Context, config *Postgre
}

// Wait for the database to be ready (long-running operation)
result, err := waiter.Wait(ctx)
result, err := waitCreateOperation(ctx, waiter.Wait)
if err != nil {
return "", nil, err
}
Expand Down
2 changes: 1 addition & 1 deletion bundle/direct/dresources/postgres_endpoint.go
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@ func (r *ResourcePostgresEndpoint) DoCreate(ctx context.Context, config *Postgre
}

// Wait for the operation to complete
result, err := waiter.Wait(ctx)
result, err := waitCreateOperation(ctx, waiter.Wait)
if err != nil {
return "", nil, err
}
Expand Down
7 changes: 5 additions & 2 deletions bundle/direct/dresources/postgres_project.go
Original file line number Diff line number Diff line change
Expand Up @@ -121,11 +121,14 @@ func (r *ResourcePostgresProject) DoCreate(ctx context.Context, config *Postgres
},
})
if err != nil {
return "", nil, err
// The project is keyed by the caller-chosen project_id, so repeating the request cannot
// create a duplicate: if the failed request did create the project, the retry fails with
// ALREADY_EXISTS, just as the next deploy would have.
return "", nil, retrySafe(err)
}

// Wait for the project to be ready (long-running operation)
result, err := waiter.Wait(ctx)
result, err := waitCreateOperation(ctx, waiter.Wait)
if err != nil {
return "", nil, err
}
Expand Down
2 changes: 1 addition & 1 deletion bundle/direct/dresources/postgres_role.go
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ func (r *ResourcePostgresRole) DoCreate(ctx context.Context, config *PostgresRol
return "", nil, err
}

result, err := waiter.Wait(ctx)
result, err := waitCreateOperation(ctx, waiter.Wait)
if err != nil {
return "", nil, err
}
Expand Down
2 changes: 1 addition & 1 deletion bundle/direct/dresources/postgres_synced_table.go
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ func (r *ResourcePostgresSyncedTable) DoCreate(ctx context.Context, config *Post
return "", nil, err
}

result, err := waiter.Wait(ctx)
result, err := waitCreateOperation(ctx, waiter.Wait)
if err != nil {
return "", nil, err
}
Expand Down
18 changes: 18 additions & 0 deletions bundle/direct/dresources/postgres_wait.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
package dresources

import (
"context"

"github.com/databricks/cli/libs/retry"
"github.com/databricks/databricks-sdk-go/experimental/api"
)

// waitCreateOperation polls a postgres create operation, polling again when a poll fails with a
// transient error (observed: 500 INTERNAL_ERROR and inactivity timeouts on GET .../operations/...).
// The create request was already accepted, so polling again cannot create a second resource,
// whereas a failure returned from DoCreate is not retried at all.
func waitCreateOperation[T any](ctx context.Context, wait func(context.Context, ...api.Option) (T, error)) (T, error) {
return retry.With(ctx, retry.IsTransientRead, func() (T, error) {
return wait(ctx)
})
}
Loading