From 01ead29154b02baf66bf935edd02280dfbfccc86 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Thu, 12 Nov 2026 13:50:56 +0000 Subject: [PATCH 1/4] direct: retry transient errors when creating postgres resources Creating a postgres project sometimes fails with 500 INTERNAL_ERROR, either on the create request or on polling the create operation. DoCreate failures are not retried, unlike update and delete. - Poll the create operation again on transient errors for all postgres resources: the create request was accepted, so polling cannot duplicate it. - Mark a failed project create request as retry-safe: it is keyed by project_id, so a repeat cannot create a second project. cloud: TestAccept/bundle/resources/postgres_projects/basic TestAccept/bundle/resources/postgres_endpoints/basic Co-authored-by: Isaac --- .../create_transient_500/databricks.yml | 12 ++++++++++ .../create_transient_500/out.test.toml | 3 +++ .../create_transient_500/output.txt | 24 +++++++++++++++++++ .../create_transient_500/script | 11 +++++++++ .../create_transient_500/test.toml | 6 +++++ bundle/direct/dresources/postgres_branch.go | 2 +- bundle/direct/dresources/postgres_catalog.go | 2 +- bundle/direct/dresources/postgres_data_api.go | 2 +- bundle/direct/dresources/postgres_database.go | 2 +- bundle/direct/dresources/postgres_endpoint.go | 2 +- bundle/direct/dresources/postgres_project.go | 7 ++++-- bundle/direct/dresources/postgres_role.go | 2 +- .../dresources/postgres_synced_table.go | 2 +- bundle/direct/dresources/postgres_wait.go | 18 ++++++++++++++ 14 files changed, 86 insertions(+), 9 deletions(-) create mode 100644 acceptance/bundle/resources/postgres_projects/create_transient_500/databricks.yml create mode 100644 acceptance/bundle/resources/postgres_projects/create_transient_500/out.test.toml create mode 100644 acceptance/bundle/resources/postgres_projects/create_transient_500/output.txt create mode 100644 acceptance/bundle/resources/postgres_projects/create_transient_500/script create mode 100644 acceptance/bundle/resources/postgres_projects/create_transient_500/test.toml create mode 100644 bundle/direct/dresources/postgres_wait.go diff --git a/acceptance/bundle/resources/postgres_projects/create_transient_500/databricks.yml b/acceptance/bundle/resources/postgres_projects/create_transient_500/databricks.yml new file mode 100644 index 00000000000..b95ebe35d2c --- /dev/null +++ b/acceptance/bundle/resources/postgres_projects/create_transient_500/databricks.yml @@ -0,0 +1,12 @@ +bundle: + name: test-bundle + +sync: + paths: [] + +resources: + postgres_projects: + my_project: + project_id: test-pg-proj + display_name: "Test Postgres Project" + pg_version: 16 diff --git a/acceptance/bundle/resources/postgres_projects/create_transient_500/out.test.toml b/acceptance/bundle/resources/postgres_projects/create_transient_500/out.test.toml new file mode 100644 index 00000000000..15c32f324ac --- /dev/null +++ b/acceptance/bundle/resources/postgres_projects/create_transient_500/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] +EnvMatrix.READPLAN = [] diff --git a/acceptance/bundle/resources/postgres_projects/create_transient_500/output.txt b/acceptance/bundle/resources/postgres_projects/create_transient_500/output.txt new file mode 100644 index 00000000000..c6170683245 --- /dev/null +++ b/acceptance/bundle/resources/postgres_projects/create_transient_500/output.txt @@ -0,0 +1,24 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Warn: deploying resources.postgres_projects.my_project: retrying after 500 Internal Server Error from POST /api/2.0/postgres/projects +Warn: deploying resources.postgres_projects.my_project: retrying after 500 Internal Server Error from GET /api/2.0/postgres/projects/test-pg-proj/operations/[UUID] +Created postgres_projects.my_project +Files: 0 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --get //postgres + 2 POST /api/2.0/postgres/projects + 2 GET /api/2.0/postgres/projects/test-pg-proj/operations/[UUID] + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.postgres_projects.my_project + +This action will result in the deletion of the following Lakebase projects along with +all their branches, databases, and endpoints. All data stored in them will be permanently lost: + delete resources.postgres_projects.my_project + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/postgres_projects/create_transient_500/script b/acceptance/bundle/resources/postgres_projects/create_transient_500/script new file mode 100644 index 00000000000..94ef430d640 --- /dev/null +++ b/acceptance/bundle/resources/postgres_projects/create_transient_500/script @@ -0,0 +1,11 @@ +# One transient 500 on the create request and one on polling the create operation; both are +# retried and the deploy succeeds. +fault.py "POST /api/2.0/postgres/projects" 500 0 1 INTERNAL_ERROR +fault.py "GET /api/2.0/postgres/projects/*/operations/*" 500 0 1 INTERNAL_ERROR + +trace $CLI bundle deploy + +# Two create requests (the 500 and the retry) and at least two operation polls (the 500 and the retry). +trace print_requests.py --get //postgres | jq -r '.method + " " + .path' | uniq -c + +trace $CLI bundle destroy --auto-approve diff --git a/acceptance/bundle/resources/postgres_projects/create_transient_500/test.toml b/acceptance/bundle/resources/postgres_projects/create_transient_500/test.toml new file mode 100644 index 00000000000..ab3f407bfa7 --- /dev/null +++ b/acceptance/bundle/resources/postgres_projects/create_transient_500/test.toml @@ -0,0 +1,6 @@ +Cloud = false +RecordRequests = true +EnvMatrix.READPLAN = [] +Env.DATABRICKS_BUNDLE_ENGINE = "direct" +Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "100" +SuppressLines.warn_retry = "" diff --git a/bundle/direct/dresources/postgres_branch.go b/bundle/direct/dresources/postgres_branch.go index 19fac982f08..00948f0ee4f 100644 --- a/bundle/direct/dresources/postgres_branch.go +++ b/bundle/direct/dresources/postgres_branch.go @@ -129,7 +129,7 @@ func (r *ResourcePostgresBranch) DoCreate(ctx context.Context, config *PostgresB } // Wait for the branch to be ready (long-running operation) - result, err := waiter.Wait(ctx) + result, err := waitCreateOperation(ctx, waiter.Wait) if err != nil { return "", nil, err } diff --git a/bundle/direct/dresources/postgres_catalog.go b/bundle/direct/dresources/postgres_catalog.go index 348ba88960b..e2b32665d85 100644 --- a/bundle/direct/dresources/postgres_catalog.go +++ b/bundle/direct/dresources/postgres_catalog.go @@ -111,7 +111,7 @@ func (r *ResourcePostgresCatalog) DoCreate(ctx context.Context, config *Postgres return "", nil, err } - result, err := waiter.Wait(ctx) + result, err := waitCreateOperation(ctx, waiter.Wait) if err != nil { return "", nil, err } diff --git a/bundle/direct/dresources/postgres_data_api.go b/bundle/direct/dresources/postgres_data_api.go index 14f9f0e6066..ba94134e0ca 100644 --- a/bundle/direct/dresources/postgres_data_api.go +++ b/bundle/direct/dresources/postgres_data_api.go @@ -135,7 +135,7 @@ func (r *ResourcePostgresDataApi) DoCreate(ctx context.Context, config *Postgres return "", nil, err } - result, err := waiter.Wait(ctx) + result, err := waitCreateOperation(ctx, waiter.Wait) if err != nil { return "", nil, err } diff --git a/bundle/direct/dresources/postgres_database.go b/bundle/direct/dresources/postgres_database.go index 154163b4c8f..b3ec6d7b439 100644 --- a/bundle/direct/dresources/postgres_database.go +++ b/bundle/direct/dresources/postgres_database.go @@ -121,7 +121,7 @@ func (r *ResourcePostgresDatabase) DoCreate(ctx context.Context, config *Postgre } // Wait for the database to be ready (long-running operation) - result, err := waiter.Wait(ctx) + result, err := waitCreateOperation(ctx, waiter.Wait) if err != nil { return "", nil, err } diff --git a/bundle/direct/dresources/postgres_endpoint.go b/bundle/direct/dresources/postgres_endpoint.go index 48ea312005a..a2a339091fe 100644 --- a/bundle/direct/dresources/postgres_endpoint.go +++ b/bundle/direct/dresources/postgres_endpoint.go @@ -162,7 +162,7 @@ func (r *ResourcePostgresEndpoint) DoCreate(ctx context.Context, config *Postgre } // Wait for the operation to complete - result, err := waiter.Wait(ctx) + result, err := waitCreateOperation(ctx, waiter.Wait) if err != nil { return "", nil, err } diff --git a/bundle/direct/dresources/postgres_project.go b/bundle/direct/dresources/postgres_project.go index e62b87906cc..1bdc8617913 100644 --- a/bundle/direct/dresources/postgres_project.go +++ b/bundle/direct/dresources/postgres_project.go @@ -121,11 +121,14 @@ func (r *ResourcePostgresProject) DoCreate(ctx context.Context, config *Postgres }, }) if err != nil { - return "", nil, err + // The project is keyed by the caller-chosen project_id, so repeating the request cannot + // create a duplicate: if the failed request did create the project, the retry fails with + // ALREADY_EXISTS, just as the next deploy would have. + return "", nil, retrySafe(err) } // Wait for the project to be ready (long-running operation) - result, err := waiter.Wait(ctx) + result, err := waitCreateOperation(ctx, waiter.Wait) if err != nil { return "", nil, err } diff --git a/bundle/direct/dresources/postgres_role.go b/bundle/direct/dresources/postgres_role.go index 43eef50a657..3f4153b91e0 100644 --- a/bundle/direct/dresources/postgres_role.go +++ b/bundle/direct/dresources/postgres_role.go @@ -145,7 +145,7 @@ func (r *ResourcePostgresRole) DoCreate(ctx context.Context, config *PostgresRol return "", nil, err } - result, err := waiter.Wait(ctx) + result, err := waitCreateOperation(ctx, waiter.Wait) if err != nil { return "", nil, err } diff --git a/bundle/direct/dresources/postgres_synced_table.go b/bundle/direct/dresources/postgres_synced_table.go index f194df54e61..a94ba9b7c19 100644 --- a/bundle/direct/dresources/postgres_synced_table.go +++ b/bundle/direct/dresources/postgres_synced_table.go @@ -117,7 +117,7 @@ func (r *ResourcePostgresSyncedTable) DoCreate(ctx context.Context, config *Post return "", nil, err } - result, err := waiter.Wait(ctx) + result, err := waitCreateOperation(ctx, waiter.Wait) if err != nil { return "", nil, err } diff --git a/bundle/direct/dresources/postgres_wait.go b/bundle/direct/dresources/postgres_wait.go new file mode 100644 index 00000000000..c902356cabf --- /dev/null +++ b/bundle/direct/dresources/postgres_wait.go @@ -0,0 +1,18 @@ +package dresources + +import ( + "context" + + "github.com/databricks/cli/libs/retry" + "github.com/databricks/databricks-sdk-go/experimental/api" +) + +// waitCreateOperation polls a postgres create operation, polling again when a poll fails with a +// transient error (observed: 500 INTERNAL_ERROR and inactivity timeouts on GET .../operations/...). +// The create request was already accepted, so polling again cannot create a second resource, +// whereas a failure returned from DoCreate is not retried at all. +func waitCreateOperation[T any](ctx context.Context, wait func(context.Context, ...api.Option) (T, error)) (T, error) { + return retry.With(ctx, retry.IsTransientRead, func() (T, error) { + return wait(ctx) + }) +} From fc59e56048b550971ada43662a85d476f729d9c8 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Thu, 12 Nov 2026 13:52:04 +0000 Subject: [PATCH 2/4] Add changelog entry cloud: TestAccept/bundle/resources/postgres_projects/basic TestAccept/bundle/resources/postgres_endpoints/basic Co-authored-by: Isaac --- .nextchanges/bundles/7557.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 .nextchanges/bundles/7557.md diff --git a/.nextchanges/bundles/7557.md b/.nextchanges/bundles/7557.md new file mode 100644 index 00000000000..bcc0193dde2 --- /dev/null +++ b/.nextchanges/bundles/7557.md @@ -0,0 +1 @@ +* Retry transient server errors when creating Lakebase (`postgres_*`) resources in the direct deployment engine. ([#7557](https://github.com/databricks/cli/pull/7557)) From a74de8b2388b8c7579cd8edcf4b5f6d581240c8e Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Thu, 12 Nov 2026 13:55:42 +0000 Subject: [PATCH 3/4] acc: fix comment in create_transient_500 cloud: TestAccept/bundle/resources/postgres_projects/basic TestAccept/bundle/resources/postgres_endpoints/basic Co-authored-by: Isaac --- .../resources/postgres_projects/create_transient_500/script | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/bundle/resources/postgres_projects/create_transient_500/script b/acceptance/bundle/resources/postgres_projects/create_transient_500/script index 94ef430d640..c77d17b235f 100644 --- a/acceptance/bundle/resources/postgres_projects/create_transient_500/script +++ b/acceptance/bundle/resources/postgres_projects/create_transient_500/script @@ -5,7 +5,7 @@ fault.py "GET /api/2.0/postgres/projects/*/operations/*" 500 0 1 INTERNAL_ERROR trace $CLI bundle deploy -# Two create requests (the 500 and the retry) and at least two operation polls (the 500 and the retry). +# Two create requests and two operation polls: the 500 and the retry of each. trace print_requests.py --get //postgres | jq -r '.method + " " + .path' | uniq -c trace $CLI bundle destroy --auto-approve From 37f3075c00f52b326b077bc7a570ff38b338cc61 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Thu, 12 Nov 2026 14:06:51 +0000 Subject: [PATCH 4/4] acc: don't depend on uniq -c padding in create_transient_500 cloud: TestAccept/bundle/resources/postgres_projects/basic TestAccept/bundle/resources/postgres_endpoints/basic Co-authored-by: Isaac --- .../postgres_projects/create_transient_500/output.txt | 6 ++++-- .../resources/postgres_projects/create_transient_500/script | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/acceptance/bundle/resources/postgres_projects/create_transient_500/output.txt b/acceptance/bundle/resources/postgres_projects/create_transient_500/output.txt index c6170683245..cc28f656da5 100644 --- a/acceptance/bundle/resources/postgres_projects/create_transient_500/output.txt +++ b/acceptance/bundle/resources/postgres_projects/create_transient_500/output.txt @@ -8,8 +8,10 @@ Files: 0 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> print_requests.py --get //postgres - 2 POST /api/2.0/postgres/projects - 2 GET /api/2.0/postgres/projects/test-pg-proj/operations/[UUID] +POST /api/2.0/postgres/projects +POST /api/2.0/postgres/projects +GET /api/2.0/postgres/projects/test-pg-proj/operations/[UUID] +GET /api/2.0/postgres/projects/test-pg-proj/operations/[UUID] >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/resources/postgres_projects/create_transient_500/script b/acceptance/bundle/resources/postgres_projects/create_transient_500/script index c77d17b235f..0f07dbbcf59 100644 --- a/acceptance/bundle/resources/postgres_projects/create_transient_500/script +++ b/acceptance/bundle/resources/postgres_projects/create_transient_500/script @@ -6,6 +6,6 @@ fault.py "GET /api/2.0/postgres/projects/*/operations/*" 500 0 1 INTERNAL_ERROR trace $CLI bundle deploy # Two create requests and two operation polls: the 500 and the retry of each. -trace print_requests.py --get //postgres | jq -r '.method + " " + .path' | uniq -c +trace print_requests.py --get //postgres | jq -r '.method + " " + .path' trace $CLI bundle destroy --auto-approve