Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
bundle:
name: vs-endpoint-504-$UNIQUE_NAME

sync:
paths: []

resources:
vector_search_endpoints:
my_endpoint:
name: vs-endpoint-$UNIQUE_NAME
endpoint_type: STANDARD
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Cloud = false
EnvMatrix.DMS = [""]
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@

=== A 504 before the endpoint is created: the create is retried

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/vs-endpoint-504-[UNIQUE_NAME]/default/files...
Warn: deploying resources.vector_search_endpoints.my_endpoint: retrying after 504 Gateway Timeout from POST /api/2.0/vector-search/endpoints
Created vector_search_endpoints.my_endpoint
Files: 0 uploaded, 0 deleted
Resources: 1 created, 0 changed, 0 deleted, 0 unchanged

>>> print_requests.py --get //vector-search/endpoints
{
"method": "POST",
"path": "/api/2.0/vector-search/endpoints",
"body": {
"endpoint_type": "STANDARD",
"name": "vs-endpoint-[UNIQUE_NAME]"
}
}
{
"method": "POST",
"path": "/api/2.0/vector-search/endpoints",
"body": {
"endpoint_type": "STANDARD",
"name": "vs-endpoint-[UNIQUE_NAME]"
}
}
{
"method": "GET",
"path": "/api/2.0/vector-search/endpoints/vs-endpoint-[UNIQUE_NAME]"
}

>>> [CLI] bundle plan
Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged

>>> [CLI] bundle destroy --auto-approve
The following resources will be deleted:
delete resources.vector_search_endpoints.my_endpoint

All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/vs-endpoint-504-[UNIQUE_NAME]/default

Destroy: 1 deleted

=== A 504 after the endpoint is created: the retry finds it and uses it

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/vs-endpoint-504-[UNIQUE_NAME]/default/files...
Warn: deploying resources.vector_search_endpoints.my_endpoint: retrying after 504 Gateway Timeout from POST /api/2.0/vector-search/endpoints
Warn: deploying resources.vector_search_endpoints.my_endpoint: vector search endpoint "vs-endpoint-[UNIQUE_NAME]" was created by an earlier attempt that failed with a transient error, using it
Created vector_search_endpoints.my_endpoint
Files: 0 uploaded, 0 deleted
Resources: 1 created, 0 changed, 0 deleted, 0 unchanged

>>> print_requests.py --get //vector-search/endpoints
{
"method": "POST",
"path": "/api/2.0/vector-search/endpoints",
"body": {
"endpoint_type": "STANDARD",
"name": "vs-endpoint-[UNIQUE_NAME]"
}
}
{
"method": "POST",
"path": "/api/2.0/vector-search/endpoints",
"body": {
"endpoint_type": "STANDARD",
"name": "vs-endpoint-[UNIQUE_NAME]"
}
}
{
"method": "GET",
"path": "/api/2.0/vector-search/endpoints/vs-endpoint-[UNIQUE_NAME]"
}
{
"method": "GET",
"path": "/api/2.0/vector-search/endpoints/vs-endpoint-[UNIQUE_NAME]"
}

>>> [CLI] bundle plan
Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged

>>> [CLI] bundle destroy --auto-approve
The following resources will be deleted:
delete resources.vector_search_endpoints.my_endpoint

All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/vs-endpoint-504-[UNIQUE_NAME]/default

Destroy: 1 deleted
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
envsubst < databricks.yml.tmpl > databricks.yml

cleanup() {
trace $CLI bundle destroy --auto-approve
rm -f "$OUT_REQUESTS"
}
trap cleanup EXIT

title "A 504 before the endpoint is created: the create is retried"
fault.py "POST /api/2.0/vector-search/endpoints" 504 0 1 TEMPORARILY_UNAVAILABLE
trace $CLI bundle deploy
trace print_requests.py --get //vector-search/endpoints
trace $CLI bundle plan
trace $CLI bundle destroy --auto-approve
rm -f "$OUT_REQUESTS"

title "A 504 after the endpoint is created: the retry finds it and uses it"
fault.py --after-handler "POST /api/2.0/vector-search/endpoints" 504 0 1 TEMPORARILY_UNAVAILABLE
trace $CLI bundle deploy
trace print_requests.py --get //vector-search/endpoints
trace $CLI bundle plan
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
# Faults are injected through the testserver.
Cloud = false

Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "100"
SuppressLines.warn_retry = ""
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,12 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged

>>> [CLI] bundle deploy --auto-approve
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/recreate-create-fails-[UNIQUE_NAME]/default/files...
Error: cannot recreate resources.vector_search_endpoints.my_endpoint: Vector search endpoint with name vs-endpoint-b-[UNIQUE_NAME] already exists (409 RESOURCE_ALREADY_EXISTS)
Error: cannot recreate resources.vector_search_endpoints.my_endpoint: AI Search endpoint vs-endpoint-b-[UNIQUE_NAME] already exists. Please try again with a different endpoint name. (409 ALREADY_EXISTS)

Endpoint: POST [DATABRICKS_URL]/api/2.0/vector-search/endpoints
HTTP Status: 409 Conflict
API error_code: RESOURCE_ALREADY_EXISTS
API message: Vector search endpoint with name vs-endpoint-b-[UNIQUE_NAME] already exists
API error_code: ALREADY_EXISTS
API message: AI Search endpoint vs-endpoint-b-[UNIQUE_NAME] already exists. Please try again with a different endpoint name.

Files: 0 uploaded, 0 deleted

Expand Down
49 changes: 46 additions & 3 deletions bundle/direct/dresources/vector_search_endpoint.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,16 @@ package dresources

import (
"context"
"errors"
"time"

"github.com/databricks/cli/bundle/config/resources"
"github.com/databricks/cli/libs/log"
"github.com/databricks/cli/libs/retry"
"github.com/databricks/cli/libs/structs/structpath"
"github.com/databricks/cli/libs/utils"
"github.com/databricks/databricks-sdk-go"
"github.com/databricks/databricks-sdk-go/apierr"
"github.com/databricks/databricks-sdk-go/marshal"
"github.com/databricks/databricks-sdk-go/service/vectorsearch"
)
Expand Down Expand Up @@ -80,13 +84,52 @@ func (r *ResourceVectorSearchEndpoint) DoRead(ctx context.Context, id string) (*
return newVectorSearchEndpointRemote(info), nil
}

// DoCreate retries transient errors itself rather than marking them retrySafe: the backend
// can answer 504 TEMPORARILY_UNAVAILABLE after it has already created the endpoint, and the
// retry is then rejected as already existing. In that case the endpoint is used if it was
// created after the first attempt started; an older one (another user's endpoint, or the
// old endpoint of a recreate that is still being deleted) keeps failing the deploy.
func (r *ResourceVectorSearchEndpoint) DoCreate(ctx context.Context, config *vectorsearch.CreateEndpoint) (string, *VectorSearchEndpointRemote, error) {
waiter, err := r.client.VectorSearchEndpoints.CreateEndpoint(ctx, *config)
start := time.Now()
afterTransient := false
info, err := retry.With(ctx, retry.IsTransient, func() (*vectorsearch.EndpointInfo, error) {
waiter, err := r.client.VectorSearchEndpoints.CreateEndpoint(ctx, *config)
if err == nil {
return waiter.Response, nil
}
if afterTransient && isEndpointAlreadyExists(err) {
info, getErr := r.client.VectorSearchEndpoints.GetEndpointByEndpointName(ctx, config.Name)
if getErr != nil {
return nil, getErr
}
if !createdSince(info, start) {
return nil, err
}
log.Warnf(ctx, "vector search endpoint %q was created by an earlier attempt that failed with a transient error, using it", config.Name)
return info, nil
}
afterTransient = afterTransient || retry.IsTransient(ctx, err)
return nil, err
})
if err != nil {
return "", nil, err
}
id := config.Name
return id, newVectorSearchEndpointRemote(waiter.Response), nil
return config.Name, newVectorSearchEndpointRemote(info), nil
}

// isEndpointAlreadyExists matches both codes: the cloud returns 409 ALREADY_EXISTS (observed on
// AWS, 2026-11), while RESOURCE_ALREADY_EXISTS is the generic code for the same conflict.
func isEndpointAlreadyExists(err error) bool {
return errors.Is(err, apierr.ErrAlreadyExists) || errors.Is(err, apierr.ErrResourceAlreadyExists)
}

// endpointClockSkew allows for the server's clock running behind the client's.
const endpointClockSkew = time.Minute

// createdSince reports whether the endpoint was created no earlier than start, the time of the
// first create attempt, so that it can only have been created by that attempt.
func createdSince(info *vectorsearch.EndpointInfo, start time.Time) bool {
return info.CreationTimestamp >= start.Add(-endpointClockSkew).UnixMilli()
}

func (r *ResourceVectorSearchEndpoint) WaitAfterCreate(ctx context.Context, id string, config *vectorsearch.CreateEndpoint) (*VectorSearchEndpointRemote, error) {
Expand Down
21 changes: 21 additions & 0 deletions bundle/direct/dresources/vector_search_endpoint_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
package dresources

import (
"testing"
"time"

"github.com/databricks/databricks-sdk-go/service/vectorsearch"
"github.com/stretchr/testify/assert"
)

func TestVectorSearchEndpointCreatedSince(t *testing.T) {
start := time.UnixMilli(1_800_000_000_000)
created := func(t time.Time) *vectorsearch.EndpointInfo {
return &vectorsearch.EndpointInfo{CreationTimestamp: t.UnixMilli()}
}

assert.True(t, createdSince(created(start.Add(time.Second)), start))
assert.True(t, createdSince(created(start.Add(-30*time.Second)), start), "within clock skew")
assert.False(t, createdSince(created(start.Add(-2*time.Minute)), start), "pre-existing endpoint")
assert.False(t, createdSince(&vectorsearch.EndpointInfo{}, start), "no creation timestamp")
}
2 changes: 1 addition & 1 deletion libs/testserver/vector_search_endpoints.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ func (s *FakeWorkspace) VectorSearchEndpointCreate(req Request) Response {
if _, exists := s.VectorSearchEndpoints[createReq.Name]; exists {
return Response{
StatusCode: http.StatusConflict,
Body: map[string]string{"error_code": "RESOURCE_ALREADY_EXISTS", "message": fmt.Sprintf("Vector search endpoint with name %s already exists", createReq.Name)},
Body: map[string]string{"error_code": "ALREADY_EXISTS", "message": fmt.Sprintf("AI Search endpoint %s already exists. Please try again with a different endpoint name.", createReq.Name)},
}
}

Expand Down
Loading