diff --git a/acceptance/bundle/resources/vector_search_endpoints/create-504/databricks.yml.tmpl b/acceptance/bundle/resources/vector_search_endpoints/create-504/databricks.yml.tmpl new file mode 100644 index 00000000000..9fe910f5f2e --- /dev/null +++ b/acceptance/bundle/resources/vector_search_endpoints/create-504/databricks.yml.tmpl @@ -0,0 +1,11 @@ +bundle: + name: vs-endpoint-504-$UNIQUE_NAME + +sync: + paths: [] + +resources: + vector_search_endpoints: + my_endpoint: + name: vs-endpoint-$UNIQUE_NAME + endpoint_type: STANDARD diff --git a/acceptance/bundle/resources/vector_search_endpoints/create-504/out.test.toml b/acceptance/bundle/resources/vector_search_endpoints/create-504/out.test.toml new file mode 100644 index 00000000000..19b5b15d11f --- /dev/null +++ b/acceptance/bundle/resources/vector_search_endpoints/create-504/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/vector_search_endpoints/create-504/output.txt b/acceptance/bundle/resources/vector_search_endpoints/create-504/output.txt new file mode 100644 index 00000000000..4fe63232863 --- /dev/null +++ b/acceptance/bundle/resources/vector_search_endpoints/create-504/output.txt @@ -0,0 +1,89 @@ + +=== A 504 before the endpoint is created: the create is retried + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/vs-endpoint-504-[UNIQUE_NAME]/default/files... +Warn: deploying resources.vector_search_endpoints.my_endpoint: retrying after 504 Gateway Timeout from POST /api/2.0/vector-search/endpoints +Created vector_search_endpoints.my_endpoint +Files: 0 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --get //vector-search/endpoints +{ + "method": "POST", + "path": "/api/2.0/vector-search/endpoints", + "body": { + "endpoint_type": "STANDARD", + "name": "vs-endpoint-[UNIQUE_NAME]" + } +} +{ + "method": "POST", + "path": "/api/2.0/vector-search/endpoints", + "body": { + "endpoint_type": "STANDARD", + "name": "vs-endpoint-[UNIQUE_NAME]" + } +} +{ + "method": "GET", + "path": "/api/2.0/vector-search/endpoints/vs-endpoint-[UNIQUE_NAME]" +} + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.vector_search_endpoints.my_endpoint + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/vs-endpoint-504-[UNIQUE_NAME]/default + +Destroy: 1 deleted + +=== A 504 after the endpoint is created: the retry finds it and uses it + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/vs-endpoint-504-[UNIQUE_NAME]/default/files... +Warn: deploying resources.vector_search_endpoints.my_endpoint: retrying after 504 Gateway Timeout from POST /api/2.0/vector-search/endpoints +Warn: deploying resources.vector_search_endpoints.my_endpoint: vector search endpoint "vs-endpoint-[UNIQUE_NAME]" was created by an earlier attempt that failed with a transient error, using it +Created vector_search_endpoints.my_endpoint +Files: 0 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --get //vector-search/endpoints +{ + "method": "POST", + "path": "/api/2.0/vector-search/endpoints", + "body": { + "endpoint_type": "STANDARD", + "name": "vs-endpoint-[UNIQUE_NAME]" + } +} +{ + "method": "POST", + "path": "/api/2.0/vector-search/endpoints", + "body": { + "endpoint_type": "STANDARD", + "name": "vs-endpoint-[UNIQUE_NAME]" + } +} +{ + "method": "GET", + "path": "/api/2.0/vector-search/endpoints/vs-endpoint-[UNIQUE_NAME]" +} +{ + "method": "GET", + "path": "/api/2.0/vector-search/endpoints/vs-endpoint-[UNIQUE_NAME]" +} + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.vector_search_endpoints.my_endpoint + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/vs-endpoint-504-[UNIQUE_NAME]/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/vector_search_endpoints/create-504/script b/acceptance/bundle/resources/vector_search_endpoints/create-504/script new file mode 100644 index 00000000000..34f9b157ce2 --- /dev/null +++ b/acceptance/bundle/resources/vector_search_endpoints/create-504/script @@ -0,0 +1,21 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f "$OUT_REQUESTS" +} +trap cleanup EXIT + +title "A 504 before the endpoint is created: the create is retried" +fault.py "POST /api/2.0/vector-search/endpoints" 504 0 1 TEMPORARILY_UNAVAILABLE +trace $CLI bundle deploy +trace print_requests.py --get //vector-search/endpoints +trace $CLI bundle plan +trace $CLI bundle destroy --auto-approve +rm -f "$OUT_REQUESTS" + +title "A 504 after the endpoint is created: the retry finds it and uses it" +fault.py --after-handler "POST /api/2.0/vector-search/endpoints" 504 0 1 TEMPORARILY_UNAVAILABLE +trace $CLI bundle deploy +trace print_requests.py --get //vector-search/endpoints +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/vector_search_endpoints/create-504/test.toml b/acceptance/bundle/resources/vector_search_endpoints/create-504/test.toml new file mode 100644 index 00000000000..e667cfdba82 --- /dev/null +++ b/acceptance/bundle/resources/vector_search_endpoints/create-504/test.toml @@ -0,0 +1,5 @@ +# Faults are injected through the testserver. +Cloud = false + +Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "100" +SuppressLines.warn_retry = "" diff --git a/acceptance/bundle/resources/vector_search_endpoints/recreate/create-fails/output.txt b/acceptance/bundle/resources/vector_search_endpoints/recreate/create-fails/output.txt index 906b8045fb6..ad477511f59 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/recreate/create-fails/output.txt +++ b/acceptance/bundle/resources/vector_search_endpoints/recreate/create-fails/output.txt @@ -18,12 +18,12 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/recreate-create-fails-[UNIQUE_NAME]/default/files... -Error: cannot recreate resources.vector_search_endpoints.my_endpoint: Vector search endpoint with name vs-endpoint-b-[UNIQUE_NAME] already exists (409 RESOURCE_ALREADY_EXISTS) +Error: cannot recreate resources.vector_search_endpoints.my_endpoint: AI Search endpoint vs-endpoint-b-[UNIQUE_NAME] already exists. Please try again with a different endpoint name. (409 ALREADY_EXISTS) Endpoint: POST [DATABRICKS_URL]/api/2.0/vector-search/endpoints HTTP Status: 409 Conflict -API error_code: RESOURCE_ALREADY_EXISTS -API message: Vector search endpoint with name vs-endpoint-b-[UNIQUE_NAME] already exists +API error_code: ALREADY_EXISTS +API message: AI Search endpoint vs-endpoint-b-[UNIQUE_NAME] already exists. Please try again with a different endpoint name. Files: 0 uploaded, 0 deleted diff --git a/bundle/direct/dresources/vector_search_endpoint.go b/bundle/direct/dresources/vector_search_endpoint.go index 12470872b62..46a229dbf6d 100644 --- a/bundle/direct/dresources/vector_search_endpoint.go +++ b/bundle/direct/dresources/vector_search_endpoint.go @@ -2,12 +2,16 @@ package dresources import ( "context" + "errors" "time" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/retry" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/utils" "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/vectorsearch" ) @@ -80,13 +84,52 @@ func (r *ResourceVectorSearchEndpoint) DoRead(ctx context.Context, id string) (* return newVectorSearchEndpointRemote(info), nil } +// DoCreate retries transient errors itself rather than marking them retrySafe: the backend +// can answer 504 TEMPORARILY_UNAVAILABLE after it has already created the endpoint, and the +// retry is then rejected as already existing. In that case the endpoint is used if it was +// created after the first attempt started; an older one (another user's endpoint, or the +// old endpoint of a recreate that is still being deleted) keeps failing the deploy. func (r *ResourceVectorSearchEndpoint) DoCreate(ctx context.Context, config *vectorsearch.CreateEndpoint) (string, *VectorSearchEndpointRemote, error) { - waiter, err := r.client.VectorSearchEndpoints.CreateEndpoint(ctx, *config) + start := time.Now() + afterTransient := false + info, err := retry.With(ctx, retry.IsTransient, func() (*vectorsearch.EndpointInfo, error) { + waiter, err := r.client.VectorSearchEndpoints.CreateEndpoint(ctx, *config) + if err == nil { + return waiter.Response, nil + } + if afterTransient && isEndpointAlreadyExists(err) { + info, getErr := r.client.VectorSearchEndpoints.GetEndpointByEndpointName(ctx, config.Name) + if getErr != nil { + return nil, getErr + } + if !createdSince(info, start) { + return nil, err + } + log.Warnf(ctx, "vector search endpoint %q was created by an earlier attempt that failed with a transient error, using it", config.Name) + return info, nil + } + afterTransient = afterTransient || retry.IsTransient(ctx, err) + return nil, err + }) if err != nil { return "", nil, err } - id := config.Name - return id, newVectorSearchEndpointRemote(waiter.Response), nil + return config.Name, newVectorSearchEndpointRemote(info), nil +} + +// isEndpointAlreadyExists matches both codes: the cloud returns 409 ALREADY_EXISTS (observed on +// AWS, 2026-11), while RESOURCE_ALREADY_EXISTS is the generic code for the same conflict. +func isEndpointAlreadyExists(err error) bool { + return errors.Is(err, apierr.ErrAlreadyExists) || errors.Is(err, apierr.ErrResourceAlreadyExists) +} + +// endpointClockSkew allows for the server's clock running behind the client's. +const endpointClockSkew = time.Minute + +// createdSince reports whether the endpoint was created no earlier than start, the time of the +// first create attempt, so that it can only have been created by that attempt. +func createdSince(info *vectorsearch.EndpointInfo, start time.Time) bool { + return info.CreationTimestamp >= start.Add(-endpointClockSkew).UnixMilli() } func (r *ResourceVectorSearchEndpoint) WaitAfterCreate(ctx context.Context, id string, config *vectorsearch.CreateEndpoint) (*VectorSearchEndpointRemote, error) { diff --git a/bundle/direct/dresources/vector_search_endpoint_test.go b/bundle/direct/dresources/vector_search_endpoint_test.go new file mode 100644 index 00000000000..450bb49df6e --- /dev/null +++ b/bundle/direct/dresources/vector_search_endpoint_test.go @@ -0,0 +1,21 @@ +package dresources + +import ( + "testing" + "time" + + "github.com/databricks/databricks-sdk-go/service/vectorsearch" + "github.com/stretchr/testify/assert" +) + +func TestVectorSearchEndpointCreatedSince(t *testing.T) { + start := time.UnixMilli(1_800_000_000_000) + created := func(t time.Time) *vectorsearch.EndpointInfo { + return &vectorsearch.EndpointInfo{CreationTimestamp: t.UnixMilli()} + } + + assert.True(t, createdSince(created(start.Add(time.Second)), start)) + assert.True(t, createdSince(created(start.Add(-30*time.Second)), start), "within clock skew") + assert.False(t, createdSince(created(start.Add(-2*time.Minute)), start), "pre-existing endpoint") + assert.False(t, createdSince(&vectorsearch.EndpointInfo{}, start), "no creation timestamp") +} diff --git a/libs/testserver/vector_search_endpoints.go b/libs/testserver/vector_search_endpoints.go index c051d599ccc..453d64b3924 100644 --- a/libs/testserver/vector_search_endpoints.go +++ b/libs/testserver/vector_search_endpoints.go @@ -22,7 +22,7 @@ func (s *FakeWorkspace) VectorSearchEndpointCreate(req Request) Response { if _, exists := s.VectorSearchEndpoints[createReq.Name]; exists { return Response{ StatusCode: http.StatusConflict, - Body: map[string]string{"error_code": "RESOURCE_ALREADY_EXISTS", "message": fmt.Sprintf("Vector search endpoint with name %s already exists", createReq.Name)}, + Body: map[string]string{"error_code": "ALREADY_EXISTS", "message": fmt.Sprintf("AI Search endpoint %s already exists. Please try again with a different endpoint name.", createReq.Name)}, } }