From bf5264670b4d7894731850229564295ccff1da8b Mon Sep 17 00:00:00 2001 From: idelafuente-oc Date: Thu, 20 Nov 2025 18:12:22 -0300 Subject: [PATCH 1/7] Adds for each validation --- bundle/config/validate/fast_validate.go | 1 + bundle/config/validate/for_each_task.go | 76 ++++++++ bundle/config/validate/for_each_task_test.go | 190 +++++++++++++++++++ 3 files changed, 267 insertions(+) create mode 100644 bundle/config/validate/for_each_task.go create mode 100644 bundle/config/validate/for_each_task_test.go diff --git a/bundle/config/validate/fast_validate.go b/bundle/config/validate/fast_validate.go index d01eb8c1491..d7d2c8eab72 100644 --- a/bundle/config/validate/fast_validate.go +++ b/bundle/config/validate/fast_validate.go @@ -29,6 +29,7 @@ func (f *fastValidate) Apply(ctx context.Context, rb *bundle.Bundle) diag.Diagno // Fast mutators with only in-memory checks JobClusterKeyDefined(), JobTaskClusterSpec(), + ForEachTask(), // Blocking mutators. Deployments will fail if these checks fail. ValidateArtifactPath(), diff --git a/bundle/config/validate/for_each_task.go b/bundle/config/validate/for_each_task.go new file mode 100644 index 00000000000..c1d68be8b5a --- /dev/null +++ b/bundle/config/validate/for_each_task.go @@ -0,0 +1,76 @@ +package validate + +import ( + "context" + "fmt" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/databricks-sdk-go/service/jobs" +) + +// ForEachTask validates constraints for for_each_task configuration +func ForEachTask() bundle.ReadOnlyMutator { + return &forEachTask{} +} + +type forEachTask struct{ bundle.RO } + +func (v *forEachTask) Name() string { + return "validate:for_each_task" +} + +func (v *forEachTask) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + diags := diag.Diagnostics{} + + jobsPath := dyn.NewPath(dyn.Key("resources"), dyn.Key("jobs")) + + for resourceName, job := range b.Config.Resources.Jobs { + resourcePath := jobsPath.Append(dyn.Key(resourceName)) + + for taskIndex, task := range job.Tasks { + taskPath := resourcePath.Append(dyn.Key("tasks"), dyn.Index(taskIndex)) + + if task.ForEachTask != nil { + diags = diags.Extend(validateForEachTask(b, task, taskPath)) + } + } + } + + return diags +} + +func validateForEachTask(b *bundle.Bundle, task jobs.Task, taskPath dyn.Path) diag.Diagnostics { + diags := diag.Diagnostics{} + + if task.MaxRetries != 0 { + diags = diags.Append(invalidRetryFieldDiag(b, task, taskPath, "max_retries", diag.Error)) + } + + if task.MinRetryIntervalMillis != 0 { + diags = diags.Append(invalidRetryFieldDiag(b, task, taskPath, "min_retry_interval_millis", diag.Warning)) + } + + if task.RetryOnTimeout { + diags = diags.Append(invalidRetryFieldDiag(b, task, taskPath, "retry_on_timeout", diag.Warning)) + } + + return diags +} + +func invalidRetryFieldDiag(b *bundle.Bundle, task jobs.Task, taskPath dyn.Path, fieldName string, severity diag.Severity) diag.Diagnostic { + detail := fmt.Sprintf( + "Task %q has %s defined at the parent level, but it uses for_each_task.\n"+ + "When using for_each_task, %s must be defined on the nested task (for_each_task.task.%s), not on the parent task.", + task.TaskKey, fieldName, fieldName, fieldName, + ) + + return diag.Diagnostic{ + Severity: severity, + Summary: fmt.Sprintf("Invalid %s configuration for for_each_task", fieldName), + Detail: detail, + Locations: b.Config.GetLocations(taskPath.String()), + Paths: []dyn.Path{taskPath}, + } +} diff --git a/bundle/config/validate/for_each_task_test.go b/bundle/config/validate/for_each_task_test.go new file mode 100644 index 00000000000..c45aa96633a --- /dev/null +++ b/bundle/config/validate/for_each_task_test.go @@ -0,0 +1,190 @@ +package validate + +import ( + "context" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/internal/bundletest" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func createBundleWithForEachTask(parentTask jobs.Task) *bundle.Bundle { + if parentTask.ForEachTask == nil { + parentTask.ForEachTask = &jobs.ForEachTask{ + Inputs: "[1, 2, 3]", + Task: jobs.Task{ + TaskKey: "child_task", + NotebookTask: &jobs.NotebookTask{ + NotebookPath: "test.py", + }, + }, + } + } + + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job1": { + JobSettings: jobs.JobSettings{ + Name: "My Job", + Tasks: []jobs.Task{parentTask}, + }, + }, + }, + }, + }, + } + + bundletest.SetLocation(b, "resources.jobs.job1.tasks[0]", []dyn.Location{{File: "job.yml", Line: 1, Column: 1}}) + return b +} + +func TestForEachTask_InvalidRetryFields(t *testing.T) { + tests := []struct { + name string + task jobs.Task + expectedSeverity diag.Severity + expectedSummary string + expectedDetail string + }{ + { + name: "max_retries on parent", + task: jobs.Task{ + TaskKey: "parent_task", + MaxRetries: 3, + }, + expectedSeverity: diag.Error, + expectedSummary: "Invalid max_retries configuration for for_each_task", + expectedDetail: "max_retries must be defined on the nested task", + }, + { + name: "min_retry_interval_millis on parent", + task: jobs.Task{ + TaskKey: "parent_task", + MinRetryIntervalMillis: 1000, + }, + expectedSeverity: diag.Warning, + expectedSummary: "Invalid min_retry_interval_millis configuration for for_each_task", + expectedDetail: "min_retry_interval_millis must be defined on the nested task", + }, + { + name: "retry_on_timeout on parent", + task: jobs.Task{ + TaskKey: "parent_task", + RetryOnTimeout: true, + }, + expectedSeverity: diag.Warning, + expectedSummary: "Invalid retry_on_timeout configuration for for_each_task", + expectedDetail: "retry_on_timeout must be defined on the nested task", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + b := createBundleWithForEachTask(tt.task) + + diags := ForEachTask().Apply(ctx, b) + + require.Len(t, diags, 1) + assert.Equal(t, tt.expectedSeverity, diags[0].Severity) + assert.Equal(t, tt.expectedSummary, diags[0].Summary) + assert.Contains(t, diags[0].Detail, tt.expectedDetail) + }) + } +} + +func TestForEachTask_MultipleRetryFieldsOnParent(t *testing.T) { + ctx := context.Background() + b := createBundleWithForEachTask(jobs.Task{ + TaskKey: "parent_task", + MaxRetries: 3, + MinRetryIntervalMillis: 1000, + RetryOnTimeout: true, + }) + + diags := ForEachTask().Apply(ctx, b) + require.Len(t, diags, 3) + + errorCount := 0 + warningCount := 0 + for _, d := range diags { + if d.Severity == diag.Error { + errorCount++ + } else if d.Severity == diag.Warning { + warningCount++ + } + } + assert.Equal(t, 1, errorCount) + assert.Equal(t, 2, warningCount) +} + +func TestForEachTask_ValidConfigurationOnChild(t *testing.T) { + ctx := context.Background() + b := createBundleWithForEachTask(jobs.Task{ + TaskKey: "parent_task", + ForEachTask: &jobs.ForEachTask{ + Inputs: "[1, 2, 3]", + Task: jobs.Task{ + TaskKey: "child_task", + MaxRetries: 3, + NotebookTask: &jobs.NotebookTask{ + NotebookPath: "test.py", + }, + }, + }, + }) + + diags := ForEachTask().Apply(ctx, b) + assert.Empty(t, diags) +} + +func TestForEachTask_NoForEachTask(t *testing.T) { + ctx := context.Background() + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job1": { + JobSettings: jobs.JobSettings{ + Name: "My Job", + Tasks: []jobs.Task{ + { + TaskKey: "simple_task", + MaxRetries: 3, + NotebookTask: &jobs.NotebookTask{ + NotebookPath: "test.py", + }, + }, + }, + }, + }, + }, + }, + }, + } + + bundletest.SetLocation(b, "resources.jobs.job1.tasks[0]", []dyn.Location{{File: "job.yml", Line: 1, Column: 1}}) + + diags := ForEachTask().Apply(ctx, b) + assert.Empty(t, diags) +} + +func TestForEachTask_RetryOnTimeoutFalse(t *testing.T) { + ctx := context.Background() + b := createBundleWithForEachTask(jobs.Task{ + TaskKey: "parent_task", + RetryOnTimeout: false, + }) + + diags := ForEachTask().Apply(ctx, b) + assert.Empty(t, diags) +} From 8fa76f03dc4630118d660808d0d39f2c505e3fcf Mon Sep 17 00:00:00 2001 From: idelafuente-oc Date: Mon, 8 Dec 2025 11:31:36 -0300 Subject: [PATCH 2/7] fix: github action lint --- bundle/config/validate/for_each_task_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bundle/config/validate/for_each_task_test.go b/bundle/config/validate/for_each_task_test.go index c45aa96633a..350da097697 100644 --- a/bundle/config/validate/for_each_task_test.go +++ b/bundle/config/validate/for_each_task_test.go @@ -117,9 +117,10 @@ func TestForEachTask_MultipleRetryFieldsOnParent(t *testing.T) { errorCount := 0 warningCount := 0 for _, d := range diags { - if d.Severity == diag.Error { + switch d.Severity { + case diag.Error: errorCount++ - } else if d.Severity == diag.Warning { + case diag.Warning: warningCount++ } } From 8f183d5904c8c840a3b7064b6059c47146bedb49 Mon Sep 17 00:00:00 2001 From: idelafuente-oc Date: Wed, 10 Dec 2025 12:20:43 -0300 Subject: [PATCH 3/7] feat: acceptance tests for for each tasks validation --- .../for_each_task_max_retries/databricks.yml | 16 ++++++++++ .../for_each_task_max_retries/out.test.toml | 5 ++++ .../for_each_task_max_retries/output.txt | 16 ++++++++++ .../validate/for_each_task_max_retries/script | 2 ++ .../for_each_task_max_retries/test.py | 2 ++ .../for_each_task_retry_fields/databricks.yml | 18 +++++++++++ .../for_each_task_retry_fields/out.test.toml | 5 ++++ .../for_each_task_retry_fields/output.txt | 30 +++++++++++++++++++ .../for_each_task_retry_fields/script | 2 ++ .../for_each_task_retry_fields/test.py | 2 ++ .../for_each_task_valid/databricks.yml | 16 ++++++++++ .../for_each_task_valid/out.test.toml | 5 ++++ .../validate/for_each_task_valid/output.txt | 7 +++++ .../validate/for_each_task_valid/script | 2 ++ .../validate/for_each_task_valid/test.py | 2 ++ 15 files changed, 130 insertions(+) create mode 100644 acceptance/bundle/validate/for_each_task_max_retries/databricks.yml create mode 100644 acceptance/bundle/validate/for_each_task_max_retries/out.test.toml create mode 100644 acceptance/bundle/validate/for_each_task_max_retries/output.txt create mode 100644 acceptance/bundle/validate/for_each_task_max_retries/script create mode 100644 acceptance/bundle/validate/for_each_task_max_retries/test.py create mode 100644 acceptance/bundle/validate/for_each_task_retry_fields/databricks.yml create mode 100644 acceptance/bundle/validate/for_each_task_retry_fields/out.test.toml create mode 100644 acceptance/bundle/validate/for_each_task_retry_fields/output.txt create mode 100644 acceptance/bundle/validate/for_each_task_retry_fields/script create mode 100644 acceptance/bundle/validate/for_each_task_retry_fields/test.py create mode 100644 acceptance/bundle/validate/for_each_task_valid/databricks.yml create mode 100644 acceptance/bundle/validate/for_each_task_valid/out.test.toml create mode 100644 acceptance/bundle/validate/for_each_task_valid/output.txt create mode 100644 acceptance/bundle/validate/for_each_task_valid/script create mode 100644 acceptance/bundle/validate/for_each_task_valid/test.py diff --git a/acceptance/bundle/validate/for_each_task_max_retries/databricks.yml b/acceptance/bundle/validate/for_each_task_max_retries/databricks.yml new file mode 100644 index 00000000000..dfe07e44844 --- /dev/null +++ b/acceptance/bundle/validate/for_each_task_max_retries/databricks.yml @@ -0,0 +1,16 @@ +bundle: + name: "for_each_task_validation" + +resources: + jobs: + test_job: + name: "Test Job" + tasks: + - task_key: "parent_task" + for_each_task: + inputs: "[1, 2, 3]" + task: + task_key: "child_task" + notebook_task: + notebook_path: "test.py" + max_retries: 3 diff --git a/acceptance/bundle/validate/for_each_task_max_retries/out.test.toml b/acceptance/bundle/validate/for_each_task_max_retries/out.test.toml new file mode 100644 index 00000000000..d560f1de043 --- /dev/null +++ b/acceptance/bundle/validate/for_each_task_max_retries/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/for_each_task_max_retries/output.txt b/acceptance/bundle/validate/for_each_task_max_retries/output.txt new file mode 100644 index 00000000000..d2f057650c4 --- /dev/null +++ b/acceptance/bundle/validate/for_each_task_max_retries/output.txt @@ -0,0 +1,16 @@ +Error: Invalid max_retries configuration for for_each_task + at resources.jobs.test_job.tasks[0] + in databricks.yml:9:11 + +Task "parent_task" has max_retries defined at the parent level, but it uses for_each_task. +When using for_each_task, max_retries must be defined on the nested task (for_each_task.task.max_retries), not on the parent task. + +Name: for_each_task_validation +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/for_each_task_validation/default + +Found 1 error + +Exit code: 1 diff --git a/acceptance/bundle/validate/for_each_task_max_retries/script b/acceptance/bundle/validate/for_each_task_max_retries/script new file mode 100644 index 00000000000..d286e548635 --- /dev/null +++ b/acceptance/bundle/validate/for_each_task_max_retries/script @@ -0,0 +1,2 @@ +#!/bin/bash +errcode $CLI bundle validate diff --git a/acceptance/bundle/validate/for_each_task_max_retries/test.py b/acceptance/bundle/validate/for_each_task_max_retries/test.py new file mode 100644 index 00000000000..4d1620a14e1 --- /dev/null +++ b/acceptance/bundle/validate/for_each_task_max_retries/test.py @@ -0,0 +1,2 @@ +# Databricks notebook source +print("test") diff --git a/acceptance/bundle/validate/for_each_task_retry_fields/databricks.yml b/acceptance/bundle/validate/for_each_task_retry_fields/databricks.yml new file mode 100644 index 00000000000..69ea50df597 --- /dev/null +++ b/acceptance/bundle/validate/for_each_task_retry_fields/databricks.yml @@ -0,0 +1,18 @@ +bundle: + name: "for_each_task_validation" + +resources: + jobs: + test_job: + name: "Test Job" + tasks: + - task_key: "parent_task" + for_each_task: + inputs: "[1, 2, 3]" + task: + task_key: "child_task" + notebook_task: + notebook_path: "test.py" + max_retries: 3 + min_retry_interval_millis: 1000 + retry_on_timeout: true diff --git a/acceptance/bundle/validate/for_each_task_retry_fields/out.test.toml b/acceptance/bundle/validate/for_each_task_retry_fields/out.test.toml new file mode 100644 index 00000000000..d560f1de043 --- /dev/null +++ b/acceptance/bundle/validate/for_each_task_retry_fields/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/for_each_task_retry_fields/output.txt b/acceptance/bundle/validate/for_each_task_retry_fields/output.txt new file mode 100644 index 00000000000..311d8559476 --- /dev/null +++ b/acceptance/bundle/validate/for_each_task_retry_fields/output.txt @@ -0,0 +1,30 @@ +Error: Invalid max_retries configuration for for_each_task + at resources.jobs.test_job.tasks[0] + in databricks.yml:9:11 + +Task "parent_task" has max_retries defined at the parent level, but it uses for_each_task. +When using for_each_task, max_retries must be defined on the nested task (for_each_task.task.max_retries), not on the parent task. + +Warning: Invalid min_retry_interval_millis configuration for for_each_task + at resources.jobs.test_job.tasks[0] + in databricks.yml:9:11 + +Task "parent_task" has min_retry_interval_millis defined at the parent level, but it uses for_each_task. +When using for_each_task, min_retry_interval_millis must be defined on the nested task (for_each_task.task.min_retry_interval_millis), not on the parent task. + +Warning: Invalid retry_on_timeout configuration for for_each_task + at resources.jobs.test_job.tasks[0] + in databricks.yml:9:11 + +Task "parent_task" has retry_on_timeout defined at the parent level, but it uses for_each_task. +When using for_each_task, retry_on_timeout must be defined on the nested task (for_each_task.task.retry_on_timeout), not on the parent task. + +Name: for_each_task_validation +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/for_each_task_validation/default + +Found 1 error and 2 warnings + +Exit code: 1 diff --git a/acceptance/bundle/validate/for_each_task_retry_fields/script b/acceptance/bundle/validate/for_each_task_retry_fields/script new file mode 100644 index 00000000000..d286e548635 --- /dev/null +++ b/acceptance/bundle/validate/for_each_task_retry_fields/script @@ -0,0 +1,2 @@ +#!/bin/bash +errcode $CLI bundle validate diff --git a/acceptance/bundle/validate/for_each_task_retry_fields/test.py b/acceptance/bundle/validate/for_each_task_retry_fields/test.py new file mode 100644 index 00000000000..4d1620a14e1 --- /dev/null +++ b/acceptance/bundle/validate/for_each_task_retry_fields/test.py @@ -0,0 +1,2 @@ +# Databricks notebook source +print("test") diff --git a/acceptance/bundle/validate/for_each_task_valid/databricks.yml b/acceptance/bundle/validate/for_each_task_valid/databricks.yml new file mode 100644 index 00000000000..a1dbd9aa56a --- /dev/null +++ b/acceptance/bundle/validate/for_each_task_valid/databricks.yml @@ -0,0 +1,16 @@ +bundle: + name: "for_each_task_validation" + +resources: + jobs: + test_job: + name: "Test Job" + tasks: + - task_key: "parent_task" + for_each_task: + inputs: "[1, 2, 3]" + task: + task_key: "child_task" + max_retries: 3 + notebook_task: + notebook_path: "test.py" diff --git a/acceptance/bundle/validate/for_each_task_valid/out.test.toml b/acceptance/bundle/validate/for_each_task_valid/out.test.toml new file mode 100644 index 00000000000..d560f1de043 --- /dev/null +++ b/acceptance/bundle/validate/for_each_task_valid/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/for_each_task_valid/output.txt b/acceptance/bundle/validate/for_each_task_valid/output.txt new file mode 100644 index 00000000000..65034254815 --- /dev/null +++ b/acceptance/bundle/validate/for_each_task_valid/output.txt @@ -0,0 +1,7 @@ +Name: for_each_task_validation +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/for_each_task_validation/default + +Validation OK! diff --git a/acceptance/bundle/validate/for_each_task_valid/script b/acceptance/bundle/validate/for_each_task_valid/script new file mode 100644 index 00000000000..c57053d9d05 --- /dev/null +++ b/acceptance/bundle/validate/for_each_task_valid/script @@ -0,0 +1,2 @@ +#!/bin/bash +$CLI bundle validate diff --git a/acceptance/bundle/validate/for_each_task_valid/test.py b/acceptance/bundle/validate/for_each_task_valid/test.py new file mode 100644 index 00000000000..4d1620a14e1 --- /dev/null +++ b/acceptance/bundle/validate/for_each_task_valid/test.py @@ -0,0 +1,2 @@ +# Databricks notebook source +print("test") From aa0c72cf3fcb2c9e5fc4f9627d1226e3aaabeb3f Mon Sep 17 00:00:00 2001 From: idelafuente-oc Date: Wed, 10 Dec 2025 12:21:14 -0300 Subject: [PATCH 4/7] refactor: simplifies unit test --- bundle/config/validate/for_each_task_test.go | 147 +++++-------------- 1 file changed, 36 insertions(+), 111 deletions(-) diff --git a/bundle/config/validate/for_each_task_test.go b/bundle/config/validate/for_each_task_test.go index 350da097697..c797db5b2ad 100644 --- a/bundle/config/validate/for_each_task_test.go +++ b/bundle/config/validate/for_each_task_test.go @@ -47,88 +47,55 @@ func createBundleWithForEachTask(parentTask jobs.Task) *bundle.Bundle { return b } -func TestForEachTask_InvalidRetryFields(t *testing.T) { - tests := []struct { - name string - task jobs.Task - expectedSeverity diag.Severity - expectedSummary string - expectedDetail string - }{ - { - name: "max_retries on parent", - task: jobs.Task{ - TaskKey: "parent_task", - MaxRetries: 3, - }, - expectedSeverity: diag.Error, - expectedSummary: "Invalid max_retries configuration for for_each_task", - expectedDetail: "max_retries must be defined on the nested task", - }, - { - name: "min_retry_interval_millis on parent", - task: jobs.Task{ - TaskKey: "parent_task", - MinRetryIntervalMillis: 1000, - }, - expectedSeverity: diag.Warning, - expectedSummary: "Invalid min_retry_interval_millis configuration for for_each_task", - expectedDetail: "min_retry_interval_millis must be defined on the nested task", - }, - { - name: "retry_on_timeout on parent", - task: jobs.Task{ - TaskKey: "parent_task", - RetryOnTimeout: true, - }, - expectedSeverity: diag.Warning, - expectedSummary: "Invalid retry_on_timeout configuration for for_each_task", - expectedDetail: "retry_on_timeout must be defined on the nested task", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ctx := context.Background() - b := createBundleWithForEachTask(tt.task) +func TestForEachTask_MaxRetriesError(t *testing.T) { + ctx := context.Background() + b := createBundleWithForEachTask(jobs.Task{ + TaskKey: "parent_task", + MaxRetries: 3, + }) - diags := ForEachTask().Apply(ctx, b) + diags := ForEachTask().Apply(ctx, b) - require.Len(t, diags, 1) - assert.Equal(t, tt.expectedSeverity, diags[0].Severity) - assert.Equal(t, tt.expectedSummary, diags[0].Summary) - assert.Contains(t, diags[0].Detail, tt.expectedDetail) - }) - } + require.Len(t, diags, 1) + assert.Equal(t, diag.Error, diags[0].Severity) + assert.Equal(t, "Invalid max_retries configuration for for_each_task", diags[0].Summary) + assert.Contains(t, diags[0].Detail, `Task "parent_task" has max_retries defined at the parent level`) + assert.Contains(t, diags[0].Detail, "for_each_task.task.max_retries") } -func TestForEachTask_MultipleRetryFieldsOnParent(t *testing.T) { +func TestForEachTask_MinRetryIntervalWarning(t *testing.T) { ctx := context.Background() b := createBundleWithForEachTask(jobs.Task{ TaskKey: "parent_task", - MaxRetries: 3, MinRetryIntervalMillis: 1000, - RetryOnTimeout: true, }) diags := ForEachTask().Apply(ctx, b) - require.Len(t, diags, 3) - - errorCount := 0 - warningCount := 0 - for _, d := range diags { - switch d.Severity { - case diag.Error: - errorCount++ - case diag.Warning: - warningCount++ - } - } - assert.Equal(t, 1, errorCount) - assert.Equal(t, 2, warningCount) + + require.Len(t, diags, 1) + assert.Equal(t, diag.Warning, diags[0].Severity) + assert.Equal(t, "Invalid min_retry_interval_millis configuration for for_each_task", diags[0].Summary) + assert.Contains(t, diags[0].Detail, `Task "parent_task" has min_retry_interval_millis defined at the parent level`) + assert.Contains(t, diags[0].Detail, "for_each_task.task.min_retry_interval_millis") +} + +func TestForEachTask_RetryOnTimeoutWarning(t *testing.T) { + ctx := context.Background() + b := createBundleWithForEachTask(jobs.Task{ + TaskKey: "parent_task", + RetryOnTimeout: true, + }) + + diags := ForEachTask().Apply(ctx, b) + + require.Len(t, diags, 1) + assert.Equal(t, diag.Warning, diags[0].Severity) + assert.Equal(t, "Invalid retry_on_timeout configuration for for_each_task", diags[0].Summary) + assert.Contains(t, diags[0].Detail, `Task "parent_task" has retry_on_timeout defined at the parent level`) + assert.Contains(t, diags[0].Detail, "for_each_task.task.retry_on_timeout") } -func TestForEachTask_ValidConfigurationOnChild(t *testing.T) { +func TestForEachTask_ValidConfiguration(t *testing.T) { ctx := context.Background() b := createBundleWithForEachTask(jobs.Task{ TaskKey: "parent_task", @@ -147,45 +114,3 @@ func TestForEachTask_ValidConfigurationOnChild(t *testing.T) { diags := ForEachTask().Apply(ctx, b) assert.Empty(t, diags) } - -func TestForEachTask_NoForEachTask(t *testing.T) { - ctx := context.Background() - b := &bundle.Bundle{ - Config: config.Root{ - Resources: config.Resources{ - Jobs: map[string]*resources.Job{ - "job1": { - JobSettings: jobs.JobSettings{ - Name: "My Job", - Tasks: []jobs.Task{ - { - TaskKey: "simple_task", - MaxRetries: 3, - NotebookTask: &jobs.NotebookTask{ - NotebookPath: "test.py", - }, - }, - }, - }, - }, - }, - }, - }, - } - - bundletest.SetLocation(b, "resources.jobs.job1.tasks[0]", []dyn.Location{{File: "job.yml", Line: 1, Column: 1}}) - - diags := ForEachTask().Apply(ctx, b) - assert.Empty(t, diags) -} - -func TestForEachTask_RetryOnTimeoutFalse(t *testing.T) { - ctx := context.Background() - b := createBundleWithForEachTask(jobs.Task{ - TaskKey: "parent_task", - RetryOnTimeout: false, - }) - - diags := ForEachTask().Apply(ctx, b) - assert.Empty(t, diags) -} From 8d36728ba122dcc58f22df4a5c0ba03f2b94aa31 Mon Sep 17 00:00:00 2001 From: idelafuente-oc Date: Thu, 11 Dec 2025 14:28:33 -0300 Subject: [PATCH 5/7] revert: deletes unit test --- bundle/config/validate/for_each_task_test.go | 116 ------------------- 1 file changed, 116 deletions(-) delete mode 100644 bundle/config/validate/for_each_task_test.go diff --git a/bundle/config/validate/for_each_task_test.go b/bundle/config/validate/for_each_task_test.go deleted file mode 100644 index c797db5b2ad..00000000000 --- a/bundle/config/validate/for_each_task_test.go +++ /dev/null @@ -1,116 +0,0 @@ -package validate - -import ( - "context" - "testing" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/bundle/internal/bundletest" - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/databricks-sdk-go/service/jobs" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func createBundleWithForEachTask(parentTask jobs.Task) *bundle.Bundle { - if parentTask.ForEachTask == nil { - parentTask.ForEachTask = &jobs.ForEachTask{ - Inputs: "[1, 2, 3]", - Task: jobs.Task{ - TaskKey: "child_task", - NotebookTask: &jobs.NotebookTask{ - NotebookPath: "test.py", - }, - }, - } - } - - b := &bundle.Bundle{ - Config: config.Root{ - Resources: config.Resources{ - Jobs: map[string]*resources.Job{ - "job1": { - JobSettings: jobs.JobSettings{ - Name: "My Job", - Tasks: []jobs.Task{parentTask}, - }, - }, - }, - }, - }, - } - - bundletest.SetLocation(b, "resources.jobs.job1.tasks[0]", []dyn.Location{{File: "job.yml", Line: 1, Column: 1}}) - return b -} - -func TestForEachTask_MaxRetriesError(t *testing.T) { - ctx := context.Background() - b := createBundleWithForEachTask(jobs.Task{ - TaskKey: "parent_task", - MaxRetries: 3, - }) - - diags := ForEachTask().Apply(ctx, b) - - require.Len(t, diags, 1) - assert.Equal(t, diag.Error, diags[0].Severity) - assert.Equal(t, "Invalid max_retries configuration for for_each_task", diags[0].Summary) - assert.Contains(t, diags[0].Detail, `Task "parent_task" has max_retries defined at the parent level`) - assert.Contains(t, diags[0].Detail, "for_each_task.task.max_retries") -} - -func TestForEachTask_MinRetryIntervalWarning(t *testing.T) { - ctx := context.Background() - b := createBundleWithForEachTask(jobs.Task{ - TaskKey: "parent_task", - MinRetryIntervalMillis: 1000, - }) - - diags := ForEachTask().Apply(ctx, b) - - require.Len(t, diags, 1) - assert.Equal(t, diag.Warning, diags[0].Severity) - assert.Equal(t, "Invalid min_retry_interval_millis configuration for for_each_task", diags[0].Summary) - assert.Contains(t, diags[0].Detail, `Task "parent_task" has min_retry_interval_millis defined at the parent level`) - assert.Contains(t, diags[0].Detail, "for_each_task.task.min_retry_interval_millis") -} - -func TestForEachTask_RetryOnTimeoutWarning(t *testing.T) { - ctx := context.Background() - b := createBundleWithForEachTask(jobs.Task{ - TaskKey: "parent_task", - RetryOnTimeout: true, - }) - - diags := ForEachTask().Apply(ctx, b) - - require.Len(t, diags, 1) - assert.Equal(t, diag.Warning, diags[0].Severity) - assert.Equal(t, "Invalid retry_on_timeout configuration for for_each_task", diags[0].Summary) - assert.Contains(t, diags[0].Detail, `Task "parent_task" has retry_on_timeout defined at the parent level`) - assert.Contains(t, diags[0].Detail, "for_each_task.task.retry_on_timeout") -} - -func TestForEachTask_ValidConfiguration(t *testing.T) { - ctx := context.Background() - b := createBundleWithForEachTask(jobs.Task{ - TaskKey: "parent_task", - ForEachTask: &jobs.ForEachTask{ - Inputs: "[1, 2, 3]", - Task: jobs.Task{ - TaskKey: "child_task", - MaxRetries: 3, - NotebookTask: &jobs.NotebookTask{ - NotebookPath: "test.py", - }, - }, - }, - }) - - diags := ForEachTask().Apply(ctx, b) - assert.Empty(t, diags) -} From dd98f35254ee28504a2b6dd5414fb64d127f11ca Mon Sep 17 00:00:00 2001 From: idelafuente-oc Date: Wed, 11 Nov 2026 10:57:41 -0300 Subject: [PATCH 6/7] Address for-each task validation review feedback Clarify API-backed retry diagnostics, use field-level locations, consolidate acceptance coverage, and add a changelog fragment. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .nextchanges/bundles/4104.md | 1 + .../validate/for_each_task/databricks.yml | 54 +++++++++++++++++++ .../validate/for_each_task/out.test.toml | 2 + .../bundle/validate/for_each_task/output.txt | 49 +++++++++++++++++ .../bundle/validate/for_each_task/script | 3 ++ .../test.py | 0 .../for_each_task_max_retries/databricks.yml | 16 ------ .../for_each_task_max_retries/out.test.toml | 5 -- .../for_each_task_max_retries/output.txt | 16 ------ .../validate/for_each_task_max_retries/script | 2 - .../for_each_task_retry_fields/databricks.yml | 18 ------- .../for_each_task_retry_fields/out.test.toml | 5 -- .../for_each_task_retry_fields/output.txt | 30 ----------- .../for_each_task_retry_fields/script | 2 - .../for_each_task_retry_fields/test.py | 2 - .../for_each_task_valid/databricks.yml | 16 ------ .../for_each_task_valid/out.test.toml | 5 -- .../validate/for_each_task_valid/output.txt | 7 --- .../validate/for_each_task_valid/script | 2 - .../validate/for_each_task_valid/test.py | 2 - bundle/config/validate/for_each_task.go | 37 ++++++------- 21 files changed, 126 insertions(+), 148 deletions(-) create mode 100644 .nextchanges/bundles/4104.md create mode 100644 acceptance/bundle/validate/for_each_task/databricks.yml create mode 100644 acceptance/bundle/validate/for_each_task/out.test.toml create mode 100644 acceptance/bundle/validate/for_each_task/output.txt create mode 100644 acceptance/bundle/validate/for_each_task/script rename acceptance/bundle/validate/{for_each_task_max_retries => for_each_task}/test.py (100%) delete mode 100644 acceptance/bundle/validate/for_each_task_max_retries/databricks.yml delete mode 100644 acceptance/bundle/validate/for_each_task_max_retries/out.test.toml delete mode 100644 acceptance/bundle/validate/for_each_task_max_retries/output.txt delete mode 100644 acceptance/bundle/validate/for_each_task_max_retries/script delete mode 100644 acceptance/bundle/validate/for_each_task_retry_fields/databricks.yml delete mode 100644 acceptance/bundle/validate/for_each_task_retry_fields/out.test.toml delete mode 100644 acceptance/bundle/validate/for_each_task_retry_fields/output.txt delete mode 100644 acceptance/bundle/validate/for_each_task_retry_fields/script delete mode 100644 acceptance/bundle/validate/for_each_task_retry_fields/test.py delete mode 100644 acceptance/bundle/validate/for_each_task_valid/databricks.yml delete mode 100644 acceptance/bundle/validate/for_each_task_valid/out.test.toml delete mode 100644 acceptance/bundle/validate/for_each_task_valid/output.txt delete mode 100644 acceptance/bundle/validate/for_each_task_valid/script delete mode 100644 acceptance/bundle/validate/for_each_task_valid/test.py diff --git a/.nextchanges/bundles/4104.md b/.nextchanges/bundles/4104.md new file mode 100644 index 00000000000..8abed8ef989 --- /dev/null +++ b/.nextchanges/bundles/4104.md @@ -0,0 +1 @@ +* Validate retry settings on `for_each_task` parent tasks, reporting unsupported retries as errors and ignored retry settings as warnings. ([#4104](https://github.com/databricks/cli/pull/4104)) diff --git a/acceptance/bundle/validate/for_each_task/databricks.yml b/acceptance/bundle/validate/for_each_task/databricks.yml new file mode 100644 index 00000000000..65c5ff83b39 --- /dev/null +++ b/acceptance/bundle/validate/for_each_task/databricks.yml @@ -0,0 +1,54 @@ +bundle: + name: "for_each_task_validation" + +resources: + jobs: + test_job: + name: "Test Job" + tasks: + - task_key: "parent_task" + for_each_task: + inputs: "[1, 2, 3]" + task: + task_key: "child_task" + notebook_task: + notebook_path: "test.py" + +targets: + max_retries: + resources: + jobs: + test_job: + tasks: + - task_key: "parent_task" + max_retries: 3 + + retry_fields: + resources: + jobs: + test_job: + tasks: + - task_key: "parent_task" + min_retry_interval_millis: 1000 + retry_on_timeout: true + + valid: + resources: + jobs: + test_job: + tasks: + - task_key: "parent_task" + max_retries: 0 + min_retry_interval_millis: 0 + retry_on_timeout: false + for_each_task: + task: + max_retries: 3 + min_retry_interval_millis: 1000 + retry_on_timeout: true + - task_key: "regular_task" + max_retries: 3 + min_retry_interval_millis: 1000 + retry_on_timeout: true + notebook_task: + notebook_path: "test.py" diff --git a/acceptance/bundle/validate/for_each_task/out.test.toml b/acceptance/bundle/validate/for_each_task/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/validate/for_each_task/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/validate/for_each_task/output.txt b/acceptance/bundle/validate/for_each_task/output.txt new file mode 100644 index 00000000000..e8b0bb25dfa --- /dev/null +++ b/acceptance/bundle/validate/for_each_task/output.txt @@ -0,0 +1,49 @@ + +>>> [CLI] bundle validate -t max_retries +Error: Invalid max_retries configuration for for_each_task + at resources.jobs.test_job.tasks[0].max_retries + in databricks.yml:24:28 + +When using for_each_task, max_retries must be defined on the nested task (for_each_task.task.max_retries), not on the parent task. + +Name: for_each_task_validation +Target: max_retries +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/for_each_task_validation/max_retries + +Found 1 error + +Exit code: 1 + +>>> [CLI] bundle validate -t retry_fields +Warning: Invalid min_retry_interval_millis configuration for for_each_task + at resources.jobs.test_job.tasks[0].min_retry_interval_millis + in databricks.yml:32:42 + +min_retry_interval_millis is ignored on a for_each_task parent task. +Define it on the nested task (for_each_task.task.min_retry_interval_millis) instead. + +Warning: Invalid retry_on_timeout configuration for for_each_task + at resources.jobs.test_job.tasks[0].retry_on_timeout + in databricks.yml:33:33 + +retry_on_timeout is ignored on a for_each_task parent task. +Define it on the nested task (for_each_task.task.retry_on_timeout) instead. + +Name: for_each_task_validation +Target: retry_fields +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/for_each_task_validation/retry_fields + +Found 2 warnings + +>>> [CLI] bundle validate -t valid +Name: for_each_task_validation +Target: valid +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/for_each_task_validation/valid + +Validation OK! diff --git a/acceptance/bundle/validate/for_each_task/script b/acceptance/bundle/validate/for_each_task/script new file mode 100644 index 00000000000..6e7edeeafa6 --- /dev/null +++ b/acceptance/bundle/validate/for_each_task/script @@ -0,0 +1,3 @@ +errcode trace $CLI bundle validate -t max_retries +trace $CLI bundle validate -t retry_fields +trace $CLI bundle validate -t valid diff --git a/acceptance/bundle/validate/for_each_task_max_retries/test.py b/acceptance/bundle/validate/for_each_task/test.py similarity index 100% rename from acceptance/bundle/validate/for_each_task_max_retries/test.py rename to acceptance/bundle/validate/for_each_task/test.py diff --git a/acceptance/bundle/validate/for_each_task_max_retries/databricks.yml b/acceptance/bundle/validate/for_each_task_max_retries/databricks.yml deleted file mode 100644 index dfe07e44844..00000000000 --- a/acceptance/bundle/validate/for_each_task_max_retries/databricks.yml +++ /dev/null @@ -1,16 +0,0 @@ -bundle: - name: "for_each_task_validation" - -resources: - jobs: - test_job: - name: "Test Job" - tasks: - - task_key: "parent_task" - for_each_task: - inputs: "[1, 2, 3]" - task: - task_key: "child_task" - notebook_task: - notebook_path: "test.py" - max_retries: 3 diff --git a/acceptance/bundle/validate/for_each_task_max_retries/out.test.toml b/acceptance/bundle/validate/for_each_task_max_retries/out.test.toml deleted file mode 100644 index d560f1de043..00000000000 --- a/acceptance/bundle/validate/for_each_task_max_retries/out.test.toml +++ /dev/null @@ -1,5 +0,0 @@ -Local = true -Cloud = false - -[EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/for_each_task_max_retries/output.txt b/acceptance/bundle/validate/for_each_task_max_retries/output.txt deleted file mode 100644 index d2f057650c4..00000000000 --- a/acceptance/bundle/validate/for_each_task_max_retries/output.txt +++ /dev/null @@ -1,16 +0,0 @@ -Error: Invalid max_retries configuration for for_each_task - at resources.jobs.test_job.tasks[0] - in databricks.yml:9:11 - -Task "parent_task" has max_retries defined at the parent level, but it uses for_each_task. -When using for_each_task, max_retries must be defined on the nested task (for_each_task.task.max_retries), not on the parent task. - -Name: for_each_task_validation -Target: default -Workspace: - User: [USERNAME] - Path: /Workspace/Users/[USERNAME]/.bundle/for_each_task_validation/default - -Found 1 error - -Exit code: 1 diff --git a/acceptance/bundle/validate/for_each_task_max_retries/script b/acceptance/bundle/validate/for_each_task_max_retries/script deleted file mode 100644 index d286e548635..00000000000 --- a/acceptance/bundle/validate/for_each_task_max_retries/script +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -errcode $CLI bundle validate diff --git a/acceptance/bundle/validate/for_each_task_retry_fields/databricks.yml b/acceptance/bundle/validate/for_each_task_retry_fields/databricks.yml deleted file mode 100644 index 69ea50df597..00000000000 --- a/acceptance/bundle/validate/for_each_task_retry_fields/databricks.yml +++ /dev/null @@ -1,18 +0,0 @@ -bundle: - name: "for_each_task_validation" - -resources: - jobs: - test_job: - name: "Test Job" - tasks: - - task_key: "parent_task" - for_each_task: - inputs: "[1, 2, 3]" - task: - task_key: "child_task" - notebook_task: - notebook_path: "test.py" - max_retries: 3 - min_retry_interval_millis: 1000 - retry_on_timeout: true diff --git a/acceptance/bundle/validate/for_each_task_retry_fields/out.test.toml b/acceptance/bundle/validate/for_each_task_retry_fields/out.test.toml deleted file mode 100644 index d560f1de043..00000000000 --- a/acceptance/bundle/validate/for_each_task_retry_fields/out.test.toml +++ /dev/null @@ -1,5 +0,0 @@ -Local = true -Cloud = false - -[EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/for_each_task_retry_fields/output.txt b/acceptance/bundle/validate/for_each_task_retry_fields/output.txt deleted file mode 100644 index 311d8559476..00000000000 --- a/acceptance/bundle/validate/for_each_task_retry_fields/output.txt +++ /dev/null @@ -1,30 +0,0 @@ -Error: Invalid max_retries configuration for for_each_task - at resources.jobs.test_job.tasks[0] - in databricks.yml:9:11 - -Task "parent_task" has max_retries defined at the parent level, but it uses for_each_task. -When using for_each_task, max_retries must be defined on the nested task (for_each_task.task.max_retries), not on the parent task. - -Warning: Invalid min_retry_interval_millis configuration for for_each_task - at resources.jobs.test_job.tasks[0] - in databricks.yml:9:11 - -Task "parent_task" has min_retry_interval_millis defined at the parent level, but it uses for_each_task. -When using for_each_task, min_retry_interval_millis must be defined on the nested task (for_each_task.task.min_retry_interval_millis), not on the parent task. - -Warning: Invalid retry_on_timeout configuration for for_each_task - at resources.jobs.test_job.tasks[0] - in databricks.yml:9:11 - -Task "parent_task" has retry_on_timeout defined at the parent level, but it uses for_each_task. -When using for_each_task, retry_on_timeout must be defined on the nested task (for_each_task.task.retry_on_timeout), not on the parent task. - -Name: for_each_task_validation -Target: default -Workspace: - User: [USERNAME] - Path: /Workspace/Users/[USERNAME]/.bundle/for_each_task_validation/default - -Found 1 error and 2 warnings - -Exit code: 1 diff --git a/acceptance/bundle/validate/for_each_task_retry_fields/script b/acceptance/bundle/validate/for_each_task_retry_fields/script deleted file mode 100644 index d286e548635..00000000000 --- a/acceptance/bundle/validate/for_each_task_retry_fields/script +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -errcode $CLI bundle validate diff --git a/acceptance/bundle/validate/for_each_task_retry_fields/test.py b/acceptance/bundle/validate/for_each_task_retry_fields/test.py deleted file mode 100644 index 4d1620a14e1..00000000000 --- a/acceptance/bundle/validate/for_each_task_retry_fields/test.py +++ /dev/null @@ -1,2 +0,0 @@ -# Databricks notebook source -print("test") diff --git a/acceptance/bundle/validate/for_each_task_valid/databricks.yml b/acceptance/bundle/validate/for_each_task_valid/databricks.yml deleted file mode 100644 index a1dbd9aa56a..00000000000 --- a/acceptance/bundle/validate/for_each_task_valid/databricks.yml +++ /dev/null @@ -1,16 +0,0 @@ -bundle: - name: "for_each_task_validation" - -resources: - jobs: - test_job: - name: "Test Job" - tasks: - - task_key: "parent_task" - for_each_task: - inputs: "[1, 2, 3]" - task: - task_key: "child_task" - max_retries: 3 - notebook_task: - notebook_path: "test.py" diff --git a/acceptance/bundle/validate/for_each_task_valid/out.test.toml b/acceptance/bundle/validate/for_each_task_valid/out.test.toml deleted file mode 100644 index d560f1de043..00000000000 --- a/acceptance/bundle/validate/for_each_task_valid/out.test.toml +++ /dev/null @@ -1,5 +0,0 @@ -Local = true -Cloud = false - -[EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/for_each_task_valid/output.txt b/acceptance/bundle/validate/for_each_task_valid/output.txt deleted file mode 100644 index 65034254815..00000000000 --- a/acceptance/bundle/validate/for_each_task_valid/output.txt +++ /dev/null @@ -1,7 +0,0 @@ -Name: for_each_task_validation -Target: default -Workspace: - User: [USERNAME] - Path: /Workspace/Users/[USERNAME]/.bundle/for_each_task_validation/default - -Validation OK! diff --git a/acceptance/bundle/validate/for_each_task_valid/script b/acceptance/bundle/validate/for_each_task_valid/script deleted file mode 100644 index c57053d9d05..00000000000 --- a/acceptance/bundle/validate/for_each_task_valid/script +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -$CLI bundle validate diff --git a/acceptance/bundle/validate/for_each_task_valid/test.py b/acceptance/bundle/validate/for_each_task_valid/test.py deleted file mode 100644 index 4d1620a14e1..00000000000 --- a/acceptance/bundle/validate/for_each_task_valid/test.py +++ /dev/null @@ -1,2 +0,0 @@ -# Databricks notebook source -print("test") diff --git a/bundle/config/validate/for_each_task.go b/bundle/config/validate/for_each_task.go index c1d68be8b5a..6e27776a268 100644 --- a/bundle/config/validate/for_each_task.go +++ b/bundle/config/validate/for_each_task.go @@ -6,11 +6,11 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/service/jobs" ) -// ForEachTask validates constraints for for_each_task configuration +// ForEachTask reports retry settings set on a for_each_task parent instead of its nested task. func ForEachTask() bundle.ReadOnlyMutator { return &forEachTask{} } @@ -24,15 +24,10 @@ func (v *forEachTask) Name() string { func (v *forEachTask) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { diags := diag.Diagnostics{} - jobsPath := dyn.NewPath(dyn.Key("resources"), dyn.Key("jobs")) - for resourceName, job := range b.Config.Resources.Jobs { - resourcePath := jobsPath.Append(dyn.Key(resourceName)) - for taskIndex, task := range job.Tasks { - taskPath := resourcePath.Append(dyn.Key("tasks"), dyn.Index(taskIndex)) - if task.ForEachTask != nil { + taskPath := structpath.NewPath(nil, "resources", "jobs", resourceName, "tasks", taskIndex) diags = diags.Extend(validateForEachTask(b, task, taskPath)) } } @@ -41,36 +36,38 @@ func (v *forEachTask) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnost return diags } -func validateForEachTask(b *bundle.Bundle, task jobs.Task, taskPath dyn.Path) diag.Diagnostics { +func validateForEachTask(b *bundle.Bundle, task jobs.Task, taskPath *structpath.PathNode) diag.Diagnostics { diags := diag.Diagnostics{} + // The Jobs API rejects nonzero max_retries on the parent. It accepts the other + // retry settings, but they have no effect because the parent cannot be retried. if task.MaxRetries != 0 { - diags = diags.Append(invalidRetryFieldDiag(b, task, taskPath, "max_retries", diag.Error)) + diags = diags.Append(invalidRetryFieldDiag(b, taskPath, "max_retries", diag.Error)) } if task.MinRetryIntervalMillis != 0 { - diags = diags.Append(invalidRetryFieldDiag(b, task, taskPath, "min_retry_interval_millis", diag.Warning)) + diags = diags.Append(invalidRetryFieldDiag(b, taskPath, "min_retry_interval_millis", diag.Warning)) } if task.RetryOnTimeout { - diags = diags.Append(invalidRetryFieldDiag(b, task, taskPath, "retry_on_timeout", diag.Warning)) + diags = diags.Append(invalidRetryFieldDiag(b, taskPath, "retry_on_timeout", diag.Warning)) } return diags } -func invalidRetryFieldDiag(b *bundle.Bundle, task jobs.Task, taskPath dyn.Path, fieldName string, severity diag.Severity) diag.Diagnostic { - detail := fmt.Sprintf( - "Task %q has %s defined at the parent level, but it uses for_each_task.\n"+ - "When using for_each_task, %s must be defined on the nested task (for_each_task.task.%s), not on the parent task.", - task.TaskKey, fieldName, fieldName, fieldName, - ) +func invalidRetryFieldDiag(b *bundle.Bundle, taskPath *structpath.PathNode, fieldName string, severity diag.Severity) diag.Diagnostic { + fieldPath := structpath.NewDotString(taskPath, fieldName) + detail := fmt.Sprintf("When using for_each_task, %s must be defined on the nested task (for_each_task.task.%s), not on the parent task.", fieldName, fieldName) + if severity == diag.Warning { + detail = fmt.Sprintf("%s is ignored on a for_each_task parent task.\nDefine it on the nested task (for_each_task.task.%s) instead.", fieldName, fieldName) + } return diag.Diagnostic{ Severity: severity, Summary: fmt.Sprintf("Invalid %s configuration for for_each_task", fieldName), Detail: detail, - Locations: b.Config.GetLocations(taskPath.String()), - Paths: []dyn.Path{taskPath}, + Locations: b.Config.GetLocationsOf(fieldPath), + Paths: []*structpath.PathNode{fieldPath}, } } From 2dbb22a4a0799fc7e5aab6039b999ab67dc463df Mon Sep 17 00:00:00 2001 From: idelafuente-oc Date: Wed, 11 Nov 2026 11:21:59 -0300 Subject: [PATCH 7/7] Clarify for-each task diagnostics Sort retry diagnostics by configuration path and describe unsupported or ineffective settings in their summaries. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../validate/for_each_task/databricks.yml | 14 +++++++++++++ .../bundle/validate/for_each_task/output.txt | 20 ++++++++++++------- bundle/config/validate/for_each_task.go | 11 +++++++++- 3 files changed, 37 insertions(+), 8 deletions(-) diff --git a/acceptance/bundle/validate/for_each_task/databricks.yml b/acceptance/bundle/validate/for_each_task/databricks.yml index 65c5ff83b39..f19549065bc 100644 --- a/acceptance/bundle/validate/for_each_task/databricks.yml +++ b/acceptance/bundle/validate/for_each_task/databricks.yml @@ -3,6 +3,16 @@ bundle: resources: jobs: + aaa_job: + name: "Job A" + tasks: + - task_key: "parent_task" + for_each_task: + inputs: "[1, 2, 3]" + task: + task_key: "child_task" + notebook_task: + notebook_path: "test.py" test_job: name: "Test Job" tasks: @@ -18,6 +28,10 @@ targets: max_retries: resources: jobs: + aaa_job: + tasks: + - task_key: "parent_task" + max_retries: 3 test_job: tasks: - task_key: "parent_task" diff --git a/acceptance/bundle/validate/for_each_task/output.txt b/acceptance/bundle/validate/for_each_task/output.txt index e8b0bb25dfa..9d5b65b9686 100644 --- a/acceptance/bundle/validate/for_each_task/output.txt +++ b/acceptance/bundle/validate/for_each_task/output.txt @@ -1,8 +1,14 @@ >>> [CLI] bundle validate -t max_retries -Error: Invalid max_retries configuration for for_each_task +Error: max_retries is not supported on a for_each_task parent task + at resources.jobs.aaa_job.tasks[0].max_retries + in databricks.yml:34:28 + +When using for_each_task, max_retries must be defined on the nested task (for_each_task.task.max_retries), not on the parent task. + +Error: max_retries is not supported on a for_each_task parent task at resources.jobs.test_job.tasks[0].max_retries - in databricks.yml:24:28 + in databricks.yml:38:28 When using for_each_task, max_retries must be defined on the nested task (for_each_task.task.max_retries), not on the parent task. @@ -12,21 +18,21 @@ Workspace: User: [USERNAME] Path: /Workspace/Users/[USERNAME]/.bundle/for_each_task_validation/max_retries -Found 1 error +Found 2 errors Exit code: 1 >>> [CLI] bundle validate -t retry_fields -Warning: Invalid min_retry_interval_millis configuration for for_each_task +Warning: min_retry_interval_millis has no effect on a for_each_task parent task at resources.jobs.test_job.tasks[0].min_retry_interval_millis - in databricks.yml:32:42 + in databricks.yml:46:42 min_retry_interval_millis is ignored on a for_each_task parent task. Define it on the nested task (for_each_task.task.min_retry_interval_millis) instead. -Warning: Invalid retry_on_timeout configuration for for_each_task +Warning: retry_on_timeout has no effect on a for_each_task parent task at resources.jobs.test_job.tasks[0].retry_on_timeout - in databricks.yml:33:33 + in databricks.yml:47:33 retry_on_timeout is ignored on a for_each_task parent task. Define it on the nested task (for_each_task.task.retry_on_timeout) instead. diff --git a/bundle/config/validate/for_each_task.go b/bundle/config/validate/for_each_task.go index 6e27776a268..1337fd78a85 100644 --- a/bundle/config/validate/for_each_task.go +++ b/bundle/config/validate/for_each_task.go @@ -1,8 +1,10 @@ package validate import ( + "cmp" "context" "fmt" + "slices" "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" @@ -33,6 +35,11 @@ func (v *forEachTask) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnost } } + // Map iteration order is randomized; sort by path for stable output. + slices.SortFunc(diags, func(x, y diag.Diagnostic) int { + return cmp.Compare(x.Paths[0].String(), y.Paths[0].String()) + }) + return diags } @@ -58,14 +65,16 @@ func validateForEachTask(b *bundle.Bundle, task jobs.Task, taskPath *structpath. func invalidRetryFieldDiag(b *bundle.Bundle, taskPath *structpath.PathNode, fieldName string, severity diag.Severity) diag.Diagnostic { fieldPath := structpath.NewDotString(taskPath, fieldName) + summary := fieldName + " is not supported on a for_each_task parent task" detail := fmt.Sprintf("When using for_each_task, %s must be defined on the nested task (for_each_task.task.%s), not on the parent task.", fieldName, fieldName) if severity == diag.Warning { + summary = fieldName + " has no effect on a for_each_task parent task" detail = fmt.Sprintf("%s is ignored on a for_each_task parent task.\nDefine it on the nested task (for_each_task.task.%s) instead.", fieldName, fieldName) } return diag.Diagnostic{ Severity: severity, - Summary: fmt.Sprintf("Invalid %s configuration for for_each_task", fieldName), + Summary: summary, Detail: detail, Locations: b.Config.GetLocationsOf(fieldPath), Paths: []*structpath.PathNode{fieldPath},