From f3d4e44060e4b553b40922b0f1fba5b0c9e43c3f Mon Sep 17 00:00:00 2001 From: Factory Local Trial Date: Tue, 3 Nov 2026 05:18:48 +0000 Subject: [PATCH 1/8] [factory] Fix qualified DABs behavior --- bundle/config/root.go | 36 ++++++++++++++++++++++++++ bundle/config/root_test.go | 52 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+) diff --git a/bundle/config/root.go b/bundle/config/root.go index a64b742630e..c400fd5e2ae 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -3,6 +3,7 @@ package config import ( "bytes" "context" + "encoding/json" "errors" "fmt" "os" @@ -15,6 +16,7 @@ import ( "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/convert" "github.com/databricks/cli/libs/dyn/dynloc" + "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/dyn/merge" "github.com/databricks/cli/libs/dyn/yamlloader" "github.com/databricks/cli/libs/log" @@ -130,6 +132,10 @@ func LoadFromBytes(path string, raw []byte) (*Root, diag.Diagnostics) { if err != nil { return nil, diag.Errorf("failed to rewrite %s: %v", path, err) } + v, err = rewriteForEachTaskInputs(v) + if err != nil { + return nil, diag.FromErr(fmt.Errorf("failed to rewrite for_each_task.inputs in %s: %w", path, err)) + } // Normalize dynamic configuration tree according to configuration type. v, diags := convert.Normalize(r, v) @@ -552,6 +558,36 @@ func rewriteShorthands(v dyn.Value) (dyn.Value, error) { })) } +func rewriteForEachTaskInputs(v dyn.Value) (dyn.Value, error) { + return dyn.Walk(v, func(p dyn.Path, input dyn.Value) (dyn.Value, error) { + if !isForEachTaskInputsPath(p) || input.Kind() != dyn.KindSequence { + return input, nil + } + + data, err := json.Marshal(input.AsAny()) + if err != nil { + return dyn.InvalidValue, fmt.Errorf("%s at %s: %w", p, input.Location(), err) + } + // Leave unresolved references for normal validation instead of interpolating into JSON text. + if dynvar.ContainsVariableReference(string(data)) { + return input, nil + } + return dyn.NewValue(string(data), input.Locations()), nil + }) +} + +func isForEachTaskInputsPath(p dyn.Path) bool { + if len(p) == 9 && (p[0] == dyn.Key("targets") || p[0] == dyn.Key("environments")) && p[1].Key() != "" { + p = p[2:] + } + return len(p) == 7 && + p.HasPrefix(dyn.NewPath(dyn.Key("resources"), dyn.Key("jobs"))) && + p[2].Key() != "" && + p[3] == dyn.Key("tasks") && + p[4].Key() == "" && + p.HasSuffix(dyn.NewPath(dyn.Key("for_each_task"), dyn.Key("inputs"))) +} + // validateVariableOverrides checks that all variables specified // in the target override are also defined in the root. func validateVariableOverrides(root, target dyn.Value) (err error) { diff --git a/bundle/config/root_test.go b/bundle/config/root_test.go index f735c44405e..f562e397992 100644 --- a/bundle/config/root_test.go +++ b/bundle/config/root_test.go @@ -31,6 +31,58 @@ func TestRootLoad(t *testing.T) { assert.Equal(t, "basic", root.Bundle.Name) } +func TestLoadForEachTaskInputs(t *testing.T) { + for _, tc := range []struct { + name string + inputs string + want string + }{ + {name: "YAML array", inputs: "[input_1, input_2]", want: `["input_1","input_2"]`}, + {name: "YAML object array", inputs: "[{name: input_1, enabled: true}]", want: `[{"enabled":true,"name":"input_1"}]`}, + {name: "empty YAML array", inputs: "[]", want: `[]`}, + {name: "JSON string", inputs: `'[ "input_1", "input_2" ]'`, want: `[ "input_1", "input_2" ]`}, + {name: "variable reference string", inputs: `${var.items}`, want: `${var.items}`}, + } { + t.Run(tc.name, func(t *testing.T) { + config := "bundle:\n name: foreach\nresources:\n jobs:\n fan:\n tasks:\n - task_key: fan\n for_each_task:\n inputs: " + tc.inputs + "\n task:\n task_key: inner\n" + root, diags := LoadFromBytes("databricks.yml", []byte(config)) + require.NoError(t, diags.Error()) + inputs := root.Resources.Jobs["fan"].Tasks[0].ForEachTask.Inputs + require.Equal(t, tc.want, inputs) + + body, err := json.Marshal(root.Resources.Jobs["fan"].Tasks[0].ForEachTask) + require.NoError(t, err) + var apiTask struct { + Inputs string `json:"inputs"` + } + require.NoError(t, json.Unmarshal(body, &apiTask)) + assert.Equal(t, tc.want, apiTask.Inputs) + }) + } +} + +func TestLoadForEachTaskInputsInTarget(t *testing.T) { + config := `bundle: + name: foreach +targets: + dev: + resources: + jobs: + fan: + tasks: + - task_key: fan + for_each_task: + inputs: [input_1, input_2] + task: + task_key: inner +` + root, diags := LoadFromBytes("databricks.yml", []byte(config)) + require.NoError(t, diags.Error()) + require.Equal(t, `["input_1","input_2"]`, root.Targets["dev"].Resources.Jobs["fan"].Tasks[0].ForEachTask.Inputs) + require.NoError(t, root.MergeTargetOverrides("dev")) + assert.Equal(t, `["input_1","input_2"]`, root.Resources.Jobs["fan"].Tasks[0].ForEachTask.Inputs) +} + func TestInitializeVariables(t *testing.T) { fooDefault := "abc" root := &Root{ From f0f4b0f5fa914ad976ab0bb14ba73b9723f0b377 Mon Sep 17 00:00:00 2001 From: Lennart Kats Date: Fri, 13 Nov 2026 10:42:37 +0000 Subject: [PATCH 2/8] Handle literal for-each task YAML inputs safely --- .../bundles/for-each-task-yaml-inputs.md | 1 + .../array_reference/databricks.yml | 19 ++ .../array_reference/out.test.toml | 2 + .../array_reference/output.txt | 6 + .../for_each_inputs/array_reference/script | 1 + .../literals/databricks.yml.tmpl | 57 ++++++ .../jobs/for_each_inputs/literals/notebook.py | 2 + .../for_each_inputs/literals/out.test.toml | 2 + .../jobs/for_each_inputs/literals/output.txt | 182 ++++++++++++++++++ .../jobs/for_each_inputs/literals/script | 17 ++ .../jobs/for_each_inputs/literals/test.toml | 1 + bundle/config/root.go | 59 +++--- bundle/config/root_test.go | 44 ++++- bundle/internal/schema/main.go | 17 ++ .../testdata/pass/for_each_task_inputs.yml | 64 ++++++ bundle/schema/jsonschema.json | 13 +- 16 files changed, 460 insertions(+), 27 deletions(-) create mode 100644 .nextchanges/bundles/for-each-task-yaml-inputs.md create mode 100644 acceptance/bundle/resources/jobs/for_each_inputs/array_reference/databricks.yml create mode 100644 acceptance/bundle/resources/jobs/for_each_inputs/array_reference/out.test.toml create mode 100644 acceptance/bundle/resources/jobs/for_each_inputs/array_reference/output.txt create mode 100644 acceptance/bundle/resources/jobs/for_each_inputs/array_reference/script create mode 100644 acceptance/bundle/resources/jobs/for_each_inputs/literals/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/jobs/for_each_inputs/literals/notebook.py create mode 100644 acceptance/bundle/resources/jobs/for_each_inputs/literals/out.test.toml create mode 100644 acceptance/bundle/resources/jobs/for_each_inputs/literals/output.txt create mode 100644 acceptance/bundle/resources/jobs/for_each_inputs/literals/script create mode 100644 acceptance/bundle/resources/jobs/for_each_inputs/literals/test.toml create mode 100644 bundle/internal/schema/testdata/pass/for_each_task_inputs.yml diff --git a/.nextchanges/bundles/for-each-task-yaml-inputs.md b/.nextchanges/bundles/for-each-task-yaml-inputs.md new file mode 100644 index 00000000000..d423c6a9a0d --- /dev/null +++ b/.nextchanges/bundles/for-each-task-yaml-inputs.md @@ -0,0 +1 @@ +* Support YAML arrays for job for-each task inputs. ([#7304](https://github.com/databricks/cli/pull/7304)) diff --git a/acceptance/bundle/resources/jobs/for_each_inputs/array_reference/databricks.yml b/acceptance/bundle/resources/jobs/for_each_inputs/array_reference/databricks.yml new file mode 100644 index 00000000000..25c57c9317c --- /dev/null +++ b/acceptance/bundle/resources/jobs/for_each_inputs/array_reference/databricks.yml @@ -0,0 +1,19 @@ +bundle: + name: for_each_inputs_array_reference + +variables: + item: + default: first + +resources: + jobs: + fan: + tasks: + - task_key: array_reference + for_each_task: + inputs: + - name: ${var.item} + task: + task_key: inner + notebook_task: + notebook_path: /notebook diff --git a/acceptance/bundle/resources/jobs/for_each_inputs/array_reference/out.test.toml b/acceptance/bundle/resources/jobs/for_each_inputs/array_reference/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/jobs/for_each_inputs/array_reference/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/jobs/for_each_inputs/array_reference/output.txt b/acceptance/bundle/resources/jobs/for_each_inputs/array_reference/output.txt new file mode 100644 index 00000000000..c66a6f0fa67 --- /dev/null +++ b/acceptance/bundle/resources/jobs/for_each_inputs/array_reference/output.txt @@ -0,0 +1,6 @@ + +>>> [CLI] bundle validate +Error: failed to rewrite for_each_task.inputs in [TEST_TMP_DIR]/databricks.yml: resources.jobs.fan.tasks[0].for_each_task.inputs at [TEST_TMP_DIR]/databricks.yml:15:15 contains variable reference ${var.item} in a YAML array; use a JSON string for interpolated inputs + + +Found 1 error diff --git a/acceptance/bundle/resources/jobs/for_each_inputs/array_reference/script b/acceptance/bundle/resources/jobs/for_each_inputs/array_reference/script new file mode 100644 index 00000000000..277355fe918 --- /dev/null +++ b/acceptance/bundle/resources/jobs/for_each_inputs/array_reference/script @@ -0,0 +1 @@ +musterr trace $CLI bundle validate diff --git a/acceptance/bundle/resources/jobs/for_each_inputs/literals/databricks.yml.tmpl b/acceptance/bundle/resources/jobs/for_each_inputs/literals/databricks.yml.tmpl new file mode 100644 index 00000000000..7d826ce8472 --- /dev/null +++ b/acceptance/bundle/resources/jobs/for_each_inputs/literals/databricks.yml.tmpl @@ -0,0 +1,57 @@ +bundle: + name: for_each_inputs + +workspace: + root_path: ~/.bundle/$UNIQUE_NAME/${bundle.target} + +variables: + inputs: + default: '[ "from_variable", "unchanged_spacing" ]' + +resources: + jobs: + fan: + name: for-each-inputs-$UNIQUE_NAME-${bundle.target} + tasks: + - task_key: array + for_each_task: + inputs: [root_first, root_second] + task: + task_key: inner + notebook_task: + notebook_path: ./notebook.py + source: WORKSPACE + - task_key: objects + for_each_task: + inputs: + - name: 'quoted "name" with \backslash' + count: 3 + enabled: true + - name: second + count: 0 + enabled: false + task: + task_key: inner + notebook_task: + notebook_path: ./notebook.py + source: WORKSPACE + - task_key: string_variable + for_each_task: + inputs: ${var.inputs} + task: + task_key: inner + notebook_task: + notebook_path: ./notebook.py + source: WORKSPACE + +targets: + default: + default: true + override: + resources: + jobs: + fan: + tasks: + - task_key: array + for_each_task: + inputs: [target_only] diff --git a/acceptance/bundle/resources/jobs/for_each_inputs/literals/notebook.py b/acceptance/bundle/resources/jobs/for_each_inputs/literals/notebook.py new file mode 100644 index 00000000000..5f2f3c08ddd --- /dev/null +++ b/acceptance/bundle/resources/jobs/for_each_inputs/literals/notebook.py @@ -0,0 +1,2 @@ +# Databricks notebook source +print("for-each inputs") diff --git a/acceptance/bundle/resources/jobs/for_each_inputs/literals/out.test.toml b/acceptance/bundle/resources/jobs/for_each_inputs/literals/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/jobs/for_each_inputs/literals/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/jobs/for_each_inputs/literals/output.txt b/acceptance/bundle/resources/jobs/for_each_inputs/literals/output.txt new file mode 100644 index 00000000000..e3342911f36 --- /dev/null +++ b/acceptance/bundle/resources/jobs/for_each_inputs/literals/output.txt @@ -0,0 +1,182 @@ + +=== Root arrays serialize to JSON strings and string variables retain their formatting +>>> [CLI] bundle validate -o json +[ + { + "task_key": "array", + "inputs": "[\"root_first\",\"root_second\"]" + }, + { + "task_key": "objects", + "inputs": "[{\"count\":3,\"enabled\":true,\"name\":\"quoted \\\"name\\\" with \\\\backslash\"},{\"count\":0,\"enabled\":false,\"name\":\"second\"}]" + }, + { + "task_key": "string_variable", + "inputs": "[ \"from_variable\", \"unchanged_spacing\" ]" + } +] + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/default/files... +Created jobs.fan +Files: 3 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //jobs --nostamp +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "for-each-inputs-[UNIQUE_NAME]-default", + "queue": { + "enabled": true + }, + "tasks": [ + { + "for_each_task": { + "inputs": "[\"root_first\",\"root_second\"]", + "task": { + "notebook_task": { + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/default/files/notebook", + "source": "WORKSPACE" + }, + "task_key": "inner" + } + }, + "task_key": "array" + }, + { + "for_each_task": { + "inputs": "[{\"count\":3,\"enabled\":true,\"name\":\"quoted \\\"name\\\" with \\\\backslash\"},{\"count\":0,\"enabled\":false,\"name\":\"second\"}]", + "task": { + "notebook_task": { + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/default/files/notebook", + "source": "WORKSPACE" + }, + "task_key": "inner" + } + }, + "task_key": "objects" + }, + { + "for_each_task": { + "inputs": "[ \"from_variable\", \"unchanged_spacing\" ]", + "task": { + "notebook_task": { + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/default/files/notebook", + "source": "WORKSPACE" + }, + "task_key": "inner" + } + }, + "task_key": "string_variable" + } + ] + } +} + +=== A target array replaces the root array for the same task key +>>> [CLI] bundle validate -o json -t override +[ + { + "task_key": "array", + "inputs": "[\"target_only\"]" + }, + { + "task_key": "objects", + "inputs": "[{\"count\":3,\"enabled\":true,\"name\":\"quoted \\\"name\\\" with \\\\backslash\"},{\"count\":0,\"enabled\":false,\"name\":\"second\"}]" + }, + { + "task_key": "string_variable", + "inputs": "[ \"from_variable\", \"unchanged_spacing\" ]" + } +] + +>>> [CLI] bundle deploy -t override +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/override/files... +Created jobs.fan +Files: 3 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //jobs --nostamp +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/override/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "for-each-inputs-[UNIQUE_NAME]-override", + "queue": { + "enabled": true + }, + "tasks": [ + { + "for_each_task": { + "inputs": "[\"target_only\"]", + "task": { + "notebook_task": { + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/override/files/notebook", + "source": "WORKSPACE" + }, + "task_key": "inner" + } + }, + "task_key": "array" + }, + { + "for_each_task": { + "inputs": "[{\"count\":3,\"enabled\":true,\"name\":\"quoted \\\"name\\\" with \\\\backslash\"},{\"count\":0,\"enabled\":false,\"name\":\"second\"}]", + "task": { + "notebook_task": { + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/override/files/notebook", + "source": "WORKSPACE" + }, + "task_key": "inner" + } + }, + "task_key": "objects" + }, + { + "for_each_task": { + "inputs": "[ \"from_variable\", \"unchanged_spacing\" ]", + "task": { + "notebook_task": { + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/override/files/notebook", + "source": "WORKSPACE" + }, + "task_key": "inner" + } + }, + "task_key": "string_variable" + } + ] + } +} + +>>> [CLI] bundle destroy --auto-approve -t override +The following resources will be deleted: + delete resources.jobs.fan + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/override + +Destroy: 1 deleted + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.fan + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/jobs/for_each_inputs/literals/script b/acceptance/bundle/resources/jobs/for_each_inputs/literals/script new file mode 100644 index 00000000000..7c265a4bb04 --- /dev/null +++ b/acceptance/bundle/resources/jobs/for_each_inputs/literals/script @@ -0,0 +1,17 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Root arrays serialize to JSON strings and string variables retain their formatting" +trace $CLI bundle validate -o json | jq '.resources.jobs.fan.tasks | map({task_key, inputs: .for_each_task.inputs})' + +trace $CLI bundle deploy +trace print_requests.py //jobs --nostamp + +title "A target array replaces the root array for the same task key" +trace $CLI bundle validate -o json -t override | jq '.resources.jobs.fan.tasks | map({task_key, inputs: .for_each_task.inputs})' + +trace $CLI bundle deploy -t override +trace print_requests.py //jobs --nostamp + +trace $CLI bundle destroy --auto-approve -t override +trace $CLI bundle destroy --auto-approve +rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/resources/jobs/for_each_inputs/literals/test.toml b/acceptance/bundle/resources/jobs/for_each_inputs/literals/test.toml new file mode 100644 index 00000000000..c7c6f58ed6e --- /dev/null +++ b/acceptance/bundle/resources/jobs/for_each_inputs/literals/test.toml @@ -0,0 +1 @@ +Cloud = true diff --git a/bundle/config/root.go b/bundle/config/root.go index c400fd5e2ae..9fd7aeb6569 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -558,34 +558,47 @@ func rewriteShorthands(v dyn.Value) (dyn.Value, error) { })) } +// rewriteForEachTaskInputs encodes literal arrays before SDK string normalization. +// References in arrays require the string form; other kinds normalize unchanged. func rewriteForEachTaskInputs(v dyn.Value) (dyn.Value, error) { - return dyn.Walk(v, func(p dyn.Path, input dyn.Value) (dyn.Value, error) { - if !isForEachTaskInputsPath(p) || input.Kind() != dyn.KindSequence { - return input, nil - } + inputs := dyn.NewPattern( + dyn.Key("resources"), dyn.Key("jobs"), dyn.AnyKey(), + dyn.Key("tasks"), dyn.AnyIndex(), dyn.Key("for_each_task"), dyn.Key("inputs"), + ) + for _, prefix := range []dyn.Pattern{ + nil, + dyn.NewPattern(dyn.Key("targets"), dyn.AnyKey()), + dyn.NewPattern(dyn.Key("environments"), dyn.AnyKey()), + } { + var err error + v, err = dyn.MapByPattern(v, prefix.Append(inputs...), func(p dyn.Path, input dyn.Value) (dyn.Value, error) { + if input.Kind() != dyn.KindSequence { + return input, nil + } - data, err := json.Marshal(input.AsAny()) + array, err := dyn.Walk(input, func(_ dyn.Path, value dyn.Value) (dyn.Value, error) { + if value.Kind() == dyn.KindTime { + return dyn.NewValue(value.MustTime().String(), value.Locations()), nil + } + return value, nil + }) + if err != nil { + return dyn.InvalidValue, err + } + data, err := json.Marshal(array.AsAny()) + if err != nil { + return dyn.InvalidValue, fmt.Errorf("%s at %s: %w", p, input.Location(), err) + } + if ref, ok := dynvar.NewRef(dyn.V(string(data))); ok { + return dyn.InvalidValue, fmt.Errorf("%s at %s contains variable reference ${%s} in a YAML array; use a JSON string for interpolated inputs", p, input.Location(), ref.References()[0]) + } + return dyn.NewValue(string(data), input.Locations()), nil + }) if err != nil { - return dyn.InvalidValue, fmt.Errorf("%s at %s: %w", p, input.Location(), err) - } - // Leave unresolved references for normal validation instead of interpolating into JSON text. - if dynvar.ContainsVariableReference(string(data)) { - return input, nil + return dyn.InvalidValue, err } - return dyn.NewValue(string(data), input.Locations()), nil - }) -} - -func isForEachTaskInputsPath(p dyn.Path) bool { - if len(p) == 9 && (p[0] == dyn.Key("targets") || p[0] == dyn.Key("environments")) && p[1].Key() != "" { - p = p[2:] } - return len(p) == 7 && - p.HasPrefix(dyn.NewPath(dyn.Key("resources"), dyn.Key("jobs"))) && - p[2].Key() != "" && - p[3] == dyn.Key("tasks") && - p[4].Key() == "" && - p.HasSuffix(dyn.NewPath(dyn.Key("for_each_task"), dyn.Key("inputs"))) + return v, nil } // validateVariableOverrides checks that all variables specified diff --git a/bundle/config/root_test.go b/bundle/config/root_test.go index f562e397992..28530772c73 100644 --- a/bundle/config/root_test.go +++ b/bundle/config/root_test.go @@ -40,6 +40,9 @@ func TestLoadForEachTaskInputs(t *testing.T) { {name: "YAML array", inputs: "[input_1, input_2]", want: `["input_1","input_2"]`}, {name: "YAML object array", inputs: "[{name: input_1, enabled: true}]", want: `[{"enabled":true,"name":"input_1"}]`}, {name: "empty YAML array", inputs: "[]", want: `[]`}, + {name: "YAML dates", inputs: "[2024-01-01, 2024-01-02]", want: `["2024-01-01","2024-01-02"]`}, + {name: "quoted YAML elements", inputs: `["a\"b", "line\nbreak"]`, want: `["a\"b","line\nbreak"]`}, + {name: "escaped reference in YAML array", inputs: `["$${var.item}"]`, want: `["$${var.item}"]`}, {name: "JSON string", inputs: `'[ "input_1", "input_2" ]'`, want: `[ "input_1", "input_2" ]`}, {name: "variable reference string", inputs: `${var.items}`, want: `${var.items}`}, } { @@ -61,9 +64,45 @@ func TestLoadForEachTaskInputs(t *testing.T) { } } +func TestLoadForEachTaskInputsRejectsArrayReferences(t *testing.T) { + for _, tc := range []struct { + name string + inputs string + prefix string + target bool + }{ + {name: "root element", inputs: `["${var.item}_x", y]`, prefix: "resources.jobs.fan"}, + {name: "nested object value", inputs: `[{name: "${var.item}"}]`, prefix: "resources.jobs.fan"}, + {name: "target override", inputs: `["${var.item}"]`, prefix: "targets.dev.resources.jobs.fan", target: true}, + } { + t.Run(tc.name, func(t *testing.T) { + var config string + if tc.target { + config = "bundle:\n name: foreach\nvariables:\n item:\n default: input_1\ntargets:\n dev:\n resources:\n jobs:\n fan:\n tasks:\n - task_key: fan\n for_each_task:\n inputs: " + tc.inputs + "\n task:\n task_key: inner\n" + } else { + config = "bundle:\n name: foreach\nvariables:\n item:\n default: input_1\nresources:\n jobs:\n fan:\n tasks:\n - task_key: fan\n for_each_task:\n inputs: " + tc.inputs + "\n task:\n task_key: inner\n" + } + root, diags := LoadFromBytes("databricks.yml", []byte(config)) + require.Nil(t, root) + require.ErrorContains(t, diags.Error(), "variable reference ${var.item} in a YAML array") + assert.ErrorContains(t, diags.Error(), tc.prefix+".tasks[0].for_each_task.inputs") + assert.ErrorContains(t, diags.Error(), "databricks.yml:") + }) + } +} + func TestLoadForEachTaskInputsInTarget(t *testing.T) { config := `bundle: name: foreach +resources: + jobs: + fan: + tasks: + - task_key: fan + for_each_task: + inputs: [root_1] + task: + task_key: inner targets: dev: resources: @@ -78,9 +117,12 @@ targets: ` root, diags := LoadFromBytes("databricks.yml", []byte(config)) require.NoError(t, diags.Error()) + require.Equal(t, `["root_1"]`, root.Resources.Jobs["fan"].Tasks[0].ForEachTask.Inputs) require.Equal(t, `["input_1","input_2"]`, root.Targets["dev"].Resources.Jobs["fan"].Tasks[0].ForEachTask.Inputs) require.NoError(t, root.MergeTargetOverrides("dev")) - assert.Equal(t, `["input_1","input_2"]`, root.Resources.Jobs["fan"].Tasks[0].ForEachTask.Inputs) + require.Len(t, root.Resources.Jobs["fan"].Tasks, 2) + assert.Equal(t, `["root_1"]`, root.Resources.Jobs["fan"].Tasks[0].ForEachTask.Inputs) + assert.Equal(t, `["input_1","input_2"]`, root.Resources.Jobs["fan"].Tasks[1].ForEachTask.Inputs) } func TestInitializeVariables(t *testing.T) { diff --git a/bundle/internal/schema/main.go b/bundle/internal/schema/main.go index 01a743933b2..10e544f3b9d 100644 --- a/bundle/internal/schema/main.go +++ b/bundle/internal/schema/main.go @@ -120,6 +120,22 @@ func removeJobsFields(typ reflect.Type, s jsonschema.Schema) jsonschema.Schema { return s } +func allowForEachTaskInputArrays(typ reflect.Type, s jsonschema.Schema) jsonschema.Schema { + if typ != reflect.TypeFor[jobs.ForEachTask]() { + return s + } + + inputs := s.Properties["inputs"] + inputs.Description = "Inputs for each task, as a literal YAML array or a string containing JSON or a reference." + stringRef := inputs.Reference + inputs.Reference = nil + inputs.OneOf = []jsonschema.Schema{ + {Reference: stringRef}, + {Type: jsonschema.ArrayType}, + } + return s +} + func removePipelineFields(typ reflect.Type, s jsonschema.Schema) jsonschema.Schema { switch typ { case reflect.TypeFor[resources.Pipeline](): @@ -287,6 +303,7 @@ func generateSchema(workdir, outputFile, cliJSONFile string, docsMode bool) { removeDeploymentFields, makeVolumeTypeOptional, a.addAnnotations, + allowForEachTaskInputArrays, removeOutputOnlyFields, } if !docsMode { diff --git a/bundle/internal/schema/testdata/pass/for_each_task_inputs.yml b/bundle/internal/schema/testdata/pass/for_each_task_inputs.yml new file mode 100644 index 00000000000..c428756a3a8 --- /dev/null +++ b/bundle/internal/schema/testdata/pass/for_each_task_inputs.yml @@ -0,0 +1,64 @@ +bundle: + name: for_each_task_inputs + +variables: + inputs: + default: '["from_variable"]' + +resources: + jobs: + fan: + name: for_each_task_inputs + tasks: + - task_key: array + for_each_task: + inputs: [first, second] + task: + task_key: inner + notebook_task: + notebook_path: /notebook + - task_key: objects + for_each_task: + inputs: + - name: first + count: 3 + enabled: true + task: + task_key: inner + notebook_task: + notebook_path: /notebook + - task_key: empty + for_each_task: + inputs: [] + task: + task_key: inner + notebook_task: + notebook_path: /notebook + - task_key: string + for_each_task: + inputs: '["first", "second"]' + task: + task_key: inner + notebook_task: + notebook_path: /notebook + - task_key: variable + for_each_task: + inputs: ${var.inputs} + task: + task_key: inner + notebook_task: + notebook_path: /notebook + +targets: + override: + resources: + jobs: + fan: + tasks: + - task_key: array + for_each_task: + inputs: [overridden] + task: + task_key: inner + notebook_task: + notebook_path: /notebook diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 959273d2586..7b15b8fcc69 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -10031,9 +10031,16 @@ "x-databricks-launch-stage": "GA" }, "inputs": { - "description": "Array for task to iterate on. This can be a JSON string or a reference to\nan array parameter.", - "$ref": "#/$defs/string", - "x-databricks-launch-stage": "GA" + "description": "Inputs for each task, as a literal YAML array or a string containing JSON or a reference.", + "x-databricks-launch-stage": "GA", + "oneOf": [ + { + "$ref": "#/$defs/string" + }, + { + "type": "array" + } + ] }, "task": { "description": "Configuration for the task that will be run for each element in the array", From 81e1ac6b52373b6f9a860e00cb8aca0b24dce6d8 Mon Sep 17 00:00:00 2001 From: Lennart Kats Date: Fri, 13 Nov 2026 10:52:30 +0000 Subject: [PATCH 3/8] Preserve variable-defined for-each tasks during array rewrite --- .../variable_containers/databricks.yml | 56 ++++++++++++ .../variable_containers/out.test.toml | 2 + .../variable_containers/output.txt | 20 ++++ .../variable_containers/script | 1 + bundle/config/root.go | 48 +++++----- bundle/config/root_test.go | 91 +++++++++++++++++++ 6 files changed, 195 insertions(+), 23 deletions(-) create mode 100644 acceptance/bundle/resources/jobs/for_each_inputs/variable_containers/databricks.yml create mode 100644 acceptance/bundle/resources/jobs/for_each_inputs/variable_containers/out.test.toml create mode 100644 acceptance/bundle/resources/jobs/for_each_inputs/variable_containers/output.txt create mode 100644 acceptance/bundle/resources/jobs/for_each_inputs/variable_containers/script diff --git a/acceptance/bundle/resources/jobs/for_each_inputs/variable_containers/databricks.yml b/acceptance/bundle/resources/jobs/for_each_inputs/variable_containers/databricks.yml new file mode 100644 index 00000000000..cf24223e69e --- /dev/null +++ b/acceptance/bundle/resources/jobs/for_each_inputs/variable_containers/databricks.yml @@ -0,0 +1,56 @@ +bundle: + name: for_each_inputs_variable_containers + +variables: + each: + type: complex + default: + inputs: '["from_each"]' + task: + task_key: inner + notebook_task: + notebook_path: /notebook + task: + type: complex + default: + task_key: fan + for_each_task: + inputs: '["from_task"]' + task: + task_key: inner + notebook_task: + notebook_path: /notebook + tasks: + type: complex + default: + - task_key: fan + for_each_task: + inputs: '["from_tasks"]' + task: + task_key: inner + notebook_task: + notebook_path: /notebook + job: + type: complex + default: + tasks: + - task_key: fan + for_each_task: + inputs: '["from_job"]' + task: + task_key: inner + notebook_task: + notebook_path: /notebook + +resources: + jobs: + each: + tasks: + - task_key: fan + for_each_task: ${var.each} + task: + tasks: + - ${var.task} + tasks: + tasks: ${var.tasks} + job: ${var.job} diff --git a/acceptance/bundle/resources/jobs/for_each_inputs/variable_containers/out.test.toml b/acceptance/bundle/resources/jobs/for_each_inputs/variable_containers/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/jobs/for_each_inputs/variable_containers/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/jobs/for_each_inputs/variable_containers/output.txt b/acceptance/bundle/resources/jobs/for_each_inputs/variable_containers/output.txt new file mode 100644 index 00000000000..7574845f2aa --- /dev/null +++ b/acceptance/bundle/resources/jobs/for_each_inputs/variable_containers/output.txt @@ -0,0 +1,20 @@ + +>>> [CLI] bundle validate -o json +[ + { + "job": "each", + "inputs": "[\"from_each\"]" + }, + { + "job": "job", + "inputs": "[\"from_job\"]" + }, + { + "job": "task", + "inputs": "[\"from_task\"]" + }, + { + "job": "tasks", + "inputs": "[\"from_tasks\"]" + } +] diff --git a/acceptance/bundle/resources/jobs/for_each_inputs/variable_containers/script b/acceptance/bundle/resources/jobs/for_each_inputs/variable_containers/script new file mode 100644 index 00000000000..41a6b09cf0c --- /dev/null +++ b/acceptance/bundle/resources/jobs/for_each_inputs/variable_containers/script @@ -0,0 +1 @@ +trace $CLI bundle validate -o json | jq '[.resources.jobs | to_entries[] | {job: .key, inputs: .value.tasks[0].for_each_task.inputs}] | sort_by(.job)' diff --git a/bundle/config/root.go b/bundle/config/root.go index 9fd7aeb6569..1213c2d2ad4 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -565,40 +565,42 @@ func rewriteForEachTaskInputs(v dyn.Value) (dyn.Value, error) { dyn.Key("resources"), dyn.Key("jobs"), dyn.AnyKey(), dyn.Key("tasks"), dyn.AnyIndex(), dyn.Key("for_each_task"), dyn.Key("inputs"), ) + paths := &dyn.TrieNode{} for _, prefix := range []dyn.Pattern{ nil, dyn.NewPattern(dyn.Key("targets"), dyn.AnyKey()), dyn.NewPattern(dyn.Key("environments"), dyn.AnyKey()), } { - var err error - v, err = dyn.MapByPattern(v, prefix.Append(inputs...), func(p dyn.Path, input dyn.Value) (dyn.Value, error) { - if input.Kind() != dyn.KindSequence { - return input, nil - } + if err := paths.Insert(prefix.Append(inputs...)); err != nil { + return dyn.InvalidValue, fmt.Errorf("failed to match for_each_task.inputs: %w", err) + } + } + return dyn.Walk(v, func(p dyn.Path, input dyn.Value) (dyn.Value, error) { + if input.Kind() != dyn.KindSequence { + return input, nil + } + if _, ok := paths.SearchPath(p); !ok { + return input, nil + } - array, err := dyn.Walk(input, func(_ dyn.Path, value dyn.Value) (dyn.Value, error) { - if value.Kind() == dyn.KindTime { - return dyn.NewValue(value.MustTime().String(), value.Locations()), nil - } - return value, nil - }) - if err != nil { - return dyn.InvalidValue, err - } - data, err := json.Marshal(array.AsAny()) - if err != nil { - return dyn.InvalidValue, fmt.Errorf("%s at %s: %w", p, input.Location(), err) + array, err := dyn.Walk(input, func(_ dyn.Path, value dyn.Value) (dyn.Value, error) { + if value.Kind() == dyn.KindTime { + return dyn.NewValue(value.MustTime().String(), value.Locations()), nil } - if ref, ok := dynvar.NewRef(dyn.V(string(data))); ok { - return dyn.InvalidValue, fmt.Errorf("%s at %s contains variable reference ${%s} in a YAML array; use a JSON string for interpolated inputs", p, input.Location(), ref.References()[0]) - } - return dyn.NewValue(string(data), input.Locations()), nil + return value, nil }) if err != nil { return dyn.InvalidValue, err } - } - return v, nil + data, err := json.Marshal(array.AsAny()) + if err != nil { + return dyn.InvalidValue, fmt.Errorf("%s at %s: %w", p, input.Location(), err) + } + if ref, ok := dynvar.NewRef(dyn.V(string(data))); ok { + return dyn.InvalidValue, fmt.Errorf("%s at %s contains variable reference ${%s} in a YAML array; use a JSON string for interpolated inputs", p, input.Location(), ref.References()[0]) + } + return dyn.NewValue(string(data), input.Locations()), nil + }) } // validateVariableOverrides checks that all variables specified diff --git a/bundle/config/root_test.go b/bundle/config/root_test.go index 28530772c73..29f7a15b01d 100644 --- a/bundle/config/root_test.go +++ b/bundle/config/root_test.go @@ -125,6 +125,97 @@ targets: assert.Equal(t, `["input_1","input_2"]`, root.Resources.Jobs["fan"].Tasks[1].ForEachTask.Inputs) } +func TestLoadForEachTaskInputsPreservesVariableContainers(t *testing.T) { + for _, tc := range []struct { + name string + config string + }{ + { + name: "for-each task reference", + config: `bundle: + name: foreach +variables: + each: + type: complex + default: + inputs: '["item"]' + task: + task_key: inner +resources: + jobs: + fan: + tasks: + - task_key: fan + for_each_task: ${var.each} +`, + }, + { + name: "task reference", + config: `bundle: + name: foreach +variables: + task: + type: complex + default: + task_key: fan + for_each_task: + inputs: '["item"]' + task: + task_key: inner +resources: + jobs: + fan: + tasks: + - ${var.task} +`, + }, + { + name: "task list reference", + config: `bundle: + name: foreach +variables: + tasks: + type: complex + default: + - task_key: fan + for_each_task: + inputs: '["item"]' + task: + task_key: inner +resources: + jobs: + fan: + tasks: ${var.tasks} +`, + }, + { + name: "job reference", + config: `bundle: + name: foreach +variables: + job: + type: complex + default: + tasks: + - task_key: fan + for_each_task: + inputs: '["item"]' + task: + task_key: inner +resources: + jobs: + fan: ${var.job} +`, + }, + } { + t.Run(tc.name, func(t *testing.T) { + root, diags := LoadFromBytes("databricks.yml", []byte(tc.config)) + require.NoError(t, diags.Error()) + require.NotNil(t, root) + }) + } +} + func TestInitializeVariables(t *testing.T) { fooDefault := "abc" root := &Root{ From 27b29682b84f0860a7e227e960a7e674d56d558d Mon Sep 17 00:00:00 2001 From: Lennart Kats Date: Fri, 13 Nov 2026 10:56:17 +0000 Subject: [PATCH 4/8] Use unique nested task keys in for-each acceptance fixture --- .../for_each_inputs/literals/databricks.yml.tmpl | 6 +++--- .../jobs/for_each_inputs/literals/output.txt | 12 ++++++------ .../schema/testdata/pass/for_each_task_inputs.yml | 12 ++++++------ 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/acceptance/bundle/resources/jobs/for_each_inputs/literals/databricks.yml.tmpl b/acceptance/bundle/resources/jobs/for_each_inputs/literals/databricks.yml.tmpl index 7d826ce8472..05064d098cc 100644 --- a/acceptance/bundle/resources/jobs/for_each_inputs/literals/databricks.yml.tmpl +++ b/acceptance/bundle/resources/jobs/for_each_inputs/literals/databricks.yml.tmpl @@ -17,7 +17,7 @@ resources: for_each_task: inputs: [root_first, root_second] task: - task_key: inner + task_key: array_inner notebook_task: notebook_path: ./notebook.py source: WORKSPACE @@ -31,7 +31,7 @@ resources: count: 0 enabled: false task: - task_key: inner + task_key: objects_inner notebook_task: notebook_path: ./notebook.py source: WORKSPACE @@ -39,7 +39,7 @@ resources: for_each_task: inputs: ${var.inputs} task: - task_key: inner + task_key: string_inner notebook_task: notebook_path: ./notebook.py source: WORKSPACE diff --git a/acceptance/bundle/resources/jobs/for_each_inputs/literals/output.txt b/acceptance/bundle/resources/jobs/for_each_inputs/literals/output.txt index e3342911f36..956ac34ac3c 100644 --- a/acceptance/bundle/resources/jobs/for_each_inputs/literals/output.txt +++ b/acceptance/bundle/resources/jobs/for_each_inputs/literals/output.txt @@ -47,7 +47,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/default/files/notebook", "source": "WORKSPACE" }, - "task_key": "inner" + "task_key": "array_inner" } }, "task_key": "array" @@ -60,7 +60,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/default/files/notebook", "source": "WORKSPACE" }, - "task_key": "inner" + "task_key": "objects_inner" } }, "task_key": "objects" @@ -73,7 +73,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/default/files/notebook", "source": "WORKSPACE" }, - "task_key": "inner" + "task_key": "string_inner" } }, "task_key": "string_variable" @@ -130,7 +130,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/override/files/notebook", "source": "WORKSPACE" }, - "task_key": "inner" + "task_key": "array_inner" } }, "task_key": "array" @@ -143,7 +143,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/override/files/notebook", "source": "WORKSPACE" }, - "task_key": "inner" + "task_key": "objects_inner" } }, "task_key": "objects" @@ -156,7 +156,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/override/files/notebook", "source": "WORKSPACE" }, - "task_key": "inner" + "task_key": "string_inner" } }, "task_key": "string_variable" diff --git a/bundle/internal/schema/testdata/pass/for_each_task_inputs.yml b/bundle/internal/schema/testdata/pass/for_each_task_inputs.yml index c428756a3a8..025c0b8039d 100644 --- a/bundle/internal/schema/testdata/pass/for_each_task_inputs.yml +++ b/bundle/internal/schema/testdata/pass/for_each_task_inputs.yml @@ -14,7 +14,7 @@ resources: for_each_task: inputs: [first, second] task: - task_key: inner + task_key: array_inner notebook_task: notebook_path: /notebook - task_key: objects @@ -24,28 +24,28 @@ resources: count: 3 enabled: true task: - task_key: inner + task_key: objects_inner notebook_task: notebook_path: /notebook - task_key: empty for_each_task: inputs: [] task: - task_key: inner + task_key: empty_inner notebook_task: notebook_path: /notebook - task_key: string for_each_task: inputs: '["first", "second"]' task: - task_key: inner + task_key: string_inner notebook_task: notebook_path: /notebook - task_key: variable for_each_task: inputs: ${var.inputs} task: - task_key: inner + task_key: variable_inner notebook_task: notebook_path: /notebook @@ -59,6 +59,6 @@ targets: for_each_task: inputs: [overridden] task: - task_key: inner + task_key: array_inner notebook_task: notebook_path: /notebook From 962678f82c9f0ad39abd944c7d583afe1b505d70 Mon Sep 17 00:00:00 2001 From: Lennart Kats Date: Fri, 13 Nov 2026 11:13:57 +0000 Subject: [PATCH 5/8] Keep for-each YAML inputs compatible with code generation --- .../literals/databricks.yml.tmpl | 10 ++++++ .../jobs/for_each_inputs/literals/output.txt | 34 +++++++++++++++++++ .../jobs/for_each_inputs/literals/script | 4 +-- bundle/config/root.go | 4 ++- bundle/config/root_test.go | 1 + bundle/internal/schema/main.go | 1 - bundle/schema/jsonschema.json | 2 +- python/codegen/codegen/jsonschema.py | 23 ++++++++++--- .../codegen/codegen_tests/test_jsonschema.py | 24 +++++++++++++ 9 files changed, 94 insertions(+), 9 deletions(-) diff --git a/acceptance/bundle/resources/jobs/for_each_inputs/literals/databricks.yml.tmpl b/acceptance/bundle/resources/jobs/for_each_inputs/literals/databricks.yml.tmpl index 05064d098cc..bb967028c4c 100644 --- a/acceptance/bundle/resources/jobs/for_each_inputs/literals/databricks.yml.tmpl +++ b/acceptance/bundle/resources/jobs/for_each_inputs/literals/databricks.yml.tmpl @@ -7,6 +7,8 @@ workspace: variables: inputs: default: '[ "from_variable", "unchanged_spacing" ]' + item: + default: from_reference resources: jobs: @@ -43,6 +45,14 @@ resources: notebook_task: notebook_path: ./notebook.py source: WORKSPACE + - task_key: string_interpolation + for_each_task: + inputs: '["${var.item}_x", "y"]' + task: + task_key: interpolation_inner + notebook_task: + notebook_path: ./notebook.py + source: WORKSPACE targets: default: diff --git a/acceptance/bundle/resources/jobs/for_each_inputs/literals/output.txt b/acceptance/bundle/resources/jobs/for_each_inputs/literals/output.txt index 956ac34ac3c..c67d7038800 100644 --- a/acceptance/bundle/resources/jobs/for_each_inputs/literals/output.txt +++ b/acceptance/bundle/resources/jobs/for_each_inputs/literals/output.txt @@ -10,6 +10,10 @@ "task_key": "objects", "inputs": "[{\"count\":3,\"enabled\":true,\"name\":\"quoted \\\"name\\\" with \\\\backslash\"},{\"count\":0,\"enabled\":false,\"name\":\"second\"}]" }, + { + "task_key": "string_interpolation", + "inputs": "[\"from_reference_x\", \"y\"]" + }, { "task_key": "string_variable", "inputs": "[ \"from_variable\", \"unchanged_spacing\" ]" @@ -65,6 +69,19 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged }, "task_key": "objects" }, + { + "for_each_task": { + "inputs": "[\"from_reference_x\", \"y\"]", + "task": { + "notebook_task": { + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/default/files/notebook", + "source": "WORKSPACE" + }, + "task_key": "interpolation_inner" + } + }, + "task_key": "string_interpolation" + }, { "for_each_task": { "inputs": "[ \"from_variable\", \"unchanged_spacing\" ]", @@ -93,6 +110,10 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "task_key": "objects", "inputs": "[{\"count\":3,\"enabled\":true,\"name\":\"quoted \\\"name\\\" with \\\\backslash\"},{\"count\":0,\"enabled\":false,\"name\":\"second\"}]" }, + { + "task_key": "string_interpolation", + "inputs": "[\"from_reference_x\", \"y\"]" + }, { "task_key": "string_variable", "inputs": "[ \"from_variable\", \"unchanged_spacing\" ]" @@ -148,6 +169,19 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged }, "task_key": "objects" }, + { + "for_each_task": { + "inputs": "[\"from_reference_x\", \"y\"]", + "task": { + "notebook_task": { + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/override/files/notebook", + "source": "WORKSPACE" + }, + "task_key": "interpolation_inner" + } + }, + "task_key": "string_interpolation" + }, { "for_each_task": { "inputs": "[ \"from_variable\", \"unchanged_spacing\" ]", diff --git a/acceptance/bundle/resources/jobs/for_each_inputs/literals/script b/acceptance/bundle/resources/jobs/for_each_inputs/literals/script index 7c265a4bb04..7d31170c837 100644 --- a/acceptance/bundle/resources/jobs/for_each_inputs/literals/script +++ b/acceptance/bundle/resources/jobs/for_each_inputs/literals/script @@ -1,12 +1,12 @@ envsubst < databricks.yml.tmpl > databricks.yml -title "Root arrays serialize to JSON strings and string variables retain their formatting" +title "Root arrays serialize to JSON strings and string variables retain their formatting" >&2 trace $CLI bundle validate -o json | jq '.resources.jobs.fan.tasks | map({task_key, inputs: .for_each_task.inputs})' trace $CLI bundle deploy trace print_requests.py //jobs --nostamp -title "A target array replaces the root array for the same task key" +title "A target array replaces the root array for the same task key" >&2 trace $CLI bundle validate -o json -t override | jq '.resources.jobs.fan.tasks | map({task_key, inputs: .for_each_task.inputs})' trace $CLI bundle deploy -t override diff --git a/bundle/config/root.go b/bundle/config/root.go index 1213c2d2ad4..a2577cdec65 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -559,7 +559,9 @@ func rewriteShorthands(v dyn.Value) (dyn.Value, error) { } // rewriteForEachTaskInputs encodes literal arrays before SDK string normalization. -// References in arrays require the string form; other kinds normalize unchanged. +// A resource-level any shadow cannot cover this nested SDK field, so this runs +// before typed normalization, unlike resource mutators after variable resolution. +// Other kinds use normal validation; array references fail here. func rewriteForEachTaskInputs(v dyn.Value) (dyn.Value, error) { inputs := dyn.NewPattern( dyn.Key("resources"), dyn.Key("jobs"), dyn.AnyKey(), diff --git a/bundle/config/root_test.go b/bundle/config/root_test.go index 29f7a15b01d..8ce561c8117 100644 --- a/bundle/config/root_test.go +++ b/bundle/config/root_test.go @@ -41,6 +41,7 @@ func TestLoadForEachTaskInputs(t *testing.T) { {name: "YAML object array", inputs: "[{name: input_1, enabled: true}]", want: `[{"enabled":true,"name":"input_1"}]`}, {name: "empty YAML array", inputs: "[]", want: `[]`}, {name: "YAML dates", inputs: "[2024-01-01, 2024-01-02]", want: `["2024-01-01","2024-01-02"]`}, + {name: "nested YAML timestamp", inputs: "[{when: 2024-01-01T01:02:03.123+02:00}]", want: `[{"when":"2024-01-01T01:02:03.123+02:00"}]`}, {name: "quoted YAML elements", inputs: `["a\"b", "line\nbreak"]`, want: `["a\"b","line\nbreak"]`}, {name: "escaped reference in YAML array", inputs: `["$${var.item}"]`, want: `["$${var.item}"]`}, {name: "JSON string", inputs: `'[ "input_1", "input_2" ]'`, want: `[ "input_1", "input_2" ]`}, diff --git a/bundle/internal/schema/main.go b/bundle/internal/schema/main.go index 10e544f3b9d..13b50ea1a09 100644 --- a/bundle/internal/schema/main.go +++ b/bundle/internal/schema/main.go @@ -126,7 +126,6 @@ func allowForEachTaskInputArrays(typ reflect.Type, s jsonschema.Schema) jsonsche } inputs := s.Properties["inputs"] - inputs.Description = "Inputs for each task, as a literal YAML array or a string containing JSON or a reference." stringRef := inputs.Reference inputs.Reference = nil inputs.OneOf = []jsonschema.Schema{ diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 7b15b8fcc69..f2b5c48ba73 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -10031,7 +10031,7 @@ "x-databricks-launch-stage": "GA" }, "inputs": { - "description": "Inputs for each task, as a literal YAML array or a string containing JSON or a reference.", + "description": "Array for task to iterate on. This can be a JSON string or a reference to\nan array parameter.", "x-databricks-launch-stage": "GA", "oneOf": [ { diff --git a/python/codegen/codegen/jsonschema.py b/python/codegen/codegen/jsonschema.py index 9ad7191a55c..72c8b40eceb 100644 --- a/python/codegen/codegen/jsonschema.py +++ b/python/codegen/codegen/jsonschema.py @@ -91,6 +91,20 @@ def _unwrap_variable(schema: dict): return None +def _property_ref(value: dict) -> Optional[str]: + if ref := value.get("$ref"): + return ref + + # CLI accepts literal YAML arrays here, while PyDABs still emits the SDK string. + if value.get("oneOf") == [ + {"$ref": "#/$defs/string"}, + {"type": "array"}, + ]: + return "#/$defs/string" + + return None + + def _parse_schema(schema: dict) -> Schema: schema = _unwrap_variable(schema) or schema properties = {} @@ -106,10 +120,11 @@ def _parse_bool(value) -> Optional[bool]: assert v.get("properties") is None assert v.get("items") is None - assert v.get("$ref") + ref = _property_ref(v) + assert ref prop = Property( - ref=v["$ref"], + ref=ref, description=v.get("description"), deprecated=_parse_bool(v.get("deprecated")), stage=v.get("x-databricks-launch-stage"), @@ -179,9 +194,9 @@ def _refs_in_raw_schema(schema: dict) -> list[str]: schema = _unwrap_variable(schema) or schema return [ - prop["$ref"].split("/")[-1] + ref.split("/")[-1] for prop in schema.get("properties", {}).values() - if prop.get("$ref") + if (ref := _property_ref(prop)) ] diff --git a/python/codegen/codegen_tests/test_jsonschema.py b/python/codegen/codegen_tests/test_jsonschema.py index 307bcc1dec9..d19c7f9e508 100644 --- a/python/codegen/codegen_tests/test_jsonschema.py +++ b/python/codegen/codegen_tests/test_jsonschema.py @@ -3,6 +3,7 @@ Schema, SchemaType, _parse_schema, + _refs_in_raw_schema, _unwrap_variable, get_schemas, ) @@ -77,3 +78,26 @@ def test_parse_schema(): ) assert _parse_schema(spec) == expected + + +def test_parse_schema_for_each_inputs_array_shorthand(): + spec = { + "type": "object", + "properties": { + "inputs": { + "description": "For-each task inputs", + "x-databricks-launch-stage": "GA", + "oneOf": [ + {"$ref": "#/$defs/string"}, + {"type": "array"}, + ], + } + }, + } + + assert _parse_schema(spec).properties["inputs"] == Property( + ref="#/$defs/string", + description="For-each task inputs", + stage="GA", + ) + assert _refs_in_raw_schema(spec) == ["string"] From 2f01d5cc37d8d578d2558b9d49819c0a1985d31a Mon Sep 17 00:00:00 2001 From: Lennart Kats Date: Fri, 13 Nov 2026 11:29:41 +0000 Subject: [PATCH 6/8] Stabilize for-each inputs acceptance output across DMS modes --- .../bundle/resources/jobs/for_each_inputs/literals/output.txt | 2 -- .../bundle/resources/jobs/for_each_inputs/literals/script | 2 -- 2 files changed, 4 deletions(-) diff --git a/acceptance/bundle/resources/jobs/for_each_inputs/literals/output.txt b/acceptance/bundle/resources/jobs/for_each_inputs/literals/output.txt index c67d7038800..8a4c0b6f1ba 100644 --- a/acceptance/bundle/resources/jobs/for_each_inputs/literals/output.txt +++ b/acceptance/bundle/resources/jobs/for_each_inputs/literals/output.txt @@ -1,5 +1,4 @@ -=== Root arrays serialize to JSON strings and string variables retain their formatting >>> [CLI] bundle validate -o json [ { @@ -99,7 +98,6 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } } -=== A target array replaces the root array for the same task key >>> [CLI] bundle validate -o json -t override [ { diff --git a/acceptance/bundle/resources/jobs/for_each_inputs/literals/script b/acceptance/bundle/resources/jobs/for_each_inputs/literals/script index 7d31170c837..ad012477038 100644 --- a/acceptance/bundle/resources/jobs/for_each_inputs/literals/script +++ b/acceptance/bundle/resources/jobs/for_each_inputs/literals/script @@ -1,12 +1,10 @@ envsubst < databricks.yml.tmpl > databricks.yml -title "Root arrays serialize to JSON strings and string variables retain their formatting" >&2 trace $CLI bundle validate -o json | jq '.resources.jobs.fan.tasks | map({task_key, inputs: .for_each_task.inputs})' trace $CLI bundle deploy trace print_requests.py //jobs --nostamp -title "A target array replaces the root array for the same task key" >&2 trace $CLI bundle validate -o json -t override | jq '.resources.jobs.fan.tasks | map({task_key, inputs: .for_each_task.inputs})' trace $CLI bundle deploy -t override From 85d6e5521aa03657544e4add013d28625d789239 Mon Sep 17 00:00:00 2001 From: Lennart Kats Date: Fri, 13 Nov 2026 12:02:55 +0000 Subject: [PATCH 7/8] Normalize Windows paths in for-each array reference acceptance output --- .../resources/jobs/for_each_inputs/array_reference/test.toml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 acceptance/bundle/resources/jobs/for_each_inputs/array_reference/test.toml diff --git a/acceptance/bundle/resources/jobs/for_each_inputs/array_reference/test.toml b/acceptance/bundle/resources/jobs/for_each_inputs/array_reference/test.toml new file mode 100644 index 00000000000..e9cede5e28a --- /dev/null +++ b/acceptance/bundle/resources/jobs/for_each_inputs/array_reference/test.toml @@ -0,0 +1,4 @@ +# Errors for invalid YAML print the full path of the file; normalize Windows separators. +[[Repls]] +Old = '\\' +New = '/' From 2e23b883d4e2ded51bcf9dccb8536e4b9f93df7e Mon Sep 17 00:00:00 2001 From: Lennart Kats Date: Fri, 13 Nov 2026 14:49:34 +0000 Subject: [PATCH 8/8] Clarify literal array support for for-each task inputs --- .nextchanges/bundles/for-each-task-yaml-inputs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/bundles/for-each-task-yaml-inputs.md b/.nextchanges/bundles/for-each-task-yaml-inputs.md index d423c6a9a0d..5ee5220d05c 100644 --- a/.nextchanges/bundles/for-each-task-yaml-inputs.md +++ b/.nextchanges/bundles/for-each-task-yaml-inputs.md @@ -1 +1 @@ -* Support YAML arrays for job for-each task inputs. ([#7304](https://github.com/databricks/cli/pull/7304)) +* Support literal YAML arrays directly in job for-each task inputs. Use the JSON string form when inputs need variable interpolation. ([#7304](https://github.com/databricks/cli/pull/7304))