From c8f091ddfaabec75c47fc7db61313d1b197fe222 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Mon, 20 Jul 2026 11:12:42 +0200 Subject: [PATCH 1/7] libs/dyn/convert: suppress "unknown field" warnings for anchor containers Bundle validation already suppresses "unknown field" warnings for standalone YAML anchors, but still warns when anchors are grouped inside a list or map, the common pattern of collecting reusable blocks under an x-* key: x-anchors: - &a { name: foo } - &b { name: bar } This adds a recursive isAnchorContainer check so a non-empty sequence or map whose elements are all anchors (or all anchor containers) is treated the same as a standalone anchor and does not trigger the warning. Empty containers and containers with any non-anchor element still warn as before. The unknown field is dropped from the normalized output regardless; only the spurious warning is suppressed. Co-authored-by: Isaac --- .../suppress-anchor-container-warnings.md | 1 + .../validate/anchor_containers/databricks.yml | 31 ++++++++ .../validate/anchor_containers/out.test.toml | 5 ++ .../validate/anchor_containers/output.txt | 9 +++ .../bundle/validate/anchor_containers/script | 1 + libs/dyn/convert/normalize.go | 32 +++++++- libs/dyn/convert/normalize_test.go | 73 +++++++++++++++++++ 7 files changed, 151 insertions(+), 1 deletion(-) create mode 100644 .nextchanges/bundles/suppress-anchor-container-warnings.md create mode 100644 acceptance/bundle/validate/anchor_containers/databricks.yml create mode 100644 acceptance/bundle/validate/anchor_containers/out.test.toml create mode 100644 acceptance/bundle/validate/anchor_containers/output.txt create mode 100644 acceptance/bundle/validate/anchor_containers/script diff --git a/.nextchanges/bundles/suppress-anchor-container-warnings.md b/.nextchanges/bundles/suppress-anchor-container-warnings.md new file mode 100644 index 00000000000..7e08214870b --- /dev/null +++ b/.nextchanges/bundles/suppress-anchor-container-warnings.md @@ -0,0 +1 @@ +* Do not emit "unknown field" warnings for YAML anchors grouped under `x-*` keys as a list or map, matching the existing suppression for standalone anchors. diff --git a/acceptance/bundle/validate/anchor_containers/databricks.yml b/acceptance/bundle/validate/anchor_containers/databricks.yml new file mode 100644 index 00000000000..ae79d990c96 --- /dev/null +++ b/acceptance/bundle/validate/anchor_containers/databricks.yml @@ -0,0 +1,31 @@ +bundle: + name: test-bundle + +# This test verifies that anchors grouped under x-* keys as a list or a map do +# not trigger "unknown field" warnings. The suppression recurses into containers +# whose elements are all anchors. + +x-task-defaults: &task_defaults + max_retries: 3 + +x-parameter-items: + - &p_root + name: root_path + default: ${workspace.root_path} + - &p_core_whl + name: core_whl + default: ${var.core_whl} + +x-parameter-map: + root: &p_root_map + name: root_path + +resources: + jobs: + my_job: + name: "test job with anchors" + tasks: + - task_key: "main" + <<: *task_defaults + notebook_task: + notebook_path: "/some/notebook" diff --git a/acceptance/bundle/validate/anchor_containers/out.test.toml b/acceptance/bundle/validate/anchor_containers/out.test.toml new file mode 100644 index 00000000000..67759662971 --- /dev/null +++ b/acceptance/bundle/validate/anchor_containers/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false +GOOSOnPR.darwin = false +GOOSOnPR.windows = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/anchor_containers/output.txt b/acceptance/bundle/validate/anchor_containers/output.txt new file mode 100644 index 00000000000..d44a21b582a --- /dev/null +++ b/acceptance/bundle/validate/anchor_containers/output.txt @@ -0,0 +1,9 @@ + +>>> [CLI] bundle validate +Name: test-bundle +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Validation OK! diff --git a/acceptance/bundle/validate/anchor_containers/script b/acceptance/bundle/validate/anchor_containers/script new file mode 100644 index 00000000000..5350876150f --- /dev/null +++ b/acceptance/bundle/validate/anchor_containers/script @@ -0,0 +1 @@ +trace $CLI bundle validate diff --git a/libs/dyn/convert/normalize.go b/libs/dyn/convert/normalize.go index 79cfee37441..70583456b85 100644 --- a/libs/dyn/convert/normalize.go +++ b/libs/dyn/convert/normalize.go @@ -87,6 +87,36 @@ func typeMismatch(expected dyn.Kind, src dyn.Value, path dyn.Path) diag.Diagnost } } +// isAnchorContainer reports whether v is a YAML anchor or a non-empty +// sequence/map composed entirely of anchor containers. Anchors define reusable +// blocks (commonly grouped under x-* keys as a list or map) and must not trigger +// "unknown field" warnings, including when nested inside a container. +func isAnchorContainer(v dyn.Value) bool { + if v.IsAnchor() { + return true + } + + var elements []dyn.Value + switch v.Kind() { + case dyn.KindSequence: + elements = v.MustSequence() + case dyn.KindMap: + elements = v.MustMap().Values() + default: + return false + } + + if len(elements) == 0 { + return false + } + for _, e := range elements { + if !isAnchorContainer(e) { + return false + } + } + return true +} + func (n normalizeOptions) normalizeStruct(typ reflect.Type, src dyn.Value, seen []reflect.Type, path dyn.Path) (dyn.Value, diag.Diagnostics) { var diags diag.Diagnostics @@ -101,7 +131,7 @@ func (n normalizeOptions) normalizeStruct(typ reflect.Type, src dyn.Value, seen fieldName := pk.MustString() index, ok := info.Fields[fieldName] if !ok { - if !pv.IsAnchor() { + if !isAnchorContainer(pv) { // Special case: provide a more helpful message for "valueFrom" vs "value_from" if fieldName == "valueFrom" { if _, hasValueFrom := info.Fields["value_from"]; hasValueFrom { diff --git a/libs/dyn/convert/normalize_test.go b/libs/dyn/convert/normalize_test.go index cf8ca061733..6019dd40d5e 100644 --- a/libs/dyn/convert/normalize_test.go +++ b/libs/dyn/convert/normalize_test.go @@ -860,6 +860,79 @@ func TestNormalizeAnchors(t *testing.T) { }, vout.AsAny()) } +func TestNormalizeAnchorContainers(t *testing.T) { + type Tmp struct { + Foo string `json:"foo"` + } + + anchor := func() dyn.Value { + return dyn.V(map[string]dyn.Value{"name": dyn.V("x")}).MarkAnchor() + } + + tcases := []struct { + name string + value dyn.Value + wantWarn bool + }{ + { + name: "list of anchors", + value: dyn.V([]dyn.Value{anchor(), anchor()}), + wantWarn: false, + }, + { + name: "map of anchors", + value: dyn.V(map[string]dyn.Value{ + "a": anchor(), + "b": anchor(), + }), + wantWarn: false, + }, + { + name: "nested list of anchors", + value: dyn.V([]dyn.Value{dyn.V([]dyn.Value{anchor()})}), + wantWarn: false, + }, + { + name: "list with a non-anchor element", + value: dyn.V([]dyn.Value{anchor(), dyn.V(map[string]dyn.Value{"name": dyn.V("x")})}), + wantWarn: true, + }, + { + name: "empty list", + value: dyn.V([]dyn.Value{}), + wantWarn: true, + }, + { + name: "empty map", + value: dyn.V(map[string]dyn.Value{}), + wantWarn: true, + }, + } + + for _, tc := range tcases { + t.Run(tc.name, func(t *testing.T) { + var typ Tmp + vin := dyn.V(map[string]dyn.Value{ + "foo": dyn.V("bar"), + "x-thing": tc.value, + }) + + vout, diags := Normalize(typ, vin) + if tc.wantWarn { + assert.Len(t, diags, 1) + assert.Equal(t, "unknown field: x-thing", diags[0].Summary) + } else { + assert.Empty(t, diags) + } + + // The unknown field is never retained regardless of the warning. + assert.Equal(t, map[string]any{ + "foo": "bar", + }, vout.AsAny()) + }) + } +} + func TestNormalizeAnyFromSlice(t *testing.T) { var typ any v1 := dyn.NewValue(1, []dyn.Location{{File: "file", Line: 1, Column: 1}}) From dd36efa13344bfb831aa948aac785fe04253bda8 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Mon, 20 Jul 2026 11:14:39 +0200 Subject: [PATCH 2/7] Make changelog entry more generic Co-authored-by: Isaac --- .nextchanges/bundles/suppress-anchor-container-warnings.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/bundles/suppress-anchor-container-warnings.md b/.nextchanges/bundles/suppress-anchor-container-warnings.md index 7e08214870b..b673d1c858d 100644 --- a/.nextchanges/bundles/suppress-anchor-container-warnings.md +++ b/.nextchanges/bundles/suppress-anchor-container-warnings.md @@ -1 +1 @@ -* Do not emit "unknown field" warnings for YAML anchors grouped under `x-*` keys as a list or map, matching the existing suppression for standalone anchors. +* Do not emit "unknown field" warnings for YAML anchors grouped in a list or map, matching the existing suppression for standalone anchors. From 12eb3c1ac6d1642673ce905115715c40fbf328df Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Mon, 20 Jul 2026 11:15:25 +0200 Subject: [PATCH 3/7] Use generic anchor keys in acceptance test The suppression does not depend on the x-* prefix, so use plain keys. Co-authored-by: Isaac --- .../bundle/validate/anchor_containers/databricks.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/acceptance/bundle/validate/anchor_containers/databricks.yml b/acceptance/bundle/validate/anchor_containers/databricks.yml index ae79d990c96..2cbbbd32fcc 100644 --- a/acceptance/bundle/validate/anchor_containers/databricks.yml +++ b/acceptance/bundle/validate/anchor_containers/databricks.yml @@ -1,14 +1,14 @@ bundle: name: test-bundle -# This test verifies that anchors grouped under x-* keys as a list or a map do -# not trigger "unknown field" warnings. The suppression recurses into containers -# whose elements are all anchors. +# This test verifies that anchors grouped in a list or a map do not trigger +# "unknown field" warnings. The suppression recurses into containers whose +# elements are all anchors. -x-task-defaults: &task_defaults +task_defaults: &task_defaults max_retries: 3 -x-parameter-items: +parameter_items: - &p_root name: root_path default: ${workspace.root_path} @@ -16,7 +16,7 @@ x-parameter-items: name: core_whl default: ${var.core_whl} -x-parameter-map: +parameter_map: root: &p_root_map name: root_path From 4ddf4912785779f3f2d31fa9f090e8e01725ef8a Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Mon, 20 Jul 2026 11:16:02 +0200 Subject: [PATCH 4/7] Use generic anchor names and values in acceptance test Co-authored-by: Isaac --- .../validate/anchor_containers/databricks.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/acceptance/bundle/validate/anchor_containers/databricks.yml b/acceptance/bundle/validate/anchor_containers/databricks.yml index 2cbbbd32fcc..b92721c2b16 100644 --- a/acceptance/bundle/validate/anchor_containers/databricks.yml +++ b/acceptance/bundle/validate/anchor_containers/databricks.yml @@ -9,16 +9,16 @@ task_defaults: &task_defaults max_retries: 3 parameter_items: - - &p_root - name: root_path - default: ${workspace.root_path} - - &p_core_whl - name: core_whl - default: ${var.core_whl} + - &p_first + name: first + default: one + - &p_second + name: second + default: two parameter_map: - root: &p_root_map - name: root_path + entry: &p_map_entry + name: first resources: jobs: From 286272a34d69e1b5d618b051f7a139f84c5c87a6 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Mon, 20 Jul 2026 11:19:15 +0200 Subject: [PATCH 5/7] Add PR link to changelog entry Co-authored-by: Isaac --- .nextchanges/bundles/suppress-anchor-container-warnings.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/bundles/suppress-anchor-container-warnings.md b/.nextchanges/bundles/suppress-anchor-container-warnings.md index b673d1c858d..f376f6da499 100644 --- a/.nextchanges/bundles/suppress-anchor-container-warnings.md +++ b/.nextchanges/bundles/suppress-anchor-container-warnings.md @@ -1 +1 @@ -* Do not emit "unknown field" warnings for YAML anchors grouped in a list or map, matching the existing suppression for standalone anchors. +* Do not emit "unknown field" warnings for YAML anchors grouped in a list or map, matching the existing suppression for standalone anchors ([#5975](https://github.com/databricks/cli/pull/5975)). From 9ee6199aa632f7f4ae280d549e90aca6c74b137b Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Tue, 21 Jul 2026 15:52:22 +0200 Subject: [PATCH 6/7] Cover nested and mixed anchor containers in acceptance test The acceptance test only exercised the no-warning path. Extend it to also cover a container nested inside a container and a mixed container that holds a non-anchor element (which is reported as an unknown field), documenting the rationale for not suppressing the mixed case. Co-authored-by: Isaac --- .../validate/anchor_containers/databricks.yml | 19 ++++++++++++++++++- .../validate/anchor_containers/output.txt | 5 ++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/validate/anchor_containers/databricks.yml b/acceptance/bundle/validate/anchor_containers/databricks.yml index b92721c2b16..a82d4fe7202 100644 --- a/acceptance/bundle/validate/anchor_containers/databricks.yml +++ b/acceptance/bundle/validate/anchor_containers/databricks.yml @@ -3,11 +3,14 @@ bundle: # This test verifies that anchors grouped in a list or a map do not trigger # "unknown field" warnings. The suppression recurses into containers whose -# elements are all anchors. +# elements are all anchors. A container that also holds a non-anchor element is +# not suppressed: it is reported as an unknown field, since it is more likely a +# misplaced configuration value than a block of reusable anchors. task_defaults: &task_defaults max_retries: 3 +# List of anchors. parameter_items: - &p_first name: first @@ -16,10 +19,24 @@ parameter_items: name: second default: two +# Map of anchors. parameter_map: entry: &p_map_entry name: first +# Container nested inside a container: a map whose value is a list of anchors. +nested_parameters: + group: + - &p_nested + name: nested + +# Mixed container: one anchor and one non-anchor element. This is reported as an +# unknown field because not all elements are anchor containers. +mixed_parameters: + - &p_mixed + name: mixed + - name: plain + resources: jobs: my_job: diff --git a/acceptance/bundle/validate/anchor_containers/output.txt b/acceptance/bundle/validate/anchor_containers/output.txt index d44a21b582a..86ae905e85f 100644 --- a/acceptance/bundle/validate/anchor_containers/output.txt +++ b/acceptance/bundle/validate/anchor_containers/output.txt @@ -1,9 +1,12 @@ >>> [CLI] bundle validate +Warning: unknown field: mixed_parameters + in databricks.yml:35:1 + Name: test-bundle Target: default Workspace: User: [USERNAME] Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default -Validation OK! +Found 1 warning From 1365452b47579c4558687c077f738e870ec62da5 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Tue, 21 Jul 2026 16:01:07 +0200 Subject: [PATCH 7/7] Drop x- prefix references from anchor container tests Remove the parenthetical about x-* keys from the isAnchorContainer doc comment and rename the unit test's unknown field from x-thing to thing, so neither uses the x- prefix. Co-authored-by: Isaac --- libs/dyn/convert/normalize.go | 4 ++-- libs/dyn/convert/normalize_test.go | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/libs/dyn/convert/normalize.go b/libs/dyn/convert/normalize.go index 70583456b85..b6a5fffd07e 100644 --- a/libs/dyn/convert/normalize.go +++ b/libs/dyn/convert/normalize.go @@ -89,8 +89,8 @@ func typeMismatch(expected dyn.Kind, src dyn.Value, path dyn.Path) diag.Diagnost // isAnchorContainer reports whether v is a YAML anchor or a non-empty // sequence/map composed entirely of anchor containers. Anchors define reusable -// blocks (commonly grouped under x-* keys as a list or map) and must not trigger -// "unknown field" warnings, including when nested inside a container. +// blocks and must not trigger "unknown field" warnings, including when nested +// inside a container. func isAnchorContainer(v dyn.Value) bool { if v.IsAnchor() { return true diff --git a/libs/dyn/convert/normalize_test.go b/libs/dyn/convert/normalize_test.go index 6019dd40d5e..1cde66473b2 100644 --- a/libs/dyn/convert/normalize_test.go +++ b/libs/dyn/convert/normalize_test.go @@ -913,14 +913,14 @@ func TestNormalizeAnchorContainers(t *testing.T) { t.Run(tc.name, func(t *testing.T) { var typ Tmp vin := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("bar"), - "x-thing": tc.value, + "foo": dyn.V("bar"), + "thing": tc.value, }) vout, diags := Normalize(typ, vin) if tc.wantWarn { assert.Len(t, diags, 1) - assert.Equal(t, "unknown field: x-thing", diags[0].Summary) + assert.Equal(t, "unknown field: thing", diags[0].Summary) } else { assert.Empty(t, diags) }