config-remote-sync: skip permissions/grants sub-resources instead of failing

acceptance/bundle/config-remote-sync/skip_permissions/databricks.yml.tmpl

@@ -0,0 +1,26 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+# Bundle-level permissions are applied to every resource on deploy but have no
+# per-resource YAML node. config-remote-sync must skip the resulting
+# permissions sub-resource instead of failing to resolve it.
+permissions:
+  - level: CAN_MANAGE
+    user_name: ${workspace.current_user.userName}
+
+resources:
+  jobs:
+    my_job:
+      max_concurrent_runs: 1
+      tasks:
+        - task_key: main
+          notebook_task:
+            notebook_path: /Users/{{workspace_user_name}}/main
+          new_cluster:
+            spark_version: $DEFAULT_SPARK_VERSION
+            node_type_id: $NODE_TYPE_ID
+            num_workers: 1
+
+targets:
+  default:
+    mode: development

acceptance/bundle/config-remote-sync/skip_permissions/output.txt

@@ -0,0 +1,38 @@
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+=== Add a permission remotely (a change on the permissions sub-resource)
+
+=== Modify a regular job field remotely
+
+=== Sync: permissions are skipped, the regular field change is written
+Detected changes in 1 resource(s):
+
+Resource: resources.jobs.my_job
+  max_concurrent_runs: replace
+
+
+
+=== Configuration changes
+
+>>> diff.py databricks.yml.backup databricks.yml
+--- databricks.yml.backup
++++ databricks.yml
+@@ -12,5 +12,5 @@
+   jobs:
+     my_job:
+-      max_concurrent_runs: 1
++      max_concurrent_runs: 5
+       tasks:
+         - task_key: main
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.jobs.my_job
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default
+
+Deleting files...
+Destroy complete!

acceptance/bundle/config-remote-sync/skip_permissions/script

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+envsubst < databricks.yml.tmpl > databricks.yml
+
+cleanup() {
+  trace $CLI bundle destroy --auto-approve
+}
+trap cleanup EXIT
+
+$CLI bundle deploy
+job_id="$(read_id.py my_job)"
+
+title "Add a permission remotely (a change on the permissions sub-resource)"
+echo
+# permissions set replaces the whole ACL, so keep the deploy identity as owner
+# and add a grantee (the built-in "users" group) that is not in config: this
+# makes the permissions sub-resource show a change on the next sync.
+$CLI permissions set jobs $job_id --json '{"access_control_list":[{"permission_level":"IS_OWNER","user_name":"'"$CURRENT_USER_NAME"'"},{"permission_level":"CAN_VIEW","group_name":"users"}]}' > /dev/null
+
+title "Modify a regular job field remotely"
+echo
+edit_resource.py jobs $job_id <<EOF
+r["max_concurrent_runs"] = 5
+EOF
+
+title "Sync: permissions are skipped, the regular field change is written"
+echo
+cp databricks.yml databricks.yml.backup
+$CLI bundle config-remote-sync --save
+
+title "Configuration changes"
+echo
+trace diff.py databricks.yml.backup databricks.yml
+rm databricks.yml.backup

acceptance/bundle/config-remote-sync/skip_permissions/test.toml

@@ -0,0 +1,10 @@
+Cloud = true
+
+RecordRequests = false
+Ignore = [".databricks", "databricks.yml", "databricks.yml.backup"]
+
+[Env]
+DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_YAML_SYNC = "true"
+
+[EnvMatrix]
+DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"]

bundle/configsync/diff.go

@@ -8,6 +8,7 @@ import (
 	"io/fs"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/config/engine"
@@ -159,6 +160,16 @@ func ExtractChanges(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan
 	changes := make(Changes)
 
 	for resourceKey, entry := range plan.Plan {
+		// permissions and grants are emitted as their own plan keys
+		// ("resources.<type>.<name>.permissions" / ".grants"; see splitResourcePath
+		// in bundle/direct/bundle_plan.go). config-remote-sync cannot write them back
+		// to YAML: their fields (e.g. object_id) are server-populated with no source
+		// location, and bundle-level permissions have no per-resource YAML node at all,
+		// so resolving them would fail the whole sync. Skip them instead.
+		if strings.HasSuffix(resourceKey, ".permissions") || strings.HasSuffix(resourceKey, ".grants") {
+			continue
+		}
+
 		resourceChanges := make(ResourceChanges)
 
 		if entry.Changes != nil {

bundle/configsync/diff_test.go

@@ -3,6 +3,8 @@ package configsync
 import (
 	"testing"
 
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config/engine"
 	"github.com/databricks/cli/bundle/deployplan"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -247,3 +249,45 @@ func TestMatchPattern(t *testing.T) {
 		})
 	}
 }
+
+// ExtractChanges must skip permissions/grants sub-resources: they are emitted as
+// their own plan keys but cannot be written back to YAML, and resolving them
+// (e.g. the server-populated object_id field, or a bundle-level permissions entry
+// with no per-resource YAML node) otherwise fails the whole sync.
+func TestExtractChangesSkipsPermissionsAndGrants(t *testing.T) {
+	ctx := t.Context()
+
+	plan := &deployplan.Plan{Plan: map[string]*deployplan.PlanEntry{
+		"resources.jobs.my_job": {
+			Changes: deployplan.Changes{
+				"description": {Action: deployplan.Update, New: nil, Remote: "remote-desc"},
+			},
+		},
+		"resources.jobs.my_job.permissions": {
+			Changes: deployplan.Changes{
+				"object_id": {Action: deployplan.Update, New: nil, Remote: "/jobs/123"},
+			},
+		},
+		"resources.schemas.my_schema.grants": {
+			Changes: deployplan.Changes{
+				"[0].privileges": {Action: deployplan.Update, New: nil, Remote: []any{"SELECT"}},
+			},
+		},
+	}}
+
+	for _, eng := range []engine.EngineType{engine.EngineDirect, engine.EngineTerraform} {
+		t.Run(string(eng), func(t *testing.T) {
+			changes, err := ExtractChanges(ctx, &bundle.Bundle{}, plan, eng)
+			require.NoError(t, err)
+
+			_, hasPerms := changes["resources.jobs.my_job.permissions"]
+			assert.False(t, hasPerms, "permissions sub-resource must be skipped")
+			_, hasGrants := changes["resources.schemas.my_schema.grants"]
+			assert.False(t, hasGrants, "grants sub-resource must be skipped")
+
+			job, hasJob := changes["resources.jobs.my_job"]
+			require.True(t, hasJob, "regular resource changes must be kept")
+			assert.Contains(t, job, "description")
+		})
+	}
+}