postgres: mask the fields of a message added to config, not the message

.nextchanges/bundles/postgres-mask-new-message.md

@@ -0,0 +1 @@
+* Fixed a Lakebase (`postgres_*`) deploy failing with `Field ... is in update_mask but not provided in request` when a nested block such as `default_endpoint_settings` was added to the configuration and left some of its fields unset. ([#6374](https://github.com/databricks/cli/pull/6374))

acceptance/bundle/resources/postgres_projects/add_default_endpoint_settings/out.requests.direct.json

@@ -0,0 +1,18 @@
+{
+  "method": "PATCH",
+  "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]",
+  "q": {
+    "update_mask": "spec.default_endpoint_settings.autoscaling_limit_max_cu,spec.default_endpoint_settings.autoscaling_limit_min_cu"
+  },
+  "body": {
+    "spec": {
+      "default_endpoint_settings": {
+        "autoscaling_limit_max_cu": 4,
+        "autoscaling_limit_min_cu": 0.5
+      },
+      "display_name": "Test Project for Adding Default Endpoint Settings",
+      "history_retention_duration": "604800s",
+      "pg_version": 16
+    }
+  }
+}

acceptance/bundle/resources/postgres_projects/add_default_endpoint_settings/out.requests.terraform.json

@@ -0,0 +1,19 @@
+{
+  "method": "PATCH",
+  "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]",
+  "q": {
+    "update_mask": "initial_branch_spec,initial_endpoint_spec,spec"
+  },
+  "body": {
+    "name": "projects/test-pg-proj-[UNIQUE_NAME]",
+    "spec": {
+      "default_endpoint_settings": {
+        "autoscaling_limit_max_cu": 4,
+        "autoscaling_limit_min_cu": 0.5
+      },
+      "display_name": "Test Project for Adding Default Endpoint Settings",
+      "history_retention_duration": "604800s",
+      "pg_version": 16
+    }
+  }
+}

acceptance/bundle/resources/postgres_projects/add_default_endpoint_settings/output.txt

@@ -8,6 +8,35 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
 
 === Add the whole default_endpoint_settings block and re-deploy
 >>> update_file.py databricks.yml #TO_ADD:  
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files...
+Updated postgres_projects.my_project
+Files: 0 uploaded, 0 deleted
+Resources: 0 created, 1 changed, 0 deleted, 0 unchanged
+
+>>> print_requests.py --del-body project_id //postgres/projects
+
+>>> [CLI] postgres get-project projects/test-pg-proj-[UNIQUE_NAME]
+{
+  "name": "projects/test-pg-proj-[UNIQUE_NAME]",
+  "project_id": "test-pg-proj-[UNIQUE_NAME]",
+  "status": {
+    "branch_logical_size_limit_bytes": [NUMID],
+    "default_branch": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production",
+    "default_endpoint_settings": {
+      "autoscaling_limit_max_cu": 4,
+      "autoscaling_limit_min_cu": 0.5,
+      "suspend_timeout_duration": "86400s"
+    },
+    "display_name": "Test Project for Adding Default Endpoint Settings",
+    "enable_pg_native_login": false,
+    "history_retention_duration": "604800s",
+    "owner": "[USERNAME]",
+    "pg_version": 16,
+    "project_id": "test-pg-proj-[UNIQUE_NAME]",
+    "synthetic_storage_size_bytes": 0
+  },
+  "uid": "[UUID]"
+}
 
 >>> [CLI] bundle destroy --auto-approve
 The following resources will be deleted:

acceptance/bundle/resources/postgres_projects/add_default_endpoint_settings/script

@@ -6,21 +6,26 @@ cleanup() {
 }
 trap cleanup EXIT
 
+project_name="projects/test-pg-proj-${UNIQUE_NAME}"
+
 title "Initial deployment with no default_endpoint_settings block"
 trace $CLI bundle deploy
 rm -f out.requests.txt
 
 title "Add the whole default_endpoint_settings block and re-deploy"
 trace update_file.py databricks.yml "#TO_ADD: " ""
 
-# The direct plan's "changes" map is the structdiff artifact: it records one
-# change at path default_endpoint_settings (the whole block) instead of descending
-# to its leaves. (The top-level plan verdict is 'update' on both engines.)
+# The block is added as a whole, so the direct plan's "changes" map records one change
+# at path default_endpoint_settings rather than at its leaves. Masking the message asks
+# the API to replace it wholesale, which then requires the suspension oneof in the body;
+# the mask is expanded to the leaves the body carries instead (see dresources/util.go).
 $CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json
 
-# The consequence shows up at deploy time, which diverges by engine, so route it to
-# per-engine files. Masking the whole message means the API requires the suspension
-# oneof in the body (see dresources/util.go), so the direct deploy FAILS with a 400;
-# terraform masks the leaf spec.default_endpoint_settings.autoscaling_limit_max_cu
-# and succeeds.
-errcode $CLI bundle deploy > out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1
+$CLI bundle plan -o json > tmp.plan.json
+# Not traced: the command line carries --plan only in the saved-plan run, and every
+# variant must produce the same output.txt.
+$CLI bundle deploy $(readplanarg tmp.plan.json)
+
+trace print_requests.py --del-body project_id '//postgres/projects' > out.requests.$DATABRICKS_BUNDLE_ENGINE.json
+
+trace $CLI postgres get-project "${project_name}" | project_fields

acceptance/bundle/resources/postgres_projects/add_default_endpoint_settings/test.toml

@@ -1,7 +1,6 @@
-# Inherits Cloud = true from the parent. On a real workspace this validates that the
-# API still rejects a message-level update_mask exactly as the testserver models it
-# (libs/testserver/postgres.go, probed 2026-08-31): the direct deploy is expected to
-# 400 there (tolerated by errcode), while terraform masks the leaf and succeeds. Runs
-# on both engines; the divergent plan/deploy are routed to per-engine files.
+# Inherits Cloud = true from the parent, so this also runs against a real workspace:
+# it confirms the API accepts the leaf mask this fix produces. Runs on both engines;
+# the plan and the recorded requests diverge by engine, so both go to per-engine files.
+EnvMatrix.READPLAN = ["", "1"]
 
-Badness = "On the direct engine, adding the whole default_endpoint_settings block produces update_mask=spec.default_endpoint_settings (the enclosing message) instead of the leaf spec.default_endpoint_settings.autoscaling_limit_max_cu, and the deploy FAILS with 400: 'Field spec.default_endpoint_settings.suspension is in update_mask but not provided in request'. Root cause: structdiff records one change at the block level instead of descending to leaves, so collectUpdatePathsWithPrefix masks the message. Terraform masks the leaf and succeeds."
+Ignore = ["tmp.plan.json"]

bundle/direct/dresources/postgres_branch.go

@@ -156,7 +156,7 @@ func (r *ResourcePostgresBranch) DoUpdate(ctx context.Context, id string, config
 	// expands to nested attributes the body would have to set too — so we
 	// can't use a static all-fields mask. The change list naturally tracks
 	// what the user actually set, so the body and mask stay consistent.
-	fieldPaths := collectUpdatePathsWithPrefix(entry.Changes, "spec.", branchOneofGroups)
+	fieldPaths := collectUpdatePathsWithPrefix(entry.Changes, "spec.", branchOneofGroups, &config.BranchSpec)
 
 	// purge_on_delete is an input-only flag consulted at delete time; it is
 	// not a spec field. Strip it from the mask so toggling it between deploys

bundle/direct/dresources/postgres_database.go

@@ -135,7 +135,7 @@ func (r *ResourcePostgresDatabase) DoUpdate(ctx context.Context, id string, conf
 	// This excludes immutable fields and fields that haven't changed.
 	// Prefix with "spec." because the API expects paths relative to the Database object,
 	// not relative to our flattened state type.
-	fieldPaths := collectUpdatePathsWithPrefix(entry.Changes, "spec.", nil)
+	fieldPaths := collectUpdatePathsWithPrefix(entry.Changes, "spec.", nil, &config.DatabaseDatabaseSpec)
 
 	waiter, err := r.client.Postgres.UpdateDatabase(ctx, postgres.UpdateDatabaseRequest{
 		Database: postgres.Database{

bundle/direct/dresources/postgres_endpoint.go

@@ -188,7 +188,7 @@ func (r *ResourcePostgresEndpoint) DoUpdate(ctx context.Context, id string, conf
 	// This excludes immutable fields and fields that haven't changed.
 	// Prefix with "spec." because the API expects paths relative to the Endpoint object,
 	// not relative to our flattened state type.
-	fieldPaths := collectUpdatePathsWithPrefix(entry.Changes, "spec.", endpointOneofGroups)
+	fieldPaths := collectUpdatePathsWithPrefix(entry.Changes, "spec.", endpointOneofGroups, &config.EndpointSpec)
 
 	waiter, err := r.client.Postgres.UpdateEndpoint(ctx, postgres.UpdateEndpointRequest{
 		Endpoint: postgres.Endpoint{

bundle/direct/dresources/postgres_project.go

@@ -150,7 +150,7 @@ func (r *ResourcePostgresProject) DoUpdate(ctx context.Context, id string, confi
 	// expands to nested attributes the body would have to set too — so we
 	// can't use a static all-fields mask. The change list naturally tracks
 	// what the user actually set, so the body and mask stay consistent.
-	fieldPaths := collectUpdatePathsWithPrefix(entry.Changes, "spec.", projectOneofGroups)
+	fieldPaths := collectUpdatePathsWithPrefix(entry.Changes, "spec.", projectOneofGroups, &config.ProjectSpec)
 
 	// purge_on_delete is an input-only flag consulted at delete time; it is
 	// not a spec field. Strip it from the mask so toggling it between deploys

bundle/direct/dresources/postgres_role.go

@@ -158,7 +158,7 @@ func (r *ResourcePostgresRole) DoUpdate(ctx context.Context, id string, config *
 	// Build update mask from fields that have action="update" in the changes map.
 	// Prefix with "spec." because the API expects paths relative to the Role
 	// object, not relative to our flattened state type.
-	fieldPaths := collectUpdatePathsWithPrefix(entry.Changes, "spec.", nil)
+	fieldPaths := collectUpdatePathsWithPrefix(entry.Changes, "spec.", nil, &config.RoleRoleSpec)
 
 	waiter, err := r.client.Postgres.UpdateRole(ctx, postgres.UpdateRoleRequest{
 		Name: id,

bundle/direct/dresources/util.go

@@ -1,11 +1,15 @@
 package dresources
 
 import (
+	"encoding/json"
 	"errors"
+	"reflect"
 	"slices"
 	"strings"
 
 	"github.com/databricks/cli/bundle/deployplan"
+	"github.com/databricks/cli/libs/structs/structaccess"
+	"github.com/databricks/cli/libs/structs/structtag"
 	"github.com/databricks/databricks-sdk-go/retries"
 )
 
@@ -37,8 +41,13 @@ func shouldRetry(err error) bool {
 // oneofGroups renames a change path to the group it belongs to, for fields the API only
 // accepts under their oneof group name; see the per-resource maps below.
 //
+// spec is the value the request body carries. A change that lands on a message is
+// expanded against it rather than against the plan's own copy of the new value,
+// because a plan read back from disk carries that copy as deserialized JSON, with the
+// types the expansion needs erased.
+//
 // Sorted, so the generated update_mask does not depend on map iteration order.
-func collectUpdatePathsWithPrefix(changes Changes, prefix string, oneofGroups map[string]string) []string {
+func collectUpdatePathsWithPrefix(changes Changes, prefix string, oneofGroups map[string]string, spec any) []string {
 	var paths []string
 	for path, change := range changes {
 		if change.Action != deployplan.Update {
@@ -56,10 +65,15 @@ func collectUpdatePathsWithPrefix(changes Changes, prefix string, oneofGroups ma
 		}
 		if !hasChild {
 			masked := maskPath(path)
-			if group, ok := oneofGroups[masked]; ok {
-				masked = group
+			// A change that lands on a message rather than on a field inside it masks the
+			// leaves the body populates, not the message: see expandMaskedMessage.
+			if leaves := expandMaskedMessage(masked, specValue(spec, masked)); len(leaves) > 0 {
+				for _, leaf := range leaves {
+					paths = append(paths, prefix+oneofGroup(oneofGroups, leaf))
+				}
+				continue
 			}
-			paths = append(paths, prefix+masked)
+			paths = append(paths, prefix+oneofGroup(oneofGroups, masked))
 		}
 	}
 	slices.Sort(paths)
@@ -68,6 +82,140 @@ func collectUpdatePathsWithPrefix(changes Changes, prefix string, oneofGroups ma
 	return slices.Compact(paths)
 }
 
+// oneofGroup returns the group a field is masked under, or the path itself when the field
+// does not belong to a oneof.
+func oneofGroup(oneofGroups map[string]string, path string) string {
+	if group, ok := oneofGroups[path]; ok {
+		return group
+	}
+	return path
+}
+
+// specValue returns the value spec carries at path, or nil when the path names
+// something that is not part of the spec, such as an input-only state field.
+func specValue(spec any, path string) any {
+	value, err := structaccess.GetByString(spec, path)
+	if err != nil {
+		return nil
+	}
+	return value
+}
+
+// expandMaskedMessage returns the paths of the leaves that value populates, relative to
+// path, or nil when value is not a nested message so that the caller keeps path as it is.
+//
+// Masking a message asks the API to replace it wholesale, and the API then requires every
+// field under it to be populated in the request body — which a bundle that declares only
+// some of them never satisfies. A change lands on the message itself, rather than on a
+// field inside it, when the block is added to the configuration as a whole. Verified
+// against a real workspace on 2026-09-03, where a config growing a
+// default_endpoint_settings block with just an autoscaling limit produced
+// update_mask=spec.default_endpoint_settings and was rejected with "Field
+// 'spec.default_endpoint_settings.suspension' is in update_mask but not provided in
+// request".
+//
+// A map or a repeated field is addressable only as a whole (see maskPath), and a wrapper
+// such as duration.Duration is a struct in Go but a scalar on the wire. Neither expands.
+func expandMaskedMessage(path string, value any) []string {
+	typ := reflect.TypeOf(value)
+	body, ok := messageBody(typ, value)
+	if !ok {
+		return nil
+	}
+	if bodyPopulatesAll(derefType(typ), body) {
+		// Nothing under the message is left unset, so no requirement the API places
+		// on a masked field can go unmet, and replacing the message wholesale is
+		// what the configuration declares. Keep the message.
+		return nil
+	}
+	var paths []string
+	appendPopulatedLeaves(&paths, path, derefType(typ), body)
+	return paths
+}
+
+// bodyPopulatesAll reports whether body carries a value for every field of typ, at
+// any depth.
+func bodyPopulatesAll(typ reflect.Type, body map[string]any) bool {
+	for field := range typ.Fields() {
+		if !field.IsExported() || field.Name == "ForceSendFields" {
+			continue
+		}
+		name := structtag.JSONTag(field.Tag.Get("json")).Name()
+		if name == "" || name == "-" {
+			continue
+		}
+		value, present := body[name]
+		if !present {
+			return false
+		}
+		nested, isMessage := value.(map[string]any)
+		fieldType := derefType(field.Type)
+		if isMessage && fieldType.Kind() == reflect.Struct && !bodyPopulatesAll(fieldType, nested) {
+			return false
+		}
+	}
+	return true
+}
+
+// messageBody returns value marshalled as a JSON object, reporting false when value is not
+// a struct or does not marshal to an object.
+func messageBody(typ reflect.Type, value any) (map[string]any, bool) {
+	if typ == nil || derefType(typ).Kind() != reflect.Struct {
+		return nil, false
+	}
+	raw, err := json.Marshal(value)
+	if err != nil {
+		return nil, false
+	}
+	var body map[string]any
+	if err := json.Unmarshal(raw, &body); err != nil {
+		return nil, false
+	}
+	return body, true
+}
+
+// appendPopulatedLeaves appends the path of every leaf present in body. A field is a leaf
+// unless it is a struct in Go and an object on the wire, which is what a nested message is.
+func appendPopulatedLeaves(paths *[]string, prefix string, typ reflect.Type, body map[string]any) {
+	for name, value := range body {
+		path := prefix + "." + name
+		fieldType, found := jsonFieldType(typ, name)
+		nested, isObject := value.(map[string]any)
+		if found && isObject && derefType(fieldType).Kind() == reflect.Struct {
+			appendPopulatedLeaves(paths, path, derefType(fieldType), nested)
+			continue
+		}
+		*paths = append(*paths, path)
+	}
+}
+
+// jsonFieldType returns the type of the field typ serializes under the given JSON name.
+func jsonFieldType(typ reflect.Type, name string) (reflect.Type, bool) {
+	for field := range typ.Fields() {
+		if field.Anonymous {
+			embedded := derefType(field.Type)
+			if embedded.Kind() != reflect.Struct {
+				continue
+			}
+			if fieldType, ok := jsonFieldType(embedded, name); ok {
+				return fieldType, true
+			}
+			continue
+		}
+		if structtag.JSONTag(field.Tag.Get("json")).Name() == name {
+			return field.Type, true
+		}
+	}
+	return nil, false
+}
+
+func derefType(typ reflect.Type) reflect.Type {
+	for typ.Kind() == reflect.Pointer {
+		typ = typ.Elem()
+	}
+	return typ
+}
+
 // maskPath converts a change path into the path the API accepts in update_mask. A map or
 // repeated field is addressable only as a whole, so everything from the first subscript
 // onwards is dropped: settings.pg_settings['work_mem'] is masked as settings.pg_settings.