From 6ad27d73191fe4b299b91fdbc053fa123fc9f07b Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Mon, 24 Aug 2026 13:20:37 +0000 Subject: [PATCH 01/55] add libraries field for Clusters --- bundle/config/resources/clusters.go | 3 +++ bundle/internal/schema/annotations.yml | 3 +++ bundle/schema/jsonschema.json | 4 ++++ 3 files changed, 10 insertions(+) diff --git a/bundle/config/resources/clusters.go b/bundle/config/resources/clusters.go index 235ea6eee1a..cb69fc6b752 100644 --- a/bundle/config/resources/clusters.go +++ b/bundle/config/resources/clusters.go @@ -18,6 +18,9 @@ type Cluster struct { // Lifecycle shadows BaseResource.Lifecycle to add support for lifecycle.started. Lifecycle *LifecycleWithStarted `json:"lifecycle,omitempty"` + // Libraries are installed via the Libraries API, not the cluster spec. + Libraries []compute.Library `json:"libraries,omitempty"` + Permissions []ClusterPermission `json:"permissions,omitempty"` } diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index e47f89c505a..e01d21460d3 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -605,6 +605,9 @@ resources: notebook_path: "./src/my_notebook.py" ``` "$fields": + "libraries": + "description": |- + A list of libraries to install on the cluster. Installed via the Libraries API after the cluster is created. Only supported in direct deployment mode. "lifecycle": "description": |- Settings that control the deployment lifecycle of the resource, such as preventing it from being destroyed. diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 425956ab06d..f32d43532f0 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -488,6 +488,10 @@ "description": "The kind of compute described by this compute specification.\n\nDepending on `kind`, different validations and default values will be applied.\n\nClusters with `kind = CLASSIC_PREVIEW` support the following fields, whereas clusters with no specified `kind` do not.\n* [is_single_node](/api/workspace/clusters/create#is_single_node)\n* [use_ml_runtime](/api/workspace/clusters/create#use_ml_runtime)\n\nBy using the [simple form](https://docs.databricks.com/compute/simple-form.html), your clusters are automatically using `kind = CLASSIC_PREVIEW`.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.Kind" }, + "libraries": { + "description": "A list of libraries to install on the cluster. Installed via the Libraries API after the cluster is created. Only supported in direct deployment mode.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/compute.Library" + }, "lifecycle": { "description": "Settings that control the deployment lifecycle of the resource, such as preventing it from being destroyed.", "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.LifecycleWithStarted" From 352669b31e2d640a4a2d139bbe7b9030d46c360d Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 25 Aug 2026 13:57:59 +0000 Subject: [PATCH 02/55] Recognize clusters.libraries as a sub-resource node Wire the direct engine to treat resources.clusters.*.libraries as a child-resource node, the same way permissions and grants are handled: node/type resolution, reference splitting, and plan node discovery. Co-authored-by: Isaac --- bundle/config/resources_types.go | 5 +++++ bundle/config/root.go | 2 +- bundle/direct/bundle_plan.go | 3 ++- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/bundle/config/resources_types.go b/bundle/config/resources_types.go index dcc91545f19..21d6d407468 100644 --- a/bundle/config/resources_types.go +++ b/bundle/config/resources_types.go @@ -43,6 +43,11 @@ var ResourcesTypes = func() map[string]reflect.Type { if resourceField.Name == "Grants" { grantsKey := name + ".grants" res[grantsKey] = resourceField.Type + continue + } + if resourceField.Name == "Libraries" { + librariesKey := name + ".libraries" + res[librariesKey] = resourceField.Type } } } diff --git a/bundle/config/root.go b/bundle/config/root.go index e13bf78bf15..0d7d03dadb3 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -620,7 +620,7 @@ func GetNodeAndType(path dyn.Path) (dyn.Path, string) { } if len(path) >= 4 { - if path[3].Key() == "permissions" || path[3].Key() == "grants" { + if path[3].Key() == "permissions" || path[3].Key() == "grants" || path[3].Key() == "libraries" { return path[:4], path[1].Key() + "." + path[3].Key() } } diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 5b8829e3f59..4d37a21e090 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -713,7 +713,7 @@ func splitResourcePath(path *structpath.PathNode) (string, *structpath.PathNode) // Check if the 4th component is "permissions" or "grants" (sub-resource) if path.Len() > 4 { first := path.SkipPrefix(3).Prefix(1) - if key, ok := first.StringKey(); ok && (key == "permissions" || key == "grants") { + if key, ok := first.StringKey(); ok && (key == "permissions" || key == "grants" || key == "libraries") { return path.Prefix(4).String(), path.SkipPrefix(4) } } @@ -930,6 +930,7 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey(), dyn.Key("permissions")), dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey(), dyn.Key("grants")), + dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey(), dyn.Key("libraries")), } // Walk? From 6d0278de8bdab2426364dc131ff8471354a85c71 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 25 Aug 2026 13:58:10 +0000 Subject: [PATCH 03/55] Add clusters.libraries direct-engine child resource Implement ResourceLibraries: installs/uninstalls cluster libraries via the Libraries API, reconciling removed libraries on update and polling for install completion on a running cluster. Registered in all.go. Note: TestAll/clusters.libraries fails until the testserver models the libraries install/uninstall/cluster-status endpoints (next step). Co-authored-by: Isaac --- bundle/direct/dresources/all.go | 3 + bundle/direct/dresources/cluster_libraries.go | 253 ++++++++++++++++++ bundle/direct/dresources/type_test.go | 1 + 3 files changed, 257 insertions(+) create mode 100644 bundle/direct/dresources/cluster_libraries.go diff --git a/bundle/direct/dresources/all.go b/bundle/direct/dresources/all.go index 391fb0684d2..4ffeed88ab2 100644 --- a/bundle/direct/dresources/all.go +++ b/bundle/direct/dresources/all.go @@ -61,6 +61,9 @@ var SupportedResources = map[string]any{ "vector_search_endpoints.permissions": (*ResourcePermissions)(nil), "instance_pools.permissions": (*ResourcePermissions)(nil), + // Libraries + "clusters.libraries": (*ResourceLibraries)(nil), + // Grants "catalogs.grants": (*ResourceGrants)(nil), "schemas.grants": (*ResourceGrants)(nil), diff --git a/bundle/direct/dresources/cluster_libraries.go b/bundle/direct/dresources/cluster_libraries.go new file mode 100644 index 00000000000..a629dfe08ee --- /dev/null +++ b/bundle/direct/dresources/cluster_libraries.go @@ -0,0 +1,253 @@ +package dresources + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/structs/structvar" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/retries" + "github.com/databricks/databricks-sdk-go/service/compute" +) + +// Corresponds to the databricks_library terraform resource: +// https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/library + +// librariesWaitTimeout bounds how long we poll for libraries to finish installing. +const librariesWaitTimeout = 15 * time.Minute + +// LibrariesState is the state for a cluster's libraries sub-resource. Libraries are installed +// via the Libraries API against the parent cluster identified by ClusterId, not through the +// cluster spec. +type LibrariesState struct { + ClusterId string `json:"cluster_id"` + // By convention EmbeddedSlice fields have the __embed__ json tag, see permissions.go. + EmbeddedSlice []compute.Library `json:"__embed__,omitempty"` +} + +type ResourceLibraries struct { + client *databricks.WorkspaceClient +} + +func (*ResourceLibraries) New(client *databricks.WorkspaceClient) *ResourceLibraries { + return &ResourceLibraries{client: client} +} + +func (r *ResourceLibraries) PrepareInputConfig(inputConfig *[]compute.Library, resourceKey string) (*structvar.StructVar, error) { + baseNode, ok := strings.CutSuffix(resourceKey, ".libraries") + if !ok { + return nil, fmt.Errorf("internal error: node %q does not end with .libraries", resourceKey) + } + + return &structvar.StructVar{ + Value: &LibrariesState{ + ClusterId: "", // Always a reference, defined in Refs below. + EmbeddedSlice: *inputConfig, + }, + Refs: map[string]string{ + "cluster_id": "${" + baseNode + ".id}", + }, + }, nil +} + +func (*ResourceLibraries) PrepareState(state *LibrariesState) *LibrariesState { + return state +} + +// IsEmptyState reports an empty libraries list as no resource at all: nothing to install, and no +// state entry is persisted for it. +func (*ResourceLibraries) IsEmptyState(state *LibrariesState) bool { + return len(state.EmbeddedSlice) == 0 +} + +// libraryKey identifies a library by its type-specific field so slices compare by identity +// rather than by index (see KeyedSlices). +func libraryKey(l compute.Library) (string, string) { + switch { + case l.Whl != "": + return "whl", l.Whl + case l.Jar != "": + return "jar", l.Jar + case l.Egg != "": + return "egg", l.Egg + case l.Requirements != "": + return "requirements", l.Requirements + case l.Pypi != nil: + return "pypi", l.Pypi.Package + case l.Maven != nil: + return "maven", l.Maven.Coordinates + case l.Cran != nil: + return "cran", l.Cran.Package + } + return "", "" +} + +func (*ResourceLibraries) KeyedSlices() map[string]any { + // Empty key because EmbeddedSlice appears at the root path of LibrariesState. + return map[string]any{ + "": libraryKey, + } +} + +func (r *ResourceLibraries) DoRead(ctx context.Context, id string) (*LibrariesState, error) { + statuses, err := r.client.Libraries.ClusterStatusByClusterId(ctx, id) + if err != nil { + return nil, err + } + + state := &LibrariesState{ClusterId: id} + for _, s := range statuses.LibraryStatuses { + // Libraries set for all clusters via the UI are not managed by the bundle + // (following the permissions convention of ignoring inherited entries). + if s.Library == nil || s.IsLibraryForAllClusters { + continue + } + // A library pending uninstall on restart is on its way out; don't report it as present. + if s.Status == compute.LibraryInstallStatusUninstallOnRestart { + continue + } + state.EmbeddedSlice = append(state.EmbeddedSlice, *s.Library) + } + return state, nil +} + +// DoCreate installs the libraries on the cluster. +// https://docs.databricks.com/api/workspace/libraries/install +func (r *ResourceLibraries) DoCreate(ctx context.Context, state *LibrariesState) (string, *LibrariesState, error) { + err := r.client.Libraries.Install(ctx, compute.InstallLibraries{ + ClusterId: state.ClusterId, + Libraries: state.EmbeddedSlice, + }) + if err != nil { + // Install is idempotent (installing an already-installed library is a no-op), + // so retrying on transient errors is safe. + return "", nil, retrySafe(err) + } + return state.ClusterId, nil, nil +} + +// DoUpdate uninstalls libraries removed from config and installs the desired set. This is two API +// calls because the Libraries API exposes install and uninstall as separate endpoints, unlike the +// single-call model most resources follow. +func (r *ResourceLibraries) DoUpdate(ctx context.Context, id string, state *LibrariesState, entry *PlanEntry) (*LibrariesState, error) { + removed := removedLibraries(state.EmbeddedSlice, entry) + if len(removed) > 0 { + err := r.client.Libraries.Uninstall(ctx, compute.UninstallLibraries{ + ClusterId: id, + Libraries: removed, + }) + if err != nil { + return nil, err + } + } + + if len(state.EmbeddedSlice) > 0 { + err := r.client.Libraries.Install(ctx, compute.InstallLibraries{ + ClusterId: id, + Libraries: state.EmbeddedSlice, + }) + if err != nil { + return nil, err + } + } + return nil, nil +} + +// DoDelete is a no-op: removing individual libraries is handled by DoUpdate's uninstall diff, and +// DoDelete only fires when the parent cluster is deleted, at which point uninstalling is moot. +func (r *ResourceLibraries) DoDelete(ctx context.Context, id string, _ *LibrariesState) error { + return nil +} + +// removedLibraries returns libraries present in the remote state but absent from the desired set. +func removedLibraries(desired []compute.Library, entry *PlanEntry) []compute.Library { + if entry == nil { + return nil + } + remote, ok := entry.RemoteState.(*LibrariesState) + if !ok || remote == nil { + return nil + } + + desiredKeys := make(map[string]struct{}, len(desired)) + for _, l := range desired { + desiredKeys[libraryMapKey(l)] = struct{}{} + } + + var result []compute.Library + for _, l := range remote.EmbeddedSlice { + if _, ok := desiredKeys[libraryMapKey(l)]; !ok { + result = append(result, l) + } + } + return result +} + +// libraryMapKey flattens libraryKey into a single string for map lookups. +func libraryMapKey(l compute.Library) string { + f, v := libraryKey(l) + return f + "=" + v +} + +func (r *ResourceLibraries) WaitAfterCreate(ctx context.Context, id string, state *LibrariesState) (*LibrariesState, error) { + return nil, r.waitForInstall(ctx, id, state.EmbeddedSlice) +} + +func (r *ResourceLibraries) WaitAfterUpdate(ctx context.Context, id string, state *LibrariesState) (*LibrariesState, error) { + return nil, r.waitForInstall(ctx, id, state.EmbeddedSlice) +} + +// waitForInstall polls until every desired library reaches a terminal installed state. It returns +// early without waiting when the cluster is not running: installs only progress on a running +// cluster and are queued until it next starts. +func (r *ResourceLibraries) waitForInstall(ctx context.Context, id string, desired []compute.Library) error { + if len(desired) == 0 { + return nil + } + + details, err := r.client.Clusters.GetByClusterId(ctx, id) + if err != nil { + return err + } + if details.State != compute.StateRunning { + log.Debugf(ctx, "cluster %s is not running (%s); skipping wait for library installation", id, details.State) + return nil + } + + desiredKeys := make(map[string]struct{}, len(desired)) + for _, l := range desired { + desiredKeys[libraryMapKey(l)] = struct{}{} + } + + _, err = retries.Poll(ctx, librariesWaitTimeout, func() (*struct{}, *retries.Err) { + statuses, err := r.client.Libraries.ClusterStatusByClusterId(ctx, id) + if err != nil { + return nil, retries.Halt(err) + } + + pending := len(desiredKeys) + for _, s := range statuses.LibraryStatuses { + if s.Library == nil { + continue + } + if _, ok := desiredKeys[libraryMapKey(*s.Library)]; !ok { + continue + } + switch s.Status { + case compute.LibraryInstallStatusFailed: + return nil, retries.Halt(fmt.Errorf("library %s failed to install: %s", libraryMapKey(*s.Library), strings.Join(s.Messages, "; "))) + case compute.LibraryInstallStatusInstalled, compute.LibraryInstallStatusSkipped, compute.LibraryInstallStatusRestored: + pending-- + } + } + + if pending > 0 { + return nil, retries.Continues(fmt.Sprintf("waiting for %d librar(ies) to install on cluster %s", pending, id)) + } + return &struct{}{}, nil + }) + return err +} diff --git a/bundle/direct/dresources/type_test.go b/bundle/direct/dresources/type_test.go index 2d5516d59c7..e471d6b2b6e 100644 --- a/bundle/direct/dresources/type_test.go +++ b/bundle/direct/dresources/type_test.go @@ -66,6 +66,7 @@ var knownMissingInRemoteType = map[string][]string{ // These are bundle-specific fields that exist in InputType but not in StateType. var commonMissingInStateType = []string{ "grants", + "libraries", "lifecycle", "permissions", } From 7c9526f94ec2ed8cac8e670c6c2e1f8ed86158d5 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 25 Aug 2026 14:07:36 +0000 Subject: [PATCH 04/55] Model cluster libraries endpoints in testserver Add stateful fakes for the Libraries API (install, uninstall, cluster-status) so clusters.libraries runs against the in-process server. Add the TestAll fixture and classify libraries as a no-op delete alongside permissions/grants; this greens TestAll/clusters.libraries. Co-authored-by: Isaac --- bundle/direct/dresources/all_test.go | 20 ++++++- libs/testserver/fake_workspace.go | 4 +- libs/testserver/handlers.go | 13 +++++ libs/testserver/libraries.go | 87 ++++++++++++++++++++++++++++ 4 files changed, 122 insertions(+), 2 deletions(-) create mode 100644 libs/testserver/libraries.go diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index 36eaa3e27df..5d326411598 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -454,6 +454,24 @@ var testDeps = map[string]prepareWorkspace{ }, nil }, + "clusters.libraries": func(ctx context.Context, client *databricks.WorkspaceClient) (any, error) { + wait, err := client.Clusters.Create(ctx, compute.CreateCluster{ + ClusterName: "libraries-cluster", + SparkVersion: "13.3.x-scala2.12", + NodeTypeId: "m5.large", + NumWorkers: 1, + }) + if err != nil { + return nil, err + } + return &LibrariesState{ + ClusterId: wait.ClusterId, + EmbeddedSlice: []compute.Library{ + {Whl: "/Workspace/Users/test/lib.whl"}, + }, + }, nil + }, + "cluster_policies.permissions": func(ctx context.Context, client *databricks.WorkspaceClient) (any, error) { return &PermissionsState{ ObjectID: "/cluster-policies/cluster-policy-permissions", @@ -1129,7 +1147,7 @@ func testCRUD(t *testing.T, group string, adapter *Adapter, client *databricks.W require.NoError(t, err) } - deleteIsNoop := strings.HasSuffix(group, "permissions") || strings.HasSuffix(group, "grants") + deleteIsNoop := strings.HasSuffix(group, "permissions") || strings.HasSuffix(group, "grants") || strings.HasSuffix(group, "libraries") // Apps DoDelete is fire-and-forget: the API returns success while the app // sits in DELETING state for up to ~20 minutes before the record is removed. // A GET on the DELETING app returns the app, not 404 -- the testserver diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index 2f077b1b631..41d015e771d 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -199,6 +199,7 @@ type FakeWorkspace struct { ModelRegistryModels map[string]ml.Model ModelRegistryModelIDs map[string]string // model name -> numeric ID Clusters map[string]compute.ClusterDetails + ClusterLibraries map[string][]compute.Library // cluster id -> installed libraries InstancePools map[string]compute.GetInstancePool ClusterPolicies map[string]compute.Policy Catalogs map[string]catalog.CatalogInfo @@ -512,7 +513,8 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { SingleUserName: TestUser.UserName, }, }, - InstancePools: map[string]compute.GetInstancePool{}, + InstancePools: map[string]compute.GetInstancePool{}, + ClusterLibraries: map[string][]compute.Library{}, ClusterPolicies: map[string]compute.Policy{ // Seeded so the stateful list keeps backing the variable-lookup tests // (e.g. acceptance/bundle/variables/env_overrides resolves these by name). diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 8c44d97eae7..e1ea0baa471 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -927,6 +927,19 @@ func AddDefaultHandlers(server *Server) { return req.Workspace.ClustersPermanentDelete(req) }) + // Cluster libraries: + server.Handle("POST", "/api/2.0/libraries/install", func(req Request) any { + return req.Workspace.LibrariesInstall(req) + }) + + server.Handle("POST", "/api/2.0/libraries/uninstall", func(req Request) any { + return req.Workspace.LibrariesUninstall(req) + }) + + server.Handle("GET", "/api/2.0/libraries/cluster-status", func(req Request) any { + return req.Workspace.LibrariesClusterStatus(req, req.URL.Query().Get("cluster_id")) + }) + // MLflow Experiments: server.Handle("GET", "/api/2.0/mlflow/experiments/get", func(req Request) any { experimentId := req.URL.Query().Get("experiment_id") diff --git a/libs/testserver/libraries.go b/libs/testserver/libraries.go new file mode 100644 index 00000000000..63042ca8e66 --- /dev/null +++ b/libs/testserver/libraries.go @@ -0,0 +1,87 @@ +package testserver + +import ( + "encoding/json" + "fmt" + "net/http" + "reflect" + + "github.com/databricks/databricks-sdk-go/service/compute" +) + +func (s *FakeWorkspace) LibrariesInstall(req Request) any { + var request compute.InstallLibraries + if err := json.Unmarshal(req.Body, &request); err != nil { + return Response{StatusCode: http.StatusBadRequest, Body: fmt.Sprintf("request parsing error: %s", err)} + } + + defer s.LockUnlock()() + + if _, ok := s.Clusters[request.ClusterId]; !ok { + return Response{StatusCode: http.StatusNotFound} + } + + // Install is additive and idempotent: installing an already-present library is a no-op. + installed := s.ClusterLibraries[request.ClusterId] + for _, lib := range request.Libraries { + if !containsLibrary(installed, lib) { + installed = append(installed, lib) + } + } + s.ClusterLibraries[request.ClusterId] = installed + + return Response{} +} + +func (s *FakeWorkspace) LibrariesUninstall(req Request) any { + var request compute.UninstallLibraries + if err := json.Unmarshal(req.Body, &request); err != nil { + return Response{StatusCode: http.StatusBadRequest, Body: fmt.Sprintf("request parsing error: %s", err)} + } + + defer s.LockUnlock()() + + installed := s.ClusterLibraries[request.ClusterId] + remaining := make([]compute.Library, 0, len(installed)) + for _, lib := range installed { + if !containsLibrary(request.Libraries, lib) { + remaining = append(remaining, lib) + } + } + s.ClusterLibraries[request.ClusterId] = remaining + + return Response{} +} + +func (s *FakeWorkspace) LibrariesClusterStatus(req Request, clusterId string) any { + defer s.LockUnlock()() + + if _, ok := s.Clusters[clusterId]; !ok { + return Response{StatusCode: http.StatusNotFound} + } + + installed := s.ClusterLibraries[clusterId] + statuses := make([]compute.LibraryFullStatus, 0, len(installed)) + for i := range installed { + statuses = append(statuses, compute.LibraryFullStatus{ + Library: &installed[i], + Status: compute.LibraryInstallStatusInstalled, + }) + } + + return Response{ + Body: compute.ClusterLibraryStatuses{ + ClusterId: clusterId, + LibraryStatuses: statuses, + }, + } +} + +func containsLibrary(libs []compute.Library, target compute.Library) bool { + for _, l := range libs { + if reflect.DeepEqual(l, target) { + return true + } + } + return false +} From da49387fb65b8c89f0c2be6bd8476ec682df993b Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 25 Aug 2026 14:16:18 +0000 Subject: [PATCH 05/55] Wire cluster libraries into the wheel build/upload pipeline Add clusterLibrariesPattern (resources.clusters.*.libraries) to glob expansion, local-library collection/upload, duplicate-name checking, and patched-wheel swapping, mirroring the job task library wiring. Local whl/jar globs now build, upload to artifact_path/.internal, and rewrite to absolute workspace paths; pypi/maven entries pass through unchanged. Co-authored-by: Isaac --- bundle/libraries/expand_glob_references.go | 11 +++++++++++ bundle/libraries/remote_path.go | 2 ++ bundle/libraries/same_name_libraries.go | 2 ++ bundle/libraries/switch_to_patched_wheels.go | 16 ++++++++++++++++ 4 files changed, 31 insertions(+) diff --git a/bundle/libraries/expand_glob_references.go b/bundle/libraries/expand_glob_references.go index 720142fe6d7..ab1da3df68d 100644 --- a/bundle/libraries/expand_glob_references.go +++ b/bundle/libraries/expand_glob_references.go @@ -198,6 +198,13 @@ var pipelineEnvDepsPattern = dyn.NewPattern( dyn.Key("dependencies"), ) +var clusterLibrariesPattern = dyn.NewPattern( + dyn.Key("resources"), + dyn.Key("clusters"), + dyn.AnyKey(), + dyn.Key("libraries"), +) + func (e *expand) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { expanders := []expandPattern{ { @@ -216,6 +223,10 @@ func (e *expand) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { pattern: pipelineEnvDepsPattern, fn: expandEnvironmentDeps, }, + { + pattern: clusterLibrariesPattern, + fn: expandLibraries, + }, } var diags diag.Diagnostics diff --git a/bundle/libraries/remote_path.go b/bundle/libraries/remote_path.go index 02a1172f36d..e40de7460df 100644 --- a/bundle/libraries/remote_path.go +++ b/bundle/libraries/remote_path.go @@ -68,6 +68,8 @@ func collectLocalLibraries(b *bundle.Bundle) (map[string][]LocationToUpdate, err taskLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")), forEachTaskLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("whl")), forEachTaskLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")), + clusterLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("whl")), + clusterLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")), envDepsPattern.Append(dyn.AnyIndex()), pipelineEnvDepsPattern.Append(dyn.AnyIndex()), // The AI Runtime task's code_source_path is a local archive (typically an diff --git a/bundle/libraries/same_name_libraries.go b/bundle/libraries/same_name_libraries.go index 49776fbd8c9..8fb140d7aa4 100644 --- a/bundle/libraries/same_name_libraries.go +++ b/bundle/libraries/same_name_libraries.go @@ -17,6 +17,8 @@ var patterns = []dyn.Pattern{ taskLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")), forEachTaskLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("whl")), forEachTaskLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")), + clusterLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("whl")), + clusterLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")), envDepsPattern.Append(dyn.AnyIndex()), pipelineEnvDepsPattern.Append(dyn.AnyIndex()), } diff --git a/bundle/libraries/switch_to_patched_wheels.go b/bundle/libraries/switch_to_patched_wheels.go index 56250d713a7..3c5ff184a97 100644 --- a/bundle/libraries/switch_to_patched_wheels.go +++ b/bundle/libraries/switch_to_patched_wheels.go @@ -79,6 +79,22 @@ func (c switchToPatchedWheels) Apply(ctx context.Context, b *bundle.Bundle) diag } } + // Update resources.clusters.*.libraries[*].whl + for clusterName, clusterRef := range b.Config.Resources.Clusters { + if clusterRef == nil { + continue + } + for libInd, lib := range clusterRef.Libraries { + repl := replacements[lib.Whl] + if repl != "" { + log.Debugf(ctx, "Updating resources.clusters.%s.libraries[%d].whl from %s to %s", clusterName, libInd, lib.Whl, repl) + clusterRef.Libraries[libInd].Whl = repl + } else { + log.Debugf(ctx, "Not updating resources.clusters.%s.libraries[%d].whl from %s. Available replacements: %v", clusterName, libInd, lib.Whl, slices.Sorted(maps.Keys(replacements))) + } + } + } + return nil } From fc3d5c0e40f187c36f333f412c8e5e0c0539e2a1 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 25 Aug 2026 14:21:34 +0000 Subject: [PATCH 06/55] Reject cluster libraries under the terraform engine Add ValidateClusterLibraries and register it in PreDeployChecks so a libraries block on a cluster errors under the terraform engine instead of being silently dropped. Cluster libraries are direct-only. Mirrors the existing lifecycle.started guard. Co-authored-by: Isaac --- .../mutator/validate_cluster_libraries.go | 43 +++++++++++++++++++ bundle/phases/plan.go | 1 + 2 files changed, 44 insertions(+) create mode 100644 bundle/config/mutator/validate_cluster_libraries.go diff --git a/bundle/config/mutator/validate_cluster_libraries.go b/bundle/config/mutator/validate_cluster_libraries.go new file mode 100644 index 00000000000..f13cd3be373 --- /dev/null +++ b/bundle/config/mutator/validate_cluster_libraries.go @@ -0,0 +1,43 @@ +package mutator + +import ( + "context" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/engine" + "github.com/databricks/cli/libs/diag" +) + +type validateClusterLibraries struct { + engine engine.EngineType +} + +// ValidateClusterLibraries returns a mutator that errors when cluster libraries are used with +// the terraform deployment engine. Cluster libraries are only supported in direct deployment mode. +func ValidateClusterLibraries(e engine.EngineType) bundle.Mutator { + return &validateClusterLibraries{engine: e} +} + +func (m *validateClusterLibraries) Name() string { + return "ValidateClusterLibraries" +} + +func (m *validateClusterLibraries) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + if m.engine.IsDirect() { + return nil + } + + var diags diag.Diagnostics + for key, cluster := range b.Config.Resources.Clusters { + if cluster == nil || len(cluster.Libraries) == 0 { + continue + } + path := "resources.clusters." + key + ".libraries" + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: "cluster libraries are only supported in direct deployment mode", + Locations: b.Config.GetLocations(path), + }) + } + return diags +} diff --git a/bundle/phases/plan.go b/bundle/phases/plan.go index 3db0864c2c4..208582a2d9d 100644 --- a/bundle/phases/plan.go +++ b/bundle/phases/plan.go @@ -28,6 +28,7 @@ func PreDeployChecks(ctx context.Context, b *bundle.Bundle, isPlan bool, engine mutator.ValidateGitDetails(), mutator.ValidateDirectOnlyResources(engine), mutator.ValidateLifecycleStarted(engine), + mutator.ValidateClusterLibraries(engine), mutator.ValidateCascadeOnDestroy(engine), mutator.ValidateJobRunTriggers(), statemgmt.CheckRunningResource(engine), From 3fa6fb17b3f6beb94366594c80574a63a7ef611e Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 25 Aug 2026 14:30:29 +0000 Subject: [PATCH 07/55] Regenerate terraform<->DABs field map for cluster libraries Adding the clusters libraries field makes DABs libraries map to the terraform databricks_cluster.library field; regenerate the mapping so reference translation and the tf-only field audit stay correct. Co-authored-by: Isaac --- bundle/terraform_dabs_map/generated.go | 32 ++++++++------------------ 1 file changed, 10 insertions(+), 22 deletions(-) diff --git a/bundle/terraform_dabs_map/generated.go b/bundle/terraform_dabs_map/generated.go index d0a783f35ac..f23c0bfa9eb 100644 --- a/bundle/terraform_dabs_map/generated.go +++ b/bundle/terraform_dabs_map/generated.go @@ -6,7 +6,8 @@ package terraform_dabs_map // alerts / databricks_alert_v2: 3 tf-only // apps / databricks_app: 6 dabs-only // apps / databricks_app: 1 tf-only -// clusters / databricks_cluster: 26 tf-only +// clusters / databricks_cluster: 1 renames +// clusters / databricks_cluster: 11 tf-only // dashboards / databricks_dashboard: 2 tf-only // database_instances / databricks_database_instance: 1 tf-only // experiments / databricks_mlflow_experiment: 1 tf-only @@ -34,6 +35,9 @@ package terraform_dabs_map // TerraformToDABsFieldMap maps DABs group name → nested TF segments → DABs segment name. // Navigate using TF field name segments; DABs is the corresponding DABs name when it differs. var TerraformToDABsFieldMap = map[string]RenameTree{ + "clusters": { + "library": {NewName: "libraries"}, + }, "jobs": { "environment": {NewName: "environments"}, "git_source": {Children: RenameTree{ @@ -168,27 +172,8 @@ var TerraformOnlyFields = map[string]FieldSet{ }, "idempotency_token": {}, "is_pinned": {}, - "library": { - "cran": { - "package": {}, // databricks_cluster.*.library.cran.package - "repo": {}, // databricks_cluster.*.library.cran.repo - }, - "egg": {}, // databricks_cluster.*.library.egg - "jar": {}, // databricks_cluster.*.library.jar - "maven": { - "coordinates": {}, // databricks_cluster.*.library.maven.coordinates - "exclusions": {}, // databricks_cluster.*.library.maven.exclusions - "repo": {}, // databricks_cluster.*.library.maven.repo - }, - "pypi": { - "package": {}, // databricks_cluster.*.library.pypi.package - "repo": {}, // databricks_cluster.*.library.pypi.repo - }, - "requirements": {}, // databricks_cluster.*.library.requirements - "whl": {}, // databricks_cluster.*.library.whl - }, - "no_wait": {}, - "url": {}, + "no_wait": {}, + "url": {}, }, "dashboards": { "dashboard_change_detected": {}, @@ -569,6 +554,9 @@ var TerraformOnlyFields = map[string]FieldSet{ // DABsToTerraformRenameMap maps DABs group name → nested DABs segments → TF segment name. // Navigate using DABs field name segments; NewName is the TF name when it differs. var DABsToTerraformRenameMap = map[string]RenameTree{ + "clusters": { + "libraries": {NewName: "library"}, + }, "jobs": { "environments": {NewName: "environment"}, "git_source": {Children: RenameTree{ From 0771bfe6f08635965d1185032cdc745c9374a5f0 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 25 Aug 2026 14:30:30 +0000 Subject: [PATCH 08/55] Add acceptance tests and changelog for cluster libraries - clusters/libraries: direct-engine deploy installs pypi + local wheel (rewritten to its uploaded path), and removing the wheel and redeploying uninstalls it. - clusters/libraries-terraform-error: bundle plan/deploy reject cluster libraries under the terraform engine. Co-authored-by: Isaac --- .nextchanges/bundles/cluster-libraries.md | 1 + .../libraries-terraform-error/databricks.yml | 13 +++++ .../libraries-terraform-error/out.test.toml | 2 + .../libraries-terraform-error/output.txt | 16 +++++ .../clusters/libraries-terraform-error/script | 5 ++ .../libraries-terraform-error/test.toml | 4 ++ .../clusters/libraries/databricks.yml | 14 +++++ .../clusters/libraries/out.test.toml | 2 + .../resources/clusters/libraries/output.txt | 58 +++++++++++++++++++ .../resources/clusters/libraries/script | 16 +++++ .../resources/clusters/libraries/test.toml | 10 ++++ 11 files changed, 141 insertions(+) create mode 100644 .nextchanges/bundles/cluster-libraries.md create mode 100644 acceptance/bundle/resources/clusters/libraries-terraform-error/databricks.yml create mode 100644 acceptance/bundle/resources/clusters/libraries-terraform-error/out.test.toml create mode 100644 acceptance/bundle/resources/clusters/libraries-terraform-error/output.txt create mode 100644 acceptance/bundle/resources/clusters/libraries-terraform-error/script create mode 100644 acceptance/bundle/resources/clusters/libraries-terraform-error/test.toml create mode 100644 acceptance/bundle/resources/clusters/libraries/databricks.yml create mode 100644 acceptance/bundle/resources/clusters/libraries/out.test.toml create mode 100644 acceptance/bundle/resources/clusters/libraries/output.txt create mode 100644 acceptance/bundle/resources/clusters/libraries/script create mode 100644 acceptance/bundle/resources/clusters/libraries/test.toml diff --git a/.nextchanges/bundles/cluster-libraries.md b/.nextchanges/bundles/cluster-libraries.md new file mode 100644 index 00000000000..4c9b1a52006 --- /dev/null +++ b/.nextchanges/bundles/cluster-libraries.md @@ -0,0 +1 @@ +Add support for a `libraries` list on the `clusters` resource type in Declarative Automation Bundles. Libraries (whl, jar, pypi, maven, cran, egg, requirements) are installed on the all-purpose cluster via the Libraries API; local wheels/jars are built and uploaded automatically. Cluster libraries are only supported in direct deployment mode. diff --git a/acceptance/bundle/resources/clusters/libraries-terraform-error/databricks.yml b/acceptance/bundle/resources/clusters/libraries-terraform-error/databricks.yml new file mode 100644 index 00000000000..e2645b5cc5e --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-terraform-error/databricks.yml @@ -0,0 +1,13 @@ +bundle: + name: cluster-libraries-terraform-error + +resources: + clusters: + mycluster: + cluster_name: mycluster + spark_version: 15.4.x-scala2.12 + node_type_id: i3.xlarge + num_workers: 1 + libraries: + - pypi: + package: requests diff --git a/acceptance/bundle/resources/clusters/libraries-terraform-error/out.test.toml b/acceptance/bundle/resources/clusters/libraries-terraform-error/out.test.toml new file mode 100644 index 00000000000..d2059b4b5d7 --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-terraform-error/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/resources/clusters/libraries-terraform-error/output.txt b/acceptance/bundle/resources/clusters/libraries-terraform-error/output.txt new file mode 100644 index 00000000000..aa0a6711e7e --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-terraform-error/output.txt @@ -0,0 +1,16 @@ + +=== bundle plan fails with cluster libraries on terraform engine +>>> errcode [CLI] bundle plan +Error: cluster libraries are only supported in direct deployment mode + in databricks.yml:12:9 + + +Exit code: 1 + +=== bundle deploy fails with cluster libraries on terraform engine +>>> errcode [CLI] bundle deploy +Error: cluster libraries are only supported in direct deployment mode + in databricks.yml:12:9 + + +Exit code: 1 diff --git a/acceptance/bundle/resources/clusters/libraries-terraform-error/script b/acceptance/bundle/resources/clusters/libraries-terraform-error/script new file mode 100644 index 00000000000..93db6e06309 --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-terraform-error/script @@ -0,0 +1,5 @@ +title "bundle plan fails with cluster libraries on terraform engine" +trace errcode $CLI bundle plan + +title "bundle deploy fails with cluster libraries on terraform engine" +trace errcode $CLI bundle deploy diff --git a/acceptance/bundle/resources/clusters/libraries-terraform-error/test.toml b/acceptance/bundle/resources/clusters/libraries-terraform-error/test.toml new file mode 100644 index 00000000000..e4a0f1c6301 --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-terraform-error/test.toml @@ -0,0 +1,4 @@ +Cloud = false +RecordRequests = false + +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/resources/clusters/libraries/databricks.yml b/acceptance/bundle/resources/clusters/libraries/databricks.yml new file mode 100644 index 00000000000..922f719fda4 --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries/databricks.yml @@ -0,0 +1,14 @@ +bundle: + name: cluster-libraries + +resources: + clusters: + mycluster: + cluster_name: mycluster + spark_version: 15.4.x-scala2.12 + node_type_id: i3.xlarge + num_workers: 1 + libraries: + - pypi: + package: requests + - whl: ./dist/*.whl diff --git a/acceptance/bundle/resources/clusters/libraries/out.test.toml b/acceptance/bundle/resources/clusters/libraries/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/clusters/libraries/output.txt b/acceptance/bundle/resources/clusters/libraries/output.txt new file mode 100644 index 00000000000..74c60339a00 --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries/output.txt @@ -0,0 +1,58 @@ + +=== Deploy a cluster with a pypi and a local wheel library +>>> [CLI] bundle deploy +Uploading dist/my_package-0.0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster-libraries/default/files... +Created clusters.mycluster +Created clusters.mycluster.libraries +Files: 6 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== Libraries installed via the Libraries API (wheel rewritten to its uploaded path) +>>> print_requests.py //libraries/install +{ + "method": "POST", + "path": "/api/2.0/libraries/install", + "body": { + "cluster_id": "[UUID]", + "libraries": [ + { + "pypi": { + "package": "requests" + } + }, + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/cluster-libraries/default/artifacts/.internal/my_package-0.0.1-py3-none-any.whl" + } + ] + } +} + +=== Removing the wheel and redeploying uninstalls it +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster-libraries/default/files... +Updated clusters.mycluster.libraries +Files: 3 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 1 unchanged + +>>> print_requests.py //libraries/uninstall +{ + "method": "POST", + "path": "/api/2.0/libraries/uninstall", + "body": { + "cluster_id": "[UUID]", + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/cluster-libraries/default/artifacts/.internal/my_package-0.0.1-py3-none-any.whl" + } + ] + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.clusters.mycluster + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/cluster-libraries/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/clusters/libraries/script b/acceptance/bundle/resources/clusters/libraries/script new file mode 100644 index 00000000000..33af4f7c5d5 --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries/script @@ -0,0 +1,16 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +title "Deploy a cluster with a pypi and a local wheel library" +trace $CLI bundle deploy + +title "Libraries installed via the Libraries API (wheel rewritten to its uploaded path)" +trace print_requests.py //libraries/install + +title "Removing the wheel and redeploying uninstalls it" +update_file.py databricks.yml " - whl: ./dist/*.whl" "" +trace $CLI bundle deploy +trace print_requests.py //libraries/uninstall diff --git a/acceptance/bundle/resources/clusters/libraries/test.toml b/acceptance/bundle/resources/clusters/libraries/test.toml new file mode 100644 index 00000000000..93cb0543245 --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries/test.toml @@ -0,0 +1,10 @@ +Cloud = false +RecordRequests = true + +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +Ignore = [".databricks"] + +[[Repls]] +Old = "[0-9]{4}-[0-9]{6}-[0-9a-z]{8}" +New = "[CLUSTER-ID]" From 8c5552a179bdc4ee871eb9e031267b269c6c0480 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 25 Aug 2026 14:41:41 +0000 Subject: [PATCH 09/55] Fix lint in cluster_libraries (exhaustive switch, exhaustruct) Enumerate the remaining LibraryInstallStatus cases in the install-wait poll and set EmbeddedSlice explicitly in DoRead's state literal. Co-authored-by: Isaac --- bundle/direct/dresources/cluster_libraries.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bundle/direct/dresources/cluster_libraries.go b/bundle/direct/dresources/cluster_libraries.go index a629dfe08ee..a3458da761d 100644 --- a/bundle/direct/dresources/cluster_libraries.go +++ b/bundle/direct/dresources/cluster_libraries.go @@ -98,7 +98,7 @@ func (r *ResourceLibraries) DoRead(ctx context.Context, id string) (*LibrariesSt return nil, err } - state := &LibrariesState{ClusterId: id} + state := &LibrariesState{ClusterId: id, EmbeddedSlice: nil} for _, s := range statuses.LibraryStatuses { // Libraries set for all clusters via the UI are not managed by the bundle // (following the permissions convention of ignoring inherited entries). @@ -241,6 +241,8 @@ func (r *ResourceLibraries) waitForInstall(ctx context.Context, id string, desir return nil, retries.Halt(fmt.Errorf("library %s failed to install: %s", libraryMapKey(*s.Library), strings.Join(s.Messages, "; "))) case compute.LibraryInstallStatusInstalled, compute.LibraryInstallStatusSkipped, compute.LibraryInstallStatusRestored: pending-- + case compute.LibraryInstallStatusPending, compute.LibraryInstallStatusResolving, compute.LibraryInstallStatusInstalling, compute.LibraryInstallStatusUninstallOnRestart: + // Still in progress (or being removed); keep polling. } } From fe179e807a1b9f8701cf18cb4d61431f010c32a2 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 25 Aug 2026 14:48:54 +0000 Subject: [PATCH 10/55] Add cloud drift check for cluster libraries clusters/libraries-drift (Cloud = true, direct engine): deploy a cluster with a pypi library, then assert the immediate re-plan is a no-op (0 to change). Verified on a real AWS workspace: the Libraries status API round-trips the library without drift, so no normalization is needed. Co-authored-by: Isaac --- .../libraries-drift/databricks.yml.tmpl | 17 ++++++++++++++++ .../clusters/libraries-drift/out.test.toml | 2 ++ .../clusters/libraries-drift/output.txt | 12 +++++++++++ .../resources/clusters/libraries-drift/script | 20 +++++++++++++++++++ .../clusters/libraries-drift/test.toml | 6 ++++++ 5 files changed, 57 insertions(+) create mode 100644 acceptance/bundle/resources/clusters/libraries-drift/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/clusters/libraries-drift/out.test.toml create mode 100644 acceptance/bundle/resources/clusters/libraries-drift/output.txt create mode 100644 acceptance/bundle/resources/clusters/libraries-drift/script create mode 100644 acceptance/bundle/resources/clusters/libraries-drift/test.toml diff --git a/acceptance/bundle/resources/clusters/libraries-drift/databricks.yml.tmpl b/acceptance/bundle/resources/clusters/libraries-drift/databricks.yml.tmpl new file mode 100644 index 00000000000..b3bf2463741 --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-drift/databricks.yml.tmpl @@ -0,0 +1,17 @@ +bundle: + name: cluster-libraries-drift-$UNIQUE_NAME + +workspace: + root_path: ~/.bundle/$UNIQUE_NAME + +resources: + clusters: + mycluster: + cluster_name: mycluster-$UNIQUE_NAME + spark_version: $DEFAULT_SPARK_VERSION + node_type_id: $NODE_TYPE_ID + instance_pool_id: $TEST_INSTANCE_POOL_ID + num_workers: 1 + libraries: + - pypi: + package: requests diff --git a/acceptance/bundle/resources/clusters/libraries-drift/out.test.toml b/acceptance/bundle/resources/clusters/libraries-drift/out.test.toml new file mode 100644 index 00000000000..c502b28221b --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-drift/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/clusters/libraries-drift/output.txt b/acceptance/bundle/resources/clusters/libraries-drift/output.txt new file mode 100644 index 00000000000..4a180dfbf0c --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-drift/output.txt @@ -0,0 +1,12 @@ + +=== Plan is a no-op immediately after deploy (no library drift) +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.clusters.mycluster + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/clusters/libraries-drift/script b/acceptance/bundle/resources/clusters/libraries-drift/script new file mode 100644 index 00000000000..b1fef7ee702 --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-drift/script @@ -0,0 +1,20 @@ +# A pypi library is used because its nested {package, repo} shape is the most +# likely to drift (the status API echoing a repo we did not set); a workspace +# wheel is covered by the local clusters/libraries test instead, since the +# shared cloud test cluster rejects libraries from /Workspace paths. +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +# Cluster provisioning and library-install output is noisy and differs between +# the fake and cloud, so route it to LOG and assert only the deterministic +# drift signal below. +$CLI bundle deploy &> LOG.deploy +cat LOG.deploy | contains.py "Created clusters.mycluster.libraries" > /dev/null + +title "Plan is a no-op immediately after deploy (no library drift)" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/clusters/libraries-drift/test.toml b/acceptance/bundle/resources/clusters/libraries-drift/test.toml new file mode 100644 index 00000000000..63a8c8a332c --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-drift/test.toml @@ -0,0 +1,6 @@ +Cloud = true +RecordRequests = false + +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +Ignore = [".databricks", "databricks.yml"] From 44a37b89abf89dad1e8e8e1c3c67bd6c2981985c Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 25 Aug 2026 14:53:23 +0000 Subject: [PATCH 11/55] Regenerate refschema and required_fields for cluster libraries Co-authored-by: Isaac --- acceptance/bundle/refschema/out.fields.txt | 34 +++++++++++++++++++ .../validation/generated/required_fields.go | 3 ++ 2 files changed, 37 insertions(+) diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index de252dc4162..5e00e1b1f7e 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -460,6 +460,23 @@ resources.clusters.*.jdbc_port int REMOTE resources.clusters.*.kind compute.Kind ALL resources.clusters.*.last_restarted_time int64 REMOTE resources.clusters.*.last_state_loss_time int64 REMOTE +resources.clusters.*.libraries []compute.Library INPUT +resources.clusters.*.libraries[*] compute.Library INPUT +resources.clusters.*.libraries[*].cran *compute.RCranLibrary INPUT +resources.clusters.*.libraries[*].cran.package string INPUT +resources.clusters.*.libraries[*].cran.repo string INPUT +resources.clusters.*.libraries[*].egg string INPUT +resources.clusters.*.libraries[*].jar string INPUT +resources.clusters.*.libraries[*].maven *compute.MavenLibrary INPUT +resources.clusters.*.libraries[*].maven.coordinates string INPUT +resources.clusters.*.libraries[*].maven.exclusions []string INPUT +resources.clusters.*.libraries[*].maven.exclusions[*] string INPUT +resources.clusters.*.libraries[*].maven.repo string INPUT +resources.clusters.*.libraries[*].pypi *compute.PythonPyPiLibrary INPUT +resources.clusters.*.libraries[*].pypi.package string INPUT +resources.clusters.*.libraries[*].pypi.repo string INPUT +resources.clusters.*.libraries[*].requirements string INPUT +resources.clusters.*.libraries[*].whl string INPUT resources.clusters.*.lifecycle *dresources.StateLifecycle REMOTE STATE resources.clusters.*.lifecycle *resources.LifecycleWithStarted INPUT resources.clusters.*.lifecycle resources.Lifecycle INPUT @@ -610,6 +627,23 @@ resources.clusters.*.workload_type *compute.WorkloadType ALL resources.clusters.*.workload_type.clients compute.ClientsTypes ALL resources.clusters.*.workload_type.clients.jobs bool ALL resources.clusters.*.workload_type.clients.notebooks bool ALL +resources.clusters.*.libraries.cluster_id string ALL +resources.clusters.*.libraries[*] compute.Library ALL +resources.clusters.*.libraries[*].cran *compute.RCranLibrary ALL +resources.clusters.*.libraries[*].cran.package string ALL +resources.clusters.*.libraries[*].cran.repo string ALL +resources.clusters.*.libraries[*].egg string ALL +resources.clusters.*.libraries[*].jar string ALL +resources.clusters.*.libraries[*].maven *compute.MavenLibrary ALL +resources.clusters.*.libraries[*].maven.coordinates string ALL +resources.clusters.*.libraries[*].maven.exclusions []string ALL +resources.clusters.*.libraries[*].maven.exclusions[*] string ALL +resources.clusters.*.libraries[*].maven.repo string ALL +resources.clusters.*.libraries[*].pypi *compute.PythonPyPiLibrary ALL +resources.clusters.*.libraries[*].pypi.package string ALL +resources.clusters.*.libraries[*].pypi.repo string ALL +resources.clusters.*.libraries[*].requirements string ALL +resources.clusters.*.libraries[*].whl string ALL resources.clusters.*.permissions.object_id string ALL resources.clusters.*.permissions[*] dresources.StatePermission ALL resources.clusters.*.permissions[*].group_name string ALL diff --git a/bundle/internal/validation/generated/required_fields.go b/bundle/internal/validation/generated/required_fields.go index 617f5a65ecc..d0937851098 100644 --- a/bundle/internal/validation/generated/required_fields.go +++ b/bundle/internal/validation/generated/required_fields.go @@ -56,6 +56,9 @@ var RequiredFields = map[string][]string{ "resources.clusters.*.init_scripts[*].s3": {"destination"}, "resources.clusters.*.init_scripts[*].volumes": {"destination"}, "resources.clusters.*.init_scripts[*].workspace": {"destination"}, + "resources.clusters.*.libraries[*].cran": {"package"}, + "resources.clusters.*.libraries[*].maven": {"coordinates"}, + "resources.clusters.*.libraries[*].pypi": {"package"}, "resources.clusters.*.permissions[*]": {"level"}, "resources.clusters.*.workload_type": {"clients"}, From 033c26d46560baa1fe51963a57a9ceea3a8d85b6 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 25 Aug 2026 15:05:40 +0000 Subject: [PATCH 12/55] Scope libraries sub-resource to clusters only Pipelines have a native libraries field that is a plain field, not a child resource. The sub-resource wiring matched resources.*.*.libraries for every resource type, so the direct engine tried to plan pipelines.libraries as a resource and failed with 'unsupported resource type: pipelines.libraries'. Scope both GetNodeAndType and the plan pattern to clusters. Co-authored-by: Isaac --- bundle/config/root.go | 11 +++++++++-- bundle/direct/bundle_plan.go | 3 ++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/bundle/config/root.go b/bundle/config/root.go index 0d7d03dadb3..c05851c3d86 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -620,8 +620,15 @@ func GetNodeAndType(path dyn.Path) (dyn.Path, string) { } if len(path) >= 4 { - if path[3].Key() == "permissions" || path[3].Key() == "grants" || path[3].Key() == "libraries" { - return path[:4], path[1].Key() + "." + path[3].Key() + sub := path[3].Key() + if sub == "permissions" || sub == "grants" { + return path[:4], path[1].Key() + "." + sub + } + // libraries is a sub-resource only for clusters. Other resource types + // (e.g. pipelines) have a native libraries field that is a plain field, + // not a child resource. + if sub == "libraries" && path[1].Key() == "clusters" { + return path[:4], path[1].Key() + "." + sub } } diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 4d37a21e090..c9e08fe798e 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -930,7 +930,8 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey(), dyn.Key("permissions")), dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey(), dyn.Key("grants")), - dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey(), dyn.Key("libraries")), + // libraries is a sub-resource only for clusters; other resource types have a native libraries field. + dyn.NewPattern(dyn.Key("resources"), dyn.Key("clusters"), dyn.AnyKey(), dyn.Key("libraries")), } // Walk? From 25f701e53ddf011c7f9226d8f1f06070a914fe34 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 25 Aug 2026 15:09:02 +0000 Subject: [PATCH 13/55] Scope libraries entry in ResourcesTypes to clusters Consistency follow-up to the GetNodeAndType/plan-pattern scoping: ResourcesTypes registered a .libraries key for every resource type with a Libraries field, spuriously adding pipelines.libraries and cluster_policies.libraries. Those keys are unreachable now that GetNodeAndType is scoped, but scope this branch too so the map stays consistent and the entries don't mislead future callers. Co-authored-by: Isaac --- bundle/config/resources_types.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bundle/config/resources_types.go b/bundle/config/resources_types.go index 21d6d407468..20dbee0222a 100644 --- a/bundle/config/resources_types.go +++ b/bundle/config/resources_types.go @@ -45,7 +45,10 @@ var ResourcesTypes = func() map[string]reflect.Type { res[grantsKey] = resourceField.Type continue } - if resourceField.Name == "Libraries" { + // libraries is a child resource only for clusters. Pipelines and + // cluster_policies have a native Libraries field that is a plain + // field, not a child resource. + if resourceField.Name == "Libraries" && name == "clusters" { librariesKey := name + ".libraries" res[librariesKey] = resourceField.Type } From 612add74d0610251357433a8c6a1c589cf750fa8 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 25 Aug 2026 15:27:36 +0000 Subject: [PATCH 14/55] Cover clusters.libraries in invariant suite; commit wheel fixture Two CI failures in the local acceptance suite: 1. bundle/resources/clusters/libraries deployed a prebuilt wheel from ./dist/*.whl, but dist/ is gitignored so the fixture was never committed. CI's clean checkout hit 'no files match pattern: ./dist/*.whl'. Force-add the dummy wheel as a committed test input. 2. TestInvariantConfigsCoverage requires every config.ResourcesTypes key to be covered. clusters.libraries had no coverage: the scanner only understood .permissions/.grants sub-resources. Teach it .libraries too, add a pypi-only cluster_libraries invariant config, and wire it into INPUT_CONFIG. Cluster libraries are direct-only, so exclude the config from the terraform-seeded migrate subtest like the other direct-only resources. Co-authored-by: Isaac --- .../configs/cluster_libraries.yml.tmpl | 14 ++++++++++ .../invariant/continue_293/out.test.toml | 1 + .../invariant/delete_idempotent/out.test.toml | 1 + .../destroy_idempotent/out.test.toml | 1 + acceptance/bundle/invariant/migrate/test.toml | 2 ++ .../bundle/invariant/no_drift/out.test.toml | 1 + acceptance/bundle/invariant/test.toml | 1 + .../dist/my_package-0.0.1-py3-none-any.whl | 1 + acceptance/invariant_test.go | 28 +++++++++++++------ 9 files changed, 41 insertions(+), 9 deletions(-) create mode 100644 acceptance/bundle/invariant/configs/cluster_libraries.yml.tmpl create mode 100644 acceptance/bundle/resources/clusters/libraries/dist/my_package-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/invariant/configs/cluster_libraries.yml.tmpl b/acceptance/bundle/invariant/configs/cluster_libraries.yml.tmpl new file mode 100644 index 00000000000..507730ec287 --- /dev/null +++ b/acceptance/bundle/invariant/configs/cluster_libraries.yml.tmpl @@ -0,0 +1,14 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + clusters: + foo: + cluster_name: test-cluster-$UNIQUE_NAME + spark_version: 13.3.x-scala2.12 + node_type_id: $NODE_TYPE_ID + instance_pool_id: $TEST_INSTANCE_POOL_ID + num_workers: 1 + libraries: + - pypi: + package: requests diff --git a/acceptance/bundle/invariant/continue_293/out.test.toml b/acceptance/bundle/invariant/continue_293/out.test.toml index f5883975fcd..f041da59ad0 100644 --- a/acceptance/bundle/invariant/continue_293/out.test.toml +++ b/acceptance/bundle/invariant/continue_293/out.test.toml @@ -7,6 +7,7 @@ EnvMatrix.INPUT_CONFIG = [ "catalog_optional_fields.yml.tmpl", "cluster.yml.tmpl", "cluster_apply_policy_default_values.yml.tmpl", + "cluster_libraries.yml.tmpl", "dashboard.yml.tmpl", "job_apply_policy_default_values_job_cluster.yml.tmpl", "job_apply_policy_default_values_task_cluster.yml.tmpl", diff --git a/acceptance/bundle/invariant/delete_idempotent/out.test.toml b/acceptance/bundle/invariant/delete_idempotent/out.test.toml index 8ed109d4820..2e4027b3b43 100644 --- a/acceptance/bundle/invariant/delete_idempotent/out.test.toml +++ b/acceptance/bundle/invariant/delete_idempotent/out.test.toml @@ -7,6 +7,7 @@ EnvMatrix.INPUT_CONFIG = [ "catalog_optional_fields.yml.tmpl", "cluster.yml.tmpl", "cluster_apply_policy_default_values.yml.tmpl", + "cluster_libraries.yml.tmpl", "cluster_policy.yml.tmpl", "dashboard.yml.tmpl", "job_apply_policy_default_values_job_cluster.yml.tmpl", diff --git a/acceptance/bundle/invariant/destroy_idempotent/out.test.toml b/acceptance/bundle/invariant/destroy_idempotent/out.test.toml index 8ed109d4820..2e4027b3b43 100644 --- a/acceptance/bundle/invariant/destroy_idempotent/out.test.toml +++ b/acceptance/bundle/invariant/destroy_idempotent/out.test.toml @@ -7,6 +7,7 @@ EnvMatrix.INPUT_CONFIG = [ "catalog_optional_fields.yml.tmpl", "cluster.yml.tmpl", "cluster_apply_policy_default_values.yml.tmpl", + "cluster_libraries.yml.tmpl", "cluster_policy.yml.tmpl", "dashboard.yml.tmpl", "job_apply_policy_default_values_job_cluster.yml.tmpl", diff --git a/acceptance/bundle/invariant/migrate/test.toml b/acceptance/bundle/invariant/migrate/test.toml index aa24bf58eed..c9a21f879a4 100644 --- a/acceptance/bundle/invariant/migrate/test.toml +++ b/acceptance/bundle/invariant/migrate/test.toml @@ -16,6 +16,8 @@ EnvMatrixExclude.no_genie_space = ["INPUT_CONFIG=genie_space.yml.tmpl"] EnvMatrixExclude.no_instance_pool = ["INPUT_CONFIG=instance_pool.yml.tmpl"] # Cluster policies are direct-only; the terraform deploy that seeds the migration fails for them. EnvMatrixExclude.no_cluster_policy = ["INPUT_CONFIG=cluster_policy.yml.tmpl"] +# Cluster libraries are direct-only; the terraform deploy that seeds the migration fails for them. +EnvMatrixExclude.no_cluster_libraries = ["INPUT_CONFIG=cluster_libraries.yml.tmpl"] # Cross-resource permission references (e.g. ${resources.jobs.job_b.permissions[0].level}) # don't work in terraform mode: the terraform interpolator converts the path to diff --git a/acceptance/bundle/invariant/no_drift/out.test.toml b/acceptance/bundle/invariant/no_drift/out.test.toml index 8ed109d4820..2e4027b3b43 100644 --- a/acceptance/bundle/invariant/no_drift/out.test.toml +++ b/acceptance/bundle/invariant/no_drift/out.test.toml @@ -7,6 +7,7 @@ EnvMatrix.INPUT_CONFIG = [ "catalog_optional_fields.yml.tmpl", "cluster.yml.tmpl", "cluster_apply_policy_default_values.yml.tmpl", + "cluster_libraries.yml.tmpl", "cluster_policy.yml.tmpl", "dashboard.yml.tmpl", "job_apply_policy_default_values_job_cluster.yml.tmpl", diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index ac4584e034d..968afe44f3e 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -25,6 +25,7 @@ EnvMatrix.INPUT_CONFIG = [ "catalog_optional_fields.yml.tmpl", "cluster.yml.tmpl", "cluster_apply_policy_default_values.yml.tmpl", + "cluster_libraries.yml.tmpl", "cluster_policy.yml.tmpl", "dashboard.yml.tmpl", "job_apply_policy_default_values_job_cluster.yml.tmpl", diff --git a/acceptance/bundle/resources/clusters/libraries/dist/my_package-0.0.1-py3-none-any.whl b/acceptance/bundle/resources/clusters/libraries/dist/my_package-0.0.1-py3-none-any.whl new file mode 100644 index 00000000000..99c37c880a6 --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries/dist/my_package-0.0.1-py3-none-any.whl @@ -0,0 +1 @@ +dummy wheel contents \ No newline at end of file diff --git a/acceptance/invariant_test.go b/acceptance/invariant_test.go index 1c8204055ed..3450728ea6f 100644 --- a/acceptance/invariant_test.go +++ b/acceptance/invariant_test.go @@ -19,8 +19,9 @@ const invariantConfigsDir = "bundle/invariant/configs" // LackingInvariantTest lists keys from config.ResourcesTypes that knowingly lack // a covering config in invariantConfigsDir. Keys match the ResourcesTypes // form: "" for the resource itself, ".permissions" / ".grants" -// for permissions/grants coverage. Add a config and remove the entry to close a gap; -// the test fails if an entry here is actually covered, so the list only shrinks. +// / ".libraries" for sub-resource coverage. Add a config and remove the entry +// to close a gap; the test fails if an entry here is actually covered, so the list +// only shrinks. var LackingInvariantTest = map[string]bool{ "quality_monitors": true, } @@ -30,10 +31,11 @@ var LackingInvariantTest = map[string]bool{ // types supporting permissions or grants have at least one config exercising them. // // config.ResourcesTypes is the source of truth: it maps each resource group -// (e.g. "jobs") to its Go type and, where the resource struct has a Permissions -// or Grants field, adds derived keys ".permissions" and ".grants". +// (e.g. "jobs") to its Go type and adds derived keys ".permissions", +// ".grants", and ".libraries" where the resource has the +// corresponding sub-resource. func TestInvariantConfigsCoverage(t *testing.T) { - present, withPermissions, withGrants := scanInvariantConfigs(t) + present, withPermissions, withGrants, withLibraries := scanInvariantConfigs(t) keys := make([]string, 0, len(config.ResourcesTypes)) for key := range config.ResourcesTypes { @@ -53,6 +55,10 @@ func TestInvariantConfigsCoverage(t *testing.T) { group := strings.TrimSuffix(key, ".grants") covered = withGrants[group] hint = "attaches grants to a " + group + " resource" + case strings.HasSuffix(key, ".libraries"): + group := strings.TrimSuffix(key, ".libraries") + covered = withLibraries[group] + hint = "attaches libraries to a " + group + " resource" default: covered = present[key] hint = "defines a " + key + " resource" @@ -69,12 +75,13 @@ func TestInvariantConfigsCoverage(t *testing.T) { } // scanInvariantConfigs parses every config in the invariant configs directory and -// returns the set of resource groups present, the groups with at least one resource -// carrying permissions, and the groups with at least one resource carrying grants. -func scanInvariantConfigs(t *testing.T) (present, withPermissions, withGrants map[string]bool) { +// returns the set of resource groups present, and the groups with at least one +// resource carrying permissions, grants, or libraries. +func scanInvariantConfigs(t *testing.T) (present, withPermissions, withGrants, withLibraries map[string]bool) { present = map[string]bool{} withPermissions = map[string]bool{} withGrants = map[string]bool{} + withLibraries = map[string]bool{} entries, err := os.ReadDir(invariantConfigsDir) require.NoError(t, err) @@ -114,9 +121,12 @@ func scanInvariantConfigs(t *testing.T) (present, withPermissions, withGrants ma if cfg.Get("grants").Kind() != dyn.KindInvalid { withGrants[groupName] = true } + if cfg.Get("libraries").Kind() != dyn.KindInvalid { + withLibraries[groupName] = true + } } } } - return present, withPermissions, withGrants + return present, withPermissions, withGrants, withLibraries } From a0b4f3a43b25748f8e22b4d9c454deef2e26e312 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Mon, 31 Aug 2026 11:34:02 +0000 Subject: [PATCH 15/55] Restart cluster on library change so it takes effect Installs are invisible to attached notebooks and uninstalls are deferred until restart, so restart the running cluster after a library change (WaitAfterUpdate/WaitAfterDelete). DoDelete now uninstalls recorded libraries. Co-authored-by: Isaac --- .../clusters/libraries-drift/output.txt | 1 + .../resources/clusters/libraries/output.txt | 2 + bundle/direct/dresources/all_test.go | 5 +- bundle/direct/dresources/cluster_libraries.go | 62 +++++++++++++++++-- libs/testserver/clusters.go | 29 +++++++++ libs/testserver/handlers.go | 4 ++ 6 files changed, 98 insertions(+), 5 deletions(-) diff --git a/acceptance/bundle/resources/clusters/libraries-drift/output.txt b/acceptance/bundle/resources/clusters/libraries-drift/output.txt index 4a180dfbf0c..c2660075a25 100644 --- a/acceptance/bundle/resources/clusters/libraries-drift/output.txt +++ b/acceptance/bundle/resources/clusters/libraries-drift/output.txt @@ -9,4 +9,5 @@ The following resources will be deleted: All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] +Restarting cluster [UUID] because its libraries changed Destroy: 1 deleted diff --git a/acceptance/bundle/resources/clusters/libraries/output.txt b/acceptance/bundle/resources/clusters/libraries/output.txt index 74c60339a00..4d825933dfa 100644 --- a/acceptance/bundle/resources/clusters/libraries/output.txt +++ b/acceptance/bundle/resources/clusters/libraries/output.txt @@ -31,6 +31,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Removing the wheel and redeploying uninstalls it >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster-libraries/default/files... +Restarting cluster [UUID] because its libraries changed Updated clusters.mycluster.libraries Files: 3 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 1 unchanged @@ -55,4 +56,5 @@ The following resources will be deleted: All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/cluster-libraries/default +Restarting cluster [UUID] because its libraries changed Destroy: 1 deleted diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index 5d326411598..b8650d225a1 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -11,6 +11,7 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/structs/structaccess" "github.com/databricks/cli/libs/structs/structdiff" "github.com/databricks/cli/libs/structs/structpath" @@ -1031,7 +1032,7 @@ func testCRUD(t *testing.T, group string, adapter *Adapter, client *databricks.W newState, err := adapter.PrepareState(inputConfig) require.NoError(t, err, "PrepareState failed") - ctx := t.Context() + ctx := cmdio.MockDiscard(t.Context()) // initial DoRead() cannot find the resource remote, err := adapter.DoRead(ctx, "1234") @@ -1147,6 +1148,8 @@ func testCRUD(t *testing.T, group string, adapter *Adapter, client *databricks.W require.NoError(t, err) } + // permissions/grants have no-op deletes; libraries' DoDelete uninstalls but leaves the parent + // cluster, so DoRead still succeeds and reports the (now empty) set rather than erroring. deleteIsNoop := strings.HasSuffix(group, "permissions") || strings.HasSuffix(group, "grants") || strings.HasSuffix(group, "libraries") // Apps DoDelete is fire-and-forget: the API returns success while the app // sits in DELETING state for up to ~20 minutes before the record is removed. diff --git a/bundle/direct/dresources/cluster_libraries.go b/bundle/direct/dresources/cluster_libraries.go index a3458da761d..c613765d1ff 100644 --- a/bundle/direct/dresources/cluster_libraries.go +++ b/bundle/direct/dresources/cluster_libraries.go @@ -6,9 +6,11 @@ import ( "strings" "time" + "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/structs/structvar" "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/retries" "github.com/databricks/databricks-sdk-go/service/compute" ) @@ -156,10 +158,24 @@ func (r *ResourceLibraries) DoUpdate(ctx context.Context, id string, state *Libr return nil, nil } -// DoDelete is a no-op: removing individual libraries is handled by DoUpdate's uninstall diff, and -// DoDelete only fires when the parent cluster is deleted, at which point uninstalling is moot. -func (r *ResourceLibraries) DoDelete(ctx context.Context, id string, _ *LibrariesState) error { - return nil +// DoDelete uninstalls the libraries recorded in state. It fires both when the whole libraries +// block is removed from config (the cluster is kept) and during bundle destroy (the cluster is +// deleted too). WaitAfterDelete then restarts the cluster so the uninstall takes effect, since the +// Libraries API defers uninstalls until the next restart. +// +// TODO: during `bundle destroy` this restart is wasteful, since the cluster is permanently deleted +// right after. The delete path only receives the cluster id, so it cannot tell "block removed, +// cluster kept" from "cluster destroyed" and the restart fires in both. Worth revisiting if the +// framework can signal that the parent is also being deleted. +func (r *ResourceLibraries) DoDelete(ctx context.Context, id string, state *LibrariesState) error { + if len(state.EmbeddedSlice) == 0 { + return nil + } + // A 404 here means the parent cluster is already gone; the framework treats that as success. + return r.client.Libraries.Uninstall(ctx, compute.UninstallLibraries{ + ClusterId: id, + Libraries: state.EmbeddedSlice, + }) } // removedLibraries returns libraries present in the remote state but absent from the desired set. @@ -197,9 +213,47 @@ func (r *ResourceLibraries) WaitAfterCreate(ctx context.Context, id string, stat } func (r *ResourceLibraries) WaitAfterUpdate(ctx context.Context, id string, state *LibrariesState) (*LibrariesState, error) { + // Restart so the change is live: newly installed libraries are invisible to attached notebooks + // until restart, and the Libraries API defers uninstalls to the next restart. + if err := r.restartIfRunning(ctx, id); err != nil { + return nil, err + } return nil, r.waitForInstall(ctx, id, state.EmbeddedSlice) } +// WaitAfterDelete restarts the cluster so libraries uninstalled by DoDelete are actually evicted +// (the Libraries API defers uninstalls until restart). +func (r *ResourceLibraries) WaitAfterDelete(ctx context.Context, id string) error { + err := r.restartIfRunning(ctx, id) + if apierr.IsMissing(err) { + // Parent cluster already deleted; nothing to restart. + return nil + } + return err +} + +// restartIfRunning restarts the cluster so library changes take effect, but only when it is +// running: a stopped cluster applies pending install/uninstall on its next start. It waits for the +// cluster to return to RUNNING before returning. +func (r *ResourceLibraries) restartIfRunning(ctx context.Context, id string) error { + details, err := r.client.Clusters.GetByClusterId(ctx, id) + if err != nil { + return err + } + if details.State != compute.StateRunning { + log.Debugf(ctx, "cluster %s is not running (%s); skipping restart for library change", id, details.State) + return nil + } + + cmdio.LogString(ctx, fmt.Sprintf("Restarting cluster %s because its libraries changed", id)) + wait, err := r.client.Clusters.Restart(ctx, compute.RestartCluster{ClusterId: id}) + if err != nil { + return err + } + _, err = wait.GetWithTimeout(clusterWaitTimeout) + return err +} + // waitForInstall polls until every desired library reaches a terminal installed state. It returns // early without waiting when the cluster is not running: installs only progress on a running // cluster and are queued until it next starts. diff --git a/libs/testserver/clusters.go b/libs/testserver/clusters.go index 80991b0091a..14a6d45eef7 100644 --- a/libs/testserver/clusters.go +++ b/libs/testserver/clusters.go @@ -222,6 +222,35 @@ func (s *FakeWorkspace) ClustersStart(req Request) any { return Response{} } +// ClustersRestart restarts a running cluster. It moves the cluster back to PENDING so the next +// ClustersGet transitions it to RUNNING, mirroring how a real restart cycles through states, and +// clears the venv cache like ClustersEdit (a restart rebuilds the library environment). +func (s *FakeWorkspace) ClustersRestart(req Request) any { + var request compute.RestartCluster + if err := json.Unmarshal(req.Body, &request); err != nil { + return Response{ + StatusCode: 400, + Body: fmt.Sprintf("request parsing error: %s", err), + } + } + defer s.LockUnlock()() + + cluster, ok := s.Clusters[request.ClusterId] + if !ok { + return Response{StatusCode: 404} + } + + cluster.State = compute.StatePending + s.Clusters[request.ClusterId] = cluster + + if env, ok := s.clusterVenvs[request.ClusterId]; ok { + os.RemoveAll(env.dir) + delete(s.clusterVenvs, request.ClusterId) + } + + return Response{} +} + func (s *FakeWorkspace) ClustersPermanentDelete(req Request) any { var request compute.PermanentDeleteCluster if err := json.Unmarshal(req.Body, &request); err != nil { diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index e1ea0baa471..7fa5776a604 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -923,6 +923,10 @@ func AddDefaultHandlers(server *Server) { return req.Workspace.ClustersStart(req) }) + server.Handle("POST", "/api/2.1/clusters/restart", func(req Request) any { + return req.Workspace.ClustersRestart(req) + }) + server.Handle("POST", "/api/2.1/clusters/permanent-delete", func(req Request) any { return req.Workspace.ClustersPermanentDelete(req) }) From 3e32ae968bb47b469f07283c29261b2993462da3 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Mon, 31 Aug 2026 11:36:59 +0000 Subject: [PATCH 16/55] Add acceptance tests for cluster library restart Cover full-block removal (Delete path: uninstall + restart) and a stopped cluster (restart skipped when the cluster is not running). Co-authored-by: Isaac --- .../libraries-remove-all/databricks.yml | 13 +++++ .../libraries-remove-all/out.test.toml | 2 + .../clusters/libraries-remove-all/output.txt | 46 +++++++++++++++ .../clusters/libraries-remove-all/script | 19 ++++++ .../clusters/libraries-remove-all/test.toml | 10 ++++ .../libraries-stopped-cluster/databricks.yml | 15 +++++ .../libraries-stopped-cluster/out.test.toml | 2 + .../libraries-stopped-cluster/output.txt | 58 +++++++++++++++++++ .../clusters/libraries-stopped-cluster/script | 17 ++++++ .../libraries-stopped-cluster/test.toml | 10 ++++ 10 files changed, 192 insertions(+) create mode 100644 acceptance/bundle/resources/clusters/libraries-remove-all/databricks.yml create mode 100644 acceptance/bundle/resources/clusters/libraries-remove-all/out.test.toml create mode 100644 acceptance/bundle/resources/clusters/libraries-remove-all/output.txt create mode 100644 acceptance/bundle/resources/clusters/libraries-remove-all/script create mode 100644 acceptance/bundle/resources/clusters/libraries-remove-all/test.toml create mode 100644 acceptance/bundle/resources/clusters/libraries-stopped-cluster/databricks.yml create mode 100644 acceptance/bundle/resources/clusters/libraries-stopped-cluster/out.test.toml create mode 100644 acceptance/bundle/resources/clusters/libraries-stopped-cluster/output.txt create mode 100644 acceptance/bundle/resources/clusters/libraries-stopped-cluster/script create mode 100644 acceptance/bundle/resources/clusters/libraries-stopped-cluster/test.toml diff --git a/acceptance/bundle/resources/clusters/libraries-remove-all/databricks.yml b/acceptance/bundle/resources/clusters/libraries-remove-all/databricks.yml new file mode 100644 index 00000000000..ae60c5f3ec0 --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-remove-all/databricks.yml @@ -0,0 +1,13 @@ +bundle: + name: cluster-libraries-remove-all + +resources: + clusters: + mycluster: + cluster_name: mycluster + spark_version: 15.4.x-scala2.12 + node_type_id: i3.xlarge + num_workers: 1 + libraries: + - pypi: + package: requests diff --git a/acceptance/bundle/resources/clusters/libraries-remove-all/out.test.toml b/acceptance/bundle/resources/clusters/libraries-remove-all/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-remove-all/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/clusters/libraries-remove-all/output.txt b/acceptance/bundle/resources/clusters/libraries-remove-all/output.txt new file mode 100644 index 00000000000..73a497f44b7 --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-remove-all/output.txt @@ -0,0 +1,46 @@ + +=== Deploy a cluster with a pypi library +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster-libraries-remove-all/default/files... +Created clusters.mycluster +Created clusters.mycluster.libraries +Files: 5 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== Removing the whole libraries block plans a delete of the libraries sub-resource +>>> [CLI] bundle plan +delete clusters.mycluster.libraries + +Plan: 0 to add, 0 to change, 1 to delete, 1 unchanged + +=== Redeploy uninstalls the libraries and restarts the running cluster +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster-libraries-remove-all/default/files... +Restarting cluster [UUID] because its libraries changed +Deleted clusters.mycluster.libraries +Files: 3 uploaded, 0 deleted +Resources: 0 created, 0 changed, 1 deleted, 1 unchanged + +>>> print_requests.py //libraries/uninstall +{ + "method": "POST", + "path": "/api/2.0/libraries/uninstall", + "body": { + "cluster_id": "[UUID]", + "libraries": [ + { + "pypi": { + "package": "requests" + } + } + ] + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.clusters.mycluster + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/cluster-libraries-remove-all/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/clusters/libraries-remove-all/script b/acceptance/bundle/resources/clusters/libraries-remove-all/script new file mode 100644 index 00000000000..e95a77288e1 --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-remove-all/script @@ -0,0 +1,19 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +title "Deploy a cluster with a pypi library" +trace $CLI bundle deploy + +title "Removing the whole libraries block plans a delete of the libraries sub-resource" +update_file.py databricks.yml " libraries: + - pypi: + package: requests +" "" +trace $CLI bundle plan + +title "Redeploy uninstalls the libraries and restarts the running cluster" +trace $CLI bundle deploy +trace print_requests.py //libraries/uninstall diff --git a/acceptance/bundle/resources/clusters/libraries-remove-all/test.toml b/acceptance/bundle/resources/clusters/libraries-remove-all/test.toml new file mode 100644 index 00000000000..93cb0543245 --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-remove-all/test.toml @@ -0,0 +1,10 @@ +Cloud = false +RecordRequests = true + +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +Ignore = [".databricks"] + +[[Repls]] +Old = "[0-9]{4}-[0-9]{6}-[0-9a-z]{8}" +New = "[CLUSTER-ID]" diff --git a/acceptance/bundle/resources/clusters/libraries-stopped-cluster/databricks.yml b/acceptance/bundle/resources/clusters/libraries-stopped-cluster/databricks.yml new file mode 100644 index 00000000000..2831955da56 --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-stopped-cluster/databricks.yml @@ -0,0 +1,15 @@ +bundle: + name: cluster-libraries-stopped + +resources: + clusters: + mycluster: + cluster_name: mycluster + spark_version: 15.4.x-scala2.12 + node_type_id: i3.xlarge + num_workers: 1 + lifecycle: + started: false + libraries: + - pypi: + package: requests diff --git a/acceptance/bundle/resources/clusters/libraries-stopped-cluster/out.test.toml b/acceptance/bundle/resources/clusters/libraries-stopped-cluster/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-stopped-cluster/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/clusters/libraries-stopped-cluster/output.txt b/acceptance/bundle/resources/clusters/libraries-stopped-cluster/output.txt new file mode 100644 index 00000000000..061c4967ef3 --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-stopped-cluster/output.txt @@ -0,0 +1,58 @@ + +=== Deploy libraries on a stopped cluster (started=false): install is queued, no restart +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster-libraries-stopped/default/files... +Created clusters.mycluster +Created clusters.mycluster.libraries +Files: 5 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== Change the libraries and redeploy: cluster is stopped, so no restart is issued +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster-libraries-stopped/default/files... +Updated clusters.mycluster.libraries +Files: 3 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 1 unchanged + +>>> print_requests.py //libraries //clusters/restart +{ + "method": "POST", + "path": "/api/2.0/libraries/install", + "body": { + "cluster_id": "[UUID]", + "libraries": [ + { + "pypi": { + "package": "requests" + } + } + ] + } +} +{ + "method": "POST", + "path": "/api/2.0/libraries/install", + "body": { + "cluster_id": "[UUID]", + "libraries": [ + { + "pypi": { + "package": "requests" + } + }, + { + "pypi": { + "package": "numpy" + } + } + ] + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.clusters.mycluster + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/cluster-libraries-stopped/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/clusters/libraries-stopped-cluster/script b/acceptance/bundle/resources/clusters/libraries-stopped-cluster/script new file mode 100644 index 00000000000..cb7cb4dbf44 --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-stopped-cluster/script @@ -0,0 +1,17 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +title "Deploy libraries on a stopped cluster (started=false): install is queued, no restart" +trace $CLI bundle deploy + +title "Change the libraries and redeploy: cluster is stopped, so no restart is issued" +update_file.py databricks.yml " - pypi: + package: requests" " - pypi: + package: requests + - pypi: + package: numpy" +trace $CLI bundle deploy +trace print_requests.py //libraries //clusters/restart diff --git a/acceptance/bundle/resources/clusters/libraries-stopped-cluster/test.toml b/acceptance/bundle/resources/clusters/libraries-stopped-cluster/test.toml new file mode 100644 index 00000000000..93cb0543245 --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-stopped-cluster/test.toml @@ -0,0 +1,10 @@ +Cloud = false +RecordRequests = true + +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +Ignore = [".databricks"] + +[[Repls]] +Old = "[0-9]{4}-[0-9]{6}-[0-9a-z]{8}" +New = "[CLUSTER-ID]" From ed672d77bbc299a2c09d658b2925bde5e3f76c26 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Mon, 31 Aug 2026 11:43:47 +0000 Subject: [PATCH 17/55] Fix exhaustruct lint on RestartCluster literal Set all fields (RestartUser, ForceSendFields) explicitly, matching the dresources exhaustruct convention. Co-authored-by: Isaac --- bundle/direct/dresources/cluster_libraries.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bundle/direct/dresources/cluster_libraries.go b/bundle/direct/dresources/cluster_libraries.go index c613765d1ff..384b5c79c48 100644 --- a/bundle/direct/dresources/cluster_libraries.go +++ b/bundle/direct/dresources/cluster_libraries.go @@ -246,7 +246,11 @@ func (r *ResourceLibraries) restartIfRunning(ctx context.Context, id string) err } cmdio.LogString(ctx, fmt.Sprintf("Restarting cluster %s because its libraries changed", id)) - wait, err := r.client.Clusters.Restart(ctx, compute.RestartCluster{ClusterId: id}) + wait, err := r.client.Clusters.Restart(ctx, compute.RestartCluster{ + ClusterId: id, + RestartUser: "", + ForceSendFields: nil, + }) if err != nil { return err } From 690aa623c788e2c146a98a19256d754ec5a1404a Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Mon, 31 Aug 2026 11:48:49 +0000 Subject: [PATCH 18/55] update comment --- bundle/direct/dresources/all_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index b8650d225a1..62f26248a60 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -1148,8 +1148,8 @@ func testCRUD(t *testing.T, group string, adapter *Adapter, client *databricks.W require.NoError(t, err) } - // permissions/grants have no-op deletes; libraries' DoDelete uninstalls but leaves the parent - // cluster, so DoRead still succeeds and reports the (now empty) set rather than erroring. + // this lists all the sub-resources that do not delete the parent resource when they + // are deleted. deleteIsNoop := strings.HasSuffix(group, "permissions") || strings.HasSuffix(group, "grants") || strings.HasSuffix(group, "libraries") // Apps DoDelete is fire-and-forget: the API returns success while the app // sits in DELETING state for up to ~20 minutes before the record is removed. From 5ee3448885df79265cf8f8e21f0758ad3e7a4082 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Mon, 31 Aug 2026 12:12:04 +0000 Subject: [PATCH 19/55] Narrow libraries-stopped-cluster request filter to fix Windows The broad //libraries filter degrades to a bare 'libraries' substring on Windows Git Bash, matching the bundle name (cluster-libraries-stopped) in file-upload paths and dumping unrelated requests. Use //libraries/install like the sibling tests so the filter stays specific across platforms. Co-authored-by: Isaac --- .../resources/clusters/libraries-stopped-cluster/output.txt | 2 +- .../bundle/resources/clusters/libraries-stopped-cluster/script | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/resources/clusters/libraries-stopped-cluster/output.txt b/acceptance/bundle/resources/clusters/libraries-stopped-cluster/output.txt index 061c4967ef3..05044c69577 100644 --- a/acceptance/bundle/resources/clusters/libraries-stopped-cluster/output.txt +++ b/acceptance/bundle/resources/clusters/libraries-stopped-cluster/output.txt @@ -14,7 +14,7 @@ Updated clusters.mycluster.libraries Files: 3 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 1 unchanged ->>> print_requests.py //libraries //clusters/restart +>>> print_requests.py //libraries/install //clusters/restart { "method": "POST", "path": "/api/2.0/libraries/install", diff --git a/acceptance/bundle/resources/clusters/libraries-stopped-cluster/script b/acceptance/bundle/resources/clusters/libraries-stopped-cluster/script index cb7cb4dbf44..ea1d6164d82 100644 --- a/acceptance/bundle/resources/clusters/libraries-stopped-cluster/script +++ b/acceptance/bundle/resources/clusters/libraries-stopped-cluster/script @@ -14,4 +14,4 @@ update_file.py databricks.yml " - pypi: - pypi: package: numpy" trace $CLI bundle deploy -trace print_requests.py //libraries //clusters/restart +trace print_requests.py //libraries/install //clusters/restart From 3ba91b8106aa110f778aef8ac6ab6a33eb6b1de5 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Mon, 31 Aug 2026 12:16:22 +0000 Subject: [PATCH 20/55] update comments --- bundle/direct/dresources/cluster_libraries.go | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/bundle/direct/dresources/cluster_libraries.go b/bundle/direct/dresources/cluster_libraries.go index 384b5c79c48..abbcbf3a345 100644 --- a/bundle/direct/dresources/cluster_libraries.go +++ b/bundle/direct/dresources/cluster_libraries.go @@ -22,8 +22,7 @@ import ( const librariesWaitTimeout = 15 * time.Minute // LibrariesState is the state for a cluster's libraries sub-resource. Libraries are installed -// via the Libraries API against the parent cluster identified by ClusterId, not through the -// cluster spec. +// via the Libraries API against the parent cluster identified by ClusterId. type LibrariesState struct { ClusterId string `json:"cluster_id"` // By convention EmbeddedSlice fields have the __embed__ json tag, see permissions.go. @@ -132,8 +131,7 @@ func (r *ResourceLibraries) DoCreate(ctx context.Context, state *LibrariesState) } // DoUpdate uninstalls libraries removed from config and installs the desired set. This is two API -// calls because the Libraries API exposes install and uninstall as separate endpoints, unlike the -// single-call model most resources follow. +// calls because the Libraries API exposes install and uninstall as separate endpoints func (r *ResourceLibraries) DoUpdate(ctx context.Context, id string, state *LibrariesState, entry *PlanEntry) (*LibrariesState, error) { removed := removedLibraries(state.EmbeddedSlice, entry) if len(removed) > 0 { @@ -163,10 +161,8 @@ func (r *ResourceLibraries) DoUpdate(ctx context.Context, id string, state *Libr // deleted too). WaitAfterDelete then restarts the cluster so the uninstall takes effect, since the // Libraries API defers uninstalls until the next restart. // -// TODO: during `bundle destroy` this restart is wasteful, since the cluster is permanently deleted -// right after. The delete path only receives the cluster id, so it cannot tell "block removed, -// cluster kept" from "cluster destroyed" and the restart fires in both. Worth revisiting if the -// framework can signal that the parent is also being deleted. +// During `bundle destroy` this restart is wasteful, but it is not possible to differentitate +// between the two cases, without adding additional logic to the framework. func (r *ResourceLibraries) DoDelete(ctx context.Context, id string, state *LibrariesState) error { if len(state.EmbeddedSlice) == 0 { return nil From f2a022a35c59d217e9cae195cedb823875bec33a Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Mon, 31 Aug 2026 12:31:22 +0000 Subject: [PATCH 21/55] Exclude cluster_libraries from continue_293 backward-compat invariant Cluster libraries postdate v0.293.0, so the old CLI cannot deploy the config; exclude it like the other post-0.293 resources. Addresses PR review feedback. Co-authored-by: Isaac --- acceptance/bundle/invariant/continue_293/out.test.toml | 1 - acceptance/bundle/invariant/continue_293/test.toml | 3 +++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/acceptance/bundle/invariant/continue_293/out.test.toml b/acceptance/bundle/invariant/continue_293/out.test.toml index f041da59ad0..f5883975fcd 100644 --- a/acceptance/bundle/invariant/continue_293/out.test.toml +++ b/acceptance/bundle/invariant/continue_293/out.test.toml @@ -7,7 +7,6 @@ EnvMatrix.INPUT_CONFIG = [ "catalog_optional_fields.yml.tmpl", "cluster.yml.tmpl", "cluster_apply_policy_default_values.yml.tmpl", - "cluster_libraries.yml.tmpl", "dashboard.yml.tmpl", "job_apply_policy_default_values_job_cluster.yml.tmpl", "job_apply_policy_default_values_task_cluster.yml.tmpl", diff --git a/acceptance/bundle/invariant/continue_293/test.toml b/acceptance/bundle/invariant/continue_293/test.toml index f174ba144c4..12d645e6b6b 100644 --- a/acceptance/bundle/invariant/continue_293/test.toml +++ b/acceptance/bundle/invariant/continue_293/test.toml @@ -19,6 +19,9 @@ EnvMatrixExclude.no_instance_pool = ["INPUT_CONFIG=instance_pool.yml.tmpl"] # cluster_policies resource is not supported on v0.293.0 EnvMatrixExclude.no_cluster_policy = ["INPUT_CONFIG=cluster_policy.yml.tmpl"] +# cluster libraries (clusters.libraries) are not supported on v0.293.0 +EnvMatrixExclude.no_cluster_libraries = ["INPUT_CONFIG=cluster_libraries.yml.tmpl"] + # job_runs resource is not supported on v0.293.0 EnvMatrixExclude.no_job_run = ["INPUT_CONFIG=job_run.yml.tmpl"] From bbbe43a915e55045e77270785cd798d06aab3bfa Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Mon, 31 Aug 2026 12:58:04 +0000 Subject: [PATCH 22/55] Scope libraries sub-resource split to clusters in splitResourcePath splitResourcePath treated any 4th-component 'libraries' as a sub-resource, but only clusters expose libraries as one; pipelines have a native top-level libraries field, so a reference into a pipeline's libraries mis-split to a non-existent node. Scope it to clusters, matching makePlan. Addresses Isaac Review finding. Co-authored-by: Isaac --- bundle/direct/bundle_plan.go | 11 ++++++++--- bundle/direct/bundle_plan_test.go | 25 +++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 6f71bd5f2ab..1618d46a201 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -717,11 +717,16 @@ func isEmptyStruct(rv reflect.Value) bool { // For regular resources like "resources.jobs.foo.name", returns ("resources.jobs.foo", "name"). // For sub-resources like "resources.jobs.foo.permissions[0].level", returns ("resources.jobs.foo.permissions", "[0].level"). func splitResourcePath(path *structpath.PathNode) (string, *structpath.PathNode) { - // Check if the 4th component is "permissions" or "grants" (sub-resource) + // permissions and grants are sub-resources for every group; libraries is a sub-resource + // only for clusters (pipelines have a native top-level libraries field), so scope it to + // clusters, matching the pattern in makePlan. if path.Len() > 4 { first := path.SkipPrefix(3).Prefix(1) - if key, ok := first.StringKey(); ok && (key == "permissions" || key == "grants" || key == "libraries") { - return path.Prefix(4).String(), path.SkipPrefix(4) + if key, ok := first.StringKey(); ok { + group, _ := path.SkipPrefix(1).Prefix(1).StringKey() + if key == "permissions" || key == "grants" || (key == "libraries" && group == "clusters") { + return path.Prefix(4).String(), path.SkipPrefix(4) + } } } return path.Prefix(3).String(), path.SkipPrefix(3) diff --git a/bundle/direct/bundle_plan_test.go b/bundle/direct/bundle_plan_test.go index bf875680c2b..d711de39e63 100644 --- a/bundle/direct/bundle_plan_test.go +++ b/bundle/direct/bundle_plan_test.go @@ -401,3 +401,28 @@ func bundleWithSkippedJobRun(t *testing.T, remote *dresources.JobRunRemote) *Dep b.RemoteStateCache.Store(jobRunKey, remote) return b } + +func TestSplitResourcePath(t *testing.T) { + tests := []struct { + path string + wantKey string + wantField string + }{ + {"resources.jobs.foo.name", "resources.jobs.foo", "name"}, + {"resources.jobs.foo.permissions[0].level", "resources.jobs.foo.permissions", "[0].level"}, + {"resources.schemas.foo.grants[0].principal", "resources.schemas.foo.grants", "[0].principal"}, + // libraries is a sub-resource only for clusters. + {"resources.clusters.foo.libraries[0].whl", "resources.clusters.foo.libraries", "[0].whl"}, + // pipelines have a native top-level libraries field, so it must not split as a sub-resource. + {"resources.pipelines.foo.libraries[0].notebook.path", "resources.pipelines.foo", "libraries[0].notebook.path"}, + } + for _, tt := range tests { + t.Run(tt.path, func(t *testing.T) { + path, err := structpath.ParsePath(tt.path) + require.NoError(t, err) + key, field := splitResourcePath(path) + assert.Equal(t, tt.wantKey, key) + assert.Equal(t, tt.wantField, field.String()) + }) + } +} From 1b80d8535390c8f52636bc517b52164a54ed98af Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Mon, 31 Aug 2026 13:43:16 +0000 Subject: [PATCH 23/55] Include repo/exclusions in cluster library identity key libraryKey ignored pypi/maven/cran repo and maven exclusions, so a change to only the repo was treated as no change and never applied. The backend hashes these into library_id_hash and round-trips them on cluster-status, so include them. Adds a Cloud=true test covering repo round-trip + repo-change detection. Co-authored-by: Isaac --- .../libraries-repo/databricks.yml.tmpl | 18 ++++++++++ .../clusters/libraries-repo/out.test.toml | 2 ++ .../clusters/libraries-repo/output.txt | 19 ++++++++++ .../resources/clusters/libraries-repo/script | 18 ++++++++++ .../clusters/libraries-repo/test.toml | 6 ++++ bundle/direct/dresources/cluster_libraries.go | 36 +++++++++++++++---- 6 files changed, 93 insertions(+), 6 deletions(-) create mode 100644 acceptance/bundle/resources/clusters/libraries-repo/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/clusters/libraries-repo/out.test.toml create mode 100644 acceptance/bundle/resources/clusters/libraries-repo/output.txt create mode 100644 acceptance/bundle/resources/clusters/libraries-repo/script create mode 100644 acceptance/bundle/resources/clusters/libraries-repo/test.toml diff --git a/acceptance/bundle/resources/clusters/libraries-repo/databricks.yml.tmpl b/acceptance/bundle/resources/clusters/libraries-repo/databricks.yml.tmpl new file mode 100644 index 00000000000..40f63c9241c --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-repo/databricks.yml.tmpl @@ -0,0 +1,18 @@ +bundle: + name: cluster-libraries-repo-$UNIQUE_NAME + +workspace: + root_path: ~/.bundle/$UNIQUE_NAME + +resources: + clusters: + mycluster: + cluster_name: mycluster-$UNIQUE_NAME + spark_version: $DEFAULT_SPARK_VERSION + node_type_id: $NODE_TYPE_ID + instance_pool_id: $TEST_INSTANCE_POOL_ID + num_workers: 1 + libraries: + - pypi: + package: requests + repo: https://pypi.org/simple diff --git a/acceptance/bundle/resources/clusters/libraries-repo/out.test.toml b/acceptance/bundle/resources/clusters/libraries-repo/out.test.toml new file mode 100644 index 00000000000..c502b28221b --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-repo/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/clusters/libraries-repo/output.txt b/acceptance/bundle/resources/clusters/libraries-repo/output.txt new file mode 100644 index 00000000000..d15fccc198d --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-repo/output.txt @@ -0,0 +1,19 @@ + +=== Plan is a no-op after deploy: the repo round-trips +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +=== Changing only the repo is planned as an update +>>> [CLI] bundle plan +update clusters.mycluster.libraries + +Plan: 0 to add, 1 to change, 0 to delete, 1 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.clusters.mycluster + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] + +Restarting cluster [UUID] because its libraries changed +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/clusters/libraries-repo/script b/acceptance/bundle/resources/clusters/libraries-repo/script new file mode 100644 index 00000000000..11cb4c56207 --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-repo/script @@ -0,0 +1,18 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +# Deploy output is noisy and differs fake vs cloud; assert only the plan signals below. +$CLI bundle deploy &> LOG.deploy +cat LOG.deploy | contains.py "Created clusters.mycluster.libraries" > /dev/null + +title "Plan is a no-op after deploy: the repo round-trips" +trace $CLI bundle plan + +title "Changing only the repo is planned as an update" +update_file.py databricks.yml "repo: https://pypi.org/simple" "repo: https://example.invalid/simple" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/clusters/libraries-repo/test.toml b/acceptance/bundle/resources/clusters/libraries-repo/test.toml new file mode 100644 index 00000000000..63a8c8a332c --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-repo/test.toml @@ -0,0 +1,6 @@ +Cloud = true +RecordRequests = false + +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +Ignore = [".databricks", "databricks.yml"] diff --git a/bundle/direct/dresources/cluster_libraries.go b/bundle/direct/dresources/cluster_libraries.go index abbcbf3a345..7510d4818ff 100644 --- a/bundle/direct/dresources/cluster_libraries.go +++ b/bundle/direct/dresources/cluster_libraries.go @@ -64,9 +64,8 @@ func (*ResourceLibraries) IsEmptyState(state *LibrariesState) bool { return len(state.EmbeddedSlice) == 0 } -// libraryKey identifies a library by its type-specific field so slices compare by identity -// rather than by index (see KeyedSlices). -func libraryKey(l compute.Library) (string, string) { +// libraryWaitKey identifies a library by its primary field only, used to match install-status reports. +func libraryWaitKey(l compute.Library) (string, string) { switch { case l.Whl != "": return "whl", l.Whl @@ -86,6 +85,20 @@ func libraryKey(l compute.Library) (string, string) { return "", "" } +// libraryKey extends libraryWaitKey with repo/exclusions so a repo-only change is detected (see KeyedSlices). +func libraryKey(l compute.Library) (string, string) { + typ, id := libraryWaitKey(l) + switch { + case l.Pypi != nil: + return typ, id + ";" + l.Pypi.Repo + case l.Maven != nil: + return typ, id + ";" + l.Maven.Repo + ";" + strings.Join(l.Maven.Exclusions, ",") + case l.Cran != nil: + return typ, id + ";" + l.Cran.Repo + } + return typ, id +} + func (*ResourceLibraries) KeyedSlices() map[string]any { // Empty key because EmbeddedSlice appears at the root path of LibrariesState. return map[string]any{ @@ -204,6 +217,12 @@ func libraryMapKey(l compute.Library) string { return f + "=" + v } +// libraryWaitMapKey flattens libraryWaitKey into a single string for map lookups. +func libraryWaitMapKey(l compute.Library) string { + f, v := libraryWaitKey(l) + return f + "=" + v +} + func (r *ResourceLibraries) WaitAfterCreate(ctx context.Context, id string, state *LibrariesState) (*LibrariesState, error) { return nil, r.waitForInstall(ctx, id, state.EmbeddedSlice) } @@ -273,7 +292,12 @@ func (r *ResourceLibraries) waitForInstall(ctx context.Context, id string, desir desiredKeys := make(map[string]struct{}, len(desired)) for _, l := range desired { - desiredKeys[libraryMapKey(l)] = struct{}{} + typ, id := libraryWaitKey(l) + if typ == "" { + // Unknown library type: it can't be matched in cluster-status, so don't wait for it. + continue + } + desiredKeys[typ+"="+id] = struct{}{} } _, err = retries.Poll(ctx, librariesWaitTimeout, func() (*struct{}, *retries.Err) { @@ -287,12 +311,12 @@ func (r *ResourceLibraries) waitForInstall(ctx context.Context, id string, desir if s.Library == nil { continue } - if _, ok := desiredKeys[libraryMapKey(*s.Library)]; !ok { + if _, ok := desiredKeys[libraryWaitMapKey(*s.Library)]; !ok { continue } switch s.Status { case compute.LibraryInstallStatusFailed: - return nil, retries.Halt(fmt.Errorf("library %s failed to install: %s", libraryMapKey(*s.Library), strings.Join(s.Messages, "; "))) + return nil, retries.Halt(fmt.Errorf("library %s failed to install: %s", libraryWaitMapKey(*s.Library), strings.Join(s.Messages, "; "))) case compute.LibraryInstallStatusInstalled, compute.LibraryInstallStatusSkipped, compute.LibraryInstallStatusRestored: pending-- case compute.LibraryInstallStatusPending, compute.LibraryInstallStatusResolving, compute.LibraryInstallStatusInstalling, compute.LibraryInstallStatusUninstallOnRestart: From 078f0225b238c777fc8320429353caec29890156 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Mon, 31 Aug 2026 13:57:04 +0000 Subject: [PATCH 24/55] Skip clusters.libraries from parent walk in refschema dump The libraries sub-resource is emitted by both the clusters adapter's input walk (as an INPUT-only block) and the clusters.libraries sub-resource adapter (as an ALL block), duplicating resources.clusters.*.libraries in the refschema dump. permissions and grants avoid this via an explicit skip filter; extend it to libraries, scoped to clusters since pipelines have a native top-level libraries field. Co-authored-by: Isaac --- acceptance/bundle/refschema/out.fields.txt | 17 ----------------- cmd/bundle/debug/refschema.go | 5 +++++ 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 5e00e1b1f7e..ca59ff7374d 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -460,23 +460,6 @@ resources.clusters.*.jdbc_port int REMOTE resources.clusters.*.kind compute.Kind ALL resources.clusters.*.last_restarted_time int64 REMOTE resources.clusters.*.last_state_loss_time int64 REMOTE -resources.clusters.*.libraries []compute.Library INPUT -resources.clusters.*.libraries[*] compute.Library INPUT -resources.clusters.*.libraries[*].cran *compute.RCranLibrary INPUT -resources.clusters.*.libraries[*].cran.package string INPUT -resources.clusters.*.libraries[*].cran.repo string INPUT -resources.clusters.*.libraries[*].egg string INPUT -resources.clusters.*.libraries[*].jar string INPUT -resources.clusters.*.libraries[*].maven *compute.MavenLibrary INPUT -resources.clusters.*.libraries[*].maven.coordinates string INPUT -resources.clusters.*.libraries[*].maven.exclusions []string INPUT -resources.clusters.*.libraries[*].maven.exclusions[*] string INPUT -resources.clusters.*.libraries[*].maven.repo string INPUT -resources.clusters.*.libraries[*].pypi *compute.PythonPyPiLibrary INPUT -resources.clusters.*.libraries[*].pypi.package string INPUT -resources.clusters.*.libraries[*].pypi.repo string INPUT -resources.clusters.*.libraries[*].requirements string INPUT -resources.clusters.*.libraries[*].whl string INPUT resources.clusters.*.lifecycle *dresources.StateLifecycle REMOTE STATE resources.clusters.*.lifecycle *resources.LifecycleWithStarted INPUT resources.clusters.*.lifecycle resources.Lifecycle INPUT diff --git a/cmd/bundle/debug/refschema.go b/cmd/bundle/debug/refschema.go index 4ba1ae999fb..f848c56c04c 100644 --- a/cmd/bundle/debug/refschema.go +++ b/cmd/bundle/debug/refschema.go @@ -73,6 +73,11 @@ func dumpRemoteSchemas(out io.Writer) error { p == "grants" || strings.HasPrefix(p, "grants.") || strings.HasPrefix(p, "grants[") { return false } + // libraries is a sub-resource adapter for clusters only (pipelines have a native + // top-level libraries field), so skip it here just for clusters. + if resourceName == "clusters" && (p == "libraries" || strings.HasPrefix(p, "libraries.") || strings.HasPrefix(p, "libraries[")) { + return false + } t := strings.ReplaceAll(fmt.Sprint(typ), "interface {}", "any") byType, ok := pathTypes[p] if !ok { From 00895fee7120c16f43d1e57f69d9df487318f048 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Mon, 31 Aug 2026 14:09:05 +0000 Subject: [PATCH 25/55] Remove redundant libraries-drift acceptance test The invariant no_drift suite now runs configs/cluster_libraries.yml.tmpl (added in 612add74d), which deploys the identical pypi-requests cluster config and asserts no drift after deploy, plus the delete/destroy/migrate idempotency variants. That makes the standalone libraries-drift cloud test a strict subset. The restart-on-destroy output it captured is already asserted in the local libraries test, and libraries-repo remains as the stronger repo round-trip check. Co-authored-by: Isaac --- .../libraries-drift/databricks.yml.tmpl | 17 ---------------- .../clusters/libraries-drift/out.test.toml | 2 -- .../clusters/libraries-drift/output.txt | 13 ------------ .../resources/clusters/libraries-drift/script | 20 ------------------- .../clusters/libraries-drift/test.toml | 6 ------ 5 files changed, 58 deletions(-) delete mode 100644 acceptance/bundle/resources/clusters/libraries-drift/databricks.yml.tmpl delete mode 100644 acceptance/bundle/resources/clusters/libraries-drift/out.test.toml delete mode 100644 acceptance/bundle/resources/clusters/libraries-drift/output.txt delete mode 100644 acceptance/bundle/resources/clusters/libraries-drift/script delete mode 100644 acceptance/bundle/resources/clusters/libraries-drift/test.toml diff --git a/acceptance/bundle/resources/clusters/libraries-drift/databricks.yml.tmpl b/acceptance/bundle/resources/clusters/libraries-drift/databricks.yml.tmpl deleted file mode 100644 index b3bf2463741..00000000000 --- a/acceptance/bundle/resources/clusters/libraries-drift/databricks.yml.tmpl +++ /dev/null @@ -1,17 +0,0 @@ -bundle: - name: cluster-libraries-drift-$UNIQUE_NAME - -workspace: - root_path: ~/.bundle/$UNIQUE_NAME - -resources: - clusters: - mycluster: - cluster_name: mycluster-$UNIQUE_NAME - spark_version: $DEFAULT_SPARK_VERSION - node_type_id: $NODE_TYPE_ID - instance_pool_id: $TEST_INSTANCE_POOL_ID - num_workers: 1 - libraries: - - pypi: - package: requests diff --git a/acceptance/bundle/resources/clusters/libraries-drift/out.test.toml b/acceptance/bundle/resources/clusters/libraries-drift/out.test.toml deleted file mode 100644 index c502b28221b..00000000000 --- a/acceptance/bundle/resources/clusters/libraries-drift/out.test.toml +++ /dev/null @@ -1,2 +0,0 @@ -Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/clusters/libraries-drift/output.txt b/acceptance/bundle/resources/clusters/libraries-drift/output.txt deleted file mode 100644 index c2660075a25..00000000000 --- a/acceptance/bundle/resources/clusters/libraries-drift/output.txt +++ /dev/null @@ -1,13 +0,0 @@ - -=== Plan is a no-op immediately after deploy (no library drift) ->>> [CLI] bundle plan -Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged - ->>> [CLI] bundle destroy --auto-approve -The following resources will be deleted: - delete resources.clusters.mycluster - -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] - -Restarting cluster [UUID] because its libraries changed -Destroy: 1 deleted diff --git a/acceptance/bundle/resources/clusters/libraries-drift/script b/acceptance/bundle/resources/clusters/libraries-drift/script deleted file mode 100644 index b1fef7ee702..00000000000 --- a/acceptance/bundle/resources/clusters/libraries-drift/script +++ /dev/null @@ -1,20 +0,0 @@ -# A pypi library is used because its nested {package, repo} shape is the most -# likely to drift (the status API echoing a repo we did not set); a workspace -# wheel is covered by the local clusters/libraries test instead, since the -# shared cloud test cluster rejects libraries from /Workspace paths. -envsubst < databricks.yml.tmpl > databricks.yml - -cleanup() { - trace $CLI bundle destroy --auto-approve - rm -f out.requests.txt -} -trap cleanup EXIT - -# Cluster provisioning and library-install output is noisy and differs between -# the fake and cloud, so route it to LOG and assert only the deterministic -# drift signal below. -$CLI bundle deploy &> LOG.deploy -cat LOG.deploy | contains.py "Created clusters.mycluster.libraries" > /dev/null - -title "Plan is a no-op immediately after deploy (no library drift)" -trace $CLI bundle plan diff --git a/acceptance/bundle/resources/clusters/libraries-drift/test.toml b/acceptance/bundle/resources/clusters/libraries-drift/test.toml deleted file mode 100644 index 63a8c8a332c..00000000000 --- a/acceptance/bundle/resources/clusters/libraries-drift/test.toml +++ /dev/null @@ -1,6 +0,0 @@ -Cloud = true -RecordRequests = false - -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] - -Ignore = [".databricks", "databricks.yml"] From 7f40ac85489ed0c45992aae9a17cbf8aadc68045 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Mon, 31 Aug 2026 14:56:01 +0000 Subject: [PATCH 26/55] Run cluster-libraries acceptance tests on cloud The three pypi/wheel tests were authored fake-only (hardcoded i3.xlarge, no $UNIQUE_NAME, no instance pool, static databricks.yml), so they could not run against a real workspace. Convert them to cloud: - libraries: build a real wheel via setup.py + an artifacts block (mirroring integration_whl/interactive_cluster) instead of the 20-byte dummy stub, so the /Workspace wheel install is exercised on a real cluster. Cloud + CloudSlow; data_security_mode=USER_ISOLATION. Deploy output routed to LOG; asserts the install request (rewritten /Workspace wheel path), uninstall, and restart. - libraries-remove-all, libraries-stopped-cluster: parameterize databricks.yml as a .tmpl with cloud vars and instance pool; assert summary/restart lines instead of exact request bodies (which the recording proxy makes redundant). - All four tests: map the cloud cluster-id format to [UUID] so cloud output matches the fake run. All four remain Cloud=true and still run locally against the fake server. Co-authored-by: Isaac --- .../libraries-remove-all/databricks.yml | 13 ----- .../libraries-remove-all/databricks.yml.tmpl | 17 +++++++ .../libraries-remove-all/out.test.toml | 2 +- .../clusters/libraries-remove-all/output.txt | 32 +----------- .../clusters/libraries-remove-all/script | 11 ++-- .../clusters/libraries-remove-all/test.toml | 8 +-- .../clusters/libraries-repo/test.toml | 4 ++ .../libraries-stopped-cluster/databricks.yml | 15 ------ .../databricks.yml.tmpl | 19 +++++++ .../libraries-stopped-cluster/out.test.toml | 2 +- .../libraries-stopped-cluster/output.txt | 50 +------------------ .../clusters/libraries-stopped-cluster/script | 12 +++-- .../libraries-stopped-cluster/test.toml | 8 +-- .../clusters/libraries/databricks.yml | 14 ------ .../clusters/libraries/databricks.yml.tmpl | 25 ++++++++++ .../dist/my_package-0.0.1-py3-none-any.whl | 1 - .../libraries/my_test_code/__init__.py | 2 + .../libraries/my_test_code/__main__.py | 16 ++++++ .../clusters/libraries/out.test.toml | 3 +- .../resources/clusters/libraries/output.txt | 23 ++------- .../resources/clusters/libraries/script | 15 ++++-- .../clusters/libraries/script.prepare | 9 ++++ .../resources/clusters/libraries/setup.py | 15 ++++++ .../resources/clusters/libraries/test.toml | 7 +-- 24 files changed, 158 insertions(+), 165 deletions(-) delete mode 100644 acceptance/bundle/resources/clusters/libraries-remove-all/databricks.yml create mode 100644 acceptance/bundle/resources/clusters/libraries-remove-all/databricks.yml.tmpl delete mode 100644 acceptance/bundle/resources/clusters/libraries-stopped-cluster/databricks.yml create mode 100644 acceptance/bundle/resources/clusters/libraries-stopped-cluster/databricks.yml.tmpl delete mode 100644 acceptance/bundle/resources/clusters/libraries/databricks.yml create mode 100644 acceptance/bundle/resources/clusters/libraries/databricks.yml.tmpl delete mode 100644 acceptance/bundle/resources/clusters/libraries/dist/my_package-0.0.1-py3-none-any.whl create mode 100644 acceptance/bundle/resources/clusters/libraries/my_test_code/__init__.py create mode 100644 acceptance/bundle/resources/clusters/libraries/my_test_code/__main__.py create mode 100644 acceptance/bundle/resources/clusters/libraries/script.prepare create mode 100644 acceptance/bundle/resources/clusters/libraries/setup.py diff --git a/acceptance/bundle/resources/clusters/libraries-remove-all/databricks.yml b/acceptance/bundle/resources/clusters/libraries-remove-all/databricks.yml deleted file mode 100644 index ae60c5f3ec0..00000000000 --- a/acceptance/bundle/resources/clusters/libraries-remove-all/databricks.yml +++ /dev/null @@ -1,13 +0,0 @@ -bundle: - name: cluster-libraries-remove-all - -resources: - clusters: - mycluster: - cluster_name: mycluster - spark_version: 15.4.x-scala2.12 - node_type_id: i3.xlarge - num_workers: 1 - libraries: - - pypi: - package: requests diff --git a/acceptance/bundle/resources/clusters/libraries-remove-all/databricks.yml.tmpl b/acceptance/bundle/resources/clusters/libraries-remove-all/databricks.yml.tmpl new file mode 100644 index 00000000000..9aa23bd6c8d --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-remove-all/databricks.yml.tmpl @@ -0,0 +1,17 @@ +bundle: + name: cluster-libraries-remove-all-$UNIQUE_NAME + +workspace: + root_path: ~/.bundle/$UNIQUE_NAME + +resources: + clusters: + mycluster: + cluster_name: mycluster-$UNIQUE_NAME + spark_version: $DEFAULT_SPARK_VERSION + node_type_id: $NODE_TYPE_ID + instance_pool_id: $TEST_INSTANCE_POOL_ID + num_workers: 1 + libraries: + - pypi: + package: requests diff --git a/acceptance/bundle/resources/clusters/libraries-remove-all/out.test.toml b/acceptance/bundle/resources/clusters/libraries-remove-all/out.test.toml index 0938e678987..c502b28221b 100644 --- a/acceptance/bundle/resources/clusters/libraries-remove-all/out.test.toml +++ b/acceptance/bundle/resources/clusters/libraries-remove-all/out.test.toml @@ -1,2 +1,2 @@ -Cloud = false +Cloud = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/clusters/libraries-remove-all/output.txt b/acceptance/bundle/resources/clusters/libraries-remove-all/output.txt index 73a497f44b7..1d5670b04e1 100644 --- a/acceptance/bundle/resources/clusters/libraries-remove-all/output.txt +++ b/acceptance/bundle/resources/clusters/libraries-remove-all/output.txt @@ -1,12 +1,5 @@ === Deploy a cluster with a pypi library ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster-libraries-remove-all/default/files... -Created clusters.mycluster -Created clusters.mycluster.libraries -Files: 5 uploaded, 0 deleted -Resources: 2 created, 0 changed, 0 deleted, 0 unchanged - === Removing the whole libraries block plans a delete of the libraries sub-resource >>> [CLI] bundle plan delete clusters.mycluster.libraries @@ -14,33 +7,10 @@ delete clusters.mycluster.libraries Plan: 0 to add, 0 to change, 1 to delete, 1 unchanged === Redeploy uninstalls the libraries and restarts the running cluster ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster-libraries-remove-all/default/files... -Restarting cluster [UUID] because its libraries changed -Deleted clusters.mycluster.libraries -Files: 3 uploaded, 0 deleted -Resources: 0 created, 0 changed, 1 deleted, 1 unchanged - ->>> print_requests.py //libraries/uninstall -{ - "method": "POST", - "path": "/api/2.0/libraries/uninstall", - "body": { - "cluster_id": "[UUID]", - "libraries": [ - { - "pypi": { - "package": "requests" - } - } - ] - } -} - >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.clusters.mycluster -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/cluster-libraries-remove-all/default +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] Destroy: 1 deleted diff --git a/acceptance/bundle/resources/clusters/libraries-remove-all/script b/acceptance/bundle/resources/clusters/libraries-remove-all/script index e95a77288e1..8f8e3a6e841 100644 --- a/acceptance/bundle/resources/clusters/libraries-remove-all/script +++ b/acceptance/bundle/resources/clusters/libraries-remove-all/script @@ -1,11 +1,16 @@ +envsubst < databricks.yml.tmpl > databricks.yml + cleanup() { trace $CLI bundle destroy --auto-approve rm -f out.requests.txt } trap cleanup EXIT +# Deploy output is noisy and differs fake vs cloud, so route it to LOG and assert +# only the deterministic signals. title "Deploy a cluster with a pypi library" -trace $CLI bundle deploy +$CLI bundle deploy &> LOG.deploy +cat LOG.deploy | contains.py "Created clusters.mycluster.libraries" > /dev/null title "Removing the whole libraries block plans a delete of the libraries sub-resource" update_file.py databricks.yml " libraries: @@ -15,5 +20,5 @@ update_file.py databricks.yml " libraries: trace $CLI bundle plan title "Redeploy uninstalls the libraries and restarts the running cluster" -trace $CLI bundle deploy -trace print_requests.py //libraries/uninstall +$CLI bundle deploy &> LOG.redeploy +cat LOG.redeploy | contains.py "Restarting cluster" > /dev/null diff --git a/acceptance/bundle/resources/clusters/libraries-remove-all/test.toml b/acceptance/bundle/resources/clusters/libraries-remove-all/test.toml index 93cb0543245..de8ea8588b8 100644 --- a/acceptance/bundle/resources/clusters/libraries-remove-all/test.toml +++ b/acceptance/bundle/resources/clusters/libraries-remove-all/test.toml @@ -1,10 +1,10 @@ -Cloud = false -RecordRequests = true +Cloud = true +RecordRequests = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -Ignore = [".databricks"] +Ignore = [".databricks", "databricks.yml"] [[Repls]] Old = "[0-9]{4}-[0-9]{6}-[0-9a-z]{8}" -New = "[CLUSTER-ID]" +New = "[UUID]" diff --git a/acceptance/bundle/resources/clusters/libraries-repo/test.toml b/acceptance/bundle/resources/clusters/libraries-repo/test.toml index 63a8c8a332c..de8ea8588b8 100644 --- a/acceptance/bundle/resources/clusters/libraries-repo/test.toml +++ b/acceptance/bundle/resources/clusters/libraries-repo/test.toml @@ -4,3 +4,7 @@ RecordRequests = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] Ignore = [".databricks", "databricks.yml"] + +[[Repls]] +Old = "[0-9]{4}-[0-9]{6}-[0-9a-z]{8}" +New = "[UUID]" diff --git a/acceptance/bundle/resources/clusters/libraries-stopped-cluster/databricks.yml b/acceptance/bundle/resources/clusters/libraries-stopped-cluster/databricks.yml deleted file mode 100644 index 2831955da56..00000000000 --- a/acceptance/bundle/resources/clusters/libraries-stopped-cluster/databricks.yml +++ /dev/null @@ -1,15 +0,0 @@ -bundle: - name: cluster-libraries-stopped - -resources: - clusters: - mycluster: - cluster_name: mycluster - spark_version: 15.4.x-scala2.12 - node_type_id: i3.xlarge - num_workers: 1 - lifecycle: - started: false - libraries: - - pypi: - package: requests diff --git a/acceptance/bundle/resources/clusters/libraries-stopped-cluster/databricks.yml.tmpl b/acceptance/bundle/resources/clusters/libraries-stopped-cluster/databricks.yml.tmpl new file mode 100644 index 00000000000..705177abe13 --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-stopped-cluster/databricks.yml.tmpl @@ -0,0 +1,19 @@ +bundle: + name: cluster-libraries-stopped-$UNIQUE_NAME + +workspace: + root_path: ~/.bundle/$UNIQUE_NAME + +resources: + clusters: + mycluster: + cluster_name: mycluster-$UNIQUE_NAME + spark_version: $DEFAULT_SPARK_VERSION + node_type_id: $NODE_TYPE_ID + instance_pool_id: $TEST_INSTANCE_POOL_ID + num_workers: 1 + lifecycle: + started: false + libraries: + - pypi: + package: requests diff --git a/acceptance/bundle/resources/clusters/libraries-stopped-cluster/out.test.toml b/acceptance/bundle/resources/clusters/libraries-stopped-cluster/out.test.toml index 0938e678987..c502b28221b 100644 --- a/acceptance/bundle/resources/clusters/libraries-stopped-cluster/out.test.toml +++ b/acceptance/bundle/resources/clusters/libraries-stopped-cluster/out.test.toml @@ -1,2 +1,2 @@ -Cloud = false +Cloud = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/clusters/libraries-stopped-cluster/output.txt b/acceptance/bundle/resources/clusters/libraries-stopped-cluster/output.txt index 05044c69577..053e34eac55 100644 --- a/acceptance/bundle/resources/clusters/libraries-stopped-cluster/output.txt +++ b/acceptance/bundle/resources/clusters/libraries-stopped-cluster/output.txt @@ -1,58 +1,10 @@ === Deploy libraries on a stopped cluster (started=false): install is queued, no restart ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster-libraries-stopped/default/files... -Created clusters.mycluster -Created clusters.mycluster.libraries -Files: 5 uploaded, 0 deleted -Resources: 2 created, 0 changed, 0 deleted, 0 unchanged - === Change the libraries and redeploy: cluster is stopped, so no restart is issued ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster-libraries-stopped/default/files... -Updated clusters.mycluster.libraries -Files: 3 uploaded, 0 deleted -Resources: 0 created, 1 changed, 0 deleted, 1 unchanged - ->>> print_requests.py //libraries/install //clusters/restart -{ - "method": "POST", - "path": "/api/2.0/libraries/install", - "body": { - "cluster_id": "[UUID]", - "libraries": [ - { - "pypi": { - "package": "requests" - } - } - ] - } -} -{ - "method": "POST", - "path": "/api/2.0/libraries/install", - "body": { - "cluster_id": "[UUID]", - "libraries": [ - { - "pypi": { - "package": "requests" - } - }, - { - "pypi": { - "package": "numpy" - } - } - ] - } -} - >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.clusters.mycluster -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/cluster-libraries-stopped/default +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] Destroy: 1 deleted diff --git a/acceptance/bundle/resources/clusters/libraries-stopped-cluster/script b/acceptance/bundle/resources/clusters/libraries-stopped-cluster/script index ea1d6164d82..df45657a425 100644 --- a/acceptance/bundle/resources/clusters/libraries-stopped-cluster/script +++ b/acceptance/bundle/resources/clusters/libraries-stopped-cluster/script @@ -1,11 +1,17 @@ +envsubst < databricks.yml.tmpl > databricks.yml + cleanup() { trace $CLI bundle destroy --auto-approve rm -f out.requests.txt } trap cleanup EXIT +# Deploy output is noisy and differs fake vs cloud, so route it to LOG and assert +# only the deterministic signals. The cluster is stopped (started=false), so no +# restart is issued: the install is queued and applied on its next start. title "Deploy libraries on a stopped cluster (started=false): install is queued, no restart" -trace $CLI bundle deploy +$CLI bundle deploy &> LOG.deploy +cat LOG.deploy | contains.py "Created clusters.mycluster.libraries" "!Restarting cluster" > /dev/null title "Change the libraries and redeploy: cluster is stopped, so no restart is issued" update_file.py databricks.yml " - pypi: @@ -13,5 +19,5 @@ update_file.py databricks.yml " - pypi: package: requests - pypi: package: numpy" -trace $CLI bundle deploy -trace print_requests.py //libraries/install //clusters/restart +$CLI bundle deploy &> LOG.redeploy +cat LOG.redeploy | contains.py "Updated clusters.mycluster.libraries" "!Restarting cluster" > /dev/null diff --git a/acceptance/bundle/resources/clusters/libraries-stopped-cluster/test.toml b/acceptance/bundle/resources/clusters/libraries-stopped-cluster/test.toml index 93cb0543245..de8ea8588b8 100644 --- a/acceptance/bundle/resources/clusters/libraries-stopped-cluster/test.toml +++ b/acceptance/bundle/resources/clusters/libraries-stopped-cluster/test.toml @@ -1,10 +1,10 @@ -Cloud = false -RecordRequests = true +Cloud = true +RecordRequests = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -Ignore = [".databricks"] +Ignore = [".databricks", "databricks.yml"] [[Repls]] Old = "[0-9]{4}-[0-9]{6}-[0-9a-z]{8}" -New = "[CLUSTER-ID]" +New = "[UUID]" diff --git a/acceptance/bundle/resources/clusters/libraries/databricks.yml b/acceptance/bundle/resources/clusters/libraries/databricks.yml deleted file mode 100644 index 922f719fda4..00000000000 --- a/acceptance/bundle/resources/clusters/libraries/databricks.yml +++ /dev/null @@ -1,14 +0,0 @@ -bundle: - name: cluster-libraries - -resources: - clusters: - mycluster: - cluster_name: mycluster - spark_version: 15.4.x-scala2.12 - node_type_id: i3.xlarge - num_workers: 1 - libraries: - - pypi: - package: requests - - whl: ./dist/*.whl diff --git a/acceptance/bundle/resources/clusters/libraries/databricks.yml.tmpl b/acceptance/bundle/resources/clusters/libraries/databricks.yml.tmpl new file mode 100644 index 00000000000..2364bafe42a --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries/databricks.yml.tmpl @@ -0,0 +1,25 @@ +bundle: + name: cluster-libraries-$UNIQUE_NAME + +workspace: + root_path: ~/.bundle/$UNIQUE_NAME + +artifacts: + my_test_code: + type: whl + path: . + # Use 'python' because 'python3' does not exist in Windows virtualenvs. + build: python setup.py bdist_wheel + +resources: + clusters: + mycluster: + cluster_name: mycluster-$UNIQUE_NAME + spark_version: $DEFAULT_SPARK_VERSION + node_type_id: $NODE_TYPE_ID + num_workers: 1 + data_security_mode: $DATA_SECURITY_MODE + libraries: + - pypi: + package: requests + - whl: ./dist/*.whl diff --git a/acceptance/bundle/resources/clusters/libraries/dist/my_package-0.0.1-py3-none-any.whl b/acceptance/bundle/resources/clusters/libraries/dist/my_package-0.0.1-py3-none-any.whl deleted file mode 100644 index 99c37c880a6..00000000000 --- a/acceptance/bundle/resources/clusters/libraries/dist/my_package-0.0.1-py3-none-any.whl +++ /dev/null @@ -1 +0,0 @@ -dummy wheel contents \ No newline at end of file diff --git a/acceptance/bundle/resources/clusters/libraries/my_test_code/__init__.py b/acceptance/bundle/resources/clusters/libraries/my_test_code/__init__.py new file mode 100644 index 00000000000..909f1f3220d --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries/my_test_code/__init__.py @@ -0,0 +1,2 @@ +__version__ = "0.0.1" +__author__ = "Databricks" diff --git a/acceptance/bundle/resources/clusters/libraries/my_test_code/__main__.py b/acceptance/bundle/resources/clusters/libraries/my_test_code/__main__.py new file mode 100644 index 00000000000..ea918ce2d53 --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries/my_test_code/__main__.py @@ -0,0 +1,16 @@ +""" +The entry point of the Python Wheel +""" + +import sys + + +def main(): + # This method will print the provided arguments + print("Hello from my func") + print("Got arguments:") + print(sys.argv) + + +if __name__ == "__main__": + main() diff --git a/acceptance/bundle/resources/clusters/libraries/out.test.toml b/acceptance/bundle/resources/clusters/libraries/out.test.toml index 0938e678987..33b0e8c26b2 100644 --- a/acceptance/bundle/resources/clusters/libraries/out.test.toml +++ b/acceptance/bundle/resources/clusters/libraries/out.test.toml @@ -1,2 +1,3 @@ -Cloud = false +Cloud = true +CloudSlow = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/clusters/libraries/output.txt b/acceptance/bundle/resources/clusters/libraries/output.txt index 4d825933dfa..9091910386d 100644 --- a/acceptance/bundle/resources/clusters/libraries/output.txt +++ b/acceptance/bundle/resources/clusters/libraries/output.txt @@ -1,13 +1,5 @@ === Deploy a cluster with a pypi and a local wheel library ->>> [CLI] bundle deploy -Uploading dist/my_package-0.0.1-py3-none-any.whl... -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster-libraries/default/files... -Created clusters.mycluster -Created clusters.mycluster.libraries -Files: 6 uploaded, 0 deleted -Resources: 2 created, 0 changed, 0 deleted, 0 unchanged - === Libraries installed via the Libraries API (wheel rewritten to its uploaded path) >>> print_requests.py //libraries/install { @@ -22,20 +14,13 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } }, { - "whl": "/Workspace/Users/[USERNAME]/.bundle/cluster-libraries/default/artifacts/.internal/my_package-0.0.1-py3-none-any.whl" + "whl": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" } ] } } -=== Removing the wheel and redeploying uninstalls it ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster-libraries/default/files... -Restarting cluster [UUID] because its libraries changed -Updated clusters.mycluster.libraries -Files: 3 uploaded, 0 deleted -Resources: 0 created, 1 changed, 0 deleted, 1 unchanged - +=== Removing the wheel and redeploying uninstalls it and restarts the running cluster >>> print_requests.py //libraries/uninstall { "method": "POST", @@ -44,7 +29,7 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged "cluster_id": "[UUID]", "libraries": [ { - "whl": "/Workspace/Users/[USERNAME]/.bundle/cluster-libraries/default/artifacts/.internal/my_package-0.0.1-py3-none-any.whl" + "whl": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" } ] } @@ -54,7 +39,7 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged The following resources will be deleted: delete resources.clusters.mycluster -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/cluster-libraries/default +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] Restarting cluster [UUID] because its libraries changed Destroy: 1 deleted diff --git a/acceptance/bundle/resources/clusters/libraries/script b/acceptance/bundle/resources/clusters/libraries/script index 33af4f7c5d5..8e248ee1931 100644 --- a/acceptance/bundle/resources/clusters/libraries/script +++ b/acceptance/bundle/resources/clusters/libraries/script @@ -1,16 +1,25 @@ +export DATA_SECURITY_MODE=USER_ISOLATION +envsubst < databricks.yml.tmpl > databricks.yml + cleanup() { trace $CLI bundle destroy --auto-approve rm -f out.requests.txt } trap cleanup EXIT +# Deploy builds the wheel from setup.py, uploads it, and rewrites the whl entry to +# its workspace path. Deploy output is noisy and differs fake vs cloud, so route it +# to LOG and assert only the deterministic signals: the created sub-resource and the +# install request (which shows the rewritten wheel path). title "Deploy a cluster with a pypi and a local wheel library" -trace $CLI bundle deploy +$CLI bundle deploy &> LOG.deploy +cat LOG.deploy | contains.py "Created clusters.mycluster.libraries" > /dev/null title "Libraries installed via the Libraries API (wheel rewritten to its uploaded path)" trace print_requests.py //libraries/install -title "Removing the wheel and redeploying uninstalls it" +title "Removing the wheel and redeploying uninstalls it and restarts the running cluster" update_file.py databricks.yml " - whl: ./dist/*.whl" "" -trace $CLI bundle deploy +$CLI bundle deploy &> LOG.redeploy +cat LOG.redeploy | contains.py "Updated clusters.mycluster.libraries" "Restarting cluster" > /dev/null trace print_requests.py //libraries/uninstall diff --git a/acceptance/bundle/resources/clusters/libraries/script.prepare b/acceptance/bundle/resources/clusters/libraries/script.prepare new file mode 100644 index 00000000000..14a888323c4 --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries/script.prepare @@ -0,0 +1,9 @@ +uv venv -q .venv +venv_activate +uv pip install -q --no-index setuptools + +# On Windows, create python3 alias since some build commands use python3 +if [[ "$OSTYPE" == "msys" || "$OSTYPE" == "cygwin" || "$OSTYPE" == "win32" ]]; then + python3() { python "$@"; } + export -f python3 +fi diff --git a/acceptance/bundle/resources/clusters/libraries/setup.py b/acceptance/bundle/resources/clusters/libraries/setup.py new file mode 100644 index 00000000000..8b48a92b4ce --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries/setup.py @@ -0,0 +1,15 @@ +from setuptools import setup, find_packages + +import my_test_code + +setup( + name="my_test_code", + version=my_test_code.__version__, + author=my_test_code.__author__, + url="https://databricks.com", + author_email="john.doe@databricks.com", + description="my example wheel", + packages=find_packages(include=["my_test_code"]), + entry_points={"group1": "run=my_test_code.__main__:main"}, + install_requires=["setuptools"], +) diff --git a/acceptance/bundle/resources/clusters/libraries/test.toml b/acceptance/bundle/resources/clusters/libraries/test.toml index 93cb0543245..3101985b060 100644 --- a/acceptance/bundle/resources/clusters/libraries/test.toml +++ b/acceptance/bundle/resources/clusters/libraries/test.toml @@ -1,10 +1,11 @@ -Cloud = false +Cloud = true +CloudSlow = true RecordRequests = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -Ignore = [".databricks"] +Ignore = [".databricks", ".venv", "build", "dist", "my_test_code.egg-info", "databricks.yml"] [[Repls]] Old = "[0-9]{4}-[0-9]{6}-[0-9a-z]{8}" -New = "[CLUSTER-ID]" +New = "[UUID]" From a9213b68938423f5d4ac0e249f7d75fb65d96eca Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Mon, 31 Aug 2026 22:36:10 +0000 Subject: [PATCH 27/55] Manage cluster libraries as part of the cluster, not a sub-resource The Aug 27 DABs core-eng meeting finalized that cluster libraries are managed as part of the cluster resource (Option 3), not as a `clusters.libraries` sub-resource. This reworks the feature accordingly: - Fold install/uninstall/restart into ResourceCluster: Libraries is now a field of ClusterState and ClusterRemote (DoRead populates it from the Libraries cluster-status API). WaitAfterCreate installs on the freshly-running cluster; DoUpdate reconciles (uninstall removed + install desired) and restarts a running cluster so the change takes effect (a cluster edit already restarts, so we only restart ourselves when no edit fired). KeyedSlices keys libraries by identity so reordering is not drift. - Delete cluster_libraries.go and remove all sub-resource wiring (all.go, resources_types.go, root.go, bundle_plan.go splitResourcePath + makePlan, refschema.go skip). Library changes now surface as `update clusters.`, and destroy no longer wastefully restarts (PermanentDelete removes the cluster). - Rewrite the clusters/libraries* acceptance tests and regenerate refschema for the part-of-cluster shape. Co-authored-by: Isaac --- acceptance/bundle/refschema/out.fields.txt | 34 +- .../clusters/libraries-remove-all/output.txt | 6 +- .../clusters/libraries-remove-all/script | 4 +- .../clusters/libraries-repo/output.txt | 7 +- .../resources/clusters/libraries-repo/script | 2 +- .../libraries-stopped-cluster/output.txt | 2 +- .../clusters/libraries-stopped-cluster/script | 10 +- .../resources/clusters/libraries/output.txt | 1 - .../resources/clusters/libraries/script | 9 +- bundle/config/resources_types.go | 7 - bundle/config/root.go | 6 - bundle/direct/bundle_plan.go | 9 +- bundle/direct/bundle_plan_test.go | 6 +- bundle/direct/dresources/all.go | 3 - bundle/direct/dresources/all_test.go | 20 +- bundle/direct/dresources/cluster.go | 261 +++++++++++++- bundle/direct/dresources/cluster_libraries.go | 333 ------------------ cmd/bundle/debug/refschema.go | 5 - 18 files changed, 299 insertions(+), 426 deletions(-) delete mode 100644 bundle/direct/dresources/cluster_libraries.go diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index ca59ff7374d..4888eb6bef0 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -460,6 +460,23 @@ resources.clusters.*.jdbc_port int REMOTE resources.clusters.*.kind compute.Kind ALL resources.clusters.*.last_restarted_time int64 REMOTE resources.clusters.*.last_state_loss_time int64 REMOTE +resources.clusters.*.libraries []compute.Library ALL +resources.clusters.*.libraries[*] compute.Library ALL +resources.clusters.*.libraries[*].cran *compute.RCranLibrary ALL +resources.clusters.*.libraries[*].cran.package string ALL +resources.clusters.*.libraries[*].cran.repo string ALL +resources.clusters.*.libraries[*].egg string ALL +resources.clusters.*.libraries[*].jar string ALL +resources.clusters.*.libraries[*].maven *compute.MavenLibrary ALL +resources.clusters.*.libraries[*].maven.coordinates string ALL +resources.clusters.*.libraries[*].maven.exclusions []string ALL +resources.clusters.*.libraries[*].maven.exclusions[*] string ALL +resources.clusters.*.libraries[*].maven.repo string ALL +resources.clusters.*.libraries[*].pypi *compute.PythonPyPiLibrary ALL +resources.clusters.*.libraries[*].pypi.package string ALL +resources.clusters.*.libraries[*].pypi.repo string ALL +resources.clusters.*.libraries[*].requirements string ALL +resources.clusters.*.libraries[*].whl string ALL resources.clusters.*.lifecycle *dresources.StateLifecycle REMOTE STATE resources.clusters.*.lifecycle *resources.LifecycleWithStarted INPUT resources.clusters.*.lifecycle resources.Lifecycle INPUT @@ -610,23 +627,6 @@ resources.clusters.*.workload_type *compute.WorkloadType ALL resources.clusters.*.workload_type.clients compute.ClientsTypes ALL resources.clusters.*.workload_type.clients.jobs bool ALL resources.clusters.*.workload_type.clients.notebooks bool ALL -resources.clusters.*.libraries.cluster_id string ALL -resources.clusters.*.libraries[*] compute.Library ALL -resources.clusters.*.libraries[*].cran *compute.RCranLibrary ALL -resources.clusters.*.libraries[*].cran.package string ALL -resources.clusters.*.libraries[*].cran.repo string ALL -resources.clusters.*.libraries[*].egg string ALL -resources.clusters.*.libraries[*].jar string ALL -resources.clusters.*.libraries[*].maven *compute.MavenLibrary ALL -resources.clusters.*.libraries[*].maven.coordinates string ALL -resources.clusters.*.libraries[*].maven.exclusions []string ALL -resources.clusters.*.libraries[*].maven.exclusions[*] string ALL -resources.clusters.*.libraries[*].maven.repo string ALL -resources.clusters.*.libraries[*].pypi *compute.PythonPyPiLibrary ALL -resources.clusters.*.libraries[*].pypi.package string ALL -resources.clusters.*.libraries[*].pypi.repo string ALL -resources.clusters.*.libraries[*].requirements string ALL -resources.clusters.*.libraries[*].whl string ALL resources.clusters.*.permissions.object_id string ALL resources.clusters.*.permissions[*] dresources.StatePermission ALL resources.clusters.*.permissions[*].group_name string ALL diff --git a/acceptance/bundle/resources/clusters/libraries-remove-all/output.txt b/acceptance/bundle/resources/clusters/libraries-remove-all/output.txt index 1d5670b04e1..4881826f6e3 100644 --- a/acceptance/bundle/resources/clusters/libraries-remove-all/output.txt +++ b/acceptance/bundle/resources/clusters/libraries-remove-all/output.txt @@ -1,10 +1,10 @@ === Deploy a cluster with a pypi library -=== Removing the whole libraries block plans a delete of the libraries sub-resource +=== Removing all libraries plans an update of the cluster >>> [CLI] bundle plan -delete clusters.mycluster.libraries +update clusters.mycluster -Plan: 0 to add, 0 to change, 1 to delete, 1 unchanged +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged === Redeploy uninstalls the libraries and restarts the running cluster >>> [CLI] bundle destroy --auto-approve diff --git a/acceptance/bundle/resources/clusters/libraries-remove-all/script b/acceptance/bundle/resources/clusters/libraries-remove-all/script index 8f8e3a6e841..00b927da388 100644 --- a/acceptance/bundle/resources/clusters/libraries-remove-all/script +++ b/acceptance/bundle/resources/clusters/libraries-remove-all/script @@ -10,9 +10,9 @@ trap cleanup EXIT # only the deterministic signals. title "Deploy a cluster with a pypi library" $CLI bundle deploy &> LOG.deploy -cat LOG.deploy | contains.py "Created clusters.mycluster.libraries" > /dev/null +cat LOG.deploy | contains.py "Created clusters.mycluster" > /dev/null -title "Removing the whole libraries block plans a delete of the libraries sub-resource" +title "Removing all libraries plans an update of the cluster" update_file.py databricks.yml " libraries: - pypi: package: requests diff --git a/acceptance/bundle/resources/clusters/libraries-repo/output.txt b/acceptance/bundle/resources/clusters/libraries-repo/output.txt index d15fccc198d..616e9015204 100644 --- a/acceptance/bundle/resources/clusters/libraries-repo/output.txt +++ b/acceptance/bundle/resources/clusters/libraries-repo/output.txt @@ -1,13 +1,13 @@ === Plan is a no-op after deploy: the repo round-trips >>> [CLI] bundle plan -Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged === Changing only the repo is planned as an update >>> [CLI] bundle plan -update clusters.mycluster.libraries +update clusters.mycluster -Plan: 0 to add, 1 to change, 0 to delete, 1 unchanged +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: @@ -15,5 +15,4 @@ The following resources will be deleted: All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] -Restarting cluster [UUID] because its libraries changed Destroy: 1 deleted diff --git a/acceptance/bundle/resources/clusters/libraries-repo/script b/acceptance/bundle/resources/clusters/libraries-repo/script index 11cb4c56207..262a43b049f 100644 --- a/acceptance/bundle/resources/clusters/libraries-repo/script +++ b/acceptance/bundle/resources/clusters/libraries-repo/script @@ -8,7 +8,7 @@ trap cleanup EXIT # Deploy output is noisy and differs fake vs cloud; assert only the plan signals below. $CLI bundle deploy &> LOG.deploy -cat LOG.deploy | contains.py "Created clusters.mycluster.libraries" > /dev/null +cat LOG.deploy | contains.py "Created clusters.mycluster" > /dev/null title "Plan is a no-op after deploy: the repo round-trips" trace $CLI bundle plan diff --git a/acceptance/bundle/resources/clusters/libraries-stopped-cluster/output.txt b/acceptance/bundle/resources/clusters/libraries-stopped-cluster/output.txt index 053e34eac55..3af56fce4ef 100644 --- a/acceptance/bundle/resources/clusters/libraries-stopped-cluster/output.txt +++ b/acceptance/bundle/resources/clusters/libraries-stopped-cluster/output.txt @@ -1,5 +1,5 @@ -=== Deploy libraries on a stopped cluster (started=false): install is queued, no restart +=== Deploy libraries on a started=false cluster: no restart === Change the libraries and redeploy: cluster is stopped, so no restart is issued >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/resources/clusters/libraries-stopped-cluster/script b/acceptance/bundle/resources/clusters/libraries-stopped-cluster/script index df45657a425..bcf7772b63e 100644 --- a/acceptance/bundle/resources/clusters/libraries-stopped-cluster/script +++ b/acceptance/bundle/resources/clusters/libraries-stopped-cluster/script @@ -7,11 +7,11 @@ cleanup() { trap cleanup EXIT # Deploy output is noisy and differs fake vs cloud, so route it to LOG and assert -# only the deterministic signals. The cluster is stopped (started=false), so no -# restart is issued: the install is queued and applied on its next start. -title "Deploy libraries on a stopped cluster (started=false): install is queued, no restart" +# only the deterministic signals. The cluster ends up stopped (started=false), so no +# restart is ever issued. +title "Deploy libraries on a started=false cluster: no restart" $CLI bundle deploy &> LOG.deploy -cat LOG.deploy | contains.py "Created clusters.mycluster.libraries" "!Restarting cluster" > /dev/null +cat LOG.deploy | contains.py "Created clusters.mycluster" "!Restarting cluster" > /dev/null title "Change the libraries and redeploy: cluster is stopped, so no restart is issued" update_file.py databricks.yml " - pypi: @@ -20,4 +20,4 @@ update_file.py databricks.yml " - pypi: - pypi: package: numpy" $CLI bundle deploy &> LOG.redeploy -cat LOG.redeploy | contains.py "Updated clusters.mycluster.libraries" "!Restarting cluster" > /dev/null +cat LOG.redeploy | contains.py "Updated clusters.mycluster" "!Restarting cluster" > /dev/null diff --git a/acceptance/bundle/resources/clusters/libraries/output.txt b/acceptance/bundle/resources/clusters/libraries/output.txt index 9091910386d..4ad3e376e64 100644 --- a/acceptance/bundle/resources/clusters/libraries/output.txt +++ b/acceptance/bundle/resources/clusters/libraries/output.txt @@ -41,5 +41,4 @@ The following resources will be deleted: All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] -Restarting cluster [UUID] because its libraries changed Destroy: 1 deleted diff --git a/acceptance/bundle/resources/clusters/libraries/script b/acceptance/bundle/resources/clusters/libraries/script index 8e248ee1931..a49f2a9d493 100644 --- a/acceptance/bundle/resources/clusters/libraries/script +++ b/acceptance/bundle/resources/clusters/libraries/script @@ -9,11 +9,12 @@ trap cleanup EXIT # Deploy builds the wheel from setup.py, uploads it, and rewrites the whl entry to # its workspace path. Deploy output is noisy and differs fake vs cloud, so route it -# to LOG and assert only the deterministic signals: the created sub-resource and the -# install request (which shows the rewritten wheel path). +# to LOG and assert only the deterministic signals: the created cluster and the +# install request (which shows the rewritten wheel path). Libraries are part of the +# cluster, so they show up under the cluster, not a separate resource. title "Deploy a cluster with a pypi and a local wheel library" $CLI bundle deploy &> LOG.deploy -cat LOG.deploy | contains.py "Created clusters.mycluster.libraries" > /dev/null +cat LOG.deploy | contains.py "Created clusters.mycluster" > /dev/null title "Libraries installed via the Libraries API (wheel rewritten to its uploaded path)" trace print_requests.py //libraries/install @@ -21,5 +22,5 @@ trace print_requests.py //libraries/install title "Removing the wheel and redeploying uninstalls it and restarts the running cluster" update_file.py databricks.yml " - whl: ./dist/*.whl" "" $CLI bundle deploy &> LOG.redeploy -cat LOG.redeploy | contains.py "Updated clusters.mycluster.libraries" "Restarting cluster" > /dev/null +cat LOG.redeploy | contains.py "Updated clusters.mycluster" "Restarting cluster" > /dev/null trace print_requests.py //libraries/uninstall diff --git a/bundle/config/resources_types.go b/bundle/config/resources_types.go index 20dbee0222a..ab7361740a2 100644 --- a/bundle/config/resources_types.go +++ b/bundle/config/resources_types.go @@ -45,13 +45,6 @@ var ResourcesTypes = func() map[string]reflect.Type { res[grantsKey] = resourceField.Type continue } - // libraries is a child resource only for clusters. Pipelines and - // cluster_policies have a native Libraries field that is a plain - // field, not a child resource. - if resourceField.Name == "Libraries" && name == "clusters" { - librariesKey := name + ".libraries" - res[librariesKey] = resourceField.Type - } } } diff --git a/bundle/config/root.go b/bundle/config/root.go index c05851c3d86..9f149506a65 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -624,12 +624,6 @@ func GetNodeAndType(path dyn.Path) (dyn.Path, string) { if sub == "permissions" || sub == "grants" { return path[:4], path[1].Key() + "." + sub } - // libraries is a sub-resource only for clusters. Other resource types - // (e.g. pipelines) have a native libraries field that is a plain field, - // not a child resource. - if sub == "libraries" && path[1].Key() == "clusters" { - return path[:4], path[1].Key() + "." + sub - } } return path[:3], path[1].Key() diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 1618d46a201..fc8e375fa2f 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -717,14 +717,11 @@ func isEmptyStruct(rv reflect.Value) bool { // For regular resources like "resources.jobs.foo.name", returns ("resources.jobs.foo", "name"). // For sub-resources like "resources.jobs.foo.permissions[0].level", returns ("resources.jobs.foo.permissions", "[0].level"). func splitResourcePath(path *structpath.PathNode) (string, *structpath.PathNode) { - // permissions and grants are sub-resources for every group; libraries is a sub-resource - // only for clusters (pipelines have a native top-level libraries field), so scope it to - // clusters, matching the pattern in makePlan. + // permissions and grants are sub-resources for every group. if path.Len() > 4 { first := path.SkipPrefix(3).Prefix(1) if key, ok := first.StringKey(); ok { - group, _ := path.SkipPrefix(1).Prefix(1).StringKey() - if key == "permissions" || key == "grants" || (key == "libraries" && group == "clusters") { + if key == "permissions" || key == "grants" { return path.Prefix(4).String(), path.SkipPrefix(4) } } @@ -942,8 +939,6 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey(), dyn.Key("permissions")), dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey(), dyn.Key("grants")), - // libraries is a sub-resource only for clusters; other resource types have a native libraries field. - dyn.NewPattern(dyn.Key("resources"), dyn.Key("clusters"), dyn.AnyKey(), dyn.Key("libraries")), } // Walk? diff --git a/bundle/direct/bundle_plan_test.go b/bundle/direct/bundle_plan_test.go index d711de39e63..e9186386d3d 100644 --- a/bundle/direct/bundle_plan_test.go +++ b/bundle/direct/bundle_plan_test.go @@ -411,9 +411,9 @@ func TestSplitResourcePath(t *testing.T) { {"resources.jobs.foo.name", "resources.jobs.foo", "name"}, {"resources.jobs.foo.permissions[0].level", "resources.jobs.foo.permissions", "[0].level"}, {"resources.schemas.foo.grants[0].principal", "resources.schemas.foo.grants", "[0].principal"}, - // libraries is a sub-resource only for clusters. - {"resources.clusters.foo.libraries[0].whl", "resources.clusters.foo.libraries", "[0].whl"}, - // pipelines have a native top-level libraries field, so it must not split as a sub-resource. + // libraries is a plain field on the cluster (managed as part of the cluster), not a sub-resource. + {"resources.clusters.foo.libraries[0].whl", "resources.clusters.foo", "libraries[0].whl"}, + // pipelines have a native top-level libraries field, likewise not a sub-resource. {"resources.pipelines.foo.libraries[0].notebook.path", "resources.pipelines.foo", "libraries[0].notebook.path"}, } for _, tt := range tests { diff --git a/bundle/direct/dresources/all.go b/bundle/direct/dresources/all.go index 4ffeed88ab2..391fb0684d2 100644 --- a/bundle/direct/dresources/all.go +++ b/bundle/direct/dresources/all.go @@ -61,9 +61,6 @@ var SupportedResources = map[string]any{ "vector_search_endpoints.permissions": (*ResourcePermissions)(nil), "instance_pools.permissions": (*ResourcePermissions)(nil), - // Libraries - "clusters.libraries": (*ResourceLibraries)(nil), - // Grants "catalogs.grants": (*ResourceGrants)(nil), "schemas.grants": (*ResourceGrants)(nil), diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index 62f26248a60..48f9de47496 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -455,24 +455,6 @@ var testDeps = map[string]prepareWorkspace{ }, nil }, - "clusters.libraries": func(ctx context.Context, client *databricks.WorkspaceClient) (any, error) { - wait, err := client.Clusters.Create(ctx, compute.CreateCluster{ - ClusterName: "libraries-cluster", - SparkVersion: "13.3.x-scala2.12", - NodeTypeId: "m5.large", - NumWorkers: 1, - }) - if err != nil { - return nil, err - } - return &LibrariesState{ - ClusterId: wait.ClusterId, - EmbeddedSlice: []compute.Library{ - {Whl: "/Workspace/Users/test/lib.whl"}, - }, - }, nil - }, - "cluster_policies.permissions": func(ctx context.Context, client *databricks.WorkspaceClient) (any, error) { return &PermissionsState{ ObjectID: "/cluster-policies/cluster-policy-permissions", @@ -1150,7 +1132,7 @@ func testCRUD(t *testing.T, group string, adapter *Adapter, client *databricks.W // this lists all the sub-resources that do not delete the parent resource when they // are deleted. - deleteIsNoop := strings.HasSuffix(group, "permissions") || strings.HasSuffix(group, "grants") || strings.HasSuffix(group, "libraries") + deleteIsNoop := strings.HasSuffix(group, "permissions") || strings.HasSuffix(group, "grants") // Apps DoDelete is fire-and-forget: the API returns success while the app // sits in DELETING state for up to ~20 minutes before the record is removed. // A GET on the DELETING app returns the app, not 404 -- the testserver diff --git a/bundle/direct/dresources/cluster.go b/bundle/direct/dresources/cluster.go index 0605d50b86c..f2c7dbdc096 100644 --- a/bundle/direct/dresources/cluster.go +++ b/bundle/direct/dresources/cluster.go @@ -4,10 +4,12 @@ import ( "context" "errors" "fmt" + "strings" "time" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/utils" @@ -18,6 +20,9 @@ import ( "github.com/databricks/databricks-sdk-go/service/compute" ) +// librariesWaitTimeout bounds how long we poll for libraries to finish installing. +const librariesWaitTimeout = 15 * time.Minute + // clusterWaitTimeout bounds how long we poll for a cluster to reach its target // state (RUNNING/TERMINATED) after create, edit, or start/stop. Provisioning can // legitimately take longer than 15 minutes on capacity-constrained workspaces @@ -33,6 +38,10 @@ type ClusterState struct { compute.ClusterSpec Lifecycle *StateLifecycle `json:"lifecycle,omitempty"` + + // Libraries are installed via the Libraries API, not the cluster spec, and managed as + // part of the cluster (see reconcileLibraries and the install in WaitAfterCreate). + Libraries []compute.Library `json:"libraries,omitempty"` } // Custom marshalers needed because embedded compute.ClusterSpec has its own MarshalJSON @@ -57,6 +66,10 @@ type ClusterRemote struct { compute.ClusterDetails ApplyPolicyDefaultValues bool `json:"apply_policy_default_values,omitempty"` Lifecycle *StateLifecycle `json:"lifecycle,omitempty"` + + // Libraries is populated by DoRead from the Libraries cluster-status API (the cluster + // GET does not return installed libraries), so it participates in drift detection. + Libraries []compute.Library `json:"libraries,omitempty"` } func (r *ClusterRemote) UnmarshalJSON(b []byte) error { @@ -81,6 +94,7 @@ func (r *ResourceCluster) PrepareState(input *resources.Cluster) *ClusterState { s := &ClusterState{ ClusterSpec: input.ClusterSpec, Lifecycle: nil, + Libraries: input.Libraries, } if input.Lifecycle != nil && input.Lifecycle.Started != nil { s.Lifecycle = &StateLifecycle{Started: input.Lifecycle.Started} @@ -132,6 +146,7 @@ func (r *ResourceCluster) RemapState(input *ClusterRemote) *ClusterState { ForceSendFields: utils.FilterFields[compute.ClusterSpec](input.ForceSendFields), }, Lifecycle: &StateLifecycle{Started: &started}, + Libraries: input.Libraries, } return spec } @@ -145,6 +160,7 @@ func (r *ResourceCluster) DoRead(ctx context.Context, id string) (*ClusterRemote ClusterDetails: *details, ApplyPolicyDefaultValues: false, Lifecycle: nil, + Libraries: nil, } // The GET response carries apply_policy_default_values only under .spec (a snapshot of the // create/edit settings), not at the top level. Promote it so RemapState is a dumb copy. @@ -162,9 +178,34 @@ func (r *ResourceCluster) DoRead(ctx context.Context, id string) (*ClusterRemote default: remote.Lifecycle = nil } + + libraries, err := r.readLibraries(ctx, id) + if err != nil { + return nil, err + } + remote.Libraries = libraries return remote, nil } +// readLibraries returns the bundle-managed libraries installed on the cluster. +// https://docs.databricks.com/api/workspace/libraries/clusterstatus +func (r *ResourceCluster) readLibraries(ctx context.Context, id string) ([]compute.Library, error) { + statuses, err := r.client.Libraries.ClusterStatusByClusterId(ctx, id) + if err != nil { + return nil, err + } + var libraries []compute.Library + for _, s := range statuses.LibraryStatuses { + // Libraries set for all clusters via the UI are not managed by the bundle; a library + // pending uninstall on restart is on its way out. Skip both. + if s.Library == nil || s.IsLibraryForAllClusters || s.Status == compute.LibraryInstallStatusUninstallOnRestart { + continue + } + libraries = append(libraries, *s.Library) + } + return libraries, nil +} + func (r *ResourceCluster) DoCreate(ctx context.Context, config *ClusterState) (string, *ClusterRemote, error) { wait, err := r.client.Clusters.Create(ctx, makeCreateCluster(&config.ClusterSpec)) if err != nil { @@ -173,14 +214,28 @@ func (r *ResourceCluster) DoCreate(ctx context.Context, config *ClusterState) (s return wait.ClusterId, nil, nil } -// hasClusterChanges reports whether the plan entry contains any Update changes -// to fields that belong to the Cluster Edit API (i.e., not lifecycle-only fields). -func hasClusterChanges(entry *PlanEntry) bool { - return entry.Changes.HasChangeExcept("lifecycle", "lifecycle.started") +// hasClusterSpecChanges reports whether the plan entry changes a Cluster Edit API field — +// anything other than lifecycle (start/stop) and libraries (handled via the Libraries API). +func hasClusterSpecChanges(entry *PlanEntry) bool { + for field, change := range entry.Changes { + if change.Action == deployplan.Skip { + continue + } + node, err := structpath.ParsePath(field) + if err != nil { + continue + } + top, _ := node.Prefix(1).StringKey() + if top != "lifecycle" && top != "libraries" { + return true + } + } + return false } func (r *ResourceCluster) DoUpdate(ctx context.Context, id string, config *ClusterState, entry *PlanEntry) (*ClusterRemote, error) { - if hasClusterChanges(entry) { + edited := hasClusterSpecChanges(entry) + if edited { // Same retry as in TF provider logic // https://github.com/databricks/terraform-provider-databricks/blob/3eecd0f90cf99d7777e79a3d03c41f9b2aafb004/clusters/resource_cluster.go#L624 _, err := retries.Poll(ctx, clusterWaitTimeout, func() (*compute.WaitGetClusterRunning[struct{}], *retries.Err) { @@ -201,6 +256,22 @@ func (r *ResourceCluster) DoUpdate(ctx context.Context, id string, config *Clust } } + if entry.Changes.HasChange(librariesPath) { + if err := r.reconcileLibraries(ctx, id, config.Libraries, entry); err != nil { + return nil, err + } + // A cluster edit restarts the cluster on its own, which applies the library change. + // Without an edit we restart so the change takes effect on a running cluster. + if !edited { + if err := r.restartIfRunning(ctx, id); err != nil { + return nil, err + } + if err := r.waitForInstall(ctx, id, config.Libraries); err != nil { + return nil, err + } + } + } + if config.Lifecycle == nil || config.Lifecycle.Started == nil { return nil, nil } @@ -245,6 +316,18 @@ func (r *ResourceCluster) WaitAfterCreate(ctx context.Context, id string, config return nil, err } + // Install libraries once the cluster is running. A freshly-created cluster has no + // attached sessions, so the install applies live without a restart. + if len(config.Libraries) > 0 { + err = r.client.Libraries.Install(ctx, compute.InstallLibraries{ClusterId: id, Libraries: config.Libraries}) + if err != nil { + return nil, err + } + if err := r.waitForInstall(ctx, id, config.Libraries); err != nil { + return nil, err + } + } + if config.Lifecycle != nil && config.Lifecycle.Started != nil && !*config.Lifecycle.Started { // started=false: terminate the cluster after it reaches RUNNING. // Note: Delete terminates the cluster; permanent removal is a separate API (permanent-delete). @@ -427,3 +510,171 @@ func makeEditCluster(id string, config *compute.ClusterSpec) compute.EditCluster return edit } + +// librariesPath is the ClusterState path of the libraries slice, used to detect library changes. +var librariesPath = structpath.MustParsePath("libraries") + +// KeyedSlices compares libraries by identity rather than index so reordering (or the +// read-order the status API returns) does not produce phantom diffs. +func (*ResourceCluster) KeyedSlices() map[string]any { + return map[string]any{"libraries": libraryKey} +} + +// reconcileLibraries uninstalls libraries dropped from config and installs the desired set. +// The Libraries API exposes install and uninstall as separate endpoints, so this is two calls. +func (r *ResourceCluster) reconcileLibraries(ctx context.Context, id string, desired []compute.Library, entry *PlanEntry) error { + removed := removedLibraries(desired, entry) + if len(removed) > 0 { + err := r.client.Libraries.Uninstall(ctx, compute.UninstallLibraries{ClusterId: id, Libraries: removed}) + if err != nil { + return err + } + } + if len(desired) > 0 { + return r.client.Libraries.Install(ctx, compute.InstallLibraries{ClusterId: id, Libraries: desired}) + } + return nil +} + +// removedLibraries returns libraries present in the remote state but absent from the desired set. +func removedLibraries(desired []compute.Library, entry *PlanEntry) []compute.Library { + remote, ok := entry.RemoteState.(*ClusterRemote) + if !ok || remote == nil { + return nil + } + desiredKeys := make(map[string]struct{}, len(desired)) + for _, l := range desired { + desiredKeys[libraryMapKey(l)] = struct{}{} + } + var result []compute.Library + for _, l := range remote.Libraries { + if _, ok := desiredKeys[libraryMapKey(l)]; !ok { + result = append(result, l) + } + } + return result +} + +// restartIfRunning restarts the cluster so a library change takes effect, but only when it is +// running: a stopped cluster applies pending install/uninstall on its next start. It waits for +// the cluster to return to RUNNING before returning. +func (r *ResourceCluster) restartIfRunning(ctx context.Context, id string) error { + details, err := r.client.Clusters.GetByClusterId(ctx, id) + if err != nil { + return err + } + if details.State != compute.StateRunning { + log.Debugf(ctx, "cluster %s is not running (%s); skipping restart for library change", id, details.State) + return nil + } + cmdio.LogString(ctx, fmt.Sprintf("Restarting cluster %s because its libraries changed", id)) + wait, err := r.client.Clusters.Restart(ctx, compute.RestartCluster{ClusterId: id, RestartUser: "", ForceSendFields: nil}) + if err != nil { + return err + } + _, err = wait.GetWithTimeout(clusterWaitTimeout) + return err +} + +// waitForInstall polls until every desired library reaches a terminal installed state. It returns +// early without waiting when the cluster is not running: installs only progress on a running +// cluster and are queued until it next starts. +func (r *ResourceCluster) waitForInstall(ctx context.Context, id string, desired []compute.Library) error { + if len(desired) == 0 { + return nil + } + details, err := r.client.Clusters.GetByClusterId(ctx, id) + if err != nil { + return err + } + if details.State != compute.StateRunning { + log.Debugf(ctx, "cluster %s is not running (%s); skipping wait for library installation", id, details.State) + return nil + } + + desiredKeys := make(map[string]struct{}, len(desired)) + for _, l := range desired { + typ, val := libraryWaitKey(l) + if typ == "" { + // Unknown library type: it can't be matched in cluster-status, so don't wait for it. + continue + } + desiredKeys[typ+"="+val] = struct{}{} + } + + _, err = retries.Poll(ctx, librariesWaitTimeout, func() (*struct{}, *retries.Err) { + statuses, err := r.client.Libraries.ClusterStatusByClusterId(ctx, id) + if err != nil { + return nil, retries.Halt(err) + } + pending := len(desiredKeys) + for _, s := range statuses.LibraryStatuses { + if s.Library == nil { + continue + } + if _, ok := desiredKeys[libraryWaitMapKey(*s.Library)]; !ok { + continue + } + switch s.Status { + case compute.LibraryInstallStatusFailed: + return nil, retries.Halt(fmt.Errorf("library %s failed to install: %s", libraryWaitMapKey(*s.Library), strings.Join(s.Messages, "; "))) + case compute.LibraryInstallStatusInstalled, compute.LibraryInstallStatusSkipped, compute.LibraryInstallStatusRestored: + pending-- + case compute.LibraryInstallStatusPending, compute.LibraryInstallStatusResolving, compute.LibraryInstallStatusInstalling, compute.LibraryInstallStatusUninstallOnRestart: + // Still in progress (or being removed); keep polling. + } + } + if pending > 0 { + return nil, retries.Continues(fmt.Sprintf("waiting for %d librar(ies) to install on cluster %s", pending, id)) + } + return &struct{}{}, nil + }) + return err +} + +// libraryWaitKey identifies a library by its primary field only, used to match install-status reports. +func libraryWaitKey(l compute.Library) (string, string) { + switch { + case l.Whl != "": + return "whl", l.Whl + case l.Jar != "": + return "jar", l.Jar + case l.Egg != "": + return "egg", l.Egg + case l.Requirements != "": + return "requirements", l.Requirements + case l.Pypi != nil: + return "pypi", l.Pypi.Package + case l.Maven != nil: + return "maven", l.Maven.Coordinates + case l.Cran != nil: + return "cran", l.Cran.Package + } + return "", "" +} + +// libraryKey extends libraryWaitKey with repo/exclusions so a repo-only change is detected. +func libraryKey(l compute.Library) (string, string) { + typ, id := libraryWaitKey(l) + switch { + case l.Pypi != nil: + return typ, id + ";" + l.Pypi.Repo + case l.Maven != nil: + return typ, id + ";" + l.Maven.Repo + ";" + strings.Join(l.Maven.Exclusions, ",") + case l.Cran != nil: + return typ, id + ";" + l.Cran.Repo + } + return typ, id +} + +// libraryMapKey flattens libraryKey into a single string for map lookups. +func libraryMapKey(l compute.Library) string { + f, v := libraryKey(l) + return f + "=" + v +} + +// libraryWaitMapKey flattens libraryWaitKey into a single string for map lookups. +func libraryWaitMapKey(l compute.Library) string { + f, v := libraryWaitKey(l) + return f + "=" + v +} diff --git a/bundle/direct/dresources/cluster_libraries.go b/bundle/direct/dresources/cluster_libraries.go deleted file mode 100644 index 7510d4818ff..00000000000 --- a/bundle/direct/dresources/cluster_libraries.go +++ /dev/null @@ -1,333 +0,0 @@ -package dresources - -import ( - "context" - "fmt" - "strings" - "time" - - "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/cli/libs/log" - "github.com/databricks/cli/libs/structs/structvar" - "github.com/databricks/databricks-sdk-go" - "github.com/databricks/databricks-sdk-go/apierr" - "github.com/databricks/databricks-sdk-go/retries" - "github.com/databricks/databricks-sdk-go/service/compute" -) - -// Corresponds to the databricks_library terraform resource: -// https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/library - -// librariesWaitTimeout bounds how long we poll for libraries to finish installing. -const librariesWaitTimeout = 15 * time.Minute - -// LibrariesState is the state for a cluster's libraries sub-resource. Libraries are installed -// via the Libraries API against the parent cluster identified by ClusterId. -type LibrariesState struct { - ClusterId string `json:"cluster_id"` - // By convention EmbeddedSlice fields have the __embed__ json tag, see permissions.go. - EmbeddedSlice []compute.Library `json:"__embed__,omitempty"` -} - -type ResourceLibraries struct { - client *databricks.WorkspaceClient -} - -func (*ResourceLibraries) New(client *databricks.WorkspaceClient) *ResourceLibraries { - return &ResourceLibraries{client: client} -} - -func (r *ResourceLibraries) PrepareInputConfig(inputConfig *[]compute.Library, resourceKey string) (*structvar.StructVar, error) { - baseNode, ok := strings.CutSuffix(resourceKey, ".libraries") - if !ok { - return nil, fmt.Errorf("internal error: node %q does not end with .libraries", resourceKey) - } - - return &structvar.StructVar{ - Value: &LibrariesState{ - ClusterId: "", // Always a reference, defined in Refs below. - EmbeddedSlice: *inputConfig, - }, - Refs: map[string]string{ - "cluster_id": "${" + baseNode + ".id}", - }, - }, nil -} - -func (*ResourceLibraries) PrepareState(state *LibrariesState) *LibrariesState { - return state -} - -// IsEmptyState reports an empty libraries list as no resource at all: nothing to install, and no -// state entry is persisted for it. -func (*ResourceLibraries) IsEmptyState(state *LibrariesState) bool { - return len(state.EmbeddedSlice) == 0 -} - -// libraryWaitKey identifies a library by its primary field only, used to match install-status reports. -func libraryWaitKey(l compute.Library) (string, string) { - switch { - case l.Whl != "": - return "whl", l.Whl - case l.Jar != "": - return "jar", l.Jar - case l.Egg != "": - return "egg", l.Egg - case l.Requirements != "": - return "requirements", l.Requirements - case l.Pypi != nil: - return "pypi", l.Pypi.Package - case l.Maven != nil: - return "maven", l.Maven.Coordinates - case l.Cran != nil: - return "cran", l.Cran.Package - } - return "", "" -} - -// libraryKey extends libraryWaitKey with repo/exclusions so a repo-only change is detected (see KeyedSlices). -func libraryKey(l compute.Library) (string, string) { - typ, id := libraryWaitKey(l) - switch { - case l.Pypi != nil: - return typ, id + ";" + l.Pypi.Repo - case l.Maven != nil: - return typ, id + ";" + l.Maven.Repo + ";" + strings.Join(l.Maven.Exclusions, ",") - case l.Cran != nil: - return typ, id + ";" + l.Cran.Repo - } - return typ, id -} - -func (*ResourceLibraries) KeyedSlices() map[string]any { - // Empty key because EmbeddedSlice appears at the root path of LibrariesState. - return map[string]any{ - "": libraryKey, - } -} - -func (r *ResourceLibraries) DoRead(ctx context.Context, id string) (*LibrariesState, error) { - statuses, err := r.client.Libraries.ClusterStatusByClusterId(ctx, id) - if err != nil { - return nil, err - } - - state := &LibrariesState{ClusterId: id, EmbeddedSlice: nil} - for _, s := range statuses.LibraryStatuses { - // Libraries set for all clusters via the UI are not managed by the bundle - // (following the permissions convention of ignoring inherited entries). - if s.Library == nil || s.IsLibraryForAllClusters { - continue - } - // A library pending uninstall on restart is on its way out; don't report it as present. - if s.Status == compute.LibraryInstallStatusUninstallOnRestart { - continue - } - state.EmbeddedSlice = append(state.EmbeddedSlice, *s.Library) - } - return state, nil -} - -// DoCreate installs the libraries on the cluster. -// https://docs.databricks.com/api/workspace/libraries/install -func (r *ResourceLibraries) DoCreate(ctx context.Context, state *LibrariesState) (string, *LibrariesState, error) { - err := r.client.Libraries.Install(ctx, compute.InstallLibraries{ - ClusterId: state.ClusterId, - Libraries: state.EmbeddedSlice, - }) - if err != nil { - // Install is idempotent (installing an already-installed library is a no-op), - // so retrying on transient errors is safe. - return "", nil, retrySafe(err) - } - return state.ClusterId, nil, nil -} - -// DoUpdate uninstalls libraries removed from config and installs the desired set. This is two API -// calls because the Libraries API exposes install and uninstall as separate endpoints -func (r *ResourceLibraries) DoUpdate(ctx context.Context, id string, state *LibrariesState, entry *PlanEntry) (*LibrariesState, error) { - removed := removedLibraries(state.EmbeddedSlice, entry) - if len(removed) > 0 { - err := r.client.Libraries.Uninstall(ctx, compute.UninstallLibraries{ - ClusterId: id, - Libraries: removed, - }) - if err != nil { - return nil, err - } - } - - if len(state.EmbeddedSlice) > 0 { - err := r.client.Libraries.Install(ctx, compute.InstallLibraries{ - ClusterId: id, - Libraries: state.EmbeddedSlice, - }) - if err != nil { - return nil, err - } - } - return nil, nil -} - -// DoDelete uninstalls the libraries recorded in state. It fires both when the whole libraries -// block is removed from config (the cluster is kept) and during bundle destroy (the cluster is -// deleted too). WaitAfterDelete then restarts the cluster so the uninstall takes effect, since the -// Libraries API defers uninstalls until the next restart. -// -// During `bundle destroy` this restart is wasteful, but it is not possible to differentitate -// between the two cases, without adding additional logic to the framework. -func (r *ResourceLibraries) DoDelete(ctx context.Context, id string, state *LibrariesState) error { - if len(state.EmbeddedSlice) == 0 { - return nil - } - // A 404 here means the parent cluster is already gone; the framework treats that as success. - return r.client.Libraries.Uninstall(ctx, compute.UninstallLibraries{ - ClusterId: id, - Libraries: state.EmbeddedSlice, - }) -} - -// removedLibraries returns libraries present in the remote state but absent from the desired set. -func removedLibraries(desired []compute.Library, entry *PlanEntry) []compute.Library { - if entry == nil { - return nil - } - remote, ok := entry.RemoteState.(*LibrariesState) - if !ok || remote == nil { - return nil - } - - desiredKeys := make(map[string]struct{}, len(desired)) - for _, l := range desired { - desiredKeys[libraryMapKey(l)] = struct{}{} - } - - var result []compute.Library - for _, l := range remote.EmbeddedSlice { - if _, ok := desiredKeys[libraryMapKey(l)]; !ok { - result = append(result, l) - } - } - return result -} - -// libraryMapKey flattens libraryKey into a single string for map lookups. -func libraryMapKey(l compute.Library) string { - f, v := libraryKey(l) - return f + "=" + v -} - -// libraryWaitMapKey flattens libraryWaitKey into a single string for map lookups. -func libraryWaitMapKey(l compute.Library) string { - f, v := libraryWaitKey(l) - return f + "=" + v -} - -func (r *ResourceLibraries) WaitAfterCreate(ctx context.Context, id string, state *LibrariesState) (*LibrariesState, error) { - return nil, r.waitForInstall(ctx, id, state.EmbeddedSlice) -} - -func (r *ResourceLibraries) WaitAfterUpdate(ctx context.Context, id string, state *LibrariesState) (*LibrariesState, error) { - // Restart so the change is live: newly installed libraries are invisible to attached notebooks - // until restart, and the Libraries API defers uninstalls to the next restart. - if err := r.restartIfRunning(ctx, id); err != nil { - return nil, err - } - return nil, r.waitForInstall(ctx, id, state.EmbeddedSlice) -} - -// WaitAfterDelete restarts the cluster so libraries uninstalled by DoDelete are actually evicted -// (the Libraries API defers uninstalls until restart). -func (r *ResourceLibraries) WaitAfterDelete(ctx context.Context, id string) error { - err := r.restartIfRunning(ctx, id) - if apierr.IsMissing(err) { - // Parent cluster already deleted; nothing to restart. - return nil - } - return err -} - -// restartIfRunning restarts the cluster so library changes take effect, but only when it is -// running: a stopped cluster applies pending install/uninstall on its next start. It waits for the -// cluster to return to RUNNING before returning. -func (r *ResourceLibraries) restartIfRunning(ctx context.Context, id string) error { - details, err := r.client.Clusters.GetByClusterId(ctx, id) - if err != nil { - return err - } - if details.State != compute.StateRunning { - log.Debugf(ctx, "cluster %s is not running (%s); skipping restart for library change", id, details.State) - return nil - } - - cmdio.LogString(ctx, fmt.Sprintf("Restarting cluster %s because its libraries changed", id)) - wait, err := r.client.Clusters.Restart(ctx, compute.RestartCluster{ - ClusterId: id, - RestartUser: "", - ForceSendFields: nil, - }) - if err != nil { - return err - } - _, err = wait.GetWithTimeout(clusterWaitTimeout) - return err -} - -// waitForInstall polls until every desired library reaches a terminal installed state. It returns -// early without waiting when the cluster is not running: installs only progress on a running -// cluster and are queued until it next starts. -func (r *ResourceLibraries) waitForInstall(ctx context.Context, id string, desired []compute.Library) error { - if len(desired) == 0 { - return nil - } - - details, err := r.client.Clusters.GetByClusterId(ctx, id) - if err != nil { - return err - } - if details.State != compute.StateRunning { - log.Debugf(ctx, "cluster %s is not running (%s); skipping wait for library installation", id, details.State) - return nil - } - - desiredKeys := make(map[string]struct{}, len(desired)) - for _, l := range desired { - typ, id := libraryWaitKey(l) - if typ == "" { - // Unknown library type: it can't be matched in cluster-status, so don't wait for it. - continue - } - desiredKeys[typ+"="+id] = struct{}{} - } - - _, err = retries.Poll(ctx, librariesWaitTimeout, func() (*struct{}, *retries.Err) { - statuses, err := r.client.Libraries.ClusterStatusByClusterId(ctx, id) - if err != nil { - return nil, retries.Halt(err) - } - - pending := len(desiredKeys) - for _, s := range statuses.LibraryStatuses { - if s.Library == nil { - continue - } - if _, ok := desiredKeys[libraryWaitMapKey(*s.Library)]; !ok { - continue - } - switch s.Status { - case compute.LibraryInstallStatusFailed: - return nil, retries.Halt(fmt.Errorf("library %s failed to install: %s", libraryWaitMapKey(*s.Library), strings.Join(s.Messages, "; "))) - case compute.LibraryInstallStatusInstalled, compute.LibraryInstallStatusSkipped, compute.LibraryInstallStatusRestored: - pending-- - case compute.LibraryInstallStatusPending, compute.LibraryInstallStatusResolving, compute.LibraryInstallStatusInstalling, compute.LibraryInstallStatusUninstallOnRestart: - // Still in progress (or being removed); keep polling. - } - } - - if pending > 0 { - return nil, retries.Continues(fmt.Sprintf("waiting for %d librar(ies) to install on cluster %s", pending, id)) - } - return &struct{}{}, nil - }) - return err -} diff --git a/cmd/bundle/debug/refschema.go b/cmd/bundle/debug/refschema.go index f848c56c04c..4ba1ae999fb 100644 --- a/cmd/bundle/debug/refschema.go +++ b/cmd/bundle/debug/refschema.go @@ -73,11 +73,6 @@ func dumpRemoteSchemas(out io.Writer) error { p == "grants" || strings.HasPrefix(p, "grants.") || strings.HasPrefix(p, "grants[") { return false } - // libraries is a sub-resource adapter for clusters only (pipelines have a native - // top-level libraries field), so skip it here just for clusters. - if resourceName == "clusters" && (p == "libraries" || strings.HasPrefix(p, "libraries.") || strings.HasPrefix(p, "libraries[")) { - return false - } t := strings.ReplaceAll(fmt.Sprint(typ), "interface {}", "any") byType, ok := pathTypes[p] if !ok { From 2219d019d8e4ad404326bba810c38892b26d5345 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 1 Sep 2026 08:01:46 +0000 Subject: [PATCH 28/55] Document unhandled same-path wheel-content change for cluster libraries A local whl/jar whose workspace path is unchanged but whose contents changed (same name+version, non-dev mode) does not trigger a restart. Note the gap and why hashing the built wheel is unsafe (non-reproducible zip mtimes); a general fix needs a source hash in state. Co-authored-by: Isaac --- bundle/direct/dresources/cluster.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bundle/direct/dresources/cluster.go b/bundle/direct/dresources/cluster.go index f2c7dbdc096..a9553bfc8f4 100644 --- a/bundle/direct/dresources/cluster.go +++ b/bundle/direct/dresources/cluster.go @@ -256,6 +256,11 @@ func (r *ResourceCluster) DoUpdate(ctx context.Context, id string, config *Clust } } + // TODO(#1860): a local whl/jar whose workspace path is unchanged but whose contents + // changed (same name+version, non-dev mode) is not detected here, so no restart fires. + // Dev mode handles this via patchwheel (a source-derived version bump); a general fix + // needs a source hash tracked in state. Hashing the built wheel is unsafe — the zip + // embeds mtimes, so a rebuild would churn a restart every deploy. if entry.Changes.HasChange(librariesPath) { if err := r.reconcileLibraries(ctx, id, config.Libraries, entry); err != nil { return nil, err From 6a2fba234e2587aeea2d2f352069a32495fce3d6 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 1 Sep 2026 08:22:06 +0000 Subject: [PATCH 29/55] Add cluster-libraries restart-matrix acceptance test Add clusters/libraries-restart (Cloud=true, pypi-only) pinning restart behaviour across the library lifecycle on a single cluster: - install (create): libraries install on the fresh cluster with NO restart - update (add a library): the running cluster is restarted - delete (remove all libraries): the running cluster is restarted Fold in clusters/libraries-remove-all, whose remove-all -> restart case is subsumed by the delete step here, keeping the Cloud=true cluster count flat. Co-authored-by: Isaac --- .../clusters/libraries-remove-all/output.txt | 16 --------- .../clusters/libraries-remove-all/script | 24 -------------- .../databricks.yml.tmpl | 2 +- .../out.test.toml | 0 .../clusters/libraries-restart/output.txt | 11 +++++++ .../clusters/libraries-restart/script | 33 +++++++++++++++++++ .../test.toml | 0 7 files changed, 45 insertions(+), 41 deletions(-) delete mode 100644 acceptance/bundle/resources/clusters/libraries-remove-all/output.txt delete mode 100644 acceptance/bundle/resources/clusters/libraries-remove-all/script rename acceptance/bundle/resources/clusters/{libraries-remove-all => libraries-restart}/databricks.yml.tmpl (87%) rename acceptance/bundle/resources/clusters/{libraries-remove-all => libraries-restart}/out.test.toml (100%) create mode 100644 acceptance/bundle/resources/clusters/libraries-restart/output.txt create mode 100644 acceptance/bundle/resources/clusters/libraries-restart/script rename acceptance/bundle/resources/clusters/{libraries-remove-all => libraries-restart}/test.toml (100%) diff --git a/acceptance/bundle/resources/clusters/libraries-remove-all/output.txt b/acceptance/bundle/resources/clusters/libraries-remove-all/output.txt deleted file mode 100644 index 4881826f6e3..00000000000 --- a/acceptance/bundle/resources/clusters/libraries-remove-all/output.txt +++ /dev/null @@ -1,16 +0,0 @@ - -=== Deploy a cluster with a pypi library -=== Removing all libraries plans an update of the cluster ->>> [CLI] bundle plan -update clusters.mycluster - -Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged - -=== Redeploy uninstalls the libraries and restarts the running cluster ->>> [CLI] bundle destroy --auto-approve -The following resources will be deleted: - delete resources.clusters.mycluster - -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] - -Destroy: 1 deleted diff --git a/acceptance/bundle/resources/clusters/libraries-remove-all/script b/acceptance/bundle/resources/clusters/libraries-remove-all/script deleted file mode 100644 index 00b927da388..00000000000 --- a/acceptance/bundle/resources/clusters/libraries-remove-all/script +++ /dev/null @@ -1,24 +0,0 @@ -envsubst < databricks.yml.tmpl > databricks.yml - -cleanup() { - trace $CLI bundle destroy --auto-approve - rm -f out.requests.txt -} -trap cleanup EXIT - -# Deploy output is noisy and differs fake vs cloud, so route it to LOG and assert -# only the deterministic signals. -title "Deploy a cluster with a pypi library" -$CLI bundle deploy &> LOG.deploy -cat LOG.deploy | contains.py "Created clusters.mycluster" > /dev/null - -title "Removing all libraries plans an update of the cluster" -update_file.py databricks.yml " libraries: - - pypi: - package: requests -" "" -trace $CLI bundle plan - -title "Redeploy uninstalls the libraries and restarts the running cluster" -$CLI bundle deploy &> LOG.redeploy -cat LOG.redeploy | contains.py "Restarting cluster" > /dev/null diff --git a/acceptance/bundle/resources/clusters/libraries-remove-all/databricks.yml.tmpl b/acceptance/bundle/resources/clusters/libraries-restart/databricks.yml.tmpl similarity index 87% rename from acceptance/bundle/resources/clusters/libraries-remove-all/databricks.yml.tmpl rename to acceptance/bundle/resources/clusters/libraries-restart/databricks.yml.tmpl index 9aa23bd6c8d..3c9a4946638 100644 --- a/acceptance/bundle/resources/clusters/libraries-remove-all/databricks.yml.tmpl +++ b/acceptance/bundle/resources/clusters/libraries-restart/databricks.yml.tmpl @@ -1,5 +1,5 @@ bundle: - name: cluster-libraries-remove-all-$UNIQUE_NAME + name: cluster-libraries-restart-$UNIQUE_NAME workspace: root_path: ~/.bundle/$UNIQUE_NAME diff --git a/acceptance/bundle/resources/clusters/libraries-remove-all/out.test.toml b/acceptance/bundle/resources/clusters/libraries-restart/out.test.toml similarity index 100% rename from acceptance/bundle/resources/clusters/libraries-remove-all/out.test.toml rename to acceptance/bundle/resources/clusters/libraries-restart/out.test.toml diff --git a/acceptance/bundle/resources/clusters/libraries-restart/output.txt b/acceptance/bundle/resources/clusters/libraries-restart/output.txt new file mode 100644 index 00000000000..d1737e6d579 --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-restart/output.txt @@ -0,0 +1,11 @@ + +=== Install (create): libraries install on the fresh cluster with no restart +=== Update: adding a library restarts the running cluster +=== Delete: removing all libraries restarts the running cluster +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.clusters.mycluster + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/clusters/libraries-restart/script b/acceptance/bundle/resources/clusters/libraries-restart/script new file mode 100644 index 00000000000..80b3a94836f --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-restart/script @@ -0,0 +1,33 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +# Deploy output is noisy and differs fake vs cloud, so route it to LOG and assert only +# the restart signal. This test pins the restart behaviour across the library lifecycle. + +title "Install (create): libraries install on the fresh cluster with no restart" +$CLI bundle deploy &> LOG.create +cat LOG.create | contains.py "Created clusters.mycluster" "!Restarting cluster" > /dev/null + +title "Update: adding a library restarts the running cluster" +update_file.py databricks.yml " - pypi: + package: requests" " - pypi: + package: requests + - pypi: + package: six" +$CLI bundle deploy &> LOG.update +cat LOG.update | contains.py "Updated clusters.mycluster" "Restarting cluster" > /dev/null + +title "Delete: removing all libraries restarts the running cluster" +update_file.py databricks.yml " libraries: + - pypi: + package: requests + - pypi: + package: six +" "" +$CLI bundle deploy &> LOG.remove +cat LOG.remove | contains.py "Updated clusters.mycluster" "Restarting cluster" > /dev/null diff --git a/acceptance/bundle/resources/clusters/libraries-remove-all/test.toml b/acceptance/bundle/resources/clusters/libraries-restart/test.toml similarity index 100% rename from acceptance/bundle/resources/clusters/libraries-remove-all/test.toml rename to acceptance/bundle/resources/clusters/libraries-restart/test.toml From aa6aa8369de7d57b63220d251c33c4d059bf36f1 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 1 Sep 2026 08:31:43 +0000 Subject: [PATCH 30/55] Add DMS env-matrix to cluster-libraries out.test.toml; drop dead continue Regenerate the cluster-libraries acceptance snapshots to carry EnvMatrix.DMS = ["", "true"] after merging main's DMS test matrix, fixing the post-test git-diff drift check. Also remove the now-dead continue at the end of the Grants loop body in resources_types.go, orphaned when the Libraries block was removed in the part-of-cluster rewrite. Co-authored-by: Isaac --- .../bundle/resources/clusters/libraries-repo/out.test.toml | 1 + .../bundle/resources/clusters/libraries-restart/out.test.toml | 1 + .../resources/clusters/libraries-stopped-cluster/out.test.toml | 1 + .../resources/clusters/libraries-terraform-error/out.test.toml | 1 + acceptance/bundle/resources/clusters/libraries/out.test.toml | 1 + bundle/config/resources_types.go | 1 - 6 files changed, 5 insertions(+), 1 deletion(-) diff --git a/acceptance/bundle/resources/clusters/libraries-repo/out.test.toml b/acceptance/bundle/resources/clusters/libraries-repo/out.test.toml index c502b28221b..ae5c7bd798f 100644 --- a/acceptance/bundle/resources/clusters/libraries-repo/out.test.toml +++ b/acceptance/bundle/resources/clusters/libraries-repo/out.test.toml @@ -1,2 +1,3 @@ Cloud = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/clusters/libraries-restart/out.test.toml b/acceptance/bundle/resources/clusters/libraries-restart/out.test.toml index c502b28221b..ae5c7bd798f 100644 --- a/acceptance/bundle/resources/clusters/libraries-restart/out.test.toml +++ b/acceptance/bundle/resources/clusters/libraries-restart/out.test.toml @@ -1,2 +1,3 @@ Cloud = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/clusters/libraries-stopped-cluster/out.test.toml b/acceptance/bundle/resources/clusters/libraries-stopped-cluster/out.test.toml index c502b28221b..ae5c7bd798f 100644 --- a/acceptance/bundle/resources/clusters/libraries-stopped-cluster/out.test.toml +++ b/acceptance/bundle/resources/clusters/libraries-stopped-cluster/out.test.toml @@ -1,2 +1,3 @@ Cloud = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/clusters/libraries-terraform-error/out.test.toml b/acceptance/bundle/resources/clusters/libraries-terraform-error/out.test.toml index d2059b4b5d7..5d0f3e4aac2 100644 --- a/acceptance/bundle/resources/clusters/libraries-terraform-error/out.test.toml +++ b/acceptance/bundle/resources/clusters/libraries-terraform-error/out.test.toml @@ -1,2 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/clusters/libraries/out.test.toml b/acceptance/bundle/resources/clusters/libraries/out.test.toml index 33b0e8c26b2..f9f4880725d 100644 --- a/acceptance/bundle/resources/clusters/libraries/out.test.toml +++ b/acceptance/bundle/resources/clusters/libraries/out.test.toml @@ -1,3 +1,4 @@ Cloud = true CloudSlow = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["", "true"] diff --git a/bundle/config/resources_types.go b/bundle/config/resources_types.go index ab7361740a2..dcc91545f19 100644 --- a/bundle/config/resources_types.go +++ b/bundle/config/resources_types.go @@ -43,7 +43,6 @@ var ResourcesTypes = func() map[string]reflect.Type { if resourceField.Name == "Grants" { grantsKey := name + ".grants" res[grantsKey] = resourceField.Type - continue } } } From cc240408c834b4636a917445cd82bbd40a34871e Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 1 Sep 2026 08:33:31 +0000 Subject: [PATCH 31/55] undo unecessary change --- bundle/config/root.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/bundle/config/root.go b/bundle/config/root.go index 9f149506a65..e13bf78bf15 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -620,9 +620,8 @@ func GetNodeAndType(path dyn.Path) (dyn.Path, string) { } if len(path) >= 4 { - sub := path[3].Key() - if sub == "permissions" || sub == "grants" { - return path[:4], path[1].Key() + "." + sub + if path[3].Key() == "permissions" || path[3].Key() == "grants" { + return path[:4], path[1].Key() + "." + path[3].Key() } } From 35a1d8a50d00dcb0d6f6581f7b775b0cc650b3fe Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 1 Sep 2026 08:36:38 +0000 Subject: [PATCH 32/55] revert unecessary changes --- bundle/direct/dresources/all_test.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index 48f9de47496..36eaa3e27df 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -11,7 +11,6 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" - "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/structs/structaccess" "github.com/databricks/cli/libs/structs/structdiff" "github.com/databricks/cli/libs/structs/structpath" @@ -1014,7 +1013,7 @@ func testCRUD(t *testing.T, group string, adapter *Adapter, client *databricks.W newState, err := adapter.PrepareState(inputConfig) require.NoError(t, err, "PrepareState failed") - ctx := cmdio.MockDiscard(t.Context()) + ctx := t.Context() // initial DoRead() cannot find the resource remote, err := adapter.DoRead(ctx, "1234") @@ -1130,8 +1129,6 @@ func testCRUD(t *testing.T, group string, adapter *Adapter, client *databricks.W require.NoError(t, err) } - // this lists all the sub-resources that do not delete the parent resource when they - // are deleted. deleteIsNoop := strings.HasSuffix(group, "permissions") || strings.HasSuffix(group, "grants") // Apps DoDelete is fire-and-forget: the API returns success while the app // sits in DELETING state for up to ~20 minutes before the record is removed. From ff961cd0091a3dd6745c87cf34c8b470064ccdd5 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 1 Sep 2026 08:38:52 +0000 Subject: [PATCH 33/55] revert pointless change --- bundle/direct/bundle_plan.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index fc8e375fa2f..7bfe7229e0a 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -717,14 +717,11 @@ func isEmptyStruct(rv reflect.Value) bool { // For regular resources like "resources.jobs.foo.name", returns ("resources.jobs.foo", "name"). // For sub-resources like "resources.jobs.foo.permissions[0].level", returns ("resources.jobs.foo.permissions", "[0].level"). func splitResourcePath(path *structpath.PathNode) (string, *structpath.PathNode) { - // permissions and grants are sub-resources for every group. + // Check if the 4th component is "permissions" or "grants" (sub-resource) if path.Len() > 4 { first := path.SkipPrefix(3).Prefix(1) - if key, ok := first.StringKey(); ok { - if key == "permissions" || key == "grants" { - return path.Prefix(4).String(), path.SkipPrefix(4) - } - } + if key, ok := first.StringKey(); ok && (key == "permissions" || key == "grants") { + return path.Prefix(4).String(), path.SkipPrefix(4) } return path.Prefix(3).String(), path.SkipPrefix(3) } From ccb2b434d26514982ffaee1f18b4710700dec6f3 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 1 Sep 2026 08:39:57 +0000 Subject: [PATCH 34/55] add bracket --- bundle/direct/bundle_plan.go | 1 + 1 file changed, 1 insertion(+) diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 7bfe7229e0a..453e588187b 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -722,6 +722,7 @@ func splitResourcePath(path *structpath.PathNode) (string, *structpath.PathNode) first := path.SkipPrefix(3).Prefix(1) if key, ok := first.StringKey(); ok && (key == "permissions" || key == "grants") { return path.Prefix(4).String(), path.SkipPrefix(4) + } } return path.Prefix(3).String(), path.SkipPrefix(3) } From c8af56d1443fcb8b98cc071c03f7bd87e2ba7dfc Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 1 Sep 2026 08:40:41 +0000 Subject: [PATCH 35/55] remove old test --- bundle/direct/bundle_plan_test.go | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/bundle/direct/bundle_plan_test.go b/bundle/direct/bundle_plan_test.go index e9186386d3d..bf875680c2b 100644 --- a/bundle/direct/bundle_plan_test.go +++ b/bundle/direct/bundle_plan_test.go @@ -401,28 +401,3 @@ func bundleWithSkippedJobRun(t *testing.T, remote *dresources.JobRunRemote) *Dep b.RemoteStateCache.Store(jobRunKey, remote) return b } - -func TestSplitResourcePath(t *testing.T) { - tests := []struct { - path string - wantKey string - wantField string - }{ - {"resources.jobs.foo.name", "resources.jobs.foo", "name"}, - {"resources.jobs.foo.permissions[0].level", "resources.jobs.foo.permissions", "[0].level"}, - {"resources.schemas.foo.grants[0].principal", "resources.schemas.foo.grants", "[0].principal"}, - // libraries is a plain field on the cluster (managed as part of the cluster), not a sub-resource. - {"resources.clusters.foo.libraries[0].whl", "resources.clusters.foo", "libraries[0].whl"}, - // pipelines have a native top-level libraries field, likewise not a sub-resource. - {"resources.pipelines.foo.libraries[0].notebook.path", "resources.pipelines.foo", "libraries[0].notebook.path"}, - } - for _, tt := range tests { - t.Run(tt.path, func(t *testing.T) { - path, err := structpath.ParsePath(tt.path) - require.NoError(t, err) - key, field := splitResourcePath(path) - assert.Equal(t, tt.wantKey, key) - assert.Equal(t, tt.wantField, field.String()) - }) - } -} From d2031283863dafaa2f46c9bb4f3aef2807c34c01 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 1 Sep 2026 08:53:42 +0000 Subject: [PATCH 36/55] Remove dead libraries sub-resource test scaffolding In the part-of-cluster model libraries is a field of ClusterState, not a sub-resource, so there is no clusters.libraries key in config.ResourcesTypes and no resource with top-level libraries missing in state. Drop the now-dead ".libraries" scanning in the invariant coverage test and the "libraries" entry in commonMissingInStateType. Co-authored-by: Isaac --- acceptance/invariant_test.go | 28 +++++++++------------------ bundle/direct/dresources/type_test.go | 1 - 2 files changed, 9 insertions(+), 20 deletions(-) diff --git a/acceptance/invariant_test.go b/acceptance/invariant_test.go index 3450728ea6f..1c8204055ed 100644 --- a/acceptance/invariant_test.go +++ b/acceptance/invariant_test.go @@ -19,9 +19,8 @@ const invariantConfigsDir = "bundle/invariant/configs" // LackingInvariantTest lists keys from config.ResourcesTypes that knowingly lack // a covering config in invariantConfigsDir. Keys match the ResourcesTypes // form: "" for the resource itself, ".permissions" / ".grants" -// / ".libraries" for sub-resource coverage. Add a config and remove the entry -// to close a gap; the test fails if an entry here is actually covered, so the list -// only shrinks. +// for permissions/grants coverage. Add a config and remove the entry to close a gap; +// the test fails if an entry here is actually covered, so the list only shrinks. var LackingInvariantTest = map[string]bool{ "quality_monitors": true, } @@ -31,11 +30,10 @@ var LackingInvariantTest = map[string]bool{ // types supporting permissions or grants have at least one config exercising them. // // config.ResourcesTypes is the source of truth: it maps each resource group -// (e.g. "jobs") to its Go type and adds derived keys ".permissions", -// ".grants", and ".libraries" where the resource has the -// corresponding sub-resource. +// (e.g. "jobs") to its Go type and, where the resource struct has a Permissions +// or Grants field, adds derived keys ".permissions" and ".grants". func TestInvariantConfigsCoverage(t *testing.T) { - present, withPermissions, withGrants, withLibraries := scanInvariantConfigs(t) + present, withPermissions, withGrants := scanInvariantConfigs(t) keys := make([]string, 0, len(config.ResourcesTypes)) for key := range config.ResourcesTypes { @@ -55,10 +53,6 @@ func TestInvariantConfigsCoverage(t *testing.T) { group := strings.TrimSuffix(key, ".grants") covered = withGrants[group] hint = "attaches grants to a " + group + " resource" - case strings.HasSuffix(key, ".libraries"): - group := strings.TrimSuffix(key, ".libraries") - covered = withLibraries[group] - hint = "attaches libraries to a " + group + " resource" default: covered = present[key] hint = "defines a " + key + " resource" @@ -75,13 +69,12 @@ func TestInvariantConfigsCoverage(t *testing.T) { } // scanInvariantConfigs parses every config in the invariant configs directory and -// returns the set of resource groups present, and the groups with at least one -// resource carrying permissions, grants, or libraries. -func scanInvariantConfigs(t *testing.T) (present, withPermissions, withGrants, withLibraries map[string]bool) { +// returns the set of resource groups present, the groups with at least one resource +// carrying permissions, and the groups with at least one resource carrying grants. +func scanInvariantConfigs(t *testing.T) (present, withPermissions, withGrants map[string]bool) { present = map[string]bool{} withPermissions = map[string]bool{} withGrants = map[string]bool{} - withLibraries = map[string]bool{} entries, err := os.ReadDir(invariantConfigsDir) require.NoError(t, err) @@ -121,12 +114,9 @@ func scanInvariantConfigs(t *testing.T) (present, withPermissions, withGrants, w if cfg.Get("grants").Kind() != dyn.KindInvalid { withGrants[groupName] = true } - if cfg.Get("libraries").Kind() != dyn.KindInvalid { - withLibraries[groupName] = true - } } } } - return present, withPermissions, withGrants, withLibraries + return present, withPermissions, withGrants } diff --git a/bundle/direct/dresources/type_test.go b/bundle/direct/dresources/type_test.go index e471d6b2b6e..2d5516d59c7 100644 --- a/bundle/direct/dresources/type_test.go +++ b/bundle/direct/dresources/type_test.go @@ -66,7 +66,6 @@ var knownMissingInRemoteType = map[string][]string{ // These are bundle-specific fields that exist in InputType but not in StateType. var commonMissingInStateType = []string{ "grants", - "libraries", "lifecycle", "permissions", } From 7b3fe0bf70837f6243e54d1998017d781d99abfe Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 1 Sep 2026 09:02:00 +0000 Subject: [PATCH 37/55] update comment --- bundle/direct/dresources/cluster.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/bundle/direct/dresources/cluster.go b/bundle/direct/dresources/cluster.go index a9553bfc8f4..2ddc2d12776 100644 --- a/bundle/direct/dresources/cluster.go +++ b/bundle/direct/dresources/cluster.go @@ -256,11 +256,9 @@ func (r *ResourceCluster) DoUpdate(ctx context.Context, id string, config *Clust } } - // TODO(#1860): a local whl/jar whose workspace path is unchanged but whose contents + // TODO: a local whl/jar whose workspace path is unchanged but whose contents // changed (same name+version, non-dev mode) is not detected here, so no restart fires. - // Dev mode handles this via patchwheel (a source-derived version bump); a general fix - // needs a source hash tracked in state. Hashing the built wheel is unsafe — the zip - // embeds mtimes, so a rebuild would churn a restart every deploy. + // Dev mode handles this via patchwheel (a source-derived version bump). if entry.Changes.HasChange(librariesPath) { if err := r.reconcileLibraries(ctx, id, config.Libraries, entry); err != nil { return nil, err From 82ea8342402a39404e6a4f19bca7ac0fbe5bae96 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 1 Sep 2026 09:23:00 +0000 Subject: [PATCH 38/55] Title the deploy phase in libraries-repo acceptance test The initial deploy step routed its output to LOG and asserted signals via contains.py, but had no title, so the phase was invisible in output.txt. Add a title so the golden documents it, matching the other cluster-libraries tests. Co-authored-by: Isaac --- acceptance/bundle/resources/clusters/libraries-repo/output.txt | 1 + acceptance/bundle/resources/clusters/libraries-repo/script | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/acceptance/bundle/resources/clusters/libraries-repo/output.txt b/acceptance/bundle/resources/clusters/libraries-repo/output.txt index 616e9015204..39c7a013d1d 100644 --- a/acceptance/bundle/resources/clusters/libraries-repo/output.txt +++ b/acceptance/bundle/resources/clusters/libraries-repo/output.txt @@ -1,4 +1,5 @@ +=== Deploy creates the cluster and installs the pypi library === Plan is a no-op after deploy: the repo round-trips >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged diff --git a/acceptance/bundle/resources/clusters/libraries-repo/script b/acceptance/bundle/resources/clusters/libraries-repo/script index 262a43b049f..976a3db9b53 100644 --- a/acceptance/bundle/resources/clusters/libraries-repo/script +++ b/acceptance/bundle/resources/clusters/libraries-repo/script @@ -6,7 +6,8 @@ cleanup() { } trap cleanup EXIT -# Deploy output is noisy and differs fake vs cloud; assert only the plan signals below. +# Deploy output is noisy and differs fake vs cloud, so route it to LOG. +title "Deploy creates the cluster and installs the pypi library" $CLI bundle deploy &> LOG.deploy cat LOG.deploy | contains.py "Created clusters.mycluster" > /dev/null From fe8d2dc1667799d4f1c3b04798127db3d5214324 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 1 Sep 2026 14:37:10 +0000 Subject: [PATCH 39/55] update nextchanges --- .nextchanges/bundles/cluster-libraries.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/bundles/cluster-libraries.md b/.nextchanges/bundles/cluster-libraries.md index 4c9b1a52006..30e04b2227b 100644 --- a/.nextchanges/bundles/cluster-libraries.md +++ b/.nextchanges/bundles/cluster-libraries.md @@ -1 +1 @@ -Add support for a `libraries` list on the `clusters` resource type in Declarative Automation Bundles. Libraries (whl, jar, pypi, maven, cran, egg, requirements) are installed on the all-purpose cluster via the Libraries API; local wheels/jars are built and uploaded automatically. Cluster libraries are only supported in direct deployment mode. +Add libraries field to clusters. [#6365](https://github.com/databricks/cli/pull/6365) From a3c0885dde35ac925e99cce333e366bf23bf7868 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 1 Sep 2026 15:02:41 +0000 Subject: [PATCH 40/55] Add READPLAN variant to cluster-libraries acceptance tests Run each cluster-libraries test through a saved plan (bundle plan -o json, applied via readplanarg) in addition to the inline deploy, mirroring clusters/readplan-lifecycle. Addresses review feedback. Deploy and plan output route to LOG/tmp.plan.json, so output.txt is unchanged; only the READPLAN matrix is added. Co-authored-by: Isaac --- .../resources/clusters/libraries-repo/out.test.toml | 1 + .../bundle/resources/clusters/libraries-repo/script | 6 ++++-- .../bundle/resources/clusters/libraries-repo/test.toml | 3 ++- .../resources/clusters/libraries-restart/out.test.toml | 1 + .../bundle/resources/clusters/libraries-restart/script | 9 ++++++--- .../resources/clusters/libraries-restart/test.toml | 3 ++- .../clusters/libraries-stopped-cluster/out.test.toml | 1 + .../resources/clusters/libraries-stopped-cluster/script | 6 ++++-- .../clusters/libraries-stopped-cluster/test.toml | 3 ++- .../bundle/resources/clusters/libraries/out.test.toml | 1 + acceptance/bundle/resources/clusters/libraries/script | 6 ++++-- acceptance/bundle/resources/clusters/libraries/test.toml | 3 ++- 12 files changed, 30 insertions(+), 13 deletions(-) diff --git a/acceptance/bundle/resources/clusters/libraries-repo/out.test.toml b/acceptance/bundle/resources/clusters/libraries-repo/out.test.toml index ae5c7bd798f..a420f50c421 100644 --- a/acceptance/bundle/resources/clusters/libraries-repo/out.test.toml +++ b/acceptance/bundle/resources/clusters/libraries-repo/out.test.toml @@ -1,3 +1,4 @@ Cloud = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.DMS = ["", "true"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/clusters/libraries-repo/script b/acceptance/bundle/resources/clusters/libraries-repo/script index 976a3db9b53..7ca90fd7d09 100644 --- a/acceptance/bundle/resources/clusters/libraries-repo/script +++ b/acceptance/bundle/resources/clusters/libraries-repo/script @@ -6,9 +6,11 @@ cleanup() { } trap cleanup EXIT -# Deploy output is noisy and differs fake vs cloud, so route it to LOG. +# Deploy output is noisy and differs fake vs cloud, so route it to LOG. The READPLAN +# variant applies a plan saved by `bundle plan` instead of one computed inline by deploy. title "Deploy creates the cluster and installs the pypi library" -$CLI bundle deploy &> LOG.deploy +$CLI bundle plan -o json > tmp.plan.json 2>>LOG.plan +$CLI bundle deploy $(readplanarg tmp.plan.json) &> LOG.deploy cat LOG.deploy | contains.py "Created clusters.mycluster" > /dev/null title "Plan is a no-op after deploy: the repo round-trips" diff --git a/acceptance/bundle/resources/clusters/libraries-repo/test.toml b/acceptance/bundle/resources/clusters/libraries-repo/test.toml index de8ea8588b8..4cd2db35d22 100644 --- a/acceptance/bundle/resources/clusters/libraries-repo/test.toml +++ b/acceptance/bundle/resources/clusters/libraries-repo/test.toml @@ -2,8 +2,9 @@ Cloud = true RecordRequests = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.READPLAN = ["", "1"] -Ignore = [".databricks", "databricks.yml"] +Ignore = [".databricks", "databricks.yml", "tmp.plan.json"] [[Repls]] Old = "[0-9]{4}-[0-9]{6}-[0-9a-z]{8}" diff --git a/acceptance/bundle/resources/clusters/libraries-restart/out.test.toml b/acceptance/bundle/resources/clusters/libraries-restart/out.test.toml index ae5c7bd798f..a420f50c421 100644 --- a/acceptance/bundle/resources/clusters/libraries-restart/out.test.toml +++ b/acceptance/bundle/resources/clusters/libraries-restart/out.test.toml @@ -1,3 +1,4 @@ Cloud = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.DMS = ["", "true"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/clusters/libraries-restart/script b/acceptance/bundle/resources/clusters/libraries-restart/script index 80b3a94836f..8355cb9feff 100644 --- a/acceptance/bundle/resources/clusters/libraries-restart/script +++ b/acceptance/bundle/resources/clusters/libraries-restart/script @@ -10,7 +10,8 @@ trap cleanup EXIT # the restart signal. This test pins the restart behaviour across the library lifecycle. title "Install (create): libraries install on the fresh cluster with no restart" -$CLI bundle deploy &> LOG.create +$CLI bundle plan -o json > tmp.plan.json 2>>LOG.plan +$CLI bundle deploy $(readplanarg tmp.plan.json) &> LOG.create cat LOG.create | contains.py "Created clusters.mycluster" "!Restarting cluster" > /dev/null title "Update: adding a library restarts the running cluster" @@ -19,7 +20,8 @@ update_file.py databricks.yml " - pypi: package: requests - pypi: package: six" -$CLI bundle deploy &> LOG.update +$CLI bundle plan -o json > tmp.plan.json 2>>LOG.plan +$CLI bundle deploy $(readplanarg tmp.plan.json) &> LOG.update cat LOG.update | contains.py "Updated clusters.mycluster" "Restarting cluster" > /dev/null title "Delete: removing all libraries restarts the running cluster" @@ -29,5 +31,6 @@ update_file.py databricks.yml " libraries: - pypi: package: six " "" -$CLI bundle deploy &> LOG.remove +$CLI bundle plan -o json > tmp.plan.json 2>>LOG.plan +$CLI bundle deploy $(readplanarg tmp.plan.json) &> LOG.remove cat LOG.remove | contains.py "Updated clusters.mycluster" "Restarting cluster" > /dev/null diff --git a/acceptance/bundle/resources/clusters/libraries-restart/test.toml b/acceptance/bundle/resources/clusters/libraries-restart/test.toml index de8ea8588b8..4cd2db35d22 100644 --- a/acceptance/bundle/resources/clusters/libraries-restart/test.toml +++ b/acceptance/bundle/resources/clusters/libraries-restart/test.toml @@ -2,8 +2,9 @@ Cloud = true RecordRequests = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.READPLAN = ["", "1"] -Ignore = [".databricks", "databricks.yml"] +Ignore = [".databricks", "databricks.yml", "tmp.plan.json"] [[Repls]] Old = "[0-9]{4}-[0-9]{6}-[0-9a-z]{8}" diff --git a/acceptance/bundle/resources/clusters/libraries-stopped-cluster/out.test.toml b/acceptance/bundle/resources/clusters/libraries-stopped-cluster/out.test.toml index ae5c7bd798f..a420f50c421 100644 --- a/acceptance/bundle/resources/clusters/libraries-stopped-cluster/out.test.toml +++ b/acceptance/bundle/resources/clusters/libraries-stopped-cluster/out.test.toml @@ -1,3 +1,4 @@ Cloud = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.DMS = ["", "true"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/clusters/libraries-stopped-cluster/script b/acceptance/bundle/resources/clusters/libraries-stopped-cluster/script index bcf7772b63e..bf76c6cb10b 100644 --- a/acceptance/bundle/resources/clusters/libraries-stopped-cluster/script +++ b/acceptance/bundle/resources/clusters/libraries-stopped-cluster/script @@ -10,7 +10,8 @@ trap cleanup EXIT # only the deterministic signals. The cluster ends up stopped (started=false), so no # restart is ever issued. title "Deploy libraries on a started=false cluster: no restart" -$CLI bundle deploy &> LOG.deploy +$CLI bundle plan -o json > tmp.plan.json 2>>LOG.plan +$CLI bundle deploy $(readplanarg tmp.plan.json) &> LOG.deploy cat LOG.deploy | contains.py "Created clusters.mycluster" "!Restarting cluster" > /dev/null title "Change the libraries and redeploy: cluster is stopped, so no restart is issued" @@ -19,5 +20,6 @@ update_file.py databricks.yml " - pypi: package: requests - pypi: package: numpy" -$CLI bundle deploy &> LOG.redeploy +$CLI bundle plan -o json > tmp.plan.json 2>>LOG.plan +$CLI bundle deploy $(readplanarg tmp.plan.json) &> LOG.redeploy cat LOG.redeploy | contains.py "Updated clusters.mycluster" "!Restarting cluster" > /dev/null diff --git a/acceptance/bundle/resources/clusters/libraries-stopped-cluster/test.toml b/acceptance/bundle/resources/clusters/libraries-stopped-cluster/test.toml index de8ea8588b8..4cd2db35d22 100644 --- a/acceptance/bundle/resources/clusters/libraries-stopped-cluster/test.toml +++ b/acceptance/bundle/resources/clusters/libraries-stopped-cluster/test.toml @@ -2,8 +2,9 @@ Cloud = true RecordRequests = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.READPLAN = ["", "1"] -Ignore = [".databricks", "databricks.yml"] +Ignore = [".databricks", "databricks.yml", "tmp.plan.json"] [[Repls]] Old = "[0-9]{4}-[0-9]{6}-[0-9a-z]{8}" diff --git a/acceptance/bundle/resources/clusters/libraries/out.test.toml b/acceptance/bundle/resources/clusters/libraries/out.test.toml index f9f4880725d..17e040e07dc 100644 --- a/acceptance/bundle/resources/clusters/libraries/out.test.toml +++ b/acceptance/bundle/resources/clusters/libraries/out.test.toml @@ -2,3 +2,4 @@ Cloud = true CloudSlow = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.DMS = ["", "true"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/clusters/libraries/script b/acceptance/bundle/resources/clusters/libraries/script index a49f2a9d493..fbfc5af91ca 100644 --- a/acceptance/bundle/resources/clusters/libraries/script +++ b/acceptance/bundle/resources/clusters/libraries/script @@ -13,7 +13,8 @@ trap cleanup EXIT # install request (which shows the rewritten wheel path). Libraries are part of the # cluster, so they show up under the cluster, not a separate resource. title "Deploy a cluster with a pypi and a local wheel library" -$CLI bundle deploy &> LOG.deploy +$CLI bundle plan -o json > tmp.plan.json 2>>LOG.plan +$CLI bundle deploy $(readplanarg tmp.plan.json) &> LOG.deploy cat LOG.deploy | contains.py "Created clusters.mycluster" > /dev/null title "Libraries installed via the Libraries API (wheel rewritten to its uploaded path)" @@ -21,6 +22,7 @@ trace print_requests.py //libraries/install title "Removing the wheel and redeploying uninstalls it and restarts the running cluster" update_file.py databricks.yml " - whl: ./dist/*.whl" "" -$CLI bundle deploy &> LOG.redeploy +$CLI bundle plan -o json > tmp.plan.json 2>>LOG.plan +$CLI bundle deploy $(readplanarg tmp.plan.json) &> LOG.redeploy cat LOG.redeploy | contains.py "Updated clusters.mycluster" "Restarting cluster" > /dev/null trace print_requests.py //libraries/uninstall diff --git a/acceptance/bundle/resources/clusters/libraries/test.toml b/acceptance/bundle/resources/clusters/libraries/test.toml index 3101985b060..ea6413633ba 100644 --- a/acceptance/bundle/resources/clusters/libraries/test.toml +++ b/acceptance/bundle/resources/clusters/libraries/test.toml @@ -3,8 +3,9 @@ CloudSlow = true RecordRequests = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.READPLAN = ["", "1"] -Ignore = [".databricks", ".venv", "build", "dist", "my_test_code.egg-info", "databricks.yml"] +Ignore = [".databricks", ".venv", "build", "dist", "my_test_code.egg-info", "databricks.yml", "tmp.plan.json"] [[Repls]] Old = "[0-9]{4}-[0-9]{6}-[0-9a-z]{8}" From 99d38d432efa6ca41a0d6f69557a84fcadfc0354 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Wed, 2 Sep 2026 11:29:28 +0000 Subject: [PATCH 41/55] make calls in parallel --- bundle/direct/dresources/cluster.go | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/bundle/direct/dresources/cluster.go b/bundle/direct/dresources/cluster.go index 2ddc2d12776..361841003d2 100644 --- a/bundle/direct/dresources/cluster.go +++ b/bundle/direct/dresources/cluster.go @@ -18,6 +18,7 @@ import ( "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/retries" "github.com/databricks/databricks-sdk-go/service/compute" + "golang.org/x/sync/errgroup" ) // librariesWaitTimeout bounds how long we poll for libraries to finish installing. @@ -152,15 +153,30 @@ func (r *ResourceCluster) RemapState(input *ClusterRemote) *ClusterState { } func (r *ResourceCluster) DoRead(ctx context.Context, id string) (*ClusterRemote, error) { - details, err := r.client.Clusters.GetByClusterId(ctx, id) - if err != nil { + var details *compute.ClusterDetails + var libraries []compute.Library + + // The cluster GET and the library-status GET are independent, so run them concurrently. + g, ctx := errgroup.WithContext(ctx) + g.Go(func() error { + var err error + details, err = r.client.Clusters.GetByClusterId(ctx, id) + return err + }) + g.Go(func() error { + var err error + libraries, err = r.readLibraries(ctx, id) + return err + }) + if err := g.Wait(); err != nil { return nil, err } + remote := &ClusterRemote{ ClusterDetails: *details, ApplyPolicyDefaultValues: false, Lifecycle: nil, - Libraries: nil, + Libraries: libraries, } // The GET response carries apply_policy_default_values only under .spec (a snapshot of the // create/edit settings), not at the top level. Promote it so RemapState is a dumb copy. @@ -179,11 +195,6 @@ func (r *ResourceCluster) DoRead(ctx context.Context, id string) (*ClusterRemote remote.Lifecycle = nil } - libraries, err := r.readLibraries(ctx, id) - if err != nil { - return nil, err - } - remote.Libraries = libraries return remote, nil } From 032ae53799479ea38de864570ab70c0dc0009151 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Wed, 2 Sep 2026 11:38:06 +0000 Subject: [PATCH 42/55] list known limitation --- bundle/direct/dresources/cluster.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bundle/direct/dresources/cluster.go b/bundle/direct/dresources/cluster.go index 361841003d2..fd5b20034ed 100644 --- a/bundle/direct/dresources/cluster.go +++ b/bundle/direct/dresources/cluster.go @@ -332,6 +332,9 @@ func (r *ResourceCluster) WaitAfterCreate(ctx context.Context, id string, config // Install libraries once the cluster is running. A freshly-created cluster has no // attached sessions, so the install applies live without a restart. + // TODO: Wait is supposed to be side effect free, but in this case moving it to + // the create will cause a wait for libraries to be installed befor the cluster is installed. + // this increases the risk of losing the cluster. This is a limitation if len(config.Libraries) > 0 { err = r.client.Libraries.Install(ctx, compute.InstallLibraries{ClusterId: id, Libraries: config.Libraries}) if err != nil { From 5ffa62049eba30bacbbd7e3188b1e31a4a01a8fd Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Wed, 2 Sep 2026 12:27:23 +0000 Subject: [PATCH 43/55] add a test for bad behaviour --- .../databricks.yml.tmpl | 23 ++++++++++++++++ .../my_test_code/__init__.py | 2 ++ .../my_test_code/__main__.py | 16 +++++++++++ .../libraries-content-change/out.test.toml | 5 ++++ .../libraries-content-change/output.txt | 26 ++++++++++++++++++ .../clusters/libraries-content-change/script | 27 +++++++++++++++++++ .../libraries-content-change/script.prepare | 9 +++++++ .../libraries-content-change/setup.py | 15 +++++++++++ .../libraries-content-change/test.toml | 14 ++++++++++ 9 files changed, 137 insertions(+) create mode 100644 acceptance/bundle/resources/clusters/libraries-content-change/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/clusters/libraries-content-change/my_test_code/__init__.py create mode 100644 acceptance/bundle/resources/clusters/libraries-content-change/my_test_code/__main__.py create mode 100644 acceptance/bundle/resources/clusters/libraries-content-change/out.test.toml create mode 100644 acceptance/bundle/resources/clusters/libraries-content-change/output.txt create mode 100644 acceptance/bundle/resources/clusters/libraries-content-change/script create mode 100644 acceptance/bundle/resources/clusters/libraries-content-change/script.prepare create mode 100644 acceptance/bundle/resources/clusters/libraries-content-change/setup.py create mode 100644 acceptance/bundle/resources/clusters/libraries-content-change/test.toml diff --git a/acceptance/bundle/resources/clusters/libraries-content-change/databricks.yml.tmpl b/acceptance/bundle/resources/clusters/libraries-content-change/databricks.yml.tmpl new file mode 100644 index 00000000000..f5e3173e888 --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-content-change/databricks.yml.tmpl @@ -0,0 +1,23 @@ +bundle: + name: cluster-libraries-$UNIQUE_NAME + +workspace: + root_path: ~/.bundle/$UNIQUE_NAME + +artifacts: + my_test_code: + type: whl + path: . + # Use 'python' because 'python3' does not exist in Windows virtualenvs. + build: python setup.py bdist_wheel + +resources: + clusters: + mycluster: + cluster_name: mycluster-$UNIQUE_NAME + spark_version: $DEFAULT_SPARK_VERSION + node_type_id: $NODE_TYPE_ID + num_workers: 1 + data_security_mode: $DATA_SECURITY_MODE + libraries: + - whl: ./dist/*.whl diff --git a/acceptance/bundle/resources/clusters/libraries-content-change/my_test_code/__init__.py b/acceptance/bundle/resources/clusters/libraries-content-change/my_test_code/__init__.py new file mode 100644 index 00000000000..909f1f3220d --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-content-change/my_test_code/__init__.py @@ -0,0 +1,2 @@ +__version__ = "0.0.1" +__author__ = "Databricks" diff --git a/acceptance/bundle/resources/clusters/libraries-content-change/my_test_code/__main__.py b/acceptance/bundle/resources/clusters/libraries-content-change/my_test_code/__main__.py new file mode 100644 index 00000000000..ea918ce2d53 --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-content-change/my_test_code/__main__.py @@ -0,0 +1,16 @@ +""" +The entry point of the Python Wheel +""" + +import sys + + +def main(): + # This method will print the provided arguments + print("Hello from my func") + print("Got arguments:") + print(sys.argv) + + +if __name__ == "__main__": + main() diff --git a/acceptance/bundle/resources/clusters/libraries-content-change/out.test.toml b/acceptance/bundle/resources/clusters/libraries-content-change/out.test.toml new file mode 100644 index 00000000000..17e040e07dc --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-content-change/out.test.toml @@ -0,0 +1,5 @@ +Cloud = true +CloudSlow = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["", "true"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/clusters/libraries-content-change/output.txt b/acceptance/bundle/resources/clusters/libraries-content-change/output.txt new file mode 100644 index 00000000000..c805142f9a7 --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-content-change/output.txt @@ -0,0 +1,26 @@ + +=== Deploy a cluster with a local wheel library +>>> print_requests.py //libraries/install +{ + "method": "POST", + "path": "/api/2.0/libraries/install", + "body": { + "cluster_id": "[UUID]", + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" + } + ] + } +} + +=== Badness: wheel code changes but version does not, so redeploy neither reinstalls nor restarts +>>> print_requests.py //libraries/install + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.clusters.mycluster + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/clusters/libraries-content-change/script b/acceptance/bundle/resources/clusters/libraries-content-change/script new file mode 100644 index 00000000000..7b0ad134255 --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-content-change/script @@ -0,0 +1,27 @@ +export DATA_SECURITY_MODE=USER_ISOLATION +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +# First deploy builds the wheel, uploads it, and installs it on the fresh cluster. +title "Deploy a cluster with a local wheel library" +$CLI bundle plan -o json > tmp.plan.json 2>>LOG.plan +$CLI bundle deploy $(readplanarg tmp.plan.json) &> LOG.deploy +cat LOG.deploy | contains.py "Created clusters.mycluster" > /dev/null +trace print_requests.py //libraries/install + +# Change only the wheel's code, keeping version 0.0.1 so the built filename - and +# thus the uploaded workspace path - is byte-for-byte identical to the first deploy. +title "Badness: wheel code changes but version does not, so redeploy neither reinstalls nor restarts" +update_file.py my_test_code/__main__.py "Hello from my func" "Hello from my func v2" +$CLI bundle plan -o json > tmp.plan.json 2>>LOG.plan +$CLI bundle deploy $(readplanarg tmp.plan.json) &> LOG.redeploy +# The libraries path is unchanged, so the plan sees no library change: the cluster is +# not restarted and no install request is issued, leaving the stale wheel running. The +# following two assertions encode that known limitation (see the TODO in cluster.go). +cat LOG.redeploy | contains.py "!Restarting cluster" > /dev/null +trace print_requests.py //libraries/install diff --git a/acceptance/bundle/resources/clusters/libraries-content-change/script.prepare b/acceptance/bundle/resources/clusters/libraries-content-change/script.prepare new file mode 100644 index 00000000000..14a888323c4 --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-content-change/script.prepare @@ -0,0 +1,9 @@ +uv venv -q .venv +venv_activate +uv pip install -q --no-index setuptools + +# On Windows, create python3 alias since some build commands use python3 +if [[ "$OSTYPE" == "msys" || "$OSTYPE" == "cygwin" || "$OSTYPE" == "win32" ]]; then + python3() { python "$@"; } + export -f python3 +fi diff --git a/acceptance/bundle/resources/clusters/libraries-content-change/setup.py b/acceptance/bundle/resources/clusters/libraries-content-change/setup.py new file mode 100644 index 00000000000..8b48a92b4ce --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-content-change/setup.py @@ -0,0 +1,15 @@ +from setuptools import setup, find_packages + +import my_test_code + +setup( + name="my_test_code", + version=my_test_code.__version__, + author=my_test_code.__author__, + url="https://databricks.com", + author_email="john.doe@databricks.com", + description="my example wheel", + packages=find_packages(include=["my_test_code"]), + entry_points={"group1": "run=my_test_code.__main__:main"}, + install_requires=["setuptools"], +) diff --git a/acceptance/bundle/resources/clusters/libraries-content-change/test.toml b/acceptance/bundle/resources/clusters/libraries-content-change/test.toml new file mode 100644 index 00000000000..1abd9f3ecba --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-content-change/test.toml @@ -0,0 +1,14 @@ +Badness = "A local wheel whose contents change but whose version (and thus workspace path) does not is not detected in non-dev mode, so redeploy neither reinstalls it nor restarts the cluster; the stale wheel keeps running. See the TODO in bundle/direct/dresources/cluster.go DoUpdate." + +Cloud = true +CloudSlow = true +RecordRequests = true + +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.READPLAN = ["", "1"] + +Ignore = [".databricks", ".venv", "build", "dist", "my_test_code.egg-info", "databricks.yml", "tmp.plan.json"] + +[[Repls]] +Old = "[0-9]{4}-[0-9]{6}-[0-9a-z]{8}" +New = "[UUID]" From 35b7b8c34e3cded7b9fca4e173891c2905c27ce9 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Wed, 2 Sep 2026 12:29:11 +0000 Subject: [PATCH 44/55] mark tests as slow --- .../bundle/resources/clusters/libraries-repo/out.test.toml | 1 + acceptance/bundle/resources/clusters/libraries-repo/test.toml | 1 + .../bundle/resources/clusters/libraries-restart/out.test.toml | 1 + acceptance/bundle/resources/clusters/libraries-restart/test.toml | 1 + .../resources/clusters/libraries-stopped-cluster/out.test.toml | 1 + .../resources/clusters/libraries-stopped-cluster/test.toml | 1 + 6 files changed, 6 insertions(+) diff --git a/acceptance/bundle/resources/clusters/libraries-repo/out.test.toml b/acceptance/bundle/resources/clusters/libraries-repo/out.test.toml index a420f50c421..17e040e07dc 100644 --- a/acceptance/bundle/resources/clusters/libraries-repo/out.test.toml +++ b/acceptance/bundle/resources/clusters/libraries-repo/out.test.toml @@ -1,4 +1,5 @@ Cloud = true +CloudSlow = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.DMS = ["", "true"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/clusters/libraries-repo/test.toml b/acceptance/bundle/resources/clusters/libraries-repo/test.toml index 4cd2db35d22..4724ab632f1 100644 --- a/acceptance/bundle/resources/clusters/libraries-repo/test.toml +++ b/acceptance/bundle/resources/clusters/libraries-repo/test.toml @@ -1,4 +1,5 @@ Cloud = true +CloudSlow = true RecordRequests = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/clusters/libraries-restart/out.test.toml b/acceptance/bundle/resources/clusters/libraries-restart/out.test.toml index a420f50c421..17e040e07dc 100644 --- a/acceptance/bundle/resources/clusters/libraries-restart/out.test.toml +++ b/acceptance/bundle/resources/clusters/libraries-restart/out.test.toml @@ -1,4 +1,5 @@ Cloud = true +CloudSlow = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.DMS = ["", "true"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/clusters/libraries-restart/test.toml b/acceptance/bundle/resources/clusters/libraries-restart/test.toml index 4cd2db35d22..4724ab632f1 100644 --- a/acceptance/bundle/resources/clusters/libraries-restart/test.toml +++ b/acceptance/bundle/resources/clusters/libraries-restart/test.toml @@ -1,4 +1,5 @@ Cloud = true +CloudSlow = true RecordRequests = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/clusters/libraries-stopped-cluster/out.test.toml b/acceptance/bundle/resources/clusters/libraries-stopped-cluster/out.test.toml index a420f50c421..17e040e07dc 100644 --- a/acceptance/bundle/resources/clusters/libraries-stopped-cluster/out.test.toml +++ b/acceptance/bundle/resources/clusters/libraries-stopped-cluster/out.test.toml @@ -1,4 +1,5 @@ Cloud = true +CloudSlow = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.DMS = ["", "true"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/clusters/libraries-stopped-cluster/test.toml b/acceptance/bundle/resources/clusters/libraries-stopped-cluster/test.toml index 4cd2db35d22..4724ab632f1 100644 --- a/acceptance/bundle/resources/clusters/libraries-stopped-cluster/test.toml +++ b/acceptance/bundle/resources/clusters/libraries-stopped-cluster/test.toml @@ -1,4 +1,5 @@ Cloud = true +CloudSlow = true RecordRequests = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] From 917219f82062b398a38a9bbe7e033e26a50efed4 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Mon, 7 Sep 2026 08:26:27 +0000 Subject: [PATCH 45/55] update nextchanges --- .nextchanges/bundles/cluster-libraries.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/bundles/cluster-libraries.md b/.nextchanges/bundles/cluster-libraries.md index 30e04b2227b..e93a1781ec7 100644 --- a/.nextchanges/bundles/cluster-libraries.md +++ b/.nextchanges/bundles/cluster-libraries.md @@ -1 +1 @@ -Add libraries field to clusters. [#6365](https://github.com/databricks/cli/pull/6365) +* Add libraries field to clusters. From 112f4c6facfa825534ba6c86e19b0bbee8ead9db Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Mon, 7 Sep 2026 08:30:40 +0000 Subject: [PATCH 46/55] update nextchanges --- .nextchanges/bundles/cluster-libraries.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/bundles/cluster-libraries.md b/.nextchanges/bundles/cluster-libraries.md index e93a1781ec7..2371d56ed46 100644 --- a/.nextchanges/bundles/cluster-libraries.md +++ b/.nextchanges/bundles/cluster-libraries.md @@ -1 +1 @@ -* Add libraries field to clusters. +* Add libraries field to clusters. [#6365](https://github.com/databricks/cli/pull/6365) From 003b59846a99e7cbe2c7f0e59f2a25625473c6bc Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Mon, 7 Sep 2026 08:33:19 +0000 Subject: [PATCH 47/55] Drop READPLAN from wheel tests; add deploy --plan upload tripwire bundle deploy --plan skips the build phase, so a cluster's local wheel library is never uploaded while the install still fires against its workspace path, failing on a real cluster. Remove the READPLAN variant from the two wheel-based tests (libraries, libraries-content-change) that hit this, and add a local tripwire (libraries-readplan-not-uploaded) that pins the missing-upload behavior until deploy --plan is fixed. Co-authored-by: Isaac --- .../libraries-content-change/out.test.toml | 1 - .../clusters/libraries-content-change/script | 6 +- .../libraries-content-change/test.toml | 3 +- .../databricks.yml | 13 ++++ .../dist/my_test_code-0.0.1-py3-none-any.whl | Bin 0 -> 1832 bytes .../out.plan.json | 25 +++++++ .../out.test.toml | 3 + .../output.txt | 68 ++++++++++++++++++ .../libraries-readplan-not-uploaded/script | 25 +++++++ .../libraries-readplan-not-uploaded/test.toml | 6 ++ .../clusters/libraries/out.test.toml | 1 - .../resources/clusters/libraries/script | 6 +- .../resources/clusters/libraries/test.toml | 3 +- 13 files changed, 146 insertions(+), 14 deletions(-) create mode 100644 acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/databricks.yml create mode 100644 acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/dist/my_test_code-0.0.1-py3-none-any.whl create mode 100644 acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/out.plan.json create mode 100644 acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/out.test.toml create mode 100644 acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/output.txt create mode 100644 acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/script create mode 100644 acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/test.toml diff --git a/acceptance/bundle/resources/clusters/libraries-content-change/out.test.toml b/acceptance/bundle/resources/clusters/libraries-content-change/out.test.toml index 17e040e07dc..f9f4880725d 100644 --- a/acceptance/bundle/resources/clusters/libraries-content-change/out.test.toml +++ b/acceptance/bundle/resources/clusters/libraries-content-change/out.test.toml @@ -2,4 +2,3 @@ Cloud = true CloudSlow = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.DMS = ["", "true"] -EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/clusters/libraries-content-change/script b/acceptance/bundle/resources/clusters/libraries-content-change/script index 7b0ad134255..e29320fa8d6 100644 --- a/acceptance/bundle/resources/clusters/libraries-content-change/script +++ b/acceptance/bundle/resources/clusters/libraries-content-change/script @@ -9,8 +9,7 @@ trap cleanup EXIT # First deploy builds the wheel, uploads it, and installs it on the fresh cluster. title "Deploy a cluster with a local wheel library" -$CLI bundle plan -o json > tmp.plan.json 2>>LOG.plan -$CLI bundle deploy $(readplanarg tmp.plan.json) &> LOG.deploy +$CLI bundle deploy &> LOG.deploy cat LOG.deploy | contains.py "Created clusters.mycluster" > /dev/null trace print_requests.py //libraries/install @@ -18,8 +17,7 @@ trace print_requests.py //libraries/install # thus the uploaded workspace path - is byte-for-byte identical to the first deploy. title "Badness: wheel code changes but version does not, so redeploy neither reinstalls nor restarts" update_file.py my_test_code/__main__.py "Hello from my func" "Hello from my func v2" -$CLI bundle plan -o json > tmp.plan.json 2>>LOG.plan -$CLI bundle deploy $(readplanarg tmp.plan.json) &> LOG.redeploy +$CLI bundle deploy &> LOG.redeploy # The libraries path is unchanged, so the plan sees no library change: the cluster is # not restarted and no install request is issued, leaving the stale wheel running. The # following two assertions encode that known limitation (see the TODO in cluster.go). diff --git a/acceptance/bundle/resources/clusters/libraries-content-change/test.toml b/acceptance/bundle/resources/clusters/libraries-content-change/test.toml index 1abd9f3ecba..afbab5783ff 100644 --- a/acceptance/bundle/resources/clusters/libraries-content-change/test.toml +++ b/acceptance/bundle/resources/clusters/libraries-content-change/test.toml @@ -5,9 +5,8 @@ CloudSlow = true RecordRequests = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.READPLAN = ["", "1"] -Ignore = [".databricks", ".venv", "build", "dist", "my_test_code.egg-info", "databricks.yml", "tmp.plan.json"] +Ignore = [".databricks", ".venv", "build", "dist", "my_test_code.egg-info", "databricks.yml"] [[Repls]] Old = "[0-9]{4}-[0-9]{6}-[0-9a-z]{8}" diff --git a/acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/databricks.yml b/acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/databricks.yml new file mode 100644 index 00000000000..db8e632173f --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/databricks.yml @@ -0,0 +1,13 @@ +bundle: + name: libraries-readplan-not-uploaded + +resources: + clusters: + mycluster: + cluster_name: mycluster + spark_version: 15.4.x-scala2.12 + node_type_id: Standard_DS3_v2 + num_workers: 1 + libraries: + # Prebuilt local wheel (no build: step) to keep the test hermetic. + - whl: ./dist/*.whl diff --git a/acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/dist/my_test_code-0.0.1-py3-none-any.whl b/acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/dist/my_test_code-0.0.1-py3-none-any.whl new file mode 100644 index 0000000000000000000000000000000000000000..4bb80477caf51393354453b18c0c88711098825e GIT binary patch literal 1832 zcmWIWW@Zs#U|`^2xLh&ryk0?NcfY>3uGYCT z=ezsEH-rWlT`<029P+5E-(N@fdGY9=0ryhVJ$tvvUNQc+{d3{D zn|B2?ro>EK#LF7E*yLqccPAsi^Am}D%e30AoeArVlYJ((&Chx9t8w97&hq5+iY(pM z?WKI0`Ng-xP8DwL5?S^qs;1iP{ao7%o!XY-3yZ&e5Ut|$zxe;x^oV^&AD++Fd7iem zR(X?DP}|ALQzYCuG|lZak1|#q*u;_lsY}J+e|O){(NT&M^DqET*~S;Ni)SUsN4C z9Dn1{?j=1*vsGR>2Ap1Urdi@ME2@w3r*1Nd0r~`ll?eGL+{4w?hx@#T@7asIkTkJ5 z$l!|cgChU4-nw2oC-pZ4d3c@F(d*P_Um1A8;DWK?CF2WExK5qdzxY+>8n1?~*V&Uk zn}RenwS1p)dHQS*(pX_~?d<9E-dBCktbX!{2`Tk}Lc4^`-&YapE?q)yOU)}Os*Eql z&&(?+)+?zf>Gsnum6bPz^n=Yu)|leb1cf?%&Q77=vb)*4}*!*?@Mi z0I?P!yGrs4;&W2VQgc8SY3q4CLw9*{w)FQ{Mg|65CI$v&LZ$_|I{OE?w4OU%)MUWm zaB=T_A&mrw*ph>*mAaB@F8Q(@M!#{WU5W^A%QRiSMW z{ozJ=^Fq0UM|XJ53fr~pZf|;Euh=Z{Z-Tx4a;F>EwmRNScHD8mvs7)8lJ+L8`I_QF zshMAec0^w`-2TiwZ^_?(T*a4Um+uz%9hJHAo6n-@<%iYQ%+*hCy{@^uvZID|#pH|s ze;h2ndvtru#jke9t4>^Ne8ji;gOB{FCkgtet-sDum^X=S@}n%?$yyAP)vvd7E<0Z8 zI`hG%7f1XLTdv`o%)7mde~0q)?pa+LTXvaz|5|P@sQzMo+2)nERD@WZf7CmNnCf|W zcinFk%sFDWcj{{;(KVwNY7ngq3`-iL;hNz^9I|F?B?v;ZHZZ*qRi2<5iJpZK zM*byaBsfE(n}?q55a!JUCTF}(M9+liMx*C4gwd?b1dK+>bLi%x=Qo(STN;B2nG4B% V0p6^j+|R%cgo;34egu^R3;+-1eXjrj literal 0 HcmV?d00001 diff --git a/acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/out.plan.json b/acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/out.plan.json new file mode 100644 index 00000000000..81accefa2b9 --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/out.plan.json @@ -0,0 +1,25 @@ +{ + "plan_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 2, + "plan": { + "resources.clusters.mycluster": { + "action": "create", + "new_state": { + "value": { + "autotermination_minutes": 60, + "cluster_name": "mycluster", + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/libraries-readplan-not-uploaded/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" + } + ], + "node_type_id": "Standard_DS3_v2", + "num_workers": 1, + "spark_version": "15.4.x-scala2.12" + } + } + } + } +} diff --git a/acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/out.test.toml b/acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/out.test.toml new file mode 100644 index 00000000000..59b56a2037c --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/output.txt b/acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/output.txt new file mode 100644 index 00000000000..a4baf653ff9 --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/output.txt @@ -0,0 +1,68 @@ + +=== bundle deploy uploads the wheel, then installs it on the cluster +>>> [CLI] bundle deploy +Uploading dist/my_test_code-0.0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/libraries-readplan-not-uploaded/default/files... +Created clusters.mycluster +Files: 6 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //artifacts/.internal //libraries/install --del-field raw_body +{ + "method": "POST", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/libraries-readplan-not-uploaded/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl", + "q": { + "overwrite": "true" + } +} +{ + "method": "POST", + "path": "/api/2.0/libraries/install", + "body": { + "cluster_id": "[UUID]", + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/libraries-readplan-not-uploaded/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" + } + ] + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.clusters.mycluster + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/libraries-readplan-not-uploaded/default + +Destroy: 1 deleted + +=== bundle deploy --plan does NOT upload the wheel (bug), yet still installs it +>>> [CLI] bundle plan -o json + +>>> [CLI] bundle deploy --plan out.plan.json +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/libraries-readplan-not-uploaded/default/files... +Created clusters.mycluster +Files: 7 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //artifacts/.internal //libraries/install --del-field raw_body +{ + "method": "POST", + "path": "/api/2.0/libraries/install", + "body": { + "cluster_id": "[UUID]", + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/libraries-readplan-not-uploaded/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" + } + ] + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.clusters.mycluster + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/libraries-readplan-not-uploaded/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/script b/acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/script new file mode 100644 index 00000000000..59acb31783c --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/script @@ -0,0 +1,25 @@ +# A cluster's local wheel library is uploaded by `bundle deploy` but NOT by +# `bundle deploy --plan`: applying a saved plan skips the build phase, which is what +# computes the artifact upload list (phases.Build -> libraries.ReplaceWithRemotePath). +# The cluster's library install still fires against the wheel's workspace path, so on a +# real cluster it fails with a missing-file error. This tripwire pins that difference; +# see test.toml Badness for what to do when it starts failing. + +title "bundle deploy uploads the wheel, then installs it on the cluster" +trace $CLI bundle deploy +# raw_body is the wheel's binary payload; drop it to keep the golden readable. +trace print_requests.py //artifacts/.internal //libraries/install --del-field raw_body + +trace $CLI bundle destroy --auto-approve +# Discard destroy's requests so the saved-plan deploy below starts from a clean recording. +rm out.requests.txt + +title "bundle deploy --plan does NOT upload the wheel (bug), yet still installs it" +trace $CLI bundle plan -o json > out.plan.json +trace $CLI bundle deploy --plan out.plan.json +# BUG: no import-file request for the wheel (the upload is missing), yet the install +# still references the same .../artifacts/.internal/ path that was never uploaded. +trace print_requests.py //artifacts/.internal //libraries/install --del-field raw_body + +trace $CLI bundle destroy --auto-approve +rm out.requests.txt diff --git a/acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/test.toml b/acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/test.toml new file mode 100644 index 00000000000..0c710249a18 --- /dev/null +++ b/acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/test.toml @@ -0,0 +1,6 @@ +Badness = "bundle deploy --plan skips the build phase, so a cluster's local wheel library is never uploaded, yet the install still fires against its (missing) workspace path. This tripwire pins that bug. When it is fixed the second install below will reference an uploaded wheel and the golden will change, failing this test: at that point delete this test and restore EnvMatrix.READPLAN = [\"\", \"1\"] on clusters/libraries and clusters/libraries-content-change." + +# --plan is direct-engine only. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +RecordRequests = true +Ignore = [".databricks"] diff --git a/acceptance/bundle/resources/clusters/libraries/out.test.toml b/acceptance/bundle/resources/clusters/libraries/out.test.toml index 17e040e07dc..f9f4880725d 100644 --- a/acceptance/bundle/resources/clusters/libraries/out.test.toml +++ b/acceptance/bundle/resources/clusters/libraries/out.test.toml @@ -2,4 +2,3 @@ Cloud = true CloudSlow = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.DMS = ["", "true"] -EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/clusters/libraries/script b/acceptance/bundle/resources/clusters/libraries/script index fbfc5af91ca..a49f2a9d493 100644 --- a/acceptance/bundle/resources/clusters/libraries/script +++ b/acceptance/bundle/resources/clusters/libraries/script @@ -13,8 +13,7 @@ trap cleanup EXIT # install request (which shows the rewritten wheel path). Libraries are part of the # cluster, so they show up under the cluster, not a separate resource. title "Deploy a cluster with a pypi and a local wheel library" -$CLI bundle plan -o json > tmp.plan.json 2>>LOG.plan -$CLI bundle deploy $(readplanarg tmp.plan.json) &> LOG.deploy +$CLI bundle deploy &> LOG.deploy cat LOG.deploy | contains.py "Created clusters.mycluster" > /dev/null title "Libraries installed via the Libraries API (wheel rewritten to its uploaded path)" @@ -22,7 +21,6 @@ trace print_requests.py //libraries/install title "Removing the wheel and redeploying uninstalls it and restarts the running cluster" update_file.py databricks.yml " - whl: ./dist/*.whl" "" -$CLI bundle plan -o json > tmp.plan.json 2>>LOG.plan -$CLI bundle deploy $(readplanarg tmp.plan.json) &> LOG.redeploy +$CLI bundle deploy &> LOG.redeploy cat LOG.redeploy | contains.py "Updated clusters.mycluster" "Restarting cluster" > /dev/null trace print_requests.py //libraries/uninstall diff --git a/acceptance/bundle/resources/clusters/libraries/test.toml b/acceptance/bundle/resources/clusters/libraries/test.toml index ea6413633ba..3101985b060 100644 --- a/acceptance/bundle/resources/clusters/libraries/test.toml +++ b/acceptance/bundle/resources/clusters/libraries/test.toml @@ -3,9 +3,8 @@ CloudSlow = true RecordRequests = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.READPLAN = ["", "1"] -Ignore = [".databricks", ".venv", "build", "dist", "my_test_code.egg-info", "databricks.yml", "tmp.plan.json"] +Ignore = [".databricks", ".venv", "build", "dist", "my_test_code.egg-info", "databricks.yml"] [[Repls]] Old = "[0-9]{4}-[0-9]{6}-[0-9a-z]{8}" From 459f91a8b2380b9a789faab99bc79ebc4d1d1ebf Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Mon, 7 Sep 2026 08:35:40 +0000 Subject: [PATCH 48/55] Fix changelog fragment format for cluster libraries Wrap the PR link in parentheses so the sentence ends with a period, as the .nextchanges validator requires. Co-authored-by: Isaac --- .nextchanges/bundles/cluster-libraries.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/bundles/cluster-libraries.md b/.nextchanges/bundles/cluster-libraries.md index 2371d56ed46..1ff0e422fa5 100644 --- a/.nextchanges/bundles/cluster-libraries.md +++ b/.nextchanges/bundles/cluster-libraries.md @@ -1 +1 @@ -* Add libraries field to clusters. [#6365](https://github.com/databricks/cli/pull/6365) +* Add libraries field to clusters. ([#6365](https://github.com/databricks/cli/pull/6365)) From dca4944da1238616682a0f7dc022048632d06a41 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Mon, 7 Sep 2026 08:37:09 +0000 Subject: [PATCH 49/55] delete useless test --- .../libraries-terraform-error/databricks.yml | 13 ------------- .../libraries-terraform-error/out.test.toml | 3 --- .../libraries-terraform-error/output.txt | 16 ---------------- .../clusters/libraries-terraform-error/script | 5 ----- .../clusters/libraries-terraform-error/test.toml | 4 ---- 5 files changed, 41 deletions(-) delete mode 100644 acceptance/bundle/resources/clusters/libraries-terraform-error/databricks.yml delete mode 100644 acceptance/bundle/resources/clusters/libraries-terraform-error/out.test.toml delete mode 100644 acceptance/bundle/resources/clusters/libraries-terraform-error/output.txt delete mode 100644 acceptance/bundle/resources/clusters/libraries-terraform-error/script delete mode 100644 acceptance/bundle/resources/clusters/libraries-terraform-error/test.toml diff --git a/acceptance/bundle/resources/clusters/libraries-terraform-error/databricks.yml b/acceptance/bundle/resources/clusters/libraries-terraform-error/databricks.yml deleted file mode 100644 index e2645b5cc5e..00000000000 --- a/acceptance/bundle/resources/clusters/libraries-terraform-error/databricks.yml +++ /dev/null @@ -1,13 +0,0 @@ -bundle: - name: cluster-libraries-terraform-error - -resources: - clusters: - mycluster: - cluster_name: mycluster - spark_version: 15.4.x-scala2.12 - node_type_id: i3.xlarge - num_workers: 1 - libraries: - - pypi: - package: requests diff --git a/acceptance/bundle/resources/clusters/libraries-terraform-error/out.test.toml b/acceptance/bundle/resources/clusters/libraries-terraform-error/out.test.toml deleted file mode 100644 index 5d0f3e4aac2..00000000000 --- a/acceptance/bundle/resources/clusters/libraries-terraform-error/out.test.toml +++ /dev/null @@ -1,3 +0,0 @@ -Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] -EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/clusters/libraries-terraform-error/output.txt b/acceptance/bundle/resources/clusters/libraries-terraform-error/output.txt deleted file mode 100644 index aa0a6711e7e..00000000000 --- a/acceptance/bundle/resources/clusters/libraries-terraform-error/output.txt +++ /dev/null @@ -1,16 +0,0 @@ - -=== bundle plan fails with cluster libraries on terraform engine ->>> errcode [CLI] bundle plan -Error: cluster libraries are only supported in direct deployment mode - in databricks.yml:12:9 - - -Exit code: 1 - -=== bundle deploy fails with cluster libraries on terraform engine ->>> errcode [CLI] bundle deploy -Error: cluster libraries are only supported in direct deployment mode - in databricks.yml:12:9 - - -Exit code: 1 diff --git a/acceptance/bundle/resources/clusters/libraries-terraform-error/script b/acceptance/bundle/resources/clusters/libraries-terraform-error/script deleted file mode 100644 index 93db6e06309..00000000000 --- a/acceptance/bundle/resources/clusters/libraries-terraform-error/script +++ /dev/null @@ -1,5 +0,0 @@ -title "bundle plan fails with cluster libraries on terraform engine" -trace errcode $CLI bundle plan - -title "bundle deploy fails with cluster libraries on terraform engine" -trace errcode $CLI bundle deploy diff --git a/acceptance/bundle/resources/clusters/libraries-terraform-error/test.toml b/acceptance/bundle/resources/clusters/libraries-terraform-error/test.toml deleted file mode 100644 index e4a0f1c6301..00000000000 --- a/acceptance/bundle/resources/clusters/libraries-terraform-error/test.toml +++ /dev/null @@ -1,4 +0,0 @@ -Cloud = false -RecordRequests = false - -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] From bb35c3aba44d0c4b42c0d03e57170c3a7618c009 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Mon, 7 Sep 2026 08:49:57 +0000 Subject: [PATCH 50/55] update test --- .../out.plan.json | 2 - .../output.txt | 54 +------------------ .../libraries-readplan-not-uploaded/script | 24 ++------- .../libraries-readplan-not-uploaded/test.toml | 6 +-- 4 files changed, 7 insertions(+), 79 deletions(-) diff --git a/acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/out.plan.json b/acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/out.plan.json index 81accefa2b9..2c5ab71c952 100644 --- a/acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/out.plan.json +++ b/acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/out.plan.json @@ -1,8 +1,6 @@ { "plan_version": 2, "cli_version": "[CLI_VERSION]", - "lineage": "[UUID]", - "serial": 2, "plan": { "resources.clusters.mycluster": { "action": "create", diff --git a/acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/output.txt b/acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/output.txt index a4baf653ff9..457d7f944cc 100644 --- a/acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/output.txt +++ b/acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/output.txt @@ -1,64 +1,12 @@ -=== bundle deploy uploads the wheel, then installs it on the cluster ->>> [CLI] bundle deploy -Uploading dist/my_test_code-0.0.1-py3-none-any.whl... -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/libraries-readplan-not-uploaded/default/files... -Created clusters.mycluster -Files: 6 uploaded, 0 deleted -Resources: 1 created, 0 changed, 0 deleted, 0 unchanged - ->>> print_requests.py //artifacts/.internal //libraries/install --del-field raw_body -{ - "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/libraries-readplan-not-uploaded/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl", - "q": { - "overwrite": "true" - } -} -{ - "method": "POST", - "path": "/api/2.0/libraries/install", - "body": { - "cluster_id": "[UUID]", - "libraries": [ - { - "whl": "/Workspace/Users/[USERNAME]/.bundle/libraries-readplan-not-uploaded/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" - } - ] - } -} - ->>> [CLI] bundle destroy --auto-approve -The following resources will be deleted: - delete resources.clusters.mycluster - -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/libraries-readplan-not-uploaded/default - -Destroy: 1 deleted - -=== bundle deploy --plan does NOT upload the wheel (bug), yet still installs it >>> [CLI] bundle plan -o json >>> [CLI] bundle deploy --plan out.plan.json Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/libraries-readplan-not-uploaded/default/files... Created clusters.mycluster -Files: 7 uploaded, 0 deleted +Files: 6 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ->>> print_requests.py //artifacts/.internal //libraries/install --del-field raw_body -{ - "method": "POST", - "path": "/api/2.0/libraries/install", - "body": { - "cluster_id": "[UUID]", - "libraries": [ - { - "whl": "/Workspace/Users/[USERNAME]/.bundle/libraries-readplan-not-uploaded/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" - } - ] - } -} - >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.clusters.mycluster diff --git a/acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/script b/acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/script index 59acb31783c..7cf4c340482 100644 --- a/acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/script +++ b/acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/script @@ -1,25 +1,7 @@ # A cluster's local wheel library is uploaded by `bundle deploy` but NOT by -# `bundle deploy --plan`: applying a saved plan skips the build phase, which is what -# computes the artifact upload list (phases.Build -> libraries.ReplaceWithRemotePath). -# The cluster's library install still fires against the wheel's workspace path, so on a -# real cluster it fails with a missing-file error. This tripwire pins that difference; -# see test.toml Badness for what to do when it starts failing. - -title "bundle deploy uploads the wheel, then installs it on the cluster" -trace $CLI bundle deploy -# raw_body is the wheel's binary payload; drop it to keep the golden readable. -trace print_requests.py //artifacts/.internal //libraries/install --del-field raw_body - -trace $CLI bundle destroy --auto-approve -# Discard destroy's requests so the saved-plan deploy below starts from a clean recording. -rm out.requests.txt - -title "bundle deploy --plan does NOT upload the wheel (bug), yet still installs it" +# `bundle deploy --plan`. This tripwire pins that difference. +# Can be removed once fixed, and other tests can be moved to READPLAN variants. +#See acceptance/bundle/deploy/readplan/whl-not-uploaded for the bug. trace $CLI bundle plan -o json > out.plan.json trace $CLI bundle deploy --plan out.plan.json -# BUG: no import-file request for the wheel (the upload is missing), yet the install -# still references the same .../artifacts/.internal/ path that was never uploaded. -trace print_requests.py //artifacts/.internal //libraries/install --del-field raw_body - trace $CLI bundle destroy --auto-approve -rm out.requests.txt diff --git a/acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/test.toml b/acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/test.toml index 0c710249a18..38aff562e99 100644 --- a/acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/test.toml +++ b/acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/test.toml @@ -1,6 +1,6 @@ -Badness = "bundle deploy --plan skips the build phase, so a cluster's local wheel library is never uploaded, yet the install still fires against its (missing) workspace path. This tripwire pins that bug. When it is fixed the second install below will reference an uploaded wheel and the golden will change, failing this test: at that point delete this test and restore EnvMatrix.READPLAN = [\"\", \"1\"] on clusters/libraries and clusters/libraries-content-change." +Badness = "bundle deploy --plan skips the build phase, so a cluster's local wheel is never uploaded (proven in deploy/readplan/whl-not-uploaded). Kept as a note: when the bug is fixed, deploy --plan will print 'Uploading dist/...' and this golden changes - then delete this test and restore EnvMatrix.READPLAN = [\"\", \"1\"] on clusters/libraries and clusters/libraries-content-change." -# --plan is direct-engine only. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -RecordRequests = true +# This test is just asserting bad behaviour so that it can be removed once that behaviour is fixed +RecordRequests = false Ignore = [".databricks"] From 42c0fa9d8a16bebee6b42a0145d2976101be4676 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 8 Sep 2026 12:02:46 +0000 Subject: [PATCH 51/55] Print a pass-line confirmation in LOG-routed cluster library tests These tests route deploy output to LOG (it differs fake vs cloud), so the assertions were invisible in output.txt. Print a short confirmation after each so the golden shows what was verified. Co-authored-by: Isaac --- .../clusters/libraries-content-change/output.txt | 4 ++++ .../resources/clusters/libraries-content-change/script | 4 +++- .../bundle/resources/clusters/libraries-repo/output.txt | 2 ++ .../bundle/resources/clusters/libraries-repo/script | 4 ++-- .../resources/clusters/libraries-restart/output.txt | 6 ++++++ .../bundle/resources/clusters/libraries-restart/script | 7 ++++--- .../clusters/libraries-stopped-cluster/output.txt | 4 ++++ .../resources/clusters/libraries-stopped-cluster/script | 6 +++--- acceptance/bundle/resources/clusters/libraries/output.txt | 4 ++++ acceptance/bundle/resources/clusters/libraries/script | 8 +++----- 10 files changed, 35 insertions(+), 14 deletions(-) diff --git a/acceptance/bundle/resources/clusters/libraries-content-change/output.txt b/acceptance/bundle/resources/clusters/libraries-content-change/output.txt index c805142f9a7..8c13e12e371 100644 --- a/acceptance/bundle/resources/clusters/libraries-content-change/output.txt +++ b/acceptance/bundle/resources/clusters/libraries-content-change/output.txt @@ -1,5 +1,7 @@ === Deploy a cluster with a local wheel library +OK: cluster created with wheel + >>> print_requests.py //libraries/install { "method": "POST", @@ -15,6 +17,8 @@ } === Badness: wheel code changes but version does not, so redeploy neither reinstalls nor restarts +OK: no restart on content-only change (known limitation) + >>> print_requests.py //libraries/install >>> [CLI] bundle destroy --auto-approve diff --git a/acceptance/bundle/resources/clusters/libraries-content-change/script b/acceptance/bundle/resources/clusters/libraries-content-change/script index e29320fa8d6..2fa40f5b891 100644 --- a/acceptance/bundle/resources/clusters/libraries-content-change/script +++ b/acceptance/bundle/resources/clusters/libraries-content-change/script @@ -7,10 +7,11 @@ cleanup() { } trap cleanup EXIT -# First deploy builds the wheel, uploads it, and installs it on the fresh cluster. +# Deploy output goes to LOG (differs fake vs cloud); echo a pass line instead. title "Deploy a cluster with a local wheel library" $CLI bundle deploy &> LOG.deploy cat LOG.deploy | contains.py "Created clusters.mycluster" > /dev/null +printf '\n%s\n' "OK: cluster created with wheel" trace print_requests.py //libraries/install # Change only the wheel's code, keeping version 0.0.1 so the built filename - and @@ -22,4 +23,5 @@ $CLI bundle deploy &> LOG.redeploy # not restarted and no install request is issued, leaving the stale wheel running. The # following two assertions encode that known limitation (see the TODO in cluster.go). cat LOG.redeploy | contains.py "!Restarting cluster" > /dev/null +printf '\n%s\n' "OK: no restart on content-only change (known limitation)" trace print_requests.py //libraries/install diff --git a/acceptance/bundle/resources/clusters/libraries-repo/output.txt b/acceptance/bundle/resources/clusters/libraries-repo/output.txt index 39c7a013d1d..03dfa68dc62 100644 --- a/acceptance/bundle/resources/clusters/libraries-repo/output.txt +++ b/acceptance/bundle/resources/clusters/libraries-repo/output.txt @@ -1,5 +1,7 @@ === Deploy creates the cluster and installs the pypi library +OK: cluster created with pypi library + === Plan is a no-op after deploy: the repo round-trips >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged diff --git a/acceptance/bundle/resources/clusters/libraries-repo/script b/acceptance/bundle/resources/clusters/libraries-repo/script index 7ca90fd7d09..c44c39e8b1f 100644 --- a/acceptance/bundle/resources/clusters/libraries-repo/script +++ b/acceptance/bundle/resources/clusters/libraries-repo/script @@ -6,12 +6,12 @@ cleanup() { } trap cleanup EXIT -# Deploy output is noisy and differs fake vs cloud, so route it to LOG. The READPLAN -# variant applies a plan saved by `bundle plan` instead of one computed inline by deploy. +# Deploy output goes to LOG (differs fake vs cloud); echo a pass line instead. title "Deploy creates the cluster and installs the pypi library" $CLI bundle plan -o json > tmp.plan.json 2>>LOG.plan $CLI bundle deploy $(readplanarg tmp.plan.json) &> LOG.deploy cat LOG.deploy | contains.py "Created clusters.mycluster" > /dev/null +printf '\n%s\n' "OK: cluster created with pypi library" title "Plan is a no-op after deploy: the repo round-trips" trace $CLI bundle plan diff --git a/acceptance/bundle/resources/clusters/libraries-restart/output.txt b/acceptance/bundle/resources/clusters/libraries-restart/output.txt index d1737e6d579..804f8e84af5 100644 --- a/acceptance/bundle/resources/clusters/libraries-restart/output.txt +++ b/acceptance/bundle/resources/clusters/libraries-restart/output.txt @@ -1,7 +1,13 @@ === Install (create): libraries install on the fresh cluster with no restart +OK: libraries installed on create, no restart + === Update: adding a library restarts the running cluster +OK: library added, cluster restarted + === Delete: removing all libraries restarts the running cluster +OK: libraries removed, cluster restarted + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.clusters.mycluster diff --git a/acceptance/bundle/resources/clusters/libraries-restart/script b/acceptance/bundle/resources/clusters/libraries-restart/script index 8355cb9feff..6fc2383b366 100644 --- a/acceptance/bundle/resources/clusters/libraries-restart/script +++ b/acceptance/bundle/resources/clusters/libraries-restart/script @@ -6,13 +6,12 @@ cleanup() { } trap cleanup EXIT -# Deploy output is noisy and differs fake vs cloud, so route it to LOG and assert only -# the restart signal. This test pins the restart behaviour across the library lifecycle. - +# Deploy output goes to LOG (differs fake vs cloud); echo a pass line instead. title "Install (create): libraries install on the fresh cluster with no restart" $CLI bundle plan -o json > tmp.plan.json 2>>LOG.plan $CLI bundle deploy $(readplanarg tmp.plan.json) &> LOG.create cat LOG.create | contains.py "Created clusters.mycluster" "!Restarting cluster" > /dev/null +printf '\n%s\n' "OK: libraries installed on create, no restart" title "Update: adding a library restarts the running cluster" update_file.py databricks.yml " - pypi: @@ -23,6 +22,7 @@ update_file.py databricks.yml " - pypi: $CLI bundle plan -o json > tmp.plan.json 2>>LOG.plan $CLI bundle deploy $(readplanarg tmp.plan.json) &> LOG.update cat LOG.update | contains.py "Updated clusters.mycluster" "Restarting cluster" > /dev/null +printf '\n%s\n' "OK: library added, cluster restarted" title "Delete: removing all libraries restarts the running cluster" update_file.py databricks.yml " libraries: @@ -34,3 +34,4 @@ update_file.py databricks.yml " libraries: $CLI bundle plan -o json > tmp.plan.json 2>>LOG.plan $CLI bundle deploy $(readplanarg tmp.plan.json) &> LOG.remove cat LOG.remove | contains.py "Updated clusters.mycluster" "Restarting cluster" > /dev/null +printf '\n%s\n' "OK: libraries removed, cluster restarted" diff --git a/acceptance/bundle/resources/clusters/libraries-stopped-cluster/output.txt b/acceptance/bundle/resources/clusters/libraries-stopped-cluster/output.txt index 3af56fce4ef..6319906394e 100644 --- a/acceptance/bundle/resources/clusters/libraries-stopped-cluster/output.txt +++ b/acceptance/bundle/resources/clusters/libraries-stopped-cluster/output.txt @@ -1,6 +1,10 @@ === Deploy libraries on a started=false cluster: no restart +OK: created on stopped cluster, no restart + === Change the libraries and redeploy: cluster is stopped, so no restart is issued +OK: libraries changed on stopped cluster, no restart + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.clusters.mycluster diff --git a/acceptance/bundle/resources/clusters/libraries-stopped-cluster/script b/acceptance/bundle/resources/clusters/libraries-stopped-cluster/script index bf76c6cb10b..02f0b8eeed6 100644 --- a/acceptance/bundle/resources/clusters/libraries-stopped-cluster/script +++ b/acceptance/bundle/resources/clusters/libraries-stopped-cluster/script @@ -6,13 +6,12 @@ cleanup() { } trap cleanup EXIT -# Deploy output is noisy and differs fake vs cloud, so route it to LOG and assert -# only the deterministic signals. The cluster ends up stopped (started=false), so no -# restart is ever issued. +# Deploy output goes to LOG (differs fake vs cloud); echo a pass line instead. title "Deploy libraries on a started=false cluster: no restart" $CLI bundle plan -o json > tmp.plan.json 2>>LOG.plan $CLI bundle deploy $(readplanarg tmp.plan.json) &> LOG.deploy cat LOG.deploy | contains.py "Created clusters.mycluster" "!Restarting cluster" > /dev/null +printf '\n%s\n' "OK: created on stopped cluster, no restart" title "Change the libraries and redeploy: cluster is stopped, so no restart is issued" update_file.py databricks.yml " - pypi: @@ -23,3 +22,4 @@ update_file.py databricks.yml " - pypi: $CLI bundle plan -o json > tmp.plan.json 2>>LOG.plan $CLI bundle deploy $(readplanarg tmp.plan.json) &> LOG.redeploy cat LOG.redeploy | contains.py "Updated clusters.mycluster" "!Restarting cluster" > /dev/null +printf '\n%s\n' "OK: libraries changed on stopped cluster, no restart" diff --git a/acceptance/bundle/resources/clusters/libraries/output.txt b/acceptance/bundle/resources/clusters/libraries/output.txt index 4ad3e376e64..1a686ac151a 100644 --- a/acceptance/bundle/resources/clusters/libraries/output.txt +++ b/acceptance/bundle/resources/clusters/libraries/output.txt @@ -1,5 +1,7 @@ === Deploy a cluster with a pypi and a local wheel library +OK: cluster created with libraries + === Libraries installed via the Libraries API (wheel rewritten to its uploaded path) >>> print_requests.py //libraries/install { @@ -21,6 +23,8 @@ } === Removing the wheel and redeploying uninstalls it and restarts the running cluster +OK: wheel uninstalled, cluster restarted + >>> print_requests.py //libraries/uninstall { "method": "POST", diff --git a/acceptance/bundle/resources/clusters/libraries/script b/acceptance/bundle/resources/clusters/libraries/script index a49f2a9d493..57484a0a77b 100644 --- a/acceptance/bundle/resources/clusters/libraries/script +++ b/acceptance/bundle/resources/clusters/libraries/script @@ -7,14 +7,11 @@ cleanup() { } trap cleanup EXIT -# Deploy builds the wheel from setup.py, uploads it, and rewrites the whl entry to -# its workspace path. Deploy output is noisy and differs fake vs cloud, so route it -# to LOG and assert only the deterministic signals: the created cluster and the -# install request (which shows the rewritten wheel path). Libraries are part of the -# cluster, so they show up under the cluster, not a separate resource. +# Deploy output goes to LOG (differs fake vs cloud); echo a pass line instead. title "Deploy a cluster with a pypi and a local wheel library" $CLI bundle deploy &> LOG.deploy cat LOG.deploy | contains.py "Created clusters.mycluster" > /dev/null +printf '\n%s\n' "OK: cluster created with libraries" title "Libraries installed via the Libraries API (wheel rewritten to its uploaded path)" trace print_requests.py //libraries/install @@ -23,4 +20,5 @@ title "Removing the wheel and redeploying uninstalls it and restarts the running update_file.py databricks.yml " - whl: ./dist/*.whl" "" $CLI bundle deploy &> LOG.redeploy cat LOG.redeploy | contains.py "Updated clusters.mycluster" "Restarting cluster" > /dev/null +printf '\n%s\n' "OK: wheel uninstalled, cluster restarted" trace print_requests.py //libraries/uninstall From b2decfe759c7d8313098ee37b562d213feb0aad9 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 8 Sep 2026 12:09:23 +0000 Subject: [PATCH 52/55] remove python3 allias --- .../clusters/libraries-content-change/script.prepare | 6 ------ .../bundle/resources/clusters/libraries/script.prepare | 6 ------ 2 files changed, 12 deletions(-) diff --git a/acceptance/bundle/resources/clusters/libraries-content-change/script.prepare b/acceptance/bundle/resources/clusters/libraries-content-change/script.prepare index 14a888323c4..31e6de1dbf5 100644 --- a/acceptance/bundle/resources/clusters/libraries-content-change/script.prepare +++ b/acceptance/bundle/resources/clusters/libraries-content-change/script.prepare @@ -1,9 +1,3 @@ uv venv -q .venv venv_activate uv pip install -q --no-index setuptools - -# On Windows, create python3 alias since some build commands use python3 -if [[ "$OSTYPE" == "msys" || "$OSTYPE" == "cygwin" || "$OSTYPE" == "win32" ]]; then - python3() { python "$@"; } - export -f python3 -fi diff --git a/acceptance/bundle/resources/clusters/libraries/script.prepare b/acceptance/bundle/resources/clusters/libraries/script.prepare index 14a888323c4..31e6de1dbf5 100644 --- a/acceptance/bundle/resources/clusters/libraries/script.prepare +++ b/acceptance/bundle/resources/clusters/libraries/script.prepare @@ -1,9 +1,3 @@ uv venv -q .venv venv_activate uv pip install -q --no-index setuptools - -# On Windows, create python3 alias since some build commands use python3 -if [[ "$OSTYPE" == "msys" || "$OSTYPE" == "cygwin" || "$OSTYPE" == "win32" ]]; then - python3() { python "$@"; } - export -f python3 -fi From 3e966a73a507526416c52909aede9ddd09ad2314 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 8 Sep 2026 12:19:17 +0000 Subject: [PATCH 53/55] Capture the cluster restart request in library tests Assert the actual POST /api/2.1/clusters/restart via print_requests instead of only grepping the 'Restarting cluster' log line, so the golden proves the restart request was sent (present on library add/remove, absent on create). Re-enable RecordRequests on libraries-restart (was disabled because it only checked the log). Co-authored-by: Isaac --- .../clusters/libraries-restart/output.txt | 20 ++++++++++++++++--- .../clusters/libraries-restart/script | 15 +++++++------- .../clusters/libraries-restart/test.toml | 1 - .../resources/clusters/libraries/output.txt | 11 +++++++--- .../resources/clusters/libraries/script | 5 ++--- 5 files changed, 35 insertions(+), 17 deletions(-) diff --git a/acceptance/bundle/resources/clusters/libraries-restart/output.txt b/acceptance/bundle/resources/clusters/libraries-restart/output.txt index 804f8e84af5..df7c44549eb 100644 --- a/acceptance/bundle/resources/clusters/libraries-restart/output.txt +++ b/acceptance/bundle/resources/clusters/libraries-restart/output.txt @@ -1,12 +1,26 @@ === Install (create): libraries install on the fresh cluster with no restart -OK: libraries installed on create, no restart +>>> print_requests.py //clusters/restart === Update: adding a library restarts the running cluster -OK: library added, cluster restarted +>>> print_requests.py //clusters/restart +{ + "method": "POST", + "path": "/api/2.1/clusters/restart", + "body": { + "cluster_id": "[UUID]" + } +} === Delete: removing all libraries restarts the running cluster -OK: libraries removed, cluster restarted +>>> print_requests.py //clusters/restart +{ + "method": "POST", + "path": "/api/2.1/clusters/restart", + "body": { + "cluster_id": "[UUID]" + } +} >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/resources/clusters/libraries-restart/script b/acceptance/bundle/resources/clusters/libraries-restart/script index 6fc2383b366..21ac92e7312 100644 --- a/acceptance/bundle/resources/clusters/libraries-restart/script +++ b/acceptance/bundle/resources/clusters/libraries-restart/script @@ -6,12 +6,13 @@ cleanup() { } trap cleanup EXIT -# Deploy output goes to LOG (differs fake vs cloud); echo a pass line instead. +# Deploy output goes to LOG (differs fake vs cloud); the captured restart request is the proof. title "Install (create): libraries install on the fresh cluster with no restart" $CLI bundle plan -o json > tmp.plan.json 2>>LOG.plan $CLI bundle deploy $(readplanarg tmp.plan.json) &> LOG.create -cat LOG.create | contains.py "Created clusters.mycluster" "!Restarting cluster" > /dev/null -printf '\n%s\n' "OK: libraries installed on create, no restart" +cat LOG.create | contains.py "Created clusters.mycluster" > /dev/null +# No restart request on create (empty below). +trace print_requests.py //clusters/restart title "Update: adding a library restarts the running cluster" update_file.py databricks.yml " - pypi: @@ -21,8 +22,8 @@ update_file.py databricks.yml " - pypi: package: six" $CLI bundle plan -o json > tmp.plan.json 2>>LOG.plan $CLI bundle deploy $(readplanarg tmp.plan.json) &> LOG.update -cat LOG.update | contains.py "Updated clusters.mycluster" "Restarting cluster" > /dev/null -printf '\n%s\n' "OK: library added, cluster restarted" +cat LOG.update | contains.py "Updated clusters.mycluster" > /dev/null +trace print_requests.py //clusters/restart title "Delete: removing all libraries restarts the running cluster" update_file.py databricks.yml " libraries: @@ -33,5 +34,5 @@ update_file.py databricks.yml " libraries: " "" $CLI bundle plan -o json > tmp.plan.json 2>>LOG.plan $CLI bundle deploy $(readplanarg tmp.plan.json) &> LOG.remove -cat LOG.remove | contains.py "Updated clusters.mycluster" "Restarting cluster" > /dev/null -printf '\n%s\n' "OK: libraries removed, cluster restarted" +cat LOG.remove | contains.py "Updated clusters.mycluster" > /dev/null +trace print_requests.py //clusters/restart diff --git a/acceptance/bundle/resources/clusters/libraries-restart/test.toml b/acceptance/bundle/resources/clusters/libraries-restart/test.toml index 4724ab632f1..24d1138996d 100644 --- a/acceptance/bundle/resources/clusters/libraries-restart/test.toml +++ b/acceptance/bundle/resources/clusters/libraries-restart/test.toml @@ -1,6 +1,5 @@ Cloud = true CloudSlow = true -RecordRequests = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/clusters/libraries/output.txt b/acceptance/bundle/resources/clusters/libraries/output.txt index 1a686ac151a..ea0567a0dca 100644 --- a/acceptance/bundle/resources/clusters/libraries/output.txt +++ b/acceptance/bundle/resources/clusters/libraries/output.txt @@ -23,9 +23,7 @@ OK: cluster created with libraries } === Removing the wheel and redeploying uninstalls it and restarts the running cluster -OK: wheel uninstalled, cluster restarted - ->>> print_requests.py //libraries/uninstall +>>> print_requests.py //libraries/uninstall //clusters/restart { "method": "POST", "path": "/api/2.0/libraries/uninstall", @@ -38,6 +36,13 @@ OK: wheel uninstalled, cluster restarted ] } } +{ + "method": "POST", + "path": "/api/2.1/clusters/restart", + "body": { + "cluster_id": "[UUID]" + } +} >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/resources/clusters/libraries/script b/acceptance/bundle/resources/clusters/libraries/script index 57484a0a77b..576d5af03b0 100644 --- a/acceptance/bundle/resources/clusters/libraries/script +++ b/acceptance/bundle/resources/clusters/libraries/script @@ -19,6 +19,5 @@ trace print_requests.py //libraries/install title "Removing the wheel and redeploying uninstalls it and restarts the running cluster" update_file.py databricks.yml " - whl: ./dist/*.whl" "" $CLI bundle deploy &> LOG.redeploy -cat LOG.redeploy | contains.py "Updated clusters.mycluster" "Restarting cluster" > /dev/null -printf '\n%s\n' "OK: wheel uninstalled, cluster restarted" -trace print_requests.py //libraries/uninstall +cat LOG.redeploy | contains.py "Updated clusters.mycluster" > /dev/null +trace print_requests.py //libraries/uninstall //clusters/restart From e25a954602ce1d633ce0a8b00ca395aeb1973aea Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 8 Sep 2026 12:37:47 +0000 Subject: [PATCH 54/55] Stop snapshotting the plan file in the readplan tripwire out.plan.json was committed as an out* golden, but its plan_version field is not identical across the DMS env-matrix variants, so the DMS=true variant failed the golden comparison. Write the plan to tmp.plan.json and ignore it, matching the convention used by the other plan-based tests; the bug proof lives in output.txt, not the plan file. Co-authored-by: Isaac --- .../out.plan.json | 23 ------------------- .../output.txt | 2 +- .../libraries-readplan-not-uploaded/script | 4 ++-- .../libraries-readplan-not-uploaded/test.toml | 2 +- 4 files changed, 4 insertions(+), 27 deletions(-) delete mode 100644 acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/out.plan.json diff --git a/acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/out.plan.json b/acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/out.plan.json deleted file mode 100644 index 2c5ab71c952..00000000000 --- a/acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/out.plan.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "plan_version": 2, - "cli_version": "[CLI_VERSION]", - "plan": { - "resources.clusters.mycluster": { - "action": "create", - "new_state": { - "value": { - "autotermination_minutes": 60, - "cluster_name": "mycluster", - "libraries": [ - { - "whl": "/Workspace/Users/[USERNAME]/.bundle/libraries-readplan-not-uploaded/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" - } - ], - "node_type_id": "Standard_DS3_v2", - "num_workers": 1, - "spark_version": "15.4.x-scala2.12" - } - } - } - } -} diff --git a/acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/output.txt b/acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/output.txt index 457d7f944cc..bbf64e49796 100644 --- a/acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/output.txt +++ b/acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/output.txt @@ -1,7 +1,7 @@ >>> [CLI] bundle plan -o json ->>> [CLI] bundle deploy --plan out.plan.json +>>> [CLI] bundle deploy --plan tmp.plan.json Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/libraries-readplan-not-uploaded/default/files... Created clusters.mycluster Files: 6 uploaded, 0 deleted diff --git a/acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/script b/acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/script index 7cf4c340482..8fff35358c4 100644 --- a/acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/script +++ b/acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/script @@ -2,6 +2,6 @@ # `bundle deploy --plan`. This tripwire pins that difference. # Can be removed once fixed, and other tests can be moved to READPLAN variants. #See acceptance/bundle/deploy/readplan/whl-not-uploaded for the bug. -trace $CLI bundle plan -o json > out.plan.json -trace $CLI bundle deploy --plan out.plan.json +trace $CLI bundle plan -o json > tmp.plan.json +trace $CLI bundle deploy --plan tmp.plan.json trace $CLI bundle destroy --auto-approve diff --git a/acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/test.toml b/acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/test.toml index 38aff562e99..764f1385977 100644 --- a/acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/test.toml +++ b/acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/test.toml @@ -3,4 +3,4 @@ Badness = "bundle deploy --plan skips the build phase, so a cluster's local whee EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] # This test is just asserting bad behaviour so that it can be removed once that behaviour is fixed RecordRequests = false -Ignore = [".databricks"] +Ignore = [".databricks", "tmp.plan.json"] From 18166f7bf1225f25d4a8dfe35aeca430d1766e17 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 8 Sep 2026 13:00:57 +0000 Subject: [PATCH 55/55] increase timeout for test --- .../bundle/resources/clusters/libraries-restart/test.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/acceptance/bundle/resources/clusters/libraries-restart/test.toml b/acceptance/bundle/resources/clusters/libraries-restart/test.toml index 24d1138996d..472e3f3459e 100644 --- a/acceptance/bundle/resources/clusters/libraries-restart/test.toml +++ b/acceptance/bundle/resources/clusters/libraries-restart/test.toml @@ -1,5 +1,8 @@ Cloud = true CloudSlow = true +# This test runs three full cluster lifecycles (create, add-library restart, remove-library +# restart); on slower clouds that exceeds the default 25m cloud timeout, so give it 50m. +TimeoutCloud = '50m' EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"]