Add libraries field to the clusters resource

.nextchanges/bundles/cluster-libraries.md

@@ -0,0 +1 @@
+* Add libraries field to clusters. ([#6365](https://github.com/databricks/cli/pull/6365))

acceptance/bundle/invariant/configs/cluster_libraries.yml.tmpl

@@ -0,0 +1,14 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+resources:
+  clusters:
+    foo:
+      cluster_name: test-cluster-$UNIQUE_NAME
+      spark_version: 13.3.x-scala2.12
+      node_type_id: $NODE_TYPE_ID
+      instance_pool_id: $TEST_INSTANCE_POOL_ID
+      num_workers: 1
+      libraries:
+        - pypi:
+            package: requests

acceptance/bundle/invariant/continue_293/test.toml

@@ -23,6 +23,9 @@ EnvMatrixExclude.no_instance_pool = ["INPUT_CONFIG=instance_pool.yml.tmpl"]
 # cluster_policies resource is not supported on v0.293.0
 EnvMatrixExclude.no_cluster_policy = ["INPUT_CONFIG=cluster_policy.yml.tmpl"]
 
+# cluster libraries (clusters.libraries) are not supported on v0.293.0
+EnvMatrixExclude.no_cluster_libraries = ["INPUT_CONFIG=cluster_libraries.yml.tmpl"]
+
 # job_runs resource is not supported on v0.293.0
 EnvMatrixExclude.no_job_run = ["INPUT_CONFIG=job_run.yml.tmpl"]
 

acceptance/bundle/invariant/migrate/test.toml

@@ -21,6 +21,8 @@ EnvMatrixExclude.no_genie_space = ["INPUT_CONFIG=genie_space.yml.tmpl"]
 EnvMatrixExclude.no_instance_pool = ["INPUT_CONFIG=instance_pool.yml.tmpl"]
 # Cluster policies are direct-only; the terraform deploy that seeds the migration fails for them.
 EnvMatrixExclude.no_cluster_policy = ["INPUT_CONFIG=cluster_policy.yml.tmpl"]
+# Cluster libraries are direct-only; the terraform deploy that seeds the migration fails for them.
+EnvMatrixExclude.no_cluster_libraries = ["INPUT_CONFIG=cluster_libraries.yml.tmpl"]
 
 # Cross-resource permission references (e.g. ${resources.jobs.job_b.permissions[0].level})
 # don't work in terraform mode: the terraform interpolator converts the path to

acceptance/bundle/invariant/test.toml

@@ -25,6 +25,7 @@ EnvMatrix.INPUT_CONFIG = [
     "catalog_optional_fields.yml.tmpl",
     "cluster.yml.tmpl",
     "cluster_apply_policy_default_values.yml.tmpl",
+    "cluster_libraries.yml.tmpl",
     "cluster_policy.yml.tmpl",
     "dashboard.yml.tmpl",
     "job_apply_policy_default_values_job_cluster.yml.tmpl",

acceptance/bundle/refschema/out.fields.txt

@@ -461,6 +461,23 @@ resources.clusters.*.jdbc_port	int	REMOTE
 resources.clusters.*.kind	compute.Kind	ALL
 resources.clusters.*.last_restarted_time	int64	REMOTE
 resources.clusters.*.last_state_loss_time	int64	REMOTE
+resources.clusters.*.libraries	[]compute.Library	ALL
+resources.clusters.*.libraries[*]	compute.Library	ALL
+resources.clusters.*.libraries[*].cran	*compute.RCranLibrary	ALL
+resources.clusters.*.libraries[*].cran.package	string	ALL
+resources.clusters.*.libraries[*].cran.repo	string	ALL
+resources.clusters.*.libraries[*].egg	string	ALL
+resources.clusters.*.libraries[*].jar	string	ALL
+resources.clusters.*.libraries[*].maven	*compute.MavenLibrary	ALL
+resources.clusters.*.libraries[*].maven.coordinates	string	ALL
+resources.clusters.*.libraries[*].maven.exclusions	[]string	ALL
+resources.clusters.*.libraries[*].maven.exclusions[*]	string	ALL
+resources.clusters.*.libraries[*].maven.repo	string	ALL
+resources.clusters.*.libraries[*].pypi	*compute.PythonPyPiLibrary	ALL
+resources.clusters.*.libraries[*].pypi.package	string	ALL
+resources.clusters.*.libraries[*].pypi.repo	string	ALL
+resources.clusters.*.libraries[*].requirements	string	ALL
+resources.clusters.*.libraries[*].whl	string	ALL
 resources.clusters.*.lifecycle	*dresources.StateLifecycle	REMOTE	STATE
 resources.clusters.*.lifecycle	*resources.LifecycleWithStarted	INPUT
 resources.clusters.*.lifecycle	resources.Lifecycle	INPUT

acceptance/bundle/resources/clusters/libraries-content-change/databricks.yml.tmpl

@@ -0,0 +1,23 @@
+bundle:
+  name: cluster-libraries-$UNIQUE_NAME
+
+workspace:
+  root_path: ~/.bundle/$UNIQUE_NAME
+
+artifacts:
+  my_test_code:
+    type: whl
+    path: .
+    # Use 'python' because 'python3' does not exist in Windows virtualenvs.
+    build: python setup.py bdist_wheel
+
+resources:
+  clusters:
+    mycluster:
+      cluster_name: mycluster-$UNIQUE_NAME
+      spark_version: $DEFAULT_SPARK_VERSION
+      node_type_id: $NODE_TYPE_ID
+      num_workers: 1
+      data_security_mode: $DATA_SECURITY_MODE
+      libraries:
+        - whl: ./dist/*.whl

acceptance/bundle/resources/clusters/libraries-content-change/my_test_code/__init__.py

@@ -0,0 +1,2 @@
+__version__ = "0.0.1"
+__author__ = "Databricks"

acceptance/bundle/resources/clusters/libraries-content-change/my_test_code/__main__.py

@@ -0,0 +1,16 @@
+"""
+The entry point of the Python Wheel
+"""
+
+import sys
+
+
+def main():
+    # This method will print the provided arguments
+    print("Hello from my func")
+    print("Got arguments:")
+    print(sys.argv)
+
+
+if __name__ == "__main__":
+    main()

acceptance/bundle/resources/clusters/libraries-content-change/output.txt

@@ -0,0 +1,30 @@
+
+=== Deploy a cluster with a local wheel library
+OK: cluster created with wheel
+
+>>> print_requests.py //libraries/install
+{
+  "method": "POST",
+  "path": "/api/2.0/libraries/install",
+  "body": {
+    "cluster_id": "[UUID]",
+    "libraries": [
+      {
+        "whl": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl"
+      }
+    ]
+  }
+}
+
+=== Badness: wheel code changes but version does not, so redeploy neither reinstalls nor restarts
+OK: no restart on content-only change (known limitation)
+
+>>> print_requests.py //libraries/install
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.clusters.mycluster
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]
+
+Destroy: 1 deleted

acceptance/bundle/resources/clusters/libraries-content-change/script

@@ -0,0 +1,27 @@
+export DATA_SECURITY_MODE=USER_ISOLATION
+envsubst < databricks.yml.tmpl > databricks.yml
+
+cleanup() {
+    trace $CLI bundle destroy --auto-approve
+    rm -f out.requests.txt
+}
+trap cleanup EXIT
+
+# Deploy output goes to LOG (differs fake vs cloud); echo a pass line instead.
+title "Deploy a cluster with a local wheel library"
+$CLI bundle deploy &> LOG.deploy
+cat LOG.deploy | contains.py "Created clusters.mycluster" > /dev/null
+printf '\n%s\n' "OK: cluster created with wheel"
+trace print_requests.py //libraries/install
+
+# Change only the wheel's code, keeping version 0.0.1 so the built filename - and
+# thus the uploaded workspace path - is byte-for-byte identical to the first deploy.
+title "Badness: wheel code changes but version does not, so redeploy neither reinstalls nor restarts"
+update_file.py my_test_code/__main__.py "Hello from my func" "Hello from my func v2"
+$CLI bundle deploy &> LOG.redeploy
+# The libraries path is unchanged, so the plan sees no library change: the cluster is
+# not restarted and no install request is issued, leaving the stale wheel running. The
+# following two assertions encode that known limitation (see the TODO in cluster.go).
+cat LOG.redeploy | contains.py "!Restarting cluster" > /dev/null
+printf '\n%s\n' "OK: no restart on content-only change (known limitation)"
+trace print_requests.py //libraries/install

acceptance/bundle/resources/clusters/libraries-content-change/script.prepare

@@ -0,0 +1,3 @@
+uv venv -q .venv
+venv_activate
+uv pip install -q --no-index setuptools

acceptance/bundle/resources/clusters/libraries-content-change/setup.py

@@ -0,0 +1,15 @@
+from setuptools import setup, find_packages
+
+import my_test_code
+
+setup(
+    name="my_test_code",
+    version=my_test_code.__version__,
+    author=my_test_code.__author__,
+    url="https://databricks.com",
+    author_email="john.doe@databricks.com",
+    description="my example wheel",
+    packages=find_packages(include=["my_test_code"]),
+    entry_points={"group1": "run=my_test_code.__main__:main"},
+    install_requires=["setuptools"],
+)

acceptance/bundle/resources/clusters/libraries-content-change/test.toml

@@ -0,0 +1,13 @@
+Badness = "A local wheel whose contents change but whose version (and thus workspace path) does not is not detected in non-dev mode, so redeploy neither reinstalls it nor restarts the cluster; the stale wheel keeps running. See the TODO in bundle/direct/dresources/cluster.go DoUpdate."
+
+Cloud = true
+CloudSlow = true
+RecordRequests = true
+
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]
+
+Ignore = [".databricks", ".venv", "build", "dist", "my_test_code.egg-info", "databricks.yml"]
+
+[[Repls]]
+Old = "[0-9]{4}-[0-9]{6}-[0-9a-z]{8}"
+New = "[UUID]"

acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/databricks.yml

@@ -0,0 +1,13 @@
+bundle:
+  name: libraries-readplan-not-uploaded
+
+resources:
+  clusters:
+    mycluster:
+      cluster_name: mycluster
+      spark_version: 15.4.x-scala2.12
+      node_type_id: Standard_DS3_v2
+      num_workers: 1
+      libraries:
+        # Prebuilt local wheel (no build: step) to keep the test hermetic.
+        - whl: ./dist/*.whl

acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/output.txt

@@ -0,0 +1,16 @@
+
+>>> [CLI] bundle plan -o json
+
+>>> [CLI] bundle deploy --plan tmp.plan.json
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/libraries-readplan-not-uploaded/default/files...
+Created clusters.mycluster
+Files: 6 uploaded, 0 deleted
+Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.clusters.mycluster
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/libraries-readplan-not-uploaded/default
+
+Destroy: 1 deleted

acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/script

@@ -0,0 +1,7 @@
+# A cluster's local wheel library is uploaded by `bundle deploy` but NOT by
+# `bundle deploy --plan`. This tripwire pins that difference.
+# Can be removed once fixed, and other tests can be moved to READPLAN variants.
+#See acceptance/bundle/deploy/readplan/whl-not-uploaded for the bug.
+trace $CLI bundle plan -o json > tmp.plan.json
+trace $CLI bundle deploy --plan tmp.plan.json
+trace $CLI bundle destroy --auto-approve

acceptance/bundle/resources/clusters/libraries-readplan-not-uploaded/test.toml

@@ -0,0 +1,6 @@
+Badness = "bundle deploy --plan skips the build phase, so a cluster's local wheel is never uploaded (proven in deploy/readplan/whl-not-uploaded). Kept as a note: when the bug is fixed, deploy --plan will print 'Uploading dist/...' and this golden changes - then delete this test and restore EnvMatrix.READPLAN = [\"\", \"1\"] on clusters/libraries and clusters/libraries-content-change."
+
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]
+# This test is just asserting bad behaviour so that it can be removed once that behaviour is fixed
+RecordRequests = false
+Ignore = [".databricks", "tmp.plan.json"]

acceptance/bundle/resources/clusters/libraries-repo/databricks.yml.tmpl

@@ -0,0 +1,18 @@
+bundle:
+  name: cluster-libraries-repo-$UNIQUE_NAME
+
+workspace:
+  root_path: ~/.bundle/$UNIQUE_NAME
+
+resources:
+  clusters:
+    mycluster:
+      cluster_name: mycluster-$UNIQUE_NAME
+      spark_version: $DEFAULT_SPARK_VERSION
+      node_type_id: $NODE_TYPE_ID
+      instance_pool_id: $TEST_INSTANCE_POOL_ID
+      num_workers: 1
+      libraries:
+        - pypi:
+            package: requests
+            repo: https://pypi.org/simple

acceptance/bundle/resources/clusters/libraries-repo/output.txt

@@ -0,0 +1,21 @@
+
+=== Deploy creates the cluster and installs the pypi library
+OK: cluster created with pypi library
+
+=== Plan is a no-op after deploy: the repo round-trips
+>>> [CLI] bundle plan
+Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
+
+=== Changing only the repo is planned as an update
+>>> [CLI] bundle plan
+update clusters.mycluster
+
+Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.clusters.mycluster
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]
+
+Destroy: 1 deleted

acceptance/bundle/resources/clusters/libraries-repo/script

@@ -0,0 +1,21 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+
+cleanup() {
+    trace $CLI bundle destroy --auto-approve
+    rm -f out.requests.txt
+}
+trap cleanup EXIT
+
+# Deploy output goes to LOG (differs fake vs cloud); echo a pass line instead.
+title "Deploy creates the cluster and installs the pypi library"
+$CLI bundle plan -o json > tmp.plan.json 2>>LOG.plan
+$CLI bundle deploy $(readplanarg tmp.plan.json) &> LOG.deploy
+cat LOG.deploy | contains.py "Created clusters.mycluster" > /dev/null
+printf '\n%s\n' "OK: cluster created with pypi library"
+
+title "Plan is a no-op after deploy: the repo round-trips"
+trace $CLI bundle plan
+
+title "Changing only the repo is planned as an update"
+update_file.py databricks.yml "repo: https://pypi.org/simple" "repo: https://example.invalid/simple"
+trace $CLI bundle plan

acceptance/bundle/resources/clusters/libraries-repo/test.toml

@@ -0,0 +1,12 @@
+Cloud = true
+CloudSlow = true
+RecordRequests = false
+
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]
+EnvMatrix.READPLAN = ["", "1"]
+
+Ignore = [".databricks", "databricks.yml", "tmp.plan.json"]
+
+[[Repls]]
+Old = "[0-9]{4}-[0-9]{6}-[0-9a-z]{8}"
+New = "[UUID]"

acceptance/bundle/resources/clusters/libraries-restart/databricks.yml.tmpl

@@ -0,0 +1,17 @@
+bundle:
+  name: cluster-libraries-restart-$UNIQUE_NAME
+
+workspace:
+  root_path: ~/.bundle/$UNIQUE_NAME
+
+resources:
+  clusters:
+    mycluster:
+      cluster_name: mycluster-$UNIQUE_NAME
+      spark_version: $DEFAULT_SPARK_VERSION
+      node_type_id: $NODE_TYPE_ID
+      instance_pool_id: $TEST_INSTANCE_POOL_ID
+      num_workers: 1
+      libraries:
+        - pypi:
+            package: requests