diff --git a/.nextchanges/bundles/serverless-environment-version-v5.md b/.nextchanges/bundles/serverless-environment-version-v5.md new file mode 100644 index 00000000000..5cb280deaf6 --- /dev/null +++ b/.nextchanges/bundles/serverless-environment-version-v5.md @@ -0,0 +1 @@ +Bundle templates now use serverless [environment version 5](https://docs.databricks.com/aws/en/release-notes/serverless/environment-version/five), which offers better performance, and `databricks-connect` 16.4. diff --git a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/resources/my_dbt_sql.job.yml b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/resources/my_dbt_sql.job.yml index d01c17860a1..9cb376adf5c 100644 --- a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/resources/my_dbt_sql.job.yml +++ b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/resources/my_dbt_sql.job.yml @@ -29,6 +29,6 @@ resources: environments: - environment_key: default spec: - environment_version: "4" + environment_version: "5" dependencies: - dbt-databricks>=1.8.0,<2.0.0 diff --git a/acceptance/bundle/templates/default-minimal/python/output/my_default_minimal/pyproject.toml b/acceptance/bundle/templates/default-minimal/python/output/my_default_minimal/pyproject.toml index 4678f9c4368..9d08d0e9641 100644 --- a/acceptance/bundle/templates/default-minimal/python/output/my_default_minimal/pyproject.toml +++ b/acceptance/bundle/templates/default-minimal/python/output/my_default_minimal/pyproject.toml @@ -2,7 +2,7 @@ name = "my_default_minimal" version = "0.0.1" authors = [{ name = "[USERNAME]" }] -requires-python = ">=3.10,<3.13" +requires-python = ">=3.12,<3.13" dependencies = [ # Any dependencies for jobs and pipelines in this project can be added here # See also https://docs.databricks.com/dev-tools/bundles/library-dependencies @@ -16,7 +16,7 @@ dev = [ "pytest", "ruff", "databricks-dlt", - "databricks-connect>=15.4,<15.5", + "databricks-connect>=16.4,<16.5", "ipykernel", ] diff --git a/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff b/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff index 35c0d8bea6e..edcb07b0fab 100644 --- a/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff +++ b/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff @@ -51,7 +51,7 @@ - environments: - - environment_key: default - spec: -- environment_version: "4" +- environment_version: "5" - dependencies: - # By default we just include the .whl file generated for the my_default_python package. - # See https://docs.databricks.com/dev-tools/bundles/library-dependencies.html diff --git a/acceptance/bundle/templates/default-python/classic/output/my_default_python/pyproject.toml b/acceptance/bundle/templates/default-python/classic/output/my_default_python/pyproject.toml index 3c4c2062763..e8bb641abd4 100644 --- a/acceptance/bundle/templates/default-python/classic/output/my_default_python/pyproject.toml +++ b/acceptance/bundle/templates/default-python/classic/output/my_default_python/pyproject.toml @@ -2,7 +2,7 @@ name = "my_default_python" version = "0.0.1" authors = [{ name = "[USERNAME]" }] -requires-python = ">=3.10,<3.13" +requires-python = ">=3.12,<3.13" dependencies = [ # Any dependencies for jobs and pipelines in this project can be added here # See also https://docs.databricks.com/dev-tools/bundles/library-dependencies @@ -16,7 +16,7 @@ dev = [ "pytest", "ruff", "databricks-dlt", - "databricks-connect>=15.4,<15.5", + "databricks-connect>=16.4,<16.5", "ipykernel", ] diff --git a/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/sample_notebook.ipynb b/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/sample_notebook.ipynb index d9e9805e869..49bbd5935d7 100644 --- a/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/sample_notebook.ipynb +++ b/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/sample_notebook.ipynb @@ -72,7 +72,7 @@ "dependencies": [ "--editable .." ], - "environment_version": "4" + "environment_version": "5" }, "language": "python", "notebookMetadata": { diff --git a/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_dev.direct.json b/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_dev.direct.json index 992402b9e54..9967e015883 100644 --- a/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_dev.direct.json +++ b/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_dev.direct.json @@ -28,7 +28,7 @@ "dependencies": [ "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1-py3-none-any.whl" ], - "environment_version": "4" + "environment_version": "5" } } ], diff --git a/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_prod.direct.json b/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_prod.direct.json index 6cb1b237e85..57a12ae5bb3 100644 --- a/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_prod.direct.json +++ b/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_prod.direct.json @@ -28,7 +28,7 @@ "dependencies": [ "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/artifacts/.internal/my_default_python-0.0.1-py3-none-any.whl" ], - "environment_version": "4" + "environment_version": "5" } } ], diff --git a/acceptance/bundle/templates/default-python/serverless/out.plan_dev.direct.json b/acceptance/bundle/templates/default-python/serverless/out.plan_dev.direct.json index 0c5b608231f..dd0e402c7df 100644 --- a/acceptance/bundle/templates/default-python/serverless/out.plan_dev.direct.json +++ b/acceptance/bundle/templates/default-python/serverless/out.plan_dev.direct.json @@ -24,7 +24,7 @@ "dependencies": [ "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1-py3-none-any.whl" ], - "environment_version": "4" + "environment_version": "5" } } ], diff --git a/acceptance/bundle/templates/default-python/serverless/out.plan_prod.direct.json b/acceptance/bundle/templates/default-python/serverless/out.plan_prod.direct.json index ef6d96a24b7..a9db4ff38f4 100644 --- a/acceptance/bundle/templates/default-python/serverless/out.plan_prod.direct.json +++ b/acceptance/bundle/templates/default-python/serverless/out.plan_prod.direct.json @@ -24,7 +24,7 @@ "dependencies": [ "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/artifacts/.internal/my_default_python-0.0.1-py3-none-any.whl" ], - "environment_version": "4" + "environment_version": "5" } } ], diff --git a/acceptance/bundle/templates/default-python/serverless/out.requests.dev.direct.txt b/acceptance/bundle/templates/default-python/serverless/out.requests.dev.direct.txt index bd7882a8b87..55dae66f070 100644 --- a/acceptance/bundle/templates/default-python/serverless/out.requests.dev.direct.txt +++ b/acceptance/bundle/templates/default-python/serverless/out.requests.dev.direct.txt @@ -111,7 +111,7 @@ "dependencies": [ "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1-py3-none-any.whl" ], - "environment_version": "4" + "environment_version": "5" } } ], diff --git a/acceptance/bundle/templates/default-python/serverless/out.requests.dev.terraform.txt b/acceptance/bundle/templates/default-python/serverless/out.requests.dev.terraform.txt index 3a31f123f60..a8cf0acc2d5 100644 --- a/acceptance/bundle/templates/default-python/serverless/out.requests.dev.terraform.txt +++ b/acceptance/bundle/templates/default-python/serverless/out.requests.dev.terraform.txt @@ -111,7 +111,7 @@ "dependencies": [ "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1-py3-none-any.whl" ], - "environment_version": "4" + "environment_version": "5" } } ], diff --git a/acceptance/bundle/templates/default-python/serverless/out.requests.prod.direct.txt b/acceptance/bundle/templates/default-python/serverless/out.requests.prod.direct.txt index d4162173ae9..b5e1fda37a3 100644 --- a/acceptance/bundle/templates/default-python/serverless/out.requests.prod.direct.txt +++ b/acceptance/bundle/templates/default-python/serverless/out.requests.prod.direct.txt @@ -114,7 +114,7 @@ "dependencies": [ "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/artifacts/.internal/my_default_python-0.0.1-py3-none-any.whl" ], - "environment_version": "4" + "environment_version": "5" } } ], diff --git a/acceptance/bundle/templates/default-python/serverless/out.requests.prod.terraform.txt b/acceptance/bundle/templates/default-python/serverless/out.requests.prod.terraform.txt index a7f84a6979b..e667d80971d 100644 --- a/acceptance/bundle/templates/default-python/serverless/out.requests.prod.terraform.txt +++ b/acceptance/bundle/templates/default-python/serverless/out.requests.prod.terraform.txt @@ -114,7 +114,7 @@ "dependencies": [ "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/artifacts/.internal/my_default_python-0.0.1-py3-none-any.whl" ], - "environment_version": "4" + "environment_version": "5" } } ], diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/pyproject.toml b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/pyproject.toml index 3c4c2062763..e8bb641abd4 100644 --- a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/pyproject.toml +++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/pyproject.toml @@ -2,7 +2,7 @@ name = "my_default_python" version = "0.0.1" authors = [{ name = "[USERNAME]" }] -requires-python = ">=3.10,<3.13" +requires-python = ">=3.12,<3.13" dependencies = [ # Any dependencies for jobs and pipelines in this project can be added here # See also https://docs.databricks.com/dev-tools/bundles/library-dependencies @@ -16,7 +16,7 @@ dev = [ "pytest", "ruff", "databricks-dlt", - "databricks-connect>=15.4,<15.5", + "databricks-connect>=16.4,<16.5", "ipykernel", ] diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/sample_job.job.yml b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/sample_job.job.yml index c64690910f4..16716edb640 100644 --- a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/sample_job.job.yml +++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/sample_job.job.yml @@ -46,7 +46,7 @@ resources: environments: - environment_key: default spec: - environment_version: "4" + environment_version: "5" dependencies: # By default we just include the .whl file generated for the my_default_python package. # See https://docs.databricks.com/dev-tools/bundles/library-dependencies.html diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/sample_notebook.ipynb b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/sample_notebook.ipynb index d9e9805e869..49bbd5935d7 100644 --- a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/sample_notebook.ipynb +++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/sample_notebook.ipynb @@ -72,7 +72,7 @@ "dependencies": [ "--editable .." ], - "environment_version": "4" + "environment_version": "5" }, "language": "python", "notebookMetadata": { diff --git a/acceptance/bundle/templates/default-scala/output/my_default_scala/resources/my_default_scala.job.yml b/acceptance/bundle/templates/default-scala/output/my_default_scala/resources/my_default_scala.job.yml index f3c9332d16f..97d9e9bbbd4 100644 --- a/acceptance/bundle/templates/default-scala/output/my_default_scala/resources/my_default_scala.job.yml +++ b/acceptance/bundle/templates/default-scala/output/my_default_scala/resources/my_default_scala.job.yml @@ -22,6 +22,6 @@ resources: environments: - environment_key: default spec: - environment_version: "4" + environment_version: "5" java_dependencies: - ${workspace.artifact_path}/.internal/my_default_scala-assembly-0.1.jar diff --git a/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/pyproject.toml b/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/pyproject.toml index e3e451122da..62b9edf788e 100644 --- a/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/pyproject.toml +++ b/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/pyproject.toml @@ -2,7 +2,7 @@ name = "my_lakeflow_pipelines" version = "0.0.1" authors = [{ name = "[USERNAME]" }] -requires-python = ">=3.10,<3.13" +requires-python = ">=3.12,<3.13" dependencies = [ # Any dependencies for jobs and pipelines in this project can be added here # See also https://docs.databricks.com/dev-tools/bundles/library-dependencies @@ -16,7 +16,7 @@ dev = [ "pytest", "ruff", "databricks-dlt", - "databricks-connect>=15.4,<15.5", + "databricks-connect>=16.4,<16.5", "ipykernel", ] diff --git a/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/resources/sample_job.job.yml b/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/resources/sample_job.job.yml index 98800715652..aa5204356ea 100644 --- a/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/resources/sample_job.job.yml +++ b/acceptance/bundle/templates/lakeflow-pipelines/python/output/my_lakeflow_pipelines/resources/sample_job.job.yml @@ -29,4 +29,4 @@ resources: environments: - environment_key: default spec: - environment_version: "4" + environment_version: "5" diff --git a/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/resources/sample_job.job.yml b/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/resources/sample_job.job.yml index 98800715652..aa5204356ea 100644 --- a/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/resources/sample_job.job.yml +++ b/acceptance/bundle/templates/lakeflow-pipelines/sql/output/my_lakeflow_pipelines/resources/sample_job.job.yml @@ -29,4 +29,4 @@ resources: environments: - environment_key: default spec: - environment_version: "4" + environment_version: "5" diff --git a/acceptance/bundle/templates/pydabs/init-classic/output/my_pydabs/pyproject.toml b/acceptance/bundle/templates/pydabs/init-classic/output/my_pydabs/pyproject.toml index 184b495a64a..bd5be00a97a 100644 --- a/acceptance/bundle/templates/pydabs/init-classic/output/my_pydabs/pyproject.toml +++ b/acceptance/bundle/templates/pydabs/init-classic/output/my_pydabs/pyproject.toml @@ -2,7 +2,7 @@ name = "my_pydabs" version = "0.0.1" authors = [{ name = "[USERNAME]" }] -requires-python = ">=3.10,<3.13" +requires-python = ">=3.12,<3.13" dependencies = [ # Any dependencies for jobs and pipelines in this project can be added here # See also https://docs.databricks.com/dev-tools/bundles/library-dependencies @@ -16,7 +16,7 @@ dev = [ "pytest", "ruff", "databricks-dlt", - "databricks-connect>=15.4,<15.5", + "databricks-connect>=16.4,<16.5", "ipykernel", "databricks-bundles==0.0.0", ] diff --git a/acceptance/pipelines/e2e/output/lakeflow_project/pyproject.toml b/acceptance/pipelines/e2e/output/lakeflow_project/pyproject.toml index 39353902f6b..141ceddab2e 100644 --- a/acceptance/pipelines/e2e/output/lakeflow_project/pyproject.toml +++ b/acceptance/pipelines/e2e/output/lakeflow_project/pyproject.toml @@ -2,7 +2,7 @@ name = "lakeflow_project" version = "0.0.1" authors = [{ name = "[USERNAME]" }] -requires-python = ">=3.10,<3.13" +requires-python = ">=3.12,<3.13" dependencies = [ # Any dependencies for jobs and pipelines in this project can be added here # See also https://docs.databricks.com/dev-tools/bundles/library-dependencies @@ -16,7 +16,7 @@ dev = [ "pytest", "ruff", "databricks-dlt", - "databricks-connect>=15.4,<15.5", + "databricks-connect>=16.4,<16.5", "ipykernel", ] diff --git a/acceptance/pipelines/e2e/output/lakeflow_project/resources/sample_job.job.yml b/acceptance/pipelines/e2e/output/lakeflow_project/resources/sample_job.job.yml index 2ba45ba9ffe..86f5510b339 100644 --- a/acceptance/pipelines/e2e/output/lakeflow_project/resources/sample_job.job.yml +++ b/acceptance/pipelines/e2e/output/lakeflow_project/resources/sample_job.job.yml @@ -29,4 +29,4 @@ resources: environments: - environment_key: default spec: - environment_version: "4" + environment_version: "5" diff --git a/acceptance/pipelines/init/python/output/my_python_project/pyproject.toml b/acceptance/pipelines/init/python/output/my_python_project/pyproject.toml index d75a0a4caa8..7b8932736c6 100644 --- a/acceptance/pipelines/init/python/output/my_python_project/pyproject.toml +++ b/acceptance/pipelines/init/python/output/my_python_project/pyproject.toml @@ -2,7 +2,7 @@ name = "my_python_project" version = "0.0.1" authors = [{ name = "[USERNAME]" }] -requires-python = ">=3.10,<3.13" +requires-python = ">=3.12,<3.13" dependencies = [ # Any dependencies for jobs and pipelines in this project can be added here # See also https://docs.databricks.com/dev-tools/bundles/library-dependencies @@ -16,7 +16,7 @@ dev = [ "pytest", "ruff", "databricks-dlt", - "databricks-connect>=15.4,<15.5", + "databricks-connect>=16.4,<16.5", "ipykernel", ] diff --git a/acceptance/pipelines/init/python/output/my_python_project/resources/sample_job.job.yml b/acceptance/pipelines/init/python/output/my_python_project/resources/sample_job.job.yml index 788216eaadb..58ffd15561f 100644 --- a/acceptance/pipelines/init/python/output/my_python_project/resources/sample_job.job.yml +++ b/acceptance/pipelines/init/python/output/my_python_project/resources/sample_job.job.yml @@ -29,4 +29,4 @@ resources: environments: - environment_key: default spec: - environment_version: "4" + environment_version: "5" diff --git a/acceptance/pipelines/init/sql/output/my_sql_project/resources/sample_job.job.yml b/acceptance/pipelines/init/sql/output/my_sql_project/resources/sample_job.job.yml index 1e1767f2f38..ebb65a80c2b 100644 --- a/acceptance/pipelines/init/sql/output/my_sql_project/resources/sample_job.job.yml +++ b/acceptance/pipelines/init/sql/output/my_sql_project/resources/sample_job.job.yml @@ -29,4 +29,4 @@ resources: environments: - environment_key: default spec: - environment_version: "4" + environment_version: "5" diff --git a/libs/template/templates/dbt-sql/library/versions.tmpl b/libs/template/templates/dbt-sql/library/versions.tmpl index cb393461f3f..7c50b969d44 100644 --- a/libs/template/templates/dbt-sql/library/versions.tmpl +++ b/libs/template/templates/dbt-sql/library/versions.tmpl @@ -2,13 +2,14 @@ 15.4.x-scala2.12 {{- end}} +{{/* TODO: unused — no dbt-sql template references this macro. Remove it. */}} {{define "latest_lts_db_connect_version_spec" -}} - >=15.4,<15.5 + >=16.4,<16.5 {{- end}} {{/* The serverless environment version to use. * See https://docs.databricks.com/aws/en/release-notes/serverless/environment-version/ */}} {{define "serverless_environment_version" -}} - 4 + 5 {{- end}} diff --git a/libs/template/templates/default-scala/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl b/libs/template/templates/default-scala/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl index dfc0fccf03a..f700ff1811a 100644 --- a/libs/template/templates/default-scala/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl +++ b/libs/template/templates/default-scala/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl @@ -31,7 +31,7 @@ resources: environments: - environment_key: default spec: - environment_version: "4" + environment_version: "5" java_dependencies: - ${workspace.artifact_path}/.internal/{{.project_name}}-assembly-{{template `version` .}}.jar {{- else}} diff --git a/libs/template/templates/default/library/versions.tmpl b/libs/template/templates/default/library/versions.tmpl index d0499ad5e7b..c3a6bed9297 100644 --- a/libs/template/templates/default/library/versions.tmpl +++ b/libs/template/templates/default/library/versions.tmpl @@ -5,40 +5,40 @@ 16.4.x-scala2.12 {{- end}} -{{/* A safe version of DB Connect that is compatible with at least half the - * clusters running in production. +{{/* A conservative version of DB Connect for local development. * - * We need to be very conservative in updating this, since a newer version can - * only connect to compute of that same version and higher. If the version is - * deemed too old, customers can update the version themselves after initializing - * the template. + * DB Connect is only forward-compatible: a client connects to compute of the + * same version and higher. We keep this conservative (low) so a freshly + * initialized project can reach the widest range of DBR versions; a high + * version would rule out customers on older DBR. We use 16.4 rather than an + * older release because DBR 15 is no longer supported and used Python 3.11, + * whereas 16.4 uses Python 3.12 (matching the serverless environment and the + * template's Python pin). Customers can move to a newer version themselves + * after initializing the template. * * See https://docs.databricks.com/dev-tools/databricks-connect/python/index.html#requirements * for DB Connect release notes and version compatibility. */}} {{define "conservative_db_connect_version_spec" -}} - >=15.4,<15.5 + >=16.4,<16.5 {{- end}} {{/* The serverless environment version to use. * See https://docs.databricks.com/release-notes/runtime/index.html for available versions. */}} {{define "serverless_environment_version" -}} - 4 + 5 {{- end}} {{/* Python version specification for the template. * This needs to be compatible with the version of DB Connect and the serverless_environment_version. * - * Serverless environment version 4 (and upcoming v5), DBR 16, and DBR 17 all use Python 3.12. - * DB Connect 15 supports all of these and is compatible with Python 3.10-3.12. - * - * We use >=3.10,<3.13 rather than pinning to 3.12 specifically because overly narrow pinning - * means users always need to use uv or similar tools to install the exact version. A broader - * range improves usability while maintaining compatibility with both DB Connect and serverless. + * Serverless environment versions 4 and 5, DBR 16, and DBR 17 all use Python 3.12. + * DB Connect 16.4 requires Python 3.12, so we pin the project to 3.12 to match both + * DB Connect and the serverless runtime. */}} {{define "python_version_spec" -}} - >=3.10,<3.13 + >=3.12,<3.13 {{- end}} {{/* The default Python version used in notebook metadata.