diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e4a3aca..3c193f57 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## Unreleased +- Kernel backend (`useKernel: true`): preserve qualified `INTERVAL MONTH` and + `INTERVAL DAY` parameter types on the SEA wire by using the kernel raw-parameter + path. - Kernel backend source builds (`useKernel: true`, built from `KERNEL_REV`): `getTypeInfo()` now matches the Thrift backend's canonical 18-column, 20-row type-info result. Customer-facing npm installs require a follow-up bump to a published native package containing this Kernel change. ([databricks-sql-kernel#291](https://github.com/databricks/databricks-sql-kernel/pull/291), PECOBLR-4166) - Kernel backend (`useKernel: true`): **Azure Entra (Azure AD) auth is now threaded through the kernel path.** On `authType: 'databricks-oauth'`: **U2M** (no secret) always routes to `OAuthU2m` — the kernel runs one cloud-blind in-house workspace-federated browser flow (it uses the workspace's OIDC-discovered authorize endpoint verbatim), which works against Azure workspaces, so Azure U2M forwards the in-house app (`databricks-sql-connector`) + `sql offline_access` scopes exactly like AWS/GCP, regardless of `useDatabricksOAuthInAzure` (verified E2E against a live Azure workspace). **M2M** (secret): `useDatabricksOAuthInAzure: true` (or non-Azure) → `OAuthM2m` (workspace-OIDC client-credentials); an Azure host with `useDatabricksOAuthInAzure` absent/`false` → the Entra-direct Azure service-principal M2M (`AzureSpM2m`, the Entra SP creds ride `oauthClientId`/`oauthClientSecret`, `azureTenantId` optional and auto-discovered when omitted). On a non-Azure host `useDatabricksOAuthInAzure` is inert. The `AzureSpM2m` path requires a `databricks-sql-kernel` native module that exposes the Azure SP surface — landed on `main` via [databricks-sql-kernel#282](https://github.com/databricks/databricks-sql-kernel/pull/282) (which the pinned `KERNEL_REV` `ef1a6f2` carries; the surface was originally proposed in [#280](https://github.com/databricks/databricks-sql-kernel/pull/280), which never reached `main`); U2M works on any kernel build. (PECOBLR-4141 / PECOBLR-4120) diff --git a/KERNEL_REV b/KERNEL_REV index 8d29e509..d89fea64 100644 --- a/KERNEL_REV +++ b/KERNEL_REV @@ -1 +1 @@ -628abd6f5045897efcadb38ec77a1e9e0c23544e +bb4a0770673926201d386d1d9295da8d4bf459aa diff --git a/bin/build-native.sh b/bin/build-native.sh index f4743b99..70961bda 100644 --- a/bin/build-native.sh +++ b/bin/build-native.sh @@ -5,6 +5,24 @@ set -euo pipefail driver_repo=$(pwd) kernel_repo=${DATABRICKS_SQL_KERNEL_REPO:-../../databricks-sql-kernel} napi_dir="${kernel_repo}/napi" +kernel_package_version=$( + node -e ' + const { optionalDependencies = {} } = require(process.argv[1]); + const versions = [...new Set( + Object.entries(optionalDependencies) + .filter(([name]) => name.startsWith("@databricks/databricks-sql-kernel-")) + .map(([, version]) => version), + )]; + + if (versions.length !== 1) { + throw new Error( + `Expected one pinned kernel package version, found: ${versions.join(", ") || "none"}`, + ); + } + + process.stdout.write(versions[0]); + ' "${driver_repo}/package.json" +) napi_major=$( cargo metadata --format-version 1 --locked --manifest-path "${napi_dir}/Cargo.toml" | node -e ' @@ -34,10 +52,13 @@ esac build_profile=${BUILD_PROFILE---release} cd "${napi_dir}" +# napi-rs normally embeds the kernel source manifest version in index.js. The +# source can advance before its native packages are published, so generate the +# loader guard from the version the driver actually installs instead. if [[ -n "${build_profile//[[:space:]]/}" ]]; then read -r -a build_profile_args <<< "${build_profile}" - "${cli[@]}" build --platform "${build_profile_args[@]}" + npm_new_version="${kernel_package_version}" "${cli[@]}" build --platform "${build_profile_args[@]}" else - "${cli[@]}" build --platform + npm_new_version="${kernel_package_version}" "${cli[@]}" build --platform fi cp index.* "${driver_repo}/native/kernel/" diff --git a/lib/kernel/KernelNativeLoader.ts b/lib/kernel/KernelNativeLoader.ts index 0b98d156..48e509e8 100644 --- a/lib/kernel/KernelNativeLoader.ts +++ b/lib/kernel/KernelNativeLoader.ts @@ -34,8 +34,7 @@ import type { ArrowBatch as NativeArrowBatch, ArrowSchema as NativeArrowSchema, ExecuteOptions as NativeExecuteOptions, - TypedValueInput as NativeTypedValueInput, - NamedTypedValueInput as NativeNamedTypedValueInput, + RawParameterInput as NativeRawParameterInput, AsyncStatement as NativeAsyncStatement, AsyncResultHandle as NativeAsyncResultHandle, CancellableExecution as NativeCancellableExecution, @@ -53,15 +52,10 @@ export type KernelArrowSchema = NativeArrowSchema; export type KernelConnection = NativeConnection; export type KernelStatement = NativeStatement; -// Per-statement execution options and bound-parameter inputs are kernel -// concerns: the napi binding generates the canonical shapes (`positionalParams` -// / `namedParams` as `TypedValueInput` / `NamedTypedValueInput`, plus -// `rowLimit`, `statementConf`, `queryTags`). We re-export -// rather than re-declare so the driver-side param codec can never drift from -// the kernel contract. +// Per-statement execution options and raw-parameter inputs come directly from +// the generated kernel contract so the driver-side codec cannot drift. export type KernelNativeExecuteOptions = NativeExecuteOptions; -export type KernelNativeTypedValueInput = NativeTypedValueInput; -export type KernelNativeNamedTypedValueInput = NativeNamedTypedValueInput; +export type KernelNativeRawParameterInput = NativeRawParameterInput; // Async-submit surface: `Connection.submitStatement` returns an // `AsyncStatement` (status / awaitResult / cancel / close); `awaitResult` diff --git a/lib/kernel/KernelPositionalParams.ts b/lib/kernel/KernelPositionalParams.ts index 758ec733..61faa460 100644 --- a/lib/kernel/KernelPositionalParams.ts +++ b/lib/kernel/KernelPositionalParams.ts @@ -14,7 +14,7 @@ import { DBSQLParameter, DBSQLParameterValue } from '../DBSQLParameter'; import ParameterError from '../errors/ParameterError'; -import { KernelNativeTypedValueInput, KernelNativeNamedTypedValueInput } from './KernelNativeLoader'; +import { KernelNativeRawParameterInput } from './KernelNativeLoader'; import assertBindableValue from './KernelInputValidation'; /** @@ -58,15 +58,14 @@ function decimalPrecisionScale(v: string): string { /** * Reduce a `DBSQLParameter | DBSQLParameterValue` to the napi - * `TypedValueInput` (`{ sqlType, value? }`) the kernel's positional-param - * codec (`parse_typed_value`) accepts. Reuses `DBSQLParameter.toSparkParameter` - * — the same type-inference + value-stringification the Thrift backend uses — - * then adapts the type name to the codec's expectations: + * `RawParameterInput` (`{ name?, sqlType, value? }`) accepted by the kernel's + * raw-parameter path. Reuses `DBSQLParameter.toSparkParameter` — the same + * type-inference + value-stringification the Thrift backend uses — then adapts + * the type name where required: * - DECIMAL → `DECIMAL(p,s)` (parenthesised form required) - * - INTERVAL * → `INTERVAL` (the codec's single interval type name) - * - a missing value ⇒ SQL NULL (`parse_typed_value` maps `value: None` to NULL). + * - a missing value ⇒ SQL NULL. */ -function toTypedValueInput(value: DBSQLParameter | DBSQLParameterValue): KernelNativeTypedValueInput { +function toRawParameterInput(value: DBSQLParameter | DBSQLParameterValue): KernelNativeRawParameterInput { const param = value instanceof DBSQLParameter ? value : new DBSQLParameter({ value }); const spark = param.toSparkParameter(); const stringValue = spark.value?.stringValue ?? undefined; @@ -81,44 +80,42 @@ function toTypedValueInput(value: DBSQLParameter | DBSQLParameterValue): KernelN const upper = sqlType.toUpperCase(); if (upper === 'DECIMAL') { sqlType = `DECIMAL(${decimalPrecisionScale(stringValue)})`; - } else if (upper.startsWith('INTERVAL')) { - sqlType = 'INTERVAL'; } return { sqlType, value: stringValue }; } /** * Convert the public `ordinalParameters` option into the napi - * `positionalParams` array (1-based `?` placeholders). Returns `undefined` + * `rawParams` array (1-based `?` placeholders). Returns `undefined` * when none were supplied, so the caller can keep the minimal no-options * call shape. */ export function buildKernelPositionalParams( ordinalParameters?: Array, -): Array | undefined { +): Array | undefined { if (ordinalParameters === undefined || ordinalParameters.length === 0) { return undefined; } return ordinalParameters.map((value, i) => { assertBindableValue(value, `ordinalParameters[${i}]`); - return toTypedValueInput(value); + return toRawParameterInput(value); }); } /** * Convert the public `namedParameters` option (`Record`) into - * the napi `namedParams` array (`:name` placeholders). Each value reuses the - * same `toTypedValueInput` mapping (DECIMAL → DECIMAL(p,s), NULL → VOID, …), + * the napi `rawParams` array (`:name` placeholders). Each value reuses the + * same `toRawParameterInput` mapping (DECIMAL → DECIMAL(p,s), NULL → VOID, …), * then carries its name. Returns `undefined` when none were supplied. */ export function buildKernelNamedParams( namedParameters?: Record, -): Array | undefined { +): Array | undefined { if (namedParameters === undefined || Object.keys(namedParameters).length === 0) { return undefined; } return Object.keys(namedParameters).map((name) => { assertBindableValue(namedParameters[name], `namedParameters[${name}]`); - return { name, ...toTypedValueInput(namedParameters[name]) }; + return { name, ...toRawParameterInput(namedParameters[name]) }; }); } diff --git a/lib/kernel/KernelSessionBackend.ts b/lib/kernel/KernelSessionBackend.ts index 84a765cb..6cbc745e 100644 --- a/lib/kernel/KernelSessionBackend.ts +++ b/lib/kernel/KernelSessionBackend.ts @@ -298,11 +298,10 @@ export default class KernelSessionBackend implements ISessionBackend { } const execOptions: KernelNativeExecuteOptions = {}; - if (positionalParams !== undefined) { - execOptions.positionalParams = positionalParams; - } - if (namedParams !== undefined) { - execOptions.namedParams = namedParams; + // Raw binding preserves qualified SQL types such as INTERVAL MONTH. + const rawParams = positionalParams ?? namedParams; + if (rawParams !== undefined) { + execOptions.rawParams = rawParams; } // NB: `queryTimeout` is intentionally NOT forwarded — it is a no-op on kernel // (SQL Warehouses use `STATEMENT_TIMEOUT`; mapping it to `wait_timeout` would diff --git a/native/kernel/index.d.ts b/native/kernel/index.d.ts index e85eb2c6..2275f9e5 100644 --- a/native/kernel/index.d.ts +++ b/native/kernel/index.d.ts @@ -686,6 +686,13 @@ export interface ConnectionOptions { * process); `true` keeps it enabled. Applies to [`AuthMode::OAuthU2m`]. */ tokenCacheEnabled?: boolean + /** + * Optional passphrase for the U2M on-disk token cache (AES-256 key). + * Omitted/blank ⇒ a machine-local derived key. Supplying one is stronger. + * A passphrase with no explicit `tokenCacheEnabled` implies enabled. + * Applies to [`AuthMode::OAuthU2m`]. + */ + tokenCachePassphrase?: string /** * Path to the PEM private-key file. Required for * [`AuthMode::OAuthM2mJwt`]. @@ -1041,6 +1048,7 @@ export interface ConnectionOptions { * `rowLimit` (SEA `row_limit`) is exposed here and threaded onto the kernel * `StatementSpec`. `positionalParams` (`?`) and `namedParams` (`:name`) * carry bound query parameters, decoded via `params::parse_typed_value`. + * `rawParams` forwards pre-marshalled SEA parameters unchanged. * (There is no `queryTimeoutSecs`: it abused the SEA `wait_timeout` inline-hold * window and was removed — a real per-statement timeout is `STATEMENT_TIMEOUT`.) * @@ -1106,6 +1114,12 @@ export interface ExecuteOptions { * mutually exclusive at the SQL level (`?` vs `:name`). */ namedParams?: Array + /** + * Pre-marshalled parameters passed through `StatementSpec::param_raw`. + * Omit `name` for positional values. Raw parameters cannot be mixed with + * typed parameters or combine named and positional markers. + */ + rawParams?: Array } /** @@ -1215,6 +1229,19 @@ export interface ProxyInput { bypassHosts?: string } +/** + * A pre-marshalled SEA parameter that preserves `sql_type` verbatim. + * Omit `name` for positional markers; the kernel assigns their ordinals. + */ +export interface RawParameterInput { + /** Named marker name. Omit for a positional marker. */ + name?: string + /** Databricks SQL type name sent verbatim to SEA. */ + sqlType: string + /** String-encoded value. `None` represents SQL NULL. */ + value?: string +} + /** * Live-retarget the bridge's level (one of * `off`/`error`/`warn`/`info`/`debug`/`trace`, case-insensitive). diff --git a/tests/e2e/kernel/execution-e2e.test.ts b/tests/e2e/kernel/execution-e2e.test.ts index d1ffc9ba..8ef9d803 100644 --- a/tests/e2e/kernel/execution-e2e.test.ts +++ b/tests/e2e/kernel/execution-e2e.test.ts @@ -13,7 +13,7 @@ // limitations under the License. import { expect } from 'chai'; -import { DBSQLClient } from '../../../lib'; +import { DBSQLClient, DBSQLParameter, DBSQLParameterType } from '../../../lib'; import { ConnectionOptions } from '../../../lib/contracts/IDBSQLClient'; import { InternalConnectionOptions } from '../../../lib/contracts/InternalConnectionOptions'; @@ -121,4 +121,117 @@ describe('kernel execution end-to-end', function e2eSuite() { await session.close(); await client.close(); }); + + it('binds ordinary positional parameters through rawParams', async () => { + const client = new DBSQLClient(); + + await client.connect({ + host: hostName as string, + path: httpPath as string, + token: token as string, + useKernel: true, + } as ConnectionOptions & InternalConnectionOptions); + + const session = await client.openSession({ initialCatalog: 'main' }); + let operation; + try { + operation = await session.executeStatement('SELECT ? AS p_int, ? AS p_string, ? AS p_bool', { + ordinalParameters: [42, 'hello', true], + }); + expect(await operation.fetchAll()).to.deep.equal([{ p_int: 42, p_string: 'hello', p_bool: true }]); + } finally { + await operation?.close(); + await session.close(); + await client.close(); + } + }); + + it('binds named NULL and empty string through rawParams on the async path', async () => { + const client = new DBSQLClient(); + + await client.connect({ + host: hostName as string, + path: httpPath as string, + token: token as string, + useKernel: true, + } as ConnectionOptions & InternalConnectionOptions); + + const session = await client.openSession({ initialCatalog: 'main' }); + let operation; + try { + operation = await session.executeStatement('SELECT :null_value AS null_value, :empty_value AS empty_value', { + namedParameters: { null_value: null, empty_value: '' }, + runAsync: true, + }); + expect(await operation.fetchAll()).to.deep.equal([{ null_value: null, empty_value: '' }]); + } finally { + await operation?.close(); + await session.close(); + await client.close(); + } + }); + + it('binds a valid INTERVAL MONTH on the SEA wire', async () => { + const client = new DBSQLClient(); + + await client.connect({ + host: hostName as string, + path: httpPath as string, + token: token as string, + useKernel: true, + } as ConnectionOptions & InternalConnectionOptions); + + const session = await client.openSession({ initialCatalog: 'main' }); + let operation; + try { + operation = await session.executeStatement("SELECT ? = INTERVAL '13' MONTH AS matches", { + ordinalParameters: [ + new DBSQLParameter({ + type: DBSQLParameterType.INTERVALMONTH, + value: '13', + }), + ], + }); + expect(await operation.fetchAll()).to.deep.equal([{ matches: true }]); + } finally { + await operation?.close(); + await session.close(); + await client.close(); + } + }); + + it('preserves INTERVAL MONTH on the SEA wire', async () => { + const client = new DBSQLClient(); + + await client.connect({ + host: hostName as string, + path: httpPath as string, + token: token as string, + useKernel: true, + } as ConnectionOptions & InternalConnectionOptions); + + const session = await client.openSession({ initialCatalog: 'main' }); + let operation; + let caught: unknown; + try { + operation = await session.executeStatement('SELECT ?', { + ordinalParameters: [ + new DBSQLParameter({ + type: DBSQLParameterType.INTERVALMONTH, + value: '2-6', + }), + ], + }); + await operation.fetchAll(); + } catch (error) { + caught = error; + } finally { + await operation?.close(); + await session.close(); + await client.close(); + } + + // "2-6" is valid YEAR TO MONTH syntax, but invalid for INTERVAL MONTH. + expect(caught).to.be.instanceOf(Error); + }); }); diff --git a/tests/unit/kernel/execution.test.ts b/tests/unit/kernel/execution.test.ts index 101d698a..706dedbf 100644 --- a/tests/unit/kernel/execution.test.ts +++ b/tests/unit/kernel/execution.test.ts @@ -27,6 +27,7 @@ import ParameterError from '../../../lib/errors/ParameterError'; import OperationStateError, { OperationStateErrorCode } from '../../../lib/errors/OperationStateError'; import { ConnectionOptions } from '../../../lib/contracts/IDBSQLClient'; import { OperationState } from '../../../lib/contracts/OperationStatus'; +import { DBSQLParameter, DBSQLParameterType } from '../../../lib/DBSQLParameter'; // ----------------------------------------------------------------------------- // Fakes — minimal stand-ins for the napi-rs generated surface and the @@ -708,26 +709,53 @@ describe('KernelSessionBackend', () => { expect(connection.statementToReturn.cancelled, 'cancel reaches the terminal statement').to.equal(true); }); - it('executeStatement forwards ordinalParameters as napi positionalParams', async () => { + it('executeStatement forwards ordinalParameters through napi rawParams', async () => { const connection = new FakeNativeConnection(); const session = makeSession(connection); await session.executeStatement('SELECT ?', { ordinalParameters: [42, 'hi'] }); - const options = connection.lastOptions as { positionalParams?: Array<{ sqlType: string; value?: string }> }; + const options = connection.lastOptions as { rawParams?: Array<{ sqlType: string; value?: string }> }; expect(options, 'options should be passed').to.not.equal(undefined); - expect(options.positionalParams).to.have.length(2); - expect(options.positionalParams?.[0]).to.deep.equal({ sqlType: 'INTEGER', value: '42' }); - expect(options.positionalParams?.[1]).to.deep.equal({ sqlType: 'STRING', value: 'hi' }); + expect(options.rawParams).to.have.length(2); + expect(options.rawParams?.[0]).to.deep.equal({ sqlType: 'INTEGER', value: '42' }); + expect(options.rawParams?.[1]).to.deep.equal({ sqlType: 'STRING', value: 'hi' }); }); - it('executeStatement forwards namedParameters as napi namedParams (:name carried)', async () => { + it('executeStatement forwards namedParameters through napi rawParams (:name carried)', async () => { const connection = new FakeNativeConnection(); const session = makeSession(connection); await session.executeStatement('SELECT :x', { namedParameters: { x: 7 } }); const options = connection.lastOptions as { - namedParams?: Array<{ name: string; sqlType: string; value?: string }>; + rawParams?: Array<{ name?: string; sqlType: string; value?: string }>; }; - expect(options.namedParams).to.have.length(1); - expect(options.namedParams?.[0]).to.deep.equal({ name: 'x', sqlType: 'INTEGER', value: '7' }); + expect(options.rawParams).to.have.length(1); + expect(options.rawParams?.[0]).to.deep.equal({ name: 'x', sqlType: 'INTEGER', value: '7' }); + }); + + it('executeStatement (runAsync: true) forwards named NULL and empty string through napi rawParams', async () => { + const connection = new FakeNativeConnection(); + const session = makeSession(connection); + await session.executeStatement('SELECT :null_value, :empty_value', { + namedParameters: { null_value: null, empty_value: '' }, + runAsync: true, + }); + const options = connection.lastOptions as { + rawParams?: Array<{ name?: string; sqlType: string; value?: string }>; + }; + expect(connection.lastAsyncStatement, 'runAsync should use submitStatement').to.not.equal(undefined); + expect(options.rawParams).to.deep.equal([ + { name: 'null_value', sqlType: 'VOID' }, + { name: 'empty_value', sqlType: 'STRING', value: '' }, + ]); + }); + + it('executeStatement preserves a qualified INTERVAL type in napi rawParams', async () => { + const connection = new FakeNativeConnection(); + const session = makeSession(connection); + await session.executeStatement('SELECT ?', { + ordinalParameters: [new DBSQLParameter({ type: DBSQLParameterType.INTERVALMONTH, value: '2-6' })], + }); + const options = connection.lastOptions as { rawParams?: Array<{ sqlType: string; value?: string }> }; + expect(options.rawParams).to.deep.equal([{ sqlType: 'INTERVAL MONTH', value: '2-6' }]); }); it('executeStatement sends no options object on the no-params path', async () => { diff --git a/tests/unit/kernel/positionalParams.test.ts b/tests/unit/kernel/positionalParams.test.ts index f6070147..960b2689 100644 --- a/tests/unit/kernel/positionalParams.test.ts +++ b/tests/unit/kernel/positionalParams.test.ts @@ -63,15 +63,15 @@ describe('KernelPositionalParams.buildKernelPositionalParams', () => { expect(decimal('')).to.throw(ParameterError, /not a plain decimal numeral/); }); - it('collapses every INTERVAL subtype to the kernel codec\'s single "INTERVAL" type name', () => { + it('preserves qualified INTERVAL types for the kernel raw binder', () => { expect( buildKernelPositionalParams([ - new DBSQLParameter({ type: DBSQLParameterType.INTERVALMONTH, value: '13' }), + new DBSQLParameter({ type: DBSQLParameterType.INTERVALMONTH, value: '2-6' }), new DBSQLParameter({ type: DBSQLParameterType.INTERVALDAY, value: '1 02:03:04' }), ]), ).to.deep.equal([ - { sqlType: 'INTERVAL', value: '13' }, - { sqlType: 'INTERVAL', value: '1 02:03:04' }, + { sqlType: 'INTERVAL MONTH', value: '2-6' }, + { sqlType: 'INTERVAL DAY', value: '1 02:03:04' }, ]); }); @@ -127,4 +127,12 @@ describe('KernelPositionalParams.buildKernelNamedParams', () => { it('maps a named NULL to a value-less VOID input (with the name)', () => { expect(buildKernelNamedParams({ x: null })).to.deep.equal([{ name: 'x', sqlType: 'VOID' }]); }); + + it('preserves a named qualified INTERVAL type', () => { + expect( + buildKernelNamedParams({ + duration: new DBSQLParameter({ type: DBSQLParameterType.INTERVALMONTH, value: '2-6' }), + }), + ).to.deep.equal([{ name: 'duration', sqlType: 'INTERVAL MONTH', value: '2-6' }]); + }); });