diff --git a/docs/en/engines/database-engines/datalake.md b/docs/en/engines/database-engines/datalake.md index 14713eb75154..3094590d073e 100644 --- a/docs/en/engines/database-engines/datalake.md +++ b/docs/en/engines/database-engines/datalake.md @@ -61,6 +61,12 @@ The following settings are supported: | `dlf_access_key_id` | Access key ID for DLF access | | `dlf_access_key_secret` | Access key Secret for DLF access | | `namespaces` | Comma-separated list of namespaces, implemented for catalog types: `rest`, `glue` and `unity` | +| `oauth_forward_user_token` | Authenticate to the catalog as the user running the query instead of as the shared service identity. Iceberg REST and Glue only. See [Forwarding the user's identity to the catalog](#user-token-forwarding) | +| `oauth_token_exchange_uri` | Empty (the default) forwards the user's token unchanged; non-empty performs an RFC 8693 token exchange at this URL first | +| `oauth_subject_token_type` | RFC 8693 `subject_token_type` of the forwarded token. Default `urn:ietf:params:oauth:token-type:access_token` | +| `oauth_requested_token_type` | RFC 8693 `requested_token_type`; empty omits the field. Default `urn:ietf:params:oauth:token-type:access_token` | +| `oauth_forward_actor_token` | Send the service principal's own token as the RFC 8693 `actor_token`. Default `0`. See [Delegation with an actor token](#user-token-forwarding-actor-token) | +| `oauth_user_token_cache_ttl` | Maximum lifetime (in seconds) of a cached exchanged session token; `0` disables caching. Default `300` | ## Examples {#examples} @@ -86,6 +92,115 @@ SELECT count() from database_name.table_name; ``` To authenticate without sharing a client secret, set `onelake_bearer_token` to a pre-obtained bearer token (scoped to `https://storage.azure.com`) instead of `onelake_client_id`/`onelake_client_secret`. ClickHouse does not refresh the token, so the database must be recreated after it expires. +## Forwarding the user's identity to the catalog {#user-token-forwarding} + +Set `oauth_forward_user_token = 1` to use the querying user's token for catalog authentication. +This requires: + +- the server-level [`enable_token_forwarding`](/operations/server-configuration-parameters/settings#enable_token_forwarding) + setting, which is `false` by default; +- `catalog_type = 'rest'` or `catalog_type = 'glue'`; +- token authentication through an `Authorization: Bearer` HTTP header or `--jwt` for the native + protocol. See [Token-based authentication](/en/operations/external-authenticators/oauth). + +:::danger Restrict `CREATE DATABASE` +Database creators choose the endpoints that receive users' tokens. Grant `CREATE DATABASE` only +to trusted users and restrict `remote_url_allow_hosts`, which applies to catalog and token-exchange +endpoints. +::: + +### Passthrough: the default {#user-token-forwarding-passthrough} + +For Iceberg REST catalogs, `oauth_forward_user_token = 1` forwards the user's bearer token +unchanged. No token endpoint or client credentials are required: + +```sql +CREATE DATABASE demo +ENGINE = DataLakeCatalog('http://lakekeeper:8181/catalog') +SETTINGS + catalog_type = 'rest', + warehouse = 'demo', + oauth_forward_user_token = 1; +``` + +The token's audience must cover both ClickHouse and the catalog. With Keycloak, add an audience +mapper to the ClickHouse client to include the catalog's audience. + +### Token exchange: opt-in {#user-token-forwarding-exchange} + +Set `oauth_token_exchange_uri` to exchange the user's token using +[RFC 8693](https://www.rfc-editor.org/rfc/rfc8693) before presenting it to the REST catalog. +Use an IdP token endpoint that issues tokens with an audience the catalog accepts: + +```sql +CREATE DATABASE demo +ENGINE = DataLakeCatalog('http://lakekeeper:8181/catalog') +SETTINGS + catalog_type = 'rest', + warehouse = 'demo', + catalog_credential = 'clickhouse:', + auth_scope = 'lakekeeper', + oauth_forward_user_token = 1, + oauth_token_exchange_uri = 'http://keycloak:8080/realms/demo/protocol/openid-connect/token'; +``` + +The exchange requires `catalog_credential`; its `client_id` and `client_secret` are sent in the +form body. `auth_scope` supplies the exchange `scope`. Override its Polaris-specific default, +`PRINCIPAL_ROLE:ALL`, for other providers. + +A catalog's `/v1/oauth/tokens` endpoint can also be used if supported. The Iceberg REST +specification deprecates this endpoint, and some catalogs, including Lakekeeper, do not implement it. + +### Delegation with an actor token {#user-token-forwarding-actor-token} + +Set `oauth_forward_actor_token = 1` to include the service principal's token as the RFC 8693 +`actor_token`. This requires `oauth_token_exchange_uri` and an endpoint that supports delegation +and can validate the actor token. + +ClickHouse obtains the actor token through a `client_credentials` grant using `catalog_credential` +at `oauth_server_uri`, or the catalog's `/v1/oauth/tokens` endpoint if that setting is empty. +The token is cached until expiry and used only for exchanges. If obtaining it fails, the query fails. + +### Glue {#user-token-forwarding-glue} + +For Glue, ClickHouse exchanges the user's token through AWS STS `AssumeRoleWithWebIdentity` +for temporary credentials of `aws_role_arn`. These credentials sign Glue and S3 requests using +SigV4. The ClickHouse user name supplies `RoleSessionName` for CloudTrail auditing. + +```sql +CREATE DATABASE glue_db +ENGINE = DataLakeCatalog +SETTINGS + catalog_type = 'glue', + region = 'us-east-1', + aws_role_arn = 'arn:aws:iam::123456789012:role/data-lake-reader', + oauth_forward_user_token = 1; +``` + +Register the token issuer as an IAM OIDC identity provider and configure the role's trust policy +to accept the users' tokens, including their `aud` and `sub` claims. + +- `aws_role_arn` is required. +- `aws_access_key_id`, `aws_secret_access_key`, and RFC 8693 exchange settings are rejected. +- The AWS STS endpoint is determined by `region`. +- Users receive the assumed role's permissions. Use separate roles or session-tag policies to + distinguish access. ClickHouse does not implement IAM Identity Center trusted identity propagation. + +### Scope and limitations {#user-token-forwarding-scope} + +- Forwarding covers catalog listings, table metadata, and write operations (`INSERT`, `ALTER`, + mutations, `DROP TABLE`, and snapshot expiry). +- Vended storage credentials and Glue sessions are cached separately for each user token. +- Requests without a user token fail with `CATALOG_USER_TOKEN_NOT_AVAILABLE`; ClickHouse does not + fall back to the service identity. Catalog listings may suppress these errors and appear empty, + depending on `database_datalake_require_metadata_access`. +- With `object_storage_cluster`, workers receive table-scoped storage credentials over the + interserver channel. Configure `interserver_https_port` or a cluster `` for cluster reads. +- For Iceberg REST, rotate `catalog_credential` with `ALTER DATABASE ... MODIFY SETTING`, + authenticated with a user token. ClickHouse validates the new credentials and reloads the catalog + configuration before applying the change, then invalidates cached session and storage credentials. + Glue settings cannot be altered. + ## Namespace filter {#namespace} By default, ClickHouse reads tables from all namespaces available in the catalog. You can limit this behavior using the `namespaces` database setting. The value should be a comma‑separated list of namespaces that are allowed to be read. diff --git a/docs/en/operations/external-authenticators/tokens.md b/docs/en/operations/external-authenticators/tokens.md index 4bc2151c2498..030b2dc88b08 100644 --- a/docs/en/operations/external-authenticators/tokens.md +++ b/docs/en/operations/external-authenticators/tokens.md @@ -300,6 +300,35 @@ To reduce number of requests to IdP, tokens are cached internally for a maximum If token expires sooner than `token_cache_lifetime`, then cache entry for this token will only be valid while token is valid. If token lifetime is longer than `token_cache_lifetime`, cache entry for this token will be valid for `token_cache_lifetime`. +## Forwarding the token to external services {#token-forwarding} + +By default, the authenticated token is not retained in the session for forwarding. Set +`enable_token_forwarding` to `1` in `config.xml` to retain it for external-service authentication: + +```xml +1 +``` + +The setting is hot-reloadable and defaults to `false`. To use it with Iceberg REST or Glue, enable +`oauth_forward_user_token` on the [`DataLakeCatalog`](/engines/database-engines/datalakecatalog) +database. See [catalog token forwarding](/engines/database-engines/datalakecatalog#user-token-forwarding) +for configuration and examples. + +:::danger Restrict `CREATE DATABASE` +Database creators choose the endpoints that receive users' tokens. Grant `CREATE DATABASE` only +to trusted users and restrict `remote_url_allow_hosts` for catalog and token-exchange endpoints. +::: + +Only the session's verified token is forwarded. It is not inherited by `EXECUTE AS` or +`DEFINER` views, forwarded to other ClickHouse nodes, or persisted to disk. + +HTTP requests authenticate separately, so a rotated token takes effect on the next request. +Native TCP sessions must reconnect with the new token. + +With `async_insert = 1`, each queued batch retains its token until the flush completes, even with +`wait_for_async_insert = 0`. Different tokens use separate batches; rotation does not change a +queued batch's token. + ## Enabling token authentication for a user in `users.xml` {#enabling-jwt-auth-in-users-xml} In order to enable token-based authentication for the user, specify `jwt` section instead of `password` or other similar sections in the user definition. diff --git a/src/Access/AccessControl.cpp b/src/Access/AccessControl.cpp index a5ff29f87202..c6b1e79ccdb8 100644 --- a/src/Access/AccessControl.cpp +++ b/src/Access/AccessControl.cpp @@ -295,6 +295,7 @@ void AccessControl::setupFromMainConfig(const Poco::Util::AbstractConfiguration setPasswordComplexityRulesFromConfig(config_); setTokenAuthEnabled(config_.getBool("enable_token_auth", true)); + setTokenForwardingEnabled(config_.getBool("enable_token_forwarding", false)); setBcryptWorkfactor(config_.getInt("bcrypt_workfactor", 12)); @@ -705,6 +706,7 @@ void AccessControl::setExternalAuthenticatorsConfig(const Poco::Util::AbstractCo /// value in place -- operators who toggle token auth off in response to an /// IdP outage or a credential leak would see no effect until restart. setTokenAuthEnabled(config.getBool("enable_token_auth", true)); + setTokenForwardingEnabled(config.getBool("enable_token_forwarding", false)); external_authenticators->setConfiguration(config, getLogger(), token_http_timeouts, isTokenAuthEnabled()); } @@ -994,4 +996,14 @@ bool AccessControl::isTokenAuthEnabled() const { return enable_token_auth; } + +void AccessControl::setTokenForwardingEnabled(bool enable) +{ + enable_token_forwarding = enable; +} + +bool AccessControl::isTokenForwardingEnabled() const +{ + return enable_token_forwarding; +} } diff --git a/src/Access/AccessControl.h b/src/Access/AccessControl.h index fa57e5c5bf80..e45405d16cc9 100644 --- a/src/Access/AccessControl.h +++ b/src/Access/AccessControl.h @@ -283,6 +283,9 @@ class AccessControl : public MultipleAccessStorage void setTokenAuthEnabled(bool enable); bool isTokenAuthEnabled() const; + void setTokenForwardingEnabled(bool enable); + bool isTokenForwardingEnabled() const; + private: class ContextAccessCache; class CustomSettingsPrefixes; @@ -320,6 +323,7 @@ class AccessControl : public MultipleAccessStorage std::atomic_bool enable_read_write_grants = false; std::atomic_bool allow_impersonate_user = false; std::atomic_bool enable_token_auth = true; + std::atomic_bool enable_token_forwarding = false; }; } diff --git a/src/Access/ForwardedAuthToken.cpp b/src/Access/ForwardedAuthToken.cpp new file mode 100644 index 000000000000..fdc3d6fa6dca --- /dev/null +++ b/src/Access/ForwardedAuthToken.cpp @@ -0,0 +1,18 @@ +#include + +#include +#include + +namespace DB +{ + +ForwardedAuthTokenPtr makeForwardedAuthToken(const TokenCredentials & credentials, const String & principal) +{ + auto result = std::make_shared(); + result->token = credentials.getToken(); + result->fingerprint = getSipHash128AsHexString(sipHash128(result->token.data(), result->token.size())); + result->principal = principal; + return result; +} + +} diff --git a/src/Access/ForwardedAuthToken.h b/src/Access/ForwardedAuthToken.h new file mode 100644 index 000000000000..62657fc7a917 --- /dev/null +++ b/src/Access/ForwardedAuthToken.h @@ -0,0 +1,25 @@ +#pragma once + +#include + +#include + +namespace DB +{ + +class TokenCredentials; + +struct ForwardedAuthToken +{ + String token; + /// Use the token fingerprint so rotation cannot reuse credentials cached for the previous token. + String fingerprint; + String principal; +}; + +using ForwardedAuthTokenPtr = std::shared_ptr; + +/// `principal` must be the canonical `AuthResult::user_name`, not the name the client sent. +ForwardedAuthTokenPtr makeForwardedAuthToken(const TokenCredentials & credentials, const String & principal); + +} diff --git a/src/Common/CurrentMetrics.cpp b/src/Common/CurrentMetrics.cpp index a36be7839914..6c835cc977ff 100644 --- a/src/Common/CurrentMetrics.cpp +++ b/src/Common/CurrentMetrics.cpp @@ -372,6 +372,10 @@ M(DNSAddressesCacheSize, "Number of cached DNS addresses") \ M(MarkCacheBytes, "Total size of mark cache in bytes") \ M(MarkCacheFiles, "Total number of mark files cached in the mark cache") \ + M(DataLakeCatalogUserTokenCacheBytes, "Total size in bytes of the per-user session tokens exchanged for data lake catalog access") \ + M(DataLakeCatalogUserTokenCacheEntries, "Total number of per-user session tokens exchanged for data lake catalog access") \ + M(DataLakeCatalogUserClientCacheBytes, "Total size in bytes of the per-user catalog clients built from forwarded user tokens") \ + M(DataLakeCatalogUserClientCacheEntries, "Total number of per-user catalog clients built from forwarded user tokens") \ M(UniqueKeyIndexCacheBytes, "Total size of UNIQUE KEY index cache in bytes") \ M(UniqueKeyIndexCacheEntries, "Total number of UNIQUE KEY index blocks cached") \ M(DeleteBitmapCacheBytes, "Total size of the UNIQUE KEY delete-bitmap cache in bytes") \ diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index cfdc94b31795..301aee43b948 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -659,6 +659,7 @@ M(777, MEMORY_RESERVATION_KILLED) \ M(778, MEMORY_RESERVATION_FAILED) \ M(779, CATALOG_NAMESPACE_DISABLED) \ + M(780, CATALOG_USER_TOKEN_NOT_AVAILABLE) \ \ M(900, DISTRIBUTED_CACHE_ERROR) \ M(901, CANNOT_USE_DISTRIBUTED_CACHE) \ diff --git a/src/Common/FormUrlEncode.cpp b/src/Common/FormUrlEncode.cpp new file mode 100644 index 000000000000..419337cc42bb --- /dev/null +++ b/src/Common/FormUrlEncode.cpp @@ -0,0 +1,15 @@ +#include + +#include + +namespace DB +{ + +std::string formUrlEncode(const std::string & value) +{ + std::string encoded; + Poco::URI::encode(value, "!$&'()*+,;=:@/?", encoded); + return encoded; +} + +} diff --git a/src/Common/FormUrlEncode.h b/src/Common/FormUrlEncode.h new file mode 100644 index 000000000000..18602a9275d4 --- /dev/null +++ b/src/Common/FormUrlEncode.h @@ -0,0 +1,11 @@ +#pragma once + +#include + +namespace DB +{ + +/// `Poco::URI::encode` leaves form delimiters unescaped unless they are explicitly reserved. +std::string formUrlEncode(const std::string & value); + +} diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index 184d4941aa3c..17fb7b0a99b6 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -1734,6 +1734,10 @@ The server successfully detected this situation and will download merged part fr M(ObjectStorageListObjectsCachePrefixMatchHits, "Number of times object storage list objects operation miss the cache using prefix matching.", ValueType::Number) \ M(DataLakeRestCatalogCredentialsVended, "Number of table metadata requests to REST catalog asking to vend storage credentials.", ValueType::Number) \ M(DataLakeRestCatalogCredentialsCacheHits, "Number of table metadata requests to REST catalog reusing cached storage credentials.", ValueType::Number) \ + M(DataLakeRestCatalogTokenExchange, "Number of RFC 8693 token exchanges performed to obtain a session token for the querying user.", ValueType::Number) \ + M(DataLakeRestCatalogTokenExchangeMicroseconds, "Total time of RFC 8693 token exchanges.", ValueType::Microseconds) \ + M(DataLakeRestCatalogTokenExchangeFailures, "Number of RFC 8693 token exchanges that failed.", ValueType::Number) \ + M(DataLakeRestCatalogUserTokenCacheHits, "Number of times a previously exchanged per-user session token was reused.", ValueType::Number) \ \ M(DataLakeRestCatalogLoadConfig, "Number of 'load config' requests to Iceberg REST catalog.", ValueType::Number) \ M(DataLakeRestCatalogLoadConfigMicroseconds, "Total time of 'load config' requests to Iceberg REST catalog.", ValueType::Microseconds) \ @@ -1772,6 +1776,10 @@ The server successfully detected this situation and will download merged part fr M(DataLakeGlueCatalogUpdateTableMicroseconds, "Total time of 'update table' requests to Iceberg Glue catalog.", ValueType::Microseconds) \ M(DataLakeGlueCatalogDropTable, "Number of 'drop table' requests to Iceberg Glue catalog.", ValueType::Number) \ M(DataLakeGlueCatalogDropTableMicroseconds, "Total time of 'drop table' requests to Iceberg Glue catalog.", ValueType::Microseconds) \ + M(DataLakeGlueCatalogAssumeRoleWithWebIdentity, "Number of AWS STS `AssumeRoleWithWebIdentity` calls made to turn a forwarded user token into credentials for the Glue catalog.", ValueType::Number) \ + M(DataLakeGlueCatalogAssumeRoleWithWebIdentityMicroseconds, "Total time of AWS STS `AssumeRoleWithWebIdentity` calls made for forwarded user tokens.", ValueType::Microseconds) \ + M(DataLakeGlueCatalogAssumeRoleWithWebIdentityFailures, "Number of AWS STS `AssumeRoleWithWebIdentity` calls that returned no credentials.", ValueType::Number) \ + M(DataLakeGlueCatalogUserClientCacheHits, "Number of times a Glue client built for a forwarded user token was reused.", ValueType::Number) \ \ M(DataLakeUnityCatalogGetTables, "Number of 'get tables' requests to Iceberg Unity catalog.", ValueType::Number) \ M(DataLakeUnityCatalogGetTablesMicroseconds, "Total time of 'get tables' requests to Iceberg Unity catalog.", ValueType::Microseconds) \ diff --git a/src/Core/ServerSettings.cpp b/src/Core/ServerSettings.cpp index 6ed2b20957d1..3a44d7de1fb0 100644 --- a/src/Core/ServerSettings.cpp +++ b/src/Core/ServerSettings.cpp @@ -890,6 +890,16 @@ namespace Default value: `true` (token authentication is enabled). )", 0) \ + DECLARE(Bool, enable_token_forwarding, false, R"( + Retain authenticated bearer tokens for forwarding through the `DataLakeCatalog` + setting `oauth_forward_user_token`. Supports Iceberg REST and AWS STS for Glue. + When disabled, tokens are not retained in sessions for forwarding. + + Database creators choose the endpoints that receive users' tokens. Grant `CREATE DATABASE` + only to trusted users and restrict `remote_url_allow_hosts`. + + Default value: `false`. + )", 0) \ DECLARE(UInt64, concurrent_threads_soft_limit_num, 0, R"( The maximum number of query processing threads, excluding threads for retrieving data from remote servers, allowed to run all queries. This is not a hard limit. In case if the limit is reached the query will still get at least one thread to run. Query can upscale to desired number of threads during execution if more threads become available. diff --git a/src/Databases/DataLake/Common.cpp b/src/Databases/DataLake/Common.cpp index 8946d3412d70..1e4c014ca01e 100644 --- a/src/Databases/DataLake/Common.cpp +++ b/src/Databases/DataLake/Common.cpp @@ -1,5 +1,7 @@ #include +#include + #include #include #include @@ -110,6 +112,13 @@ DB::DataTypePtr getType(const String & type_name, bool nullable, DB::ContextPtr : DB::Iceberg::IcebergSchemaProcessor::getSimpleType(name, context); } +DB::ForwardedAuthTokenPtr getForwardedAuthToken(const DB::ContextPtr & context) +{ + if (!context) + return {}; + return context->getForwardedAuthToken(); +} + std::pair parseTableName(const std::string & name) { auto pos = name.rfind('.'); diff --git a/src/Databases/DataLake/Common.h b/src/Databases/DataLake/Common.h index 9b0dd7c626a6..294d9d757fb4 100644 --- a/src/Databases/DataLake/Common.h +++ b/src/Databases/DataLake/Common.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -19,4 +20,6 @@ DB::DataTypePtr getType(const String & type_name, bool nullable, DB::ContextPtr /// `E` is a table name. std::pair parseTableName(const std::string & name); +DB::ForwardedAuthTokenPtr getForwardedAuthToken(const DB::ContextPtr & context); + } diff --git a/src/Databases/DataLake/DatabaseDataLake.cpp b/src/Databases/DataLake/DatabaseDataLake.cpp index 6f2a608398f6..1b731fb7816c 100644 --- a/src/Databases/DataLake/DatabaseDataLake.cpp +++ b/src/Databases/DataLake/DatabaseDataLake.cpp @@ -67,6 +67,12 @@ namespace DatabaseDataLakeSetting extern const DatabaseDataLakeSettingsS3UriStyle storage_uri_style; extern const DatabaseDataLakeSettingsString oauth_server_uri; extern const DatabaseDataLakeSettingsBool oauth_server_use_request_body; + extern const DatabaseDataLakeSettingsBool oauth_forward_user_token; + extern const DatabaseDataLakeSettingsString oauth_token_exchange_uri; + extern const DatabaseDataLakeSettingsString oauth_subject_token_type; + extern const DatabaseDataLakeSettingsString oauth_requested_token_type; + extern const DatabaseDataLakeSettingsBool oauth_forward_actor_token; + extern const DatabaseDataLakeSettingsUInt64 oauth_user_token_cache_ttl; extern const DatabaseDataLakeSettingsBool vended_credentials; extern const DatabaseDataLakeSettingsUInt64 vended_credentials_cache_ttl; extern const DatabaseDataLakeSettingsString object_storage_cluster; @@ -191,6 +197,89 @@ void DatabaseDataLake::validateSettings() ErrorCodes::BAD_ARGUMENTS, "`warehouse` setting cannot be empty. " "Please specify 'SETTINGS warehouse=' in the CREATE DATABASE query"); } + + validateTokenForwardingSettings(); +} + +void DatabaseDataLake::validateTokenForwardingSettings() const +{ + const auto settings_version = database_settings.get(); + const DatabaseDataLakeSettings & settings = *settings_version; + + if (!settings[DatabaseDataLakeSetting::oauth_forward_user_token].value) + return; + + const auto catalog_type = settings[DatabaseDataLakeSetting::catalog_type].value; + + if (catalog_type != DB::DatabaseDataLakeCatalogType::ICEBERG_REST && catalog_type != DB::DatabaseDataLakeCatalogType::GLUE) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "`oauth_forward_user_token` is only supported for `catalog_type = 'rest'` and " + "`catalog_type = 'glue'`"); + + if (catalog_type == DB::DatabaseDataLakeCatalogType::GLUE) + { + validateGlueTokenForwardingSettings(settings); + return; + } + + if (!settings[DatabaseDataLakeSetting::auth_header].value.empty()) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "`oauth_forward_user_token` cannot be combined with `auth_header`"); + + const auto & exchange_uri = settings[DatabaseDataLakeSetting::oauth_token_exchange_uri].value; + if (!exchange_uri.empty() && settings[DatabaseDataLakeSetting::catalog_credential].value.empty()) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "`oauth_token_exchange_uri` requires a non-empty `catalog_credential`"); + + static const std::array valid_token_types = { + "urn:ietf:params:oauth:token-type:access_token", + "urn:ietf:params:oauth:token-type:refresh_token", + "urn:ietf:params:oauth:token-type:id_token", + "urn:ietf:params:oauth:token-type:saml1", + "urn:ietf:params:oauth:token-type:saml2", + "urn:ietf:params:oauth:token-type:jwt", + }; + auto check_token_type = [&](std::string_view setting_name, const std::string & value, bool empty_allowed) + { + if (value.empty() && empty_allowed) + return; + if (std::find(valid_token_types.begin(), valid_token_types.end(), value) == valid_token_types.end()) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "`{}` must be one of the token type URNs defined by RFC 8693, got `{}`", + setting_name, value); + }; + check_token_type( + "oauth_subject_token_type", + settings[DatabaseDataLakeSetting::oauth_subject_token_type].value, + /* empty_allowed */ false); + check_token_type( + "oauth_requested_token_type", + settings[DatabaseDataLakeSetting::oauth_requested_token_type].value, + /* empty_allowed */ true); +} + +void DatabaseDataLake::validateGlueTokenForwardingSettings(const DatabaseDataLakeSettings & settings) +{ + if (settings[DatabaseDataLakeSetting::aws_role_arn].value.empty()) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "`oauth_forward_user_token` requires a non-empty `aws_role_arn` for a Glue catalog"); + + if (!settings[DatabaseDataLakeSetting::aws_access_key_id].value.empty() + || !settings[DatabaseDataLakeSetting::aws_secret_access_key].value.empty()) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "`oauth_forward_user_token` cannot be combined with `aws_access_key_id` / " + "`aws_secret_access_key` for a Glue catalog"); + + if (!settings[DatabaseDataLakeSetting::oauth_token_exchange_uri].value.empty()) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "`oauth_token_exchange_uri` is only supported for `catalog_type = 'rest'`"); } void DatabaseDataLake::initialize() const @@ -212,6 +301,7 @@ void DatabaseDataLake::initialize() const .aws_role_arn = settings[DatabaseDataLakeSetting::aws_role_arn].value, .aws_role_session_name = settings[DatabaseDataLakeSetting::aws_role_session_name].value, .aws_external_id = settings[DatabaseDataLakeSetting::aws_external_id].value, + .forward_user_token = settings[DatabaseDataLakeSetting::oauth_forward_user_token].value, }; switch (settings[DatabaseDataLakeSetting::catalog_type].value) @@ -227,7 +317,15 @@ void DatabaseDataLake::initialize() const settings[DatabaseDataLakeSetting::oauth_server_uri].value, settings[DatabaseDataLakeSetting::oauth_server_use_request_body].value, settings[DatabaseDataLakeSetting::namespaces].value, - Context::getGlobalContextInstance()); + Context::getGlobalContextInstance(), + DataLake::TokenForwardingConfig{ + .forward_user_token = settings[DatabaseDataLakeSetting::oauth_forward_user_token].value, + .token_exchange_uri = settings[DatabaseDataLakeSetting::oauth_token_exchange_uri].value, + .subject_token_type = settings[DatabaseDataLakeSetting::oauth_subject_token_type].value, + .requested_token_type = settings[DatabaseDataLakeSetting::oauth_requested_token_type].value, + .forward_actor_token = settings[DatabaseDataLakeSetting::oauth_forward_actor_token].value, + .user_token_cache_ttl = settings[DatabaseDataLakeSetting::oauth_user_token_cache_ttl].value, + }); break; } case DB::DatabaseDataLakeCatalogType::ICEBERG_ONELAKE: @@ -577,13 +675,13 @@ std::string DatabaseDataLake::getStorageEndpointForTable(const DataLake::TableMe bool DatabaseDataLake::empty() const { - return getCatalog()->empty(); + return getCatalog()->empty(/* auth_token */ {}); } -bool DatabaseDataLake::isTableExist(const String & name, ContextPtr /* context_ */) const +bool DatabaseDataLake::isTableExist(const String & name, ContextPtr context_) const { const auto [namespace_name, table_name] = DataLake::parseTableName(name); - return getCatalog()->existsTable(namespace_name, table_name); + return getCatalog()->existsTable(namespace_name, table_name, DataLake::getForwardedAuthToken(context_)); } StoragePtr DatabaseDataLake::tryGetTable(const String & name, ContextPtr context_) const @@ -781,7 +879,11 @@ StoragePtr DatabaseDataLake::tryGetTableImpl(const String & name, ContextPtr con auto storage_cluster = std::make_shared( cluster_name, configuration, - configuration->createObjectStorage(context_copy, /* is_readonly */ false, catalog->getCredentialsConfigurationCallback(StorageID(getDatabaseName(), name, table_uuid))), + configuration->createObjectStorage( + context_copy, + /* is_readonly */ false, + catalog->getCredentialsConfigurationCallback( + StorageID(getDatabaseName(), name, table_uuid), DataLake::getForwardedAuthToken(context_))), StorageID(getDatabaseName(), name, table_uuid), /* columns */columns, /* constraints */ConstraintsDescription{}, @@ -835,7 +937,7 @@ DatabaseTablesIteratorPtr DatabaseDataLake::getTablesIterator( throw Exception(ErrorCodes::DATALAKE_DATABASE_ERROR, "Injected catalog listing failure"); }); - iceberg_tables = getCatalog()->getTables(); + iceberg_tables = getCatalog()->getTables(DataLake::getForwardedAuthToken(context_)); } catch (...) { @@ -936,7 +1038,7 @@ std::vector DatabaseDataLake::getLightweightTablesItera throw Exception(ErrorCodes::DATALAKE_DATABASE_ERROR, "Injected catalog listing failure"); }); - iceberg_tables = getCatalog()->getTables(); + iceberg_tables = getCatalog()->getTables(DataLake::getForwardedAuthToken(context_)); } catch (...) { @@ -955,7 +1057,7 @@ std::vector DatabaseDataLake::getLightweightTablesItera return result; } -Strings DatabaseDataLake::getAllTableNames(ContextPtr /*context*/) const +Strings DatabaseDataLake::getAllTableNames(ContextPtr context_) const { Strings result; @@ -964,7 +1066,7 @@ Strings DatabaseDataLake::getAllTableNames(ContextPtr /*context*/) const /// must not fail even when the catalog is temporarily unreachable. try { - result = getCatalog()->getTables(); + result = getCatalog()->getTables(DataLake::getForwardedAuthToken(context_)); } catch (...) { @@ -983,18 +1085,18 @@ ASTPtr DatabaseDataLake::getCreateDatabaseQueryImpl() const return create_query; } -void DatabaseDataLake::checkDatabase() const +void DatabaseDataLake::checkDatabase(ContextPtr context_) const { auto catalog = getCatalog(); /// This function checks if we can access catalog and get tables list. /// We do not check if there are tables in catalog, because even if catalog is empty, it still can be valid and working. - std::ignore = catalog->empty(); + std::ignore = catalog->empty(DataLake::getForwardedAuthToken(context_)); LOG_TEST(log, "Database '{}' is OK", getDatabaseName()); } -void DatabaseDataLake::applySettingsChanges(const SettingsChanges & settings_changes, ContextPtr /*query_context*/) +void DatabaseDataLake::applySettingsChanges(const SettingsChanges & settings_changes, ContextPtr query_context) { const auto current_settings = database_settings.get(); @@ -1043,7 +1145,7 @@ void DatabaseDataLake::applySettingsChanges(const SettingsChanges & settings_cha /// fetch and the config reload may throw, and then nothing has changed yet. DataLake::ICatalog::PreparedSettingsChangesPtr prepared_catalog_changes; if (local_catalog_snapshot) - prepared_catalog_changes = local_catalog_snapshot->prepareSettingsChanges(settings_changes); + prepared_catalog_changes = local_catalog_snapshot->prepareSettingsChanges(settings_changes, DataLake::getForwardedAuthToken(query_context)); /// Persist the new metadata before publishing anything: if the write fails, the live /// state is untouched and matches the old metadata on disk. The create query is built @@ -1179,6 +1281,39 @@ void registerDatabaseDataLake(DatabaseFactory & factory) } } + /// Validate only on `CREATE` so older persisted databases can still attach at startup. + if (!args.create_query.attach) + { + const bool forwarding = database_settings[DatabaseDataLakeSetting::oauth_forward_user_token].value; + static constexpr std::array exchange_only_settings = { + "oauth_token_exchange_uri", + "oauth_subject_token_type", + "oauth_requested_token_type", + "oauth_forward_actor_token", + "oauth_user_token_cache_ttl", + }; + + const SettingsChanges changed = database_settings.allChanged(); + auto is_changed = [&](std::string_view name) + { + return std::any_of(changed.begin(), changed.end(), [&](const auto & change) { return std::string_view(change.name) == name; }); + }; + + for (const auto & name : exchange_only_settings) + { + if (!is_changed(name)) + continue; + if (!forwarding) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "`{}` has no effect without `oauth_forward_user_token = 1`", name); + if (name != "oauth_token_exchange_uri" && database_settings[DatabaseDataLakeSetting::oauth_token_exchange_uri].value.empty()) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "`{}` has no effect without `oauth_token_exchange_uri`", name); + } + } + auto catalog_type = database_settings[DB::DatabaseDataLakeSetting::catalog_type].value; /// Glue catalog is one per region, so it's fully identified by aws keys and region /// There is no URL you need to provide in constructor, even if we would want it diff --git a/src/Databases/DataLake/DatabaseDataLake.h b/src/Databases/DataLake/DatabaseDataLake.h index fc67aad2b133..6cab4caf0f58 100644 --- a/src/Databases/DataLake/DatabaseDataLake.h +++ b/src/Databases/DataLake/DatabaseDataLake.h @@ -3,6 +3,7 @@ #if USE_AVRO && USE_PARQUET +#include #include #include #include @@ -51,7 +52,7 @@ class DatabaseDataLake final : public IDatabase, WithContext Strings getAllTableNames(ContextPtr context) const override; - void checkDatabase() const override; + void checkDatabase(ContextPtr context) const override; void shutdown() override {} @@ -92,6 +93,10 @@ class DatabaseDataLake final : public IDatabase, WithContext void validateSettings(); + void validateTokenForwardingSettings() const; + + static void validateGlueTokenForwardingSettings(const DatabaseDataLakeSettings & settings); + /// Builds `catalog_impl` based on the configured catalog type. Constructing a catalog can /// validate credentials and perform network I/O (e.g. RestCatalog reads the catalog config), /// so on ATTACH (server startup) it is deferred to the first access via `getCatalog` instead diff --git a/src/Databases/DataLake/DatabaseDataLakeSettings.cpp b/src/Databases/DataLake/DatabaseDataLakeSettings.cpp index b4cf78f25a33..e32ebcb32df3 100644 --- a/src/Databases/DataLake/DatabaseDataLakeSettings.cpp +++ b/src/Databases/DataLake/DatabaseDataLakeSettings.cpp @@ -25,6 +25,12 @@ namespace ErrorCodes DECLARE(String, auth_scope, "PRINCIPAL_ROLE:ALL", "Authorization scope for client credentials or token exchange", 0) \ DECLARE(String, oauth_server_uri, "", "OAuth server uri", 0) \ DECLARE(Bool, oauth_server_use_request_body, true, "Put parameters into request body or query params", 0) \ + DECLARE(Bool, oauth_forward_user_token, false, "Authenticate to Iceberg REST or Glue using the querying user's token. Requires `enable_token_forwarding`", 0) \ + DECLARE(String, oauth_token_exchange_uri, "", "RFC 8693 token-exchange endpoint for Iceberg REST. Empty forwards the token unchanged. Requires `oauth_forward_user_token` and a non-empty `catalog_credential`", 0) \ + DECLARE(String, oauth_subject_token_type, "urn:ietf:params:oauth:token-type:access_token", "RFC 8693 `subject_token_type` of the forwarded user token. Used only when `oauth_token_exchange_uri` is set", 0) \ + DECLARE(String, oauth_requested_token_type, "urn:ietf:params:oauth:token-type:access_token", "RFC 8693 `requested_token_type`; empty omits the field. Used only when `oauth_token_exchange_uri` is set", 0) \ + DECLARE(Bool, oauth_forward_actor_token, false, "Include the service principal token as the RFC 8693 `actor_token`. Requires `oauth_token_exchange_uri` and an endpoint that supports delegation", 0) \ + DECLARE(UInt64, oauth_user_token_cache_ttl, 300, "Maximum lifetime (in seconds) of a cached per-user session token obtained by token exchange; '0' disables caching. Used only when `oauth_token_exchange_uri` is set", 0) \ DECLARE(String, warehouse, "", "Warehouse name inside the catalog", 0) \ DECLARE(String, auth_header, "", "Authorization header of format 'Authorization: '", 0) \ DECLARE(String, aws_access_key_id, "", "Key for AWS connection for Glue catalog", 0) \ diff --git a/src/Databases/DataLake/GlueCatalog.cpp b/src/Databases/DataLake/GlueCatalog.cpp index 41c343bc4842..478c14bff683 100644 --- a/src/Databases/DataLake/GlueCatalog.cpp +++ b/src/Databases/DataLake/GlueCatalog.cpp @@ -62,6 +62,7 @@ namespace DB::ErrorCodes extern const int DATALAKE_DATABASE_ERROR; extern const int FAULT_INJECTED; extern const int CATALOG_NAMESPACE_DISABLED; + extern const int CATALOG_USER_TOKEN_NOT_AVAILABLE; } namespace DB::FailPoints @@ -103,12 +104,18 @@ namespace ProfileEvents extern const Event DataLakeGlueCatalogUpdateTableMicroseconds; extern const Event DataLakeGlueCatalogDropTable; extern const Event DataLakeGlueCatalogDropTableMicroseconds; + extern const Event DataLakeGlueCatalogUserClientCacheHits; + extern const Event DataLakeGlueCatalogAssumeRoleWithWebIdentity; + extern const Event DataLakeGlueCatalogAssumeRoleWithWebIdentityMicroseconds; + extern const Event DataLakeGlueCatalogAssumeRoleWithWebIdentityFailures; } namespace CurrentMetrics { extern const Metric MarkCacheBytes; extern const Metric MarkCacheFiles; + extern const Metric DataLakeCatalogUserClientCacheBytes; + extern const Metric DataLakeCatalogUserClientCacheEntries; } namespace @@ -195,6 +202,29 @@ Poco::JSON::Object::Ptr getCurrentSchemaFromMetadata(const Poco::JSON::Object::P namespace DataLake { +namespace +{ + +std::string makeRoleSessionName(const std::string & principal) +{ + std::string result; + result.reserve(std::min(principal.size(), 64)); + for (char c : principal) + { + if (result.size() == 64) + break; + if (isalnum(static_cast(c)) || c == '_' || c == '+' || c == '=' || c == ',' || c == '.' || c == '@' || c == '-') + result += c; + } + + if (result.size() < 2) + return "ClickHouseUser"; + + return result; +} + +} + GlueCatalog::GlueCatalog( const String & endpoint, DB::ContextPtr context_, @@ -202,6 +232,10 @@ GlueCatalog::GlueCatalog( DB::ASTPtr table_engine_definition_) : ICatalog("") , DB::WithContext(context_) + , user_clients( + CurrentMetrics::DataLakeCatalogUserClientCacheBytes, + CurrentMetrics::DataLakeCatalogUserClientCacheEntries, + user_client_cache_max_entries) , log(getLogger("GlueCatalog(" + settings_.region + ")")) , region(settings_.region) , settings(settings_) @@ -249,14 +283,12 @@ GlueCatalog::GlueCatalog( client_configuration.connectTimeoutMs = static_cast(global_settings[DB::Setting::s3_connect_timeout_ms]); client_configuration.requestTimeoutMs = static_cast(global_settings[DB::Setting::s3_request_timeout_ms]); client_configuration.region = region; - auto endpoint_provider = std::make_shared(); Aws::Auth::AWSCredentials credentials(settings_.aws_access_key_id, settings_.aws_secret_access_key); /// Only for testing when we are mocking glue if (!endpoint.empty()) { client_configuration.endpointOverride = endpoint; - endpoint_provider->OverrideEndpoint(endpoint); if (credentials.IsEmpty()) { @@ -276,13 +308,85 @@ GlueCatalog::GlueCatalog( } boost::split(allowed_namespaces, settings.namespaces, boost::is_any_of(", "), boost::token_compress_on); - credentials_provider = DB::S3::getCredentialsProvider(poco_config, credentials, creds_config); - glue_client = std::make_unique(credentials_provider, endpoint_provider, client_configuration); + + /// Each `GlueClient` owns its endpoint resolver state. + auto build_glue_client = [client_configuration, endpoint](const std::shared_ptr & provider) + { + auto client_endpoint_provider = std::make_shared(); + if (!endpoint.empty()) + client_endpoint_provider->OverrideEndpoint(endpoint); + return std::make_shared(provider, client_endpoint_provider, client_configuration); + }; + + if (settings.forward_user_token) + { + make_user_client = [build_glue_client, + poco_config, + role_arn = settings.aws_role_arn, + expiration_window_seconds = creds_config.expiration_window_seconds, + logger = log](const DB::ForwardedAuthToken & auth_token) + { + auto sts_client = std::make_shared( + std::make_shared(), poco_config); + + auto provider = std::make_shared( + role_arn, + makeRoleSessionName(auth_token.principal), + auth_token.token, + expiration_window_seconds, + std::move(sts_client)); + + bool assumed = false; + { + ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogAssumeRoleWithWebIdentity); + auto timer = DB::CurrentThread::getProfileEvents().timer( + ProfileEvents::DataLakeGlueCatalogAssumeRoleWithWebIdentityMicroseconds); + assumed = !provider->GetAWSCredentials().IsEmpty(); + } + + if (!assumed) + { + ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogAssumeRoleWithWebIdentityFailures); + throw DB::Exception( + DB::ErrorCodes::CATALOG_USER_TOKEN_NOT_AVAILABLE, + "Could not assume role `{}` with the token of user `{}`: {}. " + "Check the role's trust policy for the token's issuer, `sub`, and `aud`.", + role_arn, + auth_token.principal, + provider->getLastError()); + } + + LOG_DEBUG(logger, "Assumed role {} as user {}", role_arn, auth_token.principal); + return AuthenticatedClient{build_glue_client(provider), provider}; + }; + } + else + { + auto service_credentials_provider = DB::S3::getCredentialsProvider(poco_config, credentials, creds_config); + service_client = AuthenticatedClient{build_glue_client(service_credentials_provider), service_credentials_provider}; + } +} + +GlueCatalog::AuthenticatedClient GlueCatalog::getClient(const DB::ForwardedAuthTokenPtr & auth_token) const +{ + if (!make_user_client) + return service_client; + + validateForwardedToken(getContext(), auth_token, fmt::format("Glue({})", region)); + + auto [client, outcome] = user_clients.getOrSetWithOutcome( + auth_token->fingerprint, + [&] { return std::make_shared(make_user_client(*auth_token)); }); + + if (outcome == DB::CacheGetOrSetOutcome::Hit) + ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogUserClientCacheHits); + + return *client; } GlueCatalog::~GlueCatalog() = default; -DataLake::ICatalog::Namespaces GlueCatalog::getDatabases(const std::string & prefix, size_t limit) const +DataLake::ICatalog::Namespaces GlueCatalog::getDatabases(const AuthenticatedClient & client, const std::string & prefix, size_t limit) const { DataLake::ICatalog::Namespaces result; Aws::Glue::Model::GetDatabasesRequest request; @@ -299,7 +403,7 @@ DataLake::ICatalog::Namespaces GlueCatalog::getDatabases(const std::string & pre { ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogGetDatabases); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeGlueCatalogGetDatabasesMicroseconds); - outcome = glue_client->GetDatabases(request); + outcome = client.client->GetDatabases(request); } if (outcome.IsSuccess()) @@ -333,7 +437,7 @@ DataLake::ICatalog::Namespaces GlueCatalog::getDatabases(const std::string & pre return result; } -DB::Names GlueCatalog::getTablesForDatabase(const std::string & db_name, size_t limit) const +DB::Names GlueCatalog::getTablesForDatabase(const AuthenticatedClient & client, const std::string & db_name, size_t limit) const { LOG_TEST(log, "Getting tables for database '{}' with limit {}", db_name, limit); DB::Names result; @@ -355,7 +459,7 @@ DB::Names GlueCatalog::getTablesForDatabase(const std::string & db_name, size_t { ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogGetTables); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeGlueCatalogGetTablesMicroseconds); - outcome = glue_client->GetTables(request); + outcome = client.client->GetTables(request); } if (outcome.IsSuccess()) { @@ -388,20 +492,23 @@ DB::Names GlueCatalog::getTablesForDatabase(const std::string & db_name, size_t return result; } -DB::Names GlueCatalog::getTables() const +DB::Names GlueCatalog::getTables(const DB::ForwardedAuthTokenPtr & auth_token) const { - auto databases = getDatabases(""); + auto client = getClient(auth_token); + auto databases = getDatabases(client, ""); DB::Names result; for (const auto & database : databases) { - auto tables_in_database = getTablesForDatabase(database); + auto tables_in_database = getTablesForDatabase(client, database); result.insert(result.end(), tables_in_database.begin(), tables_in_database.end()); } return result; } -bool GlueCatalog::existsTable(const std::string & database_name, const std::string & table_name) const +bool GlueCatalog::existsTable(const std::string & database_name, const std::string & table_name, const DB::ForwardedAuthTokenPtr & auth_token) const { + auto client = getClient(auth_token); + if (!isNamespaceAllowed(database_name)) throw DB::Exception(DB::ErrorCodes::CATALOG_NAMESPACE_DISABLED, "Namespace {} is filtered by `namespaces` database parameter", database_name); @@ -411,7 +518,7 @@ bool GlueCatalog::existsTable(const std::string & database_name, const std::stri ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogGetTable); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeGlueCatalogGetTableMicroseconds); - auto outcome = glue_client->GetTable(request); + auto outcome = client.client->GetTable(request); return outcome.IsSuccess(); } @@ -421,6 +528,8 @@ bool GlueCatalog::tryGetTableMetadata( DB::ContextPtr context_, TableMetadata & result) const { + auto client = getClient(getForwardedAuthToken(context_)); + if (!isNamespaceAllowed(database_name)) throw DB::Exception(DB::ErrorCodes::CATALOG_NAMESPACE_DISABLED, "Namespace {} is filtered by `namespaces` database parameter", database_name); @@ -432,7 +541,7 @@ bool GlueCatalog::tryGetTableMetadata( { ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogGetTable); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeGlueCatalogGetTableMicroseconds); - outcome = glue_client->GetTable(request); + outcome = client.client->GetTable(request); } if (outcome.IsSuccess()) { @@ -458,7 +567,7 @@ bool GlueCatalog::tryGetTableMetadata( } if (result.requiresCredentials()) - setCredentials(result); + setCredentials(client, result); auto setup_specific_properties = [&] { @@ -473,7 +582,7 @@ bool GlueCatalog::tryGetTableMetadata( if (!location_with_slash.ends_with('/')) location_with_slash += '/'; - String resolved_metadata_path = resolveMetadataPathFromTableLocation(location_with_slash, result); + String resolved_metadata_path = resolveMetadataPathFromTableLocation(client, location_with_slash, result); if (resolved_metadata_path.empty()) { result.setTableIsNotReadable(fmt::format("Could not determine metadata_location of table `{}`. ", @@ -518,7 +627,7 @@ bool GlueCatalog::tryGetTableMetadata( { if (!result.requiresDataLakeSpecificProperties()) setup_specific_properties(); - column_type = getActualTimestampType(column.GetName(), result, column_type); + column_type = getActualTimestampType(client, column.GetName(), result, column_type); } schema.push_back({column.GetName(), getType(column_type, can_be_nullable, getContext())}); @@ -535,7 +644,7 @@ bool GlueCatalog::tryGetTableMetadata( auto table_specific_properties = result.getDataLakeSpecificProperties(); if (table_specific_properties.has_value() && !table_specific_properties->iceberg_metadata_file_location.empty()) { - auto metadata_object = getOrFetchMetadataObject(table_specific_properties->iceberg_metadata_file_location, result); + auto metadata_object = getOrFetchMetadataObject(client, table_specific_properties->iceberg_metadata_file_location, result); const bool allow_geo_parser = getContext()->getSettingsRef()[DB::Setting::allow_experimental_geo_types_in_iceberg].value; auto schema_processor = DB::Iceberg::IcebergSchemaProcessor(context_, allow_geo_parser); @@ -577,13 +686,13 @@ void GlueCatalog::getTableMetadata( } } -void GlueCatalog::setCredentials(TableMetadata & metadata) const +void GlueCatalog::setCredentials(const AuthenticatedClient & client, TableMetadata & metadata) const { auto storage_type = parseStorageTypeFromLocation(metadata.getLocation()); if (storage_type == StorageType::S3) { - auto credentials = credentials_provider->GetAWSCredentials(); + auto credentials = client.credentials_provider->GetAWSCredentials(); auto s3_creds = std::make_shared(credentials.GetAWSAccessKeyId(), credentials.GetAWSSecretKey(), credentials.GetSessionToken()); metadata.setStorageCredentials(s3_creds); } @@ -594,7 +703,8 @@ void GlueCatalog::setCredentials(TableMetadata & metadata) const } } -ICatalog::CredentialsRefreshCallback GlueCatalog::getCredentialsConfigurationCallback(const DB::StorageID & storage_id) +ICatalog::CredentialsRefreshCallback GlueCatalog::getCredentialsConfigurationCallback( + const DB::StorageID & storage_id, const DB::ForwardedAuthTokenPtr & auth_token) { /// The AWS SDK credentials provider chain (instance profile, STS assume-role, /// web-identity, etc.) refreshes its cached credentials internally before @@ -604,7 +714,9 @@ ICatalog::CredentialsRefreshCallback GlueCatalog::getCredentialsConfigurationCal /// S3 client is pinned to a snapshot that goes stale on long reads. This /// callback re-asks the same provider for current credentials each time /// `ReadBufferFromS3` reports an `ExpiredToken`, letting the read recover. - return [this, storage_id]() -> std::shared_ptr + auto credentials_provider = getClient(auth_token).credentials_provider; + + return [this, storage_id, credentials_provider]() -> std::shared_ptr { LOG_DEBUG(log, "Refreshing AWS credentials for {} after expired token", storage_id.getNameForLogs()); auto credentials = credentials_provider->GetAWSCredentials(); @@ -615,22 +727,24 @@ ICatalog::CredentialsRefreshCallback GlueCatalog::getCredentialsConfigurationCal }; } -bool GlueCatalog::empty() const +bool GlueCatalog::empty(const DB::ForwardedAuthTokenPtr & auth_token) const { - auto all_databases = getDatabases(""); + auto client = getClient(auth_token); + auto all_databases = getDatabases(client, ""); for (const auto & db : all_databases) { - if (!getTablesForDatabase(db, /* limit = */ 1).empty()) + if (!getTablesForDatabase(client, db, /* limit = */ 1).empty()) return false; } return true; } -Poco::JSON::Object::Ptr GlueCatalog::getOrFetchMetadataObject(const String & metadata_uri, const TableMetadata & table_metadata) const +Poco::JSON::Object::Ptr GlueCatalog::getOrFetchMetadataObject( + const AuthenticatedClient & client, const String & metadata_uri, const TableMetadata & table_metadata) const { auto [value, _] = metadata_objects.getOrSet(metadata_uri, [&]() { - auto [object_storage, bucket_name, metadata_path] = createObjectStorageForEarlyTableAccess(metadata_uri, table_metadata); + auto [object_storage, bucket_name, metadata_path] = createObjectStorageForEarlyTableAccess(client, metadata_uri, table_metadata); auto compression_method = DB::Iceberg::getCompressionMethodFromMetadataFile(metadata_uri); auto metadata_object = DB::Iceberg::getMetadataJSONObject( metadata_path, object_storage, nullptr, getContext(), log, compression_method, std::nullopt); @@ -639,13 +753,17 @@ Poco::JSON::Object::Ptr GlueCatalog::getOrFetchMetadataObject(const String & met return *value; } -String GlueCatalog::getActualTimestampType(const String & column_name, const TableMetadata & table_metadata, const String & glue_column_type) const +String GlueCatalog::getActualTimestampType( + const AuthenticatedClient & client, + const String & column_name, + const TableMetadata & table_metadata, + const String & glue_column_type) const { auto table_specific_properties = table_metadata.getDataLakeSpecificProperties(); if (!table_specific_properties.has_value()) throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Failed to read table metadata, reason why table is unreadable: {}", table_metadata.getReasonWhyTableIsUnreadable()); - auto metadata_object = getOrFetchMetadataObject(table_specific_properties->iceberg_metadata_file_location, table_metadata); + auto metadata_object = getOrFetchMetadataObject(client, table_specific_properties->iceberg_metadata_file_location, table_metadata); return resolveTimestampTypeFromMetadata(metadata_object, column_name, glue_column_type); } @@ -674,7 +792,8 @@ String GlueCatalog::resolveTimestampTypeFromMetadata( return glue_column_type == "timestamp_nano" ? "timestamp_ns" : "timestamp"; } -GlueCatalog::ObjectStorageWithPath GlueCatalog::createObjectStorageForEarlyTableAccess(const String & s3_location, const TableMetadata & table_metadata) const +GlueCatalog::ObjectStorageWithPath GlueCatalog::createObjectStorageForEarlyTableAccess( + const AuthenticatedClient & client, const String & s3_location, const TableMetadata & table_metadata) const { DB::ASTStorage * storage = table_engine_definition->as(); DB::ASTs args = storage->engine->arguments->children; @@ -693,7 +812,7 @@ GlueCatalog::ObjectStorageWithPath GlueCatalog::createObjectStorageForEarlyTable } else { - auto credentials = credentials_provider->GetAWSCredentials(); + auto credentials = client.credentials_provider->GetAWSCredentials(); DataLake::S3Credentials(credentials.GetAWSAccessKeyId(), credentials.GetAWSSecretKey(), credentials.GetSessionToken()).addCredentialsToEngineArgs(args); } } @@ -726,9 +845,10 @@ GlueCatalog::ObjectStorageWithPath GlueCatalog::createObjectStorageForEarlyTable return {object_storage, bucket_name, table_path}; } -String GlueCatalog::resolveMetadataPathFromTableLocation(const String & table_location, const TableMetadata & table_metadata) const +String GlueCatalog::resolveMetadataPathFromTableLocation( + const AuthenticatedClient & client, const String & table_location, const TableMetadata & table_metadata) const { - auto [object_storage, bucket_name, table_path] = createObjectStorageForEarlyTableAccess(table_location, table_metadata); + auto [object_storage, bucket_name, table_path] = createObjectStorageForEarlyTableAccess(client, table_location, table_metadata); auto storage_settings = std::make_shared(); storage_settings->loadFromSettingsChanges(settings.allChanged()); @@ -749,8 +869,10 @@ String GlueCatalog::resolveMetadataPathFromTableLocation(const String & table_lo } } -void GlueCatalog::createNamespaceIfNotExists(const String & namespace_name, const String & /*location*/) const +void GlueCatalog::createNamespaceIfNotExists(const String & namespace_name, const String & /*location*/, const DB::ForwardedAuthTokenPtr & auth_token) const { + auto client = getClient(auth_token); + Aws::Glue::Model::CreateDatabaseRequest create_request; Aws::Glue::Model::DatabaseInput db_input; db_input.SetName(namespace_name); @@ -758,7 +880,7 @@ void GlueCatalog::createNamespaceIfNotExists(const String & namespace_name, cons ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogCreateDatabase); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeGlueCatalogCreateDatabaseMicroseconds); - auto outcome = glue_client->CreateDatabase(create_request); + auto outcome = client.client->CreateDatabase(create_request); if (!outcome.IsSuccess() && outcome.GetError().GetErrorType() != Aws::Glue::GlueErrors::ALREADY_EXISTS) { throw DB::Exception( @@ -768,8 +890,10 @@ void GlueCatalog::createNamespaceIfNotExists(const String & namespace_name, cons } } -void GlueCatalog::createTable(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr metadata_content) const +void GlueCatalog::createTable(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr metadata_content, const DB::ForwardedAuthTokenPtr & auth_token) const { + auto client = getClient(auth_token); + if (!isNamespaceAllowed(namespace_name)) throw DB::Exception(DB::ErrorCodes::CATALOG_NAMESPACE_DISABLED, "Failed to create table {}, namespace {} is filtered by `namespaces` database parameter", @@ -811,7 +935,7 @@ void GlueCatalog::createTable(const String & namespace_name, const String & tabl { ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogCreateTable); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeGlueCatalogCreateTableMicroseconds); - response = glue_client->CreateTable(request); + response = client.client->CreateTable(request); } if (!response.IsSuccess()) @@ -822,8 +946,11 @@ bool GlueCatalog::updateTableInGlue( const String & namespace_name, const String & table_name, const String & new_metadata_path, + const DB::ForwardedAuthTokenPtr & auth_token, const std::vector & columns) const { + auto client = getClient(auth_token); + Aws::Glue::Model::UpdateTableRequest request; request.SetDatabaseName(namespace_name); @@ -859,7 +986,7 @@ bool GlueCatalog::updateTableInGlue( { ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogUpdateTable); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeGlueCatalogUpdateTableMicroseconds); - response = glue_client->UpdateTable(request); + response = client.client->UpdateTable(request); } if (!response.IsSuccess()) @@ -868,9 +995,9 @@ bool GlueCatalog::updateTableInGlue( return true; } -bool GlueCatalog::updateMetadata(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr /*new_snapshot*/) const +bool GlueCatalog::updateMetadata(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr /*new_snapshot*/, const DB::ForwardedAuthTokenPtr & auth_token) const { - return updateTableInGlue(namespace_name, table_name, new_metadata_path); + return updateTableInGlue(namespace_name, table_name, new_metadata_path, auth_token); } bool GlueCatalog::updateSchema( @@ -880,16 +1007,19 @@ bool GlueCatalog::updateSchema( Poco::JSON::Object::Ptr new_schema, Int32 /*previous_schema_id*/, Int32 /*new_last_column_id*/, - Poco::JSON::Object::Ptr /*metadata*/) const + Poco::JSON::Object::Ptr /*metadata*/, + const DB::ForwardedAuthTokenPtr & auth_token) const { std::vector columns; if (new_schema) columns = icebergSchemaToGlueColumns(new_schema); - return updateTableInGlue(namespace_name, table_name, new_metadata_path, columns); + return updateTableInGlue(namespace_name, table_name, new_metadata_path, auth_token, columns); } -void GlueCatalog::dropTable(const String & namespace_name, const String & table_name) const +void GlueCatalog::dropTable(const String & namespace_name, const String & table_name, const DB::ForwardedAuthTokenPtr & auth_token) const { + auto client = getClient(auth_token); + if (!isNamespaceAllowed(namespace_name)) throw DB::Exception(DB::ErrorCodes::CATALOG_NAMESPACE_DISABLED, "Failed to drop table {}, namespace {} is filtered by `namespaces` database parameter", @@ -904,7 +1034,7 @@ void GlueCatalog::dropTable(const String & namespace_name, const String & table_ { ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogDropTable); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeGlueCatalogDropTableMicroseconds); - response = glue_client->DeleteTable(request); + response = client.client->DeleteTable(request); } if (!response.IsSuccess()) diff --git a/src/Databases/DataLake/GlueCatalog.h b/src/Databases/DataLake/GlueCatalog.h index 4722bcec7f54..82988ad10db1 100644 --- a/src/Databases/DataLake/GlueCatalog.h +++ b/src/Databases/DataLake/GlueCatalog.h @@ -12,6 +12,8 @@ #include #include + +#include #include namespace Aws::Glue @@ -38,11 +40,11 @@ class GlueCatalog final : public ICatalog, private DB::WithContext ~GlueCatalog() override; - bool empty() const override; + bool empty(const DB::ForwardedAuthTokenPtr & auth_token) const override; - DB::Names getTables() const override; + DB::Names getTables(const DB::ForwardedAuthTokenPtr & auth_token) const override; - bool existsTable(const std::string & database_name, const std::string & table_name) const override; + bool existsTable(const std::string & database_name, const std::string & table_name, const DB::ForwardedAuthTokenPtr & auth_token) const override; void getTableMetadata( const std::string & database_name, @@ -50,6 +52,8 @@ class GlueCatalog final : public ICatalog, private DB::WithContext DB::ContextPtr context_, TableMetadata & result) const override; + void onTokenForwardingDisabled() const override { user_clients.clear(); } + bool tryGetTableMetadata( const std::string & database_name, const std::string & table_name, @@ -67,11 +71,11 @@ class GlueCatalog final : public ICatalog, private DB::WithContext return DB::DatabaseDataLakeCatalogType::GLUE; } - void createTable(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr metadata_content) const override; + void createTable(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr metadata_content, const DB::ForwardedAuthTokenPtr & auth_token) const override; - void createNamespaceIfNotExists(const String & namespace_name, const String & location) const override; + void createNamespaceIfNotExists(const String & namespace_name, const String & location, const DB::ForwardedAuthTokenPtr & auth_token) const override; - bool updateMetadata(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr new_snapshot) const override; + bool updateMetadata(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr new_snapshot, const DB::ForwardedAuthTokenPtr & auth_token) const override; bool updateSchema( const String & namespace_name, @@ -80,15 +84,17 @@ class GlueCatalog final : public ICatalog, private DB::WithContext Poco::JSON::Object::Ptr new_schema, Int32 previous_schema_id, Int32 new_last_column_id, - Poco::JSON::Object::Ptr metadata = nullptr) const override; + Poco::JSON::Object::Ptr metadata, + const DB::ForwardedAuthTokenPtr & auth_token) const override; - void dropTable(const String & namespace_name, const String & table_name) const override; + void dropTable(const String & namespace_name, const String & table_name, const DB::ForwardedAuthTokenPtr & auth_token) const override; /// Returns a callback that re-vends fresh AWS credentials from the configured /// credentials provider chain. Invoked by `ReadBufferFromS3` when an S3 call /// fails with `ExpiredToken`, so that a long-running read can recover without /// the user having to restart the query. - ICatalog::CredentialsRefreshCallback getCredentialsConfigurationCallback(const DB::StorageID & storage_id) override; + ICatalog::CredentialsRefreshCallback getCredentialsConfigurationCallback( + const DB::StorageID & storage_id, const DB::ForwardedAuthTokenPtr & auth_token) override; /// Resolves the precise Iceberg timestamp type for `column_name` by searching the current schema /// in the Iceberg `metadata_object`. Falls back to `"timestamp_ns"` when `glue_column_type` is @@ -99,9 +105,21 @@ class GlueCatalog final : public ICatalog, private DB::WithContext const String & glue_column_type); private: - std::unique_ptr glue_client; + struct AuthenticatedClient + { + std::shared_ptr client; + std::shared_ptr credentials_provider; + }; + + AuthenticatedClient service_client; + std::function make_user_client; + + static constexpr size_t user_client_cache_max_entries = 1024; + mutable DB::CacheBase user_clients; + + AuthenticatedClient getClient(const DB::ForwardedAuthTokenPtr & auth_token) const; + const LoggerPtr log; - std::shared_ptr credentials_provider; std::string region; CatalogSettings settings; DB::ASTPtr table_engine_definition; @@ -109,16 +127,21 @@ class GlueCatalog final : public ICatalog, private DB::WithContext bool isNamespaceAllowed(const std::string & namespace_) const; - DataLake::ICatalog::Namespaces getDatabases(const std::string & prefix, size_t limit = 0) const; - DB::Names getTablesForDatabase(const std::string & db_name, size_t limit = 0) const; - void setCredentials(TableMetadata & metadata) const; + DataLake::ICatalog::Namespaces getDatabases(const AuthenticatedClient & client, const std::string & prefix, size_t limit = 0) const; + DB::Names getTablesForDatabase(const AuthenticatedClient & client, const std::string & db_name, size_t limit = 0) const; + void setCredentials(const AuthenticatedClient & client, TableMetadata & metadata) const; /// The Glue catalog does not store detailed information about the types of timestamp columns, such as whether the column is timestamp or timestamptz. /// This method allows to clarify the actual type of the timestamp column. /// `glue_column_type` is the raw Glue type (`"timestamp"` or `"timestamp_nano"`) used as a fallback when the column is not found in Iceberg metadata. - String getActualTimestampType(const String & column_name, const TableMetadata & table_metadata, const String & glue_column_type) const; + String getActualTimestampType( + const AuthenticatedClient & client, + const String & column_name, + const TableMetadata & table_metadata, + const String & glue_column_type) const; - String resolveMetadataPathFromTableLocation(const String & table_location, const TableMetadata & table_metadata) const; + String resolveMetadataPathFromTableLocation( + const AuthenticatedClient & client, const String & table_location, const TableMetadata & table_metadata) const; struct ObjectStorageWithPath { @@ -127,11 +150,13 @@ class GlueCatalog final : public ICatalog, private DB::WithContext String table_path; /// Path within bucket }; - ObjectStorageWithPath createObjectStorageForEarlyTableAccess(const String & s3_location, const TableMetadata & table_metadata) const; + ObjectStorageWithPath createObjectStorageForEarlyTableAccess( + const AuthenticatedClient & client, const String & s3_location, const TableMetadata & table_metadata) const; /// Fetches and caches the parsed Iceberg metadata JSON for `metadata_uri`. /// Returns the cached object on subsequent calls for the same URI. - Poco::JSON::Object::Ptr getOrFetchMetadataObject(const String & metadata_uri, const TableMetadata & table_metadata) const; + Poco::JSON::Object::Ptr getOrFetchMetadataObject( + const AuthenticatedClient & client, const String & metadata_uri, const TableMetadata & table_metadata) const; /// Shared implementation for updateMetadata / updateSchema that optionally /// sets StorageDescriptor columns in the Glue UpdateTable call. @@ -139,6 +164,7 @@ class GlueCatalog final : public ICatalog, private DB::WithContext const String & namespace_name, const String & table_name, const String & new_metadata_path, + const DB::ForwardedAuthTokenPtr & auth_token, const std::vector & columns = {}) const; mutable DB::CacheBase metadata_objects; diff --git a/src/Databases/DataLake/HiveCatalog.cpp b/src/Databases/DataLake/HiveCatalog.cpp index 5170e45171df..09aaf3817a60 100644 --- a/src/Databases/DataLake/HiveCatalog.cpp +++ b/src/Databases/DataLake/HiveCatalog.cpp @@ -141,7 +141,7 @@ void HiveCatalog::executeWithRetry(Func && func) const DB::ErrorCodes::NO_HIVEMETASTORE, "Hive Metastore connection failed after {} attempts. Last error: {}", max_retries, last_err_msg); } -bool HiveCatalog::empty() const +bool HiveCatalog::empty(const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { fiu_do_on(DB::FailPoints::check_database_datalake_negative, { @@ -154,7 +154,7 @@ bool HiveCatalog::empty() const return result.empty(); } -DB::Names HiveCatalog::getTables() const +DB::Names HiveCatalog::getTables(const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { DB::Names result; DB::Names databases; @@ -171,7 +171,7 @@ DB::Names HiveCatalog::getTables() const return result; } -bool HiveCatalog::existsTable(const std::string & namespace_name, const std::string & table_name) const +bool HiveCatalog::existsTable(const std::string & namespace_name, const std::string & table_name, const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { Apache::Hadoop::Hive::Table table; diff --git a/src/Databases/DataLake/HiveCatalog.h b/src/Databases/DataLake/HiveCatalog.h index d626b73c3871..9219f27b3989 100644 --- a/src/Databases/DataLake/HiveCatalog.h +++ b/src/Databases/DataLake/HiveCatalog.h @@ -32,11 +32,11 @@ class HiveCatalog final : public ICatalog, private DB::WithContext ~HiveCatalog() override = default; - bool empty() const override; + bool empty(const DB::ForwardedAuthTokenPtr & auth_token) const override; - DB::Names getTables() const override; + DB::Names getTables(const DB::ForwardedAuthTokenPtr & auth_token) const override; - bool existsTable(const std::string & namespace_name, const std::string & table_name) const override; + bool existsTable(const std::string & namespace_name, const std::string & table_name, const DB::ForwardedAuthTokenPtr & auth_token) const override; void getTableMetadata( const std::string & namespace_name, diff --git a/src/Databases/DataLake/ICatalog.cpp b/src/Databases/DataLake/ICatalog.cpp index ddeeaa25afbf..e6c5a808351a 100644 --- a/src/Databases/DataLake/ICatalog.cpp +++ b/src/Databases/DataLake/ICatalog.cpp @@ -9,11 +9,15 @@ #include #include +#include +#include + namespace DB::ErrorCodes { extern const int NOT_IMPLEMENTED; extern const int LOGICAL_ERROR; extern const int BAD_ARGUMENTS; + extern const int CATALOG_USER_TOKEN_NOT_AVAILABLE; } namespace DB::DatabaseDataLakeSetting @@ -341,17 +345,39 @@ DB::SettingsChanges CatalogSettings::allChanged() const return changes; } -void ICatalog::createTable(const String & /*namespace_name*/, const String & /*table_name*/, const String & /*new_metadata_path*/, Poco::JSON::Object::Ptr /*metadata_content*/) const +void ICatalog::validateForwardedToken( + const DB::ContextPtr & context, const DB::ForwardedAuthTokenPtr & auth_token, const std::string & catalog_description) const +{ + /// Recheck the hot-reloadable switch so existing sessions stop forwarding when it is disabled. + if (!context->getGlobalContext()->getAccessControl().isTokenForwardingEnabled()) + { + onTokenForwardingDisabled(); + + throw DB::Exception( + DB::ErrorCodes::CATALOG_USER_TOKEN_NOT_AVAILABLE, + "Catalog `{}` requires token forwarding. Set `enable_token_forwarding = 1` and reconnect.", + catalog_description); + } + + if (!auth_token || auth_token->token.empty()) + throw DB::Exception( + DB::ErrorCodes::CATALOG_USER_TOKEN_NOT_AVAILABLE, + "Cannot authenticate to catalog `{}`: this session carries no bearer token. " + "Authenticate with an `Authorization: Bearer` HTTP header or `--jwt` for the native protocol.", + catalog_description); +} + +void ICatalog::createTable(const String & /*namespace_name*/, const String & /*table_name*/, const String & /*new_metadata_path*/, Poco::JSON::Object::Ptr /*metadata_content*/, const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "createTable is not implemented"); } -void ICatalog::createNamespaceIfNotExists(const String & /*namespace_name*/, const String & /*location*/) const +void ICatalog::createNamespaceIfNotExists(const String & /*namespace_name*/, const String & /*location*/, const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "createNamespaceIfNotExists is not implemented"); } -bool ICatalog::updateMetadata(const String & /*namespace_name*/, const String & /*table_name*/, const String & /*new_metadata_path*/, Poco::JSON::Object::Ptr /*new_snapshot*/) const +bool ICatalog::updateMetadata(const String & /*namespace_name*/, const String & /*table_name*/, const String & /*new_metadata_path*/, Poco::JSON::Object::Ptr /*new_snapshot*/, const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "updateMetadata is not implemented"); } @@ -363,17 +389,19 @@ bool ICatalog::updateSchema( Poco::JSON::Object::Ptr /*new_schema*/, Int32 /*previous_schema_id*/, Int32 /*new_last_column_id*/, - Poco::JSON::Object::Ptr /*metadata*/) const + Poco::JSON::Object::Ptr /*metadata*/, + const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "updateSchema is not implemented"); } -void ICatalog::dropTable(const String & /*namespace_name*/, const String & /*table_name*/) const +void ICatalog::dropTable(const String & /*namespace_name*/, const String & /*table_name*/, const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "dropTable is not implemented"); } -ICatalog::PreparedSettingsChangesPtr ICatalog::prepareSettingsChanges(const DB::SettingsChanges & /*changes*/) +ICatalog::PreparedSettingsChangesPtr ICatalog::prepareSettingsChanges( + const DB::SettingsChanges & /*changes*/, const DB::ForwardedAuthTokenPtr & /*auth_token*/) { throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "Settings of a catalog of this type cannot be altered"); } diff --git a/src/Databases/DataLake/ICatalog.h b/src/Databases/DataLake/ICatalog.h index 9cb18c15177b..c1be073a91eb 100644 --- a/src/Databases/DataLake/ICatalog.h +++ b/src/Databases/DataLake/ICatalog.h @@ -1,5 +1,6 @@ #pragma once #include +#include #include #include #include @@ -162,6 +163,7 @@ struct CatalogSettings String aws_role_arn; String aws_role_session_name; String aws_external_id; + bool forward_user_token = false; DB::SettingsChanges allChanged() const; }; @@ -180,16 +182,17 @@ class ICatalog virtual ~ICatalog() = default; /// Does catalog have any tables? - virtual bool empty() const = 0; + virtual bool empty(const DB::ForwardedAuthTokenPtr & auth_token) const = 0; /// Fetch tables' names list. /// Contains full namespaces in names. - virtual DB::Names getTables() const = 0; + virtual DB::Names getTables(const DB::ForwardedAuthTokenPtr & auth_token) const = 0; /// Check that a table exists in a given namespace. virtual bool existsTable( const std::string & namespace_naem, - const std::string & table_name) const = 0; + const std::string & table_name, + const DB::ForwardedAuthTokenPtr & auth_token) const = 0; /// Get table metadata in the given namespace. /// Throw exception if table does not exist. @@ -214,13 +217,13 @@ class ICatalog /// Creates new table in catalog. Callers must ensure the namespace exists before /// writing any table files to storage: a catalog that shares its storage view with /// the data refuses to create a namespace over a plain directory those files create. - virtual void createTable(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr metadata_content) const; + virtual void createTable(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr metadata_content, const DB::ForwardedAuthTokenPtr & auth_token) const; /// Creates the namespace unless it already exists. - virtual void createNamespaceIfNotExists(const String & namespace_name, const String & location) const; + virtual void createNamespaceIfNotExists(const String & namespace_name, const String & location, const DB::ForwardedAuthTokenPtr & auth_token) const; /// Updates metadata in catalog. - virtual bool updateMetadata(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr new_snapshot) const; + virtual bool updateMetadata(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr new_snapshot, const DB::ForwardedAuthTokenPtr & auth_token) const; /// Commit a schema evolution (ADD/DROP/MODIFY/RENAME COLUMN) to the catalog. /// `new_metadata_path` is the path of the freshly written `vN.metadata.json`; it is used by @@ -236,10 +239,11 @@ class ICatalog Poco::JSON::Object::Ptr new_schema, Int32 previous_schema_id, Int32 new_last_column_id, - Poco::JSON::Object::Ptr metadata = nullptr) const; + Poco::JSON::Object::Ptr metadata, + const DB::ForwardedAuthTokenPtr & auth_token) const; /// Drop table from catalog. - virtual void dropTable(const String & namespace_name, const String & table_name) const; + virtual void dropTable(const String & namespace_name, const String & table_name, const DB::ForwardedAuthTokenPtr & auth_token) const; /// Does the catalog support transactions or anything like that? /// For example, the Iceberg REST catalog supports atomic operations "compare if snapshot X is equal to" and "add new snapshot Y". @@ -247,11 +251,15 @@ class ICatalog /// The Glue catalog does not support such operation. virtual bool isTransactional() const { return false; } - virtual CredentialsRefreshCallback getCredentialsConfigurationCallback(const DB::StorageID & /*storage_id*/) + /// The callback outlives the query context, so retain the token directly. + virtual CredentialsRefreshCallback getCredentialsConfigurationCallback( + const DB::StorageID & /*storage_id*/, const DB::ForwardedAuthTokenPtr & /*auth_token*/) { return std::nullopt; } + virtual void onTokenForwardingDisabled() const {} + virtual void setVendedCredentialsCacheTTL(std::chrono::seconds /*ttl*/) {} /// Result of `prepareSettingsChanges`: the new catalog state built off to the side, @@ -266,17 +274,23 @@ class ICatalog /// state without publishing anything (may throw, may do network I/O). The state /// becomes visible only after `commitSettingsChanges`, so the caller can persist /// the changes in between and abandon the prepared state on failure. - virtual PreparedSettingsChangesPtr prepareSettingsChanges(const DB::SettingsChanges & changes); + virtual PreparedSettingsChangesPtr prepareSettingsChanges( + const DB::SettingsChanges & changes, const DB::ForwardedAuthTokenPtr & auth_token = {}); /// Publish the state built by `prepareSettingsChanges`. Must not fail. virtual void commitSettingsChanges(PreparedSettingsChangesPtr prepared); - void applySettingsChanges(const DB::SettingsChanges & changes) + void applySettingsChanges(const DB::SettingsChanges & changes, const DB::ForwardedAuthTokenPtr & auth_token = {}) { - commitSettingsChanges(prepareSettingsChanges(changes)); + commitSettingsChanges(prepareSettingsChanges(changes, auth_token)); } protected: + void validateForwardedToken( + const DB::ContextPtr & context, + const DB::ForwardedAuthTokenPtr & auth_token, + const std::string & catalog_description) const; + /// Name of the warehouse, /// which is sometimes also called "catalog name". const std::string warehouse; diff --git a/src/Databases/DataLake/PaimonRestCatalog.cpp b/src/Databases/DataLake/PaimonRestCatalog.cpp index cffce96f082d..2108a143c0d1 100644 --- a/src/Databases/DataLake/PaimonRestCatalog.cpp +++ b/src/Databases/DataLake/PaimonRestCatalog.cpp @@ -413,7 +413,7 @@ void PaimonRestCatalog::forEachTables( } -bool PaimonRestCatalog::empty() const +bool PaimonRestCatalog::empty(const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { DB::Strings databases; DB::Names tables; @@ -427,7 +427,7 @@ bool PaimonRestCatalog::empty() const return tables.empty(); } -DB::Names PaimonRestCatalog::getTables() const +DB::Names PaimonRestCatalog::getTables(const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { DB::Strings databases; DB::Names tables; @@ -436,7 +436,7 @@ DB::Names PaimonRestCatalog::getTables() const return tables; } -bool PaimonRestCatalog::existsTable(const String & database_name, const String & table_name) const +bool PaimonRestCatalog::existsTable(const String & database_name, const String & table_name, const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { try { diff --git a/src/Databases/DataLake/PaimonRestCatalog.h b/src/Databases/DataLake/PaimonRestCatalog.h index 9aab6b815bdf..f157a1eceddc 100644 --- a/src/Databases/DataLake/PaimonRestCatalog.h +++ b/src/Databases/DataLake/PaimonRestCatalog.h @@ -82,11 +82,11 @@ class PaimonRestCatalog final : public ICatalog, private DB::WithContext ~PaimonRestCatalog() override = default; - bool empty() const override; + bool empty(const DB::ForwardedAuthTokenPtr & auth_token) const override; - DB::Names getTables() const override; + DB::Names getTables(const DB::ForwardedAuthTokenPtr & auth_token) const override; - bool existsTable(const String & database_name, const String & table_name) const override; + bool existsTable(const String & database_name, const String & table_name, const DB::ForwardedAuthTokenPtr & auth_token) const override; void getTableMetadata(const String & database_name, const String & table_name, DB::ContextPtr context_, TableMetadata & result) const override; diff --git a/src/Databases/DataLake/RestCatalog.cpp b/src/Databases/DataLake/RestCatalog.cpp index 1247222ce654..32bb46d8e89b 100644 --- a/src/Databases/DataLake/RestCatalog.cpp +++ b/src/Databases/DataLake/RestCatalog.cpp @@ -1,7 +1,12 @@ #include #include #include +#include +#include +#include +#include #include +#include #include #include #include @@ -70,6 +75,7 @@ namespace DB::ErrorCodes extern const int FAULT_INJECTED; extern const int NOT_IMPLEMENTED; extern const int CATALOG_NAMESPACE_DISABLED; + extern const int CATALOG_USER_TOKEN_NOT_AVAILABLE; } namespace DB::Setting @@ -88,6 +94,10 @@ namespace ProfileEvents { extern const Event DataLakeRestCatalogCredentialsVended; extern const Event DataLakeRestCatalogCredentialsCacheHits; + extern const Event DataLakeRestCatalogTokenExchange; + extern const Event DataLakeRestCatalogTokenExchangeMicroseconds; + extern const Event DataLakeRestCatalogTokenExchangeFailures; + extern const Event DataLakeRestCatalogUserTokenCacheHits; extern const Event DataLakeRestCatalogLoadConfig; extern const Event DataLakeRestCatalogLoadConfigMicroseconds; extern const Event DataLakeRestCatalogGetNamespaces; @@ -112,6 +122,12 @@ namespace ProfileEvents extern const Event DataLakeRestCatalogDropTableMicroseconds; } +namespace CurrentMetrics +{ + extern const Metric DataLakeCatalogUserTokenCacheBytes; + extern const Metric DataLakeCatalogUserTokenCacheEntries; +} + namespace DB::DatabaseDataLakeSetting { extern const DatabaseDataLakeSettingsString catalog_credential; @@ -453,7 +469,8 @@ RestCatalog::RestCatalog( const std::string & oauth_server_uri_, bool oauth_server_use_request_body_, const std::string & namespaces_, - DB::ContextPtr context_) + DB::ContextPtr context_, + const TokenForwardingConfig & token_forwarding_) : ICatalog(warehouse_) , DB::WithContext(context_) , base_url(correctAPIURI(base_url_)) @@ -461,6 +478,11 @@ RestCatalog::RestCatalog( , auth_scope(auth_scope_) , oauth_server_uri(oauth_server_uri_) , oauth_server_use_request_body(oauth_server_use_request_body_) + , token_forwarding(token_forwarding_) + , user_token_cache( + CurrentMetrics::DataLakeCatalogUserTokenCacheBytes, + CurrentMetrics::DataLakeCatalogUserTokenCacheEntries, + user_token_cache_max_entries) , allowed_namespaces(namespaces_) { CatalogState initial_state; @@ -474,7 +496,13 @@ RestCatalog::RestCatalog( initial_state.auth_header = parseAuthHeader(auth_header_); validateAuthHeaders(initial_state.auth_header.value()); } - initial_state.config = loadConfig(initial_state); + + /// Defer `/v1/config` until a query supplies the user token needed to authenticate it. + if (!token_forwarding.forward_user_token) + { + initial_state.config = loadConfig(initial_state, /* generation */ 0, /* auth_token */ {}); + initial_state.config_loaded = true; + } state.set(std::make_unique(std::move(initial_state))); } @@ -493,12 +521,45 @@ RestCatalog::RestCatalog( , auth_scope(auth_scope_) , oauth_server_uri(oauth_server_uri_) , oauth_server_use_request_body(oauth_server_use_request_body_) + , user_token_cache( + CurrentMetrics::DataLakeCatalogUserTokenCacheBytes, + CurrentMetrics::DataLakeCatalogUserTokenCacheEntries, + user_token_cache_max_entries) , allowed_namespaces(namespaces_) { } -RestCatalog::Config RestCatalog::loadConfig(const CatalogState & catalog_state, const std::optional & auth_headers) +void RestCatalog::loadConfigIfNeeded(const DB::ForwardedAuthTokenPtr & auth_token) const +{ + if (state.get()->config_loaded) + return; + + std::lock_guard lock(config_mutex); + const auto old_state = getStateSnapshot(); + if (old_state->config_loaded) + return; + + auto new_state = std::make_unique(*old_state); + new_state->config = loadConfig(*old_state, old_state.generation, auth_token); + new_state->config_loaded = true; + + /// `commitSettingsChanges` can publish during the config request. Discard a stale result + /// so it cannot restore old credentials or a warehouse resolved with them. + std::lock_guard publish_lock(auth_publish_mutex); + if (auth_generation.load(std::memory_order_acquire) != old_state.generation) + { + LOG_DEBUG(log, "Catalog credentials changed while `/v1/config` was loading; discarding it"); + return; + } + state.set(std::move(new_state)); +} + +RestCatalog::Config RestCatalog::loadConfig( + const CatalogState & catalog_state, + UInt64 generation, + const DB::ForwardedAuthTokenPtr & auth_token, + const std::optional & auth_headers) const { Poco::URI::QueryParameters params = {{"warehouse", warehouse}}; @@ -507,7 +568,7 @@ RestCatalog::Config RestCatalog::loadConfig(const CatalogState & catalog_state, { ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogLoadConfig); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogLoadConfigMicroseconds); - auto buf = createReadBuffer(catalog_state, CONFIG_ENDPOINT, params, /* headers */{}, auth_headers); + auto buf = createReadBuffer(catalog_state, generation, CONFIG_ENDPOINT, auth_token, params, /* headers */{}, auth_headers); readJSONObjectPossiblyInvalid(json_str, *buf); } @@ -553,22 +614,17 @@ void RestCatalog::validateAuthHeaders(const DB::HTTPHeaderEntry & header) const getContext()->getGlobalContext()->getHTTPHeaderFilter().checkAndNormalizeHeaders(header_to_check); } -DB::HTTPHeaderEntries RestCatalog::getAuthHeaders( - const CatalogState & catalog_state, - bool update_token, - const String & /*method*/, - const Poco::URI & /*url*/, - const DB::HTTPHeaderEntries & /*extra_headers*/, - const String & /*body*/, - bool * used_cached_oauth_token) const +DB::HTTPHeaderEntries RestCatalog::getAuthHeaders(const AuthContext & auth_context) const { fiu_do_on(DB::FailPoints::check_database_datalake_negative, { throw DB::Exception(DB::ErrorCodes::FAULT_INJECTED, "Injecting fault when checking database"); }); - if (used_cached_oauth_token) - *used_cached_oauth_token = false; + if (auth_context.used_cached_oauth_token) + *auth_context.used_cached_oauth_token = false; + + const auto & catalog_state = auth_context.catalog_state; /// Option 1: user specified auth header manually. /// Header has format: 'Authorization: '. @@ -577,7 +633,16 @@ DB::HTTPHeaderEntries RestCatalog::getAuthHeaders( return DB::HTTPHeaderEntries{catalog_state.auth_header.value()}; } - /// Option 2: user provided grant_type, client_id and client_secret. + if (token_forwarding.forward_user_token) + { + DB::HTTPHeaderEntries headers; + headers.emplace_back( + "Authorization", + "Bearer " + + getForwardedToken(catalog_state, auth_context.generation, auth_context.auth_token, auth_context.update_token)); + return headers; + } + /// We would make OAuthClientCredentialsRequest /// https://github.com/apache/iceberg/blob/3badfe0c1fcf0c0adfc7aa4a10f0b50365c48cf9/open-api/rest-catalog-open-api.yaml#L3498C5-L3498C34 if (!catalog_state.client_id.empty()) @@ -587,14 +652,14 @@ DB::HTTPHeaderEntries RestCatalog::getAuthHeaders( /// request fails with 401/403 and is retried with `update_token = true`, fetching /// a token with the snapshot's credentials. auto current = access_token.get(); - if (!current || update_token || current->isExpired()) + if (!current || auth_context.update_token || current->isExpired()) { - access_token.set(std::make_unique(retrieveAccessToken(catalog_state.client_id, catalog_state.client_secret))); - current = access_token.get(); + current = publishServiceToken( + retrieveAccessToken(catalog_state.client_id, catalog_state.client_secret), auth_context.generation); } - else if (used_cached_oauth_token) + else if (auth_context.used_cached_oauth_token) { - *used_cached_oauth_token = true; + *auth_context.used_cached_oauth_token = true; } DB::HTTPHeaderEntries headers; @@ -604,6 +669,101 @@ DB::HTTPHeaderEntries RestCatalog::getAuthHeaders( return {}; } +MultiVersion::Version RestCatalog::publishServiceToken(AccessToken minted, UInt64 generation) const +{ + auto result = std::make_shared(std::move(minted)); + + /// Check and publish under one lock so a concurrent `ALTER` cannot restore a superseded token. + std::lock_guard lock(auth_publish_mutex); + if (auth_generation.load(std::memory_order_acquire) == generation) + access_token.set(std::make_unique(*result)); + + return result; +} + +void RestCatalog::validateForwardedToken(const DB::ForwardedAuthTokenPtr & auth_token) const +{ + ICatalog::validateForwardedToken(getContext(), auth_token, warehouse); +} + +String RestCatalog::getForwardedToken( + const CatalogState & catalog_state, UInt64 generation, const DB::ForwardedAuthTokenPtr & auth_token, bool update_token) const +{ + validateForwardedToken(auth_token); + + if (!token_forwarding.exchangeEnabled()) + return auth_token->token; + + if (token_forwarding.user_token_cache_ttl == 0) + return exchangeUserToken(catalog_state, generation, *auth_token).token; + + auto exchange = [&] + { + return std::make_shared(exchangeUserToken(catalog_state, generation, *auth_token)); + }; + + const String cache_key = fmt::format("{}:{}", generation, auth_token->fingerprint); + + if (!update_token) + { + if (auto cached = user_token_cache.get(cache_key); cached && !cached->isExpired()) + { + ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogUserTokenCacheHits); + return cached->token; + } + } + + /// Remove stale entries before `getOrSetWithOutcome` so concurrent refreshes share a fresh result. + user_token_cache.remove(cache_key); + auto [session_token, outcome] = user_token_cache.getOrSetWithOutcome(cache_key, exchange); + if (outcome == DB::CacheGetOrSetOutcome::Hit) + ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogUserTokenCacheHits); + return session_token->token; +} + +AccessToken RestCatalog::exchangeUserToken( + const CatalogState & catalog_state, UInt64 generation, const DB::ForwardedAuthToken & auth_token, + const AccessToken * prepared_actor_token) const +{ + TokenRequest request; + request.grant = TokenRequest::Grant::TokenExchange; + request.url = Poco::URI(token_forwarding.token_exchange_uri); + request.scope = auth_scope; + request.client_id = catalog_state.client_id; + request.client_secret = catalog_state.client_secret; + request.subject_token = auth_token.token; + request.subject_token_type = token_forwarding.subject_token_type; + request.requested_token_type = token_forwarding.requested_token_type; + + if (token_forwarding.forward_actor_token) + request.actor_token = prepared_actor_token ? prepared_actor_token->token : getServicePrincipalToken(catalog_state, generation); + + ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogTokenExchange); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogTokenExchangeMicroseconds); + + AccessToken exchanged; + try + { + exchanged = requestToken(request); + } + catch (...) + { + ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogTokenExchangeFailures); + throw; + } + + /// Bound cached tokens even when the endpoint omits `expires_in`. + if (token_forwarding.user_token_cache_ttl > 0) + { + const auto ttl_bound = std::chrono::system_clock::now() + std::chrono::seconds(token_forwarding.user_token_cache_ttl); + if (!exchanged.expires_at.has_value() || exchanged.expires_at.value() > ttl_bound) + exchanged.expires_at = ttl_bound; + } + + LOG_DEBUG(log, "Exchanged the token of user `{}` for a catalog session token", auth_token.principal); + return exchanged; +} + OneLakeCatalog::OneLakeCatalog( const std::string & warehouse_, const std::string & base_url_, @@ -634,7 +794,8 @@ OneLakeCatalog::OneLakeCatalog( initial_state.client_secret = onelake_client_secret; update_token_if_expired = true; } - initial_state.config = loadConfig(initial_state); + initial_state.config = loadConfig(initial_state, /* generation */ 0, /* auth_token */ {}); + initial_state.config_loaded = true; state.set(std::make_unique(std::move(initial_state))); } @@ -689,22 +850,39 @@ void RestCatalog::validateSettingsChanges(const DB::SettingsChanges & changes, b struct RestCatalog::PreparedAuthChanges : ICatalog::PreparedSettingsChanges { std::unique_ptr new_state; - /// Set only when the OAuth credentials changed. std::unique_ptr new_access_token; }; -ICatalog::PreparedSettingsChangesPtr RestCatalog::prepareSettingsChanges(const DB::SettingsChanges & changes) +ICatalog::PreparedSettingsChangesPtr RestCatalog::prepareSettingsChanges( + const DB::SettingsChanges & changes, const DB::ForwardedAuthTokenPtr & auth_token) { - const auto old_state = state.get(); + if (token_forwarding.forward_user_token) + validateForwardedToken(auth_token); + const auto old_state = getStateSnapshot(); CatalogState new_state = *old_state; auto prepared = std::make_unique(); std::optional new_auth_headers; applySettingsChangesToState(changes, *old_state, new_state, new_auth_headers, prepared->new_access_token); + if (token_forwarding.forward_user_token) + { + /// Preparation may fail or be abandoned; do not publish its tokens in the live cache. + if (token_forwarding.exchangeEnabled()) + { + if (token_forwarding.forward_actor_token && !prepared->new_access_token) + prepared->new_access_token = std::make_unique(retrieveAccessToken(new_state.client_id, new_state.client_secret)); + const auto session = exchangeUserToken(new_state, old_state.generation, *auth_token, prepared->new_access_token.get()); + new_auth_headers = DB::HTTPHeaderEntries{{"Authorization", "Bearer " + session.token}}; + } + else + new_auth_headers = DB::HTTPHeaderEntries{{"Authorization", "Bearer " + auth_token->token}}; + } + /// The config was loaded with the old credentials; the new ones may resolve the /// warehouse to a different prefix or base location, so reload it before publishing. - new_state.config = loadConfig(new_state, new_auth_headers); + new_state.config = loadConfig(new_state, old_state.generation, auth_token, new_auth_headers); + new_state.config_loaded = true; prepared->new_state = std::make_unique(std::move(new_state)); return prepared; } @@ -715,9 +893,21 @@ void RestCatalog::commitSettingsChanges(ICatalog::PreparedSettingsChangesPtr pre if (!prepared_auth || !prepared_auth->new_state) throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Settings changes to commit were not prepared by this catalog"); - state.set(std::move(prepared_auth->new_state)); - if (prepared_auth->new_access_token) - access_token.set(std::move(prepared_auth->new_access_token)); + { + std::lock_guard lock(auth_publish_mutex); + state.set(std::move(prepared_auth->new_state)); + if (prepared_auth->new_access_token) + access_token.set(std::move(prepared_auth->new_access_token)); + + /// Publish the state before its generation; readers load them in the opposite order. + auth_generation.fetch_add(1, std::memory_order_release); + } + + user_token_cache.clear(); + { + std::lock_guard lock(credentials_cache_mutex); + credentials_cache.clear(); + } } void RestCatalog::applySettingsChangesToState( @@ -747,27 +937,17 @@ void RestCatalog::applySettingsChangesToState( throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Unexpected setting `{}` after validation", change.name); } - if (credential_mode && (new_state.client_id != old_state.client_id || new_state.client_secret != old_state.client_secret)) + if (credential_mode && (!token_forwarding.forward_user_token || token_forwarding.forward_actor_token) + && (new_state.client_id != old_state.client_id || new_state.client_secret != old_state.client_secret)) { - /// Eagerly fetch a token with the not-yet-published credentials: wrong credentials - /// fail the ALTER right here, and the config reload authenticates with that token - /// instead of the cached one. new_access_token = std::make_unique(retrieveAccessToken(new_state.client_id, new_state.client_secret)); new_auth_headers = DB::HTTPHeaderEntries{{"Authorization", "Bearer " + new_access_token->token}}; } } -DB::HTTPHeaderEntries OneLakeCatalog::getAuthHeaders( - const CatalogState & catalog_state, - bool update_token, - const String & method, - const Poco::URI & url, - const DB::HTTPHeaderEntries & extra_headers, - const String & body, - bool * used_cached_oauth_token) const +DB::HTTPHeaderEntries OneLakeCatalog::getAuthHeaders(const AuthContext & auth_context) const { - auto headers - = RestCatalog::getAuthHeaders(catalog_state, update_token, method, url, extra_headers, body, used_cached_oauth_token); + auto headers = RestCatalog::getAuthHeaders(auth_context); headers.emplace_back("User-Agent", fmt::format("ClickHouse/{}{} OneLake-Catalog", VERSION_STRING, VERSION_OFFICIAL)); return headers; } @@ -858,56 +1038,51 @@ namespace } -AccessToken RestCatalog::retrieveAccessToken(const std::string & client_id, const std::string & client_secret) const +AccessToken RestCatalog::requestToken(const TokenRequest & token_request) const { - ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogAuthTokenRetrieve); - auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogAuthTokenRefreshedMicroseconds); - - static constexpr auto oauth_tokens_endpoint = "oauth/tokens"; - - /// TODO: - /// 1. support oauth2-server-uri - /// https://github.com/apache/iceberg/blob/918f81f3c3f498f46afcea17c1ac9cdc6913cb5c/open-api/rest-catalog-open-api.yaml#L183C82-L183C99 - - Poco::URI url; - DB::ReadWriteBufferFromHTTP::OutStreamCallback out_stream_callback; - size_t body_size = 0; + Poco::URI url = token_request.url; String body; - if (oauth_server_uri.empty() && !oauth_server_use_request_body) + /// Do not also send bearer authentication: strict OAuth servers reject multiple client-authentication methods. + Poco::URI::QueryParameters params; + if (token_request.grant == TokenRequest::Grant::ClientCredentials) { - url = Poco::URI(base_url / oauth_tokens_endpoint); - - Poco::URI::QueryParameters params = { - {"grant_type", "client_credentials"}, - {"scope", auth_scope}, - {"client_id", client_id}, - {"client_secret", client_secret}, - }; - url.setQueryParameters(params); + params.emplace_back("grant_type", "client_credentials"); + params.emplace_back("scope", token_request.scope); } else { - String encoded_auth_scope; - String encoded_client_id; - String encoded_client_secret; - Poco::URI::encode(auth_scope, auth_scope, encoded_auth_scope); - Poco::URI::encode(client_id, client_id, encoded_client_id); - Poco::URI::encode(client_secret, client_secret, encoded_client_secret); - - body = fmt::format( - "grant_type=client_credentials&scope={}&client_id={}&client_secret={}", - encoded_auth_scope, encoded_client_id, encoded_client_secret); - body_size = body.size(); - out_stream_callback = [&](std::ostream & os) + params.emplace_back("grant_type", "urn:ietf:params:oauth:grant-type:token-exchange"); + params.emplace_back("subject_token", token_request.subject_token); + params.emplace_back("subject_token_type", token_request.subject_token_type); + if (!token_request.requested_token_type.empty()) + params.emplace_back("requested_token_type", token_request.requested_token_type); + if (!token_request.scope.empty()) + params.emplace_back("scope", token_request.scope); + if (!token_request.actor_token.empty()) { - os << body; - }; + params.emplace_back("actor_token", token_request.actor_token); + params.emplace_back("actor_token_type", "urn:ietf:params:oauth:token-type:access_token"); + } + } - if (oauth_server_uri.empty()) - url = Poco::URI(base_url / oauth_tokens_endpoint); - else - url = Poco::URI(oauth_server_uri); + params.emplace_back("client_id", token_request.client_id); + params.emplace_back("client_secret", token_request.client_secret); + + if (token_request.use_query_parameters) + url.setQueryParameters(params); + else + { + DB::WriteBufferFromOwnString wb; + bool first = true; + for (const auto & [name, value] : params) + { + if (!first) + wb << "&"; + first = false; + wb << name << "=" << DB::formUrlEncode(value); + } + body = wb.str(); } const auto & context = getContext(); @@ -918,13 +1093,10 @@ AccessToken RestCatalog::retrieveAccessToken(const std::string & client_id, cons Poco::Net::HTTPRequest request(Poco::Net::HTTPRequest::HTTP_POST, url.getPathAndQuery(), Poco::Net::HTTPMessage::HTTP_1_1); request.setContentType("application/x-www-form-urlencoded"); - request.setContentLength(body_size); + request.setContentLength(body.size()); request.set("Accept", "application/json"); - std::ostream & os = session->sendRequest(request); - /// The query-parameters flavor of the request has no body. - if (out_stream_callback) - out_stream_callback(os); + session->sendRequest(request) << body; Poco::Net::HTTPResponse response; std::istream & rs = session->receiveResponse(response); @@ -932,11 +1104,32 @@ AccessToken RestCatalog::retrieveAccessToken(const std::string & client_id, cons std::string json_str; Poco::StreamCopier::copyToString(rs, json_str); - Poco::JSON::Parser parser; - Poco::Dynamic::Var res_json = parser.parse(json_str); - const Poco::JSON::Object::Ptr & object = res_json.extract(); + /// OAuth error bodies may echo the subject token, so exclude them from exceptions. + const auto describe_endpoint = [&url, &response] + { + return fmt::format( + "OAuth token endpoint {}://{}:{}{} returned HTTP {}", + url.getScheme(), url.getHost(), url.getPort(), url.getPath(), + static_cast(response.getStatus())); + }; + + Poco::JSON::Object::Ptr object; + try + { + object = Poco::JSON::Parser().parse(json_str).extract(); + } + catch (const Poco::Exception &) + { + object = nullptr; + } + if (!object) + throw DB::Exception( + DB::ErrorCodes::DATALAKE_DATABASE_ERROR, "{} with a body that is not a JSON object", describe_endpoint()); AccessToken token; + if (!object->has("access_token")) + throw DB::Exception( + DB::ErrorCodes::DATALAKE_DATABASE_ERROR, "{} with no `access_token` field", describe_endpoint()); token.token = object->get("access_token").extract(); if (object->has("expires_in")) @@ -950,6 +1143,31 @@ AccessToken RestCatalog::retrieveAccessToken(const std::string & client_id, cons return token; } +AccessToken RestCatalog::retrieveAccessToken(const std::string & client_id, const std::string & client_secret) const +{ + static constexpr auto oauth_tokens_endpoint = "oauth/tokens"; + + TokenRequest request; + request.scope = auth_scope; + request.client_id = client_id; + request.client_secret = client_secret; + + request.url = oauth_server_uri.empty() ? Poco::URI(base_url / oauth_tokens_endpoint) : Poco::URI(oauth_server_uri); + request.use_query_parameters = oauth_server_uri.empty() && !oauth_server_use_request_body; + + ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogAuthTokenRetrieve); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogAuthTokenRefreshedMicroseconds); + return requestToken(request); +} + +String RestCatalog::getServicePrincipalToken(const CatalogState & catalog_state, UInt64 generation) const +{ + auto current = access_token.get(); + if (!current || current->isExpired()) + current = publishServiceToken(retrieveAccessToken(catalog_state.client_id, catalog_state.client_secret), generation); + return current->token; +} + BigLakeCatalog::BigLakeCatalog( const std::string & warehouse_, const std::string & base_url_, @@ -978,18 +1196,12 @@ BigLakeCatalog::BigLakeCatalog( access_token.set(std::make_unique(retrieveGoogleCloudAccessToken())); } CatalogState initial_state; - initial_state.config = loadConfig(initial_state); + initial_state.config = loadConfig(initial_state, /* generation */ 0, /* auth_token */ {}); + initial_state.config_loaded = true; state.set(std::make_unique(std::move(initial_state))); } -DB::HTTPHeaderEntries BigLakeCatalog::getAuthHeaders( - const CatalogState & catalog_state, - bool update_token, - const String & method, - const Poco::URI & url, - const DB::HTTPHeaderEntries & extra_headers, - const String & body, - bool * used_cached_oauth_token) const +DB::HTTPHeaderEntries BigLakeCatalog::getAuthHeaders(const AuthContext & auth_context) const { /// Google Cloud OAuth2 for BigLake. /// Uses GCP metadata service or Application Default Credentials to get access token. @@ -997,18 +1209,18 @@ DB::HTTPHeaderEntries BigLakeCatalog::getAuthHeaders( /// https://developers.google.com/identity/protocols/oauth2 if (!google_project_id.empty() || !google_adc_client_id.empty()) { - if (used_cached_oauth_token) - *used_cached_oauth_token = false; + if (auth_context.used_cached_oauth_token) + *auth_context.used_cached_oauth_token = false; auto current = access_token.get(); - if (!current || update_token || current->isExpired()) + if (!current || auth_context.update_token || current->isExpired()) { access_token.set(std::make_unique(retrieveGoogleCloudAccessToken())); current = access_token.get(); } - else if (used_cached_oauth_token) + else if (auth_context.used_cached_oauth_token) { - *used_cached_oauth_token = true; + *auth_context.used_cached_oauth_token = true; } DB::HTTPHeaderEntries headers; @@ -1028,7 +1240,7 @@ DB::HTTPHeaderEntries BigLakeCatalog::getAuthHeaders( return headers; } - return RestCatalog::getAuthHeaders(catalog_state, update_token, method, url, extra_headers, body, used_cached_oauth_token); + return RestCatalog::getAuthHeaders(auth_context); } AccessToken BigLakeCatalog::retrieveGoogleCloudAccessTokenFromRefreshToken() const @@ -1154,15 +1366,17 @@ AccessToken BigLakeCatalog::retrieveGoogleCloudAccessToken() const std::optional RestCatalog::getStorageType() const { - const auto state_snapshot = state.get(); - if (state_snapshot->config.default_base_location.empty()) + const auto state_snapshot = getStateSnapshot(); + if (!state_snapshot->config_loaded || state_snapshot->config.default_base_location.empty()) return std::nullopt; return parseStorageTypeFromLocation(state_snapshot->config.default_base_location); } DB::ReadWriteBufferFromHTTPPtr RestCatalog::createReadBuffer( const CatalogState & catalog_state, + UInt64 generation, const std::string & endpoint, + const DB::ForwardedAuthTokenPtr & auth_token, const Poco::URI::QueryParameters & params, const DB::HTTPHeaderEntries & headers, const std::optional & auth_headers) const @@ -1176,10 +1390,18 @@ DB::ReadWriteBufferFromHTTPPtr RestCatalog::createReadBuffer( auto create_buffer = [&](bool update_token, bool & used_cached_oauth_token) { - auto result_headers = auth_headers - ? *auth_headers - : getAuthHeaders( - catalog_state, update_token, Poco::Net::HTTPRequest::HTTP_GET, url, headers, {}, &used_cached_oauth_token); + AuthContext auth_context{ + .catalog_state = catalog_state, + .generation = generation, + .update_token = update_token, + .method = Poco::Net::HTTPRequest::HTTP_GET, + .url = url, + .extra_headers = headers, + .body = {}, + .auth_token = auth_token, + .used_cached_oauth_token = &used_cached_oauth_token, + }; + auto result_headers = auth_headers ? *auth_headers : getAuthHeaders(auth_context); std::move(headers.begin(), headers.end(), std::back_inserter(result_headers)); return DB::BuilderRWBufferFromHTTP(url) @@ -1206,20 +1428,29 @@ DB::ReadWriteBufferFromHTTPPtr RestCatalog::createReadBuffer( catch (const DB::HTTPException & e) { const auto status = e.getHTTPStatus(); - if (update_token_if_expired && - (status == Poco::Net::HTTPResponse::HTTPStatus::HTTP_UNAUTHORIZED - || status == Poco::Net::HTTPResponse::HTTPStatus::HTTP_FORBIDDEN)) - { - ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogUnauthorized); - bool used_cached_oauth_token_on_retry = false; - return create_buffer(true, used_cached_oauth_token_on_retry); - } - throw; + if (!shouldRetryWithFreshToken(status)) + throw; + + ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogUnauthorized); + bool used_cached_oauth_token_on_retry = false; + return create_buffer(true, used_cached_oauth_token_on_retry); } } -bool RestCatalog::empty() const +bool RestCatalog::shouldRetryWithFreshToken(Poco::Net::HTTPResponse::HTTPStatus status) const { + if (token_forwarding.forward_user_token) + return token_forwarding.exchangeEnabled() && status == Poco::Net::HTTPResponse::HTTPStatus::HTTP_UNAUTHORIZED; + + return update_token_if_expired + && (status == Poco::Net::HTTPResponse::HTTPStatus::HTTP_UNAUTHORIZED + || status == Poco::Net::HTTPResponse::HTTPStatus::HTTP_FORBIDDEN); +} + +bool RestCatalog::empty(const DB::ForwardedAuthTokenPtr & auth_token) const +{ + loadConfigIfNeeded(auth_token); + bool found_table = false; auto stop_condition = [&](const std::string & namespace_name) -> bool { @@ -1229,7 +1460,7 @@ bool RestCatalog::empty() const if (!allowed_namespaces.isNamespaceAllowed(namespace_name, /*nested*/ false)) return false; - const auto tables = getTables(namespace_name, /* limit */1); + const auto tables = getTablesInNamespace(namespace_name, auth_token, /* limit */1); if (!tables.empty()) found_table = true; @@ -1237,13 +1468,15 @@ bool RestCatalog::empty() const }; Namespaces namespaces; - getNamespacesRecursive("", namespaces, stop_condition, /* execute_func */{}); + getNamespacesRecursive("", namespaces, stop_condition, /* execute_func */{}, auth_token); return !found_table; } -DB::Names RestCatalog::getTables() const +DB::Names RestCatalog::getTables(const DB::ForwardedAuthTokenPtr & auth_token) const { + loadConfigIfNeeded(auth_token); + auto & pool = getContext()->getIcebergCatalogThreadpool(); DB::Names tables; std::mutex mutex; @@ -1259,7 +1492,7 @@ DB::Names RestCatalog::getTables() const runner.enqueueAndKeepTrack( [=, &tables, &mutex, this] { - auto tables_in_namespace = getTables(current_namespace); + auto tables_in_namespace = getTablesInNamespace(current_namespace, auth_token); std::lock_guard lock(mutex); std::move(tables_in_namespace.begin(), tables_in_namespace.end(), std::back_inserter(tables)); }); @@ -1270,7 +1503,8 @@ DB::Names RestCatalog::getTables() const /* base_namespace */"", /// Empty base namespace means starting from root. namespaces, /* stop_condition */{}, - /* execute_func */execute_for_each_namespace); + /* execute_func */execute_for_each_namespace, + auth_token); runner.waitForAllToFinishAndRethrowFirstError(); } @@ -1282,11 +1516,12 @@ void RestCatalog::getNamespacesRecursive( const std::string & base_namespace, Namespaces & result, StopCondition stop_condition, - ExecuteFunc func) const + ExecuteFunc func, + const DB::ForwardedAuthTokenPtr & auth_token) const { checkStackSize(); - auto namespaces = getNamespaces(base_namespace); + auto namespaces = getNamespaces(base_namespace, auth_token); result.reserve(result.size() + namespaces.size()); result.insert(result.end(), namespaces.begin(), namespaces.end()); @@ -1315,7 +1550,7 @@ void RestCatalog::getNamespacesRecursive( } if (allowed_namespaces.isNamespaceAllowed(current_namespace, /*nested*/ true)) - getNamespacesRecursive(current_namespace, result, stop_condition, func); + getNamespacesRecursive(current_namespace, result, stop_condition, func, auth_token); else { LOG_DEBUG(log, "Nested namespaces in namespace {} are filtered", current_namespace); @@ -1339,9 +1574,9 @@ Poco::URI::QueryParameters RestCatalog::createParentNamespaceParams(const std::s return {{"parent", parent_param}}; } -RestCatalog::Namespaces RestCatalog::getNamespaces(const std::string & base_namespace) const +RestCatalog::Namespaces RestCatalog::getNamespaces(const std::string & base_namespace, const DB::ForwardedAuthTokenPtr & auth_token) const { - const auto state_snapshot = state.get(); + const auto state_snapshot = getStateSnapshot(); Poco::URI::QueryParameters base_params; if (!base_namespace.empty()) @@ -1369,7 +1604,8 @@ RestCatalog::Namespaces RestCatalog::getNamespaces(const std::string & base_name ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogGetNamespaces); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogGetNamespacesMicroseconds); - auto buf = createReadBuffer(*state_snapshot, state_snapshot->config.prefix / NAMESPACES_ENDPOINT, params); + auto buf = createReadBuffer( + *state_snapshot, state_snapshot.generation, state_snapshot->config.prefix / NAMESPACES_ENDPOINT, auth_token, params); String next_page_token; auto page_namespaces = parseNamespaces(*buf, base_namespace, next_page_token); LOG_DEBUG( @@ -1496,13 +1732,13 @@ RestCatalog::Namespaces RestCatalog::parseNamespaces(DB::ReadBuffer & buf, const } } -DB::Names RestCatalog::getTables(const std::string & base_namespace, size_t limit) const +DB::Names RestCatalog::getTablesInNamespace(const std::string & base_namespace, const DB::ForwardedAuthTokenPtr & auth_token, size_t limit) const { if (!allowed_namespaces.isNamespaceAllowed(base_namespace, /*nested*/ false)) throw DB::Exception(DB::ErrorCodes::CATALOG_NAMESPACE_DISABLED, "Namespace {} is filtered by `namespaces` database parameter", base_namespace); - const auto state_snapshot = state.get(); + const auto state_snapshot = getStateSnapshot(); auto encoded_namespace = encodeNamespaceForURI(base_namespace); const std::string endpoint = std::filesystem::path(NAMESPACES_ENDPOINT) / encoded_namespace / "tables"; @@ -1527,7 +1763,7 @@ DB::Names RestCatalog::getTables(const std::string & base_namespace, size_t limi ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogGetTables); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogGetTablesMicroseconds); - auto buf = createReadBuffer(*state_snapshot, state_snapshot->config.prefix / endpoint, params); + auto buf = createReadBuffer(*state_snapshot, state_snapshot.generation, state_snapshot->config.prefix / endpoint, auth_token, params); /// Pass through the remaining limit so that single-page short-circuiting still works /// when the caller is in `empty()` (limit=1) and the first page already contains a row. @@ -1612,10 +1848,10 @@ DB::Names RestCatalog::parseTables(DB::ReadBuffer & buf, const std::string & bas } } -bool RestCatalog::existsTable(const std::string & namespace_name, const std::string & table_name) const +bool RestCatalog::existsTable(const std::string & namespace_name, const std::string & table_name, const DB::ForwardedAuthTokenPtr & auth_token) const { TableMetadata table_metadata; - return tryGetTableMetadata(namespace_name, table_name, getContext(), table_metadata); + return tryGetTableMetadataImpl(namespace_name, table_name, getContext(), table_metadata, auth_token); } bool RestCatalog::tryGetTableMetadata( @@ -1623,10 +1859,20 @@ bool RestCatalog::tryGetTableMetadata( const std::string & table_name, DB::ContextPtr context_, TableMetadata & result) const +{ + return tryGetTableMetadataImpl(namespace_name, table_name, context_, result, getForwardedAuthToken(context_)); +} + +bool RestCatalog::tryGetTableMetadataImpl( + const std::string & namespace_name, + const std::string & table_name, + DB::ContextPtr context_, + TableMetadata & result, + const DB::ForwardedAuthTokenPtr & auth_token) const { try { - return getTableMetadataImpl(namespace_name, table_name, context_, result); + return getTableMetadataImpl(namespace_name, table_name, context_, result, auth_token); } catch (const DB::HTTPException & ex) { @@ -1645,7 +1891,7 @@ void RestCatalog::getTableMetadata( DB::ContextPtr context_, TableMetadata & result) const { - if (!getTableMetadataImpl(namespace_name, table_name, context_, result)) + if (!getTableMetadataImpl(namespace_name, table_name, context_, result, getForwardedAuthToken(context_))) throw DB::Exception(DB::ErrorCodes::DATALAKE_DATABASE_ERROR, "No response from iceberg catalog"); } @@ -1736,24 +1982,31 @@ bool RestCatalog::getTableMetadataImpl( const std::string & table_name, DB::ContextPtr context_, TableMetadata & result, + const DB::ForwardedAuthTokenPtr & auth_token, bool allow_credentials_cache) const { LOG_DEBUG(log, "Checking table {} in namespace {}", table_name, namespace_name); + loadConfigIfNeeded(auth_token); + if (!allowed_namespaces.isNamespaceAllowed(namespace_name, /*nested*/ false)) throw DB::Exception(DB::ErrorCodes::CATALOG_NAMESPACE_DISABLED, "Namespace {} is filtered by `namespaces` database parameter", namespace_name); DB::HTTPHeaderEntries headers; + const auto state_snapshot = getStateSnapshot(); + const bool want_credentials = result.requiresCredentials(); + const CredentialsCacheKey credentials_key{ + state_snapshot.generation, getCredentialsCachePrincipal(auth_token), namespace_name, table_name}; /// Reuse previously vended credentials is possible std::optional cached_credentials; if (want_credentials) { if (allow_credentials_cache) - cached_credentials = tryGetCachedCredentials(namespace_name, table_name); + cached_credentials = tryGetCachedCredentials(credentials_key); /// Header `X-Iceberg-Access-Delegation` tells catalog to include storage credentials in LoadTableResponse. /// Value can be one of the two: @@ -1768,14 +2021,14 @@ bool RestCatalog::getTableMetadataImpl( } } - const auto state_snapshot = state.get(); const std::string endpoint = std::filesystem::path(NAMESPACES_ENDPOINT) / encodeNamespaceForURI(namespace_name) / "tables" / table_name; String json_str; { ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogGetTableMetadata); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogGetTableMetadataMicroseconds); - auto buf = createReadBuffer(*state_snapshot, state_snapshot->config.prefix / endpoint, /* params */{}, headers); + auto buf = createReadBuffer( + *state_snapshot, state_snapshot.generation, state_snapshot->config.prefix / endpoint, auth_token, /* params */{}, headers); if (buf->eof()) { @@ -1836,9 +2089,9 @@ bool RestCatalog::getTableMetadataImpl( { { std::lock_guard lock(credentials_cache_mutex); - credentials_cache.erase({namespace_name, table_name}); + credentials_cache.erase(credentials_key); } - return getTableMetadataImpl(namespace_name, table_name, context_, result, /* allow_credentials_cache */ false); + return getTableMetadataImpl(namespace_name, table_name, context_, result, auth_token, /* allow_credentials_cache */ false); } ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogCredentialsCacheHits); result.setStorageCredentials(cached_credentials->credentials); @@ -1852,7 +2105,7 @@ bool RestCatalog::getTableMetadataImpl( if (parsed.credentials) { result.setStorageCredentials(parsed.credentials); - cacheCredentials(namespace_name, table_name, parsed); + cacheCredentials(credentials_key, parsed); } if (!parsed.endpoint.empty()) result.setEndpoint(parsed.endpoint); @@ -1874,7 +2127,14 @@ bool RestCatalog::getTableMetadataImpl( return true; } -void RestCatalog::sendRequest(const CatalogState & catalog_state, const String & endpoint, Poco::JSON::Object::Ptr request_body, const String & method, bool ignore_result) const +void RestCatalog::sendRequest( + const CatalogState & catalog_state, + UInt64 generation, + const String & endpoint, + Poco::JSON::Object::Ptr request_body, + const DB::ForwardedAuthTokenPtr & auth_token, + const String & method, + bool ignore_result) const { std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM if (request_body) @@ -1900,8 +2160,18 @@ void RestCatalog::sendRequest(const CatalogState & catalog_state, const String & auto create_buffer = [&](bool update_token, bool & used_cached_oauth_token) { - DB::HTTPHeaderEntries headers - = getAuthHeaders(catalog_state, update_token, method, url, extra_headers, body_str, &used_cached_oauth_token); + AuthContext auth_context{ + .catalog_state = catalog_state, + .generation = generation, + .update_token = update_token, + .method = method, + .url = url, + .extra_headers = extra_headers, + .body = body_str, + .auth_token = auth_token, + .used_cached_oauth_token = &used_cached_oauth_token, + }; + DB::HTTPHeaderEntries headers = getAuthHeaders(auth_context); headers.emplace_back("Content-Type", "application/json"); return DB::BuilderRWBufferFromHTTP(url) .withConnectionGroup(DB::HTTPConnectionGroupType::HTTP) @@ -1931,31 +2201,26 @@ void RestCatalog::sendRequest(const CatalogState & catalog_state, const String & } catch (const DB::HTTPException & e) { - const auto status = e.getHTTPStatus(); - if (update_token_if_expired && - (status == Poco::Net::HTTPResponse::HTTPStatus::HTTP_UNAUTHORIZED - || status == Poco::Net::HTTPResponse::HTTPStatus::HTTP_FORBIDDEN)) - { - ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogUnauthorized); - bool used_cached_oauth_token_on_retry = false; - auto wb = create_buffer(true, used_cached_oauth_token_on_retry); + if (!shouldRetryWithFreshToken(e.getHTTPStatus())) + throw; - String response_str; - if (!ignore_result) - readJSONObjectPossiblyInvalid(response_str, *wb); - else - wb->ignoreAll(); - } + ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogUnauthorized); + bool used_cached_oauth_token_on_retry = false; + auto wb = create_buffer(true, used_cached_oauth_token_on_retry); + + String response_str; + if (!ignore_result) + readJSONObjectPossiblyInvalid(response_str, *wb); else - { - throw; - } + wb->ignoreAll(); } } -void RestCatalog::createNamespaceIfNotExists(const String & namespace_name, const String & location) const +void RestCatalog::createNamespaceIfNotExists(const String & namespace_name, const String & location, const DB::ForwardedAuthTokenPtr & auth_token) const { - const auto state_snapshot = state.get(); + loadConfigIfNeeded(auth_token); + + const auto state_snapshot = getStateSnapshot(); /// Check existence first: creation may be denied to a principal that is still /// allowed to use a pre-provisioned namespace. @@ -1963,7 +2228,9 @@ void RestCatalog::createNamespaceIfNotExists(const String & namespace_name, cons = (base_url / state_snapshot->config.prefix / NAMESPACES_ENDPOINT / encodeNamespaceForURI(namespace_name)).generic_string(); try { - sendRequest(*state_snapshot, check_endpoint, /* request_body */ nullptr, Poco::Net::HTTPRequest::HTTP_GET, /* ignore_result */ true); + sendRequest( + *state_snapshot, state_snapshot.generation, check_endpoint, /* request_body */ nullptr, auth_token, + Poco::Net::HTTPRequest::HTTP_GET, /* ignore_result */ true); return; } catch (const DB::HTTPException & e) @@ -1990,7 +2257,7 @@ void RestCatalog::createNamespaceIfNotExists(const String & namespace_name, cons { ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogCreateNamespace); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogCreateNamespaceMicroseconds); - sendRequest(*state_snapshot, endpoint, request_body); + sendRequest(*state_snapshot, state_snapshot.generation, endpoint, request_body, auth_token); } catch (const DB::HTTPException & e) { @@ -2000,13 +2267,15 @@ void RestCatalog::createNamespaceIfNotExists(const String & namespace_name, cons } } -void RestCatalog::createTable(const String & namespace_name, const String & table_name, const String & /*new_metadata_path*/, Poco::JSON::Object::Ptr metadata_content) const +void RestCatalog::createTable(const String & namespace_name, const String & table_name, const String & /*new_metadata_path*/, Poco::JSON::Object::Ptr metadata_content, const DB::ForwardedAuthTokenPtr & auth_token) const { + loadConfigIfNeeded(auth_token); + if (!allowed_namespaces.isNamespaceAllowed(namespace_name, /*nested*/ false)) throw DB::Exception(DB::ErrorCodes::CATALOG_NAMESPACE_DISABLED, "Failed to create table {}, namespace {} is filtered by `namespaces` database parameter", table_name, namespace_name); - const auto state_snapshot = state.get(); + const auto state_snapshot = getStateSnapshot(); const std::string endpoint = (base_url / state_snapshot->config.prefix / NAMESPACES_ENDPOINT / encodeNamespaceForURI(namespace_name) / "tables").generic_string(); Poco::JSON::Object::Ptr request_body = new Poco::JSON::Object; @@ -2039,7 +2308,7 @@ void RestCatalog::createTable(const String & namespace_name, const String & tabl { ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogCreateTable); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogCreateTableMicroseconds); - sendRequest(*state_snapshot, endpoint, request_body); + sendRequest(*state_snapshot, state_snapshot.generation, endpoint, request_body, auth_token); } catch (const DB::HTTPException & ex) { @@ -2048,15 +2317,17 @@ void RestCatalog::createTable(const String & namespace_name, const String & tabl } -bool RestCatalog::updateMetadata(const String & namespace_name, const String & table_name, const String & /*new_metadata_path*/, Poco::JSON::Object::Ptr new_snapshot) const +bool RestCatalog::updateMetadata(const String & namespace_name, const String & table_name, const String & /*new_metadata_path*/, Poco::JSON::Object::Ptr new_snapshot, const DB::ForwardedAuthTokenPtr & auth_token) const { + loadConfigIfNeeded(auth_token); + if (!new_snapshot) throw DB::Exception( DB::ErrorCodes::NOT_IMPLEMENTED, "REST catalog does not support metadata-only updates without a snapshot " "(required for EXPIRE SNAPSHOTS)"); - const auto state_snapshot = state.get(); + const auto state_snapshot = getStateSnapshot(); const std::string endpoint = (base_url / state_snapshot->config.prefix / NAMESPACES_ENDPOINT / encodeNamespaceForURI(namespace_name) / "tables" / table_name).generic_string(); auto request_body = buildUpdateMetadataRequestBody(namespace_name, table_name, new_snapshot); @@ -2065,7 +2336,7 @@ bool RestCatalog::updateMetadata(const String & namespace_name, const String & t { ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogUpdateTable); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogUpdateTableMicroseconds); - sendRequest(*state_snapshot, endpoint, request_body); + sendRequest(*state_snapshot, state_snapshot.generation, endpoint, request_body, auth_token); } catch (const DB::HTTPException & ex) { @@ -2088,11 +2359,14 @@ bool RestCatalog::updateSchema( Poco::JSON::Object::Ptr new_schema, Int32 previous_schema_id, Int32 new_last_column_id, - Poco::JSON::Object::Ptr metadata) const + Poco::JSON::Object::Ptr metadata, + const DB::ForwardedAuthTokenPtr & auth_token) const { fiu_do_on(DB::FailPoints::iceberg_alter_catalog_update_schema_fail, { return false; }); - const auto state_snapshot = state.get(); + loadConfigIfNeeded(auth_token); + + const auto state_snapshot = getStateSnapshot(); const std::string endpoint = (base_url / state_snapshot->config.prefix / NAMESPACES_ENDPOINT / encodeNamespaceForURI(namespace_name) / "tables" / table_name).generic_string(); auto request_body = buildUpdateSchemaRequestBody( @@ -2100,7 +2374,7 @@ bool RestCatalog::updateSchema( try { - sendRequest(*state_snapshot, endpoint, request_body); + sendRequest(*state_snapshot, state_snapshot.generation, endpoint, request_body, auth_token); } catch (const DB::HTTPException & ex) { @@ -2121,14 +2395,16 @@ bool RestCatalog::updateSchema( return true; } -void RestCatalog::dropTable(const String & namespace_name, const String & table_name) const +void RestCatalog::dropTable(const String & namespace_name, const String & table_name, const DB::ForwardedAuthTokenPtr & auth_token) const { + loadConfigIfNeeded(auth_token); + if (!allowed_namespaces.isNamespaceAllowed(namespace_name, /*nested*/ false)) throw DB::Exception(DB::ErrorCodes::CATALOG_NAMESPACE_DISABLED, "Failed to drop table {}, namespace {} is filtered by `namespaces` database parameter", table_name, namespace_name); - const auto state_snapshot = state.get(); + const auto state_snapshot = getStateSnapshot(); const std::string endpoint = (base_url / state_snapshot->config.prefix / NAMESPACES_ENDPOINT / encodeNamespaceForURI(namespace_name) / "tables" / table_name).generic_string() + "?purgeRequested=False"; @@ -2138,7 +2414,8 @@ void RestCatalog::dropTable(const String & namespace_name, const String & table_ { ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogDropTable); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogDropTableMicroseconds); - sendRequest(*state_snapshot, endpoint, request_body, Poco::Net::HTTPRequest::HTTP_DELETE, true); + sendRequest( + *state_snapshot, state_snapshot.generation, endpoint, request_body, auth_token, Poco::Net::HTTPRequest::HTTP_DELETE, true); } catch (const DB::HTTPException & ex) { @@ -2284,14 +2561,20 @@ VendedStorageCredentials RestCatalog::getCredentialsAndEndpoint(Poco::JSON::Obje return {nullptr, "", std::nullopt}; } -std::optional RestCatalog::tryGetCachedCredentials( - const std::string & namespace_name, const std::string & table_name) const +String RestCatalog::getCredentialsCachePrincipal(const DB::ForwardedAuthTokenPtr & auth_token) const +{ + if (!token_forwarding.forward_user_token || !auth_token) + return {}; + return auth_token->fingerprint; +} + +std::optional RestCatalog::tryGetCachedCredentials(const CredentialsCacheKey & key) const { if (vended_credentials_cache_ttl.load(std::memory_order_relaxed) <= std::chrono::seconds::zero()) return std::nullopt; std::lock_guard lock(credentials_cache_mutex); - auto it = credentials_cache.find({namespace_name, table_name}); + auto it = credentials_cache.find(key); if (it == credentials_cache.end()) return std::nullopt; if (std::chrono::system_clock::now() >= it->second.expires_at.value()) @@ -2303,10 +2586,7 @@ std::optional RestCatalog::tryGetCachedCredentials( return it->second; } -void RestCatalog::cacheCredentials( - const std::string & namespace_name, - const std::string & table_name, - const VendedStorageCredentials & parsed) const +void RestCatalog::cacheCredentials(const CredentialsCacheKey & key, const VendedStorageCredentials & parsed) const { const auto ttl = vended_credentials_cache_ttl.load(std::memory_order_relaxed); if (ttl <= std::chrono::seconds::zero()) @@ -2332,20 +2612,31 @@ void RestCatalog::cacheCredentials( if (credentials_cache.size() >= credentials_cache_cleanup_threshold) std::erase_if(credentials_cache, [&now](const auto & entry) { return now >= entry.second.expires_at.value(); }); - credentials_cache[{namespace_name, table_name}] + + while (credentials_cache.size() >= credentials_cache_max_entries) + { + auto oldest = std::min_element( + credentials_cache.begin(), + credentials_cache.end(), + [](const auto & lhs, const auto & rhs) { return lhs.second.expires_at.value() < rhs.second.expires_at.value(); }); + credentials_cache.erase(oldest); + } + + credentials_cache[key] = VendedStorageCredentials{parsed.credentials, parsed.endpoint, refresh_after, parsed.table_uuid}; } -ICatalog::CredentialsRefreshCallback RestCatalog::getCredentialsConfigurationCallback(const DB::StorageID & storage_id) +ICatalog::CredentialsRefreshCallback RestCatalog::getCredentialsConfigurationCallback( + const DB::StorageID & storage_id, const DB::ForwardedAuthTokenPtr & auth_token) { - return [this, storage_id] () -> std::shared_ptr + return [this, storage_id, auth_token] () -> std::shared_ptr { LOG_DEBUG(log, "Update credentials in the catalog"); DB::HTTPHeaderEntries headers; headers.emplace_back("X-Iceberg-Access-Delegation", "vended-credentials"); - const auto state_snapshot = state.get(); + const auto state_snapshot = getStateSnapshot(); const auto & table = storage_id.getTableName(); auto [namespace_name, table_name] = DataLake::parseTableName(table); const std::string endpoint = std::filesystem::path(NAMESPACES_ENDPOINT) / encodeNamespaceForURI(namespace_name) / "tables" / table_name; @@ -2354,7 +2645,8 @@ ICatalog::CredentialsRefreshCallback RestCatalog::getCredentialsConfigurationCal { ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogGetCredentials); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogGetCredentialsMicroseconds); - auto buf = createReadBuffer(*state_snapshot, state_snapshot->config.prefix / endpoint, /* params */{}, headers); + auto buf = createReadBuffer( + *state_snapshot, state_snapshot.generation, state_snapshot->config.prefix / endpoint, auth_token, /* params */{}, headers); if (buf->eof()) { @@ -2394,7 +2686,8 @@ ICatalog::CredentialsRefreshCallback RestCatalog::getCredentialsConfigurationCal if (metadata_object) parsed.table_uuid = parseTableUuid(metadata_object); /// Refresh the per-table cache so subsequent queries reuse these freshly vended credentials. - cacheCredentials(namespace_name, table_name, parsed); + cacheCredentials( + {state_snapshot.generation, getCredentialsCachePrincipal(auth_token), namespace_name, table_name}, parsed); return parsed.credentials; }; } diff --git a/src/Databases/DataLake/RestCatalog.h b/src/Databases/DataLake/RestCatalog.h index 0c54e43280c0..e730e738bdbe 100644 --- a/src/Databases/DataLake/RestCatalog.h +++ b/src/Databases/DataLake/RestCatalog.h @@ -2,8 +2,11 @@ #include "config.h" #if USE_AVRO +#include #include #include +#include +#include #include #include #include @@ -47,6 +50,48 @@ struct VendedStorageCredentials std::string table_uuid = {}; }; +struct TokenForwardingConfig +{ + bool forward_user_token = false; + String token_exchange_uri; + String subject_token_type; + String requested_token_type; + bool forward_actor_token = false; + UInt64 user_token_cache_ttl = 0; + + bool exchangeEnabled() const { return forward_user_token && !token_exchange_uri.empty(); } +}; + +struct TokenRequest +{ + enum class Grant + { + ClientCredentials, + TokenExchange, + }; + + Grant grant = Grant::ClientCredentials; + Poco::URI url; + bool use_query_parameters = false; + String scope; + String client_id; + String client_secret; + String subject_token; + String subject_token_type; + String requested_token_type; + String actor_token; +}; + +struct CredentialsCacheKey +{ + UInt64 generation = 0; + std::string principal; + std::string namespace_name; + std::string table_name; + + auto operator<=>(const CredentialsCacheKey &) const = default; +}; + class RestCatalog : public ICatalog, public DB::WithContext { public: @@ -59,15 +104,16 @@ class RestCatalog : public ICatalog, public DB::WithContext const std::string & oauth_server_uri_, bool oauth_server_use_request_body_, const std::string & namespaces_, - DB::ContextPtr context_); + DB::ContextPtr context_, + const TokenForwardingConfig & token_forwarding_ = {}); ~RestCatalog() override = default; - bool empty() const override; + bool empty(const DB::ForwardedAuthTokenPtr & auth_token) const override; - DB::Names getTables() const override; + DB::Names getTables(const DB::ForwardedAuthTokenPtr & auth_token) const override; - bool existsTable(const std::string & namespace_name, const std::string & table_name) const override; + bool existsTable(const std::string & namespace_name, const std::string & table_name, const DB::ForwardedAuthTokenPtr & auth_token) const override; void getTableMetadata( const std::string & namespace_name, @@ -88,9 +134,9 @@ class RestCatalog : public ICatalog, public DB::WithContext return DB::DatabaseDataLakeCatalogType::ICEBERG_REST; } - void createTable(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr metadata_content) const override; + void createTable(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr metadata_content, const DB::ForwardedAuthTokenPtr & auth_token) const override; - bool updateMetadata(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr new_snapshot) const override; + bool updateMetadata(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr new_snapshot, const DB::ForwardedAuthTokenPtr & auth_token) const override; bool updateSchema( const String & namespace_name, @@ -99,13 +145,19 @@ class RestCatalog : public ICatalog, public DB::WithContext Poco::JSON::Object::Ptr new_schema, Int32 previous_schema_id, Int32 new_last_column_id, - Poco::JSON::Object::Ptr metadata = nullptr) const override; + Poco::JSON::Object::Ptr metadata, + const DB::ForwardedAuthTokenPtr & auth_token) const override; bool isTransactional() const override { return true; } - void dropTable(const String & namespace_name, const String & table_name) const override; + void dropTable(const String & namespace_name, const String & table_name, const DB::ForwardedAuthTokenPtr & auth_token) const override; + + ICatalog::CredentialsRefreshCallback getCredentialsConfigurationCallback( + const DB::StorageID & storage_id, const DB::ForwardedAuthTokenPtr & auth_token) override; - ICatalog::CredentialsRefreshCallback getCredentialsConfigurationCallback(const DB::StorageID & storage_id) override; + void onTokenForwardingDisabled() const override { user_token_cache.clear(); } + + void loadConfigIfNeeded(const DB::ForwardedAuthTokenPtr & auth_token) const; void setVendedCredentialsCacheTTL(std::chrono::seconds ttl) override { vended_credentials_cache_ttl.store(ttl, std::memory_order_relaxed); } @@ -132,12 +184,43 @@ class RestCatalog : public ICatalog, public DB::WithContext std::string tenant_id; std::string bearer_token; Config config; + bool config_loaded = false; }; using CatalogStateVersion = MultiVersion::Version; - CatalogStateVersion getStateSnapshot() const { return state.get(); } + struct AuthContext + { + /// Keep the endpoint and authentication from the same state snapshot. + const CatalogState & catalog_state; + UInt64 generation = 0; + bool update_token = false; + String method; + Poco::URI url; + DB::HTTPHeaderEntries extra_headers; + String body; + DB::ForwardedAuthTokenPtr auth_token; + /// The caller records cache hits only after the catalog request succeeds. + bool * used_cached_oauth_token = nullptr; + }; + + struct StateSnapshot + { + UInt64 generation = 0; + CatalogStateVersion state; + + const CatalogState & operator*() const { return *state; } + const CatalogState * operator->() const { return state.get(); } + }; + + /// Read the generation first so an old state cannot cache credentials under a new generation. + StateSnapshot getStateSnapshot() const + { + const UInt64 generation = auth_generation.load(std::memory_order_acquire); + return StateSnapshot{generation, state.get()}; + } - ICatalog::PreparedSettingsChangesPtr prepareSettingsChanges(const DB::SettingsChanges & changes) override; + ICatalog::PreparedSettingsChangesPtr prepareSettingsChanges( + const DB::SettingsChanges & changes, const DB::ForwardedAuthTokenPtr & auth_token = {}) override; void commitSettingsChanges(ICatalog::PreparedSettingsChangesPtr prepared) override; @@ -161,30 +244,47 @@ class RestCatalog : public ICatalog, public DB::WithContext const std::string & namespaces_, DB::ContextPtr context_); - void createNamespaceIfNotExists(const String & namespace_name, const String & location) const override; + void createNamespaceIfNotExists(const String & namespace_name, const String & location, const DB::ForwardedAuthTokenPtr & auth_token) const override; const std::filesystem::path base_url; const LoggerPtr log; - MultiVersion state{std::make_unique()}; + mutable MultiVersion state{std::make_unique()}; + mutable std::mutex config_mutex; /// Parameters for OAuth (common for REST catalog). bool update_token_if_expired = false; std::string auth_scope; std::string oauth_server_uri; bool oauth_server_use_request_body; + /// Shared service or actor token; never store a user token here. mutable MultiVersion access_token; + TokenForwardingConfig token_forwarding; + + static constexpr size_t user_token_cache_max_entries = 1024; + mutable DB::CacheBase user_token_cache; + + /// Separate from `CatalogState`, which can be republished without an auth change. + /// Old requests retain their generation so their cache writes become unreachable after rotation. + std::atomic auth_generation{0}; + + /// Keep generation checks and token publication atomic with credential rotation. + /// Never hold this across a network request. + mutable std::mutex auth_publish_mutex; + /// TTL for caching vended credentials per table (0 means no caching). std::atomic vended_credentials_cache_ttl{std::chrono::seconds::zero()}; /// Sweep trigger threshold, not capacity! static constexpr size_t credentials_cache_cleanup_threshold = 1000; + static constexpr size_t credentials_cache_max_entries = 10000; + static constexpr std::chrono::seconds credentials_expiry_safety_window{60}; mutable std::mutex credentials_cache_mutex; - mutable std::map, VendedStorageCredentials> credentials_cache + mutable std::map credentials_cache TSA_GUARDED_BY(credentials_cache_mutex); public: @@ -213,7 +313,9 @@ class RestCatalog : public ICatalog, public DB::WithContext /// request never mixes the endpoint of one state version with the auth of another. DB::ReadWriteBufferFromHTTPPtr createReadBuffer( const CatalogState & catalog_state, + UInt64 generation, const std::string & endpoint, + const DB::ForwardedAuthTokenPtr & auth_token, const Poco::URI::QueryParameters & params = {}, const DB::HTTPHeaderEntries & headers = {}, const std::optional & auth_headers = std::nullopt) const; @@ -227,13 +329,14 @@ class RestCatalog : public ICatalog, public DB::WithContext const std::string & base_namespace, Namespaces & result, StopCondition stop_condition, - ExecuteFunc func) const; + ExecuteFunc func, + const DB::ForwardedAuthTokenPtr & auth_token) const; - Namespaces getNamespaces(const std::string & base_namespace) const; + Namespaces getNamespaces(const std::string & base_namespace, const DB::ForwardedAuthTokenPtr & auth_token) const; Namespaces parseNamespaces(DB::ReadBuffer & buf, const std::string & base_namespace, String & next_page_token) const; - DB::Names getTables(const std::string & base_namespace, size_t limit = 0) const; + DB::Names getTablesInNamespace(const std::string & base_namespace, const DB::ForwardedAuthTokenPtr & auth_token, size_t limit = 0) const; DB::Names parseTables(DB::ReadBuffer & buf, const std::string & base_namespace, size_t limit, String & next_page_token) const; @@ -242,21 +345,31 @@ class RestCatalog : public ICatalog, public DB::WithContext const std::string & table_name, DB::ContextPtr context_, TableMetadata & result, + const DB::ForwardedAuthTokenPtr & auth_token, bool allow_credentials_cache = true) const; + bool tryGetTableMetadataImpl( + const std::string & namespace_name, + const std::string & table_name, + DB::ContextPtr context_, + TableMetadata & result, + const DB::ForwardedAuthTokenPtr & auth_token) const; + /// Load catalog config (special http handler) utilizing information from catalog_state and auth_headers. - Config loadConfig(const CatalogState & catalog_state, const std::optional & auth_headers = std::nullopt); - /// `method`, `url`, `extra_headers` and `body` describe the request being authenticated. They are - /// used by catalogs that sign the request itself (AWS SigV4 in `S3TablesCatalog`); catalogs that - /// authenticate with a token or a static header ignore them. - virtual DB::HTTPHeaderEntries getAuthHeaders( + Config loadConfig( const CatalogState & catalog_state, - bool update_token, - const String & method = {}, - const Poco::URI & url = {}, - const DB::HTTPHeaderEntries & extra_headers = {}, - const String & body = {}, - bool * used_cached_oauth_token = nullptr) const; + UInt64 generation, + const DB::ForwardedAuthTokenPtr & auth_token, + const std::optional & auth_headers = std::nullopt) const; + + virtual DB::HTTPHeaderEntries getAuthHeaders(const AuthContext & auth_context) const; + + void validateForwardedToken(const DB::ForwardedAuthTokenPtr & auth_token) const; + + String getForwardedToken( + const CatalogState & catalog_state, UInt64 generation, const DB::ForwardedAuthTokenPtr & auth_token, bool update_token) const; + + bool shouldRetryWithFreshToken(Poco::Net::HTTPResponse::HTTPStatus status) const; void validateAuthHeaders(const DB::HTTPHeaderEntry & header) const; @@ -264,30 +377,35 @@ class RestCatalog : public ICatalog, public DB::WithContext void sendRequest( const CatalogState & catalog_state, + UInt64 generation, const String & endpoint, Poco::JSON::Object::Ptr request_body, + const DB::ForwardedAuthTokenPtr & auth_token, const String & method = Poco::Net::HTTPRequest::HTTP_POST, bool ignore_result = false) const; VendedStorageCredentials getCredentialsAndEndpoint(Poco::JSON::Object::Ptr object, const String & location) const; - std::optional tryGetCachedCredentials( - const std::string & namespace_name, const std::string & table_name) const; + String getCredentialsCachePrincipal(const DB::ForwardedAuthTokenPtr & auth_token) const; - void cacheCredentials( - const std::string & namespace_name, - const std::string & table_name, - const VendedStorageCredentials & parsed) const; + std::optional tryGetCachedCredentials(const CredentialsCacheKey & key) const; + + void cacheCredentials(const CredentialsCacheKey & key, const VendedStorageCredentials & parsed) const; + + MultiVersion::Version publishServiceToken(AccessToken minted, UInt64 generation) const; + + AccessToken requestToken(const TokenRequest & request) const; + + AccessToken exchangeUserToken( + const CatalogState & catalog_state, UInt64 generation, const DB::ForwardedAuthToken & auth_token, + const AccessToken * prepared_actor_token = nullptr) const; AccessToken retrieveAccessToken(const std::string & client_id, const std::string & client_secret) const; + String getServicePrincipalToken(const CatalogState & catalog_state, UInt64 generation) const; + struct PreparedAuthChanges; - /// Hook for `prepareSettingsChanges`: validate `changes` and apply them to `new_state`, - /// building the new auth artifacts, without publishing anything. When the OAuth - /// credentials change, the eagerly fetched token goes into `new_access_token` and - /// `new_auth_headers`, so that wrong credentials fail the ALTER right here and the - /// config reload authenticates with the new token instead of the cached one. virtual void applySettingsChangesToState( const DB::SettingsChanges & changes, const CatalogState & old_state, @@ -317,14 +435,7 @@ class OneLakeCatalog : public RestCatalog return DB::DatabaseDataLakeCatalogType::ICEBERG_ONELAKE; } - DB::HTTPHeaderEntries getAuthHeaders( - const CatalogState & catalog_state, - bool update_token, - const String & method = {}, - const Poco::URI & url = {}, - const DB::HTTPHeaderEntries & extra_headers = {}, - const String & body = {}, - bool * used_cached_oauth_token = nullptr) const override; + DB::HTTPHeaderEntries getAuthHeaders(const AuthContext & auth_context) const override; /// `bearer_mode` means the catalog authenticates with `onelake_bearer_token`, /// otherwise with the `onelake_client_id` + `onelake_client_secret` pair. @@ -361,14 +472,7 @@ class BigLakeCatalog : public RestCatalog return DB::DatabaseDataLakeCatalogType::ICEBERG_BIGLAKE; } - DB::HTTPHeaderEntries getAuthHeaders( - const CatalogState & catalog_state, - bool update_token, - const String & method = {}, - const Poco::URI & url = {}, - const DB::HTTPHeaderEntries & extra_headers = {}, - const String & body = {}, - bool * used_cached_oauth_token = nullptr) const override; + DB::HTTPHeaderEntries getAuthHeaders(const AuthContext & auth_context) const override; const std::string & getGoogleADCClientId() const { return google_adc_client_id; } const std::string & getGoogleADCClientSecret() const { return google_adc_client_secret; } diff --git a/src/Databases/DataLake/S3TablesCatalog.cpp b/src/Databases/DataLake/S3TablesCatalog.cpp index 961125be9819..d28987fe346e 100644 --- a/src/Databases/DataLake/S3TablesCatalog.cpp +++ b/src/Databases/DataLake/S3TablesCatalog.cpp @@ -114,7 +114,8 @@ S3TablesCatalog::S3TablesCatalog( /* urlEscapePath = */ false); CatalogState initial_state; - initial_state.config = loadConfig(initial_state); + initial_state.config = loadConfig(initial_state, /* generation */ 0, /* auth_token */ {}); + initial_state.config_loaded = true; if (initial_state.config.prefix.empty()) { @@ -128,9 +129,9 @@ S3TablesCatalog::S3TablesCatalog( /// S3 Tables only supports a single level of namespaces (no nesting), /// so we use flat getNamespaces() instead of the base class's getNamespacesRecursive(). -DB::Names S3TablesCatalog::getTables() const +DB::Names S3TablesCatalog::getTables(const DB::ForwardedAuthTokenPtr & auth_token) const { - auto namespaces = getNamespaces(""); + auto namespaces = getNamespaces("", auth_token); auto & pool = getContext()->getIcebergCatalogThreadpool(); DB::ThreadPoolCallbackRunnerLocal runner(pool, DB::ThreadName::DATALAKE_REST_CATALOG); @@ -142,7 +143,7 @@ DB::Names S3TablesCatalog::getTables() const runner.enqueueAndKeepTrack( [&, ns] { - auto tables_in_ns = RestCatalog::getTables(ns); + auto tables_in_ns = RestCatalog::getTablesInNamespace(ns, auth_token); std::lock_guard lock(mutex); std::move(tables_in_ns.begin(), tables_in_ns.end(), std::back_inserter(tables)); }); @@ -197,9 +198,10 @@ bool S3TablesCatalog::tryGetTableMetadata( return true; } -ICatalog::CredentialsRefreshCallback S3TablesCatalog::getCredentialsConfigurationCallback(const DB::StorageID & storage_id) +ICatalog::CredentialsRefreshCallback S3TablesCatalog::getCredentialsConfigurationCallback( + const DB::StorageID & storage_id, const DB::ForwardedAuthTokenPtr & auth_token) { - auto base_cb = RestCatalog::getCredentialsConfigurationCallback(storage_id); + auto base_cb = RestCatalog::getCredentialsConfigurationCallback(storage_id, auth_token); return [this, base_callback = std::move(base_cb)] () -> std::shared_ptr { if (base_callback) @@ -217,9 +219,9 @@ ICatalog::CredentialsRefreshCallback S3TablesCatalog::getCredentialsConfiguratio }; } -void S3TablesCatalog::dropTable(const String & namespace_name, const String & table_name) const +void S3TablesCatalog::dropTable(const String & namespace_name, const String & table_name, const DB::ForwardedAuthTokenPtr & auth_token) const { - const auto state_snapshot = state.get(); + const auto state_snapshot = getStateSnapshot(); const std::string endpoint = (base_url / state_snapshot->config.prefix / "namespaces" / namespace_name / "tables" / table_name).string() + "?purgeRequested=True"; @@ -229,7 +231,8 @@ void S3TablesCatalog::dropTable(const String & namespace_name, const String & ta { ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogDropTable); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogDropTableMicroseconds); - sendRequest(*state_snapshot, endpoint, request_body, Poco::Net::HTTPRequest::HTTP_DELETE, true); + sendRequest( + *state_snapshot, state_snapshot.generation, endpoint, request_body, auth_token, Poco::Net::HTTPRequest::HTTP_DELETE, true); } catch (const DB::HTTPException & ex) { @@ -240,17 +243,12 @@ void S3TablesCatalog::dropTable(const String & namespace_name, const String & ta } } -DB::HTTPHeaderEntries S3TablesCatalog::getAuthHeaders( - const CatalogState & /*catalog_state*/, - bool /*update_token*/, - const String & method, - const Poco::URI & url, - const DB::HTTPHeaderEntries & extra_headers, - const String & body, - bool * /*used_cached_oauth_token*/) const +DB::HTTPHeaderEntries S3TablesCatalog::getAuthHeaders(const AuthContext & auth_context) const { DB::HTTPHeaderEntries all_signed; - signRequestWithAWSV4(method, url, extra_headers, body, *signer, region, "s3tables", all_signed); + signRequestWithAWSV4( + auth_context.method, auth_context.url, auth_context.extra_headers, auth_context.body, + *signer, region, "s3tables", all_signed); DB::HTTPHeaderEntries auth_headers; for (auto & h : all_signed) diff --git a/src/Databases/DataLake/S3TablesCatalog.h b/src/Databases/DataLake/S3TablesCatalog.h index a878bb17d924..95b6890eb57e 100644 --- a/src/Databases/DataLake/S3TablesCatalog.h +++ b/src/Databases/DataLake/S3TablesCatalog.h @@ -33,7 +33,7 @@ class S3TablesCatalog final : public RestCatalog DB::DatabaseDataLakeCatalogType getCatalogType() const override { return DB::DatabaseDataLakeCatalogType::S3_TABLES; } - DB::Names getTables() const override; + DB::Names getTables(const DB::ForwardedAuthTokenPtr & auth_token) const override; bool tryGetTableMetadata( const std::string & namespace_name, @@ -41,19 +41,13 @@ class S3TablesCatalog final : public RestCatalog DB::ContextPtr context_, TableMetadata & result) const override; - void dropTable(const String & namespace_name, const String & table_name) const override; + void dropTable(const String & namespace_name, const String & table_name, const DB::ForwardedAuthTokenPtr & auth_token) const override; - ICatalog::CredentialsRefreshCallback getCredentialsConfigurationCallback(const DB::StorageID & storage_id) override; + ICatalog::CredentialsRefreshCallback getCredentialsConfigurationCallback( + const DB::StorageID & storage_id, const DB::ForwardedAuthTokenPtr & auth_token) override; protected: - DB::HTTPHeaderEntries getAuthHeaders( - const CatalogState & catalog_state, - bool update_token, - const String & method = {}, - const Poco::URI & url = {}, - const DB::HTTPHeaderEntries & extra_headers = {}, - const String & body = {}, - bool * used_cached_oauth_token = nullptr) const override; + DB::HTTPHeaderEntries getAuthHeaders(const AuthContext & auth_context) const override; private: const String region; diff --git a/src/Databases/DataLake/UnityCatalog.cpp b/src/Databases/DataLake/UnityCatalog.cpp index 57f3e02f9d09..b48306e72441 100644 --- a/src/Databases/DataLake/UnityCatalog.cpp +++ b/src/Databases/DataLake/UnityCatalog.cpp @@ -80,7 +80,7 @@ std::pair UnityCatalog::postJSONRequest(const s return makeHTTPRequestAndReadJSON(base_url / route, context, credentials, {}, {auth_header}, Poco::Net::HTTPRequest::HTTP_POST, out_stream_callaback); } -bool UnityCatalog::empty() const +bool UnityCatalog::empty(const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { auto all_schemas = getSchemas(""); for (const auto & schema : all_schemas) @@ -92,7 +92,7 @@ bool UnityCatalog::empty() const return true; } -DB::Names UnityCatalog::getTables() const +DB::Names UnityCatalog::getTables(const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { DB::Names result; @@ -314,7 +314,7 @@ bool UnityCatalog::tryGetTableMetadata( } } -bool UnityCatalog::existsTable(const std::string & schema_name, const std::string & table_name) const +bool UnityCatalog::existsTable(const std::string & schema_name, const std::string & table_name, const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { if (!isNamespaceAllowed(schema_name)) throw DB::Exception(DB::ErrorCodes::CATALOG_NAMESPACE_DISABLED, "Namespace {} is filtered by `namespaces` database parameter", schema_name); @@ -499,7 +499,8 @@ bool UnityCatalog::isNamespaceAllowed(const std::string & namespace_) const } /// getCredentialsConfigurationCallback method is supported only for S3 storage -ICatalog::CredentialsRefreshCallback UnityCatalog::getCredentialsConfigurationCallback(const DB::StorageID & table_id) +ICatalog::CredentialsRefreshCallback UnityCatalog::getCredentialsConfigurationCallback( + const DB::StorageID & table_id, const DB::ForwardedAuthTokenPtr & /*auth_token*/) { if (!table_id.hasUUID()) throw DB::Exception( diff --git a/src/Databases/DataLake/UnityCatalog.h b/src/Databases/DataLake/UnityCatalog.h index caa66cf90044..f844e389b86c 100644 --- a/src/Databases/DataLake/UnityCatalog.h +++ b/src/Databases/DataLake/UnityCatalog.h @@ -27,11 +27,11 @@ class UnityCatalog final : public ICatalog, private DB::WithContext ~UnityCatalog() override = default; - bool empty() const override; + bool empty(const DB::ForwardedAuthTokenPtr & auth_token) const override; - DB::Names getTables() const override; + DB::Names getTables(const DB::ForwardedAuthTokenPtr & auth_token) const override; - bool existsTable(const std::string & schema_name, const std::string & table_name) const override; + bool existsTable(const std::string & schema_name, const std::string & table_name, const DB::ForwardedAuthTokenPtr & auth_token) const override; void getTableMetadata( const std::string & namespace_name, @@ -82,7 +82,8 @@ class UnityCatalog final : public ICatalog, private DB::WithContext const std::string & table_name, TableMetadata & result) const; - ICatalog::CredentialsRefreshCallback getCredentialsConfigurationCallback(const DB::StorageID & table_id) override; + ICatalog::CredentialsRefreshCallback getCredentialsConfigurationCallback( + const DB::StorageID & table_id, const DB::ForwardedAuthTokenPtr & auth_token) override; }; } diff --git a/src/Databases/DataLake/tests/gtest_rest_catalog.cpp b/src/Databases/DataLake/tests/gtest_rest_catalog.cpp index 9e5c3a7f86a8..4891ee54cce7 100644 --- a/src/Databases/DataLake/tests/gtest_rest_catalog.cpp +++ b/src/Databases/DataLake/tests/gtest_rest_catalog.cpp @@ -7,33 +7,20 @@ #include #include #include -#include +#include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include +#include #include using namespace DataLake; +using namespace RestCatalogTest; namespace DB { namespace ErrorCodes { - extern const int LOGICAL_ERROR; extern const int BAD_ARGUMENTS; - extern const int NOT_IMPLEMENTED; } } @@ -47,177 +34,65 @@ enum class CatalogShape Empty, }; -void writeJSON(Poco::Net::HTTPServerResponse & response, const std::string & body) +std::string getParent(const std::string & query) { - response.setStatus(Poco::Net::HTTPResponse::HTTP_OK); - response.setContentType("application/json"); - response.setContentLength(body.size()); - response.send() << body; + Poco::URI uri; + uri.setRawQuery(query); + for (const auto & [key, value] : uri.getQueryParameters()) + if (key == "parent") + return value; + return {}; } -void writeError(Poco::Net::HTTPServerResponse & response, Poco::Net::HTTPResponse::HTTPStatus status, const std::string & body) +void installShape(ServerState & state, CatalogShape shape) { - response.setStatus(status); - response.setContentType("application/json"); - response.setContentLength(body.size()); - response.send() << body; -} - -std::string getRawPath(const std::string & uri) -{ - const auto query_pos = uri.find('?'); - if (query_pos == std::string::npos) - return uri; - return uri.substr(0, query_pos); -} - -class RestCatalogRequestHandler final : public Poco::Net::HTTPRequestHandler -{ -public: - explicit RestCatalogRequestHandler(CatalogShape shape_) - : shape(shape_) - { - } - - void handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) override + state.setRoute("/v1/namespaces", [shape](const RecordedRequest & request) { - Poco::URI uri(request.getURI()); - const auto path = getRawPath(request.getURI()); - const auto params = uri.getQueryParameters(); - - if (path == "/v1/config") - { - writeJSON(response, R"({"defaults":{},"overrides":{}})"); - return; - } - - if (path == "/v1/oauth/tokens") - { - writeJSON(response, R"({"token_type":"Bearer","expires_in":3600,"access_token":"mock-access-token"})"); - return; - } - - if (path == "/v1/namespaces") - { - const auto parent = getParent(params); - if (parent.empty()) - { - if (shape == CatalogShape::NestedTableThenEmptySibling) - writeJSON(response, R"({"namespaces":[["parent"],["empty_later"]]})"); - else - writeJSON(response, R"({"namespaces":[["namespace"]]})"); - return; - } - - if (shape == CatalogShape::NestedTableThenEmptySibling && parent == "parent") - writeJSON(response, R"({"namespaces":[["leaf_with_table"]]})"); - else - writeJSON(response, R"({"namespaces":[]})"); - return; - } - - if (path == "/v1/namespaces/namespace/tables") - { - if (shape == CatalogShape::TopLevelTable) - writeJSON(response, R"({"identifiers":[{"name":"table_a"}]})"); - else - writeJSON(response, R"({"identifiers":[]})"); - return; - } - - if (path == "/v1/namespaces/parent/tables" - || path == "/v1/namespaces/empty_later/tables") - { - writeJSON(response, R"({"identifiers":[]})"); - return; - } - - if (path == "/v1/namespaces/parent%1Fleaf_with_table/tables") - { - writeJSON(response, R"({"identifiers":[{"name":"table_a"}]})"); - return; - } - - if (path == "/v1/namespaces/namespace/tables/table_a") - { - writeJSON(response, R"({"metadata":{"table-uuid":"11111111-2222-3333-4444-555555555555"}})"); - return; - } - - if (path == "/v1/namespaces/namespace/tables/missing_table") - { - writeError(response, Poco::Net::HTTPResponse::HTTP_NOT_FOUND, R"({"error":{"message":"Table does not exist","type":"NoSuchTableException","code":404}})"); - return; - } - - if (path == "/v1/namespaces/namespace/tables/unauthorized_table") + const auto parent = getParent(request.query); + if (parent.empty()) { - writeError(response, Poco::Net::HTTPResponse::HTTP_UNAUTHORIZED, R"({"error":{"message":"The access token has expired","type":"NotAuthorizedException","code":401}})"); - return; + if (shape == CatalogShape::NestedTableThenEmptySibling) + return json(R"({"namespaces":[["parent"],["empty_later"]]})"); + return json(R"({"namespaces":[["namespace"]]})"); } - throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Unexpected request to fake Iceberg REST catalog: {}", request.getURI()); - } + if (shape == CatalogShape::NestedTableThenEmptySibling && parent == "parent") + return json(R"({"namespaces":[["leaf_with_table"]]})"); + return json(R"({"namespaces":[]})"); + }); -private: - static std::string getParent(const Poco::URI::QueryParameters & params) + state.setRoute("/v1/namespaces/namespace/tables", [shape](const RecordedRequest &) { - for (const auto & [key, value] : params) - { - if (key == "parent") - return value; - } - return {}; - } - - CatalogShape shape; -}; + if (shape == CatalogShape::TopLevelTable) + return json(R"({"identifiers":[{"name":"table_a"}]})"); + return json(R"({"identifiers":[]})"); + }); + + state.setStaticRoute("/v1/namespaces/parent/tables", R"({"identifiers":[]})"); + state.setStaticRoute("/v1/namespaces/empty_later/tables", R"({"identifiers":[]})"); + state.setStaticRoute("/v1/namespaces/parent%1Fleaf_with_table/tables", R"({"identifiers":[{"name":"table_a"}]})"); +} -class RestCatalogRequestHandlerFactory final : public Poco::Net::HTTPRequestHandlerFactory +void installTokenEndpoint(ServerState & state) { -public: - explicit RestCatalogRequestHandlerFactory(CatalogShape shape_) - : shape(shape_) - { - } - - Poco::Net::HTTPRequestHandler * createRequestHandler(const Poco::Net::HTTPServerRequest &) override - { - return new RestCatalogRequestHandler(shape); - } - -private: - CatalogShape shape; -}; + state.setStaticRoute("/v1/oauth/tokens", R"({"token_type":"Bearer","expires_in":3600,"access_token":"mock-access-token"})"); +} -class RestCatalogTestServer +void installTableRoutes(ServerState & state) { -public: - explicit RestCatalogTestServer(CatalogShape shape) - : server_socket(std::make_unique(Poco::Net::SocketAddress("127.0.0.1", 0))) - , handler_factory(new RestCatalogRequestHandlerFactory(shape)) - , server_params(new Poco::Net::HTTPServerParams()) - , server(std::make_unique(handler_factory, *server_socket, server_params)) - { - server->start(); - } - - ~RestCatalogTestServer() + state.setStaticRoute( + "/v1/namespaces/namespace/tables/table_a", R"({"metadata":{"table-uuid":"11111111-2222-3333-4444-555555555555"}})"); + state.setRoute("/v1/namespaces/namespace/tables/missing_table", [](const RecordedRequest &) { - server->stop(); - } - - std::string getUrl() const + return respondWithStatus( + 404, R"({"error":{"message":"Table does not exist","type":"NoSuchTableException","code":404}})"); + }); + state.setRoute("/v1/namespaces/namespace/tables/unauthorized_table", [](const RecordedRequest &) { - return "http://" + server_socket->address().toString(); - } - -private: - std::unique_ptr server_socket; - Poco::SharedPtr handler_factory; - Poco::AutoPtr server_params; - std::unique_ptr server; -}; + return respondWithStatus( + 401, R"({"error":{"message":"The access token has expired","type":"NotAuthorizedException","code":401}})"); + }); +} void expectThrowsCode(std::function fn, int expected_code) { @@ -234,7 +109,9 @@ void expectThrowsCode(std::function fn, int expected_code) bool restCatalogEmpty(CatalogShape shape) { - RestCatalogTestServer server(shape); + TestServer server; + installShape(*server, shape); + auto context = DB::Context::createCopy(getContext().context); context->makeQueryContext(); @@ -249,7 +126,7 @@ bool restCatalogEmpty(CatalogShape shape) /* namespaces */"*", context); - return catalog.empty(); + return catalog.empty(/* auth_token */ {}); } } @@ -271,7 +148,8 @@ TEST(RestCatalog, EmptyReturnsTrueWhenNoTablesExist) TEST(RestCatalog, ApplySettingsChangesWithoutAuthenticationRejected) { - RestCatalogTestServer server(CatalogShape::Empty); + TestServer server; + auto context = DB::Context::createCopy(getContext().context); context->makeQueryContext(); @@ -293,7 +171,9 @@ TEST(RestCatalog, ApplySettingsChangesWithoutAuthenticationRejected) TEST(RestCatalog, ApplySettingsChangesCredentialMode) { - RestCatalogTestServer server(CatalogShape::Empty); + TestServer server; + installTokenEndpoint(*server); + auto context = DB::Context::createCopy(getContext().context); context->makeQueryContext(); @@ -335,7 +215,8 @@ TEST(RestCatalog, ApplySettingsChangesCredentialMode) TEST(RestCatalog, ApplySettingsChangesAuthHeaderMode) { - RestCatalogTestServer server(CatalogShape::Empty); + TestServer server; + auto context = DB::Context::createCopy(getContext().context); context->makeQueryContext(); @@ -365,7 +246,8 @@ TEST(RestCatalog, ApplySettingsChangesAuthHeaderMode) TEST(RestCatalog, OneLakeApplySettingsChangesBearerMode) { - RestCatalogTestServer server(CatalogShape::Empty); + TestServer server; + auto context = DB::Context::createCopy(getContext().context); context->makeQueryContext(); @@ -419,7 +301,9 @@ TEST(RestCatalog, OneLakeApplySettingsChangesBearerMode) TEST(RestCatalog, TryGetTableMetadataDistinguishesMissingTableFromOtherErrors) { - RestCatalogTestServer server(CatalogShape::TopLevelTable); + TestServer server; + installTableRoutes(*server); + auto context = DB::Context::createCopy(getContext().context); context->makeQueryContext(); @@ -436,15 +320,15 @@ TEST(RestCatalog, TryGetTableMetadataDistinguishesMissingTableFromOtherErrors) TableMetadata existing; EXPECT_TRUE(catalog.tryGetTableMetadata("namespace", "table_a", context, existing)); - EXPECT_TRUE(catalog.existsTable("namespace", "table_a")); + EXPECT_TRUE(catalog.existsTable("namespace", "table_a", /* auth_token */ {})); TableMetadata missing; EXPECT_FALSE(catalog.tryGetTableMetadata("namespace", "missing_table", context, missing)); - EXPECT_FALSE(catalog.existsTable("namespace", "missing_table")); + EXPECT_FALSE(catalog.existsTable("namespace", "missing_table", /* auth_token */ {})); TableMetadata unauthorized; EXPECT_THROW(catalog.tryGetTableMetadata("namespace", "unauthorized_table", context, unauthorized), DB::HTTPException); - EXPECT_THROW(catalog.existsTable("namespace", "unauthorized_table"), DB::HTTPException); + EXPECT_THROW(catalog.existsTable("namespace", "unauthorized_table", /* auth_token */ {}), DB::HTTPException); } #endif diff --git a/src/Databases/DataLake/tests/gtest_rest_catalog_token_forwarding.cpp b/src/Databases/DataLake/tests/gtest_rest_catalog_token_forwarding.cpp new file mode 100644 index 000000000000..da4d69d6420f --- /dev/null +++ b/src/Databases/DataLake/tests/gtest_rest_catalog_token_forwarding.cpp @@ -0,0 +1,379 @@ +#include "config.h" + +#if USE_AVRO + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include +#include +#include +#include +#include +#include + +using namespace DataLake; +using namespace RestCatalogTest; + +namespace +{ + +constexpr auto CONFIG_PATH = "/v1/config"; +constexpr auto NAMESPACES_PATH = "/v1/namespaces"; +constexpr auto NS_TABLES_PATH = "/v1/namespaces/ns/tables"; +constexpr auto TABLE_PATH = "/v1/namespaces/ns/tables/t"; +constexpr auto CATALOG_TOKEN_PATH = "/v1/oauth/tokens"; +constexpr auto IDP_TOKEN_PATH = "/idp/token"; + +constexpr auto ALICE_TOKEN = "alice.jwt.token"; + +DB::ForwardedAuthTokenPtr makeToken() +{ + return DB::makeForwardedAuthToken(DB::TokenCredentials(ALICE_TOKEN), "alice"); +} + +DB::ContextMutablePtr makeQueryContext(const DB::ForwardedAuthTokenPtr & auth_token = {}) +{ + auto context = DB::Context::createCopy(getContext().context); + context->makeQueryContext(); + context->setForwardedAuthToken(auth_token); + return context; +} + +void installCatalogShape(ServerState & state) +{ + state.setRoute(NAMESPACES_PATH, [](const RecordedRequest & request) + { + if (request.query.find("parent=") != std::string::npos) + return json(R"({"namespaces":[]})"); + return json(R"({"namespaces":[["ns"]]})"); + }); + state.setStaticRoute(NS_TABLES_PATH, R"({"identifiers":[{"name":"t"}]})"); +} + +std::string loadTableResponse() +{ + const auto expires_at_ms + = std::chrono::duration_cast((std::chrono::system_clock::now() + std::chrono::hours(24)).time_since_epoch()) + .count(); + return fmt::format( + R"({{"metadata-location":"s3://bucket/t/metadata/v1.metadata.json",)" + R"("metadata":{{"table-uuid":"1e1c0e10-0000-4000-8000-000000000001","location":"s3://bucket/t","schemas":[],"current-schema-id":0}},)" + R"("config":{{"s3.access-key-id":"AKIA_VENDED","s3.secret-access-key":"secret","s3.session-token":"session",)" + R"("s3.session-token-expires-at-ms":{}}}}})", + expires_at_ms); +} + +void installTokenEndpoint(ServerState & state) +{ + state.setStaticRoute(IDP_TOKEN_PATH, R"({"access_token":"session_token","expires_in":3600})"); +} + +TokenForwardingConfig exchangeAt(const std::string & uri) +{ + return TokenForwardingConfig{ + .forward_user_token = true, + .token_exchange_uri = uri, + .subject_token_type = "urn:ietf:params:oauth:token-type:access_token", + .requested_token_type = "urn:ietf:params:oauth:token-type:access_token", + .forward_actor_token = false, + .user_token_cache_ttl = 300, + }; +} + +/// `DB::WithContext` retains only a weak pointer; callers must keep the context alive. +std::shared_ptr makeCatalog( + const TestServer & server, + const DB::ContextPtr & context, + const TokenForwardingConfig & forwarding) +{ + return std::make_shared( + "warehouse", + server.getUrl(), + "client:secret", + /* auth_scope */ "lakekeeper", + /* auth_header */ "", + /* oauth_server_uri */ "", + /* oauth_server_use_request_body */ true, + /* namespaces */ "*", + context, + forwarding); +} + +void loadTable(RestCatalog & catalog, const DB::ForwardedAuthTokenPtr & auth_token) +{ + auto query_context = makeQueryContext(auth_token); + TableMetadata metadata; + metadata.withLocation().withStorageCredentials(); + catalog.getTableMetadata("ns", "t", query_context, metadata); +} + +size_t countVendingRequests(const ServerState & state) +{ + size_t count = 0; + for (const auto & request : state.requestsTo(TABLE_PATH)) + if (request.header("X-Iceberg-Access-Delegation") == "vended-credentials") + ++count; + return count; +} + +std::map parseForm(const std::string & body) +{ + Poco::URI uri; + uri.setRawQuery(body); + const auto params = uri.getQueryParameters(); + return {params.begin(), params.end()}; +} + +} + +class RestCatalogTokenForwarding : public ::testing::Test +{ +protected: + RestCatalogTokenForwarding() + : previous(getContext().context->getAccessControl().isTokenForwardingEnabled()) + { + getContext().context->getAccessControl().setTokenForwardingEnabled(true); + } + + ~RestCatalogTokenForwarding() override + { + getContext().context->getAccessControl().setTokenForwardingEnabled(previous); + } + +private: + const bool previous; +}; + +TEST_F(RestCatalogTokenForwarding, PassesUserTokenToCatalog) +{ + TestServer server; + installCatalogShape(*server); + auto context = makeQueryContext(); + TokenForwardingConfig forwarding; + forwarding.forward_user_token = true; + auto catalog = makeCatalog(server, context, forwarding); + + ASSERT_EQ(catalog->getTables(makeToken()), DB::Names{"ns.t"}); + for (const auto * path : {CONFIG_PATH, NAMESPACES_PATH, NS_TABLES_PATH}) + { + const auto requests = server->requestsTo(path); + ASSERT_FALSE(requests.empty()); + for (const auto & request : requests) + EXPECT_EQ(request.header("Authorization"), "Bearer " + std::string(ALICE_TOKEN)); + } + EXPECT_EQ(server->countRequestsTo(CATALOG_TOKEN_PATH), 0u); +} + +TEST_F(RestCatalogTokenForwarding, ExchangesAndCachesEachUserTokenSeparately) +{ + TestServer server; + installCatalogShape(*server); + server->setRoute(IDP_TOKEN_PATH, [](const RecordedRequest & request) + { + return json(fmt::format(R"({{"access_token":"{}_session","expires_in":3600}})", parseForm(request.body).at("subject_token"))); + }); + auto context = makeQueryContext(); + auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH)); + + for (const auto * token : {ALICE_TOKEN, "rotated.alice.token"}) + { + auto auth_token = DB::makeForwardedAuthToken(DB::TokenCredentials(token), "alice"); + server->clearRequests(); + ASSERT_EQ(catalog->getTables(auth_token), DB::Names{"ns.t"}); + ASSERT_EQ(catalog->getTables(auth_token), DB::Names{"ns.t"}); + + const auto exchanges = server->requestsTo(IDP_TOKEN_PATH); + ASSERT_EQ(exchanges.size(), 1u); + const auto form = parseForm(exchanges.front().body); + EXPECT_EQ(form.at("grant_type"), "urn:ietf:params:oauth:grant-type:token-exchange"); + EXPECT_EQ(form.at("subject_token"), token); + EXPECT_EQ(form.at("subject_token_type"), "urn:ietf:params:oauth:token-type:access_token"); + for (const auto & request : server->requestsTo(NAMESPACES_PATH)) + EXPECT_EQ(request.header("Authorization"), "Bearer " + std::string(token) + "_session"); + EXPECT_EQ(server->countRequestsTo(CATALOG_TOKEN_PATH), 0u); + } +} + +TEST_F(RestCatalogTokenForwarding, AlteringCatalogCredentialDropsCachedTokensAndCredentials) +{ + TestServer server; + installTokenEndpoint(*server); + server->setStaticRoute(TABLE_PATH, loadTableResponse()); + + auto alice = makeToken(); + auto context = makeQueryContext(); + auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH)); + catalog->setVendedCredentialsCacheTTL(std::chrono::seconds(300)); + + loadTable(*catalog, alice); + ASSERT_EQ(server->countRequestsTo(IDP_TOKEN_PATH), 1u); + ASSERT_EQ(countVendingRequests(*server), 1u); + + loadTable(*catalog, alice); + ASSERT_EQ(server->countRequestsTo(IDP_TOKEN_PATH), 1u); + ASSERT_EQ(countVendingRequests(*server), 1u); + + DB::SettingsChanges changes; + changes.emplace_back("catalog_credential", "client:rotated_secret"); + catalog->applySettingsChanges(changes, alice); + + loadTable(*catalog, alice); + EXPECT_EQ(countVendingRequests(*server), 2u); + + const auto exchanges = server->requestsTo(IDP_TOKEN_PATH); + ASSERT_EQ(exchanges.size(), 3u); + EXPECT_EQ(server->countRequestsTo(CATALOG_TOKEN_PATH), 0u); + EXPECT_EQ(parseForm(exchanges.back().body).at("client_secret"), "rotated_secret"); +} + +TEST_F(RestCatalogTokenForwarding, RejectedConfigReloadDoesNotPublishPreparedUserSession) +{ + TestServer server; + installCatalogShape(*server); + server->setRoute(IDP_TOKEN_PATH, [](const RecordedRequest & request) + { + return json(fmt::format(R"({{"access_token":"{}_session","expires_in":3600}})", parseForm(request.body).at("client_secret"))); + }); + auto alice = makeToken(); + auto context = makeQueryContext(); + auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH)); + ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); + server->clearRequests(); + server->setRoute(CONFIG_PATH, [](const RecordedRequest &) { return respondWithStatus(403); }); + + DB::SettingsChanges changes; + changes.emplace_back("catalog_credential", "client:rotated_secret"); + EXPECT_THROW(catalog->prepareSettingsChanges(changes, alice), DB::Exception); + ASSERT_EQ(server->countRequestsTo(CONFIG_PATH), 1u); + EXPECT_EQ(server->requestsTo(CONFIG_PATH).front().header("Authorization"), "Bearer rotated_secret_session"); + EXPECT_EQ(server->countRequestsTo(CATALOG_TOKEN_PATH), 0u); + ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); + for (const auto & request : server->requestsTo(NAMESPACES_PATH)) + EXPECT_EQ(request.header("Authorization"), "Bearer secret_session"); +} + +class ParkedRoute +{ +public: + RestCatalogTest::ServerState::Route handler(RestCatalogTest::ServerState::Route response) + { + return [this, response](const RecordedRequest & request) + { + { + std::unique_lock lock(mutex); + if (!arrived) + { + arrived = true; + cv.notify_all(); + cv.wait(lock, [this] { return released; }); + } + } + return response(request); + }; + } + + void waitUntilParked() + { + std::unique_lock lock(mutex); + cv.wait(lock, [this] { return arrived; }); + } + + void release() + { + { + std::lock_guard lock(mutex); + released = true; + } + cv.notify_all(); + } + +private: + std::mutex mutex; + std::condition_variable cv; + bool arrived = false; + bool released = false; +}; + +TEST_F(RestCatalogTokenForwarding, InFlightVendedCredentialsDoNotOutliveTheirGeneration) +{ + ParkedRoute parked; + TestServer server; + installTokenEndpoint(*server); + server->setRoute(TABLE_PATH, parked.handler([](const RecordedRequest &) { return json(loadTableResponse()); })); + + auto alice = makeToken(); + auto context = makeQueryContext(); + auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH)); + catalog->setVendedCredentialsCacheTTL(std::chrono::seconds(300)); + + std::thread in_flight([&] { loadTable(*catalog, alice); }); + SCOPE_EXIT({ + parked.release(); + if (in_flight.joinable()) + in_flight.join(); + }); + + parked.waitUntilParked(); + + DB::SettingsChanges changes; + changes.emplace_back("catalog_credential", "client:rotated_secret"); + catalog->applySettingsChanges(changes, alice); + + parked.release(); + in_flight.join(); + + const auto vends_before = countVendingRequests(*server); + + loadTable(*catalog, alice); + EXPECT_EQ(countVendingRequests(*server), vends_before + 1); +} + +TEST_F(RestCatalogTokenForwarding, ConfigLoadDoesNotRollBackAConcurrentCredentialChange) +{ + ParkedRoute parked; + TestServer server; + installCatalogShape(*server); + installTokenEndpoint(*server); + server->setRoute("/v1/config", parked.handler([](const RecordedRequest &) { return json(R"({"defaults":{},"overrides":{}})"); })); + + auto alice = makeToken(); + auto context = makeQueryContext(); + auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH)); + + std::thread in_flight([&] { catalog->getTables(alice); }); + SCOPE_EXIT({ + parked.release(); + if (in_flight.joinable()) + in_flight.join(); + }); + + parked.waitUntilParked(); + + DB::SettingsChanges changes; + changes.emplace_back("catalog_credential", "client:rotated_secret"); + catalog->applySettingsChanges(changes, alice); + + parked.release(); + in_flight.join(); + + ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); + + const auto exchanges = server->requestsTo(IDP_TOKEN_PATH); + ASSERT_GE(exchanges.size(), 2u); + EXPECT_EQ(parseForm(exchanges.back().body).at("client_secret"), "rotated_secret"); +} + +#endif diff --git a/src/Databases/DataLake/tests/rest_catalog_test_server.h b/src/Databases/DataLake/tests/rest_catalog_test_server.h new file mode 100644 index 000000000000..3179a861c7a6 --- /dev/null +++ b/src/Databases/DataLake/tests/rest_catalog_test_server.h @@ -0,0 +1,206 @@ +#pragma once + +#include "config.h" + +#if USE_AVRO + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace RestCatalogTest +{ + +struct RecordedRequest +{ + std::string method; + std::string path; + std::string query; + std::string body; + std::map headers; + + std::string header(const std::string & name) const + { + auto it = headers.find(name); + return it == headers.end() ? std::string{} : it->second; + } +}; + +struct Response +{ + int status = 200; + std::string body; + std::string content_type = "application/json"; +}; + +inline Response json(const std::string & body) +{ + return Response{.status = 200, .body = body, .content_type = "application/json"}; +} + +inline Response respondWithStatus(int status, const std::string & body = R"({"error":{"message":"denied"}})") +{ + return Response{.status = status, .body = body, .content_type = "application/json"}; +} + +class ServerState +{ +public: + using Route = std::function; + + void setRoute(const std::string & path, Route route) + { + std::lock_guard lock(mutex); + routes[path] = std::move(route); + } + + void setStaticRoute(const std::string & path, const std::string & body) + { + setRoute(path, [body](const RecordedRequest &) { return json(body); }); + } + + std::vector requestsTo(const std::string & path) const + { + std::lock_guard lock(mutex); + std::vector result; + for (const auto & request : recorded) + if (request.path == path) + result.push_back(request); + return result; + } + + size_t countRequestsTo(const std::string & path) const { return requestsTo(path).size(); } + + void clearRequests() + { + std::lock_guard lock(mutex); + recorded.clear(); + } + + Response handle(RecordedRequest request) + { + Route route; + { + std::lock_guard lock(mutex); + recorded.push_back(request); + if (auto it = routes.find(request.path); it != routes.end()) + route = it->second; + } + + /// Report unexpected requests to the client. + if (!route) + return Response{ + .status = 599, + .body = "unexpected request to fake Iceberg REST catalog: " + request.method + " " + request.path, + .content_type = "text/plain"}; + + return route(request); + } + +private: + mutable std::mutex mutex; + std::map routes; + std::vector recorded; +}; + +class RequestHandler final : public Poco::Net::HTTPRequestHandler +{ +public: + explicit RequestHandler(std::shared_ptr state_) : state(std::move(state_)) {} + + void handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) override + { + const std::string & raw_uri = request.getURI(); + const auto query_pos = raw_uri.find('?'); + + RecordedRequest recorded; + recorded.method = request.getMethod(); + /// Keep `%1F` in nested namespace paths encoded so it matches the route keys. + recorded.path = query_pos == std::string::npos ? raw_uri : raw_uri.substr(0, query_pos); + recorded.query = query_pos == std::string::npos ? std::string{} : raw_uri.substr(query_pos + 1); + Poco::StreamCopier::copyToString(request.stream(), recorded.body); + for (const auto & [name, value] : request) + recorded.headers[name] = value; + + const auto result = state->handle(std::move(recorded)); + + response.setStatus(static_cast(result.status)); + response.setContentType(result.content_type); + response.setContentLength(result.body.size()); + response.send() << result.body; + } + +private: + std::shared_ptr state; +}; + +class RequestHandlerFactory final : public Poco::Net::HTTPRequestHandlerFactory +{ +public: + explicit RequestHandlerFactory(std::shared_ptr state_) : state(std::move(state_)) {} + + Poco::Net::HTTPRequestHandler * createRequestHandler(const Poco::Net::HTTPServerRequest &) override + { + return new RequestHandler(state); + } + +private: + std::shared_ptr state; +}; + +class TestServer +{ +public: + TestServer() + : state(std::make_shared()) + , server_socket(std::make_unique(Poco::Net::SocketAddress("127.0.0.1", 0))) + , handler_factory(new RequestHandlerFactory(state)) + , server_params(new Poco::Net::HTTPServerParams()) + , server(std::make_unique(handler_factory, *server_socket, server_params)) + { + /// Ephemeral ports can be reused; discard pooled sockets belonging to previous test servers. + DB::HTTPConnectionPools::instance().dropCache(); + + state->setStaticRoute("/v1/config", R"({"defaults":{},"overrides":{}})"); + server->start(); + } + + ~TestServer() + { + server->stop(); + DB::HTTPConnectionPools::instance().dropCache(); + } + + std::string getUrl() const { return "http://" + server_socket->address().toString(); } + + ServerState & operator*() const { return *state; } + ServerState * operator->() const { return state.get(); } + +private: + std::shared_ptr state; + std::unique_ptr server_socket; + Poco::SharedPtr handler_factory; + Poco::AutoPtr server_params; + std::unique_ptr server; +}; + +} + +#endif diff --git a/src/Databases/IDatabase.h b/src/Databases/IDatabase.h index b92e15ac35d3..e8f27e279307 100644 --- a/src/Databases/IDatabase.h +++ b/src/Databases/IDatabase.h @@ -408,7 +408,7 @@ class IDatabase : public std::enable_shared_from_this return database_name; } - virtual void checkDatabase() const + virtual void checkDatabase(ContextPtr /*context*/) const { //No-op } diff --git a/src/IO/S3/Credentials.cpp b/src/IO/S3/Credentials.cpp index c0c77fc407ca..0f46a7c09edb 100644 --- a/src/IO/S3/Credentials.cpp +++ b/src/IO/S3/Credentials.cpp @@ -57,11 +57,13 @@ namespace S3 # include # include +# include # include # include # include # include +# include # include # include @@ -1148,7 +1150,30 @@ void AssumeRoleRequest::AddQueryStringParameters(Aws::Http::URI & uri) const uri.AddQueryStringParameter("ExternalId", external_id); } -AssumeRoleResult::AssumeRoleResult(Aws::AmazonWebServiceResult result) +AssumeRoleWithWebIdentityRequest::AssumeRoleWithWebIdentityRequest( + std::string role_arn_, std::string role_session_name_, std::string web_identity_token_) + : role_arn(std::move(role_arn_)) + , role_session_name(std::move(role_session_name_)) + , web_identity_token(std::move(web_identity_token_)) +{ +} + +Aws::Http::HeaderValueCollection AssumeRoleWithWebIdentityRequest::GetHeaders() const +{ + return {{Aws::Http::HeaderValuePair(Aws::Http::CONTENT_TYPE_HEADER, Aws::FORM_CONTENT_TYPE)}}; +} + +Aws::String AssumeRoleWithWebIdentityRequest::SerializePayload() const +{ + return fmt::format( + "Action=AssumeRoleWithWebIdentity&Version=2011-06-15&RoleArn={}&RoleSessionName={}&WebIdentityToken={}", + DB::formUrlEncode(role_arn), + DB::formUrlEncode(role_session_name), + DB::formUrlEncode(web_identity_token)); +} + +AssumeRoleResult::AssumeRoleResult( + Aws::AmazonWebServiceResult result, const char * result_node_name) { using namespace Aws::Utils::Xml; const auto & xml_document = result.GetPayload(); @@ -1160,10 +1185,10 @@ AssumeRoleResult::AssumeRoleResult(Aws::AmazonWebServiceResult client_) + : role_arn(std::move(role_arn_)) + , session_name(std::move(session_name_)) + , web_identity_token(std::move(web_identity_token_)) + , expiration_window_seconds(expiration_window_seconds_) + , client(std::move(client_)) + , logger(getLogger("AwsAuthSTSAssumeRoleWithWebIdentityCredentialsProvider")) +{ +} + +Aws::Auth::AWSCredentials AwsAuthSTSAssumeRoleWithWebIdentityCredentialsProvider::GetAWSCredentials() +{ + Aws::Utils::Threading::ReaderLockGuard guard(m_reloadLock); + if (!IsSetNeedRefresh() && !areCredentialsEmptyOrExpired(credentials, expiration_window_seconds)) + return credentials; + + guard.UpgradeToWriterLock(); + if (!IsSetNeedRefresh() && !areCredentialsEmptyOrExpired(credentials, expiration_window_seconds)) // double-checked lock to avoid refreshing twice + return credentials; + + Reload(); + return credentials; +} + +void AwsAuthSTSAssumeRoleWithWebIdentityCredentialsProvider::Reload() +{ + LOG_INFO(logger, "Credentials are empty or expired, attempting to renew with AssumeRoleWithWebIdentity for role {}", role_arn); + + AssumeRoleWithWebIdentityRequest request(role_arn, session_name, web_identity_token); + auto outcome = client->assumeRoleWithWebIdentity(request); + if (!outcome.IsSuccess()) + { + credentials = Aws::Auth::AWSCredentials{}; + last_error = outcome.GetError().GetMessage(); + LOG_WARNING(logger, "Failed to get credentials using AssumeRoleWithWebIdentity. Error: {}", last_error); + return; + } + + last_error.clear(); + const auto & result = outcome.GetResult(); + credentials.SetAWSAccessKeyId(result.getAccessKeyID()); + credentials.SetAWSSecretKey(result.getSecretAccessKey()); + credentials.SetSessionToken(result.getSessionToken()); + credentials.SetExpiration(result.getExpiration()); + + AWSCredentialsProvider::Reload(); + + LOG_TRACE(logger, "Successfully retrieved credentials for role {}", role_arn); +} + +std::string AwsAuthSTSAssumeRoleWithWebIdentityCredentialsProvider::getLastError() const +{ + Aws::Utils::Threading::ReaderLockGuard guard(m_reloadLock); + return last_error; +} + std::shared_ptr getCredentialsProvider( const DB::S3::PocoHTTPClientConfiguration & configuration, const Aws::Auth::AWSCredentials & credentials, diff --git a/src/IO/S3/Credentials.h b/src/IO/S3/Credentials.h index c6e3bdfc2302..1c5aa32585e7 100644 --- a/src/IO/S3/Credentials.h +++ b/src/IO/S3/Credentials.h @@ -256,11 +256,32 @@ class AssumeRoleRequest : public Aws::AmazonSerializableWebServiceRequest std::string role_session_name; std::string external_id; }; +class AssumeRoleWithWebIdentityRequest : public Aws::AmazonSerializableWebServiceRequest +{ +public: + AssumeRoleWithWebIdentityRequest(std::string role_arn_, std::string role_session_name_, std::string web_identity_token_); + + Aws::Http::HeaderValueCollection GetHeaders() const override; + + const char * GetServiceRequestName() const override { return "AssumeRoleWithWebIdentity"; } + + Aws::String SerializePayload() const override; + +private: + std::string role_arn; + std::string role_session_name; + std::string web_identity_token; +}; + class AssumeRoleResult { public: + AssumeRoleResult() = default; + /// NOLINTNEXTLINE - AssumeRoleResult(Aws::AmazonWebServiceResult result); + AssumeRoleResult( + Aws::AmazonWebServiceResult result, + const char * result_node_name = "AssumeRoleResult"); const std::string & getAccessKeyID() const { return access_key_id; } @@ -291,10 +312,13 @@ class AWSAssumeRoleClient : public Aws::Client::AWSXMLClient AssumeRoleOutcome assumeRole(const AssumeRoleRequest & request) const; + AssumeRoleOutcome assumeRoleWithWebIdentity(const AssumeRoleWithWebIdentityRequest & request) const; + const auto & getEndpoint() const { return endpoint; } private: Aws::Endpoint::AWSEndpoint endpoint; + Aws::Endpoint::AWSEndpoint web_identity_endpoint; }; class AwsAuthSTSAssumeRoleCredentialsProvider : public Aws::Auth::AWSCredentialsProvider @@ -343,6 +367,34 @@ class AwsAuthSTSAssumeRoleCredentialsProvider : public Aws::Auth::AWSCredentials LoggerPtr logger; }; +class AwsAuthSTSAssumeRoleWithWebIdentityCredentialsProvider : public Aws::Auth::AWSCredentialsProvider +{ +public: + AwsAuthSTSAssumeRoleWithWebIdentityCredentialsProvider( + std::string role_arn_, + std::string session_name_, + std::string web_identity_token_, + uint64_t expiration_window_seconds_, + std::shared_ptr client_); + + Aws::Auth::AWSCredentials GetAWSCredentials() override; + + std::string getLastError() const; + +protected: + void Reload() override; + +private: + std::string role_arn; + std::string session_name; + std::string web_identity_token; + uint64_t expiration_window_seconds; + std::shared_ptr client; + Aws::Auth::AWSCredentials credentials; + std::string last_error; + LoggerPtr logger; +}; + std::shared_ptr getCredentialsProvider( const DB::S3::PocoHTTPClientConfiguration & configuration, const Aws::Auth::AWSCredentials & credentials, diff --git a/src/IO/S3/tests/TestPocoHTTPServer.h b/src/IO/S3/tests/TestPocoHTTPServer.h index 62a51e063ca5..02682fa04445 100644 --- a/src/IO/S3/tests/TestPocoHTTPServer.h +++ b/src/IO/S3/tests/TestPocoHTTPServer.h @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -109,12 +110,14 @@ struct StsRequestInfo { Poco::Net::MessageHeader headers; Poco::URI::QueryParameters query_params; + std::string body; }; class MockStsRequestHandler : public Poco::Net::HTTPRequestHandler { public: - explicit MockStsRequestHandler(std::optional & last_request_info_, std::string role_access_key_, std::string role_secret_key_) + explicit MockStsRequestHandler( + std::optional & last_request_info_, std::string role_access_key_, std::string role_secret_key_) : last_request_info(last_request_info_) , role_access_key(std::move(role_access_key_)) , role_secret_key(std::move(role_secret_key_)) @@ -128,20 +131,24 @@ class MockStsRequestHandler : public Poco::Net::HTTPRequestHandler Poco::URI uri(request.getURI()); last_request_info->query_params = uri.getQueryParameters(); + Poco::StreamCopier::copyToString(request.stream(), last_request_info->body); + + const bool web_identity = last_request_info->body.find("Action=AssumeRoleWithWebIdentity") != std::string::npos; + const std::string_view action = web_identity ? "AssumeRoleWithWebIdentity" : "AssumeRole"; response.setStatus(Poco::Net::HTTPResponse::HTTP_OK); auto & out = response.send(); std::string result_xml = fmt::format(R"( - - +<{0}Response xmlns="https://sts.amazonaws.com/doc/2011-06-15/"> +<{0}Result> - {} - {} + {1} + {2} session_token - -)", role_access_key, role_secret_key); + +)", action, role_access_key, role_secret_key); out << result_xml; out.flush(); } @@ -162,7 +169,8 @@ class StsHTTPRequestHandlerFactory : public Poco::Net::HTTPRequestHandlerFactory return new MockStsRequestHandler(last_request_info, role_access_key, role_secret_key); } public: - explicit StsHTTPRequestHandlerFactory(std::optional & last_request_info_, std::string role_access_key_, std::string role_secret_key_) + explicit StsHTTPRequestHandlerFactory( + std::optional & last_request_info_, std::string role_access_key_, std::string role_secret_key_) : last_request_info(last_request_info_) , role_access_key(std::move(role_access_key_)) , role_secret_key(std::move(role_secret_key_)) @@ -230,4 +238,9 @@ class TestPocoHTTPStsServer { return last_request_info->query_params; } + + const std::string & getLastBody() const + { + return last_request_info->body; + } }; diff --git a/src/IO/S3/tests/gtest_sts_assume_role_with_web_identity.cpp b/src/IO/S3/tests/gtest_sts_assume_role_with_web_identity.cpp new file mode 100644 index 000000000000..2d2fe1c0e116 --- /dev/null +++ b/src/IO/S3/tests/gtest_sts_assume_role_with_web_identity.cpp @@ -0,0 +1,91 @@ +#include + +#include "config.h" + +#if USE_AWS_S3 + +#include +#include +#include +#include + +#include + +#include + +#include +#include + +namespace +{ + +constexpr std::string_view role_access_key = "role_access_key"; +constexpr std::string_view role_secret_key = "role_secret_key"; + +DB::S3::PocoHTTPClientConfiguration makeClientConfiguration(DB::RemoteHostFilter & remote_host_filter) +{ + return DB::S3::ClientFactory::instance().createClientConfiguration( + "eu-west-1", + remote_host_filter, + /* s3_max_redirects = */ 100, + DB::S3::PocoHTTPClientConfiguration::RetryStrategy{.max_retries = 0}, + /* s3_slow_all_threads_after_network_error = */ false, + /* s3_slow_all_threads_after_retryable_error = */ false, + /* enable_s3_requests_logging = */ false, + /* for_disk_s3 = */ false, + /* opt_disk_name = */ {}, + /* request_throttler = */ {}, + "http"); +} + +std::string formParameter(const std::string & body, const std::string & name) +{ + Poco::URI uri; + uri.setRawQuery(body); + for (const auto & [key, value] : uri.getQueryParameters()) + if (key == name) + return value; + return {}; +} + +} + +TEST(STSAssumeRoleWithWebIdentity, SendsTokenInTheBody) +{ + TestPocoHTTPStsServer sts_http(std::string{role_access_key}, std::string{role_secret_key}); + + DB::RemoteHostFilter remote_host_filter; + auto client_configuration = makeClientConfiguration(remote_host_filter); + + auto client = std::make_shared( + std::make_shared(), client_configuration, sts_http.getUrl()); + + const std::string token = "header.payload.signature"; + DB::S3::AwsAuthSTSAssumeRoleWithWebIdentityCredentialsProvider provider( + "arn:aws:iam::123456789012:role/data-lake-reader", "alice", token, /* expiration_window_seconds = */ 0, client); + + auto credentials = provider.GetAWSCredentials(); + + ASSERT_TRUE(sts_http.hasLastRequest()); + + const auto & body = sts_http.getLastBody(); + EXPECT_EQ(formParameter(body, "Action"), "AssumeRoleWithWebIdentity"); + EXPECT_EQ(formParameter(body, "Version"), "2011-06-15"); + EXPECT_EQ(formParameter(body, "RoleArn"), "arn:aws:iam::123456789012:role/data-lake-reader"); + EXPECT_EQ(formParameter(body, "RoleSessionName"), "alice"); + EXPECT_EQ(formParameter(body, "WebIdentityToken"), token); + + for (const auto & [key, value] : sts_http.getLastQueryParams()) + { + EXPECT_NE(key, "WebIdentityToken"); + EXPECT_EQ(value.find(token), std::string::npos); + } + + EXPECT_FALSE(sts_http.getLastRequestHeader().has("Authorization")); + + EXPECT_EQ(credentials.GetAWSAccessKeyId(), role_access_key); + EXPECT_EQ(credentials.GetAWSSecretKey(), role_secret_key); + EXPECT_EQ(credentials.GetSessionToken(), "session_token"); +} + +#endif diff --git a/src/Interpreters/Access/InterpreterExecuteAsQuery.cpp b/src/Interpreters/Access/InterpreterExecuteAsQuery.cpp index cc095e3e0b3e..932bc5a23a23 100644 --- a/src/Interpreters/Access/InterpreterExecuteAsQuery.cpp +++ b/src/Interpreters/Access/InterpreterExecuteAsQuery.cpp @@ -64,6 +64,9 @@ namespace context->setUser(context->getAccessControl().getID(target_user_name)); + /// `EXECUTE AS` changes the session identity; the original bearer token must not survive it. + context->setForwardedAuthToken(nullptr); + /// We need to update the client info to make currentUser() return `target_user_name`. context->setCurrentUserName(target_user_name); context->setInitialUserName(target_user_name); diff --git a/src/Interpreters/AsynchronousInsertQueue.cpp b/src/Interpreters/AsynchronousInsertQueue.cpp index 068842f3c5f5..a0afe2bc51de 100644 --- a/src/Interpreters/AsynchronousInsertQueue.cpp +++ b/src/Interpreters/AsynchronousInsertQueue.cpp @@ -126,6 +126,7 @@ AsynchronousInsertQueue::InsertQuery::InsertQuery( const String & current_user_, const String & initial_user_, const String & authenticated_user_, + const ForwardedAuthTokenPtr & forwarded_auth_token_, const Settings & settings_, AsynchronousInsertQueueDataKind data_kind_) : query(query_->clone()) @@ -135,8 +136,10 @@ AsynchronousInsertQueue::InsertQuery::InsertQuery( , current_user(current_user_) , initial_user(initial_user_) , authenticated_user(authenticated_user_) + , forwarded_auth_token(forwarded_auth_token_) , settings(std::make_unique(settings_)) , data_kind(data_kind_) + , forwarded_auth_token_fingerprint(forwarded_auth_token ? forwarded_auth_token->fingerprint : String{}) { SipHash siphash; @@ -160,6 +163,9 @@ AsynchronousInsertQueue::InsertQuery::InsertQuery( siphash.update(identity_field); } + siphash.update(forwarded_auth_token_fingerprint.size()); + siphash.update(forwarded_auth_token_fingerprint); + setting_changes = settings->changes(); for (auto it = setting_changes.begin(); it != setting_changes.end();) { @@ -187,6 +193,8 @@ AsynchronousInsertQueue::InsertQuery::InsertQuery(const InsertQuery & other) current_user = other.current_user; initial_user = other.initial_user; authenticated_user = other.authenticated_user; + forwarded_auth_token = other.forwarded_auth_token; + forwarded_auth_token_fingerprint = other.forwarded_auth_token_fingerprint; settings = std::make_unique(*other.settings); data_kind = other.data_kind; hash = other.hash; @@ -205,6 +213,8 @@ AsynchronousInsertQueue::InsertQuery::operator=(const InsertQuery & other) current_user = other.current_user; initial_user = other.initial_user; authenticated_user = other.authenticated_user; + forwarded_auth_token = other.forwarded_auth_token; + forwarded_auth_token_fingerprint = other.forwarded_auth_token_fingerprint; settings = std::make_unique(*other.settings); data_kind = other.data_kind; hash = other.hash; @@ -560,6 +570,7 @@ AsynchronousInsertQueue::PushResult AsynchronousInsertQueue::pushDataChunk(ASTPt client_info.current_user, client_info.initial_user, client_info.authenticated_user, + query_context->getForwardedAuthToken(), settings, data_kind}; InsertDataPtr data_to_process; @@ -1015,6 +1026,7 @@ try insert_context->setCurrentUserName(key.current_user); insert_context->setInitialUserName(key.initial_user); insert_context->setAuthenticatedUserName(key.authenticated_user); + insert_context->setForwardedAuthToken(key.forwarded_auth_token); insert_context->setSettings(*key.settings); diff --git a/src/Interpreters/AsynchronousInsertQueue.h b/src/Interpreters/AsynchronousInsertQueue.h index 5007f121d431..f5c0b768ea5e 100644 --- a/src/Interpreters/AsynchronousInsertQueue.h +++ b/src/Interpreters/AsynchronousInsertQueue.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -95,6 +96,7 @@ class AsynchronousInsertQueue : public WithContext String current_user; String initial_user; String authenticated_user; + ForwardedAuthTokenPtr forwarded_auth_token; std::unique_ptr settings; AsynchronousInsertQueueDataKind data_kind; @@ -107,6 +109,7 @@ class AsynchronousInsertQueue : public WithContext const String & current_user_, const String & initial_user_, const String & authenticated_user_, + const ForwardedAuthTokenPtr & forwarded_auth_token_, const Settings & settings_, AsynchronousInsertQueueDataKind data_kind_); @@ -116,7 +119,14 @@ class AsynchronousInsertQueue : public WithContext StorageID getStorageID() const; private: - auto toTupleCmp() const { return std::tie(data_kind, query_str, user_id, current_roles, current_user, initial_user, authenticated_user, setting_changes); } + auto toTupleCmp() const + { + return std::tie( + data_kind, query_str, user_id, current_roles, current_user, initial_user, + authenticated_user, forwarded_auth_token_fingerprint, setting_changes); + } + + String forwarded_auth_token_fingerprint; std::vector setting_changes; }; diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 3d27e1f4a075..e3fcfc3c5fb3 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -1280,6 +1280,7 @@ ContextData::ContextData() ContextData::ContextData(const ContextData &o) : shared(o.shared), client_info(o.client_info), + forwarded_auth_token(o.forwarded_auth_token), external_tables_initializer_callback(o.external_tables_initializer_callback), input_initializer_callback(o.input_initializer_callback), input_blocks_reader(o.input_blocks_reader), @@ -7472,6 +7473,11 @@ void Context::setClientInfo(const ClientInfo & client_info_) need_recalculate_access = true; } +void Context::setForwardedAuthToken(ForwardedAuthTokenPtr token) +{ + forwarded_auth_token = std::move(token); +} + void Context::setClientName(const String & client_name) { client_info.client_name = client_name; diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 5a9943af651d..03c06f5e5f7d 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -361,6 +362,7 @@ class ContextData ContextSharedPart * shared{}; ClientInfo client_info; + ForwardedAuthTokenPtr forwarded_auth_token; ExternalTablesInitializer external_tables_initializer_callback; QueryPlanDeserializationCallback query_plan_deserialization_callback; @@ -968,6 +970,9 @@ class Context: public ContextData, public std::enable_shared_from_this /// Modify stored in the context information about the client executing a query. void setClientInfo(const ClientInfo & client_info_); + + const ForwardedAuthTokenPtr & getForwardedAuthToken() const { return forwarded_auth_token; } + void setForwardedAuthToken(ForwardedAuthTokenPtr token); void setClientName(const String & client_name); void setClientInterface(ClientInfo::Interface interface); void setClientVersion(UInt64 client_version_major, UInt64 client_version_minor, UInt64 client_version_patch, unsigned client_tcp_protocol_version); diff --git a/src/Interpreters/InterpreterCheckQuery.cpp b/src/Interpreters/InterpreterCheckQuery.cpp index 23ceb6f88bde..488db18d3b68 100644 --- a/src/Interpreters/InterpreterCheckQuery.cpp +++ b/src/Interpreters/InterpreterCheckQuery.cpp @@ -442,7 +442,7 @@ BlockIO InterpreterCheckQuery::execute() LOG_DEBUG(log, "Checking database name = {} ", database_name); context->checkAccess(AccessType::CHECK, database_name); auto database = DatabaseCatalog::instance().getDatabase(database_name); - database->checkDatabase(); + database->checkDatabase(context); BlockIO res; return res; } diff --git a/src/Interpreters/Session.cpp b/src/Interpreters/Session.cpp index 1d39c81d7a76..2af541b2e276 100644 --- a/src/Interpreters/Session.cpp +++ b/src/Interpreters/Session.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -409,6 +410,12 @@ void Session::authenticate(const Credentials & credentials_, const Poco::Net::So prepared_client_info->authenticated_user = auth_result.user_name; prepared_client_info->current_address = std::make_shared(address); prepared_client_info->connection_address = std::make_shared(connection_address ? *connection_address : address); + + if (const auto * token_credentials = typeid_cast(&credentials_)) + { + if (global_context->getAccessControl().isTokenForwardingEnabled()) + forwarded_auth_token = makeForwardedAuthToken(*token_credentials, auth_result.user_name); + } } void Session::checkIfUserIsStillValid() @@ -580,6 +587,7 @@ ContextMutablePtr Session::makeSessionContext() /// Copy prepared client info to the new session context. new_session_context->setClientInfo(*prepared_client_info); + new_session_context->setForwardedAuthToken(forwarded_auth_token); prepared_client_info.reset(); /// Set user information for the new context: current profiles, roles, access rights. @@ -651,6 +659,11 @@ ContextMutablePtr Session::makeSessionContext(const String & session_name_, std: max_sessions_for_user = max_session_for_user_field->safeGet(); } + /// Refresh reused named sessions only while they still represent the authenticated user. + /// An `EXECUTE AS` session must not regain the original token. + const bool runs_as_authenticated_user = new_session_context->getAccess()->getUserID() == user_id; + new_session_context->setForwardedAuthToken(runs_as_authenticated_user ? forwarded_auth_token : nullptr); + /// Session context is ready. session_context = std::move(new_session_context); named_session = new_named_session; @@ -711,6 +724,10 @@ ContextMutablePtr Session::makeQueryContextImpl(const ClientInfo * client_info_t else if (client_info_to_copy && (client_info_to_copy != &getClientInfo())) query_context->setClientInfo(*client_info_to_copy); + /// A session context may have cleared its token for `EXECUTE AS`; do not restore it in the query copy. + if (!from_session_context) + query_context->setForwardedAuthToken(forwarded_auth_token); + /// Copy current user's name and address if it was authenticated after query_client_info was initialized. if (prepared_client_info && !prepared_client_info->current_user.empty()) { diff --git a/src/Interpreters/Session.h b/src/Interpreters/Session.h index 4ae42dc9c993..e0a1f7b5d6a0 100644 --- a/src/Interpreters/Session.h +++ b/src/Interpreters/Session.h @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -122,6 +123,8 @@ class Session /// ClientInfo that will be copied to a session context when it's created. std::optional prepared_client_info; + ForwardedAuthTokenPtr forwarded_auth_token; + mutable UserPtr user; std::optional user_id; std::vector external_roles; diff --git a/src/Interpreters/tests/gtest_async_insert_key.cpp b/src/Interpreters/tests/gtest_async_insert_key.cpp index 327525b4db5c..4822956750ae 100644 --- a/src/Interpreters/tests/gtest_async_insert_key.cpp +++ b/src/Interpreters/tests/gtest_async_insert_key.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -37,10 +38,10 @@ TEST(AsyncInsertKey, SettingsChanges) auto kind = AsynchronousInsertQueueDataKind::Parsed; - AsynchronousInsertQueue::InsertQuery key1(query, {}, {}, {}, {}, {}, settings1, kind); - AsynchronousInsertQueue::InsertQuery key2(query, {}, {}, {}, {}, {}, settings2, kind); - AsynchronousInsertQueue::InsertQuery key3(query, {}, {}, {}, {}, {}, settings3, kind); - AsynchronousInsertQueue::InsertQuery key4(query, {}, {}, {}, {}, {}, settings4, kind); + AsynchronousInsertQueue::InsertQuery key1(query, {}, {}, {}, {}, {}, {}, settings1, kind); + AsynchronousInsertQueue::InsertQuery key2(query, {}, {}, {}, {}, {}, {}, settings2, kind); + AsynchronousInsertQueue::InsertQuery key3(query, {}, {}, {}, {}, {}, {}, settings3, kind); + AsynchronousInsertQueue::InsertQuery key4(query, {}, {}, {}, {}, {}, {}, settings4, kind); EXPECT_EQ(key1, key2); EXPECT_NE(key1, key3); @@ -62,7 +63,7 @@ TEST(AsyncInsertKey, IdentityHashIsNotAmbiguous) auto make_key = [&](const String & current_user, const String & initial_user, const String & authenticated_user) { return AsynchronousInsertQueue::InsertQuery( - query, {}, {}, current_user, initial_user, authenticated_user, settings, kind); + query, {}, {}, current_user, initial_user, authenticated_user, {}, settings, kind); }; /// The three identity fields are variable-length strings folded into the queue key hash, @@ -93,3 +94,32 @@ TEST(AsyncInsertKey, IdentityHashIsNotAmbiguous) EXPECT_NE(key_g.hash, key_h.hash); EXPECT_NE(key_g, key_h); } + +TEST(AsyncInsertKey, ForwardedTokenPartitionsBatches) +{ + String query_str = "INSERT INTO test (a) VALUES (1)"; + ParserInsertQuery parser(query_str.data() + query_str.size(), false); + ASTPtr query = parseQuery(parser, query_str, DBMS_DEFAULT_MAX_QUERY_SIZE, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS); + Settings settings; + + auto make_key = [&](const ForwardedAuthTokenPtr & token) + { + return AsynchronousInsertQueue::InsertQuery( + query, {}, {}, "alice", "alice", "alice", token, settings, AsynchronousInsertQueueDataKind::Parsed); + }; + auto token = makeForwardedAuthToken(TokenCredentials("alice-token"), "alice"); + auto same_token = makeForwardedAuthToken(TokenCredentials("alice-token"), "alice"); + auto rotated_token = makeForwardedAuthToken(TokenCredentials("alice-rotated-token"), "alice"); + + auto original = make_key(token); + auto same = make_key(same_token); + auto rotated = make_key(rotated_token); + auto no_token = make_key({}); + + EXPECT_EQ(original, same); + EXPECT_EQ(original.hash, same.hash); + EXPECT_NE(original, rotated); + EXPECT_NE(original.hash, rotated.hash); + EXPECT_NE(original, no_token); + EXPECT_NE(original.hash, no_token.hash); +} diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp index 5855504b22ac..64e743a74273 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp @@ -918,7 +918,7 @@ static bool writeConsolidatedManifestFile( { auto catalog_filename = path_resolver.resolveForCatalog(generated_metadata_info.path); const auto & [namespace_name, table_name] = DataLake::parseTableName(table_id.getTableName()); - if (!catalog->updateMetadata(namespace_name, table_name, catalog_filename, new_snapshot.snapshot)) + if (!catalog->updateMetadata(namespace_name, table_name, catalog_filename, new_snapshot.snapshot, context->getForwardedAuthToken())) { LOG_INFO(log, "Metadata commit conflict detected via catalog, cleaning up temporary files"); cleanup(); diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ExpireSnapshotsExecute.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/ExpireSnapshotsExecute.cpp index 053d86f7a4c9..fcd3594d55ce 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ExpireSnapshotsExecute.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ExpireSnapshotsExecute.cpp @@ -849,7 +849,7 @@ ExpireSnapshotsResult expireSnapshots( { auto catalog_filename = persistent_table_components.path_resolver.resolveForCatalog(metadata_info.path); const auto & [namespace_name, parsed_table_name] = DataLake::parseTableName(table_name); - if (!catalog->updateMetadata(namespace_name, parsed_table_name, catalog_filename, nullptr)) + if (!catalog->updateMetadata(namespace_name, parsed_table_name, catalog_filename, nullptr, context->getForwardedAuthToken())) { throw Exception( ErrorCodes::LOGICAL_ERROR, diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp index 7f7d7211c680..21a92089e7b8 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp @@ -801,7 +801,7 @@ void IcebergMetadata::truncate(ContextPtr context, std::shared_ptrupdateMetadata(namespace_name, table_name, catalog_filename, new_snapshot)) + if (!catalog->updateMetadata(namespace_name, table_name, catalog_filename, new_snapshot, context->getForwardedAuthToken())) throw Exception(ErrorCodes::INCORRECT_DATA, "Failed to commit Iceberg truncate update to catalog."); } @@ -959,7 +959,8 @@ void IcebergMetadata::createInitial( /// validation, so a rejected CREATE leaves no trace in the catalog): a catalog /// that shares its storage view with the data (e.g. SeaweedFS) refuses to create /// a namespace over the plain directory those files would leave behind. - catalog->createNamespaceIfNotExists(DataLake::parseTableName(table_id_.getTableName()).first, location_path); + catalog->createNamespaceIfNotExists( + DataLake::parseTableName(table_id_.getTableName()).first, location_path, local_context->getForwardedAuthToken()); } try @@ -988,7 +989,7 @@ void IcebergMetadata::createInitial( auto catalog_filename = configuration_ptr->getTypeName() + "://" + configuration_ptr->getNamespace() + "/" + configuration_ptr->getRawPath().path + "metadata/v1.metadata.json"; const auto & [namespace_name, table_name] = DataLake::parseTableName(table_id_.getTableName()); - catalog->createTable(namespace_name, table_name, catalog_filename, metadata_content_object); + catalog->createTable(namespace_name, table_name, catalog_filename, metadata_content_object, local_context->getForwardedAuthToken()); } } @@ -2044,7 +2045,7 @@ std::optional IcebergMetadata::commitImport catalog_filename = blob_storage_type_name + "://" + blob_storage_namespace_name + "/" + catalog_filename; const auto & [namespace_name, table_name] = DataLake::parseTableName(table_id.getTableName()); - if (!catalog->updateMetadata(namespace_name, table_name, catalog_filename, new_snapshot)) + if (!catalog->updateMetadata(namespace_name, table_name, catalog_filename, new_snapshot, context->getForwardedAuthToken())) { cleanup(true); return {}; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp index 0adbf4f2c0e5..f0aa0304fb9f 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp @@ -1766,7 +1766,7 @@ bool IcebergStorageSink::initializeMetadata() auto catalog_filename = resolver.resolveForCatalog(metadata_info.path); const auto & [namespace_name, table_name] = DataLake::parseTableName(table_id.getTableName()); - if (!catalog->updateMetadata(namespace_name, table_name, catalog_filename, new_snapshot)) + if (!catalog->updateMetadata(namespace_name, table_name, catalog_filename, new_snapshot, context->getForwardedAuthToken())) { cleanup(true); return false; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp index 2933f50f9296..b30f4a7a3457 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp @@ -615,7 +615,7 @@ static bool writeMetadataFiles( { auto catalog_filename = path_resolver.resolveForCatalog(metadata_info.path); const auto & [namespace_name, table_name] = DataLake::parseTableName(table_id.getTableName()); - if (!catalog->updateMetadata(namespace_name, table_name, catalog_filename, new_snapshot)) + if (!catalog->updateMetadata(namespace_name, table_name, catalog_filename, new_snapshot, context->getForwardedAuthToken())) { cleanup(); return false; @@ -972,7 +972,7 @@ void alter( metadata->getValue(Iceberg::f_last_column_id), getHighestFieldId(new_schema)); commit_attempted = true; - if (!catalog->updateSchema(namespace_name, table_name, catalog_filename, new_schema, previous_schema_id, new_last_column_id, metadata)) + if (!catalog->updateSchema(namespace_name, table_name, catalog_filename, new_schema, previous_schema_id, new_last_column_id, metadata, context->getForwardedAuthToken())) { ++i; continue; diff --git a/src/Storages/ObjectStorage/StorageObjectStorage.cpp b/src/Storages/ObjectStorage/StorageObjectStorage.cpp index e7913b4ec0fc..a74b0a392b34 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorage.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorage.cpp @@ -159,6 +159,7 @@ StorageObjectStorage::StorageObjectStorage( , is_table_function(is_table_function_) , log(getLogger(fmt::format("Storage{}({})", configuration->getEngineName(), table_id_.getFullTableName()))) , catalog(catalog_) + , catalog_auth_token(context ? context->getForwardedAuthToken() : DB::ForwardedAuthTokenPtr{}) , storage_id(table_id_) , background_operations_assignee(*this, table_id_, BackgroundJobsAssignee::Type::DataProcessing, Context::getGlobalContextInstance()) { @@ -1018,7 +1019,7 @@ void StorageObjectStorage::drop() if (catalog) { const auto [namespace_name, table_name] = DataLake::parseTableName(storage_id.getTableName()); - catalog->dropTable(namespace_name, table_name); + catalog->dropTable(namespace_name, table_name, catalog_auth_token); } /// We cannot use query context here, because drop is executed in the background. configuration->drop(Context::getGlobalContextInstance()); diff --git a/src/Storages/ObjectStorage/StorageObjectStorage.h b/src/Storages/ObjectStorage/StorageObjectStorage.h index 474e337e17ae..7497c0caa310 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorage.h +++ b/src/Storages/ObjectStorage/StorageObjectStorage.h @@ -1,4 +1,5 @@ #pragma once +#include #include #include #include @@ -270,6 +271,7 @@ class StorageObjectStorage : public IStorage, public IBackgroundOperation LoggerPtr log; std::shared_ptr catalog; + DB::ForwardedAuthTokenPtr catalog_auth_token; StorageID storage_id; BackgroundJobsAssignee background_operations_assignee; }; diff --git a/tests/integration/test_datalake_glue_token_forwarding/__init__.py b/tests/integration/test_datalake_glue_token_forwarding/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/integration/test_datalake_glue_token_forwarding/configs/token_forwarding.xml b/tests/integration/test_datalake_glue_token_forwarding/configs/token_forwarding.xml new file mode 100644 index 000000000000..ee19d7e1c33c --- /dev/null +++ b/tests/integration/test_datalake_glue_token_forwarding/configs/token_forwarding.xml @@ -0,0 +1,22 @@ + + 1 + + + jwt_static_key + HS256 + glue_token_forwarding_secret + false + true + + + + + + hs256 + default + + + + + + diff --git a/tests/integration/test_datalake_glue_token_forwarding/configs/users.xml b/tests/integration/test_datalake_glue_token_forwarding/configs/users.xml new file mode 100644 index 000000000000..98522df4d705 --- /dev/null +++ b/tests/integration/test_datalake_glue_token_forwarding/configs/users.xml @@ -0,0 +1,10 @@ + + + + + + + 1 + + + diff --git a/tests/integration/test_datalake_glue_token_forwarding/s3_mocks/mock_sts.py b/tests/integration/test_datalake_glue_token_forwarding/s3_mocks/mock_sts.py new file mode 100644 index 000000000000..fee74d5404a1 --- /dev/null +++ b/tests/integration/test_datalake_glue_token_forwarding/s3_mocks/mock_sts.py @@ -0,0 +1,73 @@ +import json +import sys +from datetime import datetime, timedelta, timezone +from urllib.parse import parse_qs + +from bottle import request, response, route, run + +recorded = [] + + +@route("/") +def ping(): + response.content_type = "text/plain" + response.set_header("Content-Length", 2) + return "OK" + + +@route("/_requests") +def list_requests(): + response.content_type = "application/json" + return json.dumps(recorded) + + +@route("/_reset") +def reset(): + recorded.clear() + response.content_type = "text/plain" + return "OK" + + +@route("/", method="POST") +def sts(): + body = request.body.read().decode() + params = {key: values[0] for key, values in parse_qs(body).items()} + + recorded.append( + { + "role_arn": params.get("RoleArn", ""), + "role_session_name": params.get("RoleSessionName", ""), + "web_identity_token": params.get("WebIdentityToken", ""), + } + ) + + if params.get("RoleSessionName") == "rejected": + response.status = 403 + response.content_type = "text/xml" + return """ + + + Sender + InvalidIdentityToken + Incorrect token audience + + + """ + + expiration = datetime.now(timezone.utc) + timedelta(hours=1) + + return f""" + + + + testing + testing + session-for-{params.get("RoleSessionName", "")} + {expiration.strftime("%Y-%m-%dT%H:%M:%SZ")} + + + + """ + + +run(host="0.0.0.0", port=int(sys.argv[1])) diff --git a/tests/integration/test_datalake_glue_token_forwarding/test.py b/tests/integration/test_datalake_glue_token_forwarding/test.py new file mode 100644 index 000000000000..6213005d0566 --- /dev/null +++ b/tests/integration/test_datalake_glue_token_forwarding/test.py @@ -0,0 +1,139 @@ +import json +import logging +import os +import uuid + +import jwt +import pytest + +from helpers.cluster import ClickHouseCluster +from helpers.mock_servers import start_mock_servers + +SECRET = "glue_token_forwarding_secret" +BASE_URL = "http://glue:3000" +ROLE_ARN = "arn:aws:iam::123456789012:role/data-lake-reader" +STS_CONTAINER = "sts.us-east-1.amazonaws.com" + +DATABASE_SETTINGS = { + "catalog_type": "glue", + "warehouse": "test", + "region": "us-east-1", + "aws_role_arn": ROLE_ARN, + "oauth_forward_user_token": "1", +} + + +def make_token(user): + return jwt.encode({"sub": user}, SECRET, algorithm="HS256") + + +@pytest.fixture(scope="module") +def started_cluster(): + cluster = ClickHouseCluster(__file__) + try: + cluster.add_instance( + "node1", + main_configs=["configs/token_forwarding.xml"], + user_configs=["configs/users.xml"], + stay_alive=True, + with_glue_catalog=True, + ) + + sts = cluster.add_instance( + name=STS_CONTAINER, + hostname=STS_CONTAINER, + image="altinityinfra/python-bottle", + tag="latest", + stay_alive=True, + ) + sts.stop_clickhouse(kill=True) + + logging.info("Starting cluster...") + cluster.start() + start_mock_servers( + cluster, + os.path.join(os.path.dirname(__file__), "s3_mocks"), + [("mock_sts.py", STS_CONTAINER, "80")], + ) + + node = cluster.instances["node1"] + node.query("CREATE ROLE IF NOT EXISTS token_users") + node.query("GRANT SHOW, SELECT ON *.* TO token_users") + + yield cluster + finally: + cluster.shutdown() + + +def sts_requests(started_cluster): + output = started_cluster.exec_in_container( + started_cluster.get_container_id(STS_CONTAINER), + [ + "python3", + "-c", + "import urllib.request;" + "print(urllib.request.urlopen('http://localhost:80/_requests').read().decode())", + ], + ) + return json.loads(output) + + +@pytest.fixture(autouse=True) +def clean_sts_log(started_cluster): + started_cluster.exec_in_container( + started_cluster.get_container_id(STS_CONTAINER), + [ + "python3", + "-c", + "import urllib.request; urllib.request.urlopen('http://localhost:80/_reset').read()", + ], + ) + + +def create_database(node, name): + node.query( + f"DROP DATABASE IF EXISTS {name}; " + f"CREATE DATABASE {name} ENGINE = DataLakeCatalog('{BASE_URL}') " + f"SETTINGS {','.join(k + '=' + repr(v) for k, v in DATABASE_SETTINGS.items())}", + settings={"allow_database_glue_catalog": 1}, + ) + + +def query_with_token(node, token, sql): + response = node.http_request( + "", method="POST", data=sql, headers={"Authorization": f"Bearer {token}"} + ) + response.raise_for_status() + + +def test_user_tokens_are_exchanged_into_separate_sts_sessions(started_cluster): + node = started_cluster.instances["node1"] + db = f"glue_{uuid.uuid4().hex[:8]}" + create_database(node, db) + + token = make_token("alice") + query_with_token(node, token, f"SHOW TABLES FROM {db}") + + query_with_token(node, make_token("bob"), f"SHOW TABLES FROM {db}") + sessions = sts_requests(started_cluster) + assert sorted(request["role_session_name"] for request in sessions) == ["alice", "bob"] + assert {request["web_identity_token"] for request in sessions} == {token, make_token("bob")} + assert all(request["role_arn"] == ROLE_ARN for request in sessions) + + +def test_rejected_token_does_not_fall_back(started_cluster): + node = started_cluster.instances["node1"] + db = f"glue_{uuid.uuid4().hex[:8]}" + create_database(node, db) + + response = node.http_request( + "", + method="POST", + data=f"SHOW TABLES FROM {db}", + headers={"Authorization": f"Bearer {make_token('rejected')}"}, + ) + assert response.status_code != 200 + assert "Could not assume role" in response.text, response.text + assert "InvalidIdentityToken" in response.text, response.text + + assert len(sts_requests(started_cluster)) == 1 diff --git a/tests/integration/test_datalake_token_forwarding/__init__.py b/tests/integration/test_datalake_token_forwarding/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/integration/test_datalake_token_forwarding/configs/token_forwarding.xml b/tests/integration/test_datalake_token_forwarding/configs/token_forwarding.xml new file mode 100644 index 000000000000..9b148d647c29 --- /dev/null +++ b/tests/integration/test_datalake_token_forwarding/configs/token_forwarding.xml @@ -0,0 +1,22 @@ + + 1 + + + jwt_static_key + HS256 + datalake_token_forwarding_secret + false + true + + + + + + hs256 + default + + + + + + diff --git a/tests/integration/test_datalake_token_forwarding/configs/users.xml b/tests/integration/test_datalake_token_forwarding/configs/users.xml new file mode 100644 index 000000000000..752bc965edb2 --- /dev/null +++ b/tests/integration/test_datalake_token_forwarding/configs/users.xml @@ -0,0 +1,15 @@ + + + + + + + 1 + + + passworduser_password + default + ::/0 + + + diff --git a/tests/integration/test_datalake_token_forwarding/test.py b/tests/integration/test_datalake_token_forwarding/test.py new file mode 100644 index 000000000000..35279ce2821f --- /dev/null +++ b/tests/integration/test_datalake_token_forwarding/test.py @@ -0,0 +1,226 @@ +import logging +import uuid + +import jwt +import pytest +import requests + +from helpers.cluster import ClickHouseCluster +from helpers.config_cluster import minio_access_key, minio_secret_key + +SECRET = "datalake_token_forwarding_secret" +BASE_URL = "http://rest:8181/v1" +CATALOG_NAME = "demo" +DATABASE_SETTINGS = { + "catalog_type": "rest", + "warehouse": "demo", + "storage_endpoint": "http://minio1:9001/warehouse-rest", + "catalog_credential": "service:principal", + "oauth_forward_user_token": 1, +} + + +def make_token(user): + return jwt.encode({"sub": user}, SECRET, algorithm="HS256") + + +@pytest.fixture(scope="module") +def started_cluster(): + cluster = ClickHouseCluster(__file__) + try: + cluster.add_instance( + "node1", + main_configs=["configs/token_forwarding.xml"], + user_configs=["configs/users.xml"], + stay_alive=True, + with_iceberg_catalog=True, + extra_parameters={ + "docker_compose_file_name": "docker_compose_iceberg_rest_catalog.yml" + }, + ) + logging.info("Starting cluster...") + cluster.start() + + node = cluster.instances["node1"] + node.query("CREATE ROLE IF NOT EXISTS token_users") + node.query("GRANT CHECK, DROP TABLE, INSERT, SELECT, SHOW ON *.* TO token_users") + node.query("GRANT S3 ON *.* TO token_users") + + yield cluster + finally: + cluster.shutdown() + + +def catalog_local_url(started_cluster): + return f"http://localhost:{started_cluster.iceberg_rest_catalog_port}/v1" + + +def create_namespace(started_cluster, namespace): + response = requests.post( + f"{catalog_local_url(started_cluster)}/namespaces", + json={"namespace": [namespace], "properties": {}}, + timeout=30, + ) + assert response.status_code in (200, 409), response.text + + +def query_with_token(node, token, sql, **kwargs): + response = node.http_request( + "", + method="POST", + data=sql, + headers={"Authorization": f"Bearer {token}"}, + **kwargs, + ) + response.raise_for_status() + return response.text + + +def create_database(node, storage_credentials=False): + arguments = f"'{BASE_URL}'" + if storage_credentials: + arguments += f", '{minio_access_key}', '{minio_secret_key}'" + node.query(f"DROP DATABASE IF EXISTS {CATALOG_NAME}") + node.query( + f"SET allow_experimental_database_iceberg=true;" + f"CREATE DATABASE {CATALOG_NAME} ENGINE = DataLakeCatalog({arguments}) " + f"SETTINGS {','.join(k + '=' + repr(v) for k, v in DATABASE_SETTINGS.items())}" + ) + + +def create_table_in_catalog(started_cluster, namespace, table): + response = requests.post( + f"{catalog_local_url(started_cluster)}/namespaces/{namespace}/tables", + json={ + "name": table, + "location": f"s3://warehouse-rest/{table}", + "schema": { + "type": "struct", + "schema-id": 0, + "fields": [{"id": 1, "name": "x", "required": False, "type": "string"}], + }, + }, + timeout=30, + ) + assert response.status_code in (200, 409), response.text + + +def catalog_tables(started_cluster, namespace): + response = requests.get( + f"{catalog_local_url(started_cluster)}/namespaces/{namespace}/tables", timeout=30 + ) + response.raise_for_status() + return {identifier["name"] for identifier in response.json()["identifiers"]} + + +def profile_event(node, query_id, event): + node.query("SYSTEM FLUSH LOGS") + return int( + node.query( + f"SELECT sum(ProfileEvents['{event}']) FROM system.query_log " + f"WHERE query_id = '{query_id}' AND type = 'QueryFinish'" + ).strip() + ) + + +def test_password_user_is_denied(started_cluster): + node = started_cluster.instances["node1"] + create_database(node) + + response = node.http_request( + "", + method="POST", + data=f"SELECT * FROM {CATALOG_NAME}.`nonexistent.table`", + params={"user": "passworduser", "password": "passworduser_password"}, + ) + assert "CATALOG_USER_TOKEN_NOT_AVAILABLE" in response.text, response.text + + +def test_native_protocol_forwards_jwt(started_cluster): + node = started_cluster.instances["node1"] + create_database(node) + + node.exec_in_container( + ["clickhouse", "client", "--jwt", make_token("alice"), "--query", f"CHECK DATABASE {CATALOG_NAME}"] + ) + + +def test_no_forwarding_without_the_server_setting(started_cluster): + node = started_cluster.instances["node1"] + + create_database(node) + + node.replace_in_config( + "/etc/clickhouse-server/config.d/token_forwarding.xml", + "1", + "0", + ) + node.query("SYSTEM RELOAD CONFIG") + try: + response = node.http_request( + "", + method="POST", + data=f"SELECT * FROM {CATALOG_NAME}.`nonexistent.table`", + headers={"Authorization": f"Bearer {make_token('alice')}"}, + ) + assert "CATALOG_USER_TOKEN_NOT_AVAILABLE" in response.text, response.text + finally: + node.replace_in_config( + "/etc/clickhouse-server/config.d/token_forwarding.xml", + "0", + "1", + ) + node.query("SYSTEM RELOAD CONFIG") + + +def write_fixture(started_cluster, node): + namespace = f"ns_{uuid.uuid4().hex[:8]}" + table = f"t_{uuid.uuid4().hex[:8]}" + create_namespace(started_cluster, namespace) + create_table_in_catalog(started_cluster, namespace, table) + create_database(node, storage_credentials=True) + return namespace, table + + +def test_async_insert_retains_the_querying_user_token(started_cluster): + node = started_cluster.instances["node1"] + namespace, table = write_fixture(started_cluster, node) + token = make_token("writer") + query_id = f"insert-{uuid.uuid4()}" + query_with_token( + node, + token, + f"INSERT INTO {CATALOG_NAME}.`{namespace}.{table}` VALUES ('written by the token user')", + params={ + "query_id": query_id, + "allow_insert_into_iceberg": 1, + "write_full_path_in_iceberg_metadata": 1, + "async_insert": 1, + "wait_for_async_insert": 0, + "async_insert_use_adaptive_busy_timeout": 0, + "async_insert_busy_timeout_ms": 60000, + }, + ) + + node.query("SYSTEM FLUSH ASYNC INSERT QUEUE") + assert profile_event(node, query_id, "AsyncInsertQuery") == 1 + assert ( + query_with_token(node, token, f"SELECT x FROM {CATALOG_NAME}.`{namespace}.{table}`").strip() + == "written by the token user" + ) + + +def test_drop_table_reaches_the_catalog_as_the_querying_user(started_cluster): + node = started_cluster.instances["node1"] + namespace, table = write_fixture(started_cluster, node) + + query_id = f"drop-{uuid.uuid4()}" + query_with_token( + node, + make_token("dropper"), + f"DROP TABLE {CATALOG_NAME}.`{namespace}.{table}`", + params={"query_id": query_id}, + ) + + assert table not in catalog_tables(started_cluster, namespace) + assert profile_event(node, query_id, "DataLakeRestCatalogAuthTokenRetrieve") == 0