diff --git a/src/Analyzer/Utils.cpp b/src/Analyzer/Utils.cpp index fe5ddbdd6d7f..cc9f77498055 100644 --- a/src/Analyzer/Utils.cpp +++ b/src/Analyzer/Utils.cpp @@ -1017,6 +1017,63 @@ void resolveAggregateFunctionNodeByName(FunctionNode & function_node, const Stri function_node.resolveAsAggregateFunction(std::move(aggregate_function)); } +namespace +{ + +class FinalizeAliasMarkersVisitor : public InDepthQueryTreeVisitor +{ +public: + explicit FinalizeAliasMarkersVisitor(ContextPtr context_) : context(std::move(context_)) {} + + /// Visit children first, so a nested marker chain is materialized from the inside out. + bool shouldTraverseTopToBottom() const { return false; } + + void visitImpl(QueryTreeNodePtr & node) + { + auto * function_node = node->as(); + if (!function_node || function_node->getFunctionName() != "__aliasMarker") + return; + + auto & arguments = function_node->getArguments().getNodes(); + if (arguments.size() != 2 || !arguments[0] || !arguments[1]) + return; + + /// Already materialized on an earlier hop. + if (const auto * id_node = arguments[1]->as(); id_node && isString(id_node->getResultType())) + return; + + const auto * column_node = arguments[1]->as(); + if (!column_node) + return; + + const auto & column_source = column_node->getColumnSourceOrNull(); + if (!column_source) + return; + + /// A lambda parameter -- `arrayMap(x -> __aliasMarker(x, x), ...)` written by hand -- has the `LambdaNode` as + /// its source. There is no table alias to build an id from, and the marker is a per-row identity rather than a + /// transport marker, so leave it as it is. A marker this pass injected inside a lambda body does not land here: + /// its id names an `ALIAS` column of a table, and the table expression is its source. + if (column_source->getNodeType() == QueryTreeNodeType::LAMBDA || !column_source->hasAlias()) + return; + + auto alias_id = column_source->getAlias() + "." + column_node->getColumnName(); + arguments[1] = std::make_shared(std::move(alias_id), std::make_shared()); + resolveOrdinaryFunctionNodeByName(*function_node, "__aliasMarker", context); + } + +private: + ContextPtr context; +}; + +} + +void finalizeAliasMarkersForDistributedSerialization(QueryTreeNodePtr & node, const ContextPtr & context) +{ + FinalizeAliasMarkersVisitor visitor(context); + visitor.visit(node); +} + std::pair getExpressionSource(const QueryTreeNodePtr & node) { if (const auto * column = node->as()) diff --git a/src/Analyzer/Utils.h b/src/Analyzer/Utils.h index 1e1182c76649..08cf5215426a 100644 --- a/src/Analyzer/Utils.h +++ b/src/Analyzer/Utils.h @@ -157,6 +157,19 @@ void resolveOrdinaryFunctionNodeByName(FunctionNode & function_node, const Strin /// Arguments and parameters are taken from the node. void resolveAggregateFunctionNodeByName(FunctionNode & function_node, const String & function_name); +/** Materialize the id argument of every `__aliasMarker` in the tree, turning the `ColumnNode` it carries into the + * `String` constant the shard will read. + * + * Call this immediately before the tree is rendered as SQL for a shard, and no earlier. The id is the column's + * analyzer identifier, and `createUniqueAliasesIfNecessary` -- which runs inside `buildQueryTreeForShard` -- is what + * settles the `__tableN` part of it. An id frozen before that point names a table alias that no longer exists by the + * time the shard header is built. + * + * A marker whose id is already a `String` constant was materialized on an earlier hop and is left alone. A marker + * carrying anything else is a user-written one, not ours, and is also left alone. + */ +void finalizeAliasMarkersForDistributedSerialization(QueryTreeNodePtr & node, const ContextPtr & context); + /// Returns single source of expression node. /// First element of pair is source node, can be nullptr if there are no sources or multiple sources. /// Second element of pair is true if there is at most one source, false if there are multiple sources. diff --git a/src/Functions/identity.cpp b/src/Functions/identity.cpp index 05d2ef870601..5245b39921b0 100644 --- a/src/Functions/identity.cpp +++ b/src/Functions/identity.cpp @@ -38,12 +38,14 @@ REGISTER_FUNCTION(AliasMarker) { factory.registerFunction(FunctionDocumentation{ .description = R"( -Internal function that marks ALIAS column expressions for the analyzer. Not intended for direct use. +Internal function. Returns its first argument unchanged. The second argument records which ALIAS column the +expression was inlined from, so that a shard names the result the way the initiator expects. Not intended for +direct use, but harmless when used directly. )", - .syntax = {"__aliasMarker(expr, alias_name)"}, + .syntax = {"__aliasMarker(expr, alias_id)"}, .arguments = { {"expr", "Expression to mark.", {"Any"}}, - {"alias_name", "Alias name attached to the expression.", {"String"}}, + {"alias_id", "Identity of the ALIAS column the expression came from.", {"Any"}}, }, .returned_value = {"Returns expr unchanged.", {"Any"}}, .introduced_in = {25, 8}, diff --git a/src/Functions/identity.h b/src/Functions/identity.h index 9c2ae607de1d..8278fff52af7 100644 --- a/src/Functions/identity.h +++ b/src/Functions/identity.h @@ -108,6 +108,35 @@ struct AliasMarkerName static constexpr auto name = "__aliasMarker"; }; +/** `__aliasMarker(expr, id)` is an internal pass-through identity. It returns `expr` untouched; `id` exists only to + * give the expression a stable name in the planner's `ActionsDAG`. + * + * It is injected when an `ALIAS` column is inlined into its defining expression for transport to a shard. The + * initiator still sees the un-inlined column, so without the marker the shard would name the output after the + * expression (`multiply(__table1.value, 2)`) while the initiator expects the column (`__table1.computed`), and the + * two headers could not be matched by name. + * + * The marker carries the low-level column identity, not the user's SQL alias. A SQL alias cannot do this job: it + * participates in user-visible query semantics, it can collide with names the user chose, and in the + * mergeable-state path the projection step that would normally apply it is skipped. + * + * This is also why the marker is not `__actionName`. `__actionName` survives into the `ActionsDAG` as a function node + * with a forced result name; `__aliasMarker` is consumed into an alias on top of its child, which is what keeps the + * expression behaving like a distinct logical column. + * + * The second argument travels in two forms. Between injection and serialization it is a `ColumnNode`, so the ordinary + * analyzer passes keep transforming it along with everything else -- in particular `createUniqueAliasesIfNecessary`, + * which assigns the final `__tableN` aliases. `finalizeAliasMarkersForDistributedSerialization` then materializes it + * into a `String` constant just before the query is rendered as SQL. Freezing the id any earlier captures a table + * alias that later changes, and the shard then emits a name the initiator never asked for. + * + * Both forms are accepted here, and so is anything else: a user can write `__aliasMarker(x, x)` and it behaves as an + * identity. Rejecting unexpected arguments in the function would turn user SQL into a server-side error for no gain, + * since the marker is harmless by construction. + * + * This is a bridge for as long as distributed transport still goes through SQL text. Once query plan serialization + * replaces that boundary, the marker should become unnecessary. + */ class FunctionAliasMarker : public IFunction { public: @@ -116,7 +145,9 @@ class FunctionAliasMarker : public IFunction String getName() const override { return name; } size_t getNumberOfArguments() const override { return 2; } - ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1}; } + /// The id argument is a constant only after finalization. Before that it is a `ColumnNode`, and between the two + /// the function must still resolve. + ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {}; } bool isSuitableForConstantFolding() const override { return false; } bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } @@ -125,9 +156,6 @@ class FunctionAliasMarker : public IFunction if (arguments.size() != 2) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Function __aliasMarker expects 2 arguments"); - if (!WhichDataType(arguments[1]).isString()) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Function __aliasMarker is internal and should not be used directly"); - return arguments.front(); } diff --git a/src/Planner/PlannerActionsVisitor.cpp b/src/Planner/PlannerActionsVisitor.cpp index cf347f8df6c4..3b2fb3c78fa9 100644 --- a/src/Planner/PlannerActionsVisitor.cpp +++ b/src/Planner/PlannerActionsVisitor.cpp @@ -95,6 +95,16 @@ String calculateActionNodeNameWithCastIfNeeded(const ConstantNode & constant_nod return buffer.str(); } +/// Return a `__aliasMarker`'s finalized id, or an empty string when it has none yet. Between injection and +/// serialization the id is a `ColumnNode`, and a user-written marker can hold anything at all. +String tryExtractAliasMarkerId(const QueryTreeNodePtr & id_argument) +{ + if (const auto * id_node = id_argument->as(); id_node && isString(id_node->getResultType())) + return id_node->getValue().safeGet(); + + return {}; +} + class ActionNodeNameHelper { public: @@ -198,18 +208,21 @@ class ActionNodeNameHelper const auto & function_node = node->as(); if (function_node.getFunctionName() == "__aliasMarker") { - /// Perform sanity check, because user may call this function with unexpected arguments const auto & function_argument_nodes = function_node.getArguments().getNodes(); - if (function_argument_nodes.size() == 2) - { - if (const auto * second_argument = function_argument_nodes.at(1)->as()) - { - if (isString(second_argument->getResultType())) - result = second_argument->getValue().safeGet(); - } - } + if (function_argument_nodes.size() != 2) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Function __aliasMarker expects 2 arguments"); - /// Empty node name is not allowed and leads to logical errors + result = tryExtractAliasMarkerId(function_argument_nodes.at(1)); + + /// No finalized id: either the marker has not reached its serialization boundary yet, or a user + /// wrote one by hand. Name the node after the expression it wraps, which is what the name would + /// have been without a marker at all. The two cases are not distinguishable here -- a hand-written + /// `__aliasMarker(x, x)` looks exactly like an injected one whose id is still a `ColumnNode` -- so + /// this cannot be turned into an assertion without rejecting valid SQL (03933). + if (result.empty()) + result = calculateActionNodeName(function_argument_nodes.at(0)); + + /// An empty node name is not allowed and leads to logical errors. if (result.empty()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Function __aliasMarker is internal and should not be used directly"); break; @@ -1259,15 +1272,14 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::vi if (function_arguments.size() != 2) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Function __aliasMarker expects 2 arguments"); - const auto * alias_id_node = function_arguments.at(1)->as(); - if (!alias_id_node || !isString(alias_id_node->getResultType())) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Function __aliasMarker is internal and should not be used directly"); + auto [child_name, levels] = visitImpl(function_arguments.at(0)); - const auto & alias_id = alias_id_node->getValue().safeGet(); + /// Without a finalized id the marker adds no identity of its own, so it resolves to its payload. That happens + /// for a marker the user wrote by hand, and for one that has not reached a serialization boundary yet. + auto alias_id = tryExtractAliasMarkerId(function_arguments.at(1)); if (alias_id.empty()) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Function __aliasMarker is internal and should not be used directly"); + alias_id = child_name; - auto [child_name, levels] = visitImpl(function_arguments.at(0)); if (alias_id == child_name) return {child_name, levels}; diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index a51044cfe76d..2bae73f364a1 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -10,7 +10,6 @@ #include #include #include -#include #include #include @@ -55,7 +54,6 @@ #include #include -#include #include #include #include @@ -213,7 +211,6 @@ namespace Setting extern const SettingsBool skip_unavailable_shards; extern const SettingsBool enable_global_with_statement; extern const SettingsBool allow_experimental_hybrid_table; - extern const SettingsBool enable_alias_marker; } namespace DistributedSetting @@ -849,73 +846,6 @@ StorageSnapshotPtr StorageDistributed::getStorageSnapshot(const StorageMetadataP namespace { -class ReplaseAliasColumnsVisitor : public InDepthQueryTreeVisitor -{ - QueryTreeNodePtr getColumnNodeAliasExpression(const QueryTreeNodePtr & node) const - { - const auto * column_node = node->as(); - if (!column_node || !column_node->hasExpression()) - return nullptr; - - const auto & column_source = column_node->getColumnSourceOrNull(); - if (!column_source || column_source->getNodeType() == QueryTreeNodeType::JOIN - || column_source->getNodeType() == QueryTreeNodeType::CROSS_JOIN - || column_source->getNodeType() == QueryTreeNodeType::ARRAY_JOIN) - return nullptr; - - auto column_expression = column_node->getExpression(); - const auto & column_name = column_node->getColumnName(); - - if (!context->getSettingsRef()[Setting::enable_alias_marker]) - { - column_expression->setAlias(column_name); - return column_expression; - } - - String alias_id; - const auto & source_alias = column_source->getAlias(); - if (!source_alias.empty()) - alias_id = source_alias + "." + column_name; - else - alias_id = column_name; - - if (auto * function_node = column_expression->as(); - function_node && function_node->getFunctionName() == "__aliasMarker") - { - auto & arguments = function_node->getArguments().getNodes(); - if (arguments.size() == 2) - arguments[1] = std::make_shared(alias_id, std::make_shared()); - - column_expression->setAlias(column_name); - return column_expression; - } - - QueryTreeNodes arguments; - arguments.reserve(2); - arguments.emplace_back(std::move(column_expression)); - arguments.emplace_back(std::make_shared(alias_id, std::make_shared())); - - auto alias_marker_node = std::make_shared("__aliasMarker"); - alias_marker_node->getArguments().getNodes() = std::move(arguments); - alias_marker_node->setAlias(column_name); - resolveOrdinaryFunctionNodeByName(*alias_marker_node, "__aliasMarker", context); - - return alias_marker_node; - } - -public: - explicit ReplaseAliasColumnsVisitor(ContextPtr context_) : context(std::move(context_)) {} - - void visitImpl(QueryTreeNodePtr & node) - { - if (auto column_expression = getColumnNodeAliasExpression(node)) - node = column_expression; - } - -private: - ContextPtr context; -}; - using ColumnNameToColumnNodeMap = std::unordered_map; ColumnNameToColumnNodeMap buildColumnNodesForTableExpression(const QueryTreeNodePtr & table_expression_node, const ContextPtr & context) @@ -1151,7 +1081,7 @@ QueryTreeNodePtr buildQueryTreeDistributed(SelectQueryInfo & query_info, * (including fully-resolved ALIAS expressions) and rewrite the whole query tree * so all references to the replaced table share the same column source and * the same alias semantics. This keeps SELECT and WHERE consistent before - * ReplaseAliasColumnsVisitor performs final alias expansion. + * inlineAliasColumns performs final alias expansion. */ ReplaceColumnNodesForTableExpressionVisitor replace_query_columns_visitor( replacement_table_expression, @@ -1160,8 +1090,7 @@ QueryTreeNodePtr buildQueryTreeDistributed(SelectQueryInfo & query_info, replace_query_columns_visitor.visit(query_tree_to_modify); } - ReplaseAliasColumnsVisitor replace_alias_columns_visitor(query_context); - replace_alias_columns_visitor.visit(query_tree_to_modify); + inlineAliasColumns(query_tree_to_modify, query_context); const auto & settings = query_context->getSettingsRef(); @@ -1174,8 +1103,9 @@ QueryTreeNodePtr buildQueryTreeDistributed(SelectQueryInfo & query_info, rewriteJoinToGlobalJoinIfNeeded(query_node.getJoinTree()); } + /// `buildQueryTreeForShard` materializes the marker ids on the way out, after the `__tableN` renumbering they are + /// built from. return buildQueryTreeForShard(query_info.planner_context, query_tree_to_modify, /*allow_global_join_for_right_table*/ false); - } std::optional> tryGetParamTypeAndName(const ASTPtr & node) diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 1fbcff907866..5fb2af1552dc 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -942,7 +943,16 @@ std::vector ReadFromMerge::createChildrenPlans(SelectQ { /// Source tables could have different but convertible types, like numeric types of different width. /// We must return streams with structure equals to structure of Merge table. - convertAndFilterSourceStream(*common_header, modified_query_info, nested_storage_snapshot, aliases, row_policy_data_opt, context, child, is_smallest_column_requested); + convertAndFilterSourceStream( + *common_header, + modified_query_info, + nested_storage_snapshot, + merge_storage_snapshot->metadata->getColumns(), + aliases, + row_policy_data_opt, + context, + child, + is_smallest_column_requested); for (const auto & filter_info : pushed_down_filters) { @@ -1627,6 +1637,7 @@ void ReadFromMerge::convertAndFilterSourceStream( const Block & header, SelectQueryInfo & modified_query_info, const StorageSnapshotPtr & snapshot, + const ColumnsDescription & merge_columns, const Aliases & aliases, const RowPolicyDataOpt & row_policy_data_opt, ContextPtr local_context, @@ -1637,12 +1648,119 @@ void ReadFromMerge::convertAndFilterSourceStream( auto pipe_columns = before_block_header->getNamesAndTypesList(); + /// TODO(storage-merge-alias): the analyzer branch below recomputes by hand what the analyzer's own column-alias + /// resolution would produce if it ran end to end on the child plan. It exists because the work is split in two: + /// + /// Step 1, in `getModifiedQueryInfo`: rewrite the query going to the child storage. `replaceColumns` swaps every + /// reference to a Merge-level ALIAS column for its resolved expression, so the child storage is asked for the + /// PHYSICAL columns those expressions need and never sees the alias names. + /// + /// Step 2, here: recompute the alias VALUES at the Merge level from those physical columns, by building a fresh + /// `ActionsDAG`, running `QueryAnalysisPass` over each alias expression and visiting it with + /// `PlannerActionsVisitor`. Each alias output is emitted under its analyzer identifier so the Merge target + /// header, which also uses analyzer identifiers, can pick it up by name. + /// + /// The awkward part is that alias values are computed AFTER the child's read, not inside it. A predicate over an + /// ALIAS column can still use the underlying physical column for index analysis, but only because Step 1 happens + /// to inline the alias expression into the predicate as well, so `KeyCondition` sees `col * 2 > 10` rather than + /// `alias > 10`. Output-side aliases are recomputed here even when the child already produced the same value -- + /// a `Distributed` child, for instance, evaluates alias expressions on the shard and returns them as + /// expression-named columns, which this then computes a second time. + /// + /// The unification would be to have Step 1 emit `__aliasMarker(, '')` instead + /// of a bare resolved expression. `PlannerActionsVisitor` resolves the marker at plan-build time and the call + /// disappears from the resulting `ActionsDAG`, leaving an ordinary action node that computes `` + /// under ``. Predicate and `KeyCondition` analysis would be unaffected, because the marker is a + /// planner-time naming device rather than a runtime expression. Step 2 would then be deletable outright: + /// `pipe_columns` would already carry the alias values under the right names. + /// + /// Left as future work. What is here is correct -- Step 1 and Step 2 together produce the right values -- just not + /// minimal. if (local_context->getSettingsRef()[Setting::allow_experimental_analyzer]) { + /// The Merge table wants its columns under analyzer identifiers (`__table1.a`, ``__table1.`n.a` ``), while alias + /// expressions and `alias.name` speak in plain logical names (`a`, `n.a`). + /// + /// The planner's `TableExpressionData` for the Merge node is not populated yet at this point -- column + /// collection happens later, in `CollectTableExpressionData` -- so the mapping cannot be looked up through + /// `PlannerContext`. Build it here instead, by matching each Merge-declared column name against the suffixes of + /// the identifier names in `header` / `pipe_columns`: a column named `.` or ``.`` `` + /// belongs to the Merge column `C`. + /// + /// Driving the match from the declared Merge schema rather than from a pattern over the analyzer's naming + /// convention is what makes dotted names work, `Nested` subcolumns and backtick-quoted names included. + + /// `getAll` builds a fresh list on every call, so take it once rather than per candidate column. + const auto all_merge_columns = merge_columns.getAll(); + + auto build_plain_to_identifier = [&merge_columns, &all_merge_columns](const auto & candidate_names) + { + std::unordered_map plain_to_identifier; + std::unordered_set ambiguous; + for (const auto & column : candidate_names) + { + /// An exact match means the column carries no analyzer prefix at all. + if (merge_columns.has(column.name)) + { + if (!plain_to_identifier.emplace(column.name, column.name).second) + ambiguous.insert(column.name); + continue; + } + + /// Otherwise look for `.` or ``.`` `` where C is a declared Merge column. Columns + /// matching no declared name are skipped; they are intermediate outputs of the child plan. + /// + /// The unquoted form of a dotted name is accepted defensively. No producer of it is known: + /// `buildColumnIdentifier` always backquotes a dotted column name, so an analyzer identifier for `n.a` + /// is always ``__tableN.`n.a` ``. + for (const auto & merge_column : all_merge_columns) + { + const bool dotted = merge_column.name.find('.') != String::npos; + const String want_raw = "." + merge_column.name; + const String want_quoted = dotted ? ("." + backQuote(merge_column.name)) : want_raw; + if (column.name.ends_with(want_quoted) || (dotted && column.name.ends_with(want_raw))) + { + if (!plain_to_identifier.emplace(merge_column.name, column.name).second) + ambiguous.insert(merge_column.name); + break; + } + } + } + for (const auto & ambiguous_name : ambiguous) + plain_to_identifier.erase(ambiguous_name); + return plain_to_identifier; + }; + + const auto header_plain_to_identifier = build_plain_to_identifier(header); + const auto pipe_plain_to_identifier = build_plain_to_identifier(pipe_columns); + for (const auto & alias : aliases) { ActionsDAG actions_dag(pipe_columns); + /// Alias expressions name their inputs in plain logical names, but the child stream's inputs carry analyzer + /// identifiers. Expose every mapped column under its plain name as well, so that `buildQueryTree` below + /// resolves a reference like `a` or `n.a` against an input named `__table1.a` or ``__table1.`n.a` ``. An + /// alias defined over another alias needs this. + for (const auto & [plain, identifier] : pipe_plain_to_identifier) + { + if (plain == identifier) + continue; + + const ActionsDAG::Node * input_node = nullptr; + for (const auto * candidate : actions_dag.getInputs()) + { + if (candidate->result_name == identifier) + { + input_node = candidate; + break; + } + } + + if (input_node) + actions_dag.addAlias(*input_node, plain); + } + QueryTreeNodePtr query_tree = buildQueryTree(alias.expression, local_context); query_tree->setAlias(alias.name); @@ -1659,7 +1777,12 @@ void ReadFromMerge::convertAndFilterSourceStream( if (nodes.size() != 1) throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected to have 1 output but got {}", nodes.size()); - actions_dag.addOrReplaceInOutputs(actions_dag.addAlias(*nodes.front(), alias.name)); + /// Emit the alias output under its analyzer identifier, so the `addMissingDefaults` step further down + /// matches it by name. Emitted under the plain name it would miss, and the expected + /// ``__tableN.`alias.name` `` column would be filled with type defaults instead. + const auto identifier_it = header_plain_to_identifier.find(alias.name); + const String & output_name = identifier_it != header_plain_to_identifier.end() ? identifier_it->second : alias.name; + actions_dag.addOrReplaceInOutputs(actions_dag.addAlias(*nodes.front(), output_name)); auto expression_step = std::make_unique(child.plan.getCurrentHeader(), std::move(actions_dag)); child.plan.addStep(std::move(expression_step)); } @@ -1733,6 +1856,8 @@ void ReadFromMerge::convertAndFilterSourceStream( } } + /// Position, because the loop above builds `converted_columns[i]` out of `current_step_columns[i]`: the pairing is + /// fixed by that construction rather than by the names, and one of the branches deliberately renames a column. auto convert_actions_dag = ActionsDAG::makeConvertingActions( current_step_columns, converted_columns, diff --git a/src/Storages/StorageMerge.h b/src/Storages/StorageMerge.h index 99d815791fea..34efb3fd5e7e 100644 --- a/src/Storages/StorageMerge.h +++ b/src/Storages/StorageMerge.h @@ -306,6 +306,7 @@ class ReadFromMerge final : public SourceStepWithFilter const Block & header, SelectQueryInfo & modified_query_info, const StorageSnapshotPtr & snapshot, + const ColumnsDescription & merge_columns, const Aliases & aliases, const RowPolicyDataOpt & row_policy_data_opt, ContextPtr context, diff --git a/src/Storages/buildQueryTreeForShard.cpp b/src/Storages/buildQueryTreeForShard.cpp index 9669cd40ad7e..0dfdb324db2a 100644 --- a/src/Storages/buildQueryTreeForShard.cpp +++ b/src/Storages/buildQueryTreeForShard.cpp @@ -62,6 +62,7 @@ namespace Setting extern const SettingsBool parallel_replicas_prefer_local_join; extern const SettingsBool prefer_global_in_and_join; extern const SettingsBool enable_add_distinct_to_in_subqueries; + extern const SettingsBool enable_alias_marker; extern const SettingsInt64 optimize_const_name_size; extern const SettingsOverflowMode transfer_overflow_mode; extern const SettingsObjectStorageClusterJoinMode object_storage_cluster_join_mode; @@ -78,6 +79,218 @@ namespace ErrorCodes namespace { +/// Return a clone of the defining expression of an inlineable `ALIAS` column node, or nullptr otherwise. +/// A JOIN / CROSS_JOIN / ARRAY_JOIN source puts a `ListNode` of the joined sides in the expression child, which is not +/// an alias body. The expression is cloned so each occurrence gets its own copy: that lets one occurrence be aliased +/// (a projection output) without mutating another (a reference in `ORDER BY`). +QueryTreeNodePtr getInlineableAliasColumnExpression(const QueryTreeNodePtr & node) +{ + const auto * column_node = node->as(); + if (!column_node || !column_node->hasExpression()) + return nullptr; + + const auto & column_source = column_node->getColumnSourceOrNull(); + if (!column_source || column_source->getNodeType() == QueryTreeNodeType::JOIN + || column_source->getNodeType() == QueryTreeNodeType::CROSS_JOIN + || column_source->getNodeType() == QueryTreeNodeType::ARRAY_JOIN) + return nullptr; + + return column_node->getExpression()->clone(); +} + +/// Wrap an inlined `ALIAS` expression in `__aliasMarker` so it carries the identity of the column it came from. +/// +/// The id goes in as a live `ColumnNode`, not as a finished string. Later analyzer passes still have to run -- +/// `createUniqueAliasesIfNecessary` in particular, which assigns the `__tableN` aliases the id is built from -- and a +/// `ColumnNode` is transformed by those passes exactly as the rest of the tree is. +/// `finalizeAliasMarkersForDistributedSerialization` turns it into a `String` constant at the serialization boundary. +/// +/// An expression that is already a marker has its id replaced rather than gaining a second wrapper. +QueryTreeNodePtr wrapInAliasMarker(QueryTreeNodePtr expression, const ColumnNode & column_node, const ContextPtr & context) +{ + auto marker_id = std::make_shared(column_node.getColumn(), column_node.getColumnSourceOrNull()); + + if (auto * function_node = expression->as(); + function_node && function_node->getFunctionName() == "__aliasMarker") + { + auto & marker_arguments = function_node->getArguments().getNodes(); + if (marker_arguments.size() == 2) + marker_arguments[1] = std::move(marker_id); + return expression; + } + + QueryTreeNodes arguments; + arguments.reserve(2); + arguments.emplace_back(std::move(expression)); + arguments.emplace_back(std::move(marker_id)); + + auto marker_node = std::make_shared("__aliasMarker"); + marker_node->getArguments().getNodes() = std::move(arguments); + resolveOrdinaryFunctionNodeByName(*marker_node, "__aliasMarker", context); + + return marker_node; +} + +/// Inlines `ALIAS` columns into their defining expressions across a query tree that is about to be shipped. +/// See `inlineAliasColumns` in the header for what the pass guarantees and why. +struct AliasColumnInliner +{ + ContextPtr context; + bool use_alias_marker; + + /// Replace one inlineable `ALIAS` column node with its body, marking the body when markers are on. + /// Returns false when `node` is not an inlineable `ALIAS` column, leaving it untouched. + bool inlineOnce(QueryTreeNodePtr & node) const + { + auto expression = getInlineableAliasColumnExpression(node); + if (!expression) + return false; + + if (use_alias_marker) + expression = wrapInAliasMarker(std::move(expression), node->as(), context); + + node = std::move(expression); + return true; + } + + /// Inline `ALIAS` columns inside an expression subtree without assigning any alias. A nested subquery goes back + /// through `inlineQuery` so its own projection columns keep their names. + void inlineExpression(QueryTreeNodePtr & node) const + { + if (node->as() || node->as()) + { + inlineQuery(node); + return; + } + + /// An `ALIAS` column may be defined over another one, so keep unwrapping. With markers on the loop stops after + /// the first step, because the wrapper is a function node; the column it wrapped is reached through the + /// children below and gets its own marker there. + /// + /// A marker that ends up nested inside the payload expression -- `computed ALIAS inner * 2` -- is kept as it + /// is. One that ends up as the payload itself -- `computed ALIAS inner` -- is collapsed by + /// `NormalizeAliasMarkerVisitor` when the tree is rendered to SQL, and nothing is lost by that: the inner id + /// names an intermediate action node, while the name the initiator matches against comes from the outer id. + while (inlineOnce(node)) + { + } + + if (auto * marker_node = node->as(); + marker_node && marker_node->getFunctionName() == "__aliasMarker") + { + auto & marker_arguments = marker_node->getArguments().getNodes(); + if (marker_arguments.size() == 2) + { + /// Descend into the payload only. The second argument is the column reference the marker exists to + /// record, and inlining it would expand the very `ALIAS` column whose identity it carries. It has to + /// reach `finalizeAliasMarkersForDistributedSerialization` as a `ColumnNode`. + inlineExpression(marker_arguments[0]); + return; + } + } + + auto * join_node = node->as(); + const bool using_join = join_node && join_node->isUsingJoinExpression(); + + for (auto & child : node->getChildren()) + { + if (!child) + continue; + + if (using_join && child == join_node->getJoinExpression()) + inlineJoinUsingKeys(child); + else + inlineExpression(child); + } + } + + /// A `JOIN USING` key is a `ColumnNode` whose expression is a `ListNode` recording how the key resolves on each + /// side, and a side's entry can itself be an `ALIAS` column. Such an entry keeps the key's name as an alias when + /// inlined, because the shipped SQL renders the entry rather than the key: `USING (x AS a)` is what lets a remote + /// server resolve a key that exists only as an `ALIAS` column of the initiator's table. `rejectUnshippableJoinUsingKeys` + /// reads the same shape to decide which keys no remote server can resolve. + void inlineJoinUsingKeys(QueryTreeNodePtr & join_expression) const + { + auto * using_list = join_expression->as(); + if (!using_list) + return; + + for (auto & using_node : using_list->getNodes()) + { + auto * using_column = using_node->as(); + if (!using_column || !using_column->hasExpression()) + continue; + + auto * key_sides = using_column->getExpression()->as(); + if (!key_sides) + continue; + + for (auto & side : key_sides->getNodes()) + { + const auto * side_column = side->as(); + auto expression = getInlineableAliasColumnExpression(side); + if (!expression) + continue; + + const String key_name = side_column->getColumnName(); + if (use_alias_marker) + expression = wrapInAliasMarker(std::move(expression), *side_column, context); + + inlineExpression(expression); + expression->setAlias(key_name); + side = std::move(expression); + } + } + } + + /// Inline `ALIAS` columns into their defining expressions, so the expression is evaluated on the shard that reads + /// the real table instead of the column being resolved there as if it were physical. + /// + /// A top-level projection item keeps the column's logical name as an alias, so the mergeable-state output column + /// keeps its name. Inside expression clauses no alias is set: otherwise two same-named `ALIAS` columns from + /// different `JOIN` sources land in one scope carrying different bodies, and the shard throws + /// `MULTIPLE_EXPRESSIONS_FOR_ALIAS` (https://github.com/ClickHouse/ClickHouse/issues/107990). + void inlineQuery(QueryTreeNodePtr & node) const + { + if (auto * union_node = node->as()) + { + for (auto & query : union_node->getQueries().getNodes()) + inlineQuery(query); + return; + } + + auto * query_node = node->as(); + if (!query_node) + { + inlineExpression(node); + return; + } + + for (auto & projection_item : query_node->getProjection().getNodes()) + { + const auto * column_node = projection_item->as(); + auto expression = getInlineableAliasColumnExpression(projection_item); + if (!expression) + { + inlineExpression(projection_item); + continue; + } + + const String output_alias = column_node->getColumnName(); + if (use_alias_marker) + expression = wrapInAliasMarker(std::move(expression), *column_node, context); + + inlineExpression(expression); + expression->setAlias(output_alias); + projection_item = std::move(expression); + } + + for (auto & child : query_node->getChildren()) + if (child && child != query_node->getProjectionNode()) + inlineExpression(child); + } +}; + /// Visitor that collect column source to columns mapping from query and all subqueries class CollectColumnSourceToColumnsVisitor : public InDepthQueryTreeVisitor { @@ -104,8 +317,24 @@ class CollectColumnSourceToColumnsVisitor : public InDepthQueryTreeVisitoras(); + function_node && function_node->getFunctionName() == "__aliasMarker") + { + const auto & marker_arguments = function_node->getArguments().getNodes(); + if (marker_arguments.size() == 2 && marker_arguments[1]) + marker_id_nodes.insert(marker_arguments[1].get()); + return; + } + auto * column_node = node->as(); - if (!column_node) + if (!column_node || marker_id_nodes.contains(node.get())) return; auto column_source = column_node->getColumnSourceOrNull(); @@ -124,6 +353,7 @@ class CollectColumnSourceToColumnsVisitor : public InDepthQueryTreeVisitor column_source_to_columns; + std::unordered_set marker_id_nodes; }; /** Visitor that rewrites IN and JOINs in query and all subqueries according to distributed_product_mode and @@ -434,7 +664,14 @@ TableNodePtr executeSubqueryNode(const QueryTreeNodePtr & subquery_node, ContextMutablePtr & mutable_context, size_t subquery_depth) { - const auto subquery_hash = subquery_node->getTreeHash(); + /// A `GLOBAL IN` / `GLOBAL JOIN` subquery is materialized here and shipped as a temporary table, so this is a + /// serialization boundary too and the marker ids have to be finalized before the subquery runs. Finalizing also + /// makes the tree hash below stable: an unmaterialized marker hashes its `ColumnNode`, whose identifier can still + /// change, which would key the same subquery under two different temporary table names. + auto subquery_node_to_execute = subquery_node->clone(); + finalizeAliasMarkersForDistributedSerialization(subquery_node_to_execute, mutable_context); + + const auto subquery_hash = subquery_node_to_execute->getTreeHash(); const auto temporary_table_name = fmt::format("_data_{}", toString(subquery_hash)); const auto & external_tables = mutable_context->getExternalTables(); @@ -452,7 +689,7 @@ TableNodePtr executeSubqueryNode(const QueryTreeNodePtr & subquery_node, auto context_copy = Context::createCopy(mutable_context); updateContextForSubqueryExecution(context_copy); - InterpreterSelectQueryAnalyzer interpreter(subquery_node, context_copy, subquery_options); + InterpreterSelectQueryAnalyzer interpreter(subquery_node_to_execute, context_copy, subquery_options); auto & query_plan = interpreter.getQueryPlan(); auto sample_block_with_unique_names = *query_plan.getCurrentHeader(); @@ -738,6 +975,12 @@ void rejectUnshippableJoinUsingKeys(const QueryTreeNodePtr & root) } +void inlineAliasColumns(QueryTreeNodePtr & query_tree_to_modify, const ContextPtr & context) +{ + const AliasColumnInliner inliner{context, context->getSettingsRef()[Setting::enable_alias_marker]}; + inliner.inlineQuery(query_tree_to_modify); +} + QueryTreeNodePtr buildQueryTreeForShard( const PlannerContextPtr & planner_context, QueryTreeNodePtr query_tree_to_modify, @@ -913,6 +1156,12 @@ QueryTreeNodePtr buildQueryTreeForShard( scalar_visitor.visit(query_tree_to_modify); } + /// Last, because `createUniqueAliasesIfNecessary` above is what settles the `__tableN` aliases the marker ids are + /// built from: an id materialized any earlier would name the table alias the initiator happened to assign before + /// the renumbering rather than the one the shard will use. Here rather than in the callers, so that every path + /// that ships a query tree finalizes whatever markers it carries. + finalizeAliasMarkersForDistributedSerialization(query_tree_to_modify, planner_context->getQueryContext()); + return query_tree_to_modify; } diff --git a/src/Storages/buildQueryTreeForShard.h b/src/Storages/buildQueryTreeForShard.h index 5e3ae678dcf6..7730c43adc6b 100644 --- a/src/Storages/buildQueryTreeForShard.h +++ b/src/Storages/buildQueryTreeForShard.h @@ -27,6 +27,23 @@ QueryTreeNodePtr buildQueryTreeForShard( bool allow_global_join_for_right_table, bool find_cross_join = false); +/** Replace every `ALIAS` column node with its defining expression, so the expression is evaluated on the shard that reads + * the real table instead of the column being resolved there as if it were physical. + * + * Apply this to any query tree that is about to be shipped, before `buildQueryTreeForShard`: that function rebuilds a + * shipped table expression from column names and types only, which drops an `ALIAS` column's resolved expression and + * leaves the remote side asking storage for a column it does not have. + * + * When `enable_alias_marker` is on, each inlined expression is also wrapped in `__aliasMarker(expr, '')`. The + * marker preserves the identity of the logical column the expression was expanded from, so the initiator -- which + * still sees the un-inlined column -- can match the shard header by name instead of by position. + * + * The column's logical name is kept as an alias on top-level projection items and on `JOIN USING` key sides only. + * Aliasing an occurrence inside `WHERE` / `GROUP BY` / `ORDER BY` / `HAVING` / `JOIN ON` makes two same-named `ALIAS` + * columns from different sources collide in one scope, and the shard then throws `MULTIPLE_EXPRESSIONS_FOR_ALIAS`. + */ +void inlineAliasColumns(QueryTreeNodePtr & query_tree_to_modify, const ContextPtr & context); + void rewriteJoinToGlobalJoin(QueryTreeNodePtr query_tree_to_modify, ContextPtr context, bool force_prefer_global_join = false); void rewriteInToGlobalIn(QueryTreeNodePtr & query_tree_to_modify, ContextPtr context, bool rewrite_for_distributed = false); diff --git a/tests/queries/0_stateless/03648_alias_marker_with_mergeable_state.reference b/tests/queries/0_stateless/03648_alias_marker_with_mergeable_state.reference index 58bf6a7ec74b..5f061a829b23 100644 --- a/tests/queries/0_stateless/03648_alias_marker_with_mergeable_state.reference +++ b/tests/queries/0_stateless/03648_alias_marker_with_mergeable_state.reference @@ -2,6 +2,8 @@ Header: sum(foo) AggregateFunction(sum, Int64) ---- stage: with_mergeable_state (analyzer=0) ---- Expected error: Function __aliasMarker is internal and supported only with the analyzer +---- explicit __aliasMarker in user query (analyzer=1) ---- +Explicit __aliasMarker call is allowed ---- stage: complete (analyzer=1) ---- Header: x Int64 ---- stage: fetch_columns (analyzer=1) ---- diff --git a/tests/queries/0_stateless/03648_alias_marker_with_mergeable_state.sh b/tests/queries/0_stateless/03648_alias_marker_with_mergeable_state.sh index 66974be38517..1125a06b3d54 100755 --- a/tests/queries/0_stateless/03648_alias_marker_with_mergeable_state.sh +++ b/tests/queries/0_stateless/03648_alias_marker_with_mergeable_state.sh @@ -4,8 +4,13 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh +# `--query_kind secondary_query` goes with every `--stage` below, because the point is to print the +# header a shard leg produces. A query marked as secondary skips AST-level optimizations +# (`PlannerContext.cpp`, `is_ast_level_optimization_allowed`) and changes how aggregation is split, +# neither of which a real shard leg would have applied. Inherited unchanged from Altinity#1844. + echo "---- stage: with_mergeable_state (analyzer=1, setting=enable_alias_marker=1) ----" -$CLICKHOUSE_CLIENT --enable_analyzer=1 --stage with_mergeable_state --multiquery 2>&1 <<'EOF' | sed -n '/^Header:/,/^ [^ ]/p' | sed '$d' +$CLICKHOUSE_CLIENT --enable_analyzer=1 --query_kind secondary_query --stage with_mergeable_state --multiquery 2>&1 <<'EOF' | sed -n '/^Header:/,/^ [^ ]/p' | sed '$d' SET enable_alias_marker=1; EXPLAIN header=1 SELECT sum(__aliasMarker(number*2-3,'foo')) AS x @@ -22,27 +27,36 @@ else echo "${alias_marker_error_output}" fi +echo "---- explicit __aliasMarker in user query (analyzer=1) ----" +if $CLICKHOUSE_CLIENT --enable_analyzer=1 --query \ + "SELECT __aliasMarker(number*2-3,'foo') FROM numbers(1)" >/dev/null 2>&1 +then + echo "Explicit __aliasMarker call is allowed" +else + echo "Unexpected error for explicit __aliasMarker call" +fi + echo "---- stage: complete (analyzer=1) ----" -$CLICKHOUSE_CLIENT --enable_analyzer=1 --stage complete --query \ +$CLICKHOUSE_CLIENT --enable_analyzer=1 --query_kind secondary_query --stage complete --query \ "EXPLAIN header=1 SELECT sum(__aliasMarker(number*2-3,'foo')) AS x FROM numbers(10)" \ 2>&1 | sed -n '/^Header:/,/^ [^ ]/p' | sed '$d' echo "---- stage: fetch_columns (analyzer=1) ----" -$CLICKHOUSE_CLIENT --enable_analyzer=1 --stage fetch_columns --query \ +$CLICKHOUSE_CLIENT --enable_analyzer=1 --query_kind secondary_query --stage fetch_columns --query \ "EXPLAIN header=1 SELECT sum(__aliasMarker(number*2-3,'foo')) AS x FROM numbers(10)" \ 2>&1 | sed -n '/^Header:/,/^ [^ ]/p' | sed '$d' echo "---- stage: with_mergeable_state (analyzer=1) ----" -$CLICKHOUSE_CLIENT --enable_analyzer=1 --stage with_mergeable_state --query \ +$CLICKHOUSE_CLIENT --enable_analyzer=1 --query_kind secondary_query --stage with_mergeable_state --query \ "EXPLAIN header=1 SELECT sum(__aliasMarker(number*2-3,'foo')) AS x FROM numbers(10)" \ 2>&1 | sed -n '/^Header:/,/^ [^ ]/p' | sed '$d' echo "---- stage: with_mergeable_state_after_aggregation (analyzer=1) ----" -$CLICKHOUSE_CLIENT --enable_analyzer=1 --stage with_mergeable_state_after_aggregation --query \ +$CLICKHOUSE_CLIENT --enable_analyzer=1 --query_kind secondary_query --stage with_mergeable_state_after_aggregation --query \ "EXPLAIN header=1 SELECT sum(__aliasMarker(number*2-3,'foo')) AS x FROM numbers(10)" \ 2>&1 | sed -n '/^Header:/,/^ [^ ]/p' | sed '$d' echo "---- stage: with_mergeable_state_after_aggregation_and_limit (analyzer=1) ----" -$CLICKHOUSE_CLIENT --enable_analyzer=1 --stage with_mergeable_state_after_aggregation_and_limit --query \ +$CLICKHOUSE_CLIENT --enable_analyzer=1 --query_kind secondary_query --stage with_mergeable_state_after_aggregation_and_limit --query \ "EXPLAIN header=1 SELECT sum(__aliasMarker(number*2-3,'foo')) AS x FROM numbers(10) GROUP BY intDiv(number,10) AS y ORDER BY y LIMIT 10" \ 2>&1 | sed -n '/^Header:/,/^ [^ ]/p' | sed '$d' diff --git a/tests/queries/0_stateless/03842_hybrid_alias_issue_1424.reference b/tests/queries/0_stateless/03842_hybrid_alias_issue_1424.reference new file mode 100644 index 000000000000..6f78da4c4f59 --- /dev/null +++ b/tests/queries/0_stateless/03842_hybrid_alias_issue_1424.reference @@ -0,0 +1,42 @@ +max in subquery +4294967294 +sum in subquery +-4921211434 +cte min with predicate +679772422 +cte with limit +-2147483648 -4294967296 +-1762862292 -574613778 +-1329695183 -1573638336 +-221724287 679772422 +0 0 +550067609 -3048000734 +1084637461 3417479706 +1169291374 -3082049462 +1899628504 -740161250 +2147483647 4294967294 +cte without limit +-2147483648 -4294967296 +-1762862292 -574613778 +-1329695183 -1573638336 +-221724287 679772422 +0 0 +550067609 -3048000734 +1084637461 3417479706 +1169291374 -3082049462 +1899628504 -740161250 +2147483647 4294967294 +group by in subquery +10 10 +intersect with order by +-221724287 679772422 +1084637461 3417479706 +2147483647 4294967294 +intersect without order by +-221724287 679772422 +1084637461 3417479706 +2147483647 4294967294 +constant alias in subquery +9 7 32 +constant alias predicate +2 diff --git a/tests/queries/0_stateless/03842_hybrid_alias_issue_1424.sql b/tests/queries/0_stateless/03842_hybrid_alias_issue_1424.sql new file mode 100644 index 000000000000..8b9cf9182896 --- /dev/null +++ b/tests/queries/0_stateless/03842_hybrid_alias_issue_1424.sql @@ -0,0 +1,202 @@ +SET allow_experimental_hybrid_table = 1, enable_analyzer = 1, enable_alias_marker = 1; + +DROP TABLE IF EXISTS test_hybrid_issue_1424; +DROP TABLE IF EXISTS test_hybrid_issue_1424_left; +DROP TABLE IF EXISTS test_hybrid_issue_1424_right; +DROP TABLE IF EXISTS test_hybrid_issue_1424_const; +DROP TABLE IF EXISTS test_hybrid_issue_1424_const_left; +DROP TABLE IF EXISTS test_hybrid_issue_1424_const_right; + +CREATE TABLE test_hybrid_issue_1424_left +( + id Int32, + value Int32, + date_col Date, + computed ALIAS value * 2 +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(date_col) +ORDER BY (date_col, id); + +INSERT INTO test_hybrid_issue_1424_left VALUES + (toInt32(2147483647), toInt32(2147483647), toDate('2149-06-06')), + (toInt32(-2147483648), toInt32(-2147483648), toDate('1970-01-01')), + (toInt32(0), toInt32(0), '1970-01-01'), + (toInt32(1084637461), toInt32(1708739853), toDate(1335613783)), + (toInt32(-221724287), toInt32(339886211), toDate(1294089763)), + (toInt32(-1762862292), toInt32(-287306889), toDate(1375707465)), + (toInt32(1169291374), toInt32(-1541024731), toDate(1082126480)), + (toInt32(-1329695183), toInt32(-786819168), toDate(1226000164)), + (toInt32(1899628504), toInt32(-370080625), toDate(1179050966)), + (toInt32(550067609), toInt32(-1524000367), toDate(1410654931)); + +CREATE TABLE test_hybrid_issue_1424_right +( + id Int32, + value Int32, + date_col Date, + computed ALIAS value * 2 +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(date_col) +ORDER BY (date_col, id); + +INSERT INTO test_hybrid_issue_1424_right VALUES + (toInt32(2147483647), toInt32(2147483647), toDate('2149-06-06')), + (toInt32(-2147483648), toInt32(-2147483648), toDate('1970-01-01')), + (toInt32(0), toInt32(0), '1970-01-01'), + (toInt32(1084637461), toInt32(1708739853), toDate(1335613783)), + (toInt32(-221724287), toInt32(339886211), toDate(1294089763)), + (toInt32(-1762862292), toInt32(-287306889), toDate(1375707465)), + (toInt32(1169291374), toInt32(-1541024731), toDate(1082126480)), + (toInt32(-1329695183), toInt32(-786819168), toDate(1226000164)), + (toInt32(1899628504), toInt32(-370080625), toDate(1179050966)), + (toInt32(550067609), toInt32(-1524000367), toDate(1410654931)); + +CREATE TABLE test_hybrid_issue_1424 +( + id Int32, + value Int32, + date_col Date, + computed Int64 +) +ENGINE = Hybrid( + remote('127.0.0.1:9000', currentDatabase(), 'test_hybrid_issue_1424_left'), date_col >= '2025-01-15', + remote('127.0.0.1:9000', currentDatabase(), 'test_hybrid_issue_1424_right'), date_col < '2025-01-15' +); + +SELECT 'max in subquery'; +SELECT max_computed FROM (SELECT max(computed) AS max_computed FROM test_hybrid_issue_1424); + +SELECT 'sum in subquery'; +SELECT sum_computed FROM (SELECT sum(computed) AS sum_computed FROM test_hybrid_issue_1424); + +SELECT 'cte min with predicate'; +WITH cte AS +( + SELECT min(computed) AS min_computed + FROM test_hybrid_issue_1424 + WHERE computed > 50 +) +SELECT * FROM cte; + +SELECT 'cte with limit'; +WITH ranked AS +( + SELECT id, computed + FROM test_hybrid_issue_1424 + LIMIT 10 +) +SELECT * +FROM ranked +ORDER BY id ASC; + +SELECT 'cte without limit'; +WITH ranked AS +( + SELECT id, computed + FROM test_hybrid_issue_1424 +) +SELECT * +FROM ranked +ORDER BY id ASC; + +SELECT 'group by in subquery'; +WITH monthly AS +( + SELECT count() AS cnt + FROM test_hybrid_issue_1424 + GROUP BY computed +) +SELECT sum(cnt), count() FROM monthly; + +SELECT 'intersect with order by'; +SELECT * +FROM +( + SELECT id, computed + FROM test_hybrid_issue_1424 + WHERE computed > 100 + INTERSECT + SELECT id, computed + FROM test_hybrid_issue_1424 + WHERE value > 50 +) +ORDER BY id; + +SELECT 'intersect without order by'; +SELECT * +FROM +( + SELECT id, computed + FROM test_hybrid_issue_1424 + WHERE computed > 100 + INTERSECT + SELECT id, computed + FROM test_hybrid_issue_1424 + WHERE value > 50 +) +ORDER BY id; + +CREATE TABLE test_hybrid_issue_1424_const_left +( + id Int32, + value Int32, + date_col Date, + computed ALIAS toInt64(7) +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(date_col) +ORDER BY (date_col, id); + +INSERT INTO test_hybrid_issue_1424_const_left VALUES + (1, 1, toDate('2025-01-15')), + (2, 2, toDate('2025-02-01')); + +CREATE TABLE test_hybrid_issue_1424_const_right +( + id Int32, + value Int32, + date_col Date, + computed ALIAS toInt64(9) +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(date_col) +ORDER BY (date_col, id); + +INSERT INTO test_hybrid_issue_1424_const_right VALUES + (3, 3, toDate('2024-12-31')), + (4, 4, toDate('2020-01-01')); + +CREATE TABLE test_hybrid_issue_1424_const +( + id Int32, + value Int32, + date_col Date, + computed Int64 +) +ENGINE = Hybrid( + remote('127.0.0.1:9000', currentDatabase(), 'test_hybrid_issue_1424_const_left'), date_col >= '2025-01-15', + remote('127.0.0.1:9000', currentDatabase(), 'test_hybrid_issue_1424_const_right'), date_col < '2025-01-15' +); + +SELECT 'constant alias in subquery'; +SELECT max_computed, min_computed, sum_computed +FROM +( + SELECT + max(computed) AS max_computed, + min(computed) AS min_computed, + sum(computed) AS sum_computed + FROM test_hybrid_issue_1424_const +); + +SELECT 'constant alias predicate'; +SELECT count() FROM test_hybrid_issue_1424_const WHERE computed = 9; + +DROP TABLE test_hybrid_issue_1424; +DROP TABLE test_hybrid_issue_1424_left; +DROP TABLE test_hybrid_issue_1424_right; +DROP TABLE test_hybrid_issue_1424_const; +DROP TABLE test_hybrid_issue_1424_const_left; +DROP TABLE test_hybrid_issue_1424_const_right; diff --git a/tests/queries/0_stateless/03844_distributed_nested_alias_marker.reference b/tests/queries/0_stateless/03844_distributed_nested_alias_marker.reference new file mode 100644 index 000000000000..7b05cb1e81a0 --- /dev/null +++ b/tests/queries/0_stateless/03844_distributed_nested_alias_marker.reference @@ -0,0 +1,4 @@ +analyzer +x x +legacy +x x diff --git a/tests/queries/0_stateless/03844_distributed_nested_alias_marker.sql b/tests/queries/0_stateless/03844_distributed_nested_alias_marker.sql new file mode 100644 index 000000000000..b725acf38949 --- /dev/null +++ b/tests/queries/0_stateless/03844_distributed_nested_alias_marker.sql @@ -0,0 +1,34 @@ +DROP TABLE IF EXISTS test_nested_alias_dist; +DROP TABLE IF EXISTS test_nested_alias_local; + +CREATE TABLE test_nested_alias_local +( + dt DateTime64(3), + base String, + a String ALIAS base, + b String ALIAS a +) +ENGINE = MergeTree() +ORDER BY dt; + +INSERT INTO test_nested_alias_local VALUES ('1999-03-29T01:15:33', 'x'); + +CREATE TABLE test_nested_alias_dist AS test_nested_alias_local +ENGINE = Distributed('test_shard_localhost', currentDatabase(), test_nested_alias_local, rand()); + +SELECT 'analyzer'; +SELECT a, b +FROM test_nested_alias_dist +ORDER BY dt +LIMIT 1 +SETTINGS enable_analyzer = 1; + +SELECT 'legacy'; +SELECT a, b +FROM test_nested_alias_dist +ORDER BY dt +LIMIT 1 +SETTINGS enable_analyzer = 0; + +DROP TABLE test_nested_alias_dist; +DROP TABLE test_nested_alias_local; diff --git a/tests/queries/0_stateless/03845_distributed_global_in_join_alias_chain.reference b/tests/queries/0_stateless/03845_distributed_global_in_join_alias_chain.reference new file mode 100644 index 000000000000..325078d71cc1 --- /dev/null +++ b/tests/queries/0_stateless/03845_distributed_global_in_join_alias_chain.reference @@ -0,0 +1,8 @@ +rewrite_in +1 +1 +rewrite_join +1 +1 +1 +1 diff --git a/tests/queries/0_stateless/03845_distributed_global_in_join_alias_chain.sql b/tests/queries/0_stateless/03845_distributed_global_in_join_alias_chain.sql new file mode 100644 index 000000000000..9bd95d72fd20 --- /dev/null +++ b/tests/queries/0_stateless/03845_distributed_global_in_join_alias_chain.sql @@ -0,0 +1,34 @@ +DROP TABLE IF EXISTS test_global_alias_chain_dist; +DROP TABLE IF EXISTS test_global_alias_chain_local; + +CREATE TABLE test_global_alias_chain_local +( + id UInt64, + base UInt64, + a UInt64 ALIAS base, + b UInt64 ALIAS a +) +ENGINE = MergeTree() +ORDER BY id; + +INSERT INTO test_global_alias_chain_local VALUES (1, 1); + +CREATE TABLE test_global_alias_chain_dist AS test_global_alias_chain_local +ENGINE = Distributed('test_cluster_two_shards', currentDatabase(), test_global_alias_chain_local, rand()); + +SELECT 'rewrite_in'; +SELECT id +FROM test_global_alias_chain_dist +WHERE id IN (SELECT b FROM test_global_alias_chain_dist) +ORDER BY id +SETTINGS enable_analyzer = 1, distributed_product_mode = 'global'; + +SELECT 'rewrite_join'; +SELECT l.id +FROM test_global_alias_chain_dist AS l +INNER JOIN (SELECT b FROM test_global_alias_chain_dist) AS r ON l.id = r.b +ORDER BY l.id +SETTINGS enable_analyzer = 1, distributed_product_mode = 'global'; + +DROP TABLE test_global_alias_chain_dist; +DROP TABLE test_global_alias_chain_local; diff --git a/tests/queries/0_stateless/03846_distributed_global_in_alias_marker_collision.reference b/tests/queries/0_stateless/03846_distributed_global_in_alias_marker_collision.reference new file mode 100644 index 000000000000..9a3a29a69ce8 --- /dev/null +++ b/tests/queries/0_stateless/03846_distributed_global_in_alias_marker_collision.reference @@ -0,0 +1,2 @@ +global_in_collision_check +1 diff --git a/tests/queries/0_stateless/03846_distributed_global_in_alias_marker_collision.sql b/tests/queries/0_stateless/03846_distributed_global_in_alias_marker_collision.sql new file mode 100644 index 000000000000..d47e6a304ba1 --- /dev/null +++ b/tests/queries/0_stateless/03846_distributed_global_in_alias_marker_collision.sql @@ -0,0 +1,56 @@ +DROP TABLE IF EXISTS test_marker_collision_dist; +DROP TABLE IF EXISTS test_marker_collision_main; +DROP TABLE IF EXISTS test_marker_collision_left; +DROP TABLE IF EXISTS test_marker_collision_right; + +CREATE TABLE test_marker_collision_main +( + id UInt64 +) +ENGINE = MergeTree() +ORDER BY id; + +INSERT INTO test_marker_collision_main VALUES (1); + +CREATE TABLE test_marker_collision_left +( + id UInt64, + x UInt64, + b UInt64 ALIAS x +) +ENGINE = MergeTree() +ORDER BY id; + +CREATE TABLE test_marker_collision_right +( + id UInt64, + y UInt64, + b UInt64 ALIAS y +) +ENGINE = MergeTree() +ORDER BY id; + +INSERT INTO test_marker_collision_left VALUES (1, 1); +INSERT INTO test_marker_collision_right VALUES (1, 20); + +CREATE TABLE test_marker_collision_dist AS test_marker_collision_main +ENGINE = Distributed('test_shard_localhost', currentDatabase(), test_marker_collision_main, rand()); + +SELECT 'global_in_collision_check'; +SELECT id +FROM test_marker_collision_dist +WHERE id GLOBAL IN +( + SELECT test_marker_collision_left.id + FROM test_marker_collision_left + INNER JOIN test_marker_collision_right + ON test_marker_collision_left.id = test_marker_collision_right.id + WHERE test_marker_collision_left.b + test_marker_collision_right.b = 21 +) +ORDER BY id +SETTINGS enable_analyzer = 1, enable_alias_marker = 1; + +DROP TABLE test_marker_collision_dist; +DROP TABLE test_marker_collision_main; +DROP TABLE test_marker_collision_left; +DROP TABLE test_marker_collision_right; diff --git a/tests/queries/0_stateless/03847_parallel_replicas_second_hop_alias_marker.reference b/tests/queries/0_stateless/03847_parallel_replicas_second_hop_alias_marker.reference new file mode 100644 index 000000000000..fbdae0d35623 --- /dev/null +++ b/tests/queries/0_stateless/03847_parallel_replicas_second_hop_alias_marker.reference @@ -0,0 +1,4 @@ +single_replica_second_hop +1999-03-29 01:15:33.000 x x +parallel_replicas_second_hop +1999-03-29 01:15:33.000 x x diff --git a/tests/queries/0_stateless/03847_parallel_replicas_second_hop_alias_marker.sql b/tests/queries/0_stateless/03847_parallel_replicas_second_hop_alias_marker.sql new file mode 100644 index 000000000000..5500c4904f82 --- /dev/null +++ b/tests/queries/0_stateless/03847_parallel_replicas_second_hop_alias_marker.sql @@ -0,0 +1,51 @@ +-- Regression coverage for materialized __aliasMarker metadata across +-- remote -> Distributed -> parallel replicas fanout. + +DROP TABLE IF EXISTS test_alias_pr_second_hop_dist; +DROP TABLE IF EXISTS test_alias_pr_second_hop_local; + +CREATE TABLE test_alias_pr_second_hop_local +( + dt DateTime64(3), + base String, + alias_base_0 String ALIAS base, + alias_base_1 String ALIAS base +) +ENGINE = MergeTree() +ORDER BY dt; + +INSERT INTO test_alias_pr_second_hop_local VALUES + ('1999-03-29T01:15:33', 'x'), + ('1999-03-29T01:15:34', 'y'); + +CREATE TABLE test_alias_pr_second_hop_dist AS test_alias_pr_second_hop_local +ENGINE = Distributed(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), test_alias_pr_second_hop_local); + +SELECT 'single_replica_second_hop'; +SELECT dt, alias_base_0, alias_base_1 +FROM remote('127.0.0.2', currentDatabase(), test_alias_pr_second_hop_dist) +ORDER BY dt +LIMIT 1 +SETTINGS + enable_analyzer = 1, + enable_alias_marker = 1, + enable_parallel_replicas = 1, + max_parallel_replicas = 1, + cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', + parallel_replicas_for_non_replicated_merge_tree = 1; + +SELECT 'parallel_replicas_second_hop'; +SELECT dt, alias_base_0, alias_base_1 +FROM remote('127.0.0.2', currentDatabase(), test_alias_pr_second_hop_dist) +ORDER BY dt +LIMIT 1 +SETTINGS + enable_analyzer = 1, + enable_alias_marker = 1, + enable_parallel_replicas = 1, + max_parallel_replicas = 3, + cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', + parallel_replicas_for_non_replicated_merge_tree = 1; + +DROP TABLE test_alias_pr_second_hop_dist; +DROP TABLE test_alias_pr_second_hop_local; diff --git a/tests/queries/0_stateless/03931_parallel_replicas_alias_swap.reference b/tests/queries/0_stateless/03931_parallel_replicas_alias_swap.reference new file mode 100644 index 000000000000..14f9c770f714 --- /dev/null +++ b/tests/queries/0_stateless/03931_parallel_replicas_alias_swap.reference @@ -0,0 +1,9 @@ +local +10 11 12 +20 21 22 +pr_ast +10 11 12 +20 21 22 +pr_plan +10 11 12 +20 21 22 diff --git a/tests/queries/0_stateless/03931_parallel_replicas_alias_swap.sql b/tests/queries/0_stateless/03931_parallel_replicas_alias_swap.sql new file mode 100644 index 000000000000..f669631889c2 --- /dev/null +++ b/tests/queries/0_stateless/03931_parallel_replicas_alias_swap.sql @@ -0,0 +1,37 @@ +-- Plain Distributed + parallel replicas (no Hybrid). Exercises the findParallelReplicasQuery +-- header reconciliation path with nested ALIAS columns. Correct result equals the single-node +-- ('local') result for both AST and serialized-plan transport. +-- +-- Determinism note: parallel replicas over a small non-replicated table can read the same rows on +-- several replicas under some (randomized) settings, duplicating output. GROUP BY x, a1, a2 +-- deduplicates that and keeps x in the required columns for the ALIAS expansion; ORDER BY x over +-- distinct values gives a total order. The test still fails if a1/a2 are swapped or wrong. +DROP TABLE IF EXISTS t_local_03931; +DROP TABLE IF EXISTS t_dist_03931; + +CREATE TABLE t_local_03931 (x UInt32, a1 UInt32 ALIAS x + 1, a2 UInt32 ALIAS a1 + 1) +ENGINE = MergeTree ORDER BY x; +INSERT INTO t_local_03931 VALUES (10), (20); + +CREATE TABLE t_dist_03931 AS t_local_03931 +ENGINE = Distributed(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), t_local_03931); + +SELECT 'local'; +SELECT x, a1, a2 FROM t_local_03931 GROUP BY x, a1, a2 ORDER BY x; + +SELECT 'pr_ast'; +SELECT x, a1, a2 FROM t_dist_03931 GROUP BY x, a1, a2 ORDER BY x +SETTINGS enable_analyzer = 1, enable_alias_marker = 1, + allow_experimental_parallel_reading_from_replicas = 1, max_parallel_replicas = 3, + cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', + serialize_query_plan = 0; + +SELECT 'pr_plan'; +SELECT x, a1, a2 FROM t_dist_03931 GROUP BY x, a1, a2 ORDER BY x +SETTINGS enable_analyzer = 1, enable_alias_marker = 1, + allow_experimental_parallel_reading_from_replicas = 1, max_parallel_replicas = 3, + cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', + serialize_query_plan = 1; + +DROP TABLE t_dist_03931; +DROP TABLE t_local_03931; diff --git a/tests/queries/0_stateless/03932_distributed_alias_strict_name.reference b/tests/queries/0_stateless/03932_distributed_alias_strict_name.reference new file mode 100644 index 000000000000..cddf594d4e31 --- /dev/null +++ b/tests/queries/0_stateless/03932_distributed_alias_strict_name.reference @@ -0,0 +1,9 @@ +local +12 11 23 +22 21 43 +dist +12 11 23 +22 21 43 +dist_plan +12 11 23 +22 21 43 diff --git a/tests/queries/0_stateless/03932_distributed_alias_strict_name.sql b/tests/queries/0_stateless/03932_distributed_alias_strict_name.sql new file mode 100644 index 000000000000..c094d28f01e0 --- /dev/null +++ b/tests/queries/0_stateless/03932_distributed_alias_strict_name.sql @@ -0,0 +1,27 @@ +-- Plain Distributed (no Hybrid). Reorders alias columns and mixes a computed expression over +-- them. With strict name-based header reconciliation (positional fallback disabled), the result +-- must equal the single-node ('local') result for both AST and serialized-plan transport, and no +-- LOGICAL_ERROR must be raised. +DROP TABLE IF EXISTS t_local_03932; +DROP TABLE IF EXISTS t_dist_03932; + +CREATE TABLE t_local_03932 (x UInt32, a1 UInt32 ALIAS x + 1, a2 UInt32 ALIAS a1 + 1) +ENGINE = MergeTree ORDER BY x; +INSERT INTO t_local_03932 VALUES (10), (20); + +CREATE TABLE t_dist_03932 AS t_local_03932 +ENGINE = Distributed(test_shard_localhost, currentDatabase(), t_local_03932); + +SELECT 'local'; +SELECT a2, a1, a1 + a2 AS s FROM t_local_03932 ORDER BY x; + +SELECT 'dist'; +SELECT a2, a1, a1 + a2 AS s FROM t_dist_03932 ORDER BY x +SETTINGS enable_analyzer = 1, enable_alias_marker = 1, prefer_localhost_replica = 0; + +SELECT 'dist_plan'; +SELECT a2, a1, a1 + a2 AS s FROM t_dist_03932 ORDER BY x +SETTINGS enable_analyzer = 1, enable_alias_marker = 1, prefer_localhost_replica = 0, serialize_query_plan = 1; + +DROP TABLE t_dist_03932; +DROP TABLE t_local_03932; diff --git a/tests/queries/0_stateless/03933_alias_marker_direct_use_no_logical_error.reference b/tests/queries/0_stateless/03933_alias_marker_direct_use_no_logical_error.reference new file mode 100644 index 000000000000..f3f736b7dea2 --- /dev/null +++ b/tests/queries/0_stateless/03933_alias_marker_direct_use_no_logical_error.reference @@ -0,0 +1,14 @@ +2arg_identity +42 +lambda_local +[1] +[2] +[3] +lambda_over_distributed +[1] +[2] +[3] +lambda_over_distributed_plan +[1] +[2] +[3] diff --git a/tests/queries/0_stateless/03933_alias_marker_direct_use_no_logical_error.sql b/tests/queries/0_stateless/03933_alias_marker_direct_use_no_logical_error.sql new file mode 100644 index 000000000000..e327c442397d --- /dev/null +++ b/tests/queries/0_stateless/03933_alias_marker_direct_use_no_logical_error.sql @@ -0,0 +1,31 @@ +-- __aliasMarker is an internal pass-through identity function. Direct use from SQL must not +-- raise a server-side LOGICAL_ERROR (which would abort under abort_on_logical_error / sanitizers), +-- in particular inside a lambda over a Distributed table where the marker's column argument +-- resolves to a lambda parameter with no table source. +DROP TABLE IF EXISTS t_local_03933; +DROP TABLE IF EXISTS t_dist_03933; + +CREATE TABLE t_local_03933 (x UInt64) ENGINE = MergeTree ORDER BY x; +INSERT INTO t_local_03933 VALUES (1), (2), (3); + +CREATE TABLE t_dist_03933 AS t_local_03933 +ENGINE = Distributed(test_shard_localhost, currentDatabase(), t_local_03933); + +SELECT '2arg_identity'; +SELECT __aliasMarker(42, 'anything'); + +SELECT 'lambda_local'; +SELECT arrayMap(lx -> __aliasMarker(lx, lx), [x]) AS arr FROM t_local_03933 ORDER BY x; + +SELECT 'lambda_over_distributed'; +SELECT arrayMap(lx -> __aliasMarker(lx, lx), [x]) AS arr +FROM t_dist_03933 ORDER BY x +SETTINGS enable_analyzer = 1, enable_alias_marker = 1, prefer_localhost_replica = 0; + +SELECT 'lambda_over_distributed_plan'; +SELECT arrayMap(lx -> __aliasMarker(lx, lx), [x]) AS arr +FROM t_dist_03933 ORDER BY x +SETTINGS enable_analyzer = 1, enable_alias_marker = 1, prefer_localhost_replica = 0, serialize_query_plan = 1; + +DROP TABLE t_dist_03933; +DROP TABLE t_local_03933; diff --git a/tests/queries/0_stateless/03934_distributed_alias_marker_setting_effect.reference b/tests/queries/0_stateless/03934_distributed_alias_marker_setting_effect.reference new file mode 100644 index 000000000000..d015dccfb64f --- /dev/null +++ b/tests/queries/0_stateless/03934_distributed_alias_marker_setting_effect.reference @@ -0,0 +1,28 @@ +marker_on +x a_str inner_c +1 aaaa 2 +1 aaaa 2 +1 aaaa 2 +1 aaaa 2 +2 aaaa 3 +2 aaaa 3 +2 aaaa 3 +2 aaaa 3 +10 aaaa 11 +10 aaaa 11 +10 aaaa 11 +10 aaaa 11 +marker_off +x a_str inner_c +1 aaaa 2 +1 aaaa 2 +1 aaaa 2 +1 aaaa 2 +2 aaaa 3 +2 aaaa 3 +2 aaaa 3 +2 aaaa 3 +10 aaaa 11 +10 aaaa 11 +10 aaaa 11 +10 aaaa 11 diff --git a/tests/queries/0_stateless/03934_distributed_alias_marker_setting_effect.sql b/tests/queries/0_stateless/03934_distributed_alias_marker_setting_effect.sql new file mode 100644 index 000000000000..648c98d7553b --- /dev/null +++ b/tests/queries/0_stateless/03934_distributed_alias_marker_setting_effect.sql @@ -0,0 +1,42 @@ +-- `enable_alias_marker` must not change results. It exists so an initiator can stop emitting +-- `__aliasMarker` for a mixed-version cluster whose shards do not understand it, which is a +-- transport concern, not a correctness one. +-- +-- Distributed-over-distributed with a String ALIAS (`a_str`) and a UInt64 ALIAS (`inner_c`). This +-- shape used to swap the two columns with the marker off, routing the String 'aaaa' into the +-- UInt64 `inner_c` slot and failing with CANNOT_PARSE_TEXT. Upstream fixed the underlying column +-- ordering, so both settings now return the same rows, and this test holds them to that. +DROP TABLE IF EXISTS t_se_local; +DROP TABLE IF EXISTS t_se_inner; +DROP TABLE IF EXISTS t_se_outer; + +CREATE TABLE t_se_local (x UInt64) ENGINE = MergeTree() ORDER BY x; +INSERT INTO t_se_local VALUES (1), (2), (10); + +CREATE TABLE t_se_inner (x UInt64, inner_c UInt64 ALIAS x + 1) +ENGINE = Distributed(test_cluster_two_shards, currentDatabase(), t_se_local); + +CREATE TABLE t_se_outer (x UInt64, inner_c UInt64, a_str String ALIAS 'aaaa') +ENGINE = Distributed(test_cluster_two_shards, currentDatabase(), t_se_inner); + +-- serialize_query_plan is pinned to 0 throughout: this test targets the AST-path alias marker. +-- On the serialized-plan path the header is reconciled by name regardless of the marker, so the +-- marker_off swap below does not occur there; the "distributed plan" CI flavor would otherwise +-- force the plan path on and the marker_off query would succeed instead of erroring. +SELECT 'marker_on'; +SELECT x, a_str, inner_c +FROM t_se_outer +ORDER BY x +SETTINGS enable_analyzer = 1, enable_alias_marker = 1, prefer_localhost_replica = 0, serialize_query_plan = 0 +FORMAT TSVWithNames; + +SELECT 'marker_off'; +SELECT x, a_str, inner_c +FROM t_se_outer +ORDER BY x +SETTINGS enable_analyzer = 1, enable_alias_marker = 0, prefer_localhost_replica = 0, serialize_query_plan = 0 +FORMAT TSVWithNames; + +DROP TABLE t_se_outer; +DROP TABLE t_se_inner; +DROP TABLE t_se_local; diff --git a/tests/queries/0_stateless/04281_storage_merge_over_distributed_alias.reference b/tests/queries/0_stateless/04281_storage_merge_over_distributed_alias.reference new file mode 100644 index 000000000000..f32381f38096 --- /dev/null +++ b/tests/queries/0_stateless/04281_storage_merge_over_distributed_alias.reference @@ -0,0 +1,20 @@ +local +1 2 3 +2 3 4 +10 11 12 +merge_prefer0 +1 2 3 +2 3 4 +10 11 12 +merge_prefer1 +1 2 3 +2 3 4 +10 11 12 +merge_prefer0_plan +1 2 3 +2 3 4 +10 11 12 +merge_prefer1_plan +1 2 3 +2 3 4 +10 11 12 diff --git a/tests/queries/0_stateless/04281_storage_merge_over_distributed_alias.sql b/tests/queries/0_stateless/04281_storage_merge_over_distributed_alias.sql new file mode 100644 index 000000000000..e1ac59428d5e --- /dev/null +++ b/tests/queries/0_stateless/04281_storage_merge_over_distributed_alias.sql @@ -0,0 +1,58 @@ +-- Plain Merge over Distributed over MergeTree without an explicit __aliasMarker call. +-- Nested ALIAS columns (b contains a's subexpression). Reading the alias columns through the +-- Merge table must reconcile the child (Distributed) header by name; a positional reconciliation +-- in StorageMerge::convertAndFilterSourceStream would swap the columns (or fill them with 0). +-- The correct result equals the single-node ('local') result. +-- +-- Determinism notes: `x` is kept in GROUP BY so the ALIAS expansion can resolve it (the alias +-- expressions are defined in terms of x); GROUP BY also deduplicates the rows the two shards +-- produce, and ORDER BY x (distinct values) gives a total order independent of the distributed +-- merge order. So every block - local and the distributed variants - yields the same rows. +DROP TABLE IF EXISTS test_merge_alias_swap_merge; +DROP TABLE IF EXISTS test_merge_alias_swap_dist; +DROP TABLE IF EXISTS test_merge_alias_swap_local; + +CREATE TABLE test_merge_alias_swap_local +( + x UInt64, + a UInt64 ALIAS x + 1, + b UInt64 ALIAS a + 1 +) +ENGINE = MergeTree() +ORDER BY x; + +INSERT INTO test_merge_alias_swap_local VALUES (1), (2), (10); + +CREATE TABLE test_merge_alias_swap_dist AS test_merge_alias_swap_local +ENGINE = Distributed(test_cluster_two_shards, currentDatabase(), test_merge_alias_swap_local); + +CREATE TABLE test_merge_alias_swap_merge +( + x UInt64, + a UInt64, + b UInt64 +) +ENGINE = Merge(currentDatabase(), '^test_merge_alias_swap_dist$'); + +SELECT 'local'; +SELECT x, a, b FROM test_merge_alias_swap_local GROUP BY x, a, b ORDER BY x; + +SELECT 'merge_prefer0'; +SELECT x, a, b FROM test_merge_alias_swap_merge GROUP BY x, a, b ORDER BY x +SETTINGS enable_analyzer = 1, prefer_localhost_replica = 0; + +SELECT 'merge_prefer1'; +SELECT x, a, b FROM test_merge_alias_swap_merge GROUP BY x, a, b ORDER BY x +SETTINGS enable_analyzer = 1, prefer_localhost_replica = 1; + +SELECT 'merge_prefer0_plan'; +SELECT x, a, b FROM test_merge_alias_swap_merge GROUP BY x, a, b ORDER BY x +SETTINGS enable_analyzer = 1, prefer_localhost_replica = 0, serialize_query_plan = 1; + +SELECT 'merge_prefer1_plan'; +SELECT x, a, b FROM test_merge_alias_swap_merge GROUP BY x, a, b ORDER BY x +SETTINGS enable_analyzer = 1, prefer_localhost_replica = 1, serialize_query_plan = 1; + +DROP TABLE test_merge_alias_swap_merge; +DROP TABLE test_merge_alias_swap_dist; +DROP TABLE test_merge_alias_swap_local; diff --git a/tests/queries/0_stateless/05053_distributed_alias_same_expression.reference b/tests/queries/0_stateless/05053_distributed_alias_same_expression.reference new file mode 100644 index 000000000000..8ddb26d0b961 --- /dev/null +++ b/tests/queries/0_stateless/05053_distributed_alias_same_expression.reference @@ -0,0 +1,16 @@ +first +1999-03-29 01:15:33.000 +second +1999-03-29 01:15:33.000 +third +1999-03-29 01:15:33.000 +fourth +1999-03-29 01:15:33.000 +fifth +1999-03-29 01:15:33.000 +sixth +query_alias_0 query_alias_1 + +seventh +alias_String_7_0 alias_String_7_1 + diff --git a/tests/queries/0_stateless/05053_distributed_alias_same_expression.sql b/tests/queries/0_stateless/05053_distributed_alias_same_expression.sql new file mode 100644 index 000000000000..20cf77930d0b --- /dev/null +++ b/tests/queries/0_stateless/05053_distributed_alias_same_expression.sql @@ -0,0 +1,76 @@ +-- Two ALIAS columns over the same expression, read through `remote` with ORDER BY. The shard's +-- ActionsDAG deduplicates the two identical expressions into one output column, so its header is a +-- column short of what the initiator expects. +-- +-- Every variant below must return the same single row. `enable_alias_marker = 0` is covered too: +-- the marker keeps the two columns distinct in transport, but it is not what makes this shape work. +-- `buildShardCollapseFanOut` reconstructs the missing column either way, so turning the marker off +-- must not change the answer. +-- +-- Related issue: https://github.com/ClickHouse/ClickHouse/issues/79916 +-- Fixed upstream by: https://github.com/ClickHouse/ClickHouse/pull/107913 + +DROP TABLE IF EXISTS test_alias_same_expr_remote; + +CREATE TABLE test_alias_same_expr_remote +( + dt DateTime64(3), + String_7 String, + alias_String_7_0 String ALIAS String_7, + alias_String_7_1 String ALIAS String_7 +) +ENGINE = MergeTree() +ORDER BY dt; + +INSERT INTO test_alias_same_expr_remote VALUES ('1999-03-29T01:15:33', ''); + +SELECT 'first'; +SELECT dt, alias_String_7_0, alias_String_7_1 +FROM remote('127.0.0.{1,2}', currentDatabase(), test_alias_same_expr_remote) +LIMIT 1; + +SELECT 'second'; +SELECT dt, alias_String_7_0, alias_String_7_1 +FROM remote('127.0.0.{1,2}', currentDatabase(), test_alias_same_expr_remote) +ORDER BY dt +LIMIT 1 +SETTINGS enable_analyzer = 0; + +SELECT 'third'; +SELECT dt, alias_String_7_0, alias_String_7_1 +FROM remote('127.0.0.{1,2}', currentDatabase(), test_alias_same_expr_remote) +ORDER BY dt +LIMIT 1 +SETTINGS enable_analyzer = 1; + +SELECT 'fourth'; +SELECT dt, alias_String_7_0, alias_String_7_1 +FROM remote('127.0.0.{1,2}', currentDatabase(), test_alias_same_expr_remote) +ORDER BY dt +LIMIT 1 +SETTINGS enable_analyzer = 1, enable_alias_marker = 0; + +SELECT 'fifth'; +SELECT dt, alias_String_7_0, alias_String_7_1 +FROM remote('127.0.0.{1,2}', currentDatabase(), test_alias_same_expr_remote) +ORDER BY dt +LIMIT 1 +SETTINGS enable_analyzer = 1, enable_alias_marker = 1, serialize_query_plan = 1; + +SELECT 'sixth'; +SELECT alias_String_7_0 AS query_alias_0, alias_String_7_1 AS query_alias_1 +FROM remote('127.0.0.{1,2}', currentDatabase(), test_alias_same_expr_remote) +ORDER BY dt +LIMIT 1 +SETTINGS enable_analyzer = 1, enable_alias_marker = 1 +FORMAT TSVWithNames; + +SELECT 'seventh'; +SELECT alias_String_7_0, alias_String_7_1 +FROM remote('127.0.0.{1,2}', currentDatabase(), test_alias_same_expr_remote) +ORDER BY dt +LIMIT 1 +SETTINGS enable_analyzer = 1, enable_alias_marker = 1 +FORMAT TSVWithNames; + +DROP TABLE test_alias_same_expr_remote; diff --git a/tests/queries/0_stateless/05054_distributed_global_alias_marker_matrix.reference b/tests/queries/0_stateless/05054_distributed_global_alias_marker_matrix.reference new file mode 100644 index 000000000000..8c2ebec53ff8 --- /dev/null +++ b/tests/queries/0_stateless/05054_distributed_global_alias_marker_matrix.reference @@ -0,0 +1,35 @@ +case1_global_in_unnamed_identical_derived_subqueries +1 +case2_global_join_unnamed_identical_derived_subqueries +id left_b0 right_b0 +1 10 20 +case3_global_join_unnamed_identical_derived_subqueries_serialize_query_plan +id left_b0 right_b0 +1 10 20 +case4_global_join_unnamed_remote_over_distributed_subqueries +id left_b0 right_b0 +1 10 20 +case5_global_join_unnamed_identical_dual_alias_columns +id left_b0 right_b1 +1 10 20 +case6_local_join_unnamed_identical_derived_subqueries +id left_b0 right_b0 +1 10 20 +case7_local_join_unnamed_identical_derived_subqueries_serialize_query_plan +id left_b0 right_b0 +1 10 20 +case8_global_join_direct_distributed_serialize_query_plan +id b0 b1 +1 10 10 +2 20 20 +case9_global_join_direct_remote_over_distributed_serialize_query_plan +id b0 b1 +1 10 10 +2 20 20 +case10_wrapper_alias_subquery_serialize_query_plan +id left_foo right_foo +1 1 20 +case11_wrapper_constant_alias_subquery_serialize_query_plan +id left_foo right_foo +1 foo foo +2 foo foo diff --git a/tests/queries/0_stateless/05054_distributed_global_alias_marker_matrix.sql b/tests/queries/0_stateless/05054_distributed_global_alias_marker_matrix.sql new file mode 100644 index 000000000000..2ac9a8a65fe3 --- /dev/null +++ b/tests/queries/0_stateless/05054_distributed_global_alias_marker_matrix.sql @@ -0,0 +1,297 @@ +DROP TABLE IF EXISTS test_marker_suite_main_dist; +DROP TABLE IF EXISTS test_marker_suite_side_dist; +DROP TABLE IF EXISTS test_marker_suite_main; +DROP TABLE IF EXISTS test_marker_suite_side; +DROP TABLE IF EXISTS test_wrapper_alias_a_dist; +DROP TABLE IF EXISTS test_wrapper_alias_b_dist; +DROP TABLE IF EXISTS test_wrapper_alias_a_local; +DROP TABLE IF EXISTS test_wrapper_alias_b_local; +DROP TABLE IF EXISTS test_wrapper_const_alias_a_dist; +DROP TABLE IF EXISTS test_wrapper_const_alias_b_dist; +DROP TABLE IF EXISTS test_wrapper_const_alias_a_local; +DROP TABLE IF EXISTS test_wrapper_const_alias_b_local; + +CREATE TABLE test_marker_suite_main +( + id UInt64 +) +ENGINE = MergeTree() +ORDER BY id; + +INSERT INTO test_marker_suite_main VALUES (1), (2); + +CREATE TABLE test_marker_suite_side +( + id UInt64, + x UInt64, + b0 UInt64 ALIAS x, + b1 UInt64 ALIAS x +) +ENGINE = MergeTree() +ORDER BY id; + +INSERT INTO test_marker_suite_side VALUES (1, 10), (2, 20); + +CREATE TABLE test_marker_suite_main_dist AS test_marker_suite_main +ENGINE = Distributed('test_shard_localhost', currentDatabase(), test_marker_suite_main, rand()); + +CREATE TABLE test_marker_suite_side_dist AS test_marker_suite_side +ENGINE = Distributed('test_shard_localhost', currentDatabase(), test_marker_suite_side, rand()); + +SELECT 'case1_global_in_unnamed_identical_derived_subqueries'; +SELECT id +FROM test_marker_suite_main_dist +WHERE id GLOBAL IN +( + SELECT left_id + FROM + (SELECT id AS left_id, b0 AS left_b0 FROM test_marker_suite_side_dist) + INNER JOIN + (SELECT id AS right_id, b0 AS right_b0 FROM test_marker_suite_side_dist) + ON left_id < right_id + WHERE left_b0 + right_b0 = 30 + SETTINGS joined_subquery_requires_alias = 0 +) +ORDER BY id +SETTINGS enable_analyzer = 1, enable_alias_marker = 1; + +SELECT 'case2_global_join_unnamed_identical_derived_subqueries'; +SELECT m.id, j.left_b0, j.right_b0 +FROM test_marker_suite_main_dist AS m +GLOBAL INNER JOIN +( + SELECT left_id AS id, left_b0, right_b0 + FROM + (SELECT id AS left_id, b0 AS left_b0 FROM test_marker_suite_side_dist) + INNER JOIN + (SELECT id AS right_id, b0 AS right_b0 FROM test_marker_suite_side_dist) + ON left_id < right_id + WHERE left_b0 + right_b0 = 30 + SETTINGS joined_subquery_requires_alias = 0 +) AS j +ON m.id = j.id +ORDER BY m.id +SETTINGS enable_analyzer = 1, enable_alias_marker = 1 +FORMAT TSVWithNames; + +SELECT 'case3_global_join_unnamed_identical_derived_subqueries_serialize_query_plan'; +SELECT m.id, j.left_b0, j.right_b0 +FROM test_marker_suite_main_dist AS m +GLOBAL INNER JOIN +( + SELECT left_id AS id, left_b0, right_b0 + FROM + (SELECT id AS left_id, b0 AS left_b0 FROM test_marker_suite_side_dist) + INNER JOIN + (SELECT id AS right_id, b0 AS right_b0 FROM test_marker_suite_side_dist) + ON left_id < right_id + WHERE left_b0 + right_b0 = 30 + SETTINGS joined_subquery_requires_alias = 0 +) AS j +ON m.id = j.id +ORDER BY m.id +SETTINGS enable_analyzer = 1, enable_alias_marker = 1, serialize_query_plan = 1 +FORMAT TSVWithNames; + +SELECT 'case4_global_join_unnamed_remote_over_distributed_subqueries'; +SELECT m.id, j.left_b0, j.right_b0 +FROM test_marker_suite_main_dist AS m +GLOBAL INNER JOIN +( + SELECT left_id AS id, left_b0, right_b0 + FROM + (SELECT id AS left_id, b0 AS left_b0 FROM remote('127.0.0.2', currentDatabase(), test_marker_suite_side_dist)) + INNER JOIN + (SELECT id AS right_id, b0 AS right_b0 FROM remote('127.0.0.2', currentDatabase(), test_marker_suite_side_dist)) + ON left_id < right_id + WHERE left_b0 + right_b0 = 30 + SETTINGS joined_subquery_requires_alias = 0 +) AS j +ON m.id = j.id +ORDER BY m.id +SETTINGS enable_analyzer = 1, enable_alias_marker = 1 +FORMAT TSVWithNames; + +SELECT 'case5_global_join_unnamed_identical_dual_alias_columns'; +SELECT m.id, j.left_b0, j.right_b1 +FROM test_marker_suite_main_dist AS m +GLOBAL INNER JOIN +( + SELECT left_id AS id, left_b0, right_b1 + FROM + (SELECT id AS left_id, b0 AS left_b0 FROM test_marker_suite_side_dist) + INNER JOIN + (SELECT id AS right_id, b1 AS right_b1 FROM test_marker_suite_side_dist) + ON left_id < right_id + WHERE left_b0 + right_b1 = 30 + SETTINGS joined_subquery_requires_alias = 0 +) AS j +ON m.id = j.id +ORDER BY m.id +SETTINGS enable_analyzer = 1, enable_alias_marker = 1 +FORMAT TSVWithNames; + +SELECT 'case6_local_join_unnamed_identical_derived_subqueries'; +SELECT m.id, j.left_b0, j.right_b0 +FROM test_marker_suite_main_dist AS m +INNER JOIN +( + SELECT left_id AS id, left_b0, right_b0 + FROM + (SELECT id AS left_id, b0 AS left_b0 FROM test_marker_suite_side_dist) + INNER JOIN + (SELECT id AS right_id, b0 AS right_b0 FROM test_marker_suite_side_dist) + ON left_id < right_id + WHERE left_b0 + right_b0 = 30 + SETTINGS joined_subquery_requires_alias = 0 +) AS j +ON m.id = j.id +ORDER BY m.id +SETTINGS enable_analyzer = 1, enable_alias_marker = 1, distributed_product_mode = 'local' +FORMAT TSVWithNames; + +SELECT 'case7_local_join_unnamed_identical_derived_subqueries_serialize_query_plan'; +SELECT m.id, j.left_b0, j.right_b0 +FROM test_marker_suite_main_dist AS m +INNER JOIN +( + SELECT left_id AS id, left_b0, right_b0 + FROM + (SELECT id AS left_id, b0 AS left_b0 FROM test_marker_suite_side_dist) + INNER JOIN + (SELECT id AS right_id, b0 AS right_b0 FROM test_marker_suite_side_dist) + ON left_id < right_id + WHERE left_b0 + right_b0 = 30 + SETTINGS joined_subquery_requires_alias = 0 +) AS j +ON m.id = j.id +ORDER BY m.id +SETTINGS enable_analyzer = 1, enable_alias_marker = 1, distributed_product_mode = 'local', serialize_query_plan = 1 +FORMAT TSVWithNames; + +SELECT 'case8_global_join_direct_distributed_serialize_query_plan'; +SELECT m.id, b0, b1 +FROM test_marker_suite_main_dist AS m +GLOBAL INNER JOIN test_marker_suite_side_dist USING (id) +ORDER BY m.id +SETTINGS enable_analyzer = 1, enable_alias_marker = 1, asterisk_include_alias_columns = 1, serialize_query_plan = 1 +FORMAT TSVWithNames; + +SELECT 'case9_global_join_direct_remote_over_distributed_serialize_query_plan'; +SELECT m.id, b0, b1 +FROM test_marker_suite_main_dist AS m +GLOBAL INNER JOIN remote('127.0.0.2', currentDatabase(), test_marker_suite_side_dist) USING (id) +ORDER BY m.id +SETTINGS enable_analyzer = 1, enable_alias_marker = 1, asterisk_include_alias_columns = 1, joined_subquery_requires_alias = 0, serialize_query_plan = 1 +FORMAT TSVWithNames; + +DROP TABLE test_marker_suite_main_dist; +DROP TABLE test_marker_suite_side_dist; +DROP TABLE test_marker_suite_main; +DROP TABLE test_marker_suite_side; + +CREATE TABLE test_wrapper_alias_a_local +( + id UInt64, + x UInt64 +) +ENGINE = MergeTree() +ORDER BY id; + +CREATE TABLE test_wrapper_alias_b_local +( + id UInt64, + x UInt64 +) +ENGINE = MergeTree() +ORDER BY id; + +INSERT INTO test_wrapper_alias_a_local VALUES (1, 1), (2, 20); +INSERT INTO test_wrapper_alias_b_local VALUES (1, 1), (2, 20); + +CREATE TABLE test_wrapper_alias_a_dist +( + id UInt64, + x UInt64, + foo UInt64 ALIAS x +) +ENGINE = Distributed('test_shard_localhost', currentDatabase(), test_wrapper_alias_a_local, rand()); + +CREATE TABLE test_wrapper_alias_b_dist +( + id UInt64, + x UInt64, + foo UInt64 ALIAS x +) +ENGINE = Distributed('test_shard_localhost', currentDatabase(), test_wrapper_alias_b_local, rand()); + +SELECT 'case10_wrapper_alias_subquery_serialize_query_plan'; +SELECT a.id, j.left_foo, j.right_foo +FROM test_wrapper_alias_a_dist AS a +GLOBAL INNER JOIN +( + SELECT l.id, l.foo AS left_foo, r.foo AS right_foo + FROM test_wrapper_alias_a_dist AS l + INNER JOIN test_wrapper_alias_b_dist AS r ON l.id < r.id + WHERE l.foo + r.foo = 21 +) AS j +ON a.id = j.id +ORDER BY a.id +SETTINGS enable_analyzer = 1, enable_alias_marker = 1, serialize_query_plan = 1 +FORMAT TSVWithNames; + +DROP TABLE test_wrapper_alias_a_dist; +DROP TABLE test_wrapper_alias_b_dist; +DROP TABLE test_wrapper_alias_a_local; +DROP TABLE test_wrapper_alias_b_local; + +CREATE TABLE test_wrapper_const_alias_a_local +( + id UInt64 +) +ENGINE = MergeTree() +ORDER BY id; + +CREATE TABLE test_wrapper_const_alias_b_local +( + id UInt64 +) +ENGINE = MergeTree() +ORDER BY id; + +INSERT INTO test_wrapper_const_alias_a_local VALUES (1), (2); +INSERT INTO test_wrapper_const_alias_b_local VALUES (1), (2); + +CREATE TABLE test_wrapper_const_alias_a_dist +( + id UInt64, + foo String ALIAS 'foo' +) +ENGINE = Distributed('test_shard_localhost', currentDatabase(), test_wrapper_const_alias_a_local, rand()); + +CREATE TABLE test_wrapper_const_alias_b_dist +( + id UInt64, + foo String ALIAS 'foo' +) +ENGINE = Distributed('test_shard_localhost', currentDatabase(), test_wrapper_const_alias_b_local, rand()); + +SELECT 'case11_wrapper_constant_alias_subquery_serialize_query_plan'; +SELECT a.id, j.left_foo, j.right_foo +FROM test_wrapper_const_alias_a_dist AS a +GLOBAL INNER JOIN +( + SELECT l.id, l.foo AS left_foo, r.foo AS right_foo + FROM test_wrapper_const_alias_a_dist AS l + INNER JOIN test_wrapper_const_alias_b_dist AS r ON l.id = r.id + WHERE l.foo = 'foo' AND r.foo = 'foo' +) AS j +ON a.id = j.id +ORDER BY a.id +SETTINGS enable_analyzer = 1, enable_alias_marker = 1, serialize_query_plan = 1 +FORMAT TSVWithNames; + +DROP TABLE test_wrapper_const_alias_a_dist; +DROP TABLE test_wrapper_const_alias_b_dist; +DROP TABLE test_wrapper_const_alias_a_local; +DROP TABLE test_wrapper_const_alias_b_local; diff --git a/tests/queries/0_stateless/05055_distributed_over_distributed_double_aliases.reference b/tests/queries/0_stateless/05055_distributed_over_distributed_double_aliases.reference new file mode 100644 index 000000000000..750abc85a605 --- /dev/null +++ b/tests/queries/0_stateless/05055_distributed_over_distributed_double_aliases.reference @@ -0,0 +1,56 @@ +prefer_localhost_replica_0 +x a b c d inner_c inner_d +1 1 1 2 2 2 2 +1 1 1 2 2 2 2 +1 1 1 2 2 2 2 +1 1 1 2 2 2 2 +2 1 1 3 3 3 3 +2 1 1 3 3 3 3 +2 1 1 3 3 3 3 +2 1 1 3 3 3 3 +10 1 1 11 11 11 11 +10 1 1 11 11 11 11 +10 1 1 11 11 11 11 +10 1 1 11 11 11 11 +prefer_localhost_replica_1 +x a b c d inner_c inner_d +1 1 1 2 2 2 2 +1 1 1 2 2 2 2 +1 1 1 2 2 2 2 +1 1 1 2 2 2 2 +2 1 1 3 3 3 3 +2 1 1 3 3 3 3 +2 1 1 3 3 3 3 +2 1 1 3 3 3 3 +10 1 1 11 11 11 11 +10 1 1 11 11 11 11 +10 1 1 11 11 11 11 +10 1 1 11 11 11 11 +prefer_localhost_replica_0_serialize_query_plan_1 +x a b c d inner_c inner_d +1 1 1 2 2 2 2 +1 1 1 2 2 2 2 +1 1 1 2 2 2 2 +1 1 1 2 2 2 2 +2 1 1 3 3 3 3 +2 1 1 3 3 3 3 +2 1 1 3 3 3 3 +2 1 1 3 3 3 3 +10 1 1 11 11 11 11 +10 1 1 11 11 11 11 +10 1 1 11 11 11 11 +10 1 1 11 11 11 11 +prefer_localhost_replica_1_serialize_query_plan_1 +x a b c d inner_c inner_d +1 1 1 2 2 2 2 +1 1 1 2 2 2 2 +1 1 1 2 2 2 2 +1 1 1 2 2 2 2 +2 1 1 3 3 3 3 +2 1 1 3 3 3 3 +2 1 1 3 3 3 3 +2 1 1 3 3 3 3 +10 1 1 11 11 11 11 +10 1 1 11 11 11 11 +10 1 1 11 11 11 11 +10 1 1 11 11 11 11 diff --git a/tests/queries/0_stateless/05055_distributed_over_distributed_double_aliases.sql b/tests/queries/0_stateless/05055_distributed_over_distributed_double_aliases.sql new file mode 100644 index 000000000000..e4bbceb7e6cf --- /dev/null +++ b/tests/queries/0_stateless/05055_distributed_over_distributed_double_aliases.sql @@ -0,0 +1,92 @@ +DROP TABLE IF EXISTS test_dod_double_alias_outer; +DROP TABLE IF EXISTS test_dod_double_alias_inner; +DROP TABLE IF EXISTS test_dod_double_alias_local; + +CREATE TABLE test_dod_double_alias_local +( + x UInt64 +) +ENGINE = MergeTree() +ORDER BY x; + +INSERT INTO test_dod_double_alias_local VALUES (1), (2), (10); + +CREATE TABLE test_dod_double_alias_inner +( + x UInt64, + a UInt64 ALIAS 2, + b UInt64 ALIAS 2, + inner_c UInt64 ALIAS x + 1, + inner_d UInt64 ALIAS x + 1 +) +ENGINE = Distributed(test_cluster_two_shards, currentDatabase(), test_dod_double_alias_local); + +CREATE TABLE test_dod_double_alias_outer +( + x UInt64, + inner_c UInt64, + a UInt64 ALIAS 1, + b UInt64 ALIAS 1, + c UInt64 ALIAS inner_c, + d UInt64 ALIAS inner_c, + inner_d UInt64 +) +ENGINE = Distributed(test_cluster_two_shards, currentDatabase(), test_dod_double_alias_inner); + +SELECT 'prefer_localhost_replica_0'; +SELECT x, a, b, c, d, inner_c, inner_d +FROM test_dod_double_alias_outer +ORDER BY x +SETTINGS + enable_analyzer = 1, + enable_alias_marker = 1, + prefer_localhost_replica = 0, + enable_parallel_replicas = 0, + max_parallel_replicas = 1, + parallel_replicas_local_plan = 0 +FORMAT TSVWithNames; + +SELECT 'prefer_localhost_replica_1'; +SELECT x, a, b, c, d, inner_c, inner_d +FROM test_dod_double_alias_outer +ORDER BY x +SETTINGS + enable_analyzer = 1, + enable_alias_marker = 1, + prefer_localhost_replica = 1, + enable_parallel_replicas = 0, + max_parallel_replicas = 1, + parallel_replicas_local_plan = 0 +FORMAT TSVWithNames; + +SELECT 'prefer_localhost_replica_0_serialize_query_plan_1'; +SELECT x, a, b, c, d, inner_c, inner_d +FROM test_dod_double_alias_outer +ORDER BY x +SETTINGS + enable_analyzer = 1, + enable_alias_marker = 1, + prefer_localhost_replica = 0, + enable_parallel_replicas = 0, + max_parallel_replicas = 1, + parallel_replicas_local_plan = 0, + serialize_query_plan = 1 +FORMAT TSVWithNames; + +SELECT 'prefer_localhost_replica_1_serialize_query_plan_1'; +SELECT x, a, b, c, d, inner_c, inner_d +FROM test_dod_double_alias_outer +ORDER BY x +SETTINGS + enable_analyzer = 1, + enable_alias_marker = 1, + prefer_localhost_replica = 1, + enable_parallel_replicas = 0, + max_parallel_replicas = 1, + parallel_replicas_local_plan = 0, + serialize_query_plan = 1 +FORMAT TSVWithNames; + +DROP TABLE test_dod_double_alias_outer; +DROP TABLE test_dod_double_alias_inner; +DROP TABLE test_dod_double_alias_local; diff --git a/tests/queries/0_stateless/05056_hybrid_unknown_table_issues_1208_1209_1422.reference b/tests/queries/0_stateless/05056_hybrid_unknown_table_issues_1208_1209_1422.reference new file mode 100644 index 000000000000..5155d27310c2 --- /dev/null +++ b/tests/queries/0_stateless/05056_hybrid_unknown_table_issues_1208_1209_1422.reference @@ -0,0 +1,8 @@ +issue_1208_self_in_subquery +5 +issue_1209_join_mode_local +6 +issue_1209_join_mode_allow +6 +issue_1422_hybrid_in_merge_tree_subquery +5 diff --git a/tests/queries/0_stateless/05056_hybrid_unknown_table_issues_1208_1209_1422.sql b/tests/queries/0_stateless/05056_hybrid_unknown_table_issues_1208_1209_1422.sql new file mode 100644 index 000000000000..c5cab11cee86 --- /dev/null +++ b/tests/queries/0_stateless/05056_hybrid_unknown_table_issues_1208_1209_1422.sql @@ -0,0 +1,118 @@ +SET allow_experimental_hybrid_table = 1, + enable_analyzer = 1, + prefer_localhost_replica = 0, + -- AST-path regression test for unknown-table issues #1208/#1209/#1422. Pin + -- serialize_query_plan=0 so the "distributed plan" CI flavor (which forces it on) does not + -- route these hybrid + IN-subquery queries through the plan path, which has a separate, + -- unrelated header-reconciliation gap. + serialize_query_plan = 0; + +DROP TABLE IF EXISTS test_hybrid_issue_1208_1209_1422; +DROP TABLE IF EXISTS test_hybrid_issue_1208_1209_1422_left; +DROP TABLE IF EXISTS test_hybrid_issue_1208_1209_1422_right; +DROP TABLE IF EXISTS test_hybrid_issue_1208_1209_1422_mt; + +CREATE TABLE test_hybrid_issue_1208_1209_1422_left +( + string_col String, + long_col Int64, + date_col Date +) +ENGINE = MergeTree +ORDER BY string_col; + +CREATE TABLE test_hybrid_issue_1208_1209_1422_right +( + string_col String, + long_col Int64, + date_col Date +) +ENGINE = MergeTree +ORDER BY string_col; + +CREATE TABLE test_hybrid_issue_1208_1209_1422_mt +( + string_col String, + long_col Int64, + date_col Date +) +ENGINE = MergeTree +ORDER BY string_col; + +INSERT INTO test_hybrid_issue_1208_1209_1422_left VALUES + ('William', 9044, toDate('2024-01-01')), + ('Oliver', 1654, toDate('2024-01-01')), + ('Frank', 8751, toDate('2024-01-01')); + +INSERT INTO test_hybrid_issue_1208_1209_1422_right VALUES + ('Louis', 1519, toDate('2024-01-02')), + ('Isaac', 3611, toDate('2024-01-02')); + +INSERT INTO test_hybrid_issue_1208_1209_1422_mt +SELECT * FROM test_hybrid_issue_1208_1209_1422_left +UNION ALL +SELECT * FROM test_hybrid_issue_1208_1209_1422_right; + +CREATE TABLE test_hybrid_issue_1208_1209_1422 +( + string_col String, + long_col Int64, + date_col Date +) +ENGINE = Hybrid( + remote('127.0.0.1:9000', currentDatabase(), 'test_hybrid_issue_1208_1209_1422_left'), date_col <= '2024-01-01', + remote('127.0.0.1:9000', currentDatabase(), 'test_hybrid_issue_1208_1209_1422_right'), date_col > '2024-01-01' +); + +SELECT 'issue_1208_self_in_subquery'; +SELECT count() +FROM +( + SELECT string_col + FROM test_hybrid_issue_1208_1209_1422 + WHERE string_col IN + ( + SELECT DISTINCT string_col + FROM test_hybrid_issue_1208_1209_1422 + WHERE long_col > 1500 + ) +); + +SELECT 'issue_1209_join_mode_local'; +SELECT uniqExact(coalesce(h_string_col, m_string_col)) +FROM +( + SELECT h.string_col AS h_string_col, m.string_col AS m_string_col, h.long_col AS hybrid_long, m.long_col AS mt_long + FROM test_hybrid_issue_1208_1209_1422 AS h + FULL OUTER JOIN test_hybrid_issue_1208_1209_1422_mt AS m ON h.string_col = m.string_col + SETTINGS object_storage_cluster_join_mode = 'local' +); + +SELECT 'issue_1209_join_mode_allow'; +SELECT uniqExact(coalesce(h_string_col, m_string_col)) +FROM +( + SELECT h.string_col AS h_string_col, m.string_col AS m_string_col, h.long_col AS hybrid_long, m.long_col AS mt_long + FROM test_hybrid_issue_1208_1209_1422 AS h + FULL OUTER JOIN test_hybrid_issue_1208_1209_1422_mt AS m ON h.string_col = m.string_col + SETTINGS object_storage_cluster_join_mode = 'allow' +); + +SELECT 'issue_1422_hybrid_in_merge_tree_subquery'; +SELECT count() +FROM +( + SELECT string_col + FROM test_hybrid_issue_1208_1209_1422 + WHERE string_col IN + ( + SELECT DISTINCT string_col + FROM test_hybrid_issue_1208_1209_1422_mt + WHERE long_col > 1500 + ) +); + +DROP TABLE test_hybrid_issue_1208_1209_1422; +DROP TABLE test_hybrid_issue_1208_1209_1422_left; +DROP TABLE test_hybrid_issue_1208_1209_1422_right; +DROP TABLE test_hybrid_issue_1208_1209_1422_mt; diff --git a/tests/queries/0_stateless/05057_merge_over_distributed_alias_marker_column_swap.reference b/tests/queries/0_stateless/05057_merge_over_distributed_alias_marker_column_swap.reference new file mode 100644 index 000000000000..f32381f38096 --- /dev/null +++ b/tests/queries/0_stateless/05057_merge_over_distributed_alias_marker_column_swap.reference @@ -0,0 +1,20 @@ +local +1 2 3 +2 3 4 +10 11 12 +merge_prefer0 +1 2 3 +2 3 4 +10 11 12 +merge_prefer1 +1 2 3 +2 3 4 +10 11 12 +merge_prefer0_plan +1 2 3 +2 3 4 +10 11 12 +merge_prefer1_plan +1 2 3 +2 3 4 +10 11 12 diff --git a/tests/queries/0_stateless/05057_merge_over_distributed_alias_marker_column_swap.sql b/tests/queries/0_stateless/05057_merge_over_distributed_alias_marker_column_swap.sql new file mode 100644 index 000000000000..c5817fd07f34 --- /dev/null +++ b/tests/queries/0_stateless/05057_merge_over_distributed_alias_marker_column_swap.sql @@ -0,0 +1,58 @@ +-- Plain Merge over Distributed over MergeTree (no Hybrid, no explicit __aliasMarker). +-- Nested ALIAS columns (b contains a's subexpression). Reading the alias columns through the +-- Merge table must reconcile the child (Distributed) header by name; a positional reconciliation +-- in StorageMerge::convertAndFilterSourceStream would swap the columns (or fill them with 0). +-- The correct result equals the single-node ('local') result. +-- +-- Determinism notes: `x` is kept in GROUP BY so the ALIAS expansion can resolve it (the alias +-- expressions are defined in terms of x); GROUP BY also deduplicates the rows the two shards +-- produce, and ORDER BY x (distinct values) gives a total order independent of the distributed +-- merge order. So every block - local and the distributed variants - yields the same rows. +DROP TABLE IF EXISTS test_merge_alias_swap_merge; +DROP TABLE IF EXISTS test_merge_alias_swap_dist; +DROP TABLE IF EXISTS test_merge_alias_swap_local; + +CREATE TABLE test_merge_alias_swap_local +( + x UInt64, + a UInt64 ALIAS x + 1, + b UInt64 ALIAS a + 1 +) +ENGINE = MergeTree() +ORDER BY x; + +INSERT INTO test_merge_alias_swap_local VALUES (1), (2), (10); + +CREATE TABLE test_merge_alias_swap_dist AS test_merge_alias_swap_local +ENGINE = Distributed(test_cluster_two_shards, currentDatabase(), test_merge_alias_swap_local); + +CREATE TABLE test_merge_alias_swap_merge +( + x UInt64, + a UInt64, + b UInt64 +) +ENGINE = Merge(currentDatabase(), '^test_merge_alias_swap_dist$'); + +SELECT 'local'; +SELECT x, a, b FROM test_merge_alias_swap_local GROUP BY x, a, b ORDER BY x; + +SELECT 'merge_prefer0'; +SELECT x, a, b FROM test_merge_alias_swap_merge GROUP BY x, a, b ORDER BY x +SETTINGS enable_analyzer = 1, enable_alias_marker = 1, prefer_localhost_replica = 0; + +SELECT 'merge_prefer1'; +SELECT x, a, b FROM test_merge_alias_swap_merge GROUP BY x, a, b ORDER BY x +SETTINGS enable_analyzer = 1, enable_alias_marker = 1, prefer_localhost_replica = 1; + +SELECT 'merge_prefer0_plan'; +SELECT x, a, b FROM test_merge_alias_swap_merge GROUP BY x, a, b ORDER BY x +SETTINGS enable_analyzer = 1, enable_alias_marker = 1, prefer_localhost_replica = 0, serialize_query_plan = 1; + +SELECT 'merge_prefer1_plan'; +SELECT x, a, b FROM test_merge_alias_swap_merge GROUP BY x, a, b ORDER BY x +SETTINGS enable_analyzer = 1, enable_alias_marker = 1, prefer_localhost_replica = 1, serialize_query_plan = 1; + +DROP TABLE test_merge_alias_swap_merge; +DROP TABLE test_merge_alias_swap_dist; +DROP TABLE test_merge_alias_swap_local; diff --git a/tests/queries/0_stateless/05058_distributed_alias_swap_planner.reference b/tests/queries/0_stateless/05058_distributed_alias_swap_planner.reference new file mode 100644 index 000000000000..402cc360bae5 --- /dev/null +++ b/tests/queries/0_stateless/05058_distributed_alias_swap_planner.reference @@ -0,0 +1,15 @@ +local +11 12 +21 22 +dist_prefer0 +11 12 +21 22 +dist_prefer1 +11 12 +21 22 +dist_prefer0_plan +11 12 +21 22 +dist_prefer1_plan +11 12 +21 22 diff --git a/tests/queries/0_stateless/05058_distributed_alias_swap_planner.sql b/tests/queries/0_stateless/05058_distributed_alias_swap_planner.sql new file mode 100644 index 000000000000..848f35b0be14 --- /dev/null +++ b/tests/queries/0_stateless/05058_distributed_alias_swap_planner.sql @@ -0,0 +1,34 @@ +-- Plain Distributed (no Hybrid). Two nested ALIAS columns: a2 contains a1's subexpression, +-- so planner CSE may reorder the remote header. Correct result must equal the single-node +-- ('local') result across every transport variant. +DROP TABLE IF EXISTS t_local_03930; +DROP TABLE IF EXISTS t_dist_03930; + +CREATE TABLE t_local_03930 (x UInt32, a1 UInt32 ALIAS x + 1, a2 UInt32 ALIAS a1 + 1) +ENGINE = MergeTree ORDER BY x; +INSERT INTO t_local_03930 VALUES (10), (20); + +CREATE TABLE t_dist_03930 AS t_local_03930 +ENGINE = Distributed(test_shard_localhost, currentDatabase(), t_local_03930); + +SELECT 'local'; +SELECT a1, a2 FROM t_local_03930 ORDER BY a1; + +SELECT 'dist_prefer0'; +SELECT a1, a2 FROM t_dist_03930 ORDER BY a1 +SETTINGS enable_analyzer = 1, enable_alias_marker = 1, prefer_localhost_replica = 0; + +SELECT 'dist_prefer1'; +SELECT a1, a2 FROM t_dist_03930 ORDER BY a1 +SETTINGS enable_analyzer = 1, enable_alias_marker = 1, prefer_localhost_replica = 1; + +SELECT 'dist_prefer0_plan'; +SELECT a1, a2 FROM t_dist_03930 ORDER BY a1 +SETTINGS enable_analyzer = 1, enable_alias_marker = 1, prefer_localhost_replica = 0, serialize_query_plan = 1; + +SELECT 'dist_prefer1_plan'; +SELECT a1, a2 FROM t_dist_03930 ORDER BY a1 +SETTINGS enable_analyzer = 1, enable_alias_marker = 1, prefer_localhost_replica = 1, serialize_query_plan = 1; + +DROP TABLE t_dist_03930; +DROP TABLE t_local_03930; diff --git a/tests/queries/0_stateless/05059_dotted_alias_merge_over_distributed.reference b/tests/queries/0_stateless/05059_dotted_alias_merge_over_distributed.reference new file mode 100644 index 000000000000..97dd73615024 --- /dev/null +++ b/tests/queries/0_stateless/05059_dotted_alias_merge_over_distributed.reference @@ -0,0 +1,9 @@ +local +1 10 100 +2 20 200 +merge_prefer0 +1 10 100 +2 20 200 +merge_prefer1 +1 10 100 +2 20 200 diff --git a/tests/queries/0_stateless/05059_dotted_alias_merge_over_distributed.sql b/tests/queries/0_stateless/05059_dotted_alias_merge_over_distributed.sql new file mode 100644 index 000000000000..c42ad63adab4 --- /dev/null +++ b/tests/queries/0_stateless/05059_dotted_alias_merge_over_distributed.sql @@ -0,0 +1,64 @@ +-- Regression for the StorageMerge alias-output-naming fix. +-- The bug: `Nested::splitName(name, reverse=true)` (used before this fix to strip the +-- analyzer's `__tableN.` prefix from header column names) splits on the LAST dot, so for +-- an analyzer identifier like `__table1.\`n.a\`` (a dotted column name wrapped in backticks +-- by the analyzer) it returns the suffix `a\`` instead of `n.a`, leaving the +-- `logical_name_to_header_name` map with broken keys. The lookup for `alias.name == "n.a"` +-- then misses, the alias output falls back to the bare name `n.a`, and the downstream +-- header-reconciliation step fills the expected `__table1.\`n.a\`` column with type +-- defaults (zeros). Silent wrong data. +-- +-- Repro shape: Merge declares dotted column names explicitly (typical when matching a +-- schema with Nested-style names), underlying storage has those columns as ALIAS, and +-- the Distributed routing forces analyzer-prefixed names in the Merge level. Using a +-- two-shard cluster with prefer_localhost_replica=0 reliably reproduces. + +DROP TABLE IF EXISTS test_04286_dotted_alias_local; +DROP TABLE IF EXISTS test_04286_dotted_alias_dist; +DROP TABLE IF EXISTS test_04286_dotted_alias_merge; + +CREATE TABLE test_04286_dotted_alias_local +( + id UInt32, + `n.a` UInt32 ALIAS id * 10, + `m.b` UInt32 ALIAS id * 100 +) +ENGINE = MergeTree +ORDER BY id; + +INSERT INTO test_04286_dotted_alias_local VALUES (1), (2); + +CREATE TABLE test_04286_dotted_alias_dist AS test_04286_dotted_alias_local +ENGINE = Distributed(test_cluster_two_shards, currentDatabase(), test_04286_dotted_alias_local); + +CREATE TABLE test_04286_dotted_alias_merge +( + id UInt32, + `n.a` UInt32, + `m.b` UInt32 +) +ENGINE = Merge(currentDatabase(), '^test_04286_dotted_alias_dist$'); + +SELECT 'local'; +SELECT id, `n.a`, `m.b` +FROM test_04286_dotted_alias_local +GROUP BY id, `n.a`, `m.b` +ORDER BY id; + +SELECT 'merge_prefer0'; +SELECT id, `n.a`, `m.b` +FROM test_04286_dotted_alias_merge +GROUP BY id, `n.a`, `m.b` +ORDER BY id +SETTINGS prefer_localhost_replica = 0; + +SELECT 'merge_prefer1'; +SELECT id, `n.a`, `m.b` +FROM test_04286_dotted_alias_merge +GROUP BY id, `n.a`, `m.b` +ORDER BY id +SETTINGS prefer_localhost_replica = 1; + +DROP TABLE test_04286_dotted_alias_merge; +DROP TABLE test_04286_dotted_alias_dist; +DROP TABLE test_04286_dotted_alias_local; diff --git a/tests/queries/0_stateless/05060_hybrid_unknown_table_exact_schema.reference b/tests/queries/0_stateless/05060_hybrid_unknown_table_exact_schema.reference new file mode 100644 index 000000000000..286fa88dd6a4 --- /dev/null +++ b/tests/queries/0_stateless/05060_hybrid_unknown_table_exact_schema.reference @@ -0,0 +1,35 @@ +merge_tree_row_count +5 +iceberg_row_count +5 +hybrid_row_count +5 +true 8751 7291.267979503492 Frank 2024-01-01 06:00:00.000000 2024-01-01 43200000000 2024-01-01 12:00:00.000000 5313 8428.52 456.78 +false 3611 4492.090462838536 Isaac 2024-01-01 06:00:00.000000 2024-01-01 43200000000 2024-01-01 12:00:00.000000 4552 1554.795 456.78 +true 1519 3799.273006373374 Louis 2024-01-01 06:00:00.000000 2024-01-01 43200000000 2024-01-01 12:00:00.000000 8785 1248.2616 456.78 +true 1654 3801.2622503916614 Oliver 2024-01-01 06:00:00.000000 2024-01-01 43200000000 2024-01-01 12:00:00.000000 3432 6701.752 456.78 +true 9044 2931.782814070929 William 2024-01-01 06:00:00.000000 2024-01-01 43200000000 2024-01-01 12:00:00.000000 3733 7730.6836 456.78 +issue_1208_join_hybrid_mt_local +Frank 8751 8751 +Isaac 3611 3611 +Louis 1519 1519 +Oliver 1654 1654 +William 9044 9044 +issue_1208_join_hybrid_mt_allow +Frank 8751 8751 +Isaac 3611 3611 +Louis 1519 1519 +Oliver 1654 1654 +William 9044 9044 +issue_1208_join_hybrid_mt_iceberg_local +Frank 8751 8751 8751 +Isaac 3611 3611 3611 +Louis 1519 1519 1519 +Oliver 1654 1654 1654 +William 9044 9044 9044 +issue_1208_join_hybrid_mt_iceberg_allow +Frank 8751 8751 8751 +Isaac 3611 3611 3611 +Louis 1519 1519 1519 +Oliver 1654 1654 1654 +William 9044 9044 9044 diff --git a/tests/queries/0_stateless/05060_hybrid_unknown_table_exact_schema.sql b/tests/queries/0_stateless/05060_hybrid_unknown_table_exact_schema.sql new file mode 100644 index 000000000000..55d247058b3f --- /dev/null +++ b/tests/queries/0_stateless/05060_hybrid_unknown_table_exact_schema.sql @@ -0,0 +1,339 @@ +-- Hybrid table joined against a MergeTree and an Iceberg table, covering Altinity#1208, #1209 and +-- #1422. Both `object_storage_cluster_join_mode` values must return the same rows: `'local'` used +-- to raise UNKNOWN_IDENTIFIER here, and no longer does, so it is held to the `'allow'` result. +-- +-- Needs the stateless S3 mock on localhost:11111, and a server listening on 127.0.0.3 as well as +-- 127.0.0.1 and 127.0.0.2, because `icebergCluster` below reads through +-- `test_cluster_one_shard_three_replicas_localhost`. Without the third address the query still +-- returns the right rows by failing over, but the connection warnings on stderr fail the test. +-- Run with `enable_parallel_blocks_marshalling = 0` until the DISTINCT-over-ColumnBLOB abort is +-- fixed. +SET allow_experimental_hybrid_table = 1, + enable_analyzer = 1, + prefer_localhost_replica = 0, + iceberg_delete_data_on_drop = 1; + +DROP TABLE IF EXISTS hybrid_table; +DROP TABLE IF EXISTS merge_tree_table_b9faf88a_d5d3_11f0_b816_e0c26496f172; +DROP TABLE IF EXISTS iceberg_table_b4bd039e_d5d3_11f0_8208_e0c26496f172; +DROP TABLE IF EXISTS merge_tree_table_3ef2c546_d5d6_11f0_b816_e0c26496f172; +DROP TABLE IF EXISTS hybrid_table_64293f1a_0cba_11f1_876b_de7b9eea3490; +DROP TABLE IF EXISTS merge_tree_table_640a9b6e_0cba_11f1_876b_de7b9eea3490; +DROP TABLE IF EXISTS database_39afd42b_d5d6_11f0_b919_e0c26496f172.`namespace_39afe1b3_d5d6_11f0_9b00_e0c26496f172.table_39afe20a_d5d6_11f0_8208_e0c26496f172`; +DROP DATABASE IF EXISTS database_39afd42b_d5d6_11f0_b919_e0c26496f172; + +CREATE TABLE merge_tree_table_b9faf88a_d5d3_11f0_b816_e0c26496f172 +( + boolean_col Nullable(Bool), + long_col Nullable(Int64), + double_col Nullable(Float64), + string_col String, + timestamp_col Nullable(DateTime64(6)), + date_col Nullable(Date), + time_col Nullable(Int64), + timestamptz_col Nullable(DateTime64(6, 'UTC')), + integer_col Nullable(Int32), + float_col Nullable(Float32), + decimal_col Nullable(Decimal(10, 2)) +) +ENGINE = MergeTree +PARTITION BY string_col +ORDER BY tuple() +SETTINGS index_granularity = 8192; + +INSERT INTO merge_tree_table_b9faf88a_d5d3_11f0_b816_e0c26496f172 VALUES + (true, 9044, 2931.782814070929, 'William', toDateTime64('2024-01-01 06:00:00', 6), toDate('2024-01-01'), 43200000000, toDateTime64('2024-01-01 12:00:00', 6, 'UTC'), 3733, 7730.6836, 456.78), + (true, 1654, 3801.2622503916614, 'Oliver', toDateTime64('2024-01-01 06:00:00', 6), toDate('2024-01-01'), 43200000000, toDateTime64('2024-01-01 12:00:00', 6, 'UTC'), 3432, 6701.752, 456.78), + (true, 8751, 7291.267979503492, 'Frank', toDateTime64('2024-01-01 06:00:00', 6), toDate('2024-01-01'), 43200000000, toDateTime64('2024-01-01 12:00:00', 6, 'UTC'), 5313, 8428.52, 456.78), + (true, 1519, 3799.273006373374, 'Louis', toDateTime64('2024-01-01 06:00:00', 6), toDate('2024-01-01'), 43200000000, toDateTime64('2024-01-01 12:00:00', 6, 'UTC'), 8785, 1248.2616, 456.78), + (false, 3611, 4492.090462838536, 'Isaac', toDateTime64('2024-01-01 06:00:00', 6), toDate('2024-01-01'), 43200000000, toDateTime64('2024-01-01 12:00:00', 6, 'UTC'), 4552, 1554.795, 456.78); + +SELECT 'merge_tree_row_count'; +SELECT count() FROM merge_tree_table_b9faf88a_d5d3_11f0_b816_e0c26496f172; + +CREATE TABLE iceberg_table_b4bd039e_d5d3_11f0_8208_e0c26496f172 +( + boolean_col Nullable(Int32), + long_col Nullable(Int64), + double_col Nullable(Float64), + string_col String, + timestamp_col Nullable(DateTime64(6)), + date_col Nullable(Date), + time_col Nullable(Int64), + timestamptz_col Nullable(DateTime64(6, 'UTC')), + integer_col Nullable(Int32), + float_col Nullable(Float32), + decimal_col Nullable(Float64) +) +ENGINE = IcebergS3( + s3_conn, + filename = concat('hybrid_unknown_table_exact_schema_03924/', currentDatabase(), '/iceberg_table') +); + +INSERT INTO iceberg_table_b4bd039e_d5d3_11f0_8208_e0c26496f172 SETTINGS allow_experimental_insert_into_iceberg = 1, write_full_path_in_iceberg_metadata = 1 VALUES + (1, 9044, 2931.782814070929, 'William', toDateTime64('2024-01-01 06:00:00', 6), toDate('2024-01-01'), 43200000000, toDateTime64('2024-01-01 12:00:00', 6, 'UTC'), 3733, 7730.6836, 456.78), + (1, 1654, 3801.2622503916614, 'Oliver', toDateTime64('2024-01-01 06:00:00', 6), toDate('2024-01-01'), 43200000000, toDateTime64('2024-01-01 12:00:00', 6, 'UTC'), 3432, 6701.752, 456.78), + (1, 8751, 7291.267979503492, 'Frank', toDateTime64('2024-01-01 06:00:00', 6), toDate('2024-01-01'), 43200000000, toDateTime64('2024-01-01 12:00:00', 6, 'UTC'), 5313, 8428.52, 456.78), + (1, 1519, 3799.273006373374, 'Louis', toDateTime64('2024-01-01 06:00:00', 6), toDate('2024-01-01'), 43200000000, toDateTime64('2024-01-01 12:00:00', 6, 'UTC'), 8785, 1248.2616, 456.78), + (0, 3611, 4492.090462838536, 'Isaac', toDateTime64('2024-01-01 06:00:00', 6), toDate('2024-01-01'), 43200000000, toDateTime64('2024-01-01 12:00:00', 6, 'UTC'), 4552, 1554.795, 456.78); + +SELECT 'iceberg_row_count'; +SELECT count() FROM iceberg_table_b4bd039e_d5d3_11f0_8208_e0c26496f172; + +CREATE TABLE hybrid_table +( + boolean_col Nullable(Bool), + long_col Nullable(Int64), + double_col Nullable(Float64), + string_col String, + timestamp_col Nullable(DateTime64(6)), + date_col Nullable(Date), + time_col Nullable(Int64), + timestamptz_col Nullable(DateTime64(6, 'UTC')), + integer_col Nullable(Int32), + float_col Nullable(Float32), + decimal_col Nullable(Decimal(10, 2)) +) +ENGINE = Hybrid( + remote('localhost', currentDatabase(), 'merge_tree_table_b9faf88a_d5d3_11f0_b816_e0c26496f172'), + date_col <= '2024-01-01', + icebergCluster( + 'test_cluster_one_shard_three_replicas_localhost', + concat('http://localhost:11111/test/hybrid_unknown_table_exact_schema_03924/', currentDatabase(), '/iceberg_table/'), + 'test', + 'testtest' + ), + date_col > '2024-01-01' +); + +SELECT 'hybrid_row_count'; +SELECT count() FROM hybrid_table; + +CREATE TABLE merge_tree_table_3ef2c546_d5d6_11f0_b816_e0c26496f172 +( + boolean_col Nullable(Bool), + long_col Nullable(Int64), + double_col Nullable(Float64), + string_col String, + timestamp_col Nullable(DateTime64(6)), + date_col Nullable(Date), + time_col Nullable(Int64), + timestamptz_col Nullable(DateTime64(6, 'UTC')), + integer_col Nullable(Int32), + float_col Nullable(Float32), + decimal_col Nullable(Decimal(10, 2)) +) +ENGINE = MergeTree +PARTITION BY string_col +ORDER BY tuple() +SETTINGS index_granularity = 8192; + +INSERT INTO merge_tree_table_3ef2c546_d5d6_11f0_b816_e0c26496f172 +SELECT * FROM merge_tree_table_b9faf88a_d5d3_11f0_b816_e0c26496f172; + +CREATE DATABASE database_39afd42b_d5d6_11f0_b919_e0c26496f172; + +CREATE TABLE database_39afd42b_d5d6_11f0_b919_e0c26496f172.`namespace_39afe1b3_d5d6_11f0_9b00_e0c26496f172.table_39afe20a_d5d6_11f0_8208_e0c26496f172` +( + boolean_col Nullable(Int32), + long_col Nullable(Int64), + double_col Nullable(Float64), + string_col String, + timestamp_col Nullable(DateTime64(6)), + date_col Nullable(Date), + time_col Nullable(Int64), + timestamptz_col Nullable(DateTime64(6, 'UTC')), + integer_col Nullable(Int32), + float_col Nullable(Float32), + decimal_col Nullable(Float64) +) +ENGINE = IcebergS3( + s3_conn, + filename = concat('hybrid_unknown_table_exact_schema_03924/', currentDatabase(), '/iceberg_table_39afe20a_d5d6_11f0_8208_e0c26496f172') +); + +INSERT INTO database_39afd42b_d5d6_11f0_b919_e0c26496f172.`namespace_39afe1b3_d5d6_11f0_9b00_e0c26496f172.table_39afe20a_d5d6_11f0_8208_e0c26496f172` +SETTINGS allow_experimental_insert_into_iceberg = 1, write_full_path_in_iceberg_metadata = 1 +SELECT + toInt32(boolean_col), + long_col, + double_col, + string_col, + timestamp_col, + date_col, + time_col, + timestamptz_col, + integer_col, + float_col, + toFloat64(decimal_col) +FROM merge_tree_table_3ef2c546_d5d6_11f0_b816_e0c26496f172; + +SELECT * +FROM hybrid_table +WHERE string_col IN +( + SELECT DISTINCT string_col + FROM hybrid_table + WHERE long_col > 1500 +) +ORDER BY string_col; + +SELECT 'issue_1208_join_hybrid_mt_local'; +SELECT + h.string_col, + h.long_col AS hybrid_long, + m.long_col AS mt_long +FROM hybrid_table AS h +FULL OUTER JOIN merge_tree_table_3ef2c546_d5d6_11f0_b816_e0c26496f172 AS m ON h.string_col = m.string_col +ORDER BY h.string_col +LIMIT 10 +SETTINGS object_storage_cluster_join_mode = 'local'; + +SELECT 'issue_1208_join_hybrid_mt_allow'; +SELECT + h.string_col, + h.long_col AS hybrid_long, + m.long_col AS mt_long +FROM hybrid_table AS h +FULL OUTER JOIN merge_tree_table_3ef2c546_d5d6_11f0_b816_e0c26496f172 AS m ON h.string_col = m.string_col +ORDER BY h.string_col +LIMIT 10 +SETTINGS object_storage_cluster_join_mode = 'allow'; + +SELECT 'issue_1208_join_hybrid_mt_iceberg_local'; +SELECT + h.string_col, + h.long_col AS hybrid_long, + m.long_col AS mt_long, + i.long_col AS iceberg_long +FROM hybrid_table AS h +FULL OUTER JOIN merge_tree_table_3ef2c546_d5d6_11f0_b816_e0c26496f172 AS m ON h.string_col = m.string_col +FULL OUTER JOIN database_39afd42b_d5d6_11f0_b919_e0c26496f172.`namespace_39afe1b3_d5d6_11f0_9b00_e0c26496f172.table_39afe20a_d5d6_11f0_8208_e0c26496f172` AS i ON h.string_col = i.string_col +ORDER BY h.string_col +LIMIT 10 +SETTINGS object_storage_cluster_join_mode = 'local'; + +SELECT 'issue_1208_join_hybrid_mt_iceberg_allow'; +SELECT + h.string_col, + h.long_col AS hybrid_long, + m.long_col AS mt_long, + i.long_col AS iceberg_long +FROM hybrid_table AS h +FULL OUTER JOIN merge_tree_table_3ef2c546_d5d6_11f0_b816_e0c26496f172 AS m ON h.string_col = m.string_col +FULL OUTER JOIN database_39afd42b_d5d6_11f0_b919_e0c26496f172.`namespace_39afe1b3_d5d6_11f0_9b00_e0c26496f172.table_39afe20a_d5d6_11f0_8208_e0c26496f172` AS i ON h.string_col = i.string_col +ORDER BY h.string_col +LIMIT 10 +SETTINGS object_storage_cluster_join_mode = 'allow'; + +-- Exact issue-shape queries (no ORDER BY), deterministic output via FORMAT Null. +SELECT + h.string_col, + h.long_col AS hybrid_long, + m.long_col AS mt_long +FROM hybrid_table AS h +FULL OUTER JOIN merge_tree_table_3ef2c546_d5d6_11f0_b816_e0c26496f172 AS m ON h.string_col = m.string_col +LIMIT 10 +SETTINGS object_storage_cluster_join_mode = 'local' +FORMAT Null; + +SELECT + h.string_col, + h.long_col AS hybrid_long, + m.long_col AS mt_long +FROM hybrid_table AS h +FULL OUTER JOIN merge_tree_table_3ef2c546_d5d6_11f0_b816_e0c26496f172 AS m ON h.string_col = m.string_col +LIMIT 10 +SETTINGS object_storage_cluster_join_mode = 'allow' +FORMAT Null; + +SELECT + h.string_col, + h.long_col AS hybrid_long, + m.long_col AS mt_long, + i.long_col AS iceberg_long +FROM hybrid_table AS h +FULL OUTER JOIN merge_tree_table_3ef2c546_d5d6_11f0_b816_e0c26496f172 AS m ON h.string_col = m.string_col +FULL OUTER JOIN database_39afd42b_d5d6_11f0_b919_e0c26496f172.`namespace_39afe1b3_d5d6_11f0_9b00_e0c26496f172.table_39afe20a_d5d6_11f0_8208_e0c26496f172` AS i ON h.string_col = i.string_col +LIMIT 10 +SETTINGS object_storage_cluster_join_mode = 'local' +FORMAT Null; + +SELECT + h.string_col, + h.long_col AS hybrid_long, + m.long_col AS mt_long, + i.long_col AS iceberg_long +FROM hybrid_table AS h +FULL OUTER JOIN merge_tree_table_3ef2c546_d5d6_11f0_b816_e0c26496f172 AS m ON h.string_col = m.string_col +FULL OUTER JOIN database_39afd42b_d5d6_11f0_b919_e0c26496f172.`namespace_39afe1b3_d5d6_11f0_9b00_e0c26496f172.table_39afe20a_d5d6_11f0_8208_e0c26496f172` AS i ON h.string_col = i.string_col +LIMIT 10 +SETTINGS object_storage_cluster_join_mode = 'allow' +FORMAT Null; + +CREATE TABLE merge_tree_table_640a9b6e_0cba_11f1_876b_de7b9eea3490 +( + boolean_col Nullable(Bool), + long_col Nullable(Int64), + double_col Nullable(Float64), + string_col String, + timestamp_col Nullable(DateTime64(6)), + date_col Nullable(Date), + time_col Nullable(Int64), + timestamptz_col Nullable(DateTime64(6, 'UTC')), + integer_col Nullable(Int32), + float_col Nullable(Float32), + decimal_col Nullable(Decimal(10, 2)) +) +ENGINE = MergeTree +PARTITION BY string_col +ORDER BY tuple() +SETTINGS index_granularity = 8192; + +INSERT INTO merge_tree_table_640a9b6e_0cba_11f1_876b_de7b9eea3490 +SELECT * FROM merge_tree_table_3ef2c546_d5d6_11f0_b816_e0c26496f172; + +CREATE TABLE hybrid_table_64293f1a_0cba_11f1_876b_de7b9eea3490 +( + boolean_col Nullable(Bool), + long_col Nullable(Int64), + double_col Nullable(Float64), + string_col String, + timestamp_col Nullable(DateTime64(6)), + date_col Nullable(Date), + time_col Nullable(Int64), + timestamptz_col Nullable(DateTime64(6, 'UTC')), + integer_col Nullable(Int32), + float_col Nullable(Float32), + decimal_col Nullable(Decimal(10, 2)) +) +ENGINE = Hybrid( + remote('localhost', currentDatabase(), 'merge_tree_table_b9faf88a_d5d3_11f0_b816_e0c26496f172'), + date_col <= '2024-01-01', + icebergCluster( + 'test_cluster_one_shard_three_replicas_localhost', + concat('http://localhost:11111/test/hybrid_unknown_table_exact_schema_03924/', currentDatabase(), '/iceberg_table/'), + 'test', + 'testtest' + ), + date_col > '2024-01-01' +); + +SELECT * +FROM hybrid_table_64293f1a_0cba_11f1_876b_de7b9eea3490 +WHERE string_col IN +( + SELECT DISTINCT string_col + FROM merge_tree_table_640a9b6e_0cba_11f1_876b_de7b9eea3490 + WHERE long_col > 1500 +) +FORMAT Null; + +DROP TABLE hybrid_table; +DROP TABLE merge_tree_table_b9faf88a_d5d3_11f0_b816_e0c26496f172; +DROP TABLE iceberg_table_b4bd039e_d5d3_11f0_8208_e0c26496f172; +DROP TABLE merge_tree_table_3ef2c546_d5d6_11f0_b816_e0c26496f172; +DROP TABLE hybrid_table_64293f1a_0cba_11f1_876b_de7b9eea3490; +DROP TABLE merge_tree_table_640a9b6e_0cba_11f1_876b_de7b9eea3490; +DROP TABLE database_39afd42b_d5d6_11f0_b919_e0c26496f172.`namespace_39afe1b3_d5d6_11f0_9b00_e0c26496f172.table_39afe20a_d5d6_11f0_8208_e0c26496f172`; +DROP DATABASE database_39afd42b_d5d6_11f0_b919_e0c26496f172; diff --git a/tests/queries/0_stateless/05061_distributed_alias_column_swap_without_marker.reference b/tests/queries/0_stateless/05061_distributed_alias_column_swap_without_marker.reference new file mode 100644 index 000000000000..1e2e9b11750a --- /dev/null +++ b/tests/queries/0_stateless/05061_distributed_alias_column_swap_without_marker.reference @@ -0,0 +1,56 @@ +prefer_localhost_replica_0_uint64 +x a_num inner_c +1 1 2 +1 1 2 +1 1 2 +1 1 2 +2 1 3 +2 1 3 +2 1 3 +2 1 3 +10 1 11 +10 1 11 +10 1 11 +10 1 11 +prefer_localhost_replica_0_string +x a_str inner_c +1 aaaa 2 +1 aaaa 2 +1 aaaa 2 +1 aaaa 2 +2 aaaa 3 +2 aaaa 3 +2 aaaa 3 +2 aaaa 3 +10 aaaa 11 +10 aaaa 11 +10 aaaa 11 +10 aaaa 11 +prefer_localhost_replica_1_uint64 +x a_num inner_c +1 1 2 +1 1 2 +1 1 2 +1 1 2 +2 1 3 +2 1 3 +2 1 3 +2 1 3 +10 1 11 +10 1 11 +10 1 11 +10 1 11 +prefer_localhost_replica_1_string +x a_str inner_c +1 aaaa 2 +1 aaaa 2 +1 aaaa 2 +1 aaaa 2 +2 aaaa 3 +2 aaaa 3 +2 aaaa 3 +2 aaaa 3 +10 aaaa 11 +10 aaaa 11 +10 aaaa 11 +10 aaaa 11 diff --git a/tests/queries/0_stateless/05061_distributed_alias_column_swap_without_marker.sql b/tests/queries/0_stateless/05061_distributed_alias_column_swap_without_marker.sql new file mode 100644 index 000000000000..84ad3cf170d0 --- /dev/null +++ b/tests/queries/0_stateless/05061_distributed_alias_column_swap_without_marker.sql @@ -0,0 +1,96 @@ +DROP TABLE IF EXISTS test_dod_alias_swap_no_marker_outer; +DROP TABLE IF EXISTS test_dod_alias_swap_no_marker_inner; +DROP TABLE IF EXISTS test_dod_alias_swap_no_marker_local; + +CREATE TABLE test_dod_alias_swap_no_marker_local +( + x UInt64 +) +ENGINE = MergeTree() +ORDER BY x; + +INSERT INTO test_dod_alias_swap_no_marker_local VALUES (1), (2), (10); + +CREATE TABLE test_dod_alias_swap_no_marker_inner +( + x UInt64, + inner_c UInt64 ALIAS x + 1 +) +ENGINE = Distributed(test_cluster_two_shards, currentDatabase(), test_dod_alias_swap_no_marker_local); + +CREATE TABLE test_dod_alias_swap_no_marker_outer +( + x UInt64, + inner_c UInt64, + a_num UInt64 ALIAS 1, + a_str String ALIAS 'aaaa' +) +ENGINE = Distributed(test_cluster_two_shards, currentDatabase(), test_dod_alias_swap_no_marker_inner); + +SELECT 'prefer_localhost_replica_0_uint64'; +SELECT + x, + a_num, + inner_c +FROM test_dod_alias_swap_no_marker_outer +ORDER BY x +SETTINGS + allow_experimental_analyzer = 1, + enable_alias_marker = 0, + prefer_localhost_replica = 0, + enable_parallel_replicas = 0, + max_parallel_replicas = 1, + parallel_replicas_local_plan = 0 +FORMAT TSVWithNames; + +SELECT 'prefer_localhost_replica_0_string'; +SELECT + x, + a_str, + inner_c +FROM test_dod_alias_swap_no_marker_outer +ORDER BY x +SETTINGS + allow_experimental_analyzer = 1, + enable_alias_marker = 0, + prefer_localhost_replica = 0, + enable_parallel_replicas = 0, + max_parallel_replicas = 1, + parallel_replicas_local_plan = 0 +FORMAT TSVWithNames; + +SELECT 'prefer_localhost_replica_1_uint64'; +SELECT + x, + a_num, + inner_c +FROM test_dod_alias_swap_no_marker_outer +ORDER BY x +SETTINGS + allow_experimental_analyzer = 1, + enable_alias_marker = 0, + prefer_localhost_replica = 1, + enable_parallel_replicas = 0, + max_parallel_replicas = 1, + parallel_replicas_local_plan = 0 +FORMAT TSVWithNames; + +SELECT 'prefer_localhost_replica_1_string'; +SELECT + x, + a_str, + inner_c +FROM test_dod_alias_swap_no_marker_outer +ORDER BY x +SETTINGS + allow_experimental_analyzer = 1, + enable_alias_marker = 0, + prefer_localhost_replica = 1, + enable_parallel_replicas = 0, + max_parallel_replicas = 1, + parallel_replicas_local_plan = 0 +FORMAT TSVWithNames; + +DROP TABLE test_dod_alias_swap_no_marker_outer; +DROP TABLE test_dod_alias_swap_no_marker_inner; +DROP TABLE test_dod_alias_swap_no_marker_local; diff --git a/tests/queries/0_stateless/05062_parallel_replicas_dod_alias_column_swap.reference b/tests/queries/0_stateless/05062_parallel_replicas_dod_alias_column_swap.reference new file mode 100644 index 000000000000..228ac5f667f7 --- /dev/null +++ b/tests/queries/0_stateless/05062_parallel_replicas_dod_alias_column_swap.reference @@ -0,0 +1,20 @@ +no_pr_uint64 +x a_num inner_c +1 1 2 +2 1 3 +10 1 11 +no_pr_string +x a_str inner_c +1 aaaa 2 +2 aaaa 3 +10 aaaa 11 +pr_uint64 +x a_num inner_c +1 1 2 +2 1 3 +10 1 11 +pr_string +x a_str inner_c +1 aaaa 2 +2 aaaa 3 +10 aaaa 11 diff --git a/tests/queries/0_stateless/05062_parallel_replicas_dod_alias_column_swap.sql b/tests/queries/0_stateless/05062_parallel_replicas_dod_alias_column_swap.sql new file mode 100644 index 000000000000..070330e98826 --- /dev/null +++ b/tests/queries/0_stateless/05062_parallel_replicas_dod_alias_column_swap.sql @@ -0,0 +1,94 @@ +DROP TABLE IF EXISTS test_pr_dod_alias_swap_outer; +DROP TABLE IF EXISTS test_pr_dod_alias_swap_inner; +DROP TABLE IF EXISTS test_pr_dod_alias_swap_local; + +CREATE TABLE test_pr_dod_alias_swap_local +( + x UInt64 +) +ENGINE = MergeTree() +ORDER BY x; + +INSERT INTO test_pr_dod_alias_swap_local VALUES (1), (2), (10); + +CREATE TABLE test_pr_dod_alias_swap_inner +( + x UInt64, + inner_c UInt64 ALIAS x + 1 +) +ENGINE = Distributed(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), test_pr_dod_alias_swap_local); + +CREATE TABLE test_pr_dod_alias_swap_outer +( + x UInt64, + inner_c UInt64, + a_num UInt64 ALIAS 1, + a_str String ALIAS 'aaaa' +) +ENGINE = Distributed(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), test_pr_dod_alias_swap_inner); + +SELECT 'no_pr_uint64'; +SELECT x, a_num, inner_c +FROM test_pr_dod_alias_swap_outer +ORDER BY x +SETTINGS + enable_analyzer = 1, + enable_alias_marker = 0, + enable_parallel_replicas = 0, + allow_experimental_parallel_reading_from_replicas = 0, + max_parallel_replicas = 1, + parallel_replicas_local_plan = 0, + parallel_replicas_for_non_replicated_merge_tree = 1, + cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost' +FORMAT TSVWithNames; + +SELECT 'no_pr_string'; +SELECT x, a_str, inner_c +FROM test_pr_dod_alias_swap_outer +ORDER BY x +SETTINGS + enable_analyzer = 1, + enable_alias_marker = 0, + enable_parallel_replicas = 0, + allow_experimental_parallel_reading_from_replicas = 0, + max_parallel_replicas = 1, + parallel_replicas_local_plan = 0, + parallel_replicas_for_non_replicated_merge_tree = 1, + cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost' +FORMAT TSVWithNames; + +SELECT 'pr_uint64'; +SELECT x, a_num, inner_c +FROM test_pr_dod_alias_swap_outer +ORDER BY x +SETTINGS + enable_analyzer = 1, + enable_alias_marker = 0, + enable_parallel_replicas = 2, + allow_experimental_parallel_reading_from_replicas = 2, + max_parallel_replicas = 3, + parallel_replicas_local_plan = 1, + parallel_replicas_for_non_replicated_merge_tree = 1, + parallel_replicas_min_number_of_rows_per_replica = 0, + cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost' +FORMAT TSVWithNames; + +SELECT 'pr_string'; +SELECT x, a_str, inner_c +FROM test_pr_dod_alias_swap_outer +ORDER BY x +SETTINGS + enable_analyzer = 1, + enable_alias_marker = 0, + enable_parallel_replicas = 2, + allow_experimental_parallel_reading_from_replicas = 2, + max_parallel_replicas = 3, + parallel_replicas_local_plan = 1, + parallel_replicas_for_non_replicated_merge_tree = 1, + parallel_replicas_min_number_of_rows_per_replica = 0, + cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost' +FORMAT TSVWithNames; + +DROP TABLE test_pr_dod_alias_swap_outer; +DROP TABLE test_pr_dod_alias_swap_inner; +DROP TABLE test_pr_dod_alias_swap_local; diff --git a/tests/queries/0_stateless/05063_distributed_alias_marker_explicit_column_swap.reference b/tests/queries/0_stateless/05063_distributed_alias_marker_explicit_column_swap.reference new file mode 100644 index 000000000000..f3797cb0ce0e --- /dev/null +++ b/tests/queries/0_stateless/05063_distributed_alias_marker_explicit_column_swap.reference @@ -0,0 +1,32 @@ +prefer_localhost_replica_0_uint64 +a inner_c +1 2 +1 2 +1 3 +1 3 +1 11 +1 11 +prefer_localhost_replica_0_string +a inner_c +aaaa 2 +aaaa 2 +aaaa 3 +aaaa 3 +aaaa 11 +aaaa 11 +prefer_localhost_replica_1_uint64 +a inner_c +1 2 +1 2 +1 3 +1 3 +1 11 +1 11 +prefer_localhost_replica_1_string +a inner_c +aaaa 2 +aaaa 2 +aaaa 3 +aaaa 3 +aaaa 11 +aaaa 11 diff --git a/tests/queries/0_stateless/05063_distributed_alias_marker_explicit_column_swap.sql b/tests/queries/0_stateless/05063_distributed_alias_marker_explicit_column_swap.sql new file mode 100644 index 000000000000..50085531b6dd --- /dev/null +++ b/tests/queries/0_stateless/05063_distributed_alias_marker_explicit_column_swap.sql @@ -0,0 +1,82 @@ +DROP TABLE IF EXISTS test_dod_alias_swap_local; +DROP TABLE IF EXISTS test_dod_alias_swap_inner; + +CREATE TABLE test_dod_alias_swap_local +( + x UInt64 +) +ENGINE = MergeTree() +ORDER BY x; + +INSERT INTO test_dod_alias_swap_local VALUES (1), (2), (10); + +CREATE TABLE test_dod_alias_swap_inner +( + x UInt64, + a UInt64 ALIAS 2, + inner_c UInt64 ALIAS x + 1 +) +ENGINE = Distributed(test_cluster_two_shards, currentDatabase(), test_dod_alias_swap_local); + +SELECT 'prefer_localhost_replica_0_uint64'; +SELECT + __aliasMarker(_CAST(1, 'UInt64'), '__table1.a') AS a, + __table1.inner_c AS inner_c +FROM test_dod_alias_swap_inner AS __table1 +ORDER BY __table1.x +SETTINGS + enable_analyzer = 1, + enable_alias_marker = 1, + prefer_localhost_replica = 0, + enable_parallel_replicas = 0, + max_parallel_replicas = 1, + parallel_replicas_local_plan = 0 +FORMAT TSVWithNames; + +SELECT 'prefer_localhost_replica_0_string'; +SELECT + __aliasMarker(_CAST('aaaa', 'String'), '__table1.a') AS a, + __table1.inner_c AS inner_c +FROM test_dod_alias_swap_inner AS __table1 +ORDER BY __table1.x +SETTINGS + enable_analyzer = 1, + enable_alias_marker = 1, + prefer_localhost_replica = 0, + enable_parallel_replicas = 0, + max_parallel_replicas = 1, + parallel_replicas_local_plan = 0 +FORMAT TSVWithNames; + +SELECT 'prefer_localhost_replica_1_uint64'; +SELECT + __aliasMarker(_CAST(1, 'UInt64'), '__table1.a') AS a, + __table1.inner_c AS inner_c +FROM test_dod_alias_swap_inner AS __table1 +ORDER BY __table1.x +SETTINGS + enable_analyzer = 1, + enable_alias_marker = 1, + prefer_localhost_replica = 1, + enable_parallel_replicas = 0, + max_parallel_replicas = 1, + parallel_replicas_local_plan = 0 +FORMAT TSVWithNames; + +SELECT 'prefer_localhost_replica_1_string'; +SELECT + __aliasMarker(_CAST('aaaa', 'String'), '__table1.a') AS a, + __table1.inner_c AS inner_c +FROM test_dod_alias_swap_inner AS __table1 +ORDER BY __table1.x +SETTINGS + enable_analyzer = 1, + enable_alias_marker = 1, + prefer_localhost_replica = 1, + enable_parallel_replicas = 0, + max_parallel_replicas = 1, + parallel_replicas_local_plan = 0 +FORMAT TSVWithNames; + +DROP TABLE test_dod_alias_swap_inner; +DROP TABLE test_dod_alias_swap_local; diff --git a/tests/queries/0_stateless/05064_alias_marker_in_lambda_over_distributed.reference b/tests/queries/0_stateless/05064_alias_marker_in_lambda_over_distributed.reference new file mode 100644 index 000000000000..8262875092b6 --- /dev/null +++ b/tests/queries/0_stateless/05064_alias_marker_in_lambda_over_distributed.reference @@ -0,0 +1,9 @@ +control +2 +6 +lambda +[12,22] +[11] +lambda_no_marker +[12,22] +[11] diff --git a/tests/queries/0_stateless/05064_alias_marker_in_lambda_over_distributed.sql b/tests/queries/0_stateless/05064_alias_marker_in_lambda_over_distributed.sql new file mode 100644 index 000000000000..848cc84b8f41 --- /dev/null +++ b/tests/queries/0_stateless/05064_alias_marker_in_lambda_over_distributed.sql @@ -0,0 +1,43 @@ +-- An `ALIAS` column referenced from inside a lambda body is inlined and marked like any other, but +-- `finalizeAliasMarkersForDistributedSerialization` used to refuse to descend into a lambda. The marker's id then +-- stayed a live `ColumnNode` and was rendered back into the shipped SQL as `__table1.computed`, so the shard had to +-- resolve the very `ALIAS` column the inlining exists to remove. Here the column is declared on the `Distributed` +-- table and not on the local one, so the shard cannot resolve it and the query fails. +-- +-- The finalize pass still leaves a hand-written marker inside a lambda alone, because its id resolves to the lambda +-- parameter rather than to a column of a table -- that is what 03933 covers. + +DROP TABLE IF EXISTS t_local_05064; +DROP TABLE IF EXISTS t_dist_05064; + +CREATE TABLE t_local_05064 (value UInt64, arr Array(UInt64)) ENGINE = MergeTree ORDER BY value; +INSERT INTO t_local_05064 VALUES (1, [10, 20]), (3, [5]); + +CREATE TABLE t_dist_05064 +( + value UInt64, + arr Array(UInt64), + computed UInt64 ALIAS value * 2 +) +ENGINE = Distributed(test_shard_localhost, currentDatabase(), t_local_05064); + +SELECT 'control'; +SELECT computed +FROM t_dist_05064 +ORDER BY value +SETTINGS enable_alias_marker = 1, prefer_localhost_replica = 0; + +SELECT 'lambda'; +SELECT arrayMap(x -> x + computed, arr) +FROM t_dist_05064 +ORDER BY value +SETTINGS enable_alias_marker = 1, prefer_localhost_replica = 0; + +SELECT 'lambda_no_marker'; +SELECT arrayMap(x -> x + computed, arr) +FROM t_dist_05064 +ORDER BY value +SETTINGS enable_alias_marker = 0, prefer_localhost_replica = 0; + +DROP TABLE t_dist_05064; +DROP TABLE t_local_05064; diff --git a/tests/queries/0_stateless/05065_shadowed_dotted_alias_merge_over_distributed.reference b/tests/queries/0_stateless/05065_shadowed_dotted_alias_merge_over_distributed.reference new file mode 100644 index 000000000000..97dd73615024 --- /dev/null +++ b/tests/queries/0_stateless/05065_shadowed_dotted_alias_merge_over_distributed.reference @@ -0,0 +1,9 @@ +local +1 10 100 +2 20 200 +merge_prefer0 +1 10 100 +2 20 200 +merge_prefer1 +1 10 100 +2 20 200 diff --git a/tests/queries/0_stateless/05065_shadowed_dotted_alias_merge_over_distributed.sql b/tests/queries/0_stateless/05065_shadowed_dotted_alias_merge_over_distributed.sql new file mode 100644 index 000000000000..79a985ba163d --- /dev/null +++ b/tests/queries/0_stateless/05065_shadowed_dotted_alias_merge_over_distributed.sql @@ -0,0 +1,55 @@ +-- Values guard for a `Merge` over `Distributed` whose schema declares both a plain column and a +-- dotted column whose tail is that plain name (`b` and `` `a.b` ``), both read as `ALIAS` columns. +-- 05059 covers dotted names on their own; this shape is covered nowhere else. +-- +-- `test_cluster_two_shards` reads the local table twice, so every query groups to dedup. + +DROP TABLE IF EXISTS t_local_05065; +DROP TABLE IF EXISTS t_dist_05065; +DROP TABLE IF EXISTS t_merge_05065; + +CREATE TABLE t_local_05065 +( + id UInt32, + b UInt32 ALIAS id * 10, + `a.b` UInt32 ALIAS id * 100 +) +ENGINE = MergeTree +ORDER BY id; + +INSERT INTO t_local_05065 VALUES (1), (2); + +CREATE TABLE t_dist_05065 AS t_local_05065 +ENGINE = Distributed(test_cluster_two_shards, currentDatabase(), t_local_05065); + +CREATE TABLE t_merge_05065 +( + id UInt32, + b UInt32, + `a.b` UInt32 +) +ENGINE = Merge(currentDatabase(), '^t_dist_05065$'); + +SELECT 'local'; +SELECT id, b, `a.b` +FROM t_local_05065 +GROUP BY id, b, `a.b` +ORDER BY id; + +SELECT 'merge_prefer0'; +SELECT id, b, `a.b` +FROM t_merge_05065 +GROUP BY id, b, `a.b` +ORDER BY id +SETTINGS prefer_localhost_replica = 0; + +SELECT 'merge_prefer1'; +SELECT id, b, `a.b` +FROM t_merge_05065 +GROUP BY id, b, `a.b` +ORDER BY id +SETTINGS prefer_localhost_replica = 1; + +DROP TABLE t_merge_05065; +DROP TABLE t_dist_05065; +DROP TABLE t_local_05065; diff --git a/tests/queries/0_stateless/05066_global_join_alias_only_on_distributed.reference b/tests/queries/0_stateless/05066_global_join_alias_only_on_distributed.reference new file mode 100644 index 000000000000..48bb2c730573 --- /dev/null +++ b/tests/queries/0_stateless/05066_global_join_alias_only_on_distributed.reference @@ -0,0 +1,18 @@ +projected_marker_on +1 20 +2 40 +3 60 +clause_only_marker_on +2 +3 +projected_marker_off +1 20 +2 40 +3 60 +clause_only_marker_off +2 +3 +implicit_global_join +1 20 +2 40 +3 60 diff --git a/tests/queries/0_stateless/05066_global_join_alias_only_on_distributed.sql b/tests/queries/0_stateless/05066_global_join_alias_only_on_distributed.sql new file mode 100644 index 000000000000..c5e86a074721 --- /dev/null +++ b/tests/queries/0_stateless/05066_global_join_alias_only_on_distributed.sql @@ -0,0 +1,75 @@ +-- A `GLOBAL JOIN` ships its right side as a temporary table, whose columns are the ones +-- `CollectColumnSourceToColumnsVisitor` gathered from the query tree. The `__aliasMarker` id is a +-- `ColumnNode` naming the `ALIAS` column an inlined expression came from, so it used to be gathered +-- as if it were a column the query reads. The temporary table's subquery is rebuilt from names and +-- types alone, which drops the alias body, and the shard is then asked for a column its local table +-- does not declare. +-- +-- `foo` exists only on the `Distributed` table here, so nothing can resolve it on the shard. + +DROP TABLE IF EXISTS t_left_local_05066; +DROP TABLE IF EXISTS t_left_dist_05066; +DROP TABLE IF EXISTS t_right_local_05066; +DROP TABLE IF EXISTS t_right_dist_05066; + +CREATE TABLE t_left_local_05066 (id UInt64) ENGINE = MergeTree ORDER BY id; +INSERT INTO t_left_local_05066 VALUES (1), (2), (3); + +CREATE TABLE t_left_dist_05066 AS t_left_local_05066 +ENGINE = Distributed(test_shard_localhost, currentDatabase(), t_left_local_05066); + +CREATE TABLE t_right_local_05066 (id UInt64, x UInt64) ENGINE = MergeTree ORDER BY id; +INSERT INTO t_right_local_05066 VALUES (1, 10), (2, 20), (3, 30); + +CREATE TABLE t_right_dist_05066 +( + id UInt64, + x UInt64, + foo UInt64 ALIAS x * 2 +) +ENGINE = Distributed(test_shard_localhost, currentDatabase(), t_right_local_05066); + +SELECT 'projected_marker_on'; +SELECT m.id, r.foo +FROM t_left_dist_05066 AS m +GLOBAL INNER JOIN t_right_dist_05066 AS r ON m.id = r.id +ORDER BY m.id +SETTINGS enable_alias_marker = 1, prefer_localhost_replica = 0; + +-- The projection path also sets an alias on the marker. Referencing the column only in a clause +-- exercises the path where no alias is set at all. +SELECT 'clause_only_marker_on'; +SELECT m.id +FROM t_left_dist_05066 AS m +GLOBAL INNER JOIN t_right_dist_05066 AS r ON m.id = r.id +WHERE r.foo > 30 +ORDER BY m.id +SETTINGS enable_alias_marker = 1, prefer_localhost_replica = 0; + +SELECT 'projected_marker_off'; +SELECT m.id, r.foo +FROM t_left_dist_05066 AS m +GLOBAL INNER JOIN t_right_dist_05066 AS r ON m.id = r.id +ORDER BY m.id +SETTINGS enable_alias_marker = 0, prefer_localhost_replica = 0; + +SELECT 'clause_only_marker_off'; +SELECT m.id +FROM t_left_dist_05066 AS m +GLOBAL INNER JOIN t_right_dist_05066 AS r ON m.id = r.id +WHERE r.foo > 30 +ORDER BY m.id +SETTINGS enable_alias_marker = 0, prefer_localhost_replica = 0; + +-- The same rewrite reaches a plain `JOIN` through `prefer_global_in_and_join`. +SELECT 'implicit_global_join'; +SELECT m.id, r.foo +FROM t_left_dist_05066 AS m +INNER JOIN t_right_dist_05066 AS r ON m.id = r.id +ORDER BY m.id +SETTINGS enable_alias_marker = 1, prefer_localhost_replica = 0, prefer_global_in_and_join = 1; + +DROP TABLE t_right_dist_05066; +DROP TABLE t_right_local_05066; +DROP TABLE t_left_dist_05066; +DROP TABLE t_left_local_05066;