diff --git a/src/Storages/IPartitionStrategy.cpp b/src/Storages/IPartitionStrategy.cpp index 01c85f481427c..3b57f9cfcf56c 100644 --- a/src/Storages/IPartitionStrategy.cpp +++ b/src/Storages/IPartitionStrategy.cpp @@ -241,6 +241,16 @@ IPartitionStrategy::getPartitionExpressionActions(ASTPtr & expression_ast) const return result; } +ColumnPtr IPartitionStrategy::computePartitionKey( + const Chunk & chunk, const PartitionExpressionActionsAndColumnName & partition_key_actions) const +{ + Block block_with_partition_by_expr = sample_block.cloneWithoutColumns(); + block_with_partition_by_expr.setColumns(chunk.getColumns()); + partition_key_actions.actions->execute(block_with_partition_by_expr); + + return block_with_partition_by_expr.getByName(partition_key_actions.column_name).column; +} + std::shared_ptr PartitionStrategyFactory::get(StrategyType strategy, ASTPtr partition_by, const NamesAndTypesList & partition_columns, @@ -298,18 +308,12 @@ WildcardPartitionStrategy::WildcardPartitionStrategy(KeyDescription partition_ke cacheDeterministicActions(cached_result, actions_with_column); } -ColumnPtr WildcardPartitionStrategy::computePartitionKey(const Chunk & chunk) const +IPartitionStrategy::PartitionExpressionActionsAndColumnName WildcardPartitionStrategy::getPartitionKeyActions() const { - auto actions_with_column = getCachedOrBuildActions( + return getCachedOrBuildActions( cached_result, *this, [&] { return buildToStringPartitionAST(partition_key_description.definition_ast); }); - - Block block_with_partition_by_expr = sample_block.cloneWithoutColumns(); - block_with_partition_by_expr.setColumns(chunk.getColumns()); - actions_with_column.actions->execute(block_with_partition_by_expr); - - return block_with_partition_by_expr.getByName(actions_with_column.column_name).column; } ColumnPtr WildcardPartitionStrategy::computePartitionKey(Block & block) const @@ -348,18 +352,12 @@ HiveStylePartitionStrategy::HiveStylePartitionStrategy( cacheDeterministicActions(cached_result, actions_with_column); } -ColumnPtr HiveStylePartitionStrategy::computePartitionKey(const Chunk & chunk) const +IPartitionStrategy::PartitionExpressionActionsAndColumnName HiveStylePartitionStrategy::getPartitionKeyActions() const { - auto actions_with_column = getCachedOrBuildActions( + return getCachedOrBuildActions( cached_result, *this, [&] { return buildHivePartitionAST(partition_key_description.definition_ast, getPartitionColumns()); }); - - Block block_with_partition_by_expr = sample_block.cloneWithoutColumns(); - block_with_partition_by_expr.setColumns(chunk.getColumns()); - actions_with_column.actions->execute(block_with_partition_by_expr); - - return block_with_partition_by_expr.getByName(actions_with_column.column_name).column; } ColumnPtr HiveStylePartitionStrategy::computePartitionKey(Block & block) const diff --git a/src/Storages/IPartitionStrategy.h b/src/Storages/IPartitionStrategy.h index 1378762c6911d..1a905d35243db 100644 --- a/src/Storages/IPartitionStrategy.h +++ b/src/Storages/IPartitionStrategy.h @@ -27,7 +27,14 @@ struct IPartitionStrategy virtual ~IPartitionStrategy() = default; - virtual ColumnPtr computePartitionKey(const Chunk & chunk) const = 0; + /// Rebuilt on every call if non-deterministic. A sink calls it once and reuses the result for all its chunks, + /// so a stateful function keeps its state across the chunks of one INSERT. + virtual PartitionExpressionActionsAndColumnName getPartitionKeyActions() const = 0; + + ColumnPtr computePartitionKey(const Chunk & chunk, const PartitionExpressionActionsAndColumnName & partition_key_actions) const; + + /// Convenience overload (used by Delta Lake sink): builds the actions on every call. + ColumnPtr computePartitionKey(const Chunk & chunk) const { return computePartitionKey(chunk, getPartitionKeyActions()); } virtual ColumnPtr computePartitionKey(Block & block) const = 0; @@ -91,7 +98,7 @@ struct WildcardPartitionStrategy : IPartitionStrategy { WildcardPartitionStrategy(KeyDescription partition_key_description_, const Block & sample_block_, ContextPtr context_); - ColumnPtr computePartitionKey(const Chunk & chunk) const override; + PartitionExpressionActionsAndColumnName getPartitionKeyActions() const override; ColumnPtr computePartitionKey(Block & block) const override; }; @@ -110,7 +117,7 @@ struct HiveStylePartitionStrategy : IPartitionStrategy const std::string & file_format_, bool partition_columns_in_data_file_); - ColumnPtr computePartitionKey(const Chunk & chunk) const override; + PartitionExpressionActionsAndColumnName getPartitionKeyActions() const override; ColumnPtr computePartitionKey(Block & block) const override; diff --git a/src/Storages/PartitionedSink.cpp b/src/Storages/PartitionedSink.cpp index fbc41f6be4d39..f44539eb5adaf 100644 --- a/src/Storages/PartitionedSink.cpp +++ b/src/Storages/PartitionedSink.cpp @@ -34,6 +34,7 @@ PartitionedSink::PartitionedSink( , sink_creator(sink_creator_) , context(context_) , source_header(source_header_) + , partition_key_actions(partition_strategy->getPartitionKeyActions()) { } @@ -52,7 +53,7 @@ SinkPtr PartitionedSink::getSinkForPartitionKey(std::string_view partition_key) void PartitionedSink::consume(Chunk & source_chunk) { - const ColumnPtr partition_by_result_column = partition_strategy->computePartitionKey(source_chunk); + const ColumnPtr partition_by_result_column = partition_strategy->computePartitionKey(source_chunk, partition_key_actions); /// Not all columns are serialized using the format writer (e.g, hive partitioning stores partition columns in the file path) const auto columns_to_consume = partition_strategy->getFormatChunkColumns(source_chunk); diff --git a/src/Storages/PartitionedSink.h b/src/Storages/PartitionedSink.h index 1f612076db92f..9bf07defb3fc2 100644 --- a/src/Storages/PartitionedSink.h +++ b/src/Storages/PartitionedSink.h @@ -51,6 +51,7 @@ class PartitionedSink : public SinkToStorage std::shared_ptr sink_creator; ContextPtr context; SharedHeader source_header; + IPartitionStrategy::PartitionExpressionActionsAndColumnName partition_key_actions; absl::flat_hash_map partition_id_to_sink; HashMapWithSavedHash partition_id_to_chunk_index; diff --git a/tests/queries/0_stateless/05459_partition_by_stateful_function_spans_chunks.reference b/tests/queries/0_stateless/05459_partition_by_stateful_function_spans_chunks.reference new file mode 100644 index 0000000000000..61a6ea609e207 --- /dev/null +++ b/tests/queries/0_stateless/05459_partition_by_stateful_function_spans_chunks.reference @@ -0,0 +1,11 @@ +05459_rn_0.csv 1000 0 999 +05459_rn_1.csv 1000 1000 1999 +05459_rn_2.csv 1000 2000 2999 +05459_rn_3.csv 1000 3000 3999 +05459_rn_4.csv 1000 4000 4999 +05459_eng_0.csv 1000 10000 10999 +05459_eng_1.csv 1000 11000 11999 +05459_eng_2.csv 1000 2000 2999 +05459_eng_3.csv 1000 3000 3999 +05459_eng_4.csv 1000 4000 4999 +1 5000 diff --git a/tests/queries/0_stateless/05459_partition_by_stateful_function_spans_chunks.sql b/tests/queries/0_stateless/05459_partition_by_stateful_function_spans_chunks.sql new file mode 100644 index 0000000000000..54420711c58df --- /dev/null +++ b/tests/queries/0_stateless/05459_partition_by_stateful_function_spans_chunks.sql @@ -0,0 +1,25 @@ +-- Tags: no-fasttest +-- Tag no-fasttest: uses S3 + +SET max_block_size = 1000, max_insert_threads = 1, engine_file_truncate_on_insert = 1; + +-- file(): the counter spans the five 1000-row chunks +INSERT INTO FUNCTION file(currentDatabase() || '/05459_rn_{_partition_id}.csv', 'CSV', 'n UInt64') +PARTITION BY intDiv(rowNumberInAllBlocks(), 1000) +SELECT number FROM numbers(5000); +SELECT _file, count(), min(n), max(n) FROM file(currentDatabase() || '/05459_rn_*.csv', 'CSV', 'n UInt64') GROUP BY _file ORDER BY _file; + +-- S3 engine (strategy shared by all INSERTs into the table): spans chunks, starts anew with the next INSERT +CREATE TABLE t_05459 (n UInt64) +ENGINE = S3(s3_conn, filename = currentDatabase() || '/05459_eng_{_partition_id}.csv', format = CSV, partition_strategy = 'wildcard') +PARTITION BY intDiv(rowNumberInAllBlocks(), 1000); +INSERT INTO t_05459 SELECT number FROM numbers(5000) SETTINGS s3_truncate_on_insert = 1, s3_create_new_file_on_insert = 0; +INSERT INTO t_05459 SELECT number + 10000 FROM numbers(2000) SETTINGS s3_truncate_on_insert = 1, s3_create_new_file_on_insert = 0; +SELECT _file, count(), min(n), max(n) FROM s3(s3_conn, filename = currentDatabase() || '/05459_eng_*.csv', format = CSV, structure = 'n UInt64') GROUP BY _file ORDER BY _file; +DROP TABLE t_05459; + +-- now64() is evaluated once per INSERT, not once per chunk +INSERT INTO FUNCTION file(currentDatabase() || '/05459_now_{_partition_id}.csv', 'CSV', 'n UInt64') +PARTITION BY toUnixTimestamp64Nano(now64(9)) +SELECT number FROM numbers(5000); +SELECT count(DISTINCT _file), count() FROM file(currentDatabase() || '/05459_now_*.csv', 'CSV', 'n UInt64');